summaryrefslogtreecommitdiffstats
path: root/third_party/libwebrtc/modules/video_coding/codecs/vp8
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
commit36d22d82aa202bb199967e9512281e9a53db42c9 (patch)
tree105e8c98ddea1c1e4784a60a5a6410fa416be2de /third_party/libwebrtc/modules/video_coding/codecs/vp8
parentInitial commit. (diff)
downloadfirefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.tar.xz
firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.zip
Adding upstream version 115.7.0esr.upstream/115.7.0esr
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/libwebrtc/modules/video_coding/codecs/vp8')
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc884
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h168
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc781
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/include/temporal_layers_checker.h63
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/include/vp8.h50
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/include/vp8_globals.h49
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc384
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h74
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc1438
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h159
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_simulcast_test.cc112
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc624
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers.h164
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc788
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/temporal_layers.h17
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/temporal_layers_checker.cc146
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc913
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/vp8_scalability.cc24
-rw-r--r--third_party/libwebrtc/modules/video_coding/codecs/vp8/vp8_scalability.h24
19 files changed, 6862 insertions, 0 deletions
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc
new file mode 100644
index 0000000000..94860da1b6
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc
@@ -0,0 +1,884 @@
+/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/vp8/default_temporal_layers.h"
+
+#include <stdlib.h>
+
+#include <algorithm>
+#include <array>
+#include <memory>
+#include <set>
+#include <utility>
+#include <vector>
+
+#include "modules/video_coding/include/video_codec_interface.h"
+#include "rtc_base/arraysize.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+#include "system_wrappers/include/field_trial.h"
+
+namespace webrtc {
+DefaultTemporalLayers::PendingFrame::PendingFrame() = default;
+DefaultTemporalLayers::PendingFrame::PendingFrame(
+ uint32_t timestamp,
+ bool expired,
+ uint8_t updated_buffers_mask,
+ const DependencyInfo& dependency_info)
+ : timestamp(timestamp),
+ expired(expired),
+ updated_buffer_mask(updated_buffers_mask),
+ dependency_info(dependency_info) {}
+
+namespace {
+using BufferFlags = Vp8FrameConfig::BufferFlags;
+using FreezeEntropy = Vp8FrameConfig::FreezeEntropy;
+using Vp8BufferReference = Vp8FrameConfig::Vp8BufferReference;
+
+constexpr BufferFlags kNone = BufferFlags::kNone;
+constexpr BufferFlags kReference = BufferFlags::kReference;
+constexpr BufferFlags kUpdate = BufferFlags::kUpdate;
+constexpr BufferFlags kReferenceAndUpdate = BufferFlags::kReferenceAndUpdate;
+constexpr FreezeEntropy kFreezeEntropy = FreezeEntropy::kFreezeEntropy;
+
+static constexpr uint8_t kUninitializedPatternIndex =
+ std::numeric_limits<uint8_t>::max();
+static constexpr std::array<Vp8BufferReference, 3> kAllBuffers = {
+ {Vp8BufferReference::kLast, Vp8BufferReference::kGolden,
+ Vp8BufferReference::kAltref}};
+
+std::vector<unsigned int> GetTemporalIds(size_t num_layers) {
+ switch (num_layers) {
+ case 1:
+ // Temporal layer structure (single layer):
+ // 0 0 0 0 ...
+ return {0};
+ case 2:
+ // Temporal layer structure:
+ // 1 1 ...
+ // 0 0 ...
+ return {0, 1};
+ case 3:
+ // Temporal layer structure:
+ // 2 2 2 2 ...
+ // 1 1 ...
+ // 0 0 ...
+ return {0, 2, 1, 2};
+ case 4:
+ // Temporal layer structure:
+ // 3 3 3 3 3 3 3 3 ...
+ // 2 2 2 2 ...
+ // 1 1 ...
+ // 0 0 ...
+ return {0, 3, 2, 3, 1, 3, 2, 3};
+ default:
+ RTC_DCHECK_NOTREACHED();
+ break;
+ }
+ RTC_DCHECK_NOTREACHED();
+ return {0};
+}
+
+uint8_t GetUpdatedBuffers(const Vp8FrameConfig& config) {
+ uint8_t flags = 0;
+ if (config.last_buffer_flags & BufferFlags::kUpdate) {
+ flags |= static_cast<uint8_t>(Vp8BufferReference::kLast);
+ }
+ if (config.golden_buffer_flags & BufferFlags::kUpdate) {
+ flags |= static_cast<uint8_t>(Vp8BufferReference::kGolden);
+ }
+ if (config.arf_buffer_flags & BufferFlags::kUpdate) {
+ flags |= static_cast<uint8_t>(Vp8BufferReference::kAltref);
+ }
+ return flags;
+}
+
+size_t BufferToIndex(Vp8BufferReference buffer) {
+ switch (buffer) {
+ case Vp8FrameConfig::Vp8BufferReference::kLast:
+ return 0;
+ case Vp8FrameConfig::Vp8BufferReference::kGolden:
+ return 1;
+ case Vp8FrameConfig::Vp8BufferReference::kAltref:
+ return 2;
+ case Vp8FrameConfig::Vp8BufferReference::kNone:
+ RTC_CHECK_NOTREACHED();
+ }
+}
+
+} // namespace
+
+constexpr size_t DefaultTemporalLayers::kNumReferenceBuffers;
+
+std::vector<DefaultTemporalLayers::DependencyInfo>
+DefaultTemporalLayers::GetDependencyInfo(size_t num_layers) {
+ // For indexing in the patterns described below (which temporal layers they
+ // belong to), see the diagram above.
+ // Layer sync is done similarly for all patterns (except single stream) and
+ // happens every 8 frames:
+ // TL1 layer syncs by periodically by only referencing TL0 ('last'), but still
+ // updating 'golden', so it can be used as a reference by future TL1 frames.
+ // TL2 layer syncs just before TL1 by only depending on TL0 (and not depending
+ // on TL1's buffer before TL1 has layer synced).
+ // TODO(pbos): Consider cyclically updating 'arf' (and 'golden' for 1TL) for
+ // the base layer in 1-3TL instead of 'last' periodically on long intervals,
+ // so that if scene changes occur (user walks between rooms or rotates webcam)
+ // the 'arf' (or 'golden' respectively) is not stuck on a no-longer relevant
+ // keyframe.
+
+ switch (num_layers) {
+ case 1:
+ // Always reference and update the same buffer.
+ return {{"S", {kReferenceAndUpdate, kNone, kNone}}};
+ case 2:
+ // All layers can reference but not update the 'alt' buffer, this means
+ // that the 'alt' buffer reference is effectively the last keyframe.
+ // TL0 also references and updates the 'last' buffer.
+ // TL1 also references 'last' and references and updates 'golden'.
+ if (!field_trial::IsDisabled("WebRTC-UseShortVP8TL2Pattern")) {
+ // Shortened 4-frame pattern:
+ // 1---1 1---1 ...
+ // / / / /
+ // 0---0---0---0 ...
+ return {{"SS", {kReferenceAndUpdate, kNone, kNone}},
+ {"-S", {kReference, kUpdate, kNone}},
+ {"SR", {kReferenceAndUpdate, kNone, kNone}},
+ {"-D", {kReference, kReference, kNone, kFreezeEntropy}}};
+ } else {
+ // "Default" 8-frame pattern:
+ // 1---1---1---1 1---1---1---1 ...
+ // / / / / / / / /
+ // 0---0---0---0---0---0---0---0 ...
+ return {{"SS", {kReferenceAndUpdate, kNone, kNone}},
+ {"-S", {kReference, kUpdate, kNone}},
+ {"SR", {kReferenceAndUpdate, kNone, kNone}},
+ {"-R", {kReference, kReferenceAndUpdate, kNone}},
+ {"SR", {kReferenceAndUpdate, kNone, kNone}},
+ {"-R", {kReference, kReferenceAndUpdate, kNone}},
+ {"SR", {kReferenceAndUpdate, kNone, kNone}},
+ {"-D", {kReference, kReference, kNone, kFreezeEntropy}}};
+ }
+ case 3:
+ if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) {
+ // This field trial is intended to check if it is worth using a shorter
+ // temporal pattern, trading some coding efficiency for less risk of
+ // dropped frames.
+ // The coding efficiency will decrease somewhat since the higher layer
+ // state is more volatile, but it will be offset slightly by updating
+ // the altref buffer with TL2 frames, instead of just referencing lower
+ // layers.
+ // If a frame is dropped in a higher layer, the jitter
+ // buffer on the receive side won't be able to decode any higher layer
+ // frame until the next sync frame. So we expect a noticeable decrease
+ // in frame drops on links with high packet loss.
+
+ // TL0 references and updates the 'last' buffer.
+ // TL1 references 'last' and references and updates 'golden'.
+ // TL2 references both 'last' & 'golden' and references and updates
+ // 'arf'.
+ // 2-------2 2-------2 2
+ // / __/ / __/ /
+ // / __1 / __1 /
+ // /___/ /___/ /
+ // 0---------------0---------------0-----
+ // 0 1 2 3 4 5 6 7 8 9 ...
+ return {{"SSS", {kReferenceAndUpdate, kNone, kNone}},
+ {"--S", {kReference, kNone, kUpdate}},
+ {"-DR", {kReference, kUpdate, kNone}},
+ {"--D", {kReference, kReference, kReference, kFreezeEntropy}}};
+ } else {
+ // All layers can reference but not update the 'alt' buffer, this means
+ // that the 'alt' buffer reference is effectively the last keyframe.
+ // TL0 also references and updates the 'last' buffer.
+ // TL1 also references 'last' and references and updates 'golden'.
+ // TL2 references both 'last' and 'golden' but updates no buffer.
+ // 2 __2 _____2 __2 2
+ // / /____/ / / /
+ // / 1---------/-----1 /
+ // /_____/ /_____/ /
+ // 0---------------0---------------0-----
+ // 0 1 2 3 4 5 6 7 8 9 ...
+ return {{"SSS", {kReferenceAndUpdate, kNone, kNone}},
+ {"--D", {kReference, kNone, kNone, kFreezeEntropy}},
+ {"-SS", {kReference, kUpdate, kNone}},
+ {"--D", {kReference, kReference, kNone, kFreezeEntropy}},
+ {"SRR", {kReferenceAndUpdate, kNone, kNone}},
+ {"--D", {kReference, kReference, kNone, kFreezeEntropy}},
+ {"-DS", {kReference, kReferenceAndUpdate, kNone}},
+ {"--D", {kReference, kReference, kNone, kFreezeEntropy}}};
+ }
+ case 4:
+ // TL0 references and updates only the 'last' buffer.
+ // TL1 references 'last' and updates and references 'golden'.
+ // TL2 references 'last' and 'golden', and references and updates 'arf'.
+ // TL3 references all buffers but update none of them.
+ // TODO(philipel): Set decode target information for this structure.
+ return {{"----", {kReferenceAndUpdate, kNone, kNone}},
+ {"----", {kReference, kNone, kNone, kFreezeEntropy}},
+ {"----", {kReference, kNone, kUpdate}},
+ {"----", {kReference, kNone, kReference, kFreezeEntropy}},
+ {"----", {kReference, kUpdate, kNone}},
+ {"----", {kReference, kReference, kReference, kFreezeEntropy}},
+ {"----", {kReference, kReference, kReferenceAndUpdate}},
+ {"----", {kReference, kReference, kReference, kFreezeEntropy}},
+ {"----", {kReferenceAndUpdate, kNone, kNone}},
+ {"----", {kReference, kReference, kReference, kFreezeEntropy}},
+ {"----", {kReference, kReference, kReferenceAndUpdate}},
+ {"----", {kReference, kReference, kReference, kFreezeEntropy}},
+ {"----", {kReference, kReferenceAndUpdate, kNone}},
+ {"----", {kReference, kReference, kReference, kFreezeEntropy}},
+ {"----", {kReference, kReference, kReferenceAndUpdate}},
+ {"----", {kReference, kReference, kReference, kFreezeEntropy}}};
+ default:
+ RTC_DCHECK_NOTREACHED();
+ break;
+ }
+ RTC_DCHECK_NOTREACHED();
+ return {{"", {kNone, kNone, kNone}}};
+}
+
+std::bitset<DefaultTemporalLayers::kNumReferenceBuffers>
+DefaultTemporalLayers::DetermineStaticBuffers(
+ const std::vector<DependencyInfo>& temporal_pattern) {
+ std::bitset<kNumReferenceBuffers> buffers;
+ buffers.set();
+ for (const DependencyInfo& info : temporal_pattern) {
+ uint8_t updated_buffers = GetUpdatedBuffers(info.frame_config);
+
+ for (Vp8BufferReference buffer : kAllBuffers) {
+ if (static_cast<uint8_t>(buffer) & updated_buffers) {
+ buffers.reset(BufferToIndex(buffer));
+ }
+ }
+ }
+ return buffers;
+}
+
+DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers)
+ : num_layers_(std::max(1, number_of_temporal_layers)),
+ temporal_ids_(GetTemporalIds(num_layers_)),
+ temporal_pattern_(GetDependencyInfo(num_layers_)),
+ is_static_buffer_(DetermineStaticBuffers(temporal_pattern_)),
+ pattern_idx_(kUninitializedPatternIndex),
+ new_bitrates_bps_(std::vector<uint32_t>(num_layers_, 0u)) {
+ RTC_CHECK_GE(kMaxTemporalStreams, number_of_temporal_layers);
+ RTC_CHECK_GE(number_of_temporal_layers, 0);
+ RTC_CHECK_LE(number_of_temporal_layers, 4);
+ // pattern_idx_ wraps around temporal_pattern_.size, this is incorrect if
+ // temporal_ids_ are ever longer. If this is no longer correct it needs to
+ // wrap at max(temporal_ids_.size(), temporal_pattern_.size()).
+ RTC_DCHECK_LE(temporal_ids_.size(), temporal_pattern_.size());
+
+ RTC_DCHECK(
+ checker_ = TemporalLayersChecker::CreateTemporalLayersChecker(
+ Vp8TemporalLayersType::kFixedPattern, number_of_temporal_layers));
+
+ // Always need to start with a keyframe, so pre-populate all frame counters.
+ frames_since_buffer_refresh_.fill(0);
+}
+
+DefaultTemporalLayers::~DefaultTemporalLayers() = default;
+
+void DefaultTemporalLayers::SetQpLimits(size_t stream_index,
+ int min_qp,
+ int max_qp) {
+ RTC_DCHECK_LT(stream_index, StreamCount());
+ // Ignore.
+}
+
+size_t DefaultTemporalLayers::StreamCount() const {
+ return 1;
+}
+
+bool DefaultTemporalLayers::SupportsEncoderFrameDropping(
+ size_t stream_index) const {
+ RTC_DCHECK_LT(stream_index, StreamCount());
+ // This class allows the encoder drop frames as it sees fit.
+ return true;
+}
+
+void DefaultTemporalLayers::OnRatesUpdated(
+ size_t stream_index,
+ const std::vector<uint32_t>& bitrates_bps,
+ int framerate_fps) {
+ RTC_DCHECK_LT(stream_index, StreamCount());
+ RTC_DCHECK_GT(bitrates_bps.size(), 0);
+ RTC_DCHECK_LE(bitrates_bps.size(), num_layers_);
+ // `bitrates_bps` uses individual rate per layer, but Vp8EncoderConfig wants
+ // the accumulated rate, so sum them up.
+ new_bitrates_bps_ = bitrates_bps;
+ new_bitrates_bps_->resize(num_layers_);
+ for (size_t i = 1; i < num_layers_; ++i) {
+ (*new_bitrates_bps_)[i] += (*new_bitrates_bps_)[i - 1];
+ }
+}
+
+Vp8EncoderConfig DefaultTemporalLayers::UpdateConfiguration(
+ size_t stream_index) {
+ RTC_DCHECK_LT(stream_index, StreamCount());
+
+ Vp8EncoderConfig config;
+
+ if (!new_bitrates_bps_) {
+ return config;
+ }
+
+ config.temporal_layer_config.emplace();
+ Vp8EncoderConfig::TemporalLayerConfig& ts_config =
+ config.temporal_layer_config.value();
+
+ for (size_t i = 0; i < num_layers_; ++i) {
+ ts_config.ts_target_bitrate[i] = (*new_bitrates_bps_)[i] / 1000;
+ // ..., 4, 2, 1
+ ts_config.ts_rate_decimator[i] = 1 << (num_layers_ - i - 1);
+ }
+
+ ts_config.ts_number_layers = num_layers_;
+ ts_config.ts_periodicity = temporal_ids_.size();
+ std::copy(temporal_ids_.begin(), temporal_ids_.end(),
+ ts_config.ts_layer_id.begin());
+
+ new_bitrates_bps_.reset();
+
+ return config;
+}
+
+bool DefaultTemporalLayers::IsSyncFrame(const Vp8FrameConfig& config) const {
+ // Since we always assign TL0 to 'last' in these patterns, we can infer layer
+ // sync by checking if temporal id > 0 and we only reference TL0 or buffers
+ // containing the last key-frame.
+ if (config.packetizer_temporal_idx == 0) {
+ // TL0 frames are per definition not sync frames.
+ return false;
+ }
+
+ if ((config.last_buffer_flags & BufferFlags::kReference) == 0) {
+ // Sync frames must reference TL0.
+ return false;
+ }
+
+ if ((config.golden_buffer_flags & BufferFlags::kReference) &&
+ !is_static_buffer_[BufferToIndex(Vp8BufferReference::kGolden)]) {
+ // Referencing a golden frame that contains a non-(base layer|key frame).
+ return false;
+ }
+ if ((config.arf_buffer_flags & BufferFlags::kReference) &&
+ !is_static_buffer_[BufferToIndex(Vp8BufferReference::kAltref)]) {
+ // Referencing an altref frame that contains a non-(base layer|key frame).
+ return false;
+ }
+
+ return true;
+}
+
+Vp8FrameConfig DefaultTemporalLayers::NextFrameConfig(size_t stream_index,
+ uint32_t timestamp) {
+ RTC_DCHECK_LT(stream_index, StreamCount());
+ RTC_DCHECK_GT(num_layers_, 0);
+ RTC_DCHECK_GT(temporal_pattern_.size(), 0);
+
+ RTC_DCHECK_GT(kUninitializedPatternIndex, temporal_pattern_.size());
+ const bool first_frame = (pattern_idx_ == kUninitializedPatternIndex);
+
+ pattern_idx_ = (pattern_idx_ + 1) % temporal_pattern_.size();
+ DependencyInfo dependency_info = temporal_pattern_[pattern_idx_];
+ Vp8FrameConfig& tl_config = dependency_info.frame_config;
+ tl_config.encoder_layer_id = tl_config.packetizer_temporal_idx =
+ temporal_ids_[pattern_idx_ % temporal_ids_.size()];
+
+ if (pattern_idx_ == 0) {
+ // Start of new pattern iteration, set up clear state by invalidating any
+ // pending frames, so that we don't make an invalid reference to a buffer
+ // containing data from a previous iteration.
+ for (auto& frame : pending_frames_) {
+ frame.expired = true;
+ }
+ }
+
+ if (first_frame) {
+ tl_config = Vp8FrameConfig::GetIntraFrameConfig();
+ } else {
+ // Last is always ok to reference as it contains the base layer. For other
+ // buffers though, we need to check if the buffer has actually been
+ // refreshed this cycle of the temporal pattern. If the encoder dropped
+ // a frame, it might not have.
+ ValidateReferences(&tl_config.golden_buffer_flags,
+ Vp8BufferReference::kGolden);
+ ValidateReferences(&tl_config.arf_buffer_flags,
+ Vp8BufferReference::kAltref);
+ // Update search order to let the encoder know which buffers contains the
+ // most recent data.
+ UpdateSearchOrder(&tl_config);
+ // Figure out if this a sync frame (non-base-layer frame with only
+ // base-layer references).
+ tl_config.layer_sync = IsSyncFrame(tl_config);
+
+ // Increment frame age, this needs to be in sync with `pattern_idx_`,
+ // so must update it here. Resetting age to 0 must be done when encoding is
+ // complete though, and so in the case of pipelining encoder it might lag.
+ // To prevent this data spill over into the next iteration,
+ // the `pedning_frames_` map is reset in loops. If delay is constant,
+ // the relative age should still be OK for the search order.
+ for (size_t& n : frames_since_buffer_refresh_) {
+ ++n;
+ }
+ }
+
+ // Add frame to set of pending frames, awaiting completion.
+ pending_frames_.emplace_back(timestamp, false, GetUpdatedBuffers(tl_config),
+ dependency_info);
+
+ // Checker does not yet support encoder frame dropping, so validate flags
+ // here before they can be dropped.
+ // TODO(sprang): Update checker to support dropping.
+ RTC_DCHECK(checker_->CheckTemporalConfig(first_frame, tl_config));
+
+ return tl_config;
+}
+
+void DefaultTemporalLayers::ValidateReferences(BufferFlags* flags,
+ Vp8BufferReference ref) const {
+ // Check if the buffer specified by `ref` is actually referenced, and if so
+ // if it also a dynamically updating one (buffers always just containing
+ // keyframes are always safe to reference).
+ if ((*flags & BufferFlags::kReference) &&
+ !is_static_buffer_[BufferToIndex(ref)]) {
+ if (NumFramesSinceBufferRefresh(ref) >= pattern_idx_) {
+ // No valid buffer state, or buffer contains frame that is older than the
+ // current pattern. This reference is not valid, so remove it.
+ *flags = static_cast<BufferFlags>(*flags & ~BufferFlags::kReference);
+ }
+ }
+}
+
+void DefaultTemporalLayers::UpdateSearchOrder(Vp8FrameConfig* config) {
+ // Figure out which of the buffers we can reference, and order them so that
+ // the most recently refreshed is first. Otherwise prioritize last first,
+ // golden second, and altref third.
+ using BufferRefAge = std::pair<Vp8BufferReference, size_t>;
+ std::vector<BufferRefAge> eligible_buffers;
+ if (config->last_buffer_flags & BufferFlags::kReference) {
+ eligible_buffers.emplace_back(
+ Vp8BufferReference::kLast,
+ NumFramesSinceBufferRefresh(Vp8BufferReference::kLast));
+ }
+ if (config->golden_buffer_flags & BufferFlags::kReference) {
+ eligible_buffers.emplace_back(
+ Vp8BufferReference::kGolden,
+ NumFramesSinceBufferRefresh(Vp8BufferReference::kGolden));
+ }
+ if (config->arf_buffer_flags & BufferFlags::kReference) {
+ eligible_buffers.emplace_back(
+ Vp8BufferReference::kAltref,
+ NumFramesSinceBufferRefresh(Vp8BufferReference::kAltref));
+ }
+
+ std::sort(eligible_buffers.begin(), eligible_buffers.end(),
+ [](const BufferRefAge& lhs, const BufferRefAge& rhs) {
+ if (lhs.second != rhs.second) {
+ // Lower count has highest precedence.
+ return lhs.second < rhs.second;
+ }
+ return lhs.first < rhs.first;
+ });
+
+ // Populate the search order fields where possible.
+ if (!eligible_buffers.empty()) {
+ config->first_reference = eligible_buffers.front().first;
+ if (eligible_buffers.size() > 1)
+ config->second_reference = eligible_buffers[1].first;
+ }
+}
+
+size_t DefaultTemporalLayers::NumFramesSinceBufferRefresh(
+ Vp8FrameConfig::Vp8BufferReference ref) const {
+ return frames_since_buffer_refresh_[BufferToIndex(ref)];
+}
+
+void DefaultTemporalLayers::ResetNumFramesSinceBufferRefresh(
+ Vp8FrameConfig::Vp8BufferReference ref) {
+ frames_since_buffer_refresh_[BufferToIndex(ref)] = 0;
+}
+
+void DefaultTemporalLayers::CullPendingFramesBefore(uint32_t timestamp) {
+ while (!pending_frames_.empty() &&
+ pending_frames_.front().timestamp != timestamp) {
+ pending_frames_.pop_front();
+ }
+}
+
+void DefaultTemporalLayers::OnEncodeDone(size_t stream_index,
+ uint32_t rtp_timestamp,
+ size_t size_bytes,
+ bool is_keyframe,
+ int qp,
+ CodecSpecificInfo* info) {
+ RTC_DCHECK_LT(stream_index, StreamCount());
+ RTC_DCHECK_GT(num_layers_, 0);
+
+ if (size_bytes == 0) {
+ RTC_LOG(LS_WARNING) << "Empty frame; treating as dropped.";
+ OnFrameDropped(stream_index, rtp_timestamp);
+ return;
+ }
+
+ CullPendingFramesBefore(rtp_timestamp);
+ RTC_CHECK(!pending_frames_.empty());
+ PendingFrame& frame = pending_frames_.front();
+ RTC_DCHECK_EQ(frame.timestamp, rtp_timestamp);
+ const Vp8FrameConfig& frame_config = frame.dependency_info.frame_config;
+ if (is_keyframe) {
+ // Signal key-frame so checker resets state.
+ RTC_DCHECK(checker_->CheckTemporalConfig(true, frame_config));
+ }
+
+ CodecSpecificInfoVP8& vp8_info = info->codecSpecific.VP8;
+ if (num_layers_ == 1) {
+ vp8_info.temporalIdx = kNoTemporalIdx;
+ vp8_info.layerSync = false;
+ } else {
+ if (is_keyframe) {
+ // Restart the temporal pattern on keyframes.
+ pattern_idx_ = 0;
+ vp8_info.temporalIdx = 0;
+ vp8_info.layerSync = true; // Keyframes are always sync frames.
+
+ for (Vp8BufferReference buffer : kAllBuffers) {
+ if (is_static_buffer_[BufferToIndex(buffer)]) {
+ // Update frame count of all kf-only buffers, regardless of state of
+ // `pending_frames_`.
+ ResetNumFramesSinceBufferRefresh(buffer);
+ } else {
+ // Key-frames update all buffers, this should be reflected when
+ // updating state in FrameEncoded().
+ frame.updated_buffer_mask |= static_cast<uint8_t>(buffer);
+ }
+ }
+ } else {
+ // Delta frame, update codec specifics with temporal id and sync flag.
+ vp8_info.temporalIdx = frame_config.packetizer_temporal_idx;
+ vp8_info.layerSync = frame_config.layer_sync;
+ }
+ }
+
+ vp8_info.useExplicitDependencies = true;
+ RTC_DCHECK_EQ(vp8_info.referencedBuffersCount, 0u);
+ RTC_DCHECK_EQ(vp8_info.updatedBuffersCount, 0u);
+
+ GenericFrameInfo& generic_frame_info = info->generic_frame_info.emplace();
+
+ for (int i = 0; i < static_cast<int>(Vp8FrameConfig::Buffer::kCount); ++i) {
+ bool references = false;
+ bool updates = is_keyframe;
+
+ if (!is_keyframe &&
+ frame_config.References(static_cast<Vp8FrameConfig::Buffer>(i))) {
+ RTC_DCHECK_LT(vp8_info.referencedBuffersCount,
+ arraysize(CodecSpecificInfoVP8::referencedBuffers));
+ references = true;
+ vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i;
+ }
+
+ if (is_keyframe ||
+ frame_config.Updates(static_cast<Vp8FrameConfig::Buffer>(i))) {
+ RTC_DCHECK_LT(vp8_info.updatedBuffersCount,
+ arraysize(CodecSpecificInfoVP8::updatedBuffers));
+ updates = true;
+ vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i;
+ }
+
+ if (references || updates) {
+ generic_frame_info.encoder_buffers.emplace_back(i, references, updates);
+ }
+ }
+
+ // The templates are always present on keyframes, and then refered to by
+ // subsequent frames.
+ if (is_keyframe) {
+ info->template_structure = GetTemplateStructure(num_layers_);
+ generic_frame_info.decode_target_indications =
+ temporal_pattern_.front().decode_target_indications;
+ generic_frame_info.temporal_id = 0;
+ } else {
+ generic_frame_info.decode_target_indications =
+ frame.dependency_info.decode_target_indications;
+ generic_frame_info.temporal_id = frame_config.packetizer_temporal_idx;
+ }
+
+ if (!frame.expired) {
+ for (Vp8BufferReference buffer : kAllBuffers) {
+ if (frame.updated_buffer_mask & static_cast<uint8_t>(buffer)) {
+ ResetNumFramesSinceBufferRefresh(buffer);
+ }
+ }
+ }
+
+ pending_frames_.pop_front();
+}
+
+void DefaultTemporalLayers::OnFrameDropped(size_t stream_index,
+ uint32_t rtp_timestamp) {
+ CullPendingFramesBefore(rtp_timestamp);
+ RTC_CHECK(!pending_frames_.empty());
+ RTC_DCHECK_EQ(pending_frames_.front().timestamp, rtp_timestamp);
+ pending_frames_.pop_front();
+}
+
+void DefaultTemporalLayers::OnPacketLossRateUpdate(float packet_loss_rate) {}
+
+void DefaultTemporalLayers::OnRttUpdate(int64_t rtt_ms) {}
+
+void DefaultTemporalLayers::OnLossNotification(
+ const VideoEncoder::LossNotification& loss_notification) {}
+
+FrameDependencyStructure DefaultTemporalLayers::GetTemplateStructure(
+ int num_layers) const {
+ RTC_CHECK_LT(num_layers, 5);
+ RTC_CHECK_GT(num_layers, 0);
+
+ FrameDependencyStructure template_structure;
+ template_structure.num_decode_targets = num_layers;
+
+ switch (num_layers) {
+ case 1: {
+ template_structure.templates.resize(2);
+ template_structure.templates[0].T(0).Dtis("S");
+ template_structure.templates[1].T(0).Dtis("S").FrameDiffs({1});
+ return template_structure;
+ }
+ case 2: {
+ template_structure.templates.resize(5);
+ template_structure.templates[0].T(0).Dtis("SS");
+ template_structure.templates[1].T(0).Dtis("SS").FrameDiffs({2});
+ template_structure.templates[2].T(0).Dtis("SR").FrameDiffs({2});
+ template_structure.templates[3].T(1).Dtis("-S").FrameDiffs({1});
+ template_structure.templates[4].T(1).Dtis("-D").FrameDiffs({2, 1});
+ return template_structure;
+ }
+ case 3: {
+ if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) {
+ template_structure.templates.resize(5);
+ template_structure.templates[0].T(0).Dtis("SSS");
+ template_structure.templates[1].T(0).Dtis("SSS").FrameDiffs({4});
+ template_structure.templates[2].T(1).Dtis("-DR").FrameDiffs({2});
+ template_structure.templates[3].T(2).Dtis("--S").FrameDiffs({1});
+ template_structure.templates[4].T(2).Dtis("--D").FrameDiffs({2, 1});
+ } else {
+ template_structure.templates.resize(7);
+ template_structure.templates[0].T(0).Dtis("SSS");
+ template_structure.templates[1].T(0).Dtis("SSS").FrameDiffs({4});
+ template_structure.templates[2].T(0).Dtis("SRR").FrameDiffs({4});
+ template_structure.templates[3].T(1).Dtis("-SS").FrameDiffs({2});
+ template_structure.templates[4].T(1).Dtis("-DS").FrameDiffs({4, 2});
+ template_structure.templates[5].T(2).Dtis("--D").FrameDiffs({1});
+ template_structure.templates[6].T(2).Dtis("--D").FrameDiffs({3, 1});
+ }
+ return template_structure;
+ }
+ case 4: {
+ template_structure.templates.resize(8);
+ template_structure.templates[0].T(0).Dtis("SSSS");
+ template_structure.templates[1].T(0).Dtis("SSSS").FrameDiffs({8});
+ template_structure.templates[2].T(1).Dtis("-SRR").FrameDiffs({4});
+ template_structure.templates[3].T(1).Dtis("-SRR").FrameDiffs({4, 8});
+ template_structure.templates[4].T(2).Dtis("--SR").FrameDiffs({2});
+ template_structure.templates[5].T(2).Dtis("--SR").FrameDiffs({2, 4});
+ template_structure.templates[6].T(3).Dtis("---D").FrameDiffs({1});
+ template_structure.templates[7].T(3).Dtis("---D").FrameDiffs({1, 3});
+ return template_structure;
+ }
+ default:
+ RTC_DCHECK_NOTREACHED();
+ // To make the compiler happy!
+ return template_structure;
+ }
+}
+
+// Returns list of temporal dependencies for each frame in the temporal pattern.
+// Values are lists of indecies in the pattern.
+std::vector<std::set<uint8_t>> GetTemporalDependencies(
+ int num_temporal_layers) {
+ switch (num_temporal_layers) {
+ case 1:
+ return {{0}};
+ case 2:
+ if (!field_trial::IsDisabled("WebRTC-UseShortVP8TL2Pattern")) {
+ return {{2}, {0}, {0}, {1, 2}};
+ } else {
+ return {{6}, {0}, {0}, {1, 2}, {2}, {3, 4}, {4}, {5, 6}};
+ }
+ case 3:
+ if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) {
+ return {{0}, {0}, {0}, {0, 1, 2}};
+ } else {
+ return {{4}, {0}, {0}, {0, 2}, {0}, {2, 4}, {2, 4}, {4, 6}};
+ }
+ case 4:
+ return {{8}, {0}, {0}, {0, 2},
+ {0}, {0, 2, 4}, {0, 2, 4}, {0, 4, 6},
+ {0}, {4, 6, 8}, {4, 6, 8}, {4, 8, 10},
+ {4, 8}, {8, 10, 12}, {8, 10, 12}, {8, 12, 14}};
+ default:
+ RTC_DCHECK_NOTREACHED();
+ return {};
+ }
+}
+
+DefaultTemporalLayersChecker::DefaultTemporalLayersChecker(
+ int num_temporal_layers)
+ : TemporalLayersChecker(num_temporal_layers),
+ num_layers_(std::max(1, num_temporal_layers)),
+ temporal_ids_(GetTemporalIds(num_layers_)),
+ temporal_dependencies_(GetTemporalDependencies(num_layers_)),
+ pattern_idx_(255) {
+ int i = 0;
+ while (temporal_ids_.size() < temporal_dependencies_.size()) {
+ temporal_ids_.push_back(temporal_ids_[i++]);
+ }
+}
+
+DefaultTemporalLayersChecker::~DefaultTemporalLayersChecker() = default;
+
+bool DefaultTemporalLayersChecker::CheckTemporalConfig(
+ bool frame_is_keyframe,
+ const Vp8FrameConfig& frame_config) {
+ if (!TemporalLayersChecker::CheckTemporalConfig(frame_is_keyframe,
+ frame_config)) {
+ return false;
+ }
+ if (frame_config.drop_frame) {
+ return true;
+ }
+
+ if (frame_is_keyframe) {
+ pattern_idx_ = 0;
+ last_ = BufferState();
+ golden_ = BufferState();
+ arf_ = BufferState();
+ return true;
+ }
+
+ ++pattern_idx_;
+ if (pattern_idx_ == temporal_ids_.size()) {
+ // All non key-frame buffers should be updated each pattern cycle.
+ if (!last_.is_keyframe && !last_.is_updated_this_cycle) {
+ RTC_LOG(LS_ERROR) << "Last buffer was not updated during pattern cycle.";
+ return false;
+ }
+ if (!arf_.is_keyframe && !arf_.is_updated_this_cycle) {
+ RTC_LOG(LS_ERROR) << "Arf buffer was not updated during pattern cycle.";
+ return false;
+ }
+ if (!golden_.is_keyframe && !golden_.is_updated_this_cycle) {
+ RTC_LOG(LS_ERROR)
+ << "Golden buffer was not updated during pattern cycle.";
+ return false;
+ }
+ last_.is_updated_this_cycle = false;
+ arf_.is_updated_this_cycle = false;
+ golden_.is_updated_this_cycle = false;
+ pattern_idx_ = 0;
+ }
+ uint8_t expected_tl_idx = temporal_ids_[pattern_idx_];
+ if (frame_config.packetizer_temporal_idx != expected_tl_idx) {
+ RTC_LOG(LS_ERROR) << "Frame has an incorrect temporal index. Expected: "
+ << static_cast<int>(expected_tl_idx) << " Actual: "
+ << static_cast<int>(frame_config.packetizer_temporal_idx);
+ return false;
+ }
+
+ bool need_sync = temporal_ids_[pattern_idx_] > 0 &&
+ temporal_ids_[pattern_idx_] != kNoTemporalIdx;
+ std::vector<int> dependencies;
+
+ if (frame_config.last_buffer_flags & BufferFlags::kReference) {
+ uint8_t referenced_layer = temporal_ids_[last_.pattern_idx];
+ if (referenced_layer > 0) {
+ need_sync = false;
+ }
+ if (!last_.is_keyframe) {
+ dependencies.push_back(last_.pattern_idx);
+ }
+ } else if (frame_config.first_reference == Vp8BufferReference::kLast ||
+ frame_config.second_reference == Vp8BufferReference::kLast) {
+ RTC_LOG(LS_ERROR)
+ << "Last buffer not referenced, but present in search order.";
+ return false;
+ }
+
+ if (frame_config.arf_buffer_flags & BufferFlags::kReference) {
+ uint8_t referenced_layer = temporal_ids_[arf_.pattern_idx];
+ if (referenced_layer > 0) {
+ need_sync = false;
+ }
+ if (!arf_.is_keyframe) {
+ dependencies.push_back(arf_.pattern_idx);
+ }
+ } else if (frame_config.first_reference == Vp8BufferReference::kAltref ||
+ frame_config.second_reference == Vp8BufferReference::kAltref) {
+ RTC_LOG(LS_ERROR)
+ << "Altret buffer not referenced, but present in search order.";
+ return false;
+ }
+
+ if (frame_config.golden_buffer_flags & BufferFlags::kReference) {
+ uint8_t referenced_layer = temporal_ids_[golden_.pattern_idx];
+ if (referenced_layer > 0) {
+ need_sync = false;
+ }
+ if (!golden_.is_keyframe) {
+ dependencies.push_back(golden_.pattern_idx);
+ }
+ } else if (frame_config.first_reference == Vp8BufferReference::kGolden ||
+ frame_config.second_reference == Vp8BufferReference::kGolden) {
+ RTC_LOG(LS_ERROR)
+ << "Golden buffer not referenced, but present in search order.";
+ return false;
+ }
+
+ if (need_sync != frame_config.layer_sync) {
+ RTC_LOG(LS_ERROR) << "Sync bit is set incorrectly on a frame. Expected: "
+ << need_sync << " Actual: " << frame_config.layer_sync;
+ return false;
+ }
+
+ if (!frame_is_keyframe) {
+ size_t i;
+ for (i = 0; i < dependencies.size(); ++i) {
+ if (temporal_dependencies_[pattern_idx_].find(dependencies[i]) ==
+ temporal_dependencies_[pattern_idx_].end()) {
+ RTC_LOG(LS_ERROR)
+ << "Illegal temporal dependency out of defined pattern "
+ "from position "
+ << static_cast<int>(pattern_idx_) << " to position "
+ << static_cast<int>(dependencies[i]);
+ return false;
+ }
+ }
+ }
+
+ if (frame_config.last_buffer_flags & BufferFlags::kUpdate) {
+ last_.is_updated_this_cycle = true;
+ last_.pattern_idx = pattern_idx_;
+ last_.is_keyframe = false;
+ }
+ if (frame_config.arf_buffer_flags & BufferFlags::kUpdate) {
+ arf_.is_updated_this_cycle = true;
+ arf_.pattern_idx = pattern_idx_;
+ arf_.is_keyframe = false;
+ }
+ if (frame_config.golden_buffer_flags & BufferFlags::kUpdate) {
+ golden_.is_updated_this_cycle = true;
+ golden_.pattern_idx = pattern_idx_;
+ golden_.is_keyframe = false;
+ }
+ return true;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h
new file mode 100644
index 0000000000..bc6574c54c
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h
@@ -0,0 +1,168 @@
+/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+/*
+ * This file defines classes for doing temporal layers with VP8.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_
+#define MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <bitset>
+#include <deque>
+#include <limits>
+#include <memory>
+#include <set>
+#include <utility>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/video_codecs/vp8_frame_config.h"
+#include "api/video_codecs/vp8_temporal_layers.h"
+#include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h"
+#include "modules/video_coding/include/video_codec_interface.h"
+
+namespace webrtc {
+
+class DefaultTemporalLayers final : public Vp8FrameBufferController {
+ public:
+ explicit DefaultTemporalLayers(int number_of_temporal_layers);
+ ~DefaultTemporalLayers() override;
+
+ void SetQpLimits(size_t stream_index, int min_qp, int max_qp) override;
+
+ size_t StreamCount() const override;
+
+ bool SupportsEncoderFrameDropping(size_t stream_index) const override;
+
+ // Returns the recommended VP8 encode flags needed. May refresh the decoder
+ // and/or update the reference buffers.
+ Vp8FrameConfig NextFrameConfig(size_t stream_index,
+ uint32_t timestamp) override;
+
+ // New target bitrate, per temporal layer.
+ void OnRatesUpdated(size_t stream_index,
+ const std::vector<uint32_t>& bitrates_bps,
+ int framerate_fps) override;
+
+ Vp8EncoderConfig UpdateConfiguration(size_t stream_index) override;
+
+ // Callbacks methods on frame completion. OnEncodeDone() or OnFrameDropped()
+ // should be called once for each NextFrameConfig() call (using the RTP
+ // timestamp as ID), and the calls MUST be in the same order.
+ void OnEncodeDone(size_t stream_index,
+ uint32_t rtp_timestamp,
+ size_t size_bytes,
+ bool is_keyframe,
+ int qp,
+ CodecSpecificInfo* info) override;
+ void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) override;
+
+ void OnPacketLossRateUpdate(float packet_loss_rate) override;
+
+ void OnRttUpdate(int64_t rtt_ms) override;
+
+ void OnLossNotification(
+ const VideoEncoder::LossNotification& loss_notification) override;
+
+ private:
+ static constexpr size_t kNumReferenceBuffers = 3; // Last, golden, altref.
+ struct DependencyInfo {
+ DependencyInfo() = default;
+ DependencyInfo(absl::string_view indication_symbols,
+ Vp8FrameConfig frame_config)
+ : decode_target_indications(
+ webrtc_impl::StringToDecodeTargetIndications(indication_symbols)),
+ frame_config(frame_config) {}
+
+ absl::InlinedVector<DecodeTargetIndication, 10> decode_target_indications;
+ Vp8FrameConfig frame_config;
+ };
+ struct PendingFrame {
+ PendingFrame();
+ PendingFrame(uint32_t timestamp,
+ bool expired,
+ uint8_t updated_buffers_mask,
+ const DependencyInfo& dependency_info);
+ uint32_t timestamp = 0;
+ // Flag indicating if this frame has expired, ie it belongs to a previous
+ // iteration of the temporal pattern.
+ bool expired = false;
+ // Bitmask of Vp8BufferReference flags, indicating which buffers this frame
+ // updates.
+ uint8_t updated_buffer_mask = 0;
+ // The frame config returned by NextFrameConfig() for this frame.
+ DependencyInfo dependency_info;
+ };
+
+ static std::vector<DependencyInfo> GetDependencyInfo(size_t num_layers);
+ static std::bitset<kNumReferenceBuffers> DetermineStaticBuffers(
+ const std::vector<DependencyInfo>& temporal_pattern);
+ bool IsSyncFrame(const Vp8FrameConfig& config) const;
+ void ValidateReferences(Vp8FrameConfig::BufferFlags* flags,
+ Vp8FrameConfig::Vp8BufferReference ref) const;
+ void UpdateSearchOrder(Vp8FrameConfig* config);
+ size_t NumFramesSinceBufferRefresh(
+ Vp8FrameConfig::Vp8BufferReference ref) const;
+ void ResetNumFramesSinceBufferRefresh(Vp8FrameConfig::Vp8BufferReference ref);
+ void CullPendingFramesBefore(uint32_t timestamp);
+
+ const size_t num_layers_;
+ const std::vector<unsigned int> temporal_ids_;
+ const std::vector<DependencyInfo> temporal_pattern_;
+ // Per reference buffer flag indicating if it is static, meaning it is only
+ // updated by key-frames.
+ const std::bitset<kNumReferenceBuffers> is_static_buffer_;
+ FrameDependencyStructure GetTemplateStructure(int num_layers) const;
+
+ uint8_t pattern_idx_;
+ // Updated cumulative bitrates, per temporal layer.
+ absl::optional<std::vector<uint32_t>> new_bitrates_bps_;
+
+ // Status for each pending frame, in
+ std::deque<PendingFrame> pending_frames_;
+
+ // One counter per reference buffer, indicating number of frames since last
+ // refresh. For non-base-layer frames (ie golden, altref buffers), this is
+ // reset when the pattern loops.
+ std::array<size_t, kNumReferenceBuffers> frames_since_buffer_refresh_;
+
+ // Optional utility used to verify reference validity.
+ std::unique_ptr<TemporalLayersChecker> checker_;
+};
+
+class DefaultTemporalLayersChecker : public TemporalLayersChecker {
+ public:
+ explicit DefaultTemporalLayersChecker(int number_of_temporal_layers);
+ ~DefaultTemporalLayersChecker() override;
+
+ bool CheckTemporalConfig(bool frame_is_keyframe,
+ const Vp8FrameConfig& frame_config) override;
+
+ private:
+ struct BufferState {
+ BufferState()
+ : is_updated_this_cycle(false), is_keyframe(true), pattern_idx(0) {}
+
+ bool is_updated_this_cycle;
+ bool is_keyframe;
+ uint8_t pattern_idx;
+ };
+ const size_t num_layers_;
+ std::vector<unsigned int> temporal_ids_;
+ const std::vector<std::set<uint8_t>> temporal_dependencies_;
+ BufferState last_;
+ BufferState arf_;
+ BufferState golden_;
+ uint8_t pattern_idx_;
+};
+
+} // namespace webrtc
+#endif // MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc
new file mode 100644
index 0000000000..ae027a9d8a
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc
@@ -0,0 +1,781 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/vp8/default_temporal_layers.h"
+
+#include <cstdint>
+#include <memory>
+
+#include "api/video/video_bitrate_allocation.h"
+#include "api/video_codecs/video_codec.h"
+#include "api/video_codecs/vp8_frame_config.h"
+#include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h"
+#include "modules/video_coding/include/video_codec_interface.h"
+#include "modules/video_coding/utility/simulcast_rate_allocator.h"
+#include "test/field_trial.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "vpx/vp8cx.h"
+
+// TODO(bugs.webrtc.org/10582): Test the behavior of UpdateConfiguration().
+
+namespace webrtc {
+namespace test {
+namespace {
+
+using ::testing::Each;
+
+enum {
+ kTemporalUpdateLast = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
+ VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF,
+ kTemporalUpdateGoldenWithoutDependency =
+ VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF |
+ VP8_EFLAG_NO_UPD_LAST,
+ kTemporalUpdateGolden =
+ VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
+ kTemporalUpdateAltrefWithoutDependency =
+ VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF |
+ VP8_EFLAG_NO_UPD_LAST,
+ kTemporalUpdateAltref = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_LAST,
+ kTemporalUpdateNone = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
+ VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
+ kTemporalUpdateNoneNoRefAltRef =
+ VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
+ VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
+ kTemporalUpdateNoneNoRefGolden =
+ VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
+ VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
+ kTemporalUpdateNoneNoRefGoldenAltRef =
+ VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_REF_ARF |
+ VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY,
+ kTemporalUpdateGoldenWithoutDependencyRefAltRef =
+ VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
+ kTemporalUpdateGoldenRefAltRef = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST,
+ kTemporalUpdateLastRefAltRef =
+ VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF,
+ kTemporalUpdateLastAndGoldenRefAltRef =
+ VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF,
+};
+
+using BufferFlags = Vp8FrameConfig::BufferFlags;
+using Vp8BufferReference = Vp8FrameConfig::Vp8BufferReference;
+
+constexpr uint8_t kNone = static_cast<uint8_t>(Vp8BufferReference::kNone);
+constexpr uint8_t kLast = static_cast<uint8_t>(Vp8BufferReference::kLast);
+constexpr uint8_t kGolden = static_cast<uint8_t>(Vp8BufferReference::kGolden);
+constexpr uint8_t kAltref = static_cast<uint8_t>(Vp8BufferReference::kAltref);
+constexpr uint8_t kAll = kLast | kGolden | kAltref;
+
+constexpr int ToVp8CodecFlags(uint8_t referenced_buffers,
+ uint8_t updated_buffers,
+ bool update_entropy) {
+ return (((referenced_buffers & kLast) == 0) ? VP8_EFLAG_NO_REF_LAST : 0) |
+ (((referenced_buffers & kGolden) == 0) ? VP8_EFLAG_NO_REF_GF : 0) |
+ (((referenced_buffers & kAltref) == 0) ? VP8_EFLAG_NO_REF_ARF : 0) |
+ (((updated_buffers & kLast) == 0) ? VP8_EFLAG_NO_UPD_LAST : 0) |
+ (((updated_buffers & kGolden) == 0) ? VP8_EFLAG_NO_UPD_GF : 0) |
+ (((updated_buffers & kAltref) == 0) ? VP8_EFLAG_NO_UPD_ARF : 0) |
+ (update_entropy ? 0 : VP8_EFLAG_NO_UPD_ENTROPY);
+}
+
+constexpr int kKeyFrameFlags = ToVp8CodecFlags(kNone, kAll, true);
+
+std::vector<uint32_t> GetTemporalLayerRates(int target_bitrate_kbps,
+ int framerate_fps,
+ int num_temporal_layers) {
+ VideoCodec codec;
+ codec.codecType = VideoCodecType::kVideoCodecVP8;
+ codec.numberOfSimulcastStreams = 1;
+ codec.maxBitrate = target_bitrate_kbps;
+ codec.maxFramerate = framerate_fps;
+ codec.simulcastStream[0].targetBitrate = target_bitrate_kbps;
+ codec.simulcastStream[0].maxBitrate = target_bitrate_kbps;
+ codec.simulcastStream[0].numberOfTemporalLayers = num_temporal_layers;
+ codec.simulcastStream[0].active = true;
+ SimulcastRateAllocator allocator(codec);
+ return allocator
+ .Allocate(
+ VideoBitrateAllocationParameters(target_bitrate_kbps, framerate_fps))
+ .GetTemporalLayerAllocation(0);
+}
+
+constexpr int kDefaultBitrateBps = 500;
+constexpr int kDefaultFramerate = 30;
+constexpr int kDefaultBytesPerFrame =
+ (kDefaultBitrateBps / 8) / kDefaultFramerate;
+constexpr int kDefaultQp = 2;
+} // namespace
+
+class TemporalLayersTest : public ::testing::Test {
+ public:
+ ~TemporalLayersTest() override = default;
+
+ CodecSpecificInfo* IgnoredCodecSpecificInfo() {
+ codec_specific_info_ = std::make_unique<CodecSpecificInfo>();
+ return codec_specific_info_.get();
+ }
+
+ private:
+ std::unique_ptr<CodecSpecificInfo> codec_specific_info_;
+};
+
+TEST_F(TemporalLayersTest, 2Layers) {
+ constexpr int kNumLayers = 2;
+ DefaultTemporalLayers tl(kNumLayers);
+ DefaultTemporalLayersChecker checker(kNumLayers);
+ tl.OnRatesUpdated(0,
+ GetTemporalLayerRates(kDefaultBytesPerFrame,
+ kDefaultFramerate, kNumLayers),
+ kDefaultFramerate);
+ tl.UpdateConfiguration(0);
+
+ constexpr size_t kPatternSize = 4;
+ constexpr size_t kRepetitions = 4;
+
+ const int expected_flags[kPatternSize] = {
+ ToVp8CodecFlags(kLast, kLast, true),
+ ToVp8CodecFlags(kLast, kGolden, true),
+ ToVp8CodecFlags(kLast, kLast, true),
+ ToVp8CodecFlags(kLast | kGolden, kNone, false),
+ };
+ const int expected_temporal_idx[kPatternSize] = {0, 1, 0, 1};
+ const bool expected_layer_sync[kPatternSize] = {false, true, false, false};
+
+ uint32_t timestamp = 0;
+ for (size_t i = 0; i < kPatternSize * kRepetitions; ++i) {
+ const size_t ind = i % kPatternSize;
+ const bool is_keyframe = (i == 0);
+ CodecSpecificInfo info;
+ Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
+ EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[ind],
+ LibvpxVp8Encoder::EncodeFlags(tl_config))
+ << i;
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe,
+ kDefaultQp, &info);
+ EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config));
+ EXPECT_EQ(expected_temporal_idx[ind], info.codecSpecific.VP8.temporalIdx);
+ EXPECT_EQ(expected_temporal_idx[ind], tl_config.packetizer_temporal_idx);
+ EXPECT_EQ(expected_temporal_idx[ind], tl_config.encoder_layer_id);
+ EXPECT_EQ(is_keyframe || expected_layer_sync[ind],
+ info.codecSpecific.VP8.layerSync);
+ EXPECT_EQ(expected_layer_sync[ind], tl_config.layer_sync);
+ timestamp += 3000;
+ }
+}
+
+TEST_F(TemporalLayersTest, 3Layers) {
+ constexpr int kNumLayers = 3;
+ DefaultTemporalLayers tl(kNumLayers);
+ DefaultTemporalLayersChecker checker(kNumLayers);
+ tl.OnRatesUpdated(0,
+ GetTemporalLayerRates(kDefaultBytesPerFrame,
+ kDefaultFramerate, kNumLayers),
+ kDefaultFramerate);
+ tl.UpdateConfiguration(0);
+
+ int expected_flags[16] = {
+ kTemporalUpdateLast,
+ kTemporalUpdateNoneNoRefGoldenAltRef,
+ kTemporalUpdateGoldenWithoutDependency,
+ kTemporalUpdateNoneNoRefAltRef,
+ kTemporalUpdateLast,
+ kTemporalUpdateNoneNoRefAltRef,
+ kTemporalUpdateGolden,
+ kTemporalUpdateNoneNoRefAltRef,
+ kTemporalUpdateLast,
+ kTemporalUpdateNoneNoRefGoldenAltRef,
+ kTemporalUpdateGoldenWithoutDependency,
+ kTemporalUpdateNoneNoRefAltRef,
+ kTemporalUpdateLast,
+ kTemporalUpdateNoneNoRefAltRef,
+ kTemporalUpdateGolden,
+ kTemporalUpdateNoneNoRefAltRef,
+ };
+ int expected_temporal_idx[16] = {0, 2, 1, 2, 0, 2, 1, 2,
+ 0, 2, 1, 2, 0, 2, 1, 2};
+
+ bool expected_layer_sync[16] = {false, true, true, false, false, false,
+ false, false, false, true, true, false,
+ false, false, false, false};
+
+ unsigned int timestamp = 0;
+ for (int i = 0; i < 16; ++i) {
+ const bool is_keyframe = (i == 0);
+ CodecSpecificInfo info;
+ Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
+ EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[i],
+ LibvpxVp8Encoder::EncodeFlags(tl_config))
+ << i;
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe,
+ kDefaultQp, &info);
+ EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config));
+ EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx);
+ EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
+ EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
+ EXPECT_EQ(is_keyframe || expected_layer_sync[i],
+ info.codecSpecific.VP8.layerSync);
+ EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
+ timestamp += 3000;
+ }
+}
+
+TEST_F(TemporalLayersTest, Alternative3Layers) {
+ constexpr int kNumLayers = 3;
+ ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
+ DefaultTemporalLayers tl(kNumLayers);
+ DefaultTemporalLayersChecker checker(kNumLayers);
+ tl.OnRatesUpdated(0,
+ GetTemporalLayerRates(kDefaultBytesPerFrame,
+ kDefaultFramerate, kNumLayers),
+ kDefaultFramerate);
+ tl.UpdateConfiguration(0);
+
+ int expected_flags[8] = {kTemporalUpdateLast,
+ kTemporalUpdateAltrefWithoutDependency,
+ kTemporalUpdateGoldenWithoutDependency,
+ kTemporalUpdateNone,
+ kTemporalUpdateLast,
+ kTemporalUpdateAltrefWithoutDependency,
+ kTemporalUpdateGoldenWithoutDependency,
+ kTemporalUpdateNone};
+ int expected_temporal_idx[8] = {0, 2, 1, 2, 0, 2, 1, 2};
+
+ bool expected_layer_sync[8] = {false, true, true, false,
+ false, true, true, false};
+
+ unsigned int timestamp = 0;
+ for (int i = 0; i < 8; ++i) {
+ const bool is_keyframe = (i == 0);
+ CodecSpecificInfo info;
+ Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
+ EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[i],
+ LibvpxVp8Encoder::EncodeFlags(tl_config))
+ << i;
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe,
+ kDefaultQp, &info);
+ EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config));
+ EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx);
+ EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx);
+ EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id);
+ EXPECT_EQ(is_keyframe || expected_layer_sync[i],
+ info.codecSpecific.VP8.layerSync);
+ EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync);
+ timestamp += 3000;
+ }
+}
+
+TEST_F(TemporalLayersTest, SearchOrder) {
+ constexpr int kNumLayers = 3;
+ ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
+ DefaultTemporalLayers tl(kNumLayers);
+ DefaultTemporalLayersChecker checker(kNumLayers);
+ tl.OnRatesUpdated(0,
+ GetTemporalLayerRates(kDefaultBytesPerFrame,
+ kDefaultFramerate, kNumLayers),
+ kDefaultFramerate);
+ tl.UpdateConfiguration(0);
+
+ // Use a repeating pattern of tl 0, 2, 1, 2.
+ // Tl 0, 1, 2 update last, golden, altref respectively.
+
+ // Start with a key-frame. tl_config flags can be ignored.
+ uint32_t timestamp = 0;
+ Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+
+ // TL2 frame. First one only references TL0. Updates altref.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
+ EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
+
+ // TL1 frame. Can only reference TL0. Updated golden.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
+ EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
+
+ // TL2 frame. Can reference all three buffers. Golden was the last to be
+ // updated, the next to last was altref.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kGolden);
+ EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kAltref);
+}
+
+TEST_F(TemporalLayersTest, SearchOrderWithDrop) {
+ constexpr int kNumLayers = 3;
+ ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
+ DefaultTemporalLayers tl(kNumLayers);
+ DefaultTemporalLayersChecker checker(kNumLayers);
+ tl.OnRatesUpdated(0,
+ GetTemporalLayerRates(kDefaultBytesPerFrame,
+ kDefaultFramerate, kNumLayers),
+ kDefaultFramerate);
+ tl.UpdateConfiguration(0);
+
+ // Use a repeating pattern of tl 0, 2, 1, 2.
+ // Tl 0, 1, 2 update last, golden, altref respectively.
+
+ // Start with a key-frame. tl_config flags can be ignored.
+ uint32_t timestamp = 0;
+ Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+
+ // TL2 frame. First one only references TL0. Updates altref.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast);
+ EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone);
+
+ // Dropped TL1 frame. Can only reference TL0. Should have updated golden.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
+
+ // TL2 frame. Can normally reference all three buffers, but golden has not
+ // been populated this cycle. Altref was last to be updated, before that last.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kAltref);
+ EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kLast);
+}
+
+TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) {
+ constexpr int kNumLayers = 3;
+ // Use a repeating pattern of tl 0, 2, 1, 2.
+ // Tl 0, 1, 2 update last, golden, altref respectively.
+ ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
+ DefaultTemporalLayers tl(kNumLayers);
+ DefaultTemporalLayersChecker checker(kNumLayers);
+ tl.OnRatesUpdated(0,
+ GetTemporalLayerRates(kDefaultBytesPerFrame,
+ kDefaultFramerate, kNumLayers),
+ kDefaultFramerate);
+ tl.UpdateConfiguration(0);
+
+ // Start with a keyframe.
+ uint32_t timestamp = 0;
+ Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+
+ // Dropped TL2 frame.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
+
+ // Dropped TL1 frame.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
+
+ // TL2 frame. Can reference all three buffers, valid since golden and altref
+ // both contain the last keyframe.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
+ EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
+ EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference);
+
+ // Restart of cycle!
+
+ // TL0 base layer frame, updating and referencing last.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+
+ // TL2 frame, updating altref.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+
+ // TL1 frame, updating golden.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+
+ // TL2 frame. Can still reference all buffer since they have been update this
+ // cycle.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
+ EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
+ EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference);
+
+ // Restart of cycle!
+
+ // TL0 base layer frame, updating and referencing last.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+
+ // Dropped TL2 frame.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
+
+ // Dropped TL1 frame.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
+
+ // TL2 frame. This time golden and altref contain data from the previous cycle
+ // and cannot be referenced.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
+ EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference);
+ EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
+}
+
+TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExist) {
+ constexpr int kNumLayers = 3;
+ // Use a repeating pattern of tl 0, 2, 1, 2.
+ // Tl 0, 1 updates last, golden respectively. Altref is always last keyframe.
+ DefaultTemporalLayers tl(kNumLayers);
+ DefaultTemporalLayersChecker checker(kNumLayers);
+ tl.OnRatesUpdated(0,
+ GetTemporalLayerRates(kDefaultBytesPerFrame,
+ kDefaultFramerate, kNumLayers),
+ kDefaultFramerate);
+ tl.UpdateConfiguration(0);
+
+ // Start with a keyframe.
+ uint32_t timestamp = 0;
+ Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+
+ // Do a full cycle of the pattern.
+ for (int i = 0; i < 7; ++i) {
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ }
+
+ // TL0 base layer frame, starting the cycle over.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+
+ // TL2 frame.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+
+ // Encoder has a hiccup and builds a queue, so frame encoding is delayed.
+ // TL1 frame, updating golden.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+
+ // TL2 frame, that should be referencing golden, but we can't be certain it's
+ // not going to be dropped, so that is not allowed.
+ tl_config = tl.NextFrameConfig(0, timestamp + 1);
+ EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
+ EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference);
+ EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
+
+ // TL0 base layer frame.
+ tl_config = tl.NextFrameConfig(0, timestamp + 2);
+
+ // The previous four enqueued frames finally get encoded, and the updated
+ // buffers are now OK to reference.
+ // Enqueued TL1 frame ready.
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ // Enqueued TL2 frame.
+ tl.OnEncodeDone(0, ++timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ // Enqueued TL0 frame.
+ tl.OnEncodeDone(0, ++timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+
+ // TL2 frame, all buffers are now in a known good state, OK to reference.
+ tl_config = tl.NextFrameConfig(0, ++timestamp + 1);
+ EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
+ EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference);
+ EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
+}
+
+TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExistLongDelay) {
+ constexpr int kNumLayers = 3;
+ // Use a repeating pattern of tl 0, 2, 1, 2.
+ // Tl 0, 1 updates last, golden, altref respectively.
+ ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/");
+ DefaultTemporalLayers tl(kNumLayers);
+ DefaultTemporalLayersChecker checker(kNumLayers);
+ tl.OnRatesUpdated(0,
+ GetTemporalLayerRates(kDefaultBytesPerFrame,
+ kDefaultFramerate, kNumLayers),
+ kDefaultFramerate);
+ tl.UpdateConfiguration(0);
+
+ // Start with a keyframe.
+ uint32_t timestamp = 0;
+ Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+
+ // Do a full cycle of the pattern.
+ for (int i = 0; i < 3; ++i) {
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ }
+
+ // TL0 base layer frame, starting the cycle over.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+
+ // TL2 frame.
+ tl_config = tl.NextFrameConfig(0, ++timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+
+ // Encoder has a hiccup and builds a queue, so frame encoding is delayed.
+ // Encoded, but delayed frames in TL 1, 2.
+ tl_config = tl.NextFrameConfig(0, timestamp + 1);
+ tl_config = tl.NextFrameConfig(0, timestamp + 2);
+
+ // Restart of the pattern!
+
+ // Encoded, but delayed frames in TL 2, 1.
+ tl_config = tl.NextFrameConfig(0, timestamp + 3);
+ tl_config = tl.NextFrameConfig(0, timestamp + 4);
+
+ // TL1 frame from last cycle is ready.
+ tl.OnEncodeDone(0, timestamp + 1, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ // TL2 frame from last cycle is ready.
+ tl.OnEncodeDone(0, timestamp + 2, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+
+ // TL2 frame, that should be referencing all buffers, but altref and golden
+ // haven not been updated this cycle. (Don't be fooled by the late frames from
+ // the last cycle!)
+ tl_config = tl.NextFrameConfig(0, timestamp + 5);
+ EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference);
+ EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference);
+ EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference);
+}
+
+TEST_F(TemporalLayersTest, KeyFrame) {
+ constexpr int kNumLayers = 3;
+ DefaultTemporalLayers tl(kNumLayers);
+ DefaultTemporalLayersChecker checker(kNumLayers);
+ tl.OnRatesUpdated(0,
+ GetTemporalLayerRates(kDefaultBytesPerFrame,
+ kDefaultFramerate, kNumLayers),
+ kDefaultFramerate);
+ tl.UpdateConfiguration(0);
+
+ int expected_flags[8] = {
+ kTemporalUpdateLastRefAltRef,
+ kTemporalUpdateNoneNoRefGoldenAltRef,
+ kTemporalUpdateGoldenWithoutDependency,
+ kTemporalUpdateNoneNoRefAltRef,
+ kTemporalUpdateLast,
+ kTemporalUpdateNoneNoRefAltRef,
+ kTemporalUpdateGolden,
+ kTemporalUpdateNone,
+ };
+ int expected_temporal_idx[8] = {0, 2, 1, 2, 0, 2, 1, 2};
+ bool expected_layer_sync[8] = {true, true, true, false,
+ false, false, false, false};
+
+ uint32_t timestamp = 0;
+ for (int i = 0; i < 7; ++i) {
+ // Temporal pattern starts from 0 after key frame. Let the first `i` - 1
+ // frames be delta frames, and the `i`th one key frame.
+ for (int j = 1; j <= i; ++j) {
+ // Since last frame was always a keyframe and thus index 0 in the pattern,
+ // this loop starts at index 1.
+ Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
+ EXPECT_EQ(expected_flags[j], LibvpxVp8Encoder::EncodeFlags(tl_config))
+ << j;
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ EXPECT_TRUE(checker.CheckTemporalConfig(false, tl_config));
+ EXPECT_EQ(expected_temporal_idx[j], tl_config.packetizer_temporal_idx);
+ EXPECT_EQ(expected_temporal_idx[j], tl_config.encoder_layer_id);
+ EXPECT_EQ(expected_layer_sync[j], tl_config.layer_sync);
+ timestamp += 3000;
+ }
+
+ CodecSpecificInfo info;
+ Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp);
+ tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp,
+ &info);
+ EXPECT_TRUE(info.codecSpecific.VP8.layerSync)
+ << "Key frame should be marked layer sync.";
+ EXPECT_EQ(0, info.codecSpecific.VP8.temporalIdx)
+ << "Key frame should always be packetized as layer 0";
+ EXPECT_EQ(0, info.generic_frame_info->temporal_id)
+ << "Key frame should always be packetized as layer 0";
+ EXPECT_THAT(info.generic_frame_info->decode_target_indications,
+ Each(DecodeTargetIndication::kSwitch))
+ << "Key frame is universal switch";
+ EXPECT_TRUE(checker.CheckTemporalConfig(true, tl_config));
+ }
+}
+
+TEST_F(TemporalLayersTest, SetsTlCountOnFirstConfigUpdate) {
+ // Create an instance and fetch config update without setting any rate.
+ constexpr int kNumLayers = 2;
+ DefaultTemporalLayers tl(kNumLayers);
+ Vp8EncoderConfig config = tl.UpdateConfiguration(0);
+
+ // Config should indicate correct number of temporal layers, but zero bitrate.
+ ASSERT_TRUE(config.temporal_layer_config.has_value());
+ EXPECT_EQ(config.temporal_layer_config->ts_number_layers,
+ uint32_t{kNumLayers});
+ std::array<uint32_t, Vp8EncoderConfig::TemporalLayerConfig::kMaxLayers>
+ kZeroRate = {};
+ EXPECT_EQ(config.temporal_layer_config->ts_target_bitrate, kZeroRate);
+
+ // On second call, no new update.
+ config = tl.UpdateConfiguration(0);
+ EXPECT_FALSE(config.temporal_layer_config.has_value());
+}
+
+class TemporalLayersReferenceTest : public TemporalLayersTest,
+ public ::testing::WithParamInterface<int> {
+ public:
+ TemporalLayersReferenceTest()
+ : timestamp_(1),
+ last_sync_timestamp_(timestamp_),
+ tl0_reference_(nullptr) {}
+ virtual ~TemporalLayersReferenceTest() {}
+
+ protected:
+ static const int kMaxPatternLength = 32;
+
+ struct BufferState {
+ BufferState() : BufferState(-1, 0, false) {}
+ BufferState(int temporal_idx, uint32_t timestamp, bool sync)
+ : temporal_idx(temporal_idx), timestamp(timestamp), sync(sync) {}
+ int temporal_idx;
+ uint32_t timestamp;
+ bool sync;
+ };
+
+ bool UpdateSyncRefState(const BufferFlags& flags, BufferState* buffer_state) {
+ if (flags & BufferFlags::kReference) {
+ if (buffer_state->temporal_idx == -1)
+ return true; // References key-frame.
+ if (buffer_state->temporal_idx == 0) {
+ // No more than one reference to TL0 frame.
+ EXPECT_EQ(nullptr, tl0_reference_);
+ tl0_reference_ = buffer_state;
+ return true;
+ }
+ return false; // References higher layer.
+ }
+ return true; // No reference, does not affect sync frame status.
+ }
+
+ void ValidateReference(const BufferFlags& flags,
+ const BufferState& buffer_state,
+ int temporal_layer) {
+ if (flags & BufferFlags::kReference) {
+ if (temporal_layer > 0 && buffer_state.timestamp > 0) {
+ // Check that high layer reference does not go past last sync frame.
+ EXPECT_GE(buffer_state.timestamp, last_sync_timestamp_);
+ }
+ // No reference to buffer in higher layer.
+ EXPECT_LE(buffer_state.temporal_idx, temporal_layer);
+ }
+ }
+
+ uint32_t timestamp_ = 1;
+ uint32_t last_sync_timestamp_ = timestamp_;
+ BufferState* tl0_reference_;
+
+ BufferState last_state;
+ BufferState golden_state;
+ BufferState altref_state;
+};
+
+INSTANTIATE_TEST_SUITE_P(DefaultTemporalLayersTest,
+ TemporalLayersReferenceTest,
+ ::testing::Range(1, kMaxTemporalStreams + 1));
+
+TEST_P(TemporalLayersReferenceTest, ValidFrameConfigs) {
+ const int num_layers = GetParam();
+ DefaultTemporalLayers tl(num_layers);
+ tl.OnRatesUpdated(
+ 0, GetTemporalLayerRates(kDefaultBytesPerFrame, kDefaultFramerate, 1),
+ kDefaultFramerate);
+ tl.UpdateConfiguration(0);
+
+ // Run through the pattern and store the frame dependencies, plus keep track
+ // of the buffer state; which buffers references which temporal layers (if
+ // (any). If a given buffer is never updated, it is legal to reference it
+ // even for sync frames. In order to be general, don't assume TL0 always
+ // updates `last`.
+ std::vector<Vp8FrameConfig> tl_configs(kMaxPatternLength);
+ for (int i = 0; i < kMaxPatternLength; ++i) {
+ Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp_);
+ tl.OnEncodeDone(0, timestamp_, kDefaultBytesPerFrame, i == 0, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ ++timestamp_;
+ EXPECT_FALSE(tl_config.drop_frame);
+ tl_configs.push_back(tl_config);
+ int temporal_idx = tl_config.encoder_layer_id;
+ // For the default layers, always keep encoder and rtp layers in sync.
+ EXPECT_EQ(tl_config.packetizer_temporal_idx, temporal_idx);
+
+ // Determine if this frame is in a higher layer but references only TL0
+ // or untouched buffers, if so verify it is marked as a layer sync.
+ bool is_sync_frame = true;
+ tl0_reference_ = nullptr;
+ if (temporal_idx <= 0) {
+ is_sync_frame = false; // TL0 by definition not a sync frame.
+ } else if (!UpdateSyncRefState(tl_config.last_buffer_flags, &last_state)) {
+ is_sync_frame = false;
+ } else if (!UpdateSyncRefState(tl_config.golden_buffer_flags,
+ &golden_state)) {
+ is_sync_frame = false;
+ } else if (!UpdateSyncRefState(tl_config.arf_buffer_flags, &altref_state)) {
+ is_sync_frame = false;
+ }
+ if (is_sync_frame) {
+ // Cache timestamp for last found sync frame, so that we can verify no
+ // references back past this frame.
+ ASSERT_TRUE(tl0_reference_);
+ last_sync_timestamp_ = tl0_reference_->timestamp;
+ }
+ EXPECT_EQ(tl_config.layer_sync, is_sync_frame);
+
+ // Validate no reference from lower to high temporal layer, or backwards
+ // past last reference frame.
+ ValidateReference(tl_config.last_buffer_flags, last_state, temporal_idx);
+ ValidateReference(tl_config.golden_buffer_flags, golden_state,
+ temporal_idx);
+ ValidateReference(tl_config.arf_buffer_flags, altref_state, temporal_idx);
+
+ // Update the current layer state.
+ BufferState state = {temporal_idx, timestamp_, is_sync_frame};
+ if (tl_config.last_buffer_flags & BufferFlags::kUpdate)
+ last_state = state;
+ if (tl_config.golden_buffer_flags & BufferFlags::kUpdate)
+ golden_state = state;
+ if (tl_config.arf_buffer_flags & BufferFlags::kUpdate)
+ altref_state = state;
+ }
+}
+} // namespace test
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/temporal_layers_checker.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/temporal_layers_checker.h
new file mode 100644
index 0000000000..3d1671a676
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/temporal_layers_checker.h
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_TEMPORAL_LAYERS_CHECKER_H_
+#define MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_TEMPORAL_LAYERS_CHECKER_H_
+
+#include <stdint.h>
+
+#include <memory>
+
+#include "api/video_codecs/vp8_frame_config.h"
+#include "api/video_codecs/vp8_temporal_layers.h"
+
+namespace webrtc {
+
+// Interface for a class that verifies correctness of temporal layer
+// configurations (dependencies, sync flag, etc).
+// Intended to be used in tests as well as with real apps in debug mode.
+class TemporalLayersChecker {
+ public:
+ explicit TemporalLayersChecker(int num_temporal_layers);
+ virtual ~TemporalLayersChecker() {}
+
+ virtual bool CheckTemporalConfig(bool frame_is_keyframe,
+ const Vp8FrameConfig& frame_config);
+
+ static std::unique_ptr<TemporalLayersChecker> CreateTemporalLayersChecker(
+ Vp8TemporalLayersType type,
+ int num_temporal_layers);
+
+ private:
+ struct BufferState {
+ BufferState() : is_keyframe(true), temporal_layer(0), sequence_number(0) {}
+ bool is_keyframe;
+ uint8_t temporal_layer;
+ uint32_t sequence_number;
+ };
+ bool CheckAndUpdateBufferState(BufferState* state,
+ bool* need_sync,
+ bool frame_is_keyframe,
+ uint8_t temporal_layer,
+ Vp8FrameConfig::BufferFlags flags,
+ uint32_t sequence_number,
+ uint32_t* lowest_sequence_referenced);
+ BufferState last_;
+ BufferState arf_;
+ BufferState golden_;
+ int num_temporal_layers_;
+ uint32_t sequence_number_;
+ uint32_t last_sync_sequence_number_;
+ uint32_t last_tl0_sequence_number_;
+};
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_TEMPORAL_LAYERS_CHECKER_H_
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/vp8.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/vp8.h
new file mode 100644
index 0000000000..2fc647874f
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/vp8.h
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_H_
+#define MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_H_
+
+#include <memory>
+#include <vector>
+
+#include "api/video_codecs/video_encoder.h"
+#include "api/video_codecs/vp8_frame_buffer_controller.h"
+#include "modules/video_coding/include/video_codec_interface.h"
+
+namespace webrtc {
+
+// TODO(brandtr): Move these interfaces to the api/ folder.
+class VP8Encoder {
+ public:
+ struct Settings {
+ // Allows for overriding the Vp8FrameBufferController used by the encoder.
+ // If unset, a default Vp8FrameBufferController will be instantiated
+ // internally.
+ std::unique_ptr<Vp8FrameBufferControllerFactory>
+ frame_buffer_controller_factory = nullptr;
+
+ // Allows for overriding the resolution/bitrate limits exposed through
+ // VideoEncoder::GetEncoderInfo(). No override is done if empty.
+ std::vector<VideoEncoder::ResolutionBitrateLimits>
+ resolution_bitrate_limits = {};
+ };
+
+ static std::unique_ptr<VideoEncoder> Create();
+ static std::unique_ptr<VideoEncoder> Create(Settings settings);
+};
+
+class VP8Decoder {
+ public:
+ static std::unique_ptr<VideoDecoder> Create();
+};
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_H_
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/vp8_globals.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/vp8_globals.h
new file mode 100644
index 0000000000..1fab5f45a6
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/vp8_globals.h
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+// This file contains codec dependent definitions that are needed in
+// order to compile the WebRTC codebase, even if this codec is not used.
+
+#ifndef MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_GLOBALS_H_
+#define MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_GLOBALS_H_
+
+#include "modules/video_coding/codecs/interface/common_constants.h"
+
+namespace webrtc {
+
+struct RTPVideoHeaderVP8 {
+ void InitRTPVideoHeaderVP8() {
+ nonReference = false;
+ pictureId = kNoPictureId;
+ tl0PicIdx = kNoTl0PicIdx;
+ temporalIdx = kNoTemporalIdx;
+ layerSync = false;
+ keyIdx = kNoKeyIdx;
+ partitionId = 0;
+ beginningOfPartition = false;
+ }
+
+ bool nonReference; // Frame is discardable.
+ int16_t pictureId; // Picture ID index, 15 bits;
+ // kNoPictureId if PictureID does not exist.
+ int16_t tl0PicIdx; // TL0PIC_IDX, 8 bits;
+ // kNoTl0PicIdx means no value provided.
+ uint8_t temporalIdx; // Temporal layer index, or kNoTemporalIdx.
+ bool layerSync; // This frame is a layer sync frame.
+ // Disabled if temporalIdx == kNoTemporalIdx.
+ int keyIdx; // 5 bits; kNoKeyIdx means not used.
+ int partitionId; // VP8 partition ID
+ bool beginningOfPartition; // True if this packet is the first
+ // in a VP8 partition. Otherwise false
+};
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_GLOBALS_H_
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc
new file mode 100644
index 0000000000..3fe86f2f85
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc
@@ -0,0 +1,384 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include <algorithm>
+#include <memory>
+#include <string>
+
+#include "absl/types/optional.h"
+#include "api/scoped_refptr.h"
+#include "api/video/i420_buffer.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_frame_buffer.h"
+#include "api/video/video_rotation.h"
+#include "modules/video_coding/codecs/vp8/include/vp8.h"
+#include "modules/video_coding/include/video_error_codes.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/numerics/exp_filter.h"
+#include "rtc_base/time_utils.h"
+#include "system_wrappers/include/field_trial.h"
+#include "system_wrappers/include/metrics.h"
+#include "third_party/libyuv/include/libyuv/convert.h"
+#include "vpx/vp8.h"
+#include "vpx/vp8dx.h"
+#include "vpx/vpx_decoder.h"
+
+namespace webrtc {
+namespace {
+constexpr int kVp8ErrorPropagationTh = 30;
+// vpx_decoder.h documentation indicates decode deadline is time in us, with
+// "Set to zero for unlimited.", but actual implementation requires this to be
+// a mode with 0 meaning allow delay and 1 not allowing it.
+constexpr long kDecodeDeadlineRealtime = 1; // NOLINT
+
+const char kVp8PostProcArmFieldTrial[] = "WebRTC-VP8-Postproc-Config-Arm";
+const char kVp8PostProcFieldTrial[] = "WebRTC-VP8-Postproc-Config";
+
+#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
+ defined(WEBRTC_ANDROID) || defined(WEBRTC_ARCH_MIPS)
+constexpr bool kIsArm = true;
+#else
+constexpr bool kIsArm = false;
+#endif
+
+absl::optional<LibvpxVp8Decoder::DeblockParams> DefaultDeblockParams() {
+ return LibvpxVp8Decoder::DeblockParams(/*max_level=*/8,
+ /*degrade_qp=*/60,
+ /*min_qp=*/30);
+}
+
+absl::optional<LibvpxVp8Decoder::DeblockParams>
+GetPostProcParamsFromFieldTrialGroup() {
+ std::string group = webrtc::field_trial::FindFullName(
+ kIsArm ? kVp8PostProcArmFieldTrial : kVp8PostProcFieldTrial);
+ if (group.empty()) {
+ return DefaultDeblockParams();
+ }
+
+ LibvpxVp8Decoder::DeblockParams params;
+ if (sscanf(group.c_str(), "Enabled-%d,%d,%d", &params.max_level,
+ &params.min_qp, &params.degrade_qp) != 3) {
+ return DefaultDeblockParams();
+ }
+
+ if (params.max_level < 0 || params.max_level > 16) {
+ return DefaultDeblockParams();
+ }
+
+ if (params.min_qp < 0 || params.degrade_qp <= params.min_qp) {
+ return DefaultDeblockParams();
+ }
+
+ return params;
+}
+
+} // namespace
+
+std::unique_ptr<VideoDecoder> VP8Decoder::Create() {
+ return std::make_unique<LibvpxVp8Decoder>();
+}
+
+class LibvpxVp8Decoder::QpSmoother {
+ public:
+ QpSmoother() : last_sample_ms_(rtc::TimeMillis()), smoother_(kAlpha) {}
+
+ int GetAvg() const {
+ float value = smoother_.filtered();
+ return (value == rtc::ExpFilter::kValueUndefined) ? 0
+ : static_cast<int>(value);
+ }
+
+ void Add(float sample) {
+ int64_t now_ms = rtc::TimeMillis();
+ smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample);
+ last_sample_ms_ = now_ms;
+ }
+
+ void Reset() { smoother_.Reset(kAlpha); }
+
+ private:
+ const float kAlpha = 0.95f;
+ int64_t last_sample_ms_;
+ rtc::ExpFilter smoother_;
+};
+
+LibvpxVp8Decoder::LibvpxVp8Decoder()
+ : use_postproc_(
+ kIsArm ? webrtc::field_trial::IsEnabled(kVp8PostProcArmFieldTrial)
+ : true),
+ buffer_pool_(false, 300 /* max_number_of_buffers*/),
+ decode_complete_callback_(NULL),
+ inited_(false),
+ decoder_(NULL),
+ propagation_cnt_(-1),
+ last_frame_width_(0),
+ last_frame_height_(0),
+ key_frame_required_(true),
+ deblock_params_(use_postproc_ ? GetPostProcParamsFromFieldTrialGroup()
+ : absl::nullopt),
+ qp_smoother_(use_postproc_ ? new QpSmoother() : nullptr) {}
+
+LibvpxVp8Decoder::~LibvpxVp8Decoder() {
+ inited_ = true; // in order to do the actual release
+ Release();
+}
+
+bool LibvpxVp8Decoder::Configure(const Settings& settings) {
+ if (Release() < 0) {
+ return false;
+ }
+ if (decoder_ == NULL) {
+ decoder_ = new vpx_codec_ctx_t;
+ memset(decoder_, 0, sizeof(*decoder_));
+ }
+ vpx_codec_dec_cfg_t cfg;
+ // Setting number of threads to a constant value (1)
+ cfg.threads = 1;
+ cfg.h = cfg.w = 0; // set after decode
+
+ vpx_codec_flags_t flags = use_postproc_ ? VPX_CODEC_USE_POSTPROC : 0;
+
+ if (vpx_codec_dec_init(decoder_, vpx_codec_vp8_dx(), &cfg, flags)) {
+ delete decoder_;
+ decoder_ = nullptr;
+ return false;
+ }
+
+ propagation_cnt_ = -1;
+ inited_ = true;
+
+ // Always start with a complete key frame.
+ key_frame_required_ = true;
+ if (absl::optional<int> buffer_pool_size = settings.buffer_pool_size()) {
+ if (!buffer_pool_.Resize(*buffer_pool_size)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+int LibvpxVp8Decoder::Decode(const EncodedImage& input_image,
+ bool missing_frames,
+ int64_t /*render_time_ms*/) {
+ if (!inited_) {
+ return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+ }
+ if (decode_complete_callback_ == NULL) {
+ return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+ }
+ if (input_image.data() == NULL && input_image.size() > 0) {
+ // Reset to avoid requesting key frames too often.
+ if (propagation_cnt_ > 0)
+ propagation_cnt_ = 0;
+ return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+ }
+
+ // Post process configurations.
+ if (use_postproc_) {
+ vp8_postproc_cfg_t ppcfg;
+ // MFQE enabled to reduce key frame popping.
+ ppcfg.post_proc_flag = VP8_MFQE;
+
+ if (kIsArm) {
+ RTC_DCHECK(deblock_params_.has_value());
+ }
+ if (deblock_params_.has_value()) {
+ // For low resolutions, use stronger deblocking filter.
+ int last_width_x_height = last_frame_width_ * last_frame_height_;
+ if (last_width_x_height > 0 && last_width_x_height <= 320 * 240) {
+ // Enable the deblock and demacroblocker based on qp thresholds.
+ RTC_DCHECK(qp_smoother_);
+ int qp = qp_smoother_->GetAvg();
+ if (qp > deblock_params_->min_qp) {
+ int level = deblock_params_->max_level;
+ if (qp < deblock_params_->degrade_qp) {
+ // Use lower level.
+ level = deblock_params_->max_level *
+ (qp - deblock_params_->min_qp) /
+ (deblock_params_->degrade_qp - deblock_params_->min_qp);
+ }
+ // Deblocking level only affects VP8_DEMACROBLOCK.
+ ppcfg.deblocking_level = std::max(level, 1);
+ ppcfg.post_proc_flag |= VP8_DEBLOCK | VP8_DEMACROBLOCK;
+ }
+ }
+ } else {
+ // Non-arm with no explicit deblock params set.
+ ppcfg.post_proc_flag |= VP8_DEBLOCK;
+ // For VGA resolutions and lower, enable the demacroblocker postproc.
+ if (last_frame_width_ * last_frame_height_ <= 640 * 360) {
+ ppcfg.post_proc_flag |= VP8_DEMACROBLOCK;
+ }
+ // Strength of deblocking filter. Valid range:[0,16]
+ ppcfg.deblocking_level = 3;
+ }
+
+ vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg);
+ }
+
+ // Always start with a complete key frame.
+ if (key_frame_required_) {
+ if (input_image._frameType != VideoFrameType::kVideoFrameKey)
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ key_frame_required_ = false;
+ }
+ // Restrict error propagation using key frame requests.
+ // Reset on a key frame refresh.
+ if (input_image._frameType == VideoFrameType::kVideoFrameKey) {
+ propagation_cnt_ = -1;
+ // Start count on first loss.
+ } else if (missing_frames && propagation_cnt_ == -1) {
+ propagation_cnt_ = 0;
+ }
+ if (propagation_cnt_ >= 0) {
+ propagation_cnt_++;
+ }
+
+ vpx_codec_iter_t iter = NULL;
+ vpx_image_t* img;
+ int ret;
+
+ // Check for missing frames.
+ if (missing_frames) {
+ // Call decoder with zero data length to signal missing frames.
+ if (vpx_codec_decode(decoder_, NULL, 0, 0, kDecodeDeadlineRealtime)) {
+ // Reset to avoid requesting key frames too often.
+ if (propagation_cnt_ > 0)
+ propagation_cnt_ = 0;
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+ img = vpx_codec_get_frame(decoder_, &iter);
+ iter = NULL;
+ }
+
+ const uint8_t* buffer = input_image.data();
+ if (input_image.size() == 0) {
+ buffer = NULL; // Triggers full frame concealment.
+ }
+ if (vpx_codec_decode(decoder_, buffer, input_image.size(), 0,
+ kDecodeDeadlineRealtime)) {
+ // Reset to avoid requesting key frames too often.
+ if (propagation_cnt_ > 0) {
+ propagation_cnt_ = 0;
+ }
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+
+ img = vpx_codec_get_frame(decoder_, &iter);
+ int qp;
+ vpx_codec_err_t vpx_ret =
+ vpx_codec_control(decoder_, VPXD_GET_LAST_QUANTIZER, &qp);
+ RTC_DCHECK_EQ(vpx_ret, VPX_CODEC_OK);
+ ret = ReturnFrame(img, input_image.Timestamp(), qp, input_image.ColorSpace());
+ if (ret != 0) {
+ // Reset to avoid requesting key frames too often.
+ if (ret < 0 && propagation_cnt_ > 0)
+ propagation_cnt_ = 0;
+ return ret;
+ }
+ // Check Vs. threshold
+ if (propagation_cnt_ > kVp8ErrorPropagationTh) {
+ // Reset to avoid requesting key frames too often.
+ propagation_cnt_ = 0;
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int LibvpxVp8Decoder::ReturnFrame(
+ const vpx_image_t* img,
+ uint32_t timestamp,
+ int qp,
+ const webrtc::ColorSpace* explicit_color_space) {
+ if (img == NULL) {
+ // Decoder OK and NULL image => No show frame
+ return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
+ }
+ if (qp_smoother_) {
+ if (last_frame_width_ != static_cast<int>(img->d_w) ||
+ last_frame_height_ != static_cast<int>(img->d_h)) {
+ qp_smoother_->Reset();
+ }
+ qp_smoother_->Add(qp);
+ }
+ last_frame_width_ = img->d_w;
+ last_frame_height_ = img->d_h;
+ // Allocate memory for decoded image.
+ rtc::scoped_refptr<VideoFrameBuffer> buffer;
+
+ rtc::scoped_refptr<I420Buffer> i420_buffer =
+ buffer_pool_.CreateI420Buffer(img->d_w, img->d_h);
+ buffer = i420_buffer;
+ if (i420_buffer.get()) {
+ libyuv::I420Copy(img->planes[VPX_PLANE_Y], img->stride[VPX_PLANE_Y],
+ img->planes[VPX_PLANE_U], img->stride[VPX_PLANE_U],
+ img->planes[VPX_PLANE_V], img->stride[VPX_PLANE_V],
+ i420_buffer->MutableDataY(), i420_buffer->StrideY(),
+ i420_buffer->MutableDataU(), i420_buffer->StrideU(),
+ i420_buffer->MutableDataV(), i420_buffer->StrideV(),
+ img->d_w, img->d_h);
+ }
+
+ if (!buffer.get()) {
+ // Pool has too many pending frames.
+ RTC_HISTOGRAM_BOOLEAN("WebRTC.Video.LibvpxVp8Decoder.TooManyPendingFrames",
+ 1);
+ return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
+ }
+
+ VideoFrame decoded_image = VideoFrame::Builder()
+ .set_video_frame_buffer(buffer)
+ .set_timestamp_rtp(timestamp)
+ .set_color_space(explicit_color_space)
+ .build();
+ decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp);
+
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int LibvpxVp8Decoder::RegisterDecodeCompleteCallback(
+ DecodedImageCallback* callback) {
+ decode_complete_callback_ = callback;
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int LibvpxVp8Decoder::Release() {
+ int ret_val = WEBRTC_VIDEO_CODEC_OK;
+
+ if (decoder_ != NULL) {
+ if (inited_) {
+ if (vpx_codec_destroy(decoder_)) {
+ ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
+ }
+ }
+ delete decoder_;
+ decoder_ = NULL;
+ }
+ buffer_pool_.Release();
+ inited_ = false;
+ return ret_val;
+}
+
+VideoDecoder::DecoderInfo LibvpxVp8Decoder::GetDecoderInfo() const {
+ DecoderInfo info;
+ info.implementation_name = "libvpx";
+ info.is_hardware_accelerated = false;
+ return info;
+}
+
+const char* LibvpxVp8Decoder::ImplementationName() const {
+ return "libvpx";
+}
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h
new file mode 100644
index 0000000000..f9acd70bad
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_DECODER_H_
+#define MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_DECODER_H_
+
+#include <memory>
+
+#include "absl/types/optional.h"
+#include "api/video/encoded_image.h"
+#include "api/video_codecs/video_decoder.h"
+#include "common_video/include/video_frame_buffer_pool.h"
+#include "modules/video_coding/codecs/vp8/include/vp8.h"
+#include "modules/video_coding/include/video_codec_interface.h"
+#include "vpx/vp8dx.h"
+#include "vpx/vpx_decoder.h"
+
+namespace webrtc {
+
+class LibvpxVp8Decoder : public VideoDecoder {
+ public:
+ LibvpxVp8Decoder();
+ ~LibvpxVp8Decoder() override;
+
+ bool Configure(const Settings& settings) override;
+ int Decode(const EncodedImage& input_image,
+ bool missing_frames,
+ int64_t /*render_time_ms*/) override;
+
+ int RegisterDecodeCompleteCallback(DecodedImageCallback* callback) override;
+ int Release() override;
+
+ DecoderInfo GetDecoderInfo() const override;
+ const char* ImplementationName() const override;
+
+ struct DeblockParams {
+ DeblockParams() : max_level(6), degrade_qp(1), min_qp(0) {}
+ DeblockParams(int max_level, int degrade_qp, int min_qp)
+ : max_level(max_level), degrade_qp(degrade_qp), min_qp(min_qp) {}
+ int max_level; // Deblocking strength: [0, 16].
+ int degrade_qp; // If QP value is below, start lowering `max_level`.
+ int min_qp; // If QP value is below, turn off deblocking.
+ };
+
+ private:
+ class QpSmoother;
+ int ReturnFrame(const vpx_image_t* img,
+ uint32_t timeStamp,
+ int qp,
+ const webrtc::ColorSpace* explicit_color_space);
+ const bool use_postproc_;
+
+ VideoFrameBufferPool buffer_pool_;
+ DecodedImageCallback* decode_complete_callback_;
+ bool inited_;
+ vpx_codec_ctx_t* decoder_;
+ int propagation_cnt_;
+ int last_frame_width_;
+ int last_frame_height_;
+ bool key_frame_required_;
+ const absl::optional<DeblockParams> deblock_params_;
+ const std::unique_ptr<QpSmoother> qp_smoother_;
+};
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_DECODER_H_
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc
new file mode 100644
index 0000000000..cc84605ce7
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc
@@ -0,0 +1,1438 @@
+/*
+ * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h"
+
+#include <string.h>
+
+#include <algorithm>
+#include <cstdint>
+#include <iterator>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/algorithm/container.h"
+#include "api/scoped_refptr.h"
+#include "api/video/video_content_type.h"
+#include "api/video/video_frame_buffer.h"
+#include "api/video/video_timing.h"
+#include "api/video_codecs/scalability_mode.h"
+#include "api/video_codecs/vp8_temporal_layers.h"
+#include "api/video_codecs/vp8_temporal_layers_factory.h"
+#include "modules/video_coding/codecs/interface/common_constants.h"
+#include "modules/video_coding/codecs/vp8/include/vp8.h"
+#include "modules/video_coding/codecs/vp8/vp8_scalability.h"
+#include "modules/video_coding/include/video_error_codes.h"
+#include "modules/video_coding/svc/scalability_mode_util.h"
+#include "modules/video_coding/utility/simulcast_rate_allocator.h"
+#include "modules/video_coding/utility/simulcast_utility.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/experiments/field_trial_parser.h"
+#include "rtc_base/experiments/field_trial_units.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/trace_event.h"
+#include "system_wrappers/include/field_trial.h"
+#include "third_party/libyuv/include/libyuv/scale.h"
+#include "vpx/vp8cx.h"
+
+namespace webrtc {
+namespace {
+#if defined(WEBRTC_IOS)
+constexpr char kVP8IosMaxNumberOfThreadFieldTrial[] =
+ "WebRTC-VP8IosMaxNumberOfThread";
+constexpr char kVP8IosMaxNumberOfThreadFieldTrialParameter[] = "max_thread";
+#endif
+
+constexpr char kVp8ForcePartitionResilience[] =
+ "WebRTC-VP8-ForcePartitionResilience";
+
+// QP is obtained from VP8-bitstream for HW, so the QP corresponds to the
+// bitstream range of [0, 127] and not the user-level range of [0,63].
+constexpr int kLowVp8QpThreshold = 29;
+constexpr int kHighVp8QpThreshold = 95;
+
+constexpr int kTokenPartitions = VP8_ONE_TOKENPARTITION;
+constexpr uint32_t kVp832ByteAlign = 32u;
+
+constexpr int kRtpTicksPerSecond = 90000;
+constexpr int kRtpTicksPerMs = kRtpTicksPerSecond / 1000;
+
+// VP8 denoiser states.
+enum denoiserState : uint32_t {
+ kDenoiserOff,
+ kDenoiserOnYOnly,
+ kDenoiserOnYUV,
+ kDenoiserOnYUVAggressive,
+ // Adaptive mode defaults to kDenoiserOnYUV on key frame, but may switch
+ // to kDenoiserOnYUVAggressive based on a computed noise metric.
+ kDenoiserOnAdaptive
+};
+
+// Greatest common divisior
+int GCD(int a, int b) {
+ int c = a % b;
+ while (c != 0) {
+ a = b;
+ b = c;
+ c = a % b;
+ }
+ return b;
+}
+
+static_assert(Vp8EncoderConfig::TemporalLayerConfig::kMaxPeriodicity ==
+ VPX_TS_MAX_PERIODICITY,
+ "Vp8EncoderConfig::kMaxPeriodicity must be kept in sync with the "
+ "constant in libvpx.");
+static_assert(Vp8EncoderConfig::TemporalLayerConfig::kMaxLayers ==
+ VPX_TS_MAX_LAYERS,
+ "Vp8EncoderConfig::kMaxLayers must be kept in sync with the "
+ "constant in libvpx.");
+
+// Allow a newer value to override a current value only if the new value
+// is set.
+template <typename T>
+bool MaybeSetNewValue(const absl::optional<T>& new_value,
+ absl::optional<T>* base_value) {
+ if (new_value.has_value() && new_value != *base_value) {
+ *base_value = new_value;
+ return true;
+ } else {
+ return false;
+ }
+}
+
+// Adds configuration from `new_config` to `base_config`. Both configs consist
+// of optionals, and only optionals which are set in `new_config` can have
+// an effect. (That is, set values in `base_config` cannot be unset.)
+// Returns `true` iff any changes were made to `base_config`.
+bool MaybeExtendVp8EncoderConfig(const Vp8EncoderConfig& new_config,
+ Vp8EncoderConfig* base_config) {
+ bool changes_made = false;
+ changes_made |= MaybeSetNewValue(new_config.temporal_layer_config,
+ &base_config->temporal_layer_config);
+ changes_made |= MaybeSetNewValue(new_config.rc_target_bitrate,
+ &base_config->rc_target_bitrate);
+ changes_made |= MaybeSetNewValue(new_config.rc_max_quantizer,
+ &base_config->rc_max_quantizer);
+ changes_made |= MaybeSetNewValue(new_config.g_error_resilient,
+ &base_config->g_error_resilient);
+ return changes_made;
+}
+
+void ApplyVp8EncoderConfigToVpxConfig(const Vp8EncoderConfig& encoder_config,
+ vpx_codec_enc_cfg_t* vpx_config) {
+ if (encoder_config.temporal_layer_config.has_value()) {
+ const Vp8EncoderConfig::TemporalLayerConfig& ts_config =
+ encoder_config.temporal_layer_config.value();
+ vpx_config->ts_number_layers = ts_config.ts_number_layers;
+ std::copy(ts_config.ts_target_bitrate.begin(),
+ ts_config.ts_target_bitrate.end(),
+ std::begin(vpx_config->ts_target_bitrate));
+ std::copy(ts_config.ts_rate_decimator.begin(),
+ ts_config.ts_rate_decimator.end(),
+ std::begin(vpx_config->ts_rate_decimator));
+ vpx_config->ts_periodicity = ts_config.ts_periodicity;
+ std::copy(ts_config.ts_layer_id.begin(), ts_config.ts_layer_id.end(),
+ std::begin(vpx_config->ts_layer_id));
+ } else {
+ vpx_config->ts_number_layers = 1;
+ vpx_config->ts_rate_decimator[0] = 1;
+ vpx_config->ts_periodicity = 1;
+ vpx_config->ts_layer_id[0] = 0;
+ }
+
+ if (encoder_config.rc_target_bitrate.has_value()) {
+ vpx_config->rc_target_bitrate = encoder_config.rc_target_bitrate.value();
+ }
+
+ if (encoder_config.rc_max_quantizer.has_value()) {
+ vpx_config->rc_max_quantizer = encoder_config.rc_max_quantizer.value();
+ }
+
+ if (encoder_config.g_error_resilient.has_value()) {
+ vpx_config->g_error_resilient = encoder_config.g_error_resilient.value();
+ }
+}
+
+bool IsCompatibleVideoFrameBufferType(VideoFrameBuffer::Type left,
+ VideoFrameBuffer::Type right) {
+ if (left == VideoFrameBuffer::Type::kI420 ||
+ left == VideoFrameBuffer::Type::kI420A) {
+ // LibvpxVp8Encoder does not care about the alpha channel, I420A and I420
+ // are considered compatible.
+ return right == VideoFrameBuffer::Type::kI420 ||
+ right == VideoFrameBuffer::Type::kI420A;
+ }
+ return left == right;
+}
+
+void SetRawImagePlanes(vpx_image_t* raw_image, VideoFrameBuffer* buffer) {
+ switch (buffer->type()) {
+ case VideoFrameBuffer::Type::kI420:
+ case VideoFrameBuffer::Type::kI420A: {
+ const I420BufferInterface* i420_buffer = buffer->GetI420();
+ RTC_DCHECK(i420_buffer);
+ raw_image->planes[VPX_PLANE_Y] =
+ const_cast<uint8_t*>(i420_buffer->DataY());
+ raw_image->planes[VPX_PLANE_U] =
+ const_cast<uint8_t*>(i420_buffer->DataU());
+ raw_image->planes[VPX_PLANE_V] =
+ const_cast<uint8_t*>(i420_buffer->DataV());
+ raw_image->stride[VPX_PLANE_Y] = i420_buffer->StrideY();
+ raw_image->stride[VPX_PLANE_U] = i420_buffer->StrideU();
+ raw_image->stride[VPX_PLANE_V] = i420_buffer->StrideV();
+ break;
+ }
+ case VideoFrameBuffer::Type::kNV12: {
+ const NV12BufferInterface* nv12_buffer = buffer->GetNV12();
+ RTC_DCHECK(nv12_buffer);
+ raw_image->planes[VPX_PLANE_Y] =
+ const_cast<uint8_t*>(nv12_buffer->DataY());
+ raw_image->planes[VPX_PLANE_U] =
+ const_cast<uint8_t*>(nv12_buffer->DataUV());
+ raw_image->planes[VPX_PLANE_V] = raw_image->planes[VPX_PLANE_U] + 1;
+ raw_image->stride[VPX_PLANE_Y] = nv12_buffer->StrideY();
+ raw_image->stride[VPX_PLANE_U] = nv12_buffer->StrideUV();
+ raw_image->stride[VPX_PLANE_V] = nv12_buffer->StrideUV();
+ break;
+ }
+ default:
+ RTC_DCHECK_NOTREACHED();
+ }
+}
+
+} // namespace
+
+std::unique_ptr<VideoEncoder> VP8Encoder::Create() {
+ return std::make_unique<LibvpxVp8Encoder>(LibvpxInterface::Create(),
+ VP8Encoder::Settings());
+}
+
+std::unique_ptr<VideoEncoder> VP8Encoder::Create(
+ VP8Encoder::Settings settings) {
+ return std::make_unique<LibvpxVp8Encoder>(LibvpxInterface::Create(),
+ std::move(settings));
+}
+
+vpx_enc_frame_flags_t LibvpxVp8Encoder::EncodeFlags(
+ const Vp8FrameConfig& references) {
+ RTC_DCHECK(!references.drop_frame);
+
+ vpx_enc_frame_flags_t flags = 0;
+
+ if ((references.last_buffer_flags &
+ Vp8FrameConfig::BufferFlags::kReference) == 0)
+ flags |= VP8_EFLAG_NO_REF_LAST;
+ if ((references.last_buffer_flags & Vp8FrameConfig::BufferFlags::kUpdate) ==
+ 0)
+ flags |= VP8_EFLAG_NO_UPD_LAST;
+ if ((references.golden_buffer_flags &
+ Vp8FrameConfig::BufferFlags::kReference) == 0)
+ flags |= VP8_EFLAG_NO_REF_GF;
+ if ((references.golden_buffer_flags & Vp8FrameConfig::BufferFlags::kUpdate) ==
+ 0)
+ flags |= VP8_EFLAG_NO_UPD_GF;
+ if ((references.arf_buffer_flags & Vp8FrameConfig::BufferFlags::kReference) ==
+ 0)
+ flags |= VP8_EFLAG_NO_REF_ARF;
+ if ((references.arf_buffer_flags & Vp8FrameConfig::BufferFlags::kUpdate) == 0)
+ flags |= VP8_EFLAG_NO_UPD_ARF;
+ if (references.freeze_entropy)
+ flags |= VP8_EFLAG_NO_UPD_ENTROPY;
+
+ return flags;
+}
+
+LibvpxVp8Encoder::LibvpxVp8Encoder(std::unique_ptr<LibvpxInterface> interface,
+ VP8Encoder::Settings settings)
+ : libvpx_(std::move(interface)),
+ rate_control_settings_(RateControlSettings::ParseFromFieldTrials()),
+ frame_buffer_controller_factory_(
+ std::move(settings.frame_buffer_controller_factory)),
+ resolution_bitrate_limits_(std::move(settings.resolution_bitrate_limits)),
+ key_frame_request_(kMaxSimulcastStreams, false),
+ variable_framerate_experiment_(ParseVariableFramerateConfig(
+ "WebRTC-VP8VariableFramerateScreenshare")),
+ framerate_controller_(variable_framerate_experiment_.framerate_limit) {
+ // TODO(eladalon/ilnik): These reservations might be wasting memory.
+ // InitEncode() is resizing to the actual size, which might be smaller.
+ raw_images_.reserve(kMaxSimulcastStreams);
+ encoded_images_.reserve(kMaxSimulcastStreams);
+ send_stream_.reserve(kMaxSimulcastStreams);
+ cpu_speed_.assign(kMaxSimulcastStreams, cpu_speed_default_);
+ encoders_.reserve(kMaxSimulcastStreams);
+ vpx_configs_.reserve(kMaxSimulcastStreams);
+ config_overrides_.reserve(kMaxSimulcastStreams);
+ downsampling_factors_.reserve(kMaxSimulcastStreams);
+}
+
+LibvpxVp8Encoder::~LibvpxVp8Encoder() {
+ Release();
+}
+
+int LibvpxVp8Encoder::Release() {
+ int ret_val = WEBRTC_VIDEO_CODEC_OK;
+
+ encoded_images_.clear();
+
+ if (inited_) {
+ for (auto it = encoders_.rbegin(); it != encoders_.rend(); ++it) {
+ if (libvpx_->codec_destroy(&*it)) {
+ ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
+ }
+ }
+ }
+ encoders_.clear();
+
+ vpx_configs_.clear();
+ config_overrides_.clear();
+ send_stream_.clear();
+ cpu_speed_.clear();
+
+ for (auto it = raw_images_.rbegin(); it != raw_images_.rend(); ++it) {
+ libvpx_->img_free(&*it);
+ }
+ raw_images_.clear();
+
+ frame_buffer_controller_.reset();
+ inited_ = false;
+ return ret_val;
+}
+
+void LibvpxVp8Encoder::SetRates(const RateControlParameters& parameters) {
+ if (!inited_) {
+ RTC_LOG(LS_WARNING) << "SetRates() while not initialize";
+ return;
+ }
+
+ if (encoders_[0].err) {
+ RTC_LOG(LS_WARNING) << "Encoder in error state.";
+ return;
+ }
+
+ if (parameters.framerate_fps < 1.0) {
+ RTC_LOG(LS_WARNING) << "Unsupported framerate (must be >= 1.0): "
+ << parameters.framerate_fps;
+ return;
+ }
+
+ if (parameters.bitrate.get_sum_bps() == 0) {
+ // Encoder paused, turn off all encoding.
+ const int num_streams = static_cast<size_t>(encoders_.size());
+ for (int i = 0; i < num_streams; ++i)
+ SetStreamState(false, i);
+ return;
+ }
+
+ codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
+
+ if (encoders_.size() > 1) {
+ // If we have more than 1 stream, reduce the qp_max for the low resolution
+ // stream if frame rate is not too low. The trade-off with lower qp_max is
+ // possibly more dropped frames, so we only do this if the frame rate is
+ // above some threshold (base temporal layer is down to 1/4 for 3 layers).
+ // We may want to condition this on bitrate later.
+ if (rate_control_settings_.Vp8BoostBaseLayerQuality() &&
+ parameters.framerate_fps > 20.0) {
+ vpx_configs_[encoders_.size() - 1].rc_max_quantizer = 45;
+ } else {
+ // Go back to default value set in InitEncode.
+ vpx_configs_[encoders_.size() - 1].rc_max_quantizer = qp_max_;
+ }
+ }
+
+ for (size_t i = 0; i < encoders_.size(); ++i) {
+ const size_t stream_idx = encoders_.size() - 1 - i;
+
+ unsigned int target_bitrate_kbps =
+ parameters.bitrate.GetSpatialLayerSum(stream_idx) / 1000;
+
+ bool send_stream = target_bitrate_kbps > 0;
+ if (send_stream || encoders_.size() > 1)
+ SetStreamState(send_stream, stream_idx);
+
+ vpx_configs_[i].rc_target_bitrate = target_bitrate_kbps;
+ if (send_stream) {
+ frame_buffer_controller_->OnRatesUpdated(
+ stream_idx, parameters.bitrate.GetTemporalLayerAllocation(stream_idx),
+ static_cast<int>(parameters.framerate_fps + 0.5));
+ }
+
+ UpdateVpxConfiguration(stream_idx);
+
+ vpx_codec_err_t err =
+ libvpx_->codec_enc_config_set(&encoders_[i], &vpx_configs_[i]);
+ if (err != VPX_CODEC_OK) {
+ RTC_LOG(LS_WARNING) << "Error configuring codec, error code: " << err
+ << ", details: "
+ << libvpx_->codec_error_detail(&encoders_[i]);
+ }
+ }
+}
+
+void LibvpxVp8Encoder::OnPacketLossRateUpdate(float packet_loss_rate) {
+ // TODO(bugs.webrtc.org/10431): Replace condition by DCHECK.
+ if (frame_buffer_controller_) {
+ frame_buffer_controller_->OnPacketLossRateUpdate(packet_loss_rate);
+ }
+}
+
+void LibvpxVp8Encoder::OnRttUpdate(int64_t rtt_ms) {
+ // TODO(bugs.webrtc.org/10431): Replace condition by DCHECK.
+ if (frame_buffer_controller_) {
+ frame_buffer_controller_->OnRttUpdate(rtt_ms);
+ }
+}
+
+void LibvpxVp8Encoder::OnLossNotification(
+ const LossNotification& loss_notification) {
+ if (frame_buffer_controller_) {
+ frame_buffer_controller_->OnLossNotification(loss_notification);
+ }
+}
+
+void LibvpxVp8Encoder::SetStreamState(bool send_stream, int stream_idx) {
+ if (send_stream && !send_stream_[stream_idx]) {
+ // Need a key frame if we have not sent this stream before.
+ key_frame_request_[stream_idx] = true;
+ }
+ send_stream_[stream_idx] = send_stream;
+}
+
+void LibvpxVp8Encoder::SetFecControllerOverride(
+ FecControllerOverride* fec_controller_override) {
+ // TODO(bugs.webrtc.org/10769): Update downstream and remove ability to
+ // pass nullptr.
+ // RTC_DCHECK(fec_controller_override);
+ RTC_DCHECK(!fec_controller_override_);
+ fec_controller_override_ = fec_controller_override;
+}
+
+// TODO(eladalon): s/inst/codec_settings/g.
+int LibvpxVp8Encoder::InitEncode(const VideoCodec* inst,
+ const VideoEncoder::Settings& settings) {
+ if (inst == NULL) {
+ return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+ }
+ if (inst->maxFramerate < 1) {
+ return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+ }
+ // allow zero to represent an unspecified maxBitRate
+ if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) {
+ return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+ }
+ if (inst->width < 1 || inst->height < 1) {
+ return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+ }
+ if (settings.number_of_cores < 1) {
+ return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+ }
+
+ if (absl::optional<ScalabilityMode> scalability_mode =
+ inst->GetScalabilityMode();
+ scalability_mode.has_value() &&
+ !VP8SupportsScalabilityMode(*scalability_mode)) {
+ return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+ }
+
+ num_active_streams_ = 0;
+ for (int i = 0; i < inst->numberOfSimulcastStreams; ++i) {
+ if (inst->simulcastStream[i].active) {
+ ++num_active_streams_;
+ }
+ }
+ if (inst->numberOfSimulcastStreams == 0 && inst->active) {
+ num_active_streams_ = 1;
+ }
+
+ if (inst->VP8().automaticResizeOn && num_active_streams_ > 1) {
+ return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+ }
+
+ // Use the previous pixel format to avoid extra image allocations.
+ vpx_img_fmt_t pixel_format =
+ raw_images_.empty() ? VPX_IMG_FMT_I420 : raw_images_[0].fmt;
+
+ int retVal = Release();
+ if (retVal < 0) {
+ return retVal;
+ }
+
+ int number_of_streams = SimulcastUtility::NumberOfSimulcastStreams(*inst);
+ if (number_of_streams > 1 &&
+ !SimulcastUtility::ValidSimulcastParameters(*inst, number_of_streams)) {
+ return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED;
+ }
+
+ RTC_DCHECK(!frame_buffer_controller_);
+ if (frame_buffer_controller_factory_) {
+ frame_buffer_controller_ = frame_buffer_controller_factory_->Create(
+ *inst, settings, fec_controller_override_);
+ } else {
+ Vp8TemporalLayersFactory factory;
+ frame_buffer_controller_ =
+ factory.Create(*inst, settings, fec_controller_override_);
+ }
+ RTC_DCHECK(frame_buffer_controller_);
+
+ number_of_cores_ = settings.number_of_cores;
+ timestamp_ = 0;
+ codec_ = *inst;
+
+ // Code expects simulcastStream resolutions to be correct, make sure they are
+ // filled even when there are no simulcast layers.
+ if (codec_.numberOfSimulcastStreams == 0) {
+ codec_.simulcastStream[0].width = codec_.width;
+ codec_.simulcastStream[0].height = codec_.height;
+ }
+
+ encoded_images_.resize(number_of_streams);
+ encoders_.resize(number_of_streams);
+ vpx_configs_.resize(number_of_streams);
+ config_overrides_.resize(number_of_streams);
+ downsampling_factors_.resize(number_of_streams);
+ raw_images_.resize(number_of_streams);
+ send_stream_.resize(number_of_streams);
+ send_stream_[0] = true; // For non-simulcast case.
+ cpu_speed_.resize(number_of_streams);
+ std::fill(key_frame_request_.begin(), key_frame_request_.end(), false);
+
+ int idx = number_of_streams - 1;
+ for (int i = 0; i < (number_of_streams - 1); ++i, --idx) {
+ int gcd = GCD(inst->simulcastStream[idx].width,
+ inst->simulcastStream[idx - 1].width);
+ downsampling_factors_[i].num = inst->simulcastStream[idx].width / gcd;
+ downsampling_factors_[i].den = inst->simulcastStream[idx - 1].width / gcd;
+ send_stream_[i] = false;
+ }
+ if (number_of_streams > 1) {
+ send_stream_[number_of_streams - 1] = false;
+ downsampling_factors_[number_of_streams - 1].num = 1;
+ downsampling_factors_[number_of_streams - 1].den = 1;
+ }
+
+ // populate encoder configuration with default values
+ if (libvpx_->codec_enc_config_default(vpx_codec_vp8_cx(), &vpx_configs_[0],
+ 0)) {
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+ // setting the time base of the codec
+ vpx_configs_[0].g_timebase.num = 1;
+ vpx_configs_[0].g_timebase.den = kRtpTicksPerSecond;
+ vpx_configs_[0].g_lag_in_frames = 0; // 0- no frame lagging
+
+ // Set the error resilience mode for temporal layers (but not simulcast).
+ vpx_configs_[0].g_error_resilient =
+ (SimulcastUtility::NumberOfTemporalLayers(*inst, 0) > 1)
+ ? VPX_ERROR_RESILIENT_DEFAULT
+ : 0;
+
+ // Override the error resilience mode if this is not simulcast, but we are
+ // using temporal layers.
+ if (field_trial::IsEnabled(kVp8ForcePartitionResilience) &&
+ (number_of_streams == 1) &&
+ (SimulcastUtility::NumberOfTemporalLayers(*inst, 0) > 1)) {
+ RTC_LOG(LS_INFO) << "Overriding g_error_resilient from "
+ << vpx_configs_[0].g_error_resilient << " to "
+ << VPX_ERROR_RESILIENT_PARTITIONS;
+ vpx_configs_[0].g_error_resilient = VPX_ERROR_RESILIENT_PARTITIONS;
+ }
+
+ // rate control settings
+ vpx_configs_[0].rc_dropframe_thresh = FrameDropThreshold(0);
+ vpx_configs_[0].rc_end_usage = VPX_CBR;
+ vpx_configs_[0].g_pass = VPX_RC_ONE_PASS;
+ // Handle resizing outside of libvpx.
+ vpx_configs_[0].rc_resize_allowed = 0;
+ vpx_configs_[0].rc_min_quantizer =
+ codec_.mode == VideoCodecMode::kScreensharing ? 12 : 2;
+ if (inst->qpMax >= vpx_configs_[0].rc_min_quantizer) {
+ qp_max_ = inst->qpMax;
+ }
+ if (rate_control_settings_.LibvpxVp8QpMax()) {
+ qp_max_ = std::max(rate_control_settings_.LibvpxVp8QpMax().value(),
+ static_cast<int>(vpx_configs_[0].rc_min_quantizer));
+ }
+ vpx_configs_[0].rc_max_quantizer = qp_max_;
+ vpx_configs_[0].rc_undershoot_pct = 100;
+ vpx_configs_[0].rc_overshoot_pct = 15;
+ vpx_configs_[0].rc_buf_initial_sz = 500;
+ vpx_configs_[0].rc_buf_optimal_sz = 600;
+ vpx_configs_[0].rc_buf_sz = 1000;
+
+ // Set the maximum target size of any key-frame.
+ rc_max_intra_target_ = MaxIntraTarget(vpx_configs_[0].rc_buf_optimal_sz);
+
+ if (inst->VP8().keyFrameInterval > 0) {
+ vpx_configs_[0].kf_mode = VPX_KF_AUTO;
+ vpx_configs_[0].kf_max_dist = inst->VP8().keyFrameInterval;
+ } else {
+ vpx_configs_[0].kf_mode = VPX_KF_DISABLED;
+ }
+
+ // Allow the user to set the complexity for the base stream.
+ switch (inst->GetVideoEncoderComplexity()) {
+ case VideoCodecComplexity::kComplexityHigh:
+ cpu_speed_[0] = -5;
+ break;
+ case VideoCodecComplexity::kComplexityHigher:
+ cpu_speed_[0] = -4;
+ break;
+ case VideoCodecComplexity::kComplexityMax:
+ cpu_speed_[0] = -3;
+ break;
+ default:
+ cpu_speed_[0] = -6;
+ break;
+ }
+ cpu_speed_default_ = cpu_speed_[0];
+ // Set encoding complexity (cpu_speed) based on resolution and/or platform.
+ cpu_speed_[0] = GetCpuSpeed(inst->width, inst->height);
+ for (int i = 1; i < number_of_streams; ++i) {
+ cpu_speed_[i] =
+ GetCpuSpeed(inst->simulcastStream[number_of_streams - 1 - i].width,
+ inst->simulcastStream[number_of_streams - 1 - i].height);
+ }
+ vpx_configs_[0].g_w = inst->width;
+ vpx_configs_[0].g_h = inst->height;
+
+ // Determine number of threads based on the image size and #cores.
+ // TODO(fbarchard): Consider number of Simulcast layers.
+ vpx_configs_[0].g_threads = NumberOfThreads(
+ vpx_configs_[0].g_w, vpx_configs_[0].g_h, settings.number_of_cores);
+
+ // Creating a wrapper to the image - setting image data to NULL.
+ // Actual pointer will be set in encode. Setting align to 1, as it
+ // is meaningless (no memory allocation is done here).
+ libvpx_->img_wrap(&raw_images_[0], pixel_format, inst->width, inst->height, 1,
+ NULL);
+
+ // Note the order we use is different from webm, we have lowest resolution
+ // at position 0 and they have highest resolution at position 0.
+ const size_t stream_idx_cfg_0 = encoders_.size() - 1;
+ SimulcastRateAllocator init_allocator(codec_);
+ VideoBitrateAllocation allocation =
+ init_allocator.Allocate(VideoBitrateAllocationParameters(
+ inst->startBitrate * 1000, inst->maxFramerate));
+ std::vector<uint32_t> stream_bitrates;
+ for (int i = 0; i == 0 || i < inst->numberOfSimulcastStreams; ++i) {
+ uint32_t bitrate = allocation.GetSpatialLayerSum(i) / 1000;
+ stream_bitrates.push_back(bitrate);
+ }
+
+ vpx_configs_[0].rc_target_bitrate = stream_bitrates[stream_idx_cfg_0];
+ if (stream_bitrates[stream_idx_cfg_0] > 0) {
+ uint32_t maxFramerate =
+ inst->simulcastStream[stream_idx_cfg_0].maxFramerate;
+ if (!maxFramerate) {
+ maxFramerate = inst->maxFramerate;
+ }
+
+ frame_buffer_controller_->OnRatesUpdated(
+ stream_idx_cfg_0,
+ allocation.GetTemporalLayerAllocation(stream_idx_cfg_0), maxFramerate);
+ }
+ frame_buffer_controller_->SetQpLimits(stream_idx_cfg_0,
+ vpx_configs_[0].rc_min_quantizer,
+ vpx_configs_[0].rc_max_quantizer);
+ UpdateVpxConfiguration(stream_idx_cfg_0);
+ vpx_configs_[0].rc_dropframe_thresh = FrameDropThreshold(stream_idx_cfg_0);
+
+ for (size_t i = 1; i < encoders_.size(); ++i) {
+ const size_t stream_idx = encoders_.size() - 1 - i;
+ memcpy(&vpx_configs_[i], &vpx_configs_[0], sizeof(vpx_configs_[0]));
+
+ vpx_configs_[i].g_w = inst->simulcastStream[stream_idx].width;
+ vpx_configs_[i].g_h = inst->simulcastStream[stream_idx].height;
+
+ // Use 1 thread for lower resolutions.
+ vpx_configs_[i].g_threads = 1;
+
+ vpx_configs_[i].rc_dropframe_thresh = FrameDropThreshold(stream_idx);
+
+ // Setting alignment to 32 - as that ensures at least 16 for all
+ // planes (32 for Y, 16 for U,V). Libvpx sets the requested stride for
+ // the y plane, but only half of it to the u and v planes.
+ libvpx_->img_alloc(
+ &raw_images_[i], pixel_format, inst->simulcastStream[stream_idx].width,
+ inst->simulcastStream[stream_idx].height, kVp832ByteAlign);
+ SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx);
+ vpx_configs_[i].rc_target_bitrate = stream_bitrates[stream_idx];
+ if (stream_bitrates[stream_idx] > 0) {
+ uint32_t maxFramerate = inst->simulcastStream[stream_idx].maxFramerate;
+ if (!maxFramerate) {
+ maxFramerate = inst->maxFramerate;
+ }
+ frame_buffer_controller_->OnRatesUpdated(
+ stream_idx, allocation.GetTemporalLayerAllocation(stream_idx),
+ maxFramerate);
+ }
+ frame_buffer_controller_->SetQpLimits(stream_idx,
+ vpx_configs_[i].rc_min_quantizer,
+ vpx_configs_[i].rc_max_quantizer);
+ UpdateVpxConfiguration(stream_idx);
+ }
+
+ return InitAndSetControlSettings();
+}
+
+int LibvpxVp8Encoder::GetCpuSpeed(int width, int height) {
+#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
+ defined(WEBRTC_ANDROID) || defined(WEBRTC_ARCH_MIPS)
+ // On mobile platform, use a lower speed setting for lower resolutions for
+ // CPUs with 4 or more cores.
+ RTC_DCHECK_GT(number_of_cores_, 0);
+ if (experimental_cpu_speed_config_arm_
+ .GetValue(width * height, number_of_cores_)
+ .has_value()) {
+ return experimental_cpu_speed_config_arm_
+ .GetValue(width * height, number_of_cores_)
+ .value();
+ }
+
+ if (number_of_cores_ <= 3)
+ return -12;
+
+ if (width * height <= 352 * 288)
+ return -8;
+ else if (width * height <= 640 * 480)
+ return -10;
+ else
+ return -12;
+#else
+ // For non-ARM, increase encoding complexity (i.e., use lower speed setting)
+ // if resolution is below CIF. Otherwise, keep the default/user setting
+ // (`cpu_speed_default_`) set on InitEncode via VP8().complexity.
+ if (width * height < 352 * 288)
+ return (cpu_speed_default_ < -4) ? -4 : cpu_speed_default_;
+ else
+ return cpu_speed_default_;
+#endif
+}
+
+int LibvpxVp8Encoder::NumberOfThreads(int width, int height, int cpus) {
+#if defined(WEBRTC_ANDROID) || defined(WEBRTC_ARCH_MIPS)
+ if (width * height >= 320 * 180) {
+ if (cpus >= 4) {
+ // 3 threads for CPUs with 4 and more cores since most of times only 4
+ // cores will be active.
+ return 3;
+ } else if (cpus == 3 || cpus == 2) {
+ return 2;
+ } else {
+ return 1;
+ }
+ }
+ return 1;
+#else
+#if defined(WEBRTC_IOS)
+ std::string trial_string =
+ field_trial::FindFullName(kVP8IosMaxNumberOfThreadFieldTrial);
+ FieldTrialParameter<int> max_thread_number(
+ kVP8IosMaxNumberOfThreadFieldTrialParameter, 0);
+ ParseFieldTrial({&max_thread_number}, trial_string);
+ if (max_thread_number.Get() > 0) {
+ if (width * height < 320 * 180) {
+ return 1; // Use single thread for small screens
+ }
+ // thread number must be less than or equal to the number of CPUs.
+ return std::min(cpus, max_thread_number.Get());
+ }
+#endif // defined(WEBRTC_IOS)
+ if (width * height >= 1920 * 1080 && cpus > 8) {
+ return 8; // 8 threads for 1080p on high perf machines.
+ } else if (width * height > 1280 * 960 && cpus >= 6) {
+ // 3 threads for 1080p.
+ return 3;
+ } else if (width * height > 640 * 480 && cpus >= 3) {
+ // Default 2 threads for qHD/HD, but allow 3 if core count is high enough,
+ // as this will allow more margin for high-core/low clock machines or if
+ // not built with highest optimization.
+ if (cpus >= 6) {
+ return 3;
+ }
+ return 2;
+ } else {
+ // 1 thread for VGA or less.
+ return 1;
+ }
+#endif
+}
+
+int LibvpxVp8Encoder::InitAndSetControlSettings() {
+ vpx_codec_flags_t flags = 0;
+ flags |= VPX_CODEC_USE_OUTPUT_PARTITION;
+
+ if (encoders_.size() > 1) {
+ int error = libvpx_->codec_enc_init_multi(
+ &encoders_[0], vpx_codec_vp8_cx(), &vpx_configs_[0], encoders_.size(),
+ flags, &downsampling_factors_[0]);
+ if (error) {
+ return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+ }
+ } else {
+ if (libvpx_->codec_enc_init(&encoders_[0], vpx_codec_vp8_cx(),
+ &vpx_configs_[0], flags)) {
+ return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+ }
+ }
+ // Enable denoising for the highest resolution stream, and for
+ // the second highest resolution if we are doing more than 2
+ // spatial layers/streams.
+ // TODO(holmer): Investigate possibility of adding a libvpx API
+ // for getting the denoised frame from the encoder and using that
+ // when encoding lower resolution streams. Would it work with the
+ // multi-res encoding feature?
+ denoiserState denoiser_state = kDenoiserOnYOnly;
+#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
+ defined(WEBRTC_ANDROID) || defined(WEBRTC_ARCH_MIPS)
+ denoiser_state = kDenoiserOnYOnly;
+#else
+ denoiser_state = kDenoiserOnAdaptive;
+#endif
+ libvpx_->codec_control(
+ &encoders_[0], VP8E_SET_NOISE_SENSITIVITY,
+ codec_.VP8()->denoisingOn ? denoiser_state : kDenoiserOff);
+ if (encoders_.size() > 2) {
+ libvpx_->codec_control(
+ &encoders_[1], VP8E_SET_NOISE_SENSITIVITY,
+ codec_.VP8()->denoisingOn ? denoiser_state : kDenoiserOff);
+ }
+ for (size_t i = 0; i < encoders_.size(); ++i) {
+ // Allow more screen content to be detected as static.
+ libvpx_->codec_control(
+ &(encoders_[i]), VP8E_SET_STATIC_THRESHOLD,
+ codec_.mode == VideoCodecMode::kScreensharing ? 100u : 1u);
+ libvpx_->codec_control(&(encoders_[i]), VP8E_SET_CPUUSED, cpu_speed_[i]);
+ libvpx_->codec_control(
+ &(encoders_[i]), VP8E_SET_TOKEN_PARTITIONS,
+ static_cast<vp8e_token_partitions>(kTokenPartitions));
+ libvpx_->codec_control(&(encoders_[i]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
+ rc_max_intra_target_);
+ // VP8E_SET_SCREEN_CONTENT_MODE 2 = screen content with more aggressive
+ // rate control (drop frames on large target bitrate overshoot)
+ libvpx_->codec_control(
+ &(encoders_[i]), VP8E_SET_SCREEN_CONTENT_MODE,
+ codec_.mode == VideoCodecMode::kScreensharing ? 2u : 0u);
+ }
+ inited_ = true;
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+uint32_t LibvpxVp8Encoder::MaxIntraTarget(uint32_t optimalBuffersize) {
+ // Set max to the optimal buffer level (normalized by target BR),
+ // and scaled by a scalePar.
+ // Max target size = scalePar * optimalBufferSize * targetBR[Kbps].
+ // This values is presented in percentage of perFrameBw:
+ // perFrameBw = targetBR[Kbps] * 1000 / frameRate.
+ // The target in % is as follows:
+
+ float scalePar = 0.5;
+ uint32_t targetPct = optimalBuffersize * scalePar * codec_.maxFramerate / 10;
+
+ // Don't go below 3 times the per frame bandwidth.
+ const uint32_t minIntraTh = 300;
+ return (targetPct < minIntraTh) ? minIntraTh : targetPct;
+}
+
+uint32_t LibvpxVp8Encoder::FrameDropThreshold(size_t spatial_idx) const {
+ if (!codec_.GetFrameDropEnabled()) {
+ return 0;
+ }
+
+ // If temporal layers are used, they get to override the frame dropping
+ // setting, as eg. ScreenshareLayers does not work as intended with frame
+ // dropping on and DefaultTemporalLayers will have performance issues with
+ // frame dropping off.
+ RTC_DCHECK(frame_buffer_controller_);
+ RTC_DCHECK_LT(spatial_idx, frame_buffer_controller_->StreamCount());
+ return frame_buffer_controller_->SupportsEncoderFrameDropping(spatial_idx)
+ ? 30
+ : 0;
+}
+
+size_t LibvpxVp8Encoder::SteadyStateSize(int sid, int tid) {
+ const int encoder_id = encoders_.size() - 1 - sid;
+ size_t bitrate_bps;
+ float fps;
+ if ((SimulcastUtility::IsConferenceModeScreenshare(codec_) && sid == 0) ||
+ vpx_configs_[encoder_id].ts_number_layers <= 1) {
+ // In conference screenshare there's no defined per temporal layer bitrate
+ // and framerate.
+ bitrate_bps = vpx_configs_[encoder_id].rc_target_bitrate * 1000;
+ fps = codec_.maxFramerate;
+ } else {
+ bitrate_bps = vpx_configs_[encoder_id].ts_target_bitrate[tid] * 1000;
+ fps = codec_.maxFramerate /
+ fmax(vpx_configs_[encoder_id].ts_rate_decimator[tid], 1.0);
+ if (tid > 0) {
+ // Layer bitrate and fps are counted as a partial sums.
+ bitrate_bps -= vpx_configs_[encoder_id].ts_target_bitrate[tid - 1] * 1000;
+ fps = codec_.maxFramerate /
+ fmax(vpx_configs_[encoder_id].ts_rate_decimator[tid - 1], 1.0);
+ }
+ }
+
+ if (fps < 1e-9)
+ return 0;
+ return static_cast<size_t>(
+ bitrate_bps / (8 * fps) *
+ (100 -
+ variable_framerate_experiment_.steady_state_undershoot_percentage) /
+ 100 +
+ 0.5);
+}
+
+bool LibvpxVp8Encoder::UpdateVpxConfiguration(size_t stream_index) {
+ RTC_DCHECK(frame_buffer_controller_);
+
+ const size_t config_index = vpx_configs_.size() - 1 - stream_index;
+
+ RTC_DCHECK_LT(config_index, config_overrides_.size());
+ Vp8EncoderConfig* config = &config_overrides_[config_index];
+
+ const Vp8EncoderConfig new_config =
+ frame_buffer_controller_->UpdateConfiguration(stream_index);
+
+ if (new_config.reset_previous_configuration_overrides) {
+ *config = new_config;
+ return true;
+ }
+
+ const bool changes_made = MaybeExtendVp8EncoderConfig(new_config, config);
+
+ // Note that overrides must be applied even if they haven't changed.
+ RTC_DCHECK_LT(config_index, vpx_configs_.size());
+ vpx_codec_enc_cfg_t* vpx_config = &vpx_configs_[config_index];
+ ApplyVp8EncoderConfigToVpxConfig(*config, vpx_config);
+
+ return changes_made;
+}
+
+int LibvpxVp8Encoder::Encode(const VideoFrame& frame,
+ const std::vector<VideoFrameType>* frame_types) {
+ RTC_DCHECK_EQ(frame.width(), codec_.width);
+ RTC_DCHECK_EQ(frame.height(), codec_.height);
+
+ if (!inited_)
+ return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+ if (encoded_complete_callback_ == NULL)
+ return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+
+ bool key_frame_requested = false;
+ for (size_t i = 0; i < key_frame_request_.size() && i < send_stream_.size();
+ ++i) {
+ if (key_frame_request_[i] && send_stream_[i]) {
+ key_frame_requested = true;
+ break;
+ }
+ }
+ if (!key_frame_requested && frame_types) {
+ for (size_t i = 0; i < frame_types->size() && i < send_stream_.size();
+ ++i) {
+ if ((*frame_types)[i] == VideoFrameType::kVideoFrameKey &&
+ send_stream_[i]) {
+ key_frame_requested = true;
+ break;
+ }
+ }
+ }
+
+ if (frame.update_rect().IsEmpty() && num_steady_state_frames_ >= 3 &&
+ !key_frame_requested) {
+ if (variable_framerate_experiment_.enabled &&
+ framerate_controller_.DropFrame(frame.timestamp() / kRtpTicksPerMs)) {
+ return WEBRTC_VIDEO_CODEC_OK;
+ }
+ framerate_controller_.AddFrame(frame.timestamp() / kRtpTicksPerMs);
+ }
+
+ bool send_key_frame = key_frame_requested;
+ bool drop_frame = false;
+ bool retransmission_allowed = true;
+ Vp8FrameConfig tl_configs[kMaxSimulcastStreams];
+ for (size_t i = 0; i < encoders_.size(); ++i) {
+ tl_configs[i] =
+ frame_buffer_controller_->NextFrameConfig(i, frame.timestamp());
+ send_key_frame |= tl_configs[i].IntraFrame();
+ drop_frame |= tl_configs[i].drop_frame;
+ RTC_DCHECK(i == 0 ||
+ retransmission_allowed == tl_configs[i].retransmission_allowed);
+ retransmission_allowed = tl_configs[i].retransmission_allowed;
+ }
+
+ if (drop_frame && !send_key_frame) {
+ return WEBRTC_VIDEO_CODEC_OK;
+ }
+
+ vpx_enc_frame_flags_t flags[kMaxSimulcastStreams];
+ for (size_t i = 0; i < encoders_.size(); ++i) {
+ flags[i] = send_key_frame ? VPX_EFLAG_FORCE_KF : EncodeFlags(tl_configs[i]);
+ }
+
+ // Scale and map buffers and set `raw_images_` to hold pointers to the result.
+ // Because `raw_images_` are set to hold pointers to the prepared buffers, we
+ // need to keep these buffers alive through reference counting until after
+ // encoding is complete.
+ std::vector<rtc::scoped_refptr<VideoFrameBuffer>> prepared_buffers =
+ PrepareBuffers(frame.video_frame_buffer());
+ if (prepared_buffers.empty()) {
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+ struct CleanUpOnExit {
+ explicit CleanUpOnExit(
+ vpx_image_t* raw_image,
+ std::vector<rtc::scoped_refptr<VideoFrameBuffer>> prepared_buffers)
+ : raw_image_(raw_image),
+ prepared_buffers_(std::move(prepared_buffers)) {}
+ ~CleanUpOnExit() {
+ raw_image_->planes[VPX_PLANE_Y] = nullptr;
+ raw_image_->planes[VPX_PLANE_U] = nullptr;
+ raw_image_->planes[VPX_PLANE_V] = nullptr;
+ }
+ vpx_image_t* raw_image_;
+ std::vector<rtc::scoped_refptr<VideoFrameBuffer>> prepared_buffers_;
+ } clean_up_on_exit(&raw_images_[0], std::move(prepared_buffers));
+
+ if (send_key_frame) {
+ // Adapt the size of the key frame when in screenshare with 1 temporal
+ // layer.
+ if (encoders_.size() == 1 &&
+ codec_.mode == VideoCodecMode::kScreensharing &&
+ codec_.VP8()->numberOfTemporalLayers <= 1) {
+ const uint32_t forceKeyFrameIntraTh = 100;
+ libvpx_->codec_control(&(encoders_[0]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
+ forceKeyFrameIntraTh);
+ }
+
+ std::fill(key_frame_request_.begin(), key_frame_request_.end(), false);
+ }
+
+ // Set the encoder frame flags and temporal layer_id for each spatial stream.
+ // Note that streams are defined starting from lowest resolution at
+ // position 0 to highest resolution at position |encoders_.size() - 1|,
+ // whereas `encoder_` is from highest to lowest resolution.
+ for (size_t i = 0; i < encoders_.size(); ++i) {
+ const size_t stream_idx = encoders_.size() - 1 - i;
+
+ if (UpdateVpxConfiguration(stream_idx)) {
+ if (libvpx_->codec_enc_config_set(&encoders_[i], &vpx_configs_[i]))
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+
+ libvpx_->codec_control(&encoders_[i], VP8E_SET_FRAME_FLAGS,
+ static_cast<int>(flags[stream_idx]));
+ libvpx_->codec_control(&encoders_[i], VP8E_SET_TEMPORAL_LAYER_ID,
+ tl_configs[i].encoder_layer_id);
+ }
+ // TODO(holmer): Ideally the duration should be the timestamp diff of this
+ // frame and the next frame to be encoded, which we don't have. Instead we
+ // would like to use the duration of the previous frame. Unfortunately the
+ // rate control seems to be off with that setup. Using the average input
+ // frame rate to calculate an average duration for now.
+ RTC_DCHECK_GT(codec_.maxFramerate, 0);
+ uint32_t duration = kRtpTicksPerSecond / codec_.maxFramerate;
+
+ int error = WEBRTC_VIDEO_CODEC_OK;
+ int num_tries = 0;
+ // If the first try returns WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT
+ // the frame must be reencoded with the same parameters again because
+ // target bitrate is exceeded and encoder state has been reset.
+ while (num_tries == 0 ||
+ (num_tries == 1 &&
+ error == WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT)) {
+ ++num_tries;
+ // Note we must pass 0 for `flags` field in encode call below since they are
+ // set above in `libvpx_interface_->vpx_codec_control_` function for each
+ // encoder/spatial layer.
+ error = libvpx_->codec_encode(&encoders_[0], &raw_images_[0], timestamp_,
+ duration, 0, VPX_DL_REALTIME);
+ // Reset specific intra frame thresholds, following the key frame.
+ if (send_key_frame) {
+ libvpx_->codec_control(&(encoders_[0]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
+ rc_max_intra_target_);
+ }
+ if (error)
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ // Examines frame timestamps only.
+ error = GetEncodedPartitions(frame, retransmission_allowed);
+ }
+ // TODO(sprang): Shouldn't we use the frame timestamp instead?
+ timestamp_ += duration;
+ return error;
+}
+
+void LibvpxVp8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
+ const vpx_codec_cx_pkt_t& pkt,
+ int stream_idx,
+ int encoder_idx,
+ uint32_t timestamp) {
+ RTC_DCHECK(codec_specific);
+ codec_specific->codecType = kVideoCodecVP8;
+ codec_specific->codecSpecific.VP8.keyIdx =
+ kNoKeyIdx; // TODO(hlundin) populate this
+ codec_specific->codecSpecific.VP8.nonReference =
+ (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) != 0;
+
+ int qp = 0;
+ vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp);
+ bool is_keyframe = (pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0;
+ frame_buffer_controller_->OnEncodeDone(stream_idx, timestamp,
+ encoded_images_[encoder_idx].size(),
+ is_keyframe, qp, codec_specific);
+ if (is_keyframe && codec_specific->template_structure != absl::nullopt) {
+ // Number of resolutions must match number of spatial layers, VP8 structures
+ // expected to use single spatial layer. Templates must be ordered by
+ // spatial_id, so assumption there is exactly one spatial layer is same as
+ // assumption last template uses spatial_id = 0.
+ // This check catches potential scenario where template_structure is shared
+ // across multiple vp8 streams and they are distinguished using spatial_id.
+ // Assigning single resolution doesn't support such scenario, i.e. assumes
+ // vp8 simulcast is sent using multiple ssrcs.
+ RTC_DCHECK(!codec_specific->template_structure->templates.empty());
+ RTC_DCHECK_EQ(
+ codec_specific->template_structure->templates.back().spatial_id, 0);
+ codec_specific->template_structure->resolutions = {
+ RenderResolution(pkt.data.frame.width[0], pkt.data.frame.height[0])};
+ }
+ switch (vpx_configs_[encoder_idx].ts_number_layers) {
+ case 1:
+ codec_specific->scalability_mode = ScalabilityMode::kL1T1;
+ break;
+ case 2:
+ codec_specific->scalability_mode = ScalabilityMode::kL1T2;
+ break;
+ case 3:
+ codec_specific->scalability_mode = ScalabilityMode::kL1T3;
+ break;
+ }
+}
+
+int LibvpxVp8Encoder::GetEncodedPartitions(const VideoFrame& input_image,
+ bool retransmission_allowed) {
+ int stream_idx = static_cast<int>(encoders_.size()) - 1;
+ int result = WEBRTC_VIDEO_CODEC_OK;
+ for (size_t encoder_idx = 0; encoder_idx < encoders_.size();
+ ++encoder_idx, --stream_idx) {
+ vpx_codec_iter_t iter = NULL;
+ encoded_images_[encoder_idx].set_size(0);
+ encoded_images_[encoder_idx]._frameType = VideoFrameType::kVideoFrameDelta;
+ CodecSpecificInfo codec_specific;
+ const vpx_codec_cx_pkt_t* pkt = NULL;
+
+ size_t encoded_size = 0;
+ while ((pkt = libvpx_->codec_get_cx_data(&encoders_[encoder_idx], &iter)) !=
+ NULL) {
+ if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) {
+ encoded_size += pkt->data.frame.sz;
+ }
+ }
+
+ auto buffer = EncodedImageBuffer::Create(encoded_size);
+
+ iter = NULL;
+ size_t encoded_pos = 0;
+ while ((pkt = libvpx_->codec_get_cx_data(&encoders_[encoder_idx], &iter)) !=
+ NULL) {
+ switch (pkt->kind) {
+ case VPX_CODEC_CX_FRAME_PKT: {
+ RTC_CHECK_LE(encoded_pos + pkt->data.frame.sz, buffer->size());
+ memcpy(&buffer->data()[encoded_pos], pkt->data.frame.buf,
+ pkt->data.frame.sz);
+ encoded_pos += pkt->data.frame.sz;
+ break;
+ }
+ default:
+ break;
+ }
+ // End of frame
+ if ((pkt->data.frame.flags & VPX_FRAME_IS_FRAGMENT) == 0) {
+ // check if encoded frame is a key frame
+ if (pkt->data.frame.flags & VPX_FRAME_IS_KEY) {
+ encoded_images_[encoder_idx]._frameType =
+ VideoFrameType::kVideoFrameKey;
+ }
+ encoded_images_[encoder_idx].SetEncodedData(buffer);
+ encoded_images_[encoder_idx].set_size(encoded_pos);
+ encoded_images_[encoder_idx].SetSpatialIndex(stream_idx);
+ PopulateCodecSpecific(&codec_specific, *pkt, stream_idx, encoder_idx,
+ input_image.timestamp());
+ if (codec_specific.codecSpecific.VP8.temporalIdx != kNoTemporalIdx) {
+ encoded_images_[encoder_idx].SetTemporalIndex(
+ codec_specific.codecSpecific.VP8.temporalIdx);
+ }
+ break;
+ }
+ }
+ encoded_images_[encoder_idx].SetTimestamp(input_image.timestamp());
+ encoded_images_[encoder_idx].SetColorSpace(input_image.color_space());
+ encoded_images_[encoder_idx].SetRetransmissionAllowed(
+ retransmission_allowed);
+
+ if (send_stream_[stream_idx]) {
+ if (encoded_images_[encoder_idx].size() > 0) {
+ TRACE_COUNTER_ID1("webrtc", "EncodedFrameSize", encoder_idx,
+ encoded_images_[encoder_idx].size());
+ encoded_images_[encoder_idx]._encodedHeight =
+ codec_.simulcastStream[stream_idx].height;
+ encoded_images_[encoder_idx]._encodedWidth =
+ codec_.simulcastStream[stream_idx].width;
+ int qp_128 = -1;
+ libvpx_->codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER,
+ &qp_128);
+ encoded_images_[encoder_idx].qp_ = qp_128;
+ encoded_complete_callback_->OnEncodedImage(encoded_images_[encoder_idx],
+ &codec_specific);
+ const size_t steady_state_size = SteadyStateSize(
+ stream_idx, codec_specific.codecSpecific.VP8.temporalIdx);
+ if (qp_128 > variable_framerate_experiment_.steady_state_qp ||
+ encoded_images_[encoder_idx].size() > steady_state_size) {
+ num_steady_state_frames_ = 0;
+ } else {
+ ++num_steady_state_frames_;
+ }
+ } else if (!frame_buffer_controller_->SupportsEncoderFrameDropping(
+ stream_idx)) {
+ result = WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT;
+ if (encoded_images_[encoder_idx].size() == 0) {
+ // Dropped frame that will be re-encoded.
+ frame_buffer_controller_->OnFrameDropped(stream_idx,
+ input_image.timestamp());
+ }
+ }
+ }
+ }
+ return result;
+}
+
+VideoEncoder::EncoderInfo LibvpxVp8Encoder::GetEncoderInfo() const {
+ EncoderInfo info;
+ info.supports_native_handle = false;
+ info.implementation_name = "libvpx";
+ info.has_trusted_rate_controller =
+ rate_control_settings_.LibvpxVp8TrustedRateController();
+ info.is_hardware_accelerated = false;
+ info.supports_simulcast = true;
+ if (!resolution_bitrate_limits_.empty()) {
+ info.resolution_bitrate_limits = resolution_bitrate_limits_;
+ }
+ if (encoder_info_override_.requested_resolution_alignment()) {
+ info.requested_resolution_alignment =
+ *encoder_info_override_.requested_resolution_alignment();
+ info.apply_alignment_to_all_simulcast_layers =
+ encoder_info_override_.apply_alignment_to_all_simulcast_layers();
+ }
+ if (!encoder_info_override_.resolution_bitrate_limits().empty()) {
+ info.resolution_bitrate_limits =
+ encoder_info_override_.resolution_bitrate_limits();
+ }
+
+ const bool enable_scaling =
+ num_active_streams_ == 1 &&
+ (vpx_configs_.empty() || vpx_configs_[0].rc_dropframe_thresh > 0) &&
+ codec_.VP8().automaticResizeOn;
+
+ info.scaling_settings = enable_scaling
+ ? VideoEncoder::ScalingSettings(
+ kLowVp8QpThreshold, kHighVp8QpThreshold)
+ : VideoEncoder::ScalingSettings::kOff;
+ if (rate_control_settings_.LibvpxVp8MinPixels()) {
+ info.scaling_settings.min_pixels_per_frame =
+ rate_control_settings_.LibvpxVp8MinPixels().value();
+ }
+ info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420,
+ VideoFrameBuffer::Type::kNV12};
+
+ if (inited_) {
+ // `encoder_idx` is libvpx index where 0 is highest resolution.
+ // `si` is simulcast index, where 0 is lowest resolution.
+ for (size_t si = 0, encoder_idx = encoders_.size() - 1;
+ si < encoders_.size(); ++si, --encoder_idx) {
+ info.fps_allocation[si].clear();
+ if ((codec_.numberOfSimulcastStreams > si &&
+ !codec_.simulcastStream[si].active) ||
+ (si == 0 && SimulcastUtility::IsConferenceModeScreenshare(codec_))) {
+ // No defined frame rate fractions if not active or if using
+ // ScreenshareLayers, leave vector empty and continue;
+ continue;
+ }
+ if (vpx_configs_[encoder_idx].ts_number_layers <= 1) {
+ info.fps_allocation[si].push_back(EncoderInfo::kMaxFramerateFraction);
+ } else {
+ for (size_t ti = 0; ti < vpx_configs_[encoder_idx].ts_number_layers;
+ ++ti) {
+ RTC_DCHECK_GT(vpx_configs_[encoder_idx].ts_rate_decimator[ti], 0);
+ info.fps_allocation[si].push_back(rtc::saturated_cast<uint8_t>(
+ EncoderInfo::kMaxFramerateFraction /
+ vpx_configs_[encoder_idx].ts_rate_decimator[ti] +
+ 0.5));
+ }
+ }
+ }
+ }
+
+ return info;
+}
+
+int LibvpxVp8Encoder::RegisterEncodeCompleteCallback(
+ EncodedImageCallback* callback) {
+ encoded_complete_callback_ = callback;
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+void LibvpxVp8Encoder::MaybeUpdatePixelFormat(vpx_img_fmt fmt) {
+ RTC_DCHECK(!raw_images_.empty());
+ if (raw_images_[0].fmt == fmt) {
+ RTC_DCHECK(std::all_of(
+ std::next(raw_images_.begin()), raw_images_.end(),
+ [fmt](const vpx_image_t& raw_img) { return raw_img.fmt == fmt; }))
+ << "Not all raw images had the right format!";
+ return;
+ }
+ RTC_LOG(LS_INFO) << "Updating vp8 encoder pixel format to "
+ << (fmt == VPX_IMG_FMT_NV12 ? "NV12" : "I420");
+ for (size_t i = 0; i < raw_images_.size(); ++i) {
+ vpx_image_t& img = raw_images_[i];
+ auto d_w = img.d_w;
+ auto d_h = img.d_h;
+ libvpx_->img_free(&img);
+ // First image is wrapping the input frame, the rest are allocated.
+ if (i == 0) {
+ libvpx_->img_wrap(&img, fmt, d_w, d_h, 1, NULL);
+ } else {
+ libvpx_->img_alloc(&img, fmt, d_w, d_h, kVp832ByteAlign);
+ }
+ }
+}
+
+std::vector<rtc::scoped_refptr<VideoFrameBuffer>>
+LibvpxVp8Encoder::PrepareBuffers(rtc::scoped_refptr<VideoFrameBuffer> buffer) {
+ RTC_DCHECK_EQ(buffer->width(), raw_images_[0].d_w);
+ RTC_DCHECK_EQ(buffer->height(), raw_images_[0].d_h);
+ absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
+ supported_formats = {VideoFrameBuffer::Type::kI420,
+ VideoFrameBuffer::Type::kNV12};
+
+ rtc::scoped_refptr<VideoFrameBuffer> mapped_buffer;
+ if (buffer->type() != VideoFrameBuffer::Type::kNative) {
+ // `buffer` is already mapped.
+ mapped_buffer = buffer;
+ } else {
+ // Attempt to map to one of the supported formats.
+ mapped_buffer = buffer->GetMappedFrameBuffer(supported_formats);
+ }
+ if (!mapped_buffer ||
+ (absl::c_find(supported_formats, mapped_buffer->type()) ==
+ supported_formats.end() &&
+ mapped_buffer->type() != VideoFrameBuffer::Type::kI420A)) {
+ // Unknown pixel format or unable to map, convert to I420 and prepare that
+ // buffer instead to ensure Scale() is safe to use.
+ auto converted_buffer = buffer->ToI420();
+ if (!converted_buffer) {
+ RTC_LOG(LS_ERROR) << "Failed to convert "
+ << VideoFrameBufferTypeToString(buffer->type())
+ << " image to I420. Can't encode frame.";
+ return {};
+ }
+ RTC_CHECK(converted_buffer->type() == VideoFrameBuffer::Type::kI420 ||
+ converted_buffer->type() == VideoFrameBuffer::Type::kI420A);
+
+ // Because `buffer` had to be converted, use `converted_buffer` instead...
+ buffer = mapped_buffer = converted_buffer;
+ }
+
+ // Maybe update pixel format.
+ absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
+ mapped_type = {mapped_buffer->type()};
+ switch (mapped_buffer->type()) {
+ case VideoFrameBuffer::Type::kI420:
+ case VideoFrameBuffer::Type::kI420A:
+ MaybeUpdatePixelFormat(VPX_IMG_FMT_I420);
+ break;
+ case VideoFrameBuffer::Type::kNV12:
+ MaybeUpdatePixelFormat(VPX_IMG_FMT_NV12);
+ break;
+ default:
+ RTC_DCHECK_NOTREACHED();
+ }
+
+ // Prepare `raw_images_` from `mapped_buffer` and, if simulcast, scaled
+ // versions of `buffer`.
+ std::vector<rtc::scoped_refptr<VideoFrameBuffer>> prepared_buffers;
+ SetRawImagePlanes(&raw_images_[0], mapped_buffer.get());
+ prepared_buffers.push_back(mapped_buffer);
+ for (size_t i = 1; i < encoders_.size(); ++i) {
+ // Native buffers should implement optimized scaling and is the preferred
+ // buffer to scale. But if the buffer isn't native, it should be cheaper to
+ // scale from the previously prepared buffer which is smaller than `buffer`.
+ VideoFrameBuffer* buffer_to_scale =
+ buffer->type() == VideoFrameBuffer::Type::kNative
+ ? buffer.get()
+ : prepared_buffers.back().get();
+
+ auto scaled_buffer =
+ buffer_to_scale->Scale(raw_images_[i].d_w, raw_images_[i].d_h);
+ if (scaled_buffer->type() == VideoFrameBuffer::Type::kNative) {
+ auto mapped_scaled_buffer =
+ scaled_buffer->GetMappedFrameBuffer(mapped_type);
+ RTC_DCHECK(mapped_scaled_buffer) << "Unable to map the scaled buffer.";
+ if (!mapped_scaled_buffer) {
+ RTC_LOG(LS_ERROR) << "Failed to map scaled "
+ << VideoFrameBufferTypeToString(scaled_buffer->type())
+ << " image to "
+ << VideoFrameBufferTypeToString(mapped_buffer->type())
+ << ". Can't encode frame.";
+ return {};
+ }
+ scaled_buffer = mapped_scaled_buffer;
+ }
+ if (!IsCompatibleVideoFrameBufferType(scaled_buffer->type(),
+ mapped_buffer->type())) {
+ RTC_LOG(LS_ERROR) << "When scaling "
+ << VideoFrameBufferTypeToString(buffer_to_scale->type())
+ << ", the image was unexpectedly converted to "
+ << VideoFrameBufferTypeToString(scaled_buffer->type())
+ << " instead of "
+ << VideoFrameBufferTypeToString(mapped_buffer->type())
+ << ". Can't encode frame.";
+ RTC_DCHECK_NOTREACHED()
+ << "Scaled buffer type "
+ << VideoFrameBufferTypeToString(scaled_buffer->type())
+ << " is not compatible with mapped buffer type "
+ << VideoFrameBufferTypeToString(mapped_buffer->type());
+ return {};
+ }
+ SetRawImagePlanes(&raw_images_[i], scaled_buffer.get());
+ prepared_buffers.push_back(scaled_buffer);
+ }
+ return prepared_buffers;
+}
+
+// static
+LibvpxVp8Encoder::VariableFramerateExperiment
+LibvpxVp8Encoder::ParseVariableFramerateConfig(std::string group_name) {
+ FieldTrialFlag disabled = FieldTrialFlag("Disabled");
+ FieldTrialParameter<double> framerate_limit("min_fps", 5.0);
+ FieldTrialParameter<int> qp("min_qp", 15);
+ FieldTrialParameter<int> undershoot_percentage("undershoot", 30);
+ ParseFieldTrial({&disabled, &framerate_limit, &qp, &undershoot_percentage},
+ field_trial::FindFullName(group_name));
+ VariableFramerateExperiment config;
+ config.enabled = !disabled.Get();
+ config.framerate_limit = framerate_limit.Get();
+ config.steady_state_qp = qp.Get();
+ config.steady_state_undershoot_percentage = undershoot_percentage.Get();
+
+ return config;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h
new file mode 100644
index 0000000000..74477eac7e
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h
@@ -0,0 +1,159 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_ENCODER_H_
+#define MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_ENCODER_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "api/fec_controller_override.h"
+#include "api/video/encoded_image.h"
+#include "api/video/video_frame.h"
+#include "api/video_codecs/video_encoder.h"
+#include "api/video_codecs/vp8_frame_buffer_controller.h"
+#include "api/video_codecs/vp8_frame_config.h"
+#include "modules/video_coding/codecs/interface/libvpx_interface.h"
+#include "modules/video_coding/codecs/vp8/include/vp8.h"
+#include "modules/video_coding/include/video_codec_interface.h"
+#include "modules/video_coding/utility/framerate_controller_deprecated.h"
+#include "modules/video_coding/utility/vp8_constants.h"
+#include "rtc_base/experiments/cpu_speed_experiment.h"
+#include "rtc_base/experiments/encoder_info_settings.h"
+#include "rtc_base/experiments/rate_control_settings.h"
+#include "vpx/vp8cx.h"
+#include "vpx/vpx_encoder.h"
+
+namespace webrtc {
+
+class LibvpxVp8Encoder : public VideoEncoder {
+ public:
+ LibvpxVp8Encoder(std::unique_ptr<LibvpxInterface> interface,
+ VP8Encoder::Settings settings);
+ ~LibvpxVp8Encoder() override;
+
+ int Release() override;
+
+ void SetFecControllerOverride(
+ FecControllerOverride* fec_controller_override) override;
+
+ int InitEncode(const VideoCodec* codec_settings,
+ const VideoEncoder::Settings& settings) override;
+
+ int Encode(const VideoFrame& input_image,
+ const std::vector<VideoFrameType>* frame_types) override;
+
+ int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override;
+
+ void SetRates(const RateControlParameters& parameters) override;
+
+ void OnPacketLossRateUpdate(float packet_loss_rate) override;
+
+ void OnRttUpdate(int64_t rtt_ms) override;
+
+ void OnLossNotification(const LossNotification& loss_notification) override;
+
+ EncoderInfo GetEncoderInfo() const override;
+
+ static vpx_enc_frame_flags_t EncodeFlags(const Vp8FrameConfig& references);
+
+ private:
+ // Get the cpu_speed setting for encoder based on resolution and/or platform.
+ int GetCpuSpeed(int width, int height);
+
+ // Determine number of encoder threads to use.
+ int NumberOfThreads(int width, int height, int number_of_cores);
+
+ // Call encoder initialize function and set control settings.
+ int InitAndSetControlSettings();
+
+ void PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
+ const vpx_codec_cx_pkt& pkt,
+ int stream_idx,
+ int encoder_idx,
+ uint32_t timestamp);
+
+ int GetEncodedPartitions(const VideoFrame& input_image,
+ bool retransmission_allowed);
+
+ // Set the stream state for stream `stream_idx`.
+ void SetStreamState(bool send_stream, int stream_idx);
+
+ uint32_t MaxIntraTarget(uint32_t optimal_buffer_size);
+
+ uint32_t FrameDropThreshold(size_t spatial_idx) const;
+
+ size_t SteadyStateSize(int sid, int tid);
+
+ bool UpdateVpxConfiguration(size_t stream_index);
+
+ void MaybeUpdatePixelFormat(vpx_img_fmt fmt);
+ // Prepares `raw_image_` to reference image data of `buffer`, or of mapped or
+ // scaled versions of `buffer`. Returns a list of buffers that got referenced
+ // as a result, allowing the caller to keep references to them until after
+ // encoding has finished. On failure to convert the buffer, an empty list is
+ // returned.
+ std::vector<rtc::scoped_refptr<VideoFrameBuffer>> PrepareBuffers(
+ rtc::scoped_refptr<VideoFrameBuffer> buffer);
+
+ const std::unique_ptr<LibvpxInterface> libvpx_;
+
+ const CpuSpeedExperiment experimental_cpu_speed_config_arm_;
+ const RateControlSettings rate_control_settings_;
+
+ EncodedImageCallback* encoded_complete_callback_ = nullptr;
+ VideoCodec codec_;
+ bool inited_ = false;
+ int64_t timestamp_ = 0;
+ int qp_max_ = 56;
+ int cpu_speed_default_ = -6;
+ int number_of_cores_ = 0;
+ uint32_t rc_max_intra_target_ = 0;
+ int num_active_streams_ = 0;
+ const std::unique_ptr<Vp8FrameBufferControllerFactory>
+ frame_buffer_controller_factory_;
+ std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_;
+ const std::vector<VideoEncoder::ResolutionBitrateLimits>
+ resolution_bitrate_limits_;
+ std::vector<bool> key_frame_request_;
+ std::vector<bool> send_stream_;
+ std::vector<int> cpu_speed_;
+ std::vector<vpx_image_t> raw_images_;
+ std::vector<EncodedImage> encoded_images_;
+ std::vector<vpx_codec_ctx_t> encoders_;
+ std::vector<vpx_codec_enc_cfg_t> vpx_configs_;
+ std::vector<Vp8EncoderConfig> config_overrides_;
+ std::vector<vpx_rational_t> downsampling_factors_;
+
+ // Variable frame-rate screencast related fields and methods.
+ const struct VariableFramerateExperiment {
+ bool enabled = false;
+ // Framerate is limited to this value in steady state.
+ float framerate_limit = 5.0;
+ // This qp or below is considered a steady state.
+ int steady_state_qp = kVp8SteadyStateQpThreshold;
+ // Frames of at least this percentage below ideal for configured bitrate are
+ // considered in a steady state.
+ int steady_state_undershoot_percentage = 30;
+ } variable_framerate_experiment_;
+ static VariableFramerateExperiment ParseVariableFramerateConfig(
+ std::string group_name);
+ FramerateControllerDeprecated framerate_controller_;
+ int num_steady_state_frames_ = 0;
+
+ FecControllerOverride* fec_controller_override_ = nullptr;
+
+ const LibvpxVp8EncoderInfoSettings encoder_info_override_;
+};
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_ENCODER_H_
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_simulcast_test.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_simulcast_test.cc
new file mode 100644
index 0000000000..4ca3de20d5
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_simulcast_test.cc
@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <memory>
+
+#include "api/test/create_simulcast_test_fixture.h"
+#include "api/test/simulcast_test_fixture.h"
+#include "api/test/video/function_video_decoder_factory.h"
+#include "api/test/video/function_video_encoder_factory.h"
+#include "modules/video_coding/codecs/vp8/include/vp8.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace test {
+
+namespace {
+std::unique_ptr<SimulcastTestFixture> CreateSpecificSimulcastTestFixture() {
+ std::unique_ptr<VideoEncoderFactory> encoder_factory =
+ std::make_unique<FunctionVideoEncoderFactory>(
+ []() { return VP8Encoder::Create(); });
+ std::unique_ptr<VideoDecoderFactory> decoder_factory =
+ std::make_unique<FunctionVideoDecoderFactory>(
+ []() { return VP8Decoder::Create(); });
+ return CreateSimulcastTestFixture(std::move(encoder_factory),
+ std::move(decoder_factory),
+ SdpVideoFormat("VP8"));
+}
+} // namespace
+
+TEST(LibvpxVp8SimulcastTest, TestKeyFrameRequestsOnAllStreams) {
+ auto fixture = CreateSpecificSimulcastTestFixture();
+ fixture->TestKeyFrameRequestsOnAllStreams();
+}
+
+TEST(LibvpxVp8SimulcastTest, TestKeyFrameRequestsOnSpecificStreams) {
+ GTEST_SKIP() << "Not applicable to VP8.";
+}
+
+TEST(LibvpxVp8SimulcastTest, TestPaddingAllStreams) {
+ auto fixture = CreateSpecificSimulcastTestFixture();
+ fixture->TestPaddingAllStreams();
+}
+
+TEST(LibvpxVp8SimulcastTest, TestPaddingTwoStreams) {
+ auto fixture = CreateSpecificSimulcastTestFixture();
+ fixture->TestPaddingTwoStreams();
+}
+
+TEST(LibvpxVp8SimulcastTest, TestPaddingTwoStreamsOneMaxedOut) {
+ auto fixture = CreateSpecificSimulcastTestFixture();
+ fixture->TestPaddingTwoStreamsOneMaxedOut();
+}
+
+TEST(LibvpxVp8SimulcastTest, TestPaddingOneStream) {
+ auto fixture = CreateSpecificSimulcastTestFixture();
+ fixture->TestPaddingOneStream();
+}
+
+TEST(LibvpxVp8SimulcastTest, TestPaddingOneStreamTwoMaxedOut) {
+ auto fixture = CreateSpecificSimulcastTestFixture();
+ fixture->TestPaddingOneStreamTwoMaxedOut();
+}
+
+TEST(LibvpxVp8SimulcastTest, TestSendAllStreams) {
+ auto fixture = CreateSpecificSimulcastTestFixture();
+ fixture->TestSendAllStreams();
+}
+
+TEST(LibvpxVp8SimulcastTest, TestDisablingStreams) {
+ auto fixture = CreateSpecificSimulcastTestFixture();
+ fixture->TestDisablingStreams();
+}
+
+TEST(LibvpxVp8SimulcastTest, TestActiveStreams) {
+ auto fixture = CreateSpecificSimulcastTestFixture();
+ fixture->TestActiveStreams();
+}
+
+TEST(LibvpxVp8SimulcastTest, TestSwitchingToOneStream) {
+ auto fixture = CreateSpecificSimulcastTestFixture();
+ fixture->TestSwitchingToOneStream();
+}
+
+TEST(LibvpxVp8SimulcastTest, TestSwitchingToOneOddStream) {
+ auto fixture = CreateSpecificSimulcastTestFixture();
+ fixture->TestSwitchingToOneOddStream();
+}
+
+TEST(LibvpxVp8SimulcastTest, TestSwitchingToOneSmallStream) {
+ auto fixture = CreateSpecificSimulcastTestFixture();
+ fixture->TestSwitchingToOneSmallStream();
+}
+
+TEST(LibvpxVp8SimulcastTest, TestSpatioTemporalLayers333PatternEncoder) {
+ auto fixture = CreateSpecificSimulcastTestFixture();
+ fixture->TestSpatioTemporalLayers333PatternEncoder();
+}
+
+TEST(LibvpxVp8SimulcastTest, TestStrideEncodeDecode) {
+ auto fixture = CreateSpecificSimulcastTestFixture();
+ fixture->TestStrideEncodeDecode();
+}
+
+} // namespace test
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc
new file mode 100644
index 0000000000..71db0b22c2
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc
@@ -0,0 +1,624 @@
+/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/vp8/screenshare_layers.h"
+
+#include <stdlib.h>
+
+#include <algorithm>
+#include <memory>
+
+#include "modules/video_coding/include/video_codec_interface.h"
+#include "rtc_base/arraysize.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/time_utils.h"
+#include "system_wrappers/include/metrics.h"
+
+namespace webrtc {
+namespace {
+using BufferFlags = Vp8FrameConfig::BufferFlags;
+
+constexpr BufferFlags kNone = Vp8FrameConfig::BufferFlags::kNone;
+constexpr BufferFlags kReference = Vp8FrameConfig::BufferFlags::kReference;
+constexpr BufferFlags kUpdate = Vp8FrameConfig::BufferFlags::kUpdate;
+constexpr BufferFlags kReferenceAndUpdate =
+ Vp8FrameConfig::BufferFlags::kReferenceAndUpdate;
+
+constexpr int kOneSecond90Khz = 90000;
+constexpr int kMinTimeBetweenSyncs = kOneSecond90Khz * 2;
+constexpr int kMaxTimeBetweenSyncs = kOneSecond90Khz * 4;
+constexpr int kQpDeltaThresholdForSync = 8;
+constexpr int kMinBitrateKbpsForQpBoost = 500;
+constexpr auto kSwitch = DecodeTargetIndication::kSwitch;
+} // namespace
+
+const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5;
+const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0;
+
+constexpr int ScreenshareLayers::kMaxNumTemporalLayers;
+
+// Always emit a frame with certain interval, even if bitrate targets have
+// been exceeded. This prevents needless keyframe requests.
+const int ScreenshareLayers::kMaxFrameIntervalMs = 2750;
+
+ScreenshareLayers::ScreenshareLayers(int num_temporal_layers)
+ : number_of_temporal_layers_(
+ std::min(kMaxNumTemporalLayers, num_temporal_layers)),
+ active_layer_(-1),
+ last_timestamp_(-1),
+ last_sync_timestamp_(-1),
+ last_emitted_tl0_timestamp_(-1),
+ last_frame_time_ms_(-1),
+ max_debt_bytes_(0),
+ encode_framerate_(1000.0f, 1000.0f), // 1 second window, second scale.
+ bitrate_updated_(false),
+ checker_(TemporalLayersChecker::CreateTemporalLayersChecker(
+ Vp8TemporalLayersType::kBitrateDynamic,
+ num_temporal_layers)) {
+ RTC_CHECK_GT(number_of_temporal_layers_, 0);
+ RTC_CHECK_LE(number_of_temporal_layers_, kMaxNumTemporalLayers);
+}
+
+ScreenshareLayers::~ScreenshareLayers() {
+ UpdateHistograms();
+}
+
+void ScreenshareLayers::SetQpLimits(size_t stream_index,
+ int min_qp,
+ int max_qp) {
+ RTC_DCHECK_LT(stream_index, StreamCount());
+ // 0 < min_qp <= max_qp
+ RTC_DCHECK_LT(0, min_qp);
+ RTC_DCHECK_LE(min_qp, max_qp);
+
+ RTC_DCHECK_EQ(min_qp_.has_value(), max_qp_.has_value());
+ if (!min_qp_.has_value()) {
+ min_qp_ = min_qp;
+ max_qp_ = max_qp;
+ } else {
+ RTC_DCHECK_EQ(min_qp, min_qp_.value());
+ RTC_DCHECK_EQ(max_qp, max_qp_.value());
+ }
+}
+
+size_t ScreenshareLayers::StreamCount() const {
+ return 1;
+}
+
+bool ScreenshareLayers::SupportsEncoderFrameDropping(
+ size_t stream_index) const {
+ RTC_DCHECK_LT(stream_index, StreamCount());
+ // Frame dropping is handled internally by this class.
+ return false;
+}
+
+Vp8FrameConfig ScreenshareLayers::NextFrameConfig(size_t stream_index,
+ uint32_t timestamp) {
+ RTC_DCHECK_LT(stream_index, StreamCount());
+
+ auto it = pending_frame_configs_.find(timestamp);
+ if (it != pending_frame_configs_.end()) {
+ // Drop and re-encode, reuse the previous config.
+ return it->second.frame_config;
+ }
+
+ if (number_of_temporal_layers_ <= 1) {
+ // No flags needed for 1 layer screenshare.
+ // TODO(pbos): Consider updating only last, and not all buffers.
+ DependencyInfo dependency_info{
+ "S", {kReferenceAndUpdate, kReferenceAndUpdate, kReferenceAndUpdate}};
+ pending_frame_configs_[timestamp] = dependency_info;
+ return dependency_info.frame_config;
+ }
+
+ const int64_t now_ms = rtc::TimeMillis();
+
+ int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
+ int64_t ts_diff;
+ if (last_timestamp_ == -1) {
+ ts_diff = kOneSecond90Khz / capture_framerate_.value_or(*target_framerate_);
+ } else {
+ ts_diff = unwrapped_timestamp - last_timestamp_;
+ }
+
+ if (target_framerate_) {
+ // If input frame rate exceeds target frame rate, either over a one second
+ // averaging window, or if frame interval is below 90% of desired value,
+ // drop frame.
+ if (encode_framerate_.Rate(now_ms).value_or(0) > *target_framerate_)
+ return Vp8FrameConfig(kNone, kNone, kNone);
+
+ // Primarily check if frame interval is too short using frame timestamps,
+ // as if they are correct they won't be affected by queuing in webrtc.
+ const int64_t expected_frame_interval_90khz =
+ kOneSecond90Khz / *target_framerate_;
+ if (last_timestamp_ != -1 && ts_diff > 0) {
+ if (ts_diff < 85 * expected_frame_interval_90khz / 100) {
+ return Vp8FrameConfig(kNone, kNone, kNone);
+ }
+ } else {
+ // Timestamps looks off, use realtime clock here instead.
+ const int64_t expected_frame_interval_ms = 1000 / *target_framerate_;
+ if (last_frame_time_ms_ != -1 &&
+ now_ms - last_frame_time_ms_ <
+ (85 * expected_frame_interval_ms) / 100) {
+ return Vp8FrameConfig(kNone, kNone, kNone);
+ }
+ }
+ }
+
+ if (stats_.first_frame_time_ms_ == -1)
+ stats_.first_frame_time_ms_ = now_ms;
+
+ // Make sure both frame droppers leak out bits.
+ layers_[0].UpdateDebt(ts_diff / 90);
+ layers_[1].UpdateDebt(ts_diff / 90);
+ last_timestamp_ = timestamp;
+ last_frame_time_ms_ = now_ms;
+
+ TemporalLayerState layer_state = TemporalLayerState::kDrop;
+
+ if (active_layer_ == -1 ||
+ layers_[active_layer_].state != TemporalLayer::State::kDropped) {
+ if (last_emitted_tl0_timestamp_ != -1 &&
+ (unwrapped_timestamp - last_emitted_tl0_timestamp_) / 90 >
+ kMaxFrameIntervalMs) {
+ // Too long time has passed since the last frame was emitted, cancel
+ // enough debt to allow a single frame.
+ layers_[0].debt_bytes_ = max_debt_bytes_ - 1;
+ }
+ if (layers_[0].debt_bytes_ > max_debt_bytes_) {
+ // Must drop TL0, encode TL1 instead.
+ if (layers_[1].debt_bytes_ > max_debt_bytes_) {
+ // Must drop both TL0 and TL1.
+ active_layer_ = -1;
+ } else {
+ active_layer_ = 1;
+ }
+ } else {
+ active_layer_ = 0;
+ }
+ }
+
+ switch (active_layer_) {
+ case 0:
+ layer_state = TemporalLayerState::kTl0;
+ last_emitted_tl0_timestamp_ = unwrapped_timestamp;
+ break;
+ case 1:
+ if (layers_[1].state != TemporalLayer::State::kDropped) {
+ if (TimeToSync(unwrapped_timestamp) ||
+ layers_[1].state == TemporalLayer::State::kKeyFrame) {
+ last_sync_timestamp_ = unwrapped_timestamp;
+ layer_state = TemporalLayerState::kTl1Sync;
+ } else {
+ layer_state = TemporalLayerState::kTl1;
+ }
+ } else {
+ layer_state = last_sync_timestamp_ == unwrapped_timestamp
+ ? TemporalLayerState::kTl1Sync
+ : TemporalLayerState::kTl1;
+ }
+ break;
+ case -1:
+ layer_state = TemporalLayerState::kDrop;
+ ++stats_.num_dropped_frames_;
+ break;
+ default:
+ RTC_DCHECK_NOTREACHED();
+ }
+
+ DependencyInfo dependency_info;
+ // TODO(pbos): Consider referencing but not updating the 'alt' buffer for all
+ // layers.
+ switch (layer_state) {
+ case TemporalLayerState::kDrop:
+ dependency_info = {"", {kNone, kNone, kNone}};
+ break;
+ case TemporalLayerState::kTl0:
+ // TL0 only references and updates 'last'.
+ dependency_info = {"SS", {kReferenceAndUpdate, kNone, kNone}};
+ dependency_info.frame_config.packetizer_temporal_idx = 0;
+ break;
+ case TemporalLayerState::kTl1:
+ // TL1 references both 'last' and 'golden' but only updates 'golden'.
+ dependency_info = {"-R", {kReference, kReferenceAndUpdate, kNone}};
+ dependency_info.frame_config.packetizer_temporal_idx = 1;
+ break;
+ case TemporalLayerState::kTl1Sync:
+ // Predict from only TL0 to allow participants to switch to the high
+ // bitrate stream. Updates 'golden' so that TL1 can continue to refer to
+ // and update 'golden' from this point on.
+ dependency_info = {"-S", {kReference, kUpdate, kNone}};
+ dependency_info.frame_config.packetizer_temporal_idx = 1;
+ dependency_info.frame_config.layer_sync = true;
+ break;
+ }
+
+ pending_frame_configs_[timestamp] = dependency_info;
+ return dependency_info.frame_config;
+}
+
+void ScreenshareLayers::OnRatesUpdated(
+ size_t stream_index,
+ const std::vector<uint32_t>& bitrates_bps,
+ int framerate_fps) {
+ RTC_DCHECK_LT(stream_index, StreamCount());
+ RTC_DCHECK_GT(framerate_fps, 0);
+ RTC_DCHECK_GE(bitrates_bps.size(), 1);
+ RTC_DCHECK_LE(bitrates_bps.size(), 2);
+
+ // `bitrates_bps` uses individual rates per layer, but we want to use the
+ // accumulated rate here.
+ uint32_t tl0_kbps = bitrates_bps[0] / 1000;
+ uint32_t tl1_kbps = tl0_kbps;
+ if (bitrates_bps.size() > 1) {
+ tl1_kbps += bitrates_bps[1] / 1000;
+ }
+
+ if (!target_framerate_) {
+ // First OnRatesUpdated() is called during construction, with the
+ // configured targets as parameters.
+ target_framerate_ = framerate_fps;
+ capture_framerate_ = target_framerate_;
+ bitrate_updated_ = true;
+ } else {
+ if ((capture_framerate_ &&
+ framerate_fps != static_cast<int>(*capture_framerate_)) ||
+ (tl0_kbps != layers_[0].target_rate_kbps_) ||
+ (tl1_kbps != layers_[1].target_rate_kbps_)) {
+ bitrate_updated_ = true;
+ }
+
+ if (framerate_fps < 0) {
+ capture_framerate_.reset();
+ } else {
+ capture_framerate_ = framerate_fps;
+ }
+ }
+
+ layers_[0].target_rate_kbps_ = tl0_kbps;
+ layers_[1].target_rate_kbps_ = tl1_kbps;
+}
+
+void ScreenshareLayers::OnEncodeDone(size_t stream_index,
+ uint32_t rtp_timestamp,
+ size_t size_bytes,
+ bool is_keyframe,
+ int qp,
+ CodecSpecificInfo* info) {
+ RTC_DCHECK_LT(stream_index, StreamCount());
+
+ if (size_bytes == 0) {
+ RTC_LOG(LS_WARNING) << "Empty frame; treating as dropped.";
+ OnFrameDropped(stream_index, rtp_timestamp);
+ return;
+ }
+
+ absl::optional<DependencyInfo> dependency_info;
+ auto it = pending_frame_configs_.find(rtp_timestamp);
+ if (it != pending_frame_configs_.end()) {
+ dependency_info = it->second;
+ pending_frame_configs_.erase(it);
+
+ if (checker_) {
+ RTC_DCHECK(checker_->CheckTemporalConfig(is_keyframe,
+ dependency_info->frame_config));
+ }
+ }
+
+ CodecSpecificInfoVP8& vp8_info = info->codecSpecific.VP8;
+ GenericFrameInfo& generic_frame_info = info->generic_frame_info.emplace();
+
+ if (number_of_temporal_layers_ == 1) {
+ vp8_info.temporalIdx = kNoTemporalIdx;
+ vp8_info.layerSync = false;
+ generic_frame_info.temporal_id = 0;
+ generic_frame_info.decode_target_indications = {kSwitch};
+ generic_frame_info.encoder_buffers.emplace_back(
+ 0, /*referenced=*/!is_keyframe, /*updated=*/true);
+ } else {
+ int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(rtp_timestamp);
+ if (dependency_info) {
+ vp8_info.temporalIdx =
+ dependency_info->frame_config.packetizer_temporal_idx;
+ vp8_info.layerSync = dependency_info->frame_config.layer_sync;
+ generic_frame_info.temporal_id = vp8_info.temporalIdx;
+ generic_frame_info.decode_target_indications =
+ dependency_info->decode_target_indications;
+ } else {
+ RTC_DCHECK(is_keyframe);
+ }
+
+ if (is_keyframe) {
+ vp8_info.temporalIdx = 0;
+ last_sync_timestamp_ = unwrapped_timestamp;
+ vp8_info.layerSync = true;
+ layers_[0].state = TemporalLayer::State::kKeyFrame;
+ layers_[1].state = TemporalLayer::State::kKeyFrame;
+ active_layer_ = 1;
+ info->template_structure =
+ GetTemplateStructure(number_of_temporal_layers_);
+ generic_frame_info.temporal_id = vp8_info.temporalIdx;
+ generic_frame_info.decode_target_indications = {kSwitch, kSwitch};
+ } else if (active_layer_ >= 0 && layers_[active_layer_].state ==
+ TemporalLayer::State::kKeyFrame) {
+ layers_[active_layer_].state = TemporalLayer::State::kNormal;
+ }
+
+ vp8_info.useExplicitDependencies = true;
+ RTC_DCHECK_EQ(vp8_info.referencedBuffersCount, 0u);
+ RTC_DCHECK_EQ(vp8_info.updatedBuffersCount, 0u);
+
+ // Note that `frame_config` is not derefernced if `is_keyframe`,
+ // meaning it's never dereferenced if the optional may be unset.
+ for (int i = 0; i < static_cast<int>(Vp8FrameConfig::Buffer::kCount); ++i) {
+ bool references = false;
+ bool updates = is_keyframe;
+ if (!is_keyframe && dependency_info->frame_config.References(
+ static_cast<Vp8FrameConfig::Buffer>(i))) {
+ RTC_DCHECK_LT(vp8_info.referencedBuffersCount,
+ arraysize(CodecSpecificInfoVP8::referencedBuffers));
+ references = true;
+ vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i;
+ }
+
+ if (is_keyframe || dependency_info->frame_config.Updates(
+ static_cast<Vp8FrameConfig::Buffer>(i))) {
+ RTC_DCHECK_LT(vp8_info.updatedBuffersCount,
+ arraysize(CodecSpecificInfoVP8::updatedBuffers));
+ updates = true;
+ vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i;
+ }
+
+ if (references || updates)
+ generic_frame_info.encoder_buffers.emplace_back(i, references, updates);
+ }
+ }
+
+ encode_framerate_.Update(1, rtc::TimeMillis());
+
+ if (number_of_temporal_layers_ == 1)
+ return;
+
+ RTC_DCHECK_NE(-1, active_layer_);
+ if (layers_[active_layer_].state == TemporalLayer::State::kDropped) {
+ layers_[active_layer_].state = TemporalLayer::State::kQualityBoost;
+ }
+
+ if (qp != -1)
+ layers_[active_layer_].last_qp = qp;
+
+ if (active_layer_ == 0) {
+ layers_[0].debt_bytes_ += size_bytes;
+ layers_[1].debt_bytes_ += size_bytes;
+ ++stats_.num_tl0_frames_;
+ stats_.tl0_target_bitrate_sum_ += layers_[0].target_rate_kbps_;
+ stats_.tl0_qp_sum_ += qp;
+ } else if (active_layer_ == 1) {
+ layers_[1].debt_bytes_ += size_bytes;
+ ++stats_.num_tl1_frames_;
+ stats_.tl1_target_bitrate_sum_ += layers_[1].target_rate_kbps_;
+ stats_.tl1_qp_sum_ += qp;
+ }
+}
+
+void ScreenshareLayers::OnFrameDropped(size_t stream_index,
+ uint32_t rtp_timestamp) {
+ layers_[active_layer_].state = TemporalLayer::State::kDropped;
+ ++stats_.num_overshoots_;
+}
+
+void ScreenshareLayers::OnPacketLossRateUpdate(float packet_loss_rate) {}
+
+void ScreenshareLayers::OnRttUpdate(int64_t rtt_ms) {}
+
+void ScreenshareLayers::OnLossNotification(
+ const VideoEncoder::LossNotification& loss_notification) {}
+
+FrameDependencyStructure ScreenshareLayers::GetTemplateStructure(
+ int num_layers) const {
+ RTC_CHECK_LT(num_layers, 3);
+ RTC_CHECK_GT(num_layers, 0);
+
+ FrameDependencyStructure template_structure;
+ template_structure.num_decode_targets = num_layers;
+
+ switch (num_layers) {
+ case 1: {
+ template_structure.templates.resize(2);
+ template_structure.templates[0].T(0).Dtis("S");
+ template_structure.templates[1].T(0).Dtis("S").FrameDiffs({1});
+ return template_structure;
+ }
+ case 2: {
+ template_structure.templates.resize(3);
+ template_structure.templates[0].T(0).Dtis("SS");
+ template_structure.templates[1].T(0).Dtis("SS").FrameDiffs({1});
+ template_structure.templates[2].T(1).Dtis("-S").FrameDiffs({1});
+ return template_structure;
+ }
+ default:
+ RTC_DCHECK_NOTREACHED();
+ // To make the compiler happy!
+ return template_structure;
+ }
+}
+
+bool ScreenshareLayers::TimeToSync(int64_t timestamp) const {
+ RTC_DCHECK_EQ(1, active_layer_);
+ RTC_DCHECK_NE(-1, layers_[0].last_qp);
+ if (layers_[1].last_qp == -1) {
+ // First frame in TL1 should only depend on TL0 since there are no
+ // previous frames in TL1.
+ return true;
+ }
+
+ RTC_DCHECK_NE(-1, last_sync_timestamp_);
+ int64_t timestamp_diff = timestamp - last_sync_timestamp_;
+ if (timestamp_diff > kMaxTimeBetweenSyncs) {
+ // After a certain time, force a sync frame.
+ return true;
+ } else if (timestamp_diff < kMinTimeBetweenSyncs) {
+ // If too soon from previous sync frame, don't issue a new one.
+ return false;
+ }
+ // Issue a sync frame if difference in quality between TL0 and TL1 isn't too
+ // large.
+ if (layers_[0].last_qp - layers_[1].last_qp < kQpDeltaThresholdForSync)
+ return true;
+ return false;
+}
+
+uint32_t ScreenshareLayers::GetCodecTargetBitrateKbps() const {
+ uint32_t target_bitrate_kbps = layers_[0].target_rate_kbps_;
+
+ if (number_of_temporal_layers_ > 1) {
+ // Calculate a codec target bitrate. This may be higher than TL0, gaining
+ // quality at the expense of frame rate at TL0. Constraints:
+ // - TL0 frame rate no less than framerate / kMaxTL0FpsReduction.
+ // - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate.
+ target_bitrate_kbps =
+ std::min(layers_[0].target_rate_kbps_ * kMaxTL0FpsReduction,
+ layers_[1].target_rate_kbps_ / kAcceptableTargetOvershoot);
+ }
+
+ return std::max(layers_[0].target_rate_kbps_, target_bitrate_kbps);
+}
+
+Vp8EncoderConfig ScreenshareLayers::UpdateConfiguration(size_t stream_index) {
+ RTC_DCHECK_LT(stream_index, StreamCount());
+ RTC_DCHECK(min_qp_.has_value());
+ RTC_DCHECK(max_qp_.has_value());
+
+ const uint32_t target_bitrate_kbps = GetCodecTargetBitrateKbps();
+
+ // TODO(sprang): We _really_ need to make an overhaul of this class. :(
+ // If we're dropping frames in order to meet a target framerate, adjust the
+ // bitrate assigned to the encoder so the total average bitrate is correct.
+ float encoder_config_bitrate_kbps = target_bitrate_kbps;
+ if (target_framerate_ && capture_framerate_ &&
+ *target_framerate_ < *capture_framerate_) {
+ encoder_config_bitrate_kbps *=
+ static_cast<float>(*capture_framerate_) / *target_framerate_;
+ }
+
+ if (bitrate_updated_ ||
+ encoder_config_.rc_target_bitrate !=
+ absl::make_optional(encoder_config_bitrate_kbps)) {
+ encoder_config_.rc_target_bitrate = encoder_config_bitrate_kbps;
+
+ // Don't reconfigure qp limits during quality boost frames.
+ if (active_layer_ == -1 ||
+ layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) {
+ const int min_qp = min_qp_.value();
+ const int max_qp = max_qp_.value();
+
+ // After a dropped frame, a frame with max qp will be encoded and the
+ // quality will then ramp up from there. To boost the speed of recovery,
+ // encode the next frame with lower max qp, if there is sufficient
+ // bandwidth to do so without causing excessive delay.
+ // TL0 is the most important to improve since the errors in this layer
+ // will propagate to TL1.
+ // Currently, reduce max qp by 20% for TL0 and 15% for TL1.
+ if (layers_[1].target_rate_kbps_ >= kMinBitrateKbpsForQpBoost) {
+ layers_[0].enhanced_max_qp = min_qp + (((max_qp - min_qp) * 80) / 100);
+ layers_[1].enhanced_max_qp = min_qp + (((max_qp - min_qp) * 85) / 100);
+ } else {
+ layers_[0].enhanced_max_qp = -1;
+ layers_[1].enhanced_max_qp = -1;
+ }
+ }
+
+ if (capture_framerate_) {
+ int avg_frame_size =
+ (target_bitrate_kbps * 1000) / (8 * *capture_framerate_);
+ // Allow max debt to be the size of a single optimal frame.
+ // TODO(sprang): Determine if this needs to be adjusted by some factor.
+ // (Lower values may cause more frame drops, higher may lead to queuing
+ // delays.)
+ max_debt_bytes_ = avg_frame_size;
+ }
+
+ bitrate_updated_ = false;
+ }
+
+ // Don't try to update boosts state if not active yet.
+ if (active_layer_ == -1)
+ return encoder_config_;
+
+ if (number_of_temporal_layers_ <= 1)
+ return encoder_config_;
+
+ // If layer is in the quality boost state (following a dropped frame), update
+ // the configuration with the adjusted (lower) qp and set the state back to
+ // normal.
+ unsigned int adjusted_max_qp = max_qp_.value(); // Set the normal max qp.
+ if (layers_[active_layer_].state == TemporalLayer::State::kQualityBoost) {
+ if (layers_[active_layer_].enhanced_max_qp != -1) {
+ // Bitrate is high enough for quality boost, update max qp.
+ adjusted_max_qp = layers_[active_layer_].enhanced_max_qp;
+ }
+ // Regardless of qp, reset the boost state for the next frame.
+ layers_[active_layer_].state = TemporalLayer::State::kNormal;
+ }
+ encoder_config_.rc_max_quantizer = adjusted_max_qp;
+
+ return encoder_config_;
+}
+
+void ScreenshareLayers::TemporalLayer::UpdateDebt(int64_t delta_ms) {
+ uint32_t debt_reduction_bytes = target_rate_kbps_ * delta_ms / 8;
+ if (debt_reduction_bytes >= debt_bytes_) {
+ debt_bytes_ = 0;
+ } else {
+ debt_bytes_ -= debt_reduction_bytes;
+ }
+}
+
+void ScreenshareLayers::UpdateHistograms() {
+ if (stats_.first_frame_time_ms_ == -1)
+ return;
+ int64_t duration_sec =
+ (rtc::TimeMillis() - stats_.first_frame_time_ms_ + 500) / 1000;
+ if (duration_sec >= metrics::kMinRunTimeInSeconds) {
+ RTC_HISTOGRAM_COUNTS_10000(
+ "WebRTC.Video.Screenshare.Layer0.FrameRate",
+ (stats_.num_tl0_frames_ + (duration_sec / 2)) / duration_sec);
+ RTC_HISTOGRAM_COUNTS_10000(
+ "WebRTC.Video.Screenshare.Layer1.FrameRate",
+ (stats_.num_tl1_frames_ + (duration_sec / 2)) / duration_sec);
+ int total_frames = stats_.num_tl0_frames_ + stats_.num_tl1_frames_;
+ RTC_HISTOGRAM_COUNTS_10000(
+ "WebRTC.Video.Screenshare.FramesPerDrop",
+ (stats_.num_dropped_frames_ == 0
+ ? 0
+ : total_frames / stats_.num_dropped_frames_));
+ RTC_HISTOGRAM_COUNTS_10000(
+ "WebRTC.Video.Screenshare.FramesPerOvershoot",
+ (stats_.num_overshoots_ == 0 ? 0
+ : total_frames / stats_.num_overshoots_));
+ if (stats_.num_tl0_frames_ > 0) {
+ RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.Layer0.Qp",
+ stats_.tl0_qp_sum_ / stats_.num_tl0_frames_);
+ RTC_HISTOGRAM_COUNTS_10000(
+ "WebRTC.Video.Screenshare.Layer0.TargetBitrate",
+ stats_.tl0_target_bitrate_sum_ / stats_.num_tl0_frames_);
+ }
+ if (stats_.num_tl1_frames_ > 0) {
+ RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.Layer1.Qp",
+ stats_.tl1_qp_sum_ / stats_.num_tl1_frames_);
+ RTC_HISTOGRAM_COUNTS_10000(
+ "WebRTC.Video.Screenshare.Layer1.TargetBitrate",
+ stats_.tl1_target_bitrate_sum_ / stats_.num_tl1_frames_);
+ }
+ }
+}
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers.h
new file mode 100644
index 0000000000..47d6b401f4
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers.h
@@ -0,0 +1,164 @@
+/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_
+#define MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_
+
+#include <map>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "api/video_codecs/vp8_frame_config.h"
+#include "api/video_codecs/vp8_temporal_layers.h"
+#include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h"
+#include "modules/video_coding/include/video_codec_interface.h"
+#include "modules/video_coding/utility/frame_dropper.h"
+#include "rtc_base/numerics/sequence_number_unwrapper.h"
+#include "rtc_base/rate_statistics.h"
+
+namespace webrtc {
+
+struct CodecSpecificInfoVP8;
+class Clock;
+
+class ScreenshareLayers final : public Vp8FrameBufferController {
+ public:
+ static const double kMaxTL0FpsReduction;
+ static const double kAcceptableTargetOvershoot;
+ static const int kMaxFrameIntervalMs;
+
+ explicit ScreenshareLayers(int num_temporal_layers);
+ ~ScreenshareLayers() override;
+
+ void SetQpLimits(size_t stream_index, int min_qp, int max_qp) override;
+
+ size_t StreamCount() const override;
+
+ bool SupportsEncoderFrameDropping(size_t stream_index) const override;
+
+ // Returns the recommended VP8 encode flags needed. May refresh the decoder
+ // and/or update the reference buffers.
+ Vp8FrameConfig NextFrameConfig(size_t stream_index,
+ uint32_t rtp_timestamp) override;
+
+ // New target bitrate, per temporal layer.
+ void OnRatesUpdated(size_t stream_index,
+ const std::vector<uint32_t>& bitrates_bps,
+ int framerate_fps) override;
+
+ Vp8EncoderConfig UpdateConfiguration(size_t stream_index) override;
+
+ void OnEncodeDone(size_t stream_index,
+ uint32_t rtp_timestamp,
+ size_t size_bytes,
+ bool is_keyframe,
+ int qp,
+ CodecSpecificInfo* info) override;
+
+ void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) override;
+
+ void OnPacketLossRateUpdate(float packet_loss_rate) override;
+
+ void OnRttUpdate(int64_t rtt_ms) override;
+
+ void OnLossNotification(
+ const VideoEncoder::LossNotification& loss_notification) override;
+
+ private:
+ enum class TemporalLayerState : int { kDrop, kTl0, kTl1, kTl1Sync };
+
+ struct DependencyInfo {
+ DependencyInfo() = default;
+ DependencyInfo(absl::string_view indication_symbols,
+ Vp8FrameConfig frame_config)
+ : decode_target_indications(
+ webrtc_impl::StringToDecodeTargetIndications(indication_symbols)),
+ frame_config(frame_config) {}
+
+ absl::InlinedVector<DecodeTargetIndication, 10> decode_target_indications;
+ Vp8FrameConfig frame_config;
+ };
+
+ bool TimeToSync(int64_t timestamp) const;
+ uint32_t GetCodecTargetBitrateKbps() const;
+
+ const int number_of_temporal_layers_;
+
+ // TODO(eladalon/sprang): These should be made into const-int set in the ctor.
+ absl::optional<int> min_qp_;
+ absl::optional<int> max_qp_;
+
+ int active_layer_;
+ int64_t last_timestamp_;
+ int64_t last_sync_timestamp_;
+ int64_t last_emitted_tl0_timestamp_;
+ int64_t last_frame_time_ms_;
+ RtpTimestampUnwrapper time_wrap_handler_;
+ uint32_t max_debt_bytes_;
+
+ std::map<uint32_t, DependencyInfo> pending_frame_configs_;
+
+ // Configured max framerate.
+ absl::optional<uint32_t> target_framerate_;
+ // Incoming framerate from capturer.
+ absl::optional<uint32_t> capture_framerate_;
+
+ // Tracks what framerate we actually encode, and drops frames on overshoot.
+ RateStatistics encode_framerate_;
+ bool bitrate_updated_;
+
+ static constexpr int kMaxNumTemporalLayers = 2;
+ struct TemporalLayer {
+ TemporalLayer()
+ : state(State::kNormal),
+ enhanced_max_qp(-1),
+ last_qp(-1),
+ debt_bytes_(0),
+ target_rate_kbps_(0) {}
+
+ enum class State {
+ kNormal,
+ kDropped,
+ kReencoded,
+ kQualityBoost,
+ kKeyFrame
+ } state;
+
+ int enhanced_max_qp;
+ int last_qp;
+ uint32_t debt_bytes_;
+ uint32_t target_rate_kbps_;
+
+ void UpdateDebt(int64_t delta_ms);
+ } layers_[kMaxNumTemporalLayers];
+
+ void UpdateHistograms();
+ FrameDependencyStructure GetTemplateStructure(int num_layers) const;
+
+ // Data for histogram statistics.
+ struct Stats {
+ int64_t first_frame_time_ms_ = -1;
+ int64_t num_tl0_frames_ = 0;
+ int64_t num_tl1_frames_ = 0;
+ int64_t num_dropped_frames_ = 0;
+ int64_t num_overshoots_ = 0;
+ int64_t tl0_qp_sum_ = 0;
+ int64_t tl1_qp_sum_ = 0;
+ int64_t tl0_target_bitrate_sum_ = 0;
+ int64_t tl1_target_bitrate_sum_ = 0;
+ } stats_;
+
+ Vp8EncoderConfig encoder_config_;
+
+ // Optional utility used to verify reference validity.
+ std::unique_ptr<TemporalLayersChecker> checker_;
+};
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc
new file mode 100644
index 0000000000..e5b3bd4fdf
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc
@@ -0,0 +1,788 @@
+/*
+ * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/vp8/screenshare_layers.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+#include "api/video_codecs/vp8_frame_config.h"
+#include "modules/video_coding/codecs/interface/common_constants.h"
+#include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h"
+#include "modules/video_coding/include/video_codec_interface.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/fake_clock.h"
+#include "system_wrappers/include/metrics.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "vpx/vp8cx.h"
+
+using ::testing::_;
+using ::testing::ElementsAre;
+using ::testing::NiceMock;
+
+namespace webrtc {
+namespace {
+// 5 frames per second at 90 kHz.
+const uint32_t kTimestampDelta5Fps = 90000 / 5;
+const int kDefaultQp = 54;
+const int kDefaultTl0BitrateKbps = 200;
+const int kDefaultTl1BitrateKbps = 2000;
+const int kFrameRate = 5;
+const int kSyncPeriodSeconds = 2;
+const int kMaxSyncPeriodSeconds = 4;
+
+// Expected flags for corresponding temporal layers.
+const int kTl0Flags = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
+ VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF;
+const int kTl1Flags =
+ VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST;
+const int kTl1SyncFlags = VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF |
+ VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST;
+const std::vector<uint32_t> kDefault2TlBitratesBps = {
+ kDefaultTl0BitrateKbps * 1000,
+ (kDefaultTl1BitrateKbps - kDefaultTl0BitrateKbps) * 1000};
+
+} // namespace
+
+class ScreenshareLayerTest : public ::testing::Test {
+ protected:
+ ScreenshareLayerTest()
+ : min_qp_(2),
+ max_qp_(kDefaultQp),
+ frame_size_(-1),
+ timestamp_(90),
+ config_updated_(false) {}
+ virtual ~ScreenshareLayerTest() {}
+
+ void SetUp() override {
+ layers_.reset(new ScreenshareLayers(2));
+ cfg_ = ConfigureBitrates();
+ }
+
+ int EncodeFrame(bool base_sync, CodecSpecificInfo* info = nullptr) {
+ CodecSpecificInfo ignored_info;
+ if (!info) {
+ info = &ignored_info;
+ }
+
+ int flags = ConfigureFrame(base_sync);
+ if (flags != -1)
+ layers_->OnEncodeDone(0, timestamp_, frame_size_, base_sync, kDefaultQp,
+ info);
+ return flags;
+ }
+
+ int ConfigureFrame(bool key_frame) {
+ tl_config_ = NextFrameConfig(0, timestamp_);
+ EXPECT_EQ(0, tl_config_.encoder_layer_id)
+ << "ScreenshareLayers always encodes using the bitrate allocator for "
+ "layer 0, but may reference different buffers and packetize "
+ "differently.";
+ if (tl_config_.drop_frame) {
+ return -1;
+ }
+ const uint32_t prev_rc_target_bitrate = cfg_.rc_target_bitrate.value_or(-1);
+ const uint32_t prev_rc_max_quantizer = cfg_.rc_max_quantizer.value_or(-1);
+
+ cfg_ = layers_->UpdateConfiguration(0);
+
+ config_updated_ =
+ cfg_.temporal_layer_config.has_value() ||
+ (cfg_.rc_target_bitrate.has_value() &&
+ cfg_.rc_target_bitrate.value() != prev_rc_target_bitrate) ||
+ (cfg_.rc_max_quantizer.has_value() &&
+ cfg_.rc_max_quantizer.value() != prev_rc_max_quantizer) ||
+ cfg_.g_error_resilient.has_value();
+
+ int flags = LibvpxVp8Encoder::EncodeFlags(tl_config_);
+ EXPECT_NE(-1, frame_size_);
+ return flags;
+ }
+
+ Vp8FrameConfig NextFrameConfig(size_t stream_index, uint32_t timestamp) {
+ int64_t timestamp_ms = timestamp / 90;
+ clock_.AdvanceTime(TimeDelta::Millis(timestamp_ms - rtc::TimeMillis()));
+ return layers_->NextFrameConfig(stream_index, timestamp);
+ }
+
+ int FrameSizeForBitrate(int bitrate_kbps) {
+ return ((bitrate_kbps * 1000) / 8) / kFrameRate;
+ }
+
+ Vp8EncoderConfig ConfigureBitrates() {
+ layers_->SetQpLimits(0, min_qp_, max_qp_);
+ layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, kFrameRate);
+ const Vp8EncoderConfig vp8_cfg = layers_->UpdateConfiguration(0);
+ EXPECT_TRUE(vp8_cfg.rc_target_bitrate.has_value());
+ frame_size_ = FrameSizeForBitrate(vp8_cfg.rc_target_bitrate.value());
+ return vp8_cfg;
+ }
+
+ void WithQpLimits(int min_qp, int max_qp) {
+ min_qp_ = min_qp;
+ max_qp_ = max_qp;
+ }
+
+ // Runs a few initial frames and makes sure we have seen frames on both
+ // temporal layers, including sync and non-sync frames.
+ bool RunGracePeriod() {
+ bool got_tl0 = false;
+ bool got_tl1 = false;
+ bool got_tl1_sync = false;
+ for (int i = 0; i < 10; ++i) {
+ CodecSpecificInfo info;
+ EXPECT_NE(-1, EncodeFrame(false, &info));
+ timestamp_ += kTimestampDelta5Fps;
+ if (info.codecSpecific.VP8.temporalIdx == 0) {
+ got_tl0 = true;
+ } else if (info.codecSpecific.VP8.layerSync) {
+ got_tl1_sync = true;
+ } else {
+ got_tl1 = true;
+ }
+ if (got_tl0 && got_tl1 && got_tl1_sync)
+ return true;
+ }
+ return false;
+ }
+
+ // Adds frames until we get one in the specified temporal layer. The last
+ // FrameEncoded() call will be omitted and needs to be done by the caller.
+ // Returns the flags for the last frame.
+ int SkipUntilTl(int layer) {
+ return SkipUntilTlAndSync(layer, absl::nullopt);
+ }
+
+ // Same as SkipUntilTl, but also waits until the sync bit condition is met.
+ int SkipUntilTlAndSync(int layer, absl::optional<bool> sync) {
+ int flags = 0;
+ const int kMaxFramesToSkip =
+ 1 + (sync.value_or(false) ? kMaxSyncPeriodSeconds : 1) * kFrameRate;
+ for (int i = 0; i < kMaxFramesToSkip; ++i) {
+ flags = ConfigureFrame(false);
+ if (tl_config_.packetizer_temporal_idx != layer ||
+ (sync && *sync != tl_config_.layer_sync)) {
+ if (flags != -1) {
+ // If flags do not request a frame drop, report some default values
+ // for frame size etc.
+ CodecSpecificInfo info;
+ layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
+ &info);
+ }
+ timestamp_ += kTimestampDelta5Fps;
+ } else {
+ // Found frame from sought after layer.
+ return flags;
+ }
+ }
+ ADD_FAILURE() << "Did not get a frame of TL" << layer << " in time.";
+ return -1;
+ }
+
+ int min_qp_;
+ uint32_t max_qp_;
+ int frame_size_;
+ rtc::ScopedFakeClock clock_;
+ std::unique_ptr<ScreenshareLayers> layers_;
+
+ uint32_t timestamp_;
+ Vp8FrameConfig tl_config_;
+ Vp8EncoderConfig cfg_;
+ bool config_updated_;
+
+ CodecSpecificInfo* IgnoredCodecSpecificInfo() {
+ ignored_codec_specific_info_ = std::make_unique<CodecSpecificInfo>();
+ return ignored_codec_specific_info_.get();
+ }
+
+ private:
+ std::unique_ptr<CodecSpecificInfo> ignored_codec_specific_info_;
+};
+
+TEST_F(ScreenshareLayerTest, 1Layer) {
+ layers_.reset(new ScreenshareLayers(1));
+ ConfigureBitrates();
+ // One layer screenshare should not use the frame dropper as all frames will
+ // belong to the base layer.
+ const int kSingleLayerFlags = 0;
+ auto info = std::make_unique<CodecSpecificInfo>();
+ int flags = EncodeFrame(/*base_sync=*/false, info.get());
+ timestamp_ += kTimestampDelta5Fps;
+ EXPECT_EQ(static_cast<uint8_t>(kNoTemporalIdx),
+ info->codecSpecific.VP8.temporalIdx);
+ EXPECT_FALSE(info->codecSpecific.VP8.layerSync);
+ EXPECT_EQ(info->generic_frame_info->temporal_id, 0);
+
+ info = std::make_unique<CodecSpecificInfo>();
+ flags = EncodeFrame(/*base_sync=*/false, info.get());
+ EXPECT_EQ(kSingleLayerFlags, flags);
+ EXPECT_EQ(static_cast<uint8_t>(kNoTemporalIdx),
+ info->codecSpecific.VP8.temporalIdx);
+ EXPECT_FALSE(info->codecSpecific.VP8.layerSync);
+ EXPECT_EQ(info->generic_frame_info->temporal_id, 0);
+}
+
+TEST_F(ScreenshareLayerTest, 2LayersPeriodicSync) {
+ std::vector<int> sync_times;
+ const int kNumFrames = kSyncPeriodSeconds * kFrameRate * 2 - 1;
+ for (int i = 0; i < kNumFrames; ++i) {
+ CodecSpecificInfo info;
+ EncodeFrame(false, &info);
+ timestamp_ += kTimestampDelta5Fps;
+ if (info.codecSpecific.VP8.temporalIdx == 1 &&
+ info.codecSpecific.VP8.layerSync) {
+ sync_times.push_back(timestamp_);
+ }
+ }
+
+ ASSERT_EQ(2u, sync_times.size());
+ EXPECT_GE(sync_times[1] - sync_times[0], 90000 * kSyncPeriodSeconds);
+}
+
+TEST_F(ScreenshareLayerTest, 2LayersSyncAfterTimeout) {
+ std::vector<int> sync_times;
+ const int kNumFrames = kMaxSyncPeriodSeconds * kFrameRate * 2 - 1;
+ for (int i = 0; i < kNumFrames; ++i) {
+ CodecSpecificInfo info;
+
+ tl_config_ = NextFrameConfig(0, timestamp_);
+ cfg_ = layers_->UpdateConfiguration(0);
+
+ // Simulate TL1 being at least 8 qp steps better.
+ if (tl_config_.packetizer_temporal_idx == 0) {
+ layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
+ &info);
+ } else {
+ layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp - 8,
+ &info);
+ }
+
+ if (info.codecSpecific.VP8.temporalIdx == 1 &&
+ info.codecSpecific.VP8.layerSync)
+ sync_times.push_back(timestamp_);
+
+ timestamp_ += kTimestampDelta5Fps;
+ }
+
+ ASSERT_EQ(2u, sync_times.size());
+ EXPECT_GE(sync_times[1] - sync_times[0], 90000 * kMaxSyncPeriodSeconds);
+}
+
+TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) {
+ std::vector<int> sync_times;
+
+ const int kNumFrames = (kSyncPeriodSeconds +
+ ((kMaxSyncPeriodSeconds - kSyncPeriodSeconds) / 2)) *
+ kFrameRate;
+ for (int i = 0; i < kNumFrames; ++i) {
+ CodecSpecificInfo info;
+
+ ConfigureFrame(false);
+
+ // Simulate TL1 being at least 8 qp steps better.
+ if (tl_config_.packetizer_temporal_idx == 0) {
+ layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
+ &info);
+ } else {
+ layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp - 8,
+ &info);
+ }
+
+ if (info.codecSpecific.VP8.temporalIdx == 1 &&
+ info.codecSpecific.VP8.layerSync)
+ sync_times.push_back(timestamp_);
+
+ timestamp_ += kTimestampDelta5Fps;
+ }
+
+ ASSERT_EQ(1u, sync_times.size());
+
+ bool bumped_tl0_quality = false;
+ for (int i = 0; i < 3; ++i) {
+ CodecSpecificInfo info;
+
+ int flags = ConfigureFrame(false);
+ layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp - 8,
+ &info);
+ if (info.codecSpecific.VP8.temporalIdx == 0) {
+ // Bump TL0 to same quality as TL1.
+ bumped_tl0_quality = true;
+ } else {
+ if (bumped_tl0_quality) {
+ EXPECT_TRUE(info.codecSpecific.VP8.layerSync);
+ EXPECT_EQ(kTl1SyncFlags, flags);
+ return;
+ }
+ }
+ timestamp_ += kTimestampDelta5Fps;
+ }
+ ADD_FAILURE() << "No TL1 frame arrived within time limit.";
+}
+
+TEST_F(ScreenshareLayerTest, 2LayersToggling) {
+ EXPECT_TRUE(RunGracePeriod());
+
+ // Insert 50 frames. 2/5 should be TL0.
+ int tl0_frames = 0;
+ int tl1_frames = 0;
+ for (int i = 0; i < 50; ++i) {
+ CodecSpecificInfo info;
+ EncodeFrame(/*base_sync=*/false, &info);
+ EXPECT_EQ(info.codecSpecific.VP8.temporalIdx,
+ info.generic_frame_info->temporal_id);
+ timestamp_ += kTimestampDelta5Fps;
+ switch (info.codecSpecific.VP8.temporalIdx) {
+ case 0:
+ ++tl0_frames;
+ break;
+ case 1:
+ ++tl1_frames;
+ break;
+ default:
+ abort();
+ }
+ }
+ EXPECT_EQ(20, tl0_frames);
+ EXPECT_EQ(30, tl1_frames);
+}
+
+TEST_F(ScreenshareLayerTest, AllFitsLayer0) {
+ frame_size_ = FrameSizeForBitrate(kDefaultTl0BitrateKbps);
+
+ // Insert 50 frames, small enough that all fits in TL0.
+ for (int i = 0; i < 50; ++i) {
+ CodecSpecificInfo info;
+ int flags = EncodeFrame(false, &info);
+ timestamp_ += kTimestampDelta5Fps;
+ EXPECT_EQ(kTl0Flags, flags);
+ EXPECT_EQ(0, info.codecSpecific.VP8.temporalIdx);
+ }
+}
+
+TEST_F(ScreenshareLayerTest, TooHighBitrate) {
+ frame_size_ = 2 * FrameSizeForBitrate(kDefaultTl1BitrateKbps);
+
+ // Insert 100 frames. Half should be dropped.
+ int tl0_frames = 0;
+ int tl1_frames = 0;
+ int dropped_frames = 0;
+ for (int i = 0; i < 100; ++i) {
+ CodecSpecificInfo info;
+ int flags = EncodeFrame(false, &info);
+ timestamp_ += kTimestampDelta5Fps;
+ if (flags == -1) {
+ ++dropped_frames;
+ } else {
+ switch (info.codecSpecific.VP8.temporalIdx) {
+ case 0:
+ ++tl0_frames;
+ break;
+ case 1:
+ ++tl1_frames;
+ break;
+ default:
+ ADD_FAILURE() << "Unexpected temporal id";
+ }
+ }
+ }
+
+ EXPECT_NEAR(50, tl0_frames + tl1_frames, 1);
+ EXPECT_NEAR(50, dropped_frames, 1);
+}
+
+TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL0) {
+ const int kTl0_kbps = 100;
+ const int kTl1_kbps = 1000;
+ const std::vector<uint32_t> layer_rates = {kTl0_kbps * 1000,
+ (kTl1_kbps - kTl0_kbps) * 1000};
+ layers_->OnRatesUpdated(0, layer_rates, kFrameRate);
+ cfg_ = layers_->UpdateConfiguration(0);
+
+ EXPECT_EQ(static_cast<unsigned int>(
+ ScreenshareLayers::kMaxTL0FpsReduction * kTl0_kbps + 0.5),
+ cfg_.rc_target_bitrate);
+}
+
+TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL1) {
+ const int kTl0_kbps = 100;
+ const int kTl1_kbps = 450;
+ const std::vector<uint32_t> layer_rates = {kTl0_kbps * 1000,
+ (kTl1_kbps - kTl0_kbps) * 1000};
+ layers_->OnRatesUpdated(0, layer_rates, kFrameRate);
+ cfg_ = layers_->UpdateConfiguration(0);
+
+ EXPECT_EQ(static_cast<unsigned int>(
+ kTl1_kbps / ScreenshareLayers::kAcceptableTargetOvershoot),
+ cfg_.rc_target_bitrate);
+}
+
+TEST_F(ScreenshareLayerTest, TargetBitrateBelowTL0) {
+ const int kTl0_kbps = 100;
+ const std::vector<uint32_t> layer_rates = {kTl0_kbps * 1000};
+ layers_->OnRatesUpdated(0, layer_rates, kFrameRate);
+ cfg_ = layers_->UpdateConfiguration(0);
+
+ EXPECT_EQ(static_cast<uint32_t>(kTl0_kbps), cfg_.rc_target_bitrate);
+}
+
+TEST_F(ScreenshareLayerTest, EncoderDrop) {
+ EXPECT_TRUE(RunGracePeriod());
+ SkipUntilTl(0);
+
+ // Size 0 indicates dropped frame.
+ layers_->OnEncodeDone(0, timestamp_, 0, false, 0, IgnoredCodecSpecificInfo());
+
+ // Re-encode frame (so don't advance timestamp).
+ int flags = EncodeFrame(false);
+ timestamp_ += kTimestampDelta5Fps;
+ EXPECT_FALSE(config_updated_);
+ EXPECT_EQ(kTl0Flags, flags);
+
+ // Next frame should have boosted quality...
+ SkipUntilTl(0);
+ EXPECT_TRUE(config_updated_);
+ EXPECT_LT(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
+ layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ timestamp_ += kTimestampDelta5Fps;
+
+ // ...then back to standard setup.
+ SkipUntilTl(0);
+ layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ timestamp_ += kTimestampDelta5Fps;
+ EXPECT_EQ(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
+
+ // Next drop in TL1.
+ SkipUntilTl(1);
+ layers_->OnEncodeDone(0, timestamp_, 0, false, 0, IgnoredCodecSpecificInfo());
+
+ // Re-encode frame (so don't advance timestamp).
+ flags = EncodeFrame(false);
+ timestamp_ += kTimestampDelta5Fps;
+ EXPECT_FALSE(config_updated_);
+ EXPECT_EQ(kTl1Flags, flags);
+
+ // Next frame should have boosted QP.
+ SkipUntilTl(1);
+ EXPECT_TRUE(config_updated_);
+ EXPECT_LT(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
+ layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ timestamp_ += kTimestampDelta5Fps;
+
+ // ...and back to normal.
+ SkipUntilTl(1);
+ EXPECT_EQ(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
+ layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ timestamp_ += kTimestampDelta5Fps;
+}
+
+TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) {
+ const int kLowBitrateKbps = 50;
+ const int kLargeFrameSizeBytes = 100000;
+ const uint32_t kStartTimestamp = 1234;
+
+ const std::vector<uint32_t> layer_rates = {kLowBitrateKbps * 1000};
+ layers_->OnRatesUpdated(0, layer_rates, kFrameRate);
+ cfg_ = layers_->UpdateConfiguration(0);
+
+ EXPECT_EQ(kTl0Flags,
+ LibvpxVp8Encoder::EncodeFlags(NextFrameConfig(0, kStartTimestamp)));
+ layers_->OnEncodeDone(0, kStartTimestamp, kLargeFrameSizeBytes, false,
+ kDefaultQp, IgnoredCodecSpecificInfo());
+
+ const uint32_t kTwoSecondsLater =
+ kStartTimestamp + (ScreenshareLayers::kMaxFrameIntervalMs * 90);
+
+ // Sanity check, repayment time should exceed kMaxFrameIntervalMs.
+ ASSERT_GT(kStartTimestamp + 90 * (kLargeFrameSizeBytes * 8) / kLowBitrateKbps,
+ kStartTimestamp + (ScreenshareLayers::kMaxFrameIntervalMs * 90));
+
+ // Expect drop one frame interval before the two second timeout. If we try
+ // any later, the frame will be dropped anyway by the frame rate throttling
+ // logic.
+ EXPECT_TRUE(
+ NextFrameConfig(0, kTwoSecondsLater - kTimestampDelta5Fps).drop_frame);
+
+ // More than two seconds has passed since last frame, one should be emitted
+ // even if bitrate target is then exceeded.
+ EXPECT_EQ(kTl0Flags, LibvpxVp8Encoder::EncodeFlags(
+ NextFrameConfig(0, kTwoSecondsLater + 90)));
+}
+
+TEST_F(ScreenshareLayerTest, UpdatesHistograms) {
+ metrics::Reset();
+ bool trigger_drop = false;
+ bool dropped_frame = false;
+ bool overshoot = false;
+ const int kTl0Qp = 35;
+ const int kTl1Qp = 30;
+ for (int64_t timestamp = 0;
+ timestamp < kTimestampDelta5Fps * 5 * metrics::kMinRunTimeInSeconds;
+ timestamp += kTimestampDelta5Fps) {
+ tl_config_ = NextFrameConfig(0, timestamp);
+ if (tl_config_.drop_frame) {
+ dropped_frame = true;
+ continue;
+ }
+ int flags = LibvpxVp8Encoder::EncodeFlags(tl_config_);
+ if (flags != -1)
+ cfg_ = layers_->UpdateConfiguration(0);
+
+ if (timestamp >= kTimestampDelta5Fps * 5 && !overshoot && flags != -1) {
+ // Simulate one overshoot.
+ layers_->OnEncodeDone(0, timestamp, 0, false, 0, nullptr);
+ overshoot = true;
+ }
+
+ if (flags == kTl0Flags) {
+ if (timestamp >= kTimestampDelta5Fps * 20 && !trigger_drop) {
+ // Simulate a too large frame, to cause frame drop.
+ layers_->OnEncodeDone(0, timestamp, frame_size_ * 10, false, kTl0Qp,
+ IgnoredCodecSpecificInfo());
+ trigger_drop = true;
+ } else {
+ layers_->OnEncodeDone(0, timestamp, frame_size_, false, kTl0Qp,
+ IgnoredCodecSpecificInfo());
+ }
+ } else if (flags == kTl1Flags || flags == kTl1SyncFlags) {
+ layers_->OnEncodeDone(0, timestamp, frame_size_, false, kTl1Qp,
+ IgnoredCodecSpecificInfo());
+ } else if (flags == -1) {
+ dropped_frame = true;
+ } else {
+ RTC_DCHECK_NOTREACHED() << "Unexpected flags";
+ }
+ clock_.AdvanceTime(TimeDelta::Millis(1000 / 5));
+ }
+
+ EXPECT_TRUE(overshoot);
+ EXPECT_TRUE(dropped_frame);
+
+ layers_.reset(); // Histograms are reported on destruction.
+
+ EXPECT_METRIC_EQ(
+ 1, metrics::NumSamples("WebRTC.Video.Screenshare.Layer0.FrameRate"));
+ EXPECT_METRIC_EQ(
+ 1, metrics::NumSamples("WebRTC.Video.Screenshare.Layer1.FrameRate"));
+ EXPECT_METRIC_EQ(
+ 1, metrics::NumSamples("WebRTC.Video.Screenshare.FramesPerDrop"));
+ EXPECT_METRIC_EQ(
+ 1, metrics::NumSamples("WebRTC.Video.Screenshare.FramesPerOvershoot"));
+ EXPECT_METRIC_EQ(1,
+ metrics::NumSamples("WebRTC.Video.Screenshare.Layer0.Qp"));
+ EXPECT_METRIC_EQ(1,
+ metrics::NumSamples("WebRTC.Video.Screenshare.Layer1.Qp"));
+ EXPECT_METRIC_EQ(
+ 1, metrics::NumSamples("WebRTC.Video.Screenshare.Layer0.TargetBitrate"));
+ EXPECT_METRIC_EQ(
+ 1, metrics::NumSamples("WebRTC.Video.Screenshare.Layer1.TargetBitrate"));
+
+ EXPECT_METRIC_GT(
+ metrics::MinSample("WebRTC.Video.Screenshare.Layer0.FrameRate"), 1);
+ EXPECT_METRIC_GT(
+ metrics::MinSample("WebRTC.Video.Screenshare.Layer1.FrameRate"), 1);
+ EXPECT_METRIC_GT(metrics::MinSample("WebRTC.Video.Screenshare.FramesPerDrop"),
+ 1);
+ EXPECT_METRIC_GT(
+ metrics::MinSample("WebRTC.Video.Screenshare.FramesPerOvershoot"), 1);
+ EXPECT_METRIC_EQ(
+ 1, metrics::NumEvents("WebRTC.Video.Screenshare.Layer0.Qp", kTl0Qp));
+ EXPECT_METRIC_EQ(
+ 1, metrics::NumEvents("WebRTC.Video.Screenshare.Layer1.Qp", kTl1Qp));
+ EXPECT_METRIC_EQ(
+ 1, metrics::NumEvents("WebRTC.Video.Screenshare.Layer0.TargetBitrate",
+ kDefaultTl0BitrateKbps));
+ EXPECT_METRIC_EQ(
+ 1, metrics::NumEvents("WebRTC.Video.Screenshare.Layer1.TargetBitrate",
+ kDefaultTl1BitrateKbps));
+}
+
+TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) {
+ int64_t kTestSpanMs = 2000;
+ int64_t kFrameIntervalsMs = 1000 / kFrameRate;
+
+ uint32_t timestamp = 1234;
+ int num_input_frames = 0;
+ int num_discarded_frames = 0;
+
+ // Send at regular rate - no drops expected.
+ for (int64_t i = 0; i < kTestSpanMs; i += kFrameIntervalsMs) {
+ if (NextFrameConfig(0, timestamp).drop_frame) {
+ ++num_discarded_frames;
+ } else {
+ size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8;
+ layers_->OnEncodeDone(0, timestamp, frame_size_bytes, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ }
+ timestamp += kFrameIntervalsMs * 90;
+ clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalsMs));
+
+ ++num_input_frames;
+ }
+ EXPECT_EQ(0, num_discarded_frames);
+
+ // Send at twice the configured rate - drop every other frame.
+ num_input_frames = 0;
+ num_discarded_frames = 0;
+ for (int64_t i = 0; i < kTestSpanMs; i += kFrameIntervalsMs / 2) {
+ if (NextFrameConfig(0, timestamp).drop_frame) {
+ ++num_discarded_frames;
+ } else {
+ size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8;
+ layers_->OnEncodeDone(0, timestamp, frame_size_bytes, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+ }
+ timestamp += kFrameIntervalsMs * 90 / 2;
+ clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalsMs));
+ ++num_input_frames;
+ }
+
+ // Allow for some rounding errors in the measurements.
+ EXPECT_NEAR(num_discarded_frames, num_input_frames / 2, 2);
+}
+
+TEST_F(ScreenshareLayerTest, 2LayersSyncAtOvershootDrop) {
+ // Run grace period so we have existing frames in both TL0 and Tl1.
+ EXPECT_TRUE(RunGracePeriod());
+
+ // Move ahead until we have a sync frame in TL1.
+ EXPECT_EQ(kTl1SyncFlags, SkipUntilTlAndSync(1, true));
+ ASSERT_TRUE(tl_config_.layer_sync);
+
+ // Simulate overshoot of this frame.
+ layers_->OnEncodeDone(0, timestamp_, 0, false, 0, nullptr);
+
+ cfg_ = layers_->UpdateConfiguration(0);
+ EXPECT_EQ(kTl1SyncFlags, LibvpxVp8Encoder::EncodeFlags(tl_config_));
+
+ CodecSpecificInfo new_info;
+ layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
+ &new_info);
+ EXPECT_TRUE(new_info.codecSpecific.VP8.layerSync);
+}
+
+TEST_F(ScreenshareLayerTest, DropOnTooShortFrameInterval) {
+ // Run grace period so we have existing frames in both TL0 and Tl1.
+ EXPECT_TRUE(RunGracePeriod());
+
+ // Add a large gap, so there's plenty of room in the rate tracker.
+ timestamp_ += kTimestampDelta5Fps * 3;
+ EXPECT_FALSE(NextFrameConfig(0, timestamp_).drop_frame);
+ layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp,
+ IgnoredCodecSpecificInfo());
+
+ // Frame interval below 90% if desired time is not allowed, try inserting
+ // frame just before this limit.
+ const int64_t kMinFrameInterval = (kTimestampDelta5Fps * 85) / 100;
+ timestamp_ += kMinFrameInterval - 90;
+ EXPECT_TRUE(NextFrameConfig(0, timestamp_).drop_frame);
+
+ // Try again at the limit, now it should pass.
+ timestamp_ += 90;
+ EXPECT_FALSE(NextFrameConfig(0, timestamp_).drop_frame);
+}
+
+TEST_F(ScreenshareLayerTest, AdjustsBitrateWhenDroppingFrames) {
+ const uint32_t kTimestampDelta10Fps = kTimestampDelta5Fps / 2;
+ const int kNumFrames = 30;
+ ASSERT_TRUE(cfg_.rc_target_bitrate.has_value());
+ const uint32_t default_bitrate = cfg_.rc_target_bitrate.value();
+ layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, 10);
+
+ int num_dropped_frames = 0;
+ for (int i = 0; i < kNumFrames; ++i) {
+ if (EncodeFrame(false) == -1)
+ ++num_dropped_frames;
+ timestamp_ += kTimestampDelta10Fps;
+ }
+ cfg_ = layers_->UpdateConfiguration(0);
+
+ EXPECT_EQ(num_dropped_frames, kNumFrames / 2);
+ EXPECT_EQ(cfg_.rc_target_bitrate, default_bitrate * 2);
+}
+
+TEST_F(ScreenshareLayerTest, UpdatesConfigurationAfterRateChange) {
+ // Set inital rate again, no need to update configuration.
+ layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, kFrameRate);
+ cfg_ = layers_->UpdateConfiguration(0);
+
+ // Rate changed, now update config.
+ std::vector<uint32_t> bitrates = kDefault2TlBitratesBps;
+ bitrates[1] -= 100000;
+ layers_->OnRatesUpdated(0, bitrates, 5);
+ cfg_ = layers_->UpdateConfiguration(0);
+
+ // Changed rate, but then set changed rate again before trying to update
+ // configuration, update should still apply.
+ bitrates[1] -= 100000;
+ layers_->OnRatesUpdated(0, bitrates, 5);
+ layers_->OnRatesUpdated(0, bitrates, 5);
+ cfg_ = layers_->UpdateConfiguration(0);
+}
+
+TEST_F(ScreenshareLayerTest, MaxQpRestoredAfterDoubleDrop) {
+ // Run grace period so we have existing frames in both TL0 and Tl1.
+ EXPECT_TRUE(RunGracePeriod());
+
+ // Move ahead until we have a sync frame in TL1.
+ EXPECT_EQ(kTl1SyncFlags, SkipUntilTlAndSync(1, true));
+ ASSERT_TRUE(tl_config_.layer_sync);
+
+ // Simulate overshoot of this frame.
+ layers_->OnEncodeDone(0, timestamp_, 0, false, -1, nullptr);
+
+ // Simulate re-encoded frame.
+ layers_->OnEncodeDone(0, timestamp_, 1, false, max_qp_,
+ IgnoredCodecSpecificInfo());
+
+ // Next frame, expect boosted quality.
+ // Slightly alter bitrate between each frame.
+ std::vector<uint32_t> kDefault2TlBitratesBpsAlt = kDefault2TlBitratesBps;
+ kDefault2TlBitratesBpsAlt[1] += 4000;
+ layers_->OnRatesUpdated(0, kDefault2TlBitratesBpsAlt, kFrameRate);
+ EXPECT_EQ(kTl1Flags, SkipUntilTlAndSync(1, false));
+ EXPECT_TRUE(config_updated_);
+ EXPECT_LT(cfg_.rc_max_quantizer, max_qp_);
+ ASSERT_TRUE(cfg_.rc_max_quantizer.has_value());
+ const uint32_t adjusted_qp = cfg_.rc_max_quantizer.value();
+
+ // Simulate overshoot of this frame.
+ layers_->OnEncodeDone(0, timestamp_, 0, false, -1, nullptr);
+
+ // Simulate re-encoded frame.
+ layers_->OnEncodeDone(0, timestamp_, frame_size_, false, max_qp_,
+ IgnoredCodecSpecificInfo());
+
+ // A third frame, expect boosted quality.
+ layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, kFrameRate);
+ EXPECT_EQ(kTl1Flags, SkipUntilTlAndSync(1, false));
+ EXPECT_TRUE(config_updated_);
+ EXPECT_LT(cfg_.rc_max_quantizer, max_qp_);
+ EXPECT_EQ(adjusted_qp, cfg_.rc_max_quantizer);
+
+ // Frame encoded.
+ layers_->OnEncodeDone(0, timestamp_, frame_size_, false, max_qp_,
+ IgnoredCodecSpecificInfo());
+
+ // A fourth frame, max qp should be restored.
+ layers_->OnRatesUpdated(0, kDefault2TlBitratesBpsAlt, kFrameRate);
+ EXPECT_EQ(kTl1Flags, SkipUntilTlAndSync(1, false));
+ EXPECT_EQ(cfg_.rc_max_quantizer, max_qp_);
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/temporal_layers.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/temporal_layers.h
new file mode 100644
index 0000000000..9576fb27be
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/temporal_layers.h
@@ -0,0 +1,17 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_
+#define MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_
+
+// TODO(webrtc:9012) Remove this file when downstream projects have updated.
+#include "api/video_codecs/vp8_temporal_layers.h"
+
+#endif // MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/temporal_layers_checker.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/temporal_layers_checker.cc
new file mode 100644
index 0000000000..5aebd2c526
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/temporal_layers_checker.cc
@@ -0,0 +1,146 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h"
+
+#include <memory>
+
+#include "modules/video_coding/codecs/interface/common_constants.h"
+#include "modules/video_coding/codecs/vp8/default_temporal_layers.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+
+std::unique_ptr<TemporalLayersChecker>
+TemporalLayersChecker::CreateTemporalLayersChecker(Vp8TemporalLayersType type,
+ int num_temporal_layers) {
+ switch (type) {
+ case Vp8TemporalLayersType::kFixedPattern:
+ return std::make_unique<DefaultTemporalLayersChecker>(
+ num_temporal_layers);
+ case Vp8TemporalLayersType::kBitrateDynamic:
+ // Conference mode temporal layering for screen content in base stream.
+ return std::make_unique<TemporalLayersChecker>(num_temporal_layers);
+ }
+ RTC_CHECK_NOTREACHED();
+}
+
+TemporalLayersChecker::TemporalLayersChecker(int num_temporal_layers)
+ : num_temporal_layers_(num_temporal_layers),
+ sequence_number_(0),
+ last_sync_sequence_number_(0),
+ last_tl0_sequence_number_(0) {}
+
+bool TemporalLayersChecker::CheckAndUpdateBufferState(
+ BufferState* state,
+ bool* need_sync,
+ bool frame_is_keyframe,
+ uint8_t temporal_layer,
+ Vp8FrameConfig::BufferFlags flags,
+ uint32_t sequence_number,
+ uint32_t* lowest_sequence_referenced) {
+ if (flags & Vp8FrameConfig::BufferFlags::kReference) {
+ if (state->temporal_layer > 0 && !state->is_keyframe) {
+ *need_sync = false;
+ }
+ if (!state->is_keyframe && !frame_is_keyframe &&
+ state->sequence_number < *lowest_sequence_referenced) {
+ *lowest_sequence_referenced = state->sequence_number;
+ }
+ if (!frame_is_keyframe && !state->is_keyframe &&
+ state->temporal_layer > temporal_layer) {
+ RTC_LOG(LS_ERROR) << "Frame is referencing higher temporal layer.";
+ return false;
+ }
+ }
+ if ((flags & Vp8FrameConfig::BufferFlags::kUpdate)) {
+ state->temporal_layer = temporal_layer;
+ state->sequence_number = sequence_number;
+ state->is_keyframe = frame_is_keyframe;
+ }
+ if (frame_is_keyframe)
+ state->is_keyframe = true;
+ return true;
+}
+
+bool TemporalLayersChecker::CheckTemporalConfig(
+ bool frame_is_keyframe,
+ const Vp8FrameConfig& frame_config) {
+ if (frame_config.drop_frame ||
+ frame_config.packetizer_temporal_idx == kNoTemporalIdx) {
+ return true;
+ }
+ ++sequence_number_;
+ if (frame_config.packetizer_temporal_idx >= num_temporal_layers_ ||
+ (frame_config.packetizer_temporal_idx == kNoTemporalIdx &&
+ num_temporal_layers_ > 1)) {
+ RTC_LOG(LS_ERROR) << "Incorrect temporal layer set for frame: "
+ << frame_config.packetizer_temporal_idx
+ << " num_temporal_layers: " << num_temporal_layers_;
+ return false;
+ }
+
+ uint32_t lowest_sequence_referenced = sequence_number_;
+ bool need_sync = frame_config.packetizer_temporal_idx > 0 &&
+ frame_config.packetizer_temporal_idx != kNoTemporalIdx;
+
+ if (!CheckAndUpdateBufferState(
+ &last_, &need_sync, frame_is_keyframe,
+ frame_config.packetizer_temporal_idx, frame_config.last_buffer_flags,
+ sequence_number_, &lowest_sequence_referenced)) {
+ RTC_LOG(LS_ERROR) << "Error in the Last buffer";
+ return false;
+ }
+ if (!CheckAndUpdateBufferState(&golden_, &need_sync, frame_is_keyframe,
+ frame_config.packetizer_temporal_idx,
+ frame_config.golden_buffer_flags,
+ sequence_number_,
+ &lowest_sequence_referenced)) {
+ RTC_LOG(LS_ERROR) << "Error in the Golden buffer";
+ return false;
+ }
+ if (!CheckAndUpdateBufferState(
+ &arf_, &need_sync, frame_is_keyframe,
+ frame_config.packetizer_temporal_idx, frame_config.arf_buffer_flags,
+ sequence_number_, &lowest_sequence_referenced)) {
+ RTC_LOG(LS_ERROR) << "Error in the Arf buffer";
+ return false;
+ }
+
+ if (lowest_sequence_referenced < last_sync_sequence_number_ &&
+ !frame_is_keyframe) {
+ RTC_LOG(LS_ERROR) << "Reference past the last sync frame. Referenced "
+ << lowest_sequence_referenced << ", but sync was at "
+ << last_sync_sequence_number_;
+ return false;
+ }
+
+ if (frame_config.packetizer_temporal_idx == 0) {
+ last_tl0_sequence_number_ = sequence_number_;
+ }
+
+ if (frame_is_keyframe) {
+ last_sync_sequence_number_ = sequence_number_;
+ }
+
+ if (need_sync) {
+ last_sync_sequence_number_ = last_tl0_sequence_number_;
+ }
+
+ // Ignore sync flag on key-frames as it really doesn't matter.
+ if (need_sync != frame_config.layer_sync && !frame_is_keyframe) {
+ RTC_LOG(LS_ERROR) << "Sync bit is set incorrectly on a frame. Expected: "
+ << need_sync << " Actual: " << frame_config.layer_sync;
+ return false;
+ }
+ return true;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc
new file mode 100644
index 0000000000..c5a8b659c4
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc
@@ -0,0 +1,913 @@
+/*
+ * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <stdio.h>
+
+#include <memory>
+
+#include "api/test/create_frame_generator.h"
+#include "api/test/frame_generator_interface.h"
+#include "api/test/mock_video_decoder.h"
+#include "api/test/mock_video_encoder.h"
+#include "api/video_codecs/video_encoder.h"
+#include "api/video_codecs/vp8_temporal_layers.h"
+#include "common_video/libyuv/include/webrtc_libyuv.h"
+#include "common_video/test/utilities.h"
+#include "modules/video_coding/codecs/interface/mock_libvpx_interface.h"
+#include "modules/video_coding/codecs/test/video_codec_unittest.h"
+#include "modules/video_coding/codecs/vp8/include/vp8.h"
+#include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h"
+#include "modules/video_coding/utility/vp8_header_parser.h"
+#include "rtc_base/time_utils.h"
+#include "test/field_trial.h"
+#include "test/mappable_native_buffer.h"
+#include "test/video_codec_settings.h"
+
+namespace webrtc {
+
+using ::testing::_;
+using ::testing::AllOf;
+using ::testing::ElementsAre;
+using ::testing::ElementsAreArray;
+using ::testing::Field;
+using ::testing::Invoke;
+using ::testing::NiceMock;
+using ::testing::Return;
+using EncoderInfo = webrtc::VideoEncoder::EncoderInfo;
+using FramerateFractions =
+ absl::InlinedVector<uint8_t, webrtc::kMaxTemporalStreams>;
+
+namespace {
+constexpr uint32_t kLegacyScreenshareTl0BitrateKbps = 200;
+constexpr uint32_t kLegacyScreenshareTl1BitrateKbps = 1000;
+constexpr uint32_t kInitialTimestampRtp = 123;
+constexpr int64_t kTestNtpTimeMs = 456;
+constexpr int64_t kInitialTimestampMs = 789;
+constexpr int kNumCores = 1;
+constexpr size_t kMaxPayloadSize = 1440;
+constexpr int kWidth = 172;
+constexpr int kHeight = 144;
+constexpr float kFramerateFps = 30;
+
+const VideoEncoder::Capabilities kCapabilities(false);
+const VideoEncoder::Settings kSettings(kCapabilities,
+ kNumCores,
+ kMaxPayloadSize);
+} // namespace
+
+class TestVp8Impl : public VideoCodecUnitTest {
+ protected:
+ std::unique_ptr<VideoEncoder> CreateEncoder() override {
+ return VP8Encoder::Create();
+ }
+
+ std::unique_ptr<VideoDecoder> CreateDecoder() override {
+ return VP8Decoder::Create();
+ }
+
+ void ModifyCodecSettings(VideoCodec* codec_settings) override {
+ webrtc::test::CodecSettings(kVideoCodecVP8, codec_settings);
+ codec_settings->width = kWidth;
+ codec_settings->height = kHeight;
+ codec_settings->SetVideoEncoderComplexity(
+ VideoCodecComplexity::kComplexityNormal);
+ }
+
+ void EncodeAndWaitForFrame(const VideoFrame& input_frame,
+ EncodedImage* encoded_frame,
+ CodecSpecificInfo* codec_specific_info,
+ bool keyframe = false) {
+ std::vector<VideoFrameType> frame_types;
+ if (keyframe) {
+ frame_types.emplace_back(VideoFrameType::kVideoFrameKey);
+ } else {
+ frame_types.emplace_back(VideoFrameType::kVideoFrameDelta);
+ }
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->Encode(input_frame, &frame_types));
+ ASSERT_TRUE(WaitForEncodedFrame(encoded_frame, codec_specific_info));
+ VerifyQpParser(*encoded_frame);
+ EXPECT_EQ(kVideoCodecVP8, codec_specific_info->codecType);
+ EXPECT_EQ(0, encoded_frame->SpatialIndex());
+ }
+
+ void EncodeAndExpectFrameWith(const VideoFrame& input_frame,
+ uint8_t temporal_idx,
+ bool keyframe = false) {
+ EncodedImage encoded_frame;
+ CodecSpecificInfo codec_specific_info;
+ EncodeAndWaitForFrame(input_frame, &encoded_frame, &codec_specific_info,
+ keyframe);
+ EXPECT_EQ(temporal_idx, codec_specific_info.codecSpecific.VP8.temporalIdx);
+ }
+
+ void VerifyQpParser(const EncodedImage& encoded_frame) const {
+ int qp;
+ EXPECT_GT(encoded_frame.size(), 0u);
+ ASSERT_TRUE(vp8::GetQp(encoded_frame.data(), encoded_frame.size(), &qp));
+ EXPECT_EQ(encoded_frame.qp_, qp) << "Encoder QP != parsed bitstream QP.";
+ }
+};
+
+TEST_F(TestVp8Impl, ErrorResilienceDisabledForNoTemporalLayers) {
+ codec_settings_.simulcastStream[0].numberOfTemporalLayers = 1;
+
+ auto* const vpx = new NiceMock<MockLibvpxInterface>();
+ LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
+ VP8Encoder::Settings());
+ EXPECT_CALL(*vpx,
+ codec_enc_init(
+ _, _, Field(&vpx_codec_enc_cfg_t::g_error_resilient, 0), _));
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder.InitEncode(&codec_settings_, kSettings));
+}
+
+TEST_F(TestVp8Impl, DefaultErrorResilienceEnabledForTemporalLayers) {
+ codec_settings_.simulcastStream[0].numberOfTemporalLayers = 2;
+ codec_settings_.VP8()->numberOfTemporalLayers = 2;
+
+ auto* const vpx = new NiceMock<MockLibvpxInterface>();
+ LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
+ VP8Encoder::Settings());
+ EXPECT_CALL(*vpx,
+ codec_enc_init(_, _,
+ Field(&vpx_codec_enc_cfg_t::g_error_resilient,
+ VPX_ERROR_RESILIENT_DEFAULT),
+ _));
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder.InitEncode(&codec_settings_, kSettings));
+}
+
+TEST_F(TestVp8Impl,
+ PartitionErrorResilienceEnabledForTemporalLayersWithFieldTrial) {
+ test::ScopedFieldTrials field_trials(
+ "WebRTC-VP8-ForcePartitionResilience/Enabled/");
+ codec_settings_.simulcastStream[0].numberOfTemporalLayers = 2;
+ codec_settings_.VP8()->numberOfTemporalLayers = 2;
+
+ auto* const vpx = new NiceMock<MockLibvpxInterface>();
+ LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
+ VP8Encoder::Settings());
+ EXPECT_CALL(*vpx,
+ codec_enc_init(_, _,
+ Field(&vpx_codec_enc_cfg_t::g_error_resilient,
+ VPX_ERROR_RESILIENT_PARTITIONS),
+ _));
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder.InitEncode(&codec_settings_, kSettings));
+}
+
+TEST_F(TestVp8Impl, SetRates) {
+ codec_settings_.SetFrameDropEnabled(true);
+ auto* const vpx = new NiceMock<MockLibvpxInterface>();
+ LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
+ VP8Encoder::Settings());
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder.InitEncode(&codec_settings_,
+ VideoEncoder::Settings(kCapabilities, 1, 1000)));
+
+ const uint32_t kBitrateBps = 300000;
+ VideoBitrateAllocation bitrate_allocation;
+ bitrate_allocation.SetBitrate(0, 0, kBitrateBps);
+ EXPECT_CALL(
+ *vpx,
+ codec_enc_config_set(
+ _, AllOf(Field(&vpx_codec_enc_cfg_t::rc_target_bitrate,
+ kBitrateBps / 1000),
+ Field(&vpx_codec_enc_cfg_t::rc_undershoot_pct, 100u),
+ Field(&vpx_codec_enc_cfg_t::rc_overshoot_pct, 15u),
+ Field(&vpx_codec_enc_cfg_t::rc_buf_sz, 1000u),
+ Field(&vpx_codec_enc_cfg_t::rc_buf_optimal_sz, 600u),
+ Field(&vpx_codec_enc_cfg_t::rc_dropframe_thresh, 30u))))
+ .WillOnce(Return(VPX_CODEC_OK));
+ encoder.SetRates(VideoEncoder::RateControlParameters(
+ bitrate_allocation, static_cast<double>(codec_settings_.maxFramerate)));
+}
+
+TEST_F(TestVp8Impl, EncodeFrameAndRelease) {
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+
+ EncodedImage encoded_frame;
+ CodecSpecificInfo codec_specific_info;
+ EncodeAndWaitForFrame(NextInputFrame(), &encoded_frame, &codec_specific_info);
+
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_UNINITIALIZED,
+ encoder_->Encode(NextInputFrame(), nullptr));
+}
+
+TEST_F(TestVp8Impl, EncodeNv12FrameSimulcast) {
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+
+ EncodedImage encoded_frame;
+ CodecSpecificInfo codec_specific_info;
+ input_frame_generator_ = test::CreateSquareFrameGenerator(
+ kWidth, kHeight, test::FrameGeneratorInterface::OutputType::kNV12,
+ absl::nullopt);
+ EncodeAndWaitForFrame(NextInputFrame(), &encoded_frame, &codec_specific_info);
+
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_UNINITIALIZED,
+ encoder_->Encode(NextInputFrame(), nullptr));
+}
+
+TEST_F(TestVp8Impl, EncodeI420FrameAfterNv12Frame) {
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+
+ EncodedImage encoded_frame;
+ CodecSpecificInfo codec_specific_info;
+ input_frame_generator_ = test::CreateSquareFrameGenerator(
+ kWidth, kHeight, test::FrameGeneratorInterface::OutputType::kNV12,
+ absl::nullopt);
+ EncodeAndWaitForFrame(NextInputFrame(), &encoded_frame, &codec_specific_info);
+ input_frame_generator_ = test::CreateSquareFrameGenerator(
+ kWidth, kHeight, test::FrameGeneratorInterface::OutputType::kI420,
+ absl::nullopt);
+ EncodeAndWaitForFrame(NextInputFrame(), &encoded_frame, &codec_specific_info);
+
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_UNINITIALIZED,
+ encoder_->Encode(NextInputFrame(), nullptr));
+}
+
+TEST_F(TestVp8Impl, Configure) {
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release());
+ EXPECT_TRUE(decoder_->Configure({}));
+}
+
+TEST_F(TestVp8Impl, OnEncodedImageReportsInfo) {
+ VideoFrame input_frame = NextInputFrame();
+ input_frame.set_timestamp(kInitialTimestampRtp);
+ input_frame.set_timestamp_us(kInitialTimestampMs *
+ rtc::kNumMicrosecsPerMillisec);
+ EncodedImage encoded_frame;
+ CodecSpecificInfo codec_specific_info;
+ EncodeAndWaitForFrame(input_frame, &encoded_frame, &codec_specific_info);
+
+ EXPECT_EQ(kInitialTimestampRtp, encoded_frame.Timestamp());
+ EXPECT_EQ(kWidth, static_cast<int>(encoded_frame._encodedWidth));
+ EXPECT_EQ(kHeight, static_cast<int>(encoded_frame._encodedHeight));
+}
+
+TEST_F(TestVp8Impl,
+ EncoderFillsResolutionInCodecAgnosticSectionOfCodecSpecificInfo) {
+ EncodedImage encoded_frame;
+ CodecSpecificInfo codec_specific_info;
+ EncodeAndWaitForFrame(NextInputFrame(), &encoded_frame, &codec_specific_info);
+
+ ASSERT_TRUE(codec_specific_info.template_structure);
+ EXPECT_THAT(codec_specific_info.template_structure->resolutions,
+ ElementsAre(RenderResolution(kWidth, kHeight)));
+}
+
+TEST_F(TestVp8Impl, DecodedQpEqualsEncodedQp) {
+ VideoFrame input_frame = NextInputFrame();
+ EncodedImage encoded_frame;
+ CodecSpecificInfo codec_specific_info;
+ EncodeAndWaitForFrame(input_frame, &encoded_frame, &codec_specific_info);
+
+ // First frame should be a key frame.
+ encoded_frame._frameType = VideoFrameType::kVideoFrameKey;
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, false, -1));
+ std::unique_ptr<VideoFrame> decoded_frame;
+ absl::optional<uint8_t> decoded_qp;
+ ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
+ ASSERT_TRUE(decoded_frame);
+ ASSERT_TRUE(decoded_qp);
+ EXPECT_GT(I420PSNR(&input_frame, decoded_frame.get()), 36);
+ EXPECT_EQ(encoded_frame.qp_, *decoded_qp);
+}
+
+TEST_F(TestVp8Impl, ChecksSimulcastSettings) {
+ codec_settings_.numberOfSimulcastStreams = 2;
+ // Resolutions are not in ascending order, temporal layers do not match.
+ codec_settings_.simulcastStream[0] = {.width = kWidth,
+ .height = kHeight,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 2,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ codec_settings_.simulcastStream[1] = {.width = kWidth / 2,
+ .height = kHeight / 2,
+ .maxFramerate = 30,
+ .numberOfTemporalLayers = 3,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+ codec_settings_.numberOfSimulcastStreams = 3;
+ // Resolutions are not in ascending order.
+ codec_settings_.simulcastStream[0] = {.width = kWidth / 2,
+ .height = kHeight / 2,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ codec_settings_.simulcastStream[1] = {.width = kWidth / 2 - 1,
+ .height = kHeight / 2 - 1,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ codec_settings_.simulcastStream[2] = {.width = kWidth,
+ .height = kHeight,
+ .maxFramerate = 30,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+ // Resolutions are not in ascending order.
+ codec_settings_.simulcastStream[0] = {.width = kWidth,
+ .height = kHeight,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ codec_settings_.simulcastStream[1] = {.width = kWidth,
+ .height = kHeight,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ codec_settings_.simulcastStream[2] = {.width = kWidth - 1,
+ .height = kHeight - 1,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+ // Temporal layers do not match.
+ codec_settings_.simulcastStream[0] = {.width = kWidth / 4,
+ .height = kHeight / 4,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ codec_settings_.simulcastStream[1] = {.width = kWidth / 2,
+ .height = kHeight / 2,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 2,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ codec_settings_.simulcastStream[2] = {.width = kWidth,
+ .height = kHeight,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 3,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+ // Resolutions do not match codec config.
+ codec_settings_.simulcastStream[0] = {.width = kWidth / 4 + 1,
+ .height = kHeight / 4 + 1,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ codec_settings_.simulcastStream[1] = {.width = kWidth / 2 + 2,
+ .height = kHeight / 2 + 2,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ codec_settings_.simulcastStream[2] = {.width = kWidth + 4,
+ .height = kHeight + 4,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+ // Everything fine: scaling by 2, top resolution matches video, temporal
+ // settings are the same for all layers.
+ codec_settings_.simulcastStream[0] = {.width = kWidth / 4,
+ .height = kHeight / 4,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ codec_settings_.simulcastStream[1] = {.width = kWidth / 2,
+ .height = kHeight / 2,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ codec_settings_.simulcastStream[2] = {.width = kWidth,
+ .height = kHeight,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+ // Everything fine: custom scaling, top resolution matches video, temporal
+ // settings are the same for all layers.
+ codec_settings_.simulcastStream[0] = {.width = kWidth / 4,
+ .height = kHeight / 4,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ codec_settings_.simulcastStream[1] = {.width = kWidth,
+ .height = kHeight,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ codec_settings_.simulcastStream[2] = {.width = kWidth,
+ .height = kHeight,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80};
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+}
+
+#if defined(WEBRTC_ANDROID)
+#define MAYBE_AlignedStrideEncodeDecode DISABLED_AlignedStrideEncodeDecode
+#else
+#define MAYBE_AlignedStrideEncodeDecode AlignedStrideEncodeDecode
+#endif
+TEST_F(TestVp8Impl, MAYBE_AlignedStrideEncodeDecode) {
+ VideoFrame input_frame = NextInputFrame();
+ input_frame.set_timestamp(kInitialTimestampRtp);
+ input_frame.set_timestamp_us(kInitialTimestampMs *
+ rtc::kNumMicrosecsPerMillisec);
+ EncodedImage encoded_frame;
+ CodecSpecificInfo codec_specific_info;
+ EncodeAndWaitForFrame(input_frame, &encoded_frame, &codec_specific_info);
+
+ // First frame should be a key frame.
+ encoded_frame._frameType = VideoFrameType::kVideoFrameKey;
+ encoded_frame.ntp_time_ms_ = kTestNtpTimeMs;
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, false, -1));
+
+ std::unique_ptr<VideoFrame> decoded_frame;
+ absl::optional<uint8_t> decoded_qp;
+ ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
+ ASSERT_TRUE(decoded_frame);
+ // Compute PSNR on all planes (faster than SSIM).
+ EXPECT_GT(I420PSNR(&input_frame, decoded_frame.get()), 36);
+ EXPECT_EQ(kInitialTimestampRtp, decoded_frame->timestamp());
+}
+
+TEST_F(TestVp8Impl, EncoderWith2TemporalLayers) {
+ codec_settings_.VP8()->numberOfTemporalLayers = 2;
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+
+ // Temporal layer 0.
+ EncodedImage encoded_frame;
+ CodecSpecificInfo codec_specific_info;
+ EncodeAndWaitForFrame(NextInputFrame(), &encoded_frame, &codec_specific_info);
+
+ EXPECT_EQ(0, codec_specific_info.codecSpecific.VP8.temporalIdx);
+ // Temporal layer 1.
+ EncodeAndExpectFrameWith(NextInputFrame(), 1);
+ // Temporal layer 0.
+ EncodeAndExpectFrameWith(NextInputFrame(), 0);
+ // Temporal layer 1.
+ EncodeAndExpectFrameWith(NextInputFrame(), 1);
+}
+
+TEST_F(TestVp8Impl, ScalingDisabledIfAutomaticResizeOff) {
+ codec_settings_.VP8()->automaticResizeOn = false;
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+
+ VideoEncoder::ScalingSettings settings =
+ encoder_->GetEncoderInfo().scaling_settings;
+ EXPECT_FALSE(settings.thresholds.has_value());
+}
+
+TEST_F(TestVp8Impl, ScalingEnabledIfAutomaticResizeOn) {
+ codec_settings_.SetFrameDropEnabled(true);
+ codec_settings_.VP8()->automaticResizeOn = true;
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+
+ VideoEncoder::ScalingSettings settings =
+ encoder_->GetEncoderInfo().scaling_settings;
+ EXPECT_TRUE(settings.thresholds.has_value());
+ EXPECT_EQ(kDefaultMinPixelsPerFrame, settings.min_pixels_per_frame);
+}
+
+TEST_F(TestVp8Impl, DontDropKeyframes) {
+ // Set very high resolution to trigger overuse more easily.
+ const int kScreenWidth = 1920;
+ const int kScreenHeight = 1080;
+
+ codec_settings_.width = kScreenWidth;
+ codec_settings_.height = kScreenHeight;
+
+ // Screensharing has the internal frame dropper off, and instead per frame
+ // asks ScreenshareLayers to decide if it should be dropped or not.
+ codec_settings_.SetFrameDropEnabled(false);
+ codec_settings_.mode = VideoCodecMode::kScreensharing;
+ // ScreenshareLayers triggers on 2 temporal layers and 1000kbps max bitrate.
+ codec_settings_.VP8()->numberOfTemporalLayers = 2;
+ codec_settings_.maxBitrate = 1000;
+
+ // Reset the frame generator with large number of squares, leading to lots of
+ // details and high probability of overshoot.
+ input_frame_generator_ = test::CreateSquareFrameGenerator(
+ codec_settings_.width, codec_settings_.height,
+ test::FrameGeneratorInterface::OutputType::kI420,
+ /* num_squares = */ absl::optional<int>(300));
+
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+
+ VideoBitrateAllocation bitrate_allocation;
+ // Bitrate only enough for TL0.
+ bitrate_allocation.SetBitrate(0, 0, 200000);
+ encoder_->SetRates(
+ VideoEncoder::RateControlParameters(bitrate_allocation, 5.0));
+
+ EncodedImage encoded_frame;
+ CodecSpecificInfo codec_specific_info;
+ EncodeAndWaitForFrame(NextInputFrame(), &encoded_frame, &codec_specific_info,
+ true);
+ EncodeAndExpectFrameWith(NextInputFrame(), 0, true);
+ EncodeAndExpectFrameWith(NextInputFrame(), 0, true);
+ EncodeAndExpectFrameWith(NextInputFrame(), 0, true);
+}
+
+TEST_F(TestVp8Impl, KeepsTimestampOnReencode) {
+ auto* const vpx = new NiceMock<MockLibvpxInterface>();
+ LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
+ VP8Encoder::Settings());
+
+ // Settings needed to trigger ScreenshareLayers usage, which is required for
+ // overshoot-drop-reencode logic.
+ codec_settings_.maxBitrate = 1000;
+ codec_settings_.mode = VideoCodecMode::kScreensharing;
+ codec_settings_.VP8()->numberOfTemporalLayers = 2;
+ codec_settings_.legacy_conference_mode = true;
+
+ EXPECT_CALL(*vpx, img_wrap(_, _, _, _, _, _))
+ .WillOnce(Invoke([](vpx_image_t* img, vpx_img_fmt_t fmt, unsigned int d_w,
+ unsigned int d_h, unsigned int stride_align,
+ unsigned char* img_data) {
+ img->fmt = fmt;
+ img->d_w = d_w;
+ img->d_h = d_h;
+ img->img_data = img_data;
+ return img;
+ }));
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder.InitEncode(&codec_settings_,
+ VideoEncoder::Settings(kCapabilities, 1, 1000)));
+ MockEncodedImageCallback callback;
+ encoder.RegisterEncodeCompleteCallback(&callback);
+
+ // Simulate overshoot drop, re-encode: encode function will be called twice
+ // with the same parameters. codec_get_cx_data() will by default return no
+ // image data and be interpreted as drop.
+ EXPECT_CALL(*vpx, codec_encode(_, _, /* pts = */ 0, _, _, _))
+ .Times(2)
+ .WillRepeatedly(Return(vpx_codec_err_t::VPX_CODEC_OK));
+
+ auto delta_frame =
+ std::vector<VideoFrameType>{VideoFrameType::kVideoFrameDelta};
+ encoder.Encode(NextInputFrame(), &delta_frame);
+}
+
+TEST(LibvpxVp8EncoderTest, GetEncoderInfoReturnsStaticInformation) {
+ auto* const vpx = new NiceMock<MockLibvpxInterface>();
+ LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
+ VP8Encoder::Settings());
+
+ const auto info = encoder.GetEncoderInfo();
+
+ EXPECT_FALSE(info.supports_native_handle);
+ EXPECT_FALSE(info.is_hardware_accelerated);
+ EXPECT_TRUE(info.supports_simulcast);
+ EXPECT_EQ(info.implementation_name, "libvpx");
+ EXPECT_EQ(info.requested_resolution_alignment, 1u);
+ EXPECT_THAT(info.preferred_pixel_formats,
+ testing::UnorderedElementsAre(VideoFrameBuffer::Type::kNV12,
+ VideoFrameBuffer::Type::kI420));
+}
+
+TEST(LibvpxVp8EncoderTest, RequestedResolutionAlignmentFromFieldTrial) {
+ test::ScopedFieldTrials field_trials(
+ "WebRTC-VP8-GetEncoderInfoOverride/"
+ "requested_resolution_alignment:10/");
+
+ auto* const vpx = new NiceMock<MockLibvpxInterface>();
+ LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
+ VP8Encoder::Settings());
+
+ EXPECT_EQ(encoder.GetEncoderInfo().requested_resolution_alignment, 10u);
+ EXPECT_FALSE(
+ encoder.GetEncoderInfo().apply_alignment_to_all_simulcast_layers);
+ EXPECT_TRUE(encoder.GetEncoderInfo().resolution_bitrate_limits.empty());
+}
+
+TEST(LibvpxVp8EncoderTest, ResolutionBitrateLimitsFromFieldTrial) {
+ test::ScopedFieldTrials field_trials(
+ "WebRTC-VP8-GetEncoderInfoOverride/"
+ "frame_size_pixels:123|456|789,"
+ "min_start_bitrate_bps:11000|22000|33000,"
+ "min_bitrate_bps:44000|55000|66000,"
+ "max_bitrate_bps:77000|88000|99000/");
+
+ auto* const vpx = new NiceMock<MockLibvpxInterface>();
+ LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
+ VP8Encoder::Settings());
+
+ EXPECT_THAT(
+ encoder.GetEncoderInfo().resolution_bitrate_limits,
+ ::testing::ElementsAre(
+ VideoEncoder::ResolutionBitrateLimits{123, 11000, 44000, 77000},
+ VideoEncoder::ResolutionBitrateLimits{456, 22000, 55000, 88000},
+ VideoEncoder::ResolutionBitrateLimits{789, 33000, 66000, 99000}));
+}
+
+TEST(LibvpxVp8EncoderTest,
+ GetEncoderInfoReturnsEmptyResolutionBitrateLimitsByDefault) {
+ auto* const vpx = new NiceMock<MockLibvpxInterface>();
+ LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
+ VP8Encoder::Settings());
+
+ const auto info = encoder.GetEncoderInfo();
+
+ EXPECT_TRUE(info.resolution_bitrate_limits.empty());
+}
+
+TEST(LibvpxVp8EncoderTest,
+ GetEncoderInfoReturnsResolutionBitrateLimitsAsConfigured) {
+ std::vector<VideoEncoder::ResolutionBitrateLimits> resolution_bitrate_limits =
+ {VideoEncoder::ResolutionBitrateLimits(/*frame_size_pixels=*/640 * 360,
+ /*min_start_bitrate_bps=*/300,
+ /*min_bitrate_bps=*/100,
+ /*max_bitrate_bps=*/1000),
+ VideoEncoder::ResolutionBitrateLimits(320 * 180, 100, 30, 500)};
+ VP8Encoder::Settings settings;
+ settings.resolution_bitrate_limits = resolution_bitrate_limits;
+
+ auto* const vpx = new NiceMock<MockLibvpxInterface>();
+ LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)),
+ std::move(settings));
+
+ const auto info = encoder.GetEncoderInfo();
+
+ EXPECT_EQ(info.resolution_bitrate_limits, resolution_bitrate_limits);
+}
+
+TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationNoLayers) {
+ FramerateFractions expected_fps_allocation[kMaxSpatialLayers] = {
+ FramerateFractions(1, EncoderInfo::kMaxFramerateFraction)};
+
+ EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
+ ::testing::ElementsAreArray(expected_fps_allocation));
+}
+
+TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationTwoTemporalLayers) {
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
+ codec_settings_.numberOfSimulcastStreams = 1;
+ codec_settings_.simulcastStream[0].active = true;
+ codec_settings_.simulcastStream[0].targetBitrate = 100;
+ codec_settings_.simulcastStream[0].maxBitrate = 100;
+ codec_settings_.simulcastStream[0].numberOfTemporalLayers = 2;
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+
+ FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
+ expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2);
+ expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction);
+
+ EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
+ ::testing::ElementsAreArray(expected_fps_allocation));
+}
+
+TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationThreeTemporalLayers) {
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
+ codec_settings_.numberOfSimulcastStreams = 1;
+ codec_settings_.simulcastStream[0].active = true;
+ codec_settings_.simulcastStream[0].targetBitrate = 100;
+ codec_settings_.simulcastStream[0].maxBitrate = 100;
+ codec_settings_.simulcastStream[0].numberOfTemporalLayers = 3;
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+
+ FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
+ expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 4);
+ expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2);
+ expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction);
+
+ EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
+ ::testing::ElementsAreArray(expected_fps_allocation));
+}
+
+TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationScreenshareLayers) {
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
+ codec_settings_.numberOfSimulcastStreams = 1;
+ codec_settings_.mode = VideoCodecMode::kScreensharing;
+ codec_settings_.simulcastStream[0].active = true;
+ codec_settings_.simulcastStream[0].minBitrate = 30;
+ codec_settings_.simulcastStream[0].targetBitrate =
+ kLegacyScreenshareTl0BitrateKbps;
+ codec_settings_.simulcastStream[0].maxBitrate =
+ kLegacyScreenshareTl1BitrateKbps;
+ codec_settings_.simulcastStream[0].numberOfTemporalLayers = 2;
+ codec_settings_.legacy_conference_mode = true;
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+
+ // Expect empty vector, since this mode doesn't have a fixed framerate.
+ FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
+ EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
+ ::testing::ElementsAreArray(expected_fps_allocation));
+}
+
+TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationSimulcastVideo) {
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
+
+ // Set up three simulcast streams with three temporal layers each.
+ codec_settings_.numberOfSimulcastStreams = 3;
+ for (int i = 0; i < codec_settings_.numberOfSimulcastStreams; ++i) {
+ codec_settings_.simulcastStream[i].active = true;
+ codec_settings_.simulcastStream[i].minBitrate = 30;
+ codec_settings_.simulcastStream[i].targetBitrate = 30;
+ codec_settings_.simulcastStream[i].maxBitrate = 30;
+ codec_settings_.simulcastStream[i].numberOfTemporalLayers = 3;
+ codec_settings_.simulcastStream[i].width =
+ codec_settings_.width >>
+ (codec_settings_.numberOfSimulcastStreams - i - 1);
+ codec_settings_.simulcastStream[i].height =
+ codec_settings_.height >>
+ (codec_settings_.numberOfSimulcastStreams - i - 1);
+ }
+
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+
+ FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
+ expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 4);
+ expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2);
+ expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction);
+ expected_fps_allocation[1] = expected_fps_allocation[0];
+ expected_fps_allocation[2] = expected_fps_allocation[0];
+ EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
+ ::testing::ElementsAreArray(expected_fps_allocation));
+
+ // Release encoder and re-init without temporal layers.
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
+
+ // Sanity check fps allocation when not inited.
+ FramerateFractions default_fps_fraction[kMaxSpatialLayers];
+ default_fps_fraction[0].push_back(EncoderInfo::kMaxFramerateFraction);
+ EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
+ ::testing::ElementsAreArray(default_fps_fraction));
+
+ for (int i = 0; i < codec_settings_.numberOfSimulcastStreams; ++i) {
+ codec_settings_.simulcastStream[i].numberOfTemporalLayers = 1;
+ }
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+
+ for (size_t i = 0; i < 3; ++i) {
+ expected_fps_allocation[i].clear();
+ expected_fps_allocation[i].push_back(EncoderInfo::kMaxFramerateFraction);
+ }
+ EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
+ ::testing::ElementsAreArray(expected_fps_allocation));
+}
+
+class TestVp8ImplForPixelFormat
+ : public TestVp8Impl,
+ public ::testing::WithParamInterface<VideoFrameBuffer::Type> {
+ public:
+ TestVp8ImplForPixelFormat() : TestVp8Impl(), mappable_type_(GetParam()) {}
+
+ protected:
+ VideoFrameBuffer::Type mappable_type_;
+};
+
+TEST_P(TestVp8ImplForPixelFormat, EncodeNativeFrameSimulcast) {
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
+
+ // Configure simulcast.
+ codec_settings_.numberOfSimulcastStreams = 3;
+ codec_settings_.simulcastStream[0] = {.width = kWidth / 4,
+ .height = kHeight / 4,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80,
+ .active = true};
+ codec_settings_.simulcastStream[1] = {.width = kWidth / 2,
+ .height = kHeight / 2,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80,
+ .active = true};
+ codec_settings_.simulcastStream[2] = {.width = kWidth,
+ .height = kHeight,
+ .maxFramerate = kFramerateFps,
+ .numberOfTemporalLayers = 1,
+ .maxBitrate = 4000,
+ .targetBitrate = 3000,
+ .minBitrate = 2000,
+ .qpMax = 80,
+ .active = true};
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+ encoder_->InitEncode(&codec_settings_, kSettings));
+
+ // Create a zero-conversion NV12 frame (calling ToI420 on it crashes).
+ VideoFrame input_frame =
+ test::CreateMappableNativeFrame(1, mappable_type_, kWidth, kHeight);
+
+ EncodedImage encoded_frame;
+ CodecSpecificInfo codec_specific_info;
+ EncodeAndWaitForFrame(input_frame, &encoded_frame, &codec_specific_info);
+
+ // After encoding, we expect one mapping per simulcast layer.
+ rtc::scoped_refptr<test::MappableNativeBuffer> mappable_buffer =
+ test::GetMappableNativeBufferFromVideoFrame(input_frame);
+ std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_buffers =
+ mappable_buffer->GetMappedFramedBuffers();
+ ASSERT_EQ(mapped_buffers.size(), 3u);
+ EXPECT_EQ(mapped_buffers[0]->type(), mappable_type_);
+ EXPECT_EQ(mapped_buffers[0]->width(), kWidth);
+ EXPECT_EQ(mapped_buffers[0]->height(), kHeight);
+ EXPECT_EQ(mapped_buffers[1]->type(), mappable_type_);
+ EXPECT_EQ(mapped_buffers[1]->width(), kWidth / 2);
+ EXPECT_EQ(mapped_buffers[1]->height(), kHeight / 2);
+ EXPECT_EQ(mapped_buffers[2]->type(), mappable_type_);
+ EXPECT_EQ(mapped_buffers[2]->width(), kWidth / 4);
+ EXPECT_EQ(mapped_buffers[2]->height(), kHeight / 4);
+ EXPECT_FALSE(mappable_buffer->DidConvertToI420());
+
+ EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
+}
+
+INSTANTIATE_TEST_SUITE_P(All,
+ TestVp8ImplForPixelFormat,
+ ::testing::Values(VideoFrameBuffer::Type::kI420,
+ VideoFrameBuffer::Type::kNV12));
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/vp8_scalability.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/vp8_scalability.cc
new file mode 100644
index 0000000000..9c7495ddf7
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/vp8_scalability.cc
@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/vp8/vp8_scalability.h"
+
+namespace webrtc {
+
+bool VP8SupportsScalabilityMode(ScalabilityMode scalability_mode) {
+ for (const auto& entry : kVP8SupportedScalabilityModes) {
+ if (entry == scalability_mode) {
+ return true;
+ }
+ }
+ return false;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/vp8_scalability.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/vp8_scalability.h
new file mode 100644
index 0000000000..923f159118
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/vp8_scalability.h
@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_CODECS_VP8_VP8_SCALABILITY_H_
+#define MODULES_VIDEO_CODING_CODECS_VP8_VP8_SCALABILITY_H_
+
+#include "api/video_codecs/scalability_mode.h"
+
+namespace webrtc {
+
+inline constexpr ScalabilityMode kVP8SupportedScalabilityModes[] = {
+ ScalabilityMode::kL1T1, ScalabilityMode::kL1T2, ScalabilityMode::kL1T3};
+bool VP8SupportsScalabilityMode(ScalabilityMode scalability_mode);
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_CODECS_VP8_VP8_SCALABILITY_H_