diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 19:33:14 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 19:33:14 +0000 |
commit | 36d22d82aa202bb199967e9512281e9a53db42c9 (patch) | |
tree | 105e8c98ddea1c1e4784a60a5a6410fa416be2de /third_party/libwebrtc/modules/video_coding/codecs/vp8 | |
parent | Initial commit. (diff) | |
download | firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.tar.xz firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.zip |
Adding upstream version 115.7.0esr.upstream/115.7.0esr
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/libwebrtc/modules/video_coding/codecs/vp8')
19 files changed, 6862 insertions, 0 deletions
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc new file mode 100644 index 0000000000..94860da1b6 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -0,0 +1,884 @@ +/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/vp8/default_temporal_layers.h" + +#include <stdlib.h> + +#include <algorithm> +#include <array> +#include <memory> +#include <set> +#include <utility> +#include <vector> + +#include "modules/video_coding/include/video_codec_interface.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { +DefaultTemporalLayers::PendingFrame::PendingFrame() = default; +DefaultTemporalLayers::PendingFrame::PendingFrame( + uint32_t timestamp, + bool expired, + uint8_t updated_buffers_mask, + const DependencyInfo& dependency_info) + : timestamp(timestamp), + expired(expired), + updated_buffer_mask(updated_buffers_mask), + dependency_info(dependency_info) {} + +namespace { +using BufferFlags = Vp8FrameConfig::BufferFlags; +using FreezeEntropy = Vp8FrameConfig::FreezeEntropy; +using Vp8BufferReference = Vp8FrameConfig::Vp8BufferReference; + +constexpr BufferFlags kNone = BufferFlags::kNone; +constexpr BufferFlags kReference = BufferFlags::kReference; +constexpr BufferFlags kUpdate = BufferFlags::kUpdate; +constexpr BufferFlags kReferenceAndUpdate = BufferFlags::kReferenceAndUpdate; +constexpr FreezeEntropy kFreezeEntropy = FreezeEntropy::kFreezeEntropy; + +static constexpr uint8_t kUninitializedPatternIndex = + std::numeric_limits<uint8_t>::max(); +static constexpr std::array<Vp8BufferReference, 3> kAllBuffers = { + {Vp8BufferReference::kLast, Vp8BufferReference::kGolden, + Vp8BufferReference::kAltref}}; + +std::vector<unsigned int> GetTemporalIds(size_t num_layers) { + switch (num_layers) { + case 1: + // Temporal layer structure (single layer): + // 0 0 0 0 ... + return {0}; + case 2: + // Temporal layer structure: + // 1 1 ... + // 0 0 ... + return {0, 1}; + case 3: + // Temporal layer structure: + // 2 2 2 2 ... + // 1 1 ... + // 0 0 ... + return {0, 2, 1, 2}; + case 4: + // Temporal layer structure: + // 3 3 3 3 3 3 3 3 ... + // 2 2 2 2 ... + // 1 1 ... + // 0 0 ... + return {0, 3, 2, 3, 1, 3, 2, 3}; + default: + RTC_DCHECK_NOTREACHED(); + break; + } + RTC_DCHECK_NOTREACHED(); + return {0}; +} + +uint8_t GetUpdatedBuffers(const Vp8FrameConfig& config) { + uint8_t flags = 0; + if (config.last_buffer_flags & BufferFlags::kUpdate) { + flags |= static_cast<uint8_t>(Vp8BufferReference::kLast); + } + if (config.golden_buffer_flags & BufferFlags::kUpdate) { + flags |= static_cast<uint8_t>(Vp8BufferReference::kGolden); + } + if (config.arf_buffer_flags & BufferFlags::kUpdate) { + flags |= static_cast<uint8_t>(Vp8BufferReference::kAltref); + } + return flags; +} + +size_t BufferToIndex(Vp8BufferReference buffer) { + switch (buffer) { + case Vp8FrameConfig::Vp8BufferReference::kLast: + return 0; + case Vp8FrameConfig::Vp8BufferReference::kGolden: + return 1; + case Vp8FrameConfig::Vp8BufferReference::kAltref: + return 2; + case Vp8FrameConfig::Vp8BufferReference::kNone: + RTC_CHECK_NOTREACHED(); + } +} + +} // namespace + +constexpr size_t DefaultTemporalLayers::kNumReferenceBuffers; + +std::vector<DefaultTemporalLayers::DependencyInfo> +DefaultTemporalLayers::GetDependencyInfo(size_t num_layers) { + // For indexing in the patterns described below (which temporal layers they + // belong to), see the diagram above. + // Layer sync is done similarly for all patterns (except single stream) and + // happens every 8 frames: + // TL1 layer syncs by periodically by only referencing TL0 ('last'), but still + // updating 'golden', so it can be used as a reference by future TL1 frames. + // TL2 layer syncs just before TL1 by only depending on TL0 (and not depending + // on TL1's buffer before TL1 has layer synced). + // TODO(pbos): Consider cyclically updating 'arf' (and 'golden' for 1TL) for + // the base layer in 1-3TL instead of 'last' periodically on long intervals, + // so that if scene changes occur (user walks between rooms or rotates webcam) + // the 'arf' (or 'golden' respectively) is not stuck on a no-longer relevant + // keyframe. + + switch (num_layers) { + case 1: + // Always reference and update the same buffer. + return {{"S", {kReferenceAndUpdate, kNone, kNone}}}; + case 2: + // All layers can reference but not update the 'alt' buffer, this means + // that the 'alt' buffer reference is effectively the last keyframe. + // TL0 also references and updates the 'last' buffer. + // TL1 also references 'last' and references and updates 'golden'. + if (!field_trial::IsDisabled("WebRTC-UseShortVP8TL2Pattern")) { + // Shortened 4-frame pattern: + // 1---1 1---1 ... + // / / / / + // 0---0---0---0 ... + return {{"SS", {kReferenceAndUpdate, kNone, kNone}}, + {"-S", {kReference, kUpdate, kNone}}, + {"SR", {kReferenceAndUpdate, kNone, kNone}}, + {"-D", {kReference, kReference, kNone, kFreezeEntropy}}}; + } else { + // "Default" 8-frame pattern: + // 1---1---1---1 1---1---1---1 ... + // / / / / / / / / + // 0---0---0---0---0---0---0---0 ... + return {{"SS", {kReferenceAndUpdate, kNone, kNone}}, + {"-S", {kReference, kUpdate, kNone}}, + {"SR", {kReferenceAndUpdate, kNone, kNone}}, + {"-R", {kReference, kReferenceAndUpdate, kNone}}, + {"SR", {kReferenceAndUpdate, kNone, kNone}}, + {"-R", {kReference, kReferenceAndUpdate, kNone}}, + {"SR", {kReferenceAndUpdate, kNone, kNone}}, + {"-D", {kReference, kReference, kNone, kFreezeEntropy}}}; + } + case 3: + if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) { + // This field trial is intended to check if it is worth using a shorter + // temporal pattern, trading some coding efficiency for less risk of + // dropped frames. + // The coding efficiency will decrease somewhat since the higher layer + // state is more volatile, but it will be offset slightly by updating + // the altref buffer with TL2 frames, instead of just referencing lower + // layers. + // If a frame is dropped in a higher layer, the jitter + // buffer on the receive side won't be able to decode any higher layer + // frame until the next sync frame. So we expect a noticeable decrease + // in frame drops on links with high packet loss. + + // TL0 references and updates the 'last' buffer. + // TL1 references 'last' and references and updates 'golden'. + // TL2 references both 'last' & 'golden' and references and updates + // 'arf'. + // 2-------2 2-------2 2 + // / __/ / __/ / + // / __1 / __1 / + // /___/ /___/ / + // 0---------------0---------------0----- + // 0 1 2 3 4 5 6 7 8 9 ... + return {{"SSS", {kReferenceAndUpdate, kNone, kNone}}, + {"--S", {kReference, kNone, kUpdate}}, + {"-DR", {kReference, kUpdate, kNone}}, + {"--D", {kReference, kReference, kReference, kFreezeEntropy}}}; + } else { + // All layers can reference but not update the 'alt' buffer, this means + // that the 'alt' buffer reference is effectively the last keyframe. + // TL0 also references and updates the 'last' buffer. + // TL1 also references 'last' and references and updates 'golden'. + // TL2 references both 'last' and 'golden' but updates no buffer. + // 2 __2 _____2 __2 2 + // / /____/ / / / + // / 1---------/-----1 / + // /_____/ /_____/ / + // 0---------------0---------------0----- + // 0 1 2 3 4 5 6 7 8 9 ... + return {{"SSS", {kReferenceAndUpdate, kNone, kNone}}, + {"--D", {kReference, kNone, kNone, kFreezeEntropy}}, + {"-SS", {kReference, kUpdate, kNone}}, + {"--D", {kReference, kReference, kNone, kFreezeEntropy}}, + {"SRR", {kReferenceAndUpdate, kNone, kNone}}, + {"--D", {kReference, kReference, kNone, kFreezeEntropy}}, + {"-DS", {kReference, kReferenceAndUpdate, kNone}}, + {"--D", {kReference, kReference, kNone, kFreezeEntropy}}}; + } + case 4: + // TL0 references and updates only the 'last' buffer. + // TL1 references 'last' and updates and references 'golden'. + // TL2 references 'last' and 'golden', and references and updates 'arf'. + // TL3 references all buffers but update none of them. + // TODO(philipel): Set decode target information for this structure. + return {{"----", {kReferenceAndUpdate, kNone, kNone}}, + {"----", {kReference, kNone, kNone, kFreezeEntropy}}, + {"----", {kReference, kNone, kUpdate}}, + {"----", {kReference, kNone, kReference, kFreezeEntropy}}, + {"----", {kReference, kUpdate, kNone}}, + {"----", {kReference, kReference, kReference, kFreezeEntropy}}, + {"----", {kReference, kReference, kReferenceAndUpdate}}, + {"----", {kReference, kReference, kReference, kFreezeEntropy}}, + {"----", {kReferenceAndUpdate, kNone, kNone}}, + {"----", {kReference, kReference, kReference, kFreezeEntropy}}, + {"----", {kReference, kReference, kReferenceAndUpdate}}, + {"----", {kReference, kReference, kReference, kFreezeEntropy}}, + {"----", {kReference, kReferenceAndUpdate, kNone}}, + {"----", {kReference, kReference, kReference, kFreezeEntropy}}, + {"----", {kReference, kReference, kReferenceAndUpdate}}, + {"----", {kReference, kReference, kReference, kFreezeEntropy}}}; + default: + RTC_DCHECK_NOTREACHED(); + break; + } + RTC_DCHECK_NOTREACHED(); + return {{"", {kNone, kNone, kNone}}}; +} + +std::bitset<DefaultTemporalLayers::kNumReferenceBuffers> +DefaultTemporalLayers::DetermineStaticBuffers( + const std::vector<DependencyInfo>& temporal_pattern) { + std::bitset<kNumReferenceBuffers> buffers; + buffers.set(); + for (const DependencyInfo& info : temporal_pattern) { + uint8_t updated_buffers = GetUpdatedBuffers(info.frame_config); + + for (Vp8BufferReference buffer : kAllBuffers) { + if (static_cast<uint8_t>(buffer) & updated_buffers) { + buffers.reset(BufferToIndex(buffer)); + } + } + } + return buffers; +} + +DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers) + : num_layers_(std::max(1, number_of_temporal_layers)), + temporal_ids_(GetTemporalIds(num_layers_)), + temporal_pattern_(GetDependencyInfo(num_layers_)), + is_static_buffer_(DetermineStaticBuffers(temporal_pattern_)), + pattern_idx_(kUninitializedPatternIndex), + new_bitrates_bps_(std::vector<uint32_t>(num_layers_, 0u)) { + RTC_CHECK_GE(kMaxTemporalStreams, number_of_temporal_layers); + RTC_CHECK_GE(number_of_temporal_layers, 0); + RTC_CHECK_LE(number_of_temporal_layers, 4); + // pattern_idx_ wraps around temporal_pattern_.size, this is incorrect if + // temporal_ids_ are ever longer. If this is no longer correct it needs to + // wrap at max(temporal_ids_.size(), temporal_pattern_.size()). + RTC_DCHECK_LE(temporal_ids_.size(), temporal_pattern_.size()); + + RTC_DCHECK( + checker_ = TemporalLayersChecker::CreateTemporalLayersChecker( + Vp8TemporalLayersType::kFixedPattern, number_of_temporal_layers)); + + // Always need to start with a keyframe, so pre-populate all frame counters. + frames_since_buffer_refresh_.fill(0); +} + +DefaultTemporalLayers::~DefaultTemporalLayers() = default; + +void DefaultTemporalLayers::SetQpLimits(size_t stream_index, + int min_qp, + int max_qp) { + RTC_DCHECK_LT(stream_index, StreamCount()); + // Ignore. +} + +size_t DefaultTemporalLayers::StreamCount() const { + return 1; +} + +bool DefaultTemporalLayers::SupportsEncoderFrameDropping( + size_t stream_index) const { + RTC_DCHECK_LT(stream_index, StreamCount()); + // This class allows the encoder drop frames as it sees fit. + return true; +} + +void DefaultTemporalLayers::OnRatesUpdated( + size_t stream_index, + const std::vector<uint32_t>& bitrates_bps, + int framerate_fps) { + RTC_DCHECK_LT(stream_index, StreamCount()); + RTC_DCHECK_GT(bitrates_bps.size(), 0); + RTC_DCHECK_LE(bitrates_bps.size(), num_layers_); + // `bitrates_bps` uses individual rate per layer, but Vp8EncoderConfig wants + // the accumulated rate, so sum them up. + new_bitrates_bps_ = bitrates_bps; + new_bitrates_bps_->resize(num_layers_); + for (size_t i = 1; i < num_layers_; ++i) { + (*new_bitrates_bps_)[i] += (*new_bitrates_bps_)[i - 1]; + } +} + +Vp8EncoderConfig DefaultTemporalLayers::UpdateConfiguration( + size_t stream_index) { + RTC_DCHECK_LT(stream_index, StreamCount()); + + Vp8EncoderConfig config; + + if (!new_bitrates_bps_) { + return config; + } + + config.temporal_layer_config.emplace(); + Vp8EncoderConfig::TemporalLayerConfig& ts_config = + config.temporal_layer_config.value(); + + for (size_t i = 0; i < num_layers_; ++i) { + ts_config.ts_target_bitrate[i] = (*new_bitrates_bps_)[i] / 1000; + // ..., 4, 2, 1 + ts_config.ts_rate_decimator[i] = 1 << (num_layers_ - i - 1); + } + + ts_config.ts_number_layers = num_layers_; + ts_config.ts_periodicity = temporal_ids_.size(); + std::copy(temporal_ids_.begin(), temporal_ids_.end(), + ts_config.ts_layer_id.begin()); + + new_bitrates_bps_.reset(); + + return config; +} + +bool DefaultTemporalLayers::IsSyncFrame(const Vp8FrameConfig& config) const { + // Since we always assign TL0 to 'last' in these patterns, we can infer layer + // sync by checking if temporal id > 0 and we only reference TL0 or buffers + // containing the last key-frame. + if (config.packetizer_temporal_idx == 0) { + // TL0 frames are per definition not sync frames. + return false; + } + + if ((config.last_buffer_flags & BufferFlags::kReference) == 0) { + // Sync frames must reference TL0. + return false; + } + + if ((config.golden_buffer_flags & BufferFlags::kReference) && + !is_static_buffer_[BufferToIndex(Vp8BufferReference::kGolden)]) { + // Referencing a golden frame that contains a non-(base layer|key frame). + return false; + } + if ((config.arf_buffer_flags & BufferFlags::kReference) && + !is_static_buffer_[BufferToIndex(Vp8BufferReference::kAltref)]) { + // Referencing an altref frame that contains a non-(base layer|key frame). + return false; + } + + return true; +} + +Vp8FrameConfig DefaultTemporalLayers::NextFrameConfig(size_t stream_index, + uint32_t timestamp) { + RTC_DCHECK_LT(stream_index, StreamCount()); + RTC_DCHECK_GT(num_layers_, 0); + RTC_DCHECK_GT(temporal_pattern_.size(), 0); + + RTC_DCHECK_GT(kUninitializedPatternIndex, temporal_pattern_.size()); + const bool first_frame = (pattern_idx_ == kUninitializedPatternIndex); + + pattern_idx_ = (pattern_idx_ + 1) % temporal_pattern_.size(); + DependencyInfo dependency_info = temporal_pattern_[pattern_idx_]; + Vp8FrameConfig& tl_config = dependency_info.frame_config; + tl_config.encoder_layer_id = tl_config.packetizer_temporal_idx = + temporal_ids_[pattern_idx_ % temporal_ids_.size()]; + + if (pattern_idx_ == 0) { + // Start of new pattern iteration, set up clear state by invalidating any + // pending frames, so that we don't make an invalid reference to a buffer + // containing data from a previous iteration. + for (auto& frame : pending_frames_) { + frame.expired = true; + } + } + + if (first_frame) { + tl_config = Vp8FrameConfig::GetIntraFrameConfig(); + } else { + // Last is always ok to reference as it contains the base layer. For other + // buffers though, we need to check if the buffer has actually been + // refreshed this cycle of the temporal pattern. If the encoder dropped + // a frame, it might not have. + ValidateReferences(&tl_config.golden_buffer_flags, + Vp8BufferReference::kGolden); + ValidateReferences(&tl_config.arf_buffer_flags, + Vp8BufferReference::kAltref); + // Update search order to let the encoder know which buffers contains the + // most recent data. + UpdateSearchOrder(&tl_config); + // Figure out if this a sync frame (non-base-layer frame with only + // base-layer references). + tl_config.layer_sync = IsSyncFrame(tl_config); + + // Increment frame age, this needs to be in sync with `pattern_idx_`, + // so must update it here. Resetting age to 0 must be done when encoding is + // complete though, and so in the case of pipelining encoder it might lag. + // To prevent this data spill over into the next iteration, + // the `pedning_frames_` map is reset in loops. If delay is constant, + // the relative age should still be OK for the search order. + for (size_t& n : frames_since_buffer_refresh_) { + ++n; + } + } + + // Add frame to set of pending frames, awaiting completion. + pending_frames_.emplace_back(timestamp, false, GetUpdatedBuffers(tl_config), + dependency_info); + + // Checker does not yet support encoder frame dropping, so validate flags + // here before they can be dropped. + // TODO(sprang): Update checker to support dropping. + RTC_DCHECK(checker_->CheckTemporalConfig(first_frame, tl_config)); + + return tl_config; +} + +void DefaultTemporalLayers::ValidateReferences(BufferFlags* flags, + Vp8BufferReference ref) const { + // Check if the buffer specified by `ref` is actually referenced, and if so + // if it also a dynamically updating one (buffers always just containing + // keyframes are always safe to reference). + if ((*flags & BufferFlags::kReference) && + !is_static_buffer_[BufferToIndex(ref)]) { + if (NumFramesSinceBufferRefresh(ref) >= pattern_idx_) { + // No valid buffer state, or buffer contains frame that is older than the + // current pattern. This reference is not valid, so remove it. + *flags = static_cast<BufferFlags>(*flags & ~BufferFlags::kReference); + } + } +} + +void DefaultTemporalLayers::UpdateSearchOrder(Vp8FrameConfig* config) { + // Figure out which of the buffers we can reference, and order them so that + // the most recently refreshed is first. Otherwise prioritize last first, + // golden second, and altref third. + using BufferRefAge = std::pair<Vp8BufferReference, size_t>; + std::vector<BufferRefAge> eligible_buffers; + if (config->last_buffer_flags & BufferFlags::kReference) { + eligible_buffers.emplace_back( + Vp8BufferReference::kLast, + NumFramesSinceBufferRefresh(Vp8BufferReference::kLast)); + } + if (config->golden_buffer_flags & BufferFlags::kReference) { + eligible_buffers.emplace_back( + Vp8BufferReference::kGolden, + NumFramesSinceBufferRefresh(Vp8BufferReference::kGolden)); + } + if (config->arf_buffer_flags & BufferFlags::kReference) { + eligible_buffers.emplace_back( + Vp8BufferReference::kAltref, + NumFramesSinceBufferRefresh(Vp8BufferReference::kAltref)); + } + + std::sort(eligible_buffers.begin(), eligible_buffers.end(), + [](const BufferRefAge& lhs, const BufferRefAge& rhs) { + if (lhs.second != rhs.second) { + // Lower count has highest precedence. + return lhs.second < rhs.second; + } + return lhs.first < rhs.first; + }); + + // Populate the search order fields where possible. + if (!eligible_buffers.empty()) { + config->first_reference = eligible_buffers.front().first; + if (eligible_buffers.size() > 1) + config->second_reference = eligible_buffers[1].first; + } +} + +size_t DefaultTemporalLayers::NumFramesSinceBufferRefresh( + Vp8FrameConfig::Vp8BufferReference ref) const { + return frames_since_buffer_refresh_[BufferToIndex(ref)]; +} + +void DefaultTemporalLayers::ResetNumFramesSinceBufferRefresh( + Vp8FrameConfig::Vp8BufferReference ref) { + frames_since_buffer_refresh_[BufferToIndex(ref)] = 0; +} + +void DefaultTemporalLayers::CullPendingFramesBefore(uint32_t timestamp) { + while (!pending_frames_.empty() && + pending_frames_.front().timestamp != timestamp) { + pending_frames_.pop_front(); + } +} + +void DefaultTemporalLayers::OnEncodeDone(size_t stream_index, + uint32_t rtp_timestamp, + size_t size_bytes, + bool is_keyframe, + int qp, + CodecSpecificInfo* info) { + RTC_DCHECK_LT(stream_index, StreamCount()); + RTC_DCHECK_GT(num_layers_, 0); + + if (size_bytes == 0) { + RTC_LOG(LS_WARNING) << "Empty frame; treating as dropped."; + OnFrameDropped(stream_index, rtp_timestamp); + return; + } + + CullPendingFramesBefore(rtp_timestamp); + RTC_CHECK(!pending_frames_.empty()); + PendingFrame& frame = pending_frames_.front(); + RTC_DCHECK_EQ(frame.timestamp, rtp_timestamp); + const Vp8FrameConfig& frame_config = frame.dependency_info.frame_config; + if (is_keyframe) { + // Signal key-frame so checker resets state. + RTC_DCHECK(checker_->CheckTemporalConfig(true, frame_config)); + } + + CodecSpecificInfoVP8& vp8_info = info->codecSpecific.VP8; + if (num_layers_ == 1) { + vp8_info.temporalIdx = kNoTemporalIdx; + vp8_info.layerSync = false; + } else { + if (is_keyframe) { + // Restart the temporal pattern on keyframes. + pattern_idx_ = 0; + vp8_info.temporalIdx = 0; + vp8_info.layerSync = true; // Keyframes are always sync frames. + + for (Vp8BufferReference buffer : kAllBuffers) { + if (is_static_buffer_[BufferToIndex(buffer)]) { + // Update frame count of all kf-only buffers, regardless of state of + // `pending_frames_`. + ResetNumFramesSinceBufferRefresh(buffer); + } else { + // Key-frames update all buffers, this should be reflected when + // updating state in FrameEncoded(). + frame.updated_buffer_mask |= static_cast<uint8_t>(buffer); + } + } + } else { + // Delta frame, update codec specifics with temporal id and sync flag. + vp8_info.temporalIdx = frame_config.packetizer_temporal_idx; + vp8_info.layerSync = frame_config.layer_sync; + } + } + + vp8_info.useExplicitDependencies = true; + RTC_DCHECK_EQ(vp8_info.referencedBuffersCount, 0u); + RTC_DCHECK_EQ(vp8_info.updatedBuffersCount, 0u); + + GenericFrameInfo& generic_frame_info = info->generic_frame_info.emplace(); + + for (int i = 0; i < static_cast<int>(Vp8FrameConfig::Buffer::kCount); ++i) { + bool references = false; + bool updates = is_keyframe; + + if (!is_keyframe && + frame_config.References(static_cast<Vp8FrameConfig::Buffer>(i))) { + RTC_DCHECK_LT(vp8_info.referencedBuffersCount, + arraysize(CodecSpecificInfoVP8::referencedBuffers)); + references = true; + vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i; + } + + if (is_keyframe || + frame_config.Updates(static_cast<Vp8FrameConfig::Buffer>(i))) { + RTC_DCHECK_LT(vp8_info.updatedBuffersCount, + arraysize(CodecSpecificInfoVP8::updatedBuffers)); + updates = true; + vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i; + } + + if (references || updates) { + generic_frame_info.encoder_buffers.emplace_back(i, references, updates); + } + } + + // The templates are always present on keyframes, and then refered to by + // subsequent frames. + if (is_keyframe) { + info->template_structure = GetTemplateStructure(num_layers_); + generic_frame_info.decode_target_indications = + temporal_pattern_.front().decode_target_indications; + generic_frame_info.temporal_id = 0; + } else { + generic_frame_info.decode_target_indications = + frame.dependency_info.decode_target_indications; + generic_frame_info.temporal_id = frame_config.packetizer_temporal_idx; + } + + if (!frame.expired) { + for (Vp8BufferReference buffer : kAllBuffers) { + if (frame.updated_buffer_mask & static_cast<uint8_t>(buffer)) { + ResetNumFramesSinceBufferRefresh(buffer); + } + } + } + + pending_frames_.pop_front(); +} + +void DefaultTemporalLayers::OnFrameDropped(size_t stream_index, + uint32_t rtp_timestamp) { + CullPendingFramesBefore(rtp_timestamp); + RTC_CHECK(!pending_frames_.empty()); + RTC_DCHECK_EQ(pending_frames_.front().timestamp, rtp_timestamp); + pending_frames_.pop_front(); +} + +void DefaultTemporalLayers::OnPacketLossRateUpdate(float packet_loss_rate) {} + +void DefaultTemporalLayers::OnRttUpdate(int64_t rtt_ms) {} + +void DefaultTemporalLayers::OnLossNotification( + const VideoEncoder::LossNotification& loss_notification) {} + +FrameDependencyStructure DefaultTemporalLayers::GetTemplateStructure( + int num_layers) const { + RTC_CHECK_LT(num_layers, 5); + RTC_CHECK_GT(num_layers, 0); + + FrameDependencyStructure template_structure; + template_structure.num_decode_targets = num_layers; + + switch (num_layers) { + case 1: { + template_structure.templates.resize(2); + template_structure.templates[0].T(0).Dtis("S"); + template_structure.templates[1].T(0).Dtis("S").FrameDiffs({1}); + return template_structure; + } + case 2: { + template_structure.templates.resize(5); + template_structure.templates[0].T(0).Dtis("SS"); + template_structure.templates[1].T(0).Dtis("SS").FrameDiffs({2}); + template_structure.templates[2].T(0).Dtis("SR").FrameDiffs({2}); + template_structure.templates[3].T(1).Dtis("-S").FrameDiffs({1}); + template_structure.templates[4].T(1).Dtis("-D").FrameDiffs({2, 1}); + return template_structure; + } + case 3: { + if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) { + template_structure.templates.resize(5); + template_structure.templates[0].T(0).Dtis("SSS"); + template_structure.templates[1].T(0).Dtis("SSS").FrameDiffs({4}); + template_structure.templates[2].T(1).Dtis("-DR").FrameDiffs({2}); + template_structure.templates[3].T(2).Dtis("--S").FrameDiffs({1}); + template_structure.templates[4].T(2).Dtis("--D").FrameDiffs({2, 1}); + } else { + template_structure.templates.resize(7); + template_structure.templates[0].T(0).Dtis("SSS"); + template_structure.templates[1].T(0).Dtis("SSS").FrameDiffs({4}); + template_structure.templates[2].T(0).Dtis("SRR").FrameDiffs({4}); + template_structure.templates[3].T(1).Dtis("-SS").FrameDiffs({2}); + template_structure.templates[4].T(1).Dtis("-DS").FrameDiffs({4, 2}); + template_structure.templates[5].T(2).Dtis("--D").FrameDiffs({1}); + template_structure.templates[6].T(2).Dtis("--D").FrameDiffs({3, 1}); + } + return template_structure; + } + case 4: { + template_structure.templates.resize(8); + template_structure.templates[0].T(0).Dtis("SSSS"); + template_structure.templates[1].T(0).Dtis("SSSS").FrameDiffs({8}); + template_structure.templates[2].T(1).Dtis("-SRR").FrameDiffs({4}); + template_structure.templates[3].T(1).Dtis("-SRR").FrameDiffs({4, 8}); + template_structure.templates[4].T(2).Dtis("--SR").FrameDiffs({2}); + template_structure.templates[5].T(2).Dtis("--SR").FrameDiffs({2, 4}); + template_structure.templates[6].T(3).Dtis("---D").FrameDiffs({1}); + template_structure.templates[7].T(3).Dtis("---D").FrameDiffs({1, 3}); + return template_structure; + } + default: + RTC_DCHECK_NOTREACHED(); + // To make the compiler happy! + return template_structure; + } +} + +// Returns list of temporal dependencies for each frame in the temporal pattern. +// Values are lists of indecies in the pattern. +std::vector<std::set<uint8_t>> GetTemporalDependencies( + int num_temporal_layers) { + switch (num_temporal_layers) { + case 1: + return {{0}}; + case 2: + if (!field_trial::IsDisabled("WebRTC-UseShortVP8TL2Pattern")) { + return {{2}, {0}, {0}, {1, 2}}; + } else { + return {{6}, {0}, {0}, {1, 2}, {2}, {3, 4}, {4}, {5, 6}}; + } + case 3: + if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) { + return {{0}, {0}, {0}, {0, 1, 2}}; + } else { + return {{4}, {0}, {0}, {0, 2}, {0}, {2, 4}, {2, 4}, {4, 6}}; + } + case 4: + return {{8}, {0}, {0}, {0, 2}, + {0}, {0, 2, 4}, {0, 2, 4}, {0, 4, 6}, + {0}, {4, 6, 8}, {4, 6, 8}, {4, 8, 10}, + {4, 8}, {8, 10, 12}, {8, 10, 12}, {8, 12, 14}}; + default: + RTC_DCHECK_NOTREACHED(); + return {}; + } +} + +DefaultTemporalLayersChecker::DefaultTemporalLayersChecker( + int num_temporal_layers) + : TemporalLayersChecker(num_temporal_layers), + num_layers_(std::max(1, num_temporal_layers)), + temporal_ids_(GetTemporalIds(num_layers_)), + temporal_dependencies_(GetTemporalDependencies(num_layers_)), + pattern_idx_(255) { + int i = 0; + while (temporal_ids_.size() < temporal_dependencies_.size()) { + temporal_ids_.push_back(temporal_ids_[i++]); + } +} + +DefaultTemporalLayersChecker::~DefaultTemporalLayersChecker() = default; + +bool DefaultTemporalLayersChecker::CheckTemporalConfig( + bool frame_is_keyframe, + const Vp8FrameConfig& frame_config) { + if (!TemporalLayersChecker::CheckTemporalConfig(frame_is_keyframe, + frame_config)) { + return false; + } + if (frame_config.drop_frame) { + return true; + } + + if (frame_is_keyframe) { + pattern_idx_ = 0; + last_ = BufferState(); + golden_ = BufferState(); + arf_ = BufferState(); + return true; + } + + ++pattern_idx_; + if (pattern_idx_ == temporal_ids_.size()) { + // All non key-frame buffers should be updated each pattern cycle. + if (!last_.is_keyframe && !last_.is_updated_this_cycle) { + RTC_LOG(LS_ERROR) << "Last buffer was not updated during pattern cycle."; + return false; + } + if (!arf_.is_keyframe && !arf_.is_updated_this_cycle) { + RTC_LOG(LS_ERROR) << "Arf buffer was not updated during pattern cycle."; + return false; + } + if (!golden_.is_keyframe && !golden_.is_updated_this_cycle) { + RTC_LOG(LS_ERROR) + << "Golden buffer was not updated during pattern cycle."; + return false; + } + last_.is_updated_this_cycle = false; + arf_.is_updated_this_cycle = false; + golden_.is_updated_this_cycle = false; + pattern_idx_ = 0; + } + uint8_t expected_tl_idx = temporal_ids_[pattern_idx_]; + if (frame_config.packetizer_temporal_idx != expected_tl_idx) { + RTC_LOG(LS_ERROR) << "Frame has an incorrect temporal index. Expected: " + << static_cast<int>(expected_tl_idx) << " Actual: " + << static_cast<int>(frame_config.packetizer_temporal_idx); + return false; + } + + bool need_sync = temporal_ids_[pattern_idx_] > 0 && + temporal_ids_[pattern_idx_] != kNoTemporalIdx; + std::vector<int> dependencies; + + if (frame_config.last_buffer_flags & BufferFlags::kReference) { + uint8_t referenced_layer = temporal_ids_[last_.pattern_idx]; + if (referenced_layer > 0) { + need_sync = false; + } + if (!last_.is_keyframe) { + dependencies.push_back(last_.pattern_idx); + } + } else if (frame_config.first_reference == Vp8BufferReference::kLast || + frame_config.second_reference == Vp8BufferReference::kLast) { + RTC_LOG(LS_ERROR) + << "Last buffer not referenced, but present in search order."; + return false; + } + + if (frame_config.arf_buffer_flags & BufferFlags::kReference) { + uint8_t referenced_layer = temporal_ids_[arf_.pattern_idx]; + if (referenced_layer > 0) { + need_sync = false; + } + if (!arf_.is_keyframe) { + dependencies.push_back(arf_.pattern_idx); + } + } else if (frame_config.first_reference == Vp8BufferReference::kAltref || + frame_config.second_reference == Vp8BufferReference::kAltref) { + RTC_LOG(LS_ERROR) + << "Altret buffer not referenced, but present in search order."; + return false; + } + + if (frame_config.golden_buffer_flags & BufferFlags::kReference) { + uint8_t referenced_layer = temporal_ids_[golden_.pattern_idx]; + if (referenced_layer > 0) { + need_sync = false; + } + if (!golden_.is_keyframe) { + dependencies.push_back(golden_.pattern_idx); + } + } else if (frame_config.first_reference == Vp8BufferReference::kGolden || + frame_config.second_reference == Vp8BufferReference::kGolden) { + RTC_LOG(LS_ERROR) + << "Golden buffer not referenced, but present in search order."; + return false; + } + + if (need_sync != frame_config.layer_sync) { + RTC_LOG(LS_ERROR) << "Sync bit is set incorrectly on a frame. Expected: " + << need_sync << " Actual: " << frame_config.layer_sync; + return false; + } + + if (!frame_is_keyframe) { + size_t i; + for (i = 0; i < dependencies.size(); ++i) { + if (temporal_dependencies_[pattern_idx_].find(dependencies[i]) == + temporal_dependencies_[pattern_idx_].end()) { + RTC_LOG(LS_ERROR) + << "Illegal temporal dependency out of defined pattern " + "from position " + << static_cast<int>(pattern_idx_) << " to position " + << static_cast<int>(dependencies[i]); + return false; + } + } + } + + if (frame_config.last_buffer_flags & BufferFlags::kUpdate) { + last_.is_updated_this_cycle = true; + last_.pattern_idx = pattern_idx_; + last_.is_keyframe = false; + } + if (frame_config.arf_buffer_flags & BufferFlags::kUpdate) { + arf_.is_updated_this_cycle = true; + arf_.pattern_idx = pattern_idx_; + arf_.is_keyframe = false; + } + if (frame_config.golden_buffer_flags & BufferFlags::kUpdate) { + golden_.is_updated_this_cycle = true; + golden_.pattern_idx = pattern_idx_; + golden_.is_keyframe = false; + } + return true; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h new file mode 100644 index 0000000000..bc6574c54c --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h @@ -0,0 +1,168 @@ +/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +/* + * This file defines classes for doing temporal layers with VP8. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_ +#define MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <bitset> +#include <deque> +#include <limits> +#include <memory> +#include <set> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/video_codecs/vp8_frame_config.h" +#include "api/video_codecs/vp8_temporal_layers.h" +#include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h" +#include "modules/video_coding/include/video_codec_interface.h" + +namespace webrtc { + +class DefaultTemporalLayers final : public Vp8FrameBufferController { + public: + explicit DefaultTemporalLayers(int number_of_temporal_layers); + ~DefaultTemporalLayers() override; + + void SetQpLimits(size_t stream_index, int min_qp, int max_qp) override; + + size_t StreamCount() const override; + + bool SupportsEncoderFrameDropping(size_t stream_index) const override; + + // Returns the recommended VP8 encode flags needed. May refresh the decoder + // and/or update the reference buffers. + Vp8FrameConfig NextFrameConfig(size_t stream_index, + uint32_t timestamp) override; + + // New target bitrate, per temporal layer. + void OnRatesUpdated(size_t stream_index, + const std::vector<uint32_t>& bitrates_bps, + int framerate_fps) override; + + Vp8EncoderConfig UpdateConfiguration(size_t stream_index) override; + + // Callbacks methods on frame completion. OnEncodeDone() or OnFrameDropped() + // should be called once for each NextFrameConfig() call (using the RTP + // timestamp as ID), and the calls MUST be in the same order. + void OnEncodeDone(size_t stream_index, + uint32_t rtp_timestamp, + size_t size_bytes, + bool is_keyframe, + int qp, + CodecSpecificInfo* info) override; + void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) override; + + void OnPacketLossRateUpdate(float packet_loss_rate) override; + + void OnRttUpdate(int64_t rtt_ms) override; + + void OnLossNotification( + const VideoEncoder::LossNotification& loss_notification) override; + + private: + static constexpr size_t kNumReferenceBuffers = 3; // Last, golden, altref. + struct DependencyInfo { + DependencyInfo() = default; + DependencyInfo(absl::string_view indication_symbols, + Vp8FrameConfig frame_config) + : decode_target_indications( + webrtc_impl::StringToDecodeTargetIndications(indication_symbols)), + frame_config(frame_config) {} + + absl::InlinedVector<DecodeTargetIndication, 10> decode_target_indications; + Vp8FrameConfig frame_config; + }; + struct PendingFrame { + PendingFrame(); + PendingFrame(uint32_t timestamp, + bool expired, + uint8_t updated_buffers_mask, + const DependencyInfo& dependency_info); + uint32_t timestamp = 0; + // Flag indicating if this frame has expired, ie it belongs to a previous + // iteration of the temporal pattern. + bool expired = false; + // Bitmask of Vp8BufferReference flags, indicating which buffers this frame + // updates. + uint8_t updated_buffer_mask = 0; + // The frame config returned by NextFrameConfig() for this frame. + DependencyInfo dependency_info; + }; + + static std::vector<DependencyInfo> GetDependencyInfo(size_t num_layers); + static std::bitset<kNumReferenceBuffers> DetermineStaticBuffers( + const std::vector<DependencyInfo>& temporal_pattern); + bool IsSyncFrame(const Vp8FrameConfig& config) const; + void ValidateReferences(Vp8FrameConfig::BufferFlags* flags, + Vp8FrameConfig::Vp8BufferReference ref) const; + void UpdateSearchOrder(Vp8FrameConfig* config); + size_t NumFramesSinceBufferRefresh( + Vp8FrameConfig::Vp8BufferReference ref) const; + void ResetNumFramesSinceBufferRefresh(Vp8FrameConfig::Vp8BufferReference ref); + void CullPendingFramesBefore(uint32_t timestamp); + + const size_t num_layers_; + const std::vector<unsigned int> temporal_ids_; + const std::vector<DependencyInfo> temporal_pattern_; + // Per reference buffer flag indicating if it is static, meaning it is only + // updated by key-frames. + const std::bitset<kNumReferenceBuffers> is_static_buffer_; + FrameDependencyStructure GetTemplateStructure(int num_layers) const; + + uint8_t pattern_idx_; + // Updated cumulative bitrates, per temporal layer. + absl::optional<std::vector<uint32_t>> new_bitrates_bps_; + + // Status for each pending frame, in + std::deque<PendingFrame> pending_frames_; + + // One counter per reference buffer, indicating number of frames since last + // refresh. For non-base-layer frames (ie golden, altref buffers), this is + // reset when the pattern loops. + std::array<size_t, kNumReferenceBuffers> frames_since_buffer_refresh_; + + // Optional utility used to verify reference validity. + std::unique_ptr<TemporalLayersChecker> checker_; +}; + +class DefaultTemporalLayersChecker : public TemporalLayersChecker { + public: + explicit DefaultTemporalLayersChecker(int number_of_temporal_layers); + ~DefaultTemporalLayersChecker() override; + + bool CheckTemporalConfig(bool frame_is_keyframe, + const Vp8FrameConfig& frame_config) override; + + private: + struct BufferState { + BufferState() + : is_updated_this_cycle(false), is_keyframe(true), pattern_idx(0) {} + + bool is_updated_this_cycle; + bool is_keyframe; + uint8_t pattern_idx; + }; + const size_t num_layers_; + std::vector<unsigned int> temporal_ids_; + const std::vector<std::set<uint8_t>> temporal_dependencies_; + BufferState last_; + BufferState arf_; + BufferState golden_; + uint8_t pattern_idx_; +}; + +} // namespace webrtc +#endif // MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_ diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc new file mode 100644 index 0000000000..ae027a9d8a --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc @@ -0,0 +1,781 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/vp8/default_temporal_layers.h" + +#include <cstdint> +#include <memory> + +#include "api/video/video_bitrate_allocation.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/vp8_frame_config.h" +#include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/utility/simulcast_rate_allocator.h" +#include "test/field_trial.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "vpx/vp8cx.h" + +// TODO(bugs.webrtc.org/10582): Test the behavior of UpdateConfiguration(). + +namespace webrtc { +namespace test { +namespace { + +using ::testing::Each; + +enum { + kTemporalUpdateLast = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF, + kTemporalUpdateGoldenWithoutDependency = + VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_UPD_LAST, + kTemporalUpdateGolden = + VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST, + kTemporalUpdateAltrefWithoutDependency = + VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | + VP8_EFLAG_NO_UPD_LAST, + kTemporalUpdateAltref = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_LAST, + kTemporalUpdateNone = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY, + kTemporalUpdateNoneNoRefAltRef = + VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY, + kTemporalUpdateNoneNoRefGolden = + VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY, + kTemporalUpdateNoneNoRefGoldenAltRef = + VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_REF_ARF | + VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_UPD_ENTROPY, + kTemporalUpdateGoldenWithoutDependencyRefAltRef = + VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST, + kTemporalUpdateGoldenRefAltRef = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST, + kTemporalUpdateLastRefAltRef = + VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF, + kTemporalUpdateLastAndGoldenRefAltRef = + VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF, +}; + +using BufferFlags = Vp8FrameConfig::BufferFlags; +using Vp8BufferReference = Vp8FrameConfig::Vp8BufferReference; + +constexpr uint8_t kNone = static_cast<uint8_t>(Vp8BufferReference::kNone); +constexpr uint8_t kLast = static_cast<uint8_t>(Vp8BufferReference::kLast); +constexpr uint8_t kGolden = static_cast<uint8_t>(Vp8BufferReference::kGolden); +constexpr uint8_t kAltref = static_cast<uint8_t>(Vp8BufferReference::kAltref); +constexpr uint8_t kAll = kLast | kGolden | kAltref; + +constexpr int ToVp8CodecFlags(uint8_t referenced_buffers, + uint8_t updated_buffers, + bool update_entropy) { + return (((referenced_buffers & kLast) == 0) ? VP8_EFLAG_NO_REF_LAST : 0) | + (((referenced_buffers & kGolden) == 0) ? VP8_EFLAG_NO_REF_GF : 0) | + (((referenced_buffers & kAltref) == 0) ? VP8_EFLAG_NO_REF_ARF : 0) | + (((updated_buffers & kLast) == 0) ? VP8_EFLAG_NO_UPD_LAST : 0) | + (((updated_buffers & kGolden) == 0) ? VP8_EFLAG_NO_UPD_GF : 0) | + (((updated_buffers & kAltref) == 0) ? VP8_EFLAG_NO_UPD_ARF : 0) | + (update_entropy ? 0 : VP8_EFLAG_NO_UPD_ENTROPY); +} + +constexpr int kKeyFrameFlags = ToVp8CodecFlags(kNone, kAll, true); + +std::vector<uint32_t> GetTemporalLayerRates(int target_bitrate_kbps, + int framerate_fps, + int num_temporal_layers) { + VideoCodec codec; + codec.codecType = VideoCodecType::kVideoCodecVP8; + codec.numberOfSimulcastStreams = 1; + codec.maxBitrate = target_bitrate_kbps; + codec.maxFramerate = framerate_fps; + codec.simulcastStream[0].targetBitrate = target_bitrate_kbps; + codec.simulcastStream[0].maxBitrate = target_bitrate_kbps; + codec.simulcastStream[0].numberOfTemporalLayers = num_temporal_layers; + codec.simulcastStream[0].active = true; + SimulcastRateAllocator allocator(codec); + return allocator + .Allocate( + VideoBitrateAllocationParameters(target_bitrate_kbps, framerate_fps)) + .GetTemporalLayerAllocation(0); +} + +constexpr int kDefaultBitrateBps = 500; +constexpr int kDefaultFramerate = 30; +constexpr int kDefaultBytesPerFrame = + (kDefaultBitrateBps / 8) / kDefaultFramerate; +constexpr int kDefaultQp = 2; +} // namespace + +class TemporalLayersTest : public ::testing::Test { + public: + ~TemporalLayersTest() override = default; + + CodecSpecificInfo* IgnoredCodecSpecificInfo() { + codec_specific_info_ = std::make_unique<CodecSpecificInfo>(); + return codec_specific_info_.get(); + } + + private: + std::unique_ptr<CodecSpecificInfo> codec_specific_info_; +}; + +TEST_F(TemporalLayersTest, 2Layers) { + constexpr int kNumLayers = 2; + DefaultTemporalLayers tl(kNumLayers); + DefaultTemporalLayersChecker checker(kNumLayers); + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, + kDefaultFramerate, kNumLayers), + kDefaultFramerate); + tl.UpdateConfiguration(0); + + constexpr size_t kPatternSize = 4; + constexpr size_t kRepetitions = 4; + + const int expected_flags[kPatternSize] = { + ToVp8CodecFlags(kLast, kLast, true), + ToVp8CodecFlags(kLast, kGolden, true), + ToVp8CodecFlags(kLast, kLast, true), + ToVp8CodecFlags(kLast | kGolden, kNone, false), + }; + const int expected_temporal_idx[kPatternSize] = {0, 1, 0, 1}; + const bool expected_layer_sync[kPatternSize] = {false, true, false, false}; + + uint32_t timestamp = 0; + for (size_t i = 0; i < kPatternSize * kRepetitions; ++i) { + const size_t ind = i % kPatternSize; + const bool is_keyframe = (i == 0); + CodecSpecificInfo info; + Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp); + EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[ind], + LibvpxVp8Encoder::EncodeFlags(tl_config)) + << i; + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe, + kDefaultQp, &info); + EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config)); + EXPECT_EQ(expected_temporal_idx[ind], info.codecSpecific.VP8.temporalIdx); + EXPECT_EQ(expected_temporal_idx[ind], tl_config.packetizer_temporal_idx); + EXPECT_EQ(expected_temporal_idx[ind], tl_config.encoder_layer_id); + EXPECT_EQ(is_keyframe || expected_layer_sync[ind], + info.codecSpecific.VP8.layerSync); + EXPECT_EQ(expected_layer_sync[ind], tl_config.layer_sync); + timestamp += 3000; + } +} + +TEST_F(TemporalLayersTest, 3Layers) { + constexpr int kNumLayers = 3; + DefaultTemporalLayers tl(kNumLayers); + DefaultTemporalLayersChecker checker(kNumLayers); + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, + kDefaultFramerate, kNumLayers), + kDefaultFramerate); + tl.UpdateConfiguration(0); + + int expected_flags[16] = { + kTemporalUpdateLast, + kTemporalUpdateNoneNoRefGoldenAltRef, + kTemporalUpdateGoldenWithoutDependency, + kTemporalUpdateNoneNoRefAltRef, + kTemporalUpdateLast, + kTemporalUpdateNoneNoRefAltRef, + kTemporalUpdateGolden, + kTemporalUpdateNoneNoRefAltRef, + kTemporalUpdateLast, + kTemporalUpdateNoneNoRefGoldenAltRef, + kTemporalUpdateGoldenWithoutDependency, + kTemporalUpdateNoneNoRefAltRef, + kTemporalUpdateLast, + kTemporalUpdateNoneNoRefAltRef, + kTemporalUpdateGolden, + kTemporalUpdateNoneNoRefAltRef, + }; + int expected_temporal_idx[16] = {0, 2, 1, 2, 0, 2, 1, 2, + 0, 2, 1, 2, 0, 2, 1, 2}; + + bool expected_layer_sync[16] = {false, true, true, false, false, false, + false, false, false, true, true, false, + false, false, false, false}; + + unsigned int timestamp = 0; + for (int i = 0; i < 16; ++i) { + const bool is_keyframe = (i == 0); + CodecSpecificInfo info; + Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp); + EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[i], + LibvpxVp8Encoder::EncodeFlags(tl_config)) + << i; + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe, + kDefaultQp, &info); + EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config)); + EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx); + EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx); + EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id); + EXPECT_EQ(is_keyframe || expected_layer_sync[i], + info.codecSpecific.VP8.layerSync); + EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync); + timestamp += 3000; + } +} + +TEST_F(TemporalLayersTest, Alternative3Layers) { + constexpr int kNumLayers = 3; + ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/"); + DefaultTemporalLayers tl(kNumLayers); + DefaultTemporalLayersChecker checker(kNumLayers); + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, + kDefaultFramerate, kNumLayers), + kDefaultFramerate); + tl.UpdateConfiguration(0); + + int expected_flags[8] = {kTemporalUpdateLast, + kTemporalUpdateAltrefWithoutDependency, + kTemporalUpdateGoldenWithoutDependency, + kTemporalUpdateNone, + kTemporalUpdateLast, + kTemporalUpdateAltrefWithoutDependency, + kTemporalUpdateGoldenWithoutDependency, + kTemporalUpdateNone}; + int expected_temporal_idx[8] = {0, 2, 1, 2, 0, 2, 1, 2}; + + bool expected_layer_sync[8] = {false, true, true, false, + false, true, true, false}; + + unsigned int timestamp = 0; + for (int i = 0; i < 8; ++i) { + const bool is_keyframe = (i == 0); + CodecSpecificInfo info; + Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp); + EXPECT_EQ(is_keyframe ? kKeyFrameFlags : expected_flags[i], + LibvpxVp8Encoder::EncodeFlags(tl_config)) + << i; + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, is_keyframe, + kDefaultQp, &info); + EXPECT_TRUE(checker.CheckTemporalConfig(is_keyframe, tl_config)); + EXPECT_EQ(expected_temporal_idx[i], info.codecSpecific.VP8.temporalIdx); + EXPECT_EQ(expected_temporal_idx[i], tl_config.packetizer_temporal_idx); + EXPECT_EQ(expected_temporal_idx[i], tl_config.encoder_layer_id); + EXPECT_EQ(is_keyframe || expected_layer_sync[i], + info.codecSpecific.VP8.layerSync); + EXPECT_EQ(expected_layer_sync[i], tl_config.layer_sync); + timestamp += 3000; + } +} + +TEST_F(TemporalLayersTest, SearchOrder) { + constexpr int kNumLayers = 3; + ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/"); + DefaultTemporalLayers tl(kNumLayers); + DefaultTemporalLayersChecker checker(kNumLayers); + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, + kDefaultFramerate, kNumLayers), + kDefaultFramerate); + tl.UpdateConfiguration(0); + + // Use a repeating pattern of tl 0, 2, 1, 2. + // Tl 0, 1, 2 update last, golden, altref respectively. + + // Start with a key-frame. tl_config flags can be ignored. + uint32_t timestamp = 0; + Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp, + IgnoredCodecSpecificInfo()); + + // TL2 frame. First one only references TL0. Updates altref. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast); + EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone); + + // TL1 frame. Can only reference TL0. Updated golden. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast); + EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone); + + // TL2 frame. Can reference all three buffers. Golden was the last to be + // updated, the next to last was altref. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kGolden); + EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kAltref); +} + +TEST_F(TemporalLayersTest, SearchOrderWithDrop) { + constexpr int kNumLayers = 3; + ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/"); + DefaultTemporalLayers tl(kNumLayers); + DefaultTemporalLayersChecker checker(kNumLayers); + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, + kDefaultFramerate, kNumLayers), + kDefaultFramerate); + tl.UpdateConfiguration(0); + + // Use a repeating pattern of tl 0, 2, 1, 2. + // Tl 0, 1, 2 update last, golden, altref respectively. + + // Start with a key-frame. tl_config flags can be ignored. + uint32_t timestamp = 0; + Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp, + IgnoredCodecSpecificInfo()); + + // TL2 frame. First one only references TL0. Updates altref. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kLast); + EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kNone); + + // Dropped TL1 frame. Can only reference TL0. Should have updated golden. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr); + + // TL2 frame. Can normally reference all three buffers, but golden has not + // been populated this cycle. Altref was last to be updated, before that last. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + EXPECT_EQ(tl_config.first_reference, Vp8BufferReference::kAltref); + EXPECT_EQ(tl_config.second_reference, Vp8BufferReference::kLast); +} + +TEST_F(TemporalLayersTest, DoesNotReferenceDroppedFrames) { + constexpr int kNumLayers = 3; + // Use a repeating pattern of tl 0, 2, 1, 2. + // Tl 0, 1, 2 update last, golden, altref respectively. + ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/"); + DefaultTemporalLayers tl(kNumLayers); + DefaultTemporalLayersChecker checker(kNumLayers); + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, + kDefaultFramerate, kNumLayers), + kDefaultFramerate); + tl.UpdateConfiguration(0); + + // Start with a keyframe. + uint32_t timestamp = 0; + Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp, + IgnoredCodecSpecificInfo()); + + // Dropped TL2 frame. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr); + + // Dropped TL1 frame. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr); + + // TL2 frame. Can reference all three buffers, valid since golden and altref + // both contain the last keyframe. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference); + EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference); + EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference); + + // Restart of cycle! + + // TL0 base layer frame, updating and referencing last. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + + // TL2 frame, updating altref. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + + // TL1 frame, updating golden. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + + // TL2 frame. Can still reference all buffer since they have been update this + // cycle. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference); + EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference); + EXPECT_TRUE(tl_config.arf_buffer_flags & BufferFlags::kReference); + + // Restart of cycle! + + // TL0 base layer frame, updating and referencing last. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + + // Dropped TL2 frame. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr); + + // Dropped TL1 frame. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, 0, false, 0, nullptr); + + // TL2 frame. This time golden and altref contain data from the previous cycle + // and cannot be referenced. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference); + EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference); + EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference); +} + +TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExist) { + constexpr int kNumLayers = 3; + // Use a repeating pattern of tl 0, 2, 1, 2. + // Tl 0, 1 updates last, golden respectively. Altref is always last keyframe. + DefaultTemporalLayers tl(kNumLayers); + DefaultTemporalLayersChecker checker(kNumLayers); + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, + kDefaultFramerate, kNumLayers), + kDefaultFramerate); + tl.UpdateConfiguration(0); + + // Start with a keyframe. + uint32_t timestamp = 0; + Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp, + IgnoredCodecSpecificInfo()); + + // Do a full cycle of the pattern. + for (int i = 0; i < 7; ++i) { + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + } + + // TL0 base layer frame, starting the cycle over. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + + // TL2 frame. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + + // Encoder has a hiccup and builds a queue, so frame encoding is delayed. + // TL1 frame, updating golden. + tl_config = tl.NextFrameConfig(0, ++timestamp); + + // TL2 frame, that should be referencing golden, but we can't be certain it's + // not going to be dropped, so that is not allowed. + tl_config = tl.NextFrameConfig(0, timestamp + 1); + EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference); + EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference); + EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference); + + // TL0 base layer frame. + tl_config = tl.NextFrameConfig(0, timestamp + 2); + + // The previous four enqueued frames finally get encoded, and the updated + // buffers are now OK to reference. + // Enqueued TL1 frame ready. + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + // Enqueued TL2 frame. + tl.OnEncodeDone(0, ++timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + // Enqueued TL0 frame. + tl.OnEncodeDone(0, ++timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + + // TL2 frame, all buffers are now in a known good state, OK to reference. + tl_config = tl.NextFrameConfig(0, ++timestamp + 1); + EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference); + EXPECT_TRUE(tl_config.golden_buffer_flags & BufferFlags::kReference); + EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference); +} + +TEST_F(TemporalLayersTest, DoesNotReferenceUnlessGuaranteedToExistLongDelay) { + constexpr int kNumLayers = 3; + // Use a repeating pattern of tl 0, 2, 1, 2. + // Tl 0, 1 updates last, golden, altref respectively. + ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/"); + DefaultTemporalLayers tl(kNumLayers); + DefaultTemporalLayersChecker checker(kNumLayers); + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, + kDefaultFramerate, kNumLayers), + kDefaultFramerate); + tl.UpdateConfiguration(0); + + // Start with a keyframe. + uint32_t timestamp = 0; + Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp, + IgnoredCodecSpecificInfo()); + + // Do a full cycle of the pattern. + for (int i = 0; i < 3; ++i) { + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + } + + // TL0 base layer frame, starting the cycle over. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + + // TL2 frame. + tl_config = tl.NextFrameConfig(0, ++timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + + // Encoder has a hiccup and builds a queue, so frame encoding is delayed. + // Encoded, but delayed frames in TL 1, 2. + tl_config = tl.NextFrameConfig(0, timestamp + 1); + tl_config = tl.NextFrameConfig(0, timestamp + 2); + + // Restart of the pattern! + + // Encoded, but delayed frames in TL 2, 1. + tl_config = tl.NextFrameConfig(0, timestamp + 3); + tl_config = tl.NextFrameConfig(0, timestamp + 4); + + // TL1 frame from last cycle is ready. + tl.OnEncodeDone(0, timestamp + 1, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + // TL2 frame from last cycle is ready. + tl.OnEncodeDone(0, timestamp + 2, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + + // TL2 frame, that should be referencing all buffers, but altref and golden + // haven not been updated this cycle. (Don't be fooled by the late frames from + // the last cycle!) + tl_config = tl.NextFrameConfig(0, timestamp + 5); + EXPECT_TRUE(tl_config.last_buffer_flags & BufferFlags::kReference); + EXPECT_FALSE(tl_config.golden_buffer_flags & BufferFlags::kReference); + EXPECT_FALSE(tl_config.arf_buffer_flags & BufferFlags::kReference); +} + +TEST_F(TemporalLayersTest, KeyFrame) { + constexpr int kNumLayers = 3; + DefaultTemporalLayers tl(kNumLayers); + DefaultTemporalLayersChecker checker(kNumLayers); + tl.OnRatesUpdated(0, + GetTemporalLayerRates(kDefaultBytesPerFrame, + kDefaultFramerate, kNumLayers), + kDefaultFramerate); + tl.UpdateConfiguration(0); + + int expected_flags[8] = { + kTemporalUpdateLastRefAltRef, + kTemporalUpdateNoneNoRefGoldenAltRef, + kTemporalUpdateGoldenWithoutDependency, + kTemporalUpdateNoneNoRefAltRef, + kTemporalUpdateLast, + kTemporalUpdateNoneNoRefAltRef, + kTemporalUpdateGolden, + kTemporalUpdateNone, + }; + int expected_temporal_idx[8] = {0, 2, 1, 2, 0, 2, 1, 2}; + bool expected_layer_sync[8] = {true, true, true, false, + false, false, false, false}; + + uint32_t timestamp = 0; + for (int i = 0; i < 7; ++i) { + // Temporal pattern starts from 0 after key frame. Let the first `i` - 1 + // frames be delta frames, and the `i`th one key frame. + for (int j = 1; j <= i; ++j) { + // Since last frame was always a keyframe and thus index 0 in the pattern, + // this loop starts at index 1. + Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp); + EXPECT_EQ(expected_flags[j], LibvpxVp8Encoder::EncodeFlags(tl_config)) + << j; + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + EXPECT_TRUE(checker.CheckTemporalConfig(false, tl_config)); + EXPECT_EQ(expected_temporal_idx[j], tl_config.packetizer_temporal_idx); + EXPECT_EQ(expected_temporal_idx[j], tl_config.encoder_layer_id); + EXPECT_EQ(expected_layer_sync[j], tl_config.layer_sync); + timestamp += 3000; + } + + CodecSpecificInfo info; + Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp); + tl.OnEncodeDone(0, timestamp, kDefaultBytesPerFrame, true, kDefaultQp, + &info); + EXPECT_TRUE(info.codecSpecific.VP8.layerSync) + << "Key frame should be marked layer sync."; + EXPECT_EQ(0, info.codecSpecific.VP8.temporalIdx) + << "Key frame should always be packetized as layer 0"; + EXPECT_EQ(0, info.generic_frame_info->temporal_id) + << "Key frame should always be packetized as layer 0"; + EXPECT_THAT(info.generic_frame_info->decode_target_indications, + Each(DecodeTargetIndication::kSwitch)) + << "Key frame is universal switch"; + EXPECT_TRUE(checker.CheckTemporalConfig(true, tl_config)); + } +} + +TEST_F(TemporalLayersTest, SetsTlCountOnFirstConfigUpdate) { + // Create an instance and fetch config update without setting any rate. + constexpr int kNumLayers = 2; + DefaultTemporalLayers tl(kNumLayers); + Vp8EncoderConfig config = tl.UpdateConfiguration(0); + + // Config should indicate correct number of temporal layers, but zero bitrate. + ASSERT_TRUE(config.temporal_layer_config.has_value()); + EXPECT_EQ(config.temporal_layer_config->ts_number_layers, + uint32_t{kNumLayers}); + std::array<uint32_t, Vp8EncoderConfig::TemporalLayerConfig::kMaxLayers> + kZeroRate = {}; + EXPECT_EQ(config.temporal_layer_config->ts_target_bitrate, kZeroRate); + + // On second call, no new update. + config = tl.UpdateConfiguration(0); + EXPECT_FALSE(config.temporal_layer_config.has_value()); +} + +class TemporalLayersReferenceTest : public TemporalLayersTest, + public ::testing::WithParamInterface<int> { + public: + TemporalLayersReferenceTest() + : timestamp_(1), + last_sync_timestamp_(timestamp_), + tl0_reference_(nullptr) {} + virtual ~TemporalLayersReferenceTest() {} + + protected: + static const int kMaxPatternLength = 32; + + struct BufferState { + BufferState() : BufferState(-1, 0, false) {} + BufferState(int temporal_idx, uint32_t timestamp, bool sync) + : temporal_idx(temporal_idx), timestamp(timestamp), sync(sync) {} + int temporal_idx; + uint32_t timestamp; + bool sync; + }; + + bool UpdateSyncRefState(const BufferFlags& flags, BufferState* buffer_state) { + if (flags & BufferFlags::kReference) { + if (buffer_state->temporal_idx == -1) + return true; // References key-frame. + if (buffer_state->temporal_idx == 0) { + // No more than one reference to TL0 frame. + EXPECT_EQ(nullptr, tl0_reference_); + tl0_reference_ = buffer_state; + return true; + } + return false; // References higher layer. + } + return true; // No reference, does not affect sync frame status. + } + + void ValidateReference(const BufferFlags& flags, + const BufferState& buffer_state, + int temporal_layer) { + if (flags & BufferFlags::kReference) { + if (temporal_layer > 0 && buffer_state.timestamp > 0) { + // Check that high layer reference does not go past last sync frame. + EXPECT_GE(buffer_state.timestamp, last_sync_timestamp_); + } + // No reference to buffer in higher layer. + EXPECT_LE(buffer_state.temporal_idx, temporal_layer); + } + } + + uint32_t timestamp_ = 1; + uint32_t last_sync_timestamp_ = timestamp_; + BufferState* tl0_reference_; + + BufferState last_state; + BufferState golden_state; + BufferState altref_state; +}; + +INSTANTIATE_TEST_SUITE_P(DefaultTemporalLayersTest, + TemporalLayersReferenceTest, + ::testing::Range(1, kMaxTemporalStreams + 1)); + +TEST_P(TemporalLayersReferenceTest, ValidFrameConfigs) { + const int num_layers = GetParam(); + DefaultTemporalLayers tl(num_layers); + tl.OnRatesUpdated( + 0, GetTemporalLayerRates(kDefaultBytesPerFrame, kDefaultFramerate, 1), + kDefaultFramerate); + tl.UpdateConfiguration(0); + + // Run through the pattern and store the frame dependencies, plus keep track + // of the buffer state; which buffers references which temporal layers (if + // (any). If a given buffer is never updated, it is legal to reference it + // even for sync frames. In order to be general, don't assume TL0 always + // updates `last`. + std::vector<Vp8FrameConfig> tl_configs(kMaxPatternLength); + for (int i = 0; i < kMaxPatternLength; ++i) { + Vp8FrameConfig tl_config = tl.NextFrameConfig(0, timestamp_); + tl.OnEncodeDone(0, timestamp_, kDefaultBytesPerFrame, i == 0, kDefaultQp, + IgnoredCodecSpecificInfo()); + ++timestamp_; + EXPECT_FALSE(tl_config.drop_frame); + tl_configs.push_back(tl_config); + int temporal_idx = tl_config.encoder_layer_id; + // For the default layers, always keep encoder and rtp layers in sync. + EXPECT_EQ(tl_config.packetizer_temporal_idx, temporal_idx); + + // Determine if this frame is in a higher layer but references only TL0 + // or untouched buffers, if so verify it is marked as a layer sync. + bool is_sync_frame = true; + tl0_reference_ = nullptr; + if (temporal_idx <= 0) { + is_sync_frame = false; // TL0 by definition not a sync frame. + } else if (!UpdateSyncRefState(tl_config.last_buffer_flags, &last_state)) { + is_sync_frame = false; + } else if (!UpdateSyncRefState(tl_config.golden_buffer_flags, + &golden_state)) { + is_sync_frame = false; + } else if (!UpdateSyncRefState(tl_config.arf_buffer_flags, &altref_state)) { + is_sync_frame = false; + } + if (is_sync_frame) { + // Cache timestamp for last found sync frame, so that we can verify no + // references back past this frame. + ASSERT_TRUE(tl0_reference_); + last_sync_timestamp_ = tl0_reference_->timestamp; + } + EXPECT_EQ(tl_config.layer_sync, is_sync_frame); + + // Validate no reference from lower to high temporal layer, or backwards + // past last reference frame. + ValidateReference(tl_config.last_buffer_flags, last_state, temporal_idx); + ValidateReference(tl_config.golden_buffer_flags, golden_state, + temporal_idx); + ValidateReference(tl_config.arf_buffer_flags, altref_state, temporal_idx); + + // Update the current layer state. + BufferState state = {temporal_idx, timestamp_, is_sync_frame}; + if (tl_config.last_buffer_flags & BufferFlags::kUpdate) + last_state = state; + if (tl_config.golden_buffer_flags & BufferFlags::kUpdate) + golden_state = state; + if (tl_config.arf_buffer_flags & BufferFlags::kUpdate) + altref_state = state; + } +} +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/temporal_layers_checker.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/temporal_layers_checker.h new file mode 100644 index 0000000000..3d1671a676 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/temporal_layers_checker.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_TEMPORAL_LAYERS_CHECKER_H_ +#define MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_TEMPORAL_LAYERS_CHECKER_H_ + +#include <stdint.h> + +#include <memory> + +#include "api/video_codecs/vp8_frame_config.h" +#include "api/video_codecs/vp8_temporal_layers.h" + +namespace webrtc { + +// Interface for a class that verifies correctness of temporal layer +// configurations (dependencies, sync flag, etc). +// Intended to be used in tests as well as with real apps in debug mode. +class TemporalLayersChecker { + public: + explicit TemporalLayersChecker(int num_temporal_layers); + virtual ~TemporalLayersChecker() {} + + virtual bool CheckTemporalConfig(bool frame_is_keyframe, + const Vp8FrameConfig& frame_config); + + static std::unique_ptr<TemporalLayersChecker> CreateTemporalLayersChecker( + Vp8TemporalLayersType type, + int num_temporal_layers); + + private: + struct BufferState { + BufferState() : is_keyframe(true), temporal_layer(0), sequence_number(0) {} + bool is_keyframe; + uint8_t temporal_layer; + uint32_t sequence_number; + }; + bool CheckAndUpdateBufferState(BufferState* state, + bool* need_sync, + bool frame_is_keyframe, + uint8_t temporal_layer, + Vp8FrameConfig::BufferFlags flags, + uint32_t sequence_number, + uint32_t* lowest_sequence_referenced); + BufferState last_; + BufferState arf_; + BufferState golden_; + int num_temporal_layers_; + uint32_t sequence_number_; + uint32_t last_sync_sequence_number_; + uint32_t last_tl0_sequence_number_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_TEMPORAL_LAYERS_CHECKER_H_ diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/vp8.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/vp8.h new file mode 100644 index 0000000000..2fc647874f --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/vp8.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_H_ +#define MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_H_ + +#include <memory> +#include <vector> + +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/vp8_frame_buffer_controller.h" +#include "modules/video_coding/include/video_codec_interface.h" + +namespace webrtc { + +// TODO(brandtr): Move these interfaces to the api/ folder. +class VP8Encoder { + public: + struct Settings { + // Allows for overriding the Vp8FrameBufferController used by the encoder. + // If unset, a default Vp8FrameBufferController will be instantiated + // internally. + std::unique_ptr<Vp8FrameBufferControllerFactory> + frame_buffer_controller_factory = nullptr; + + // Allows for overriding the resolution/bitrate limits exposed through + // VideoEncoder::GetEncoderInfo(). No override is done if empty. + std::vector<VideoEncoder::ResolutionBitrateLimits> + resolution_bitrate_limits = {}; + }; + + static std::unique_ptr<VideoEncoder> Create(); + static std::unique_ptr<VideoEncoder> Create(Settings settings); +}; + +class VP8Decoder { + public: + static std::unique_ptr<VideoDecoder> Create(); +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_H_ diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/vp8_globals.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/vp8_globals.h new file mode 100644 index 0000000000..1fab5f45a6 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/vp8_globals.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +// This file contains codec dependent definitions that are needed in +// order to compile the WebRTC codebase, even if this codec is not used. + +#ifndef MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_GLOBALS_H_ +#define MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_GLOBALS_H_ + +#include "modules/video_coding/codecs/interface/common_constants.h" + +namespace webrtc { + +struct RTPVideoHeaderVP8 { + void InitRTPVideoHeaderVP8() { + nonReference = false; + pictureId = kNoPictureId; + tl0PicIdx = kNoTl0PicIdx; + temporalIdx = kNoTemporalIdx; + layerSync = false; + keyIdx = kNoKeyIdx; + partitionId = 0; + beginningOfPartition = false; + } + + bool nonReference; // Frame is discardable. + int16_t pictureId; // Picture ID index, 15 bits; + // kNoPictureId if PictureID does not exist. + int16_t tl0PicIdx; // TL0PIC_IDX, 8 bits; + // kNoTl0PicIdx means no value provided. + uint8_t temporalIdx; // Temporal layer index, or kNoTemporalIdx. + bool layerSync; // This frame is a layer sync frame. + // Disabled if temporalIdx == kNoTemporalIdx. + int keyIdx; // 5 bits; kNoKeyIdx means not used. + int partitionId; // VP8 partition ID + bool beginningOfPartition; // True if this packet is the first + // in a VP8 partition. Otherwise false +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_GLOBALS_H_ diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc new file mode 100644 index 0000000000..3fe86f2f85 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc @@ -0,0 +1,384 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h" + +#include <stdio.h> +#include <string.h> + +#include <algorithm> +#include <memory> +#include <string> + +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_buffer.h" +#include "api/video/video_rotation.h" +#include "modules/video_coding/codecs/vp8/include/vp8.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/checks.h" +#include "rtc_base/numerics/exp_filter.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/field_trial.h" +#include "system_wrappers/include/metrics.h" +#include "third_party/libyuv/include/libyuv/convert.h" +#include "vpx/vp8.h" +#include "vpx/vp8dx.h" +#include "vpx/vpx_decoder.h" + +namespace webrtc { +namespace { +constexpr int kVp8ErrorPropagationTh = 30; +// vpx_decoder.h documentation indicates decode deadline is time in us, with +// "Set to zero for unlimited.", but actual implementation requires this to be +// a mode with 0 meaning allow delay and 1 not allowing it. +constexpr long kDecodeDeadlineRealtime = 1; // NOLINT + +const char kVp8PostProcArmFieldTrial[] = "WebRTC-VP8-Postproc-Config-Arm"; +const char kVp8PostProcFieldTrial[] = "WebRTC-VP8-Postproc-Config"; + +#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \ + defined(WEBRTC_ANDROID) || defined(WEBRTC_ARCH_MIPS) +constexpr bool kIsArm = true; +#else +constexpr bool kIsArm = false; +#endif + +absl::optional<LibvpxVp8Decoder::DeblockParams> DefaultDeblockParams() { + return LibvpxVp8Decoder::DeblockParams(/*max_level=*/8, + /*degrade_qp=*/60, + /*min_qp=*/30); +} + +absl::optional<LibvpxVp8Decoder::DeblockParams> +GetPostProcParamsFromFieldTrialGroup() { + std::string group = webrtc::field_trial::FindFullName( + kIsArm ? kVp8PostProcArmFieldTrial : kVp8PostProcFieldTrial); + if (group.empty()) { + return DefaultDeblockParams(); + } + + LibvpxVp8Decoder::DeblockParams params; + if (sscanf(group.c_str(), "Enabled-%d,%d,%d", ¶ms.max_level, + ¶ms.min_qp, ¶ms.degrade_qp) != 3) { + return DefaultDeblockParams(); + } + + if (params.max_level < 0 || params.max_level > 16) { + return DefaultDeblockParams(); + } + + if (params.min_qp < 0 || params.degrade_qp <= params.min_qp) { + return DefaultDeblockParams(); + } + + return params; +} + +} // namespace + +std::unique_ptr<VideoDecoder> VP8Decoder::Create() { + return std::make_unique<LibvpxVp8Decoder>(); +} + +class LibvpxVp8Decoder::QpSmoother { + public: + QpSmoother() : last_sample_ms_(rtc::TimeMillis()), smoother_(kAlpha) {} + + int GetAvg() const { + float value = smoother_.filtered(); + return (value == rtc::ExpFilter::kValueUndefined) ? 0 + : static_cast<int>(value); + } + + void Add(float sample) { + int64_t now_ms = rtc::TimeMillis(); + smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample); + last_sample_ms_ = now_ms; + } + + void Reset() { smoother_.Reset(kAlpha); } + + private: + const float kAlpha = 0.95f; + int64_t last_sample_ms_; + rtc::ExpFilter smoother_; +}; + +LibvpxVp8Decoder::LibvpxVp8Decoder() + : use_postproc_( + kIsArm ? webrtc::field_trial::IsEnabled(kVp8PostProcArmFieldTrial) + : true), + buffer_pool_(false, 300 /* max_number_of_buffers*/), + decode_complete_callback_(NULL), + inited_(false), + decoder_(NULL), + propagation_cnt_(-1), + last_frame_width_(0), + last_frame_height_(0), + key_frame_required_(true), + deblock_params_(use_postproc_ ? GetPostProcParamsFromFieldTrialGroup() + : absl::nullopt), + qp_smoother_(use_postproc_ ? new QpSmoother() : nullptr) {} + +LibvpxVp8Decoder::~LibvpxVp8Decoder() { + inited_ = true; // in order to do the actual release + Release(); +} + +bool LibvpxVp8Decoder::Configure(const Settings& settings) { + if (Release() < 0) { + return false; + } + if (decoder_ == NULL) { + decoder_ = new vpx_codec_ctx_t; + memset(decoder_, 0, sizeof(*decoder_)); + } + vpx_codec_dec_cfg_t cfg; + // Setting number of threads to a constant value (1) + cfg.threads = 1; + cfg.h = cfg.w = 0; // set after decode + + vpx_codec_flags_t flags = use_postproc_ ? VPX_CODEC_USE_POSTPROC : 0; + + if (vpx_codec_dec_init(decoder_, vpx_codec_vp8_dx(), &cfg, flags)) { + delete decoder_; + decoder_ = nullptr; + return false; + } + + propagation_cnt_ = -1; + inited_ = true; + + // Always start with a complete key frame. + key_frame_required_ = true; + if (absl::optional<int> buffer_pool_size = settings.buffer_pool_size()) { + if (!buffer_pool_.Resize(*buffer_pool_size)) { + return false; + } + } + return true; +} + +int LibvpxVp8Decoder::Decode(const EncodedImage& input_image, + bool missing_frames, + int64_t /*render_time_ms*/) { + if (!inited_) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (decode_complete_callback_ == NULL) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (input_image.data() == NULL && input_image.size() > 0) { + // Reset to avoid requesting key frames too often. + if (propagation_cnt_ > 0) + propagation_cnt_ = 0; + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + // Post process configurations. + if (use_postproc_) { + vp8_postproc_cfg_t ppcfg; + // MFQE enabled to reduce key frame popping. + ppcfg.post_proc_flag = VP8_MFQE; + + if (kIsArm) { + RTC_DCHECK(deblock_params_.has_value()); + } + if (deblock_params_.has_value()) { + // For low resolutions, use stronger deblocking filter. + int last_width_x_height = last_frame_width_ * last_frame_height_; + if (last_width_x_height > 0 && last_width_x_height <= 320 * 240) { + // Enable the deblock and demacroblocker based on qp thresholds. + RTC_DCHECK(qp_smoother_); + int qp = qp_smoother_->GetAvg(); + if (qp > deblock_params_->min_qp) { + int level = deblock_params_->max_level; + if (qp < deblock_params_->degrade_qp) { + // Use lower level. + level = deblock_params_->max_level * + (qp - deblock_params_->min_qp) / + (deblock_params_->degrade_qp - deblock_params_->min_qp); + } + // Deblocking level only affects VP8_DEMACROBLOCK. + ppcfg.deblocking_level = std::max(level, 1); + ppcfg.post_proc_flag |= VP8_DEBLOCK | VP8_DEMACROBLOCK; + } + } + } else { + // Non-arm with no explicit deblock params set. + ppcfg.post_proc_flag |= VP8_DEBLOCK; + // For VGA resolutions and lower, enable the demacroblocker postproc. + if (last_frame_width_ * last_frame_height_ <= 640 * 360) { + ppcfg.post_proc_flag |= VP8_DEMACROBLOCK; + } + // Strength of deblocking filter. Valid range:[0,16] + ppcfg.deblocking_level = 3; + } + + vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg); + } + + // Always start with a complete key frame. + if (key_frame_required_) { + if (input_image._frameType != VideoFrameType::kVideoFrameKey) + return WEBRTC_VIDEO_CODEC_ERROR; + key_frame_required_ = false; + } + // Restrict error propagation using key frame requests. + // Reset on a key frame refresh. + if (input_image._frameType == VideoFrameType::kVideoFrameKey) { + propagation_cnt_ = -1; + // Start count on first loss. + } else if (missing_frames && propagation_cnt_ == -1) { + propagation_cnt_ = 0; + } + if (propagation_cnt_ >= 0) { + propagation_cnt_++; + } + + vpx_codec_iter_t iter = NULL; + vpx_image_t* img; + int ret; + + // Check for missing frames. + if (missing_frames) { + // Call decoder with zero data length to signal missing frames. + if (vpx_codec_decode(decoder_, NULL, 0, 0, kDecodeDeadlineRealtime)) { + // Reset to avoid requesting key frames too often. + if (propagation_cnt_ > 0) + propagation_cnt_ = 0; + return WEBRTC_VIDEO_CODEC_ERROR; + } + img = vpx_codec_get_frame(decoder_, &iter); + iter = NULL; + } + + const uint8_t* buffer = input_image.data(); + if (input_image.size() == 0) { + buffer = NULL; // Triggers full frame concealment. + } + if (vpx_codec_decode(decoder_, buffer, input_image.size(), 0, + kDecodeDeadlineRealtime)) { + // Reset to avoid requesting key frames too often. + if (propagation_cnt_ > 0) { + propagation_cnt_ = 0; + } + return WEBRTC_VIDEO_CODEC_ERROR; + } + + img = vpx_codec_get_frame(decoder_, &iter); + int qp; + vpx_codec_err_t vpx_ret = + vpx_codec_control(decoder_, VPXD_GET_LAST_QUANTIZER, &qp); + RTC_DCHECK_EQ(vpx_ret, VPX_CODEC_OK); + ret = ReturnFrame(img, input_image.Timestamp(), qp, input_image.ColorSpace()); + if (ret != 0) { + // Reset to avoid requesting key frames too often. + if (ret < 0 && propagation_cnt_ > 0) + propagation_cnt_ = 0; + return ret; + } + // Check Vs. threshold + if (propagation_cnt_ > kVp8ErrorPropagationTh) { + // Reset to avoid requesting key frames too often. + propagation_cnt_ = 0; + return WEBRTC_VIDEO_CODEC_ERROR; + } + return WEBRTC_VIDEO_CODEC_OK; +} + +int LibvpxVp8Decoder::ReturnFrame( + const vpx_image_t* img, + uint32_t timestamp, + int qp, + const webrtc::ColorSpace* explicit_color_space) { + if (img == NULL) { + // Decoder OK and NULL image => No show frame + return WEBRTC_VIDEO_CODEC_NO_OUTPUT; + } + if (qp_smoother_) { + if (last_frame_width_ != static_cast<int>(img->d_w) || + last_frame_height_ != static_cast<int>(img->d_h)) { + qp_smoother_->Reset(); + } + qp_smoother_->Add(qp); + } + last_frame_width_ = img->d_w; + last_frame_height_ = img->d_h; + // Allocate memory for decoded image. + rtc::scoped_refptr<VideoFrameBuffer> buffer; + + rtc::scoped_refptr<I420Buffer> i420_buffer = + buffer_pool_.CreateI420Buffer(img->d_w, img->d_h); + buffer = i420_buffer; + if (i420_buffer.get()) { + libyuv::I420Copy(img->planes[VPX_PLANE_Y], img->stride[VPX_PLANE_Y], + img->planes[VPX_PLANE_U], img->stride[VPX_PLANE_U], + img->planes[VPX_PLANE_V], img->stride[VPX_PLANE_V], + i420_buffer->MutableDataY(), i420_buffer->StrideY(), + i420_buffer->MutableDataU(), i420_buffer->StrideU(), + i420_buffer->MutableDataV(), i420_buffer->StrideV(), + img->d_w, img->d_h); + } + + if (!buffer.get()) { + // Pool has too many pending frames. + RTC_HISTOGRAM_BOOLEAN("WebRTC.Video.LibvpxVp8Decoder.TooManyPendingFrames", + 1); + return WEBRTC_VIDEO_CODEC_NO_OUTPUT; + } + + VideoFrame decoded_image = VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp) + .set_color_space(explicit_color_space) + .build(); + decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp); + + return WEBRTC_VIDEO_CODEC_OK; +} + +int LibvpxVp8Decoder::RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) { + decode_complete_callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; +} + +int LibvpxVp8Decoder::Release() { + int ret_val = WEBRTC_VIDEO_CODEC_OK; + + if (decoder_ != NULL) { + if (inited_) { + if (vpx_codec_destroy(decoder_)) { + ret_val = WEBRTC_VIDEO_CODEC_MEMORY; + } + } + delete decoder_; + decoder_ = NULL; + } + buffer_pool_.Release(); + inited_ = false; + return ret_val; +} + +VideoDecoder::DecoderInfo LibvpxVp8Decoder::GetDecoderInfo() const { + DecoderInfo info; + info.implementation_name = "libvpx"; + info.is_hardware_accelerated = false; + return info; +} + +const char* LibvpxVp8Decoder::ImplementationName() const { + return "libvpx"; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h new file mode 100644 index 0000000000..f9acd70bad --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_DECODER_H_ +#define MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_DECODER_H_ + +#include <memory> + +#include "absl/types/optional.h" +#include "api/video/encoded_image.h" +#include "api/video_codecs/video_decoder.h" +#include "common_video/include/video_frame_buffer_pool.h" +#include "modules/video_coding/codecs/vp8/include/vp8.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "vpx/vp8dx.h" +#include "vpx/vpx_decoder.h" + +namespace webrtc { + +class LibvpxVp8Decoder : public VideoDecoder { + public: + LibvpxVp8Decoder(); + ~LibvpxVp8Decoder() override; + + bool Configure(const Settings& settings) override; + int Decode(const EncodedImage& input_image, + bool missing_frames, + int64_t /*render_time_ms*/) override; + + int RegisterDecodeCompleteCallback(DecodedImageCallback* callback) override; + int Release() override; + + DecoderInfo GetDecoderInfo() const override; + const char* ImplementationName() const override; + + struct DeblockParams { + DeblockParams() : max_level(6), degrade_qp(1), min_qp(0) {} + DeblockParams(int max_level, int degrade_qp, int min_qp) + : max_level(max_level), degrade_qp(degrade_qp), min_qp(min_qp) {} + int max_level; // Deblocking strength: [0, 16]. + int degrade_qp; // If QP value is below, start lowering `max_level`. + int min_qp; // If QP value is below, turn off deblocking. + }; + + private: + class QpSmoother; + int ReturnFrame(const vpx_image_t* img, + uint32_t timeStamp, + int qp, + const webrtc::ColorSpace* explicit_color_space); + const bool use_postproc_; + + VideoFrameBufferPool buffer_pool_; + DecodedImageCallback* decode_complete_callback_; + bool inited_; + vpx_codec_ctx_t* decoder_; + int propagation_cnt_; + int last_frame_width_; + int last_frame_height_; + bool key_frame_required_; + const absl::optional<DeblockParams> deblock_params_; + const std::unique_ptr<QpSmoother> qp_smoother_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_DECODER_H_ diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc new file mode 100644 index 0000000000..cc84605ce7 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc @@ -0,0 +1,1438 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h" + +#include <string.h> + +#include <algorithm> +#include <cstdint> +#include <iterator> +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "absl/algorithm/container.h" +#include "api/scoped_refptr.h" +#include "api/video/video_content_type.h" +#include "api/video/video_frame_buffer.h" +#include "api/video/video_timing.h" +#include "api/video_codecs/scalability_mode.h" +#include "api/video_codecs/vp8_temporal_layers.h" +#include "api/video_codecs/vp8_temporal_layers_factory.h" +#include "modules/video_coding/codecs/interface/common_constants.h" +#include "modules/video_coding/codecs/vp8/include/vp8.h" +#include "modules/video_coding/codecs/vp8/vp8_scalability.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/svc/scalability_mode_util.h" +#include "modules/video_coding/utility/simulcast_rate_allocator.h" +#include "modules/video_coding/utility/simulcast_utility.h" +#include "rtc_base/checks.h" +#include "rtc_base/experiments/field_trial_parser.h" +#include "rtc_base/experiments/field_trial_units.h" +#include "rtc_base/logging.h" +#include "rtc_base/trace_event.h" +#include "system_wrappers/include/field_trial.h" +#include "third_party/libyuv/include/libyuv/scale.h" +#include "vpx/vp8cx.h" + +namespace webrtc { +namespace { +#if defined(WEBRTC_IOS) +constexpr char kVP8IosMaxNumberOfThreadFieldTrial[] = + "WebRTC-VP8IosMaxNumberOfThread"; +constexpr char kVP8IosMaxNumberOfThreadFieldTrialParameter[] = "max_thread"; +#endif + +constexpr char kVp8ForcePartitionResilience[] = + "WebRTC-VP8-ForcePartitionResilience"; + +// QP is obtained from VP8-bitstream for HW, so the QP corresponds to the +// bitstream range of [0, 127] and not the user-level range of [0,63]. +constexpr int kLowVp8QpThreshold = 29; +constexpr int kHighVp8QpThreshold = 95; + +constexpr int kTokenPartitions = VP8_ONE_TOKENPARTITION; +constexpr uint32_t kVp832ByteAlign = 32u; + +constexpr int kRtpTicksPerSecond = 90000; +constexpr int kRtpTicksPerMs = kRtpTicksPerSecond / 1000; + +// VP8 denoiser states. +enum denoiserState : uint32_t { + kDenoiserOff, + kDenoiserOnYOnly, + kDenoiserOnYUV, + kDenoiserOnYUVAggressive, + // Adaptive mode defaults to kDenoiserOnYUV on key frame, but may switch + // to kDenoiserOnYUVAggressive based on a computed noise metric. + kDenoiserOnAdaptive +}; + +// Greatest common divisior +int GCD(int a, int b) { + int c = a % b; + while (c != 0) { + a = b; + b = c; + c = a % b; + } + return b; +} + +static_assert(Vp8EncoderConfig::TemporalLayerConfig::kMaxPeriodicity == + VPX_TS_MAX_PERIODICITY, + "Vp8EncoderConfig::kMaxPeriodicity must be kept in sync with the " + "constant in libvpx."); +static_assert(Vp8EncoderConfig::TemporalLayerConfig::kMaxLayers == + VPX_TS_MAX_LAYERS, + "Vp8EncoderConfig::kMaxLayers must be kept in sync with the " + "constant in libvpx."); + +// Allow a newer value to override a current value only if the new value +// is set. +template <typename T> +bool MaybeSetNewValue(const absl::optional<T>& new_value, + absl::optional<T>* base_value) { + if (new_value.has_value() && new_value != *base_value) { + *base_value = new_value; + return true; + } else { + return false; + } +} + +// Adds configuration from `new_config` to `base_config`. Both configs consist +// of optionals, and only optionals which are set in `new_config` can have +// an effect. (That is, set values in `base_config` cannot be unset.) +// Returns `true` iff any changes were made to `base_config`. +bool MaybeExtendVp8EncoderConfig(const Vp8EncoderConfig& new_config, + Vp8EncoderConfig* base_config) { + bool changes_made = false; + changes_made |= MaybeSetNewValue(new_config.temporal_layer_config, + &base_config->temporal_layer_config); + changes_made |= MaybeSetNewValue(new_config.rc_target_bitrate, + &base_config->rc_target_bitrate); + changes_made |= MaybeSetNewValue(new_config.rc_max_quantizer, + &base_config->rc_max_quantizer); + changes_made |= MaybeSetNewValue(new_config.g_error_resilient, + &base_config->g_error_resilient); + return changes_made; +} + +void ApplyVp8EncoderConfigToVpxConfig(const Vp8EncoderConfig& encoder_config, + vpx_codec_enc_cfg_t* vpx_config) { + if (encoder_config.temporal_layer_config.has_value()) { + const Vp8EncoderConfig::TemporalLayerConfig& ts_config = + encoder_config.temporal_layer_config.value(); + vpx_config->ts_number_layers = ts_config.ts_number_layers; + std::copy(ts_config.ts_target_bitrate.begin(), + ts_config.ts_target_bitrate.end(), + std::begin(vpx_config->ts_target_bitrate)); + std::copy(ts_config.ts_rate_decimator.begin(), + ts_config.ts_rate_decimator.end(), + std::begin(vpx_config->ts_rate_decimator)); + vpx_config->ts_periodicity = ts_config.ts_periodicity; + std::copy(ts_config.ts_layer_id.begin(), ts_config.ts_layer_id.end(), + std::begin(vpx_config->ts_layer_id)); + } else { + vpx_config->ts_number_layers = 1; + vpx_config->ts_rate_decimator[0] = 1; + vpx_config->ts_periodicity = 1; + vpx_config->ts_layer_id[0] = 0; + } + + if (encoder_config.rc_target_bitrate.has_value()) { + vpx_config->rc_target_bitrate = encoder_config.rc_target_bitrate.value(); + } + + if (encoder_config.rc_max_quantizer.has_value()) { + vpx_config->rc_max_quantizer = encoder_config.rc_max_quantizer.value(); + } + + if (encoder_config.g_error_resilient.has_value()) { + vpx_config->g_error_resilient = encoder_config.g_error_resilient.value(); + } +} + +bool IsCompatibleVideoFrameBufferType(VideoFrameBuffer::Type left, + VideoFrameBuffer::Type right) { + if (left == VideoFrameBuffer::Type::kI420 || + left == VideoFrameBuffer::Type::kI420A) { + // LibvpxVp8Encoder does not care about the alpha channel, I420A and I420 + // are considered compatible. + return right == VideoFrameBuffer::Type::kI420 || + right == VideoFrameBuffer::Type::kI420A; + } + return left == right; +} + +void SetRawImagePlanes(vpx_image_t* raw_image, VideoFrameBuffer* buffer) { + switch (buffer->type()) { + case VideoFrameBuffer::Type::kI420: + case VideoFrameBuffer::Type::kI420A: { + const I420BufferInterface* i420_buffer = buffer->GetI420(); + RTC_DCHECK(i420_buffer); + raw_image->planes[VPX_PLANE_Y] = + const_cast<uint8_t*>(i420_buffer->DataY()); + raw_image->planes[VPX_PLANE_U] = + const_cast<uint8_t*>(i420_buffer->DataU()); + raw_image->planes[VPX_PLANE_V] = + const_cast<uint8_t*>(i420_buffer->DataV()); + raw_image->stride[VPX_PLANE_Y] = i420_buffer->StrideY(); + raw_image->stride[VPX_PLANE_U] = i420_buffer->StrideU(); + raw_image->stride[VPX_PLANE_V] = i420_buffer->StrideV(); + break; + } + case VideoFrameBuffer::Type::kNV12: { + const NV12BufferInterface* nv12_buffer = buffer->GetNV12(); + RTC_DCHECK(nv12_buffer); + raw_image->planes[VPX_PLANE_Y] = + const_cast<uint8_t*>(nv12_buffer->DataY()); + raw_image->planes[VPX_PLANE_U] = + const_cast<uint8_t*>(nv12_buffer->DataUV()); + raw_image->planes[VPX_PLANE_V] = raw_image->planes[VPX_PLANE_U] + 1; + raw_image->stride[VPX_PLANE_Y] = nv12_buffer->StrideY(); + raw_image->stride[VPX_PLANE_U] = nv12_buffer->StrideUV(); + raw_image->stride[VPX_PLANE_V] = nv12_buffer->StrideUV(); + break; + } + default: + RTC_DCHECK_NOTREACHED(); + } +} + +} // namespace + +std::unique_ptr<VideoEncoder> VP8Encoder::Create() { + return std::make_unique<LibvpxVp8Encoder>(LibvpxInterface::Create(), + VP8Encoder::Settings()); +} + +std::unique_ptr<VideoEncoder> VP8Encoder::Create( + VP8Encoder::Settings settings) { + return std::make_unique<LibvpxVp8Encoder>(LibvpxInterface::Create(), + std::move(settings)); +} + +vpx_enc_frame_flags_t LibvpxVp8Encoder::EncodeFlags( + const Vp8FrameConfig& references) { + RTC_DCHECK(!references.drop_frame); + + vpx_enc_frame_flags_t flags = 0; + + if ((references.last_buffer_flags & + Vp8FrameConfig::BufferFlags::kReference) == 0) + flags |= VP8_EFLAG_NO_REF_LAST; + if ((references.last_buffer_flags & Vp8FrameConfig::BufferFlags::kUpdate) == + 0) + flags |= VP8_EFLAG_NO_UPD_LAST; + if ((references.golden_buffer_flags & + Vp8FrameConfig::BufferFlags::kReference) == 0) + flags |= VP8_EFLAG_NO_REF_GF; + if ((references.golden_buffer_flags & Vp8FrameConfig::BufferFlags::kUpdate) == + 0) + flags |= VP8_EFLAG_NO_UPD_GF; + if ((references.arf_buffer_flags & Vp8FrameConfig::BufferFlags::kReference) == + 0) + flags |= VP8_EFLAG_NO_REF_ARF; + if ((references.arf_buffer_flags & Vp8FrameConfig::BufferFlags::kUpdate) == 0) + flags |= VP8_EFLAG_NO_UPD_ARF; + if (references.freeze_entropy) + flags |= VP8_EFLAG_NO_UPD_ENTROPY; + + return flags; +} + +LibvpxVp8Encoder::LibvpxVp8Encoder(std::unique_ptr<LibvpxInterface> interface, + VP8Encoder::Settings settings) + : libvpx_(std::move(interface)), + rate_control_settings_(RateControlSettings::ParseFromFieldTrials()), + frame_buffer_controller_factory_( + std::move(settings.frame_buffer_controller_factory)), + resolution_bitrate_limits_(std::move(settings.resolution_bitrate_limits)), + key_frame_request_(kMaxSimulcastStreams, false), + variable_framerate_experiment_(ParseVariableFramerateConfig( + "WebRTC-VP8VariableFramerateScreenshare")), + framerate_controller_(variable_framerate_experiment_.framerate_limit) { + // TODO(eladalon/ilnik): These reservations might be wasting memory. + // InitEncode() is resizing to the actual size, which might be smaller. + raw_images_.reserve(kMaxSimulcastStreams); + encoded_images_.reserve(kMaxSimulcastStreams); + send_stream_.reserve(kMaxSimulcastStreams); + cpu_speed_.assign(kMaxSimulcastStreams, cpu_speed_default_); + encoders_.reserve(kMaxSimulcastStreams); + vpx_configs_.reserve(kMaxSimulcastStreams); + config_overrides_.reserve(kMaxSimulcastStreams); + downsampling_factors_.reserve(kMaxSimulcastStreams); +} + +LibvpxVp8Encoder::~LibvpxVp8Encoder() { + Release(); +} + +int LibvpxVp8Encoder::Release() { + int ret_val = WEBRTC_VIDEO_CODEC_OK; + + encoded_images_.clear(); + + if (inited_) { + for (auto it = encoders_.rbegin(); it != encoders_.rend(); ++it) { + if (libvpx_->codec_destroy(&*it)) { + ret_val = WEBRTC_VIDEO_CODEC_MEMORY; + } + } + } + encoders_.clear(); + + vpx_configs_.clear(); + config_overrides_.clear(); + send_stream_.clear(); + cpu_speed_.clear(); + + for (auto it = raw_images_.rbegin(); it != raw_images_.rend(); ++it) { + libvpx_->img_free(&*it); + } + raw_images_.clear(); + + frame_buffer_controller_.reset(); + inited_ = false; + return ret_val; +} + +void LibvpxVp8Encoder::SetRates(const RateControlParameters& parameters) { + if (!inited_) { + RTC_LOG(LS_WARNING) << "SetRates() while not initialize"; + return; + } + + if (encoders_[0].err) { + RTC_LOG(LS_WARNING) << "Encoder in error state."; + return; + } + + if (parameters.framerate_fps < 1.0) { + RTC_LOG(LS_WARNING) << "Unsupported framerate (must be >= 1.0): " + << parameters.framerate_fps; + return; + } + + if (parameters.bitrate.get_sum_bps() == 0) { + // Encoder paused, turn off all encoding. + const int num_streams = static_cast<size_t>(encoders_.size()); + for (int i = 0; i < num_streams; ++i) + SetStreamState(false, i); + return; + } + + codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5); + + if (encoders_.size() > 1) { + // If we have more than 1 stream, reduce the qp_max for the low resolution + // stream if frame rate is not too low. The trade-off with lower qp_max is + // possibly more dropped frames, so we only do this if the frame rate is + // above some threshold (base temporal layer is down to 1/4 for 3 layers). + // We may want to condition this on bitrate later. + if (rate_control_settings_.Vp8BoostBaseLayerQuality() && + parameters.framerate_fps > 20.0) { + vpx_configs_[encoders_.size() - 1].rc_max_quantizer = 45; + } else { + // Go back to default value set in InitEncode. + vpx_configs_[encoders_.size() - 1].rc_max_quantizer = qp_max_; + } + } + + for (size_t i = 0; i < encoders_.size(); ++i) { + const size_t stream_idx = encoders_.size() - 1 - i; + + unsigned int target_bitrate_kbps = + parameters.bitrate.GetSpatialLayerSum(stream_idx) / 1000; + + bool send_stream = target_bitrate_kbps > 0; + if (send_stream || encoders_.size() > 1) + SetStreamState(send_stream, stream_idx); + + vpx_configs_[i].rc_target_bitrate = target_bitrate_kbps; + if (send_stream) { + frame_buffer_controller_->OnRatesUpdated( + stream_idx, parameters.bitrate.GetTemporalLayerAllocation(stream_idx), + static_cast<int>(parameters.framerate_fps + 0.5)); + } + + UpdateVpxConfiguration(stream_idx); + + vpx_codec_err_t err = + libvpx_->codec_enc_config_set(&encoders_[i], &vpx_configs_[i]); + if (err != VPX_CODEC_OK) { + RTC_LOG(LS_WARNING) << "Error configuring codec, error code: " << err + << ", details: " + << libvpx_->codec_error_detail(&encoders_[i]); + } + } +} + +void LibvpxVp8Encoder::OnPacketLossRateUpdate(float packet_loss_rate) { + // TODO(bugs.webrtc.org/10431): Replace condition by DCHECK. + if (frame_buffer_controller_) { + frame_buffer_controller_->OnPacketLossRateUpdate(packet_loss_rate); + } +} + +void LibvpxVp8Encoder::OnRttUpdate(int64_t rtt_ms) { + // TODO(bugs.webrtc.org/10431): Replace condition by DCHECK. + if (frame_buffer_controller_) { + frame_buffer_controller_->OnRttUpdate(rtt_ms); + } +} + +void LibvpxVp8Encoder::OnLossNotification( + const LossNotification& loss_notification) { + if (frame_buffer_controller_) { + frame_buffer_controller_->OnLossNotification(loss_notification); + } +} + +void LibvpxVp8Encoder::SetStreamState(bool send_stream, int stream_idx) { + if (send_stream && !send_stream_[stream_idx]) { + // Need a key frame if we have not sent this stream before. + key_frame_request_[stream_idx] = true; + } + send_stream_[stream_idx] = send_stream; +} + +void LibvpxVp8Encoder::SetFecControllerOverride( + FecControllerOverride* fec_controller_override) { + // TODO(bugs.webrtc.org/10769): Update downstream and remove ability to + // pass nullptr. + // RTC_DCHECK(fec_controller_override); + RTC_DCHECK(!fec_controller_override_); + fec_controller_override_ = fec_controller_override; +} + +// TODO(eladalon): s/inst/codec_settings/g. +int LibvpxVp8Encoder::InitEncode(const VideoCodec* inst, + const VideoEncoder::Settings& settings) { + if (inst == NULL) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (inst->maxFramerate < 1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + // allow zero to represent an unspecified maxBitRate + if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (inst->width < 1 || inst->height < 1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (settings.number_of_cores < 1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + if (absl::optional<ScalabilityMode> scalability_mode = + inst->GetScalabilityMode(); + scalability_mode.has_value() && + !VP8SupportsScalabilityMode(*scalability_mode)) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + num_active_streams_ = 0; + for (int i = 0; i < inst->numberOfSimulcastStreams; ++i) { + if (inst->simulcastStream[i].active) { + ++num_active_streams_; + } + } + if (inst->numberOfSimulcastStreams == 0 && inst->active) { + num_active_streams_ = 1; + } + + if (inst->VP8().automaticResizeOn && num_active_streams_ > 1) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + // Use the previous pixel format to avoid extra image allocations. + vpx_img_fmt_t pixel_format = + raw_images_.empty() ? VPX_IMG_FMT_I420 : raw_images_[0].fmt; + + int retVal = Release(); + if (retVal < 0) { + return retVal; + } + + int number_of_streams = SimulcastUtility::NumberOfSimulcastStreams(*inst); + if (number_of_streams > 1 && + !SimulcastUtility::ValidSimulcastParameters(*inst, number_of_streams)) { + return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; + } + + RTC_DCHECK(!frame_buffer_controller_); + if (frame_buffer_controller_factory_) { + frame_buffer_controller_ = frame_buffer_controller_factory_->Create( + *inst, settings, fec_controller_override_); + } else { + Vp8TemporalLayersFactory factory; + frame_buffer_controller_ = + factory.Create(*inst, settings, fec_controller_override_); + } + RTC_DCHECK(frame_buffer_controller_); + + number_of_cores_ = settings.number_of_cores; + timestamp_ = 0; + codec_ = *inst; + + // Code expects simulcastStream resolutions to be correct, make sure they are + // filled even when there are no simulcast layers. + if (codec_.numberOfSimulcastStreams == 0) { + codec_.simulcastStream[0].width = codec_.width; + codec_.simulcastStream[0].height = codec_.height; + } + + encoded_images_.resize(number_of_streams); + encoders_.resize(number_of_streams); + vpx_configs_.resize(number_of_streams); + config_overrides_.resize(number_of_streams); + downsampling_factors_.resize(number_of_streams); + raw_images_.resize(number_of_streams); + send_stream_.resize(number_of_streams); + send_stream_[0] = true; // For non-simulcast case. + cpu_speed_.resize(number_of_streams); + std::fill(key_frame_request_.begin(), key_frame_request_.end(), false); + + int idx = number_of_streams - 1; + for (int i = 0; i < (number_of_streams - 1); ++i, --idx) { + int gcd = GCD(inst->simulcastStream[idx].width, + inst->simulcastStream[idx - 1].width); + downsampling_factors_[i].num = inst->simulcastStream[idx].width / gcd; + downsampling_factors_[i].den = inst->simulcastStream[idx - 1].width / gcd; + send_stream_[i] = false; + } + if (number_of_streams > 1) { + send_stream_[number_of_streams - 1] = false; + downsampling_factors_[number_of_streams - 1].num = 1; + downsampling_factors_[number_of_streams - 1].den = 1; + } + + // populate encoder configuration with default values + if (libvpx_->codec_enc_config_default(vpx_codec_vp8_cx(), &vpx_configs_[0], + 0)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + // setting the time base of the codec + vpx_configs_[0].g_timebase.num = 1; + vpx_configs_[0].g_timebase.den = kRtpTicksPerSecond; + vpx_configs_[0].g_lag_in_frames = 0; // 0- no frame lagging + + // Set the error resilience mode for temporal layers (but not simulcast). + vpx_configs_[0].g_error_resilient = + (SimulcastUtility::NumberOfTemporalLayers(*inst, 0) > 1) + ? VPX_ERROR_RESILIENT_DEFAULT + : 0; + + // Override the error resilience mode if this is not simulcast, but we are + // using temporal layers. + if (field_trial::IsEnabled(kVp8ForcePartitionResilience) && + (number_of_streams == 1) && + (SimulcastUtility::NumberOfTemporalLayers(*inst, 0) > 1)) { + RTC_LOG(LS_INFO) << "Overriding g_error_resilient from " + << vpx_configs_[0].g_error_resilient << " to " + << VPX_ERROR_RESILIENT_PARTITIONS; + vpx_configs_[0].g_error_resilient = VPX_ERROR_RESILIENT_PARTITIONS; + } + + // rate control settings + vpx_configs_[0].rc_dropframe_thresh = FrameDropThreshold(0); + vpx_configs_[0].rc_end_usage = VPX_CBR; + vpx_configs_[0].g_pass = VPX_RC_ONE_PASS; + // Handle resizing outside of libvpx. + vpx_configs_[0].rc_resize_allowed = 0; + vpx_configs_[0].rc_min_quantizer = + codec_.mode == VideoCodecMode::kScreensharing ? 12 : 2; + if (inst->qpMax >= vpx_configs_[0].rc_min_quantizer) { + qp_max_ = inst->qpMax; + } + if (rate_control_settings_.LibvpxVp8QpMax()) { + qp_max_ = std::max(rate_control_settings_.LibvpxVp8QpMax().value(), + static_cast<int>(vpx_configs_[0].rc_min_quantizer)); + } + vpx_configs_[0].rc_max_quantizer = qp_max_; + vpx_configs_[0].rc_undershoot_pct = 100; + vpx_configs_[0].rc_overshoot_pct = 15; + vpx_configs_[0].rc_buf_initial_sz = 500; + vpx_configs_[0].rc_buf_optimal_sz = 600; + vpx_configs_[0].rc_buf_sz = 1000; + + // Set the maximum target size of any key-frame. + rc_max_intra_target_ = MaxIntraTarget(vpx_configs_[0].rc_buf_optimal_sz); + + if (inst->VP8().keyFrameInterval > 0) { + vpx_configs_[0].kf_mode = VPX_KF_AUTO; + vpx_configs_[0].kf_max_dist = inst->VP8().keyFrameInterval; + } else { + vpx_configs_[0].kf_mode = VPX_KF_DISABLED; + } + + // Allow the user to set the complexity for the base stream. + switch (inst->GetVideoEncoderComplexity()) { + case VideoCodecComplexity::kComplexityHigh: + cpu_speed_[0] = -5; + break; + case VideoCodecComplexity::kComplexityHigher: + cpu_speed_[0] = -4; + break; + case VideoCodecComplexity::kComplexityMax: + cpu_speed_[0] = -3; + break; + default: + cpu_speed_[0] = -6; + break; + } + cpu_speed_default_ = cpu_speed_[0]; + // Set encoding complexity (cpu_speed) based on resolution and/or platform. + cpu_speed_[0] = GetCpuSpeed(inst->width, inst->height); + for (int i = 1; i < number_of_streams; ++i) { + cpu_speed_[i] = + GetCpuSpeed(inst->simulcastStream[number_of_streams - 1 - i].width, + inst->simulcastStream[number_of_streams - 1 - i].height); + } + vpx_configs_[0].g_w = inst->width; + vpx_configs_[0].g_h = inst->height; + + // Determine number of threads based on the image size and #cores. + // TODO(fbarchard): Consider number of Simulcast layers. + vpx_configs_[0].g_threads = NumberOfThreads( + vpx_configs_[0].g_w, vpx_configs_[0].g_h, settings.number_of_cores); + + // Creating a wrapper to the image - setting image data to NULL. + // Actual pointer will be set in encode. Setting align to 1, as it + // is meaningless (no memory allocation is done here). + libvpx_->img_wrap(&raw_images_[0], pixel_format, inst->width, inst->height, 1, + NULL); + + // Note the order we use is different from webm, we have lowest resolution + // at position 0 and they have highest resolution at position 0. + const size_t stream_idx_cfg_0 = encoders_.size() - 1; + SimulcastRateAllocator init_allocator(codec_); + VideoBitrateAllocation allocation = + init_allocator.Allocate(VideoBitrateAllocationParameters( + inst->startBitrate * 1000, inst->maxFramerate)); + std::vector<uint32_t> stream_bitrates; + for (int i = 0; i == 0 || i < inst->numberOfSimulcastStreams; ++i) { + uint32_t bitrate = allocation.GetSpatialLayerSum(i) / 1000; + stream_bitrates.push_back(bitrate); + } + + vpx_configs_[0].rc_target_bitrate = stream_bitrates[stream_idx_cfg_0]; + if (stream_bitrates[stream_idx_cfg_0] > 0) { + uint32_t maxFramerate = + inst->simulcastStream[stream_idx_cfg_0].maxFramerate; + if (!maxFramerate) { + maxFramerate = inst->maxFramerate; + } + + frame_buffer_controller_->OnRatesUpdated( + stream_idx_cfg_0, + allocation.GetTemporalLayerAllocation(stream_idx_cfg_0), maxFramerate); + } + frame_buffer_controller_->SetQpLimits(stream_idx_cfg_0, + vpx_configs_[0].rc_min_quantizer, + vpx_configs_[0].rc_max_quantizer); + UpdateVpxConfiguration(stream_idx_cfg_0); + vpx_configs_[0].rc_dropframe_thresh = FrameDropThreshold(stream_idx_cfg_0); + + for (size_t i = 1; i < encoders_.size(); ++i) { + const size_t stream_idx = encoders_.size() - 1 - i; + memcpy(&vpx_configs_[i], &vpx_configs_[0], sizeof(vpx_configs_[0])); + + vpx_configs_[i].g_w = inst->simulcastStream[stream_idx].width; + vpx_configs_[i].g_h = inst->simulcastStream[stream_idx].height; + + // Use 1 thread for lower resolutions. + vpx_configs_[i].g_threads = 1; + + vpx_configs_[i].rc_dropframe_thresh = FrameDropThreshold(stream_idx); + + // Setting alignment to 32 - as that ensures at least 16 for all + // planes (32 for Y, 16 for U,V). Libvpx sets the requested stride for + // the y plane, but only half of it to the u and v planes. + libvpx_->img_alloc( + &raw_images_[i], pixel_format, inst->simulcastStream[stream_idx].width, + inst->simulcastStream[stream_idx].height, kVp832ByteAlign); + SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx); + vpx_configs_[i].rc_target_bitrate = stream_bitrates[stream_idx]; + if (stream_bitrates[stream_idx] > 0) { + uint32_t maxFramerate = inst->simulcastStream[stream_idx].maxFramerate; + if (!maxFramerate) { + maxFramerate = inst->maxFramerate; + } + frame_buffer_controller_->OnRatesUpdated( + stream_idx, allocation.GetTemporalLayerAllocation(stream_idx), + maxFramerate); + } + frame_buffer_controller_->SetQpLimits(stream_idx, + vpx_configs_[i].rc_min_quantizer, + vpx_configs_[i].rc_max_quantizer); + UpdateVpxConfiguration(stream_idx); + } + + return InitAndSetControlSettings(); +} + +int LibvpxVp8Encoder::GetCpuSpeed(int width, int height) { +#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \ + defined(WEBRTC_ANDROID) || defined(WEBRTC_ARCH_MIPS) + // On mobile platform, use a lower speed setting for lower resolutions for + // CPUs with 4 or more cores. + RTC_DCHECK_GT(number_of_cores_, 0); + if (experimental_cpu_speed_config_arm_ + .GetValue(width * height, number_of_cores_) + .has_value()) { + return experimental_cpu_speed_config_arm_ + .GetValue(width * height, number_of_cores_) + .value(); + } + + if (number_of_cores_ <= 3) + return -12; + + if (width * height <= 352 * 288) + return -8; + else if (width * height <= 640 * 480) + return -10; + else + return -12; +#else + // For non-ARM, increase encoding complexity (i.e., use lower speed setting) + // if resolution is below CIF. Otherwise, keep the default/user setting + // (`cpu_speed_default_`) set on InitEncode via VP8().complexity. + if (width * height < 352 * 288) + return (cpu_speed_default_ < -4) ? -4 : cpu_speed_default_; + else + return cpu_speed_default_; +#endif +} + +int LibvpxVp8Encoder::NumberOfThreads(int width, int height, int cpus) { +#if defined(WEBRTC_ANDROID) || defined(WEBRTC_ARCH_MIPS) + if (width * height >= 320 * 180) { + if (cpus >= 4) { + // 3 threads for CPUs with 4 and more cores since most of times only 4 + // cores will be active. + return 3; + } else if (cpus == 3 || cpus == 2) { + return 2; + } else { + return 1; + } + } + return 1; +#else +#if defined(WEBRTC_IOS) + std::string trial_string = + field_trial::FindFullName(kVP8IosMaxNumberOfThreadFieldTrial); + FieldTrialParameter<int> max_thread_number( + kVP8IosMaxNumberOfThreadFieldTrialParameter, 0); + ParseFieldTrial({&max_thread_number}, trial_string); + if (max_thread_number.Get() > 0) { + if (width * height < 320 * 180) { + return 1; // Use single thread for small screens + } + // thread number must be less than or equal to the number of CPUs. + return std::min(cpus, max_thread_number.Get()); + } +#endif // defined(WEBRTC_IOS) + if (width * height >= 1920 * 1080 && cpus > 8) { + return 8; // 8 threads for 1080p on high perf machines. + } else if (width * height > 1280 * 960 && cpus >= 6) { + // 3 threads for 1080p. + return 3; + } else if (width * height > 640 * 480 && cpus >= 3) { + // Default 2 threads for qHD/HD, but allow 3 if core count is high enough, + // as this will allow more margin for high-core/low clock machines or if + // not built with highest optimization. + if (cpus >= 6) { + return 3; + } + return 2; + } else { + // 1 thread for VGA or less. + return 1; + } +#endif +} + +int LibvpxVp8Encoder::InitAndSetControlSettings() { + vpx_codec_flags_t flags = 0; + flags |= VPX_CODEC_USE_OUTPUT_PARTITION; + + if (encoders_.size() > 1) { + int error = libvpx_->codec_enc_init_multi( + &encoders_[0], vpx_codec_vp8_cx(), &vpx_configs_[0], encoders_.size(), + flags, &downsampling_factors_[0]); + if (error) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + } else { + if (libvpx_->codec_enc_init(&encoders_[0], vpx_codec_vp8_cx(), + &vpx_configs_[0], flags)) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + } + // Enable denoising for the highest resolution stream, and for + // the second highest resolution if we are doing more than 2 + // spatial layers/streams. + // TODO(holmer): Investigate possibility of adding a libvpx API + // for getting the denoised frame from the encoder and using that + // when encoding lower resolution streams. Would it work with the + // multi-res encoding feature? + denoiserState denoiser_state = kDenoiserOnYOnly; +#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \ + defined(WEBRTC_ANDROID) || defined(WEBRTC_ARCH_MIPS) + denoiser_state = kDenoiserOnYOnly; +#else + denoiser_state = kDenoiserOnAdaptive; +#endif + libvpx_->codec_control( + &encoders_[0], VP8E_SET_NOISE_SENSITIVITY, + codec_.VP8()->denoisingOn ? denoiser_state : kDenoiserOff); + if (encoders_.size() > 2) { + libvpx_->codec_control( + &encoders_[1], VP8E_SET_NOISE_SENSITIVITY, + codec_.VP8()->denoisingOn ? denoiser_state : kDenoiserOff); + } + for (size_t i = 0; i < encoders_.size(); ++i) { + // Allow more screen content to be detected as static. + libvpx_->codec_control( + &(encoders_[i]), VP8E_SET_STATIC_THRESHOLD, + codec_.mode == VideoCodecMode::kScreensharing ? 100u : 1u); + libvpx_->codec_control(&(encoders_[i]), VP8E_SET_CPUUSED, cpu_speed_[i]); + libvpx_->codec_control( + &(encoders_[i]), VP8E_SET_TOKEN_PARTITIONS, + static_cast<vp8e_token_partitions>(kTokenPartitions)); + libvpx_->codec_control(&(encoders_[i]), VP8E_SET_MAX_INTRA_BITRATE_PCT, + rc_max_intra_target_); + // VP8E_SET_SCREEN_CONTENT_MODE 2 = screen content with more aggressive + // rate control (drop frames on large target bitrate overshoot) + libvpx_->codec_control( + &(encoders_[i]), VP8E_SET_SCREEN_CONTENT_MODE, + codec_.mode == VideoCodecMode::kScreensharing ? 2u : 0u); + } + inited_ = true; + return WEBRTC_VIDEO_CODEC_OK; +} + +uint32_t LibvpxVp8Encoder::MaxIntraTarget(uint32_t optimalBuffersize) { + // Set max to the optimal buffer level (normalized by target BR), + // and scaled by a scalePar. + // Max target size = scalePar * optimalBufferSize * targetBR[Kbps]. + // This values is presented in percentage of perFrameBw: + // perFrameBw = targetBR[Kbps] * 1000 / frameRate. + // The target in % is as follows: + + float scalePar = 0.5; + uint32_t targetPct = optimalBuffersize * scalePar * codec_.maxFramerate / 10; + + // Don't go below 3 times the per frame bandwidth. + const uint32_t minIntraTh = 300; + return (targetPct < minIntraTh) ? minIntraTh : targetPct; +} + +uint32_t LibvpxVp8Encoder::FrameDropThreshold(size_t spatial_idx) const { + if (!codec_.GetFrameDropEnabled()) { + return 0; + } + + // If temporal layers are used, they get to override the frame dropping + // setting, as eg. ScreenshareLayers does not work as intended with frame + // dropping on and DefaultTemporalLayers will have performance issues with + // frame dropping off. + RTC_DCHECK(frame_buffer_controller_); + RTC_DCHECK_LT(spatial_idx, frame_buffer_controller_->StreamCount()); + return frame_buffer_controller_->SupportsEncoderFrameDropping(spatial_idx) + ? 30 + : 0; +} + +size_t LibvpxVp8Encoder::SteadyStateSize(int sid, int tid) { + const int encoder_id = encoders_.size() - 1 - sid; + size_t bitrate_bps; + float fps; + if ((SimulcastUtility::IsConferenceModeScreenshare(codec_) && sid == 0) || + vpx_configs_[encoder_id].ts_number_layers <= 1) { + // In conference screenshare there's no defined per temporal layer bitrate + // and framerate. + bitrate_bps = vpx_configs_[encoder_id].rc_target_bitrate * 1000; + fps = codec_.maxFramerate; + } else { + bitrate_bps = vpx_configs_[encoder_id].ts_target_bitrate[tid] * 1000; + fps = codec_.maxFramerate / + fmax(vpx_configs_[encoder_id].ts_rate_decimator[tid], 1.0); + if (tid > 0) { + // Layer bitrate and fps are counted as a partial sums. + bitrate_bps -= vpx_configs_[encoder_id].ts_target_bitrate[tid - 1] * 1000; + fps = codec_.maxFramerate / + fmax(vpx_configs_[encoder_id].ts_rate_decimator[tid - 1], 1.0); + } + } + + if (fps < 1e-9) + return 0; + return static_cast<size_t>( + bitrate_bps / (8 * fps) * + (100 - + variable_framerate_experiment_.steady_state_undershoot_percentage) / + 100 + + 0.5); +} + +bool LibvpxVp8Encoder::UpdateVpxConfiguration(size_t stream_index) { + RTC_DCHECK(frame_buffer_controller_); + + const size_t config_index = vpx_configs_.size() - 1 - stream_index; + + RTC_DCHECK_LT(config_index, config_overrides_.size()); + Vp8EncoderConfig* config = &config_overrides_[config_index]; + + const Vp8EncoderConfig new_config = + frame_buffer_controller_->UpdateConfiguration(stream_index); + + if (new_config.reset_previous_configuration_overrides) { + *config = new_config; + return true; + } + + const bool changes_made = MaybeExtendVp8EncoderConfig(new_config, config); + + // Note that overrides must be applied even if they haven't changed. + RTC_DCHECK_LT(config_index, vpx_configs_.size()); + vpx_codec_enc_cfg_t* vpx_config = &vpx_configs_[config_index]; + ApplyVp8EncoderConfigToVpxConfig(*config, vpx_config); + + return changes_made; +} + +int LibvpxVp8Encoder::Encode(const VideoFrame& frame, + const std::vector<VideoFrameType>* frame_types) { + RTC_DCHECK_EQ(frame.width(), codec_.width); + RTC_DCHECK_EQ(frame.height(), codec_.height); + + if (!inited_) + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + if (encoded_complete_callback_ == NULL) + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + + bool key_frame_requested = false; + for (size_t i = 0; i < key_frame_request_.size() && i < send_stream_.size(); + ++i) { + if (key_frame_request_[i] && send_stream_[i]) { + key_frame_requested = true; + break; + } + } + if (!key_frame_requested && frame_types) { + for (size_t i = 0; i < frame_types->size() && i < send_stream_.size(); + ++i) { + if ((*frame_types)[i] == VideoFrameType::kVideoFrameKey && + send_stream_[i]) { + key_frame_requested = true; + break; + } + } + } + + if (frame.update_rect().IsEmpty() && num_steady_state_frames_ >= 3 && + !key_frame_requested) { + if (variable_framerate_experiment_.enabled && + framerate_controller_.DropFrame(frame.timestamp() / kRtpTicksPerMs)) { + return WEBRTC_VIDEO_CODEC_OK; + } + framerate_controller_.AddFrame(frame.timestamp() / kRtpTicksPerMs); + } + + bool send_key_frame = key_frame_requested; + bool drop_frame = false; + bool retransmission_allowed = true; + Vp8FrameConfig tl_configs[kMaxSimulcastStreams]; + for (size_t i = 0; i < encoders_.size(); ++i) { + tl_configs[i] = + frame_buffer_controller_->NextFrameConfig(i, frame.timestamp()); + send_key_frame |= tl_configs[i].IntraFrame(); + drop_frame |= tl_configs[i].drop_frame; + RTC_DCHECK(i == 0 || + retransmission_allowed == tl_configs[i].retransmission_allowed); + retransmission_allowed = tl_configs[i].retransmission_allowed; + } + + if (drop_frame && !send_key_frame) { + return WEBRTC_VIDEO_CODEC_OK; + } + + vpx_enc_frame_flags_t flags[kMaxSimulcastStreams]; + for (size_t i = 0; i < encoders_.size(); ++i) { + flags[i] = send_key_frame ? VPX_EFLAG_FORCE_KF : EncodeFlags(tl_configs[i]); + } + + // Scale and map buffers and set `raw_images_` to hold pointers to the result. + // Because `raw_images_` are set to hold pointers to the prepared buffers, we + // need to keep these buffers alive through reference counting until after + // encoding is complete. + std::vector<rtc::scoped_refptr<VideoFrameBuffer>> prepared_buffers = + PrepareBuffers(frame.video_frame_buffer()); + if (prepared_buffers.empty()) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + struct CleanUpOnExit { + explicit CleanUpOnExit( + vpx_image_t* raw_image, + std::vector<rtc::scoped_refptr<VideoFrameBuffer>> prepared_buffers) + : raw_image_(raw_image), + prepared_buffers_(std::move(prepared_buffers)) {} + ~CleanUpOnExit() { + raw_image_->planes[VPX_PLANE_Y] = nullptr; + raw_image_->planes[VPX_PLANE_U] = nullptr; + raw_image_->planes[VPX_PLANE_V] = nullptr; + } + vpx_image_t* raw_image_; + std::vector<rtc::scoped_refptr<VideoFrameBuffer>> prepared_buffers_; + } clean_up_on_exit(&raw_images_[0], std::move(prepared_buffers)); + + if (send_key_frame) { + // Adapt the size of the key frame when in screenshare with 1 temporal + // layer. + if (encoders_.size() == 1 && + codec_.mode == VideoCodecMode::kScreensharing && + codec_.VP8()->numberOfTemporalLayers <= 1) { + const uint32_t forceKeyFrameIntraTh = 100; + libvpx_->codec_control(&(encoders_[0]), VP8E_SET_MAX_INTRA_BITRATE_PCT, + forceKeyFrameIntraTh); + } + + std::fill(key_frame_request_.begin(), key_frame_request_.end(), false); + } + + // Set the encoder frame flags and temporal layer_id for each spatial stream. + // Note that streams are defined starting from lowest resolution at + // position 0 to highest resolution at position |encoders_.size() - 1|, + // whereas `encoder_` is from highest to lowest resolution. + for (size_t i = 0; i < encoders_.size(); ++i) { + const size_t stream_idx = encoders_.size() - 1 - i; + + if (UpdateVpxConfiguration(stream_idx)) { + if (libvpx_->codec_enc_config_set(&encoders_[i], &vpx_configs_[i])) + return WEBRTC_VIDEO_CODEC_ERROR; + } + + libvpx_->codec_control(&encoders_[i], VP8E_SET_FRAME_FLAGS, + static_cast<int>(flags[stream_idx])); + libvpx_->codec_control(&encoders_[i], VP8E_SET_TEMPORAL_LAYER_ID, + tl_configs[i].encoder_layer_id); + } + // TODO(holmer): Ideally the duration should be the timestamp diff of this + // frame and the next frame to be encoded, which we don't have. Instead we + // would like to use the duration of the previous frame. Unfortunately the + // rate control seems to be off with that setup. Using the average input + // frame rate to calculate an average duration for now. + RTC_DCHECK_GT(codec_.maxFramerate, 0); + uint32_t duration = kRtpTicksPerSecond / codec_.maxFramerate; + + int error = WEBRTC_VIDEO_CODEC_OK; + int num_tries = 0; + // If the first try returns WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT + // the frame must be reencoded with the same parameters again because + // target bitrate is exceeded and encoder state has been reset. + while (num_tries == 0 || + (num_tries == 1 && + error == WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT)) { + ++num_tries; + // Note we must pass 0 for `flags` field in encode call below since they are + // set above in `libvpx_interface_->vpx_codec_control_` function for each + // encoder/spatial layer. + error = libvpx_->codec_encode(&encoders_[0], &raw_images_[0], timestamp_, + duration, 0, VPX_DL_REALTIME); + // Reset specific intra frame thresholds, following the key frame. + if (send_key_frame) { + libvpx_->codec_control(&(encoders_[0]), VP8E_SET_MAX_INTRA_BITRATE_PCT, + rc_max_intra_target_); + } + if (error) + return WEBRTC_VIDEO_CODEC_ERROR; + // Examines frame timestamps only. + error = GetEncodedPartitions(frame, retransmission_allowed); + } + // TODO(sprang): Shouldn't we use the frame timestamp instead? + timestamp_ += duration; + return error; +} + +void LibvpxVp8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, + const vpx_codec_cx_pkt_t& pkt, + int stream_idx, + int encoder_idx, + uint32_t timestamp) { + RTC_DCHECK(codec_specific); + codec_specific->codecType = kVideoCodecVP8; + codec_specific->codecSpecific.VP8.keyIdx = + kNoKeyIdx; // TODO(hlundin) populate this + codec_specific->codecSpecific.VP8.nonReference = + (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) != 0; + + int qp = 0; + vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp); + bool is_keyframe = (pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0; + frame_buffer_controller_->OnEncodeDone(stream_idx, timestamp, + encoded_images_[encoder_idx].size(), + is_keyframe, qp, codec_specific); + if (is_keyframe && codec_specific->template_structure != absl::nullopt) { + // Number of resolutions must match number of spatial layers, VP8 structures + // expected to use single spatial layer. Templates must be ordered by + // spatial_id, so assumption there is exactly one spatial layer is same as + // assumption last template uses spatial_id = 0. + // This check catches potential scenario where template_structure is shared + // across multiple vp8 streams and they are distinguished using spatial_id. + // Assigning single resolution doesn't support such scenario, i.e. assumes + // vp8 simulcast is sent using multiple ssrcs. + RTC_DCHECK(!codec_specific->template_structure->templates.empty()); + RTC_DCHECK_EQ( + codec_specific->template_structure->templates.back().spatial_id, 0); + codec_specific->template_structure->resolutions = { + RenderResolution(pkt.data.frame.width[0], pkt.data.frame.height[0])}; + } + switch (vpx_configs_[encoder_idx].ts_number_layers) { + case 1: + codec_specific->scalability_mode = ScalabilityMode::kL1T1; + break; + case 2: + codec_specific->scalability_mode = ScalabilityMode::kL1T2; + break; + case 3: + codec_specific->scalability_mode = ScalabilityMode::kL1T3; + break; + } +} + +int LibvpxVp8Encoder::GetEncodedPartitions(const VideoFrame& input_image, + bool retransmission_allowed) { + int stream_idx = static_cast<int>(encoders_.size()) - 1; + int result = WEBRTC_VIDEO_CODEC_OK; + for (size_t encoder_idx = 0; encoder_idx < encoders_.size(); + ++encoder_idx, --stream_idx) { + vpx_codec_iter_t iter = NULL; + encoded_images_[encoder_idx].set_size(0); + encoded_images_[encoder_idx]._frameType = VideoFrameType::kVideoFrameDelta; + CodecSpecificInfo codec_specific; + const vpx_codec_cx_pkt_t* pkt = NULL; + + size_t encoded_size = 0; + while ((pkt = libvpx_->codec_get_cx_data(&encoders_[encoder_idx], &iter)) != + NULL) { + if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { + encoded_size += pkt->data.frame.sz; + } + } + + auto buffer = EncodedImageBuffer::Create(encoded_size); + + iter = NULL; + size_t encoded_pos = 0; + while ((pkt = libvpx_->codec_get_cx_data(&encoders_[encoder_idx], &iter)) != + NULL) { + switch (pkt->kind) { + case VPX_CODEC_CX_FRAME_PKT: { + RTC_CHECK_LE(encoded_pos + pkt->data.frame.sz, buffer->size()); + memcpy(&buffer->data()[encoded_pos], pkt->data.frame.buf, + pkt->data.frame.sz); + encoded_pos += pkt->data.frame.sz; + break; + } + default: + break; + } + // End of frame + if ((pkt->data.frame.flags & VPX_FRAME_IS_FRAGMENT) == 0) { + // check if encoded frame is a key frame + if (pkt->data.frame.flags & VPX_FRAME_IS_KEY) { + encoded_images_[encoder_idx]._frameType = + VideoFrameType::kVideoFrameKey; + } + encoded_images_[encoder_idx].SetEncodedData(buffer); + encoded_images_[encoder_idx].set_size(encoded_pos); + encoded_images_[encoder_idx].SetSpatialIndex(stream_idx); + PopulateCodecSpecific(&codec_specific, *pkt, stream_idx, encoder_idx, + input_image.timestamp()); + if (codec_specific.codecSpecific.VP8.temporalIdx != kNoTemporalIdx) { + encoded_images_[encoder_idx].SetTemporalIndex( + codec_specific.codecSpecific.VP8.temporalIdx); + } + break; + } + } + encoded_images_[encoder_idx].SetTimestamp(input_image.timestamp()); + encoded_images_[encoder_idx].SetColorSpace(input_image.color_space()); + encoded_images_[encoder_idx].SetRetransmissionAllowed( + retransmission_allowed); + + if (send_stream_[stream_idx]) { + if (encoded_images_[encoder_idx].size() > 0) { + TRACE_COUNTER_ID1("webrtc", "EncodedFrameSize", encoder_idx, + encoded_images_[encoder_idx].size()); + encoded_images_[encoder_idx]._encodedHeight = + codec_.simulcastStream[stream_idx].height; + encoded_images_[encoder_idx]._encodedWidth = + codec_.simulcastStream[stream_idx].width; + int qp_128 = -1; + libvpx_->codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER, + &qp_128); + encoded_images_[encoder_idx].qp_ = qp_128; + encoded_complete_callback_->OnEncodedImage(encoded_images_[encoder_idx], + &codec_specific); + const size_t steady_state_size = SteadyStateSize( + stream_idx, codec_specific.codecSpecific.VP8.temporalIdx); + if (qp_128 > variable_framerate_experiment_.steady_state_qp || + encoded_images_[encoder_idx].size() > steady_state_size) { + num_steady_state_frames_ = 0; + } else { + ++num_steady_state_frames_; + } + } else if (!frame_buffer_controller_->SupportsEncoderFrameDropping( + stream_idx)) { + result = WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT; + if (encoded_images_[encoder_idx].size() == 0) { + // Dropped frame that will be re-encoded. + frame_buffer_controller_->OnFrameDropped(stream_idx, + input_image.timestamp()); + } + } + } + } + return result; +} + +VideoEncoder::EncoderInfo LibvpxVp8Encoder::GetEncoderInfo() const { + EncoderInfo info; + info.supports_native_handle = false; + info.implementation_name = "libvpx"; + info.has_trusted_rate_controller = + rate_control_settings_.LibvpxVp8TrustedRateController(); + info.is_hardware_accelerated = false; + info.supports_simulcast = true; + if (!resolution_bitrate_limits_.empty()) { + info.resolution_bitrate_limits = resolution_bitrate_limits_; + } + if (encoder_info_override_.requested_resolution_alignment()) { + info.requested_resolution_alignment = + *encoder_info_override_.requested_resolution_alignment(); + info.apply_alignment_to_all_simulcast_layers = + encoder_info_override_.apply_alignment_to_all_simulcast_layers(); + } + if (!encoder_info_override_.resolution_bitrate_limits().empty()) { + info.resolution_bitrate_limits = + encoder_info_override_.resolution_bitrate_limits(); + } + + const bool enable_scaling = + num_active_streams_ == 1 && + (vpx_configs_.empty() || vpx_configs_[0].rc_dropframe_thresh > 0) && + codec_.VP8().automaticResizeOn; + + info.scaling_settings = enable_scaling + ? VideoEncoder::ScalingSettings( + kLowVp8QpThreshold, kHighVp8QpThreshold) + : VideoEncoder::ScalingSettings::kOff; + if (rate_control_settings_.LibvpxVp8MinPixels()) { + info.scaling_settings.min_pixels_per_frame = + rate_control_settings_.LibvpxVp8MinPixels().value(); + } + info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420, + VideoFrameBuffer::Type::kNV12}; + + if (inited_) { + // `encoder_idx` is libvpx index where 0 is highest resolution. + // `si` is simulcast index, where 0 is lowest resolution. + for (size_t si = 0, encoder_idx = encoders_.size() - 1; + si < encoders_.size(); ++si, --encoder_idx) { + info.fps_allocation[si].clear(); + if ((codec_.numberOfSimulcastStreams > si && + !codec_.simulcastStream[si].active) || + (si == 0 && SimulcastUtility::IsConferenceModeScreenshare(codec_))) { + // No defined frame rate fractions if not active or if using + // ScreenshareLayers, leave vector empty and continue; + continue; + } + if (vpx_configs_[encoder_idx].ts_number_layers <= 1) { + info.fps_allocation[si].push_back(EncoderInfo::kMaxFramerateFraction); + } else { + for (size_t ti = 0; ti < vpx_configs_[encoder_idx].ts_number_layers; + ++ti) { + RTC_DCHECK_GT(vpx_configs_[encoder_idx].ts_rate_decimator[ti], 0); + info.fps_allocation[si].push_back(rtc::saturated_cast<uint8_t>( + EncoderInfo::kMaxFramerateFraction / + vpx_configs_[encoder_idx].ts_rate_decimator[ti] + + 0.5)); + } + } + } + } + + return info; +} + +int LibvpxVp8Encoder::RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) { + encoded_complete_callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; +} + +void LibvpxVp8Encoder::MaybeUpdatePixelFormat(vpx_img_fmt fmt) { + RTC_DCHECK(!raw_images_.empty()); + if (raw_images_[0].fmt == fmt) { + RTC_DCHECK(std::all_of( + std::next(raw_images_.begin()), raw_images_.end(), + [fmt](const vpx_image_t& raw_img) { return raw_img.fmt == fmt; })) + << "Not all raw images had the right format!"; + return; + } + RTC_LOG(LS_INFO) << "Updating vp8 encoder pixel format to " + << (fmt == VPX_IMG_FMT_NV12 ? "NV12" : "I420"); + for (size_t i = 0; i < raw_images_.size(); ++i) { + vpx_image_t& img = raw_images_[i]; + auto d_w = img.d_w; + auto d_h = img.d_h; + libvpx_->img_free(&img); + // First image is wrapping the input frame, the rest are allocated. + if (i == 0) { + libvpx_->img_wrap(&img, fmt, d_w, d_h, 1, NULL); + } else { + libvpx_->img_alloc(&img, fmt, d_w, d_h, kVp832ByteAlign); + } + } +} + +std::vector<rtc::scoped_refptr<VideoFrameBuffer>> +LibvpxVp8Encoder::PrepareBuffers(rtc::scoped_refptr<VideoFrameBuffer> buffer) { + RTC_DCHECK_EQ(buffer->width(), raw_images_[0].d_w); + RTC_DCHECK_EQ(buffer->height(), raw_images_[0].d_h); + absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats> + supported_formats = {VideoFrameBuffer::Type::kI420, + VideoFrameBuffer::Type::kNV12}; + + rtc::scoped_refptr<VideoFrameBuffer> mapped_buffer; + if (buffer->type() != VideoFrameBuffer::Type::kNative) { + // `buffer` is already mapped. + mapped_buffer = buffer; + } else { + // Attempt to map to one of the supported formats. + mapped_buffer = buffer->GetMappedFrameBuffer(supported_formats); + } + if (!mapped_buffer || + (absl::c_find(supported_formats, mapped_buffer->type()) == + supported_formats.end() && + mapped_buffer->type() != VideoFrameBuffer::Type::kI420A)) { + // Unknown pixel format or unable to map, convert to I420 and prepare that + // buffer instead to ensure Scale() is safe to use. + auto converted_buffer = buffer->ToI420(); + if (!converted_buffer) { + RTC_LOG(LS_ERROR) << "Failed to convert " + << VideoFrameBufferTypeToString(buffer->type()) + << " image to I420. Can't encode frame."; + return {}; + } + RTC_CHECK(converted_buffer->type() == VideoFrameBuffer::Type::kI420 || + converted_buffer->type() == VideoFrameBuffer::Type::kI420A); + + // Because `buffer` had to be converted, use `converted_buffer` instead... + buffer = mapped_buffer = converted_buffer; + } + + // Maybe update pixel format. + absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats> + mapped_type = {mapped_buffer->type()}; + switch (mapped_buffer->type()) { + case VideoFrameBuffer::Type::kI420: + case VideoFrameBuffer::Type::kI420A: + MaybeUpdatePixelFormat(VPX_IMG_FMT_I420); + break; + case VideoFrameBuffer::Type::kNV12: + MaybeUpdatePixelFormat(VPX_IMG_FMT_NV12); + break; + default: + RTC_DCHECK_NOTREACHED(); + } + + // Prepare `raw_images_` from `mapped_buffer` and, if simulcast, scaled + // versions of `buffer`. + std::vector<rtc::scoped_refptr<VideoFrameBuffer>> prepared_buffers; + SetRawImagePlanes(&raw_images_[0], mapped_buffer.get()); + prepared_buffers.push_back(mapped_buffer); + for (size_t i = 1; i < encoders_.size(); ++i) { + // Native buffers should implement optimized scaling and is the preferred + // buffer to scale. But if the buffer isn't native, it should be cheaper to + // scale from the previously prepared buffer which is smaller than `buffer`. + VideoFrameBuffer* buffer_to_scale = + buffer->type() == VideoFrameBuffer::Type::kNative + ? buffer.get() + : prepared_buffers.back().get(); + + auto scaled_buffer = + buffer_to_scale->Scale(raw_images_[i].d_w, raw_images_[i].d_h); + if (scaled_buffer->type() == VideoFrameBuffer::Type::kNative) { + auto mapped_scaled_buffer = + scaled_buffer->GetMappedFrameBuffer(mapped_type); + RTC_DCHECK(mapped_scaled_buffer) << "Unable to map the scaled buffer."; + if (!mapped_scaled_buffer) { + RTC_LOG(LS_ERROR) << "Failed to map scaled " + << VideoFrameBufferTypeToString(scaled_buffer->type()) + << " image to " + << VideoFrameBufferTypeToString(mapped_buffer->type()) + << ". Can't encode frame."; + return {}; + } + scaled_buffer = mapped_scaled_buffer; + } + if (!IsCompatibleVideoFrameBufferType(scaled_buffer->type(), + mapped_buffer->type())) { + RTC_LOG(LS_ERROR) << "When scaling " + << VideoFrameBufferTypeToString(buffer_to_scale->type()) + << ", the image was unexpectedly converted to " + << VideoFrameBufferTypeToString(scaled_buffer->type()) + << " instead of " + << VideoFrameBufferTypeToString(mapped_buffer->type()) + << ". Can't encode frame."; + RTC_DCHECK_NOTREACHED() + << "Scaled buffer type " + << VideoFrameBufferTypeToString(scaled_buffer->type()) + << " is not compatible with mapped buffer type " + << VideoFrameBufferTypeToString(mapped_buffer->type()); + return {}; + } + SetRawImagePlanes(&raw_images_[i], scaled_buffer.get()); + prepared_buffers.push_back(scaled_buffer); + } + return prepared_buffers; +} + +// static +LibvpxVp8Encoder::VariableFramerateExperiment +LibvpxVp8Encoder::ParseVariableFramerateConfig(std::string group_name) { + FieldTrialFlag disabled = FieldTrialFlag("Disabled"); + FieldTrialParameter<double> framerate_limit("min_fps", 5.0); + FieldTrialParameter<int> qp("min_qp", 15); + FieldTrialParameter<int> undershoot_percentage("undershoot", 30); + ParseFieldTrial({&disabled, &framerate_limit, &qp, &undershoot_percentage}, + field_trial::FindFullName(group_name)); + VariableFramerateExperiment config; + config.enabled = !disabled.Get(); + config.framerate_limit = framerate_limit.Get(); + config.steady_state_qp = qp.Get(); + config.steady_state_undershoot_percentage = undershoot_percentage.Get(); + + return config; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h new file mode 100644 index 0000000000..74477eac7e --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_ENCODER_H_ +#define MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_ENCODER_H_ + +#include <memory> +#include <string> +#include <vector> + +#include "api/fec_controller_override.h" +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/vp8_frame_buffer_controller.h" +#include "api/video_codecs/vp8_frame_config.h" +#include "modules/video_coding/codecs/interface/libvpx_interface.h" +#include "modules/video_coding/codecs/vp8/include/vp8.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/utility/framerate_controller_deprecated.h" +#include "modules/video_coding/utility/vp8_constants.h" +#include "rtc_base/experiments/cpu_speed_experiment.h" +#include "rtc_base/experiments/encoder_info_settings.h" +#include "rtc_base/experiments/rate_control_settings.h" +#include "vpx/vp8cx.h" +#include "vpx/vpx_encoder.h" + +namespace webrtc { + +class LibvpxVp8Encoder : public VideoEncoder { + public: + LibvpxVp8Encoder(std::unique_ptr<LibvpxInterface> interface, + VP8Encoder::Settings settings); + ~LibvpxVp8Encoder() override; + + int Release() override; + + void SetFecControllerOverride( + FecControllerOverride* fec_controller_override) override; + + int InitEncode(const VideoCodec* codec_settings, + const VideoEncoder::Settings& settings) override; + + int Encode(const VideoFrame& input_image, + const std::vector<VideoFrameType>* frame_types) override; + + int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override; + + void SetRates(const RateControlParameters& parameters) override; + + void OnPacketLossRateUpdate(float packet_loss_rate) override; + + void OnRttUpdate(int64_t rtt_ms) override; + + void OnLossNotification(const LossNotification& loss_notification) override; + + EncoderInfo GetEncoderInfo() const override; + + static vpx_enc_frame_flags_t EncodeFlags(const Vp8FrameConfig& references); + + private: + // Get the cpu_speed setting for encoder based on resolution and/or platform. + int GetCpuSpeed(int width, int height); + + // Determine number of encoder threads to use. + int NumberOfThreads(int width, int height, int number_of_cores); + + // Call encoder initialize function and set control settings. + int InitAndSetControlSettings(); + + void PopulateCodecSpecific(CodecSpecificInfo* codec_specific, + const vpx_codec_cx_pkt& pkt, + int stream_idx, + int encoder_idx, + uint32_t timestamp); + + int GetEncodedPartitions(const VideoFrame& input_image, + bool retransmission_allowed); + + // Set the stream state for stream `stream_idx`. + void SetStreamState(bool send_stream, int stream_idx); + + uint32_t MaxIntraTarget(uint32_t optimal_buffer_size); + + uint32_t FrameDropThreshold(size_t spatial_idx) const; + + size_t SteadyStateSize(int sid, int tid); + + bool UpdateVpxConfiguration(size_t stream_index); + + void MaybeUpdatePixelFormat(vpx_img_fmt fmt); + // Prepares `raw_image_` to reference image data of `buffer`, or of mapped or + // scaled versions of `buffer`. Returns a list of buffers that got referenced + // as a result, allowing the caller to keep references to them until after + // encoding has finished. On failure to convert the buffer, an empty list is + // returned. + std::vector<rtc::scoped_refptr<VideoFrameBuffer>> PrepareBuffers( + rtc::scoped_refptr<VideoFrameBuffer> buffer); + + const std::unique_ptr<LibvpxInterface> libvpx_; + + const CpuSpeedExperiment experimental_cpu_speed_config_arm_; + const RateControlSettings rate_control_settings_; + + EncodedImageCallback* encoded_complete_callback_ = nullptr; + VideoCodec codec_; + bool inited_ = false; + int64_t timestamp_ = 0; + int qp_max_ = 56; + int cpu_speed_default_ = -6; + int number_of_cores_ = 0; + uint32_t rc_max_intra_target_ = 0; + int num_active_streams_ = 0; + const std::unique_ptr<Vp8FrameBufferControllerFactory> + frame_buffer_controller_factory_; + std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_; + const std::vector<VideoEncoder::ResolutionBitrateLimits> + resolution_bitrate_limits_; + std::vector<bool> key_frame_request_; + std::vector<bool> send_stream_; + std::vector<int> cpu_speed_; + std::vector<vpx_image_t> raw_images_; + std::vector<EncodedImage> encoded_images_; + std::vector<vpx_codec_ctx_t> encoders_; + std::vector<vpx_codec_enc_cfg_t> vpx_configs_; + std::vector<Vp8EncoderConfig> config_overrides_; + std::vector<vpx_rational_t> downsampling_factors_; + + // Variable frame-rate screencast related fields and methods. + const struct VariableFramerateExperiment { + bool enabled = false; + // Framerate is limited to this value in steady state. + float framerate_limit = 5.0; + // This qp or below is considered a steady state. + int steady_state_qp = kVp8SteadyStateQpThreshold; + // Frames of at least this percentage below ideal for configured bitrate are + // considered in a steady state. + int steady_state_undershoot_percentage = 30; + } variable_framerate_experiment_; + static VariableFramerateExperiment ParseVariableFramerateConfig( + std::string group_name); + FramerateControllerDeprecated framerate_controller_; + int num_steady_state_frames_ = 0; + + FecControllerOverride* fec_controller_override_ = nullptr; + + const LibvpxVp8EncoderInfoSettings encoder_info_override_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_VP8_LIBVPX_VP8_ENCODER_H_ diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_simulcast_test.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_simulcast_test.cc new file mode 100644 index 0000000000..4ca3de20d5 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_simulcast_test.cc @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <memory> + +#include "api/test/create_simulcast_test_fixture.h" +#include "api/test/simulcast_test_fixture.h" +#include "api/test/video/function_video_decoder_factory.h" +#include "api/test/video/function_video_encoder_factory.h" +#include "modules/video_coding/codecs/vp8/include/vp8.h" +#include "test/gtest.h" + +namespace webrtc { +namespace test { + +namespace { +std::unique_ptr<SimulcastTestFixture> CreateSpecificSimulcastTestFixture() { + std::unique_ptr<VideoEncoderFactory> encoder_factory = + std::make_unique<FunctionVideoEncoderFactory>( + []() { return VP8Encoder::Create(); }); + std::unique_ptr<VideoDecoderFactory> decoder_factory = + std::make_unique<FunctionVideoDecoderFactory>( + []() { return VP8Decoder::Create(); }); + return CreateSimulcastTestFixture(std::move(encoder_factory), + std::move(decoder_factory), + SdpVideoFormat("VP8")); +} +} // namespace + +TEST(LibvpxVp8SimulcastTest, TestKeyFrameRequestsOnAllStreams) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestKeyFrameRequestsOnAllStreams(); +} + +TEST(LibvpxVp8SimulcastTest, TestKeyFrameRequestsOnSpecificStreams) { + GTEST_SKIP() << "Not applicable to VP8."; +} + +TEST(LibvpxVp8SimulcastTest, TestPaddingAllStreams) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestPaddingAllStreams(); +} + +TEST(LibvpxVp8SimulcastTest, TestPaddingTwoStreams) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestPaddingTwoStreams(); +} + +TEST(LibvpxVp8SimulcastTest, TestPaddingTwoStreamsOneMaxedOut) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestPaddingTwoStreamsOneMaxedOut(); +} + +TEST(LibvpxVp8SimulcastTest, TestPaddingOneStream) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestPaddingOneStream(); +} + +TEST(LibvpxVp8SimulcastTest, TestPaddingOneStreamTwoMaxedOut) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestPaddingOneStreamTwoMaxedOut(); +} + +TEST(LibvpxVp8SimulcastTest, TestSendAllStreams) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestSendAllStreams(); +} + +TEST(LibvpxVp8SimulcastTest, TestDisablingStreams) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestDisablingStreams(); +} + +TEST(LibvpxVp8SimulcastTest, TestActiveStreams) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestActiveStreams(); +} + +TEST(LibvpxVp8SimulcastTest, TestSwitchingToOneStream) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestSwitchingToOneStream(); +} + +TEST(LibvpxVp8SimulcastTest, TestSwitchingToOneOddStream) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestSwitchingToOneOddStream(); +} + +TEST(LibvpxVp8SimulcastTest, TestSwitchingToOneSmallStream) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestSwitchingToOneSmallStream(); +} + +TEST(LibvpxVp8SimulcastTest, TestSpatioTemporalLayers333PatternEncoder) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestSpatioTemporalLayers333PatternEncoder(); +} + +TEST(LibvpxVp8SimulcastTest, TestStrideEncodeDecode) { + auto fixture = CreateSpecificSimulcastTestFixture(); + fixture->TestStrideEncodeDecode(); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc new file mode 100644 index 0000000000..71db0b22c2 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -0,0 +1,624 @@ +/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/vp8/screenshare_layers.h" + +#include <stdlib.h> + +#include <algorithm> +#include <memory> + +#include "modules/video_coding/include/video_codec_interface.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { +namespace { +using BufferFlags = Vp8FrameConfig::BufferFlags; + +constexpr BufferFlags kNone = Vp8FrameConfig::BufferFlags::kNone; +constexpr BufferFlags kReference = Vp8FrameConfig::BufferFlags::kReference; +constexpr BufferFlags kUpdate = Vp8FrameConfig::BufferFlags::kUpdate; +constexpr BufferFlags kReferenceAndUpdate = + Vp8FrameConfig::BufferFlags::kReferenceAndUpdate; + +constexpr int kOneSecond90Khz = 90000; +constexpr int kMinTimeBetweenSyncs = kOneSecond90Khz * 2; +constexpr int kMaxTimeBetweenSyncs = kOneSecond90Khz * 4; +constexpr int kQpDeltaThresholdForSync = 8; +constexpr int kMinBitrateKbpsForQpBoost = 500; +constexpr auto kSwitch = DecodeTargetIndication::kSwitch; +} // namespace + +const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5; +const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0; + +constexpr int ScreenshareLayers::kMaxNumTemporalLayers; + +// Always emit a frame with certain interval, even if bitrate targets have +// been exceeded. This prevents needless keyframe requests. +const int ScreenshareLayers::kMaxFrameIntervalMs = 2750; + +ScreenshareLayers::ScreenshareLayers(int num_temporal_layers) + : number_of_temporal_layers_( + std::min(kMaxNumTemporalLayers, num_temporal_layers)), + active_layer_(-1), + last_timestamp_(-1), + last_sync_timestamp_(-1), + last_emitted_tl0_timestamp_(-1), + last_frame_time_ms_(-1), + max_debt_bytes_(0), + encode_framerate_(1000.0f, 1000.0f), // 1 second window, second scale. + bitrate_updated_(false), + checker_(TemporalLayersChecker::CreateTemporalLayersChecker( + Vp8TemporalLayersType::kBitrateDynamic, + num_temporal_layers)) { + RTC_CHECK_GT(number_of_temporal_layers_, 0); + RTC_CHECK_LE(number_of_temporal_layers_, kMaxNumTemporalLayers); +} + +ScreenshareLayers::~ScreenshareLayers() { + UpdateHistograms(); +} + +void ScreenshareLayers::SetQpLimits(size_t stream_index, + int min_qp, + int max_qp) { + RTC_DCHECK_LT(stream_index, StreamCount()); + // 0 < min_qp <= max_qp + RTC_DCHECK_LT(0, min_qp); + RTC_DCHECK_LE(min_qp, max_qp); + + RTC_DCHECK_EQ(min_qp_.has_value(), max_qp_.has_value()); + if (!min_qp_.has_value()) { + min_qp_ = min_qp; + max_qp_ = max_qp; + } else { + RTC_DCHECK_EQ(min_qp, min_qp_.value()); + RTC_DCHECK_EQ(max_qp, max_qp_.value()); + } +} + +size_t ScreenshareLayers::StreamCount() const { + return 1; +} + +bool ScreenshareLayers::SupportsEncoderFrameDropping( + size_t stream_index) const { + RTC_DCHECK_LT(stream_index, StreamCount()); + // Frame dropping is handled internally by this class. + return false; +} + +Vp8FrameConfig ScreenshareLayers::NextFrameConfig(size_t stream_index, + uint32_t timestamp) { + RTC_DCHECK_LT(stream_index, StreamCount()); + + auto it = pending_frame_configs_.find(timestamp); + if (it != pending_frame_configs_.end()) { + // Drop and re-encode, reuse the previous config. + return it->second.frame_config; + } + + if (number_of_temporal_layers_ <= 1) { + // No flags needed for 1 layer screenshare. + // TODO(pbos): Consider updating only last, and not all buffers. + DependencyInfo dependency_info{ + "S", {kReferenceAndUpdate, kReferenceAndUpdate, kReferenceAndUpdate}}; + pending_frame_configs_[timestamp] = dependency_info; + return dependency_info.frame_config; + } + + const int64_t now_ms = rtc::TimeMillis(); + + int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp); + int64_t ts_diff; + if (last_timestamp_ == -1) { + ts_diff = kOneSecond90Khz / capture_framerate_.value_or(*target_framerate_); + } else { + ts_diff = unwrapped_timestamp - last_timestamp_; + } + + if (target_framerate_) { + // If input frame rate exceeds target frame rate, either over a one second + // averaging window, or if frame interval is below 90% of desired value, + // drop frame. + if (encode_framerate_.Rate(now_ms).value_or(0) > *target_framerate_) + return Vp8FrameConfig(kNone, kNone, kNone); + + // Primarily check if frame interval is too short using frame timestamps, + // as if they are correct they won't be affected by queuing in webrtc. + const int64_t expected_frame_interval_90khz = + kOneSecond90Khz / *target_framerate_; + if (last_timestamp_ != -1 && ts_diff > 0) { + if (ts_diff < 85 * expected_frame_interval_90khz / 100) { + return Vp8FrameConfig(kNone, kNone, kNone); + } + } else { + // Timestamps looks off, use realtime clock here instead. + const int64_t expected_frame_interval_ms = 1000 / *target_framerate_; + if (last_frame_time_ms_ != -1 && + now_ms - last_frame_time_ms_ < + (85 * expected_frame_interval_ms) / 100) { + return Vp8FrameConfig(kNone, kNone, kNone); + } + } + } + + if (stats_.first_frame_time_ms_ == -1) + stats_.first_frame_time_ms_ = now_ms; + + // Make sure both frame droppers leak out bits. + layers_[0].UpdateDebt(ts_diff / 90); + layers_[1].UpdateDebt(ts_diff / 90); + last_timestamp_ = timestamp; + last_frame_time_ms_ = now_ms; + + TemporalLayerState layer_state = TemporalLayerState::kDrop; + + if (active_layer_ == -1 || + layers_[active_layer_].state != TemporalLayer::State::kDropped) { + if (last_emitted_tl0_timestamp_ != -1 && + (unwrapped_timestamp - last_emitted_tl0_timestamp_) / 90 > + kMaxFrameIntervalMs) { + // Too long time has passed since the last frame was emitted, cancel + // enough debt to allow a single frame. + layers_[0].debt_bytes_ = max_debt_bytes_ - 1; + } + if (layers_[0].debt_bytes_ > max_debt_bytes_) { + // Must drop TL0, encode TL1 instead. + if (layers_[1].debt_bytes_ > max_debt_bytes_) { + // Must drop both TL0 and TL1. + active_layer_ = -1; + } else { + active_layer_ = 1; + } + } else { + active_layer_ = 0; + } + } + + switch (active_layer_) { + case 0: + layer_state = TemporalLayerState::kTl0; + last_emitted_tl0_timestamp_ = unwrapped_timestamp; + break; + case 1: + if (layers_[1].state != TemporalLayer::State::kDropped) { + if (TimeToSync(unwrapped_timestamp) || + layers_[1].state == TemporalLayer::State::kKeyFrame) { + last_sync_timestamp_ = unwrapped_timestamp; + layer_state = TemporalLayerState::kTl1Sync; + } else { + layer_state = TemporalLayerState::kTl1; + } + } else { + layer_state = last_sync_timestamp_ == unwrapped_timestamp + ? TemporalLayerState::kTl1Sync + : TemporalLayerState::kTl1; + } + break; + case -1: + layer_state = TemporalLayerState::kDrop; + ++stats_.num_dropped_frames_; + break; + default: + RTC_DCHECK_NOTREACHED(); + } + + DependencyInfo dependency_info; + // TODO(pbos): Consider referencing but not updating the 'alt' buffer for all + // layers. + switch (layer_state) { + case TemporalLayerState::kDrop: + dependency_info = {"", {kNone, kNone, kNone}}; + break; + case TemporalLayerState::kTl0: + // TL0 only references and updates 'last'. + dependency_info = {"SS", {kReferenceAndUpdate, kNone, kNone}}; + dependency_info.frame_config.packetizer_temporal_idx = 0; + break; + case TemporalLayerState::kTl1: + // TL1 references both 'last' and 'golden' but only updates 'golden'. + dependency_info = {"-R", {kReference, kReferenceAndUpdate, kNone}}; + dependency_info.frame_config.packetizer_temporal_idx = 1; + break; + case TemporalLayerState::kTl1Sync: + // Predict from only TL0 to allow participants to switch to the high + // bitrate stream. Updates 'golden' so that TL1 can continue to refer to + // and update 'golden' from this point on. + dependency_info = {"-S", {kReference, kUpdate, kNone}}; + dependency_info.frame_config.packetizer_temporal_idx = 1; + dependency_info.frame_config.layer_sync = true; + break; + } + + pending_frame_configs_[timestamp] = dependency_info; + return dependency_info.frame_config; +} + +void ScreenshareLayers::OnRatesUpdated( + size_t stream_index, + const std::vector<uint32_t>& bitrates_bps, + int framerate_fps) { + RTC_DCHECK_LT(stream_index, StreamCount()); + RTC_DCHECK_GT(framerate_fps, 0); + RTC_DCHECK_GE(bitrates_bps.size(), 1); + RTC_DCHECK_LE(bitrates_bps.size(), 2); + + // `bitrates_bps` uses individual rates per layer, but we want to use the + // accumulated rate here. + uint32_t tl0_kbps = bitrates_bps[0] / 1000; + uint32_t tl1_kbps = tl0_kbps; + if (bitrates_bps.size() > 1) { + tl1_kbps += bitrates_bps[1] / 1000; + } + + if (!target_framerate_) { + // First OnRatesUpdated() is called during construction, with the + // configured targets as parameters. + target_framerate_ = framerate_fps; + capture_framerate_ = target_framerate_; + bitrate_updated_ = true; + } else { + if ((capture_framerate_ && + framerate_fps != static_cast<int>(*capture_framerate_)) || + (tl0_kbps != layers_[0].target_rate_kbps_) || + (tl1_kbps != layers_[1].target_rate_kbps_)) { + bitrate_updated_ = true; + } + + if (framerate_fps < 0) { + capture_framerate_.reset(); + } else { + capture_framerate_ = framerate_fps; + } + } + + layers_[0].target_rate_kbps_ = tl0_kbps; + layers_[1].target_rate_kbps_ = tl1_kbps; +} + +void ScreenshareLayers::OnEncodeDone(size_t stream_index, + uint32_t rtp_timestamp, + size_t size_bytes, + bool is_keyframe, + int qp, + CodecSpecificInfo* info) { + RTC_DCHECK_LT(stream_index, StreamCount()); + + if (size_bytes == 0) { + RTC_LOG(LS_WARNING) << "Empty frame; treating as dropped."; + OnFrameDropped(stream_index, rtp_timestamp); + return; + } + + absl::optional<DependencyInfo> dependency_info; + auto it = pending_frame_configs_.find(rtp_timestamp); + if (it != pending_frame_configs_.end()) { + dependency_info = it->second; + pending_frame_configs_.erase(it); + + if (checker_) { + RTC_DCHECK(checker_->CheckTemporalConfig(is_keyframe, + dependency_info->frame_config)); + } + } + + CodecSpecificInfoVP8& vp8_info = info->codecSpecific.VP8; + GenericFrameInfo& generic_frame_info = info->generic_frame_info.emplace(); + + if (number_of_temporal_layers_ == 1) { + vp8_info.temporalIdx = kNoTemporalIdx; + vp8_info.layerSync = false; + generic_frame_info.temporal_id = 0; + generic_frame_info.decode_target_indications = {kSwitch}; + generic_frame_info.encoder_buffers.emplace_back( + 0, /*referenced=*/!is_keyframe, /*updated=*/true); + } else { + int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(rtp_timestamp); + if (dependency_info) { + vp8_info.temporalIdx = + dependency_info->frame_config.packetizer_temporal_idx; + vp8_info.layerSync = dependency_info->frame_config.layer_sync; + generic_frame_info.temporal_id = vp8_info.temporalIdx; + generic_frame_info.decode_target_indications = + dependency_info->decode_target_indications; + } else { + RTC_DCHECK(is_keyframe); + } + + if (is_keyframe) { + vp8_info.temporalIdx = 0; + last_sync_timestamp_ = unwrapped_timestamp; + vp8_info.layerSync = true; + layers_[0].state = TemporalLayer::State::kKeyFrame; + layers_[1].state = TemporalLayer::State::kKeyFrame; + active_layer_ = 1; + info->template_structure = + GetTemplateStructure(number_of_temporal_layers_); + generic_frame_info.temporal_id = vp8_info.temporalIdx; + generic_frame_info.decode_target_indications = {kSwitch, kSwitch}; + } else if (active_layer_ >= 0 && layers_[active_layer_].state == + TemporalLayer::State::kKeyFrame) { + layers_[active_layer_].state = TemporalLayer::State::kNormal; + } + + vp8_info.useExplicitDependencies = true; + RTC_DCHECK_EQ(vp8_info.referencedBuffersCount, 0u); + RTC_DCHECK_EQ(vp8_info.updatedBuffersCount, 0u); + + // Note that `frame_config` is not derefernced if `is_keyframe`, + // meaning it's never dereferenced if the optional may be unset. + for (int i = 0; i < static_cast<int>(Vp8FrameConfig::Buffer::kCount); ++i) { + bool references = false; + bool updates = is_keyframe; + if (!is_keyframe && dependency_info->frame_config.References( + static_cast<Vp8FrameConfig::Buffer>(i))) { + RTC_DCHECK_LT(vp8_info.referencedBuffersCount, + arraysize(CodecSpecificInfoVP8::referencedBuffers)); + references = true; + vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i; + } + + if (is_keyframe || dependency_info->frame_config.Updates( + static_cast<Vp8FrameConfig::Buffer>(i))) { + RTC_DCHECK_LT(vp8_info.updatedBuffersCount, + arraysize(CodecSpecificInfoVP8::updatedBuffers)); + updates = true; + vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i; + } + + if (references || updates) + generic_frame_info.encoder_buffers.emplace_back(i, references, updates); + } + } + + encode_framerate_.Update(1, rtc::TimeMillis()); + + if (number_of_temporal_layers_ == 1) + return; + + RTC_DCHECK_NE(-1, active_layer_); + if (layers_[active_layer_].state == TemporalLayer::State::kDropped) { + layers_[active_layer_].state = TemporalLayer::State::kQualityBoost; + } + + if (qp != -1) + layers_[active_layer_].last_qp = qp; + + if (active_layer_ == 0) { + layers_[0].debt_bytes_ += size_bytes; + layers_[1].debt_bytes_ += size_bytes; + ++stats_.num_tl0_frames_; + stats_.tl0_target_bitrate_sum_ += layers_[0].target_rate_kbps_; + stats_.tl0_qp_sum_ += qp; + } else if (active_layer_ == 1) { + layers_[1].debt_bytes_ += size_bytes; + ++stats_.num_tl1_frames_; + stats_.tl1_target_bitrate_sum_ += layers_[1].target_rate_kbps_; + stats_.tl1_qp_sum_ += qp; + } +} + +void ScreenshareLayers::OnFrameDropped(size_t stream_index, + uint32_t rtp_timestamp) { + layers_[active_layer_].state = TemporalLayer::State::kDropped; + ++stats_.num_overshoots_; +} + +void ScreenshareLayers::OnPacketLossRateUpdate(float packet_loss_rate) {} + +void ScreenshareLayers::OnRttUpdate(int64_t rtt_ms) {} + +void ScreenshareLayers::OnLossNotification( + const VideoEncoder::LossNotification& loss_notification) {} + +FrameDependencyStructure ScreenshareLayers::GetTemplateStructure( + int num_layers) const { + RTC_CHECK_LT(num_layers, 3); + RTC_CHECK_GT(num_layers, 0); + + FrameDependencyStructure template_structure; + template_structure.num_decode_targets = num_layers; + + switch (num_layers) { + case 1: { + template_structure.templates.resize(2); + template_structure.templates[0].T(0).Dtis("S"); + template_structure.templates[1].T(0).Dtis("S").FrameDiffs({1}); + return template_structure; + } + case 2: { + template_structure.templates.resize(3); + template_structure.templates[0].T(0).Dtis("SS"); + template_structure.templates[1].T(0).Dtis("SS").FrameDiffs({1}); + template_structure.templates[2].T(1).Dtis("-S").FrameDiffs({1}); + return template_structure; + } + default: + RTC_DCHECK_NOTREACHED(); + // To make the compiler happy! + return template_structure; + } +} + +bool ScreenshareLayers::TimeToSync(int64_t timestamp) const { + RTC_DCHECK_EQ(1, active_layer_); + RTC_DCHECK_NE(-1, layers_[0].last_qp); + if (layers_[1].last_qp == -1) { + // First frame in TL1 should only depend on TL0 since there are no + // previous frames in TL1. + return true; + } + + RTC_DCHECK_NE(-1, last_sync_timestamp_); + int64_t timestamp_diff = timestamp - last_sync_timestamp_; + if (timestamp_diff > kMaxTimeBetweenSyncs) { + // After a certain time, force a sync frame. + return true; + } else if (timestamp_diff < kMinTimeBetweenSyncs) { + // If too soon from previous sync frame, don't issue a new one. + return false; + } + // Issue a sync frame if difference in quality between TL0 and TL1 isn't too + // large. + if (layers_[0].last_qp - layers_[1].last_qp < kQpDeltaThresholdForSync) + return true; + return false; +} + +uint32_t ScreenshareLayers::GetCodecTargetBitrateKbps() const { + uint32_t target_bitrate_kbps = layers_[0].target_rate_kbps_; + + if (number_of_temporal_layers_ > 1) { + // Calculate a codec target bitrate. This may be higher than TL0, gaining + // quality at the expense of frame rate at TL0. Constraints: + // - TL0 frame rate no less than framerate / kMaxTL0FpsReduction. + // - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate. + target_bitrate_kbps = + std::min(layers_[0].target_rate_kbps_ * kMaxTL0FpsReduction, + layers_[1].target_rate_kbps_ / kAcceptableTargetOvershoot); + } + + return std::max(layers_[0].target_rate_kbps_, target_bitrate_kbps); +} + +Vp8EncoderConfig ScreenshareLayers::UpdateConfiguration(size_t stream_index) { + RTC_DCHECK_LT(stream_index, StreamCount()); + RTC_DCHECK(min_qp_.has_value()); + RTC_DCHECK(max_qp_.has_value()); + + const uint32_t target_bitrate_kbps = GetCodecTargetBitrateKbps(); + + // TODO(sprang): We _really_ need to make an overhaul of this class. :( + // If we're dropping frames in order to meet a target framerate, adjust the + // bitrate assigned to the encoder so the total average bitrate is correct. + float encoder_config_bitrate_kbps = target_bitrate_kbps; + if (target_framerate_ && capture_framerate_ && + *target_framerate_ < *capture_framerate_) { + encoder_config_bitrate_kbps *= + static_cast<float>(*capture_framerate_) / *target_framerate_; + } + + if (bitrate_updated_ || + encoder_config_.rc_target_bitrate != + absl::make_optional(encoder_config_bitrate_kbps)) { + encoder_config_.rc_target_bitrate = encoder_config_bitrate_kbps; + + // Don't reconfigure qp limits during quality boost frames. + if (active_layer_ == -1 || + layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) { + const int min_qp = min_qp_.value(); + const int max_qp = max_qp_.value(); + + // After a dropped frame, a frame with max qp will be encoded and the + // quality will then ramp up from there. To boost the speed of recovery, + // encode the next frame with lower max qp, if there is sufficient + // bandwidth to do so without causing excessive delay. + // TL0 is the most important to improve since the errors in this layer + // will propagate to TL1. + // Currently, reduce max qp by 20% for TL0 and 15% for TL1. + if (layers_[1].target_rate_kbps_ >= kMinBitrateKbpsForQpBoost) { + layers_[0].enhanced_max_qp = min_qp + (((max_qp - min_qp) * 80) / 100); + layers_[1].enhanced_max_qp = min_qp + (((max_qp - min_qp) * 85) / 100); + } else { + layers_[0].enhanced_max_qp = -1; + layers_[1].enhanced_max_qp = -1; + } + } + + if (capture_framerate_) { + int avg_frame_size = + (target_bitrate_kbps * 1000) / (8 * *capture_framerate_); + // Allow max debt to be the size of a single optimal frame. + // TODO(sprang): Determine if this needs to be adjusted by some factor. + // (Lower values may cause more frame drops, higher may lead to queuing + // delays.) + max_debt_bytes_ = avg_frame_size; + } + + bitrate_updated_ = false; + } + + // Don't try to update boosts state if not active yet. + if (active_layer_ == -1) + return encoder_config_; + + if (number_of_temporal_layers_ <= 1) + return encoder_config_; + + // If layer is in the quality boost state (following a dropped frame), update + // the configuration with the adjusted (lower) qp and set the state back to + // normal. + unsigned int adjusted_max_qp = max_qp_.value(); // Set the normal max qp. + if (layers_[active_layer_].state == TemporalLayer::State::kQualityBoost) { + if (layers_[active_layer_].enhanced_max_qp != -1) { + // Bitrate is high enough for quality boost, update max qp. + adjusted_max_qp = layers_[active_layer_].enhanced_max_qp; + } + // Regardless of qp, reset the boost state for the next frame. + layers_[active_layer_].state = TemporalLayer::State::kNormal; + } + encoder_config_.rc_max_quantizer = adjusted_max_qp; + + return encoder_config_; +} + +void ScreenshareLayers::TemporalLayer::UpdateDebt(int64_t delta_ms) { + uint32_t debt_reduction_bytes = target_rate_kbps_ * delta_ms / 8; + if (debt_reduction_bytes >= debt_bytes_) { + debt_bytes_ = 0; + } else { + debt_bytes_ -= debt_reduction_bytes; + } +} + +void ScreenshareLayers::UpdateHistograms() { + if (stats_.first_frame_time_ms_ == -1) + return; + int64_t duration_sec = + (rtc::TimeMillis() - stats_.first_frame_time_ms_ + 500) / 1000; + if (duration_sec >= metrics::kMinRunTimeInSeconds) { + RTC_HISTOGRAM_COUNTS_10000( + "WebRTC.Video.Screenshare.Layer0.FrameRate", + (stats_.num_tl0_frames_ + (duration_sec / 2)) / duration_sec); + RTC_HISTOGRAM_COUNTS_10000( + "WebRTC.Video.Screenshare.Layer1.FrameRate", + (stats_.num_tl1_frames_ + (duration_sec / 2)) / duration_sec); + int total_frames = stats_.num_tl0_frames_ + stats_.num_tl1_frames_; + RTC_HISTOGRAM_COUNTS_10000( + "WebRTC.Video.Screenshare.FramesPerDrop", + (stats_.num_dropped_frames_ == 0 + ? 0 + : total_frames / stats_.num_dropped_frames_)); + RTC_HISTOGRAM_COUNTS_10000( + "WebRTC.Video.Screenshare.FramesPerOvershoot", + (stats_.num_overshoots_ == 0 ? 0 + : total_frames / stats_.num_overshoots_)); + if (stats_.num_tl0_frames_ > 0) { + RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.Layer0.Qp", + stats_.tl0_qp_sum_ / stats_.num_tl0_frames_); + RTC_HISTOGRAM_COUNTS_10000( + "WebRTC.Video.Screenshare.Layer0.TargetBitrate", + stats_.tl0_target_bitrate_sum_ / stats_.num_tl0_frames_); + } + if (stats_.num_tl1_frames_ > 0) { + RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.Layer1.Qp", + stats_.tl1_qp_sum_ / stats_.num_tl1_frames_); + RTC_HISTOGRAM_COUNTS_10000( + "WebRTC.Video.Screenshare.Layer1.TargetBitrate", + stats_.tl1_target_bitrate_sum_ / stats_.num_tl1_frames_); + } + } +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers.h new file mode 100644 index 0000000000..47d6b401f4 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -0,0 +1,164 @@ +/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_ +#define MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_ + +#include <map> +#include <memory> +#include <utility> +#include <vector> + +#include "api/video_codecs/vp8_frame_config.h" +#include "api/video_codecs/vp8_temporal_layers.h" +#include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/utility/frame_dropper.h" +#include "rtc_base/numerics/sequence_number_unwrapper.h" +#include "rtc_base/rate_statistics.h" + +namespace webrtc { + +struct CodecSpecificInfoVP8; +class Clock; + +class ScreenshareLayers final : public Vp8FrameBufferController { + public: + static const double kMaxTL0FpsReduction; + static const double kAcceptableTargetOvershoot; + static const int kMaxFrameIntervalMs; + + explicit ScreenshareLayers(int num_temporal_layers); + ~ScreenshareLayers() override; + + void SetQpLimits(size_t stream_index, int min_qp, int max_qp) override; + + size_t StreamCount() const override; + + bool SupportsEncoderFrameDropping(size_t stream_index) const override; + + // Returns the recommended VP8 encode flags needed. May refresh the decoder + // and/or update the reference buffers. + Vp8FrameConfig NextFrameConfig(size_t stream_index, + uint32_t rtp_timestamp) override; + + // New target bitrate, per temporal layer. + void OnRatesUpdated(size_t stream_index, + const std::vector<uint32_t>& bitrates_bps, + int framerate_fps) override; + + Vp8EncoderConfig UpdateConfiguration(size_t stream_index) override; + + void OnEncodeDone(size_t stream_index, + uint32_t rtp_timestamp, + size_t size_bytes, + bool is_keyframe, + int qp, + CodecSpecificInfo* info) override; + + void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) override; + + void OnPacketLossRateUpdate(float packet_loss_rate) override; + + void OnRttUpdate(int64_t rtt_ms) override; + + void OnLossNotification( + const VideoEncoder::LossNotification& loss_notification) override; + + private: + enum class TemporalLayerState : int { kDrop, kTl0, kTl1, kTl1Sync }; + + struct DependencyInfo { + DependencyInfo() = default; + DependencyInfo(absl::string_view indication_symbols, + Vp8FrameConfig frame_config) + : decode_target_indications( + webrtc_impl::StringToDecodeTargetIndications(indication_symbols)), + frame_config(frame_config) {} + + absl::InlinedVector<DecodeTargetIndication, 10> decode_target_indications; + Vp8FrameConfig frame_config; + }; + + bool TimeToSync(int64_t timestamp) const; + uint32_t GetCodecTargetBitrateKbps() const; + + const int number_of_temporal_layers_; + + // TODO(eladalon/sprang): These should be made into const-int set in the ctor. + absl::optional<int> min_qp_; + absl::optional<int> max_qp_; + + int active_layer_; + int64_t last_timestamp_; + int64_t last_sync_timestamp_; + int64_t last_emitted_tl0_timestamp_; + int64_t last_frame_time_ms_; + RtpTimestampUnwrapper time_wrap_handler_; + uint32_t max_debt_bytes_; + + std::map<uint32_t, DependencyInfo> pending_frame_configs_; + + // Configured max framerate. + absl::optional<uint32_t> target_framerate_; + // Incoming framerate from capturer. + absl::optional<uint32_t> capture_framerate_; + + // Tracks what framerate we actually encode, and drops frames on overshoot. + RateStatistics encode_framerate_; + bool bitrate_updated_; + + static constexpr int kMaxNumTemporalLayers = 2; + struct TemporalLayer { + TemporalLayer() + : state(State::kNormal), + enhanced_max_qp(-1), + last_qp(-1), + debt_bytes_(0), + target_rate_kbps_(0) {} + + enum class State { + kNormal, + kDropped, + kReencoded, + kQualityBoost, + kKeyFrame + } state; + + int enhanced_max_qp; + int last_qp; + uint32_t debt_bytes_; + uint32_t target_rate_kbps_; + + void UpdateDebt(int64_t delta_ms); + } layers_[kMaxNumTemporalLayers]; + + void UpdateHistograms(); + FrameDependencyStructure GetTemplateStructure(int num_layers) const; + + // Data for histogram statistics. + struct Stats { + int64_t first_frame_time_ms_ = -1; + int64_t num_tl0_frames_ = 0; + int64_t num_tl1_frames_ = 0; + int64_t num_dropped_frames_ = 0; + int64_t num_overshoots_ = 0; + int64_t tl0_qp_sum_ = 0; + int64_t tl1_qp_sum_ = 0; + int64_t tl0_target_bitrate_sum_ = 0; + int64_t tl1_target_bitrate_sum_ = 0; + } stats_; + + Vp8EncoderConfig encoder_config_; + + // Optional utility used to verify reference validity. + std::unique_ptr<TemporalLayersChecker> checker_; +}; +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_ diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc new file mode 100644 index 0000000000..e5b3bd4fdf --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc @@ -0,0 +1,788 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/vp8/screenshare_layers.h" + +#include <stdlib.h> +#include <string.h> + +#include <cstdint> +#include <memory> +#include <vector> + +#include "api/video_codecs/vp8_frame_config.h" +#include "modules/video_coding/codecs/interface/common_constants.h" +#include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "rtc_base/checks.h" +#include "rtc_base/fake_clock.h" +#include "system_wrappers/include/metrics.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "vpx/vp8cx.h" + +using ::testing::_; +using ::testing::ElementsAre; +using ::testing::NiceMock; + +namespace webrtc { +namespace { +// 5 frames per second at 90 kHz. +const uint32_t kTimestampDelta5Fps = 90000 / 5; +const int kDefaultQp = 54; +const int kDefaultTl0BitrateKbps = 200; +const int kDefaultTl1BitrateKbps = 2000; +const int kFrameRate = 5; +const int kSyncPeriodSeconds = 2; +const int kMaxSyncPeriodSeconds = 4; + +// Expected flags for corresponding temporal layers. +const int kTl0Flags = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF; +const int kTl1Flags = + VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST; +const int kTl1SyncFlags = VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | + VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST; +const std::vector<uint32_t> kDefault2TlBitratesBps = { + kDefaultTl0BitrateKbps * 1000, + (kDefaultTl1BitrateKbps - kDefaultTl0BitrateKbps) * 1000}; + +} // namespace + +class ScreenshareLayerTest : public ::testing::Test { + protected: + ScreenshareLayerTest() + : min_qp_(2), + max_qp_(kDefaultQp), + frame_size_(-1), + timestamp_(90), + config_updated_(false) {} + virtual ~ScreenshareLayerTest() {} + + void SetUp() override { + layers_.reset(new ScreenshareLayers(2)); + cfg_ = ConfigureBitrates(); + } + + int EncodeFrame(bool base_sync, CodecSpecificInfo* info = nullptr) { + CodecSpecificInfo ignored_info; + if (!info) { + info = &ignored_info; + } + + int flags = ConfigureFrame(base_sync); + if (flags != -1) + layers_->OnEncodeDone(0, timestamp_, frame_size_, base_sync, kDefaultQp, + info); + return flags; + } + + int ConfigureFrame(bool key_frame) { + tl_config_ = NextFrameConfig(0, timestamp_); + EXPECT_EQ(0, tl_config_.encoder_layer_id) + << "ScreenshareLayers always encodes using the bitrate allocator for " + "layer 0, but may reference different buffers and packetize " + "differently."; + if (tl_config_.drop_frame) { + return -1; + } + const uint32_t prev_rc_target_bitrate = cfg_.rc_target_bitrate.value_or(-1); + const uint32_t prev_rc_max_quantizer = cfg_.rc_max_quantizer.value_or(-1); + + cfg_ = layers_->UpdateConfiguration(0); + + config_updated_ = + cfg_.temporal_layer_config.has_value() || + (cfg_.rc_target_bitrate.has_value() && + cfg_.rc_target_bitrate.value() != prev_rc_target_bitrate) || + (cfg_.rc_max_quantizer.has_value() && + cfg_.rc_max_quantizer.value() != prev_rc_max_quantizer) || + cfg_.g_error_resilient.has_value(); + + int flags = LibvpxVp8Encoder::EncodeFlags(tl_config_); + EXPECT_NE(-1, frame_size_); + return flags; + } + + Vp8FrameConfig NextFrameConfig(size_t stream_index, uint32_t timestamp) { + int64_t timestamp_ms = timestamp / 90; + clock_.AdvanceTime(TimeDelta::Millis(timestamp_ms - rtc::TimeMillis())); + return layers_->NextFrameConfig(stream_index, timestamp); + } + + int FrameSizeForBitrate(int bitrate_kbps) { + return ((bitrate_kbps * 1000) / 8) / kFrameRate; + } + + Vp8EncoderConfig ConfigureBitrates() { + layers_->SetQpLimits(0, min_qp_, max_qp_); + layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, kFrameRate); + const Vp8EncoderConfig vp8_cfg = layers_->UpdateConfiguration(0); + EXPECT_TRUE(vp8_cfg.rc_target_bitrate.has_value()); + frame_size_ = FrameSizeForBitrate(vp8_cfg.rc_target_bitrate.value()); + return vp8_cfg; + } + + void WithQpLimits(int min_qp, int max_qp) { + min_qp_ = min_qp; + max_qp_ = max_qp; + } + + // Runs a few initial frames and makes sure we have seen frames on both + // temporal layers, including sync and non-sync frames. + bool RunGracePeriod() { + bool got_tl0 = false; + bool got_tl1 = false; + bool got_tl1_sync = false; + for (int i = 0; i < 10; ++i) { + CodecSpecificInfo info; + EXPECT_NE(-1, EncodeFrame(false, &info)); + timestamp_ += kTimestampDelta5Fps; + if (info.codecSpecific.VP8.temporalIdx == 0) { + got_tl0 = true; + } else if (info.codecSpecific.VP8.layerSync) { + got_tl1_sync = true; + } else { + got_tl1 = true; + } + if (got_tl0 && got_tl1 && got_tl1_sync) + return true; + } + return false; + } + + // Adds frames until we get one in the specified temporal layer. The last + // FrameEncoded() call will be omitted and needs to be done by the caller. + // Returns the flags for the last frame. + int SkipUntilTl(int layer) { + return SkipUntilTlAndSync(layer, absl::nullopt); + } + + // Same as SkipUntilTl, but also waits until the sync bit condition is met. + int SkipUntilTlAndSync(int layer, absl::optional<bool> sync) { + int flags = 0; + const int kMaxFramesToSkip = + 1 + (sync.value_or(false) ? kMaxSyncPeriodSeconds : 1) * kFrameRate; + for (int i = 0; i < kMaxFramesToSkip; ++i) { + flags = ConfigureFrame(false); + if (tl_config_.packetizer_temporal_idx != layer || + (sync && *sync != tl_config_.layer_sync)) { + if (flags != -1) { + // If flags do not request a frame drop, report some default values + // for frame size etc. + CodecSpecificInfo info; + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp, + &info); + } + timestamp_ += kTimestampDelta5Fps; + } else { + // Found frame from sought after layer. + return flags; + } + } + ADD_FAILURE() << "Did not get a frame of TL" << layer << " in time."; + return -1; + } + + int min_qp_; + uint32_t max_qp_; + int frame_size_; + rtc::ScopedFakeClock clock_; + std::unique_ptr<ScreenshareLayers> layers_; + + uint32_t timestamp_; + Vp8FrameConfig tl_config_; + Vp8EncoderConfig cfg_; + bool config_updated_; + + CodecSpecificInfo* IgnoredCodecSpecificInfo() { + ignored_codec_specific_info_ = std::make_unique<CodecSpecificInfo>(); + return ignored_codec_specific_info_.get(); + } + + private: + std::unique_ptr<CodecSpecificInfo> ignored_codec_specific_info_; +}; + +TEST_F(ScreenshareLayerTest, 1Layer) { + layers_.reset(new ScreenshareLayers(1)); + ConfigureBitrates(); + // One layer screenshare should not use the frame dropper as all frames will + // belong to the base layer. + const int kSingleLayerFlags = 0; + auto info = std::make_unique<CodecSpecificInfo>(); + int flags = EncodeFrame(/*base_sync=*/false, info.get()); + timestamp_ += kTimestampDelta5Fps; + EXPECT_EQ(static_cast<uint8_t>(kNoTemporalIdx), + info->codecSpecific.VP8.temporalIdx); + EXPECT_FALSE(info->codecSpecific.VP8.layerSync); + EXPECT_EQ(info->generic_frame_info->temporal_id, 0); + + info = std::make_unique<CodecSpecificInfo>(); + flags = EncodeFrame(/*base_sync=*/false, info.get()); + EXPECT_EQ(kSingleLayerFlags, flags); + EXPECT_EQ(static_cast<uint8_t>(kNoTemporalIdx), + info->codecSpecific.VP8.temporalIdx); + EXPECT_FALSE(info->codecSpecific.VP8.layerSync); + EXPECT_EQ(info->generic_frame_info->temporal_id, 0); +} + +TEST_F(ScreenshareLayerTest, 2LayersPeriodicSync) { + std::vector<int> sync_times; + const int kNumFrames = kSyncPeriodSeconds * kFrameRate * 2 - 1; + for (int i = 0; i < kNumFrames; ++i) { + CodecSpecificInfo info; + EncodeFrame(false, &info); + timestamp_ += kTimestampDelta5Fps; + if (info.codecSpecific.VP8.temporalIdx == 1 && + info.codecSpecific.VP8.layerSync) { + sync_times.push_back(timestamp_); + } + } + + ASSERT_EQ(2u, sync_times.size()); + EXPECT_GE(sync_times[1] - sync_times[0], 90000 * kSyncPeriodSeconds); +} + +TEST_F(ScreenshareLayerTest, 2LayersSyncAfterTimeout) { + std::vector<int> sync_times; + const int kNumFrames = kMaxSyncPeriodSeconds * kFrameRate * 2 - 1; + for (int i = 0; i < kNumFrames; ++i) { + CodecSpecificInfo info; + + tl_config_ = NextFrameConfig(0, timestamp_); + cfg_ = layers_->UpdateConfiguration(0); + + // Simulate TL1 being at least 8 qp steps better. + if (tl_config_.packetizer_temporal_idx == 0) { + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp, + &info); + } else { + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp - 8, + &info); + } + + if (info.codecSpecific.VP8.temporalIdx == 1 && + info.codecSpecific.VP8.layerSync) + sync_times.push_back(timestamp_); + + timestamp_ += kTimestampDelta5Fps; + } + + ASSERT_EQ(2u, sync_times.size()); + EXPECT_GE(sync_times[1] - sync_times[0], 90000 * kMaxSyncPeriodSeconds); +} + +TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) { + std::vector<int> sync_times; + + const int kNumFrames = (kSyncPeriodSeconds + + ((kMaxSyncPeriodSeconds - kSyncPeriodSeconds) / 2)) * + kFrameRate; + for (int i = 0; i < kNumFrames; ++i) { + CodecSpecificInfo info; + + ConfigureFrame(false); + + // Simulate TL1 being at least 8 qp steps better. + if (tl_config_.packetizer_temporal_idx == 0) { + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp, + &info); + } else { + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp - 8, + &info); + } + + if (info.codecSpecific.VP8.temporalIdx == 1 && + info.codecSpecific.VP8.layerSync) + sync_times.push_back(timestamp_); + + timestamp_ += kTimestampDelta5Fps; + } + + ASSERT_EQ(1u, sync_times.size()); + + bool bumped_tl0_quality = false; + for (int i = 0; i < 3; ++i) { + CodecSpecificInfo info; + + int flags = ConfigureFrame(false); + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp - 8, + &info); + if (info.codecSpecific.VP8.temporalIdx == 0) { + // Bump TL0 to same quality as TL1. + bumped_tl0_quality = true; + } else { + if (bumped_tl0_quality) { + EXPECT_TRUE(info.codecSpecific.VP8.layerSync); + EXPECT_EQ(kTl1SyncFlags, flags); + return; + } + } + timestamp_ += kTimestampDelta5Fps; + } + ADD_FAILURE() << "No TL1 frame arrived within time limit."; +} + +TEST_F(ScreenshareLayerTest, 2LayersToggling) { + EXPECT_TRUE(RunGracePeriod()); + + // Insert 50 frames. 2/5 should be TL0. + int tl0_frames = 0; + int tl1_frames = 0; + for (int i = 0; i < 50; ++i) { + CodecSpecificInfo info; + EncodeFrame(/*base_sync=*/false, &info); + EXPECT_EQ(info.codecSpecific.VP8.temporalIdx, + info.generic_frame_info->temporal_id); + timestamp_ += kTimestampDelta5Fps; + switch (info.codecSpecific.VP8.temporalIdx) { + case 0: + ++tl0_frames; + break; + case 1: + ++tl1_frames; + break; + default: + abort(); + } + } + EXPECT_EQ(20, tl0_frames); + EXPECT_EQ(30, tl1_frames); +} + +TEST_F(ScreenshareLayerTest, AllFitsLayer0) { + frame_size_ = FrameSizeForBitrate(kDefaultTl0BitrateKbps); + + // Insert 50 frames, small enough that all fits in TL0. + for (int i = 0; i < 50; ++i) { + CodecSpecificInfo info; + int flags = EncodeFrame(false, &info); + timestamp_ += kTimestampDelta5Fps; + EXPECT_EQ(kTl0Flags, flags); + EXPECT_EQ(0, info.codecSpecific.VP8.temporalIdx); + } +} + +TEST_F(ScreenshareLayerTest, TooHighBitrate) { + frame_size_ = 2 * FrameSizeForBitrate(kDefaultTl1BitrateKbps); + + // Insert 100 frames. Half should be dropped. + int tl0_frames = 0; + int tl1_frames = 0; + int dropped_frames = 0; + for (int i = 0; i < 100; ++i) { + CodecSpecificInfo info; + int flags = EncodeFrame(false, &info); + timestamp_ += kTimestampDelta5Fps; + if (flags == -1) { + ++dropped_frames; + } else { + switch (info.codecSpecific.VP8.temporalIdx) { + case 0: + ++tl0_frames; + break; + case 1: + ++tl1_frames; + break; + default: + ADD_FAILURE() << "Unexpected temporal id"; + } + } + } + + EXPECT_NEAR(50, tl0_frames + tl1_frames, 1); + EXPECT_NEAR(50, dropped_frames, 1); +} + +TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL0) { + const int kTl0_kbps = 100; + const int kTl1_kbps = 1000; + const std::vector<uint32_t> layer_rates = {kTl0_kbps * 1000, + (kTl1_kbps - kTl0_kbps) * 1000}; + layers_->OnRatesUpdated(0, layer_rates, kFrameRate); + cfg_ = layers_->UpdateConfiguration(0); + + EXPECT_EQ(static_cast<unsigned int>( + ScreenshareLayers::kMaxTL0FpsReduction * kTl0_kbps + 0.5), + cfg_.rc_target_bitrate); +} + +TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL1) { + const int kTl0_kbps = 100; + const int kTl1_kbps = 450; + const std::vector<uint32_t> layer_rates = {kTl0_kbps * 1000, + (kTl1_kbps - kTl0_kbps) * 1000}; + layers_->OnRatesUpdated(0, layer_rates, kFrameRate); + cfg_ = layers_->UpdateConfiguration(0); + + EXPECT_EQ(static_cast<unsigned int>( + kTl1_kbps / ScreenshareLayers::kAcceptableTargetOvershoot), + cfg_.rc_target_bitrate); +} + +TEST_F(ScreenshareLayerTest, TargetBitrateBelowTL0) { + const int kTl0_kbps = 100; + const std::vector<uint32_t> layer_rates = {kTl0_kbps * 1000}; + layers_->OnRatesUpdated(0, layer_rates, kFrameRate); + cfg_ = layers_->UpdateConfiguration(0); + + EXPECT_EQ(static_cast<uint32_t>(kTl0_kbps), cfg_.rc_target_bitrate); +} + +TEST_F(ScreenshareLayerTest, EncoderDrop) { + EXPECT_TRUE(RunGracePeriod()); + SkipUntilTl(0); + + // Size 0 indicates dropped frame. + layers_->OnEncodeDone(0, timestamp_, 0, false, 0, IgnoredCodecSpecificInfo()); + + // Re-encode frame (so don't advance timestamp). + int flags = EncodeFrame(false); + timestamp_ += kTimestampDelta5Fps; + EXPECT_FALSE(config_updated_); + EXPECT_EQ(kTl0Flags, flags); + + // Next frame should have boosted quality... + SkipUntilTl(0); + EXPECT_TRUE(config_updated_); + EXPECT_LT(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp)); + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + timestamp_ += kTimestampDelta5Fps; + + // ...then back to standard setup. + SkipUntilTl(0); + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + timestamp_ += kTimestampDelta5Fps; + EXPECT_EQ(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp)); + + // Next drop in TL1. + SkipUntilTl(1); + layers_->OnEncodeDone(0, timestamp_, 0, false, 0, IgnoredCodecSpecificInfo()); + + // Re-encode frame (so don't advance timestamp). + flags = EncodeFrame(false); + timestamp_ += kTimestampDelta5Fps; + EXPECT_FALSE(config_updated_); + EXPECT_EQ(kTl1Flags, flags); + + // Next frame should have boosted QP. + SkipUntilTl(1); + EXPECT_TRUE(config_updated_); + EXPECT_LT(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp)); + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + timestamp_ += kTimestampDelta5Fps; + + // ...and back to normal. + SkipUntilTl(1); + EXPECT_EQ(cfg_.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp)); + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + timestamp_ += kTimestampDelta5Fps; +} + +TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) { + const int kLowBitrateKbps = 50; + const int kLargeFrameSizeBytes = 100000; + const uint32_t kStartTimestamp = 1234; + + const std::vector<uint32_t> layer_rates = {kLowBitrateKbps * 1000}; + layers_->OnRatesUpdated(0, layer_rates, kFrameRate); + cfg_ = layers_->UpdateConfiguration(0); + + EXPECT_EQ(kTl0Flags, + LibvpxVp8Encoder::EncodeFlags(NextFrameConfig(0, kStartTimestamp))); + layers_->OnEncodeDone(0, kStartTimestamp, kLargeFrameSizeBytes, false, + kDefaultQp, IgnoredCodecSpecificInfo()); + + const uint32_t kTwoSecondsLater = + kStartTimestamp + (ScreenshareLayers::kMaxFrameIntervalMs * 90); + + // Sanity check, repayment time should exceed kMaxFrameIntervalMs. + ASSERT_GT(kStartTimestamp + 90 * (kLargeFrameSizeBytes * 8) / kLowBitrateKbps, + kStartTimestamp + (ScreenshareLayers::kMaxFrameIntervalMs * 90)); + + // Expect drop one frame interval before the two second timeout. If we try + // any later, the frame will be dropped anyway by the frame rate throttling + // logic. + EXPECT_TRUE( + NextFrameConfig(0, kTwoSecondsLater - kTimestampDelta5Fps).drop_frame); + + // More than two seconds has passed since last frame, one should be emitted + // even if bitrate target is then exceeded. + EXPECT_EQ(kTl0Flags, LibvpxVp8Encoder::EncodeFlags( + NextFrameConfig(0, kTwoSecondsLater + 90))); +} + +TEST_F(ScreenshareLayerTest, UpdatesHistograms) { + metrics::Reset(); + bool trigger_drop = false; + bool dropped_frame = false; + bool overshoot = false; + const int kTl0Qp = 35; + const int kTl1Qp = 30; + for (int64_t timestamp = 0; + timestamp < kTimestampDelta5Fps * 5 * metrics::kMinRunTimeInSeconds; + timestamp += kTimestampDelta5Fps) { + tl_config_ = NextFrameConfig(0, timestamp); + if (tl_config_.drop_frame) { + dropped_frame = true; + continue; + } + int flags = LibvpxVp8Encoder::EncodeFlags(tl_config_); + if (flags != -1) + cfg_ = layers_->UpdateConfiguration(0); + + if (timestamp >= kTimestampDelta5Fps * 5 && !overshoot && flags != -1) { + // Simulate one overshoot. + layers_->OnEncodeDone(0, timestamp, 0, false, 0, nullptr); + overshoot = true; + } + + if (flags == kTl0Flags) { + if (timestamp >= kTimestampDelta5Fps * 20 && !trigger_drop) { + // Simulate a too large frame, to cause frame drop. + layers_->OnEncodeDone(0, timestamp, frame_size_ * 10, false, kTl0Qp, + IgnoredCodecSpecificInfo()); + trigger_drop = true; + } else { + layers_->OnEncodeDone(0, timestamp, frame_size_, false, kTl0Qp, + IgnoredCodecSpecificInfo()); + } + } else if (flags == kTl1Flags || flags == kTl1SyncFlags) { + layers_->OnEncodeDone(0, timestamp, frame_size_, false, kTl1Qp, + IgnoredCodecSpecificInfo()); + } else if (flags == -1) { + dropped_frame = true; + } else { + RTC_DCHECK_NOTREACHED() << "Unexpected flags"; + } + clock_.AdvanceTime(TimeDelta::Millis(1000 / 5)); + } + + EXPECT_TRUE(overshoot); + EXPECT_TRUE(dropped_frame); + + layers_.reset(); // Histograms are reported on destruction. + + EXPECT_METRIC_EQ( + 1, metrics::NumSamples("WebRTC.Video.Screenshare.Layer0.FrameRate")); + EXPECT_METRIC_EQ( + 1, metrics::NumSamples("WebRTC.Video.Screenshare.Layer1.FrameRate")); + EXPECT_METRIC_EQ( + 1, metrics::NumSamples("WebRTC.Video.Screenshare.FramesPerDrop")); + EXPECT_METRIC_EQ( + 1, metrics::NumSamples("WebRTC.Video.Screenshare.FramesPerOvershoot")); + EXPECT_METRIC_EQ(1, + metrics::NumSamples("WebRTC.Video.Screenshare.Layer0.Qp")); + EXPECT_METRIC_EQ(1, + metrics::NumSamples("WebRTC.Video.Screenshare.Layer1.Qp")); + EXPECT_METRIC_EQ( + 1, metrics::NumSamples("WebRTC.Video.Screenshare.Layer0.TargetBitrate")); + EXPECT_METRIC_EQ( + 1, metrics::NumSamples("WebRTC.Video.Screenshare.Layer1.TargetBitrate")); + + EXPECT_METRIC_GT( + metrics::MinSample("WebRTC.Video.Screenshare.Layer0.FrameRate"), 1); + EXPECT_METRIC_GT( + metrics::MinSample("WebRTC.Video.Screenshare.Layer1.FrameRate"), 1); + EXPECT_METRIC_GT(metrics::MinSample("WebRTC.Video.Screenshare.FramesPerDrop"), + 1); + EXPECT_METRIC_GT( + metrics::MinSample("WebRTC.Video.Screenshare.FramesPerOvershoot"), 1); + EXPECT_METRIC_EQ( + 1, metrics::NumEvents("WebRTC.Video.Screenshare.Layer0.Qp", kTl0Qp)); + EXPECT_METRIC_EQ( + 1, metrics::NumEvents("WebRTC.Video.Screenshare.Layer1.Qp", kTl1Qp)); + EXPECT_METRIC_EQ( + 1, metrics::NumEvents("WebRTC.Video.Screenshare.Layer0.TargetBitrate", + kDefaultTl0BitrateKbps)); + EXPECT_METRIC_EQ( + 1, metrics::NumEvents("WebRTC.Video.Screenshare.Layer1.TargetBitrate", + kDefaultTl1BitrateKbps)); +} + +TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) { + int64_t kTestSpanMs = 2000; + int64_t kFrameIntervalsMs = 1000 / kFrameRate; + + uint32_t timestamp = 1234; + int num_input_frames = 0; + int num_discarded_frames = 0; + + // Send at regular rate - no drops expected. + for (int64_t i = 0; i < kTestSpanMs; i += kFrameIntervalsMs) { + if (NextFrameConfig(0, timestamp).drop_frame) { + ++num_discarded_frames; + } else { + size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8; + layers_->OnEncodeDone(0, timestamp, frame_size_bytes, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + } + timestamp += kFrameIntervalsMs * 90; + clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalsMs)); + + ++num_input_frames; + } + EXPECT_EQ(0, num_discarded_frames); + + // Send at twice the configured rate - drop every other frame. + num_input_frames = 0; + num_discarded_frames = 0; + for (int64_t i = 0; i < kTestSpanMs; i += kFrameIntervalsMs / 2) { + if (NextFrameConfig(0, timestamp).drop_frame) { + ++num_discarded_frames; + } else { + size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8; + layers_->OnEncodeDone(0, timestamp, frame_size_bytes, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + } + timestamp += kFrameIntervalsMs * 90 / 2; + clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalsMs)); + ++num_input_frames; + } + + // Allow for some rounding errors in the measurements. + EXPECT_NEAR(num_discarded_frames, num_input_frames / 2, 2); +} + +TEST_F(ScreenshareLayerTest, 2LayersSyncAtOvershootDrop) { + // Run grace period so we have existing frames in both TL0 and Tl1. + EXPECT_TRUE(RunGracePeriod()); + + // Move ahead until we have a sync frame in TL1. + EXPECT_EQ(kTl1SyncFlags, SkipUntilTlAndSync(1, true)); + ASSERT_TRUE(tl_config_.layer_sync); + + // Simulate overshoot of this frame. + layers_->OnEncodeDone(0, timestamp_, 0, false, 0, nullptr); + + cfg_ = layers_->UpdateConfiguration(0); + EXPECT_EQ(kTl1SyncFlags, LibvpxVp8Encoder::EncodeFlags(tl_config_)); + + CodecSpecificInfo new_info; + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp, + &new_info); + EXPECT_TRUE(new_info.codecSpecific.VP8.layerSync); +} + +TEST_F(ScreenshareLayerTest, DropOnTooShortFrameInterval) { + // Run grace period so we have existing frames in both TL0 and Tl1. + EXPECT_TRUE(RunGracePeriod()); + + // Add a large gap, so there's plenty of room in the rate tracker. + timestamp_ += kTimestampDelta5Fps * 3; + EXPECT_FALSE(NextFrameConfig(0, timestamp_).drop_frame); + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, kDefaultQp, + IgnoredCodecSpecificInfo()); + + // Frame interval below 90% if desired time is not allowed, try inserting + // frame just before this limit. + const int64_t kMinFrameInterval = (kTimestampDelta5Fps * 85) / 100; + timestamp_ += kMinFrameInterval - 90; + EXPECT_TRUE(NextFrameConfig(0, timestamp_).drop_frame); + + // Try again at the limit, now it should pass. + timestamp_ += 90; + EXPECT_FALSE(NextFrameConfig(0, timestamp_).drop_frame); +} + +TEST_F(ScreenshareLayerTest, AdjustsBitrateWhenDroppingFrames) { + const uint32_t kTimestampDelta10Fps = kTimestampDelta5Fps / 2; + const int kNumFrames = 30; + ASSERT_TRUE(cfg_.rc_target_bitrate.has_value()); + const uint32_t default_bitrate = cfg_.rc_target_bitrate.value(); + layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, 10); + + int num_dropped_frames = 0; + for (int i = 0; i < kNumFrames; ++i) { + if (EncodeFrame(false) == -1) + ++num_dropped_frames; + timestamp_ += kTimestampDelta10Fps; + } + cfg_ = layers_->UpdateConfiguration(0); + + EXPECT_EQ(num_dropped_frames, kNumFrames / 2); + EXPECT_EQ(cfg_.rc_target_bitrate, default_bitrate * 2); +} + +TEST_F(ScreenshareLayerTest, UpdatesConfigurationAfterRateChange) { + // Set inital rate again, no need to update configuration. + layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, kFrameRate); + cfg_ = layers_->UpdateConfiguration(0); + + // Rate changed, now update config. + std::vector<uint32_t> bitrates = kDefault2TlBitratesBps; + bitrates[1] -= 100000; + layers_->OnRatesUpdated(0, bitrates, 5); + cfg_ = layers_->UpdateConfiguration(0); + + // Changed rate, but then set changed rate again before trying to update + // configuration, update should still apply. + bitrates[1] -= 100000; + layers_->OnRatesUpdated(0, bitrates, 5); + layers_->OnRatesUpdated(0, bitrates, 5); + cfg_ = layers_->UpdateConfiguration(0); +} + +TEST_F(ScreenshareLayerTest, MaxQpRestoredAfterDoubleDrop) { + // Run grace period so we have existing frames in both TL0 and Tl1. + EXPECT_TRUE(RunGracePeriod()); + + // Move ahead until we have a sync frame in TL1. + EXPECT_EQ(kTl1SyncFlags, SkipUntilTlAndSync(1, true)); + ASSERT_TRUE(tl_config_.layer_sync); + + // Simulate overshoot of this frame. + layers_->OnEncodeDone(0, timestamp_, 0, false, -1, nullptr); + + // Simulate re-encoded frame. + layers_->OnEncodeDone(0, timestamp_, 1, false, max_qp_, + IgnoredCodecSpecificInfo()); + + // Next frame, expect boosted quality. + // Slightly alter bitrate between each frame. + std::vector<uint32_t> kDefault2TlBitratesBpsAlt = kDefault2TlBitratesBps; + kDefault2TlBitratesBpsAlt[1] += 4000; + layers_->OnRatesUpdated(0, kDefault2TlBitratesBpsAlt, kFrameRate); + EXPECT_EQ(kTl1Flags, SkipUntilTlAndSync(1, false)); + EXPECT_TRUE(config_updated_); + EXPECT_LT(cfg_.rc_max_quantizer, max_qp_); + ASSERT_TRUE(cfg_.rc_max_quantizer.has_value()); + const uint32_t adjusted_qp = cfg_.rc_max_quantizer.value(); + + // Simulate overshoot of this frame. + layers_->OnEncodeDone(0, timestamp_, 0, false, -1, nullptr); + + // Simulate re-encoded frame. + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, max_qp_, + IgnoredCodecSpecificInfo()); + + // A third frame, expect boosted quality. + layers_->OnRatesUpdated(0, kDefault2TlBitratesBps, kFrameRate); + EXPECT_EQ(kTl1Flags, SkipUntilTlAndSync(1, false)); + EXPECT_TRUE(config_updated_); + EXPECT_LT(cfg_.rc_max_quantizer, max_qp_); + EXPECT_EQ(adjusted_qp, cfg_.rc_max_quantizer); + + // Frame encoded. + layers_->OnEncodeDone(0, timestamp_, frame_size_, false, max_qp_, + IgnoredCodecSpecificInfo()); + + // A fourth frame, max qp should be restored. + layers_->OnRatesUpdated(0, kDefault2TlBitratesBpsAlt, kFrameRate); + EXPECT_EQ(kTl1Flags, SkipUntilTlAndSync(1, false)); + EXPECT_EQ(cfg_.rc_max_quantizer, max_qp_); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/temporal_layers.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/temporal_layers.h new file mode 100644 index 0000000000..9576fb27be --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/temporal_layers.h @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_ +#define MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_ + +// TODO(webrtc:9012) Remove this file when downstream projects have updated. +#include "api/video_codecs/vp8_temporal_layers.h" + +#endif // MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_ diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/temporal_layers_checker.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/temporal_layers_checker.cc new file mode 100644 index 0000000000..5aebd2c526 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/temporal_layers_checker.cc @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h" + +#include <memory> + +#include "modules/video_coding/codecs/interface/common_constants.h" +#include "modules/video_coding/codecs/vp8/default_temporal_layers.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +std::unique_ptr<TemporalLayersChecker> +TemporalLayersChecker::CreateTemporalLayersChecker(Vp8TemporalLayersType type, + int num_temporal_layers) { + switch (type) { + case Vp8TemporalLayersType::kFixedPattern: + return std::make_unique<DefaultTemporalLayersChecker>( + num_temporal_layers); + case Vp8TemporalLayersType::kBitrateDynamic: + // Conference mode temporal layering for screen content in base stream. + return std::make_unique<TemporalLayersChecker>(num_temporal_layers); + } + RTC_CHECK_NOTREACHED(); +} + +TemporalLayersChecker::TemporalLayersChecker(int num_temporal_layers) + : num_temporal_layers_(num_temporal_layers), + sequence_number_(0), + last_sync_sequence_number_(0), + last_tl0_sequence_number_(0) {} + +bool TemporalLayersChecker::CheckAndUpdateBufferState( + BufferState* state, + bool* need_sync, + bool frame_is_keyframe, + uint8_t temporal_layer, + Vp8FrameConfig::BufferFlags flags, + uint32_t sequence_number, + uint32_t* lowest_sequence_referenced) { + if (flags & Vp8FrameConfig::BufferFlags::kReference) { + if (state->temporal_layer > 0 && !state->is_keyframe) { + *need_sync = false; + } + if (!state->is_keyframe && !frame_is_keyframe && + state->sequence_number < *lowest_sequence_referenced) { + *lowest_sequence_referenced = state->sequence_number; + } + if (!frame_is_keyframe && !state->is_keyframe && + state->temporal_layer > temporal_layer) { + RTC_LOG(LS_ERROR) << "Frame is referencing higher temporal layer."; + return false; + } + } + if ((flags & Vp8FrameConfig::BufferFlags::kUpdate)) { + state->temporal_layer = temporal_layer; + state->sequence_number = sequence_number; + state->is_keyframe = frame_is_keyframe; + } + if (frame_is_keyframe) + state->is_keyframe = true; + return true; +} + +bool TemporalLayersChecker::CheckTemporalConfig( + bool frame_is_keyframe, + const Vp8FrameConfig& frame_config) { + if (frame_config.drop_frame || + frame_config.packetizer_temporal_idx == kNoTemporalIdx) { + return true; + } + ++sequence_number_; + if (frame_config.packetizer_temporal_idx >= num_temporal_layers_ || + (frame_config.packetizer_temporal_idx == kNoTemporalIdx && + num_temporal_layers_ > 1)) { + RTC_LOG(LS_ERROR) << "Incorrect temporal layer set for frame: " + << frame_config.packetizer_temporal_idx + << " num_temporal_layers: " << num_temporal_layers_; + return false; + } + + uint32_t lowest_sequence_referenced = sequence_number_; + bool need_sync = frame_config.packetizer_temporal_idx > 0 && + frame_config.packetizer_temporal_idx != kNoTemporalIdx; + + if (!CheckAndUpdateBufferState( + &last_, &need_sync, frame_is_keyframe, + frame_config.packetizer_temporal_idx, frame_config.last_buffer_flags, + sequence_number_, &lowest_sequence_referenced)) { + RTC_LOG(LS_ERROR) << "Error in the Last buffer"; + return false; + } + if (!CheckAndUpdateBufferState(&golden_, &need_sync, frame_is_keyframe, + frame_config.packetizer_temporal_idx, + frame_config.golden_buffer_flags, + sequence_number_, + &lowest_sequence_referenced)) { + RTC_LOG(LS_ERROR) << "Error in the Golden buffer"; + return false; + } + if (!CheckAndUpdateBufferState( + &arf_, &need_sync, frame_is_keyframe, + frame_config.packetizer_temporal_idx, frame_config.arf_buffer_flags, + sequence_number_, &lowest_sequence_referenced)) { + RTC_LOG(LS_ERROR) << "Error in the Arf buffer"; + return false; + } + + if (lowest_sequence_referenced < last_sync_sequence_number_ && + !frame_is_keyframe) { + RTC_LOG(LS_ERROR) << "Reference past the last sync frame. Referenced " + << lowest_sequence_referenced << ", but sync was at " + << last_sync_sequence_number_; + return false; + } + + if (frame_config.packetizer_temporal_idx == 0) { + last_tl0_sequence_number_ = sequence_number_; + } + + if (frame_is_keyframe) { + last_sync_sequence_number_ = sequence_number_; + } + + if (need_sync) { + last_sync_sequence_number_ = last_tl0_sequence_number_; + } + + // Ignore sync flag on key-frames as it really doesn't matter. + if (need_sync != frame_config.layer_sync && !frame_is_keyframe) { + RTC_LOG(LS_ERROR) << "Sync bit is set incorrectly on a frame. Expected: " + << need_sync << " Actual: " << frame_config.layer_sync; + return false; + } + return true; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc new file mode 100644 index 0000000000..c5a8b659c4 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc @@ -0,0 +1,913 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <stdio.h> + +#include <memory> + +#include "api/test/create_frame_generator.h" +#include "api/test/frame_generator_interface.h" +#include "api/test/mock_video_decoder.h" +#include "api/test/mock_video_encoder.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/vp8_temporal_layers.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "common_video/test/utilities.h" +#include "modules/video_coding/codecs/interface/mock_libvpx_interface.h" +#include "modules/video_coding/codecs/test/video_codec_unittest.h" +#include "modules/video_coding/codecs/vp8/include/vp8.h" +#include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h" +#include "modules/video_coding/utility/vp8_header_parser.h" +#include "rtc_base/time_utils.h" +#include "test/field_trial.h" +#include "test/mappable_native_buffer.h" +#include "test/video_codec_settings.h" + +namespace webrtc { + +using ::testing::_; +using ::testing::AllOf; +using ::testing::ElementsAre; +using ::testing::ElementsAreArray; +using ::testing::Field; +using ::testing::Invoke; +using ::testing::NiceMock; +using ::testing::Return; +using EncoderInfo = webrtc::VideoEncoder::EncoderInfo; +using FramerateFractions = + absl::InlinedVector<uint8_t, webrtc::kMaxTemporalStreams>; + +namespace { +constexpr uint32_t kLegacyScreenshareTl0BitrateKbps = 200; +constexpr uint32_t kLegacyScreenshareTl1BitrateKbps = 1000; +constexpr uint32_t kInitialTimestampRtp = 123; +constexpr int64_t kTestNtpTimeMs = 456; +constexpr int64_t kInitialTimestampMs = 789; +constexpr int kNumCores = 1; +constexpr size_t kMaxPayloadSize = 1440; +constexpr int kWidth = 172; +constexpr int kHeight = 144; +constexpr float kFramerateFps = 30; + +const VideoEncoder::Capabilities kCapabilities(false); +const VideoEncoder::Settings kSettings(kCapabilities, + kNumCores, + kMaxPayloadSize); +} // namespace + +class TestVp8Impl : public VideoCodecUnitTest { + protected: + std::unique_ptr<VideoEncoder> CreateEncoder() override { + return VP8Encoder::Create(); + } + + std::unique_ptr<VideoDecoder> CreateDecoder() override { + return VP8Decoder::Create(); + } + + void ModifyCodecSettings(VideoCodec* codec_settings) override { + webrtc::test::CodecSettings(kVideoCodecVP8, codec_settings); + codec_settings->width = kWidth; + codec_settings->height = kHeight; + codec_settings->SetVideoEncoderComplexity( + VideoCodecComplexity::kComplexityNormal); + } + + void EncodeAndWaitForFrame(const VideoFrame& input_frame, + EncodedImage* encoded_frame, + CodecSpecificInfo* codec_specific_info, + bool keyframe = false) { + std::vector<VideoFrameType> frame_types; + if (keyframe) { + frame_types.emplace_back(VideoFrameType::kVideoFrameKey); + } else { + frame_types.emplace_back(VideoFrameType::kVideoFrameDelta); + } + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->Encode(input_frame, &frame_types)); + ASSERT_TRUE(WaitForEncodedFrame(encoded_frame, codec_specific_info)); + VerifyQpParser(*encoded_frame); + EXPECT_EQ(kVideoCodecVP8, codec_specific_info->codecType); + EXPECT_EQ(0, encoded_frame->SpatialIndex()); + } + + void EncodeAndExpectFrameWith(const VideoFrame& input_frame, + uint8_t temporal_idx, + bool keyframe = false) { + EncodedImage encoded_frame; + CodecSpecificInfo codec_specific_info; + EncodeAndWaitForFrame(input_frame, &encoded_frame, &codec_specific_info, + keyframe); + EXPECT_EQ(temporal_idx, codec_specific_info.codecSpecific.VP8.temporalIdx); + } + + void VerifyQpParser(const EncodedImage& encoded_frame) const { + int qp; + EXPECT_GT(encoded_frame.size(), 0u); + ASSERT_TRUE(vp8::GetQp(encoded_frame.data(), encoded_frame.size(), &qp)); + EXPECT_EQ(encoded_frame.qp_, qp) << "Encoder QP != parsed bitstream QP."; + } +}; + +TEST_F(TestVp8Impl, ErrorResilienceDisabledForNoTemporalLayers) { + codec_settings_.simulcastStream[0].numberOfTemporalLayers = 1; + + auto* const vpx = new NiceMock<MockLibvpxInterface>(); + LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)), + VP8Encoder::Settings()); + EXPECT_CALL(*vpx, + codec_enc_init( + _, _, Field(&vpx_codec_enc_cfg_t::g_error_resilient, 0), _)); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder.InitEncode(&codec_settings_, kSettings)); +} + +TEST_F(TestVp8Impl, DefaultErrorResilienceEnabledForTemporalLayers) { + codec_settings_.simulcastStream[0].numberOfTemporalLayers = 2; + codec_settings_.VP8()->numberOfTemporalLayers = 2; + + auto* const vpx = new NiceMock<MockLibvpxInterface>(); + LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)), + VP8Encoder::Settings()); + EXPECT_CALL(*vpx, + codec_enc_init(_, _, + Field(&vpx_codec_enc_cfg_t::g_error_resilient, + VPX_ERROR_RESILIENT_DEFAULT), + _)); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder.InitEncode(&codec_settings_, kSettings)); +} + +TEST_F(TestVp8Impl, + PartitionErrorResilienceEnabledForTemporalLayersWithFieldTrial) { + test::ScopedFieldTrials field_trials( + "WebRTC-VP8-ForcePartitionResilience/Enabled/"); + codec_settings_.simulcastStream[0].numberOfTemporalLayers = 2; + codec_settings_.VP8()->numberOfTemporalLayers = 2; + + auto* const vpx = new NiceMock<MockLibvpxInterface>(); + LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)), + VP8Encoder::Settings()); + EXPECT_CALL(*vpx, + codec_enc_init(_, _, + Field(&vpx_codec_enc_cfg_t::g_error_resilient, + VPX_ERROR_RESILIENT_PARTITIONS), + _)); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder.InitEncode(&codec_settings_, kSettings)); +} + +TEST_F(TestVp8Impl, SetRates) { + codec_settings_.SetFrameDropEnabled(true); + auto* const vpx = new NiceMock<MockLibvpxInterface>(); + LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)), + VP8Encoder::Settings()); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder.InitEncode(&codec_settings_, + VideoEncoder::Settings(kCapabilities, 1, 1000))); + + const uint32_t kBitrateBps = 300000; + VideoBitrateAllocation bitrate_allocation; + bitrate_allocation.SetBitrate(0, 0, kBitrateBps); + EXPECT_CALL( + *vpx, + codec_enc_config_set( + _, AllOf(Field(&vpx_codec_enc_cfg_t::rc_target_bitrate, + kBitrateBps / 1000), + Field(&vpx_codec_enc_cfg_t::rc_undershoot_pct, 100u), + Field(&vpx_codec_enc_cfg_t::rc_overshoot_pct, 15u), + Field(&vpx_codec_enc_cfg_t::rc_buf_sz, 1000u), + Field(&vpx_codec_enc_cfg_t::rc_buf_optimal_sz, 600u), + Field(&vpx_codec_enc_cfg_t::rc_dropframe_thresh, 30u)))) + .WillOnce(Return(VPX_CODEC_OK)); + encoder.SetRates(VideoEncoder::RateControlParameters( + bitrate_allocation, static_cast<double>(codec_settings_.maxFramerate))); +} + +TEST_F(TestVp8Impl, EncodeFrameAndRelease) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kSettings)); + + EncodedImage encoded_frame; + CodecSpecificInfo codec_specific_info; + EncodeAndWaitForFrame(NextInputFrame(), &encoded_frame, &codec_specific_info); + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_UNINITIALIZED, + encoder_->Encode(NextInputFrame(), nullptr)); +} + +TEST_F(TestVp8Impl, EncodeNv12FrameSimulcast) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kSettings)); + + EncodedImage encoded_frame; + CodecSpecificInfo codec_specific_info; + input_frame_generator_ = test::CreateSquareFrameGenerator( + kWidth, kHeight, test::FrameGeneratorInterface::OutputType::kNV12, + absl::nullopt); + EncodeAndWaitForFrame(NextInputFrame(), &encoded_frame, &codec_specific_info); + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_UNINITIALIZED, + encoder_->Encode(NextInputFrame(), nullptr)); +} + +TEST_F(TestVp8Impl, EncodeI420FrameAfterNv12Frame) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kSettings)); + + EncodedImage encoded_frame; + CodecSpecificInfo codec_specific_info; + input_frame_generator_ = test::CreateSquareFrameGenerator( + kWidth, kHeight, test::FrameGeneratorInterface::OutputType::kNV12, + absl::nullopt); + EncodeAndWaitForFrame(NextInputFrame(), &encoded_frame, &codec_specific_info); + input_frame_generator_ = test::CreateSquareFrameGenerator( + kWidth, kHeight, test::FrameGeneratorInterface::OutputType::kI420, + absl::nullopt); + EncodeAndWaitForFrame(NextInputFrame(), &encoded_frame, &codec_specific_info); + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_UNINITIALIZED, + encoder_->Encode(NextInputFrame(), nullptr)); +} + +TEST_F(TestVp8Impl, Configure) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release()); + EXPECT_TRUE(decoder_->Configure({})); +} + +TEST_F(TestVp8Impl, OnEncodedImageReportsInfo) { + VideoFrame input_frame = NextInputFrame(); + input_frame.set_timestamp(kInitialTimestampRtp); + input_frame.set_timestamp_us(kInitialTimestampMs * + rtc::kNumMicrosecsPerMillisec); + EncodedImage encoded_frame; + CodecSpecificInfo codec_specific_info; + EncodeAndWaitForFrame(input_frame, &encoded_frame, &codec_specific_info); + + EXPECT_EQ(kInitialTimestampRtp, encoded_frame.Timestamp()); + EXPECT_EQ(kWidth, static_cast<int>(encoded_frame._encodedWidth)); + EXPECT_EQ(kHeight, static_cast<int>(encoded_frame._encodedHeight)); +} + +TEST_F(TestVp8Impl, + EncoderFillsResolutionInCodecAgnosticSectionOfCodecSpecificInfo) { + EncodedImage encoded_frame; + CodecSpecificInfo codec_specific_info; + EncodeAndWaitForFrame(NextInputFrame(), &encoded_frame, &codec_specific_info); + + ASSERT_TRUE(codec_specific_info.template_structure); + EXPECT_THAT(codec_specific_info.template_structure->resolutions, + ElementsAre(RenderResolution(kWidth, kHeight))); +} + +TEST_F(TestVp8Impl, DecodedQpEqualsEncodedQp) { + VideoFrame input_frame = NextInputFrame(); + EncodedImage encoded_frame; + CodecSpecificInfo codec_specific_info; + EncodeAndWaitForFrame(input_frame, &encoded_frame, &codec_specific_info); + + // First frame should be a key frame. + encoded_frame._frameType = VideoFrameType::kVideoFrameKey; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, false, -1)); + std::unique_ptr<VideoFrame> decoded_frame; + absl::optional<uint8_t> decoded_qp; + ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp)); + ASSERT_TRUE(decoded_frame); + ASSERT_TRUE(decoded_qp); + EXPECT_GT(I420PSNR(&input_frame, decoded_frame.get()), 36); + EXPECT_EQ(encoded_frame.qp_, *decoded_qp); +} + +TEST_F(TestVp8Impl, ChecksSimulcastSettings) { + codec_settings_.numberOfSimulcastStreams = 2; + // Resolutions are not in ascending order, temporal layers do not match. + codec_settings_.simulcastStream[0] = {.width = kWidth, + .height = kHeight, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 2, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + codec_settings_.simulcastStream[1] = {.width = kWidth / 2, + .height = kHeight / 2, + .maxFramerate = 30, + .numberOfTemporalLayers = 3, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED, + encoder_->InitEncode(&codec_settings_, kSettings)); + codec_settings_.numberOfSimulcastStreams = 3; + // Resolutions are not in ascending order. + codec_settings_.simulcastStream[0] = {.width = kWidth / 2, + .height = kHeight / 2, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + codec_settings_.simulcastStream[1] = {.width = kWidth / 2 - 1, + .height = kHeight / 2 - 1, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + codec_settings_.simulcastStream[2] = {.width = kWidth, + .height = kHeight, + .maxFramerate = 30, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED, + encoder_->InitEncode(&codec_settings_, kSettings)); + // Resolutions are not in ascending order. + codec_settings_.simulcastStream[0] = {.width = kWidth, + .height = kHeight, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + codec_settings_.simulcastStream[1] = {.width = kWidth, + .height = kHeight, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + codec_settings_.simulcastStream[2] = {.width = kWidth - 1, + .height = kHeight - 1, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED, + encoder_->InitEncode(&codec_settings_, kSettings)); + // Temporal layers do not match. + codec_settings_.simulcastStream[0] = {.width = kWidth / 4, + .height = kHeight / 4, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + codec_settings_.simulcastStream[1] = {.width = kWidth / 2, + .height = kHeight / 2, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 2, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + codec_settings_.simulcastStream[2] = {.width = kWidth, + .height = kHeight, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 3, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED, + encoder_->InitEncode(&codec_settings_, kSettings)); + // Resolutions do not match codec config. + codec_settings_.simulcastStream[0] = {.width = kWidth / 4 + 1, + .height = kHeight / 4 + 1, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + codec_settings_.simulcastStream[1] = {.width = kWidth / 2 + 2, + .height = kHeight / 2 + 2, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + codec_settings_.simulcastStream[2] = {.width = kWidth + 4, + .height = kHeight + 4, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED, + encoder_->InitEncode(&codec_settings_, kSettings)); + // Everything fine: scaling by 2, top resolution matches video, temporal + // settings are the same for all layers. + codec_settings_.simulcastStream[0] = {.width = kWidth / 4, + .height = kHeight / 4, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + codec_settings_.simulcastStream[1] = {.width = kWidth / 2, + .height = kHeight / 2, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + codec_settings_.simulcastStream[2] = {.width = kWidth, + .height = kHeight, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kSettings)); + // Everything fine: custom scaling, top resolution matches video, temporal + // settings are the same for all layers. + codec_settings_.simulcastStream[0] = {.width = kWidth / 4, + .height = kHeight / 4, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + codec_settings_.simulcastStream[1] = {.width = kWidth, + .height = kHeight, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + codec_settings_.simulcastStream[2] = {.width = kWidth, + .height = kHeight, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80}; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kSettings)); +} + +#if defined(WEBRTC_ANDROID) +#define MAYBE_AlignedStrideEncodeDecode DISABLED_AlignedStrideEncodeDecode +#else +#define MAYBE_AlignedStrideEncodeDecode AlignedStrideEncodeDecode +#endif +TEST_F(TestVp8Impl, MAYBE_AlignedStrideEncodeDecode) { + VideoFrame input_frame = NextInputFrame(); + input_frame.set_timestamp(kInitialTimestampRtp); + input_frame.set_timestamp_us(kInitialTimestampMs * + rtc::kNumMicrosecsPerMillisec); + EncodedImage encoded_frame; + CodecSpecificInfo codec_specific_info; + EncodeAndWaitForFrame(input_frame, &encoded_frame, &codec_specific_info); + + // First frame should be a key frame. + encoded_frame._frameType = VideoFrameType::kVideoFrameKey; + encoded_frame.ntp_time_ms_ = kTestNtpTimeMs; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Decode(encoded_frame, false, -1)); + + std::unique_ptr<VideoFrame> decoded_frame; + absl::optional<uint8_t> decoded_qp; + ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp)); + ASSERT_TRUE(decoded_frame); + // Compute PSNR on all planes (faster than SSIM). + EXPECT_GT(I420PSNR(&input_frame, decoded_frame.get()), 36); + EXPECT_EQ(kInitialTimestampRtp, decoded_frame->timestamp()); +} + +TEST_F(TestVp8Impl, EncoderWith2TemporalLayers) { + codec_settings_.VP8()->numberOfTemporalLayers = 2; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kSettings)); + + // Temporal layer 0. + EncodedImage encoded_frame; + CodecSpecificInfo codec_specific_info; + EncodeAndWaitForFrame(NextInputFrame(), &encoded_frame, &codec_specific_info); + + EXPECT_EQ(0, codec_specific_info.codecSpecific.VP8.temporalIdx); + // Temporal layer 1. + EncodeAndExpectFrameWith(NextInputFrame(), 1); + // Temporal layer 0. + EncodeAndExpectFrameWith(NextInputFrame(), 0); + // Temporal layer 1. + EncodeAndExpectFrameWith(NextInputFrame(), 1); +} + +TEST_F(TestVp8Impl, ScalingDisabledIfAutomaticResizeOff) { + codec_settings_.VP8()->automaticResizeOn = false; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kSettings)); + + VideoEncoder::ScalingSettings settings = + encoder_->GetEncoderInfo().scaling_settings; + EXPECT_FALSE(settings.thresholds.has_value()); +} + +TEST_F(TestVp8Impl, ScalingEnabledIfAutomaticResizeOn) { + codec_settings_.SetFrameDropEnabled(true); + codec_settings_.VP8()->automaticResizeOn = true; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kSettings)); + + VideoEncoder::ScalingSettings settings = + encoder_->GetEncoderInfo().scaling_settings; + EXPECT_TRUE(settings.thresholds.has_value()); + EXPECT_EQ(kDefaultMinPixelsPerFrame, settings.min_pixels_per_frame); +} + +TEST_F(TestVp8Impl, DontDropKeyframes) { + // Set very high resolution to trigger overuse more easily. + const int kScreenWidth = 1920; + const int kScreenHeight = 1080; + + codec_settings_.width = kScreenWidth; + codec_settings_.height = kScreenHeight; + + // Screensharing has the internal frame dropper off, and instead per frame + // asks ScreenshareLayers to decide if it should be dropped or not. + codec_settings_.SetFrameDropEnabled(false); + codec_settings_.mode = VideoCodecMode::kScreensharing; + // ScreenshareLayers triggers on 2 temporal layers and 1000kbps max bitrate. + codec_settings_.VP8()->numberOfTemporalLayers = 2; + codec_settings_.maxBitrate = 1000; + + // Reset the frame generator with large number of squares, leading to lots of + // details and high probability of overshoot. + input_frame_generator_ = test::CreateSquareFrameGenerator( + codec_settings_.width, codec_settings_.height, + test::FrameGeneratorInterface::OutputType::kI420, + /* num_squares = */ absl::optional<int>(300)); + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kSettings)); + + VideoBitrateAllocation bitrate_allocation; + // Bitrate only enough for TL0. + bitrate_allocation.SetBitrate(0, 0, 200000); + encoder_->SetRates( + VideoEncoder::RateControlParameters(bitrate_allocation, 5.0)); + + EncodedImage encoded_frame; + CodecSpecificInfo codec_specific_info; + EncodeAndWaitForFrame(NextInputFrame(), &encoded_frame, &codec_specific_info, + true); + EncodeAndExpectFrameWith(NextInputFrame(), 0, true); + EncodeAndExpectFrameWith(NextInputFrame(), 0, true); + EncodeAndExpectFrameWith(NextInputFrame(), 0, true); +} + +TEST_F(TestVp8Impl, KeepsTimestampOnReencode) { + auto* const vpx = new NiceMock<MockLibvpxInterface>(); + LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)), + VP8Encoder::Settings()); + + // Settings needed to trigger ScreenshareLayers usage, which is required for + // overshoot-drop-reencode logic. + codec_settings_.maxBitrate = 1000; + codec_settings_.mode = VideoCodecMode::kScreensharing; + codec_settings_.VP8()->numberOfTemporalLayers = 2; + codec_settings_.legacy_conference_mode = true; + + EXPECT_CALL(*vpx, img_wrap(_, _, _, _, _, _)) + .WillOnce(Invoke([](vpx_image_t* img, vpx_img_fmt_t fmt, unsigned int d_w, + unsigned int d_h, unsigned int stride_align, + unsigned char* img_data) { + img->fmt = fmt; + img->d_w = d_w; + img->d_h = d_h; + img->img_data = img_data; + return img; + })); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder.InitEncode(&codec_settings_, + VideoEncoder::Settings(kCapabilities, 1, 1000))); + MockEncodedImageCallback callback; + encoder.RegisterEncodeCompleteCallback(&callback); + + // Simulate overshoot drop, re-encode: encode function will be called twice + // with the same parameters. codec_get_cx_data() will by default return no + // image data and be interpreted as drop. + EXPECT_CALL(*vpx, codec_encode(_, _, /* pts = */ 0, _, _, _)) + .Times(2) + .WillRepeatedly(Return(vpx_codec_err_t::VPX_CODEC_OK)); + + auto delta_frame = + std::vector<VideoFrameType>{VideoFrameType::kVideoFrameDelta}; + encoder.Encode(NextInputFrame(), &delta_frame); +} + +TEST(LibvpxVp8EncoderTest, GetEncoderInfoReturnsStaticInformation) { + auto* const vpx = new NiceMock<MockLibvpxInterface>(); + LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)), + VP8Encoder::Settings()); + + const auto info = encoder.GetEncoderInfo(); + + EXPECT_FALSE(info.supports_native_handle); + EXPECT_FALSE(info.is_hardware_accelerated); + EXPECT_TRUE(info.supports_simulcast); + EXPECT_EQ(info.implementation_name, "libvpx"); + EXPECT_EQ(info.requested_resolution_alignment, 1u); + EXPECT_THAT(info.preferred_pixel_formats, + testing::UnorderedElementsAre(VideoFrameBuffer::Type::kNV12, + VideoFrameBuffer::Type::kI420)); +} + +TEST(LibvpxVp8EncoderTest, RequestedResolutionAlignmentFromFieldTrial) { + test::ScopedFieldTrials field_trials( + "WebRTC-VP8-GetEncoderInfoOverride/" + "requested_resolution_alignment:10/"); + + auto* const vpx = new NiceMock<MockLibvpxInterface>(); + LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)), + VP8Encoder::Settings()); + + EXPECT_EQ(encoder.GetEncoderInfo().requested_resolution_alignment, 10u); + EXPECT_FALSE( + encoder.GetEncoderInfo().apply_alignment_to_all_simulcast_layers); + EXPECT_TRUE(encoder.GetEncoderInfo().resolution_bitrate_limits.empty()); +} + +TEST(LibvpxVp8EncoderTest, ResolutionBitrateLimitsFromFieldTrial) { + test::ScopedFieldTrials field_trials( + "WebRTC-VP8-GetEncoderInfoOverride/" + "frame_size_pixels:123|456|789," + "min_start_bitrate_bps:11000|22000|33000," + "min_bitrate_bps:44000|55000|66000," + "max_bitrate_bps:77000|88000|99000/"); + + auto* const vpx = new NiceMock<MockLibvpxInterface>(); + LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)), + VP8Encoder::Settings()); + + EXPECT_THAT( + encoder.GetEncoderInfo().resolution_bitrate_limits, + ::testing::ElementsAre( + VideoEncoder::ResolutionBitrateLimits{123, 11000, 44000, 77000}, + VideoEncoder::ResolutionBitrateLimits{456, 22000, 55000, 88000}, + VideoEncoder::ResolutionBitrateLimits{789, 33000, 66000, 99000})); +} + +TEST(LibvpxVp8EncoderTest, + GetEncoderInfoReturnsEmptyResolutionBitrateLimitsByDefault) { + auto* const vpx = new NiceMock<MockLibvpxInterface>(); + LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)), + VP8Encoder::Settings()); + + const auto info = encoder.GetEncoderInfo(); + + EXPECT_TRUE(info.resolution_bitrate_limits.empty()); +} + +TEST(LibvpxVp8EncoderTest, + GetEncoderInfoReturnsResolutionBitrateLimitsAsConfigured) { + std::vector<VideoEncoder::ResolutionBitrateLimits> resolution_bitrate_limits = + {VideoEncoder::ResolutionBitrateLimits(/*frame_size_pixels=*/640 * 360, + /*min_start_bitrate_bps=*/300, + /*min_bitrate_bps=*/100, + /*max_bitrate_bps=*/1000), + VideoEncoder::ResolutionBitrateLimits(320 * 180, 100, 30, 500)}; + VP8Encoder::Settings settings; + settings.resolution_bitrate_limits = resolution_bitrate_limits; + + auto* const vpx = new NiceMock<MockLibvpxInterface>(); + LibvpxVp8Encoder encoder((std::unique_ptr<LibvpxInterface>(vpx)), + std::move(settings)); + + const auto info = encoder.GetEncoderInfo(); + + EXPECT_EQ(info.resolution_bitrate_limits, resolution_bitrate_limits); +} + +TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationNoLayers) { + FramerateFractions expected_fps_allocation[kMaxSpatialLayers] = { + FramerateFractions(1, EncoderInfo::kMaxFramerateFraction)}; + + EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation, + ::testing::ElementsAreArray(expected_fps_allocation)); +} + +TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationTwoTemporalLayers) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + codec_settings_.numberOfSimulcastStreams = 1; + codec_settings_.simulcastStream[0].active = true; + codec_settings_.simulcastStream[0].targetBitrate = 100; + codec_settings_.simulcastStream[0].maxBitrate = 100; + codec_settings_.simulcastStream[0].numberOfTemporalLayers = 2; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kSettings)); + + FramerateFractions expected_fps_allocation[kMaxSpatialLayers]; + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2); + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction); + + EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation, + ::testing::ElementsAreArray(expected_fps_allocation)); +} + +TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationThreeTemporalLayers) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + codec_settings_.numberOfSimulcastStreams = 1; + codec_settings_.simulcastStream[0].active = true; + codec_settings_.simulcastStream[0].targetBitrate = 100; + codec_settings_.simulcastStream[0].maxBitrate = 100; + codec_settings_.simulcastStream[0].numberOfTemporalLayers = 3; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kSettings)); + + FramerateFractions expected_fps_allocation[kMaxSpatialLayers]; + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 4); + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2); + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction); + + EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation, + ::testing::ElementsAreArray(expected_fps_allocation)); +} + +TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationScreenshareLayers) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + codec_settings_.numberOfSimulcastStreams = 1; + codec_settings_.mode = VideoCodecMode::kScreensharing; + codec_settings_.simulcastStream[0].active = true; + codec_settings_.simulcastStream[0].minBitrate = 30; + codec_settings_.simulcastStream[0].targetBitrate = + kLegacyScreenshareTl0BitrateKbps; + codec_settings_.simulcastStream[0].maxBitrate = + kLegacyScreenshareTl1BitrateKbps; + codec_settings_.simulcastStream[0].numberOfTemporalLayers = 2; + codec_settings_.legacy_conference_mode = true; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kSettings)); + + // Expect empty vector, since this mode doesn't have a fixed framerate. + FramerateFractions expected_fps_allocation[kMaxSpatialLayers]; + EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation, + ::testing::ElementsAreArray(expected_fps_allocation)); +} + +TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationSimulcastVideo) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + + // Set up three simulcast streams with three temporal layers each. + codec_settings_.numberOfSimulcastStreams = 3; + for (int i = 0; i < codec_settings_.numberOfSimulcastStreams; ++i) { + codec_settings_.simulcastStream[i].active = true; + codec_settings_.simulcastStream[i].minBitrate = 30; + codec_settings_.simulcastStream[i].targetBitrate = 30; + codec_settings_.simulcastStream[i].maxBitrate = 30; + codec_settings_.simulcastStream[i].numberOfTemporalLayers = 3; + codec_settings_.simulcastStream[i].width = + codec_settings_.width >> + (codec_settings_.numberOfSimulcastStreams - i - 1); + codec_settings_.simulcastStream[i].height = + codec_settings_.height >> + (codec_settings_.numberOfSimulcastStreams - i - 1); + } + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kSettings)); + + FramerateFractions expected_fps_allocation[kMaxSpatialLayers]; + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 4); + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2); + expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction); + expected_fps_allocation[1] = expected_fps_allocation[0]; + expected_fps_allocation[2] = expected_fps_allocation[0]; + EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation, + ::testing::ElementsAreArray(expected_fps_allocation)); + + // Release encoder and re-init without temporal layers. + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + + // Sanity check fps allocation when not inited. + FramerateFractions default_fps_fraction[kMaxSpatialLayers]; + default_fps_fraction[0].push_back(EncoderInfo::kMaxFramerateFraction); + EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation, + ::testing::ElementsAreArray(default_fps_fraction)); + + for (int i = 0; i < codec_settings_.numberOfSimulcastStreams; ++i) { + codec_settings_.simulcastStream[i].numberOfTemporalLayers = 1; + } + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kSettings)); + + for (size_t i = 0; i < 3; ++i) { + expected_fps_allocation[i].clear(); + expected_fps_allocation[i].push_back(EncoderInfo::kMaxFramerateFraction); + } + EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation, + ::testing::ElementsAreArray(expected_fps_allocation)); +} + +class TestVp8ImplForPixelFormat + : public TestVp8Impl, + public ::testing::WithParamInterface<VideoFrameBuffer::Type> { + public: + TestVp8ImplForPixelFormat() : TestVp8Impl(), mappable_type_(GetParam()) {} + + protected: + VideoFrameBuffer::Type mappable_type_; +}; + +TEST_P(TestVp8ImplForPixelFormat, EncodeNativeFrameSimulcast) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); + + // Configure simulcast. + codec_settings_.numberOfSimulcastStreams = 3; + codec_settings_.simulcastStream[0] = {.width = kWidth / 4, + .height = kHeight / 4, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80, + .active = true}; + codec_settings_.simulcastStream[1] = {.width = kWidth / 2, + .height = kHeight / 2, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80, + .active = true}; + codec_settings_.simulcastStream[2] = {.width = kWidth, + .height = kHeight, + .maxFramerate = kFramerateFps, + .numberOfTemporalLayers = 1, + .maxBitrate = 4000, + .targetBitrate = 3000, + .minBitrate = 2000, + .qpMax = 80, + .active = true}; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + encoder_->InitEncode(&codec_settings_, kSettings)); + + // Create a zero-conversion NV12 frame (calling ToI420 on it crashes). + VideoFrame input_frame = + test::CreateMappableNativeFrame(1, mappable_type_, kWidth, kHeight); + + EncodedImage encoded_frame; + CodecSpecificInfo codec_specific_info; + EncodeAndWaitForFrame(input_frame, &encoded_frame, &codec_specific_info); + + // After encoding, we expect one mapping per simulcast layer. + rtc::scoped_refptr<test::MappableNativeBuffer> mappable_buffer = + test::GetMappableNativeBufferFromVideoFrame(input_frame); + std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_buffers = + mappable_buffer->GetMappedFramedBuffers(); + ASSERT_EQ(mapped_buffers.size(), 3u); + EXPECT_EQ(mapped_buffers[0]->type(), mappable_type_); + EXPECT_EQ(mapped_buffers[0]->width(), kWidth); + EXPECT_EQ(mapped_buffers[0]->height(), kHeight); + EXPECT_EQ(mapped_buffers[1]->type(), mappable_type_); + EXPECT_EQ(mapped_buffers[1]->width(), kWidth / 2); + EXPECT_EQ(mapped_buffers[1]->height(), kHeight / 2); + EXPECT_EQ(mapped_buffers[2]->type(), mappable_type_); + EXPECT_EQ(mapped_buffers[2]->width(), kWidth / 4); + EXPECT_EQ(mapped_buffers[2]->height(), kHeight / 4); + EXPECT_FALSE(mappable_buffer->DidConvertToI420()); + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); +} + +INSTANTIATE_TEST_SUITE_P(All, + TestVp8ImplForPixelFormat, + ::testing::Values(VideoFrameBuffer::Type::kI420, + VideoFrameBuffer::Type::kNV12)); + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/vp8_scalability.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/vp8_scalability.cc new file mode 100644 index 0000000000..9c7495ddf7 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/vp8_scalability.cc @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/vp8/vp8_scalability.h" + +namespace webrtc { + +bool VP8SupportsScalabilityMode(ScalabilityMode scalability_mode) { + for (const auto& entry : kVP8SupportedScalabilityModes) { + if (entry == scalability_mode) { + return true; + } + } + return false; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/vp8_scalability.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/vp8_scalability.h new file mode 100644 index 0000000000..923f159118 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/vp8_scalability.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_VP8_VP8_SCALABILITY_H_ +#define MODULES_VIDEO_CODING_CODECS_VP8_VP8_SCALABILITY_H_ + +#include "api/video_codecs/scalability_mode.h" + +namespace webrtc { + +inline constexpr ScalabilityMode kVP8SupportedScalabilityModes[] = { + ScalabilityMode::kL1T1, ScalabilityMode::kL1T2, ScalabilityMode::kL1T3}; +bool VP8SupportsScalabilityMode(ScalabilityMode scalability_mode); + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_VP8_VP8_SCALABILITY_H_ |