diff options
Diffstat (limited to 'third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc')
| -rw-r--r-- | third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc | 884 |
1 files changed, 884 insertions, 0 deletions
diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc new file mode 100644 index 0000000000..94860da1b6 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -0,0 +1,884 @@ +/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/vp8/default_temporal_layers.h" + +#include <stdlib.h> + +#include <algorithm> +#include <array> +#include <memory> +#include <set> +#include <utility> +#include <vector> + +#include "modules/video_coding/include/video_codec_interface.h" +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { +DefaultTemporalLayers::PendingFrame::PendingFrame() = default; +DefaultTemporalLayers::PendingFrame::PendingFrame( + uint32_t timestamp, + bool expired, + uint8_t updated_buffers_mask, + const DependencyInfo& dependency_info) + : timestamp(timestamp), + expired(expired), + updated_buffer_mask(updated_buffers_mask), + dependency_info(dependency_info) {} + +namespace { +using BufferFlags = Vp8FrameConfig::BufferFlags; +using FreezeEntropy = Vp8FrameConfig::FreezeEntropy; +using Vp8BufferReference = Vp8FrameConfig::Vp8BufferReference; + +constexpr BufferFlags kNone = BufferFlags::kNone; +constexpr BufferFlags kReference = BufferFlags::kReference; +constexpr BufferFlags kUpdate = BufferFlags::kUpdate; +constexpr BufferFlags kReferenceAndUpdate = BufferFlags::kReferenceAndUpdate; +constexpr FreezeEntropy kFreezeEntropy = FreezeEntropy::kFreezeEntropy; + +static constexpr uint8_t kUninitializedPatternIndex = + std::numeric_limits<uint8_t>::max(); +static constexpr std::array<Vp8BufferReference, 3> kAllBuffers = { + {Vp8BufferReference::kLast, Vp8BufferReference::kGolden, + Vp8BufferReference::kAltref}}; + +std::vector<unsigned int> GetTemporalIds(size_t num_layers) { + switch (num_layers) { + case 1: + // Temporal layer structure (single layer): + // 0 0 0 0 ... + return {0}; + case 2: + // Temporal layer structure: + // 1 1 ... + // 0 0 ... + return {0, 1}; + case 3: + // Temporal layer structure: + // 2 2 2 2 ... + // 1 1 ... + // 0 0 ... + return {0, 2, 1, 2}; + case 4: + // Temporal layer structure: + // 3 3 3 3 3 3 3 3 ... + // 2 2 2 2 ... + // 1 1 ... + // 0 0 ... + return {0, 3, 2, 3, 1, 3, 2, 3}; + default: + RTC_DCHECK_NOTREACHED(); + break; + } + RTC_DCHECK_NOTREACHED(); + return {0}; +} + +uint8_t GetUpdatedBuffers(const Vp8FrameConfig& config) { + uint8_t flags = 0; + if (config.last_buffer_flags & BufferFlags::kUpdate) { + flags |= static_cast<uint8_t>(Vp8BufferReference::kLast); + } + if (config.golden_buffer_flags & BufferFlags::kUpdate) { + flags |= static_cast<uint8_t>(Vp8BufferReference::kGolden); + } + if (config.arf_buffer_flags & BufferFlags::kUpdate) { + flags |= static_cast<uint8_t>(Vp8BufferReference::kAltref); + } + return flags; +} + +size_t BufferToIndex(Vp8BufferReference buffer) { + switch (buffer) { + case Vp8FrameConfig::Vp8BufferReference::kLast: + return 0; + case Vp8FrameConfig::Vp8BufferReference::kGolden: + return 1; + case Vp8FrameConfig::Vp8BufferReference::kAltref: + return 2; + case Vp8FrameConfig::Vp8BufferReference::kNone: + RTC_CHECK_NOTREACHED(); + } +} + +} // namespace + +constexpr size_t DefaultTemporalLayers::kNumReferenceBuffers; + +std::vector<DefaultTemporalLayers::DependencyInfo> +DefaultTemporalLayers::GetDependencyInfo(size_t num_layers) { + // For indexing in the patterns described below (which temporal layers they + // belong to), see the diagram above. + // Layer sync is done similarly for all patterns (except single stream) and + // happens every 8 frames: + // TL1 layer syncs by periodically by only referencing TL0 ('last'), but still + // updating 'golden', so it can be used as a reference by future TL1 frames. + // TL2 layer syncs just before TL1 by only depending on TL0 (and not depending + // on TL1's buffer before TL1 has layer synced). + // TODO(pbos): Consider cyclically updating 'arf' (and 'golden' for 1TL) for + // the base layer in 1-3TL instead of 'last' periodically on long intervals, + // so that if scene changes occur (user walks between rooms or rotates webcam) + // the 'arf' (or 'golden' respectively) is not stuck on a no-longer relevant + // keyframe. + + switch (num_layers) { + case 1: + // Always reference and update the same buffer. + return {{"S", {kReferenceAndUpdate, kNone, kNone}}}; + case 2: + // All layers can reference but not update the 'alt' buffer, this means + // that the 'alt' buffer reference is effectively the last keyframe. + // TL0 also references and updates the 'last' buffer. + // TL1 also references 'last' and references and updates 'golden'. + if (!field_trial::IsDisabled("WebRTC-UseShortVP8TL2Pattern")) { + // Shortened 4-frame pattern: + // 1---1 1---1 ... + // / / / / + // 0---0---0---0 ... + return {{"SS", {kReferenceAndUpdate, kNone, kNone}}, + {"-S", {kReference, kUpdate, kNone}}, + {"SR", {kReferenceAndUpdate, kNone, kNone}}, + {"-D", {kReference, kReference, kNone, kFreezeEntropy}}}; + } else { + // "Default" 8-frame pattern: + // 1---1---1---1 1---1---1---1 ... + // / / / / / / / / + // 0---0---0---0---0---0---0---0 ... + return {{"SS", {kReferenceAndUpdate, kNone, kNone}}, + {"-S", {kReference, kUpdate, kNone}}, + {"SR", {kReferenceAndUpdate, kNone, kNone}}, + {"-R", {kReference, kReferenceAndUpdate, kNone}}, + {"SR", {kReferenceAndUpdate, kNone, kNone}}, + {"-R", {kReference, kReferenceAndUpdate, kNone}}, + {"SR", {kReferenceAndUpdate, kNone, kNone}}, + {"-D", {kReference, kReference, kNone, kFreezeEntropy}}}; + } + case 3: + if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) { + // This field trial is intended to check if it is worth using a shorter + // temporal pattern, trading some coding efficiency for less risk of + // dropped frames. + // The coding efficiency will decrease somewhat since the higher layer + // state is more volatile, but it will be offset slightly by updating + // the altref buffer with TL2 frames, instead of just referencing lower + // layers. + // If a frame is dropped in a higher layer, the jitter + // buffer on the receive side won't be able to decode any higher layer + // frame until the next sync frame. So we expect a noticeable decrease + // in frame drops on links with high packet loss. + + // TL0 references and updates the 'last' buffer. + // TL1 references 'last' and references and updates 'golden'. + // TL2 references both 'last' & 'golden' and references and updates + // 'arf'. + // 2-------2 2-------2 2 + // / __/ / __/ / + // / __1 / __1 / + // /___/ /___/ / + // 0---------------0---------------0----- + // 0 1 2 3 4 5 6 7 8 9 ... + return {{"SSS", {kReferenceAndUpdate, kNone, kNone}}, + {"--S", {kReference, kNone, kUpdate}}, + {"-DR", {kReference, kUpdate, kNone}}, + {"--D", {kReference, kReference, kReference, kFreezeEntropy}}}; + } else { + // All layers can reference but not update the 'alt' buffer, this means + // that the 'alt' buffer reference is effectively the last keyframe. + // TL0 also references and updates the 'last' buffer. + // TL1 also references 'last' and references and updates 'golden'. + // TL2 references both 'last' and 'golden' but updates no buffer. + // 2 __2 _____2 __2 2 + // / /____/ / / / + // / 1---------/-----1 / + // /_____/ /_____/ / + // 0---------------0---------------0----- + // 0 1 2 3 4 5 6 7 8 9 ... + return {{"SSS", {kReferenceAndUpdate, kNone, kNone}}, + {"--D", {kReference, kNone, kNone, kFreezeEntropy}}, + {"-SS", {kReference, kUpdate, kNone}}, + {"--D", {kReference, kReference, kNone, kFreezeEntropy}}, + {"SRR", {kReferenceAndUpdate, kNone, kNone}}, + {"--D", {kReference, kReference, kNone, kFreezeEntropy}}, + {"-DS", {kReference, kReferenceAndUpdate, kNone}}, + {"--D", {kReference, kReference, kNone, kFreezeEntropy}}}; + } + case 4: + // TL0 references and updates only the 'last' buffer. + // TL1 references 'last' and updates and references 'golden'. + // TL2 references 'last' and 'golden', and references and updates 'arf'. + // TL3 references all buffers but update none of them. + // TODO(philipel): Set decode target information for this structure. + return {{"----", {kReferenceAndUpdate, kNone, kNone}}, + {"----", {kReference, kNone, kNone, kFreezeEntropy}}, + {"----", {kReference, kNone, kUpdate}}, + {"----", {kReference, kNone, kReference, kFreezeEntropy}}, + {"----", {kReference, kUpdate, kNone}}, + {"----", {kReference, kReference, kReference, kFreezeEntropy}}, + {"----", {kReference, kReference, kReferenceAndUpdate}}, + {"----", {kReference, kReference, kReference, kFreezeEntropy}}, + {"----", {kReferenceAndUpdate, kNone, kNone}}, + {"----", {kReference, kReference, kReference, kFreezeEntropy}}, + {"----", {kReference, kReference, kReferenceAndUpdate}}, + {"----", {kReference, kReference, kReference, kFreezeEntropy}}, + {"----", {kReference, kReferenceAndUpdate, kNone}}, + {"----", {kReference, kReference, kReference, kFreezeEntropy}}, + {"----", {kReference, kReference, kReferenceAndUpdate}}, + {"----", {kReference, kReference, kReference, kFreezeEntropy}}}; + default: + RTC_DCHECK_NOTREACHED(); + break; + } + RTC_DCHECK_NOTREACHED(); + return {{"", {kNone, kNone, kNone}}}; +} + +std::bitset<DefaultTemporalLayers::kNumReferenceBuffers> +DefaultTemporalLayers::DetermineStaticBuffers( + const std::vector<DependencyInfo>& temporal_pattern) { + std::bitset<kNumReferenceBuffers> buffers; + buffers.set(); + for (const DependencyInfo& info : temporal_pattern) { + uint8_t updated_buffers = GetUpdatedBuffers(info.frame_config); + + for (Vp8BufferReference buffer : kAllBuffers) { + if (static_cast<uint8_t>(buffer) & updated_buffers) { + buffers.reset(BufferToIndex(buffer)); + } + } + } + return buffers; +} + +DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers) + : num_layers_(std::max(1, number_of_temporal_layers)), + temporal_ids_(GetTemporalIds(num_layers_)), + temporal_pattern_(GetDependencyInfo(num_layers_)), + is_static_buffer_(DetermineStaticBuffers(temporal_pattern_)), + pattern_idx_(kUninitializedPatternIndex), + new_bitrates_bps_(std::vector<uint32_t>(num_layers_, 0u)) { + RTC_CHECK_GE(kMaxTemporalStreams, number_of_temporal_layers); + RTC_CHECK_GE(number_of_temporal_layers, 0); + RTC_CHECK_LE(number_of_temporal_layers, 4); + // pattern_idx_ wraps around temporal_pattern_.size, this is incorrect if + // temporal_ids_ are ever longer. If this is no longer correct it needs to + // wrap at max(temporal_ids_.size(), temporal_pattern_.size()). + RTC_DCHECK_LE(temporal_ids_.size(), temporal_pattern_.size()); + + RTC_DCHECK( + checker_ = TemporalLayersChecker::CreateTemporalLayersChecker( + Vp8TemporalLayersType::kFixedPattern, number_of_temporal_layers)); + + // Always need to start with a keyframe, so pre-populate all frame counters. + frames_since_buffer_refresh_.fill(0); +} + +DefaultTemporalLayers::~DefaultTemporalLayers() = default; + +void DefaultTemporalLayers::SetQpLimits(size_t stream_index, + int min_qp, + int max_qp) { + RTC_DCHECK_LT(stream_index, StreamCount()); + // Ignore. +} + +size_t DefaultTemporalLayers::StreamCount() const { + return 1; +} + +bool DefaultTemporalLayers::SupportsEncoderFrameDropping( + size_t stream_index) const { + RTC_DCHECK_LT(stream_index, StreamCount()); + // This class allows the encoder drop frames as it sees fit. + return true; +} + +void DefaultTemporalLayers::OnRatesUpdated( + size_t stream_index, + const std::vector<uint32_t>& bitrates_bps, + int framerate_fps) { + RTC_DCHECK_LT(stream_index, StreamCount()); + RTC_DCHECK_GT(bitrates_bps.size(), 0); + RTC_DCHECK_LE(bitrates_bps.size(), num_layers_); + // `bitrates_bps` uses individual rate per layer, but Vp8EncoderConfig wants + // the accumulated rate, so sum them up. + new_bitrates_bps_ = bitrates_bps; + new_bitrates_bps_->resize(num_layers_); + for (size_t i = 1; i < num_layers_; ++i) { + (*new_bitrates_bps_)[i] += (*new_bitrates_bps_)[i - 1]; + } +} + +Vp8EncoderConfig DefaultTemporalLayers::UpdateConfiguration( + size_t stream_index) { + RTC_DCHECK_LT(stream_index, StreamCount()); + + Vp8EncoderConfig config; + + if (!new_bitrates_bps_) { + return config; + } + + config.temporal_layer_config.emplace(); + Vp8EncoderConfig::TemporalLayerConfig& ts_config = + config.temporal_layer_config.value(); + + for (size_t i = 0; i < num_layers_; ++i) { + ts_config.ts_target_bitrate[i] = (*new_bitrates_bps_)[i] / 1000; + // ..., 4, 2, 1 + ts_config.ts_rate_decimator[i] = 1 << (num_layers_ - i - 1); + } + + ts_config.ts_number_layers = num_layers_; + ts_config.ts_periodicity = temporal_ids_.size(); + std::copy(temporal_ids_.begin(), temporal_ids_.end(), + ts_config.ts_layer_id.begin()); + + new_bitrates_bps_.reset(); + + return config; +} + +bool DefaultTemporalLayers::IsSyncFrame(const Vp8FrameConfig& config) const { + // Since we always assign TL0 to 'last' in these patterns, we can infer layer + // sync by checking if temporal id > 0 and we only reference TL0 or buffers + // containing the last key-frame. + if (config.packetizer_temporal_idx == 0) { + // TL0 frames are per definition not sync frames. + return false; + } + + if ((config.last_buffer_flags & BufferFlags::kReference) == 0) { + // Sync frames must reference TL0. + return false; + } + + if ((config.golden_buffer_flags & BufferFlags::kReference) && + !is_static_buffer_[BufferToIndex(Vp8BufferReference::kGolden)]) { + // Referencing a golden frame that contains a non-(base layer|key frame). + return false; + } + if ((config.arf_buffer_flags & BufferFlags::kReference) && + !is_static_buffer_[BufferToIndex(Vp8BufferReference::kAltref)]) { + // Referencing an altref frame that contains a non-(base layer|key frame). + return false; + } + + return true; +} + +Vp8FrameConfig DefaultTemporalLayers::NextFrameConfig(size_t stream_index, + uint32_t timestamp) { + RTC_DCHECK_LT(stream_index, StreamCount()); + RTC_DCHECK_GT(num_layers_, 0); + RTC_DCHECK_GT(temporal_pattern_.size(), 0); + + RTC_DCHECK_GT(kUninitializedPatternIndex, temporal_pattern_.size()); + const bool first_frame = (pattern_idx_ == kUninitializedPatternIndex); + + pattern_idx_ = (pattern_idx_ + 1) % temporal_pattern_.size(); + DependencyInfo dependency_info = temporal_pattern_[pattern_idx_]; + Vp8FrameConfig& tl_config = dependency_info.frame_config; + tl_config.encoder_layer_id = tl_config.packetizer_temporal_idx = + temporal_ids_[pattern_idx_ % temporal_ids_.size()]; + + if (pattern_idx_ == 0) { + // Start of new pattern iteration, set up clear state by invalidating any + // pending frames, so that we don't make an invalid reference to a buffer + // containing data from a previous iteration. + for (auto& frame : pending_frames_) { + frame.expired = true; + } + } + + if (first_frame) { + tl_config = Vp8FrameConfig::GetIntraFrameConfig(); + } else { + // Last is always ok to reference as it contains the base layer. For other + // buffers though, we need to check if the buffer has actually been + // refreshed this cycle of the temporal pattern. If the encoder dropped + // a frame, it might not have. + ValidateReferences(&tl_config.golden_buffer_flags, + Vp8BufferReference::kGolden); + ValidateReferences(&tl_config.arf_buffer_flags, + Vp8BufferReference::kAltref); + // Update search order to let the encoder know which buffers contains the + // most recent data. + UpdateSearchOrder(&tl_config); + // Figure out if this a sync frame (non-base-layer frame with only + // base-layer references). + tl_config.layer_sync = IsSyncFrame(tl_config); + + // Increment frame age, this needs to be in sync with `pattern_idx_`, + // so must update it here. Resetting age to 0 must be done when encoding is + // complete though, and so in the case of pipelining encoder it might lag. + // To prevent this data spill over into the next iteration, + // the `pedning_frames_` map is reset in loops. If delay is constant, + // the relative age should still be OK for the search order. + for (size_t& n : frames_since_buffer_refresh_) { + ++n; + } + } + + // Add frame to set of pending frames, awaiting completion. + pending_frames_.emplace_back(timestamp, false, GetUpdatedBuffers(tl_config), + dependency_info); + + // Checker does not yet support encoder frame dropping, so validate flags + // here before they can be dropped. + // TODO(sprang): Update checker to support dropping. + RTC_DCHECK(checker_->CheckTemporalConfig(first_frame, tl_config)); + + return tl_config; +} + +void DefaultTemporalLayers::ValidateReferences(BufferFlags* flags, + Vp8BufferReference ref) const { + // Check if the buffer specified by `ref` is actually referenced, and if so + // if it also a dynamically updating one (buffers always just containing + // keyframes are always safe to reference). + if ((*flags & BufferFlags::kReference) && + !is_static_buffer_[BufferToIndex(ref)]) { + if (NumFramesSinceBufferRefresh(ref) >= pattern_idx_) { + // No valid buffer state, or buffer contains frame that is older than the + // current pattern. This reference is not valid, so remove it. + *flags = static_cast<BufferFlags>(*flags & ~BufferFlags::kReference); + } + } +} + +void DefaultTemporalLayers::UpdateSearchOrder(Vp8FrameConfig* config) { + // Figure out which of the buffers we can reference, and order them so that + // the most recently refreshed is first. Otherwise prioritize last first, + // golden second, and altref third. + using BufferRefAge = std::pair<Vp8BufferReference, size_t>; + std::vector<BufferRefAge> eligible_buffers; + if (config->last_buffer_flags & BufferFlags::kReference) { + eligible_buffers.emplace_back( + Vp8BufferReference::kLast, + NumFramesSinceBufferRefresh(Vp8BufferReference::kLast)); + } + if (config->golden_buffer_flags & BufferFlags::kReference) { + eligible_buffers.emplace_back( + Vp8BufferReference::kGolden, + NumFramesSinceBufferRefresh(Vp8BufferReference::kGolden)); + } + if (config->arf_buffer_flags & BufferFlags::kReference) { + eligible_buffers.emplace_back( + Vp8BufferReference::kAltref, + NumFramesSinceBufferRefresh(Vp8BufferReference::kAltref)); + } + + std::sort(eligible_buffers.begin(), eligible_buffers.end(), + [](const BufferRefAge& lhs, const BufferRefAge& rhs) { + if (lhs.second != rhs.second) { + // Lower count has highest precedence. + return lhs.second < rhs.second; + } + return lhs.first < rhs.first; + }); + + // Populate the search order fields where possible. + if (!eligible_buffers.empty()) { + config->first_reference = eligible_buffers.front().first; + if (eligible_buffers.size() > 1) + config->second_reference = eligible_buffers[1].first; + } +} + +size_t DefaultTemporalLayers::NumFramesSinceBufferRefresh( + Vp8FrameConfig::Vp8BufferReference ref) const { + return frames_since_buffer_refresh_[BufferToIndex(ref)]; +} + +void DefaultTemporalLayers::ResetNumFramesSinceBufferRefresh( + Vp8FrameConfig::Vp8BufferReference ref) { + frames_since_buffer_refresh_[BufferToIndex(ref)] = 0; +} + +void DefaultTemporalLayers::CullPendingFramesBefore(uint32_t timestamp) { + while (!pending_frames_.empty() && + pending_frames_.front().timestamp != timestamp) { + pending_frames_.pop_front(); + } +} + +void DefaultTemporalLayers::OnEncodeDone(size_t stream_index, + uint32_t rtp_timestamp, + size_t size_bytes, + bool is_keyframe, + int qp, + CodecSpecificInfo* info) { + RTC_DCHECK_LT(stream_index, StreamCount()); + RTC_DCHECK_GT(num_layers_, 0); + + if (size_bytes == 0) { + RTC_LOG(LS_WARNING) << "Empty frame; treating as dropped."; + OnFrameDropped(stream_index, rtp_timestamp); + return; + } + + CullPendingFramesBefore(rtp_timestamp); + RTC_CHECK(!pending_frames_.empty()); + PendingFrame& frame = pending_frames_.front(); + RTC_DCHECK_EQ(frame.timestamp, rtp_timestamp); + const Vp8FrameConfig& frame_config = frame.dependency_info.frame_config; + if (is_keyframe) { + // Signal key-frame so checker resets state. + RTC_DCHECK(checker_->CheckTemporalConfig(true, frame_config)); + } + + CodecSpecificInfoVP8& vp8_info = info->codecSpecific.VP8; + if (num_layers_ == 1) { + vp8_info.temporalIdx = kNoTemporalIdx; + vp8_info.layerSync = false; + } else { + if (is_keyframe) { + // Restart the temporal pattern on keyframes. + pattern_idx_ = 0; + vp8_info.temporalIdx = 0; + vp8_info.layerSync = true; // Keyframes are always sync frames. + + for (Vp8BufferReference buffer : kAllBuffers) { + if (is_static_buffer_[BufferToIndex(buffer)]) { + // Update frame count of all kf-only buffers, regardless of state of + // `pending_frames_`. + ResetNumFramesSinceBufferRefresh(buffer); + } else { + // Key-frames update all buffers, this should be reflected when + // updating state in FrameEncoded(). + frame.updated_buffer_mask |= static_cast<uint8_t>(buffer); + } + } + } else { + // Delta frame, update codec specifics with temporal id and sync flag. + vp8_info.temporalIdx = frame_config.packetizer_temporal_idx; + vp8_info.layerSync = frame_config.layer_sync; + } + } + + vp8_info.useExplicitDependencies = true; + RTC_DCHECK_EQ(vp8_info.referencedBuffersCount, 0u); + RTC_DCHECK_EQ(vp8_info.updatedBuffersCount, 0u); + + GenericFrameInfo& generic_frame_info = info->generic_frame_info.emplace(); + + for (int i = 0; i < static_cast<int>(Vp8FrameConfig::Buffer::kCount); ++i) { + bool references = false; + bool updates = is_keyframe; + + if (!is_keyframe && + frame_config.References(static_cast<Vp8FrameConfig::Buffer>(i))) { + RTC_DCHECK_LT(vp8_info.referencedBuffersCount, + arraysize(CodecSpecificInfoVP8::referencedBuffers)); + references = true; + vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i; + } + + if (is_keyframe || + frame_config.Updates(static_cast<Vp8FrameConfig::Buffer>(i))) { + RTC_DCHECK_LT(vp8_info.updatedBuffersCount, + arraysize(CodecSpecificInfoVP8::updatedBuffers)); + updates = true; + vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i; + } + + if (references || updates) { + generic_frame_info.encoder_buffers.emplace_back(i, references, updates); + } + } + + // The templates are always present on keyframes, and then refered to by + // subsequent frames. + if (is_keyframe) { + info->template_structure = GetTemplateStructure(num_layers_); + generic_frame_info.decode_target_indications = + temporal_pattern_.front().decode_target_indications; + generic_frame_info.temporal_id = 0; + } else { + generic_frame_info.decode_target_indications = + frame.dependency_info.decode_target_indications; + generic_frame_info.temporal_id = frame_config.packetizer_temporal_idx; + } + + if (!frame.expired) { + for (Vp8BufferReference buffer : kAllBuffers) { + if (frame.updated_buffer_mask & static_cast<uint8_t>(buffer)) { + ResetNumFramesSinceBufferRefresh(buffer); + } + } + } + + pending_frames_.pop_front(); +} + +void DefaultTemporalLayers::OnFrameDropped(size_t stream_index, + uint32_t rtp_timestamp) { + CullPendingFramesBefore(rtp_timestamp); + RTC_CHECK(!pending_frames_.empty()); + RTC_DCHECK_EQ(pending_frames_.front().timestamp, rtp_timestamp); + pending_frames_.pop_front(); +} + +void DefaultTemporalLayers::OnPacketLossRateUpdate(float packet_loss_rate) {} + +void DefaultTemporalLayers::OnRttUpdate(int64_t rtt_ms) {} + +void DefaultTemporalLayers::OnLossNotification( + const VideoEncoder::LossNotification& loss_notification) {} + +FrameDependencyStructure DefaultTemporalLayers::GetTemplateStructure( + int num_layers) const { + RTC_CHECK_LT(num_layers, 5); + RTC_CHECK_GT(num_layers, 0); + + FrameDependencyStructure template_structure; + template_structure.num_decode_targets = num_layers; + + switch (num_layers) { + case 1: { + template_structure.templates.resize(2); + template_structure.templates[0].T(0).Dtis("S"); + template_structure.templates[1].T(0).Dtis("S").FrameDiffs({1}); + return template_structure; + } + case 2: { + template_structure.templates.resize(5); + template_structure.templates[0].T(0).Dtis("SS"); + template_structure.templates[1].T(0).Dtis("SS").FrameDiffs({2}); + template_structure.templates[2].T(0).Dtis("SR").FrameDiffs({2}); + template_structure.templates[3].T(1).Dtis("-S").FrameDiffs({1}); + template_structure.templates[4].T(1).Dtis("-D").FrameDiffs({2, 1}); + return template_structure; + } + case 3: { + if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) { + template_structure.templates.resize(5); + template_structure.templates[0].T(0).Dtis("SSS"); + template_structure.templates[1].T(0).Dtis("SSS").FrameDiffs({4}); + template_structure.templates[2].T(1).Dtis("-DR").FrameDiffs({2}); + template_structure.templates[3].T(2).Dtis("--S").FrameDiffs({1}); + template_structure.templates[4].T(2).Dtis("--D").FrameDiffs({2, 1}); + } else { + template_structure.templates.resize(7); + template_structure.templates[0].T(0).Dtis("SSS"); + template_structure.templates[1].T(0).Dtis("SSS").FrameDiffs({4}); + template_structure.templates[2].T(0).Dtis("SRR").FrameDiffs({4}); + template_structure.templates[3].T(1).Dtis("-SS").FrameDiffs({2}); + template_structure.templates[4].T(1).Dtis("-DS").FrameDiffs({4, 2}); + template_structure.templates[5].T(2).Dtis("--D").FrameDiffs({1}); + template_structure.templates[6].T(2).Dtis("--D").FrameDiffs({3, 1}); + } + return template_structure; + } + case 4: { + template_structure.templates.resize(8); + template_structure.templates[0].T(0).Dtis("SSSS"); + template_structure.templates[1].T(0).Dtis("SSSS").FrameDiffs({8}); + template_structure.templates[2].T(1).Dtis("-SRR").FrameDiffs({4}); + template_structure.templates[3].T(1).Dtis("-SRR").FrameDiffs({4, 8}); + template_structure.templates[4].T(2).Dtis("--SR").FrameDiffs({2}); + template_structure.templates[5].T(2).Dtis("--SR").FrameDiffs({2, 4}); + template_structure.templates[6].T(3).Dtis("---D").FrameDiffs({1}); + template_structure.templates[7].T(3).Dtis("---D").FrameDiffs({1, 3}); + return template_structure; + } + default: + RTC_DCHECK_NOTREACHED(); + // To make the compiler happy! + return template_structure; + } +} + +// Returns list of temporal dependencies for each frame in the temporal pattern. +// Values are lists of indecies in the pattern. +std::vector<std::set<uint8_t>> GetTemporalDependencies( + int num_temporal_layers) { + switch (num_temporal_layers) { + case 1: + return {{0}}; + case 2: + if (!field_trial::IsDisabled("WebRTC-UseShortVP8TL2Pattern")) { + return {{2}, {0}, {0}, {1, 2}}; + } else { + return {{6}, {0}, {0}, {1, 2}, {2}, {3, 4}, {4}, {5, 6}}; + } + case 3: + if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) { + return {{0}, {0}, {0}, {0, 1, 2}}; + } else { + return {{4}, {0}, {0}, {0, 2}, {0}, {2, 4}, {2, 4}, {4, 6}}; + } + case 4: + return {{8}, {0}, {0}, {0, 2}, + {0}, {0, 2, 4}, {0, 2, 4}, {0, 4, 6}, + {0}, {4, 6, 8}, {4, 6, 8}, {4, 8, 10}, + {4, 8}, {8, 10, 12}, {8, 10, 12}, {8, 12, 14}}; + default: + RTC_DCHECK_NOTREACHED(); + return {}; + } +} + +DefaultTemporalLayersChecker::DefaultTemporalLayersChecker( + int num_temporal_layers) + : TemporalLayersChecker(num_temporal_layers), + num_layers_(std::max(1, num_temporal_layers)), + temporal_ids_(GetTemporalIds(num_layers_)), + temporal_dependencies_(GetTemporalDependencies(num_layers_)), + pattern_idx_(255) { + int i = 0; + while (temporal_ids_.size() < temporal_dependencies_.size()) { + temporal_ids_.push_back(temporal_ids_[i++]); + } +} + +DefaultTemporalLayersChecker::~DefaultTemporalLayersChecker() = default; + +bool DefaultTemporalLayersChecker::CheckTemporalConfig( + bool frame_is_keyframe, + const Vp8FrameConfig& frame_config) { + if (!TemporalLayersChecker::CheckTemporalConfig(frame_is_keyframe, + frame_config)) { + return false; + } + if (frame_config.drop_frame) { + return true; + } + + if (frame_is_keyframe) { + pattern_idx_ = 0; + last_ = BufferState(); + golden_ = BufferState(); + arf_ = BufferState(); + return true; + } + + ++pattern_idx_; + if (pattern_idx_ == temporal_ids_.size()) { + // All non key-frame buffers should be updated each pattern cycle. + if (!last_.is_keyframe && !last_.is_updated_this_cycle) { + RTC_LOG(LS_ERROR) << "Last buffer was not updated during pattern cycle."; + return false; + } + if (!arf_.is_keyframe && !arf_.is_updated_this_cycle) { + RTC_LOG(LS_ERROR) << "Arf buffer was not updated during pattern cycle."; + return false; + } + if (!golden_.is_keyframe && !golden_.is_updated_this_cycle) { + RTC_LOG(LS_ERROR) + << "Golden buffer was not updated during pattern cycle."; + return false; + } + last_.is_updated_this_cycle = false; + arf_.is_updated_this_cycle = false; + golden_.is_updated_this_cycle = false; + pattern_idx_ = 0; + } + uint8_t expected_tl_idx = temporal_ids_[pattern_idx_]; + if (frame_config.packetizer_temporal_idx != expected_tl_idx) { + RTC_LOG(LS_ERROR) << "Frame has an incorrect temporal index. Expected: " + << static_cast<int>(expected_tl_idx) << " Actual: " + << static_cast<int>(frame_config.packetizer_temporal_idx); + return false; + } + + bool need_sync = temporal_ids_[pattern_idx_] > 0 && + temporal_ids_[pattern_idx_] != kNoTemporalIdx; + std::vector<int> dependencies; + + if (frame_config.last_buffer_flags & BufferFlags::kReference) { + uint8_t referenced_layer = temporal_ids_[last_.pattern_idx]; + if (referenced_layer > 0) { + need_sync = false; + } + if (!last_.is_keyframe) { + dependencies.push_back(last_.pattern_idx); + } + } else if (frame_config.first_reference == Vp8BufferReference::kLast || + frame_config.second_reference == Vp8BufferReference::kLast) { + RTC_LOG(LS_ERROR) + << "Last buffer not referenced, but present in search order."; + return false; + } + + if (frame_config.arf_buffer_flags & BufferFlags::kReference) { + uint8_t referenced_layer = temporal_ids_[arf_.pattern_idx]; + if (referenced_layer > 0) { + need_sync = false; + } + if (!arf_.is_keyframe) { + dependencies.push_back(arf_.pattern_idx); + } + } else if (frame_config.first_reference == Vp8BufferReference::kAltref || + frame_config.second_reference == Vp8BufferReference::kAltref) { + RTC_LOG(LS_ERROR) + << "Altret buffer not referenced, but present in search order."; + return false; + } + + if (frame_config.golden_buffer_flags & BufferFlags::kReference) { + uint8_t referenced_layer = temporal_ids_[golden_.pattern_idx]; + if (referenced_layer > 0) { + need_sync = false; + } + if (!golden_.is_keyframe) { + dependencies.push_back(golden_.pattern_idx); + } + } else if (frame_config.first_reference == Vp8BufferReference::kGolden || + frame_config.second_reference == Vp8BufferReference::kGolden) { + RTC_LOG(LS_ERROR) + << "Golden buffer not referenced, but present in search order."; + return false; + } + + if (need_sync != frame_config.layer_sync) { + RTC_LOG(LS_ERROR) << "Sync bit is set incorrectly on a frame. Expected: " + << need_sync << " Actual: " << frame_config.layer_sync; + return false; + } + + if (!frame_is_keyframe) { + size_t i; + for (i = 0; i < dependencies.size(); ++i) { + if (temporal_dependencies_[pattern_idx_].find(dependencies[i]) == + temporal_dependencies_[pattern_idx_].end()) { + RTC_LOG(LS_ERROR) + << "Illegal temporal dependency out of defined pattern " + "from position " + << static_cast<int>(pattern_idx_) << " to position " + << static_cast<int>(dependencies[i]); + return false; + } + } + } + + if (frame_config.last_buffer_flags & BufferFlags::kUpdate) { + last_.is_updated_this_cycle = true; + last_.pattern_idx = pattern_idx_; + last_.is_keyframe = false; + } + if (frame_config.arf_buffer_flags & BufferFlags::kUpdate) { + arf_.is_updated_this_cycle = true; + arf_.pattern_idx = pattern_idx_; + arf_.is_keyframe = false; + } + if (frame_config.golden_buffer_flags & BufferFlags::kUpdate) { + golden_.is_updated_this_cycle = true; + golden_.pattern_idx = pattern_idx_; + golden_.is_keyframe = false; + } + return true; +} + +} // namespace webrtc |