/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "modules/video_coding/codecs/vp8/default_temporal_layers.h" #include #include #include #include #include #include #include #include "modules/video_coding/include/video_codec_interface.h" #include "rtc_base/arraysize.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "system_wrappers/include/field_trial.h" namespace webrtc { DefaultTemporalLayers::PendingFrame::PendingFrame() = default; DefaultTemporalLayers::PendingFrame::PendingFrame( uint32_t timestamp, bool expired, uint8_t updated_buffers_mask, const DependencyInfo& dependency_info) : timestamp(timestamp), expired(expired), updated_buffer_mask(updated_buffers_mask), dependency_info(dependency_info) {} namespace { using BufferFlags = Vp8FrameConfig::BufferFlags; using FreezeEntropy = Vp8FrameConfig::FreezeEntropy; using Vp8BufferReference = Vp8FrameConfig::Vp8BufferReference; constexpr BufferFlags kNone = BufferFlags::kNone; constexpr BufferFlags kReference = BufferFlags::kReference; constexpr BufferFlags kUpdate = BufferFlags::kUpdate; constexpr BufferFlags kReferenceAndUpdate = BufferFlags::kReferenceAndUpdate; constexpr FreezeEntropy kFreezeEntropy = FreezeEntropy::kFreezeEntropy; static constexpr uint8_t kUninitializedPatternIndex = std::numeric_limits::max(); static constexpr std::array kAllBuffers = { {Vp8BufferReference::kLast, Vp8BufferReference::kGolden, Vp8BufferReference::kAltref}}; std::vector GetTemporalIds(size_t num_layers) { switch (num_layers) { case 1: // Temporal layer structure (single layer): // 0 0 0 0 ... return {0}; case 2: // Temporal layer structure: // 1 1 ... // 0 0 ... return {0, 1}; case 3: // Temporal layer structure: // 2 2 2 2 ... // 1 1 ... // 0 0 ... return {0, 2, 1, 2}; case 4: // Temporal layer structure: // 3 3 3 3 3 3 3 3 ... // 2 2 2 2 ... // 1 1 ... // 0 0 ... return {0, 3, 2, 3, 1, 3, 2, 3}; default: RTC_DCHECK_NOTREACHED(); break; } RTC_DCHECK_NOTREACHED(); return {0}; } uint8_t GetUpdatedBuffers(const Vp8FrameConfig& config) { uint8_t flags = 0; if (config.last_buffer_flags & BufferFlags::kUpdate) { flags |= static_cast(Vp8BufferReference::kLast); } if (config.golden_buffer_flags & BufferFlags::kUpdate) { flags |= static_cast(Vp8BufferReference::kGolden); } if (config.arf_buffer_flags & BufferFlags::kUpdate) { flags |= static_cast(Vp8BufferReference::kAltref); } return flags; } size_t BufferToIndex(Vp8BufferReference buffer) { switch (buffer) { case Vp8FrameConfig::Vp8BufferReference::kLast: return 0; case Vp8FrameConfig::Vp8BufferReference::kGolden: return 1; case Vp8FrameConfig::Vp8BufferReference::kAltref: return 2; case Vp8FrameConfig::Vp8BufferReference::kNone: RTC_CHECK_NOTREACHED(); } } } // namespace constexpr size_t DefaultTemporalLayers::kNumReferenceBuffers; std::vector DefaultTemporalLayers::GetDependencyInfo(size_t num_layers) { // For indexing in the patterns described below (which temporal layers they // belong to), see the diagram above. // Layer sync is done similarly for all patterns (except single stream) and // happens every 8 frames: // TL1 layer syncs by periodically by only referencing TL0 ('last'), but still // updating 'golden', so it can be used as a reference by future TL1 frames. // TL2 layer syncs just before TL1 by only depending on TL0 (and not depending // on TL1's buffer before TL1 has layer synced). // TODO(pbos): Consider cyclically updating 'arf' (and 'golden' for 1TL) for // the base layer in 1-3TL instead of 'last' periodically on long intervals, // so that if scene changes occur (user walks between rooms or rotates webcam) // the 'arf' (or 'golden' respectively) is not stuck on a no-longer relevant // keyframe. switch (num_layers) { case 1: // Always reference and update the same buffer. return {{"S", {kReferenceAndUpdate, kNone, kNone}}}; case 2: // All layers can reference but not update the 'alt' buffer, this means // that the 'alt' buffer reference is effectively the last keyframe. // TL0 also references and updates the 'last' buffer. // TL1 also references 'last' and references and updates 'golden'. if (!field_trial::IsDisabled("WebRTC-UseShortVP8TL2Pattern")) { // Shortened 4-frame pattern: // 1---1 1---1 ... // / / / / // 0---0---0---0 ... return {{"SS", {kReferenceAndUpdate, kNone, kNone}}, {"-S", {kReference, kUpdate, kNone}}, {"SR", {kReferenceAndUpdate, kNone, kNone}}, {"-D", {kReference, kReference, kNone, kFreezeEntropy}}}; } else { // "Default" 8-frame pattern: // 1---1---1---1 1---1---1---1 ... // / / / / / / / / // 0---0---0---0---0---0---0---0 ... return {{"SS", {kReferenceAndUpdate, kNone, kNone}}, {"-S", {kReference, kUpdate, kNone}}, {"SR", {kReferenceAndUpdate, kNone, kNone}}, {"-R", {kReference, kReferenceAndUpdate, kNone}}, {"SR", {kReferenceAndUpdate, kNone, kNone}}, {"-R", {kReference, kReferenceAndUpdate, kNone}}, {"SR", {kReferenceAndUpdate, kNone, kNone}}, {"-D", {kReference, kReference, kNone, kFreezeEntropy}}}; } case 3: if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) { // This field trial is intended to check if it is worth using a shorter // temporal pattern, trading some coding efficiency for less risk of // dropped frames. // The coding efficiency will decrease somewhat since the higher layer // state is more volatile, but it will be offset slightly by updating // the altref buffer with TL2 frames, instead of just referencing lower // layers. // If a frame is dropped in a higher layer, the jitter // buffer on the receive side won't be able to decode any higher layer // frame until the next sync frame. So we expect a noticeable decrease // in frame drops on links with high packet loss. // TL0 references and updates the 'last' buffer. // TL1 references 'last' and references and updates 'golden'. // TL2 references both 'last' & 'golden' and references and updates // 'arf'. // 2-------2 2-------2 2 // / __/ / __/ / // / __1 / __1 / // /___/ /___/ / // 0---------------0---------------0----- // 0 1 2 3 4 5 6 7 8 9 ... return {{"SSS", {kReferenceAndUpdate, kNone, kNone}}, {"--S", {kReference, kNone, kUpdate}}, {"-DR", {kReference, kUpdate, kNone}}, {"--D", {kReference, kReference, kReference, kFreezeEntropy}}}; } else { // All layers can reference but not update the 'alt' buffer, this means // that the 'alt' buffer reference is effectively the last keyframe. // TL0 also references and updates the 'last' buffer. // TL1 also references 'last' and references and updates 'golden'. // TL2 references both 'last' and 'golden' but updates no buffer. // 2 __2 _____2 __2 2 // / /____/ / / / // / 1---------/-----1 / // /_____/ /_____/ / // 0---------------0---------------0----- // 0 1 2 3 4 5 6 7 8 9 ... return {{"SSS", {kReferenceAndUpdate, kNone, kNone}}, {"--D", {kReference, kNone, kNone, kFreezeEntropy}}, {"-SS", {kReference, kUpdate, kNone}}, {"--D", {kReference, kReference, kNone, kFreezeEntropy}}, {"SRR", {kReferenceAndUpdate, kNone, kNone}}, {"--D", {kReference, kReference, kNone, kFreezeEntropy}}, {"-DS", {kReference, kReferenceAndUpdate, kNone}}, {"--D", {kReference, kReference, kNone, kFreezeEntropy}}}; } case 4: // TL0 references and updates only the 'last' buffer. // TL1 references 'last' and updates and references 'golden'. // TL2 references 'last' and 'golden', and references and updates 'arf'. // TL3 references all buffers but update none of them. // TODO(philipel): Set decode target information for this structure. return {{"----", {kReferenceAndUpdate, kNone, kNone}}, {"----", {kReference, kNone, kNone, kFreezeEntropy}}, {"----", {kReference, kNone, kUpdate}}, {"----", {kReference, kNone, kReference, kFreezeEntropy}}, {"----", {kReference, kUpdate, kNone}}, {"----", {kReference, kReference, kReference, kFreezeEntropy}}, {"----", {kReference, kReference, kReferenceAndUpdate}}, {"----", {kReference, kReference, kReference, kFreezeEntropy}}, {"----", {kReferenceAndUpdate, kNone, kNone}}, {"----", {kReference, kReference, kReference, kFreezeEntropy}}, {"----", {kReference, kReference, kReferenceAndUpdate}}, {"----", {kReference, kReference, kReference, kFreezeEntropy}}, {"----", {kReference, kReferenceAndUpdate, kNone}}, {"----", {kReference, kReference, kReference, kFreezeEntropy}}, {"----", {kReference, kReference, kReferenceAndUpdate}}, {"----", {kReference, kReference, kReference, kFreezeEntropy}}}; default: RTC_DCHECK_NOTREACHED(); break; } RTC_DCHECK_NOTREACHED(); return {{"", {kNone, kNone, kNone}}}; } std::bitset DefaultTemporalLayers::DetermineStaticBuffers( const std::vector& temporal_pattern) { std::bitset buffers; buffers.set(); for (const DependencyInfo& info : temporal_pattern) { uint8_t updated_buffers = GetUpdatedBuffers(info.frame_config); for (Vp8BufferReference buffer : kAllBuffers) { if (static_cast(buffer) & updated_buffers) { buffers.reset(BufferToIndex(buffer)); } } } return buffers; } DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers) : num_layers_(std::max(1, number_of_temporal_layers)), temporal_ids_(GetTemporalIds(num_layers_)), temporal_pattern_(GetDependencyInfo(num_layers_)), is_static_buffer_(DetermineStaticBuffers(temporal_pattern_)), pattern_idx_(kUninitializedPatternIndex), new_bitrates_bps_(std::vector(num_layers_, 0u)) { RTC_CHECK_GE(kMaxTemporalStreams, number_of_temporal_layers); RTC_CHECK_GE(number_of_temporal_layers, 0); RTC_CHECK_LE(number_of_temporal_layers, 4); // pattern_idx_ wraps around temporal_pattern_.size, this is incorrect if // temporal_ids_ are ever longer. If this is no longer correct it needs to // wrap at max(temporal_ids_.size(), temporal_pattern_.size()). RTC_DCHECK_LE(temporal_ids_.size(), temporal_pattern_.size()); RTC_DCHECK( checker_ = TemporalLayersChecker::CreateTemporalLayersChecker( Vp8TemporalLayersType::kFixedPattern, number_of_temporal_layers)); // Always need to start with a keyframe, so pre-populate all frame counters. frames_since_buffer_refresh_.fill(0); } DefaultTemporalLayers::~DefaultTemporalLayers() = default; void DefaultTemporalLayers::SetQpLimits(size_t stream_index, int min_qp, int max_qp) { RTC_DCHECK_LT(stream_index, StreamCount()); // Ignore. } size_t DefaultTemporalLayers::StreamCount() const { return 1; } bool DefaultTemporalLayers::SupportsEncoderFrameDropping( size_t stream_index) const { RTC_DCHECK_LT(stream_index, StreamCount()); // This class allows the encoder drop frames as it sees fit. return true; } void DefaultTemporalLayers::OnRatesUpdated( size_t stream_index, const std::vector& bitrates_bps, int framerate_fps) { RTC_DCHECK_LT(stream_index, StreamCount()); RTC_DCHECK_GT(bitrates_bps.size(), 0); RTC_DCHECK_LE(bitrates_bps.size(), num_layers_); // `bitrates_bps` uses individual rate per layer, but Vp8EncoderConfig wants // the accumulated rate, so sum them up. new_bitrates_bps_ = bitrates_bps; new_bitrates_bps_->resize(num_layers_); for (size_t i = 1; i < num_layers_; ++i) { (*new_bitrates_bps_)[i] += (*new_bitrates_bps_)[i - 1]; } } Vp8EncoderConfig DefaultTemporalLayers::UpdateConfiguration( size_t stream_index) { RTC_DCHECK_LT(stream_index, StreamCount()); Vp8EncoderConfig config; if (!new_bitrates_bps_) { return config; } config.temporal_layer_config.emplace(); Vp8EncoderConfig::TemporalLayerConfig& ts_config = config.temporal_layer_config.value(); for (size_t i = 0; i < num_layers_; ++i) { ts_config.ts_target_bitrate[i] = (*new_bitrates_bps_)[i] / 1000; // ..., 4, 2, 1 ts_config.ts_rate_decimator[i] = 1 << (num_layers_ - i - 1); } ts_config.ts_number_layers = num_layers_; ts_config.ts_periodicity = temporal_ids_.size(); std::copy(temporal_ids_.begin(), temporal_ids_.end(), ts_config.ts_layer_id.begin()); new_bitrates_bps_.reset(); return config; } bool DefaultTemporalLayers::IsSyncFrame(const Vp8FrameConfig& config) const { // Since we always assign TL0 to 'last' in these patterns, we can infer layer // sync by checking if temporal id > 0 and we only reference TL0 or buffers // containing the last key-frame. if (config.packetizer_temporal_idx == 0) { // TL0 frames are per definition not sync frames. return false; } if ((config.last_buffer_flags & BufferFlags::kReference) == 0) { // Sync frames must reference TL0. return false; } if ((config.golden_buffer_flags & BufferFlags::kReference) && !is_static_buffer_[BufferToIndex(Vp8BufferReference::kGolden)]) { // Referencing a golden frame that contains a non-(base layer|key frame). return false; } if ((config.arf_buffer_flags & BufferFlags::kReference) && !is_static_buffer_[BufferToIndex(Vp8BufferReference::kAltref)]) { // Referencing an altref frame that contains a non-(base layer|key frame). return false; } return true; } Vp8FrameConfig DefaultTemporalLayers::NextFrameConfig(size_t stream_index, uint32_t timestamp) { RTC_DCHECK_LT(stream_index, StreamCount()); RTC_DCHECK_GT(num_layers_, 0); RTC_DCHECK_GT(temporal_pattern_.size(), 0); RTC_DCHECK_GT(kUninitializedPatternIndex, temporal_pattern_.size()); const bool first_frame = (pattern_idx_ == kUninitializedPatternIndex); pattern_idx_ = (pattern_idx_ + 1) % temporal_pattern_.size(); DependencyInfo dependency_info = temporal_pattern_[pattern_idx_]; Vp8FrameConfig& tl_config = dependency_info.frame_config; tl_config.encoder_layer_id = tl_config.packetizer_temporal_idx = temporal_ids_[pattern_idx_ % temporal_ids_.size()]; if (pattern_idx_ == 0) { // Start of new pattern iteration, set up clear state by invalidating any // pending frames, so that we don't make an invalid reference to a buffer // containing data from a previous iteration. for (auto& frame : pending_frames_) { frame.expired = true; } } if (first_frame) { tl_config = Vp8FrameConfig::GetIntraFrameConfig(); } else { // Last is always ok to reference as it contains the base layer. For other // buffers though, we need to check if the buffer has actually been // refreshed this cycle of the temporal pattern. If the encoder dropped // a frame, it might not have. ValidateReferences(&tl_config.golden_buffer_flags, Vp8BufferReference::kGolden); ValidateReferences(&tl_config.arf_buffer_flags, Vp8BufferReference::kAltref); // Update search order to let the encoder know which buffers contains the // most recent data. UpdateSearchOrder(&tl_config); // Figure out if this a sync frame (non-base-layer frame with only // base-layer references). tl_config.layer_sync = IsSyncFrame(tl_config); // Increment frame age, this needs to be in sync with `pattern_idx_`, // so must update it here. Resetting age to 0 must be done when encoding is // complete though, and so in the case of pipelining encoder it might lag. // To prevent this data spill over into the next iteration, // the `pedning_frames_` map is reset in loops. If delay is constant, // the relative age should still be OK for the search order. for (size_t& n : frames_since_buffer_refresh_) { ++n; } } // Add frame to set of pending frames, awaiting completion. pending_frames_.emplace_back(timestamp, false, GetUpdatedBuffers(tl_config), dependency_info); // Checker does not yet support encoder frame dropping, so validate flags // here before they can be dropped. // TODO(sprang): Update checker to support dropping. RTC_DCHECK(checker_->CheckTemporalConfig(first_frame, tl_config)); return tl_config; } void DefaultTemporalLayers::ValidateReferences(BufferFlags* flags, Vp8BufferReference ref) const { // Check if the buffer specified by `ref` is actually referenced, and if so // if it also a dynamically updating one (buffers always just containing // keyframes are always safe to reference). if ((*flags & BufferFlags::kReference) && !is_static_buffer_[BufferToIndex(ref)]) { if (NumFramesSinceBufferRefresh(ref) >= pattern_idx_) { // No valid buffer state, or buffer contains frame that is older than the // current pattern. This reference is not valid, so remove it. *flags = static_cast(*flags & ~BufferFlags::kReference); } } } void DefaultTemporalLayers::UpdateSearchOrder(Vp8FrameConfig* config) { // Figure out which of the buffers we can reference, and order them so that // the most recently refreshed is first. Otherwise prioritize last first, // golden second, and altref third. using BufferRefAge = std::pair; std::vector eligible_buffers; if (config->last_buffer_flags & BufferFlags::kReference) { eligible_buffers.emplace_back( Vp8BufferReference::kLast, NumFramesSinceBufferRefresh(Vp8BufferReference::kLast)); } if (config->golden_buffer_flags & BufferFlags::kReference) { eligible_buffers.emplace_back( Vp8BufferReference::kGolden, NumFramesSinceBufferRefresh(Vp8BufferReference::kGolden)); } if (config->arf_buffer_flags & BufferFlags::kReference) { eligible_buffers.emplace_back( Vp8BufferReference::kAltref, NumFramesSinceBufferRefresh(Vp8BufferReference::kAltref)); } std::sort(eligible_buffers.begin(), eligible_buffers.end(), [](const BufferRefAge& lhs, const BufferRefAge& rhs) { if (lhs.second != rhs.second) { // Lower count has highest precedence. return lhs.second < rhs.second; } return lhs.first < rhs.first; }); // Populate the search order fields where possible. if (!eligible_buffers.empty()) { config->first_reference = eligible_buffers.front().first; if (eligible_buffers.size() > 1) config->second_reference = eligible_buffers[1].first; } } size_t DefaultTemporalLayers::NumFramesSinceBufferRefresh( Vp8FrameConfig::Vp8BufferReference ref) const { return frames_since_buffer_refresh_[BufferToIndex(ref)]; } void DefaultTemporalLayers::ResetNumFramesSinceBufferRefresh( Vp8FrameConfig::Vp8BufferReference ref) { frames_since_buffer_refresh_[BufferToIndex(ref)] = 0; } void DefaultTemporalLayers::CullPendingFramesBefore(uint32_t timestamp) { while (!pending_frames_.empty() && pending_frames_.front().timestamp != timestamp) { pending_frames_.pop_front(); } } void DefaultTemporalLayers::OnEncodeDone(size_t stream_index, uint32_t rtp_timestamp, size_t size_bytes, bool is_keyframe, int qp, CodecSpecificInfo* info) { RTC_DCHECK_LT(stream_index, StreamCount()); RTC_DCHECK_GT(num_layers_, 0); if (size_bytes == 0) { RTC_LOG(LS_WARNING) << "Empty frame; treating as dropped."; OnFrameDropped(stream_index, rtp_timestamp); return; } CullPendingFramesBefore(rtp_timestamp); RTC_CHECK(!pending_frames_.empty()); PendingFrame& frame = pending_frames_.front(); RTC_DCHECK_EQ(frame.timestamp, rtp_timestamp); const Vp8FrameConfig& frame_config = frame.dependency_info.frame_config; if (is_keyframe) { // Signal key-frame so checker resets state. RTC_DCHECK(checker_->CheckTemporalConfig(true, frame_config)); } CodecSpecificInfoVP8& vp8_info = info->codecSpecific.VP8; if (num_layers_ == 1) { vp8_info.temporalIdx = kNoTemporalIdx; vp8_info.layerSync = false; } else { if (is_keyframe) { // Restart the temporal pattern on keyframes. pattern_idx_ = 0; vp8_info.temporalIdx = 0; vp8_info.layerSync = true; // Keyframes are always sync frames. for (Vp8BufferReference buffer : kAllBuffers) { if (is_static_buffer_[BufferToIndex(buffer)]) { // Update frame count of all kf-only buffers, regardless of state of // `pending_frames_`. ResetNumFramesSinceBufferRefresh(buffer); } else { // Key-frames update all buffers, this should be reflected when // updating state in FrameEncoded(). frame.updated_buffer_mask |= static_cast(buffer); } } } else { // Delta frame, update codec specifics with temporal id and sync flag. vp8_info.temporalIdx = frame_config.packetizer_temporal_idx; vp8_info.layerSync = frame_config.layer_sync; } } vp8_info.useExplicitDependencies = true; RTC_DCHECK_EQ(vp8_info.referencedBuffersCount, 0u); RTC_DCHECK_EQ(vp8_info.updatedBuffersCount, 0u); GenericFrameInfo& generic_frame_info = info->generic_frame_info.emplace(); for (int i = 0; i < static_cast(Vp8FrameConfig::Buffer::kCount); ++i) { bool references = false; bool updates = is_keyframe; if (!is_keyframe && frame_config.References(static_cast(i))) { RTC_DCHECK_LT(vp8_info.referencedBuffersCount, arraysize(CodecSpecificInfoVP8::referencedBuffers)); references = true; vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i; } if (is_keyframe || frame_config.Updates(static_cast(i))) { RTC_DCHECK_LT(vp8_info.updatedBuffersCount, arraysize(CodecSpecificInfoVP8::updatedBuffers)); updates = true; vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i; } if (references || updates) { generic_frame_info.encoder_buffers.emplace_back(i, references, updates); } } // The templates are always present on keyframes, and then refered to by // subsequent frames. if (is_keyframe) { info->template_structure = GetTemplateStructure(num_layers_); generic_frame_info.decode_target_indications = temporal_pattern_.front().decode_target_indications; generic_frame_info.temporal_id = 0; } else { generic_frame_info.decode_target_indications = frame.dependency_info.decode_target_indications; generic_frame_info.temporal_id = frame_config.packetizer_temporal_idx; } if (!frame.expired) { for (Vp8BufferReference buffer : kAllBuffers) { if (frame.updated_buffer_mask & static_cast(buffer)) { ResetNumFramesSinceBufferRefresh(buffer); } } } pending_frames_.pop_front(); } void DefaultTemporalLayers::OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) { CullPendingFramesBefore(rtp_timestamp); RTC_CHECK(!pending_frames_.empty()); RTC_DCHECK_EQ(pending_frames_.front().timestamp, rtp_timestamp); pending_frames_.pop_front(); } void DefaultTemporalLayers::OnPacketLossRateUpdate(float packet_loss_rate) {} void DefaultTemporalLayers::OnRttUpdate(int64_t rtt_ms) {} void DefaultTemporalLayers::OnLossNotification( const VideoEncoder::LossNotification& loss_notification) {} FrameDependencyStructure DefaultTemporalLayers::GetTemplateStructure( int num_layers) const { RTC_CHECK_LT(num_layers, 5); RTC_CHECK_GT(num_layers, 0); FrameDependencyStructure template_structure; template_structure.num_decode_targets = num_layers; switch (num_layers) { case 1: { template_structure.templates.resize(2); template_structure.templates[0].T(0).Dtis("S"); template_structure.templates[1].T(0).Dtis("S").FrameDiffs({1}); return template_structure; } case 2: { template_structure.templates.resize(5); template_structure.templates[0].T(0).Dtis("SS"); template_structure.templates[1].T(0).Dtis("SS").FrameDiffs({2}); template_structure.templates[2].T(0).Dtis("SR").FrameDiffs({2}); template_structure.templates[3].T(1).Dtis("-S").FrameDiffs({1}); template_structure.templates[4].T(1).Dtis("-D").FrameDiffs({2, 1}); return template_structure; } case 3: { if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) { template_structure.templates.resize(5); template_structure.templates[0].T(0).Dtis("SSS"); template_structure.templates[1].T(0).Dtis("SSS").FrameDiffs({4}); template_structure.templates[2].T(1).Dtis("-DR").FrameDiffs({2}); template_structure.templates[3].T(2).Dtis("--S").FrameDiffs({1}); template_structure.templates[4].T(2).Dtis("--D").FrameDiffs({2, 1}); } else { template_structure.templates.resize(7); template_structure.templates[0].T(0).Dtis("SSS"); template_structure.templates[1].T(0).Dtis("SSS").FrameDiffs({4}); template_structure.templates[2].T(0).Dtis("SRR").FrameDiffs({4}); template_structure.templates[3].T(1).Dtis("-SS").FrameDiffs({2}); template_structure.templates[4].T(1).Dtis("-DS").FrameDiffs({4, 2}); template_structure.templates[5].T(2).Dtis("--D").FrameDiffs({1}); template_structure.templates[6].T(2).Dtis("--D").FrameDiffs({3, 1}); } return template_structure; } case 4: { template_structure.templates.resize(8); template_structure.templates[0].T(0).Dtis("SSSS"); template_structure.templates[1].T(0).Dtis("SSSS").FrameDiffs({8}); template_structure.templates[2].T(1).Dtis("-SRR").FrameDiffs({4}); template_structure.templates[3].T(1).Dtis("-SRR").FrameDiffs({4, 8}); template_structure.templates[4].T(2).Dtis("--SR").FrameDiffs({2}); template_structure.templates[5].T(2).Dtis("--SR").FrameDiffs({2, 4}); template_structure.templates[6].T(3).Dtis("---D").FrameDiffs({1}); template_structure.templates[7].T(3).Dtis("---D").FrameDiffs({1, 3}); return template_structure; } default: RTC_DCHECK_NOTREACHED(); // To make the compiler happy! return template_structure; } } // Returns list of temporal dependencies for each frame in the temporal pattern. // Values are lists of indecies in the pattern. std::vector> GetTemporalDependencies( int num_temporal_layers) { switch (num_temporal_layers) { case 1: return {{0}}; case 2: if (!field_trial::IsDisabled("WebRTC-UseShortVP8TL2Pattern")) { return {{2}, {0}, {0}, {1, 2}}; } else { return {{6}, {0}, {0}, {1, 2}, {2}, {3, 4}, {4}, {5, 6}}; } case 3: if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) { return {{0}, {0}, {0}, {0, 1, 2}}; } else { return {{4}, {0}, {0}, {0, 2}, {0}, {2, 4}, {2, 4}, {4, 6}}; } case 4: return {{8}, {0}, {0}, {0, 2}, {0}, {0, 2, 4}, {0, 2, 4}, {0, 4, 6}, {0}, {4, 6, 8}, {4, 6, 8}, {4, 8, 10}, {4, 8}, {8, 10, 12}, {8, 10, 12}, {8, 12, 14}}; default: RTC_DCHECK_NOTREACHED(); return {}; } } DefaultTemporalLayersChecker::DefaultTemporalLayersChecker( int num_temporal_layers) : TemporalLayersChecker(num_temporal_layers), num_layers_(std::max(1, num_temporal_layers)), temporal_ids_(GetTemporalIds(num_layers_)), temporal_dependencies_(GetTemporalDependencies(num_layers_)), pattern_idx_(255) { int i = 0; while (temporal_ids_.size() < temporal_dependencies_.size()) { temporal_ids_.push_back(temporal_ids_[i++]); } } DefaultTemporalLayersChecker::~DefaultTemporalLayersChecker() = default; bool DefaultTemporalLayersChecker::CheckTemporalConfig( bool frame_is_keyframe, const Vp8FrameConfig& frame_config) { if (!TemporalLayersChecker::CheckTemporalConfig(frame_is_keyframe, frame_config)) { return false; } if (frame_config.drop_frame) { return true; } if (frame_is_keyframe) { pattern_idx_ = 0; last_ = BufferState(); golden_ = BufferState(); arf_ = BufferState(); return true; } ++pattern_idx_; if (pattern_idx_ == temporal_ids_.size()) { // All non key-frame buffers should be updated each pattern cycle. if (!last_.is_keyframe && !last_.is_updated_this_cycle) { RTC_LOG(LS_ERROR) << "Last buffer was not updated during pattern cycle."; return false; } if (!arf_.is_keyframe && !arf_.is_updated_this_cycle) { RTC_LOG(LS_ERROR) << "Arf buffer was not updated during pattern cycle."; return false; } if (!golden_.is_keyframe && !golden_.is_updated_this_cycle) { RTC_LOG(LS_ERROR) << "Golden buffer was not updated during pattern cycle."; return false; } last_.is_updated_this_cycle = false; arf_.is_updated_this_cycle = false; golden_.is_updated_this_cycle = false; pattern_idx_ = 0; } uint8_t expected_tl_idx = temporal_ids_[pattern_idx_]; if (frame_config.packetizer_temporal_idx != expected_tl_idx) { RTC_LOG(LS_ERROR) << "Frame has an incorrect temporal index. Expected: " << static_cast(expected_tl_idx) << " Actual: " << static_cast(frame_config.packetizer_temporal_idx); return false; } bool need_sync = temporal_ids_[pattern_idx_] > 0 && temporal_ids_[pattern_idx_] != kNoTemporalIdx; std::vector dependencies; if (frame_config.last_buffer_flags & BufferFlags::kReference) { uint8_t referenced_layer = temporal_ids_[last_.pattern_idx]; if (referenced_layer > 0) { need_sync = false; } if (!last_.is_keyframe) { dependencies.push_back(last_.pattern_idx); } } else if (frame_config.first_reference == Vp8BufferReference::kLast || frame_config.second_reference == Vp8BufferReference::kLast) { RTC_LOG(LS_ERROR) << "Last buffer not referenced, but present in search order."; return false; } if (frame_config.arf_buffer_flags & BufferFlags::kReference) { uint8_t referenced_layer = temporal_ids_[arf_.pattern_idx]; if (referenced_layer > 0) { need_sync = false; } if (!arf_.is_keyframe) { dependencies.push_back(arf_.pattern_idx); } } else if (frame_config.first_reference == Vp8BufferReference::kAltref || frame_config.second_reference == Vp8BufferReference::kAltref) { RTC_LOG(LS_ERROR) << "Altret buffer not referenced, but present in search order."; return false; } if (frame_config.golden_buffer_flags & BufferFlags::kReference) { uint8_t referenced_layer = temporal_ids_[golden_.pattern_idx]; if (referenced_layer > 0) { need_sync = false; } if (!golden_.is_keyframe) { dependencies.push_back(golden_.pattern_idx); } } else if (frame_config.first_reference == Vp8BufferReference::kGolden || frame_config.second_reference == Vp8BufferReference::kGolden) { RTC_LOG(LS_ERROR) << "Golden buffer not referenced, but present in search order."; return false; } if (need_sync != frame_config.layer_sync) { RTC_LOG(LS_ERROR) << "Sync bit is set incorrectly on a frame. Expected: " << need_sync << " Actual: " << frame_config.layer_sync; return false; } if (!frame_is_keyframe) { size_t i; for (i = 0; i < dependencies.size(); ++i) { if (temporal_dependencies_[pattern_idx_].find(dependencies[i]) == temporal_dependencies_[pattern_idx_].end()) { RTC_LOG(LS_ERROR) << "Illegal temporal dependency out of defined pattern " "from position " << static_cast(pattern_idx_) << " to position " << static_cast(dependencies[i]); return false; } } } if (frame_config.last_buffer_flags & BufferFlags::kUpdate) { last_.is_updated_this_cycle = true; last_.pattern_idx = pattern_idx_; last_.is_keyframe = false; } if (frame_config.arf_buffer_flags & BufferFlags::kUpdate) { arf_.is_updated_this_cycle = true; arf_.pattern_idx = pattern_idx_; arf_.is_keyframe = false; } if (frame_config.golden_buffer_flags & BufferFlags::kUpdate) { golden_.is_updated_this_cycle = true; golden_.pattern_idx = pattern_idx_; golden_.is_keyframe = false; } return true; } } // namespace webrtc