/* * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "modules/video_coding/timing/timing.h" #include #include "api/units/time_delta.h" #include "modules/video_coding/timing/decode_time_percentile_filter.h" #include "modules/video_coding/timing/timestamp_extrapolator.h" #include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/logging.h" #include "system_wrappers/include/clock.h" namespace webrtc { namespace { // Default pacing that is used for the low-latency renderer path. constexpr TimeDelta kZeroPlayoutDelayDefaultMinPacing = TimeDelta::Millis(8); constexpr TimeDelta kLowLatencyStreamMaxPlayoutDelayThreshold = TimeDelta::Millis(500); void CheckDelaysValid(TimeDelta min_delay, TimeDelta max_delay) { if (min_delay > max_delay) { RTC_LOG(LS_ERROR) << "Playout delays set incorrectly: min playout delay (" << min_delay << ") > max playout delay (" << max_delay << "). This is undefined behaviour. Application writers should " "ensure that the min delay is always less than or equals max " "delay. If trying to use the playout delay header extensions " "described in " "https://webrtc.googlesource.com/src/+/refs/heads/main/docs/" "native-code/rtp-hdrext/playout-delay/, be careful that a playout " "delay hint or A/V sync settings may have caused this conflict."; } } } // namespace VCMTiming::VCMTiming(Clock* clock, const FieldTrialsView& field_trials) : clock_(clock), ts_extrapolator_( std::make_unique(clock_->CurrentTime())), decode_time_filter_(std::make_unique()), render_delay_(kDefaultRenderDelay), min_playout_delay_(TimeDelta::Zero()), max_playout_delay_(TimeDelta::Seconds(10)), jitter_delay_(TimeDelta::Zero()), current_delay_(TimeDelta::Zero()), prev_frame_timestamp_(0), num_decoded_frames_(0), zero_playout_delay_min_pacing_("min_pacing", kZeroPlayoutDelayDefaultMinPacing), last_decode_scheduled_(Timestamp::Zero()) { ParseFieldTrial({&zero_playout_delay_min_pacing_}, field_trials.Lookup("WebRTC-ZeroPlayoutDelay")); } void VCMTiming::Reset() { MutexLock lock(&mutex_); ts_extrapolator_->Reset(clock_->CurrentTime()); decode_time_filter_ = std::make_unique(); render_delay_ = kDefaultRenderDelay; min_playout_delay_ = TimeDelta::Zero(); jitter_delay_ = TimeDelta::Zero(); current_delay_ = TimeDelta::Zero(); prev_frame_timestamp_ = 0; } void VCMTiming::set_render_delay(TimeDelta render_delay) { MutexLock lock(&mutex_); render_delay_ = render_delay; } TimeDelta VCMTiming::min_playout_delay() const { MutexLock lock(&mutex_); return min_playout_delay_; } void VCMTiming::set_min_playout_delay(TimeDelta min_playout_delay) { MutexLock lock(&mutex_); if (min_playout_delay_ != min_playout_delay) { CheckDelaysValid(min_playout_delay, max_playout_delay_); min_playout_delay_ = min_playout_delay; } } void VCMTiming::set_max_playout_delay(TimeDelta max_playout_delay) { MutexLock lock(&mutex_); if (max_playout_delay_ != max_playout_delay) { CheckDelaysValid(min_playout_delay_, max_playout_delay); max_playout_delay_ = max_playout_delay; } } void VCMTiming::SetJitterDelay(TimeDelta jitter_delay) { MutexLock lock(&mutex_); if (jitter_delay != jitter_delay_) { jitter_delay_ = jitter_delay; // When in initial state, set current delay to minimum delay. if (current_delay_.IsZero()) { current_delay_ = jitter_delay_; } } } void VCMTiming::UpdateCurrentDelay(uint32_t frame_timestamp) { MutexLock lock(&mutex_); TimeDelta target_delay = TargetDelayInternal(); if (current_delay_.IsZero()) { // Not initialized, set current delay to target. current_delay_ = target_delay; } else if (target_delay != current_delay_) { TimeDelta delay_diff = target_delay - current_delay_; // Never change the delay with more than 100 ms every second. If we're // changing the delay in too large steps we will get noticeable freezes. By // limiting the change we can increase the delay in smaller steps, which // will be experienced as the video is played in slow motion. When lowering // the delay the video will be played at a faster pace. TimeDelta max_change = TimeDelta::Zero(); if (frame_timestamp < 0x0000ffff && prev_frame_timestamp_ > 0xffff0000) { // wrap max_change = TimeDelta::Millis(kDelayMaxChangeMsPerS * (frame_timestamp + (static_cast(1) << 32) - prev_frame_timestamp_) / 90000); } else { max_change = TimeDelta::Millis(kDelayMaxChangeMsPerS * (frame_timestamp - prev_frame_timestamp_) / 90000); } if (max_change <= TimeDelta::Zero()) { // Any changes less than 1 ms are truncated and will be postponed. // Negative change will be due to reordering and should be ignored. return; } delay_diff = std::max(delay_diff, -max_change); delay_diff = std::min(delay_diff, max_change); current_delay_ = current_delay_ + delay_diff; } prev_frame_timestamp_ = frame_timestamp; } void VCMTiming::UpdateCurrentDelay(Timestamp render_time, Timestamp actual_decode_time) { MutexLock lock(&mutex_); TimeDelta target_delay = TargetDelayInternal(); TimeDelta delayed = (actual_decode_time - render_time) + EstimatedMaxDecodeTime() + render_delay_; // Only consider `delayed` as negative by more than a few microseconds. if (delayed.ms() < 0) { return; } if (current_delay_ + delayed <= target_delay) { current_delay_ += delayed; } else { current_delay_ = target_delay; } } void VCMTiming::StopDecodeTimer(TimeDelta decode_time, Timestamp now) { MutexLock lock(&mutex_); decode_time_filter_->AddTiming(decode_time.ms(), now.ms()); RTC_DCHECK_GE(decode_time, TimeDelta::Zero()); ++num_decoded_frames_; } void VCMTiming::IncomingTimestamp(uint32_t rtp_timestamp, Timestamp now) { MutexLock lock(&mutex_); ts_extrapolator_->Update(now, rtp_timestamp); } Timestamp VCMTiming::RenderTime(uint32_t frame_timestamp, Timestamp now) const { MutexLock lock(&mutex_); return RenderTimeInternal(frame_timestamp, now); } void VCMTiming::SetLastDecodeScheduledTimestamp( Timestamp last_decode_scheduled) { MutexLock lock(&mutex_); last_decode_scheduled_ = last_decode_scheduled; } Timestamp VCMTiming::RenderTimeInternal(uint32_t frame_timestamp, Timestamp now) const { if (UseLowLatencyRendering()) { // Render as soon as possible or with low-latency renderer algorithm. return Timestamp::Zero(); } // Note that TimestampExtrapolator::ExtrapolateLocalTime is not a const // method; it mutates the object's wraparound state. Timestamp estimated_complete_time = ts_extrapolator_->ExtrapolateLocalTime(frame_timestamp).value_or(now); // Make sure the actual delay stays in the range of `min_playout_delay_` // and `max_playout_delay_`. TimeDelta actual_delay = current_delay_.Clamped(min_playout_delay_, max_playout_delay_); return estimated_complete_time + actual_delay; } TimeDelta VCMTiming::EstimatedMaxDecodeTime() const { const int decode_time_ms = decode_time_filter_->RequiredDecodeTimeMs(); RTC_DCHECK_GE(decode_time_ms, 0); return TimeDelta::Millis(decode_time_ms); } TimeDelta VCMTiming::MaxWaitingTime(Timestamp render_time, Timestamp now, bool too_many_frames_queued) const { MutexLock lock(&mutex_); if (render_time.IsZero() && zero_playout_delay_min_pacing_->us() > 0 && min_playout_delay_.IsZero() && max_playout_delay_ > TimeDelta::Zero()) { // `render_time` == 0 indicates that the frame should be decoded and // rendered as soon as possible. However, the decoder can be choked if too // many frames are sent at once. Therefore, limit the interframe delay to // |zero_playout_delay_min_pacing_| unless too many frames are queued in // which case the frames are sent to the decoder at once. if (too_many_frames_queued) { return TimeDelta::Zero(); } Timestamp earliest_next_decode_start_time = last_decode_scheduled_ + zero_playout_delay_min_pacing_; TimeDelta max_wait_time = now >= earliest_next_decode_start_time ? TimeDelta::Zero() : earliest_next_decode_start_time - now; return max_wait_time; } return render_time - now - EstimatedMaxDecodeTime() - render_delay_; } TimeDelta VCMTiming::TargetVideoDelay() const { MutexLock lock(&mutex_); return TargetDelayInternal(); } TimeDelta VCMTiming::TargetDelayInternal() const { return std::max(min_playout_delay_, jitter_delay_ + EstimatedMaxDecodeTime() + render_delay_); } // TODO(crbug.com/webrtc/15197): Centralize delay arithmetic. TimeDelta VCMTiming::StatsTargetDelayInternal() const { TimeDelta stats_target_delay = TargetDelayInternal() - (EstimatedMaxDecodeTime() + render_delay_); return std::max(TimeDelta::Zero(), stats_target_delay); } VideoFrame::RenderParameters VCMTiming::RenderParameters() const { MutexLock lock(&mutex_); return {.use_low_latency_rendering = UseLowLatencyRendering(), .max_composition_delay_in_frames = max_composition_delay_in_frames_}; } bool VCMTiming::UseLowLatencyRendering() const { // min_playout_delay_==0, // max_playout_delay_<=kLowLatencyStreamMaxPlayoutDelayThreshold indicates // that the low-latency path should be used, which means that frames should be // decoded and rendered as soon as possible. return min_playout_delay_.IsZero() && max_playout_delay_ <= kLowLatencyStreamMaxPlayoutDelayThreshold; } VCMTiming::VideoDelayTimings VCMTiming::GetTimings() const { MutexLock lock(&mutex_); return VideoDelayTimings{ .num_decoded_frames = num_decoded_frames_, .minimum_delay = jitter_delay_, .estimated_max_decode_time = EstimatedMaxDecodeTime(), .render_delay = render_delay_, .min_playout_delay = min_playout_delay_, .max_playout_delay = max_playout_delay_, .target_delay = StatsTargetDelayInternal(), .current_delay = current_delay_}; } void VCMTiming::SetTimingFrameInfo(const TimingFrameInfo& info) { MutexLock lock(&mutex_); timing_frame_info_.emplace(info); } absl::optional VCMTiming::GetTimingFrameInfo() { MutexLock lock(&mutex_); return timing_frame_info_; } void VCMTiming::SetMaxCompositionDelayInFrames( absl::optional max_composition_delay_in_frames) { MutexLock lock(&mutex_); max_composition_delay_in_frames_ = max_composition_delay_in_frames; } } // namespace webrtc