diff options
Diffstat (limited to 'third_party/libwebrtc/video/frame_cadence_adapter.cc')
| -rw-r--r-- | third_party/libwebrtc/video/frame_cadence_adapter.cc | 803 |
1 files changed, 803 insertions, 0 deletions
diff --git a/third_party/libwebrtc/video/frame_cadence_adapter.cc b/third_party/libwebrtc/video/frame_cadence_adapter.cc new file mode 100644 index 0000000000..efffa9672a --- /dev/null +++ b/third_party/libwebrtc/video/frame_cadence_adapter.cc @@ -0,0 +1,803 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/frame_cadence_adapter.h" + +#include <atomic> +#include <deque> +#include <memory> +#include <utility> +#include <vector> + +#include "absl/algorithm/container.h" +#include "absl/base/attributes.h" +#include "api/sequence_checker.h" +#include "api/task_queue/pending_task_safety_flag.h" +#include "api/task_queue/task_queue_base.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "api/video/video_frame.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/race_checker.h" +#include "rtc_base/rate_statistics.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/system/no_unique_address.h" +#include "rtc_base/task_utils/repeating_task.h" +#include "rtc_base/thread_annotations.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/clock.h" +#include "system_wrappers/include/metrics.h" +#include "system_wrappers/include/ntp_time.h" + +namespace webrtc { +namespace { + +// Abstracts concrete modes of the cadence adapter. +class AdapterMode { + public: + virtual ~AdapterMode() = default; + + // Called on the worker thread for every frame that enters. + virtual void OnFrame(Timestamp post_time, + int frames_scheduled_for_processing, + const VideoFrame& frame) = 0; + + // Returns the currently estimated input framerate. + virtual absl::optional<uint32_t> GetInputFrameRateFps() = 0; + + // Updates the frame rate. + virtual void UpdateFrameRate() = 0; +}; + +// Implements a pass-through adapter. Single-threaded. +class PassthroughAdapterMode : public AdapterMode { + public: + PassthroughAdapterMode(Clock* clock, + FrameCadenceAdapterInterface::Callback* callback) + : clock_(clock), callback_(callback) { + sequence_checker_.Detach(); + } + + // Adapter overrides. + void OnFrame(Timestamp post_time, + int frames_scheduled_for_processing, + const VideoFrame& frame) override { + RTC_DCHECK_RUN_ON(&sequence_checker_); + callback_->OnFrame(post_time, frames_scheduled_for_processing, frame); + } + + absl::optional<uint32_t> GetInputFrameRateFps() override { + RTC_DCHECK_RUN_ON(&sequence_checker_); + return input_framerate_.Rate(clock_->TimeInMilliseconds()); + } + + void UpdateFrameRate() override { + RTC_DCHECK_RUN_ON(&sequence_checker_); + input_framerate_.Update(1, clock_->TimeInMilliseconds()); + } + + private: + Clock* const clock_; + FrameCadenceAdapterInterface::Callback* const callback_; + RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; + // Input frame rate statistics for use when not in zero-hertz mode. + RateStatistics input_framerate_ RTC_GUARDED_BY(sequence_checker_){ + FrameCadenceAdapterInterface::kFrameRateAveragingWindowSizeMs, 1000}; +}; + +// Implements a frame cadence adapter supporting zero-hertz input. +class ZeroHertzAdapterMode : public AdapterMode { + public: + ZeroHertzAdapterMode(TaskQueueBase* queue, + Clock* clock, + FrameCadenceAdapterInterface::Callback* callback, + double max_fps); + + // Reconfigures according to parameters. + // All spatial layer trackers are initialized as unconverged by this method. + void ReconfigureParameters( + const FrameCadenceAdapterInterface::ZeroHertzModeParams& params); + + // Updates spatial layer quality convergence status. + void UpdateLayerQualityConvergence(size_t spatial_index, + bool quality_converged); + + // Updates spatial layer enabled status. + void UpdateLayerStatus(size_t spatial_index, bool enabled); + + // Adapter overrides. + void OnFrame(Timestamp post_time, + int frames_scheduled_for_processing, + const VideoFrame& frame) override; + absl::optional<uint32_t> GetInputFrameRateFps() override; + void UpdateFrameRate() override {} + + // Notified on dropped frames. + void OnDiscardedFrame(); + + // Conditionally requests a refresh frame via + // Callback::RequestRefreshFrame. + void ProcessKeyFrameRequest(); + + private: + // The tracking state of each spatial layer. Used for determining when to + // stop repeating frames. + struct SpatialLayerTracker { + // If unset, the layer is disabled. Otherwise carries the quality + // convergence status of the layer. + absl::optional<bool> quality_converged; + }; + // The state of a scheduled repeat. + struct ScheduledRepeat { + ScheduledRepeat(Timestamp origin, + int64_t origin_timestamp_us, + int64_t origin_ntp_time_ms) + : scheduled(origin), + idle(false), + origin(origin), + origin_timestamp_us(origin_timestamp_us), + origin_ntp_time_ms(origin_ntp_time_ms) {} + // The instant when the repeat was scheduled. + Timestamp scheduled; + // True if the repeat was scheduled as an idle repeat (long), false + // otherwise. + bool idle; + // The moment we decided to start repeating. + Timestamp origin; + // The timestamp_us of the frame when we started repeating. + int64_t origin_timestamp_us; + // The ntp_times_ms of the frame when we started repeating. + int64_t origin_ntp_time_ms; + }; + + // Returns true if all spatial layers can be considered to be converged in + // terms of quality. + // Convergence means QP has dropped to a low-enough level to warrant ceasing + // to send identical frames at high frequency. + bool HasQualityConverged() const RTC_RUN_ON(sequence_checker_); + // Resets quality convergence information. HasQualityConverged() returns false + // after this call. + void ResetQualityConvergenceInfo() RTC_RUN_ON(sequence_checker_); + // Processes incoming frames on a delayed cadence. + void ProcessOnDelayedCadence() RTC_RUN_ON(sequence_checker_); + // Schedules a later repeat with delay depending on state of layer trackers. + // If true is passed in `idle_repeat`, the repeat is going to be + // kZeroHertzIdleRepeatRatePeriod. Otherwise it'll be the value of + // `frame_delay`. + void ScheduleRepeat(int frame_id, bool idle_repeat) + RTC_RUN_ON(sequence_checker_); + // Repeats a frame in the abscence of incoming frames. Slows down when quality + // convergence is attained, and stops the cadence terminally when new frames + // have arrived. + void ProcessRepeatedFrameOnDelayedCadence(int frame_id) + RTC_RUN_ON(sequence_checker_); + // Sends a frame, updating the timestamp to the current time. + void SendFrameNow(const VideoFrame& frame) const + RTC_RUN_ON(sequence_checker_); + // Returns the repeat duration depending on if it's an idle repeat or not. + TimeDelta RepeatDuration(bool idle_repeat) const + RTC_RUN_ON(sequence_checker_); + // Unless timer already running, starts repeatedly requesting refresh frames + // after a grace_period. If a frame appears before the grace_period has + // passed, the request is cancelled. + void MaybeStartRefreshFrameRequester() RTC_RUN_ON(sequence_checker_); + + TaskQueueBase* const queue_; + Clock* const clock_; + FrameCadenceAdapterInterface::Callback* const callback_; + + // The configured max_fps. + // TODO(crbug.com/1255737): support max_fps updates. + const double max_fps_; + // How much the incoming frame sequence is delayed by. + const TimeDelta frame_delay_ = TimeDelta::Seconds(1) / max_fps_; + + RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; + // A queue of incoming frames and repeated frames. + std::deque<VideoFrame> queued_frames_ RTC_GUARDED_BY(sequence_checker_); + // The current frame ID to use when starting to repeat frames. This is used + // for cancelling deferred repeated frame processing happening. + int current_frame_id_ RTC_GUARDED_BY(sequence_checker_) = 0; + // Has content when we are repeating frames. + absl::optional<ScheduledRepeat> scheduled_repeat_ + RTC_GUARDED_BY(sequence_checker_); + // Convergent state of each of the configured simulcast layers. + std::vector<SpatialLayerTracker> layer_trackers_ + RTC_GUARDED_BY(sequence_checker_); + // Repeating task handle used for requesting refresh frames until arrival, as + // they can be dropped in various places in the capture pipeline. + RepeatingTaskHandle refresh_frame_requester_ + RTC_GUARDED_BY(sequence_checker_); + + ScopedTaskSafety safety_; +}; + +class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface { + public: + FrameCadenceAdapterImpl(Clock* clock, + TaskQueueBase* queue, + const FieldTrialsView& field_trials); + ~FrameCadenceAdapterImpl(); + + // FrameCadenceAdapterInterface overrides. + void Initialize(Callback* callback) override; + void SetZeroHertzModeEnabled( + absl::optional<ZeroHertzModeParams> params) override; + absl::optional<uint32_t> GetInputFrameRateFps() override; + void UpdateFrameRate() override; + void UpdateLayerQualityConvergence(size_t spatial_index, + bool quality_converged) override; + void UpdateLayerStatus(size_t spatial_index, bool enabled) override; + void ProcessKeyFrameRequest() override; + + // VideoFrameSink overrides. + void OnFrame(const VideoFrame& frame) override; + void OnDiscardedFrame() override; + void OnConstraintsChanged( + const VideoTrackSourceConstraints& constraints) override; + + private: + // Called from OnFrame in zero-hertz mode. + void OnFrameOnMainQueue(Timestamp post_time, + int frames_scheduled_for_processing, + const VideoFrame& frame) RTC_RUN_ON(queue_); + + // Returns true under all of the following conditions: + // - constraints min fps set to 0 + // - constraints max fps set and greater than 0, + // - field trial enabled + // - zero-hertz mode enabled + bool IsZeroHertzScreenshareEnabled() const RTC_RUN_ON(queue_); + + // Handles adapter creation on configuration changes. + void MaybeReconfigureAdapters(bool was_zero_hertz_enabled) RTC_RUN_ON(queue_); + + // Called to report on constraint UMAs. + void MaybeReportFrameRateConstraintUmas() RTC_RUN_ON(queue_); + + Clock* const clock_; + TaskQueueBase* const queue_; + + // True if we support frame entry for screenshare with a minimum frequency of + // 0 Hz. + const bool zero_hertz_screenshare_enabled_; + + // The two possible modes we're under. + absl::optional<PassthroughAdapterMode> passthrough_adapter_; + absl::optional<ZeroHertzAdapterMode> zero_hertz_adapter_; + // If set, zero-hertz mode has been enabled. + absl::optional<ZeroHertzModeParams> zero_hertz_params_; + // Cache for the current adapter mode. + AdapterMode* current_adapter_mode_ = nullptr; + + // Timestamp for statistics reporting. + absl::optional<Timestamp> zero_hertz_adapter_created_timestamp_ + RTC_GUARDED_BY(queue_); + + // Set up during Initialize. + Callback* callback_ = nullptr; + + // The source's constraints. + absl::optional<VideoTrackSourceConstraints> source_constraints_ + RTC_GUARDED_BY(queue_); + + // Race checker for incoming frames. This is the network thread in chromium, + // but may vary from test contexts. + rtc::RaceChecker incoming_frame_race_checker_; + bool has_reported_screenshare_frame_rate_umas_ RTC_GUARDED_BY(queue_) = false; + + // Number of frames that are currently scheduled for processing on the + // `queue_`. + std::atomic<int> frames_scheduled_for_processing_{0}; + + ScopedTaskSafetyDetached safety_; +}; + +ZeroHertzAdapterMode::ZeroHertzAdapterMode( + TaskQueueBase* queue, + Clock* clock, + FrameCadenceAdapterInterface::Callback* callback, + double max_fps) + : queue_(queue), clock_(clock), callback_(callback), max_fps_(max_fps) { + sequence_checker_.Detach(); + MaybeStartRefreshFrameRequester(); +} + +void ZeroHertzAdapterMode::ReconfigureParameters( + const FrameCadenceAdapterInterface::ZeroHertzModeParams& params) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DLOG(LS_INFO) << __func__ << " this " << this << " num_simulcast_layers " + << params.num_simulcast_layers; + + // Start as unconverged. + layer_trackers_.clear(); + layer_trackers_.resize(params.num_simulcast_layers, + SpatialLayerTracker{false}); +} + +void ZeroHertzAdapterMode::UpdateLayerQualityConvergence( + size_t spatial_index, + bool quality_converged) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_LOG(LS_INFO) << __func__ << " this " << this << " layer " << spatial_index + << " quality has converged: " << quality_converged; + if (spatial_index >= layer_trackers_.size()) + return; + if (layer_trackers_[spatial_index].quality_converged.has_value()) + layer_trackers_[spatial_index].quality_converged = quality_converged; +} + +void ZeroHertzAdapterMode::UpdateLayerStatus(size_t spatial_index, + bool enabled) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + if (spatial_index >= layer_trackers_.size()) + return; + if (enabled) { + if (!layer_trackers_[spatial_index].quality_converged.has_value()) { + // Assume quality has not converged until hearing otherwise. + layer_trackers_[spatial_index].quality_converged = false; + } + } else { + layer_trackers_[spatial_index].quality_converged = absl::nullopt; + } + RTC_LOG(LS_INFO) + << __func__ << " this " << this << " layer " << spatial_index + << (enabled + ? (layer_trackers_[spatial_index].quality_converged.has_value() + ? " enabled." + : " enabled and it's assumed quality has not converged.") + : " disabled."); +} + +void ZeroHertzAdapterMode::OnFrame(Timestamp post_time, + int frames_scheduled_for_processing, + const VideoFrame& frame) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DLOG(LS_VERBOSE) << "ZeroHertzAdapterMode::" << __func__ << " this " + << this; + refresh_frame_requester_.Stop(); + + // Assume all enabled layers are unconverged after frame entry. + ResetQualityConvergenceInfo(); + + // Remove stored repeating frame if needed. + if (scheduled_repeat_.has_value()) { + RTC_DCHECK(queued_frames_.size() == 1); + RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this + << " cancel repeat and restart with original"; + queued_frames_.pop_front(); + } + + // Store the frame in the queue and schedule deferred processing. + queued_frames_.push_back(frame); + current_frame_id_++; + scheduled_repeat_ = absl::nullopt; + queue_->PostDelayedHighPrecisionTask( + SafeTask(safety_.flag(), + [this] { + RTC_DCHECK_RUN_ON(&sequence_checker_); + ProcessOnDelayedCadence(); + }), + frame_delay_); +} + +void ZeroHertzAdapterMode::OnDiscardedFrame() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DLOG(LS_VERBOSE) << "ZeroHertzAdapterMode::" << __func__; + + // Under zero hertz source delivery, a discarded frame ending a sequence of + // frames which happened to contain important information can be seen as a + // capture freeze. Avoid this by starting requesting refresh frames after a + // grace period. + MaybeStartRefreshFrameRequester(); +} + +absl::optional<uint32_t> ZeroHertzAdapterMode::GetInputFrameRateFps() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + return max_fps_; +} + +void ZeroHertzAdapterMode::ProcessKeyFrameRequest() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + + // If we're new and don't have a frame, there's no need to request refresh + // frames as this was being triggered for us when zero-hz mode was set up. + // + // The next frame encoded will be a key frame. Reset quality convergence so we + // don't get idle repeats shortly after, because key frames need a lot of + // refinement frames. + ResetQualityConvergenceInfo(); + + // If we're not repeating, or we're repeating with short duration, we will + // very soon send out a frame and don't need a refresh frame. + if (!scheduled_repeat_.has_value() || !scheduled_repeat_->idle) { + RTC_LOG(LS_INFO) << __func__ << " this " << this + << " not requesting refresh frame because of recently " + "incoming frame or short repeating."; + return; + } + + // If the repeat is scheduled within a short (i.e. frame_delay_) interval, we + // will very soon send out a frame and don't need a refresh frame. + Timestamp now = clock_->CurrentTime(); + if (scheduled_repeat_->scheduled + RepeatDuration(/*idle_repeat=*/true) - + now <= + frame_delay_) { + RTC_LOG(LS_INFO) << __func__ << " this " << this + << " not requesting refresh frame because of soon " + "happening idle repeat"; + return; + } + + // Cancel the current repeat and reschedule a short repeat now. No need for a + // new refresh frame. + RTC_LOG(LS_INFO) << __func__ << " this " << this + << " not requesting refresh frame and scheduling a short " + "repeat due to key frame request"; + ScheduleRepeat(++current_frame_id_, /*idle_repeat=*/false); + return; +} + +bool ZeroHertzAdapterMode::HasQualityConverged() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); + // 1. Define ourselves as unconverged with no spatial layers configured. This + // is to keep short repeating until the layer configuration comes. + // 2. Unset layers implicitly imply that they're converged to support + // disabling layers when they're not needed. + const bool quality_converged = + !layer_trackers_.empty() && + absl::c_all_of(layer_trackers_, [](const SpatialLayerTracker& tracker) { + return tracker.quality_converged.value_or(true); + }); + return quality_converged; +} + +void ZeroHertzAdapterMode::ResetQualityConvergenceInfo() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DLOG(LS_INFO) << __func__ << " this " << this; + for (auto& layer_tracker : layer_trackers_) { + if (layer_tracker.quality_converged.has_value()) + layer_tracker.quality_converged = false; + } +} + +void ZeroHertzAdapterMode::ProcessOnDelayedCadence() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(!queued_frames_.empty()); + RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this; + + SendFrameNow(queued_frames_.front()); + + // If there were two or more frames stored, we do not have to schedule repeats + // of the front frame. + if (queued_frames_.size() > 1) { + queued_frames_.pop_front(); + return; + } + + // There's only one frame to send. Schedule a repeat sequence, which is + // cancelled by `current_frame_id_` getting incremented should new frames + // arrive. + ScheduleRepeat(current_frame_id_, HasQualityConverged()); +} + +void ZeroHertzAdapterMode::ScheduleRepeat(int frame_id, bool idle_repeat) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this << " frame_id " + << frame_id; + Timestamp now = clock_->CurrentTime(); + if (!scheduled_repeat_.has_value()) { + scheduled_repeat_.emplace(now, queued_frames_.front().timestamp_us(), + queued_frames_.front().ntp_time_ms()); + } + scheduled_repeat_->scheduled = now; + scheduled_repeat_->idle = idle_repeat; + + TimeDelta repeat_delay = RepeatDuration(idle_repeat); + queue_->PostDelayedHighPrecisionTask( + SafeTask(safety_.flag(), + [this, frame_id] { + RTC_DCHECK_RUN_ON(&sequence_checker_); + ProcessRepeatedFrameOnDelayedCadence(frame_id); + }), + repeat_delay); +} + +void ZeroHertzAdapterMode::ProcessRepeatedFrameOnDelayedCadence(int frame_id) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this << " frame_id " + << frame_id; + RTC_DCHECK(!queued_frames_.empty()); + + // Cancel this invocation if new frames turned up. + if (frame_id != current_frame_id_) + return; + RTC_DCHECK(scheduled_repeat_.has_value()); + + VideoFrame& frame = queued_frames_.front(); + + // Since this is a repeated frame, nothing changed compared to before. + VideoFrame::UpdateRect empty_update_rect; + empty_update_rect.MakeEmptyUpdate(); + frame.set_update_rect(empty_update_rect); + + // Adjust timestamps of the frame of the repeat, accounting for the actual + // delay since we started repeating. + // + // NOTE: No need to update the RTP timestamp as the VideoStreamEncoder + // overwrites it based on its chosen NTP timestamp source. + TimeDelta total_delay = clock_->CurrentTime() - scheduled_repeat_->origin; + if (frame.timestamp_us() > 0) { + frame.set_timestamp_us(scheduled_repeat_->origin_timestamp_us + + total_delay.us()); + } + if (frame.ntp_time_ms()) { + frame.set_ntp_time_ms(scheduled_repeat_->origin_ntp_time_ms + + total_delay.ms()); + } + SendFrameNow(frame); + + // Schedule another repeat. + ScheduleRepeat(frame_id, HasQualityConverged()); +} + +void ZeroHertzAdapterMode::SendFrameNow(const VideoFrame& frame) const { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this << " timestamp " + << frame.timestamp() << " timestamp_us " + << frame.timestamp_us() << " ntp_time_ms " + << frame.ntp_time_ms(); + // TODO(crbug.com/1255737): figure out if frames_scheduled_for_processing + // makes sense to compute in this implementation. + callback_->OnFrame(/*post_time=*/clock_->CurrentTime(), + /*frames_scheduled_for_processing=*/1, frame); +} + +TimeDelta ZeroHertzAdapterMode::RepeatDuration(bool idle_repeat) const { + RTC_DCHECK_RUN_ON(&sequence_checker_); + return idle_repeat + ? FrameCadenceAdapterInterface::kZeroHertzIdleRepeatRatePeriod + : frame_delay_; +} + +void ZeroHertzAdapterMode::MaybeStartRefreshFrameRequester() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DLOG(LS_VERBOSE) << __func__; + if (!refresh_frame_requester_.Running()) { + refresh_frame_requester_ = RepeatingTaskHandle::DelayedStart( + queue_, + FrameCadenceAdapterInterface::kOnDiscardedFrameRefreshFramePeriod * + frame_delay_, + [this] { + RTC_DLOG(LS_VERBOSE) << __func__ << " RequestRefreshFrame"; + if (callback_) + callback_->RequestRefreshFrame(); + return frame_delay_; + }); + } +} + +FrameCadenceAdapterImpl::FrameCadenceAdapterImpl( + Clock* clock, + TaskQueueBase* queue, + const FieldTrialsView& field_trials) + : clock_(clock), + queue_(queue), + zero_hertz_screenshare_enabled_( + !field_trials.IsDisabled("WebRTC-ZeroHertzScreenshare")) {} + +FrameCadenceAdapterImpl::~FrameCadenceAdapterImpl() { + RTC_DLOG(LS_VERBOSE) << __func__ << " this " << this; +} + +void FrameCadenceAdapterImpl::Initialize(Callback* callback) { + callback_ = callback; + passthrough_adapter_.emplace(clock_, callback); + current_adapter_mode_ = &passthrough_adapter_.value(); +} + +void FrameCadenceAdapterImpl::SetZeroHertzModeEnabled( + absl::optional<ZeroHertzModeParams> params) { + RTC_DCHECK_RUN_ON(queue_); + bool was_zero_hertz_enabled = zero_hertz_params_.has_value(); + if (params.has_value() && !was_zero_hertz_enabled) + has_reported_screenshare_frame_rate_umas_ = false; + zero_hertz_params_ = params; + MaybeReconfigureAdapters(was_zero_hertz_enabled); +} + +absl::optional<uint32_t> FrameCadenceAdapterImpl::GetInputFrameRateFps() { + RTC_DCHECK_RUN_ON(queue_); + return current_adapter_mode_->GetInputFrameRateFps(); +} + +void FrameCadenceAdapterImpl::UpdateFrameRate() { + RTC_DCHECK_RUN_ON(queue_); + // The frame rate need not be updated for the zero-hertz adapter. The + // passthrough adapter however uses it. Always pass frames into the + // passthrough to keep the estimation alive should there be an adapter switch. + passthrough_adapter_->UpdateFrameRate(); +} + +void FrameCadenceAdapterImpl::UpdateLayerQualityConvergence( + size_t spatial_index, + bool quality_converged) { + if (zero_hertz_adapter_.has_value()) + zero_hertz_adapter_->UpdateLayerQualityConvergence(spatial_index, + quality_converged); +} + +void FrameCadenceAdapterImpl::UpdateLayerStatus(size_t spatial_index, + bool enabled) { + if (zero_hertz_adapter_.has_value()) + zero_hertz_adapter_->UpdateLayerStatus(spatial_index, enabled); +} + +void FrameCadenceAdapterImpl::ProcessKeyFrameRequest() { + RTC_DCHECK_RUN_ON(queue_); + if (zero_hertz_adapter_) + zero_hertz_adapter_->ProcessKeyFrameRequest(); +} + +void FrameCadenceAdapterImpl::OnFrame(const VideoFrame& frame) { + // This method is called on the network thread under Chromium, or other + // various contexts in test. + RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); + RTC_DLOG(LS_VERBOSE) << "FrameCadenceAdapterImpl::" << __func__ << " this " + << this; + + // Local time in webrtc time base. + Timestamp post_time = clock_->CurrentTime(); + frames_scheduled_for_processing_.fetch_add(1, std::memory_order_relaxed); + queue_->PostTask(SafeTask(safety_.flag(), [this, post_time, frame] { + RTC_DCHECK_RUN_ON(queue_); + if (zero_hertz_adapter_created_timestamp_.has_value()) { + TimeDelta time_until_first_frame = + clock_->CurrentTime() - *zero_hertz_adapter_created_timestamp_; + zero_hertz_adapter_created_timestamp_ = absl::nullopt; + RTC_HISTOGRAM_COUNTS_10000( + "WebRTC.Screenshare.ZeroHz.TimeUntilFirstFrameMs", + time_until_first_frame.ms()); + } + + const int frames_scheduled_for_processing = + frames_scheduled_for_processing_.fetch_sub(1, + std::memory_order_relaxed); + OnFrameOnMainQueue(post_time, frames_scheduled_for_processing, + std::move(frame)); + MaybeReportFrameRateConstraintUmas(); + })); +} + +void FrameCadenceAdapterImpl::OnDiscardedFrame() { + callback_->OnDiscardedFrame(); + queue_->PostTask(SafeTask(safety_.flag(), [this] { + RTC_DCHECK_RUN_ON(queue_); + if (zero_hertz_adapter_) { + zero_hertz_adapter_->OnDiscardedFrame(); + } + })); +} + +void FrameCadenceAdapterImpl::OnConstraintsChanged( + const VideoTrackSourceConstraints& constraints) { + RTC_LOG(LS_INFO) << __func__ << " this " << this << " min_fps " + << constraints.min_fps.value_or(-1) << " max_fps " + << constraints.max_fps.value_or(-1); + queue_->PostTask(SafeTask(safety_.flag(), [this, constraints] { + RTC_DCHECK_RUN_ON(queue_); + bool was_zero_hertz_enabled = IsZeroHertzScreenshareEnabled(); + source_constraints_ = constraints; + MaybeReconfigureAdapters(was_zero_hertz_enabled); + })); +} + +void FrameCadenceAdapterImpl::OnFrameOnMainQueue( + Timestamp post_time, + int frames_scheduled_for_processing, + const VideoFrame& frame) { + RTC_DCHECK_RUN_ON(queue_); + current_adapter_mode_->OnFrame(post_time, frames_scheduled_for_processing, + frame); +} + +bool FrameCadenceAdapterImpl::IsZeroHertzScreenshareEnabled() const { + RTC_DCHECK_RUN_ON(queue_); + return zero_hertz_screenshare_enabled_ && source_constraints_.has_value() && + source_constraints_->max_fps.value_or(-1) > 0 && + source_constraints_->min_fps.value_or(-1) == 0 && + zero_hertz_params_.has_value(); +} + +void FrameCadenceAdapterImpl::MaybeReconfigureAdapters( + bool was_zero_hertz_enabled) { + RTC_DCHECK_RUN_ON(queue_); + bool is_zero_hertz_enabled = IsZeroHertzScreenshareEnabled(); + if (is_zero_hertz_enabled) { + if (!was_zero_hertz_enabled) { + zero_hertz_adapter_.emplace(queue_, clock_, callback_, + source_constraints_->max_fps.value()); + RTC_LOG(LS_INFO) << "Zero hertz mode activated."; + zero_hertz_adapter_created_timestamp_ = clock_->CurrentTime(); + } + zero_hertz_adapter_->ReconfigureParameters(zero_hertz_params_.value()); + current_adapter_mode_ = &zero_hertz_adapter_.value(); + } else { + if (was_zero_hertz_enabled) + zero_hertz_adapter_ = absl::nullopt; + current_adapter_mode_ = &passthrough_adapter_.value(); + } +} + +void FrameCadenceAdapterImpl::MaybeReportFrameRateConstraintUmas() { + RTC_DCHECK_RUN_ON(queue_); + if (has_reported_screenshare_frame_rate_umas_) + return; + has_reported_screenshare_frame_rate_umas_ = true; + if (!zero_hertz_params_.has_value()) + return; + RTC_HISTOGRAM_BOOLEAN("WebRTC.Screenshare.FrameRateConstraints.Exists", + source_constraints_.has_value()); + if (!source_constraints_.has_value()) + return; + RTC_HISTOGRAM_BOOLEAN("WebRTC.Screenshare.FrameRateConstraints.Min.Exists", + source_constraints_->min_fps.has_value()); + if (source_constraints_->min_fps.has_value()) { + RTC_HISTOGRAM_COUNTS_100( + "WebRTC.Screenshare.FrameRateConstraints.Min.Value", + source_constraints_->min_fps.value()); + } + RTC_HISTOGRAM_BOOLEAN("WebRTC.Screenshare.FrameRateConstraints.Max.Exists", + source_constraints_->max_fps.has_value()); + if (source_constraints_->max_fps.has_value()) { + RTC_HISTOGRAM_COUNTS_100( + "WebRTC.Screenshare.FrameRateConstraints.Max.Value", + source_constraints_->max_fps.value()); + } + if (!source_constraints_->min_fps.has_value()) { + if (source_constraints_->max_fps.has_value()) { + RTC_HISTOGRAM_COUNTS_100( + "WebRTC.Screenshare.FrameRateConstraints.MinUnset.Max", + source_constraints_->max_fps.value()); + } + } else if (source_constraints_->max_fps.has_value()) { + if (source_constraints_->min_fps.value() < + source_constraints_->max_fps.value()) { + RTC_HISTOGRAM_COUNTS_100( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Min", + source_constraints_->min_fps.value()); + RTC_HISTOGRAM_COUNTS_100( + "WebRTC.Screenshare.FrameRateConstraints.MinLessThanMax.Max", + source_constraints_->max_fps.value()); + } + // Multi-dimensional histogram for min and max FPS making it possible to + // uncover min and max combinations. See + // https://chromium.googlesource.com/chromium/src.git/+/HEAD/tools/metrics/histograms/README.md#multidimensional-histograms + constexpr int kMaxBucketCount = + 60 * /*max min_fps=*/60 + /*max max_fps=*/60 - 1; + RTC_HISTOGRAM_ENUMERATION_SPARSE( + "WebRTC.Screenshare.FrameRateConstraints.60MinPlusMaxMinusOne", + source_constraints_->min_fps.value() * 60 + + source_constraints_->max_fps.value() - 1, + /*boundary=*/kMaxBucketCount); + } +} + +} // namespace + +std::unique_ptr<FrameCadenceAdapterInterface> +FrameCadenceAdapterInterface::Create(Clock* clock, + TaskQueueBase* queue, + const FieldTrialsView& field_trials) { + return std::make_unique<FrameCadenceAdapterImpl>(clock, queue, field_trials); +} + +} // namespace webrtc |