/* * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #ifndef VIDEO_VIDEO_STREAM_BUFFER_CONTROLLER_H_ #define VIDEO_VIDEO_STREAM_BUFFER_CONTROLLER_H_ #include #include "api/field_trials_view.h" #include "api/task_queue/task_queue_base.h" #include "api/video/encoded_frame.h" #include "api/video/frame_buffer.h" #include "modules/video_coding/include/video_coding_defines.h" #include "modules/video_coding/timing/inter_frame_delay_variation_calculator.h" #include "modules/video_coding/timing/jitter_estimator.h" #include "modules/video_coding/timing/timing.h" #include "rtc_base/experiments/rtt_mult_experiment.h" #include "system_wrappers/include/clock.h" #include "video/decode_synchronizer.h" #include "video/video_receive_stream_timeout_tracker.h" namespace webrtc { class FrameSchedulingReceiver { public: virtual ~FrameSchedulingReceiver() = default; virtual void OnEncodedFrame(std::unique_ptr frame) = 0; virtual void OnDecodableFrameTimeout(TimeDelta wait_time) = 0; }; class VideoStreamBufferControllerStatsObserver { public: virtual ~VideoStreamBufferControllerStatsObserver() = default; virtual void OnCompleteFrame(bool is_keyframe, size_t size_bytes, VideoContentType content_type) = 0; virtual void OnDroppedFrames(uint32_t frames_dropped) = 0; // `jitter_buffer_delay` is the delay experienced by a single frame, // whereas `target_delay` and `minimum_delay` are the current delays // applied by the jitter buffer. virtual void OnDecodableFrame(TimeDelta jitter_buffer_delay, TimeDelta target_delay, TimeDelta minimum_delay) = 0; virtual void OnDiscardedPackets(uint32_t packets_discarded) = 0; // Various jitter buffer delays determined by VCMTiming. virtual void OnFrameBufferTimingsUpdated(int estimated_max_decode_time_ms, int current_delay_ms, int target_delay_ms, int jitter_delay_ms, int min_playout_delay_ms, int render_delay_ms) = 0; virtual void OnTimingFrameInfoUpdated(const TimingFrameInfo& info) = 0; }; class VideoStreamBufferController { public: VideoStreamBufferController( Clock* clock, TaskQueueBase* worker_queue, VCMTiming* timing, VideoStreamBufferControllerStatsObserver* stats_proxy, FrameSchedulingReceiver* receiver, TimeDelta max_wait_for_keyframe, TimeDelta max_wait_for_frame, std::unique_ptr frame_decode_scheduler, const FieldTrialsView& field_trials); virtual ~VideoStreamBufferController() = default; void Stop(); void SetProtectionMode(VCMVideoProtection protection_mode); void Clear(); absl::optional InsertFrame(std::unique_ptr frame); void UpdateRtt(int64_t max_rtt_ms); void SetMaxWaits(TimeDelta max_wait_for_keyframe, TimeDelta max_wait_for_frame); void StartNextDecode(bool keyframe_required); int Size(); private: void OnFrameReady( absl::InlinedVector, 4> frames, Timestamp render_time); void OnTimeout(TimeDelta delay); void FrameReadyForDecode(uint32_t rtp_timestamp, Timestamp render_time); void UpdateDroppedFrames() RTC_RUN_ON(&worker_sequence_checker_); void UpdateDiscardedPackets() RTC_RUN_ON(&worker_sequence_checker_); void UpdateFrameBufferTimings(Timestamp min_receive_time, Timestamp now); void UpdateTimingFrameInfo(); bool IsTooManyFramesQueued() const RTC_RUN_ON(&worker_sequence_checker_); void ForceKeyFrameReleaseImmediately() RTC_RUN_ON(&worker_sequence_checker_); void MaybeScheduleFrameForRelease() RTC_RUN_ON(&worker_sequence_checker_); RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_sequence_checker_; const FieldTrialsView& field_trials_; const absl::optional rtt_mult_settings_ = RttMultExperiment::GetRttMultValue(); Clock* const clock_; VideoStreamBufferControllerStatsObserver* const stats_proxy_; FrameSchedulingReceiver* const receiver_; VCMTiming* const timing_; const std::unique_ptr frame_decode_scheduler_ RTC_GUARDED_BY(&worker_sequence_checker_); JitterEstimator jitter_estimator_ RTC_GUARDED_BY(&worker_sequence_checker_); InterFrameDelayVariationCalculator ifdv_calculator_ RTC_GUARDED_BY(&worker_sequence_checker_); bool keyframe_required_ RTC_GUARDED_BY(&worker_sequence_checker_) = false; std::unique_ptr buffer_ RTC_GUARDED_BY(&worker_sequence_checker_); FrameDecodeTiming decode_timing_ RTC_GUARDED_BY(&worker_sequence_checker_); VideoReceiveStreamTimeoutTracker timeout_tracker_ RTC_GUARDED_BY(&worker_sequence_checker_); int frames_dropped_before_last_new_frame_ RTC_GUARDED_BY(&worker_sequence_checker_) = 0; int packets_discarded_before_last_new_frame_ RTC_GUARDED_BY(&worker_sequence_checker_) = 0; VCMVideoProtection protection_mode_ RTC_GUARDED_BY(&worker_sequence_checker_) = kProtectionNack; // This flag guards frames from queuing in front of the decoder. Without this // guard, encoded frames will not wait for the decoder to finish decoding a // frame and just queue up, meaning frames will not be dropped or // fast-forwarded when the decoder is slow or hangs. bool decoder_ready_for_new_frame_ RTC_GUARDED_BY(&worker_sequence_checker_) = false; // Maximum number of frames in the decode queue to allow pacing. If the // queue grows beyond the max limit, pacing will be disabled and frames will // be pushed to the decoder as soon as possible. This only has an effect // when the low-latency rendering path is active, which is indicated by // the frame's render time == 0. FieldTrialParameter zero_playout_delay_max_decode_queue_size_; ScopedTaskSafety worker_safety_; }; } // namespace webrtc #endif // VIDEO_VIDEO_STREAM_BUFFER_CONTROLLER_H_