/* * Copyright 2020 The WebRTC Project Authors. All rights reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #ifndef CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_ #define CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_ #include #include #include #include "absl/types/optional.h" #include "absl/types/variant.h" #include "api/adaptation/resource.h" #include "api/field_trials_view.h" #include "api/rtp_parameters.h" #include "api/video/video_adaptation_counters.h" #include "call/adaptation/adaptation_constraint.h" #include "call/adaptation/degradation_preference_provider.h" #include "call/adaptation/video_source_restrictions.h" #include "call/adaptation/video_stream_input_state.h" #include "call/adaptation/video_stream_input_state_provider.h" #include "modules/video_coding/utility/quality_scaler.h" #include "rtc_base/experiments/balanced_degradation_settings.h" #include "rtc_base/system/no_unique_address.h" #include "rtc_base/thread_annotations.h" #include "video/video_stream_encoder_observer.h" namespace webrtc { // The listener is responsible for carrying out the reconfiguration of the video // source such that the VideoSourceRestrictions are fulfilled. class VideoSourceRestrictionsListener { public: virtual ~VideoSourceRestrictionsListener(); // The `restrictions` are filtered by degradation preference but not the // `adaptation_counters`, which are currently only reported for legacy stats // calculation purposes. virtual void OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, rtc::scoped_refptr reason, const VideoSourceRestrictions& unfiltered_restrictions) = 0; }; class VideoStreamAdapter; extern const int kMinFrameRateFps; VideoSourceRestrictions FilterRestrictionsByDegradationPreference( VideoSourceRestrictions source_restrictions, DegradationPreference degradation_preference); int GetLowerResolutionThan(int pixel_count); int GetHigherResolutionThan(int pixel_count); // Either represents the next VideoSourceRestrictions the VideoStreamAdapter // will take, or provides a Status code indicating the reason for not adapting // if the adaptation is not valid. class Adaptation final { public: enum class Status { // Applying this adaptation will have an effect. All other Status codes // indicate that adaptation is not possible and why. kValid, // Cannot adapt. The minimum or maximum adaptation has already been reached. // There are no more steps to take. kLimitReached, // Cannot adapt. The resolution or frame rate requested by a recent // adaptation has not yet been reflected in the input resolution or frame // rate; adaptation is refused to avoid "double-adapting". kAwaitingPreviousAdaptation, // Not enough input. kInsufficientInput, // Adaptation disabled via degradation preference. kAdaptationDisabled, // Adaptation up was rejected by a VideoAdaptationConstraint. kRejectedByConstraint, }; static const char* StatusToString(Status status); Status status() const; const VideoStreamInputState& input_state() const; const VideoSourceRestrictions& restrictions() const; const VideoAdaptationCounters& counters() const; private: friend class VideoStreamAdapter; // Constructs with a valid adaptation. Status is kValid. Adaptation(int validation_id, VideoSourceRestrictions restrictions, VideoAdaptationCounters counters, VideoStreamInputState input_state); // Constructor when adaptation is not valid. Status MUST NOT be kValid. Adaptation(int validation_id, Status invalid_status); // An Adaptation can become invalidated if the state of VideoStreamAdapter is // modified before the Adaptation is applied. To guard against this, this ID // has to match VideoStreamAdapter::adaptation_validation_id_ when applied. // TODO(https://crbug.com/webrtc/11700): Remove the validation_id_. const int validation_id_; const Status status_; // Input state when adaptation was made. const VideoStreamInputState input_state_; const VideoSourceRestrictions restrictions_; const VideoAdaptationCounters counters_; }; // Owns the VideoSourceRestriction for a single stream and is responsible for // adapting it up or down when told to do so. This class serves the following // purposes: // 1. Keep track of a stream's restrictions. // 2. Provide valid ways to adapt up or down the stream's restrictions. // 3. Modify the stream's restrictions in one of the valid ways. class VideoStreamAdapter { public: VideoStreamAdapter(VideoStreamInputStateProvider* input_state_provider, VideoStreamEncoderObserver* encoder_stats_observer, const FieldTrialsView& field_trials); ~VideoStreamAdapter(); VideoSourceRestrictions source_restrictions() const; const VideoAdaptationCounters& adaptation_counters() const; void ClearRestrictions(); void AddRestrictionsListener( VideoSourceRestrictionsListener* restrictions_listener); void RemoveRestrictionsListener( VideoSourceRestrictionsListener* restrictions_listener); void AddAdaptationConstraint(AdaptationConstraint* adaptation_constraint); void RemoveAdaptationConstraint(AdaptationConstraint* adaptation_constraint); // TODO(hbos): Setting the degradation preference should not clear // restrictions! This is not defined in the spec and is unexpected, there is a // tiny risk that people would discover and rely on this behavior. void SetDegradationPreference(DegradationPreference degradation_preference); // Returns an adaptation that we are guaranteed to be able to apply, or a // status code indicating the reason why we cannot adapt. Adaptation GetAdaptationUp(); Adaptation GetAdaptationDown(); Adaptation GetAdaptationTo(const VideoAdaptationCounters& counters, const VideoSourceRestrictions& restrictions); // Tries to adapt the resolution one step. This is used for initial frame // dropping. Does nothing if the degradation preference is not BALANCED or // MAINTAIN_FRAMERATE. In the case of BALANCED, it will try twice to reduce // the resolution. If it fails twice it gives up. Adaptation GetAdaptDownResolution(); // Updates source_restrictions() the Adaptation. void ApplyAdaptation(const Adaptation& adaptation, rtc::scoped_refptr resource); struct RestrictionsWithCounters { VideoSourceRestrictions restrictions; VideoAdaptationCounters counters; }; static absl::optional GetSingleActiveLayerPixels( const VideoCodec& codec); private: void BroadcastVideoRestrictionsUpdate( const VideoStreamInputState& input_state, const rtc::scoped_refptr& resource); bool HasSufficientInputForAdaptation(const VideoStreamInputState& input_state) const RTC_RUN_ON(&sequence_checker_); using RestrictionsOrState = absl::variant; RestrictionsOrState GetAdaptationUpStep( const VideoStreamInputState& input_state) const RTC_RUN_ON(&sequence_checker_); RestrictionsOrState GetAdaptationDownStep( const VideoStreamInputState& input_state, const RestrictionsWithCounters& current_restrictions) const RTC_RUN_ON(&sequence_checker_); RestrictionsOrState GetAdaptDownResolutionStepForBalanced( const VideoStreamInputState& input_state) const RTC_RUN_ON(&sequence_checker_); RestrictionsOrState AdaptIfFpsDiffInsufficient( const VideoStreamInputState& input_state, const RestrictionsWithCounters& restrictions) const RTC_RUN_ON(&sequence_checker_); Adaptation GetAdaptationUp(const VideoStreamInputState& input_state) const RTC_RUN_ON(&sequence_checker_); Adaptation GetAdaptationDown(const VideoStreamInputState& input_state) const RTC_RUN_ON(&sequence_checker_); static RestrictionsOrState DecreaseResolution( const VideoStreamInputState& input_state, const RestrictionsWithCounters& current_restrictions); static RestrictionsOrState IncreaseResolution( const VideoStreamInputState& input_state, const RestrictionsWithCounters& current_restrictions); // Framerate methods are member functions because they need internal state // if the degradation preference is BALANCED. RestrictionsOrState DecreaseFramerate( const VideoStreamInputState& input_state, const RestrictionsWithCounters& current_restrictions) const RTC_RUN_ON(&sequence_checker_); RestrictionsOrState IncreaseFramerate( const VideoStreamInputState& input_state, const RestrictionsWithCounters& current_restrictions) const RTC_RUN_ON(&sequence_checker_); struct RestrictionsOrStateVisitor; Adaptation RestrictionsOrStateToAdaptation( RestrictionsOrState step_or_state, const VideoStreamInputState& input_state) const RTC_RUN_ON(&sequence_checker_); RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_ RTC_GUARDED_BY(&sequence_checker_); // Gets the input state which is the basis of all adaptations. // Thread safe. VideoStreamInputStateProvider* input_state_provider_; // Used to signal when min pixel limit has been reached. VideoStreamEncoderObserver* const encoder_stats_observer_; // Decides the next adaptation target in DegradationPreference::BALANCED. const BalancedDegradationSettings balanced_settings_; // To guard against applying adaptations that have become invalidated, an // Adaptation that is applied has to have a matching validation ID. int adaptation_validation_id_ RTC_GUARDED_BY(&sequence_checker_); // When deciding the next target up or down, different strategies are used // depending on the DegradationPreference. // https://w3c.github.io/mst-content-hint/#dom-rtcdegradationpreference DegradationPreference degradation_preference_ RTC_GUARDED_BY(&sequence_checker_); // Used to avoid adapting twice. Stores the resolution at the time of the last // adaptation. // TODO(hbos): Can we implement a more general "cooldown" mechanism of // resources intead? If we already have adapted it seems like we should wait // a while before adapting again, so that we are not acting on usage // measurements that are made obsolete/unreliable by an "ongoing" adaptation. struct AwaitingFrameSizeChange { AwaitingFrameSizeChange(bool pixels_increased, int frame_size); const bool pixels_increased; const int frame_size_pixels; }; absl::optional awaiting_frame_size_change_ RTC_GUARDED_BY(&sequence_checker_); // The previous restrictions value. Starts as unrestricted. VideoSourceRestrictions last_video_source_restrictions_ RTC_GUARDED_BY(&sequence_checker_); VideoSourceRestrictions last_filtered_restrictions_ RTC_GUARDED_BY(&sequence_checker_); std::vector restrictions_listeners_ RTC_GUARDED_BY(&sequence_checker_); std::vector adaptation_constraints_ RTC_GUARDED_BY(&sequence_checker_); RestrictionsWithCounters current_restrictions_ RTC_GUARDED_BY(&sequence_checker_); }; } // namespace webrtc #endif // CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_