Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 271 insertions, 0 deletions

diff --git a/third_party/libwebrtc/call/adaptation/video_stream_adapter.h b/third_party/libwebrtc/call/adaptation/video_stream_adapter.h
new file mode 100644
index 0000000000..5c174178e4
--- /dev/null
+++ b/third_party/libwebrtc/call/adaptation/video_stream_adapter.h
@@ -0,0 +1,271 @@
+/*
+ *  Copyright 2020 The WebRTC Project Authors. All rights reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_
+#define CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_
+
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "absl/types/variant.h"
+#include "api/adaptation/resource.h"
+#include "api/field_trials_view.h"
+#include "api/rtp_parameters.h"
+#include "api/video/video_adaptation_counters.h"
+#include "call/adaptation/adaptation_constraint.h"
+#include "call/adaptation/degradation_preference_provider.h"
+#include "call/adaptation/video_source_restrictions.h"
+#include "call/adaptation/video_stream_input_state.h"
+#include "call/adaptation/video_stream_input_state_provider.h"
+#include "modules/video_coding/utility/quality_scaler.h"
+#include "rtc_base/experiments/balanced_degradation_settings.h"
+#include "rtc_base/system/no_unique_address.h"
+#include "rtc_base/thread_annotations.h"
+#include "video/video_stream_encoder_observer.h"
+
+namespace webrtc {
+
+// The listener is responsible for carrying out the reconfiguration of the video
+// source such that the VideoSourceRestrictions are fulfilled.
+class VideoSourceRestrictionsListener {
+ public:
+  virtual ~VideoSourceRestrictionsListener();
+
+  // The `restrictions` are filtered by degradation preference but not the
+  // `adaptation_counters`, which are currently only reported for legacy stats
+  // calculation purposes.
+  virtual void OnVideoSourceRestrictionsUpdated(
+      VideoSourceRestrictions restrictions,
+      const VideoAdaptationCounters& adaptation_counters,
+      rtc::scoped_refptr<Resource> reason,
+      const VideoSourceRestrictions& unfiltered_restrictions) = 0;
+};
+
+class VideoStreamAdapter;
+
+extern const int kMinFrameRateFps;
+
+VideoSourceRestrictions FilterRestrictionsByDegradationPreference(
+    VideoSourceRestrictions source_restrictions,
+    DegradationPreference degradation_preference);
+
+int GetLowerResolutionThan(int pixel_count);
+int GetHigherResolutionThan(int pixel_count);
+
+// Either represents the next VideoSourceRestrictions the VideoStreamAdapter
+// will take, or provides a Status code indicating the reason for not adapting
+// if the adaptation is not valid.
+class Adaptation final {
+ public:
+  enum class Status {
+    // Applying this adaptation will have an effect. All other Status codes
+    // indicate that adaptation is not possible and why.
+    kValid,
+    // Cannot adapt. The minimum or maximum adaptation has already been reached.
+    // There are no more steps to take.
+    kLimitReached,
+    // Cannot adapt. The resolution or frame rate requested by a recent
+    // adaptation has not yet been reflected in the input resolution or frame
+    // rate; adaptation is refused to avoid "double-adapting".
+    kAwaitingPreviousAdaptation,
+    // Not enough input.
+    kInsufficientInput,
+    // Adaptation disabled via degradation preference.
+    kAdaptationDisabled,
+    // Adaptation up was rejected by a VideoAdaptationConstraint.
+    kRejectedByConstraint,
+  };
+
+  static const char* StatusToString(Status status);
+
+  Status status() const;
+  const VideoStreamInputState& input_state() const;
+  const VideoSourceRestrictions& restrictions() const;
+  const VideoAdaptationCounters& counters() const;
+
+ private:
+  friend class VideoStreamAdapter;
+
+  // Constructs with a valid adaptation. Status is kValid.
+  Adaptation(int validation_id,
+             VideoSourceRestrictions restrictions,
+             VideoAdaptationCounters counters,
+             VideoStreamInputState input_state);
+  // Constructor when adaptation is not valid. Status MUST NOT be kValid.
+  Adaptation(int validation_id, Status invalid_status);
+
+  // An Adaptation can become invalidated if the state of VideoStreamAdapter is
+  // modified before the Adaptation is applied. To guard against this, this ID
+  // has to match VideoStreamAdapter::adaptation_validation_id_ when applied.
+  // TODO(https://crbug.com/webrtc/11700): Remove the validation_id_.
+  const int validation_id_;
+  const Status status_;
+  // Input state when adaptation was made.
+  const VideoStreamInputState input_state_;
+  const VideoSourceRestrictions restrictions_;
+  const VideoAdaptationCounters counters_;
+};
+
+// Owns the VideoSourceRestriction for a single stream and is responsible for
+// adapting it up or down when told to do so. This class serves the following
+// purposes:
+// 1. Keep track of a stream's restrictions.
+// 2. Provide valid ways to adapt up or down the stream's restrictions.
+// 3. Modify the stream's restrictions in one of the valid ways.
+class VideoStreamAdapter {
+ public:
+  VideoStreamAdapter(VideoStreamInputStateProvider* input_state_provider,
+                     VideoStreamEncoderObserver* encoder_stats_observer,
+                     const FieldTrialsView& field_trials);
+  ~VideoStreamAdapter();
+
+  VideoSourceRestrictions source_restrictions() const;
+  const VideoAdaptationCounters& adaptation_counters() const;
+  void ClearRestrictions();
+
+  void AddRestrictionsListener(
+      VideoSourceRestrictionsListener* restrictions_listener);
+  void RemoveRestrictionsListener(
+      VideoSourceRestrictionsListener* restrictions_listener);
+  void AddAdaptationConstraint(AdaptationConstraint* adaptation_constraint);
+  void RemoveAdaptationConstraint(AdaptationConstraint* adaptation_constraint);
+
+  // TODO(hbos): Setting the degradation preference should not clear
+  // restrictions! This is not defined in the spec and is unexpected, there is a
+  // tiny risk that people would discover and rely on this behavior.
+  void SetDegradationPreference(DegradationPreference degradation_preference);
+
+  // Returns an adaptation that we are guaranteed to be able to apply, or a
+  // status code indicating the reason why we cannot adapt.
+  Adaptation GetAdaptationUp();
+  Adaptation GetAdaptationDown();
+  Adaptation GetAdaptationTo(const VideoAdaptationCounters& counters,
+                             const VideoSourceRestrictions& restrictions);
+  // Tries to adapt the resolution one step. This is used for initial frame
+  // dropping. Does nothing if the degradation preference is not BALANCED or
+  // MAINTAIN_FRAMERATE. In the case of BALANCED, it will try twice to reduce
+  // the resolution. If it fails twice it gives up.
+  Adaptation GetAdaptDownResolution();
+
+  // Updates source_restrictions() the Adaptation.
+  void ApplyAdaptation(const Adaptation& adaptation,
+                       rtc::scoped_refptr<Resource> resource);
+
+  struct RestrictionsWithCounters {
+    VideoSourceRestrictions restrictions;
+    VideoAdaptationCounters counters;
+  };
+
+  static absl::optional<uint32_t> GetSingleActiveLayerPixels(
+      const VideoCodec& codec);
+
+ private:
+  void BroadcastVideoRestrictionsUpdate(
+      const VideoStreamInputState& input_state,
+      const rtc::scoped_refptr<Resource>& resource);
+
+  bool HasSufficientInputForAdaptation(const VideoStreamInputState& input_state)
+      const RTC_RUN_ON(&sequence_checker_);
+
+  using RestrictionsOrState =
+      absl::variant<RestrictionsWithCounters, Adaptation::Status>;
+  RestrictionsOrState GetAdaptationUpStep(
+      const VideoStreamInputState& input_state) const
+      RTC_RUN_ON(&sequence_checker_);
+  RestrictionsOrState GetAdaptationDownStep(
+      const VideoStreamInputState& input_state,
+      const RestrictionsWithCounters& current_restrictions) const
+      RTC_RUN_ON(&sequence_checker_);
+  RestrictionsOrState GetAdaptDownResolutionStepForBalanced(
+      const VideoStreamInputState& input_state) const
+      RTC_RUN_ON(&sequence_checker_);
+  RestrictionsOrState AdaptIfFpsDiffInsufficient(
+      const VideoStreamInputState& input_state,
+      const RestrictionsWithCounters& restrictions) const
+      RTC_RUN_ON(&sequence_checker_);
+
+  Adaptation GetAdaptationUp(const VideoStreamInputState& input_state) const
+      RTC_RUN_ON(&sequence_checker_);
+  Adaptation GetAdaptationDown(const VideoStreamInputState& input_state) const
+      RTC_RUN_ON(&sequence_checker_);
+
+  static RestrictionsOrState DecreaseResolution(
+      const VideoStreamInputState& input_state,
+      const RestrictionsWithCounters& current_restrictions);
+  static RestrictionsOrState IncreaseResolution(
+      const VideoStreamInputState& input_state,
+      const RestrictionsWithCounters& current_restrictions);
+  // Framerate methods are member functions because they need internal state
+  // if the degradation preference is BALANCED.
+  RestrictionsOrState DecreaseFramerate(
+      const VideoStreamInputState& input_state,
+      const RestrictionsWithCounters& current_restrictions) const
+      RTC_RUN_ON(&sequence_checker_);
+  RestrictionsOrState IncreaseFramerate(
+      const VideoStreamInputState& input_state,
+      const RestrictionsWithCounters& current_restrictions) const
+      RTC_RUN_ON(&sequence_checker_);
+
+  struct RestrictionsOrStateVisitor;
+  Adaptation RestrictionsOrStateToAdaptation(
+      RestrictionsOrState step_or_state,
+      const VideoStreamInputState& input_state) const
+      RTC_RUN_ON(&sequence_checker_);
+
+  RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_
+      RTC_GUARDED_BY(&sequence_checker_);
+  // Gets the input state which is the basis of all adaptations.
+  // Thread safe.
+  VideoStreamInputStateProvider* input_state_provider_;
+  // Used to signal when min pixel limit has been reached.
+  VideoStreamEncoderObserver* const encoder_stats_observer_;
+  // Decides the next adaptation target in DegradationPreference::BALANCED.
+  const BalancedDegradationSettings balanced_settings_;
+  // To guard against applying adaptations that have become invalidated, an
+  // Adaptation that is applied has to have a matching validation ID.
+  int adaptation_validation_id_ RTC_GUARDED_BY(&sequence_checker_);
+  // When deciding the next target up or down, different strategies are used
+  // depending on the DegradationPreference.
+  // https://w3c.github.io/mst-content-hint/#dom-rtcdegradationpreference
+  DegradationPreference degradation_preference_
+      RTC_GUARDED_BY(&sequence_checker_);
+  // Used to avoid adapting twice. Stores the resolution at the time of the last
+  // adaptation.
+  // TODO(hbos): Can we implement a more general "cooldown" mechanism of
+  // resources intead? If we already have adapted it seems like we should wait
+  // a while before adapting again, so that we are not acting on usage
+  // measurements that are made obsolete/unreliable by an "ongoing" adaptation.
+  struct AwaitingFrameSizeChange {
+    AwaitingFrameSizeChange(bool pixels_increased, int frame_size);
+    const bool pixels_increased;
+    const int frame_size_pixels;
+  };
+  absl::optional<AwaitingFrameSizeChange> awaiting_frame_size_change_
+      RTC_GUARDED_BY(&sequence_checker_);
+  // The previous restrictions value. Starts as unrestricted.
+  VideoSourceRestrictions last_video_source_restrictions_
+      RTC_GUARDED_BY(&sequence_checker_);
+  VideoSourceRestrictions last_filtered_restrictions_
+      RTC_GUARDED_BY(&sequence_checker_);
+
+  std::vector<VideoSourceRestrictionsListener*> restrictions_listeners_
+      RTC_GUARDED_BY(&sequence_checker_);
+  std::vector<AdaptationConstraint*> adaptation_constraints_
+      RTC_GUARDED_BY(&sequence_checker_);
+
+  RestrictionsWithCounters current_restrictions_
+      RTC_GUARDED_BY(&sequence_checker_);
+};
+
+}  // namespace webrtc
+
+#endif  // CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_