1 files changed, 336 insertions, 0 deletions
diff --git a/third_party/libwebrtc/modules/video_coding/utility/quality_scaler.cc b/third_party/libwebrtc/modules/video_coding/utility/quality_scaler.cc
new file mode 100644
index 0000000000..7ecb340ea4
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/utility/quality_scaler.cc
@@ -0,0 +1,336 @@
+/*
+ *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/utility/quality_scaler.h"
+
+#include <memory>
+#include <utility>
+
+#include "api/field_trials_view.h"
+#include "api/units/time_delta.h"
+#include "api/video/video_adaptation_reason.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/experiments/quality_scaler_settings.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/numerics/exp_filter.h"
+#include "rtc_base/weak_ptr.h"
+
+namespace webrtc {
+
+namespace {
+// Threshold constant used until first downscale (to permit fast rampup).
+static const int kMeasureMs = 2000;
+static const float kSamplePeriodScaleFactor = 2.5;
+static const int kFramedropPercentThreshold = 60;
+static const size_t kMinFramesNeededToScale = 2 * 30;
+
+}  // namespace
+
+class QualityScaler::QpSmoother {
+ public:
+  explicit QpSmoother(float alpha)
+      : alpha_(alpha),
+        // The initial value of last_sample_ms doesn't matter since the smoother
+        // will ignore the time delta for the first update.
+        last_sample_ms_(0),
+        smoother_(alpha) {}
+
+  absl::optional<int> GetAvg() const {
+    float value = smoother_.filtered();
+    if (value == rtc::ExpFilter::kValueUndefined) {
+      return absl::nullopt;
+    }
+    return static_cast<int>(value);
+  }
+
+  void Add(float sample, int64_t time_sent_us) {
+    int64_t now_ms = time_sent_us / 1000;
+    smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample);
+    last_sample_ms_ = now_ms;
+  }
+
+  void Reset() { smoother_.Reset(alpha_); }
+
+ private:
+  const float alpha_;
+  int64_t last_sample_ms_;
+  rtc::ExpFilter smoother_;
+};
+
+// The QualityScaler checks for QP periodically by queuing CheckQpTasks. The
+// task will either run to completion and trigger a new task being queued, or it
+// will be destroyed because the QualityScaler is destroyed.
+//
+// When high or low QP is reported, the task will be pending until a callback is
+// invoked. This lets the QualityScalerQpUsageHandlerInterface react to QP usage
+// asynchronously and prevents checking for QP until the stream has potentially
+// been reconfigured.
+class QualityScaler::CheckQpTask {
+ public:
+  // The result of one CheckQpTask may influence the delay of the next
+  // CheckQpTask.
+  struct Result {
+    bool observed_enough_frames = false;
+    bool qp_usage_reported = false;
+  };
+
+  CheckQpTask(QualityScaler* quality_scaler, Result previous_task_result)
+      : quality_scaler_(quality_scaler),
+        state_(State::kNotStarted),
+        previous_task_result_(previous_task_result),
+        weak_ptr_factory_(this) {}
+
+  void StartDelayedTask() {
+    RTC_DCHECK_EQ(state_, State::kNotStarted);
+    state_ = State::kCheckingQp;
+    TaskQueueBase::Current()->PostDelayedTask(
+        [this_weak_ptr = weak_ptr_factory_.GetWeakPtr(), this] {
+          if (!this_weak_ptr) {
+            // The task has been cancelled through destruction.
+            return;
+          }
+          RTC_DCHECK_EQ(state_, State::kCheckingQp);
+          RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_);
+          switch (quality_scaler_->CheckQp()) {
+            case QualityScaler::CheckQpResult::kInsufficientSamples: {
+              result_.observed_enough_frames = false;
+              // After this line, `this` may be deleted.
+              break;
+            }
+            case QualityScaler::CheckQpResult::kNormalQp: {
+              result_.observed_enough_frames = true;
+              break;
+            }
+            case QualityScaler::CheckQpResult::kHighQp: {
+              result_.observed_enough_frames = true;
+              result_.qp_usage_reported = true;
+              quality_scaler_->fast_rampup_ = false;
+              quality_scaler_->handler_->OnReportQpUsageHigh();
+              quality_scaler_->ClearSamples();
+              break;
+            }
+            case QualityScaler::CheckQpResult::kLowQp: {
+              result_.observed_enough_frames = true;
+              result_.qp_usage_reported = true;
+              quality_scaler_->handler_->OnReportQpUsageLow();
+              quality_scaler_->ClearSamples();
+              break;
+            }
+          }
+          state_ = State::kCompleted;
+          // Starting the next task deletes the pending task. After this line,
+          // `this` has been deleted.
+          quality_scaler_->StartNextCheckQpTask();
+        },
+        TimeDelta::Millis(GetCheckingQpDelayMs()));
+  }
+
+  bool HasCompletedTask() const { return state_ == State::kCompleted; }
+
+  Result result() const {
+    RTC_DCHECK(HasCompletedTask());
+    return result_;
+  }
+
+ private:
+  enum class State {
+    kNotStarted,
+    kCheckingQp,
+    kCompleted,
+  };
+
+  // Determines the sampling period of CheckQpTasks.
+  int64_t GetCheckingQpDelayMs() const {
+    RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_);
+    if (quality_scaler_->fast_rampup_) {
+      return quality_scaler_->sampling_period_ms_;
+    }
+    if (quality_scaler_->experiment_enabled_ &&
+        !previous_task_result_.observed_enough_frames) {
+      // Use half the interval while waiting for enough frames.
+      return quality_scaler_->sampling_period_ms_ / 2;
+    }
+    if (quality_scaler_->scale_factor_ &&
+        !previous_task_result_.qp_usage_reported) {
+      // Last CheckQp did not call AdaptDown/Up, possibly reduce interval.
+      return quality_scaler_->sampling_period_ms_ *
+             quality_scaler_->scale_factor_.value();
+    }
+    return quality_scaler_->sampling_period_ms_ *
+           quality_scaler_->initial_scale_factor_;
+  }
+
+  QualityScaler* const quality_scaler_;
+  State state_;
+  const Result previous_task_result_;
+  Result result_;
+
+  rtc::WeakPtrFactory<CheckQpTask> weak_ptr_factory_;
+};
+
+QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
+                             VideoEncoder::QpThresholds thresholds,
+                             const FieldTrialsView& field_trials)
+    : QualityScaler(handler, thresholds, field_trials, kMeasureMs) {}
+
+// Protected ctor, should not be called directly.
+QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
+                             VideoEncoder::QpThresholds thresholds,
+                             const FieldTrialsView& field_trials,
+                             int64_t default_sampling_period_ms)
+    : handler_(handler),
+      thresholds_(thresholds),
+      sampling_period_ms_(QualityScalerSettings(field_trials)
+                              .SamplingPeriodMs()
+                              .value_or(default_sampling_period_ms)),
+      fast_rampup_(true),
+      // Arbitrarily choose size based on 30 fps for 5 seconds.
+      average_qp_(QualityScalerSettings(field_trials)
+                      .AverageQpWindow()
+                      .value_or(5 * 30)),
+      framedrop_percent_media_opt_(5 * 30),
+      framedrop_percent_all_(5 * 30),
+      experiment_enabled_(QualityScalingExperiment::Enabled(field_trials)),
+      min_frames_needed_(QualityScalerSettings(field_trials)
+                             .MinFrames()
+                             .value_or(kMinFramesNeededToScale)),
+      initial_scale_factor_(QualityScalerSettings(field_trials)
+                                .InitialScaleFactor()
+                                .value_or(kSamplePeriodScaleFactor)),
+      scale_factor_(QualityScalerSettings(field_trials).ScaleFactor()) {
+  RTC_DCHECK_RUN_ON(&task_checker_);
+  if (experiment_enabled_) {
+    config_ = QualityScalingExperiment::GetConfig(field_trials);
+    qp_smoother_high_.reset(new QpSmoother(config_.alpha_high));
+    qp_smoother_low_.reset(new QpSmoother(config_.alpha_low));
+  }
+  RTC_DCHECK(handler_ != nullptr);
+  StartNextCheckQpTask();
+  RTC_LOG(LS_INFO) << "QP thresholds: low: " << thresholds_.low
+                   << ", high: " << thresholds_.high;
+}
+
+QualityScaler::~QualityScaler() {
+  RTC_DCHECK_RUN_ON(&task_checker_);
+}
+
+void QualityScaler::StartNextCheckQpTask() {
+  RTC_DCHECK_RUN_ON(&task_checker_);
+  RTC_DCHECK(!pending_qp_task_ || pending_qp_task_->HasCompletedTask())
+      << "A previous CheckQpTask has not completed yet!";
+  CheckQpTask::Result previous_task_result;
+  if (pending_qp_task_) {
+    previous_task_result = pending_qp_task_->result();
+  }
+  pending_qp_task_ = std::make_unique<CheckQpTask>(this, previous_task_result);
+  pending_qp_task_->StartDelayedTask();
+}
+
+void QualityScaler::SetQpThresholds(VideoEncoder::QpThresholds thresholds) {
+  RTC_DCHECK_RUN_ON(&task_checker_);
+  thresholds_ = thresholds;
+}
+
+void QualityScaler::ReportDroppedFrameByMediaOpt() {
+  RTC_DCHECK_RUN_ON(&task_checker_);
+  framedrop_percent_media_opt_.AddSample(100);
+  framedrop_percent_all_.AddSample(100);
+}
+
+void QualityScaler::ReportDroppedFrameByEncoder() {
+  RTC_DCHECK_RUN_ON(&task_checker_);
+  framedrop_percent_all_.AddSample(100);
+}
+
+void QualityScaler::ReportQp(int qp, int64_t time_sent_us) {
+  RTC_DCHECK_RUN_ON(&task_checker_);
+  framedrop_percent_media_opt_.AddSample(0);
+  framedrop_percent_all_.AddSample(0);
+  average_qp_.AddSample(qp);
+  if (qp_smoother_high_)
+    qp_smoother_high_->Add(qp, time_sent_us);
+  if (qp_smoother_low_)
+    qp_smoother_low_->Add(qp, time_sent_us);
+}
+
+bool QualityScaler::QpFastFilterLow() const {
+  RTC_DCHECK_RUN_ON(&task_checker_);
+  size_t num_frames = config_.use_all_drop_reasons
+                          ? framedrop_percent_all_.Size()
+                          : framedrop_percent_media_opt_.Size();
+  const size_t kMinNumFrames = 10;
+  if (num_frames < kMinNumFrames) {
+    return false;  // Wait for more frames before making a decision.
+  }
+  absl::optional<int> avg_qp_high = qp_smoother_high_
+                                        ? qp_smoother_high_->GetAvg()
+                                        : average_qp_.GetAverageRoundedDown();
+  return (avg_qp_high) ? (avg_qp_high.value() <= thresholds_.low) : false;
+}
+
+QualityScaler::CheckQpResult QualityScaler::CheckQp() const {
+  RTC_DCHECK_RUN_ON(&task_checker_);
+  // Should be set through InitEncode -> Should be set by now.
+  RTC_DCHECK_GE(thresholds_.low, 0);
+
+  // If we have not observed at least this many frames we can't make a good
+  // scaling decision.
+  const size_t frames = config_.use_all_drop_reasons
+                            ? framedrop_percent_all_.Size()
+                            : framedrop_percent_media_opt_.Size();
+  if (frames < min_frames_needed_) {
+    return CheckQpResult::kInsufficientSamples;
+  }
+
+  // Check if we should scale down due to high frame drop.
+  const absl::optional<int> drop_rate =
+      config_.use_all_drop_reasons
+          ? framedrop_percent_all_.GetAverageRoundedDown()
+          : framedrop_percent_media_opt_.GetAverageRoundedDown();
+  if (drop_rate && *drop_rate >= kFramedropPercentThreshold) {
+    RTC_LOG(LS_INFO) << "Reporting high QP, framedrop percent " << *drop_rate;
+    return CheckQpResult::kHighQp;
+  }
+
+  // Check if we should scale up or down based on QP.
+  const absl::optional<int> avg_qp_high =
+      qp_smoother_high_ ? qp_smoother_high_->GetAvg()
+                        : average_qp_.GetAverageRoundedDown();
+  const absl::optional<int> avg_qp_low =
+      qp_smoother_low_ ? qp_smoother_low_->GetAvg()
+                       : average_qp_.GetAverageRoundedDown();
+  if (avg_qp_high && avg_qp_low) {
+    RTC_LOG(LS_INFO) << "Checking average QP " << *avg_qp_high << " ("
+                     << *avg_qp_low << ").";
+    if (*avg_qp_high > thresholds_.high) {
+      return CheckQpResult::kHighQp;
+    }
+    if (*avg_qp_low <= thresholds_.low) {
+      // QP has been low. We want to try a higher resolution.
+      return CheckQpResult::kLowQp;
+    }
+  }
+  return CheckQpResult::kNormalQp;
+}
+
+void QualityScaler::ClearSamples() {
+  RTC_DCHECK_RUN_ON(&task_checker_);
+  framedrop_percent_media_opt_.Reset();
+  framedrop_percent_all_.Reset();
+  average_qp_.Reset();
+  if (qp_smoother_high_)
+    qp_smoother_high_->Reset();
+  if (qp_smoother_low_)
+    qp_smoother_low_->Reset();
+}
+
+QualityScalerQpUsageHandlerInterface::~QualityScalerQpUsageHandlerInterface() {}
+
+}  // namespace webrtc