From 26a029d407be480d791972afb5975cf62c9360a6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 02:47:55 +0200 Subject: Adding upstream version 124.0.1. Signed-off-by: Daniel Baumann --- .../modules/video_coding/utility/quality_scaler.cc | 336 +++++++++++++++++++++ 1 file changed, 336 insertions(+) create mode 100644 third_party/libwebrtc/modules/video_coding/utility/quality_scaler.cc (limited to 'third_party/libwebrtc/modules/video_coding/utility/quality_scaler.cc') diff --git a/third_party/libwebrtc/modules/video_coding/utility/quality_scaler.cc b/third_party/libwebrtc/modules/video_coding/utility/quality_scaler.cc new file mode 100644 index 0000000000..7ecb340ea4 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/utility/quality_scaler.cc @@ -0,0 +1,336 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/quality_scaler.h" + +#include +#include + +#include "api/field_trials_view.h" +#include "api/units/time_delta.h" +#include "api/video/video_adaptation_reason.h" +#include "rtc_base/checks.h" +#include "rtc_base/experiments/quality_scaler_settings.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/exp_filter.h" +#include "rtc_base/weak_ptr.h" + +namespace webrtc { + +namespace { +// Threshold constant used until first downscale (to permit fast rampup). +static const int kMeasureMs = 2000; +static const float kSamplePeriodScaleFactor = 2.5; +static const int kFramedropPercentThreshold = 60; +static const size_t kMinFramesNeededToScale = 2 * 30; + +} // namespace + +class QualityScaler::QpSmoother { + public: + explicit QpSmoother(float alpha) + : alpha_(alpha), + // The initial value of last_sample_ms doesn't matter since the smoother + // will ignore the time delta for the first update. + last_sample_ms_(0), + smoother_(alpha) {} + + absl::optional GetAvg() const { + float value = smoother_.filtered(); + if (value == rtc::ExpFilter::kValueUndefined) { + return absl::nullopt; + } + return static_cast(value); + } + + void Add(float sample, int64_t time_sent_us) { + int64_t now_ms = time_sent_us / 1000; + smoother_.Apply(static_cast(now_ms - last_sample_ms_), sample); + last_sample_ms_ = now_ms; + } + + void Reset() { smoother_.Reset(alpha_); } + + private: + const float alpha_; + int64_t last_sample_ms_; + rtc::ExpFilter smoother_; +}; + +// The QualityScaler checks for QP periodically by queuing CheckQpTasks. The +// task will either run to completion and trigger a new task being queued, or it +// will be destroyed because the QualityScaler is destroyed. +// +// When high or low QP is reported, the task will be pending until a callback is +// invoked. This lets the QualityScalerQpUsageHandlerInterface react to QP usage +// asynchronously and prevents checking for QP until the stream has potentially +// been reconfigured. +class QualityScaler::CheckQpTask { + public: + // The result of one CheckQpTask may influence the delay of the next + // CheckQpTask. + struct Result { + bool observed_enough_frames = false; + bool qp_usage_reported = false; + }; + + CheckQpTask(QualityScaler* quality_scaler, Result previous_task_result) + : quality_scaler_(quality_scaler), + state_(State::kNotStarted), + previous_task_result_(previous_task_result), + weak_ptr_factory_(this) {} + + void StartDelayedTask() { + RTC_DCHECK_EQ(state_, State::kNotStarted); + state_ = State::kCheckingQp; + TaskQueueBase::Current()->PostDelayedTask( + [this_weak_ptr = weak_ptr_factory_.GetWeakPtr(), this] { + if (!this_weak_ptr) { + // The task has been cancelled through destruction. + return; + } + RTC_DCHECK_EQ(state_, State::kCheckingQp); + RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_); + switch (quality_scaler_->CheckQp()) { + case QualityScaler::CheckQpResult::kInsufficientSamples: { + result_.observed_enough_frames = false; + // After this line, `this` may be deleted. + break; + } + case QualityScaler::CheckQpResult::kNormalQp: { + result_.observed_enough_frames = true; + break; + } + case QualityScaler::CheckQpResult::kHighQp: { + result_.observed_enough_frames = true; + result_.qp_usage_reported = true; + quality_scaler_->fast_rampup_ = false; + quality_scaler_->handler_->OnReportQpUsageHigh(); + quality_scaler_->ClearSamples(); + break; + } + case QualityScaler::CheckQpResult::kLowQp: { + result_.observed_enough_frames = true; + result_.qp_usage_reported = true; + quality_scaler_->handler_->OnReportQpUsageLow(); + quality_scaler_->ClearSamples(); + break; + } + } + state_ = State::kCompleted; + // Starting the next task deletes the pending task. After this line, + // `this` has been deleted. + quality_scaler_->StartNextCheckQpTask(); + }, + TimeDelta::Millis(GetCheckingQpDelayMs())); + } + + bool HasCompletedTask() const { return state_ == State::kCompleted; } + + Result result() const { + RTC_DCHECK(HasCompletedTask()); + return result_; + } + + private: + enum class State { + kNotStarted, + kCheckingQp, + kCompleted, + }; + + // Determines the sampling period of CheckQpTasks. + int64_t GetCheckingQpDelayMs() const { + RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_); + if (quality_scaler_->fast_rampup_) { + return quality_scaler_->sampling_period_ms_; + } + if (quality_scaler_->experiment_enabled_ && + !previous_task_result_.observed_enough_frames) { + // Use half the interval while waiting for enough frames. + return quality_scaler_->sampling_period_ms_ / 2; + } + if (quality_scaler_->scale_factor_ && + !previous_task_result_.qp_usage_reported) { + // Last CheckQp did not call AdaptDown/Up, possibly reduce interval. + return quality_scaler_->sampling_period_ms_ * + quality_scaler_->scale_factor_.value(); + } + return quality_scaler_->sampling_period_ms_ * + quality_scaler_->initial_scale_factor_; + } + + QualityScaler* const quality_scaler_; + State state_; + const Result previous_task_result_; + Result result_; + + rtc::WeakPtrFactory weak_ptr_factory_; +}; + +QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler, + VideoEncoder::QpThresholds thresholds, + const FieldTrialsView& field_trials) + : QualityScaler(handler, thresholds, field_trials, kMeasureMs) {} + +// Protected ctor, should not be called directly. +QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler, + VideoEncoder::QpThresholds thresholds, + const FieldTrialsView& field_trials, + int64_t default_sampling_period_ms) + : handler_(handler), + thresholds_(thresholds), + sampling_period_ms_(QualityScalerSettings(field_trials) + .SamplingPeriodMs() + .value_or(default_sampling_period_ms)), + fast_rampup_(true), + // Arbitrarily choose size based on 30 fps for 5 seconds. + average_qp_(QualityScalerSettings(field_trials) + .AverageQpWindow() + .value_or(5 * 30)), + framedrop_percent_media_opt_(5 * 30), + framedrop_percent_all_(5 * 30), + experiment_enabled_(QualityScalingExperiment::Enabled(field_trials)), + min_frames_needed_(QualityScalerSettings(field_trials) + .MinFrames() + .value_or(kMinFramesNeededToScale)), + initial_scale_factor_(QualityScalerSettings(field_trials) + .InitialScaleFactor() + .value_or(kSamplePeriodScaleFactor)), + scale_factor_(QualityScalerSettings(field_trials).ScaleFactor()) { + RTC_DCHECK_RUN_ON(&task_checker_); + if (experiment_enabled_) { + config_ = QualityScalingExperiment::GetConfig(field_trials); + qp_smoother_high_.reset(new QpSmoother(config_.alpha_high)); + qp_smoother_low_.reset(new QpSmoother(config_.alpha_low)); + } + RTC_DCHECK(handler_ != nullptr); + StartNextCheckQpTask(); + RTC_LOG(LS_INFO) << "QP thresholds: low: " << thresholds_.low + << ", high: " << thresholds_.high; +} + +QualityScaler::~QualityScaler() { + RTC_DCHECK_RUN_ON(&task_checker_); +} + +void QualityScaler::StartNextCheckQpTask() { + RTC_DCHECK_RUN_ON(&task_checker_); + RTC_DCHECK(!pending_qp_task_ || pending_qp_task_->HasCompletedTask()) + << "A previous CheckQpTask has not completed yet!"; + CheckQpTask::Result previous_task_result; + if (pending_qp_task_) { + previous_task_result = pending_qp_task_->result(); + } + pending_qp_task_ = std::make_unique(this, previous_task_result); + pending_qp_task_->StartDelayedTask(); +} + +void QualityScaler::SetQpThresholds(VideoEncoder::QpThresholds thresholds) { + RTC_DCHECK_RUN_ON(&task_checker_); + thresholds_ = thresholds; +} + +void QualityScaler::ReportDroppedFrameByMediaOpt() { + RTC_DCHECK_RUN_ON(&task_checker_); + framedrop_percent_media_opt_.AddSample(100); + framedrop_percent_all_.AddSample(100); +} + +void QualityScaler::ReportDroppedFrameByEncoder() { + RTC_DCHECK_RUN_ON(&task_checker_); + framedrop_percent_all_.AddSample(100); +} + +void QualityScaler::ReportQp(int qp, int64_t time_sent_us) { + RTC_DCHECK_RUN_ON(&task_checker_); + framedrop_percent_media_opt_.AddSample(0); + framedrop_percent_all_.AddSample(0); + average_qp_.AddSample(qp); + if (qp_smoother_high_) + qp_smoother_high_->Add(qp, time_sent_us); + if (qp_smoother_low_) + qp_smoother_low_->Add(qp, time_sent_us); +} + +bool QualityScaler::QpFastFilterLow() const { + RTC_DCHECK_RUN_ON(&task_checker_); + size_t num_frames = config_.use_all_drop_reasons + ? framedrop_percent_all_.Size() + : framedrop_percent_media_opt_.Size(); + const size_t kMinNumFrames = 10; + if (num_frames < kMinNumFrames) { + return false; // Wait for more frames before making a decision. + } + absl::optional avg_qp_high = qp_smoother_high_ + ? qp_smoother_high_->GetAvg() + : average_qp_.GetAverageRoundedDown(); + return (avg_qp_high) ? (avg_qp_high.value() <= thresholds_.low) : false; +} + +QualityScaler::CheckQpResult QualityScaler::CheckQp() const { + RTC_DCHECK_RUN_ON(&task_checker_); + // Should be set through InitEncode -> Should be set by now. + RTC_DCHECK_GE(thresholds_.low, 0); + + // If we have not observed at least this many frames we can't make a good + // scaling decision. + const size_t frames = config_.use_all_drop_reasons + ? framedrop_percent_all_.Size() + : framedrop_percent_media_opt_.Size(); + if (frames < min_frames_needed_) { + return CheckQpResult::kInsufficientSamples; + } + + // Check if we should scale down due to high frame drop. + const absl::optional drop_rate = + config_.use_all_drop_reasons + ? framedrop_percent_all_.GetAverageRoundedDown() + : framedrop_percent_media_opt_.GetAverageRoundedDown(); + if (drop_rate && *drop_rate >= kFramedropPercentThreshold) { + RTC_LOG(LS_INFO) << "Reporting high QP, framedrop percent " << *drop_rate; + return CheckQpResult::kHighQp; + } + + // Check if we should scale up or down based on QP. + const absl::optional avg_qp_high = + qp_smoother_high_ ? qp_smoother_high_->GetAvg() + : average_qp_.GetAverageRoundedDown(); + const absl::optional avg_qp_low = + qp_smoother_low_ ? qp_smoother_low_->GetAvg() + : average_qp_.GetAverageRoundedDown(); + if (avg_qp_high && avg_qp_low) { + RTC_LOG(LS_INFO) << "Checking average QP " << *avg_qp_high << " (" + << *avg_qp_low << ")."; + if (*avg_qp_high > thresholds_.high) { + return CheckQpResult::kHighQp; + } + if (*avg_qp_low <= thresholds_.low) { + // QP has been low. We want to try a higher resolution. + return CheckQpResult::kLowQp; + } + } + return CheckQpResult::kNormalQp; +} + +void QualityScaler::ClearSamples() { + RTC_DCHECK_RUN_ON(&task_checker_); + framedrop_percent_media_opt_.Reset(); + framedrop_percent_all_.Reset(); + average_qp_.Reset(); + if (qp_smoother_high_) + qp_smoother_high_->Reset(); + if (qp_smoother_low_) + qp_smoother_low_->Reset(); +} + +QualityScalerQpUsageHandlerInterface::~QualityScalerQpUsageHandlerInterface() {} + +} // namespace webrtc -- cgit v1.2.3