diff options
Diffstat (limited to 'third_party/libwebrtc/video/adaptation')
27 files changed, 5087 insertions, 0 deletions
diff --git a/third_party/libwebrtc/video/adaptation/BUILD.gn b/third_party/libwebrtc/video/adaptation/BUILD.gn new file mode 100644 index 0000000000..d206909853 --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/BUILD.gn @@ -0,0 +1,125 @@ +# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") + +rtc_library("video_adaptation") { + sources = [ + "balanced_constraint.cc", + "balanced_constraint.h", + "bandwidth_quality_scaler_resource.cc", + "bandwidth_quality_scaler_resource.h", + "bitrate_constraint.cc", + "bitrate_constraint.h", + "encode_usage_resource.cc", + "encode_usage_resource.h", + "overuse_frame_detector.cc", + "overuse_frame_detector.h", + "pixel_limit_resource.cc", + "pixel_limit_resource.h", + "quality_rampup_experiment_helper.cc", + "quality_rampup_experiment_helper.h", + "quality_scaler_resource.cc", + "quality_scaler_resource.h", + "video_stream_encoder_resource.cc", + "video_stream_encoder_resource.h", + "video_stream_encoder_resource_manager.cc", + "video_stream_encoder_resource_manager.h", + ] + + deps = [ + "../../api:field_trials_view", + "../../api:rtp_parameters", + "../../api:scoped_refptr", + "../../api:sequence_checker", + "../../api/adaptation:resource_adaptation_api", + "../../api/task_queue:task_queue", + "../../api/units:data_rate", + "../../api/units:time_delta", + "../../api/video:video_adaptation", + "../../api/video:video_frame", + "../../api/video:video_stream_encoder", + "../../api/video_codecs:video_codecs_api", + "../../call/adaptation:resource_adaptation", + "../../modules/video_coding:video_coding_utility", + "../../modules/video_coding/svc:scalability_mode_util", + "../../rtc_base:checks", + "../../rtc_base:event_tracer", + "../../rtc_base:logging", + "../../rtc_base:macromagic", + "../../rtc_base:refcount", + "../../rtc_base:rtc_event", + "../../rtc_base:rtc_numerics", + "../../rtc_base:safe_conversions", + "../../rtc_base:stringutils", + "../../rtc_base:timeutils", + "../../rtc_base/experiments:balanced_degradation_settings", + "../../rtc_base/experiments:field_trial_parser", + "../../rtc_base/experiments:quality_rampup_experiment", + "../../rtc_base/experiments:quality_scaler_settings", + "../../rtc_base/synchronization:mutex", + "../../rtc_base/system:no_unique_address", + "../../rtc_base/task_utils:repeating_task", + "../../system_wrappers:field_trial", + "../../system_wrappers:system_wrappers", + "../../video:video_stream_encoder_interface", + "../../video/config:encoder_config", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/base:core_headers", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +if (rtc_include_tests) { + rtc_library("video_adaptation_tests") { + testonly = true + + defines = [] + sources = [ + "bitrate_constraint_unittest.cc", + "overuse_frame_detector_unittest.cc", + "pixel_limit_resource_unittest.cc", + "quality_scaler_resource_unittest.cc", + ] + deps = [ + ":video_adaptation", + "../../api:field_trials_view", + "../../api:scoped_refptr", + "../../api/task_queue:task_queue", + "../../api/units:time_delta", + "../../api/units:timestamp", + "../../api/video:encoded_image", + "../../api/video:video_adaptation", + "../../api/video:video_frame", + "../../api/video_codecs:scalability_mode", + "../../api/video_codecs:video_codecs_api", + "../../call/adaptation:resource_adaptation", + "../../call/adaptation:resource_adaptation_test_utilities", + "../../modules/video_coding:video_coding_utility", + "../../rtc_base:checks", + "../../rtc_base:logging", + "../../rtc_base:random", + "../../rtc_base:rtc_base_tests_utils", + "../../rtc_base:rtc_event", + "../../rtc_base:rtc_numerics", + "../../rtc_base:task_queue_for_test", + "../../rtc_base:threading", + "../../test:field_trial", + "../../test:rtc_expect_death", + "../../test:scoped_key_value_config", + "../../test:test_support", + "../../test/time_controller:time_controller", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/functional:any_invocable", + "//third_party/abseil-cpp/absl/types:optional", + ] + } +} diff --git a/third_party/libwebrtc/video/adaptation/OWNERS b/third_party/libwebrtc/video/adaptation/OWNERS new file mode 100644 index 0000000000..bd56595d2e --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/OWNERS @@ -0,0 +1,3 @@ +eshr@webrtc.org +hbos@webrtc.org +ilnik@webrtc.org diff --git a/third_party/libwebrtc/video/adaptation/balanced_constraint.cc b/third_party/libwebrtc/video/adaptation/balanced_constraint.cc new file mode 100644 index 0000000000..f9ee08ac87 --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/balanced_constraint.cc @@ -0,0 +1,62 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/adaptation/balanced_constraint.h" + +#include <string> +#include <utility> + +#include "api/sequence_checker.h" + +namespace webrtc { + +BalancedConstraint::BalancedConstraint( + DegradationPreferenceProvider* degradation_preference_provider, + const FieldTrialsView& field_trials) + : encoder_target_bitrate_bps_(absl::nullopt), + balanced_settings_(field_trials), + degradation_preference_provider_(degradation_preference_provider) { + RTC_DCHECK(degradation_preference_provider_); + sequence_checker_.Detach(); +} + +void BalancedConstraint::OnEncoderTargetBitrateUpdated( + absl::optional<uint32_t> encoder_target_bitrate_bps) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + encoder_target_bitrate_bps_ = std::move(encoder_target_bitrate_bps); +} + +bool BalancedConstraint::IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after) const { + RTC_DCHECK_RUN_ON(&sequence_checker_); + // Don't adapt if BalancedDegradationSettings applies and determines this will + // exceed bitrate constraints. + if (degradation_preference_provider_->degradation_preference() == + DegradationPreference::BALANCED) { + int frame_size_pixels = input_state.single_active_stream_pixels().value_or( + input_state.frame_size_pixels().value()); + if (!balanced_settings_.CanAdaptUp( + input_state.video_codec_type(), frame_size_pixels, + encoder_target_bitrate_bps_.value_or(0))) { + return false; + } + if (DidIncreaseResolution(restrictions_before, restrictions_after) && + !balanced_settings_.CanAdaptUpResolution( + input_state.video_codec_type(), frame_size_pixels, + encoder_target_bitrate_bps_.value_or(0))) { + return false; + } + } + return true; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/video/adaptation/balanced_constraint.h b/third_party/libwebrtc/video/adaptation/balanced_constraint.h new file mode 100644 index 0000000000..22c7d2923c --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/balanced_constraint.h @@ -0,0 +1,53 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_ADAPTATION_BALANCED_CONSTRAINT_H_ +#define VIDEO_ADAPTATION_BALANCED_CONSTRAINT_H_ + +#include <string> + +#include "absl/types/optional.h" +#include "api/field_trials_view.h" +#include "api/sequence_checker.h" +#include "call/adaptation/adaptation_constraint.h" +#include "call/adaptation/degradation_preference_provider.h" +#include "rtc_base/experiments/balanced_degradation_settings.h" +#include "rtc_base/system/no_unique_address.h" + +namespace webrtc { + +class BalancedConstraint : public AdaptationConstraint { + public: + BalancedConstraint( + DegradationPreferenceProvider* degradation_preference_provider, + const FieldTrialsView& field_trials); + ~BalancedConstraint() override = default; + + void OnEncoderTargetBitrateUpdated( + absl::optional<uint32_t> encoder_target_bitrate_bps); + + // AdaptationConstraint implementation. + std::string Name() const override { return "BalancedConstraint"; } + bool IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after) const override; + + private: + RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; + absl::optional<uint32_t> encoder_target_bitrate_bps_ + RTC_GUARDED_BY(&sequence_checker_); + const BalancedDegradationSettings balanced_settings_; + const DegradationPreferenceProvider* degradation_preference_provider_; +}; + +} // namespace webrtc + +#endif // VIDEO_ADAPTATION_BALANCED_CONSTRAINT_H_ diff --git a/third_party/libwebrtc/video/adaptation/bandwidth_quality_scaler_resource.cc b/third_party/libwebrtc/video/adaptation/bandwidth_quality_scaler_resource.cc new file mode 100644 index 0000000000..485019f309 --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/bandwidth_quality_scaler_resource.cc @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/adaptation/bandwidth_quality_scaler_resource.h" + +#include <utility> + +#include "rtc_base/checks.h" +#include "rtc_base/experiments/balanced_degradation_settings.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" + +namespace webrtc { + +// static +rtc::scoped_refptr<BandwidthQualityScalerResource> +BandwidthQualityScalerResource::Create() { + return rtc::make_ref_counted<BandwidthQualityScalerResource>(); +} + +BandwidthQualityScalerResource::BandwidthQualityScalerResource() + : VideoStreamEncoderResource("BandwidthQualityScalerResource"), + bandwidth_quality_scaler_(nullptr) {} + +BandwidthQualityScalerResource::~BandwidthQualityScalerResource() { + RTC_DCHECK(!bandwidth_quality_scaler_); +} + +bool BandwidthQualityScalerResource::is_started() const { + RTC_DCHECK_RUN_ON(encoder_queue()); + return bandwidth_quality_scaler_.get(); +} + +void BandwidthQualityScalerResource::StartCheckForOveruse( + const std::vector<VideoEncoder::ResolutionBitrateLimits>& + resolution_bitrate_limits) { + RTC_DCHECK_RUN_ON(encoder_queue()); + RTC_DCHECK(!is_started()); + bandwidth_quality_scaler_ = std::make_unique<BandwidthQualityScaler>(this); + + // If the configuration parameters more than one, we should define and + // declare the function BandwidthQualityScaler::Initialize() and call it. + bandwidth_quality_scaler_->SetResolutionBitrateLimits( + resolution_bitrate_limits); +} + +void BandwidthQualityScalerResource::StopCheckForOveruse() { + RTC_DCHECK_RUN_ON(encoder_queue()); + RTC_DCHECK(is_started()); + // Ensure we have no pending callbacks. This makes it safe to destroy the + // BandwidthQualityScaler and even task queues with tasks in-flight. + bandwidth_quality_scaler_.reset(); +} + +void BandwidthQualityScalerResource::OnReportUsageBandwidthHigh() { + OnResourceUsageStateMeasured(ResourceUsageState::kOveruse); +} + +void BandwidthQualityScalerResource::OnReportUsageBandwidthLow() { + OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse); +} + +void BandwidthQualityScalerResource::OnEncodeCompleted( + const EncodedImage& encoded_image, + int64_t time_sent_in_us, + int64_t encoded_image_size_bytes) { + RTC_DCHECK_RUN_ON(encoder_queue()); + + if (bandwidth_quality_scaler_) { + bandwidth_quality_scaler_->ReportEncodeInfo( + encoded_image_size_bytes, time_sent_in_us / 1000, + encoded_image._encodedWidth, encoded_image._encodedHeight); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/video/adaptation/bandwidth_quality_scaler_resource.h b/third_party/libwebrtc/video/adaptation/bandwidth_quality_scaler_resource.h new file mode 100644 index 0000000000..a57c9907a4 --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/bandwidth_quality_scaler_resource.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_ADAPTATION_BANDWIDTH_QUALITY_SCALER_RESOURCE_H_ +#define VIDEO_ADAPTATION_BANDWIDTH_QUALITY_SCALER_RESOURCE_H_ + +#include <memory> +#include <queue> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "api/video/video_adaptation_reason.h" +#include "api/video_codecs/video_encoder.h" +#include "call/adaptation/degradation_preference_provider.h" +#include "call/adaptation/resource_adaptation_processor_interface.h" +#include "modules/video_coding/utility/bandwidth_quality_scaler.h" +#include "video/adaptation/video_stream_encoder_resource.h" + +namespace webrtc { + +// Handles interaction with the BandwidthQualityScaler. +class BandwidthQualityScalerResource + : public VideoStreamEncoderResource, + public BandwidthQualityScalerUsageHandlerInterface { + public: + static rtc::scoped_refptr<BandwidthQualityScalerResource> Create(); + + BandwidthQualityScalerResource(); + ~BandwidthQualityScalerResource() override; + + bool is_started() const; + + void OnEncodeCompleted(const EncodedImage& encoded_image, + int64_t time_sent_in_us, + int64_t encoded_image_size_bytes); + + void StartCheckForOveruse( + const std::vector<VideoEncoder::ResolutionBitrateLimits>& + resolution_bitrate_limits); + void StopCheckForOveruse(); + + // BandwidthScalerQpUsageHandlerInterface implementation. + void OnReportUsageBandwidthHigh() override; + void OnReportUsageBandwidthLow() override; + + private: + std::unique_ptr<BandwidthQualityScaler> bandwidth_quality_scaler_ + RTC_GUARDED_BY(encoder_queue()); +}; + +} // namespace webrtc + +#endif // VIDEO_ADAPTATION_BANDWIDTH_QUALITY_SCALER_RESOURCE_H_ diff --git a/third_party/libwebrtc/video/adaptation/bitrate_constraint.cc b/third_party/libwebrtc/video/adaptation/bitrate_constraint.cc new file mode 100644 index 0000000000..bc36723d48 --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/bitrate_constraint.cc @@ -0,0 +1,87 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/adaptation/bitrate_constraint.h" + +#include <utility> +#include <vector> + +#include "api/sequence_checker.h" +#include "call/adaptation/video_stream_adapter.h" +#include "video/adaptation/video_stream_encoder_resource_manager.h" + +namespace webrtc { + +BitrateConstraint::BitrateConstraint() + : encoder_settings_(absl::nullopt), + encoder_target_bitrate_bps_(absl::nullopt) { + sequence_checker_.Detach(); +} + +void BitrateConstraint::OnEncoderSettingsUpdated( + absl::optional<EncoderSettings> encoder_settings) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + encoder_settings_ = std::move(encoder_settings); +} + +void BitrateConstraint::OnEncoderTargetBitrateUpdated( + absl::optional<uint32_t> encoder_target_bitrate_bps) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + encoder_target_bitrate_bps_ = std::move(encoder_target_bitrate_bps); +} + +// Checks if resolution is allowed to adapt up based on the current bitrate and +// ResolutionBitrateLimits.min_start_bitrate_bps for the next higher resolution. +// Bitrate limits usage is restricted to a single active stream/layer (e.g. when +// quality scaling is enabled). +bool BitrateConstraint::IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after) const { + RTC_DCHECK_RUN_ON(&sequence_checker_); + // Make sure bitrate limits are not violated. + if (DidIncreaseResolution(restrictions_before, restrictions_after)) { + if (!encoder_settings_.has_value()) { + return true; + } + + uint32_t bitrate_bps = encoder_target_bitrate_bps_.value_or(0); + if (bitrate_bps == 0) { + return true; + } + + if (VideoStreamEncoderResourceManager::IsSimulcastOrMultipleSpatialLayers( + encoder_settings_->encoder_config())) { + // Resolution bitrate limits usage is restricted to singlecast. + return true; + } + + absl::optional<int> current_frame_size_px = + input_state.single_active_stream_pixels(); + if (!current_frame_size_px.has_value()) { + return true; + } + + absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits = + encoder_settings_->encoder_info().GetEncoderBitrateLimitsForResolution( + // Need some sort of expected resulting pixels to be used + // instead of unrestricted. + GetHigherResolutionThan(*current_frame_size_px)); + + if (bitrate_limits.has_value()) { + RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, *current_frame_size_px); + return bitrate_bps >= + static_cast<uint32_t>(bitrate_limits->min_start_bitrate_bps); + } + } + return true; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/video/adaptation/bitrate_constraint.h b/third_party/libwebrtc/video/adaptation/bitrate_constraint.h new file mode 100644 index 0000000000..a608e5db5d --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/bitrate_constraint.h @@ -0,0 +1,53 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_ADAPTATION_BITRATE_CONSTRAINT_H_ +#define VIDEO_ADAPTATION_BITRATE_CONSTRAINT_H_ + +#include <string> + +#include "absl/types/optional.h" +#include "api/sequence_checker.h" +#include "call/adaptation/adaptation_constraint.h" +#include "call/adaptation/encoder_settings.h" +#include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_input_state.h" +#include "rtc_base/system/no_unique_address.h" + +namespace webrtc { + +class BitrateConstraint : public AdaptationConstraint { + public: + BitrateConstraint(); + ~BitrateConstraint() override = default; + + void OnEncoderSettingsUpdated( + absl::optional<EncoderSettings> encoder_settings); + void OnEncoderTargetBitrateUpdated( + absl::optional<uint32_t> encoder_target_bitrate_bps); + + // AdaptationConstraint implementation. + std::string Name() const override { return "BitrateConstraint"; } + bool IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after) const override; + + private: + RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; + absl::optional<EncoderSettings> encoder_settings_ + RTC_GUARDED_BY(&sequence_checker_); + absl::optional<uint32_t> encoder_target_bitrate_bps_ + RTC_GUARDED_BY(&sequence_checker_); +}; + +} // namespace webrtc + +#endif // VIDEO_ADAPTATION_BITRATE_CONSTRAINT_H_ diff --git a/third_party/libwebrtc/video/adaptation/bitrate_constraint_unittest.cc b/third_party/libwebrtc/video/adaptation/bitrate_constraint_unittest.cc new file mode 100644 index 0000000000..f9cb87e3c1 --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/bitrate_constraint_unittest.cc @@ -0,0 +1,320 @@ +/* + * Copyright 2021 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/adaptation/bitrate_constraint.h" + +#include <utility> +#include <vector> + +#include "api/video_codecs/scalability_mode.h" +#include "api/video_codecs/video_encoder.h" +#include "call/adaptation/encoder_settings.h" +#include "call/adaptation/test/fake_frame_rate_provider.h" +#include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_input_state_provider.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { +const VideoSourceRestrictions k180p{/*max_pixels_per_frame=*/320 * 180, + /*target_pixels_per_frame=*/320 * 180, + /*max_frame_rate=*/30}; +const VideoSourceRestrictions k360p{/*max_pixels_per_frame=*/640 * 360, + /*target_pixels_per_frame=*/640 * 360, + /*max_frame_rate=*/30}; +const VideoSourceRestrictions k720p{/*max_pixels_per_frame=*/1280 * 720, + /*target_pixels_per_frame=*/1280 * 720, + /*max_frame_rate=*/30}; + +struct TestParams { + bool active; + absl::optional<ScalabilityMode> scalability_mode; +}; + +void FillCodecConfig(VideoCodec* video_codec, + VideoEncoderConfig* encoder_config, + int width_px, + int height_px, + const std::vector<TestParams>& params, + bool svc) { + size_t num_layers = params.size(); + video_codec->codecType = kVideoCodecVP8; + video_codec->numberOfSimulcastStreams = num_layers; + + encoder_config->number_of_streams = svc ? 1 : num_layers; + encoder_config->simulcast_layers.resize(num_layers); + + for (size_t layer_idx = 0; layer_idx < num_layers; ++layer_idx) { + int layer_width_px = width_px >> (num_layers - 1 - layer_idx); + int layer_height_px = height_px >> (num_layers - 1 - layer_idx); + + if (params[layer_idx].scalability_mode) + video_codec->SetScalabilityMode(*params[layer_idx].scalability_mode); + video_codec->simulcastStream[layer_idx].active = params[layer_idx].active; + video_codec->simulcastStream[layer_idx].width = layer_width_px; + video_codec->simulcastStream[layer_idx].height = layer_height_px; + + encoder_config->simulcast_layers[layer_idx].scalability_mode = + params[layer_idx].scalability_mode; + encoder_config->simulcast_layers[layer_idx].active = + params[layer_idx].active; + encoder_config->simulcast_layers[layer_idx].width = layer_width_px; + encoder_config->simulcast_layers[layer_idx].height = layer_height_px; + } +} + +constexpr int kStartBitrateBps360p = 500000; +constexpr int kStartBitrateBps720p = 1000000; + +VideoEncoder::EncoderInfo MakeEncoderInfo() { + VideoEncoder::EncoderInfo encoder_info; + encoder_info.resolution_bitrate_limits = { + {640 * 360, kStartBitrateBps360p, 0, 5000000}, + {1280 * 720, kStartBitrateBps720p, 0, 5000000}, + {1920 * 1080, 2000000, 0, 5000000}}; + return encoder_info; +} + +} // namespace + +class BitrateConstraintTest : public ::testing::Test { + public: + BitrateConstraintTest() + : frame_rate_provider_(), input_state_provider_(&frame_rate_provider_) {} + + protected: + void OnEncoderSettingsUpdated(int width_px, + int height_px, + const std::vector<TestParams>& params, + bool svc = false) { + VideoCodec video_codec; + VideoEncoderConfig encoder_config; + FillCodecConfig(&video_codec, &encoder_config, width_px, height_px, params, + svc); + + EncoderSettings encoder_settings(MakeEncoderInfo(), + std::move(encoder_config), video_codec); + bitrate_constraint_.OnEncoderSettingsUpdated(encoder_settings); + input_state_provider_.OnEncoderSettingsChanged(encoder_settings); + } + + FakeFrameRateProvider frame_rate_provider_; + VideoStreamInputStateProvider input_state_provider_; + BitrateConstraint bitrate_constraint_; +}; + +TEST_F(BitrateConstraintTest, AdaptUpAllowedAtSinglecastIfBitrateIsEnough) { + OnEncoderSettingsUpdated(/*width_px=*/640, /*height_px=*/360, + {{.active = true}}); + + bitrate_constraint_.OnEncoderTargetBitrateUpdated(kStartBitrateBps720p); + + EXPECT_TRUE(bitrate_constraint_.IsAdaptationUpAllowed( + input_state_provider_.InputState(), + /*restrictions_before=*/k360p, + /*restrictions_after=*/k720p)); +} + +TEST_F(BitrateConstraintTest, + AdaptUpDisallowedAtSinglecastIfBitrateIsNotEnough) { + OnEncoderSettingsUpdated(/*width_px=*/640, /*height_px=*/360, + {{.active = true}}); + + // 1 bps less than needed for 720p. + bitrate_constraint_.OnEncoderTargetBitrateUpdated(kStartBitrateBps720p - 1); + + EXPECT_FALSE(bitrate_constraint_.IsAdaptationUpAllowed( + input_state_provider_.InputState(), + /*restrictions_before=*/k360p, + /*restrictions_after=*/k720p)); +} + +TEST_F(BitrateConstraintTest, + AdaptUpAllowedAtSinglecastIfBitrateIsEnoughForOneSpatialLayer) { + OnEncoderSettingsUpdated( + /*width_px=*/640, /*height_px=*/360, + {{.active = true, .scalability_mode = ScalabilityMode::kL1T1}}); + + bitrate_constraint_.OnEncoderTargetBitrateUpdated(kStartBitrateBps720p); + + EXPECT_TRUE(bitrate_constraint_.IsAdaptationUpAllowed( + input_state_provider_.InputState(), + /*restrictions_before=*/k360p, + /*restrictions_after=*/k720p)); +} + +TEST_F(BitrateConstraintTest, + AdaptUpDisallowedAtSinglecastIfBitrateIsNotEnoughForOneSpatialLayer) { + OnEncoderSettingsUpdated( + /*width_px=*/640, /*height_px=*/360, + {{.active = true, .scalability_mode = ScalabilityMode::kL1T1}}); + + // 1 bps less than needed for 720p. + bitrate_constraint_.OnEncoderTargetBitrateUpdated(kStartBitrateBps720p - 1); + + EXPECT_FALSE(bitrate_constraint_.IsAdaptationUpAllowed( + input_state_provider_.InputState(), + /*restrictions_before=*/k360p, + /*restrictions_after=*/k720p)); +} + +TEST_F(BitrateConstraintTest, + AdaptUpAllowedAtSinglecastIfBitrateIsNotEnoughForMultipleSpatialLayers) { + OnEncoderSettingsUpdated( + /*width_px=*/640, /*height_px=*/360, + {{.active = true, .scalability_mode = ScalabilityMode::kL2T1}}); + + // 1 bps less than needed for 720p. + bitrate_constraint_.OnEncoderTargetBitrateUpdated(kStartBitrateBps720p - 1); + + EXPECT_TRUE(bitrate_constraint_.IsAdaptationUpAllowed( + input_state_provider_.InputState(), + /*restrictions_before=*/k360p, + /*restrictions_after=*/k720p)); +} + +TEST_F(BitrateConstraintTest, + AdaptUpAllowedAtSinglecastUpperLayerActiveIfBitrateIsEnough) { + OnEncoderSettingsUpdated( + /*width_px=*/640, /*height_px=*/360, + {{.active = false, .scalability_mode = ScalabilityMode::kL2T1}, + {.active = true}}); + + bitrate_constraint_.OnEncoderTargetBitrateUpdated(kStartBitrateBps720p); + + EXPECT_TRUE(bitrate_constraint_.IsAdaptationUpAllowed( + input_state_provider_.InputState(), + /*restrictions_before=*/k360p, + /*restrictions_after=*/k720p)); +} + +TEST_F(BitrateConstraintTest, + AdaptUpDisallowedAtSinglecastUpperLayerActiveIfBitrateIsNotEnough) { + OnEncoderSettingsUpdated( + /*width_px=*/640, /*height_px=*/360, + {{.active = false, .scalability_mode = ScalabilityMode::kL2T1}, + {.active = true}}); + + // 1 bps less than needed for 720p. + bitrate_constraint_.OnEncoderTargetBitrateUpdated(kStartBitrateBps720p - 1); + + EXPECT_FALSE(bitrate_constraint_.IsAdaptationUpAllowed( + input_state_provider_.InputState(), + /*restrictions_before=*/k360p, + /*restrictions_after=*/k720p)); +} + +TEST_F(BitrateConstraintTest, AdaptUpAllowedLowestActiveIfBitrateIsNotEnough) { + OnEncoderSettingsUpdated(/*width_px=*/640, /*height_px=*/360, + {{.active = true}, {.active = false}}); + + // 1 bps less than needed for 360p. + bitrate_constraint_.OnEncoderTargetBitrateUpdated(kStartBitrateBps360p - 1); + + EXPECT_TRUE(bitrate_constraint_.IsAdaptationUpAllowed( + input_state_provider_.InputState(), + /*restrictions_before=*/k180p, + /*restrictions_after=*/k360p)); +} + +TEST_F(BitrateConstraintTest, + AdaptUpAllowedLowestActiveIfBitrateIsNotEnoughForOneSpatialLayer) { + OnEncoderSettingsUpdated( + /*width_px=*/640, /*height_px=*/360, + {{.active = true, .scalability_mode = ScalabilityMode::kL1T2}, + {.active = false}}); + + // 1 bps less than needed for 360p. + bitrate_constraint_.OnEncoderTargetBitrateUpdated(kStartBitrateBps360p - 1); + + EXPECT_TRUE(bitrate_constraint_.IsAdaptationUpAllowed( + input_state_provider_.InputState(), + /*restrictions_before=*/k180p, + /*restrictions_after=*/k360p)); +} + +TEST_F(BitrateConstraintTest, + AdaptUpAllowedLowestActiveIfBitrateIsEnoughForOneSpatialLayerSvc) { + OnEncoderSettingsUpdated( + /*width_px=*/640, /*height_px=*/360, + {{.active = true, .scalability_mode = ScalabilityMode::kL1T1}, + {.active = false}}, + /*svc=*/true); + + bitrate_constraint_.OnEncoderTargetBitrateUpdated(kStartBitrateBps360p); + + EXPECT_TRUE(bitrate_constraint_.IsAdaptationUpAllowed( + input_state_provider_.InputState(), + /*restrictions_before=*/k180p, + /*restrictions_after=*/k360p)); +} + +TEST_F(BitrateConstraintTest, + AdaptUpDisallowedLowestActiveIfBitrateIsNotEnoughForOneSpatialLayerSvc) { + OnEncoderSettingsUpdated( + /*width_px=*/640, /*height_px=*/360, + {{.active = true, .scalability_mode = ScalabilityMode::kL1T1}, + {.active = false}}, + /*svc=*/true); + + // 1 bps less than needed for 360p. + bitrate_constraint_.OnEncoderTargetBitrateUpdated(kStartBitrateBps360p - 1); + + EXPECT_FALSE(bitrate_constraint_.IsAdaptationUpAllowed( + input_state_provider_.InputState(), + /*restrictions_before=*/k180p, + /*restrictions_after=*/k360p)); +} + +TEST_F(BitrateConstraintTest, + AdaptUpAllowedLowestActiveIfBitrateIsNotEnoughForTwoSpatialLayersSvc) { + OnEncoderSettingsUpdated( + /*width_px=*/640, /*height_px=*/360, + {{.active = true, .scalability_mode = ScalabilityMode::kL2T1}, + {.active = false}}, + /*svc=*/true); + + // 1 bps less than needed for 360p. + bitrate_constraint_.OnEncoderTargetBitrateUpdated(kStartBitrateBps360p - 1); + + EXPECT_TRUE(bitrate_constraint_.IsAdaptationUpAllowed( + input_state_provider_.InputState(), + /*restrictions_before=*/k180p, + /*restrictions_after=*/k360p)); +} + +TEST_F(BitrateConstraintTest, AdaptUpAllowedAtSimulcastIfBitrateIsNotEnough) { + OnEncoderSettingsUpdated(/*width_px=*/640, /*height_px=*/360, + {{.active = true}, {.active = true}}); + + // 1 bps less than needed for 720p. + bitrate_constraint_.OnEncoderTargetBitrateUpdated(kStartBitrateBps720p - 1); + + EXPECT_TRUE(bitrate_constraint_.IsAdaptationUpAllowed( + input_state_provider_.InputState(), + /*restrictions_before=*/k360p, + /*restrictions_after=*/k720p)); +} + +TEST_F(BitrateConstraintTest, + AdaptUpInFpsAllowedAtNoResolutionIncreaseIfBitrateIsNotEnough) { + OnEncoderSettingsUpdated(/*width_px=*/640, /*height_px=*/360, + {{.active = true}}); + + bitrate_constraint_.OnEncoderTargetBitrateUpdated(1); + + EXPECT_TRUE(bitrate_constraint_.IsAdaptationUpAllowed( + input_state_provider_.InputState(), + /*restrictions_before=*/k360p, + /*restrictions_after=*/k360p)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/video/adaptation/encode_usage_resource.cc b/third_party/libwebrtc/video/adaptation/encode_usage_resource.cc new file mode 100644 index 0000000000..4a97881b04 --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/encode_usage_resource.cc @@ -0,0 +1,105 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/adaptation/encode_usage_resource.h" + +#include <limits> +#include <utility> + +#include "rtc_base/checks.h" + +namespace webrtc { + +// static +rtc::scoped_refptr<EncodeUsageResource> EncodeUsageResource::Create( + std::unique_ptr<OveruseFrameDetector> overuse_detector) { + return rtc::make_ref_counted<EncodeUsageResource>( + std::move(overuse_detector)); +} + +EncodeUsageResource::EncodeUsageResource( + std::unique_ptr<OveruseFrameDetector> overuse_detector) + : VideoStreamEncoderResource("EncoderUsageResource"), + overuse_detector_(std::move(overuse_detector)), + is_started_(false), + target_frame_rate_(absl::nullopt) { + RTC_DCHECK(overuse_detector_); +} + +EncodeUsageResource::~EncodeUsageResource() {} + +bool EncodeUsageResource::is_started() const { + RTC_DCHECK_RUN_ON(encoder_queue()); + return is_started_; +} + +void EncodeUsageResource::StartCheckForOveruse(CpuOveruseOptions options) { + RTC_DCHECK_RUN_ON(encoder_queue()); + RTC_DCHECK(!is_started_); + overuse_detector_->StartCheckForOveruse(TaskQueueBase::Current(), + std::move(options), this); + is_started_ = true; + overuse_detector_->OnTargetFramerateUpdated(TargetFrameRateAsInt()); +} + +void EncodeUsageResource::StopCheckForOveruse() { + RTC_DCHECK_RUN_ON(encoder_queue()); + overuse_detector_->StopCheckForOveruse(); + is_started_ = false; +} + +void EncodeUsageResource::SetTargetFrameRate( + absl::optional<double> target_frame_rate) { + RTC_DCHECK_RUN_ON(encoder_queue()); + if (target_frame_rate == target_frame_rate_) + return; + target_frame_rate_ = target_frame_rate; + if (is_started_) + overuse_detector_->OnTargetFramerateUpdated(TargetFrameRateAsInt()); +} + +void EncodeUsageResource::OnEncodeStarted(const VideoFrame& cropped_frame, + int64_t time_when_first_seen_us) { + RTC_DCHECK_RUN_ON(encoder_queue()); + // TODO(hbos): Rename FrameCaptured() to something more appropriate (e.g. + // "OnEncodeStarted"?) or revise usage. + overuse_detector_->FrameCaptured(cropped_frame, time_when_first_seen_us); +} + +void EncodeUsageResource::OnEncodeCompleted( + uint32_t timestamp, + int64_t time_sent_in_us, + int64_t capture_time_us, + absl::optional<int> encode_duration_us) { + RTC_DCHECK_RUN_ON(encoder_queue()); + // TODO(hbos): Rename FrameSent() to something more appropriate (e.g. + // "OnEncodeCompleted"?). + overuse_detector_->FrameSent(timestamp, time_sent_in_us, capture_time_us, + encode_duration_us); +} + +void EncodeUsageResource::AdaptUp() { + RTC_DCHECK_RUN_ON(encoder_queue()); + OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse); +} + +void EncodeUsageResource::AdaptDown() { + RTC_DCHECK_RUN_ON(encoder_queue()); + OnResourceUsageStateMeasured(ResourceUsageState::kOveruse); +} + +int EncodeUsageResource::TargetFrameRateAsInt() { + RTC_DCHECK_RUN_ON(encoder_queue()); + return target_frame_rate_.has_value() + ? static_cast<int>(target_frame_rate_.value()) + : std::numeric_limits<int>::max(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/video/adaptation/encode_usage_resource.h b/third_party/libwebrtc/video/adaptation/encode_usage_resource.h new file mode 100644 index 0000000000..c391132e57 --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/encode_usage_resource.h @@ -0,0 +1,68 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_ADAPTATION_ENCODE_USAGE_RESOURCE_H_ +#define VIDEO_ADAPTATION_ENCODE_USAGE_RESOURCE_H_ + +#include <memory> +#include <string> + +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "api/video/video_adaptation_reason.h" +#include "video/adaptation/overuse_frame_detector.h" +#include "video/adaptation/video_stream_encoder_resource.h" + +namespace webrtc { + +// Handles interaction with the OveruseDetector. +// TODO(hbos): Add unittests specific to this class, it is currently only tested +// indirectly by usage in the ResourceAdaptationProcessor (which is only tested +// because of its usage in VideoStreamEncoder); all tests are currently in +// video_stream_encoder_unittest.cc. +class EncodeUsageResource : public VideoStreamEncoderResource, + public OveruseFrameDetectorObserverInterface { + public: + static rtc::scoped_refptr<EncodeUsageResource> Create( + std::unique_ptr<OveruseFrameDetector> overuse_detector); + + explicit EncodeUsageResource( + std::unique_ptr<OveruseFrameDetector> overuse_detector); + ~EncodeUsageResource() override; + + bool is_started() const; + + void StartCheckForOveruse(CpuOveruseOptions options); + void StopCheckForOveruse(); + + void SetTargetFrameRate(absl::optional<double> target_frame_rate); + void OnEncodeStarted(const VideoFrame& cropped_frame, + int64_t time_when_first_seen_us); + void OnEncodeCompleted(uint32_t timestamp, + int64_t time_sent_in_us, + int64_t capture_time_us, + absl::optional<int> encode_duration_us); + + // OveruseFrameDetectorObserverInterface implementation. + void AdaptUp() override; + void AdaptDown() override; + + private: + int TargetFrameRateAsInt(); + + const std::unique_ptr<OveruseFrameDetector> overuse_detector_ + RTC_GUARDED_BY(encoder_queue()); + bool is_started_ RTC_GUARDED_BY(encoder_queue()); + absl::optional<double> target_frame_rate_ RTC_GUARDED_BY(encoder_queue()); +}; + +} // namespace webrtc + +#endif // VIDEO_ADAPTATION_ENCODE_USAGE_RESOURCE_H_ diff --git a/third_party/libwebrtc/video/adaptation/overuse_frame_detector.cc b/third_party/libwebrtc/video/adaptation/overuse_frame_detector.cc new file mode 100644 index 0000000000..9836a466b5 --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/overuse_frame_detector.cc @@ -0,0 +1,722 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/adaptation/overuse_frame_detector.h" + +#include <math.h> +#include <stdio.h> + +#include <algorithm> +#include <list> +#include <map> +#include <memory> +#include <string> +#include <utility> + +#include "api/video/video_frame.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/exp_filter.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/field_trial.h" + +#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) +#include <mach/mach.h> +#endif // defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) + +namespace webrtc { + +namespace { +const int64_t kCheckForOveruseIntervalMs = 5000; +const int64_t kTimeToFirstCheckForOveruseMs = 100; + +// Delay between consecutive rampups. (Used for quick recovery.) +const int kQuickRampUpDelayMs = 10 * 1000; +// Delay between rampup attempts. Initially uses standard, scales up to max. +const int kStandardRampUpDelayMs = 40 * 1000; +const int kMaxRampUpDelayMs = 240 * 1000; +// Expontential back-off factor, to prevent annoying up-down behaviour. +const double kRampUpBackoffFactor = 2.0; + +// Max number of overuses detected before always applying the rampup delay. +const int kMaxOverusesBeforeApplyRampupDelay = 4; + +// The maximum exponent to use in VCMExpFilter. +const float kMaxExp = 7.0f; +// Default value used before first reconfiguration. +const int kDefaultFrameRate = 30; +// Default sample diff, default frame rate. +const float kDefaultSampleDiffMs = 1000.0f / kDefaultFrameRate; +// A factor applied to the sample diff on OnTargetFramerateUpdated to determine +// a max limit for the sample diff. For instance, with a framerate of 30fps, +// the sample diff is capped to (1000 / 30) * 1.35 = 45ms. This prevents +// triggering too soon if there are individual very large outliers. +const float kMaxSampleDiffMarginFactor = 1.35f; +// Minimum framerate allowed for usage calculation. This prevents crazy long +// encode times from being accepted if the frame rate happens to be low. +const int kMinFramerate = 7; +const int kMaxFramerate = 30; + +// Class for calculating the processing usage on the send-side (the average +// processing time of a frame divided by the average time difference between +// captured frames). +class SendProcessingUsage1 : public OveruseFrameDetector::ProcessingUsage { + public: + explicit SendProcessingUsage1(const CpuOveruseOptions& options) + : kWeightFactorFrameDiff(0.998f), + kWeightFactorProcessing(0.995f), + kInitialSampleDiffMs(40.0f), + options_(options), + count_(0), + last_processed_capture_time_us_(-1), + max_sample_diff_ms_(kDefaultSampleDiffMs * kMaxSampleDiffMarginFactor), + filtered_processing_ms_(new rtc::ExpFilter(kWeightFactorProcessing)), + filtered_frame_diff_ms_(new rtc::ExpFilter(kWeightFactorFrameDiff)) { + Reset(); + } + ~SendProcessingUsage1() override {} + + void Reset() override { + frame_timing_.clear(); + count_ = 0; + last_processed_capture_time_us_ = -1; + max_sample_diff_ms_ = kDefaultSampleDiffMs * kMaxSampleDiffMarginFactor; + filtered_frame_diff_ms_->Reset(kWeightFactorFrameDiff); + filtered_frame_diff_ms_->Apply(1.0f, kInitialSampleDiffMs); + filtered_processing_ms_->Reset(kWeightFactorProcessing); + filtered_processing_ms_->Apply(1.0f, InitialProcessingMs()); + } + + void SetMaxSampleDiffMs(float diff_ms) override { + max_sample_diff_ms_ = diff_ms; + } + + void FrameCaptured(const VideoFrame& frame, + int64_t time_when_first_seen_us, + int64_t last_capture_time_us) override { + if (last_capture_time_us != -1) + AddCaptureSample(1e-3 * (time_when_first_seen_us - last_capture_time_us)); + + frame_timing_.push_back(FrameTiming(frame.timestamp_us(), frame.timestamp(), + time_when_first_seen_us)); + } + + absl::optional<int> FrameSent( + uint32_t timestamp, + int64_t time_sent_in_us, + int64_t /* capture_time_us */, + absl::optional<int> /* encode_duration_us */) override { + absl::optional<int> encode_duration_us; + // Delay before reporting actual encoding time, used to have the ability to + // detect total encoding time when encoding more than one layer. Encoding is + // here assumed to finish within a second (or that we get enough long-time + // samples before one second to trigger an overuse even when this is not the + // case). + static const int64_t kEncodingTimeMeasureWindowMs = 1000; + for (auto& it : frame_timing_) { + if (it.timestamp == timestamp) { + it.last_send_us = time_sent_in_us; + break; + } + } + // TODO(pbos): Handle the case/log errors when not finding the corresponding + // frame (either very slow encoding or incorrect wrong timestamps returned + // from the encoder). + // This is currently the case for all frames on ChromeOS, so logging them + // would be spammy, and triggering overuse would be wrong. + // https://crbug.com/350106 + while (!frame_timing_.empty()) { + FrameTiming timing = frame_timing_.front(); + if (time_sent_in_us - timing.capture_us < + kEncodingTimeMeasureWindowMs * rtc::kNumMicrosecsPerMillisec) { + break; + } + if (timing.last_send_us != -1) { + encode_duration_us.emplace( + static_cast<int>(timing.last_send_us - timing.capture_us)); + + if (last_processed_capture_time_us_ != -1) { + int64_t diff_us = timing.capture_us - last_processed_capture_time_us_; + AddSample(1e-3 * (*encode_duration_us), 1e-3 * diff_us); + } + last_processed_capture_time_us_ = timing.capture_us; + } + frame_timing_.pop_front(); + } + return encode_duration_us; + } + + int Value() override { + if (count_ < static_cast<uint32_t>(options_.min_frame_samples)) { + return static_cast<int>(InitialUsageInPercent() + 0.5f); + } + float frame_diff_ms = std::max(filtered_frame_diff_ms_->filtered(), 1.0f); + frame_diff_ms = std::min(frame_diff_ms, max_sample_diff_ms_); + float encode_usage_percent = + 100.0f * filtered_processing_ms_->filtered() / frame_diff_ms; + return static_cast<int>(encode_usage_percent + 0.5); + } + + private: + struct FrameTiming { + FrameTiming(int64_t capture_time_us, uint32_t timestamp, int64_t now) + : capture_time_us(capture_time_us), + timestamp(timestamp), + capture_us(now), + last_send_us(-1) {} + int64_t capture_time_us; + uint32_t timestamp; + int64_t capture_us; + int64_t last_send_us; + }; + + void AddCaptureSample(float sample_ms) { + float exp = sample_ms / kDefaultSampleDiffMs; + exp = std::min(exp, kMaxExp); + filtered_frame_diff_ms_->Apply(exp, sample_ms); + } + + void AddSample(float processing_ms, int64_t diff_last_sample_ms) { + ++count_; + float exp = diff_last_sample_ms / kDefaultSampleDiffMs; + exp = std::min(exp, kMaxExp); + filtered_processing_ms_->Apply(exp, processing_ms); + } + + float InitialUsageInPercent() const { + // Start in between the underuse and overuse threshold. + return (options_.low_encode_usage_threshold_percent + + options_.high_encode_usage_threshold_percent) / + 2.0f; + } + + float InitialProcessingMs() const { + return InitialUsageInPercent() * kInitialSampleDiffMs / 100; + } + + const float kWeightFactorFrameDiff; + const float kWeightFactorProcessing; + const float kInitialSampleDiffMs; + + const CpuOveruseOptions options_; + std::list<FrameTiming> frame_timing_; + uint64_t count_; + int64_t last_processed_capture_time_us_; + float max_sample_diff_ms_; + std::unique_ptr<rtc::ExpFilter> filtered_processing_ms_; + std::unique_ptr<rtc::ExpFilter> filtered_frame_diff_ms_; +}; + +// New cpu load estimator. +// TODO(bugs.webrtc.org/8504): For some period of time, we need to +// switch between the two versions of the estimator for experiments. +// When problems are sorted out, the old estimator should be deleted. +class SendProcessingUsage2 : public OveruseFrameDetector::ProcessingUsage { + public: + explicit SendProcessingUsage2(const CpuOveruseOptions& options) + : options_(options) { + Reset(); + } + ~SendProcessingUsage2() override = default; + + void Reset() override { + prev_time_us_ = -1; + // Start in between the underuse and overuse threshold. + load_estimate_ = (options_.low_encode_usage_threshold_percent + + options_.high_encode_usage_threshold_percent) / + 200.0; + } + + void SetMaxSampleDiffMs(float /* diff_ms */) override {} + + void FrameCaptured(const VideoFrame& frame, + int64_t time_when_first_seen_us, + int64_t last_capture_time_us) override {} + + absl::optional<int> FrameSent( + uint32_t /* timestamp */, + int64_t /* time_sent_in_us */, + int64_t capture_time_us, + absl::optional<int> encode_duration_us) override { + if (encode_duration_us) { + int duration_per_frame_us = + DurationPerInputFrame(capture_time_us, *encode_duration_us); + if (prev_time_us_ != -1) { + if (capture_time_us < prev_time_us_) { + // The weighting in AddSample assumes that samples are processed with + // non-decreasing measurement timestamps. We could implement + // appropriate weights for samples arriving late, but since it is a + // rare case, keep things simple, by just pushing those measurements a + // bit forward in time. + capture_time_us = prev_time_us_; + } + AddSample(1e-6 * duration_per_frame_us, + 1e-6 * (capture_time_us - prev_time_us_)); + } + } + prev_time_us_ = capture_time_us; + + return encode_duration_us; + } + + private: + void AddSample(double encode_time, double diff_time) { + RTC_CHECK_GE(diff_time, 0.0); + + // Use the filter update + // + // load <-- x/d (1-exp (-d/T)) + exp (-d/T) load + // + // where we must take care for small d, using the proper limit + // (1 - exp(-d/tau)) / d = 1/tau - d/2tau^2 + O(d^2) + double tau = (1e-3 * options_.filter_time_ms); + double e = diff_time / tau; + double c; + if (e < 0.0001) { + c = (1 - e / 2) / tau; + } else { + c = -expm1(-e) / diff_time; + } + load_estimate_ = c * encode_time + exp(-e) * load_estimate_; + } + + int64_t DurationPerInputFrame(int64_t capture_time_us, + int64_t encode_time_us) { + // Discard data on old frames; limit 2 seconds. + static constexpr int64_t kMaxAge = 2 * rtc::kNumMicrosecsPerSec; + for (auto it = max_encode_time_per_input_frame_.begin(); + it != max_encode_time_per_input_frame_.end() && + it->first < capture_time_us - kMaxAge;) { + it = max_encode_time_per_input_frame_.erase(it); + } + + std::map<int64_t, int>::iterator it; + bool inserted; + std::tie(it, inserted) = max_encode_time_per_input_frame_.emplace( + capture_time_us, encode_time_us); + if (inserted) { + // First encoded frame for this input frame. + return encode_time_us; + } + if (encode_time_us <= it->second) { + // Shorter encode time than previous frame (unlikely). Count it as being + // done in parallel. + return 0; + } + // Record new maximum encode time, and return increase from previous max. + int increase = encode_time_us - it->second; + it->second = encode_time_us; + return increase; + } + + int Value() override { + return static_cast<int>(100.0 * load_estimate_ + 0.5); + } + + const CpuOveruseOptions options_; + // Indexed by the capture timestamp, used as frame id. + std::map<int64_t, int> max_encode_time_per_input_frame_; + + int64_t prev_time_us_ = -1; + double load_estimate_; +}; + +// Class used for manual testing of overuse, enabled via field trial flag. +class OverdoseInjector : public OveruseFrameDetector::ProcessingUsage { + public: + OverdoseInjector(std::unique_ptr<OveruseFrameDetector::ProcessingUsage> usage, + int64_t normal_period_ms, + int64_t overuse_period_ms, + int64_t underuse_period_ms) + : usage_(std::move(usage)), + normal_period_ms_(normal_period_ms), + overuse_period_ms_(overuse_period_ms), + underuse_period_ms_(underuse_period_ms), + state_(State::kNormal), + last_toggling_ms_(-1) { + RTC_DCHECK_GT(overuse_period_ms, 0); + RTC_DCHECK_GT(normal_period_ms, 0); + RTC_LOG(LS_INFO) << "Simulating overuse with intervals " << normal_period_ms + << "ms normal mode, " << overuse_period_ms + << "ms overuse mode."; + } + + ~OverdoseInjector() override {} + + void Reset() override { usage_->Reset(); } + + void SetMaxSampleDiffMs(float diff_ms) override { + usage_->SetMaxSampleDiffMs(diff_ms); + } + + void FrameCaptured(const VideoFrame& frame, + int64_t time_when_first_seen_us, + int64_t last_capture_time_us) override { + usage_->FrameCaptured(frame, time_when_first_seen_us, last_capture_time_us); + } + + absl::optional<int> FrameSent( + // These two argument used by old estimator. + uint32_t timestamp, + int64_t time_sent_in_us, + // And these two by the new estimator. + int64_t capture_time_us, + absl::optional<int> encode_duration_us) override { + return usage_->FrameSent(timestamp, time_sent_in_us, capture_time_us, + encode_duration_us); + } + + int Value() override { + int64_t now_ms = rtc::TimeMillis(); + if (last_toggling_ms_ == -1) { + last_toggling_ms_ = now_ms; + } else { + switch (state_) { + case State::kNormal: + if (now_ms > last_toggling_ms_ + normal_period_ms_) { + state_ = State::kOveruse; + last_toggling_ms_ = now_ms; + RTC_LOG(LS_INFO) << "Simulating CPU overuse."; + } + break; + case State::kOveruse: + if (now_ms > last_toggling_ms_ + overuse_period_ms_) { + state_ = State::kUnderuse; + last_toggling_ms_ = now_ms; + RTC_LOG(LS_INFO) << "Simulating CPU underuse."; + } + break; + case State::kUnderuse: + if (now_ms > last_toggling_ms_ + underuse_period_ms_) { + state_ = State::kNormal; + last_toggling_ms_ = now_ms; + RTC_LOG(LS_INFO) << "Actual CPU overuse measurements in effect."; + } + break; + } + } + + absl::optional<int> overried_usage_value; + switch (state_) { + case State::kNormal: + break; + case State::kOveruse: + overried_usage_value.emplace(250); + break; + case State::kUnderuse: + overried_usage_value.emplace(5); + break; + } + + return overried_usage_value.value_or(usage_->Value()); + } + + private: + const std::unique_ptr<OveruseFrameDetector::ProcessingUsage> usage_; + const int64_t normal_period_ms_; + const int64_t overuse_period_ms_; + const int64_t underuse_period_ms_; + enum class State { kNormal, kOveruse, kUnderuse } state_; + int64_t last_toggling_ms_; +}; + +} // namespace + +CpuOveruseOptions::CpuOveruseOptions(const FieldTrialsView& field_trials) + : high_encode_usage_threshold_percent(85), + frame_timeout_interval_ms(1500), + min_frame_samples(120), + min_process_count(3), + high_threshold_consecutive_count(2), + // Disabled by default. + filter_time_ms(0) { +#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) + // Kill switch for re-enabling special adaptation rules for macOS. + // TODO(bugs.webrtc.org/14138): Remove once removal is deemed safe. + if (field_trials.IsEnabled( + "WebRTC-MacSpecialOveruseRulesRemovalKillSwitch")) { + // This is proof-of-concept code for letting the physical core count affect + // the interval into which we attempt to scale. For now, the code is Mac OS + // specific, since that's the platform were we saw most problems. + // TODO(torbjorng): Enhance SystemInfo to return this metric. + + mach_port_t mach_host = mach_host_self(); + host_basic_info hbi = {}; + mach_msg_type_number_t info_count = HOST_BASIC_INFO_COUNT; + kern_return_t kr = + host_info(mach_host, HOST_BASIC_INFO, + reinterpret_cast<host_info_t>(&hbi), &info_count); + mach_port_deallocate(mach_task_self(), mach_host); + + int n_physical_cores; + if (kr != KERN_SUCCESS) { + // If we couldn't get # of physical CPUs, don't panic. Assume we have 1. + n_physical_cores = 1; + RTC_LOG(LS_ERROR) + << "Failed to determine number of physical cores, assuming 1"; + } else { + n_physical_cores = hbi.physical_cpu; + RTC_LOG(LS_INFO) << "Number of physical cores:" << n_physical_cores; + } + + // Change init list default for few core systems. The assumption here is + // that encoding, which we measure here, takes about 1/4 of the processing + // of a two-way call. This is roughly true for x86 using both vp8 and vp9 + // without hardware encoding. Since we don't affect the incoming stream + // here, we only control about 1/2 of the total processing needs, but this + // is not taken into account. + if (n_physical_cores == 1) + high_encode_usage_threshold_percent = 20; // Roughly 1/4 of 100%. + else if (n_physical_cores == 2) + high_encode_usage_threshold_percent = 40; // Roughly 1/4 of 200%. + } +#endif // defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) + // Note that we make the interval 2x+epsilon wide, since libyuv scaling steps + // are close to that (when squared). This wide interval makes sure that + // scaling up or down does not jump all the way across the interval. + low_encode_usage_threshold_percent = + (high_encode_usage_threshold_percent - 1) / 2; +} + +std::unique_ptr<OveruseFrameDetector::ProcessingUsage> +OveruseFrameDetector::CreateProcessingUsage(const CpuOveruseOptions& options) { + std::unique_ptr<ProcessingUsage> instance; + if (options.filter_time_ms > 0) { + instance = std::make_unique<SendProcessingUsage2>(options); + } else { + instance = std::make_unique<SendProcessingUsage1>(options); + } + std::string toggling_interval = + field_trial::FindFullName("WebRTC-ForceSimulatedOveruseIntervalMs"); + if (!toggling_interval.empty()) { + int normal_period_ms = 0; + int overuse_period_ms = 0; + int underuse_period_ms = 0; + if (sscanf(toggling_interval.c_str(), "%d-%d-%d", &normal_period_ms, + &overuse_period_ms, &underuse_period_ms) == 3) { + if (normal_period_ms > 0 && overuse_period_ms > 0 && + underuse_period_ms > 0) { + instance = std::make_unique<OverdoseInjector>( + std::move(instance), normal_period_ms, overuse_period_ms, + underuse_period_ms); + } else { + RTC_LOG(LS_WARNING) + << "Invalid (non-positive) normal/overuse/underuse periods: " + << normal_period_ms << " / " << overuse_period_ms << " / " + << underuse_period_ms; + } + } else { + RTC_LOG(LS_WARNING) << "Malformed toggling interval: " + << toggling_interval; + } + } + return instance; +} + +OveruseFrameDetector::OveruseFrameDetector( + CpuOveruseMetricsObserver* metrics_observer, + const FieldTrialsView& field_trials) + : options_(field_trials), + metrics_observer_(metrics_observer), + num_process_times_(0), + // TODO(bugs.webrtc.org/9078): Use absl::optional + last_capture_time_us_(-1), + num_pixels_(0), + max_framerate_(kDefaultFrameRate), + last_overuse_time_ms_(-1), + checks_above_threshold_(0), + num_overuse_detections_(0), + last_rampup_time_ms_(-1), + in_quick_rampup_(false), + current_rampup_delay_ms_(kStandardRampUpDelayMs) { + task_checker_.Detach(); + ParseFieldTrial({&filter_time_constant_}, + field_trial::FindFullName("WebRTC-CpuLoadEstimator")); +} + +OveruseFrameDetector::~OveruseFrameDetector() {} + +void OveruseFrameDetector::StartCheckForOveruse( + TaskQueueBase* task_queue_base, + const CpuOveruseOptions& options, + OveruseFrameDetectorObserverInterface* overuse_observer) { + RTC_DCHECK_RUN_ON(&task_checker_); + RTC_DCHECK(!check_overuse_task_.Running()); + RTC_DCHECK(overuse_observer != nullptr); + + SetOptions(options); + check_overuse_task_ = RepeatingTaskHandle::DelayedStart( + task_queue_base, TimeDelta::Millis(kTimeToFirstCheckForOveruseMs), + [this, overuse_observer] { + CheckForOveruse(overuse_observer); + return TimeDelta::Millis(kCheckForOveruseIntervalMs); + }); +} +void OveruseFrameDetector::StopCheckForOveruse() { + RTC_DCHECK_RUN_ON(&task_checker_); + check_overuse_task_.Stop(); +} + +void OveruseFrameDetector::EncodedFrameTimeMeasured(int encode_duration_ms) { + RTC_DCHECK_RUN_ON(&task_checker_); + encode_usage_percent_ = usage_->Value(); + + metrics_observer_->OnEncodedFrameTimeMeasured(encode_duration_ms, + *encode_usage_percent_); +} + +bool OveruseFrameDetector::FrameSizeChanged(int num_pixels) const { + RTC_DCHECK_RUN_ON(&task_checker_); + if (num_pixels != num_pixels_) { + return true; + } + return false; +} + +bool OveruseFrameDetector::FrameTimeoutDetected(int64_t now_us) const { + RTC_DCHECK_RUN_ON(&task_checker_); + if (last_capture_time_us_ == -1) + return false; + return (now_us - last_capture_time_us_) > + options_.frame_timeout_interval_ms * rtc::kNumMicrosecsPerMillisec; +} + +void OveruseFrameDetector::ResetAll(int num_pixels) { + // Reset state, as a result resolution being changed. Do not however change + // the current frame rate back to the default. + RTC_DCHECK_RUN_ON(&task_checker_); + num_pixels_ = num_pixels; + usage_->Reset(); + last_capture_time_us_ = -1; + num_process_times_ = 0; + encode_usage_percent_ = absl::nullopt; + OnTargetFramerateUpdated(max_framerate_); +} + +void OveruseFrameDetector::OnTargetFramerateUpdated(int framerate_fps) { + RTC_DCHECK_RUN_ON(&task_checker_); + RTC_DCHECK_GE(framerate_fps, 0); + max_framerate_ = std::min(kMaxFramerate, framerate_fps); + usage_->SetMaxSampleDiffMs((1000 / std::max(kMinFramerate, max_framerate_)) * + kMaxSampleDiffMarginFactor); +} + +void OveruseFrameDetector::FrameCaptured(const VideoFrame& frame, + int64_t time_when_first_seen_us) { + RTC_DCHECK_RUN_ON(&task_checker_); + + if (FrameSizeChanged(frame.width() * frame.height()) || + FrameTimeoutDetected(time_when_first_seen_us)) { + ResetAll(frame.width() * frame.height()); + } + + usage_->FrameCaptured(frame, time_when_first_seen_us, last_capture_time_us_); + last_capture_time_us_ = time_when_first_seen_us; +} + +void OveruseFrameDetector::FrameSent(uint32_t timestamp, + int64_t time_sent_in_us, + int64_t capture_time_us, + absl::optional<int> encode_duration_us) { + RTC_DCHECK_RUN_ON(&task_checker_); + encode_duration_us = usage_->FrameSent(timestamp, time_sent_in_us, + capture_time_us, encode_duration_us); + + if (encode_duration_us) { + EncodedFrameTimeMeasured(*encode_duration_us / + rtc::kNumMicrosecsPerMillisec); + } +} + +void OveruseFrameDetector::CheckForOveruse( + OveruseFrameDetectorObserverInterface* observer) { + RTC_DCHECK_RUN_ON(&task_checker_); + RTC_DCHECK(observer); + ++num_process_times_; + if (num_process_times_ <= options_.min_process_count || + !encode_usage_percent_) + return; + + int64_t now_ms = rtc::TimeMillis(); + + if (IsOverusing(*encode_usage_percent_)) { + // If the last thing we did was going up, and now have to back down, we need + // to check if this peak was short. If so we should back off to avoid going + // back and forth between this load, the system doesn't seem to handle it. + bool check_for_backoff = last_rampup_time_ms_ > last_overuse_time_ms_; + if (check_for_backoff) { + if (now_ms - last_rampup_time_ms_ < kStandardRampUpDelayMs || + num_overuse_detections_ > kMaxOverusesBeforeApplyRampupDelay) { + // Going up was not ok for very long, back off. + current_rampup_delay_ms_ *= kRampUpBackoffFactor; + if (current_rampup_delay_ms_ > kMaxRampUpDelayMs) + current_rampup_delay_ms_ = kMaxRampUpDelayMs; + } else { + // Not currently backing off, reset rampup delay. + current_rampup_delay_ms_ = kStandardRampUpDelayMs; + } + } + + last_overuse_time_ms_ = now_ms; + in_quick_rampup_ = false; + checks_above_threshold_ = 0; + ++num_overuse_detections_; + + observer->AdaptDown(); + } else if (IsUnderusing(*encode_usage_percent_, now_ms)) { + last_rampup_time_ms_ = now_ms; + in_quick_rampup_ = true; + + observer->AdaptUp(); + } + + int rampup_delay = + in_quick_rampup_ ? kQuickRampUpDelayMs : current_rampup_delay_ms_; + + RTC_LOG(LS_VERBOSE) << " Frame stats: " + " encode usage " + << *encode_usage_percent_ << " overuse detections " + << num_overuse_detections_ << " rampup delay " + << rampup_delay; +} + +void OveruseFrameDetector::SetOptions(const CpuOveruseOptions& options) { + RTC_DCHECK_RUN_ON(&task_checker_); + options_ = options; + + // Time constant config overridable by field trial. + if (filter_time_constant_) { + options_.filter_time_ms = filter_time_constant_->ms(); + } + // Force reset with next frame. + num_pixels_ = 0; + usage_ = CreateProcessingUsage(options); +} + +bool OveruseFrameDetector::IsOverusing(int usage_percent) { + RTC_DCHECK_RUN_ON(&task_checker_); + + if (usage_percent >= options_.high_encode_usage_threshold_percent) { + ++checks_above_threshold_; + } else { + checks_above_threshold_ = 0; + } + return checks_above_threshold_ >= options_.high_threshold_consecutive_count; +} + +bool OveruseFrameDetector::IsUnderusing(int usage_percent, int64_t time_now) { + RTC_DCHECK_RUN_ON(&task_checker_); + int delay = in_quick_rampup_ ? kQuickRampUpDelayMs : current_rampup_delay_ms_; + if (time_now < last_rampup_time_ms_ + delay) + return false; + + return usage_percent < options_.low_encode_usage_threshold_percent; +} +} // namespace webrtc diff --git a/third_party/libwebrtc/video/adaptation/overuse_frame_detector.h b/third_party/libwebrtc/video/adaptation/overuse_frame_detector.h new file mode 100644 index 0000000000..4e1f6a83a4 --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/overuse_frame_detector.h @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_ +#define VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_ + +#include <list> +#include <memory> + +#include "absl/types/optional.h" +#include "api/field_trials_view.h" +#include "api/sequence_checker.h" +#include "api/task_queue/task_queue_base.h" +#include "rtc_base/experiments/field_trial_parser.h" +#include "rtc_base/numerics/exp_filter.h" +#include "rtc_base/system/no_unique_address.h" +#include "rtc_base/task_utils/repeating_task.h" +#include "rtc_base/thread_annotations.h" +#include "video/video_stream_encoder_observer.h" + +namespace webrtc { + +class VideoFrame; + +struct CpuOveruseOptions { + explicit CpuOveruseOptions(const FieldTrialsView& field_trials); + + int low_encode_usage_threshold_percent; // Threshold for triggering underuse. + int high_encode_usage_threshold_percent; // Threshold for triggering overuse. + // General settings. + int frame_timeout_interval_ms; // The maximum allowed interval between two + // frames before resetting estimations. + int min_frame_samples; // The minimum number of frames required. + int min_process_count; // The number of initial process times required before + // triggering an overuse/underuse. + int high_threshold_consecutive_count; // The number of consecutive checks + // above the high threshold before + // triggering an overuse. + // New estimator enabled if this is set non-zero. + int filter_time_ms; // Time constant for averaging +}; + +class OveruseFrameDetectorObserverInterface { + public: + // Called to signal that we can handle larger or more frequent frames. + virtual void AdaptUp() = 0; + // Called to signal that the source should reduce the resolution or framerate. + virtual void AdaptDown() = 0; + + protected: + virtual ~OveruseFrameDetectorObserverInterface() {} +}; + +// Use to detect system overuse based on the send-side processing time of +// incoming frames. All methods must be called on a single task queue but it can +// be created and destroyed on an arbitrary thread. +// OveruseFrameDetector::StartCheckForOveruse must be called to periodically +// check for overuse. +class OveruseFrameDetector { + public: + explicit OveruseFrameDetector(CpuOveruseMetricsObserver* metrics_observer, + const FieldTrialsView& field_trials); + virtual ~OveruseFrameDetector(); + + OveruseFrameDetector(const OveruseFrameDetector&) = delete; + OveruseFrameDetector& operator=(const OveruseFrameDetector&) = delete; + + // Start to periodically check for overuse. + void StartCheckForOveruse( + TaskQueueBase* task_queue_base, + const CpuOveruseOptions& options, + OveruseFrameDetectorObserverInterface* overuse_observer); + + // StopCheckForOveruse must be called before destruction if + // StartCheckForOveruse has been called. + void StopCheckForOveruse(); + + // Defines the current maximum framerate targeted by the capturer. This is + // used to make sure the encode usage percent doesn't drop unduly if the + // capturer has quiet periods (for instance caused by screen capturers with + // variable capture rate depending on content updates), otherwise we might + // experience adaptation toggling. + virtual void OnTargetFramerateUpdated(int framerate_fps); + + // Called for each captured frame. + void FrameCaptured(const VideoFrame& frame, int64_t time_when_first_seen_us); + + // Called for each sent frame. + void FrameSent(uint32_t timestamp, + int64_t time_sent_in_us, + int64_t capture_time_us, + absl::optional<int> encode_duration_us); + + // Interface for cpu load estimation. Intended for internal use only. + class ProcessingUsage { + public: + virtual void Reset() = 0; + virtual void SetMaxSampleDiffMs(float diff_ms) = 0; + virtual void FrameCaptured(const VideoFrame& frame, + int64_t time_when_first_seen_us, + int64_t last_capture_time_us) = 0; + // Returns encode_time in us, if there's a new measurement. + virtual absl::optional<int> FrameSent( + // These two argument used by old estimator. + uint32_t timestamp, + int64_t time_sent_in_us, + // And these two by the new estimator. + int64_t capture_time_us, + absl::optional<int> encode_duration_us) = 0; + + virtual int Value() = 0; + virtual ~ProcessingUsage() = default; + }; + + protected: + // Protected for test purposes. + void CheckForOveruse(OveruseFrameDetectorObserverInterface* overuse_observer); + void SetOptions(const CpuOveruseOptions& options); + + CpuOveruseOptions options_; + + private: + void EncodedFrameTimeMeasured(int encode_duration_ms); + bool IsOverusing(int encode_usage_percent); + bool IsUnderusing(int encode_usage_percent, int64_t time_now); + + bool FrameTimeoutDetected(int64_t now) const; + bool FrameSizeChanged(int num_pixels) const; + + void ResetAll(int num_pixels); + + static std::unique_ptr<ProcessingUsage> CreateProcessingUsage( + const CpuOveruseOptions& options); + + RTC_NO_UNIQUE_ADDRESS SequenceChecker task_checker_; + // Owned by the task queue from where StartCheckForOveruse is called. + RepeatingTaskHandle check_overuse_task_ RTC_GUARDED_BY(task_checker_); + + // Stats metrics. + CpuOveruseMetricsObserver* const metrics_observer_; + absl::optional<int> encode_usage_percent_ RTC_GUARDED_BY(task_checker_); + + int64_t num_process_times_ RTC_GUARDED_BY(task_checker_); + + int64_t last_capture_time_us_ RTC_GUARDED_BY(task_checker_); + + // Number of pixels of last captured frame. + int num_pixels_ RTC_GUARDED_BY(task_checker_); + int max_framerate_ RTC_GUARDED_BY(task_checker_); + int64_t last_overuse_time_ms_ RTC_GUARDED_BY(task_checker_); + int checks_above_threshold_ RTC_GUARDED_BY(task_checker_); + int num_overuse_detections_ RTC_GUARDED_BY(task_checker_); + int64_t last_rampup_time_ms_ RTC_GUARDED_BY(task_checker_); + bool in_quick_rampup_ RTC_GUARDED_BY(task_checker_); + int current_rampup_delay_ms_ RTC_GUARDED_BY(task_checker_); + + std::unique_ptr<ProcessingUsage> usage_ RTC_PT_GUARDED_BY(task_checker_); + + // If set by field trial, overrides CpuOveruseOptions::filter_time_ms. + FieldTrialOptional<TimeDelta> filter_time_constant_{"tau"}; +}; + +} // namespace webrtc + +#endif // VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_ diff --git a/third_party/libwebrtc/video/adaptation/overuse_frame_detector_unittest.cc b/third_party/libwebrtc/video/adaptation/overuse_frame_detector_unittest.cc new file mode 100644 index 0000000000..5098c9c2ec --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/overuse_frame_detector_unittest.cc @@ -0,0 +1,1023 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/adaptation/overuse_frame_detector.h" + +#include <memory> + +#include "api/field_trials_view.h" +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_adaptation_reason.h" +#include "modules/video_coding/utility/quality_scaler.h" +#include "rtc_base/event.h" +#include "rtc_base/fake_clock.h" +#include "rtc_base/random.h" +#include "rtc_base/task_queue_for_test.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/scoped_key_value_config.h" + +namespace webrtc { + +using ::testing::_; +using ::testing::InvokeWithoutArgs; + +namespace { +const int kWidth = 640; +const int kHeight = 480; +// Corresponds to load of 15% +const int kFrameIntervalUs = 33 * rtc::kNumMicrosecsPerMillisec; +const int kProcessTimeUs = 5 * rtc::kNumMicrosecsPerMillisec; +const test::ScopedKeyValueConfig kFieldTrials; +} // namespace + +class MockCpuOveruseObserver : public OveruseFrameDetectorObserverInterface { + public: + MockCpuOveruseObserver() {} + virtual ~MockCpuOveruseObserver() {} + + MOCK_METHOD(void, AdaptUp, (), (override)); + MOCK_METHOD(void, AdaptDown, (), (override)); +}; + +class CpuOveruseObserverImpl : public OveruseFrameDetectorObserverInterface { + public: + CpuOveruseObserverImpl() : overuse_(0), normaluse_(0) {} + virtual ~CpuOveruseObserverImpl() {} + + void AdaptDown() override { ++overuse_; } + void AdaptUp() override { ++normaluse_; } + + int overuse_; + int normaluse_; +}; + +class OveruseFrameDetectorUnderTest : public OveruseFrameDetector { + public: + explicit OveruseFrameDetectorUnderTest( + CpuOveruseMetricsObserver* metrics_observer) + : OveruseFrameDetector(metrics_observer, kFieldTrials) {} + ~OveruseFrameDetectorUnderTest() {} + + using OveruseFrameDetector::CheckForOveruse; + using OveruseFrameDetector::SetOptions; +}; + +class OveruseFrameDetectorTest : public ::testing::Test, + public CpuOveruseMetricsObserver { + protected: + OveruseFrameDetectorTest() : options_(kFieldTrials) {} + + void SetUp() override { + observer_ = &mock_observer_; + options_.min_process_count = 0; + overuse_detector_ = std::make_unique<OveruseFrameDetectorUnderTest>(this); + // Unfortunately, we can't call SetOptions here, since that would break + // single-threading requirements in the RunOnTqNormalUsage test. + } + + void OnEncodedFrameTimeMeasured(int encode_time_ms, + int encode_usage_percent) override { + encode_usage_percent_ = encode_usage_percent; + } + + int InitialUsage() { + return ((options_.low_encode_usage_threshold_percent + + options_.high_encode_usage_threshold_percent) / + 2.0f) + + 0.5; + } + + virtual void InsertAndSendFramesWithInterval(int num_frames, + int interval_us, + int width, + int height, + int delay_us) { + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(I420Buffer::Create(width, height)) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(0) + .build(); + uint32_t timestamp = 0; + while (num_frames-- > 0) { + frame.set_timestamp(timestamp); + int64_t capture_time_us = rtc::TimeMicros(); + overuse_detector_->FrameCaptured(frame, capture_time_us); + clock_.AdvanceTime(TimeDelta::Micros(delay_us)); + overuse_detector_->FrameSent(timestamp, rtc::TimeMicros(), + capture_time_us, delay_us); + clock_.AdvanceTime(TimeDelta::Micros(interval_us - delay_us)); + timestamp += interval_us * 90 / 1000; + } + } + + virtual void InsertAndSendSimulcastFramesWithInterval( + int num_frames, + int interval_us, + int width, + int height, + // One element per layer + rtc::ArrayView<const int> delays_us) { + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(I420Buffer::Create(width, height)) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(0) + .build(); + uint32_t timestamp = 0; + while (num_frames-- > 0) { + frame.set_timestamp(timestamp); + int64_t capture_time_us = rtc::TimeMicros(); + overuse_detector_->FrameCaptured(frame, capture_time_us); + int max_delay_us = 0; + for (int delay_us : delays_us) { + if (delay_us > max_delay_us) { + clock_.AdvanceTime(TimeDelta::Micros(delay_us - max_delay_us)); + max_delay_us = delay_us; + } + + overuse_detector_->FrameSent(timestamp, rtc::TimeMicros(), + capture_time_us, delay_us); + } + overuse_detector_->CheckForOveruse(observer_); + clock_.AdvanceTime(TimeDelta::Micros(interval_us - max_delay_us)); + timestamp += interval_us * 90 / 1000; + } + } + + virtual void InsertAndSendFramesWithRandomInterval(int num_frames, + int min_interval_us, + int max_interval_us, + int width, + int height, + int delay_us) { + webrtc::Random random(17); + + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(I420Buffer::Create(width, height)) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(0) + .build(); + uint32_t timestamp = 0; + while (num_frames-- > 0) { + frame.set_timestamp(timestamp); + int interval_us = random.Rand(min_interval_us, max_interval_us); + int64_t capture_time_us = rtc::TimeMicros(); + overuse_detector_->FrameCaptured(frame, capture_time_us); + clock_.AdvanceTime(TimeDelta::Micros(delay_us)); + overuse_detector_->FrameSent(timestamp, rtc::TimeMicros(), + capture_time_us, + absl::optional<int>(delay_us)); + + overuse_detector_->CheckForOveruse(observer_); + // Avoid turning clock backwards. + if (interval_us > delay_us) + clock_.AdvanceTime(TimeDelta::Micros(interval_us - delay_us)); + + timestamp += interval_us * 90 / 1000; + } + } + + virtual void ForceUpdate(int width, int height) { + // Insert one frame, wait a second and then put in another to force update + // the usage. From the tests where these are used, adding another sample + // doesn't affect the expected outcome (this is mainly to check initial + // values and whether the overuse detector has been reset or not). + InsertAndSendFramesWithInterval(2, rtc::kNumMicrosecsPerSec, width, height, + kFrameIntervalUs); + } + void TriggerOveruse(int num_times) { + const int kDelayUs = 32 * rtc::kNumMicrosecsPerMillisec; + for (int i = 0; i < num_times; ++i) { + InsertAndSendFramesWithInterval(1000, kFrameIntervalUs, kWidth, kHeight, + kDelayUs); + overuse_detector_->CheckForOveruse(observer_); + } + } + + void TriggerUnderuse() { + const int kDelayUs1 = 5000; + const int kDelayUs2 = 6000; + InsertAndSendFramesWithInterval(1300, kFrameIntervalUs, kWidth, kHeight, + kDelayUs1); + InsertAndSendFramesWithInterval(1, kFrameIntervalUs, kWidth, kHeight, + kDelayUs2); + overuse_detector_->CheckForOveruse(observer_); + } + + int UsagePercent() { return encode_usage_percent_; } + + int64_t OveruseProcessingTimeLimitForFramerate(int fps) const { + int64_t frame_interval = rtc::kNumMicrosecsPerSec / fps; + int64_t max_processing_time_us = + (frame_interval * options_.high_encode_usage_threshold_percent) / 100; + return max_processing_time_us; + } + + int64_t UnderuseProcessingTimeLimitForFramerate(int fps) const { + int64_t frame_interval = rtc::kNumMicrosecsPerSec / fps; + int64_t max_processing_time_us = + (frame_interval * options_.low_encode_usage_threshold_percent) / 100; + return max_processing_time_us; + } + + CpuOveruseOptions options_; + rtc::ScopedFakeClock clock_; + MockCpuOveruseObserver mock_observer_; + OveruseFrameDetectorObserverInterface* observer_; + std::unique_ptr<OveruseFrameDetectorUnderTest> overuse_detector_; + int encode_usage_percent_ = -1; +}; + +// UsagePercent() > high_encode_usage_threshold_percent => overuse. +// UsagePercent() < low_encode_usage_threshold_percent => underuse. +TEST_F(OveruseFrameDetectorTest, TriggerOveruse) { + // usage > high => overuse + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); + TriggerOveruse(options_.high_threshold_consecutive_count); +} + +TEST_F(OveruseFrameDetectorTest, OveruseAndRecover) { + // usage > high => overuse + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); + TriggerOveruse(options_.high_threshold_consecutive_count); + // usage < low => underuse + EXPECT_CALL(mock_observer_, AdaptUp()).Times(::testing::AtLeast(1)); + TriggerUnderuse(); +} + +TEST_F(OveruseFrameDetectorTest, DoubleOveruseAndRecover) { + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(2); + TriggerOveruse(options_.high_threshold_consecutive_count); + TriggerOveruse(options_.high_threshold_consecutive_count); + EXPECT_CALL(mock_observer_, AdaptUp()).Times(::testing::AtLeast(1)); + TriggerUnderuse(); +} + +TEST_F(OveruseFrameDetectorTest, TriggerUnderuseWithMinProcessCount) { + const int kProcessIntervalUs = 5 * rtc::kNumMicrosecsPerSec; + options_.min_process_count = 1; + CpuOveruseObserverImpl overuse_observer; + observer_ = nullptr; + overuse_detector_->SetOptions(options_); + InsertAndSendFramesWithInterval(1200, kFrameIntervalUs, kWidth, kHeight, + kProcessTimeUs); + overuse_detector_->CheckForOveruse(&overuse_observer); + EXPECT_EQ(0, overuse_observer.normaluse_); + clock_.AdvanceTime(TimeDelta::Micros(kProcessIntervalUs)); + overuse_detector_->CheckForOveruse(&overuse_observer); + EXPECT_EQ(1, overuse_observer.normaluse_); +} + +TEST_F(OveruseFrameDetectorTest, ConstantOveruseGivesNoNormalUsage) { + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptUp()).Times(0); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(64); + for (size_t i = 0; i < 64; ++i) { + TriggerOveruse(options_.high_threshold_consecutive_count); + } +} + +TEST_F(OveruseFrameDetectorTest, ConsecutiveCountTriggersOveruse) { + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); + options_.high_threshold_consecutive_count = 2; + overuse_detector_->SetOptions(options_); + TriggerOveruse(2); +} + +TEST_F(OveruseFrameDetectorTest, IncorrectConsecutiveCountTriggersNoOveruse) { + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); + options_.high_threshold_consecutive_count = 2; + overuse_detector_->SetOptions(options_); + TriggerOveruse(1); +} + +TEST_F(OveruseFrameDetectorTest, ProcessingUsage) { + overuse_detector_->SetOptions(options_); + InsertAndSendFramesWithInterval(1000, kFrameIntervalUs, kWidth, kHeight, + kProcessTimeUs); + EXPECT_EQ(kProcessTimeUs * 100 / kFrameIntervalUs, UsagePercent()); +} + +TEST_F(OveruseFrameDetectorTest, ResetAfterResolutionChange) { + overuse_detector_->SetOptions(options_); + ForceUpdate(kWidth, kHeight); + EXPECT_EQ(InitialUsage(), UsagePercent()); + InsertAndSendFramesWithInterval(1000, kFrameIntervalUs, kWidth, kHeight, + kProcessTimeUs); + EXPECT_NE(InitialUsage(), UsagePercent()); + // Verify reset (with new width/height). + ForceUpdate(kWidth, kHeight + 1); + EXPECT_EQ(InitialUsage(), UsagePercent()); +} + +TEST_F(OveruseFrameDetectorTest, ResetAfterFrameTimeout) { + overuse_detector_->SetOptions(options_); + ForceUpdate(kWidth, kHeight); + EXPECT_EQ(InitialUsage(), UsagePercent()); + InsertAndSendFramesWithInterval(1000, kFrameIntervalUs, kWidth, kHeight, + kProcessTimeUs); + EXPECT_NE(InitialUsage(), UsagePercent()); + InsertAndSendFramesWithInterval( + 2, options_.frame_timeout_interval_ms * rtc::kNumMicrosecsPerMillisec, + kWidth, kHeight, kProcessTimeUs); + EXPECT_NE(InitialUsage(), UsagePercent()); + // Verify reset. + InsertAndSendFramesWithInterval( + 2, + (options_.frame_timeout_interval_ms + 1) * rtc::kNumMicrosecsPerMillisec, + kWidth, kHeight, kProcessTimeUs); + ForceUpdate(kWidth, kHeight); + EXPECT_EQ(InitialUsage(), UsagePercent()); +} + +TEST_F(OveruseFrameDetectorTest, MinFrameSamplesBeforeUpdating) { + options_.min_frame_samples = 40; + overuse_detector_->SetOptions(options_); + InsertAndSendFramesWithInterval(40, kFrameIntervalUs, kWidth, kHeight, + kProcessTimeUs); + EXPECT_EQ(InitialUsage(), UsagePercent()); + // Pass time far enough to digest all previous samples. + clock_.AdvanceTime(TimeDelta::Seconds(1)); + InsertAndSendFramesWithInterval(1, kFrameIntervalUs, kWidth, kHeight, + kProcessTimeUs); + // The last sample has not been processed here. + EXPECT_EQ(InitialUsage(), UsagePercent()); + + // Pass time far enough to digest all previous samples, 41 in total. + clock_.AdvanceTime(TimeDelta::Seconds(1)); + InsertAndSendFramesWithInterval(1, kFrameIntervalUs, kWidth, kHeight, + kProcessTimeUs); + EXPECT_NE(InitialUsage(), UsagePercent()); +} + +TEST_F(OveruseFrameDetectorTest, InitialProcessingUsage) { + overuse_detector_->SetOptions(options_); + ForceUpdate(kWidth, kHeight); + EXPECT_EQ(InitialUsage(), UsagePercent()); +} + +TEST_F(OveruseFrameDetectorTest, MeasuresMultipleConcurrentSamples) { + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(::testing::AtLeast(1)); + static const int kIntervalUs = 33 * rtc::kNumMicrosecsPerMillisec; + static const size_t kNumFramesEncodingDelay = 3; + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(I420Buffer::Create(kWidth, kHeight)) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(0) + .build(); + for (size_t i = 0; i < 1000; ++i) { + // Unique timestamps. + frame.set_timestamp(static_cast<uint32_t>(i)); + int64_t capture_time_us = rtc::TimeMicros(); + overuse_detector_->FrameCaptured(frame, capture_time_us); + clock_.AdvanceTime(TimeDelta::Micros(kIntervalUs)); + if (i > kNumFramesEncodingDelay) { + overuse_detector_->FrameSent( + static_cast<uint32_t>(i - kNumFramesEncodingDelay), rtc::TimeMicros(), + capture_time_us, kIntervalUs); + } + overuse_detector_->CheckForOveruse(observer_); + } +} + +TEST_F(OveruseFrameDetectorTest, UpdatesExistingSamples) { + // >85% encoding time should trigger overuse. + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(::testing::AtLeast(1)); + static const int kIntervalUs = 33 * rtc::kNumMicrosecsPerMillisec; + static const int kDelayUs = 30 * rtc::kNumMicrosecsPerMillisec; + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(I420Buffer::Create(kWidth, kHeight)) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(0) + .build(); + uint32_t timestamp = 0; + for (size_t i = 0; i < 1000; ++i) { + frame.set_timestamp(timestamp); + int64_t capture_time_us = rtc::TimeMicros(); + overuse_detector_->FrameCaptured(frame, capture_time_us); + // Encode and send first parts almost instantly. + clock_.AdvanceTime(TimeDelta::Millis(1)); + overuse_detector_->FrameSent(timestamp, rtc::TimeMicros(), capture_time_us, + rtc::kNumMicrosecsPerMillisec); + // Encode heavier part, resulting in >85% usage total. + clock_.AdvanceTime(TimeDelta::Micros(kDelayUs) - TimeDelta::Millis(1)); + overuse_detector_->FrameSent(timestamp, rtc::TimeMicros(), capture_time_us, + kDelayUs); + clock_.AdvanceTime(TimeDelta::Micros(kIntervalUs - kDelayUs)); + timestamp += kIntervalUs * 90 / 1000; + overuse_detector_->CheckForOveruse(observer_); + } +} + +TEST_F(OveruseFrameDetectorTest, RunOnTqNormalUsage) { + TaskQueueForTest queue("OveruseFrameDetectorTestQueue"); + + queue.SendTask([&] { + overuse_detector_->StartCheckForOveruse(queue.Get(), options_, observer_); + }); + + rtc::Event event; + // Expect NormalUsage(). When called, stop the `overuse_detector_` and then + // set `event` to end the test. + EXPECT_CALL(mock_observer_, AdaptUp()) + .WillOnce(InvokeWithoutArgs([this, &event] { + overuse_detector_->StopCheckForOveruse(); + event.Set(); + })); + + queue.PostTask([this] { + const int kDelayUs1 = 5 * rtc::kNumMicrosecsPerMillisec; + const int kDelayUs2 = 6 * rtc::kNumMicrosecsPerMillisec; + InsertAndSendFramesWithInterval(1300, kFrameIntervalUs, kWidth, kHeight, + kDelayUs1); + InsertAndSendFramesWithInterval(1, kFrameIntervalUs, kWidth, kHeight, + kDelayUs2); + }); + + EXPECT_TRUE(event.Wait(TimeDelta::Seconds(10))); +} + +// TODO(crbug.com/webrtc/12846): investigate why the test fails on MAC bots. +#if !defined(WEBRTC_MAC) +TEST_F(OveruseFrameDetectorTest, MaxIntervalScalesWithFramerate) { + const int kCapturerMaxFrameRate = 30; + const int kEncodeMaxFrameRate = 20; // Maximum fps the encoder can sustain. + + overuse_detector_->SetOptions(options_); + // Trigger overuse. + int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kCapturerMaxFrameRate; + // Processing time just below over use limit given kEncodeMaxFrameRate. + int64_t processing_time_us = + (98 * OveruseProcessingTimeLimitForFramerate(kEncodeMaxFrameRate)) / 100; + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(observer_); + } + + // Simulate frame rate reduction and normal usage. + frame_interval_us = rtc::kNumMicrosecsPerSec / kEncodeMaxFrameRate; + overuse_detector_->OnTargetFramerateUpdated(kEncodeMaxFrameRate); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(observer_); + } + + // Reduce processing time to trigger underuse. + processing_time_us = + (98 * UnderuseProcessingTimeLimitForFramerate(kEncodeMaxFrameRate)) / 100; + EXPECT_CALL(mock_observer_, AdaptUp()).Times(1); + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(observer_); +} +#endif + +TEST_F(OveruseFrameDetectorTest, RespectsMinFramerate) { + const int kMinFrameRate = 7; // Minimum fps allowed by current detector impl. + overuse_detector_->SetOptions(options_); + overuse_detector_->OnTargetFramerateUpdated(kMinFrameRate); + + // Normal usage just at the limit. + int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kMinFrameRate; + // Processing time just below over use limit given kEncodeMaxFrameRate. + int64_t processing_time_us = + (98 * OveruseProcessingTimeLimitForFramerate(kMinFrameRate)) / 100; + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(observer_); + } + + // Over the limit to overuse. + processing_time_us = + (102 * OveruseProcessingTimeLimitForFramerate(kMinFrameRate)) / 100; + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(observer_); + } + + // Reduce input frame rate. Should still trigger overuse. + overuse_detector_->OnTargetFramerateUpdated(kMinFrameRate - 1); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, frame_interval_us, kWidth, kHeight, + processing_time_us); + overuse_detector_->CheckForOveruse(observer_); + } +} + +TEST_F(OveruseFrameDetectorTest, LimitsMaxFrameInterval) { + const int kMaxFrameRate = 20; + overuse_detector_->SetOptions(options_); + overuse_detector_->OnTargetFramerateUpdated(kMaxFrameRate); + int64_t frame_interval_us = rtc::kNumMicrosecsPerSec / kMaxFrameRate; + // Maximum frame interval allowed is 35% above ideal. + int64_t max_frame_interval_us = (135 * frame_interval_us) / 100; + // Maximum processing time, without triggering overuse, allowed with the above + // frame interval. + int64_t max_processing_time_us = + (max_frame_interval_us * options_.high_encode_usage_threshold_percent) / + 100; + + // Processing time just below overuse limit given kMaxFrameRate. + int64_t processing_time_us = (98 * max_processing_time_us) / 100; + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth, + kHeight, processing_time_us); + overuse_detector_->CheckForOveruse(observer_); + } + + // Go above limit, trigger overuse. + processing_time_us = (102 * max_processing_time_us) / 100; + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth, + kHeight, processing_time_us); + overuse_detector_->CheckForOveruse(observer_); + } + + // Increase frame interval, should still trigger overuse. + max_frame_interval_us *= 2; + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); + for (int i = 0; i < options_.high_threshold_consecutive_count; ++i) { + InsertAndSendFramesWithInterval(1200, max_frame_interval_us, kWidth, + kHeight, processing_time_us); + overuse_detector_->CheckForOveruse(observer_); + } +} + +// Models screencast, with irregular arrival of frames which are heavy +// to encode. +TEST_F(OveruseFrameDetectorTest, NoOveruseForLargeRandomFrameInterval) { + // TODO(bugs.webrtc.org/8504): When new estimator is relanded, + // behavior is improved in this scenario, with only AdaptUp events, + // and estimated load closer to the true average. + + // EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); + // EXPECT_CALL(mock_observer_, AdaptUp()) + // .Times(::testing::AtLeast(1)); + overuse_detector_->SetOptions(options_); + + const int kNumFrames = 500; + const int kEncodeTimeUs = 100 * rtc::kNumMicrosecsPerMillisec; + + const int kMinIntervalUs = 30 * rtc::kNumMicrosecsPerMillisec; + const int kMaxIntervalUs = 1000 * rtc::kNumMicrosecsPerMillisec; + + const int kTargetFramerate = 5; + + overuse_detector_->OnTargetFramerateUpdated(kTargetFramerate); + + InsertAndSendFramesWithRandomInterval(kNumFrames, kMinIntervalUs, + kMaxIntervalUs, kWidth, kHeight, + kEncodeTimeUs); + // Average usage 19%. Check that estimate is in the right ball park. + // EXPECT_NEAR(UsagePercent(), 20, 10); + EXPECT_NEAR(UsagePercent(), 20, 35); +} + +// Models screencast, with irregular arrival of frames, often +// exceeding the timeout interval. +TEST_F(OveruseFrameDetectorTest, NoOveruseForRandomFrameIntervalWithReset) { + // TODO(bugs.webrtc.org/8504): When new estimator is relanded, + // behavior is improved in this scenario, and we get AdaptUp events. + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); + // EXPECT_CALL(mock_observer_, AdaptUp()) + // .Times(::testing::AtLeast(1)); + + const int kNumFrames = 500; + const int kEncodeTimeUs = 100 * rtc::kNumMicrosecsPerMillisec; + + const int kMinIntervalUs = 30 * rtc::kNumMicrosecsPerMillisec; + const int kMaxIntervalUs = 3000 * rtc::kNumMicrosecsPerMillisec; + + const int kTargetFramerate = 5; + + overuse_detector_->OnTargetFramerateUpdated(kTargetFramerate); + + InsertAndSendFramesWithRandomInterval(kNumFrames, kMinIntervalUs, + kMaxIntervalUs, kWidth, kHeight, + kEncodeTimeUs); + + // Average usage 6.6%, but since the frame_timeout_interval_ms is + // only 1500 ms, we often reset the estimate to the initial value. + // Check that estimate is in the right ball park. + EXPECT_GE(UsagePercent(), 1); + EXPECT_LE(UsagePercent(), InitialUsage() + 5); +} + +// Models simulcast, with multiple encoded frames for each input frame. +// Load estimate should be based on the maximum encode time per input frame. +TEST_F(OveruseFrameDetectorTest, NoOveruseForSimulcast) { + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); + + constexpr int kNumFrames = 500; + constexpr int kEncodeTimesUs[] = { + 10 * rtc::kNumMicrosecsPerMillisec, + 8 * rtc::kNumMicrosecsPerMillisec, + 12 * rtc::kNumMicrosecsPerMillisec, + }; + constexpr int kIntervalUs = 30 * rtc::kNumMicrosecsPerMillisec; + + InsertAndSendSimulcastFramesWithInterval(kNumFrames, kIntervalUs, kWidth, + kHeight, kEncodeTimesUs); + + // Average usage 40%. 12 ms / 30 ms. + EXPECT_GE(UsagePercent(), 35); + EXPECT_LE(UsagePercent(), 45); +} + +// Tests using new cpu load estimator +class OveruseFrameDetectorTest2 : public OveruseFrameDetectorTest { + protected: + void SetUp() override { + options_.filter_time_ms = 5 * rtc::kNumMillisecsPerSec; + OveruseFrameDetectorTest::SetUp(); + } + + void InsertAndSendFramesWithInterval(int num_frames, + int interval_us, + int width, + int height, + int delay_us) override { + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(I420Buffer::Create(width, height)) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(0) + .build(); + while (num_frames-- > 0) { + int64_t capture_time_us = rtc::TimeMicros(); + overuse_detector_->FrameCaptured(frame, capture_time_us /* ignored */); + overuse_detector_->FrameSent(0 /* ignored timestamp */, + 0 /* ignored send_time_us */, + capture_time_us, delay_us); + clock_.AdvanceTime(TimeDelta::Micros(interval_us)); + } + } + + void InsertAndSendFramesWithRandomInterval(int num_frames, + int min_interval_us, + int max_interval_us, + int width, + int height, + int delay_us) override { + webrtc::Random random(17); + + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(I420Buffer::Create(width, height)) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(0) + .build(); + for (int i = 0; i < num_frames; i++) { + int interval_us = random.Rand(min_interval_us, max_interval_us); + int64_t capture_time_us = rtc::TimeMicros(); + overuse_detector_->FrameCaptured(frame, capture_time_us); + overuse_detector_->FrameSent(0 /* ignored timestamp */, + 0 /* ignored send_time_us */, + capture_time_us, delay_us); + + overuse_detector_->CheckForOveruse(observer_); + clock_.AdvanceTime(TimeDelta::Micros(interval_us)); + } + } + + void ForceUpdate(int width, int height) override { + // This is mainly to check initial values and whether the overuse + // detector has been reset or not. + InsertAndSendFramesWithInterval(1, rtc::kNumMicrosecsPerSec, width, height, + kFrameIntervalUs); + } +}; + +// UsagePercent() > high_encode_usage_threshold_percent => overuse. +// UsagePercent() < low_encode_usage_threshold_percent => underuse. +TEST_F(OveruseFrameDetectorTest2, TriggerOveruse) { + // usage > high => overuse + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); + TriggerOveruse(options_.high_threshold_consecutive_count); +} + +TEST_F(OveruseFrameDetectorTest2, OveruseAndRecover) { + // usage > high => overuse + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); + TriggerOveruse(options_.high_threshold_consecutive_count); + // usage < low => underuse + EXPECT_CALL(mock_observer_, AdaptUp()).Times(::testing::AtLeast(1)); + TriggerUnderuse(); +} + +TEST_F(OveruseFrameDetectorTest2, DoubleOveruseAndRecover) { + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(2); + TriggerOveruse(options_.high_threshold_consecutive_count); + TriggerOveruse(options_.high_threshold_consecutive_count); + EXPECT_CALL(mock_observer_, AdaptUp()).Times(::testing::AtLeast(1)); + TriggerUnderuse(); +} + +TEST_F(OveruseFrameDetectorTest2, TriggerUnderuseWithMinProcessCount) { + const int kProcessIntervalUs = 5 * rtc::kNumMicrosecsPerSec; + options_.min_process_count = 1; + CpuOveruseObserverImpl overuse_observer; + observer_ = nullptr; + overuse_detector_->SetOptions(options_); + InsertAndSendFramesWithInterval(1200, kFrameIntervalUs, kWidth, kHeight, + kProcessTimeUs); + overuse_detector_->CheckForOveruse(&overuse_observer); + EXPECT_EQ(0, overuse_observer.normaluse_); + clock_.AdvanceTime(TimeDelta::Micros(kProcessIntervalUs)); + overuse_detector_->CheckForOveruse(&overuse_observer); + EXPECT_EQ(1, overuse_observer.normaluse_); +} + +TEST_F(OveruseFrameDetectorTest2, ConstantOveruseGivesNoNormalUsage) { + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptUp()).Times(0); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(64); + for (size_t i = 0; i < 64; ++i) { + TriggerOveruse(options_.high_threshold_consecutive_count); + } +} + +TEST_F(OveruseFrameDetectorTest2, ConsecutiveCountTriggersOveruse) { + EXPECT_CALL(mock_observer_, AdaptDown()).Times(1); + options_.high_threshold_consecutive_count = 2; + overuse_detector_->SetOptions(options_); + TriggerOveruse(2); +} + +TEST_F(OveruseFrameDetectorTest2, IncorrectConsecutiveCountTriggersNoOveruse) { + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); + options_.high_threshold_consecutive_count = 2; + overuse_detector_->SetOptions(options_); + TriggerOveruse(1); +} + +TEST_F(OveruseFrameDetectorTest2, ProcessingUsage) { + overuse_detector_->SetOptions(options_); + InsertAndSendFramesWithInterval(1000, kFrameIntervalUs, kWidth, kHeight, + kProcessTimeUs); + EXPECT_EQ(kProcessTimeUs * 100 / kFrameIntervalUs, UsagePercent()); +} + +TEST_F(OveruseFrameDetectorTest2, ResetAfterResolutionChange) { + overuse_detector_->SetOptions(options_); + ForceUpdate(kWidth, kHeight); + EXPECT_EQ(InitialUsage(), UsagePercent()); + InsertAndSendFramesWithInterval(1000, kFrameIntervalUs, kWidth, kHeight, + kProcessTimeUs); + EXPECT_NE(InitialUsage(), UsagePercent()); + // Verify reset (with new width/height). + ForceUpdate(kWidth, kHeight + 1); + EXPECT_EQ(InitialUsage(), UsagePercent()); +} + +TEST_F(OveruseFrameDetectorTest2, ResetAfterFrameTimeout) { + overuse_detector_->SetOptions(options_); + ForceUpdate(kWidth, kHeight); + EXPECT_EQ(InitialUsage(), UsagePercent()); + InsertAndSendFramesWithInterval(1000, kFrameIntervalUs, kWidth, kHeight, + kProcessTimeUs); + EXPECT_NE(InitialUsage(), UsagePercent()); + InsertAndSendFramesWithInterval( + 2, options_.frame_timeout_interval_ms * rtc::kNumMicrosecsPerMillisec, + kWidth, kHeight, kProcessTimeUs); + EXPECT_NE(InitialUsage(), UsagePercent()); + // Verify reset. + InsertAndSendFramesWithInterval( + 2, + (options_.frame_timeout_interval_ms + 1) * rtc::kNumMicrosecsPerMillisec, + kWidth, kHeight, kProcessTimeUs); + ForceUpdate(kWidth, kHeight); + EXPECT_EQ(InitialUsage(), UsagePercent()); +} + +TEST_F(OveruseFrameDetectorTest2, ConvergesSlowly) { + overuse_detector_->SetOptions(options_); + InsertAndSendFramesWithInterval(1, kFrameIntervalUs, kWidth, kHeight, + kProcessTimeUs); + // No update for the first sample. + EXPECT_EQ(InitialUsage(), UsagePercent()); + + // Total time approximately 40 * 33ms = 1.3s, significantly less + // than the 5s time constant. + InsertAndSendFramesWithInterval(40, kFrameIntervalUs, kWidth, kHeight, + kProcessTimeUs); + + // Should have started to approach correct load of 15%, but not very far. + EXPECT_LT(UsagePercent(), InitialUsage()); + EXPECT_GT(UsagePercent(), (InitialUsage() * 3 + 8) / 4); + + // Run for roughly 10s more, should now be closer. + InsertAndSendFramesWithInterval(300, kFrameIntervalUs, kWidth, kHeight, + kProcessTimeUs); + EXPECT_NEAR(UsagePercent(), 20, 5); +} + +TEST_F(OveruseFrameDetectorTest2, InitialProcessingUsage) { + overuse_detector_->SetOptions(options_); + ForceUpdate(kWidth, kHeight); + EXPECT_EQ(InitialUsage(), UsagePercent()); +} + +TEST_F(OveruseFrameDetectorTest2, MeasuresMultipleConcurrentSamples) { + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(::testing::AtLeast(1)); + static const int kIntervalUs = 33 * rtc::kNumMicrosecsPerMillisec; + static const size_t kNumFramesEncodingDelay = 3; + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(I420Buffer::Create(kWidth, kHeight)) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(0) + .build(); + for (size_t i = 0; i < 1000; ++i) { + // Unique timestamps. + frame.set_timestamp(static_cast<uint32_t>(i)); + int64_t capture_time_us = rtc::TimeMicros(); + overuse_detector_->FrameCaptured(frame, capture_time_us); + clock_.AdvanceTime(TimeDelta::Micros(kIntervalUs)); + if (i > kNumFramesEncodingDelay) { + overuse_detector_->FrameSent( + static_cast<uint32_t>(i - kNumFramesEncodingDelay), rtc::TimeMicros(), + capture_time_us, kIntervalUs); + } + overuse_detector_->CheckForOveruse(observer_); + } +} + +TEST_F(OveruseFrameDetectorTest2, UpdatesExistingSamples) { + // >85% encoding time should trigger overuse. + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(::testing::AtLeast(1)); + static const int kIntervalUs = 33 * rtc::kNumMicrosecsPerMillisec; + static const int kDelayUs = 30 * rtc::kNumMicrosecsPerMillisec; + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(I420Buffer::Create(kWidth, kHeight)) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(0) + .build(); + uint32_t timestamp = 0; + for (size_t i = 0; i < 1000; ++i) { + frame.set_timestamp(timestamp); + int64_t capture_time_us = rtc::TimeMicros(); + overuse_detector_->FrameCaptured(frame, capture_time_us); + // Encode and send first parts almost instantly. + clock_.AdvanceTime(TimeDelta::Millis(1)); + overuse_detector_->FrameSent(timestamp, rtc::TimeMicros(), capture_time_us, + rtc::kNumMicrosecsPerMillisec); + // Encode heavier part, resulting in >85% usage total. + clock_.AdvanceTime(TimeDelta::Micros(kDelayUs) - TimeDelta::Millis(1)); + overuse_detector_->FrameSent(timestamp, rtc::TimeMicros(), capture_time_us, + kDelayUs); + clock_.AdvanceTime(TimeDelta::Micros(kIntervalUs - kDelayUs)); + timestamp += kIntervalUs * 90 / 1000; + overuse_detector_->CheckForOveruse(observer_); + } +} + +TEST_F(OveruseFrameDetectorTest2, RunOnTqNormalUsage) { + TaskQueueForTest queue("OveruseFrameDetectorTestQueue"); + + queue.SendTask([&] { + overuse_detector_->StartCheckForOveruse(queue.Get(), options_, observer_); + }); + + rtc::Event event; + // Expect NormalUsage(). When called, stop the `overuse_detector_` and then + // set `event` to end the test. + EXPECT_CALL(mock_observer_, AdaptUp()) + .WillOnce(InvokeWithoutArgs([this, &event] { + overuse_detector_->StopCheckForOveruse(); + event.Set(); + })); + + queue.PostTask([this] { + const int kDelayUs1 = 5 * rtc::kNumMicrosecsPerMillisec; + const int kDelayUs2 = 6 * rtc::kNumMicrosecsPerMillisec; + InsertAndSendFramesWithInterval(1300, kFrameIntervalUs, kWidth, kHeight, + kDelayUs1); + InsertAndSendFramesWithInterval(1, kFrameIntervalUs, kWidth, kHeight, + kDelayUs2); + }); + + EXPECT_TRUE(event.Wait(TimeDelta::Seconds(10))); +} + +// Models screencast, with irregular arrival of frames which are heavy +// to encode. +TEST_F(OveruseFrameDetectorTest2, NoOveruseForLargeRandomFrameInterval) { + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); + EXPECT_CALL(mock_observer_, AdaptUp()).Times(::testing::AtLeast(1)); + + const int kNumFrames = 500; + const int kEncodeTimeUs = 100 * rtc::kNumMicrosecsPerMillisec; + + const int kMinIntervalUs = 30 * rtc::kNumMicrosecsPerMillisec; + const int kMaxIntervalUs = 1000 * rtc::kNumMicrosecsPerMillisec; + + InsertAndSendFramesWithRandomInterval(kNumFrames, kMinIntervalUs, + kMaxIntervalUs, kWidth, kHeight, + kEncodeTimeUs); + // Average usage 19%. Check that estimate is in the right ball park. + EXPECT_NEAR(UsagePercent(), 20, 10); +} + +// Models screencast, with irregular arrival of frames, often +// exceeding the timeout interval. +TEST_F(OveruseFrameDetectorTest2, NoOveruseForRandomFrameIntervalWithReset) { + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); + EXPECT_CALL(mock_observer_, AdaptUp()).Times(::testing::AtLeast(1)); + + const int kNumFrames = 500; + const int kEncodeTimeUs = 100 * rtc::kNumMicrosecsPerMillisec; + + const int kMinIntervalUs = 30 * rtc::kNumMicrosecsPerMillisec; + const int kMaxIntervalUs = 3000 * rtc::kNumMicrosecsPerMillisec; + + InsertAndSendFramesWithRandomInterval(kNumFrames, kMinIntervalUs, + kMaxIntervalUs, kWidth, kHeight, + kEncodeTimeUs); + + // Average usage 6.6%, but since the frame_timeout_interval_ms is + // only 1500 ms, we often reset the estimate to the initial value. + // Check that estimate is in the right ball park. + EXPECT_GE(UsagePercent(), 1); + EXPECT_LE(UsagePercent(), InitialUsage() + 5); +} + +TEST_F(OveruseFrameDetectorTest2, ToleratesOutOfOrderFrames) { + overuse_detector_->SetOptions(options_); + // Represents a cpu utilization close to 100%. First input frame results in + // three encoded frames, and the last of those isn't finished until after the + // first encoded frame corresponding to the next input frame. + const int kEncodeTimeUs = 30 * rtc::kNumMicrosecsPerMillisec; + const int kCaptureTimesMs[] = {33, 33, 66, 33}; + + for (int capture_time_ms : kCaptureTimesMs) { + overuse_detector_->FrameSent( + 0, 0, capture_time_ms * rtc::kNumMicrosecsPerMillisec, kEncodeTimeUs); + } + EXPECT_GE(UsagePercent(), InitialUsage()); +} + +// Models simulcast, with multiple encoded frames for each input frame. +// Load estimate should be based on the maximum encode time per input frame. +TEST_F(OveruseFrameDetectorTest2, NoOveruseForSimulcast) { + overuse_detector_->SetOptions(options_); + EXPECT_CALL(mock_observer_, AdaptDown()).Times(0); + + constexpr int kNumFrames = 500; + constexpr int kEncodeTimesUs[] = { + 10 * rtc::kNumMicrosecsPerMillisec, + 8 * rtc::kNumMicrosecsPerMillisec, + 12 * rtc::kNumMicrosecsPerMillisec, + }; + constexpr int kIntervalUs = 30 * rtc::kNumMicrosecsPerMillisec; + + InsertAndSendSimulcastFramesWithInterval(kNumFrames, kIntervalUs, kWidth, + kHeight, kEncodeTimesUs); + + // Average usage 40%. 12 ms / 30 ms. + EXPECT_GE(UsagePercent(), 35); + EXPECT_LE(UsagePercent(), 45); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/video/adaptation/pixel_limit_resource.cc b/third_party/libwebrtc/video/adaptation/pixel_limit_resource.cc new file mode 100644 index 0000000000..872e169879 --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/pixel_limit_resource.cc @@ -0,0 +1,101 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/adaptation/pixel_limit_resource.h" + +#include "api/sequence_checker.h" +#include "api/units/time_delta.h" +#include "call/adaptation/video_stream_adapter.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +namespace { + +constexpr TimeDelta kResourceUsageCheckIntervalMs = TimeDelta::Seconds(5); + +} // namespace + +// static +rtc::scoped_refptr<PixelLimitResource> PixelLimitResource::Create( + TaskQueueBase* task_queue, + VideoStreamInputStateProvider* input_state_provider) { + return rtc::make_ref_counted<PixelLimitResource>(task_queue, + input_state_provider); +} + +PixelLimitResource::PixelLimitResource( + TaskQueueBase* task_queue, + VideoStreamInputStateProvider* input_state_provider) + : task_queue_(task_queue), + input_state_provider_(input_state_provider), + max_pixels_(absl::nullopt) { + RTC_DCHECK(task_queue_); + RTC_DCHECK(input_state_provider_); +} + +PixelLimitResource::~PixelLimitResource() { + RTC_DCHECK(!listener_); + RTC_DCHECK(!repeating_task_.Running()); +} + +void PixelLimitResource::SetMaxPixels(int max_pixels) { + RTC_DCHECK_RUN_ON(task_queue_); + max_pixels_ = max_pixels; +} + +void PixelLimitResource::SetResourceListener(ResourceListener* listener) { + RTC_DCHECK_RUN_ON(task_queue_); + listener_ = listener; + if (listener_) { + repeating_task_.Stop(); + repeating_task_ = RepeatingTaskHandle::Start(task_queue_, [&] { + RTC_DCHECK_RUN_ON(task_queue_); + if (!listener_) { + // We don't have a listener so resource adaptation must not be running, + // try again later. + return kResourceUsageCheckIntervalMs; + } + if (!max_pixels_.has_value()) { + // No pixel limit configured yet, try again later. + return kResourceUsageCheckIntervalMs; + } + absl::optional<int> frame_size_pixels = + input_state_provider_->InputState().frame_size_pixels(); + if (!frame_size_pixels.has_value()) { + // We haven't observed a frame yet so we don't know if it's going to be + // too big or too small, try again later. + return kResourceUsageCheckIntervalMs; + } + int current_pixels = frame_size_pixels.value(); + int target_pixel_upper_bounds = max_pixels_.value(); + // To avoid toggling, we allow any resolutions between + // `target_pixel_upper_bounds` and video_stream_adapter.h's + // GetLowerResolutionThan(). This is the pixels we end up if we adapt down + // from `target_pixel_upper_bounds`. + int target_pixels_lower_bounds = + GetLowerResolutionThan(target_pixel_upper_bounds); + if (current_pixels > target_pixel_upper_bounds) { + listener_->OnResourceUsageStateMeasured( + rtc::scoped_refptr<Resource>(this), ResourceUsageState::kOveruse); + } else if (current_pixels < target_pixels_lower_bounds) { + listener_->OnResourceUsageStateMeasured( + rtc::scoped_refptr<Resource>(this), ResourceUsageState::kUnderuse); + } + return kResourceUsageCheckIntervalMs; + }); + } else { + repeating_task_.Stop(); + } + // The task must be running if we have a listener. + RTC_DCHECK(repeating_task_.Running() || !listener_); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/video/adaptation/pixel_limit_resource.h b/third_party/libwebrtc/video/adaptation/pixel_limit_resource.h new file mode 100644 index 0000000000..b42f92434f --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/pixel_limit_resource.h @@ -0,0 +1,60 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_ADAPTATION_PIXEL_LIMIT_RESOURCE_H_ +#define VIDEO_ADAPTATION_PIXEL_LIMIT_RESOURCE_H_ + +#include <string> + +#include "absl/types/optional.h" +#include "api/adaptation/resource.h" +#include "api/scoped_refptr.h" +#include "call/adaptation/video_stream_input_state_provider.h" +#include "rtc_base/task_utils/repeating_task.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { + +// An adaptation resource designed to be used in the TestBed. Used to simulate +// being CPU limited. +// +// Periodically reports "overuse" or "underuse" (every 5 seconds) until the +// stream is within the bounds specified in terms of a maximum resolution and +// one resolution step lower than that (this avoids toggling when this is the +// only resource in play). When multiple resources come in to play some amount +// of toggling is still possible in edge cases but that is OK for testing +// purposes. +class PixelLimitResource : public Resource { + public: + static rtc::scoped_refptr<PixelLimitResource> Create( + TaskQueueBase* task_queue, + VideoStreamInputStateProvider* input_state_provider); + + PixelLimitResource(TaskQueueBase* task_queue, + VideoStreamInputStateProvider* input_state_provider); + ~PixelLimitResource() override; + + void SetMaxPixels(int max_pixels); + + // Resource implementation. + std::string Name() const override { return "PixelLimitResource"; } + void SetResourceListener(ResourceListener* listener) override; + + private: + TaskQueueBase* const task_queue_; + VideoStreamInputStateProvider* const input_state_provider_; + absl::optional<int> max_pixels_ RTC_GUARDED_BY(task_queue_); + webrtc::ResourceListener* listener_ RTC_GUARDED_BY(task_queue_); + RepeatingTaskHandle repeating_task_ RTC_GUARDED_BY(task_queue_); +}; + +} // namespace webrtc + +#endif // VIDEO_ADAPTATION_PIXEL_LIMIT_RESOURCE_H_ diff --git a/third_party/libwebrtc/video/adaptation/pixel_limit_resource_unittest.cc b/third_party/libwebrtc/video/adaptation/pixel_limit_resource_unittest.cc new file mode 100644 index 0000000000..28eb19b1aa --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/pixel_limit_resource_unittest.cc @@ -0,0 +1,147 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/adaptation/pixel_limit_resource.h" + +#include <memory> +#include <utility> + +#include "absl/functional/any_invocable.h" +#include "api/units/timestamp.h" +#include "call/adaptation/test/fake_video_stream_input_state_provider.h" +#include "call/adaptation/test/mock_resource_listener.h" +#include "call/adaptation/video_stream_adapter.h" +#include "rtc_base/task_queue_for_test.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/time_controller/simulated_time_controller.h" + +using testing::_; + +namespace webrtc { + +namespace { + +constexpr TimeDelta kResourceUsageCheckIntervalMs = TimeDelta::Seconds(5); + +} // namespace + +class PixelLimitResourceTest : public ::testing::Test { + public: + PixelLimitResourceTest() + : time_controller_(Timestamp::Micros(1234)), + task_queue_(time_controller_.GetTaskQueueFactory()->CreateTaskQueue( + "TestQueue", + TaskQueueFactory::Priority::NORMAL)), + input_state_provider_() {} + + void SetCurrentPixels(int current_pixels) { + input_state_provider_.SetInputState(current_pixels, 30, current_pixels); + } + + void RunTaskOnTaskQueue(absl::AnyInvocable<void() &&> task) { + task_queue_->PostTask(std::move(task)); + time_controller_.AdvanceTime(TimeDelta::Zero()); + } + + protected: + // Posted tasks, including repeated tasks, are executed when simulated time is + // advanced by time_controller_.AdvanceTime(). + GlobalSimulatedTimeController time_controller_; + std::unique_ptr<TaskQueueBase, TaskQueueDeleter> task_queue_; + FakeVideoStreamInputStateProvider input_state_provider_; +}; + +TEST_F(PixelLimitResourceTest, ResourceIsSilentByDefault) { + // Because our mock is strick, the test would fail if + // OnResourceUsageStateMeasured() is invoked. + testing::StrictMock<MockResourceListener> resource_listener; + RunTaskOnTaskQueue([&]() { + rtc::scoped_refptr<PixelLimitResource> pixel_limit_resource = + PixelLimitResource::Create(task_queue_.get(), &input_state_provider_); + pixel_limit_resource->SetResourceListener(&resource_listener); + // Set a current pixel count. + SetCurrentPixels(1280 * 720); + // Advance a significant amount of time. + time_controller_.AdvanceTime(kResourceUsageCheckIntervalMs * 10); + pixel_limit_resource->SetResourceListener(nullptr); + }); +} + +TEST_F(PixelLimitResourceTest, + OveruseIsReportedWhileCurrentPixelsIsGreaterThanMaxPixels) { + constexpr int kMaxPixels = 640 * 480; + testing::StrictMock<MockResourceListener> resource_listener; + RunTaskOnTaskQueue([&]() { + rtc::scoped_refptr<PixelLimitResource> pixel_limit_resource = + PixelLimitResource::Create(task_queue_.get(), &input_state_provider_); + pixel_limit_resource->SetResourceListener(&resource_listener); + time_controller_.AdvanceTime(TimeDelta::Zero()); + + pixel_limit_resource->SetMaxPixels(kMaxPixels); + SetCurrentPixels(kMaxPixels + 1); + EXPECT_CALL(resource_listener, + OnResourceUsageStateMeasured(_, ResourceUsageState::kOveruse)) + .Times(1); + time_controller_.AdvanceTime(kResourceUsageCheckIntervalMs); + + // As long as the current pixels has not updated, the overuse signal is + // repeated at a fixed interval. + EXPECT_CALL(resource_listener, + OnResourceUsageStateMeasured(_, ResourceUsageState::kOveruse)) + .Times(3); + time_controller_.AdvanceTime(kResourceUsageCheckIntervalMs * 3); + + // When the overuse signal has resulted in a lower resolution, the overuse + // signals stops. + SetCurrentPixels(kMaxPixels); + EXPECT_CALL(resource_listener, OnResourceUsageStateMeasured(_, _)).Times(0); + time_controller_.AdvanceTime(kResourceUsageCheckIntervalMs * 3); + + pixel_limit_resource->SetResourceListener(nullptr); + }); +} + +TEST_F(PixelLimitResourceTest, + UnderuseIsReportedWhileCurrentPixelsIsLessThanMinPixels) { + constexpr int kMaxPixels = 640 * 480; + const int kMinPixels = GetLowerResolutionThan(kMaxPixels); + testing::StrictMock<MockResourceListener> resource_listener; + RunTaskOnTaskQueue([&]() { + rtc::scoped_refptr<PixelLimitResource> pixel_limit_resource = + PixelLimitResource::Create(task_queue_.get(), &input_state_provider_); + pixel_limit_resource->SetResourceListener(&resource_listener); + time_controller_.AdvanceTime(TimeDelta::Zero()); + + pixel_limit_resource->SetMaxPixels(kMaxPixels); + SetCurrentPixels(kMinPixels - 1); + EXPECT_CALL(resource_listener, + OnResourceUsageStateMeasured(_, ResourceUsageState::kUnderuse)) + .Times(1); + time_controller_.AdvanceTime(kResourceUsageCheckIntervalMs); + + // As long as the current pixels has not updated, the underuse signal is + // repeated at a fixed interval. + EXPECT_CALL(resource_listener, + OnResourceUsageStateMeasured(_, ResourceUsageState::kUnderuse)) + .Times(3); + time_controller_.AdvanceTime(kResourceUsageCheckIntervalMs * 3); + + // When the underuse signal has resulted in a higher resolution, the + // underuse signals stops. + SetCurrentPixels(kMinPixels); + EXPECT_CALL(resource_listener, OnResourceUsageStateMeasured(_, _)).Times(0); + time_controller_.AdvanceTime(kResourceUsageCheckIntervalMs * 3); + + pixel_limit_resource->SetResourceListener(nullptr); + }); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/video/adaptation/quality_rampup_experiment_helper.cc b/third_party/libwebrtc/video/adaptation/quality_rampup_experiment_helper.cc new file mode 100644 index 0000000000..adcad40c03 --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/quality_rampup_experiment_helper.cc @@ -0,0 +1,88 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/adaptation/quality_rampup_experiment_helper.h" + +#include <memory> +#include <utility> + +#include "rtc_base/logging.h" + +namespace webrtc { + +QualityRampUpExperimentHelper::QualityRampUpExperimentHelper( + QualityRampUpExperimentListener* experiment_listener, + Clock* clock, + QualityRampupExperiment experiment) + : experiment_listener_(experiment_listener), + clock_(clock), + quality_rampup_experiment_(std::move(experiment)), + cpu_adapted_(false), + qp_resolution_adaptations_(0) { + RTC_DCHECK(experiment_listener_); + RTC_DCHECK(clock_); +} + +std::unique_ptr<QualityRampUpExperimentHelper> +QualityRampUpExperimentHelper::CreateIfEnabled( + QualityRampUpExperimentListener* experiment_listener, + Clock* clock) { + QualityRampupExperiment experiment = QualityRampupExperiment::ParseSettings(); + if (experiment.Enabled()) { + return std::unique_ptr<QualityRampUpExperimentHelper>( + new QualityRampUpExperimentHelper(experiment_listener, clock, + experiment)); + } + return nullptr; +} + +void QualityRampUpExperimentHelper::ConfigureQualityRampupExperiment( + bool reset, + absl::optional<uint32_t> pixels, + absl::optional<DataRate> max_bitrate) { + if (reset) + quality_rampup_experiment_.Reset(); + if (pixels && max_bitrate) + quality_rampup_experiment_.SetMaxBitrate(*pixels, max_bitrate->kbps()); +} + +void QualityRampUpExperimentHelper::PerformQualityRampupExperiment( + rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource, + DataRate bandwidth, + DataRate encoder_target_bitrate, + absl::optional<DataRate> max_bitrate) { + if (!quality_scaler_resource->is_started() || !max_bitrate) + return; + + int64_t now_ms = clock_->TimeInMilliseconds(); + + bool try_quality_rampup = false; + if (quality_rampup_experiment_.BwHigh(now_ms, bandwidth.kbps())) { + // Verify that encoder is at max bitrate and the QP is low. + if (encoder_target_bitrate == *max_bitrate && + quality_scaler_resource->QpFastFilterLow()) { + try_quality_rampup = true; + } + } + if (try_quality_rampup && qp_resolution_adaptations_ > 0 && !cpu_adapted_) { + experiment_listener_->OnQualityRampUp(); + } +} + +void QualityRampUpExperimentHelper::cpu_adapted(bool cpu_adapted) { + cpu_adapted_ = cpu_adapted; +} + +void QualityRampUpExperimentHelper::qp_resolution_adaptations( + int qp_resolution_adaptations) { + qp_resolution_adaptations_ = qp_resolution_adaptations; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/video/adaptation/quality_rampup_experiment_helper.h b/third_party/libwebrtc/video/adaptation/quality_rampup_experiment_helper.h new file mode 100644 index 0000000000..4fe1f24876 --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/quality_rampup_experiment_helper.h @@ -0,0 +1,71 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_ADAPTATION_QUALITY_RAMPUP_EXPERIMENT_HELPER_H_ +#define VIDEO_ADAPTATION_QUALITY_RAMPUP_EXPERIMENT_HELPER_H_ + +#include <memory> + +#include "api/scoped_refptr.h" +#include "api/units/data_rate.h" +#include "rtc_base/experiments/quality_rampup_experiment.h" +#include "system_wrappers/include/clock.h" +#include "video/adaptation/quality_scaler_resource.h" + +namespace webrtc { + +class QualityRampUpExperimentListener { + public: + virtual ~QualityRampUpExperimentListener() = default; + virtual void OnQualityRampUp() = 0; +}; + +// Helper class for orchestrating the WebRTC-Video-QualityRampupSettings +// experiment. +class QualityRampUpExperimentHelper { + public: + // Returns a QualityRampUpExperimentHelper if the experiment is enabled, + // an nullptr otherwise. + static std::unique_ptr<QualityRampUpExperimentHelper> CreateIfEnabled( + QualityRampUpExperimentListener* experiment_listener, + Clock* clock); + + QualityRampUpExperimentHelper(const QualityRampUpExperimentHelper&) = delete; + QualityRampUpExperimentHelper& operator=( + const QualityRampUpExperimentHelper&) = delete; + + void cpu_adapted(bool cpu_adapted); + void qp_resolution_adaptations(int qp_adaptations); + + void ConfigureQualityRampupExperiment(bool reset, + absl::optional<uint32_t> pixels, + absl::optional<DataRate> max_bitrate); + + void PerformQualityRampupExperiment( + rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource, + DataRate bandwidth, + DataRate encoder_target_bitrate, + absl::optional<DataRate> max_bitrate); + + private: + QualityRampUpExperimentHelper( + QualityRampUpExperimentListener* experiment_listener, + Clock* clock, + QualityRampupExperiment experiment); + QualityRampUpExperimentListener* const experiment_listener_; + Clock* clock_; + QualityRampupExperiment quality_rampup_experiment_; + bool cpu_adapted_; + int qp_resolution_adaptations_; +}; + +} // namespace webrtc + +#endif // VIDEO_ADAPTATION_QUALITY_RAMPUP_EXPERIMENT_HELPER_H_ diff --git a/third_party/libwebrtc/video/adaptation/quality_scaler_resource.cc b/third_party/libwebrtc/video/adaptation/quality_scaler_resource.cc new file mode 100644 index 0000000000..68d56fe29e --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/quality_scaler_resource.cc @@ -0,0 +1,99 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/adaptation/quality_scaler_resource.h" + +#include <utility> + +#include "rtc_base/checks.h" +#include "rtc_base/experiments/balanced_degradation_settings.h" +#include "rtc_base/time_utils.h" + +namespace webrtc { + +// static +rtc::scoped_refptr<QualityScalerResource> QualityScalerResource::Create() { + return rtc::make_ref_counted<QualityScalerResource>(); +} + +QualityScalerResource::QualityScalerResource() + : VideoStreamEncoderResource("QualityScalerResource"), + quality_scaler_(nullptr) {} + +QualityScalerResource::~QualityScalerResource() { + RTC_DCHECK(!quality_scaler_); +} + +bool QualityScalerResource::is_started() const { + RTC_DCHECK_RUN_ON(encoder_queue()); + return quality_scaler_.get(); +} + +void QualityScalerResource::StartCheckForOveruse( + VideoEncoder::QpThresholds qp_thresholds) { + RTC_DCHECK_RUN_ON(encoder_queue()); + RTC_DCHECK(!is_started()); + quality_scaler_ = + std::make_unique<QualityScaler>(this, std::move(qp_thresholds)); +} + +void QualityScalerResource::StopCheckForOveruse() { + RTC_DCHECK_RUN_ON(encoder_queue()); + RTC_DCHECK(is_started()); + // Ensure we have no pending callbacks. This makes it safe to destroy the + // QualityScaler and even task queues with tasks in-flight. + quality_scaler_.reset(); +} + +void QualityScalerResource::SetQpThresholds( + VideoEncoder::QpThresholds qp_thresholds) { + RTC_DCHECK_RUN_ON(encoder_queue()); + RTC_DCHECK(is_started()); + quality_scaler_->SetQpThresholds(std::move(qp_thresholds)); +} + +bool QualityScalerResource::QpFastFilterLow() { + RTC_DCHECK_RUN_ON(encoder_queue()); + RTC_DCHECK(is_started()); + return quality_scaler_->QpFastFilterLow(); +} + +void QualityScalerResource::OnEncodeCompleted(const EncodedImage& encoded_image, + int64_t time_sent_in_us) { + RTC_DCHECK_RUN_ON(encoder_queue()); + if (quality_scaler_ && encoded_image.qp_ >= 0) { + quality_scaler_->ReportQp(encoded_image.qp_, time_sent_in_us); + } +} + +void QualityScalerResource::OnFrameDropped( + EncodedImageCallback::DropReason reason) { + RTC_DCHECK_RUN_ON(encoder_queue()); + if (!quality_scaler_) + return; + switch (reason) { + case EncodedImageCallback::DropReason::kDroppedByMediaOptimizations: + quality_scaler_->ReportDroppedFrameByMediaOpt(); + break; + case EncodedImageCallback::DropReason::kDroppedByEncoder: + quality_scaler_->ReportDroppedFrameByEncoder(); + break; + } +} + +void QualityScalerResource::OnReportQpUsageHigh() { + OnResourceUsageStateMeasured(ResourceUsageState::kOveruse); +} + +void QualityScalerResource::OnReportQpUsageLow() { + OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/video/adaptation/quality_scaler_resource.h b/third_party/libwebrtc/video/adaptation/quality_scaler_resource.h new file mode 100644 index 0000000000..cbb6d3d06f --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/quality_scaler_resource.h @@ -0,0 +1,59 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_ADAPTATION_QUALITY_SCALER_RESOURCE_H_ +#define VIDEO_ADAPTATION_QUALITY_SCALER_RESOURCE_H_ + +#include <memory> +#include <queue> +#include <string> + +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "api/video/video_adaptation_reason.h" +#include "api/video_codecs/video_encoder.h" +#include "call/adaptation/degradation_preference_provider.h" +#include "call/adaptation/resource_adaptation_processor_interface.h" +#include "modules/video_coding/utility/quality_scaler.h" +#include "video/adaptation/video_stream_encoder_resource.h" + +namespace webrtc { + +// Handles interaction with the QualityScaler. +class QualityScalerResource : public VideoStreamEncoderResource, + public QualityScalerQpUsageHandlerInterface { + public: + static rtc::scoped_refptr<QualityScalerResource> Create(); + + QualityScalerResource(); + ~QualityScalerResource() override; + + bool is_started() const; + + void StartCheckForOveruse(VideoEncoder::QpThresholds qp_thresholds); + void StopCheckForOveruse(); + void SetQpThresholds(VideoEncoder::QpThresholds qp_thresholds); + bool QpFastFilterLow(); + void OnEncodeCompleted(const EncodedImage& encoded_image, + int64_t time_sent_in_us); + void OnFrameDropped(EncodedImageCallback::DropReason reason); + + // QualityScalerQpUsageHandlerInterface implementation. + void OnReportQpUsageHigh() override; + void OnReportQpUsageLow() override; + + private: + std::unique_ptr<QualityScaler> quality_scaler_ + RTC_GUARDED_BY(encoder_queue()); +}; + +} // namespace webrtc + +#endif // VIDEO_ADAPTATION_QUALITY_SCALER_RESOURCE_H_ diff --git a/third_party/libwebrtc/video/adaptation/quality_scaler_resource_unittest.cc b/third_party/libwebrtc/video/adaptation/quality_scaler_resource_unittest.cc new file mode 100644 index 0000000000..70d297588f --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/quality_scaler_resource_unittest.cc @@ -0,0 +1,76 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/adaptation/quality_scaler_resource.h" + +#include <memory> + +#include "absl/types/optional.h" +#include "api/task_queue/task_queue_base.h" +#include "api/video_codecs/video_encoder.h" +#include "call/adaptation/test/mock_resource_listener.h" +#include "rtc_base/thread.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +using testing::_; +using testing::Eq; +using testing::StrictMock; + +namespace { + +class FakeDegradationPreferenceProvider : public DegradationPreferenceProvider { + public: + ~FakeDegradationPreferenceProvider() override = default; + + DegradationPreference degradation_preference() const override { + return DegradationPreference::MAINTAIN_FRAMERATE; + } +}; + +} // namespace + +class QualityScalerResourceTest : public ::testing::Test { + public: + QualityScalerResourceTest() + : quality_scaler_resource_(QualityScalerResource::Create()) { + quality_scaler_resource_->RegisterEncoderTaskQueue( + TaskQueueBase::Current()); + quality_scaler_resource_->SetResourceListener(&fake_resource_listener_); + } + + ~QualityScalerResourceTest() override { + quality_scaler_resource_->SetResourceListener(nullptr); + } + + protected: + rtc::AutoThread main_thread_; + StrictMock<MockResourceListener> fake_resource_listener_; + FakeDegradationPreferenceProvider degradation_preference_provider_; + rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource_; +}; + +TEST_F(QualityScalerResourceTest, ReportQpHigh) { + EXPECT_CALL(fake_resource_listener_, + OnResourceUsageStateMeasured(Eq(quality_scaler_resource_), + Eq(ResourceUsageState::kOveruse))); + quality_scaler_resource_->OnReportQpUsageHigh(); +} + +TEST_F(QualityScalerResourceTest, ReportQpLow) { + EXPECT_CALL(fake_resource_listener_, + OnResourceUsageStateMeasured(Eq(quality_scaler_resource_), + Eq(ResourceUsageState::kUnderuse))); + quality_scaler_resource_->OnReportQpUsageLow(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/video/adaptation/video_adaptation_gn/moz.build b/third_party/libwebrtc/video/adaptation/video_adaptation_gn/moz.build new file mode 100644 index 0000000000..e0f103cc6c --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/video_adaptation_gn/moz.build @@ -0,0 +1,241 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/video/adaptation/balanced_constraint.cc", + "/third_party/libwebrtc/video/adaptation/bandwidth_quality_scaler_resource.cc", + "/third_party/libwebrtc/video/adaptation/bitrate_constraint.cc", + "/third_party/libwebrtc/video/adaptation/encode_usage_resource.cc", + "/third_party/libwebrtc/video/adaptation/overuse_frame_detector.cc", + "/third_party/libwebrtc/video/adaptation/pixel_limit_resource.cc", + "/third_party/libwebrtc/video/adaptation/quality_rampup_experiment_helper.cc", + "/third_party/libwebrtc/video/adaptation/quality_scaler_resource.cc", + "/third_party/libwebrtc/video/adaptation/video_stream_encoder_resource.cc", + "/third_party/libwebrtc/video/adaptation/video_stream_encoder_resource_manager.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("video_adaptation_gn") diff --git a/third_party/libwebrtc/video/adaptation/video_stream_encoder_resource.cc b/third_party/libwebrtc/video/adaptation/video_stream_encoder_resource.cc new file mode 100644 index 0000000000..ad89aef52a --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/video_stream_encoder_resource.cc @@ -0,0 +1,63 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/adaptation/video_stream_encoder_resource.h" + +#include <algorithm> +#include <utility> + +namespace webrtc { + +VideoStreamEncoderResource::VideoStreamEncoderResource(std::string name) + : lock_(), + name_(std::move(name)), + encoder_queue_(nullptr), + listener_(nullptr) {} + +VideoStreamEncoderResource::~VideoStreamEncoderResource() { + RTC_DCHECK(!listener_) + << "There is a listener depending on a VideoStreamEncoderResource being " + << "destroyed."; +} + +void VideoStreamEncoderResource::RegisterEncoderTaskQueue( + TaskQueueBase* encoder_queue) { + RTC_DCHECK(!encoder_queue_); + RTC_DCHECK(encoder_queue); + encoder_queue_ = encoder_queue; +} + +void VideoStreamEncoderResource::SetResourceListener( + ResourceListener* listener) { + // If you want to change listener you need to unregister the old listener by + // setting it to null first. + MutexLock crit(&lock_); + RTC_DCHECK(!listener_ || !listener) << "A listener is already set"; + listener_ = listener; +} + +std::string VideoStreamEncoderResource::Name() const { + return name_; +} + +void VideoStreamEncoderResource::OnResourceUsageStateMeasured( + ResourceUsageState usage_state) { + MutexLock crit(&lock_); + if (listener_) { + listener_->OnResourceUsageStateMeasured(rtc::scoped_refptr<Resource>(this), + usage_state); + } +} + +TaskQueueBase* VideoStreamEncoderResource::encoder_queue() const { + return encoder_queue_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/video/adaptation/video_stream_encoder_resource.h b/third_party/libwebrtc/video/adaptation/video_stream_encoder_resource.h new file mode 100644 index 0000000000..e10f595757 --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/video_stream_encoder_resource.h @@ -0,0 +1,55 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_H_ +#define VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_H_ + +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/adaptation/resource.h" +#include "api/sequence_checker.h" +#include "api/task_queue/task_queue_base.h" +#include "call/adaptation/adaptation_constraint.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { + +class VideoStreamEncoderResource : public Resource { + public: + ~VideoStreamEncoderResource() override; + + // Registering task queues must be performed as part of initialization. + void RegisterEncoderTaskQueue(TaskQueueBase* encoder_queue); + + // Resource implementation. + std::string Name() const override; + void SetResourceListener(ResourceListener* listener) override; + + protected: + explicit VideoStreamEncoderResource(std::string name); + + void OnResourceUsageStateMeasured(ResourceUsageState usage_state); + + // The caller is responsible for ensuring the task queue is still valid. + TaskQueueBase* encoder_queue() const; + + private: + mutable Mutex lock_; + const std::string name_; + // Treated as const after initialization. + TaskQueueBase* encoder_queue_; + ResourceListener* listener_ RTC_GUARDED_BY(lock_); +}; + +} // namespace webrtc + +#endif // VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_H_ diff --git a/third_party/libwebrtc/video/adaptation/video_stream_encoder_resource_manager.cc b/third_party/libwebrtc/video/adaptation/video_stream_encoder_resource_manager.cc new file mode 100644 index 0000000000..2470bc8893 --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/video_stream_encoder_resource_manager.cc @@ -0,0 +1,851 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/adaptation/video_stream_encoder_resource_manager.h" + +#include <stdio.h> + +#include <algorithm> +#include <cmath> +#include <limits> +#include <memory> +#include <utility> + +#include "absl/algorithm/container.h" +#include "absl/base/macros.h" +#include "api/adaptation/resource.h" +#include "api/sequence_checker.h" +#include "api/task_queue/task_queue_base.h" +#include "api/video/video_adaptation_reason.h" +#include "api/video/video_source_interface.h" +#include "call/adaptation/video_source_restrictions.h" +#include "modules/video_coding/svc/scalability_mode_util.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/time_utils.h" +#include "rtc_base/trace_event.h" +#include "video/adaptation/quality_scaler_resource.h" + +namespace webrtc { + +const int kDefaultInputPixelsWidth = 176; +const int kDefaultInputPixelsHeight = 144; + +namespace { + +constexpr const char* kPixelLimitResourceFieldTrialName = + "WebRTC-PixelLimitResource"; + +bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) { + return degradation_preference == DegradationPreference::MAINTAIN_FRAMERATE || + degradation_preference == DegradationPreference::BALANCED; +} + +bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) { + return degradation_preference == DegradationPreference::MAINTAIN_RESOLUTION || + degradation_preference == DegradationPreference::BALANCED; +} + +std::string ToString(VideoAdaptationReason reason) { + switch (reason) { + case VideoAdaptationReason::kQuality: + return "quality"; + case VideoAdaptationReason::kCpu: + return "cpu"; + } + RTC_CHECK_NOTREACHED(); +} + +std::vector<bool> GetActiveLayersFlags(const VideoCodec& codec) { + std::vector<bool> flags; + if (codec.codecType == VideoCodecType::kVideoCodecVP9) { + flags.resize(codec.VP9().numberOfSpatialLayers); + for (size_t i = 0; i < flags.size(); ++i) { + flags[i] = codec.spatialLayers[i].active; + } + } else { + flags.resize(codec.numberOfSimulcastStreams); + for (size_t i = 0; i < flags.size(); ++i) { + flags[i] = codec.simulcastStream[i].active; + } + } + return flags; +} + +bool EqualFlags(const std::vector<bool>& a, const std::vector<bool>& b) { + if (a.size() != b.size()) + return false; + return std::equal(a.begin(), a.end(), b.begin()); +} + +absl::optional<DataRate> GetSingleActiveLayerMaxBitrate( + const VideoCodec& codec) { + int num_active = 0; + absl::optional<DataRate> max_bitrate; + if (codec.codecType == VideoCodecType::kVideoCodecVP9) { + for (int i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) { + if (codec.spatialLayers[i].active) { + ++num_active; + max_bitrate = + DataRate::KilobitsPerSec(codec.spatialLayers[i].maxBitrate); + } + } + } else { + for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) { + if (codec.simulcastStream[i].active) { + ++num_active; + max_bitrate = + DataRate::KilobitsPerSec(codec.simulcastStream[i].maxBitrate); + } + } + } + return (num_active > 1) ? absl::nullopt : max_bitrate; +} + +} // namespace + +class VideoStreamEncoderResourceManager::InitialFrameDropper { + public: + explicit InitialFrameDropper( + rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource) + : quality_scaler_resource_(quality_scaler_resource), + quality_scaler_settings_(QualityScalerSettings::ParseFromFieldTrials()), + has_seen_first_bwe_drop_(false), + set_start_bitrate_(DataRate::Zero()), + set_start_bitrate_time_ms_(0), + initial_framedrop_(0), + use_bandwidth_allocation_(false), + bandwidth_allocation_(DataRate::Zero()), + last_input_width_(0), + last_input_height_(0), + last_stream_configuration_changed_(false) { + RTC_DCHECK(quality_scaler_resource_); + } + + // Output signal. + bool DropInitialFrames() const { + return initial_framedrop_ < kMaxInitialFramedrop; + } + + absl::optional<uint32_t> single_active_stream_pixels() const { + return single_active_stream_pixels_; + } + + absl::optional<uint32_t> UseBandwidthAllocationBps() const { + return (use_bandwidth_allocation_ && + bandwidth_allocation_ > DataRate::Zero()) + ? absl::optional<uint32_t>(bandwidth_allocation_.bps()) + : absl::nullopt; + } + + bool last_stream_configuration_changed() const { + return last_stream_configuration_changed_; + } + + // Input signals. + void SetStartBitrate(DataRate start_bitrate, int64_t now_ms) { + set_start_bitrate_ = start_bitrate; + set_start_bitrate_time_ms_ = now_ms; + } + + void SetBandwidthAllocation(DataRate bandwidth_allocation) { + bandwidth_allocation_ = bandwidth_allocation; + } + + void SetTargetBitrate(DataRate target_bitrate, int64_t now_ms) { + if (set_start_bitrate_ > DataRate::Zero() && !has_seen_first_bwe_drop_ && + quality_scaler_resource_->is_started() && + quality_scaler_settings_.InitialBitrateIntervalMs() && + quality_scaler_settings_.InitialBitrateFactor()) { + int64_t diff_ms = now_ms - set_start_bitrate_time_ms_; + if (diff_ms < + quality_scaler_settings_.InitialBitrateIntervalMs().value() && + (target_bitrate < + (set_start_bitrate_ * + quality_scaler_settings_.InitialBitrateFactor().value()))) { + RTC_LOG(LS_INFO) << "Reset initial_framedrop_. Start bitrate: " + << set_start_bitrate_.bps() + << ", target bitrate: " << target_bitrate.bps(); + initial_framedrop_ = 0; + has_seen_first_bwe_drop_ = true; + } + } + } + + void OnEncoderSettingsUpdated( + const VideoCodec& codec, + const VideoAdaptationCounters& adaptation_counters) { + last_stream_configuration_changed_ = false; + std::vector<bool> active_flags = GetActiveLayersFlags(codec); + // Check if the source resolution has changed for the external reasons, + // i.e. without any adaptation from WebRTC. + const bool source_resolution_changed = + (last_input_width_ != codec.width || + last_input_height_ != codec.height) && + adaptation_counters.resolution_adaptations == + last_adaptation_counters_.resolution_adaptations; + if (!EqualFlags(active_flags, last_active_flags_) || + source_resolution_changed) { + // Streams configuration has changed. + last_stream_configuration_changed_ = true; + // Initial frame drop must be enabled because BWE might be way too low + // for the selected resolution. + if (quality_scaler_resource_->is_started()) { + RTC_LOG(LS_INFO) << "Resetting initial_framedrop_ due to changed " + "stream parameters"; + initial_framedrop_ = 0; + if (single_active_stream_pixels_ && + VideoStreamAdapter::GetSingleActiveLayerPixels(codec) > + *single_active_stream_pixels_) { + // Resolution increased. + use_bandwidth_allocation_ = true; + } + } + } + last_adaptation_counters_ = adaptation_counters; + last_active_flags_ = active_flags; + last_input_width_ = codec.width; + last_input_height_ = codec.height; + single_active_stream_pixels_ = + VideoStreamAdapter::GetSingleActiveLayerPixels(codec); + } + + void OnFrameDroppedDueToSize() { ++initial_framedrop_; } + + void Disable() { + initial_framedrop_ = kMaxInitialFramedrop; + use_bandwidth_allocation_ = false; + } + + void OnQualityScalerSettingsUpdated() { + if (quality_scaler_resource_->is_started()) { + // Restart frame drops due to size. + initial_framedrop_ = 0; + } else { + // Quality scaling disabled so we shouldn't drop initial frames. + Disable(); + } + } + + private: + // The maximum number of frames to drop at beginning of stream to try and + // achieve desired bitrate. + static const int kMaxInitialFramedrop = 4; + + const rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource_; + const QualityScalerSettings quality_scaler_settings_; + bool has_seen_first_bwe_drop_; + DataRate set_start_bitrate_; + int64_t set_start_bitrate_time_ms_; + // Counts how many frames we've dropped in the initial framedrop phase. + int initial_framedrop_; + absl::optional<uint32_t> single_active_stream_pixels_; + bool use_bandwidth_allocation_; + DataRate bandwidth_allocation_; + + std::vector<bool> last_active_flags_; + VideoAdaptationCounters last_adaptation_counters_; + int last_input_width_; + int last_input_height_; + bool last_stream_configuration_changed_; +}; + +VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager( + VideoStreamInputStateProvider* input_state_provider, + VideoStreamEncoderObserver* encoder_stats_observer, + Clock* clock, + bool experiment_cpu_load_estimator, + std::unique_ptr<OveruseFrameDetector> overuse_detector, + DegradationPreferenceProvider* degradation_preference_provider, + const FieldTrialsView& field_trials) + : field_trials_(field_trials), + degradation_preference_provider_(degradation_preference_provider), + bitrate_constraint_(std::make_unique<BitrateConstraint>()), + balanced_constraint_( + std::make_unique<BalancedConstraint>(degradation_preference_provider_, + field_trials)), + encode_usage_resource_( + EncodeUsageResource::Create(std::move(overuse_detector))), + quality_scaler_resource_(QualityScalerResource::Create()), + pixel_limit_resource_(nullptr), + bandwidth_quality_scaler_resource_( + BandwidthQualityScalerResource::Create()), + encoder_queue_(nullptr), + input_state_provider_(input_state_provider), + adaptation_processor_(nullptr), + encoder_stats_observer_(encoder_stats_observer), + degradation_preference_(DegradationPreference::DISABLED), + video_source_restrictions_(), + balanced_settings_(field_trials), + clock_(clock), + experiment_cpu_load_estimator_(experiment_cpu_load_estimator), + initial_frame_dropper_( + std::make_unique<InitialFrameDropper>(quality_scaler_resource_)), + quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), + pixel_limit_resource_experiment_enabled_( + field_trials.IsEnabled(kPixelLimitResourceFieldTrialName)), + encoder_target_bitrate_bps_(absl::nullopt), + quality_rampup_experiment_( + QualityRampUpExperimentHelper::CreateIfEnabled(this, clock_)), + encoder_settings_(absl::nullopt) { + TRACE_EVENT0( + "webrtc", + "VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager"); + RTC_CHECK(degradation_preference_provider_); + RTC_CHECK(encoder_stats_observer_); +} + +VideoStreamEncoderResourceManager::~VideoStreamEncoderResourceManager() = + default; + +void VideoStreamEncoderResourceManager::Initialize( + TaskQueueBase* encoder_queue) { + RTC_DCHECK(!encoder_queue_); + RTC_DCHECK(encoder_queue); + encoder_queue_ = encoder_queue; + encode_usage_resource_->RegisterEncoderTaskQueue(encoder_queue_); + quality_scaler_resource_->RegisterEncoderTaskQueue(encoder_queue_); + bandwidth_quality_scaler_resource_->RegisterEncoderTaskQueue(encoder_queue_); +} + +void VideoStreamEncoderResourceManager::SetAdaptationProcessor( + ResourceAdaptationProcessorInterface* adaptation_processor, + VideoStreamAdapter* stream_adapter) { + RTC_DCHECK_RUN_ON(encoder_queue_); + adaptation_processor_ = adaptation_processor; + stream_adapter_ = stream_adapter; +} + +void VideoStreamEncoderResourceManager::SetDegradationPreferences( + DegradationPreference degradation_preference) { + RTC_DCHECK_RUN_ON(encoder_queue_); + degradation_preference_ = degradation_preference; + UpdateStatsAdaptationSettings(); +} + +DegradationPreference +VideoStreamEncoderResourceManager::degradation_preference() const { + RTC_DCHECK_RUN_ON(encoder_queue_); + return degradation_preference_; +} + +void VideoStreamEncoderResourceManager::ConfigureEncodeUsageResource() { + RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK(encoder_settings_.has_value()); + if (encode_usage_resource_->is_started()) { + encode_usage_resource_->StopCheckForOveruse(); + } else { + // If the resource has not yet started then it needs to be added. + AddResource(encode_usage_resource_, VideoAdaptationReason::kCpu); + } + encode_usage_resource_->StartCheckForOveruse(GetCpuOveruseOptions()); +} + +void VideoStreamEncoderResourceManager::MaybeInitializePixelLimitResource() { + RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK(adaptation_processor_); + RTC_DCHECK(!pixel_limit_resource_); + if (!pixel_limit_resource_experiment_enabled_) { + // The field trial is not running. + return; + } + int max_pixels = 0; + std::string pixel_limit_field_trial = + field_trials_.Lookup(kPixelLimitResourceFieldTrialName); + if (sscanf(pixel_limit_field_trial.c_str(), "Enabled-%d", &max_pixels) != 1) { + RTC_LOG(LS_ERROR) << "Couldn't parse " << kPixelLimitResourceFieldTrialName + << " trial config: " << pixel_limit_field_trial; + return; + } + RTC_LOG(LS_INFO) << "Running field trial " + << kPixelLimitResourceFieldTrialName << " configured to " + << max_pixels << " max pixels"; + // Configure the specified max pixels from the field trial. The pixel limit + // resource is active for the lifetme of the stream (until + // StopManagedResources() is called). + pixel_limit_resource_ = + PixelLimitResource::Create(encoder_queue_, input_state_provider_); + pixel_limit_resource_->SetMaxPixels(max_pixels); + AddResource(pixel_limit_resource_, VideoAdaptationReason::kCpu); +} + +void VideoStreamEncoderResourceManager::StopManagedResources() { + RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK(adaptation_processor_); + if (encode_usage_resource_->is_started()) { + encode_usage_resource_->StopCheckForOveruse(); + RemoveResource(encode_usage_resource_); + } + if (quality_scaler_resource_->is_started()) { + quality_scaler_resource_->StopCheckForOveruse(); + RemoveResource(quality_scaler_resource_); + } + if (pixel_limit_resource_) { + RemoveResource(pixel_limit_resource_); + pixel_limit_resource_ = nullptr; + } + if (bandwidth_quality_scaler_resource_->is_started()) { + bandwidth_quality_scaler_resource_->StopCheckForOveruse(); + RemoveResource(bandwidth_quality_scaler_resource_); + } +} + +void VideoStreamEncoderResourceManager::AddResource( + rtc::scoped_refptr<Resource> resource, + VideoAdaptationReason reason) { + RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK(resource); + bool inserted; + std::tie(std::ignore, inserted) = resources_.emplace(resource, reason); + RTC_DCHECK(inserted) << "Resource " << resource->Name() + << " already was inserted"; + adaptation_processor_->AddResource(resource); +} + +void VideoStreamEncoderResourceManager::RemoveResource( + rtc::scoped_refptr<Resource> resource) { + { + RTC_DCHECK_RUN_ON(encoder_queue_); + RTC_DCHECK(resource); + const auto& it = resources_.find(resource); + RTC_DCHECK(it != resources_.end()) + << "Resource \"" << resource->Name() << "\" not found."; + resources_.erase(it); + } + adaptation_processor_->RemoveResource(resource); +} + +std::vector<AdaptationConstraint*> +VideoStreamEncoderResourceManager::AdaptationConstraints() const { + RTC_DCHECK_RUN_ON(encoder_queue_); + return {bitrate_constraint_.get(), balanced_constraint_.get()}; +} + +void VideoStreamEncoderResourceManager::SetEncoderSettings( + EncoderSettings encoder_settings) { + RTC_DCHECK_RUN_ON(encoder_queue_); + encoder_settings_ = std::move(encoder_settings); + bitrate_constraint_->OnEncoderSettingsUpdated(encoder_settings_); + initial_frame_dropper_->OnEncoderSettingsUpdated( + encoder_settings_->video_codec(), current_adaptation_counters_); + MaybeUpdateTargetFrameRate(); + if (quality_rampup_experiment_) { + quality_rampup_experiment_->ConfigureQualityRampupExperiment( + initial_frame_dropper_->last_stream_configuration_changed(), + initial_frame_dropper_->single_active_stream_pixels(), + GetSingleActiveLayerMaxBitrate(encoder_settings_->video_codec())); + } +} + +void VideoStreamEncoderResourceManager::SetStartBitrate( + DataRate start_bitrate) { + RTC_DCHECK_RUN_ON(encoder_queue_); + if (!start_bitrate.IsZero()) { + encoder_target_bitrate_bps_ = start_bitrate.bps(); + bitrate_constraint_->OnEncoderTargetBitrateUpdated( + encoder_target_bitrate_bps_); + balanced_constraint_->OnEncoderTargetBitrateUpdated( + encoder_target_bitrate_bps_); + } + initial_frame_dropper_->SetStartBitrate(start_bitrate, + clock_->TimeInMicroseconds()); +} + +void VideoStreamEncoderResourceManager::SetTargetBitrate( + DataRate target_bitrate) { + RTC_DCHECK_RUN_ON(encoder_queue_); + if (!target_bitrate.IsZero()) { + encoder_target_bitrate_bps_ = target_bitrate.bps(); + bitrate_constraint_->OnEncoderTargetBitrateUpdated( + encoder_target_bitrate_bps_); + balanced_constraint_->OnEncoderTargetBitrateUpdated( + encoder_target_bitrate_bps_); + } + initial_frame_dropper_->SetTargetBitrate(target_bitrate, + clock_->TimeInMilliseconds()); +} + +void VideoStreamEncoderResourceManager::SetEncoderRates( + const VideoEncoder::RateControlParameters& encoder_rates) { + RTC_DCHECK_RUN_ON(encoder_queue_); + encoder_rates_ = encoder_rates; + initial_frame_dropper_->SetBandwidthAllocation( + encoder_rates.bandwidth_allocation); +} + +void VideoStreamEncoderResourceManager::OnFrameDroppedDueToSize() { + RTC_DCHECK_RUN_ON(encoder_queue_); + initial_frame_dropper_->OnFrameDroppedDueToSize(); + Adaptation reduce_resolution = stream_adapter_->GetAdaptDownResolution(); + if (reduce_resolution.status() == Adaptation::Status::kValid) { + stream_adapter_->ApplyAdaptation(reduce_resolution, + quality_scaler_resource_); + } +} + +void VideoStreamEncoderResourceManager::OnEncodeStarted( + const VideoFrame& cropped_frame, + int64_t time_when_first_seen_us) { + RTC_DCHECK_RUN_ON(encoder_queue_); + encode_usage_resource_->OnEncodeStarted(cropped_frame, + time_when_first_seen_us); +} + +void VideoStreamEncoderResourceManager::OnEncodeCompleted( + const EncodedImage& encoded_image, + int64_t time_sent_in_us, + absl::optional<int> encode_duration_us, + DataSize frame_size) { + RTC_DCHECK_RUN_ON(encoder_queue_); + // Inform `encode_usage_resource_` of the encode completed event. + uint32_t timestamp = encoded_image.Timestamp(); + int64_t capture_time_us = + encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec; + encode_usage_resource_->OnEncodeCompleted( + timestamp, time_sent_in_us, capture_time_us, encode_duration_us); + quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us); + bandwidth_quality_scaler_resource_->OnEncodeCompleted( + encoded_image, time_sent_in_us, frame_size.bytes()); +} + +void VideoStreamEncoderResourceManager::OnFrameDropped( + EncodedImageCallback::DropReason reason) { + RTC_DCHECK_RUN_ON(encoder_queue_); + quality_scaler_resource_->OnFrameDropped(reason); +} + +bool VideoStreamEncoderResourceManager::DropInitialFrames() const { + RTC_DCHECK_RUN_ON(encoder_queue_); + return initial_frame_dropper_->DropInitialFrames(); +} + +absl::optional<uint32_t> +VideoStreamEncoderResourceManager::SingleActiveStreamPixels() const { + RTC_DCHECK_RUN_ON(encoder_queue_); + return initial_frame_dropper_->single_active_stream_pixels(); +} + +absl::optional<uint32_t> +VideoStreamEncoderResourceManager::UseBandwidthAllocationBps() const { + RTC_DCHECK_RUN_ON(encoder_queue_); + return initial_frame_dropper_->UseBandwidthAllocationBps(); +} + +void VideoStreamEncoderResourceManager::OnMaybeEncodeFrame() { + RTC_DCHECK_RUN_ON(encoder_queue_); + initial_frame_dropper_->Disable(); + if (quality_rampup_experiment_ && quality_scaler_resource_->is_started()) { + DataRate bandwidth = encoder_rates_.has_value() + ? encoder_rates_->bandwidth_allocation + : DataRate::Zero(); + quality_rampup_experiment_->PerformQualityRampupExperiment( + quality_scaler_resource_, bandwidth, + DataRate::BitsPerSec(encoder_target_bitrate_bps_.value_or(0)), + GetSingleActiveLayerMaxBitrate(encoder_settings_->video_codec())); + } +} + +void VideoStreamEncoderResourceManager::UpdateQualityScalerSettings( + absl::optional<VideoEncoder::QpThresholds> qp_thresholds) { + RTC_DCHECK_RUN_ON(encoder_queue_); + if (qp_thresholds.has_value()) { + if (quality_scaler_resource_->is_started()) { + quality_scaler_resource_->SetQpThresholds(qp_thresholds.value()); + } else { + quality_scaler_resource_->StartCheckForOveruse(qp_thresholds.value()); + AddResource(quality_scaler_resource_, VideoAdaptationReason::kQuality); + } + } else if (quality_scaler_resource_->is_started()) { + quality_scaler_resource_->StopCheckForOveruse(); + RemoveResource(quality_scaler_resource_); + } + initial_frame_dropper_->OnQualityScalerSettingsUpdated(); +} + +void VideoStreamEncoderResourceManager::UpdateBandwidthQualityScalerSettings( + bool bandwidth_quality_scaling_allowed, + const std::vector<VideoEncoder::ResolutionBitrateLimits>& + resolution_bitrate_limits) { + RTC_DCHECK_RUN_ON(encoder_queue_); + + if (!bandwidth_quality_scaling_allowed) { + if (bandwidth_quality_scaler_resource_->is_started()) { + bandwidth_quality_scaler_resource_->StopCheckForOveruse(); + RemoveResource(bandwidth_quality_scaler_resource_); + } + } else { + if (!bandwidth_quality_scaler_resource_->is_started()) { + // Before executing "StartCheckForOveruse",we must execute "AddResource" + // firstly,because it can make the listener valid. + AddResource(bandwidth_quality_scaler_resource_, + webrtc::VideoAdaptationReason::kQuality); + bandwidth_quality_scaler_resource_->StartCheckForOveruse( + resolution_bitrate_limits); + } + } +} + +void VideoStreamEncoderResourceManager::ConfigureQualityScaler( + const VideoEncoder::EncoderInfo& encoder_info) { + RTC_DCHECK_RUN_ON(encoder_queue_); + const auto scaling_settings = encoder_info.scaling_settings; + const bool quality_scaling_allowed = + IsResolutionScalingEnabled(degradation_preference_) && + (scaling_settings.thresholds.has_value() || + (encoder_settings_.has_value() && + encoder_settings_->encoder_config().is_quality_scaling_allowed)) && + encoder_info.is_qp_trusted.value_or(true); + + // TODO(https://crbug.com/webrtc/11222): Should this move to + // QualityScalerResource? + if (quality_scaling_allowed) { + if (!quality_scaler_resource_->is_started()) { + // Quality scaler has not already been configured. + + // Use experimental thresholds if available. + absl::optional<VideoEncoder::QpThresholds> experimental_thresholds; + if (quality_scaling_experiment_enabled_) { + experimental_thresholds = QualityScalingExperiment::GetQpThresholds( + GetVideoCodecTypeOrGeneric(encoder_settings_)); + } + UpdateQualityScalerSettings(experimental_thresholds.has_value() + ? experimental_thresholds + : scaling_settings.thresholds); + } + } else { + UpdateQualityScalerSettings(absl::nullopt); + } + + // Set the qp-thresholds to the balanced settings if balanced mode. + if (degradation_preference_ == DegradationPreference::BALANCED && + quality_scaler_resource_->is_started()) { + absl::optional<VideoEncoder::QpThresholds> thresholds = + balanced_settings_.GetQpThresholds( + GetVideoCodecTypeOrGeneric(encoder_settings_), + LastFrameSizeOrDefault()); + if (thresholds) { + quality_scaler_resource_->SetQpThresholds(*thresholds); + } + } + UpdateStatsAdaptationSettings(); +} + +void VideoStreamEncoderResourceManager::ConfigureBandwidthQualityScaler( + const VideoEncoder::EncoderInfo& encoder_info) { + RTC_DCHECK_RUN_ON(encoder_queue_); + const bool bandwidth_quality_scaling_allowed = + IsResolutionScalingEnabled(degradation_preference_) && + (encoder_settings_.has_value() && + encoder_settings_->encoder_config().is_quality_scaling_allowed) && + !encoder_info.is_qp_trusted.value_or(true); + + UpdateBandwidthQualityScalerSettings(bandwidth_quality_scaling_allowed, + encoder_info.resolution_bitrate_limits); + UpdateStatsAdaptationSettings(); +} + +VideoAdaptationReason VideoStreamEncoderResourceManager::GetReasonFromResource( + rtc::scoped_refptr<Resource> resource) const { + RTC_DCHECK_RUN_ON(encoder_queue_); + const auto& registered_resource = resources_.find(resource); + RTC_DCHECK(registered_resource != resources_.end()) + << resource->Name() << " not found."; + return registered_resource->second; +} + +// TODO(pbos): Lower these thresholds (to closer to 100%) when we handle +// pipelining encoders better (multiple input frames before something comes +// out). This should effectively turn off CPU adaptations for systems that +// remotely cope with the load right now. +CpuOveruseOptions VideoStreamEncoderResourceManager::GetCpuOveruseOptions() + const { + RTC_DCHECK_RUN_ON(encoder_queue_); + // This is already ensured by the only caller of this method: + // StartResourceAdaptation(). + RTC_DCHECK(encoder_settings_.has_value()); + CpuOveruseOptions options(field_trials_); + // Hardware accelerated encoders are assumed to be pipelined; give them + // additional overuse time. + if (encoder_settings_->encoder_info().is_hardware_accelerated) { + options.low_encode_usage_threshold_percent = 150; + options.high_encode_usage_threshold_percent = 200; + } + if (experiment_cpu_load_estimator_) { + options.filter_time_ms = 5 * rtc::kNumMillisecsPerSec; + } + return options; +} + +int VideoStreamEncoderResourceManager::LastFrameSizeOrDefault() const { + RTC_DCHECK_RUN_ON(encoder_queue_); + return input_state_provider_->InputState() + .single_active_stream_pixels() + .value_or( + input_state_provider_->InputState().frame_size_pixels().value_or( + kDefaultInputPixelsWidth * kDefaultInputPixelsHeight)); +} + +void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated( + VideoSourceRestrictions restrictions, + const VideoAdaptationCounters& adaptation_counters, + rtc::scoped_refptr<Resource> reason, + const VideoSourceRestrictions& unfiltered_restrictions) { + RTC_DCHECK_RUN_ON(encoder_queue_); + current_adaptation_counters_ = adaptation_counters; + + // TODO(bugs.webrtc.org/11553) Remove reason parameter and add reset callback. + if (!reason && adaptation_counters.Total() == 0) { + // Adaptation was manually reset - clear the per-reason counters too. + encoder_stats_observer_->ClearAdaptationStats(); + } + + video_source_restrictions_ = FilterRestrictionsByDegradationPreference( + restrictions, degradation_preference_); + MaybeUpdateTargetFrameRate(); +} + +void VideoStreamEncoderResourceManager::OnResourceLimitationChanged( + rtc::scoped_refptr<Resource> resource, + const std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters>& + resource_limitations) { + RTC_DCHECK_RUN_ON(encoder_queue_); + if (!resource) { + encoder_stats_observer_->ClearAdaptationStats(); + return; + } + + std::map<VideoAdaptationReason, VideoAdaptationCounters> limitations; + for (auto& resource_counter : resource_limitations) { + std::map<VideoAdaptationReason, VideoAdaptationCounters>::iterator it; + bool inserted; + std::tie(it, inserted) = limitations.emplace( + GetReasonFromResource(resource_counter.first), resource_counter.second); + if (!inserted && it->second.Total() < resource_counter.second.Total()) { + it->second = resource_counter.second; + } + } + + VideoAdaptationReason adaptation_reason = GetReasonFromResource(resource); + encoder_stats_observer_->OnAdaptationChanged( + adaptation_reason, limitations[VideoAdaptationReason::kCpu], + limitations[VideoAdaptationReason::kQuality]); + + if (quality_rampup_experiment_) { + bool cpu_limited = limitations.at(VideoAdaptationReason::kCpu).Total() > 0; + auto qp_resolution_adaptations = + limitations.at(VideoAdaptationReason::kQuality).resolution_adaptations; + quality_rampup_experiment_->cpu_adapted(cpu_limited); + quality_rampup_experiment_->qp_resolution_adaptations( + qp_resolution_adaptations); + } + + RTC_LOG(LS_INFO) << ActiveCountsToString(limitations); +} + +void VideoStreamEncoderResourceManager::MaybeUpdateTargetFrameRate() { + RTC_DCHECK_RUN_ON(encoder_queue_); + absl::optional<double> codec_max_frame_rate = + encoder_settings_.has_value() + ? absl::optional<double>( + encoder_settings_->video_codec().maxFramerate) + : absl::nullopt; + // The current target framerate is the maximum frame rate as specified by + // the current codec configuration or any limit imposed by the adaptation + // module. This is used to make sure overuse detection doesn't needlessly + // trigger in low and/or variable framerate scenarios. + absl::optional<double> target_frame_rate = + video_source_restrictions_.max_frame_rate(); + if (!target_frame_rate.has_value() || + (codec_max_frame_rate.has_value() && + codec_max_frame_rate.value() < target_frame_rate.value())) { + target_frame_rate = codec_max_frame_rate; + } + encode_usage_resource_->SetTargetFrameRate(target_frame_rate); +} + +void VideoStreamEncoderResourceManager::UpdateStatsAdaptationSettings() const { + RTC_DCHECK_RUN_ON(encoder_queue_); + VideoStreamEncoderObserver::AdaptationSettings cpu_settings( + IsResolutionScalingEnabled(degradation_preference_), + IsFramerateScalingEnabled(degradation_preference_)); + + VideoStreamEncoderObserver::AdaptationSettings quality_settings = + (quality_scaler_resource_->is_started() || + bandwidth_quality_scaler_resource_->is_started()) + ? cpu_settings + : VideoStreamEncoderObserver::AdaptationSettings(); + encoder_stats_observer_->UpdateAdaptationSettings(cpu_settings, + quality_settings); +} + +// static +std::string VideoStreamEncoderResourceManager::ActiveCountsToString( + const std::map<VideoAdaptationReason, VideoAdaptationCounters>& + active_counts) { + rtc::StringBuilder ss; + + ss << "Downgrade counts: fps: {"; + for (auto& reason_count : active_counts) { + ss << ToString(reason_count.first) << ":"; + ss << reason_count.second.fps_adaptations; + } + ss << "}, resolution {"; + for (auto& reason_count : active_counts) { + ss << ToString(reason_count.first) << ":"; + ss << reason_count.second.resolution_adaptations; + } + ss << "}"; + + return ss.Release(); +} + +void VideoStreamEncoderResourceManager::OnQualityRampUp() { + RTC_DCHECK_RUN_ON(encoder_queue_); + stream_adapter_->ClearRestrictions(); + quality_rampup_experiment_.reset(); +} + +bool VideoStreamEncoderResourceManager::IsSimulcastOrMultipleSpatialLayers( + const VideoEncoderConfig& encoder_config) { + const std::vector<VideoStream>& simulcast_layers = + encoder_config.simulcast_layers; + if (simulcast_layers.empty()) { + return false; + } + + absl::optional<int> num_spatial_layers; + if (simulcast_layers[0].scalability_mode.has_value() && + encoder_config.number_of_streams == 1) { + num_spatial_layers = ScalabilityModeToNumSpatialLayers( + *simulcast_layers[0].scalability_mode); + } + + if (simulcast_layers.size() == 1) { + // Check if multiple spatial layers are used. + return num_spatial_layers && *num_spatial_layers > 1; + } + + bool svc_with_one_spatial_layer = + num_spatial_layers && *num_spatial_layers == 1; + if (simulcast_layers[0].active && !svc_with_one_spatial_layer) { + // We can't distinguish between simulcast and singlecast when only the + // lowest spatial layer is active. Treat this case as simulcast. + return true; + } + + int num_active_layers = + std::count_if(simulcast_layers.begin(), simulcast_layers.end(), + [](const VideoStream& layer) { return layer.active; }); + return num_active_layers > 1; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/video/adaptation/video_stream_encoder_resource_manager.h b/third_party/libwebrtc/video/adaptation/video_stream_encoder_resource_manager.h new file mode 100644 index 0000000000..e0de3f7d19 --- /dev/null +++ b/third_party/libwebrtc/video/adaptation/video_stream_encoder_resource_manager.h @@ -0,0 +1,238 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_ +#define VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_ + +#include <atomic> +#include <map> +#include <memory> +#include <string> +#include <unordered_map> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/adaptation/resource.h" +#include "api/field_trials_view.h" +#include "api/rtp_parameters.h" +#include "api/scoped_refptr.h" +#include "api/task_queue/task_queue_base.h" +#include "api/video/video_adaptation_counters.h" +#include "api/video/video_adaptation_reason.h" +#include "api/video/video_frame.h" +#include "api/video/video_source_interface.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "call/adaptation/resource_adaptation_processor_interface.h" +#include "call/adaptation/video_stream_adapter.h" +#include "call/adaptation/video_stream_input_state_provider.h" +#include "rtc_base/experiments/quality_scaler_settings.h" +#include "rtc_base/ref_count.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/clock.h" +#include "video/adaptation/balanced_constraint.h" +#include "video/adaptation/bandwidth_quality_scaler_resource.h" +#include "video/adaptation/bitrate_constraint.h" +#include "video/adaptation/encode_usage_resource.h" +#include "video/adaptation/overuse_frame_detector.h" +#include "video/adaptation/pixel_limit_resource.h" +#include "video/adaptation/quality_rampup_experiment_helper.h" +#include "video/adaptation/quality_scaler_resource.h" +#include "video/adaptation/video_stream_encoder_resource.h" +#include "video/config/video_encoder_config.h" +#include "video/video_stream_encoder_observer.h" + +namespace webrtc { + +// The assumed input frame size if we have not yet received a frame. +// TODO(hbos): This is 144p - why are we assuming super low quality? Seems like +// a bad heuristic. +extern const int kDefaultInputPixelsWidth; +extern const int kDefaultInputPixelsHeight; + +// Owns adaptation-related Resources pertaining to a single VideoStreamEncoder +// and passes on the relevant input from the encoder to the resources. The +// resources provide resource usage states to the ResourceAdaptationProcessor +// which is responsible for reconfiguring streams in order not to overuse +// resources. +// +// The manager is also involved with various mitigations not part of the +// ResourceAdaptationProcessor code such as the initial frame dropping. +class VideoStreamEncoderResourceManager + : public VideoSourceRestrictionsListener, + public ResourceLimitationsListener, + public QualityRampUpExperimentListener { + public: + VideoStreamEncoderResourceManager( + VideoStreamInputStateProvider* input_state_provider, + VideoStreamEncoderObserver* encoder_stats_observer, + Clock* clock, + bool experiment_cpu_load_estimator, + std::unique_ptr<OveruseFrameDetector> overuse_detector, + DegradationPreferenceProvider* degradation_preference_provider, + const FieldTrialsView& field_trials); + ~VideoStreamEncoderResourceManager() override; + + void Initialize(TaskQueueBase* encoder_queue); + void SetAdaptationProcessor( + ResourceAdaptationProcessorInterface* adaptation_processor, + VideoStreamAdapter* stream_adapter); + + // TODO(https://crbug.com/webrtc/11563): The degradation preference is a + // setting of the Processor, it does not belong to the Manager - can we get + // rid of this? + void SetDegradationPreferences(DegradationPreference degradation_preference); + DegradationPreference degradation_preference() const; + + void ConfigureEncodeUsageResource(); + // Initializes the pixel limit resource if the "WebRTC-PixelLimitResource" + // field trial is enabled. This can be used for testing. + void MaybeInitializePixelLimitResource(); + // Stops the encode usage and quality scaler resources if not already stopped. + // If the pixel limit resource was created it is also stopped and nulled. + void StopManagedResources(); + + // Settings that affect the VideoStreamEncoder-specific resources. + void SetEncoderSettings(EncoderSettings encoder_settings); + void SetStartBitrate(DataRate start_bitrate); + void SetTargetBitrate(DataRate target_bitrate); + void SetEncoderRates( + const VideoEncoder::RateControlParameters& encoder_rates); + // TODO(https://crbug.com/webrtc/11338): This can be made private if we + // configure on SetDegredationPreference and SetEncoderSettings. + void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info); + void ConfigureBandwidthQualityScaler( + const VideoEncoder::EncoderInfo& encoder_info); + + // Methods corresponding to different points in the encoding pipeline. + void OnFrameDroppedDueToSize(); + void OnMaybeEncodeFrame(); + void OnEncodeStarted(const VideoFrame& cropped_frame, + int64_t time_when_first_seen_us); + void OnEncodeCompleted(const EncodedImage& encoded_image, + int64_t time_sent_in_us, + absl::optional<int> encode_duration_us, + DataSize frame_size); + void OnFrameDropped(EncodedImageCallback::DropReason reason); + + // Resources need to be mapped to an AdaptReason (kCpu or kQuality) in order + // to update legacy getStats(). + void AddResource(rtc::scoped_refptr<Resource> resource, + VideoAdaptationReason reason); + void RemoveResource(rtc::scoped_refptr<Resource> resource); + std::vector<AdaptationConstraint*> AdaptationConstraints() const; + // If true, the VideoStreamEncoder should execute its logic to maybe drop + // frames based on size and bitrate. + bool DropInitialFrames() const; + absl::optional<uint32_t> SingleActiveStreamPixels() const; + absl::optional<uint32_t> UseBandwidthAllocationBps() const; + + // VideoSourceRestrictionsListener implementation. + // Updates `video_source_restrictions_`. + void OnVideoSourceRestrictionsUpdated( + VideoSourceRestrictions restrictions, + const VideoAdaptationCounters& adaptation_counters, + rtc::scoped_refptr<Resource> reason, + const VideoSourceRestrictions& unfiltered_restrictions) override; + void OnResourceLimitationChanged( + rtc::scoped_refptr<Resource> resource, + const std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters>& + resource_limitations) override; + + // QualityRampUpExperimentListener implementation. + void OnQualityRampUp() override; + + static bool IsSimulcastOrMultipleSpatialLayers( + const VideoEncoderConfig& encoder_config); + + private: + class InitialFrameDropper; + + VideoAdaptationReason GetReasonFromResource( + rtc::scoped_refptr<Resource> resource) const; + + CpuOveruseOptions GetCpuOveruseOptions() const; + int LastFrameSizeOrDefault() const; + + // Calculates an up-to-date value of the target frame rate and informs the + // `encode_usage_resource_` of the new value. + void MaybeUpdateTargetFrameRate(); + + // Use nullopt to disable quality scaling. + void UpdateQualityScalerSettings( + absl::optional<VideoEncoder::QpThresholds> qp_thresholds); + + void UpdateBandwidthQualityScalerSettings( + bool bandwidth_quality_scaling_allowed, + const std::vector<VideoEncoder::ResolutionBitrateLimits>& + resolution_bitrate_limits); + + void UpdateStatsAdaptationSettings() const; + + static std::string ActiveCountsToString( + const std::map<VideoAdaptationReason, VideoAdaptationCounters>& + active_counts); + + const FieldTrialsView& field_trials_; + DegradationPreferenceProvider* const degradation_preference_provider_; + std::unique_ptr<BitrateConstraint> bitrate_constraint_ + RTC_GUARDED_BY(encoder_queue_); + const std::unique_ptr<BalancedConstraint> balanced_constraint_ + RTC_GUARDED_BY(encoder_queue_); + const rtc::scoped_refptr<EncodeUsageResource> encode_usage_resource_; + const rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource_; + rtc::scoped_refptr<PixelLimitResource> pixel_limit_resource_; + const rtc::scoped_refptr<BandwidthQualityScalerResource> + bandwidth_quality_scaler_resource_; + + TaskQueueBase* encoder_queue_; + VideoStreamInputStateProvider* const input_state_provider_ + RTC_GUARDED_BY(encoder_queue_); + ResourceAdaptationProcessorInterface* adaptation_processor_; + VideoStreamAdapter* stream_adapter_ RTC_GUARDED_BY(encoder_queue_); + // Thread-safe. + VideoStreamEncoderObserver* const encoder_stats_observer_; + + DegradationPreference degradation_preference_ RTC_GUARDED_BY(encoder_queue_); + VideoSourceRestrictions video_source_restrictions_ + RTC_GUARDED_BY(encoder_queue_); + + VideoAdaptationCounters current_adaptation_counters_ + RTC_GUARDED_BY(encoder_queue_); + + const BalancedDegradationSettings balanced_settings_; + Clock* clock_ RTC_GUARDED_BY(encoder_queue_); + const bool experiment_cpu_load_estimator_ RTC_GUARDED_BY(encoder_queue_); + const std::unique_ptr<InitialFrameDropper> initial_frame_dropper_ + RTC_GUARDED_BY(encoder_queue_); + const bool quality_scaling_experiment_enabled_ RTC_GUARDED_BY(encoder_queue_); + const bool pixel_limit_resource_experiment_enabled_ + RTC_GUARDED_BY(encoder_queue_); + absl::optional<uint32_t> encoder_target_bitrate_bps_ + RTC_GUARDED_BY(encoder_queue_); + absl::optional<VideoEncoder::RateControlParameters> encoder_rates_ + RTC_GUARDED_BY(encoder_queue_); + std::unique_ptr<QualityRampUpExperimentHelper> quality_rampup_experiment_ + RTC_GUARDED_BY(encoder_queue_); + absl::optional<EncoderSettings> encoder_settings_ + RTC_GUARDED_BY(encoder_queue_); + + // Ties a resource to a reason for statistical reporting. This AdaptReason is + // also used by this module to make decisions about how to adapt up/down. + std::map<rtc::scoped_refptr<Resource>, VideoAdaptationReason> resources_ + RTC_GUARDED_BY(encoder_queue_); +}; + +} // namespace webrtc + +#endif // VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_ |