From 36d22d82aa202bb199967e9512281e9a53db42c9 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 21:33:14 +0200 Subject: Adding upstream version 115.7.0esr. Signed-off-by: Daniel Baumann --- third_party/libwebrtc/call/adaptation/BUILD.gn | 136 +++ third_party/libwebrtc/call/adaptation/OWNERS | 3 + .../call/adaptation/adaptation_constraint.cc | 17 + .../call/adaptation/adaptation_constraint.h | 41 + .../call/adaptation/broadcast_resource_listener.cc | 122 +++ .../call/adaptation/broadcast_resource_listener.h | 75 ++ .../broadcast_resource_listener_unittest.cc | 121 +++ .../adaptation/degradation_preference_provider.cc | 14 + .../adaptation/degradation_preference_provider.h | 27 + .../libwebrtc/call/adaptation/encoder_settings.cc | 54 ++ .../libwebrtc/call/adaptation/encoder_settings.h | 48 ++ .../adaptation/resource_adaptation_gn/moz.build | 241 ++++++ .../adaptation/resource_adaptation_processor.cc | 378 ++++++++ .../adaptation/resource_adaptation_processor.h | 167 ++++ .../resource_adaptation_processor_interface.cc | 20 + .../resource_adaptation_processor_interface.h | 67 ++ .../resource_adaptation_processor_unittest.cc | 740 ++++++++++++++++ .../libwebrtc/call/adaptation/resource_unittest.cc | 55 ++ .../adaptation/test/fake_adaptation_constraint.cc | 40 + .../adaptation/test/fake_adaptation_constraint.h | 42 + .../adaptation/test/fake_frame_rate_provider.cc | 27 + .../adaptation/test/fake_frame_rate_provider.h | 69 ++ .../call/adaptation/test/fake_resource.cc | 46 + .../libwebrtc/call/adaptation/test/fake_resource.h | 45 + .../test/fake_video_stream_input_state_provider.cc | 35 + .../test/fake_video_stream_input_state_provider.h | 32 + .../call/adaptation/test/mock_resource_listener.h | 31 + .../call/adaptation/video_source_restrictions.cc | 173 ++++ .../call/adaptation/video_source_restrictions.h | 89 ++ .../video_source_restrictions_unittest.cc | 146 ++++ .../call/adaptation/video_stream_adapter.cc | 742 ++++++++++++++++ .../call/adaptation/video_stream_adapter.h | 271 ++++++ .../adaptation/video_stream_adapter_unittest.cc | 951 +++++++++++++++++++++ .../call/adaptation/video_stream_input_state.cc | 80 ++ .../call/adaptation/video_stream_input_state.h | 53 ++ .../video_stream_input_state_provider.cc | 54 ++ .../adaptation/video_stream_input_state_provider.h | 41 + .../video_stream_input_state_provider_unittest.cc | 62 ++ 38 files changed, 5355 insertions(+) create mode 100644 third_party/libwebrtc/call/adaptation/BUILD.gn create mode 100644 third_party/libwebrtc/call/adaptation/OWNERS create mode 100644 third_party/libwebrtc/call/adaptation/adaptation_constraint.cc create mode 100644 third_party/libwebrtc/call/adaptation/adaptation_constraint.h create mode 100644 third_party/libwebrtc/call/adaptation/broadcast_resource_listener.cc create mode 100644 third_party/libwebrtc/call/adaptation/broadcast_resource_listener.h create mode 100644 third_party/libwebrtc/call/adaptation/broadcast_resource_listener_unittest.cc create mode 100644 third_party/libwebrtc/call/adaptation/degradation_preference_provider.cc create mode 100644 third_party/libwebrtc/call/adaptation/degradation_preference_provider.h create mode 100644 third_party/libwebrtc/call/adaptation/encoder_settings.cc create mode 100644 third_party/libwebrtc/call/adaptation/encoder_settings.h create mode 100644 third_party/libwebrtc/call/adaptation/resource_adaptation_gn/moz.build create mode 100644 third_party/libwebrtc/call/adaptation/resource_adaptation_processor.cc create mode 100644 third_party/libwebrtc/call/adaptation/resource_adaptation_processor.h create mode 100644 third_party/libwebrtc/call/adaptation/resource_adaptation_processor_interface.cc create mode 100644 third_party/libwebrtc/call/adaptation/resource_adaptation_processor_interface.h create mode 100644 third_party/libwebrtc/call/adaptation/resource_adaptation_processor_unittest.cc create mode 100644 third_party/libwebrtc/call/adaptation/resource_unittest.cc create mode 100644 third_party/libwebrtc/call/adaptation/test/fake_adaptation_constraint.cc create mode 100644 third_party/libwebrtc/call/adaptation/test/fake_adaptation_constraint.h create mode 100644 third_party/libwebrtc/call/adaptation/test/fake_frame_rate_provider.cc create mode 100644 third_party/libwebrtc/call/adaptation/test/fake_frame_rate_provider.h create mode 100644 third_party/libwebrtc/call/adaptation/test/fake_resource.cc create mode 100644 third_party/libwebrtc/call/adaptation/test/fake_resource.h create mode 100644 third_party/libwebrtc/call/adaptation/test/fake_video_stream_input_state_provider.cc create mode 100644 third_party/libwebrtc/call/adaptation/test/fake_video_stream_input_state_provider.h create mode 100644 third_party/libwebrtc/call/adaptation/test/mock_resource_listener.h create mode 100644 third_party/libwebrtc/call/adaptation/video_source_restrictions.cc create mode 100644 third_party/libwebrtc/call/adaptation/video_source_restrictions.h create mode 100644 third_party/libwebrtc/call/adaptation/video_source_restrictions_unittest.cc create mode 100644 third_party/libwebrtc/call/adaptation/video_stream_adapter.cc create mode 100644 third_party/libwebrtc/call/adaptation/video_stream_adapter.h create mode 100644 third_party/libwebrtc/call/adaptation/video_stream_adapter_unittest.cc create mode 100644 third_party/libwebrtc/call/adaptation/video_stream_input_state.cc create mode 100644 third_party/libwebrtc/call/adaptation/video_stream_input_state.h create mode 100644 third_party/libwebrtc/call/adaptation/video_stream_input_state_provider.cc create mode 100644 third_party/libwebrtc/call/adaptation/video_stream_input_state_provider.h create mode 100644 third_party/libwebrtc/call/adaptation/video_stream_input_state_provider_unittest.cc (limited to 'third_party/libwebrtc/call/adaptation') diff --git a/third_party/libwebrtc/call/adaptation/BUILD.gn b/third_party/libwebrtc/call/adaptation/BUILD.gn new file mode 100644 index 0000000000..58fadc421d --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/BUILD.gn @@ -0,0 +1,136 @@ +# Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") + +rtc_library("resource_adaptation") { + sources = [ + "adaptation_constraint.cc", + "adaptation_constraint.h", + "broadcast_resource_listener.cc", + "broadcast_resource_listener.h", + "degradation_preference_provider.cc", + "degradation_preference_provider.h", + "encoder_settings.cc", + "encoder_settings.h", + "resource_adaptation_processor.cc", + "resource_adaptation_processor.h", + "resource_adaptation_processor_interface.cc", + "resource_adaptation_processor_interface.h", + "video_source_restrictions.cc", + "video_source_restrictions.h", + "video_stream_adapter.cc", + "video_stream_adapter.h", + "video_stream_input_state.cc", + "video_stream_input_state.h", + "video_stream_input_state_provider.cc", + "video_stream_input_state_provider.h", + ] + deps = [ + "../../api:field_trials_view", + "../../api:make_ref_counted", + "../../api:rtp_parameters", + "../../api:scoped_refptr", + "../../api:sequence_checker", + "../../api/adaptation:resource_adaptation_api", + "../../api/task_queue:task_queue", + "../../api/video:video_adaptation", + "../../api/video:video_frame", + "../../api/video:video_stream_encoder", + "../../api/video_codecs:video_codecs_api", + "../../modules/video_coding:video_coding_utility", + "../../rtc_base:checks", + "../../rtc_base:logging", + "../../rtc_base:macromagic", + "../../rtc_base:refcount", + "../../rtc_base:rtc_task_queue", + "../../rtc_base:safe_conversions", + "../../rtc_base:stringutils", + "../../rtc_base/experiments:balanced_degradation_settings", + "../../rtc_base/synchronization:mutex", + "../../rtc_base/system:no_unique_address", + "../../video:video_stream_encoder_interface", + "../../video/config:encoder_config", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + "//third_party/abseil-cpp/absl/types:variant", + ] +} + +if (rtc_include_tests) { + rtc_library("resource_adaptation_tests") { + testonly = true + + sources = [ + "broadcast_resource_listener_unittest.cc", + "resource_adaptation_processor_unittest.cc", + "resource_unittest.cc", + "video_source_restrictions_unittest.cc", + "video_stream_adapter_unittest.cc", + "video_stream_input_state_provider_unittest.cc", + ] + deps = [ + ":resource_adaptation", + ":resource_adaptation_test_utilities", + "../../api:scoped_refptr", + "../../api/adaptation:resource_adaptation_api", + "../../api/task_queue:default_task_queue_factory", + "../../api/task_queue:task_queue", + "../../api/video:video_adaptation", + "../../api/video_codecs:video_codecs_api", + "../../rtc_base:checks", + "../../rtc_base:gunit_helpers", + "../../rtc_base:rtc_event", + "../../rtc_base:rtc_task_queue", + "../../rtc_base:stringutils", + "../../rtc_base:task_queue_for_test", + "../../rtc_base/synchronization:mutex", + "../../test:field_trial", + "../../test:rtc_expect_death", + "../../test:scoped_key_value_config", + "../../test:test_support", + "../../video/config:encoder_config", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_source_set("resource_adaptation_test_utilities") { + testonly = true + + sources = [ + "test/fake_adaptation_constraint.cc", + "test/fake_adaptation_constraint.h", + "test/fake_frame_rate_provider.cc", + "test/fake_frame_rate_provider.h", + "test/fake_resource.cc", + "test/fake_resource.h", + "test/fake_video_stream_input_state_provider.cc", + "test/fake_video_stream_input_state_provider.h", + "test/mock_resource_listener.h", + ] + deps = [ + ":resource_adaptation", + "../../api:make_ref_counted", + "../../api:scoped_refptr", + "../../api:sequence_checker", + "../../api/adaptation:resource_adaptation_api", + "../../api/task_queue:task_queue", + "../../api/video:video_stream_encoder", + "../../test:test_support", + "../../video:video_stream_encoder_interface", + "../../video/config:encoder_config", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + } +} diff --git a/third_party/libwebrtc/call/adaptation/OWNERS b/third_party/libwebrtc/call/adaptation/OWNERS new file mode 100644 index 0000000000..bd56595d2e --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/OWNERS @@ -0,0 +1,3 @@ +eshr@webrtc.org +hbos@webrtc.org +ilnik@webrtc.org diff --git a/third_party/libwebrtc/call/adaptation/adaptation_constraint.cc b/third_party/libwebrtc/call/adaptation/adaptation_constraint.cc new file mode 100644 index 0000000000..d62bb74f87 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/adaptation_constraint.cc @@ -0,0 +1,17 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/adaptation_constraint.h" + +namespace webrtc { + +AdaptationConstraint::~AdaptationConstraint() {} + +} // namespace webrtc diff --git a/third_party/libwebrtc/call/adaptation/adaptation_constraint.h b/third_party/libwebrtc/call/adaptation/adaptation_constraint.h new file mode 100644 index 0000000000..9ad6414cd1 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/adaptation_constraint.h @@ -0,0 +1,41 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_ADAPTATION_CONSTRAINT_H_ +#define CALL_ADAPTATION_ADAPTATION_CONSTRAINT_H_ + +#include + +#include "api/adaptation/resource.h" +#include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_input_state.h" + +namespace webrtc { + +// Adaptation constraints have the ability to prevent applying a proposed +// adaptation (expressed as restrictions before/after adaptation). +class AdaptationConstraint { + public: + virtual ~AdaptationConstraint(); + + virtual std::string Name() const = 0; + + // TODO(https://crbug.com/webrtc/11172): When we have multi-stream adaptation + // support, this interface needs to indicate which stream the adaptation + // applies to. + virtual bool IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after) const = 0; +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_ADAPTATION_CONSTRAINT_H_ diff --git a/third_party/libwebrtc/call/adaptation/broadcast_resource_listener.cc b/third_party/libwebrtc/call/adaptation/broadcast_resource_listener.cc new file mode 100644 index 0000000000..505036db3d --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/broadcast_resource_listener.cc @@ -0,0 +1,122 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/broadcast_resource_listener.h" + +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "api/make_ref_counted.h" +#include "rtc_base/checks.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { + +// The AdapterResource redirects resource usage measurements from its parent to +// a single ResourceListener. +class BroadcastResourceListener::AdapterResource : public Resource { + public: + explicit AdapterResource(absl::string_view name) : name_(std::move(name)) {} + ~AdapterResource() override { RTC_DCHECK(!listener_); } + + // The parent is letting us know we have a usage neasurement. + void OnResourceUsageStateMeasured(ResourceUsageState usage_state) { + MutexLock lock(&lock_); + if (!listener_) + return; + listener_->OnResourceUsageStateMeasured(rtc::scoped_refptr(this), + usage_state); + } + + // Resource implementation. + std::string Name() const override { return name_; } + void SetResourceListener(ResourceListener* listener) override { + MutexLock lock(&lock_); + RTC_DCHECK(!listener_ || !listener); + listener_ = listener; + } + + private: + const std::string name_; + Mutex lock_; + ResourceListener* listener_ RTC_GUARDED_BY(lock_) = nullptr; +}; + +BroadcastResourceListener::BroadcastResourceListener( + rtc::scoped_refptr source_resource) + : source_resource_(source_resource), is_listening_(false) { + RTC_DCHECK(source_resource_); +} + +BroadcastResourceListener::~BroadcastResourceListener() { + RTC_DCHECK(!is_listening_); +} + +rtc::scoped_refptr BroadcastResourceListener::SourceResource() const { + return source_resource_; +} + +void BroadcastResourceListener::StartListening() { + MutexLock lock(&lock_); + RTC_DCHECK(!is_listening_); + source_resource_->SetResourceListener(this); + is_listening_ = true; +} + +void BroadcastResourceListener::StopListening() { + MutexLock lock(&lock_); + RTC_DCHECK(is_listening_); + RTC_DCHECK(adapters_.empty()); + source_resource_->SetResourceListener(nullptr); + is_listening_ = false; +} + +rtc::scoped_refptr +BroadcastResourceListener::CreateAdapterResource() { + MutexLock lock(&lock_); + RTC_DCHECK(is_listening_); + rtc::scoped_refptr adapter = + rtc::make_ref_counted(source_resource_->Name() + + "Adapter"); + adapters_.push_back(adapter); + return adapter; +} + +void BroadcastResourceListener::RemoveAdapterResource( + rtc::scoped_refptr resource) { + MutexLock lock(&lock_); + auto it = std::find(adapters_.begin(), adapters_.end(), resource); + RTC_DCHECK(it != adapters_.end()); + adapters_.erase(it); +} + +std::vector> +BroadcastResourceListener::GetAdapterResources() { + std::vector> resources; + MutexLock lock(&lock_); + for (const auto& adapter : adapters_) { + resources.push_back(adapter); + } + return resources; +} + +void BroadcastResourceListener::OnResourceUsageStateMeasured( + rtc::scoped_refptr resource, + ResourceUsageState usage_state) { + RTC_DCHECK_EQ(resource, source_resource_); + MutexLock lock(&lock_); + for (const auto& adapter : adapters_) { + adapter->OnResourceUsageStateMeasured(usage_state); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/call/adaptation/broadcast_resource_listener.h b/third_party/libwebrtc/call/adaptation/broadcast_resource_listener.h new file mode 100644 index 0000000000..2c5a5c703b --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/broadcast_resource_listener.h @@ -0,0 +1,75 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_BROADCAST_RESOURCE_LISTENER_H_ +#define CALL_ADAPTATION_BROADCAST_RESOURCE_LISTENER_H_ + +#include + +#include "api/adaptation/resource.h" +#include "api/scoped_refptr.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { + +// Responsible for forwarding 1 resource usage measurement to N listeners by +// creating N "adapter" resources. +// +// Example: +// If we have ResourceA, ResourceListenerX and ResourceListenerY we can create a +// BroadcastResourceListener that listens to ResourceA, use CreateAdapter() to +// spawn adapter resources ResourceX and ResourceY and let ResourceListenerX +// listen to ResourceX and ResourceListenerY listen to ResourceY. When ResourceA +// makes a measurement it will be echoed by both ResourceX and ResourceY. +// +// TODO(https://crbug.com/webrtc/11565): When the ResourceAdaptationProcessor is +// moved to call there will only be one ResourceAdaptationProcessor that needs +// to listen to the injected resources. When this is the case, delete this class +// and DCHECK that a Resource's listener is never overwritten. +class BroadcastResourceListener : public ResourceListener { + public: + explicit BroadcastResourceListener( + rtc::scoped_refptr source_resource); + ~BroadcastResourceListener() override; + + rtc::scoped_refptr SourceResource() const; + void StartListening(); + void StopListening(); + + // Creates a Resource that redirects any resource usage measurements that + // BroadcastResourceListener receives to its listener. + rtc::scoped_refptr CreateAdapterResource(); + + // Unregister the adapter from the BroadcastResourceListener; it will no + // longer receive resource usage measurement and will no longer be referenced. + // Use this to prevent memory leaks of old adapters. + void RemoveAdapterResource(rtc::scoped_refptr resource); + std::vector> GetAdapterResources(); + + // ResourceListener implementation. + void OnResourceUsageStateMeasured(rtc::scoped_refptr resource, + ResourceUsageState usage_state) override; + + private: + class AdapterResource; + friend class AdapterResource; + + const rtc::scoped_refptr source_resource_; + Mutex lock_; + bool is_listening_ RTC_GUARDED_BY(lock_); + // The AdapterResource unregisters itself prior to destruction, guaranteeing + // that these pointers are safe to use. + std::vector> adapters_ + RTC_GUARDED_BY(lock_); +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_BROADCAST_RESOURCE_LISTENER_H_ diff --git a/third_party/libwebrtc/call/adaptation/broadcast_resource_listener_unittest.cc b/third_party/libwebrtc/call/adaptation/broadcast_resource_listener_unittest.cc new file mode 100644 index 0000000000..9cd80500c2 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/broadcast_resource_listener_unittest.cc @@ -0,0 +1,121 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/broadcast_resource_listener.h" + +#include "call/adaptation/test/fake_resource.h" +#include "call/adaptation/test/mock_resource_listener.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +using ::testing::_; +using ::testing::StrictMock; + +TEST(BroadcastResourceListenerTest, CreateAndRemoveAdapterResource) { + rtc::scoped_refptr source_resource = + FakeResource::Create("SourceResource"); + BroadcastResourceListener broadcast_resource_listener(source_resource); + broadcast_resource_listener.StartListening(); + + EXPECT_TRUE(broadcast_resource_listener.GetAdapterResources().empty()); + rtc::scoped_refptr adapter = + broadcast_resource_listener.CreateAdapterResource(); + StrictMock listener; + adapter->SetResourceListener(&listener); + EXPECT_EQ(std::vector>{adapter}, + broadcast_resource_listener.GetAdapterResources()); + + // The removed adapter is not referenced by the broadcaster. + broadcast_resource_listener.RemoveAdapterResource(adapter); + EXPECT_TRUE(broadcast_resource_listener.GetAdapterResources().empty()); + // The removed adapter is not forwarding measurements. + EXPECT_CALL(listener, OnResourceUsageStateMeasured(_, _)).Times(0); + source_resource->SetUsageState(ResourceUsageState::kOveruse); + // Cleanup. + adapter->SetResourceListener(nullptr); + broadcast_resource_listener.StopListening(); +} + +TEST(BroadcastResourceListenerTest, AdapterNameIsBasedOnSourceResourceName) { + rtc::scoped_refptr source_resource = + FakeResource::Create("FooBarResource"); + BroadcastResourceListener broadcast_resource_listener(source_resource); + broadcast_resource_listener.StartListening(); + + rtc::scoped_refptr adapter = + broadcast_resource_listener.CreateAdapterResource(); + EXPECT_EQ("FooBarResourceAdapter", adapter->Name()); + + broadcast_resource_listener.RemoveAdapterResource(adapter); + broadcast_resource_listener.StopListening(); +} + +TEST(BroadcastResourceListenerTest, AdaptersForwardsUsageMeasurements) { + rtc::scoped_refptr source_resource = + FakeResource::Create("SourceResource"); + BroadcastResourceListener broadcast_resource_listener(source_resource); + broadcast_resource_listener.StartListening(); + + StrictMock destination_listener1; + StrictMock destination_listener2; + rtc::scoped_refptr adapter1 = + broadcast_resource_listener.CreateAdapterResource(); + adapter1->SetResourceListener(&destination_listener1); + rtc::scoped_refptr adapter2 = + broadcast_resource_listener.CreateAdapterResource(); + adapter2->SetResourceListener(&destination_listener2); + + // Expect kOveruse to be echoed. + EXPECT_CALL(destination_listener1, OnResourceUsageStateMeasured(_, _)) + .Times(1) + .WillOnce([adapter1](rtc::scoped_refptr resource, + ResourceUsageState usage_state) { + EXPECT_EQ(adapter1, resource); + EXPECT_EQ(ResourceUsageState::kOveruse, usage_state); + }); + EXPECT_CALL(destination_listener2, OnResourceUsageStateMeasured(_, _)) + .Times(1) + .WillOnce([adapter2](rtc::scoped_refptr resource, + ResourceUsageState usage_state) { + EXPECT_EQ(adapter2, resource); + EXPECT_EQ(ResourceUsageState::kOveruse, usage_state); + }); + source_resource->SetUsageState(ResourceUsageState::kOveruse); + + // Expect kUnderuse to be echoed. + EXPECT_CALL(destination_listener1, OnResourceUsageStateMeasured(_, _)) + .Times(1) + .WillOnce([adapter1](rtc::scoped_refptr resource, + ResourceUsageState usage_state) { + EXPECT_EQ(adapter1, resource); + EXPECT_EQ(ResourceUsageState::kUnderuse, usage_state); + }); + EXPECT_CALL(destination_listener2, OnResourceUsageStateMeasured(_, _)) + .Times(1) + .WillOnce([adapter2](rtc::scoped_refptr resource, + ResourceUsageState usage_state) { + EXPECT_EQ(adapter2, resource); + EXPECT_EQ(ResourceUsageState::kUnderuse, usage_state); + }); + source_resource->SetUsageState(ResourceUsageState::kUnderuse); + + // Adapters have to be unregistered before they or the broadcaster is + // destroyed, ensuring safe use of raw pointers. + adapter1->SetResourceListener(nullptr); + adapter2->SetResourceListener(nullptr); + + broadcast_resource_listener.RemoveAdapterResource(adapter1); + broadcast_resource_listener.RemoveAdapterResource(adapter2); + broadcast_resource_listener.StopListening(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/call/adaptation/degradation_preference_provider.cc b/third_party/libwebrtc/call/adaptation/degradation_preference_provider.cc new file mode 100644 index 0000000000..c87e49f366 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/degradation_preference_provider.cc @@ -0,0 +1,14 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/degradation_preference_provider.h" + +webrtc::DegradationPreferenceProvider::~DegradationPreferenceProvider() = + default; diff --git a/third_party/libwebrtc/call/adaptation/degradation_preference_provider.h b/third_party/libwebrtc/call/adaptation/degradation_preference_provider.h new file mode 100644 index 0000000000..1f75901cc5 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/degradation_preference_provider.h @@ -0,0 +1,27 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_DEGRADATION_PREFERENCE_PROVIDER_H_ +#define CALL_ADAPTATION_DEGRADATION_PREFERENCE_PROVIDER_H_ + +#include "api/rtp_parameters.h" + +namespace webrtc { + +class DegradationPreferenceProvider { + public: + virtual ~DegradationPreferenceProvider(); + + virtual DegradationPreference degradation_preference() const = 0; +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_DEGRADATION_PREFERENCE_PROVIDER_H_ diff --git a/third_party/libwebrtc/call/adaptation/encoder_settings.cc b/third_party/libwebrtc/call/adaptation/encoder_settings.cc new file mode 100644 index 0000000000..c894e833ed --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/encoder_settings.cc @@ -0,0 +1,54 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/encoder_settings.h" + +#include + +namespace webrtc { + +EncoderSettings::EncoderSettings(VideoEncoder::EncoderInfo encoder_info, + VideoEncoderConfig encoder_config, + VideoCodec video_codec) + : encoder_info_(std::move(encoder_info)), + encoder_config_(std::move(encoder_config)), + video_codec_(std::move(video_codec)) {} + +EncoderSettings::EncoderSettings(const EncoderSettings& other) + : encoder_info_(other.encoder_info_), + encoder_config_(other.encoder_config_.Copy()), + video_codec_(other.video_codec_) {} + +EncoderSettings& EncoderSettings::operator=(const EncoderSettings& other) { + encoder_info_ = other.encoder_info_; + encoder_config_ = other.encoder_config_.Copy(); + video_codec_ = other.video_codec_; + return *this; +} + +const VideoEncoder::EncoderInfo& EncoderSettings::encoder_info() const { + return encoder_info_; +} + +const VideoEncoderConfig& EncoderSettings::encoder_config() const { + return encoder_config_; +} + +const VideoCodec& EncoderSettings::video_codec() const { + return video_codec_; +} + +VideoCodecType GetVideoCodecTypeOrGeneric( + const absl::optional& settings) { + return settings.has_value() ? settings->encoder_config().codec_type + : kVideoCodecGeneric; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/call/adaptation/encoder_settings.h b/third_party/libwebrtc/call/adaptation/encoder_settings.h new file mode 100644 index 0000000000..30ce0a05bc --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/encoder_settings.h @@ -0,0 +1,48 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_ENCODER_SETTINGS_H_ +#define CALL_ADAPTATION_ENCODER_SETTINGS_H_ + +#include "absl/types/optional.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "video/config/video_encoder_config.h" + +namespace webrtc { + +// Information about an encoder available when reconfiguring the encoder. +class EncoderSettings { + public: + EncoderSettings(VideoEncoder::EncoderInfo encoder_info, + VideoEncoderConfig encoder_config, + VideoCodec video_codec); + EncoderSettings(const EncoderSettings& other); + EncoderSettings& operator=(const EncoderSettings& other); + + // Encoder capabilities, implementation info, etc. + const VideoEncoder::EncoderInfo& encoder_info() const; + // Configuration parameters, ultimately coming from the API and negotiation. + const VideoEncoderConfig& encoder_config() const; + // Lower level config, heavily based on the VideoEncoderConfig. + const VideoCodec& video_codec() const; + + private: + VideoEncoder::EncoderInfo encoder_info_; + VideoEncoderConfig encoder_config_; + VideoCodec video_codec_; +}; + +VideoCodecType GetVideoCodecTypeOrGeneric( + const absl::optional& settings); + +} // namespace webrtc + +#endif // CALL_ADAPTATION_ENCODER_SETTINGS_H_ diff --git a/third_party/libwebrtc/call/adaptation/resource_adaptation_gn/moz.build b/third_party/libwebrtc/call/adaptation/resource_adaptation_gn/moz.build new file mode 100644 index 0000000000..2e88747157 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/resource_adaptation_gn/moz.build @@ -0,0 +1,241 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/call/adaptation/adaptation_constraint.cc", + "/third_party/libwebrtc/call/adaptation/broadcast_resource_listener.cc", + "/third_party/libwebrtc/call/adaptation/degradation_preference_provider.cc", + "/third_party/libwebrtc/call/adaptation/encoder_settings.cc", + "/third_party/libwebrtc/call/adaptation/resource_adaptation_processor.cc", + "/third_party/libwebrtc/call/adaptation/resource_adaptation_processor_interface.cc", + "/third_party/libwebrtc/call/adaptation/video_source_restrictions.cc", + "/third_party/libwebrtc/call/adaptation/video_stream_adapter.cc", + "/third_party/libwebrtc/call/adaptation/video_stream_input_state.cc", + "/third_party/libwebrtc/call/adaptation/video_stream_input_state_provider.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("resource_adaptation_gn") diff --git a/third_party/libwebrtc/call/adaptation/resource_adaptation_processor.cc b/third_party/libwebrtc/call/adaptation/resource_adaptation_processor.cc new file mode 100644 index 0000000000..f4d1bf3538 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/resource_adaptation_processor.cc @@ -0,0 +1,378 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/resource_adaptation_processor.h" + +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/strings/string_view.h" +#include "api/sequence_checker.h" +#include "api/video/video_adaptation_counters.h" +#include "call/adaptation/video_stream_adapter.h" +#include "rtc_base/logging.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { + +ResourceAdaptationProcessor::ResourceListenerDelegate::ResourceListenerDelegate( + ResourceAdaptationProcessor* processor) + : task_queue_(TaskQueueBase::Current()), processor_(processor) { + RTC_DCHECK(task_queue_); +} + +void ResourceAdaptationProcessor::ResourceListenerDelegate:: + OnProcessorDestroyed() { + RTC_DCHECK_RUN_ON(task_queue_); + processor_ = nullptr; +} + +void ResourceAdaptationProcessor::ResourceListenerDelegate:: + OnResourceUsageStateMeasured(rtc::scoped_refptr resource, + ResourceUsageState usage_state) { + if (!task_queue_->IsCurrent()) { + task_queue_->PostTask( + [this_ref = rtc::scoped_refptr(this), + resource, usage_state] { + this_ref->OnResourceUsageStateMeasured(resource, usage_state); + }); + return; + } + RTC_DCHECK_RUN_ON(task_queue_); + if (processor_) { + processor_->OnResourceUsageStateMeasured(resource, usage_state); + } +} + +ResourceAdaptationProcessor::MitigationResultAndLogMessage:: + MitigationResultAndLogMessage() + : result(MitigationResult::kAdaptationApplied), message() {} + +ResourceAdaptationProcessor::MitigationResultAndLogMessage:: + MitigationResultAndLogMessage(MitigationResult result, + absl::string_view message) + : result(result), message(message) {} + +ResourceAdaptationProcessor::ResourceAdaptationProcessor( + VideoStreamAdapter* stream_adapter) + : task_queue_(TaskQueueBase::Current()), + resource_listener_delegate_( + rtc::make_ref_counted(this)), + resources_(), + stream_adapter_(stream_adapter), + last_reported_source_restrictions_(), + previous_mitigation_results_() { + RTC_DCHECK(task_queue_); + stream_adapter_->AddRestrictionsListener(this); +} + +ResourceAdaptationProcessor::~ResourceAdaptationProcessor() { + RTC_DCHECK_RUN_ON(task_queue_); + RTC_DCHECK(resources_.empty()) + << "There are resource(s) attached to a ResourceAdaptationProcessor " + << "being destroyed."; + stream_adapter_->RemoveRestrictionsListener(this); + resource_listener_delegate_->OnProcessorDestroyed(); +} + +void ResourceAdaptationProcessor::AddResourceLimitationsListener( + ResourceLimitationsListener* limitations_listener) { + RTC_DCHECK_RUN_ON(task_queue_); + RTC_DCHECK(std::find(resource_limitations_listeners_.begin(), + resource_limitations_listeners_.end(), + limitations_listener) == + resource_limitations_listeners_.end()); + resource_limitations_listeners_.push_back(limitations_listener); +} + +void ResourceAdaptationProcessor::RemoveResourceLimitationsListener( + ResourceLimitationsListener* limitations_listener) { + RTC_DCHECK_RUN_ON(task_queue_); + auto it = + std::find(resource_limitations_listeners_.begin(), + resource_limitations_listeners_.end(), limitations_listener); + RTC_DCHECK(it != resource_limitations_listeners_.end()); + resource_limitations_listeners_.erase(it); +} + +void ResourceAdaptationProcessor::AddResource( + rtc::scoped_refptr resource) { + RTC_DCHECK(resource); + { + MutexLock crit(&resources_lock_); + RTC_DCHECK(absl::c_find(resources_, resource) == resources_.end()) + << "Resource \"" << resource->Name() << "\" was already registered."; + resources_.push_back(resource); + } + resource->SetResourceListener(resource_listener_delegate_.get()); + RTC_LOG(LS_INFO) << "Registered resource \"" << resource->Name() << "\"."; +} + +std::vector> +ResourceAdaptationProcessor::GetResources() const { + MutexLock crit(&resources_lock_); + return resources_; +} + +void ResourceAdaptationProcessor::RemoveResource( + rtc::scoped_refptr resource) { + RTC_DCHECK(resource); + RTC_LOG(LS_INFO) << "Removing resource \"" << resource->Name() << "\"."; + resource->SetResourceListener(nullptr); + { + MutexLock crit(&resources_lock_); + auto it = absl::c_find(resources_, resource); + RTC_DCHECK(it != resources_.end()) << "Resource \"" << resource->Name() + << "\" was not a registered resource."; + resources_.erase(it); + } + RemoveLimitationsImposedByResource(std::move(resource)); +} + +void ResourceAdaptationProcessor::RemoveLimitationsImposedByResource( + rtc::scoped_refptr resource) { + if (!task_queue_->IsCurrent()) { + task_queue_->PostTask( + [this, resource]() { RemoveLimitationsImposedByResource(resource); }); + return; + } + RTC_DCHECK_RUN_ON(task_queue_); + auto resource_adaptation_limits = + adaptation_limits_by_resources_.find(resource); + if (resource_adaptation_limits != adaptation_limits_by_resources_.end()) { + VideoStreamAdapter::RestrictionsWithCounters adaptation_limits = + resource_adaptation_limits->second; + adaptation_limits_by_resources_.erase(resource_adaptation_limits); + if (adaptation_limits_by_resources_.empty()) { + // Only the resource being removed was adapted so clear restrictions. + stream_adapter_->ClearRestrictions(); + return; + } + + VideoStreamAdapter::RestrictionsWithCounters most_limited = + FindMostLimitedResources().second; + + if (adaptation_limits.counters.Total() <= most_limited.counters.Total()) { + // The removed limitations were less limited than the most limited + // resource. Don't change the current restrictions. + return; + } + + // Apply the new most limited resource as the next restrictions. + Adaptation adapt_to = stream_adapter_->GetAdaptationTo( + most_limited.counters, most_limited.restrictions); + RTC_DCHECK_EQ(adapt_to.status(), Adaptation::Status::kValid); + stream_adapter_->ApplyAdaptation(adapt_to, nullptr); + + RTC_LOG(LS_INFO) + << "Most limited resource removed. Restoring restrictions to " + "next most limited restrictions: " + << most_limited.restrictions.ToString() << " with counters " + << most_limited.counters.ToString(); + } +} + +void ResourceAdaptationProcessor::OnResourceUsageStateMeasured( + rtc::scoped_refptr resource, + ResourceUsageState usage_state) { + RTC_DCHECK_RUN_ON(task_queue_); + RTC_DCHECK(resource); + // `resource` could have been removed after signalling. + { + MutexLock crit(&resources_lock_); + if (absl::c_find(resources_, resource) == resources_.end()) { + RTC_LOG(LS_INFO) << "Ignoring signal from removed resource \"" + << resource->Name() << "\"."; + return; + } + } + MitigationResultAndLogMessage result_and_message; + switch (usage_state) { + case ResourceUsageState::kOveruse: + result_and_message = OnResourceOveruse(resource); + break; + case ResourceUsageState::kUnderuse: + result_and_message = OnResourceUnderuse(resource); + break; + } + // Maybe log the result of the operation. + auto it = previous_mitigation_results_.find(resource.get()); + if (it != previous_mitigation_results_.end() && + it->second == result_and_message.result) { + // This resource has previously reported the same result and we haven't + // successfully adapted since - don't log to avoid spam. + return; + } + RTC_LOG(LS_INFO) << "Resource \"" << resource->Name() << "\" signalled " + << ResourceUsageStateToString(usage_state) << ". " + << result_and_message.message; + if (result_and_message.result == MitigationResult::kAdaptationApplied) { + previous_mitigation_results_.clear(); + } else { + previous_mitigation_results_.insert( + std::make_pair(resource.get(), result_and_message.result)); + } +} + +ResourceAdaptationProcessor::MitigationResultAndLogMessage +ResourceAdaptationProcessor::OnResourceUnderuse( + rtc::scoped_refptr reason_resource) { + RTC_DCHECK_RUN_ON(task_queue_); + // How can this stream be adapted up? + Adaptation adaptation = stream_adapter_->GetAdaptationUp(); + if (adaptation.status() != Adaptation::Status::kValid) { + rtc::StringBuilder message; + message << "Not adapting up because VideoStreamAdapter returned " + << Adaptation::StatusToString(adaptation.status()); + return MitigationResultAndLogMessage(MitigationResult::kRejectedByAdapter, + message.Release()); + } + // Check that resource is most limited. + std::vector> most_limited_resources; + VideoStreamAdapter::RestrictionsWithCounters most_limited_restrictions; + std::tie(most_limited_resources, most_limited_restrictions) = + FindMostLimitedResources(); + + // If the most restricted resource is less limited than current restrictions + // then proceed with adapting up. + if (!most_limited_resources.empty() && + most_limited_restrictions.counters.Total() >= + stream_adapter_->adaptation_counters().Total()) { + // If `reason_resource` is not one of the most limiting resources then abort + // adaptation. + if (absl::c_find(most_limited_resources, reason_resource) == + most_limited_resources.end()) { + rtc::StringBuilder message; + message << "Resource \"" << reason_resource->Name() + << "\" was not the most limited resource."; + return MitigationResultAndLogMessage( + MitigationResult::kNotMostLimitedResource, message.Release()); + } + + if (most_limited_resources.size() > 1) { + // If there are multiple most limited resources, all must signal underuse + // before the adaptation is applied. + UpdateResourceLimitations(reason_resource, adaptation.restrictions(), + adaptation.counters()); + rtc::StringBuilder message; + message << "Resource \"" << reason_resource->Name() + << "\" was not the only most limited resource."; + return MitigationResultAndLogMessage( + MitigationResult::kSharedMostLimitedResource, message.Release()); + } + } + // Apply adaptation. + stream_adapter_->ApplyAdaptation(adaptation, reason_resource); + rtc::StringBuilder message; + message << "Adapted up successfully. Unfiltered adaptations: " + << stream_adapter_->adaptation_counters().ToString(); + return MitigationResultAndLogMessage(MitigationResult::kAdaptationApplied, + message.Release()); +} + +ResourceAdaptationProcessor::MitigationResultAndLogMessage +ResourceAdaptationProcessor::OnResourceOveruse( + rtc::scoped_refptr reason_resource) { + RTC_DCHECK_RUN_ON(task_queue_); + // How can this stream be adapted up? + Adaptation adaptation = stream_adapter_->GetAdaptationDown(); + if (adaptation.status() == Adaptation::Status::kLimitReached) { + // Add resource as most limited. + VideoStreamAdapter::RestrictionsWithCounters restrictions; + std::tie(std::ignore, restrictions) = FindMostLimitedResources(); + UpdateResourceLimitations(reason_resource, restrictions.restrictions, + restrictions.counters); + } + if (adaptation.status() != Adaptation::Status::kValid) { + rtc::StringBuilder message; + message << "Not adapting down because VideoStreamAdapter returned " + << Adaptation::StatusToString(adaptation.status()); + return MitigationResultAndLogMessage(MitigationResult::kRejectedByAdapter, + message.Release()); + } + // Apply adaptation. + UpdateResourceLimitations(reason_resource, adaptation.restrictions(), + adaptation.counters()); + stream_adapter_->ApplyAdaptation(adaptation, reason_resource); + rtc::StringBuilder message; + message << "Adapted down successfully. Unfiltered adaptations: " + << stream_adapter_->adaptation_counters().ToString(); + return MitigationResultAndLogMessage(MitigationResult::kAdaptationApplied, + message.Release()); +} + +std::pair>, + VideoStreamAdapter::RestrictionsWithCounters> +ResourceAdaptationProcessor::FindMostLimitedResources() const { + std::vector> most_limited_resources; + VideoStreamAdapter::RestrictionsWithCounters most_limited_restrictions{ + VideoSourceRestrictions(), VideoAdaptationCounters()}; + + for (const auto& resource_and_adaptation_limit_ : + adaptation_limits_by_resources_) { + const auto& restrictions_with_counters = + resource_and_adaptation_limit_.second; + if (restrictions_with_counters.counters.Total() > + most_limited_restrictions.counters.Total()) { + most_limited_restrictions = restrictions_with_counters; + most_limited_resources.clear(); + most_limited_resources.push_back(resource_and_adaptation_limit_.first); + } else if (most_limited_restrictions.counters == + restrictions_with_counters.counters) { + most_limited_resources.push_back(resource_and_adaptation_limit_.first); + } + } + return std::make_pair(std::move(most_limited_resources), + most_limited_restrictions); +} + +void ResourceAdaptationProcessor::UpdateResourceLimitations( + rtc::scoped_refptr reason_resource, + const VideoSourceRestrictions& restrictions, + const VideoAdaptationCounters& counters) { + auto& adaptation_limits = adaptation_limits_by_resources_[reason_resource]; + if (adaptation_limits.restrictions == restrictions && + adaptation_limits.counters == counters) { + return; + } + adaptation_limits = {restrictions, counters}; + + std::map, VideoAdaptationCounters> limitations; + for (const auto& p : adaptation_limits_by_resources_) { + limitations.insert(std::make_pair(p.first, p.second.counters)); + } + for (auto limitations_listener : resource_limitations_listeners_) { + limitations_listener->OnResourceLimitationChanged(reason_resource, + limitations); + } +} + +void ResourceAdaptationProcessor::OnVideoSourceRestrictionsUpdated( + VideoSourceRestrictions restrictions, + const VideoAdaptationCounters& adaptation_counters, + rtc::scoped_refptr reason, + const VideoSourceRestrictions& unfiltered_restrictions) { + RTC_DCHECK_RUN_ON(task_queue_); + if (reason) { + UpdateResourceLimitations(reason, unfiltered_restrictions, + adaptation_counters); + } else if (adaptation_counters.Total() == 0) { + // Adaptations are cleared. + adaptation_limits_by_resources_.clear(); + previous_mitigation_results_.clear(); + for (auto limitations_listener : resource_limitations_listeners_) { + limitations_listener->OnResourceLimitationChanged(nullptr, {}); + } + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/call/adaptation/resource_adaptation_processor.h b/third_party/libwebrtc/call/adaptation/resource_adaptation_processor.h new file mode 100644 index 0000000000..db3b4c2506 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/resource_adaptation_processor.h @@ -0,0 +1,167 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ +#define CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ + +#include +#include +#include +#include +#include + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/adaptation/resource.h" +#include "api/rtp_parameters.h" +#include "api/scoped_refptr.h" +#include "api/task_queue/task_queue_base.h" +#include "api/video/video_adaptation_counters.h" +#include "api/video/video_frame.h" +#include "call/adaptation/resource_adaptation_processor_interface.h" +#include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_adapter.h" +#include "call/adaptation/video_stream_input_state.h" +#include "call/adaptation/video_stream_input_state_provider.h" +#include "video/video_stream_encoder_observer.h" + +namespace webrtc { + +// The Resource Adaptation Processor is responsible for reacting to resource +// usage measurements (e.g. overusing or underusing CPU). When a resource is +// overused the Processor is responsible for performing mitigations in order to +// consume less resources. +// +// Today we have one Processor per VideoStreamEncoder and the Processor is only +// capable of restricting resolution or frame rate of the encoded stream. In the +// future we should have a single Processor responsible for all encoded streams, +// and it should be capable of reconfiguring other things than just +// VideoSourceRestrictions (e.g. reduce render frame rate). +// See Resource-Adaptation hotlist: +// https://bugs.chromium.org/u/590058293/hotlists/Resource-Adaptation +// +// The ResourceAdaptationProcessor is single-threaded. It may be constructed on +// any thread but MUST subsequently be used and destroyed on a single sequence, +// i.e. the "resource adaptation task queue". Resources can be added and removed +// from any thread. +class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, + public VideoSourceRestrictionsListener, + public ResourceListener { + public: + explicit ResourceAdaptationProcessor( + VideoStreamAdapter* video_stream_adapter); + ~ResourceAdaptationProcessor() override; + + // ResourceAdaptationProcessorInterface implementation. + void AddResourceLimitationsListener( + ResourceLimitationsListener* limitations_listener) override; + void RemoveResourceLimitationsListener( + ResourceLimitationsListener* limitations_listener) override; + void AddResource(rtc::scoped_refptr resource) override; + std::vector> GetResources() const override; + void RemoveResource(rtc::scoped_refptr resource) override; + + // ResourceListener implementation. + // Triggers OnResourceUnderuse() or OnResourceOveruse(). + void OnResourceUsageStateMeasured(rtc::scoped_refptr resource, + ResourceUsageState usage_state) override; + + // VideoSourceRestrictionsListener implementation. + void OnVideoSourceRestrictionsUpdated( + VideoSourceRestrictions restrictions, + const VideoAdaptationCounters& adaptation_counters, + rtc::scoped_refptr reason, + const VideoSourceRestrictions& unfiltered_restrictions) override; + + private: + // If resource usage measurements happens off the adaptation task queue, this + // class takes care of posting the measurement for the processor to handle it + // on the adaptation task queue. + class ResourceListenerDelegate : public rtc::RefCountInterface, + public ResourceListener { + public: + explicit ResourceListenerDelegate(ResourceAdaptationProcessor* processor); + + void OnProcessorDestroyed(); + + // ResourceListener implementation. + void OnResourceUsageStateMeasured(rtc::scoped_refptr resource, + ResourceUsageState usage_state) override; + + private: + TaskQueueBase* task_queue_; + ResourceAdaptationProcessor* processor_ RTC_GUARDED_BY(task_queue_); + }; + + enum class MitigationResult { + kNotMostLimitedResource, + kSharedMostLimitedResource, + kRejectedByAdapter, + kAdaptationApplied, + }; + + struct MitigationResultAndLogMessage { + MitigationResultAndLogMessage(); + MitigationResultAndLogMessage(MitigationResult result, + absl::string_view message); + MitigationResult result; + std::string message; + }; + + // Performs the adaptation by getting the next target, applying it and + // informing listeners of the new VideoSourceRestriction and adaptation + // counters. + MitigationResultAndLogMessage OnResourceUnderuse( + rtc::scoped_refptr reason_resource); + MitigationResultAndLogMessage OnResourceOveruse( + rtc::scoped_refptr reason_resource); + + void UpdateResourceLimitations(rtc::scoped_refptr reason_resource, + const VideoSourceRestrictions& restrictions, + const VideoAdaptationCounters& counters) + RTC_RUN_ON(task_queue_); + + // Searches `adaptation_limits_by_resources_` for each resource with the + // highest total adaptation counts. Adaptation up may only occur if the + // resource performing the adaptation is the only most limited resource. This + // function returns the list of all most limited resources as well as the + // corresponding adaptation of that resource. + std::pair>, + VideoStreamAdapter::RestrictionsWithCounters> + FindMostLimitedResources() const RTC_RUN_ON(task_queue_); + + void RemoveLimitationsImposedByResource( + rtc::scoped_refptr resource); + + TaskQueueBase* task_queue_; + rtc::scoped_refptr resource_listener_delegate_; + // Input and output. + mutable Mutex resources_lock_; + std::vector> resources_ + RTC_GUARDED_BY(resources_lock_); + std::vector resource_limitations_listeners_ + RTC_GUARDED_BY(task_queue_); + // Purely used for statistics, does not ensure mapped resources stay alive. + std::map, + VideoStreamAdapter::RestrictionsWithCounters> + adaptation_limits_by_resources_ RTC_GUARDED_BY(task_queue_); + // Responsible for generating and applying possible adaptations. + VideoStreamAdapter* const stream_adapter_ RTC_GUARDED_BY(task_queue_); + VideoSourceRestrictions last_reported_source_restrictions_ + RTC_GUARDED_BY(task_queue_); + // Keeps track of previous mitigation results per resource since the last + // successful adaptation. Used to avoid RTC_LOG spam. + std::map previous_mitigation_results_ + RTC_GUARDED_BY(task_queue_); +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_H_ diff --git a/third_party/libwebrtc/call/adaptation/resource_adaptation_processor_interface.cc b/third_party/libwebrtc/call/adaptation/resource_adaptation_processor_interface.cc new file mode 100644 index 0000000000..79f099b267 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/resource_adaptation_processor_interface.cc @@ -0,0 +1,20 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/resource_adaptation_processor_interface.h" + +namespace webrtc { + +ResourceAdaptationProcessorInterface::~ResourceAdaptationProcessorInterface() = + default; + +ResourceLimitationsListener::~ResourceLimitationsListener() = default; + +} // namespace webrtc diff --git a/third_party/libwebrtc/call/adaptation/resource_adaptation_processor_interface.h b/third_party/libwebrtc/call/adaptation/resource_adaptation_processor_interface.h new file mode 100644 index 0000000000..4729488150 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/resource_adaptation_processor_interface.h @@ -0,0 +1,67 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_INTERFACE_H_ +#define CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_INTERFACE_H_ + +#include +#include + +#include "absl/types/optional.h" +#include "api/adaptation/resource.h" +#include "api/rtp_parameters.h" +#include "api/scoped_refptr.h" +#include "api/task_queue/task_queue_base.h" +#include "api/video/video_adaptation_counters.h" +#include "api/video/video_frame.h" +#include "call/adaptation/adaptation_constraint.h" +#include "call/adaptation/encoder_settings.h" +#include "call/adaptation/video_source_restrictions.h" + +namespace webrtc { + +class ResourceLimitationsListener { + public: + virtual ~ResourceLimitationsListener(); + + // The limitations on a resource were changed. This does not mean the current + // video restrictions have changed. + virtual void OnResourceLimitationChanged( + rtc::scoped_refptr resource, + const std::map, VideoAdaptationCounters>& + resource_limitations) = 0; +}; + +// The Resource Adaptation Processor is responsible for reacting to resource +// usage measurements (e.g. overusing or underusing CPU). When a resource is +// overused the Processor is responsible for performing mitigations in order to +// consume less resources. +class ResourceAdaptationProcessorInterface { + public: + virtual ~ResourceAdaptationProcessorInterface(); + + virtual void AddResourceLimitationsListener( + ResourceLimitationsListener* limitations_listener) = 0; + virtual void RemoveResourceLimitationsListener( + ResourceLimitationsListener* limitations_listener) = 0; + // Starts or stops listening to resources, effectively enabling or disabling + // processing. May be called from anywhere. + // TODO(https://crbug.com/webrtc/11172): Automatically register and unregister + // with AddResource() and RemoveResource() instead. When the processor is + // multi-stream aware, stream-specific resouces will get added and removed + // over time. + virtual void AddResource(rtc::scoped_refptr resource) = 0; + virtual std::vector> GetResources() const = 0; + virtual void RemoveResource(rtc::scoped_refptr resource) = 0; +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_INTERFACE_H_ diff --git a/third_party/libwebrtc/call/adaptation/resource_adaptation_processor_unittest.cc b/third_party/libwebrtc/call/adaptation/resource_adaptation_processor_unittest.cc new file mode 100644 index 0000000000..ccccd3fe04 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/resource_adaptation_processor_unittest.cc @@ -0,0 +1,740 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/resource_adaptation_processor.h" + +#include "api/adaptation/resource.h" +#include "api/scoped_refptr.h" +#include "api/video/video_adaptation_counters.h" +#include "call/adaptation/resource_adaptation_processor_interface.h" +#include "call/adaptation/test/fake_frame_rate_provider.h" +#include "call/adaptation/test/fake_resource.h" +#include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_input_state_provider.h" +#include "rtc_base/event.h" +#include "rtc_base/gunit.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/task_queue_for_test.h" +#include "test/gtest.h" +#include "test/scoped_key_value_config.h" + +namespace webrtc { + +namespace { + +const int kDefaultFrameRate = 30; +const int kDefaultFrameSize = 1280 * 720; +constexpr TimeDelta kDefaultTimeout = TimeDelta::Seconds(5); + +class VideoSourceRestrictionsListenerForTesting + : public VideoSourceRestrictionsListener { + public: + VideoSourceRestrictionsListenerForTesting() + : restrictions_updated_count_(0), + restrictions_(), + adaptation_counters_(), + reason_(nullptr) {} + ~VideoSourceRestrictionsListenerForTesting() override {} + + size_t restrictions_updated_count() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); + return restrictions_updated_count_; + } + VideoSourceRestrictions restrictions() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); + return restrictions_; + } + VideoAdaptationCounters adaptation_counters() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); + return adaptation_counters_; + } + rtc::scoped_refptr reason() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); + return reason_; + } + + // VideoSourceRestrictionsListener implementation. + void OnVideoSourceRestrictionsUpdated( + VideoSourceRestrictions restrictions, + const VideoAdaptationCounters& adaptation_counters, + rtc::scoped_refptr reason, + const VideoSourceRestrictions& unfiltered_restrictions) override { + RTC_DCHECK_RUN_ON(&sequence_checker_); + ++restrictions_updated_count_; + restrictions_ = restrictions; + adaptation_counters_ = adaptation_counters; + reason_ = reason; + } + + private: + SequenceChecker sequence_checker_; + size_t restrictions_updated_count_ RTC_GUARDED_BY(&sequence_checker_); + VideoSourceRestrictions restrictions_ RTC_GUARDED_BY(&sequence_checker_); + VideoAdaptationCounters adaptation_counters_ + RTC_GUARDED_BY(&sequence_checker_); + rtc::scoped_refptr reason_ RTC_GUARDED_BY(&sequence_checker_); +}; + +class ResourceAdaptationProcessorTest : public ::testing::Test { + public: + ResourceAdaptationProcessorTest() + : frame_rate_provider_(), + input_state_provider_(&frame_rate_provider_), + resource_(FakeResource::Create("FakeResource")), + other_resource_(FakeResource::Create("OtherFakeResource")), + video_stream_adapter_( + std::make_unique(&input_state_provider_, + &frame_rate_provider_, + field_trials_)), + processor_(std::make_unique( + video_stream_adapter_.get())) { + video_stream_adapter_->AddRestrictionsListener(&restrictions_listener_); + processor_->AddResource(resource_); + processor_->AddResource(other_resource_); + } + ~ResourceAdaptationProcessorTest() override { + if (processor_) { + DestroyProcessor(); + } + } + + void SetInputStates(bool has_input, int fps, int frame_size) { + input_state_provider_.OnHasInputChanged(has_input); + frame_rate_provider_.set_fps(fps); + input_state_provider_.OnFrameSizeObserved(frame_size); + } + + void RestrictSource(VideoSourceRestrictions restrictions) { + SetInputStates( + true, restrictions.max_frame_rate().value_or(kDefaultFrameRate), + restrictions.target_pixels_per_frame().has_value() + ? restrictions.target_pixels_per_frame().value() + : restrictions.max_pixels_per_frame().value_or(kDefaultFrameSize)); + } + + void DestroyProcessor() { + if (resource_) { + processor_->RemoveResource(resource_); + } + if (other_resource_) { + processor_->RemoveResource(other_resource_); + } + video_stream_adapter_->RemoveRestrictionsListener(&restrictions_listener_); + processor_.reset(); + } + + static void WaitUntilTaskQueueIdle() { + ASSERT_TRUE(rtc::Thread::Current()->ProcessMessages(0)); + } + + protected: + rtc::AutoThread main_thread_; + webrtc::test::ScopedKeyValueConfig field_trials_; + FakeFrameRateProvider frame_rate_provider_; + VideoStreamInputStateProvider input_state_provider_; + rtc::scoped_refptr resource_; + rtc::scoped_refptr other_resource_; + std::unique_ptr video_stream_adapter_; + std::unique_ptr processor_; + VideoSourceRestrictionsListenerForTesting restrictions_listener_; +}; + +} // namespace + +TEST_F(ResourceAdaptationProcessorTest, DisabledByDefault) { + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + // Adaptation does not happen when disabled. + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); +} + +TEST_F(ResourceAdaptationProcessorTest, InsufficientInput) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + // Adaptation does not happen if input is insufficient. + // When frame size is missing (OnFrameSizeObserved not called yet). + input_state_provider_.OnHasInputChanged(true); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); + // When "has input" is missing. + SetInputStates(false, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); + // Note: frame rate cannot be missing, if unset it is 0. +} + +// These tests verify that restrictions are applied, but not exactly how much +// the source is restricted. This ensures that the VideoStreamAdapter is wired +// up correctly but not exactly how the VideoStreamAdapter generates +// restrictions. For that, see video_stream_adapter_unittest.cc. +TEST_F(ResourceAdaptationProcessorTest, + OveruseTriggersRestrictingResolutionInMaintainFrameRate) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); + EXPECT_TRUE( + restrictions_listener_.restrictions().max_pixels_per_frame().has_value()); +} + +TEST_F(ResourceAdaptationProcessorTest, + OveruseTriggersRestrictingFrameRateInMaintainResolution) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_RESOLUTION); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); + EXPECT_TRUE( + restrictions_listener_.restrictions().max_frame_rate().has_value()); +} + +TEST_F(ResourceAdaptationProcessorTest, + OveruseTriggersRestrictingFrameRateAndResolutionInBalanced) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::BALANCED); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + // Adapting multiple times eventually resticts both frame rate and + // resolution. Exactly many times we need to adapt depends on + // BalancedDegradationSettings, VideoStreamAdapter and default input + // states. This test requires it to be achieved within 4 adaptations. + for (size_t i = 0; i < 4; ++i) { + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(i + 1, restrictions_listener_.restrictions_updated_count()); + RestrictSource(restrictions_listener_.restrictions()); + } + EXPECT_TRUE( + restrictions_listener_.restrictions().max_pixels_per_frame().has_value()); + EXPECT_TRUE( + restrictions_listener_.restrictions().max_frame_rate().has_value()); +} + +TEST_F(ResourceAdaptationProcessorTest, AwaitingPreviousAdaptation) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); + // If we don't restrict the source then adaptation will not happen again + // due to "awaiting previous adaptation". This prevents "double-adapt". + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); +} + +TEST_F(ResourceAdaptationProcessorTest, CannotAdaptUpWhenUnrestricted) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); +} + +TEST_F(ResourceAdaptationProcessorTest, UnderuseTakesUsBackToUnrestricted) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(2u, restrictions_listener_.restrictions_updated_count()); + EXPECT_EQ(VideoSourceRestrictions(), restrictions_listener_.restrictions()); +} + +TEST_F(ResourceAdaptationProcessorTest, + ResourcesCanNotAdaptUpIfNeverAdaptedDown) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); + RestrictSource(restrictions_listener_.restrictions()); + + // Other resource signals under-use + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); +} + +TEST_F(ResourceAdaptationProcessorTest, + ResourcesCanNotAdaptUpIfNotAdaptedDownAfterReset) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); + + video_stream_adapter_->ClearRestrictions(); + EXPECT_EQ(0, restrictions_listener_.adaptation_counters().Total()); + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + + // resource_ did not overuse after we reset the restrictions, so adapt + // up should be disallowed. + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); +} + +TEST_F(ResourceAdaptationProcessorTest, OnlyMostLimitedResourceMayAdaptUp) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + + // `other_resource_` is most limited, resource_ can't adapt up. + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + + // `resource_` and `other_resource_` are now most limited, so both must + // signal underuse to adapt up. + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(0, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); +} + +TEST_F(ResourceAdaptationProcessorTest, + MultipleResourcesCanTriggerMultipleAdaptations) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(3, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + + // resource_ is not most limited so can't adapt from underuse. + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(3, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + // resource_ is still not most limited so can't adapt from underuse. + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + + // However it will be after overuse + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(3, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + + // Now other_resource_ can't adapt up as it is not most restricted. + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(3, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + + // resource_ is limited at 3 adaptations and other_resource_ 2. + // With the most limited resource signalling underuse in the following + // order we get back to unrestricted video. + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + // Both resource_ and other_resource_ are most limited. + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + // Again both are most limited. + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(0, restrictions_listener_.adaptation_counters().Total()); +} + +TEST_F(ResourceAdaptationProcessorTest, + MostLimitedResourceAdaptationWorksAfterChangingDegradataionPreference) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + // Adapt down until we can't anymore. + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + int last_total = restrictions_listener_.adaptation_counters().Total(); + + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_RESOLUTION); + // resource_ can not adapt up since we have never reduced FPS. + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(last_total, restrictions_listener_.adaptation_counters().Total()); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(last_total + 1, + restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + // other_resource_ is most limited so should be able to adapt up. + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(last_total, restrictions_listener_.adaptation_counters().Total()); +} + +TEST_F(ResourceAdaptationProcessorTest, + AdaptsDownWhenOtherResourceIsAlwaysUnderused) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + // Does not trigger adapataion because there's no restriction. + EXPECT_EQ(0, restrictions_listener_.adaptation_counters().Total()); + + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kOveruse); + // Adapts down even if other resource asked for adapting up. + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + + RestrictSource(restrictions_listener_.restrictions()); + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + // Doesn't adapt up because adaptation is due to another resource. + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); +} + +TEST_F(ResourceAdaptationProcessorTest, + TriggerOveruseNotOnAdaptationTaskQueue) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + TaskQueueForTest resource_task_queue("ResourceTaskQueue"); + resource_task_queue.PostTask( + [&]() { resource_->SetUsageState(ResourceUsageState::kOveruse); }); + + EXPECT_EQ_WAIT(1u, restrictions_listener_.restrictions_updated_count(), + kDefaultTimeout.ms()); +} + +TEST_F(ResourceAdaptationProcessorTest, + DestroyProcessorWhileResourceListenerDelegateHasTaskInFlight) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + // Wait for `resource_` to signal oversue first so we know that the delegate + // has passed it on to the processor's task queue. + rtc::Event resource_event; + TaskQueueForTest resource_task_queue("ResourceTaskQueue"); + resource_task_queue.PostTask([&]() { + resource_->SetUsageState(ResourceUsageState::kOveruse); + resource_event.Set(); + }); + + EXPECT_TRUE(resource_event.Wait(kDefaultTimeout)); + // Now destroy the processor while handling the overuse is in flight. + DestroyProcessor(); + + // Because the processor was destroyed by the time the delegate's task ran, + // the overuse signal must not have been handled. + EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); +} + +TEST_F(ResourceAdaptationProcessorTest, + ResourceOveruseIgnoredWhenSignalledDuringRemoval) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + rtc::Event overuse_event; + TaskQueueForTest resource_task_queue("ResourceTaskQueue"); + // Queues task for `resource_` overuse while `processor_` is still listening. + resource_task_queue.PostTask([&]() { + resource_->SetUsageState(ResourceUsageState::kOveruse); + overuse_event.Set(); + }); + EXPECT_TRUE(overuse_event.Wait(kDefaultTimeout)); + // Once we know the overuse task is queued, remove `resource_` so that + // `processor_` is not listening to it. + processor_->RemoveResource(resource_); + + // Runs the queued task so `processor_` gets signalled kOveruse from + // `resource_` even though `processor_` was not listening. + WaitUntilTaskQueueIdle(); + + // No restrictions should change even though `resource_` signaled `kOveruse`. + EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); + + // Delete `resource_` for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingOnlyAdaptedResourceResetsAdaptation) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + + processor_->RemoveResource(resource_); + EXPECT_EQ(0, restrictions_listener_.adaptation_counters().Total()); + + // Delete `resource_` for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingMostLimitedResourceSetsAdaptationToNextLimitedLevel) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::BALANCED); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + VideoSourceRestrictions next_limited_restrictions = + restrictions_listener_.restrictions(); + VideoAdaptationCounters next_limited_counters = + restrictions_listener_.adaptation_counters(); + + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + + // Removing most limited `resource_` should revert us back to + processor_->RemoveResource(resource_); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + EXPECT_EQ(next_limited_restrictions, restrictions_listener_.restrictions()); + EXPECT_EQ(next_limited_counters, + restrictions_listener_.adaptation_counters()); + + // Delete `resource_` for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingMostLimitedResourceSetsAdaptationIfInputStateUnchanged) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + VideoSourceRestrictions next_limited_restrictions = + restrictions_listener_.restrictions(); + VideoAdaptationCounters next_limited_counters = + restrictions_listener_.adaptation_counters(); + + // Overuse twice and underuse once. After the underuse we don't restrict the + // source. Normally this would block future underuses. + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + + // Removing most limited `resource_` should revert us back to, even though we + // did not call RestrictSource() after `resource_` was overused. Normally + // adaptation for MAINTAIN_FRAMERATE would be blocked here but for removal we + // allow this anyways. + processor_->RemoveResource(resource_); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + EXPECT_EQ(next_limited_restrictions, restrictions_listener_.restrictions()); + EXPECT_EQ(next_limited_counters, + restrictions_listener_.adaptation_counters()); + + // Delete `resource_` for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingResourceNotMostLimitedHasNoEffectOnLimitations) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::BALANCED); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + VideoSourceRestrictions current_restrictions = + restrictions_listener_.restrictions(); + VideoAdaptationCounters current_counters = + restrictions_listener_.adaptation_counters(); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + + // Removing most limited `resource_` should revert us back to + processor_->RemoveResource(other_resource_); + EXPECT_EQ(current_restrictions, restrictions_listener_.restrictions()); + EXPECT_EQ(current_counters, restrictions_listener_.adaptation_counters()); + + // Delete `other_resource_` for cleanup. + other_resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingMostLimitedResourceAfterSwitchingDegradationPreferences) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + VideoSourceRestrictions next_limited_restrictions = + restrictions_listener_.restrictions(); + VideoAdaptationCounters next_limited_counters = + restrictions_listener_.adaptation_counters(); + + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_RESOLUTION); + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + + // Revert to `other_resource_` when removing `resource_` even though the + // degradation preference was different when it was overused. + processor_->RemoveResource(resource_); + EXPECT_EQ(next_limited_counters, + restrictions_listener_.adaptation_counters()); + + // After switching back to MAINTAIN_FRAMERATE, the next most limited settings + // are restored. + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_EQ(next_limited_restrictions, restrictions_listener_.restrictions()); + + // Delete `resource_` for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingMostLimitedResourceSetsNextLimitationsInDisabled) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + VideoSourceRestrictions next_limited_restrictions = + restrictions_listener_.restrictions(); + VideoAdaptationCounters next_limited_counters = + restrictions_listener_.adaptation_counters(); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::DISABLED); + + // Revert to `other_resource_` when removing `resource_` even though the + // current degradataion preference is disabled. + processor_->RemoveResource(resource_); + + // After switching back to MAINTAIN_FRAMERATE, the next most limited settings + // are restored. + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_EQ(next_limited_restrictions, restrictions_listener_.restrictions()); + EXPECT_EQ(next_limited_counters, + restrictions_listener_.adaptation_counters()); + + // Delete `resource_` for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovedResourceSignalsIgnoredByProcessor) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + processor_->RemoveResource(resource_); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); + + // Delete `resource_` for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingResourceWhenMultipleMostLimtedHasNoEffect) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + // Adapt `resource_` up and then down so that both resource's are most + // limited at 1 adaptation. + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kUnderuse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + + // Removing `resource_` has no effect since both `resource_` and + // `other_resource_` are most limited. + processor_->RemoveResource(resource_); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + + // Delete `resource_` for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + ResourceOverusedAtLimitReachedWillShareMostLimited) { + video_stream_adapter_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + bool has_reached_min_pixels = false; + ON_CALL(frame_rate_provider_, OnMinPixelLimitReached()) + .WillByDefault(testing::Assign(&has_reached_min_pixels, true)); + + // Adapt 10 times, which should make us hit the limit. + for (int i = 0; i < 10; ++i) { + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + } + EXPECT_TRUE(has_reached_min_pixels); + auto last_update_count = restrictions_listener_.restrictions_updated_count(); + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + // Now both `resource_` and `other_resource_` are most limited. Underuse of + // `resource_` will not adapt up. + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(last_update_count, + restrictions_listener_.restrictions_updated_count()); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/call/adaptation/resource_unittest.cc b/third_party/libwebrtc/call/adaptation/resource_unittest.cc new file mode 100644 index 0000000000..a2291dfdce --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/resource_unittest.cc @@ -0,0 +1,55 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/adaptation/resource.h" + +#include + +#include "api/scoped_refptr.h" +#include "call/adaptation/test/fake_resource.h" +#include "call/adaptation/test/mock_resource_listener.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +using ::testing::_; +using ::testing::StrictMock; + +class ResourceTest : public ::testing::Test { + public: + ResourceTest() : fake_resource_(FakeResource::Create("FakeResource")) {} + + protected: + rtc::scoped_refptr fake_resource_; +}; + +TEST_F(ResourceTest, RegisteringListenerReceivesCallbacks) { + StrictMock resource_listener; + fake_resource_->SetResourceListener(&resource_listener); + EXPECT_CALL(resource_listener, OnResourceUsageStateMeasured(_, _)) + .Times(1) + .WillOnce([](rtc::scoped_refptr resource, + ResourceUsageState usage_state) { + EXPECT_EQ(ResourceUsageState::kOveruse, usage_state); + }); + fake_resource_->SetUsageState(ResourceUsageState::kOveruse); + fake_resource_->SetResourceListener(nullptr); +} + +TEST_F(ResourceTest, UnregisteringListenerStopsCallbacks) { + StrictMock resource_listener; + fake_resource_->SetResourceListener(&resource_listener); + fake_resource_->SetResourceListener(nullptr); + EXPECT_CALL(resource_listener, OnResourceUsageStateMeasured(_, _)).Times(0); + fake_resource_->SetUsageState(ResourceUsageState::kOveruse); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/call/adaptation/test/fake_adaptation_constraint.cc b/third_party/libwebrtc/call/adaptation/test/fake_adaptation_constraint.cc new file mode 100644 index 0000000000..dbb31f0d3b --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/test/fake_adaptation_constraint.cc @@ -0,0 +1,40 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/test/fake_adaptation_constraint.h" + +#include + +#include "absl/strings/string_view.h" + +namespace webrtc { + +FakeAdaptationConstraint::FakeAdaptationConstraint(absl::string_view name) + : name_(name), is_adaptation_up_allowed_(true) {} + +FakeAdaptationConstraint::~FakeAdaptationConstraint() = default; + +void FakeAdaptationConstraint::set_is_adaptation_up_allowed( + bool is_adaptation_up_allowed) { + is_adaptation_up_allowed_ = is_adaptation_up_allowed; +} + +std::string FakeAdaptationConstraint::Name() const { + return name_; +} + +bool FakeAdaptationConstraint::IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after) const { + return is_adaptation_up_allowed_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/call/adaptation/test/fake_adaptation_constraint.h b/third_party/libwebrtc/call/adaptation/test/fake_adaptation_constraint.h new file mode 100644 index 0000000000..5c684335f2 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/test/fake_adaptation_constraint.h @@ -0,0 +1,42 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_TEST_FAKE_ADAPTATION_CONSTRAINT_H_ +#define CALL_ADAPTATION_TEST_FAKE_ADAPTATION_CONSTRAINT_H_ + +#include + +#include "absl/strings/string_view.h" +#include "call/adaptation/adaptation_constraint.h" + +namespace webrtc { + +class FakeAdaptationConstraint : public AdaptationConstraint { + public: + explicit FakeAdaptationConstraint(absl::string_view name); + ~FakeAdaptationConstraint() override; + + void set_is_adaptation_up_allowed(bool is_adaptation_up_allowed); + + // AdaptationConstraint implementation. + std::string Name() const override; + bool IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after) const override; + + private: + const std::string name_; + bool is_adaptation_up_allowed_; +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_TEST_FAKE_ADAPTATION_CONSTRAINT_H_ diff --git a/third_party/libwebrtc/call/adaptation/test/fake_frame_rate_provider.cc b/third_party/libwebrtc/call/adaptation/test/fake_frame_rate_provider.cc new file mode 100644 index 0000000000..65fee6a7ba --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/test/fake_frame_rate_provider.cc @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/test/fake_frame_rate_provider.h" + +#include "test/gmock.h" + +using ::testing::Return; + +namespace webrtc { + +FakeFrameRateProvider::FakeFrameRateProvider() { + set_fps(0); +} + +void FakeFrameRateProvider::set_fps(int fps) { + EXPECT_CALL(*this, GetInputFrameRate()).WillRepeatedly(Return(fps)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/call/adaptation/test/fake_frame_rate_provider.h b/third_party/libwebrtc/call/adaptation/test/fake_frame_rate_provider.h new file mode 100644 index 0000000000..b8815f592a --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/test/fake_frame_rate_provider.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_TEST_FAKE_FRAME_RATE_PROVIDER_H_ +#define CALL_ADAPTATION_TEST_FAKE_FRAME_RATE_PROVIDER_H_ + +#include +#include + +#include "test/gmock.h" +#include "video/video_stream_encoder_observer.h" + +namespace webrtc { + +class MockVideoStreamEncoderObserver : public VideoStreamEncoderObserver { + public: + MOCK_METHOD(void, OnEncodedFrameTimeMeasured, (int, int), (override)); + MOCK_METHOD(void, OnIncomingFrame, (int, int), (override)); + MOCK_METHOD(void, + OnSendEncodedImage, + (const EncodedImage&, const CodecSpecificInfo*), + (override)); + MOCK_METHOD(void, + OnEncoderImplementationChanged, + (EncoderImplementation), + (override)); + MOCK_METHOD(void, OnFrameDropped, (DropReason), (override)); + MOCK_METHOD(void, + OnEncoderReconfigured, + (const VideoEncoderConfig&, const std::vector&), + (override)); + MOCK_METHOD(void, + OnAdaptationChanged, + (VideoAdaptationReason, + const VideoAdaptationCounters&, + const VideoAdaptationCounters&), + (override)); + MOCK_METHOD(void, ClearAdaptationStats, (), (override)); + MOCK_METHOD(void, + UpdateAdaptationSettings, + (AdaptationSettings, AdaptationSettings), + (override)); + MOCK_METHOD(void, OnMinPixelLimitReached, (), (override)); + MOCK_METHOD(void, OnInitialQualityResolutionAdaptDown, (), (override)); + MOCK_METHOD(void, OnSuspendChange, (bool), (override)); + MOCK_METHOD(void, + OnBitrateAllocationUpdated, + (const VideoCodec&, const VideoBitrateAllocation&), + (override)); + MOCK_METHOD(void, OnEncoderInternalScalerUpdate, (bool), (override)); + MOCK_METHOD(int, GetInputFrameRate, (), (const, override)); +}; + +class FakeFrameRateProvider : public MockVideoStreamEncoderObserver { + public: + FakeFrameRateProvider(); + void set_fps(int fps); +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_TEST_FAKE_FRAME_RATE_PROVIDER_H_ diff --git a/third_party/libwebrtc/call/adaptation/test/fake_resource.cc b/third_party/libwebrtc/call/adaptation/test/fake_resource.cc new file mode 100644 index 0000000000..48b4768550 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/test/fake_resource.cc @@ -0,0 +1,46 @@ +/* + * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/test/fake_resource.h" + +#include +#include + +#include "absl/strings/string_view.h" +#include "api/make_ref_counted.h" + +namespace webrtc { + +// static +rtc::scoped_refptr FakeResource::Create(absl::string_view name) { + return rtc::make_ref_counted(name); +} + +FakeResource::FakeResource(absl::string_view name) + : Resource(), name_(name), listener_(nullptr) {} + +FakeResource::~FakeResource() {} + +void FakeResource::SetUsageState(ResourceUsageState usage_state) { + if (listener_) { + listener_->OnResourceUsageStateMeasured(rtc::scoped_refptr(this), + usage_state); + } +} + +std::string FakeResource::Name() const { + return name_; +} + +void FakeResource::SetResourceListener(ResourceListener* listener) { + listener_ = listener; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/call/adaptation/test/fake_resource.h b/third_party/libwebrtc/call/adaptation/test/fake_resource.h new file mode 100644 index 0000000000..1119a9614f --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/test/fake_resource.h @@ -0,0 +1,45 @@ +/* + * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_TEST_FAKE_RESOURCE_H_ +#define CALL_ADAPTATION_TEST_FAKE_RESOURCE_H_ + +#include +#include + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/adaptation/resource.h" +#include "api/scoped_refptr.h" + +namespace webrtc { + +// Fake resource used for testing. +class FakeResource : public Resource { + public: + static rtc::scoped_refptr Create(absl::string_view name); + + explicit FakeResource(absl::string_view name); + ~FakeResource() override; + + void SetUsageState(ResourceUsageState usage_state); + + // Resource implementation. + std::string Name() const override; + void SetResourceListener(ResourceListener* listener) override; + + private: + const std::string name_; + ResourceListener* listener_; +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_TEST_FAKE_RESOURCE_H_ diff --git a/third_party/libwebrtc/call/adaptation/test/fake_video_stream_input_state_provider.cc b/third_party/libwebrtc/call/adaptation/test/fake_video_stream_input_state_provider.cc new file mode 100644 index 0000000000..ce92dfb204 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/test/fake_video_stream_input_state_provider.cc @@ -0,0 +1,35 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/test/fake_video_stream_input_state_provider.h" + +namespace webrtc { + +FakeVideoStreamInputStateProvider::FakeVideoStreamInputStateProvider() + : VideoStreamInputStateProvider(nullptr) {} + +FakeVideoStreamInputStateProvider::~FakeVideoStreamInputStateProvider() = + default; + +void FakeVideoStreamInputStateProvider::SetInputState( + int input_pixels, + int input_fps, + int min_pixels_per_frame) { + fake_input_state_.set_has_input(true); + fake_input_state_.set_frame_size_pixels(input_pixels); + fake_input_state_.set_frames_per_second(input_fps); + fake_input_state_.set_min_pixels_per_frame(min_pixels_per_frame); +} + +VideoStreamInputState FakeVideoStreamInputStateProvider::InputState() { + return fake_input_state_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/call/adaptation/test/fake_video_stream_input_state_provider.h b/third_party/libwebrtc/call/adaptation/test/fake_video_stream_input_state_provider.h new file mode 100644 index 0000000000..93f7dba7e6 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/test/fake_video_stream_input_state_provider.h @@ -0,0 +1,32 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_TEST_FAKE_VIDEO_STREAM_INPUT_STATE_PROVIDER_H_ +#define CALL_ADAPTATION_TEST_FAKE_VIDEO_STREAM_INPUT_STATE_PROVIDER_H_ + +#include "call/adaptation/video_stream_input_state_provider.h" + +namespace webrtc { + +class FakeVideoStreamInputStateProvider : public VideoStreamInputStateProvider { + public: + FakeVideoStreamInputStateProvider(); + virtual ~FakeVideoStreamInputStateProvider(); + + void SetInputState(int input_pixels, int input_fps, int min_pixels_per_frame); + VideoStreamInputState InputState() override; + + private: + VideoStreamInputState fake_input_state_; +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_TEST_FAKE_VIDEO_STREAM_INPUT_STATE_PROVIDER_H_ diff --git a/third_party/libwebrtc/call/adaptation/test/mock_resource_listener.h b/third_party/libwebrtc/call/adaptation/test/mock_resource_listener.h new file mode 100644 index 0000000000..f0f998f2e3 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/test/mock_resource_listener.h @@ -0,0 +1,31 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_TEST_MOCK_RESOURCE_LISTENER_H_ +#define CALL_ADAPTATION_TEST_MOCK_RESOURCE_LISTENER_H_ + +#include "api/adaptation/resource.h" + +#include "test/gmock.h" + +namespace webrtc { + +class MockResourceListener : public ResourceListener { + public: + MOCK_METHOD(void, + OnResourceUsageStateMeasured, + (rtc::scoped_refptr resource, + ResourceUsageState usage_state), + (override)); +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_TEST_MOCK_RESOURCE_LISTENER_H_ diff --git a/third_party/libwebrtc/call/adaptation/video_source_restrictions.cc b/third_party/libwebrtc/call/adaptation/video_source_restrictions.cc new file mode 100644 index 0000000000..719bc53278 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/video_source_restrictions.cc @@ -0,0 +1,173 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/video_source_restrictions.h" + +#include +#include + +#include "rtc_base/checks.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { + +VideoSourceRestrictions::VideoSourceRestrictions() + : max_pixels_per_frame_(absl::nullopt), + target_pixels_per_frame_(absl::nullopt), + max_frame_rate_(absl::nullopt) {} + +VideoSourceRestrictions::VideoSourceRestrictions( + absl::optional max_pixels_per_frame, + absl::optional target_pixels_per_frame, + absl::optional max_frame_rate) + : max_pixels_per_frame_(std::move(max_pixels_per_frame)), + target_pixels_per_frame_(std::move(target_pixels_per_frame)), + max_frame_rate_(std::move(max_frame_rate)) { + RTC_DCHECK(!max_pixels_per_frame_.has_value() || + max_pixels_per_frame_.value() < + static_cast(std::numeric_limits::max())); + RTC_DCHECK(!max_frame_rate_.has_value() || + max_frame_rate_.value() < std::numeric_limits::max()); + RTC_DCHECK(!max_frame_rate_.has_value() || max_frame_rate_.value() > 0.0); +} + +std::string VideoSourceRestrictions::ToString() const { + rtc::StringBuilder ss; + ss << "{"; + if (max_frame_rate_) + ss << " max_fps=" << max_frame_rate_.value(); + if (max_pixels_per_frame_) + ss << " max_pixels_per_frame=" << max_pixels_per_frame_.value(); + if (target_pixels_per_frame_) + ss << " target_pixels_per_frame=" << target_pixels_per_frame_.value(); + ss << " }"; + return ss.Release(); +} + +const absl::optional& VideoSourceRestrictions::max_pixels_per_frame() + const { + return max_pixels_per_frame_; +} + +const absl::optional& VideoSourceRestrictions::target_pixels_per_frame() + const { + return target_pixels_per_frame_; +} + +const absl::optional& VideoSourceRestrictions::max_frame_rate() const { + return max_frame_rate_; +} + +void VideoSourceRestrictions::set_max_pixels_per_frame( + absl::optional max_pixels_per_frame) { + max_pixels_per_frame_ = std::move(max_pixels_per_frame); +} + +void VideoSourceRestrictions::set_target_pixels_per_frame( + absl::optional target_pixels_per_frame) { + target_pixels_per_frame_ = std::move(target_pixels_per_frame); +} + +void VideoSourceRestrictions::set_max_frame_rate( + absl::optional max_frame_rate) { + max_frame_rate_ = std::move(max_frame_rate); +} + +void VideoSourceRestrictions::UpdateMin(const VideoSourceRestrictions& other) { + if (max_pixels_per_frame_.has_value()) { + max_pixels_per_frame_ = std::min(*max_pixels_per_frame_, + other.max_pixels_per_frame().value_or( + std::numeric_limits::max())); + } else { + max_pixels_per_frame_ = other.max_pixels_per_frame(); + } + if (target_pixels_per_frame_.has_value()) { + target_pixels_per_frame_ = std::min( + *target_pixels_per_frame_, other.target_pixels_per_frame().value_or( + std::numeric_limits::max())); + } else { + target_pixels_per_frame_ = other.target_pixels_per_frame(); + } + if (max_frame_rate_.has_value()) { + max_frame_rate_ = std::min( + *max_frame_rate_, + other.max_frame_rate().value_or(std::numeric_limits::max())); + } else { + max_frame_rate_ = other.max_frame_rate(); + } +} + +bool DidRestrictionsIncrease(VideoSourceRestrictions before, + VideoSourceRestrictions after) { + bool decreased_resolution = DidDecreaseResolution(before, after); + bool decreased_framerate = DidDecreaseFrameRate(before, after); + bool same_resolution = + before.max_pixels_per_frame() == after.max_pixels_per_frame(); + bool same_framerate = before.max_frame_rate() == after.max_frame_rate(); + + return (decreased_resolution && decreased_framerate) || + (decreased_resolution && same_framerate) || + (same_resolution && decreased_framerate); +} + +bool DidRestrictionsDecrease(VideoSourceRestrictions before, + VideoSourceRestrictions after) { + bool increased_resolution = DidIncreaseResolution(before, after); + bool increased_framerate = DidIncreaseFrameRate(before, after); + bool same_resolution = + before.max_pixels_per_frame() == after.max_pixels_per_frame(); + bool same_framerate = before.max_frame_rate() == after.max_frame_rate(); + + return (increased_resolution && increased_framerate) || + (increased_resolution && same_framerate) || + (same_resolution && increased_framerate); +} + +bool DidIncreaseResolution(VideoSourceRestrictions restrictions_before, + VideoSourceRestrictions restrictions_after) { + if (!restrictions_before.max_pixels_per_frame().has_value()) + return false; + if (!restrictions_after.max_pixels_per_frame().has_value()) + return true; + return restrictions_after.max_pixels_per_frame().value() > + restrictions_before.max_pixels_per_frame().value(); +} + +bool DidDecreaseResolution(VideoSourceRestrictions restrictions_before, + VideoSourceRestrictions restrictions_after) { + if (!restrictions_after.max_pixels_per_frame().has_value()) + return false; + if (!restrictions_before.max_pixels_per_frame().has_value()) + return true; + return restrictions_after.max_pixels_per_frame().value() < + restrictions_before.max_pixels_per_frame().value(); +} + +bool DidIncreaseFrameRate(VideoSourceRestrictions restrictions_before, + VideoSourceRestrictions restrictions_after) { + if (!restrictions_before.max_frame_rate().has_value()) + return false; + if (!restrictions_after.max_frame_rate().has_value()) + return true; + return restrictions_after.max_frame_rate().value() > + restrictions_before.max_frame_rate().value(); +} + +bool DidDecreaseFrameRate(VideoSourceRestrictions restrictions_before, + VideoSourceRestrictions restrictions_after) { + if (!restrictions_after.max_frame_rate().has_value()) + return false; + if (!restrictions_before.max_frame_rate().has_value()) + return true; + return restrictions_after.max_frame_rate().value() < + restrictions_before.max_frame_rate().value(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/call/adaptation/video_source_restrictions.h b/third_party/libwebrtc/call/adaptation/video_source_restrictions.h new file mode 100644 index 0000000000..be8520a385 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/video_source_restrictions.h @@ -0,0 +1,89 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_VIDEO_SOURCE_RESTRICTIONS_H_ +#define CALL_ADAPTATION_VIDEO_SOURCE_RESTRICTIONS_H_ + +#include +#include + +#include "absl/types/optional.h" + +namespace webrtc { + +// Describes optional restrictions to the resolution and frame rate of a video +// source. +class VideoSourceRestrictions { + public: + // Constructs without any restrictions. + VideoSourceRestrictions(); + // All values must be positive or nullopt. + // TODO(hbos): Support expressing "disable this stream"? + VideoSourceRestrictions(absl::optional max_pixels_per_frame, + absl::optional target_pixels_per_frame, + absl::optional max_frame_rate); + + bool operator==(const VideoSourceRestrictions& rhs) const { + return max_pixels_per_frame_ == rhs.max_pixels_per_frame_ && + target_pixels_per_frame_ == rhs.target_pixels_per_frame_ && + max_frame_rate_ == rhs.max_frame_rate_; + } + bool operator!=(const VideoSourceRestrictions& rhs) const { + return !(*this == rhs); + } + + std::string ToString() const; + + // The source must produce a resolution less than or equal to + // max_pixels_per_frame(). + const absl::optional& max_pixels_per_frame() const; + // The source should produce a resolution as close to the + // target_pixels_per_frame() as possible, provided this does not exceed + // max_pixels_per_frame(). + // The actual pixel count selected depends on the capabilities of the source. + // TODO(hbos): Clarify how "target" is used. One possible implementation: open + // the camera in the smallest resolution that is greater than or equal to the + // target and scale it down to the target if it is greater. Is this an + // accurate description of what this does today, or do we do something else? + const absl::optional& target_pixels_per_frame() const; + const absl::optional& max_frame_rate() const; + + void set_max_pixels_per_frame(absl::optional max_pixels_per_frame); + void set_target_pixels_per_frame( + absl::optional target_pixels_per_frame); + void set_max_frame_rate(absl::optional max_frame_rate); + + // Update `this` with min(`this`, `other`). + void UpdateMin(const VideoSourceRestrictions& other); + + private: + // These map to rtc::VideoSinkWants's `max_pixel_count` and + // `target_pixel_count`. + absl::optional max_pixels_per_frame_; + absl::optional target_pixels_per_frame_; + absl::optional max_frame_rate_; +}; + +bool DidRestrictionsIncrease(VideoSourceRestrictions before, + VideoSourceRestrictions after); +bool DidRestrictionsDecrease(VideoSourceRestrictions before, + VideoSourceRestrictions after); +bool DidIncreaseResolution(VideoSourceRestrictions restrictions_before, + VideoSourceRestrictions restrictions_after); +bool DidDecreaseResolution(VideoSourceRestrictions restrictions_before, + VideoSourceRestrictions restrictions_after); +bool DidIncreaseFrameRate(VideoSourceRestrictions restrictions_before, + VideoSourceRestrictions restrictions_after); +bool DidDecreaseFrameRate(VideoSourceRestrictions restrictions_before, + VideoSourceRestrictions restrictions_after); + +} // namespace webrtc + +#endif // CALL_ADAPTATION_VIDEO_SOURCE_RESTRICTIONS_H_ diff --git a/third_party/libwebrtc/call/adaptation/video_source_restrictions_unittest.cc b/third_party/libwebrtc/call/adaptation/video_source_restrictions_unittest.cc new file mode 100644 index 0000000000..8c1ae4c896 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/video_source_restrictions_unittest.cc @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/video_source_restrictions.h" + +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +const size_t kHdPixels = 1280 * 720; + +const VideoSourceRestrictions kUnlimited; +const VideoSourceRestrictions k15fps(absl::nullopt, absl::nullopt, 15.0); +const VideoSourceRestrictions kHd(kHdPixels, kHdPixels, absl::nullopt); +const VideoSourceRestrictions kHd15fps(kHdPixels, kHdPixels, 15.0); +const VideoSourceRestrictions kVga7fps(kHdPixels / 2, kHdPixels / 2, 7.0); + +VideoSourceRestrictions RestrictionsFromMaxPixelsPerFrame( + size_t max_pixels_per_frame) { + return VideoSourceRestrictions(max_pixels_per_frame, absl::nullopt, + absl::nullopt); +} + +VideoSourceRestrictions RestrictionsFromMaxFrameRate(double max_frame_rate) { + return VideoSourceRestrictions(absl::nullopt, absl::nullopt, max_frame_rate); +} + +} // namespace + +TEST(VideoSourceRestrictionsTest, DidIncreaseResolution) { + // smaller restrictions -> larger restrictions + EXPECT_TRUE(DidIncreaseResolution(RestrictionsFromMaxPixelsPerFrame(10), + RestrictionsFromMaxPixelsPerFrame(11))); + // unrestricted -> restricted + EXPECT_FALSE(DidIncreaseResolution(VideoSourceRestrictions(), + RestrictionsFromMaxPixelsPerFrame(10))); + // restricted -> unrestricted + EXPECT_TRUE(DidIncreaseResolution(RestrictionsFromMaxPixelsPerFrame(10), + VideoSourceRestrictions())); + // restricted -> equally restricted + EXPECT_FALSE(DidIncreaseResolution(RestrictionsFromMaxPixelsPerFrame(10), + RestrictionsFromMaxPixelsPerFrame(10))); + // unrestricted -> unrestricted + EXPECT_FALSE(DidIncreaseResolution(VideoSourceRestrictions(), + VideoSourceRestrictions())); + // larger restrictions -> smaller restrictions + EXPECT_FALSE(DidIncreaseResolution(RestrictionsFromMaxPixelsPerFrame(10), + RestrictionsFromMaxPixelsPerFrame(9))); +} + +TEST(VideoSourceRestrictionsTest, DidDecreaseFrameRate) { + // samller restrictions -> larger restrictions + EXPECT_FALSE(DidDecreaseFrameRate(RestrictionsFromMaxFrameRate(10), + RestrictionsFromMaxFrameRate(11))); + // unrestricted -> restricted + EXPECT_TRUE(DidDecreaseFrameRate(VideoSourceRestrictions(), + RestrictionsFromMaxFrameRate(10))); + // restricted -> unrestricted + EXPECT_FALSE(DidDecreaseFrameRate(RestrictionsFromMaxFrameRate(10), + VideoSourceRestrictions())); + // restricted -> equally restricted + EXPECT_FALSE(DidDecreaseFrameRate(RestrictionsFromMaxFrameRate(10), + RestrictionsFromMaxFrameRate(10))); + // unrestricted -> unrestricted + EXPECT_FALSE(DidDecreaseFrameRate(VideoSourceRestrictions(), + VideoSourceRestrictions())); + // larger restrictions -> samller restrictions + EXPECT_TRUE(DidDecreaseFrameRate(RestrictionsFromMaxFrameRate(10), + RestrictionsFromMaxFrameRate(9))); +} + +TEST(VideoSourceRestrictionsTest, DidRestrictionsChangeFalseForSame) { + EXPECT_FALSE(DidRestrictionsDecrease(kUnlimited, kUnlimited)); + EXPECT_FALSE(DidRestrictionsIncrease(kUnlimited, kUnlimited)); + + // Both resolution and fps restricted. + EXPECT_FALSE(DidRestrictionsDecrease(kHd15fps, kHd15fps)); + EXPECT_FALSE(DidRestrictionsIncrease(kHd15fps, kHd15fps)); +} + +TEST(VideoSourceRestrictions, + DidRestrictionsIncreaseTrueWhenPixelsOrFrameRateDecreased) { + // Unlimited > Limited resolution. + EXPECT_TRUE(DidRestrictionsIncrease(kUnlimited, kHd)); + // Unlimited > limited fps. + EXPECT_TRUE(DidRestrictionsIncrease(kUnlimited, k15fps)); + // Unlimited > limited resolution + limited fps. + EXPECT_TRUE(DidRestrictionsIncrease(kUnlimited, kHd15fps)); + // Limited resolution > limited resolution + limited fps. + EXPECT_TRUE(DidRestrictionsIncrease(kHd, kHd15fps)); + // Limited fps > limited resolution + limited fps. + EXPECT_TRUE(DidRestrictionsIncrease(k15fps, kHd15fps)); + // Limited resolution + fps > More limited resolution + more limited fps + EXPECT_TRUE(DidRestrictionsIncrease(kHd15fps, kVga7fps)); +} + +TEST(VideoSourceRestrictions, + DidRestrictionsDecreaseTrueWhenPixelsOrFrameRateIncreased) { + // Limited resolution < Unlimited. + EXPECT_TRUE(DidRestrictionsDecrease(kHd, kUnlimited)); + // Limited fps < Unlimited. + EXPECT_TRUE(DidRestrictionsDecrease(k15fps, kUnlimited)); + // Limited resolution + limited fps < unlimited. + EXPECT_TRUE(DidRestrictionsDecrease(kHd15fps, kUnlimited)); + // Limited resolution + limited fps < limited resolution. + EXPECT_TRUE(DidRestrictionsDecrease(kHd15fps, kHd)); + // Limited resolution + limited fps < limited fps. + EXPECT_TRUE(DidRestrictionsDecrease(kHd15fps, k15fps)); + // More limited resolution + more limited fps < limited resolution + fps + EXPECT_TRUE(DidRestrictionsDecrease(kVga7fps, kHd15fps)); +} + +TEST(VideoSourceRestrictions, + DidRestrictionsChangeFalseWhenFrameRateAndPixelsChangeDifferently) { + // One changed framerate, the other resolution; not an increase or decrease. + EXPECT_FALSE(DidRestrictionsIncrease(kHd, k15fps)); + EXPECT_FALSE(DidRestrictionsDecrease(kHd, k15fps)); +} + +TEST(VideoSourceRestrictions, UpdateMin) { + VideoSourceRestrictions one(kHdPixels / 2, kHdPixels, 7.0); + VideoSourceRestrictions two(kHdPixels, kHdPixels / 3, 15.0); + + one.UpdateMin(two); + + EXPECT_EQ(one.max_pixels_per_frame(), kHdPixels / 2); + EXPECT_EQ(one.target_pixels_per_frame(), kHdPixels / 3); + EXPECT_EQ(one.max_frame_rate(), 7.0); + + two.UpdateMin(one); + + EXPECT_EQ(two.max_pixels_per_frame(), kHdPixels / 2); + EXPECT_EQ(two.target_pixels_per_frame(), kHdPixels / 3); + EXPECT_EQ(two.max_frame_rate(), 7.0); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/call/adaptation/video_stream_adapter.cc b/third_party/libwebrtc/call/adaptation/video_stream_adapter.cc new file mode 100644 index 0000000000..f30a4d7abb --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/video_stream_adapter.cc @@ -0,0 +1,742 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/video_stream_adapter.h" + +#include +#include +#include + +#include "absl/types/optional.h" +#include "absl/types/variant.h" +#include "api/sequence_checker.h" +#include "api/video/video_adaptation_counters.h" +#include "api/video/video_adaptation_reason.h" +#include "api/video_codecs/video_encoder.h" +#include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_input_state.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +const int kMinFrameRateFps = 2; + +namespace { + +// For frame rate, the steps we take are 2/3 (down) and 3/2 (up). +int GetLowerFrameRateThan(int fps) { + RTC_DCHECK(fps != std::numeric_limits::max()); + return (fps * 2) / 3; +} +// TODO(hbos): Use absl::optional<> instead? +int GetHigherFrameRateThan(int fps) { + return fps != std::numeric_limits::max() + ? (fps * 3) / 2 + : std::numeric_limits::max(); +} + +int GetIncreasedMaxPixelsWanted(int target_pixels) { + if (target_pixels == std::numeric_limits::max()) + return std::numeric_limits::max(); + // When we decrease resolution, we go down to at most 3/5 of current pixels. + // Thus to increase resolution, we need 3/5 to get back to where we started. + // When going up, the desired max_pixels_per_frame() has to be significantly + // higher than the target because the source's native resolutions might not + // match the target. We pick 12/5 of the target. + // + // (This value was historically 4 times the old target, which is (3/5)*4 of + // the new target - or 12/5 - assuming the target is adjusted according to + // the above steps.) + RTC_DCHECK(target_pixels != std::numeric_limits::max()); + return (target_pixels * 12) / 5; +} + +bool CanDecreaseResolutionTo(int target_pixels, + int target_pixels_min, + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions) { + int max_pixels_per_frame = + rtc::dchecked_cast(restrictions.max_pixels_per_frame().value_or( + std::numeric_limits::max())); + return target_pixels < max_pixels_per_frame && + target_pixels_min >= input_state.min_pixels_per_frame(); +} + +bool CanIncreaseResolutionTo(int target_pixels, + const VideoSourceRestrictions& restrictions) { + int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels); + int max_pixels_per_frame = + rtc::dchecked_cast(restrictions.max_pixels_per_frame().value_or( + std::numeric_limits::max())); + return max_pixels_wanted > max_pixels_per_frame; +} + +bool CanDecreaseFrameRateTo(int max_frame_rate, + const VideoSourceRestrictions& restrictions) { + const int fps_wanted = std::max(kMinFrameRateFps, max_frame_rate); + return fps_wanted < + rtc::dchecked_cast(restrictions.max_frame_rate().value_or( + std::numeric_limits::max())); +} + +bool CanIncreaseFrameRateTo(int max_frame_rate, + const VideoSourceRestrictions& restrictions) { + return max_frame_rate > + rtc::dchecked_cast(restrictions.max_frame_rate().value_or( + std::numeric_limits::max())); +} + +bool MinPixelLimitReached(const VideoStreamInputState& input_state) { + if (input_state.single_active_stream_pixels().has_value()) { + return GetLowerResolutionThan( + input_state.single_active_stream_pixels().value()) < + input_state.min_pixels_per_frame(); + } + return input_state.frame_size_pixels().has_value() && + GetLowerResolutionThan(input_state.frame_size_pixels().value()) < + input_state.min_pixels_per_frame(); +} + +} // namespace + +VideoSourceRestrictionsListener::~VideoSourceRestrictionsListener() = default; + +VideoSourceRestrictions FilterRestrictionsByDegradationPreference( + VideoSourceRestrictions source_restrictions, + DegradationPreference degradation_preference) { + switch (degradation_preference) { + case DegradationPreference::BALANCED: + break; + case DegradationPreference::MAINTAIN_FRAMERATE: + source_restrictions.set_max_frame_rate(absl::nullopt); + break; + case DegradationPreference::MAINTAIN_RESOLUTION: + source_restrictions.set_max_pixels_per_frame(absl::nullopt); + source_restrictions.set_target_pixels_per_frame(absl::nullopt); + break; + case DegradationPreference::DISABLED: + source_restrictions.set_max_pixels_per_frame(absl::nullopt); + source_restrictions.set_target_pixels_per_frame(absl::nullopt); + source_restrictions.set_max_frame_rate(absl::nullopt); + } + return source_restrictions; +} + +// For resolution, the steps we take are 3/5 (down) and 5/3 (up). +// Notice the asymmetry of which restriction property is set depending on if +// we are adapting up or down: +// - VideoSourceRestrictor::DecreaseResolution() sets the max_pixels_per_frame() +// to the desired target and target_pixels_per_frame() to null. +// - VideoSourceRestrictor::IncreaseResolutionTo() sets the +// target_pixels_per_frame() to the desired target, and max_pixels_per_frame() +// is set according to VideoSourceRestrictor::GetIncreasedMaxPixelsWanted(). +int GetLowerResolutionThan(int pixel_count) { + RTC_DCHECK(pixel_count != std::numeric_limits::max()); + return (pixel_count * 3) / 5; +} + +// TODO(hbos): Use absl::optional<> instead? +int GetHigherResolutionThan(int pixel_count) { + return pixel_count != std::numeric_limits::max() + ? (pixel_count * 5) / 3 + : std::numeric_limits::max(); +} + +// static +const char* Adaptation::StatusToString(Adaptation::Status status) { + switch (status) { + case Adaptation::Status::kValid: + return "kValid"; + case Adaptation::Status::kLimitReached: + return "kLimitReached"; + case Adaptation::Status::kAwaitingPreviousAdaptation: + return "kAwaitingPreviousAdaptation"; + case Status::kInsufficientInput: + return "kInsufficientInput"; + case Status::kAdaptationDisabled: + return "kAdaptationDisabled"; + case Status::kRejectedByConstraint: + return "kRejectedByConstraint"; + } + RTC_CHECK_NOTREACHED(); +} + +Adaptation::Adaptation(int validation_id, + VideoSourceRestrictions restrictions, + VideoAdaptationCounters counters, + VideoStreamInputState input_state) + : validation_id_(validation_id), + status_(Status::kValid), + input_state_(std::move(input_state)), + restrictions_(std::move(restrictions)), + counters_(std::move(counters)) {} + +Adaptation::Adaptation(int validation_id, Status invalid_status) + : validation_id_(validation_id), status_(invalid_status) { + RTC_DCHECK_NE(status_, Status::kValid); +} + +Adaptation::Status Adaptation::status() const { + return status_; +} + +const VideoStreamInputState& Adaptation::input_state() const { + return input_state_; +} + +const VideoSourceRestrictions& Adaptation::restrictions() const { + return restrictions_; +} + +const VideoAdaptationCounters& Adaptation::counters() const { + return counters_; +} + +VideoStreamAdapter::VideoStreamAdapter( + VideoStreamInputStateProvider* input_state_provider, + VideoStreamEncoderObserver* encoder_stats_observer, + const FieldTrialsView& field_trials) + : input_state_provider_(input_state_provider), + encoder_stats_observer_(encoder_stats_observer), + balanced_settings_(field_trials), + adaptation_validation_id_(0), + degradation_preference_(DegradationPreference::DISABLED), + awaiting_frame_size_change_(absl::nullopt) { + sequence_checker_.Detach(); + RTC_DCHECK(input_state_provider_); + RTC_DCHECK(encoder_stats_observer_); +} + +VideoStreamAdapter::~VideoStreamAdapter() { + RTC_DCHECK(adaptation_constraints_.empty()) + << "There are constaint(s) attached to a VideoStreamAdapter being " + "destroyed."; +} + +VideoSourceRestrictions VideoStreamAdapter::source_restrictions() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); + return current_restrictions_.restrictions; +} + +const VideoAdaptationCounters& VideoStreamAdapter::adaptation_counters() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); + return current_restrictions_.counters; +} + +void VideoStreamAdapter::ClearRestrictions() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + // Invalidate any previously returned Adaptation. + RTC_LOG(LS_INFO) << "Resetting restrictions"; + ++adaptation_validation_id_; + current_restrictions_ = {VideoSourceRestrictions(), + VideoAdaptationCounters()}; + awaiting_frame_size_change_ = absl::nullopt; + BroadcastVideoRestrictionsUpdate(input_state_provider_->InputState(), + nullptr); +} + +void VideoStreamAdapter::AddRestrictionsListener( + VideoSourceRestrictionsListener* restrictions_listener) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(std::find(restrictions_listeners_.begin(), + restrictions_listeners_.end(), + restrictions_listener) == restrictions_listeners_.end()); + restrictions_listeners_.push_back(restrictions_listener); +} + +void VideoStreamAdapter::RemoveRestrictionsListener( + VideoSourceRestrictionsListener* restrictions_listener) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + auto it = std::find(restrictions_listeners_.begin(), + restrictions_listeners_.end(), restrictions_listener); + RTC_DCHECK(it != restrictions_listeners_.end()); + restrictions_listeners_.erase(it); +} + +void VideoStreamAdapter::AddAdaptationConstraint( + AdaptationConstraint* adaptation_constraint) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(std::find(adaptation_constraints_.begin(), + adaptation_constraints_.end(), + adaptation_constraint) == adaptation_constraints_.end()); + adaptation_constraints_.push_back(adaptation_constraint); +} + +void VideoStreamAdapter::RemoveAdaptationConstraint( + AdaptationConstraint* adaptation_constraint) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + auto it = std::find(adaptation_constraints_.begin(), + adaptation_constraints_.end(), adaptation_constraint); + RTC_DCHECK(it != adaptation_constraints_.end()); + adaptation_constraints_.erase(it); +} + +void VideoStreamAdapter::SetDegradationPreference( + DegradationPreference degradation_preference) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + if (degradation_preference_ == degradation_preference) + return; + // Invalidate any previously returned Adaptation. + ++adaptation_validation_id_; + bool balanced_switch = + degradation_preference == DegradationPreference::BALANCED || + degradation_preference_ == DegradationPreference::BALANCED; + degradation_preference_ = degradation_preference; + if (balanced_switch) { + // ClearRestrictions() calls BroadcastVideoRestrictionsUpdate(nullptr). + ClearRestrictions(); + } else { + BroadcastVideoRestrictionsUpdate(input_state_provider_->InputState(), + nullptr); + } +} + +struct VideoStreamAdapter::RestrictionsOrStateVisitor { + Adaptation operator()(const RestrictionsWithCounters& r) const { + return Adaptation(adaptation_validation_id, r.restrictions, r.counters, + input_state); + } + Adaptation operator()(const Adaptation::Status& status) const { + RTC_DCHECK_NE(status, Adaptation::Status::kValid); + return Adaptation(adaptation_validation_id, status); + } + + const int adaptation_validation_id; + const VideoStreamInputState& input_state; +}; + +Adaptation VideoStreamAdapter::RestrictionsOrStateToAdaptation( + VideoStreamAdapter::RestrictionsOrState step_or_state, + const VideoStreamInputState& input_state) const { + RTC_DCHECK(!step_or_state.valueless_by_exception()); + return absl::visit( + RestrictionsOrStateVisitor{adaptation_validation_id_, input_state}, + step_or_state); +} + +Adaptation VideoStreamAdapter::GetAdaptationUp( + const VideoStreamInputState& input_state) const { + RestrictionsOrState step = GetAdaptationUpStep(input_state); + // If an adaptation proposed, check with the constraints that it is ok. + if (absl::holds_alternative(step)) { + RestrictionsWithCounters restrictions = + absl::get(step); + for (const auto* constraint : adaptation_constraints_) { + if (!constraint->IsAdaptationUpAllowed(input_state, + current_restrictions_.restrictions, + restrictions.restrictions)) { + RTC_LOG(LS_INFO) << "Not adapting up because constraint \"" + << constraint->Name() << "\" disallowed it"; + step = Adaptation::Status::kRejectedByConstraint; + } + } + } + return RestrictionsOrStateToAdaptation(step, input_state); +} + +Adaptation VideoStreamAdapter::GetAdaptationUp() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoStreamInputState input_state = input_state_provider_->InputState(); + ++adaptation_validation_id_; + Adaptation adaptation = GetAdaptationUp(input_state); + return adaptation; +} + +VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::GetAdaptationUpStep( + const VideoStreamInputState& input_state) const { + if (!HasSufficientInputForAdaptation(input_state)) { + return Adaptation::Status::kInsufficientInput; + } + // Don't adapt if we're awaiting a previous adaptation to have an effect. + if (awaiting_frame_size_change_ && + awaiting_frame_size_change_->pixels_increased && + degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE && + input_state.frame_size_pixels().value() <= + awaiting_frame_size_change_->frame_size_pixels) { + return Adaptation::Status::kAwaitingPreviousAdaptation; + } + + // Maybe propose targets based on degradation preference. + switch (degradation_preference_) { + case DegradationPreference::BALANCED: { + // Attempt to increase target frame rate. + RestrictionsOrState increase_frame_rate = + IncreaseFramerate(input_state, current_restrictions_); + if (absl::holds_alternative( + increase_frame_rate)) { + return increase_frame_rate; + } + // else, increase resolution. + [[fallthrough]]; + } + case DegradationPreference::MAINTAIN_FRAMERATE: { + // Attempt to increase pixel count. + return IncreaseResolution(input_state, current_restrictions_); + } + case DegradationPreference::MAINTAIN_RESOLUTION: { + // Scale up framerate. + return IncreaseFramerate(input_state, current_restrictions_); + } + case DegradationPreference::DISABLED: + return Adaptation::Status::kAdaptationDisabled; + } + RTC_CHECK_NOTREACHED(); +} + +Adaptation VideoStreamAdapter::GetAdaptationDown() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoStreamInputState input_state = input_state_provider_->InputState(); + ++adaptation_validation_id_; + RestrictionsOrState restrictions_or_state = + GetAdaptationDownStep(input_state, current_restrictions_); + if (MinPixelLimitReached(input_state)) { + encoder_stats_observer_->OnMinPixelLimitReached(); + } + // Check for min_fps + if (degradation_preference_ == DegradationPreference::BALANCED && + absl::holds_alternative( + restrictions_or_state)) { + restrictions_or_state = AdaptIfFpsDiffInsufficient( + input_state, + absl::get(restrictions_or_state)); + } + return RestrictionsOrStateToAdaptation(restrictions_or_state, input_state); +} + +VideoStreamAdapter::RestrictionsOrState +VideoStreamAdapter::AdaptIfFpsDiffInsufficient( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& restrictions) const { + RTC_DCHECK_EQ(degradation_preference_, DegradationPreference::BALANCED); + int frame_size_pixels = input_state.single_active_stream_pixels().value_or( + input_state.frame_size_pixels().value()); + absl::optional min_fps_diff = + balanced_settings_.MinFpsDiff(frame_size_pixels); + if (current_restrictions_.counters.fps_adaptations < + restrictions.counters.fps_adaptations && + min_fps_diff && input_state.frames_per_second() > 0) { + int fps_diff = input_state.frames_per_second() - + restrictions.restrictions.max_frame_rate().value(); + if (fps_diff < min_fps_diff.value()) { + return GetAdaptationDownStep(input_state, restrictions); + } + } + return restrictions; +} + +VideoStreamAdapter::RestrictionsOrState +VideoStreamAdapter::GetAdaptationDownStep( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions) const { + if (!HasSufficientInputForAdaptation(input_state)) { + return Adaptation::Status::kInsufficientInput; + } + // Don't adapt if we're awaiting a previous adaptation to have an effect or + // if we switched degradation preference. + if (awaiting_frame_size_change_ && + !awaiting_frame_size_change_->pixels_increased && + degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE && + input_state.frame_size_pixels().value() >= + awaiting_frame_size_change_->frame_size_pixels) { + return Adaptation::Status::kAwaitingPreviousAdaptation; + } + // Maybe propose targets based on degradation preference. + switch (degradation_preference_) { + case DegradationPreference::BALANCED: { + // Try scale down framerate, if lower. + RestrictionsOrState decrease_frame_rate = + DecreaseFramerate(input_state, current_restrictions); + if (absl::holds_alternative( + decrease_frame_rate)) { + return decrease_frame_rate; + } + // else, decrease resolution. + [[fallthrough]]; + } + case DegradationPreference::MAINTAIN_FRAMERATE: { + return DecreaseResolution(input_state, current_restrictions); + } + case DegradationPreference::MAINTAIN_RESOLUTION: { + return DecreaseFramerate(input_state, current_restrictions); + } + case DegradationPreference::DISABLED: + return Adaptation::Status::kAdaptationDisabled; + } + RTC_CHECK_NOTREACHED(); +} + +VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::DecreaseResolution( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions) { + int target_pixels = + GetLowerResolutionThan(input_state.frame_size_pixels().value()); + // Use single active stream if set, this stream could be lower than the input. + int target_pixels_min = + GetLowerResolutionThan(input_state.single_active_stream_pixels().value_or( + input_state.frame_size_pixels().value())); + if (!CanDecreaseResolutionTo(target_pixels, target_pixels_min, input_state, + current_restrictions.restrictions)) { + return Adaptation::Status::kLimitReached; + } + RestrictionsWithCounters new_restrictions = current_restrictions; + RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: " << target_pixels; + new_restrictions.restrictions.set_max_pixels_per_frame( + target_pixels != std::numeric_limits::max() + ? absl::optional(target_pixels) + : absl::nullopt); + new_restrictions.restrictions.set_target_pixels_per_frame(absl::nullopt); + ++new_restrictions.counters.resolution_adaptations; + return new_restrictions; +} + +VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::DecreaseFramerate( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions) const { + int max_frame_rate; + if (degradation_preference_ == DegradationPreference::MAINTAIN_RESOLUTION) { + max_frame_rate = GetLowerFrameRateThan(input_state.frames_per_second()); + } else if (degradation_preference_ == DegradationPreference::BALANCED) { + int frame_size_pixels = input_state.single_active_stream_pixels().value_or( + input_state.frame_size_pixels().value()); + max_frame_rate = balanced_settings_.MinFps(input_state.video_codec_type(), + frame_size_pixels); + } else { + RTC_DCHECK_NOTREACHED(); + max_frame_rate = GetLowerFrameRateThan(input_state.frames_per_second()); + } + if (!CanDecreaseFrameRateTo(max_frame_rate, + current_restrictions.restrictions)) { + return Adaptation::Status::kLimitReached; + } + RestrictionsWithCounters new_restrictions = current_restrictions; + max_frame_rate = std::max(kMinFrameRateFps, max_frame_rate); + RTC_LOG(LS_INFO) << "Scaling down framerate: " << max_frame_rate; + new_restrictions.restrictions.set_max_frame_rate( + max_frame_rate != std::numeric_limits::max() + ? absl::optional(max_frame_rate) + : absl::nullopt); + ++new_restrictions.counters.fps_adaptations; + return new_restrictions; +} + +VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::IncreaseResolution( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions) { + int target_pixels = input_state.frame_size_pixels().value(); + if (current_restrictions.counters.resolution_adaptations == 1) { + RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting."; + target_pixels = std::numeric_limits::max(); + } + target_pixels = GetHigherResolutionThan(target_pixels); + if (!CanIncreaseResolutionTo(target_pixels, + current_restrictions.restrictions)) { + return Adaptation::Status::kLimitReached; + } + int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels); + RestrictionsWithCounters new_restrictions = current_restrictions; + RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: " + << max_pixels_wanted; + new_restrictions.restrictions.set_max_pixels_per_frame( + max_pixels_wanted != std::numeric_limits::max() + ? absl::optional(max_pixels_wanted) + : absl::nullopt); + new_restrictions.restrictions.set_target_pixels_per_frame( + max_pixels_wanted != std::numeric_limits::max() + ? absl::optional(target_pixels) + : absl::nullopt); + --new_restrictions.counters.resolution_adaptations; + RTC_DCHECK_GE(new_restrictions.counters.resolution_adaptations, 0); + return new_restrictions; +} + +VideoStreamAdapter::RestrictionsOrState VideoStreamAdapter::IncreaseFramerate( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions) const { + int max_frame_rate; + if (degradation_preference_ == DegradationPreference::MAINTAIN_RESOLUTION) { + max_frame_rate = GetHigherFrameRateThan(input_state.frames_per_second()); + } else if (degradation_preference_ == DegradationPreference::BALANCED) { + int frame_size_pixels = input_state.single_active_stream_pixels().value_or( + input_state.frame_size_pixels().value()); + max_frame_rate = balanced_settings_.MaxFps(input_state.video_codec_type(), + frame_size_pixels); + // Temporary fix for cases when there are fewer framerate adaptation steps + // up than down. Make number of down/up steps equal. + if (max_frame_rate == std::numeric_limits::max() && + current_restrictions.counters.fps_adaptations > 1) { + // Do not unrestrict framerate to allow additional adaptation up steps. + RTC_LOG(LS_INFO) << "Modifying framerate due to remaining fps count."; + max_frame_rate -= current_restrictions.counters.fps_adaptations; + } + // In BALANCED, the max_frame_rate must be checked before proceeding. This + // is because the MaxFps might be the current Fps and so the balanced + // settings may want to scale up the resolution. + if (!CanIncreaseFrameRateTo(max_frame_rate, + current_restrictions.restrictions)) { + return Adaptation::Status::kLimitReached; + } + } else { + RTC_DCHECK_NOTREACHED(); + max_frame_rate = GetHigherFrameRateThan(input_state.frames_per_second()); + } + if (current_restrictions.counters.fps_adaptations == 1) { + RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; + max_frame_rate = std::numeric_limits::max(); + } + if (!CanIncreaseFrameRateTo(max_frame_rate, + current_restrictions.restrictions)) { + return Adaptation::Status::kLimitReached; + } + RTC_LOG(LS_INFO) << "Scaling up framerate: " << max_frame_rate; + RestrictionsWithCounters new_restrictions = current_restrictions; + new_restrictions.restrictions.set_max_frame_rate( + max_frame_rate != std::numeric_limits::max() + ? absl::optional(max_frame_rate) + : absl::nullopt); + --new_restrictions.counters.fps_adaptations; + RTC_DCHECK_GE(new_restrictions.counters.fps_adaptations, 0); + return new_restrictions; +} + +Adaptation VideoStreamAdapter::GetAdaptDownResolution() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoStreamInputState input_state = input_state_provider_->InputState(); + switch (degradation_preference_) { + case DegradationPreference::DISABLED: + return RestrictionsOrStateToAdaptation( + Adaptation::Status::kAdaptationDisabled, input_state); + case DegradationPreference::MAINTAIN_RESOLUTION: + return RestrictionsOrStateToAdaptation(Adaptation::Status::kLimitReached, + input_state); + case DegradationPreference::MAINTAIN_FRAMERATE: + return GetAdaptationDown(); + case DegradationPreference::BALANCED: { + return RestrictionsOrStateToAdaptation( + GetAdaptDownResolutionStepForBalanced(input_state), input_state); + } + } + RTC_CHECK_NOTREACHED(); +} + +VideoStreamAdapter::RestrictionsOrState +VideoStreamAdapter::GetAdaptDownResolutionStepForBalanced( + const VideoStreamInputState& input_state) const { + // Adapt twice if the first adaptation did not decrease resolution. + auto first_step = GetAdaptationDownStep(input_state, current_restrictions_); + if (!absl::holds_alternative(first_step)) { + return first_step; + } + auto first_restrictions = absl::get(first_step); + if (first_restrictions.counters.resolution_adaptations > + current_restrictions_.counters.resolution_adaptations) { + return first_step; + } + // We didn't decrease resolution so force it; amend a resolution resuction + // to the existing framerate reduction in `first_restrictions`. + auto second_step = DecreaseResolution(input_state, first_restrictions); + if (absl::holds_alternative(second_step)) { + return second_step; + } + // If the second step was not successful then settle for the first one. + return first_step; +} + +void VideoStreamAdapter::ApplyAdaptation( + const Adaptation& adaptation, + rtc::scoped_refptr resource) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK_EQ(adaptation.validation_id_, adaptation_validation_id_); + if (adaptation.status() != Adaptation::Status::kValid) + return; + // Remember the input pixels and fps of this adaptation. Used to avoid + // adapting again before this adaptation has had an effect. + if (DidIncreaseResolution(current_restrictions_.restrictions, + adaptation.restrictions())) { + awaiting_frame_size_change_.emplace( + true, adaptation.input_state().frame_size_pixels().value()); + } else if (DidDecreaseResolution(current_restrictions_.restrictions, + adaptation.restrictions())) { + awaiting_frame_size_change_.emplace( + false, adaptation.input_state().frame_size_pixels().value()); + } else { + awaiting_frame_size_change_ = absl::nullopt; + } + current_restrictions_ = {adaptation.restrictions(), adaptation.counters()}; + BroadcastVideoRestrictionsUpdate(adaptation.input_state(), resource); +} + +Adaptation VideoStreamAdapter::GetAdaptationTo( + const VideoAdaptationCounters& counters, + const VideoSourceRestrictions& restrictions) { + // Adapts up/down from the current levels so counters are equal. + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoStreamInputState input_state = input_state_provider_->InputState(); + return Adaptation(adaptation_validation_id_, restrictions, counters, + input_state); +} + +void VideoStreamAdapter::BroadcastVideoRestrictionsUpdate( + const VideoStreamInputState& input_state, + const rtc::scoped_refptr& resource) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoSourceRestrictions filtered = FilterRestrictionsByDegradationPreference( + source_restrictions(), degradation_preference_); + if (last_filtered_restrictions_ == filtered) { + return; + } + for (auto* restrictions_listener : restrictions_listeners_) { + restrictions_listener->OnVideoSourceRestrictionsUpdated( + filtered, current_restrictions_.counters, resource, + source_restrictions()); + } + last_video_source_restrictions_ = current_restrictions_.restrictions; + last_filtered_restrictions_ = filtered; +} + +bool VideoStreamAdapter::HasSufficientInputForAdaptation( + const VideoStreamInputState& input_state) const { + return input_state.HasInputFrameSizeAndFramesPerSecond() && + (degradation_preference_ != + DegradationPreference::MAINTAIN_RESOLUTION || + input_state.frames_per_second() >= kMinFrameRateFps); +} + +VideoStreamAdapter::AwaitingFrameSizeChange::AwaitingFrameSizeChange( + bool pixels_increased, + int frame_size_pixels) + : pixels_increased(pixels_increased), + frame_size_pixels(frame_size_pixels) {} + +absl::optional VideoStreamAdapter::GetSingleActiveLayerPixels( + const VideoCodec& codec) { + int num_active = 0; + absl::optional pixels; + if (codec.codecType == VideoCodecType::kVideoCodecVP9) { + for (int i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) { + if (codec.spatialLayers[i].active) { + ++num_active; + pixels = codec.spatialLayers[i].width * codec.spatialLayers[i].height; + } + } + } else { + for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) { + if (codec.simulcastStream[i].active) { + ++num_active; + pixels = + codec.simulcastStream[i].width * codec.simulcastStream[i].height; + } + } + } + return (num_active > 1) ? absl::nullopt : pixels; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/call/adaptation/video_stream_adapter.h b/third_party/libwebrtc/call/adaptation/video_stream_adapter.h new file mode 100644 index 0000000000..5c174178e4 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/video_stream_adapter.h @@ -0,0 +1,271 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_ +#define CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_ + +#include +#include +#include + +#include "absl/types/optional.h" +#include "absl/types/variant.h" +#include "api/adaptation/resource.h" +#include "api/field_trials_view.h" +#include "api/rtp_parameters.h" +#include "api/video/video_adaptation_counters.h" +#include "call/adaptation/adaptation_constraint.h" +#include "call/adaptation/degradation_preference_provider.h" +#include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_input_state.h" +#include "call/adaptation/video_stream_input_state_provider.h" +#include "modules/video_coding/utility/quality_scaler.h" +#include "rtc_base/experiments/balanced_degradation_settings.h" +#include "rtc_base/system/no_unique_address.h" +#include "rtc_base/thread_annotations.h" +#include "video/video_stream_encoder_observer.h" + +namespace webrtc { + +// The listener is responsible for carrying out the reconfiguration of the video +// source such that the VideoSourceRestrictions are fulfilled. +class VideoSourceRestrictionsListener { + public: + virtual ~VideoSourceRestrictionsListener(); + + // The `restrictions` are filtered by degradation preference but not the + // `adaptation_counters`, which are currently only reported for legacy stats + // calculation purposes. + virtual void OnVideoSourceRestrictionsUpdated( + VideoSourceRestrictions restrictions, + const VideoAdaptationCounters& adaptation_counters, + rtc::scoped_refptr reason, + const VideoSourceRestrictions& unfiltered_restrictions) = 0; +}; + +class VideoStreamAdapter; + +extern const int kMinFrameRateFps; + +VideoSourceRestrictions FilterRestrictionsByDegradationPreference( + VideoSourceRestrictions source_restrictions, + DegradationPreference degradation_preference); + +int GetLowerResolutionThan(int pixel_count); +int GetHigherResolutionThan(int pixel_count); + +// Either represents the next VideoSourceRestrictions the VideoStreamAdapter +// will take, or provides a Status code indicating the reason for not adapting +// if the adaptation is not valid. +class Adaptation final { + public: + enum class Status { + // Applying this adaptation will have an effect. All other Status codes + // indicate that adaptation is not possible and why. + kValid, + // Cannot adapt. The minimum or maximum adaptation has already been reached. + // There are no more steps to take. + kLimitReached, + // Cannot adapt. The resolution or frame rate requested by a recent + // adaptation has not yet been reflected in the input resolution or frame + // rate; adaptation is refused to avoid "double-adapting". + kAwaitingPreviousAdaptation, + // Not enough input. + kInsufficientInput, + // Adaptation disabled via degradation preference. + kAdaptationDisabled, + // Adaptation up was rejected by a VideoAdaptationConstraint. + kRejectedByConstraint, + }; + + static const char* StatusToString(Status status); + + Status status() const; + const VideoStreamInputState& input_state() const; + const VideoSourceRestrictions& restrictions() const; + const VideoAdaptationCounters& counters() const; + + private: + friend class VideoStreamAdapter; + + // Constructs with a valid adaptation. Status is kValid. + Adaptation(int validation_id, + VideoSourceRestrictions restrictions, + VideoAdaptationCounters counters, + VideoStreamInputState input_state); + // Constructor when adaptation is not valid. Status MUST NOT be kValid. + Adaptation(int validation_id, Status invalid_status); + + // An Adaptation can become invalidated if the state of VideoStreamAdapter is + // modified before the Adaptation is applied. To guard against this, this ID + // has to match VideoStreamAdapter::adaptation_validation_id_ when applied. + // TODO(https://crbug.com/webrtc/11700): Remove the validation_id_. + const int validation_id_; + const Status status_; + // Input state when adaptation was made. + const VideoStreamInputState input_state_; + const VideoSourceRestrictions restrictions_; + const VideoAdaptationCounters counters_; +}; + +// Owns the VideoSourceRestriction for a single stream and is responsible for +// adapting it up or down when told to do so. This class serves the following +// purposes: +// 1. Keep track of a stream's restrictions. +// 2. Provide valid ways to adapt up or down the stream's restrictions. +// 3. Modify the stream's restrictions in one of the valid ways. +class VideoStreamAdapter { + public: + VideoStreamAdapter(VideoStreamInputStateProvider* input_state_provider, + VideoStreamEncoderObserver* encoder_stats_observer, + const FieldTrialsView& field_trials); + ~VideoStreamAdapter(); + + VideoSourceRestrictions source_restrictions() const; + const VideoAdaptationCounters& adaptation_counters() const; + void ClearRestrictions(); + + void AddRestrictionsListener( + VideoSourceRestrictionsListener* restrictions_listener); + void RemoveRestrictionsListener( + VideoSourceRestrictionsListener* restrictions_listener); + void AddAdaptationConstraint(AdaptationConstraint* adaptation_constraint); + void RemoveAdaptationConstraint(AdaptationConstraint* adaptation_constraint); + + // TODO(hbos): Setting the degradation preference should not clear + // restrictions! This is not defined in the spec and is unexpected, there is a + // tiny risk that people would discover and rely on this behavior. + void SetDegradationPreference(DegradationPreference degradation_preference); + + // Returns an adaptation that we are guaranteed to be able to apply, or a + // status code indicating the reason why we cannot adapt. + Adaptation GetAdaptationUp(); + Adaptation GetAdaptationDown(); + Adaptation GetAdaptationTo(const VideoAdaptationCounters& counters, + const VideoSourceRestrictions& restrictions); + // Tries to adapt the resolution one step. This is used for initial frame + // dropping. Does nothing if the degradation preference is not BALANCED or + // MAINTAIN_FRAMERATE. In the case of BALANCED, it will try twice to reduce + // the resolution. If it fails twice it gives up. + Adaptation GetAdaptDownResolution(); + + // Updates source_restrictions() the Adaptation. + void ApplyAdaptation(const Adaptation& adaptation, + rtc::scoped_refptr resource); + + struct RestrictionsWithCounters { + VideoSourceRestrictions restrictions; + VideoAdaptationCounters counters; + }; + + static absl::optional GetSingleActiveLayerPixels( + const VideoCodec& codec); + + private: + void BroadcastVideoRestrictionsUpdate( + const VideoStreamInputState& input_state, + const rtc::scoped_refptr& resource); + + bool HasSufficientInputForAdaptation(const VideoStreamInputState& input_state) + const RTC_RUN_ON(&sequence_checker_); + + using RestrictionsOrState = + absl::variant; + RestrictionsOrState GetAdaptationUpStep( + const VideoStreamInputState& input_state) const + RTC_RUN_ON(&sequence_checker_); + RestrictionsOrState GetAdaptationDownStep( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions) const + RTC_RUN_ON(&sequence_checker_); + RestrictionsOrState GetAdaptDownResolutionStepForBalanced( + const VideoStreamInputState& input_state) const + RTC_RUN_ON(&sequence_checker_); + RestrictionsOrState AdaptIfFpsDiffInsufficient( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& restrictions) const + RTC_RUN_ON(&sequence_checker_); + + Adaptation GetAdaptationUp(const VideoStreamInputState& input_state) const + RTC_RUN_ON(&sequence_checker_); + Adaptation GetAdaptationDown(const VideoStreamInputState& input_state) const + RTC_RUN_ON(&sequence_checker_); + + static RestrictionsOrState DecreaseResolution( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions); + static RestrictionsOrState IncreaseResolution( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions); + // Framerate methods are member functions because they need internal state + // if the degradation preference is BALANCED. + RestrictionsOrState DecreaseFramerate( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions) const + RTC_RUN_ON(&sequence_checker_); + RestrictionsOrState IncreaseFramerate( + const VideoStreamInputState& input_state, + const RestrictionsWithCounters& current_restrictions) const + RTC_RUN_ON(&sequence_checker_); + + struct RestrictionsOrStateVisitor; + Adaptation RestrictionsOrStateToAdaptation( + RestrictionsOrState step_or_state, + const VideoStreamInputState& input_state) const + RTC_RUN_ON(&sequence_checker_); + + RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_ + RTC_GUARDED_BY(&sequence_checker_); + // Gets the input state which is the basis of all adaptations. + // Thread safe. + VideoStreamInputStateProvider* input_state_provider_; + // Used to signal when min pixel limit has been reached. + VideoStreamEncoderObserver* const encoder_stats_observer_; + // Decides the next adaptation target in DegradationPreference::BALANCED. + const BalancedDegradationSettings balanced_settings_; + // To guard against applying adaptations that have become invalidated, an + // Adaptation that is applied has to have a matching validation ID. + int adaptation_validation_id_ RTC_GUARDED_BY(&sequence_checker_); + // When deciding the next target up or down, different strategies are used + // depending on the DegradationPreference. + // https://w3c.github.io/mst-content-hint/#dom-rtcdegradationpreference + DegradationPreference degradation_preference_ + RTC_GUARDED_BY(&sequence_checker_); + // Used to avoid adapting twice. Stores the resolution at the time of the last + // adaptation. + // TODO(hbos): Can we implement a more general "cooldown" mechanism of + // resources intead? If we already have adapted it seems like we should wait + // a while before adapting again, so that we are not acting on usage + // measurements that are made obsolete/unreliable by an "ongoing" adaptation. + struct AwaitingFrameSizeChange { + AwaitingFrameSizeChange(bool pixels_increased, int frame_size); + const bool pixels_increased; + const int frame_size_pixels; + }; + absl::optional awaiting_frame_size_change_ + RTC_GUARDED_BY(&sequence_checker_); + // The previous restrictions value. Starts as unrestricted. + VideoSourceRestrictions last_video_source_restrictions_ + RTC_GUARDED_BY(&sequence_checker_); + VideoSourceRestrictions last_filtered_restrictions_ + RTC_GUARDED_BY(&sequence_checker_); + + std::vector restrictions_listeners_ + RTC_GUARDED_BY(&sequence_checker_); + std::vector adaptation_constraints_ + RTC_GUARDED_BY(&sequence_checker_); + + RestrictionsWithCounters current_restrictions_ + RTC_GUARDED_BY(&sequence_checker_); +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_ diff --git a/third_party/libwebrtc/call/adaptation/video_stream_adapter_unittest.cc b/third_party/libwebrtc/call/adaptation/video_stream_adapter_unittest.cc new file mode 100644 index 0000000000..d4bc650856 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/video_stream_adapter_unittest.cc @@ -0,0 +1,951 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/video_stream_adapter.h" + +#include +#include + +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "api/video/video_adaptation_reason.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "call/adaptation/adaptation_constraint.h" +#include "call/adaptation/encoder_settings.h" +#include "call/adaptation/test/fake_frame_rate_provider.h" +#include "call/adaptation/test/fake_resource.h" +#include "call/adaptation/test/fake_video_stream_input_state_provider.h" +#include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_input_state.h" +#include "rtc_base/string_encode.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/scoped_key_value_config.h" +#include "test/testsupport/rtc_expect_death.h" +#include "video/config/video_encoder_config.h" + +namespace webrtc { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::Return; +using ::testing::SaveArg; + +namespace { + +const int kBalancedHighResolutionPixels = 1280 * 720; +const int kBalancedHighFrameRateFps = 30; + +const int kBalancedMediumResolutionPixels = 640 * 480; +const int kBalancedMediumFrameRateFps = 20; + +const int kBalancedLowResolutionPixels = 320 * 240; +const int kBalancedLowFrameRateFps = 10; + +std::string BalancedFieldTrialConfig() { + return "WebRTC-Video-BalancedDegradationSettings/pixels:" + + rtc::ToString(kBalancedLowResolutionPixels) + "|" + + rtc::ToString(kBalancedMediumResolutionPixels) + "|" + + rtc::ToString(kBalancedHighResolutionPixels) + + ",fps:" + rtc::ToString(kBalancedLowFrameRateFps) + "|" + + rtc::ToString(kBalancedMediumFrameRateFps) + "|" + + rtc::ToString(kBalancedHighFrameRateFps) + "/"; +} + +// Responsible for adjusting the inputs to VideoStreamAdapter (SetInput), such +// as pixels and frame rate, according to the most recent source restrictions. +// This helps tests that apply adaptations multiple times: if the input is not +// adjusted between adaptations, the subsequent adaptations fail with +// kAwaitingPreviousAdaptation. +class FakeVideoStream { + public: + FakeVideoStream(VideoStreamAdapter* adapter, + FakeVideoStreamInputStateProvider* provider, + int input_pixels, + int input_fps, + int min_pixels_per_frame) + : adapter_(adapter), + provider_(provider), + input_pixels_(input_pixels), + input_fps_(input_fps), + min_pixels_per_frame_(min_pixels_per_frame) { + provider_->SetInputState(input_pixels_, input_fps_, min_pixels_per_frame_); + } + + int input_pixels() const { return input_pixels_; } + int input_fps() const { return input_fps_; } + + // Performs ApplyAdaptation() followed by SetInput() with input pixels and + // frame rate adjusted according to the resulting restrictions. + void ApplyAdaptation(Adaptation adaptation) { + adapter_->ApplyAdaptation(adaptation, nullptr); + // Update input pixels and fps according to the resulting restrictions. + auto restrictions = adapter_->source_restrictions(); + if (restrictions.target_pixels_per_frame().has_value()) { + RTC_DCHECK(!restrictions.max_pixels_per_frame().has_value() || + restrictions.max_pixels_per_frame().value() >= + restrictions.target_pixels_per_frame().value()); + input_pixels_ = restrictions.target_pixels_per_frame().value(); + } else if (restrictions.max_pixels_per_frame().has_value()) { + input_pixels_ = restrictions.max_pixels_per_frame().value(); + } + if (restrictions.max_frame_rate().has_value()) { + input_fps_ = restrictions.max_frame_rate().value(); + } + provider_->SetInputState(input_pixels_, input_fps_, min_pixels_per_frame_); + } + + private: + VideoStreamAdapter* adapter_; + FakeVideoStreamInputStateProvider* provider_; + int input_pixels_; + int input_fps_; + int min_pixels_per_frame_; +}; + +class FakeVideoStreamAdapterListner : public VideoSourceRestrictionsListener { + public: + void OnVideoSourceRestrictionsUpdated( + VideoSourceRestrictions restrictions, + const VideoAdaptationCounters& adaptation_counters, + rtc::scoped_refptr reason, + const VideoSourceRestrictions& unfiltered_restrictions) override { + calls_++; + last_restrictions_ = unfiltered_restrictions; + } + + int calls() const { return calls_; } + + VideoSourceRestrictions last_restrictions() const { + return last_restrictions_; + } + + private: + int calls_ = 0; + VideoSourceRestrictions last_restrictions_; +}; + +class MockAdaptationConstraint : public AdaptationConstraint { + public: + MOCK_METHOD(bool, + IsAdaptationUpAllowed, + (const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after), + (const, override)); + + // MOCK_METHOD(std::string, Name, (), (const, override)); + std::string Name() const override { return "MockAdaptationConstraint"; } +}; + +} // namespace + +class VideoStreamAdapterTest : public ::testing::Test { + public: + VideoStreamAdapterTest() + : field_trials_(BalancedFieldTrialConfig()), + resource_(FakeResource::Create("FakeResource")), + adapter_(&input_state_provider_, + &encoder_stats_observer_, + field_trials_) {} + + protected: + webrtc::test::ScopedKeyValueConfig field_trials_; + FakeVideoStreamInputStateProvider input_state_provider_; + rtc::scoped_refptr resource_; + testing::StrictMock encoder_stats_observer_; + VideoStreamAdapter adapter_; +}; + +TEST_F(VideoStreamAdapterTest, NoRestrictionsByDefault) { + EXPECT_EQ(VideoSourceRestrictions(), adapter_.source_restrictions()); + EXPECT_EQ(0, adapter_.adaptation_counters().Total()); +} + +TEST_F(VideoStreamAdapterTest, MaintainFramerate_DecreasesPixelsToThreeFifths) { + const int kInputPixels = 1280 * 720; + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + input_state_provider_.SetInputState(kInputPixels, 30, + kDefaultMinPixelsPerFrame); + Adaptation adaptation = adapter_.GetAdaptationDown(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + adapter_.ApplyAdaptation(adaptation, nullptr); + EXPECT_EQ(static_cast((kInputPixels * 3) / 5), + adapter_.source_restrictions().max_pixels_per_frame()); + EXPECT_EQ(absl::nullopt, + adapter_.source_restrictions().target_pixels_per_frame()); + EXPECT_EQ(absl::nullopt, adapter_.source_restrictions().max_frame_rate()); + EXPECT_EQ(1, adapter_.adaptation_counters().resolution_adaptations); +} + +TEST_F(VideoStreamAdapterTest, + MaintainFramerate_DecreasesPixelsToLimitReached) { + const int kMinPixelsPerFrame = 640 * 480; + + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + input_state_provider_.SetInputState(kMinPixelsPerFrame + 1, 30, + kMinPixelsPerFrame); + EXPECT_CALL(encoder_stats_observer_, OnMinPixelLimitReached()); + // Even though we are above kMinPixelsPerFrame, because adapting down would + // have exceeded the limit, we are said to have reached the limit already. + // This differs from the frame rate adaptation logic, which would have clamped + // to the limit in the first step and reported kLimitReached in the second + // step. + Adaptation adaptation = adapter_.GetAdaptationDown(); + EXPECT_EQ(Adaptation::Status::kLimitReached, adaptation.status()); +} + +TEST_F(VideoStreamAdapterTest, MaintainFramerate_IncreasePixelsToFiveThirds) { + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // Go down twice, ensuring going back up is still a restricted resolution. + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + EXPECT_EQ(2, adapter_.adaptation_counters().resolution_adaptations); + int input_pixels = fake_stream.input_pixels(); + // Go up once. The target is 5/3 and the max is 12/5 of the target. + const int target = (input_pixels * 5) / 3; + fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp()); + EXPECT_EQ(static_cast((target * 12) / 5), + adapter_.source_restrictions().max_pixels_per_frame()); + EXPECT_EQ(static_cast(target), + adapter_.source_restrictions().target_pixels_per_frame()); + EXPECT_EQ(absl::nullopt, adapter_.source_restrictions().max_frame_rate()); + EXPECT_EQ(1, adapter_.adaptation_counters().resolution_adaptations); +} + +TEST_F(VideoStreamAdapterTest, MaintainFramerate_IncreasePixelsToUnrestricted) { + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // We are unrestricted by default and should not be able to adapt up. + EXPECT_EQ(Adaptation::Status::kLimitReached, + adapter_.GetAdaptationUp().status()); + // If we go down once and then back up we should not have any restrictions. + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + EXPECT_EQ(1, adapter_.adaptation_counters().resolution_adaptations); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp()); + EXPECT_EQ(VideoSourceRestrictions(), adapter_.source_restrictions()); + EXPECT_EQ(0, adapter_.adaptation_counters().Total()); +} + +TEST_F(VideoStreamAdapterTest, MaintainResolution_DecreasesFpsToTwoThirds) { + const int kInputFps = 30; + + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); + input_state_provider_.SetInputState(1280 * 720, kInputFps, + kDefaultMinPixelsPerFrame); + Adaptation adaptation = adapter_.GetAdaptationDown(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + adapter_.ApplyAdaptation(adaptation, nullptr); + EXPECT_EQ(absl::nullopt, + adapter_.source_restrictions().max_pixels_per_frame()); + EXPECT_EQ(absl::nullopt, + adapter_.source_restrictions().target_pixels_per_frame()); + EXPECT_EQ(static_cast((kInputFps * 2) / 3), + adapter_.source_restrictions().max_frame_rate()); + EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); +} + +TEST_F(VideoStreamAdapterTest, MaintainResolution_DecreasesFpsToLimitReached) { + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, + kMinFrameRateFps + 1, kDefaultMinPixelsPerFrame); + // If we are not yet at the limit and the next step would exceed it, the step + // is clamped such that we end up exactly on the limit. + Adaptation adaptation = adapter_.GetAdaptationDown(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + fake_stream.ApplyAdaptation(adaptation); + EXPECT_EQ(static_cast(kMinFrameRateFps), + adapter_.source_restrictions().max_frame_rate()); + EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); + // Having reached the limit, the next adaptation down is not valid. + EXPECT_EQ(Adaptation::Status::kLimitReached, + adapter_.GetAdaptationDown().status()); +} + +TEST_F(VideoStreamAdapterTest, MaintainResolution_IncreaseFpsToThreeHalves) { + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // Go down twice, ensuring going back up is still a restricted frame rate. + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + EXPECT_EQ(2, adapter_.adaptation_counters().fps_adaptations); + int input_fps = fake_stream.input_fps(); + // Go up once. The target is 3/2 of the input. + Adaptation adaptation = adapter_.GetAdaptationUp(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + fake_stream.ApplyAdaptation(adaptation); + EXPECT_EQ(absl::nullopt, + adapter_.source_restrictions().max_pixels_per_frame()); + EXPECT_EQ(absl::nullopt, + adapter_.source_restrictions().target_pixels_per_frame()); + EXPECT_EQ(static_cast((input_fps * 3) / 2), + adapter_.source_restrictions().max_frame_rate()); + EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); +} + +TEST_F(VideoStreamAdapterTest, MaintainResolution_IncreaseFpsToUnrestricted) { + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // We are unrestricted by default and should not be able to adapt up. + EXPECT_EQ(Adaptation::Status::kLimitReached, + adapter_.GetAdaptationUp().status()); + // If we go down once and then back up we should not have any restrictions. + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp()); + EXPECT_EQ(VideoSourceRestrictions(), adapter_.source_restrictions()); + EXPECT_EQ(0, adapter_.adaptation_counters().Total()); +} + +TEST_F(VideoStreamAdapterTest, Balanced_DecreaseFrameRate) { + adapter_.SetDegradationPreference(DegradationPreference::BALANCED); + input_state_provider_.SetInputState(kBalancedMediumResolutionPixels, + kBalancedHighFrameRateFps, + kDefaultMinPixelsPerFrame); + // If our frame rate is higher than the frame rate associated with our + // resolution we should try to adapt to the frame rate associated with our + // resolution: kBalancedMediumFrameRateFps. + Adaptation adaptation = adapter_.GetAdaptationDown(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + adapter_.ApplyAdaptation(adaptation, nullptr); + EXPECT_EQ(absl::nullopt, + adapter_.source_restrictions().max_pixels_per_frame()); + EXPECT_EQ(absl::nullopt, + adapter_.source_restrictions().target_pixels_per_frame()); + EXPECT_EQ(static_cast(kBalancedMediumFrameRateFps), + adapter_.source_restrictions().max_frame_rate()); + EXPECT_EQ(0, adapter_.adaptation_counters().resolution_adaptations); + EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); +} + +TEST_F(VideoStreamAdapterTest, Balanced_DecreaseResolution) { + adapter_.SetDegradationPreference(DegradationPreference::BALANCED); + FakeVideoStream fake_stream( + &adapter_, &input_state_provider_, kBalancedHighResolutionPixels, + kBalancedHighFrameRateFps, kDefaultMinPixelsPerFrame); + // If we are not below the current resolution's frame rate limit, we should + // adapt resolution according to "maintain-framerate" logic (three fifths). + // + // However, since we are unlimited at the start and input frame rate is not + // below kBalancedHighFrameRateFps, we first restrict the frame rate to + // kBalancedHighFrameRateFps even though that is our current frame rate. This + // does prevent the source from going higher, though, so it's technically not + // a NO-OP. + { + Adaptation adaptation = adapter_.GetAdaptationDown(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + fake_stream.ApplyAdaptation(adaptation); + } + EXPECT_EQ(absl::nullopt, + adapter_.source_restrictions().max_pixels_per_frame()); + EXPECT_EQ(absl::nullopt, + adapter_.source_restrictions().target_pixels_per_frame()); + EXPECT_EQ(static_cast(kBalancedHighFrameRateFps), + adapter_.source_restrictions().max_frame_rate()); + EXPECT_EQ(0, adapter_.adaptation_counters().resolution_adaptations); + EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); + // Verify "maintain-framerate" logic the second time we adapt: Frame rate + // restrictions remains the same and resolution goes down. + { + Adaptation adaptation = adapter_.GetAdaptationDown(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + fake_stream.ApplyAdaptation(adaptation); + } + constexpr size_t kReducedPixelsFirstStep = + static_cast((kBalancedHighResolutionPixels * 3) / 5); + EXPECT_EQ(kReducedPixelsFirstStep, + adapter_.source_restrictions().max_pixels_per_frame()); + EXPECT_EQ(absl::nullopt, + adapter_.source_restrictions().target_pixels_per_frame()); + EXPECT_EQ(static_cast(kBalancedHighFrameRateFps), + adapter_.source_restrictions().max_frame_rate()); + EXPECT_EQ(1, adapter_.adaptation_counters().resolution_adaptations); + EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); + // If we adapt again, because the balanced settings' proposed frame rate is + // still kBalancedHighFrameRateFps, "maintain-framerate" will trigger again. + static_assert(kReducedPixelsFirstStep > kBalancedMediumResolutionPixels, + "The reduced resolution is still greater than the next lower " + "balanced setting resolution"); + constexpr size_t kReducedPixelsSecondStep = (kReducedPixelsFirstStep * 3) / 5; + { + Adaptation adaptation = adapter_.GetAdaptationDown(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + fake_stream.ApplyAdaptation(adaptation); + } + EXPECT_EQ(kReducedPixelsSecondStep, + adapter_.source_restrictions().max_pixels_per_frame()); + EXPECT_EQ(absl::nullopt, + adapter_.source_restrictions().target_pixels_per_frame()); + EXPECT_EQ(static_cast(kBalancedHighFrameRateFps), + adapter_.source_restrictions().max_frame_rate()); + EXPECT_EQ(2, adapter_.adaptation_counters().resolution_adaptations); + EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); +} + +// Testing when to adapt frame rate and when to adapt resolution is quite +// entangled, so this test covers both cases. +// +// There is an asymmetry: When we adapt down we do it in one order, but when we +// adapt up we don't do it in the reverse order. Instead we always try to adapt +// frame rate first according to balanced settings' configs and only when the +// frame rate is already achieved do we adjust the resolution. +TEST_F(VideoStreamAdapterTest, Balanced_IncreaseFrameRateAndResolution) { + adapter_.SetDegradationPreference(DegradationPreference::BALANCED); + FakeVideoStream fake_stream( + &adapter_, &input_state_provider_, kBalancedHighResolutionPixels, + kBalancedHighFrameRateFps, kDefaultMinPixelsPerFrame); + // The desired starting point of this test is having adapted frame rate twice. + // This requires performing a number of adaptations. + constexpr size_t kReducedPixelsFirstStep = + static_cast((kBalancedHighResolutionPixels * 3) / 5); + constexpr size_t kReducedPixelsSecondStep = (kReducedPixelsFirstStep * 3) / 5; + constexpr size_t kReducedPixelsThirdStep = (kReducedPixelsSecondStep * 3) / 5; + static_assert(kReducedPixelsFirstStep > kBalancedMediumResolutionPixels, + "The first pixel reduction is greater than the balanced " + "settings' medium pixel configuration"); + static_assert(kReducedPixelsSecondStep > kBalancedMediumResolutionPixels, + "The second pixel reduction is greater than the balanced " + "settings' medium pixel configuration"); + static_assert(kReducedPixelsThirdStep <= kBalancedMediumResolutionPixels, + "The third pixel reduction is NOT greater than the balanced " + "settings' medium pixel configuration"); + // The first adaptation should affect the frame rate: See + // Balanced_DecreaseResolution for explanation why. + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + EXPECT_EQ(static_cast(kBalancedHighFrameRateFps), + adapter_.source_restrictions().max_frame_rate()); + // The next three adaptations affects the resolution, because we have to reach + // kBalancedMediumResolutionPixels before a lower frame rate is considered by + // BalancedDegradationSettings. The number three is derived from the + // static_asserts above. + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + EXPECT_EQ(kReducedPixelsFirstStep, + adapter_.source_restrictions().max_pixels_per_frame()); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + EXPECT_EQ(kReducedPixelsSecondStep, + adapter_.source_restrictions().max_pixels_per_frame()); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + EXPECT_EQ(kReducedPixelsThirdStep, + adapter_.source_restrictions().max_pixels_per_frame()); + // Thus, the next adaptation will reduce frame rate to + // kBalancedMediumFrameRateFps. + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + EXPECT_EQ(static_cast(kBalancedMediumFrameRateFps), + adapter_.source_restrictions().max_frame_rate()); + EXPECT_EQ(3, adapter_.adaptation_counters().resolution_adaptations); + EXPECT_EQ(2, adapter_.adaptation_counters().fps_adaptations); + // Adapt up! + // While our resolution is in the medium-range, the frame rate associated with + // the next resolution configuration up ("high") is kBalancedHighFrameRateFps + // and "balanced" prefers adapting frame rate if not already applied. + { + Adaptation adaptation = adapter_.GetAdaptationUp(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + fake_stream.ApplyAdaptation(adaptation); + EXPECT_EQ(static_cast(kBalancedHighFrameRateFps), + adapter_.source_restrictions().max_frame_rate()); + EXPECT_EQ(3, adapter_.adaptation_counters().resolution_adaptations); + EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); + } + // Now that we have already achieved the next frame rate up, we act according + // to "maintain-framerate". We go back up in resolution. Due to rounding + // errors we don't end up back at kReducedPixelsSecondStep. Rather we get to + // kReducedPixelsSecondStepUp, which is off by one compared to + // kReducedPixelsSecondStep. + constexpr size_t kReducedPixelsSecondStepUp = + (kReducedPixelsThirdStep * 5) / 3; + { + Adaptation adaptation = adapter_.GetAdaptationUp(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + fake_stream.ApplyAdaptation(adaptation); + EXPECT_EQ(kReducedPixelsSecondStepUp, + adapter_.source_restrictions().target_pixels_per_frame()); + EXPECT_EQ(2, adapter_.adaptation_counters().resolution_adaptations); + EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); + } + // Now that our resolution is back in the high-range, the next frame rate to + // try out is "unlimited". + { + Adaptation adaptation = adapter_.GetAdaptationUp(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + fake_stream.ApplyAdaptation(adaptation); + EXPECT_EQ(absl::nullopt, adapter_.source_restrictions().max_frame_rate()); + EXPECT_EQ(2, adapter_.adaptation_counters().resolution_adaptations); + EXPECT_EQ(0, adapter_.adaptation_counters().fps_adaptations); + } + // Now only adapting resolution remains. + constexpr size_t kReducedPixelsFirstStepUp = + (kReducedPixelsSecondStepUp * 5) / 3; + { + Adaptation adaptation = adapter_.GetAdaptationUp(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + fake_stream.ApplyAdaptation(adaptation); + EXPECT_EQ(kReducedPixelsFirstStepUp, + adapter_.source_restrictions().target_pixels_per_frame()); + EXPECT_EQ(1, adapter_.adaptation_counters().resolution_adaptations); + EXPECT_EQ(0, adapter_.adaptation_counters().fps_adaptations); + } + // The last step up should make us entirely unrestricted. + { + Adaptation adaptation = adapter_.GetAdaptationUp(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + fake_stream.ApplyAdaptation(adaptation); + EXPECT_EQ(VideoSourceRestrictions(), adapter_.source_restrictions()); + EXPECT_EQ(0, adapter_.adaptation_counters().Total()); + } +} + +TEST_F(VideoStreamAdapterTest, Balanced_LimitReached) { + adapter_.SetDegradationPreference(DegradationPreference::BALANCED); + FakeVideoStream fake_stream( + &adapter_, &input_state_provider_, kBalancedLowResolutionPixels, + kBalancedLowFrameRateFps, kDefaultMinPixelsPerFrame); + // Attempting to adapt up while unrestricted should result in kLimitReached. + EXPECT_EQ(Adaptation::Status::kLimitReached, + adapter_.GetAdaptationUp().status()); + // Adapting down once result in restricted frame rate, in this case we reach + // the lowest possible frame rate immediately: kBalancedLowFrameRateFps. + EXPECT_CALL(encoder_stats_observer_, OnMinPixelLimitReached()).Times(2); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + EXPECT_EQ(static_cast(kBalancedLowFrameRateFps), + adapter_.source_restrictions().max_frame_rate()); + EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); + // Any further adaptation must follow "maintain-framerate" rules (these are + // covered in more depth by the MaintainFramerate tests). This test does not + // assert exactly how resolution is adjusted, only that resolution always + // decreases and that we eventually reach kLimitReached. + size_t previous_resolution = kBalancedLowResolutionPixels; + bool did_reach_limit = false; + // If we have not reached the limit within 5 adaptations something is wrong... + for (int i = 0; i < 5; i++) { + Adaptation adaptation = adapter_.GetAdaptationDown(); + if (adaptation.status() == Adaptation::Status::kLimitReached) { + did_reach_limit = true; + break; + } + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + fake_stream.ApplyAdaptation(adaptation); + EXPECT_LT(adapter_.source_restrictions().max_pixels_per_frame().value(), + previous_resolution); + previous_resolution = + adapter_.source_restrictions().max_pixels_per_frame().value(); + } + EXPECT_TRUE(did_reach_limit); + // Frame rate restrictions are the same as before. + EXPECT_EQ(static_cast(kBalancedLowFrameRateFps), + adapter_.source_restrictions().max_frame_rate()); + EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); +} + +// kAwaitingPreviousAdaptation is only supported in "maintain-framerate". +TEST_F(VideoStreamAdapterTest, + MaintainFramerate_AwaitingPreviousAdaptationDown) { + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + input_state_provider_.SetInputState(1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // Adapt down once, but don't update the input. + adapter_.ApplyAdaptation(adapter_.GetAdaptationDown(), nullptr); + EXPECT_EQ(1, adapter_.adaptation_counters().resolution_adaptations); + { + // Having performed the adaptation, but not updated the input based on the + // new restrictions, adapting again in the same direction will not work. + Adaptation adaptation = adapter_.GetAdaptationDown(); + EXPECT_EQ(Adaptation::Status::kAwaitingPreviousAdaptation, + adaptation.status()); + } +} + +// kAwaitingPreviousAdaptation is only supported in "maintain-framerate". +TEST_F(VideoStreamAdapterTest, MaintainFramerate_AwaitingPreviousAdaptationUp) { + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // Perform two adaptation down so that adapting up twice is possible. + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + EXPECT_EQ(2, adapter_.adaptation_counters().resolution_adaptations); + // Adapt up once, but don't update the input. + adapter_.ApplyAdaptation(adapter_.GetAdaptationUp(), nullptr); + EXPECT_EQ(1, adapter_.adaptation_counters().resolution_adaptations); + { + // Having performed the adaptation, but not updated the input based on the + // new restrictions, adapting again in the same direction will not work. + Adaptation adaptation = adapter_.GetAdaptationUp(); + EXPECT_EQ(Adaptation::Status::kAwaitingPreviousAdaptation, + adaptation.status()); + } +} + +TEST_F(VideoStreamAdapterTest, + MaintainResolution_AdaptsUpAfterSwitchingDegradationPreference) { + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // Adapt down in fps for later. + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); + + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp()); + EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); + EXPECT_EQ(0, adapter_.adaptation_counters().resolution_adaptations); + + // We should be able to adapt in framerate one last time after the change of + // degradation preference. + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); + Adaptation adaptation = adapter_.GetAdaptationUp(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp()); + EXPECT_EQ(0, adapter_.adaptation_counters().fps_adaptations); +} + +TEST_F(VideoStreamAdapterTest, + MaintainFramerate_AdaptsUpAfterSwitchingDegradationPreference) { + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // Adapt down in resolution for later. + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + EXPECT_EQ(1, adapter_.adaptation_counters().resolution_adaptations); + + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp()); + EXPECT_EQ(1, adapter_.adaptation_counters().resolution_adaptations); + EXPECT_EQ(0, adapter_.adaptation_counters().fps_adaptations); + + // We should be able to adapt in framerate one last time after the change of + // degradation preference. + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + Adaptation adaptation = adapter_.GetAdaptationUp(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationUp()); + EXPECT_EQ(0, adapter_.adaptation_counters().resolution_adaptations); +} + +TEST_F(VideoStreamAdapterTest, + PendingResolutionIncreaseAllowsAdaptUpAfterSwitchToMaintainResolution) { + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // Adapt fps down so we can adapt up later in the test. + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + // Apply adaptation up but don't update input. + adapter_.ApplyAdaptation(adapter_.GetAdaptationUp(), nullptr); + EXPECT_EQ(Adaptation::Status::kAwaitingPreviousAdaptation, + adapter_.GetAdaptationUp().status()); + + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); + Adaptation adaptation = adapter_.GetAdaptationUp(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); +} + +TEST_F(VideoStreamAdapterTest, + MaintainFramerate_AdaptsDownAfterSwitchingDegradationPreference) { + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // Adapt down once, should change FPS. + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); + + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + // Adaptation down should apply after the degradation prefs change. + Adaptation adaptation = adapter_.GetAdaptationDown(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + fake_stream.ApplyAdaptation(adaptation); + EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); + EXPECT_EQ(1, adapter_.adaptation_counters().resolution_adaptations); +} + +TEST_F(VideoStreamAdapterTest, + MaintainResolution_AdaptsDownAfterSwitchingDegradationPreference) { + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // Adapt down once, should change FPS. + fake_stream.ApplyAdaptation(adapter_.GetAdaptationDown()); + EXPECT_EQ(1, adapter_.adaptation_counters().resolution_adaptations); + + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); + Adaptation adaptation = adapter_.GetAdaptationDown(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + fake_stream.ApplyAdaptation(adaptation); + + EXPECT_EQ(1, adapter_.adaptation_counters().fps_adaptations); + EXPECT_EQ(1, adapter_.adaptation_counters().resolution_adaptations); +} + +TEST_F( + VideoStreamAdapterTest, + PendingResolutionDecreaseAllowsAdaptDownAfterSwitchToMaintainResolution) { + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // Apply adaptation but don't update the input. + adapter_.ApplyAdaptation(adapter_.GetAdaptationDown(), nullptr); + EXPECT_EQ(Adaptation::Status::kAwaitingPreviousAdaptation, + adapter_.GetAdaptationDown().status()); + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); + Adaptation adaptation = adapter_.GetAdaptationDown(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); +} + +TEST_F(VideoStreamAdapterTest, RestrictionBroadcasted) { + FakeVideoStreamAdapterListner listener; + adapter_.AddRestrictionsListener(&listener); + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // Not broadcast on invalid ApplyAdaptation. + { + Adaptation adaptation = adapter_.GetAdaptationUp(); + adapter_.ApplyAdaptation(adaptation, nullptr); + EXPECT_EQ(0, listener.calls()); + } + + // Broadcast on ApplyAdaptation. + { + Adaptation adaptation = adapter_.GetAdaptationDown(); + fake_stream.ApplyAdaptation(adaptation); + EXPECT_EQ(1, listener.calls()); + EXPECT_EQ(adaptation.restrictions(), listener.last_restrictions()); + } + + // Broadcast on ClearRestrictions(). + adapter_.ClearRestrictions(); + EXPECT_EQ(2, listener.calls()); + EXPECT_EQ(VideoSourceRestrictions(), listener.last_restrictions()); +} + +TEST_F(VideoStreamAdapterTest, AdaptationHasNextRestrcitions) { + // Any non-disabled DegradationPreference will do. + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // When adaptation is not possible. + { + Adaptation adaptation = adapter_.GetAdaptationUp(); + EXPECT_EQ(Adaptation::Status::kLimitReached, adaptation.status()); + EXPECT_EQ(adaptation.restrictions(), adapter_.source_restrictions()); + EXPECT_EQ(0, adaptation.counters().Total()); + } + // When we adapt down. + { + Adaptation adaptation = adapter_.GetAdaptationDown(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + fake_stream.ApplyAdaptation(adaptation); + EXPECT_EQ(adaptation.restrictions(), adapter_.source_restrictions()); + EXPECT_EQ(adaptation.counters(), adapter_.adaptation_counters()); + } + // When we adapt up. + { + Adaptation adaptation = adapter_.GetAdaptationUp(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + fake_stream.ApplyAdaptation(adaptation); + EXPECT_EQ(adaptation.restrictions(), adapter_.source_restrictions()); + EXPECT_EQ(adaptation.counters(), adapter_.adaptation_counters()); + } +} + +TEST_F(VideoStreamAdapterTest, + SetDegradationPreferenceToOrFromBalancedClearsRestrictions) { + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + input_state_provider_.SetInputState(1280 * 720, 30, + kDefaultMinPixelsPerFrame); + adapter_.ApplyAdaptation(adapter_.GetAdaptationDown(), nullptr); + EXPECT_NE(VideoSourceRestrictions(), adapter_.source_restrictions()); + EXPECT_NE(0, adapter_.adaptation_counters().Total()); + // Changing from non-balanced to balanced clears the restrictions. + adapter_.SetDegradationPreference(DegradationPreference::BALANCED); + EXPECT_EQ(VideoSourceRestrictions(), adapter_.source_restrictions()); + EXPECT_EQ(0, adapter_.adaptation_counters().Total()); + // Apply adaptation again. + adapter_.ApplyAdaptation(adapter_.GetAdaptationDown(), nullptr); + EXPECT_NE(VideoSourceRestrictions(), adapter_.source_restrictions()); + EXPECT_NE(0, adapter_.adaptation_counters().Total()); + // Changing from balanced to non-balanced clears the restrictions. + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); + EXPECT_EQ(VideoSourceRestrictions(), adapter_.source_restrictions()); + EXPECT_EQ(0, adapter_.adaptation_counters().Total()); +} + +TEST_F(VideoStreamAdapterTest, + GetAdaptDownResolutionAdaptsResolutionInMaintainFramerate) { + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + input_state_provider_.SetInputState(1280 * 720, 30, + kDefaultMinPixelsPerFrame); + + auto adaptation = adapter_.GetAdaptDownResolution(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + EXPECT_EQ(1, adaptation.counters().resolution_adaptations); + EXPECT_EQ(0, adaptation.counters().fps_adaptations); +} + +TEST_F(VideoStreamAdapterTest, + GetAdaptDownResolutionReturnsWithStatusInDisabledAndMaintainResolution) { + adapter_.SetDegradationPreference(DegradationPreference::DISABLED); + input_state_provider_.SetInputState(1280 * 720, 30, + kDefaultMinPixelsPerFrame); + EXPECT_EQ(Adaptation::Status::kAdaptationDisabled, + adapter_.GetAdaptDownResolution().status()); + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); + EXPECT_EQ(Adaptation::Status::kLimitReached, + adapter_.GetAdaptDownResolution().status()); +} + +TEST_F(VideoStreamAdapterTest, + GetAdaptDownResolutionAdaptsFpsAndResolutionInBalanced) { + // Note: This test depends on BALANCED implementation, but with current + // implementation and input state settings, BALANCED will adapt resolution and + // frame rate once. + adapter_.SetDegradationPreference(DegradationPreference::BALANCED); + input_state_provider_.SetInputState(1280 * 720, 30, + kDefaultMinPixelsPerFrame); + + auto adaptation = adapter_.GetAdaptDownResolution(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + EXPECT_EQ(1, adaptation.counters().resolution_adaptations); + EXPECT_EQ(1, adaptation.counters().fps_adaptations); +} + +TEST_F( + VideoStreamAdapterTest, + GetAdaptDownResolutionAdaptsOnlyResolutionIfFpsAlreadyAdapterInBalanced) { + // Note: This test depends on BALANCED implementation, but with current + // implementation and input state settings, BALANCED will adapt resolution + // only. + adapter_.SetDegradationPreference(DegradationPreference::BALANCED); + input_state_provider_.SetInputState(1280 * 720, 5, kDefaultMinPixelsPerFrame); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + + auto first_adaptation = adapter_.GetAdaptationDown(); + fake_stream.ApplyAdaptation(first_adaptation); + + auto adaptation = adapter_.GetAdaptDownResolution(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + EXPECT_EQ(1, adaptation.counters().resolution_adaptations); + EXPECT_EQ(first_adaptation.counters().fps_adaptations, + adaptation.counters().fps_adaptations); +} + +TEST_F(VideoStreamAdapterTest, + GetAdaptDownResolutionAdaptsOnlyFpsIfResolutionLowInBalanced) { + // Note: This test depends on BALANCED implementation, but with current + // implementation and input state settings, BALANCED will adapt resolution + // only. + adapter_.SetDegradationPreference(DegradationPreference::BALANCED); + input_state_provider_.SetInputState(kDefaultMinPixelsPerFrame, 30, + kDefaultMinPixelsPerFrame); + + auto adaptation = adapter_.GetAdaptDownResolution(); + EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); + EXPECT_EQ(0, adaptation.counters().resolution_adaptations); + EXPECT_EQ(1, adaptation.counters().fps_adaptations); +} + +TEST_F(VideoStreamAdapterTest, + AdaptationDisabledStatusAlwaysWhenDegradationPreferenceDisabled) { + adapter_.SetDegradationPreference(DegradationPreference::DISABLED); + input_state_provider_.SetInputState(1280 * 720, 30, + kDefaultMinPixelsPerFrame); + EXPECT_EQ(Adaptation::Status::kAdaptationDisabled, + adapter_.GetAdaptationDown().status()); + EXPECT_EQ(Adaptation::Status::kAdaptationDisabled, + adapter_.GetAdaptationUp().status()); + EXPECT_EQ(Adaptation::Status::kAdaptationDisabled, + adapter_.GetAdaptDownResolution().status()); +} + +TEST_F(VideoStreamAdapterTest, AdaptationConstraintAllowsAdaptationsUp) { + testing::StrictMock adaptation_constraint; + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + adapter_.AddAdaptationConstraint(&adaptation_constraint); + input_state_provider_.SetInputState(1280 * 720, 30, + kDefaultMinPixelsPerFrame); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // Adapt down once so we can adapt up later. + auto first_adaptation = adapter_.GetAdaptationDown(); + fake_stream.ApplyAdaptation(first_adaptation); + + EXPECT_CALL(adaptation_constraint, + IsAdaptationUpAllowed(_, first_adaptation.restrictions(), _)) + .WillOnce(Return(true)); + EXPECT_EQ(Adaptation::Status::kValid, adapter_.GetAdaptationUp().status()); + adapter_.RemoveAdaptationConstraint(&adaptation_constraint); +} + +TEST_F(VideoStreamAdapterTest, AdaptationConstraintDisallowsAdaptationsUp) { + testing::StrictMock adaptation_constraint; + adapter_.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + adapter_.AddAdaptationConstraint(&adaptation_constraint); + input_state_provider_.SetInputState(1280 * 720, 30, + kDefaultMinPixelsPerFrame); + FakeVideoStream fake_stream(&adapter_, &input_state_provider_, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // Adapt down once so we can adapt up later. + auto first_adaptation = adapter_.GetAdaptationDown(); + fake_stream.ApplyAdaptation(first_adaptation); + + EXPECT_CALL(adaptation_constraint, + IsAdaptationUpAllowed(_, first_adaptation.restrictions(), _)) + .WillOnce(Return(false)); + EXPECT_EQ(Adaptation::Status::kRejectedByConstraint, + adapter_.GetAdaptationUp().status()); + adapter_.RemoveAdaptationConstraint(&adaptation_constraint); +} + +// Death tests. +// Disabled on Android because death tests misbehave on Android, see +// base/test/gtest_util.h. +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +TEST(VideoStreamAdapterDeathTest, + SetDegradationPreferenceInvalidatesAdaptations) { + webrtc::test::ScopedKeyValueConfig field_trials; + FakeVideoStreamInputStateProvider input_state_provider; + testing::StrictMock encoder_stats_observer_; + VideoStreamAdapter adapter(&input_state_provider, &encoder_stats_observer_, + field_trials); + adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + input_state_provider.SetInputState(1280 * 720, 30, kDefaultMinPixelsPerFrame); + Adaptation adaptation = adapter.GetAdaptationDown(); + adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); + EXPECT_DEATH(adapter.ApplyAdaptation(adaptation, nullptr), ""); +} + +TEST(VideoStreamAdapterDeathTest, AdaptDownInvalidatesAdaptations) { + webrtc::test::ScopedKeyValueConfig field_trials; + FakeVideoStreamInputStateProvider input_state_provider; + testing::StrictMock encoder_stats_observer_; + VideoStreamAdapter adapter(&input_state_provider, &encoder_stats_observer_, + field_trials); + adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); + input_state_provider.SetInputState(1280 * 720, 30, kDefaultMinPixelsPerFrame); + Adaptation adaptation = adapter.GetAdaptationDown(); + adapter.GetAdaptationDown(); + EXPECT_DEATH(adapter.ApplyAdaptation(adaptation, nullptr), ""); +} + +#endif // RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +} // namespace webrtc diff --git a/third_party/libwebrtc/call/adaptation/video_stream_input_state.cc b/third_party/libwebrtc/call/adaptation/video_stream_input_state.cc new file mode 100644 index 0000000000..9c0d475902 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/video_stream_input_state.cc @@ -0,0 +1,80 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/video_stream_input_state.h" + +#include "api/video_codecs/video_encoder.h" + +namespace webrtc { + +VideoStreamInputState::VideoStreamInputState() + : has_input_(false), + frame_size_pixels_(absl::nullopt), + frames_per_second_(0), + video_codec_type_(VideoCodecType::kVideoCodecGeneric), + min_pixels_per_frame_(kDefaultMinPixelsPerFrame), + single_active_stream_pixels_(absl::nullopt) {} + +void VideoStreamInputState::set_has_input(bool has_input) { + has_input_ = has_input; +} + +void VideoStreamInputState::set_frame_size_pixels( + absl::optional frame_size_pixels) { + frame_size_pixels_ = frame_size_pixels; +} + +void VideoStreamInputState::set_frames_per_second(int frames_per_second) { + frames_per_second_ = frames_per_second; +} + +void VideoStreamInputState::set_video_codec_type( + VideoCodecType video_codec_type) { + video_codec_type_ = video_codec_type; +} + +void VideoStreamInputState::set_min_pixels_per_frame(int min_pixels_per_frame) { + min_pixels_per_frame_ = min_pixels_per_frame; +} + +void VideoStreamInputState::set_single_active_stream_pixels( + absl::optional single_active_stream_pixels) { + single_active_stream_pixels_ = single_active_stream_pixels; +} + +bool VideoStreamInputState::has_input() const { + return has_input_; +} + +absl::optional VideoStreamInputState::frame_size_pixels() const { + return frame_size_pixels_; +} + +int VideoStreamInputState::frames_per_second() const { + return frames_per_second_; +} + +VideoCodecType VideoStreamInputState::video_codec_type() const { + return video_codec_type_; +} + +int VideoStreamInputState::min_pixels_per_frame() const { + return min_pixels_per_frame_; +} + +absl::optional VideoStreamInputState::single_active_stream_pixels() const { + return single_active_stream_pixels_; +} + +bool VideoStreamInputState::HasInputFrameSizeAndFramesPerSecond() const { + return has_input_ && frame_size_pixels_.has_value(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/call/adaptation/video_stream_input_state.h b/third_party/libwebrtc/call/adaptation/video_stream_input_state.h new file mode 100644 index 0000000000..191e22386a --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/video_stream_input_state.h @@ -0,0 +1,53 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_VIDEO_STREAM_INPUT_STATE_H_ +#define CALL_ADAPTATION_VIDEO_STREAM_INPUT_STATE_H_ + +#include "absl/types/optional.h" +#include "api/video/video_codec_type.h" + +namespace webrtc { + +// The source resolution, frame rate and other properties of a +// VideoStreamEncoder. +class VideoStreamInputState { + public: + VideoStreamInputState(); + + void set_has_input(bool has_input); + void set_frame_size_pixels(absl::optional frame_size_pixels); + void set_frames_per_second(int frames_per_second); + void set_video_codec_type(VideoCodecType video_codec_type); + void set_min_pixels_per_frame(int min_pixels_per_frame); + void set_single_active_stream_pixels( + absl::optional single_active_stream_pixels); + + bool has_input() const; + absl::optional frame_size_pixels() const; + int frames_per_second() const; + VideoCodecType video_codec_type() const; + int min_pixels_per_frame() const; + absl::optional single_active_stream_pixels() const; + + bool HasInputFrameSizeAndFramesPerSecond() const; + + private: + bool has_input_; + absl::optional frame_size_pixels_; + int frames_per_second_; + VideoCodecType video_codec_type_; + int min_pixels_per_frame_; + absl::optional single_active_stream_pixels_; +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_VIDEO_STREAM_INPUT_STATE_H_ diff --git a/third_party/libwebrtc/call/adaptation/video_stream_input_state_provider.cc b/third_party/libwebrtc/call/adaptation/video_stream_input_state_provider.cc new file mode 100644 index 0000000000..3261af39ea --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/video_stream_input_state_provider.cc @@ -0,0 +1,54 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/video_stream_input_state_provider.h" + +#include "call/adaptation/video_stream_adapter.h" + +namespace webrtc { + +VideoStreamInputStateProvider::VideoStreamInputStateProvider( + VideoStreamEncoderObserver* frame_rate_provider) + : frame_rate_provider_(frame_rate_provider) {} + +VideoStreamInputStateProvider::~VideoStreamInputStateProvider() {} + +void VideoStreamInputStateProvider::OnHasInputChanged(bool has_input) { + MutexLock lock(&mutex_); + input_state_.set_has_input(has_input); +} + +void VideoStreamInputStateProvider::OnFrameSizeObserved(int frame_size_pixels) { + RTC_DCHECK_GT(frame_size_pixels, 0); + MutexLock lock(&mutex_); + input_state_.set_frame_size_pixels(frame_size_pixels); +} + +void VideoStreamInputStateProvider::OnEncoderSettingsChanged( + EncoderSettings encoder_settings) { + MutexLock lock(&mutex_); + input_state_.set_video_codec_type( + encoder_settings.encoder_config().codec_type); + input_state_.set_min_pixels_per_frame( + encoder_settings.encoder_info().scaling_settings.min_pixels_per_frame); + input_state_.set_single_active_stream_pixels( + VideoStreamAdapter::GetSingleActiveLayerPixels( + encoder_settings.video_codec())); +} + +VideoStreamInputState VideoStreamInputStateProvider::InputState() { + // GetInputFrameRate() is thread-safe. + int input_fps = frame_rate_provider_->GetInputFrameRate(); + MutexLock lock(&mutex_); + input_state_.set_frames_per_second(input_fps); + return input_state_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/call/adaptation/video_stream_input_state_provider.h b/third_party/libwebrtc/call/adaptation/video_stream_input_state_provider.h new file mode 100644 index 0000000000..81996e6eb9 --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/video_stream_input_state_provider.h @@ -0,0 +1,41 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_VIDEO_STREAM_INPUT_STATE_PROVIDER_H_ +#define CALL_ADAPTATION_VIDEO_STREAM_INPUT_STATE_PROVIDER_H_ + +#include "call/adaptation/encoder_settings.h" +#include "call/adaptation/video_stream_input_state.h" +#include "rtc_base/synchronization/mutex.h" +#include "video/video_stream_encoder_observer.h" + +namespace webrtc { + +class VideoStreamInputStateProvider { + public: + VideoStreamInputStateProvider( + VideoStreamEncoderObserver* frame_rate_provider); + virtual ~VideoStreamInputStateProvider(); + + void OnHasInputChanged(bool has_input); + void OnFrameSizeObserved(int frame_size_pixels); + void OnEncoderSettingsChanged(EncoderSettings encoder_settings); + + virtual VideoStreamInputState InputState(); + + private: + Mutex mutex_; + VideoStreamEncoderObserver* const frame_rate_provider_; + VideoStreamInputState input_state_ RTC_GUARDED_BY(mutex_); +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_VIDEO_STREAM_INPUT_STATE_PROVIDER_H_ diff --git a/third_party/libwebrtc/call/adaptation/video_stream_input_state_provider_unittest.cc b/third_party/libwebrtc/call/adaptation/video_stream_input_state_provider_unittest.cc new file mode 100644 index 0000000000..5da2ef21cd --- /dev/null +++ b/third_party/libwebrtc/call/adaptation/video_stream_input_state_provider_unittest.cc @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/video_stream_input_state_provider.h" + +#include + +#include "api/video_codecs/video_encoder.h" +#include "call/adaptation/encoder_settings.h" +#include "call/adaptation/test/fake_frame_rate_provider.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(VideoStreamInputStateProviderTest, DefaultValues) { + FakeFrameRateProvider frame_rate_provider; + VideoStreamInputStateProvider input_state_provider(&frame_rate_provider); + VideoStreamInputState input_state = input_state_provider.InputState(); + EXPECT_EQ(false, input_state.has_input()); + EXPECT_EQ(absl::nullopt, input_state.frame_size_pixels()); + EXPECT_EQ(0, input_state.frames_per_second()); + EXPECT_EQ(VideoCodecType::kVideoCodecGeneric, input_state.video_codec_type()); + EXPECT_EQ(kDefaultMinPixelsPerFrame, input_state.min_pixels_per_frame()); + EXPECT_EQ(absl::nullopt, input_state.single_active_stream_pixels()); +} + +TEST(VideoStreamInputStateProviderTest, ValuesSet) { + FakeFrameRateProvider frame_rate_provider; + VideoStreamInputStateProvider input_state_provider(&frame_rate_provider); + input_state_provider.OnHasInputChanged(true); + input_state_provider.OnFrameSizeObserved(42); + frame_rate_provider.set_fps(123); + VideoEncoder::EncoderInfo encoder_info; + encoder_info.scaling_settings.min_pixels_per_frame = 1337; + VideoEncoderConfig encoder_config; + encoder_config.codec_type = VideoCodecType::kVideoCodecVP9; + VideoCodec video_codec; + video_codec.codecType = VideoCodecType::kVideoCodecVP8; + video_codec.numberOfSimulcastStreams = 2; + video_codec.simulcastStream[0].active = false; + video_codec.simulcastStream[1].active = true; + video_codec.simulcastStream[1].width = 111; + video_codec.simulcastStream[1].height = 222; + input_state_provider.OnEncoderSettingsChanged(EncoderSettings( + std::move(encoder_info), std::move(encoder_config), video_codec)); + VideoStreamInputState input_state = input_state_provider.InputState(); + EXPECT_EQ(true, input_state.has_input()); + EXPECT_EQ(42, input_state.frame_size_pixels()); + EXPECT_EQ(123, input_state.frames_per_second()); + EXPECT_EQ(VideoCodecType::kVideoCodecVP9, input_state.video_codec_type()); + EXPECT_EQ(1337, input_state.min_pixels_per_frame()); + EXPECT_EQ(111 * 222, input_state.single_active_stream_pixels()); +} + +} // namespace webrtc -- cgit v1.2.3