1 files changed, 378 insertions, 0 deletions
diff --git a/third_party/libwebrtc/call/adaptation/resource_adaptation_processor.cc b/third_party/libwebrtc/call/adaptation/resource_adaptation_processor.cc
new file mode 100644
index 0000000000..f4d1bf3538
--- /dev/null
+++ b/third_party/libwebrtc/call/adaptation/resource_adaptation_processor.cc
@@ -0,0 +1,378 @@
+/*
+ *  Copyright 2020 The WebRTC Project Authors. All rights reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "call/adaptation/resource_adaptation_processor.h"
+
+#include <algorithm>
+#include <string>
+#include <utility>
+
+#include "absl/algorithm/container.h"
+#include "absl/strings/string_view.h"
+#include "api/sequence_checker.h"
+#include "api/video/video_adaptation_counters.h"
+#include "call/adaptation/video_stream_adapter.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/strings/string_builder.h"
+
+namespace webrtc {
+
+ResourceAdaptationProcessor::ResourceListenerDelegate::ResourceListenerDelegate(
+    ResourceAdaptationProcessor* processor)
+    : task_queue_(TaskQueueBase::Current()), processor_(processor) {
+  RTC_DCHECK(task_queue_);
+}
+
+void ResourceAdaptationProcessor::ResourceListenerDelegate::
+    OnProcessorDestroyed() {
+  RTC_DCHECK_RUN_ON(task_queue_);
+  processor_ = nullptr;
+}
+
+void ResourceAdaptationProcessor::ResourceListenerDelegate::
+    OnResourceUsageStateMeasured(rtc::scoped_refptr<Resource> resource,
+                                 ResourceUsageState usage_state) {
+  if (!task_queue_->IsCurrent()) {
+    task_queue_->PostTask(
+        [this_ref = rtc::scoped_refptr<ResourceListenerDelegate>(this),
+         resource, usage_state] {
+          this_ref->OnResourceUsageStateMeasured(resource, usage_state);
+        });
+    return;
+  }
+  RTC_DCHECK_RUN_ON(task_queue_);
+  if (processor_) {
+    processor_->OnResourceUsageStateMeasured(resource, usage_state);
+  }
+}
+
+ResourceAdaptationProcessor::MitigationResultAndLogMessage::
+    MitigationResultAndLogMessage()
+    : result(MitigationResult::kAdaptationApplied), message() {}
+
+ResourceAdaptationProcessor::MitigationResultAndLogMessage::
+    MitigationResultAndLogMessage(MitigationResult result,
+                                  absl::string_view message)
+    : result(result), message(message) {}
+
+ResourceAdaptationProcessor::ResourceAdaptationProcessor(
+    VideoStreamAdapter* stream_adapter)
+    : task_queue_(TaskQueueBase::Current()),
+      resource_listener_delegate_(
+          rtc::make_ref_counted<ResourceListenerDelegate>(this)),
+      resources_(),
+      stream_adapter_(stream_adapter),
+      last_reported_source_restrictions_(),
+      previous_mitigation_results_() {
+  RTC_DCHECK(task_queue_);
+  stream_adapter_->AddRestrictionsListener(this);
+}
+
+ResourceAdaptationProcessor::~ResourceAdaptationProcessor() {
+  RTC_DCHECK_RUN_ON(task_queue_);
+  RTC_DCHECK(resources_.empty())
+      << "There are resource(s) attached to a ResourceAdaptationProcessor "
+      << "being destroyed.";
+  stream_adapter_->RemoveRestrictionsListener(this);
+  resource_listener_delegate_->OnProcessorDestroyed();
+}
+
+void ResourceAdaptationProcessor::AddResourceLimitationsListener(
+    ResourceLimitationsListener* limitations_listener) {
+  RTC_DCHECK_RUN_ON(task_queue_);
+  RTC_DCHECK(std::find(resource_limitations_listeners_.begin(),
+                       resource_limitations_listeners_.end(),
+                       limitations_listener) ==
+             resource_limitations_listeners_.end());
+  resource_limitations_listeners_.push_back(limitations_listener);
+}
+
+void ResourceAdaptationProcessor::RemoveResourceLimitationsListener(
+    ResourceLimitationsListener* limitations_listener) {
+  RTC_DCHECK_RUN_ON(task_queue_);
+  auto it =
+      std::find(resource_limitations_listeners_.begin(),
+                resource_limitations_listeners_.end(), limitations_listener);
+  RTC_DCHECK(it != resource_limitations_listeners_.end());
+  resource_limitations_listeners_.erase(it);
+}
+
+void ResourceAdaptationProcessor::AddResource(
+    rtc::scoped_refptr<Resource> resource) {
+  RTC_DCHECK(resource);
+  {
+    MutexLock crit(&resources_lock_);
+    RTC_DCHECK(absl::c_find(resources_, resource) == resources_.end())
+        << "Resource \"" << resource->Name() << "\" was already registered.";
+    resources_.push_back(resource);
+  }
+  resource->SetResourceListener(resource_listener_delegate_.get());
+  RTC_LOG(LS_INFO) << "Registered resource \"" << resource->Name() << "\".";
+}
+
+std::vector<rtc::scoped_refptr<Resource>>
+ResourceAdaptationProcessor::GetResources() const {
+  MutexLock crit(&resources_lock_);
+  return resources_;
+}
+
+void ResourceAdaptationProcessor::RemoveResource(
+    rtc::scoped_refptr<Resource> resource) {
+  RTC_DCHECK(resource);
+  RTC_LOG(LS_INFO) << "Removing resource \"" << resource->Name() << "\".";
+  resource->SetResourceListener(nullptr);
+  {
+    MutexLock crit(&resources_lock_);
+    auto it = absl::c_find(resources_, resource);
+    RTC_DCHECK(it != resources_.end()) << "Resource \"" << resource->Name()
+                                       << "\" was not a registered resource.";
+    resources_.erase(it);
+  }
+  RemoveLimitationsImposedByResource(std::move(resource));
+}
+
+void ResourceAdaptationProcessor::RemoveLimitationsImposedByResource(
+    rtc::scoped_refptr<Resource> resource) {
+  if (!task_queue_->IsCurrent()) {
+    task_queue_->PostTask(
+        [this, resource]() { RemoveLimitationsImposedByResource(resource); });
+    return;
+  }
+  RTC_DCHECK_RUN_ON(task_queue_);
+  auto resource_adaptation_limits =
+      adaptation_limits_by_resources_.find(resource);
+  if (resource_adaptation_limits != adaptation_limits_by_resources_.end()) {
+    VideoStreamAdapter::RestrictionsWithCounters adaptation_limits =
+        resource_adaptation_limits->second;
+    adaptation_limits_by_resources_.erase(resource_adaptation_limits);
+    if (adaptation_limits_by_resources_.empty()) {
+      // Only the resource being removed was adapted so clear restrictions.
+      stream_adapter_->ClearRestrictions();
+      return;
+    }
+
+    VideoStreamAdapter::RestrictionsWithCounters most_limited =
+        FindMostLimitedResources().second;
+
+    if (adaptation_limits.counters.Total() <= most_limited.counters.Total()) {
+      // The removed limitations were less limited than the most limited
+      // resource. Don't change the current restrictions.
+      return;
+    }
+
+    // Apply the new most limited resource as the next restrictions.
+    Adaptation adapt_to = stream_adapter_->GetAdaptationTo(
+        most_limited.counters, most_limited.restrictions);
+    RTC_DCHECK_EQ(adapt_to.status(), Adaptation::Status::kValid);
+    stream_adapter_->ApplyAdaptation(adapt_to, nullptr);
+
+    RTC_LOG(LS_INFO)
+        << "Most limited resource removed. Restoring restrictions to "
+           "next most limited restrictions: "
+        << most_limited.restrictions.ToString() << " with counters "
+        << most_limited.counters.ToString();
+  }
+}
+
+void ResourceAdaptationProcessor::OnResourceUsageStateMeasured(
+    rtc::scoped_refptr<Resource> resource,
+    ResourceUsageState usage_state) {
+  RTC_DCHECK_RUN_ON(task_queue_);
+  RTC_DCHECK(resource);
+  // `resource` could have been removed after signalling.
+  {
+    MutexLock crit(&resources_lock_);
+    if (absl::c_find(resources_, resource) == resources_.end()) {
+      RTC_LOG(LS_INFO) << "Ignoring signal from removed resource \""
+                       << resource->Name() << "\".";
+      return;
+    }
+  }
+  MitigationResultAndLogMessage result_and_message;
+  switch (usage_state) {
+    case ResourceUsageState::kOveruse:
+      result_and_message = OnResourceOveruse(resource);
+      break;
+    case ResourceUsageState::kUnderuse:
+      result_and_message = OnResourceUnderuse(resource);
+      break;
+  }
+  // Maybe log the result of the operation.
+  auto it = previous_mitigation_results_.find(resource.get());
+  if (it != previous_mitigation_results_.end() &&
+      it->second == result_and_message.result) {
+    // This resource has previously reported the same result and we haven't
+    // successfully adapted since - don't log to avoid spam.
+    return;
+  }
+  RTC_LOG(LS_INFO) << "Resource \"" << resource->Name() << "\" signalled "
+                   << ResourceUsageStateToString(usage_state) << ". "
+                   << result_and_message.message;
+  if (result_and_message.result == MitigationResult::kAdaptationApplied) {
+    previous_mitigation_results_.clear();
+  } else {
+    previous_mitigation_results_.insert(
+        std::make_pair(resource.get(), result_and_message.result));
+  }
+}
+
+ResourceAdaptationProcessor::MitigationResultAndLogMessage
+ResourceAdaptationProcessor::OnResourceUnderuse(
+    rtc::scoped_refptr<Resource> reason_resource) {
+  RTC_DCHECK_RUN_ON(task_queue_);
+  // How can this stream be adapted up?
+  Adaptation adaptation = stream_adapter_->GetAdaptationUp();
+  if (adaptation.status() != Adaptation::Status::kValid) {
+    rtc::StringBuilder message;
+    message << "Not adapting up because VideoStreamAdapter returned "
+            << Adaptation::StatusToString(adaptation.status());
+    return MitigationResultAndLogMessage(MitigationResult::kRejectedByAdapter,
+                                         message.Release());
+  }
+  // Check that resource is most limited.
+  std::vector<rtc::scoped_refptr<Resource>> most_limited_resources;
+  VideoStreamAdapter::RestrictionsWithCounters most_limited_restrictions;
+  std::tie(most_limited_resources, most_limited_restrictions) =
+      FindMostLimitedResources();
+
+  // If the most restricted resource is less limited than current restrictions
+  // then proceed with adapting up.
+  if (!most_limited_resources.empty() &&
+      most_limited_restrictions.counters.Total() >=
+          stream_adapter_->adaptation_counters().Total()) {
+    // If `reason_resource` is not one of the most limiting resources then abort
+    // adaptation.
+    if (absl::c_find(most_limited_resources, reason_resource) ==
+        most_limited_resources.end()) {
+      rtc::StringBuilder message;
+      message << "Resource \"" << reason_resource->Name()
+              << "\" was not the most limited resource.";
+      return MitigationResultAndLogMessage(
+          MitigationResult::kNotMostLimitedResource, message.Release());
+    }
+
+    if (most_limited_resources.size() > 1) {
+      // If there are multiple most limited resources, all must signal underuse
+      // before the adaptation is applied.
+      UpdateResourceLimitations(reason_resource, adaptation.restrictions(),
+                                adaptation.counters());
+      rtc::StringBuilder message;
+      message << "Resource \"" << reason_resource->Name()
+              << "\" was not the only most limited resource.";
+      return MitigationResultAndLogMessage(
+          MitigationResult::kSharedMostLimitedResource, message.Release());
+    }
+  }
+  // Apply adaptation.
+  stream_adapter_->ApplyAdaptation(adaptation, reason_resource);
+  rtc::StringBuilder message;
+  message << "Adapted up successfully. Unfiltered adaptations: "
+          << stream_adapter_->adaptation_counters().ToString();
+  return MitigationResultAndLogMessage(MitigationResult::kAdaptationApplied,
+                                       message.Release());
+}
+
+ResourceAdaptationProcessor::MitigationResultAndLogMessage
+ResourceAdaptationProcessor::OnResourceOveruse(
+    rtc::scoped_refptr<Resource> reason_resource) {
+  RTC_DCHECK_RUN_ON(task_queue_);
+  // How can this stream be adapted up?
+  Adaptation adaptation = stream_adapter_->GetAdaptationDown();
+  if (adaptation.status() == Adaptation::Status::kLimitReached) {
+    // Add resource as most limited.
+    VideoStreamAdapter::RestrictionsWithCounters restrictions;
+    std::tie(std::ignore, restrictions) = FindMostLimitedResources();
+    UpdateResourceLimitations(reason_resource, restrictions.restrictions,
+                              restrictions.counters);
+  }
+  if (adaptation.status() != Adaptation::Status::kValid) {
+    rtc::StringBuilder message;
+    message << "Not adapting down because VideoStreamAdapter returned "
+            << Adaptation::StatusToString(adaptation.status());
+    return MitigationResultAndLogMessage(MitigationResult::kRejectedByAdapter,
+                                         message.Release());
+  }
+  // Apply adaptation.
+  UpdateResourceLimitations(reason_resource, adaptation.restrictions(),
+                            adaptation.counters());
+  stream_adapter_->ApplyAdaptation(adaptation, reason_resource);
+  rtc::StringBuilder message;
+  message << "Adapted down successfully. Unfiltered adaptations: "
+          << stream_adapter_->adaptation_counters().ToString();
+  return MitigationResultAndLogMessage(MitigationResult::kAdaptationApplied,
+                                       message.Release());
+}
+
+std::pair<std::vector<rtc::scoped_refptr<Resource>>,
+          VideoStreamAdapter::RestrictionsWithCounters>
+ResourceAdaptationProcessor::FindMostLimitedResources() const {
+  std::vector<rtc::scoped_refptr<Resource>> most_limited_resources;
+  VideoStreamAdapter::RestrictionsWithCounters most_limited_restrictions{
+      VideoSourceRestrictions(), VideoAdaptationCounters()};
+
+  for (const auto& resource_and_adaptation_limit_ :
+       adaptation_limits_by_resources_) {
+    const auto& restrictions_with_counters =
+        resource_and_adaptation_limit_.second;
+    if (restrictions_with_counters.counters.Total() >
+        most_limited_restrictions.counters.Total()) {
+      most_limited_restrictions = restrictions_with_counters;
+      most_limited_resources.clear();
+      most_limited_resources.push_back(resource_and_adaptation_limit_.first);
+    } else if (most_limited_restrictions.counters ==
+               restrictions_with_counters.counters) {
+      most_limited_resources.push_back(resource_and_adaptation_limit_.first);
+    }
+  }
+  return std::make_pair(std::move(most_limited_resources),
+                        most_limited_restrictions);
+}
+
+void ResourceAdaptationProcessor::UpdateResourceLimitations(
+    rtc::scoped_refptr<Resource> reason_resource,
+    const VideoSourceRestrictions& restrictions,
+    const VideoAdaptationCounters& counters) {
+  auto& adaptation_limits = adaptation_limits_by_resources_[reason_resource];
+  if (adaptation_limits.restrictions == restrictions &&
+      adaptation_limits.counters == counters) {
+    return;
+  }
+  adaptation_limits = {restrictions, counters};
+
+  std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters> limitations;
+  for (const auto& p : adaptation_limits_by_resources_) {
+    limitations.insert(std::make_pair(p.first, p.second.counters));
+  }
+  for (auto limitations_listener : resource_limitations_listeners_) {
+    limitations_listener->OnResourceLimitationChanged(reason_resource,
+                                                      limitations);
+  }
+}
+
+void ResourceAdaptationProcessor::OnVideoSourceRestrictionsUpdated(
+    VideoSourceRestrictions restrictions,
+    const VideoAdaptationCounters& adaptation_counters,
+    rtc::scoped_refptr<Resource> reason,
+    const VideoSourceRestrictions& unfiltered_restrictions) {
+  RTC_DCHECK_RUN_ON(task_queue_);
+  if (reason) {
+    UpdateResourceLimitations(reason, unfiltered_restrictions,
+                              adaptation_counters);
+  } else if (adaptation_counters.Total() == 0) {
+    // Adaptations are cleared.
+    adaptation_limits_by_resources_.clear();
+    previous_mitigation_results_.clear();
+    for (auto limitations_listener : resource_limitations_listeners_) {
+      limitations_listener->OnResourceLimitationChanged(nullptr, {});
+    }
+  }
+}
+
+}  // namespace webrtc