Adding upstream version 124.0.1.upstream/124.0.1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 560 insertions, 0 deletions

diff --git a/dom/media/webrtc/MediaTrackConstraints.cpp b/dom/media/webrtc/MediaTrackConstraints.cpp
new file mode 100644
index 0000000000..101cd0240a
--- /dev/null
+++ b/dom/media/webrtc/MediaTrackConstraints.cpp
@@ -0,0 +1,560 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "MediaTrackConstraints.h"
+
+#include <limits>
+#include <algorithm>
+#include <iterator>
+
+#include "MediaEngineSource.h"
+#include "mozilla/dom/MediaStreamTrackBinding.h"
+#include "mozilla/MediaManager.h"
+
+#ifdef MOZ_WEBRTC
+namespace mozilla {
+extern LazyLogModule gMediaManagerLog;
+}
+#else
+static mozilla::LazyLogModule gMediaManagerLog("MediaManager");
+#endif
+#define LOG(...) MOZ_LOG(gMediaManagerLog, LogLevel::Debug, (__VA_ARGS__))
+
+namespace mozilla {
+
+using dom::CallerType;
+using dom::ConstrainBooleanParameters;
+
+template <class ValueType>
+template <class ConstrainRange>
+void NormalizedConstraintSet::Range<ValueType>::SetFrom(
+    const ConstrainRange& aOther) {
+  if (aOther.mIdeal.WasPassed()) {
+    mIdeal.emplace(aOther.mIdeal.Value());
+  }
+  if (aOther.mExact.WasPassed()) {
+    mMin = aOther.mExact.Value();
+    mMax = aOther.mExact.Value();
+  } else {
+    if (aOther.mMin.WasPassed()) {
+      mMin = aOther.mMin.Value();
+    }
+    if (aOther.mMax.WasPassed()) {
+      mMax = aOther.mMax.Value();
+    }
+  }
+}
+
+// The Range code works surprisingly well for bool, except when averaging
+// ideals.
+template <>
+bool NormalizedConstraintSet::Range<bool>::Merge(const Range& aOther) {
+  if (!Intersects(aOther)) {
+    return false;
+  }
+  Intersect(aOther);
+
+  // To avoid "unsafe use of type 'bool'", we keep counter in mMergeDenominator
+  uint32_t counter = mMergeDenominator >> 16;
+  uint32_t denominator = mMergeDenominator & 0xffff;
+
+  if (aOther.mIdeal.isSome()) {
+    if (mIdeal.isNothing()) {
+      mIdeal.emplace(aOther.Get(false));
+      counter = aOther.Get(false);
+      denominator = 1;
+    } else {
+      if (!denominator) {
+        counter = Get(false);
+        denominator = 1;
+      }
+      counter += aOther.Get(false);
+      denominator++;
+    }
+  }
+  mMergeDenominator = ((counter & 0xffff) << 16) + (denominator & 0xffff);
+  return true;
+}
+
+template <>
+void NormalizedConstraintSet::Range<bool>::FinalizeMerge() {
+  if (mMergeDenominator) {
+    uint32_t counter = mMergeDenominator >> 16;
+    uint32_t denominator = mMergeDenominator & 0xffff;
+
+    *mIdeal = !!(counter / denominator);
+    mMergeDenominator = 0;
+  }
+}
+
+NormalizedConstraintSet::LongRange::LongRange(
+    LongPtrType aMemberPtr, const char* aName,
+    const dom::Optional<dom::OwningLongOrConstrainLongRange>& aOther,
+    bool advanced, nsTArray<MemberPtrType>* aList)
+    : Range<int32_t>((MemberPtrType)aMemberPtr, aName, 1 + INT32_MIN,
+                     INT32_MAX,  // +1 avoids Windows compiler bug
+                     aList) {
+  if (!aOther.WasPassed()) {
+    return;
+  }
+  auto& other = aOther.Value();
+  if (other.IsLong()) {
+    if (advanced) {
+      mMin = mMax = other.GetAsLong();
+    } else {
+      mIdeal.emplace(other.GetAsLong());
+    }
+  } else {
+    SetFrom(other.GetAsConstrainLongRange());
+  }
+}
+
+NormalizedConstraintSet::LongLongRange::LongLongRange(
+    LongLongPtrType aMemberPtr, const char* aName, const long long& aOther,
+    nsTArray<MemberPtrType>* aList)
+    : Range<int64_t>((MemberPtrType)aMemberPtr, aName, 1 + INT64_MIN,
+                     INT64_MAX,  // +1 avoids Windows compiler bug
+                     aList) {
+  mIdeal.emplace(aOther);
+}
+
+NormalizedConstraintSet::DoubleRange::DoubleRange(
+    DoublePtrType aMemberPtr, const char* aName,
+    const dom::Optional<dom::OwningDoubleOrConstrainDoubleRange>& aOther,
+    bool advanced, nsTArray<MemberPtrType>* aList)
+    : Range<double>((MemberPtrType)aMemberPtr, aName,
+                    -std::numeric_limits<double>::infinity(),
+                    std::numeric_limits<double>::infinity(), aList) {
+  if (!aOther.WasPassed()) {
+    return;
+  }
+  auto& other = aOther.Value();
+  if (other.IsDouble()) {
+    if (advanced) {
+      mMin = mMax = other.GetAsDouble();
+    } else {
+      mIdeal.emplace(other.GetAsDouble());
+    }
+  } else {
+    SetFrom(other.GetAsConstrainDoubleRange());
+  }
+}
+
+NormalizedConstraintSet::BooleanRange::BooleanRange(
+    BooleanPtrType aMemberPtr, const char* aName,
+    const dom::Optional<dom::OwningBooleanOrConstrainBooleanParameters>& aOther,
+    bool advanced, nsTArray<MemberPtrType>* aList)
+    : Range<bool>((MemberPtrType)aMemberPtr, aName, false, true, aList) {
+  if (!aOther.WasPassed()) {
+    return;
+  }
+  auto& other = aOther.Value();
+  if (other.IsBoolean()) {
+    if (advanced) {
+      mMin = mMax = other.GetAsBoolean();
+    } else {
+      mIdeal.emplace(other.GetAsBoolean());
+    }
+  } else {
+    auto& r = other.GetAsConstrainBooleanParameters();
+    if (r.mIdeal.WasPassed()) {
+      mIdeal.emplace(r.mIdeal.Value());
+    }
+    if (r.mExact.WasPassed()) {
+      mMin = r.mExact.Value();
+      mMax = r.mExact.Value();
+    }
+  }
+}
+
+NormalizedConstraintSet::StringRange::StringRange(
+    StringPtrType aMemberPtr, const char* aName,
+    const dom::Optional<
+        dom::OwningStringOrStringSequenceOrConstrainDOMStringParameters>&
+        aOther,
+    bool advanced, nsTArray<MemberPtrType>* aList)
+    : BaseRange((MemberPtrType)aMemberPtr, aName, aList) {
+  if (!aOther.WasPassed()) {
+    return;
+  }
+  auto& other = aOther.Value();
+  if (other.IsString()) {
+    if (advanced) {
+      mExact.insert(other.GetAsString());
+    } else {
+      mIdeal.insert(other.GetAsString());
+    }
+  } else if (other.IsStringSequence()) {
+    if (advanced) {
+      mExact.clear();
+      for (auto& str : other.GetAsStringSequence()) {
+        mExact.insert(str);
+      }
+    } else {
+      mIdeal.clear();
+      for (auto& str : other.GetAsStringSequence()) {
+        mIdeal.insert(str);
+      }
+    }
+  } else {
+    SetFrom(other.GetAsConstrainDOMStringParameters());
+  }
+}
+
+void NormalizedConstraintSet::StringRange::SetFrom(
+    const dom::ConstrainDOMStringParameters& aOther) {
+  if (aOther.mIdeal.WasPassed()) {
+    mIdeal.clear();
+    if (aOther.mIdeal.Value().IsString()) {
+      mIdeal.insert(aOther.mIdeal.Value().GetAsString());
+    } else {
+      for (auto& str : aOther.mIdeal.Value().GetAsStringSequence()) {
+        mIdeal.insert(str);
+      }
+    }
+  }
+  if (aOther.mExact.WasPassed()) {
+    mExact.clear();
+    if (aOther.mExact.Value().IsString()) {
+      mExact.insert(aOther.mExact.Value().GetAsString());
+    } else {
+      for (auto& str : aOther.mExact.Value().GetAsStringSequence()) {
+        mExact.insert(str);
+      }
+    }
+  }
+}
+
+auto NormalizedConstraintSet::StringRange::Clamp(const ValueType& n) const
+    -> ValueType {
+  if (mExact.empty()) {
+    return n;
+  }
+  ValueType result;
+  for (auto& entry : n) {
+    if (mExact.find(entry) != mExact.end()) {
+      result.insert(entry);
+    }
+  }
+  return result;
+}
+
+bool NormalizedConstraintSet::StringRange::Intersects(
+    const StringRange& aOther) const {
+  if (mExact.empty() || aOther.mExact.empty()) {
+    return true;
+  }
+
+  ValueType intersection;
+  set_intersection(mExact.begin(), mExact.end(), aOther.mExact.begin(),
+                   aOther.mExact.end(),
+                   std::inserter(intersection, intersection.begin()));
+  return !intersection.empty();
+}
+
+void NormalizedConstraintSet::StringRange::Intersect(
+    const StringRange& aOther) {
+  if (aOther.mExact.empty()) {
+    return;
+  }
+
+  ValueType intersection;
+  set_intersection(mExact.begin(), mExact.end(), aOther.mExact.begin(),
+                   aOther.mExact.end(),
+                   std::inserter(intersection, intersection.begin()));
+  mExact = intersection;
+}
+
+bool NormalizedConstraintSet::StringRange::Merge(const StringRange& aOther) {
+  if (!Intersects(aOther)) {
+    return false;
+  }
+  Intersect(aOther);
+
+  ValueType unioned;
+  set_union(mIdeal.begin(), mIdeal.end(), aOther.mIdeal.begin(),
+            aOther.mIdeal.end(), std::inserter(unioned, unioned.begin()));
+  mIdeal = unioned;
+  return true;
+}
+
+NormalizedConstraints::NormalizedConstraints(
+    const dom::MediaTrackConstraints& aOther, nsTArray<MemberPtrType>* aList)
+    : NormalizedConstraintSet(aOther, false, aList), mBadConstraint(nullptr) {
+  if (aOther.mAdvanced.WasPassed()) {
+    for (auto& entry : aOther.mAdvanced.Value()) {
+      mAdvanced.push_back(NormalizedConstraintSet(entry, true));
+    }
+  }
+}
+
+FlattenedConstraints::FlattenedConstraints(const NormalizedConstraints& aOther)
+    : NormalizedConstraintSet(aOther) {
+  for (auto& set : aOther.mAdvanced) {
+    // Must only apply compatible i.e. inherently non-overconstraining sets
+    // This rule is pretty much why this code is centralized here.
+    if (mWidth.Intersects(set.mWidth) && mHeight.Intersects(set.mHeight) &&
+        mFrameRate.Intersects(set.mFrameRate)) {
+      mWidth.Intersect(set.mWidth);
+      mHeight.Intersect(set.mHeight);
+      mFrameRate.Intersect(set.mFrameRate);
+    }
+    if (mEchoCancellation.Intersects(set.mEchoCancellation)) {
+      mEchoCancellation.Intersect(set.mEchoCancellation);
+    }
+    if (mNoiseSuppression.Intersects(set.mNoiseSuppression)) {
+      mNoiseSuppression.Intersect(set.mNoiseSuppression);
+    }
+    if (mAutoGainControl.Intersects(set.mAutoGainControl)) {
+      mAutoGainControl.Intersect(set.mAutoGainControl);
+    }
+    if (mChannelCount.Intersects(set.mChannelCount)) {
+      mChannelCount.Intersect(set.mChannelCount);
+    }
+  }
+}
+
+// MediaEngine helper
+//
+// The full algorithm for all devices.
+//
+// Fitness distance returned as integer math * 1000. Infinity = UINT32_MAX
+
+// First, all devices have a minimum distance based on their deviceId.
+// If you have no other constraints, use this one. Reused by all device types.
+
+/* static */
+bool MediaConstraintsHelper::SomeSettingsFit(
+    const NormalizedConstraints& aConstraints,
+    const nsTArray<RefPtr<LocalMediaDevice>>& aDevices) {
+  nsTArray<const NormalizedConstraintSet*> sets;
+  sets.AppendElement(&aConstraints);
+
+  MOZ_ASSERT(!aDevices.IsEmpty());
+  for (auto& device : aDevices) {
+    auto distance = device->GetBestFitnessDistance(sets, CallerType::NonSystem);
+    if (distance != UINT32_MAX) {
+      return true;
+    }
+  }
+  return false;
+}
+
+// Fitness distance returned as integer math * 1000. Infinity = UINT32_MAX
+
+/* static */
+uint32_t MediaConstraintsHelper::FitnessDistance(
+    const Maybe<nsString>& aN,
+    const NormalizedConstraintSet::StringRange& aParams) {
+  if (!aParams.mExact.empty() &&
+      (aN.isNothing() || aParams.mExact.find(*aN) == aParams.mExact.end())) {
+    return UINT32_MAX;
+  }
+  if (!aParams.mIdeal.empty() &&
+      (aN.isNothing() || aParams.mIdeal.find(*aN) == aParams.mIdeal.end())) {
+    return 1000;
+  }
+  return 0;
+}
+
+/* static */ const char* MediaConstraintsHelper::SelectSettings(
+    const NormalizedConstraints& aConstraints,
+    nsTArray<RefPtr<LocalMediaDevice>>& aDevices, CallerType aCallerType) {
+  auto& c = aConstraints;
+  LogConstraints(c);
+
+  // First apply top-level constraints.
+
+  // Stack constraintSets that pass, starting with the required one, because the
+  // whole stack must be re-satisfied each time a capability-set is ruled out
+  // (this avoids storing state or pushing algorithm into the lower-level code).
+  nsTArray<RefPtr<LocalMediaDevice>> unsatisfactory;
+  nsTArray<const NormalizedConstraintSet*> aggregateConstraints;
+  aggregateConstraints.AppendElement(&c);
+
+  std::multimap<uint32_t, RefPtr<LocalMediaDevice>> ordered;
+
+  for (uint32_t i = 0; i < aDevices.Length();) {
+    uint32_t distance =
+        aDevices[i]->GetBestFitnessDistance(aggregateConstraints, aCallerType);
+    if (distance == UINT32_MAX) {
+      unsatisfactory.AppendElement(std::move(aDevices[i]));
+      aDevices.RemoveElementAt(i);
+    } else {
+      ordered.insert(std::make_pair(distance, aDevices[i]));
+      ++i;
+    }
+  }
+  if (aDevices.IsEmpty()) {
+    return FindBadConstraint(c, unsatisfactory);
+  }
+
+  // Order devices by shortest distance
+  for (auto& ordinal : ordered) {
+    aDevices.RemoveElement(ordinal.second);
+    aDevices.AppendElement(ordinal.second);
+  }
+
+  // Then apply advanced constraints.
+
+  for (const auto& advanced : c.mAdvanced) {
+    aggregateConstraints.AppendElement(&advanced);
+    nsTArray<RefPtr<LocalMediaDevice>> rejects;
+    for (uint32_t j = 0; j < aDevices.Length();) {
+      uint32_t distance = aDevices[j]->GetBestFitnessDistance(
+          aggregateConstraints, aCallerType);
+      if (distance == UINT32_MAX) {
+        rejects.AppendElement(std::move(aDevices[j]));
+        aDevices.RemoveElementAt(j);
+      } else {
+        ++j;
+      }
+    }
+    if (aDevices.IsEmpty()) {
+      aDevices.AppendElements(std::move(rejects));
+      aggregateConstraints.RemoveLastElement();
+    }
+  }
+  return nullptr;
+}
+
+/* static */ const char* MediaConstraintsHelper::FindBadConstraint(
+    const NormalizedConstraints& aConstraints,
+    const nsTArray<RefPtr<LocalMediaDevice>>& aDevices) {
+  // The spec says to report a constraint that satisfies NONE
+  // of the sources. Unfortunately, this is a bit laborious to find out, and
+  // requires updating as new constraints are added!
+  auto& c = aConstraints;
+  dom::MediaTrackConstraints empty;
+
+  if (aDevices.IsEmpty() ||
+      !SomeSettingsFit(NormalizedConstraints(empty), aDevices)) {
+    return "";
+  }
+  {
+    NormalizedConstraints fresh(empty);
+    fresh.mDeviceId = c.mDeviceId;
+    if (!SomeSettingsFit(fresh, aDevices)) {
+      return "deviceId";
+    }
+  }
+  {
+    NormalizedConstraints fresh(empty);
+    fresh.mGroupId = c.mGroupId;
+    if (!SomeSettingsFit(fresh, aDevices)) {
+      return "groupId";
+    }
+  }
+  {
+    NormalizedConstraints fresh(empty);
+    fresh.mWidth = c.mWidth;
+    if (!SomeSettingsFit(fresh, aDevices)) {
+      return "width";
+    }
+  }
+  {
+    NormalizedConstraints fresh(empty);
+    fresh.mHeight = c.mHeight;
+    if (!SomeSettingsFit(fresh, aDevices)) {
+      return "height";
+    }
+  }
+  {
+    NormalizedConstraints fresh(empty);
+    fresh.mFrameRate = c.mFrameRate;
+    if (!SomeSettingsFit(fresh, aDevices)) {
+      return "frameRate";
+    }
+  }
+  {
+    NormalizedConstraints fresh(empty);
+    fresh.mFacingMode = c.mFacingMode;
+    if (!SomeSettingsFit(fresh, aDevices)) {
+      return "facingMode";
+    }
+  }
+  return "";
+}
+
+/* static */
+const char* MediaConstraintsHelper::FindBadConstraint(
+    const NormalizedConstraints& aConstraints,
+    const MediaDevice* aMediaDevice) {
+  NormalizedConstraints c(aConstraints);
+  NormalizedConstraints empty((dom::MediaTrackConstraints()));
+  c.mDeviceId = empty.mDeviceId;
+  c.mGroupId = empty.mGroupId;
+  AutoTArray<RefPtr<LocalMediaDevice>, 1> devices;
+  devices.EmplaceBack(
+      new LocalMediaDevice(aMediaDevice, u""_ns, u""_ns, u""_ns));
+  return FindBadConstraint(c, devices);
+}
+
+static void LogConstraintStringRange(
+    const NormalizedConstraintSet::StringRange& aRange) {
+  if (aRange.mExact.size() <= 1 && aRange.mIdeal.size() <= 1) {
+    LOG("  %s: { exact: [%s], ideal: [%s] }", aRange.mName,
+        (aRange.mExact.empty()
+             ? ""
+             : NS_ConvertUTF16toUTF8(*aRange.mExact.begin()).get()),
+        (aRange.mIdeal.empty()
+             ? ""
+             : NS_ConvertUTF16toUTF8(*aRange.mIdeal.begin()).get()));
+  } else {
+    LOG("  %s: { exact: [", aRange.mName);
+    for (auto& entry : aRange.mExact) {
+      LOG("      %s,", NS_ConvertUTF16toUTF8(entry).get());
+    }
+    LOG("    ], ideal: [");
+    for (auto& entry : aRange.mIdeal) {
+      LOG("      %s,", NS_ConvertUTF16toUTF8(entry).get());
+    }
+    LOG("    ]}");
+  }
+}
+
+template <typename T>
+static void LogConstraintRange(
+    const NormalizedConstraintSet::Range<T>& aRange) {
+  if (aRange.mIdeal.isSome()) {
+    LOG("  %s: { min: %d, max: %d, ideal: %d }", aRange.mName, aRange.mMin,
+        aRange.mMax, aRange.mIdeal.valueOr(0));
+  } else {
+    LOG("  %s: { min: %d, max: %d }", aRange.mName, aRange.mMin, aRange.mMax);
+  }
+}
+
+template <>
+void LogConstraintRange(const NormalizedConstraintSet::Range<double>& aRange) {
+  if (aRange.mIdeal.isSome()) {
+    LOG("  %s: { min: %f, max: %f, ideal: %f }", aRange.mName, aRange.mMin,
+        aRange.mMax, aRange.mIdeal.valueOr(0));
+  } else {
+    LOG("  %s: { min: %f, max: %f }", aRange.mName, aRange.mMin, aRange.mMax);
+  }
+}
+
+/* static */
+void MediaConstraintsHelper::LogConstraints(
+    const NormalizedConstraintSet& aConstraints) {
+  auto& c = aConstraints;
+  LOG("Constraints: {");
+  LOG("%s", [&]() {
+    LogConstraintRange(c.mWidth);
+    LogConstraintRange(c.mHeight);
+    LogConstraintRange(c.mFrameRate);
+    LogConstraintStringRange(c.mMediaSource);
+    LogConstraintStringRange(c.mFacingMode);
+    LogConstraintStringRange(c.mDeviceId);
+    LogConstraintStringRange(c.mGroupId);
+    LogConstraintRange(c.mEchoCancellation);
+    LogConstraintRange(c.mAutoGainControl);
+    LogConstraintRange(c.mNoiseSuppression);
+    LogConstraintRange(c.mChannelCount);
+    return "}";
+  }());
+}
+
+}  // namespace mozilla