summaryrefslogtreecommitdiffstats
path: root/dom/media/webrtc/MediaTrackConstraints.cpp
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
commit26a029d407be480d791972afb5975cf62c9360a6 (patch)
treef435a8308119effd964b339f76abb83a57c29483 /dom/media/webrtc/MediaTrackConstraints.cpp
parentInitial commit. (diff)
downloadfirefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz
firefox-26a029d407be480d791972afb5975cf62c9360a6.zip
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/media/webrtc/MediaTrackConstraints.cpp')
-rw-r--r--dom/media/webrtc/MediaTrackConstraints.cpp560
1 files changed, 560 insertions, 0 deletions
diff --git a/dom/media/webrtc/MediaTrackConstraints.cpp b/dom/media/webrtc/MediaTrackConstraints.cpp
new file mode 100644
index 0000000000..101cd0240a
--- /dev/null
+++ b/dom/media/webrtc/MediaTrackConstraints.cpp
@@ -0,0 +1,560 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "MediaTrackConstraints.h"
+
+#include <limits>
+#include <algorithm>
+#include <iterator>
+
+#include "MediaEngineSource.h"
+#include "mozilla/dom/MediaStreamTrackBinding.h"
+#include "mozilla/MediaManager.h"
+
+#ifdef MOZ_WEBRTC
+namespace mozilla {
+extern LazyLogModule gMediaManagerLog;
+}
+#else
+static mozilla::LazyLogModule gMediaManagerLog("MediaManager");
+#endif
+#define LOG(...) MOZ_LOG(gMediaManagerLog, LogLevel::Debug, (__VA_ARGS__))
+
+namespace mozilla {
+
+using dom::CallerType;
+using dom::ConstrainBooleanParameters;
+
+template <class ValueType>
+template <class ConstrainRange>
+void NormalizedConstraintSet::Range<ValueType>::SetFrom(
+ const ConstrainRange& aOther) {
+ if (aOther.mIdeal.WasPassed()) {
+ mIdeal.emplace(aOther.mIdeal.Value());
+ }
+ if (aOther.mExact.WasPassed()) {
+ mMin = aOther.mExact.Value();
+ mMax = aOther.mExact.Value();
+ } else {
+ if (aOther.mMin.WasPassed()) {
+ mMin = aOther.mMin.Value();
+ }
+ if (aOther.mMax.WasPassed()) {
+ mMax = aOther.mMax.Value();
+ }
+ }
+}
+
+// The Range code works surprisingly well for bool, except when averaging
+// ideals.
+template <>
+bool NormalizedConstraintSet::Range<bool>::Merge(const Range& aOther) {
+ if (!Intersects(aOther)) {
+ return false;
+ }
+ Intersect(aOther);
+
+ // To avoid "unsafe use of type 'bool'", we keep counter in mMergeDenominator
+ uint32_t counter = mMergeDenominator >> 16;
+ uint32_t denominator = mMergeDenominator & 0xffff;
+
+ if (aOther.mIdeal.isSome()) {
+ if (mIdeal.isNothing()) {
+ mIdeal.emplace(aOther.Get(false));
+ counter = aOther.Get(false);
+ denominator = 1;
+ } else {
+ if (!denominator) {
+ counter = Get(false);
+ denominator = 1;
+ }
+ counter += aOther.Get(false);
+ denominator++;
+ }
+ }
+ mMergeDenominator = ((counter & 0xffff) << 16) + (denominator & 0xffff);
+ return true;
+}
+
+template <>
+void NormalizedConstraintSet::Range<bool>::FinalizeMerge() {
+ if (mMergeDenominator) {
+ uint32_t counter = mMergeDenominator >> 16;
+ uint32_t denominator = mMergeDenominator & 0xffff;
+
+ *mIdeal = !!(counter / denominator);
+ mMergeDenominator = 0;
+ }
+}
+
+NormalizedConstraintSet::LongRange::LongRange(
+ LongPtrType aMemberPtr, const char* aName,
+ const dom::Optional<dom::OwningLongOrConstrainLongRange>& aOther,
+ bool advanced, nsTArray<MemberPtrType>* aList)
+ : Range<int32_t>((MemberPtrType)aMemberPtr, aName, 1 + INT32_MIN,
+ INT32_MAX, // +1 avoids Windows compiler bug
+ aList) {
+ if (!aOther.WasPassed()) {
+ return;
+ }
+ auto& other = aOther.Value();
+ if (other.IsLong()) {
+ if (advanced) {
+ mMin = mMax = other.GetAsLong();
+ } else {
+ mIdeal.emplace(other.GetAsLong());
+ }
+ } else {
+ SetFrom(other.GetAsConstrainLongRange());
+ }
+}
+
+NormalizedConstraintSet::LongLongRange::LongLongRange(
+ LongLongPtrType aMemberPtr, const char* aName, const long long& aOther,
+ nsTArray<MemberPtrType>* aList)
+ : Range<int64_t>((MemberPtrType)aMemberPtr, aName, 1 + INT64_MIN,
+ INT64_MAX, // +1 avoids Windows compiler bug
+ aList) {
+ mIdeal.emplace(aOther);
+}
+
+NormalizedConstraintSet::DoubleRange::DoubleRange(
+ DoublePtrType aMemberPtr, const char* aName,
+ const dom::Optional<dom::OwningDoubleOrConstrainDoubleRange>& aOther,
+ bool advanced, nsTArray<MemberPtrType>* aList)
+ : Range<double>((MemberPtrType)aMemberPtr, aName,
+ -std::numeric_limits<double>::infinity(),
+ std::numeric_limits<double>::infinity(), aList) {
+ if (!aOther.WasPassed()) {
+ return;
+ }
+ auto& other = aOther.Value();
+ if (other.IsDouble()) {
+ if (advanced) {
+ mMin = mMax = other.GetAsDouble();
+ } else {
+ mIdeal.emplace(other.GetAsDouble());
+ }
+ } else {
+ SetFrom(other.GetAsConstrainDoubleRange());
+ }
+}
+
+NormalizedConstraintSet::BooleanRange::BooleanRange(
+ BooleanPtrType aMemberPtr, const char* aName,
+ const dom::Optional<dom::OwningBooleanOrConstrainBooleanParameters>& aOther,
+ bool advanced, nsTArray<MemberPtrType>* aList)
+ : Range<bool>((MemberPtrType)aMemberPtr, aName, false, true, aList) {
+ if (!aOther.WasPassed()) {
+ return;
+ }
+ auto& other = aOther.Value();
+ if (other.IsBoolean()) {
+ if (advanced) {
+ mMin = mMax = other.GetAsBoolean();
+ } else {
+ mIdeal.emplace(other.GetAsBoolean());
+ }
+ } else {
+ auto& r = other.GetAsConstrainBooleanParameters();
+ if (r.mIdeal.WasPassed()) {
+ mIdeal.emplace(r.mIdeal.Value());
+ }
+ if (r.mExact.WasPassed()) {
+ mMin = r.mExact.Value();
+ mMax = r.mExact.Value();
+ }
+ }
+}
+
+NormalizedConstraintSet::StringRange::StringRange(
+ StringPtrType aMemberPtr, const char* aName,
+ const dom::Optional<
+ dom::OwningStringOrStringSequenceOrConstrainDOMStringParameters>&
+ aOther,
+ bool advanced, nsTArray<MemberPtrType>* aList)
+ : BaseRange((MemberPtrType)aMemberPtr, aName, aList) {
+ if (!aOther.WasPassed()) {
+ return;
+ }
+ auto& other = aOther.Value();
+ if (other.IsString()) {
+ if (advanced) {
+ mExact.insert(other.GetAsString());
+ } else {
+ mIdeal.insert(other.GetAsString());
+ }
+ } else if (other.IsStringSequence()) {
+ if (advanced) {
+ mExact.clear();
+ for (auto& str : other.GetAsStringSequence()) {
+ mExact.insert(str);
+ }
+ } else {
+ mIdeal.clear();
+ for (auto& str : other.GetAsStringSequence()) {
+ mIdeal.insert(str);
+ }
+ }
+ } else {
+ SetFrom(other.GetAsConstrainDOMStringParameters());
+ }
+}
+
+void NormalizedConstraintSet::StringRange::SetFrom(
+ const dom::ConstrainDOMStringParameters& aOther) {
+ if (aOther.mIdeal.WasPassed()) {
+ mIdeal.clear();
+ if (aOther.mIdeal.Value().IsString()) {
+ mIdeal.insert(aOther.mIdeal.Value().GetAsString());
+ } else {
+ for (auto& str : aOther.mIdeal.Value().GetAsStringSequence()) {
+ mIdeal.insert(str);
+ }
+ }
+ }
+ if (aOther.mExact.WasPassed()) {
+ mExact.clear();
+ if (aOther.mExact.Value().IsString()) {
+ mExact.insert(aOther.mExact.Value().GetAsString());
+ } else {
+ for (auto& str : aOther.mExact.Value().GetAsStringSequence()) {
+ mExact.insert(str);
+ }
+ }
+ }
+}
+
+auto NormalizedConstraintSet::StringRange::Clamp(const ValueType& n) const
+ -> ValueType {
+ if (mExact.empty()) {
+ return n;
+ }
+ ValueType result;
+ for (auto& entry : n) {
+ if (mExact.find(entry) != mExact.end()) {
+ result.insert(entry);
+ }
+ }
+ return result;
+}
+
+bool NormalizedConstraintSet::StringRange::Intersects(
+ const StringRange& aOther) const {
+ if (mExact.empty() || aOther.mExact.empty()) {
+ return true;
+ }
+
+ ValueType intersection;
+ set_intersection(mExact.begin(), mExact.end(), aOther.mExact.begin(),
+ aOther.mExact.end(),
+ std::inserter(intersection, intersection.begin()));
+ return !intersection.empty();
+}
+
+void NormalizedConstraintSet::StringRange::Intersect(
+ const StringRange& aOther) {
+ if (aOther.mExact.empty()) {
+ return;
+ }
+
+ ValueType intersection;
+ set_intersection(mExact.begin(), mExact.end(), aOther.mExact.begin(),
+ aOther.mExact.end(),
+ std::inserter(intersection, intersection.begin()));
+ mExact = intersection;
+}
+
+bool NormalizedConstraintSet::StringRange::Merge(const StringRange& aOther) {
+ if (!Intersects(aOther)) {
+ return false;
+ }
+ Intersect(aOther);
+
+ ValueType unioned;
+ set_union(mIdeal.begin(), mIdeal.end(), aOther.mIdeal.begin(),
+ aOther.mIdeal.end(), std::inserter(unioned, unioned.begin()));
+ mIdeal = unioned;
+ return true;
+}
+
+NormalizedConstraints::NormalizedConstraints(
+ const dom::MediaTrackConstraints& aOther, nsTArray<MemberPtrType>* aList)
+ : NormalizedConstraintSet(aOther, false, aList), mBadConstraint(nullptr) {
+ if (aOther.mAdvanced.WasPassed()) {
+ for (auto& entry : aOther.mAdvanced.Value()) {
+ mAdvanced.push_back(NormalizedConstraintSet(entry, true));
+ }
+ }
+}
+
+FlattenedConstraints::FlattenedConstraints(const NormalizedConstraints& aOther)
+ : NormalizedConstraintSet(aOther) {
+ for (auto& set : aOther.mAdvanced) {
+ // Must only apply compatible i.e. inherently non-overconstraining sets
+ // This rule is pretty much why this code is centralized here.
+ if (mWidth.Intersects(set.mWidth) && mHeight.Intersects(set.mHeight) &&
+ mFrameRate.Intersects(set.mFrameRate)) {
+ mWidth.Intersect(set.mWidth);
+ mHeight.Intersect(set.mHeight);
+ mFrameRate.Intersect(set.mFrameRate);
+ }
+ if (mEchoCancellation.Intersects(set.mEchoCancellation)) {
+ mEchoCancellation.Intersect(set.mEchoCancellation);
+ }
+ if (mNoiseSuppression.Intersects(set.mNoiseSuppression)) {
+ mNoiseSuppression.Intersect(set.mNoiseSuppression);
+ }
+ if (mAutoGainControl.Intersects(set.mAutoGainControl)) {
+ mAutoGainControl.Intersect(set.mAutoGainControl);
+ }
+ if (mChannelCount.Intersects(set.mChannelCount)) {
+ mChannelCount.Intersect(set.mChannelCount);
+ }
+ }
+}
+
+// MediaEngine helper
+//
+// The full algorithm for all devices.
+//
+// Fitness distance returned as integer math * 1000. Infinity = UINT32_MAX
+
+// First, all devices have a minimum distance based on their deviceId.
+// If you have no other constraints, use this one. Reused by all device types.
+
+/* static */
+bool MediaConstraintsHelper::SomeSettingsFit(
+ const NormalizedConstraints& aConstraints,
+ const nsTArray<RefPtr<LocalMediaDevice>>& aDevices) {
+ nsTArray<const NormalizedConstraintSet*> sets;
+ sets.AppendElement(&aConstraints);
+
+ MOZ_ASSERT(!aDevices.IsEmpty());
+ for (auto& device : aDevices) {
+ auto distance = device->GetBestFitnessDistance(sets, CallerType::NonSystem);
+ if (distance != UINT32_MAX) {
+ return true;
+ }
+ }
+ return false;
+}
+
+// Fitness distance returned as integer math * 1000. Infinity = UINT32_MAX
+
+/* static */
+uint32_t MediaConstraintsHelper::FitnessDistance(
+ const Maybe<nsString>& aN,
+ const NormalizedConstraintSet::StringRange& aParams) {
+ if (!aParams.mExact.empty() &&
+ (aN.isNothing() || aParams.mExact.find(*aN) == aParams.mExact.end())) {
+ return UINT32_MAX;
+ }
+ if (!aParams.mIdeal.empty() &&
+ (aN.isNothing() || aParams.mIdeal.find(*aN) == aParams.mIdeal.end())) {
+ return 1000;
+ }
+ return 0;
+}
+
+/* static */ const char* MediaConstraintsHelper::SelectSettings(
+ const NormalizedConstraints& aConstraints,
+ nsTArray<RefPtr<LocalMediaDevice>>& aDevices, CallerType aCallerType) {
+ auto& c = aConstraints;
+ LogConstraints(c);
+
+ // First apply top-level constraints.
+
+ // Stack constraintSets that pass, starting with the required one, because the
+ // whole stack must be re-satisfied each time a capability-set is ruled out
+ // (this avoids storing state or pushing algorithm into the lower-level code).
+ nsTArray<RefPtr<LocalMediaDevice>> unsatisfactory;
+ nsTArray<const NormalizedConstraintSet*> aggregateConstraints;
+ aggregateConstraints.AppendElement(&c);
+
+ std::multimap<uint32_t, RefPtr<LocalMediaDevice>> ordered;
+
+ for (uint32_t i = 0; i < aDevices.Length();) {
+ uint32_t distance =
+ aDevices[i]->GetBestFitnessDistance(aggregateConstraints, aCallerType);
+ if (distance == UINT32_MAX) {
+ unsatisfactory.AppendElement(std::move(aDevices[i]));
+ aDevices.RemoveElementAt(i);
+ } else {
+ ordered.insert(std::make_pair(distance, aDevices[i]));
+ ++i;
+ }
+ }
+ if (aDevices.IsEmpty()) {
+ return FindBadConstraint(c, unsatisfactory);
+ }
+
+ // Order devices by shortest distance
+ for (auto& ordinal : ordered) {
+ aDevices.RemoveElement(ordinal.second);
+ aDevices.AppendElement(ordinal.second);
+ }
+
+ // Then apply advanced constraints.
+
+ for (const auto& advanced : c.mAdvanced) {
+ aggregateConstraints.AppendElement(&advanced);
+ nsTArray<RefPtr<LocalMediaDevice>> rejects;
+ for (uint32_t j = 0; j < aDevices.Length();) {
+ uint32_t distance = aDevices[j]->GetBestFitnessDistance(
+ aggregateConstraints, aCallerType);
+ if (distance == UINT32_MAX) {
+ rejects.AppendElement(std::move(aDevices[j]));
+ aDevices.RemoveElementAt(j);
+ } else {
+ ++j;
+ }
+ }
+ if (aDevices.IsEmpty()) {
+ aDevices.AppendElements(std::move(rejects));
+ aggregateConstraints.RemoveLastElement();
+ }
+ }
+ return nullptr;
+}
+
+/* static */ const char* MediaConstraintsHelper::FindBadConstraint(
+ const NormalizedConstraints& aConstraints,
+ const nsTArray<RefPtr<LocalMediaDevice>>& aDevices) {
+ // The spec says to report a constraint that satisfies NONE
+ // of the sources. Unfortunately, this is a bit laborious to find out, and
+ // requires updating as new constraints are added!
+ auto& c = aConstraints;
+ dom::MediaTrackConstraints empty;
+
+ if (aDevices.IsEmpty() ||
+ !SomeSettingsFit(NormalizedConstraints(empty), aDevices)) {
+ return "";
+ }
+ {
+ NormalizedConstraints fresh(empty);
+ fresh.mDeviceId = c.mDeviceId;
+ if (!SomeSettingsFit(fresh, aDevices)) {
+ return "deviceId";
+ }
+ }
+ {
+ NormalizedConstraints fresh(empty);
+ fresh.mGroupId = c.mGroupId;
+ if (!SomeSettingsFit(fresh, aDevices)) {
+ return "groupId";
+ }
+ }
+ {
+ NormalizedConstraints fresh(empty);
+ fresh.mWidth = c.mWidth;
+ if (!SomeSettingsFit(fresh, aDevices)) {
+ return "width";
+ }
+ }
+ {
+ NormalizedConstraints fresh(empty);
+ fresh.mHeight = c.mHeight;
+ if (!SomeSettingsFit(fresh, aDevices)) {
+ return "height";
+ }
+ }
+ {
+ NormalizedConstraints fresh(empty);
+ fresh.mFrameRate = c.mFrameRate;
+ if (!SomeSettingsFit(fresh, aDevices)) {
+ return "frameRate";
+ }
+ }
+ {
+ NormalizedConstraints fresh(empty);
+ fresh.mFacingMode = c.mFacingMode;
+ if (!SomeSettingsFit(fresh, aDevices)) {
+ return "facingMode";
+ }
+ }
+ return "";
+}
+
+/* static */
+const char* MediaConstraintsHelper::FindBadConstraint(
+ const NormalizedConstraints& aConstraints,
+ const MediaDevice* aMediaDevice) {
+ NormalizedConstraints c(aConstraints);
+ NormalizedConstraints empty((dom::MediaTrackConstraints()));
+ c.mDeviceId = empty.mDeviceId;
+ c.mGroupId = empty.mGroupId;
+ AutoTArray<RefPtr<LocalMediaDevice>, 1> devices;
+ devices.EmplaceBack(
+ new LocalMediaDevice(aMediaDevice, u""_ns, u""_ns, u""_ns));
+ return FindBadConstraint(c, devices);
+}
+
+static void LogConstraintStringRange(
+ const NormalizedConstraintSet::StringRange& aRange) {
+ if (aRange.mExact.size() <= 1 && aRange.mIdeal.size() <= 1) {
+ LOG(" %s: { exact: [%s], ideal: [%s] }", aRange.mName,
+ (aRange.mExact.empty()
+ ? ""
+ : NS_ConvertUTF16toUTF8(*aRange.mExact.begin()).get()),
+ (aRange.mIdeal.empty()
+ ? ""
+ : NS_ConvertUTF16toUTF8(*aRange.mIdeal.begin()).get()));
+ } else {
+ LOG(" %s: { exact: [", aRange.mName);
+ for (auto& entry : aRange.mExact) {
+ LOG(" %s,", NS_ConvertUTF16toUTF8(entry).get());
+ }
+ LOG(" ], ideal: [");
+ for (auto& entry : aRange.mIdeal) {
+ LOG(" %s,", NS_ConvertUTF16toUTF8(entry).get());
+ }
+ LOG(" ]}");
+ }
+}
+
+template <typename T>
+static void LogConstraintRange(
+ const NormalizedConstraintSet::Range<T>& aRange) {
+ if (aRange.mIdeal.isSome()) {
+ LOG(" %s: { min: %d, max: %d, ideal: %d }", aRange.mName, aRange.mMin,
+ aRange.mMax, aRange.mIdeal.valueOr(0));
+ } else {
+ LOG(" %s: { min: %d, max: %d }", aRange.mName, aRange.mMin, aRange.mMax);
+ }
+}
+
+template <>
+void LogConstraintRange(const NormalizedConstraintSet::Range<double>& aRange) {
+ if (aRange.mIdeal.isSome()) {
+ LOG(" %s: { min: %f, max: %f, ideal: %f }", aRange.mName, aRange.mMin,
+ aRange.mMax, aRange.mIdeal.valueOr(0));
+ } else {
+ LOG(" %s: { min: %f, max: %f }", aRange.mName, aRange.mMin, aRange.mMax);
+ }
+}
+
+/* static */
+void MediaConstraintsHelper::LogConstraints(
+ const NormalizedConstraintSet& aConstraints) {
+ auto& c = aConstraints;
+ LOG("Constraints: {");
+ LOG("%s", [&]() {
+ LogConstraintRange(c.mWidth);
+ LogConstraintRange(c.mHeight);
+ LogConstraintRange(c.mFrameRate);
+ LogConstraintStringRange(c.mMediaSource);
+ LogConstraintStringRange(c.mFacingMode);
+ LogConstraintStringRange(c.mDeviceId);
+ LogConstraintStringRange(c.mGroupId);
+ LogConstraintRange(c.mEchoCancellation);
+ LogConstraintRange(c.mAutoGainControl);
+ LogConstraintRange(c.mNoiseSuppression);
+ LogConstraintRange(c.mChannelCount);
+ return "}";
+ }());
+}
+
+} // namespace mozilla