diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 19:33:14 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 19:33:14 +0000 |
commit | 36d22d82aa202bb199967e9512281e9a53db42c9 (patch) | |
tree | 105e8c98ddea1c1e4784a60a5a6410fa416be2de /dom/media/mediacapabilities | |
parent | Initial commit. (diff) | |
download | firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.tar.xz firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.zip |
Adding upstream version 115.7.0esr.upstream/115.7.0esr
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/media/mediacapabilities')
-rw-r--r-- | dom/media/mediacapabilities/BenchmarkStorageChild.cpp | 34 | ||||
-rw-r--r-- | dom/media/mediacapabilities/BenchmarkStorageChild.h | 28 | ||||
-rw-r--r-- | dom/media/mediacapabilities/BenchmarkStorageParent.cpp | 130 | ||||
-rw-r--r-- | dom/media/mediacapabilities/BenchmarkStorageParent.h | 43 | ||||
-rw-r--r-- | dom/media/mediacapabilities/DecoderBenchmark.cpp | 243 | ||||
-rw-r--r-- | dom/media/mediacapabilities/DecoderBenchmark.h | 77 | ||||
-rw-r--r-- | dom/media/mediacapabilities/KeyValueStorage.cpp | 234 | ||||
-rw-r--r-- | dom/media/mediacapabilities/KeyValueStorage.h | 48 | ||||
-rw-r--r-- | dom/media/mediacapabilities/MediaCapabilities.cpp | 658 | ||||
-rw-r--r-- | dom/media/mediacapabilities/MediaCapabilities.h | 104 | ||||
-rw-r--r-- | dom/media/mediacapabilities/PBenchmarkStorage.ipdl | 23 | ||||
-rw-r--r-- | dom/media/mediacapabilities/moz.build | 32 |
12 files changed, 1654 insertions, 0 deletions
diff --git a/dom/media/mediacapabilities/BenchmarkStorageChild.cpp b/dom/media/mediacapabilities/BenchmarkStorageChild.cpp new file mode 100644 index 0000000000..9482f5bffe --- /dev/null +++ b/dom/media/mediacapabilities/BenchmarkStorageChild.cpp @@ -0,0 +1,34 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "BenchmarkStorageChild.h" +#include "mozilla/dom/ContentChild.h" + +namespace mozilla { + +static PBenchmarkStorageChild* sChild = nullptr; + +/* static */ +PBenchmarkStorageChild* BenchmarkStorageChild::Instance() { + MOZ_ASSERT(NS_IsMainThread()); + if (!sChild) { + sChild = new BenchmarkStorageChild(); + PContentChild* contentChild = dom::ContentChild::GetSingleton(); + MOZ_ASSERT(contentChild); + contentChild->SendPBenchmarkStorageConstructor(); + } + MOZ_ASSERT(sChild); + return sChild; +} + +BenchmarkStorageChild::~BenchmarkStorageChild() { + MOZ_ASSERT(NS_IsMainThread()); + if (sChild == this) { + sChild = nullptr; + } +} + +} // namespace mozilla diff --git a/dom/media/mediacapabilities/BenchmarkStorageChild.h b/dom/media/mediacapabilities/BenchmarkStorageChild.h new file mode 100644 index 0000000000..c53571e13b --- /dev/null +++ b/dom/media/mediacapabilities/BenchmarkStorageChild.h @@ -0,0 +1,28 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef include_dom_media_mediacapabilities_BenchmarkStorageChild_h +#define include_dom_media_mediacapabilities_BenchmarkStorageChild_h + +#include "mozilla/PBenchmarkStorageChild.h" + +namespace mozilla { + +class BenchmarkStorageChild : public PBenchmarkStorageChild { + public: + /* Singleton class to avoid recreating the protocol every time we need access + * to the storage. */ + static PBenchmarkStorageChild* Instance(); + + ~BenchmarkStorageChild(); + + private: + BenchmarkStorageChild() = default; +}; + +} // namespace mozilla + +#endif // include_dom_media_mediacapabilities_BenchmarkStorageChild_h diff --git a/dom/media/mediacapabilities/BenchmarkStorageParent.cpp b/dom/media/mediacapabilities/BenchmarkStorageParent.cpp new file mode 100644 index 0000000000..64ae6ddd44 --- /dev/null +++ b/dom/media/mediacapabilities/BenchmarkStorageParent.cpp @@ -0,0 +1,130 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "BenchmarkStorageParent.h" +#include "KeyValueStorage.h" + +namespace mozilla { + +/* Moving average window size. */ +const int32_t AVG_WINDOW = 20; +/* Calculate the moving average for the new value aValue for the current window + * aWindow and the already existing average aAverage. When the method returns + * aAverage will contain the new average and aWindow will contain the new + * window.*/ +void BenchmarkStorageParent::MovingAverage(int32_t& aAverage, int32_t& aWindow, + const int32_t aValue) { + if (aWindow < AVG_WINDOW) { + aAverage = (aAverage * aWindow + aValue) / (aWindow + 1); + aWindow++; + return; + } + MOZ_ASSERT(aWindow == AVG_WINDOW); + aAverage = (aAverage - aAverage / aWindow) + (aValue / aWindow); +} + +/* In order to decrease the number of times the database is accessed when the + * moving average is stored or retrieved we use the same value to store _both_ + * the window and the average. The range of the average is limited since it is + * a percentage (0-100), and the range of the window is limited + * (1-20). Thus the number that is stored in the database is in the form + * (digits): WWAAA. For example, the value stored when an average(A) of 88 + * corresponds to a window(W) 7 is 7088. The average of 100 that corresponds to + * a window of 20 is 20100. The following methods are helpers to extract or + * construct the stored value according to the above. */ + +/* Stored value will be in the form WWAAA(19098). We need to extract the window + * (19) and the average score (98). The aValue will + * be parsed, the aWindow will contain the window (of the moving average) and + * the return value will contain the average itself. */ +int32_t BenchmarkStorageParent::ParseStoredValue(int32_t aValue, + int32_t& aWindow) { + MOZ_ASSERT(aValue > 999); + MOZ_ASSERT(aValue < 100000); + + int32_t score = aValue % 1000; + aWindow = (aValue / 1000) % 100; + return score; +} + +int32_t BenchmarkStorageParent::PrepareStoredValue(int32_t aScore, + int32_t aWindow) { + MOZ_ASSERT(aScore >= 0); + MOZ_ASSERT(aScore <= 100); + MOZ_ASSERT(aWindow > 0); + MOZ_ASSERT(aWindow < 21); + + return aWindow * 1000 + aScore; +} + +BenchmarkStorageParent::BenchmarkStorageParent() + : mStorage(new KeyValueStorage) {} + +IPCResult BenchmarkStorageParent::RecvPut(const nsCString& aDbName, + const nsCString& aKey, + const int32_t& aValue) { + // In order to calculate and store the new moving average, we need to get the + // stored value and window first, to calculate the new score and window, and + // then to store the new aggregated value. + mStorage->Get(aDbName, aKey) + ->Then( + GetCurrentSerialEventTarget(), __func__, + [storage = mStorage, aDbName, aKey, aValue](int32_t aResult) { + int32_t window = 0; + int32_t average = 0; + if (aResult >= 0) { + // The key found. + average = ParseStoredValue(aResult, window); + } + MovingAverage(average, window, aValue); + int32_t newValue = PrepareStoredValue(average, window); + // Avoid storing if the values are the same. This is an optimization + // to minimize the disk usage. + if (aResult != newValue) { + storage->Put(aDbName, aKey, newValue); + } + }, + [](nsresult rv) { /*do nothing*/ }); + + return IPC_OK(); +} + +IPCResult BenchmarkStorageParent::RecvGet(const nsCString& aDbName, + const nsCString& aKey, + GetResolver&& aResolve) { + mStorage->Get(aDbName, aKey) + ->Then( + GetCurrentSerialEventTarget(), __func__, + [aResolve](int32_t aResult) { + int32_t window = 0; // not used + aResolve(aResult < 0 ? -1 : ParseStoredValue(aResult, window)); + }, + [aResolve](nsresult rv) { aResolve(-1); }); + + return IPC_OK(); +} + +IPCResult BenchmarkStorageParent::RecvCheckVersion(const nsCString& aDbName, + int32_t aVersion) { + mStorage->Get(aDbName, "Version"_ns) + ->Then( + GetCurrentSerialEventTarget(), __func__, + [storage = mStorage, aDbName, aVersion](int32_t aResult) { + if (aVersion != aResult) { + storage->Clear(aDbName)->Then( + GetCurrentSerialEventTarget(), __func__, + [storage, aDbName, aVersion](bool) { + storage->Put(aDbName, "Version"_ns, aVersion); + }, + [](nsresult rv) { /*do nothing*/ }); + } + }, + [](nsresult rv) { /*do nothing*/ }); + + return IPC_OK(); +} + +}; // namespace mozilla diff --git a/dom/media/mediacapabilities/BenchmarkStorageParent.h b/dom/media/mediacapabilities/BenchmarkStorageParent.h new file mode 100644 index 0000000000..e9c8f9dd9b --- /dev/null +++ b/dom/media/mediacapabilities/BenchmarkStorageParent.h @@ -0,0 +1,43 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef include_dom_media_mediacapabilities_BenchmarkStorageParent_h +#define include_dom_media_mediacapabilities_BenchmarkStorageParent_h + +#include "mozilla/PBenchmarkStorageParent.h" + +namespace mozilla { +class KeyValueStorage; + +using mozilla::ipc::IPCResult; + +class BenchmarkStorageParent : public PBenchmarkStorageParent { + friend class PBenchmarkStorageParent; + + public: + BenchmarkStorageParent(); + + IPCResult RecvPut(const nsCString& aDbName, const nsCString& aKey, + const int32_t& aValue); + + IPCResult RecvGet(const nsCString& aDbName, const nsCString& aKey, + GetResolver&& aResolve); + + IPCResult RecvCheckVersion(const nsCString& aDbName, int32_t aVersion); + + /* Helper methods exposed here to be tested via gtest. */ + static void MovingAverage(int32_t& aAverage, int32_t& aWindow, + const int32_t aValue); + static int32_t ParseStoredValue(int32_t aValue, int32_t& aWindow); + static int32_t PrepareStoredValue(int32_t aScore, int32_t aWindow); + + private: + RefPtr<KeyValueStorage> mStorage; +}; + +} // namespace mozilla + +#endif // include_dom_media_mediacapabilities_BenchmarkStorageParent_h diff --git a/dom/media/mediacapabilities/DecoderBenchmark.cpp b/dom/media/mediacapabilities/DecoderBenchmark.cpp new file mode 100644 index 0000000000..b5c9a2a693 --- /dev/null +++ b/dom/media/mediacapabilities/DecoderBenchmark.cpp @@ -0,0 +1,243 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "DecoderBenchmark.h" +#include "mozilla/BenchmarkStorageChild.h" +#include "mozilla/media/MediaUtils.h" +#include "mozilla/StaticPrefs_media.h" + +namespace mozilla { + +void DecoderBenchmark::StoreScore(const nsACString& aDecoderName, + const nsACString& aKey, + RefPtr<FrameStatistics> aStats) { + FrameStatisticsData statsData = aStats->GetFrameStatisticsData(); + uint64_t totalFrames = FrameStatistics::GetTotalFrames(statsData); + uint64_t droppedFrames = FrameStatistics::GetDroppedFrames(statsData); + + MOZ_ASSERT(droppedFrames <= totalFrames); + MOZ_ASSERT(totalFrames >= mLastTotalFrames); + MOZ_ASSERT(droppedFrames >= mLastDroppedFrames); + + uint64_t diffTotalFrames = totalFrames - mLastTotalFrames; + uint64_t diffDroppedFrames = droppedFrames - mLastDroppedFrames; + + /* Update now in case the method returns at the if check bellow. */ + mLastTotalFrames = totalFrames; + mLastDroppedFrames = droppedFrames; + + /* A minimum number of 10 frames is required to store the score. */ + if (diffTotalFrames < 10) { + return; + } + + int32_t percentage = + 100 - 100 * float(diffDroppedFrames) / float(diffTotalFrames); + + MOZ_ASSERT(percentage >= 0); + + Put(aDecoderName, aKey, percentage); +} + +void DecoderBenchmark::Put(const nsACString& aDecoderName, + const nsACString& aKey, int32_t aValue) { + MOZ_ASSERT(NS_IsMainThread()); + const nsCString name(aDecoderName); + const nsCString key(aKey); + BenchmarkStorageChild::Instance()->SendPut(name, key, aValue); +} + +RefPtr<BenchmarkScorePromise> DecoderBenchmark::GetScore( + const nsACString& aDecoderName, const nsACString& aKey) { + if (NS_IsMainThread()) { + return Get(aDecoderName, aKey); + } + + RefPtr<DecoderBenchmark> self = this; + const nsCString decoderName(aDecoderName); + const nsCString key(aKey); + return InvokeAsync( + GetMainThreadSerialEventTarget(), __func__, + [self, decoderName, key] { return self->Get(decoderName, key); }); +} + +RefPtr<BenchmarkScorePromise> DecoderBenchmark::Get( + const nsACString& aDecoderName, const nsACString& aKey) { + MOZ_ASSERT(NS_IsMainThread()); + + const nsCString name(aDecoderName); + const nsCString key(aKey); + return BenchmarkStorageChild::Instance()->SendGet(name, key)->Then( + GetCurrentSerialEventTarget(), __func__, + [](int32_t aResult) { + return BenchmarkScorePromise::CreateAndResolve(aResult, __func__); + }, + [](ipc::ResponseRejectReason&&) { + return BenchmarkScorePromise::CreateAndReject(NS_ERROR_FAILURE, + __func__); + }); +} + +/* The key string consists of the video properties resolution, framerate, + * and bitdepth. There are various levels for each of them. The key is + * formated by the closest level, for example, a video with width=1920, + * height=1080, frameRate=24, and bitdepth=8bit will have the key: + * "ResolutionLevel5-FrameRateLevel1-8bit". */ + +#define NELEMS(x) (sizeof(x) / sizeof((x)[0])) + +/* Keep them sorted */ +const uint32_t PixelLevels[] = { + /* 256x144 =*/36864, + /* 426x240 =*/102240, + /* 640x360 =*/230400, + /* 854x480 =*/409920, + /* 1280x720 =*/921600, + /* 1920x1080 =*/2073600, + /* 2560x1440 =*/3686400, + /* 3840x2160 =*/8294400, +}; +const size_t PixelLevelsSize = NELEMS(PixelLevels); + +const uint32_t FrameRateLevels[] = { + 15, 24, 30, 50, 60, +}; +const size_t FrameRateLevelsSize = NELEMS(FrameRateLevels); + +/* static */ +nsCString KeyUtil::FindLevel(const uint32_t aLevels[], const size_t length, + uint32_t aValue) { + MOZ_ASSERT(aValue); + if (aValue <= aLevels[0]) { + return "Level0"_ns; + } + nsAutoCString level("Level"); + size_t lastIndex = length - 1; + if (aValue >= aLevels[lastIndex]) { + level.AppendInt(static_cast<uint32_t>(lastIndex)); + return std::move(level); + } + for (size_t i = 0; i < lastIndex; ++i) { + if (aValue >= aLevels[i + 1]) { + continue; + } + if (aValue - aLevels[i] < aLevels[i + 1] - aValue) { + level.AppendInt(static_cast<uint32_t>(i)); + return std::move(level); + } + level.AppendInt(static_cast<uint32_t>(i + 1)); + return std::move(level); + } + MOZ_CRASH("Array is not sorted"); + return ""_ns; +} + +/* static */ +nsCString KeyUtil::BitDepthToStr(uint8_t aBitDepth) { + switch (aBitDepth) { + case 8: // ColorDepth::COLOR_8 + return "-8bit"_ns; + case 10: // ColorDepth::COLOR_10 + case 12: // ColorDepth::COLOR_12 + case 16: // ColorDepth::COLOR_16 + return "-non8bit"_ns; + } + MOZ_ASSERT_UNREACHABLE("invalid color depth value"); + return ""_ns; +} + +/* static */ +nsCString KeyUtil::CreateKey(const DecoderBenchmarkInfo& aBenchInfo) { + nsAutoCString key("Resolution"); + key.Append(FindLevel(PixelLevels, PixelLevelsSize, + aBenchInfo.mWidth * aBenchInfo.mHeight)); + + key.Append("-FrameRate"); + key.Append( + FindLevel(FrameRateLevels, FrameRateLevelsSize, aBenchInfo.mFrameRate)); + + key.Append(BitDepthToStr(aBenchInfo.mBitDepth)); + + return std::move(key); +} + +void DecoderBenchmark::Store(const DecoderBenchmarkInfo& aBenchInfo, + RefPtr<FrameStatistics> aStats) { + if (!XRE_IsContentProcess()) { + NS_WARNING( + "Storing a benchmark is only allowed only from the content process."); + return; + } + StoreScore(aBenchInfo.mContentType, KeyUtil::CreateKey(aBenchInfo), aStats); +} + +/* static */ +RefPtr<BenchmarkScorePromise> DecoderBenchmark::Get( + const DecoderBenchmarkInfo& aBenchInfo) { + if (!XRE_IsContentProcess()) { + NS_WARNING( + "Getting a benchmark is only allowed only from the content process."); + return BenchmarkScorePromise::CreateAndReject(NS_ERROR_FAILURE, __func__); + } + // There is no need for any of the data members to query the database, thus + // it can be a static method. + auto bench = MakeRefPtr<DecoderBenchmark>(); + return bench->GetScore(aBenchInfo.mContentType, + KeyUtil::CreateKey(aBenchInfo)); +} + +static nsTHashMap<nsCStringHashKey, int32_t> DecoderVersionTable() { + nsTHashMap<nsCStringHashKey, int32_t> decoderVersionTable; + + /* + * For the decoders listed here, the benchmark version number will be checked. + * If the version number does not exist in the database or is different than + * the version number listed here, all the benchmark entries for this decoder + * will be erased. An example of assigning the version number `1` for AV1 + * decoder is: + * + * decoderVersionTable.InsertOrUpdate("video/av1"_ns, 1); + * + * For the decoders not listed here the `CheckVersion` method exits early, to + * avoid sending unecessary IPC messages. + */ + + return decoderVersionTable; +} + +/* static */ +void DecoderBenchmark::CheckVersion(const nsACString& aDecoderName) { + if (!XRE_IsContentProcess()) { + NS_WARNING( + "Checking version is only allowed only from the content process."); + return; + } + + if (!StaticPrefs::media_mediacapabilities_from_database()) { + return; + } + + nsCString name(aDecoderName); + int32_t version; + if (!DecoderVersionTable().Get(name, &version)) { + // A version is not set for that decoder ignore. + return; + } + + if (NS_IsMainThread()) { + BenchmarkStorageChild::Instance()->SendCheckVersion(name, version); + return; + } + + DebugOnly<nsresult> rv = + GetMainThreadSerialEventTarget()->Dispatch(NS_NewRunnableFunction( + "DecoderBenchmark::CheckVersion", [name, version]() { + BenchmarkStorageChild::Instance()->SendCheckVersion(name, version); + })); + MOZ_ASSERT(NS_SUCCEEDED(rv)); +} + +} // namespace mozilla diff --git a/dom/media/mediacapabilities/DecoderBenchmark.h b/dom/media/mediacapabilities/DecoderBenchmark.h new file mode 100644 index 0000000000..af730fcef3 --- /dev/null +++ b/dom/media/mediacapabilities/DecoderBenchmark.h @@ -0,0 +1,77 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_DECODER_BENCHMARK_H +#define MOZILLA_DECODER_BENCHMARK_H + +#include "FrameStatistics.h" +#include "mozilla/BenchmarkStorageChild.h" +#include "mozilla/KeyValueStorage.h" + +namespace mozilla { + +typedef KeyValueStorage::GetPromise BenchmarkScorePromise; + +struct DecoderBenchmarkInfo final { + const nsCString mContentType; + const int32_t mWidth; + const int32_t mHeight; + const int32_t mFrameRate; + const uint32_t mBitDepth; +}; + +class DecoderBenchmark final { + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DecoderBenchmark) + + public: + void Store(const DecoderBenchmarkInfo& aBenchInfo, + RefPtr<FrameStatistics> aStats); + + static RefPtr<BenchmarkScorePromise> Get( + const DecoderBenchmarkInfo& aBenchInfo); + + /* For the specific decoder, specified by aDecoderName, it compares the + * version number, from a static list of versions, to the version number + * found in the database. If those numbers are different all benchmark + * entries for that decoder are deleted. */ + static void CheckVersion(const nsACString& aDecoderName); + + private: + void StoreScore(const nsACString& aDecoderName, const nsACString& aKey, + RefPtr<FrameStatistics> aStats); + + RefPtr<BenchmarkScorePromise> GetScore(const nsACString& aDecoderName, + const nsACString& aKey); + + void Put(const nsACString& aDecoderName, const nsACString& aKey, + int32_t aValue); + + RefPtr<BenchmarkScorePromise> Get(const nsACString& aDecoderName, + const nsACString& aKey); + ~DecoderBenchmark() = default; + + // Keep the last TotalFrames and DroppedFrames from FrameStatistics. + // FrameStatistics keep an ever-increasing counter across the entire video and + // even when there are resolution changes. This code is called whenever there + // is a resolution change and we need to calculate the benchmark since the + // last call. + uint64_t mLastTotalFrames = 0; + uint64_t mLastDroppedFrames = 0; +}; + +class KeyUtil { + public: + static nsCString CreateKey(const DecoderBenchmarkInfo& aBenchInfo); + + private: + static nsCString BitDepthToStr(uint8_t aBitDepth); + static nsCString FindLevel(const uint32_t aLevels[], const size_t length, + uint32_t aValue); +}; + +} // namespace mozilla + +#endif // MOZILLA_DECODER_BENCHMARK_H diff --git a/dom/media/mediacapabilities/KeyValueStorage.cpp b/dom/media/mediacapabilities/KeyValueStorage.cpp new file mode 100644 index 0000000000..f0ac0aad7d --- /dev/null +++ b/dom/media/mediacapabilities/KeyValueStorage.cpp @@ -0,0 +1,234 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "KeyValueStorage.h" + +#include "nsAppDirectoryServiceDefs.h" +#include "nsDirectoryServiceUtils.h" +#include "nsServiceManagerUtils.h" +#include "nsVariant.h" + +namespace mozilla { + +class DatabaseCallback final : public nsIKeyValueDatabaseCallback { + public: + NS_DECL_ISUPPORTS + + explicit DatabaseCallback(RefPtr<nsIKeyValueDatabase>& aDatabase) + : mDatabase(aDatabase) {} + NS_IMETHOD Resolve(nsIKeyValueDatabase* aDatabase) override { + if (!aDatabase) { + mResultPromise.Reject(NS_ERROR_FAILURE, __func__); + } + mDatabase = aDatabase; + mResultPromise.Resolve(true, __func__); + return NS_OK; + } + NS_IMETHOD Reject(const nsACString&) override { + mResultPromise.Reject(NS_ERROR_FAILURE, __func__); + return NS_OK; + } + RefPtr<GenericPromise> Ensure() { return mResultPromise.Ensure(__func__); } + + protected: + ~DatabaseCallback() = default; + RefPtr<nsIKeyValueDatabase>& mDatabase; + MozPromiseHolder<GenericPromise> mResultPromise; +}; +NS_IMPL_ISUPPORTS(DatabaseCallback, nsIKeyValueDatabaseCallback); + +RefPtr<GenericPromise> KeyValueStorage::Init() { + MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(!mDatabaseName.IsEmpty()); + nsresult rv; + + nsCOMPtr<nsIFile> profileDir; + rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(profileDir)); + if (NS_WARN_IF(NS_FAILED(rv))) { + return GenericPromise::CreateAndReject(rv, __func__); + } + MOZ_ASSERT(profileDir); + + rv = profileDir->AppendNative("mediacapabilities"_ns); + if (NS_WARN_IF(NS_FAILED(rv))) { + return GenericPromise::CreateAndReject(rv, __func__); + } + + rv = profileDir->Create(nsIFile::DIRECTORY_TYPE, 0700); + if (rv != NS_ERROR_FILE_ALREADY_EXISTS && NS_WARN_IF(NS_FAILED(rv))) { + return GenericPromise::CreateAndReject(rv, __func__); + } + + nsCOMPtr<nsIKeyValueService> keyValueService = + do_GetService("@mozilla.org/key-value-service;1", &rv); + if (NS_WARN_IF(NS_FAILED(rv))) { + return GenericPromise::CreateAndReject(rv, __func__); + } + MOZ_ASSERT(keyValueService); + + auto callback = MakeRefPtr<DatabaseCallback>(mDatabase); + + nsString path; + profileDir->GetPath(path); + keyValueService->GetOrCreate(callback, NS_ConvertUTF16toUTF8(path), + mDatabaseName); + return callback->Ensure(); +} + +class VoidCallback final : public nsIKeyValueVoidCallback { + public: + NS_DECL_ISUPPORTS + + explicit VoidCallback(const RefPtr<KeyValueStorage>& aOwner) + : nsIKeyValueVoidCallback(), mOwner(aOwner) {} + + NS_IMETHOD Resolve() override { + mResultPromise.Resolve(true, __func__); + return NS_OK; + } + NS_IMETHOD Reject(const nsACString&) override { + mResultPromise.Reject(NS_ERROR_FAILURE, __func__); + return NS_OK; + } + RefPtr<GenericPromise> Ensure(const char* aMethodName) { + return mResultPromise.Ensure(aMethodName); + } + + protected: + ~VoidCallback() = default; + MozPromiseHolder<GenericPromise> mResultPromise; + RefPtr<KeyValueStorage> mOwner; +}; +NS_IMPL_ISUPPORTS(VoidCallback, nsIKeyValueVoidCallback); + +RefPtr<GenericPromise> KeyValueStorage::Put(const nsACString& aKey, + int32_t aValue) { + MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(mDatabase); + MOZ_ASSERT(!mDatabaseName.IsEmpty()); + + auto value = MakeRefPtr<nsVariant>(); + nsresult rv = value->SetAsInt32(aValue); + if (NS_WARN_IF(NS_FAILED(rv))) { + return GenericPromise::CreateAndReject(rv, __func__); + } + + auto callback = MakeRefPtr<VoidCallback>(this); + rv = mDatabase->Put(callback, aKey, value); + if (NS_WARN_IF(NS_FAILED(rv))) { + return GenericPromise::CreateAndReject(rv, __func__); + } + return callback->Ensure(__func__); +} + +RefPtr<GenericPromise> KeyValueStorage::Put(const nsACString& aName, + const nsACString& aKey, + int32_t aValue) { + if (!mDatabase || !mDatabaseName.Equals(aName)) { + mDatabaseName = aName; + RefPtr<KeyValueStorage> self = this; + const nsCString key(aKey); + return Init()->Then( + GetCurrentSerialEventTarget(), __func__, + [self, key, aValue](bool) { return self->Put(key, aValue); }, + [](nsresult rv) { + return GenericPromise::CreateAndReject(rv, __func__); + }); + } + return Put(aKey, aValue); +} + +class GetValueCallback final : public nsIKeyValueVariantCallback { + public: + NS_DECL_ISUPPORTS + + NS_IMETHOD Resolve(nsIVariant* aResult) override { + int32_t value = 0; + Unused << aResult->GetAsInt32(&value); + mResultPromise.Resolve(value, __func__); + return NS_OK; + } + NS_IMETHOD Reject(const nsACString&) override { + mResultPromise.Reject(NS_ERROR_FAILURE, __func__); + return NS_OK; + } + RefPtr<KeyValueStorage::GetPromise> Ensure() { + return mResultPromise.Ensure(__func__); + } + + protected: + ~GetValueCallback() = default; + + private: + MozPromiseHolder<KeyValueStorage::GetPromise> mResultPromise; +}; +NS_IMPL_ISUPPORTS(GetValueCallback, nsIKeyValueVariantCallback); + +RefPtr<KeyValueStorage::GetPromise> KeyValueStorage::Get( + const nsACString& aKey) { + MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(mDatabase); + MOZ_ASSERT(!mDatabaseName.IsEmpty()); + + RefPtr<nsVariant> defaultValue = new nsVariant; + nsresult rv = defaultValue->SetAsInt32(-1); + if (NS_WARN_IF(NS_FAILED(rv))) { + return KeyValueStorage::GetPromise::CreateAndReject(rv, __func__); + } + + auto callback = MakeRefPtr<GetValueCallback>(); + rv = mDatabase->Get(callback, aKey, defaultValue); + if (NS_WARN_IF(NS_FAILED(rv))) { + return KeyValueStorage::GetPromise::CreateAndReject(rv, __func__); + } + return callback->Ensure(); +} + +RefPtr<KeyValueStorage::GetPromise> KeyValueStorage::Get( + const nsACString& aName, const nsACString& aKey) { + if (!mDatabase || !mDatabaseName.Equals(aName)) { + mDatabaseName = aName; + RefPtr<KeyValueStorage> self = this; + const nsCString key(aKey); + return Init()->Then( + GetCurrentSerialEventTarget(), __func__, + [self, key](bool) { return self->Get(key); }, + [](nsresult rv) { + return KeyValueStorage::GetPromise::CreateAndReject(rv, __func__); + }); + } + return Get(aKey); +} + +RefPtr<GenericPromise> KeyValueStorage::Clear() { + MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(mDatabase); + MOZ_ASSERT(!mDatabaseName.IsEmpty()); + + auto callback = MakeRefPtr<VoidCallback>(this); + nsresult rv = mDatabase->Clear(callback); + if (NS_WARN_IF(NS_FAILED(rv))) { + return GenericPromise::CreateAndReject(rv, __func__); + } + return callback->Ensure(__func__); +} + +RefPtr<GenericPromise> KeyValueStorage::Clear(const nsACString& aName) { + if (!mDatabase || !mDatabaseName.Equals(aName)) { + mDatabaseName = aName; + RefPtr<KeyValueStorage> self = this; + return Init()->Then( + GetCurrentSerialEventTarget(), __func__, + [self](bool) { return self->Clear(); }, + [](nsresult rv) { + return GenericPromise::CreateAndReject(rv, __func__); + }); + } + return Clear(); +} + +} // namespace mozilla diff --git a/dom/media/mediacapabilities/KeyValueStorage.h b/dom/media/mediacapabilities/KeyValueStorage.h new file mode 100644 index 0000000000..60d72a6c6d --- /dev/null +++ b/dom/media/mediacapabilities/KeyValueStorage.h @@ -0,0 +1,48 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_KEY_VALUE_STORAGE_H +#define MOZILLA_KEY_VALUE_STORAGE_H + +#include "mozilla/MozPromise.h" +#include "nsIKeyValue.h" + +namespace mozilla { + +/* A wrapper class around kv store service, which allows storing a pair of key + * value permanently. The class must be used from the parent process, where + * there is no sandbox because it requires access to the directory that the + * database is located. */ +class KeyValueStorage final { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(KeyValueStorage) + + /* Store permanently the value in the Key. */ + RefPtr<GenericPromise> Put(const nsACString& aName, const nsACString& aKey, + int32_t aValue); + /* Get the value stored in the aKey. If the aKey does not exist the promise is + * resolved with the value -1. */ + typedef MozPromise<int32_t, nsresult, true> GetPromise; + RefPtr<GetPromise> Get(const nsACString& aName, const nsACString& aKey); + + /* Clear all the key/value pairs from the aName database. */ + RefPtr<GenericPromise> Clear(const nsACString& aName); + + private: + /* Create, if doesn't exist, and initialize the database with a given name. */ + RefPtr<GenericPromise> Init(); + RefPtr<GenericPromise> Put(const nsACString& aKey, int32_t aValue); + RefPtr<GetPromise> Get(const nsACString& aKey); + RefPtr<GenericPromise> Clear(); + ~KeyValueStorage() = default; + + RefPtr<nsIKeyValueDatabase> mDatabase; + nsCString mDatabaseName; +}; + +} // namespace mozilla + +#endif // MOZILLA_KEY_VALUE_STORAGE_H diff --git a/dom/media/mediacapabilities/MediaCapabilities.cpp b/dom/media/mediacapabilities/MediaCapabilities.cpp new file mode 100644 index 0000000000..904e9f4f54 --- /dev/null +++ b/dom/media/mediacapabilities/MediaCapabilities.cpp @@ -0,0 +1,658 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MediaCapabilities.h" + +#include <inttypes.h> + +#include <utility> + +#include "AllocationPolicy.h" +#include "Benchmark.h" +#include "DecoderBenchmark.h" +#include "DecoderTraits.h" +#include "MediaInfo.h" +#include "MediaRecorder.h" +#include "PDMFactory.h" +#include "VPXDecoder.h" +#include "mozilla/ClearOnShutdown.h" +#include "mozilla/SchedulerGroup.h" +#include "mozilla/StaticPrefs_media.h" +#include "mozilla/TaskQueue.h" +#include "mozilla/dom/Document.h" +#include "mozilla/dom/DOMMozPromiseRequestHolder.h" +#include "mozilla/dom/MediaCapabilitiesBinding.h" +#include "mozilla/dom/MediaSource.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/WorkerPrivate.h" +#include "mozilla/dom/WorkerRef.h" +#include "mozilla/layers/KnowsCompositor.h" +#include "nsContentUtils.h" +#include "WindowRenderer.h" + +static mozilla::LazyLogModule sMediaCapabilitiesLog("MediaCapabilities"); + +#define LOG(msg, ...) \ + DDMOZ_LOG(sMediaCapabilitiesLog, LogLevel::Debug, msg, ##__VA_ARGS__) + +namespace mozilla::dom { + +static nsCString VideoConfigurationToStr(const VideoConfiguration* aConfig) { + if (!aConfig) { + return nsCString(); + } + + nsCString hdrMetaType( + aConfig->mHdrMetadataType.WasPassed() + ? HdrMetadataTypeValues::GetString(aConfig->mHdrMetadataType.Value()) + : "?"); + + nsCString colorGamut( + aConfig->mColorGamut.WasPassed() + ? ColorGamutValues::GetString(aConfig->mColorGamut.Value()) + : "?"); + + nsCString transferFunction(aConfig->mTransferFunction.WasPassed() + ? TransferFunctionValues::GetString( + aConfig->mTransferFunction.Value()) + : "?"); + + auto str = nsPrintfCString( + "[contentType:%s width:%d height:%d bitrate:%" PRIu64 + " framerate:%lf hasAlphaChannel:%s hdrMetadataType:%s colorGamut:%s " + "transferFunction:%s scalabilityMode:%s]", + NS_ConvertUTF16toUTF8(aConfig->mContentType).get(), aConfig->mWidth, + aConfig->mHeight, aConfig->mBitrate, aConfig->mFramerate, + aConfig->mHasAlphaChannel.WasPassed() + ? aConfig->mHasAlphaChannel.Value() ? "true" : "false" + : "?", + hdrMetaType.get(), colorGamut.get(), transferFunction.get(), + aConfig->mScalabilityMode.WasPassed() + ? NS_ConvertUTF16toUTF8(aConfig->mScalabilityMode.Value()).get() + : "?"); + return std::move(str); +} + +static nsCString AudioConfigurationToStr(const AudioConfiguration* aConfig) { + if (!aConfig) { + return nsCString(); + } + auto str = nsPrintfCString( + "[contentType:%s channels:%s bitrate:%" PRIu64 " samplerate:%d]", + NS_ConvertUTF16toUTF8(aConfig->mContentType).get(), + aConfig->mChannels.WasPassed() + ? NS_ConvertUTF16toUTF8(aConfig->mChannels.Value()).get() + : "?", + aConfig->mBitrate.WasPassed() ? aConfig->mBitrate.Value() : 0, + aConfig->mSamplerate.WasPassed() ? aConfig->mSamplerate.Value() : 0); + return std::move(str); +} + +static nsCString MediaCapabilitiesInfoToStr( + const MediaCapabilitiesInfo* aInfo) { + if (!aInfo) { + return nsCString(); + } + auto str = nsPrintfCString("[supported:%s smooth:%s powerEfficient:%s]", + aInfo->Supported() ? "true" : "false", + aInfo->Smooth() ? "true" : "false", + aInfo->PowerEfficient() ? "true" : "false"); + return std::move(str); +} + +static nsCString MediaDecodingConfigurationToStr( + const MediaDecodingConfiguration& aConfig) { + nsCString str; + str += "["_ns; + if (aConfig.mVideo.WasPassed()) { + str += "video:"_ns + VideoConfigurationToStr(&aConfig.mVideo.Value()); + if (aConfig.mAudio.WasPassed()) { + str += " "_ns; + } + } + if (aConfig.mAudio.WasPassed()) { + str += "audio:"_ns + AudioConfigurationToStr(&aConfig.mAudio.Value()); + } + str += "]"_ns; + return str; +} + +MediaCapabilities::MediaCapabilities(nsIGlobalObject* aParent) + : mParent(aParent) {} + +already_AddRefed<Promise> MediaCapabilities::DecodingInfo( + const MediaDecodingConfiguration& aConfiguration, ErrorResult& aRv) { + RefPtr<Promise> promise = Promise::Create(mParent, aRv); + if (aRv.Failed()) { + return nullptr; + } + + // If configuration is not a valid MediaConfiguration, return a Promise + // rejected with a TypeError. + if (!aConfiguration.mVideo.WasPassed() && + !aConfiguration.mAudio.WasPassed()) { + aRv.ThrowTypeError<MSG_MISSING_REQUIRED_DICTIONARY_MEMBER>( + "'audio' or 'video' member of argument of " + "MediaCapabilities.decodingInfo"); + return nullptr; + } + + LOG("Processing %s", MediaDecodingConfigurationToStr(aConfiguration).get()); + + bool supported = true; + Maybe<MediaContainerType> videoContainer; + Maybe<MediaContainerType> audioContainer; + + // If configuration.video is present and is not a valid video configuration, + // return a Promise rejected with a TypeError. + if (aConfiguration.mVideo.WasPassed()) { + videoContainer = CheckVideoConfiguration(aConfiguration.mVideo.Value()); + if (!videoContainer) { + aRv.ThrowTypeError<MSG_INVALID_MEDIA_VIDEO_CONFIGURATION>(); + return nullptr; + } + + // We have a video configuration and it is valid. Check if it is supported. + supported &= + aConfiguration.mType == MediaDecodingType::File + ? CheckTypeForFile(aConfiguration.mVideo.Value().mContentType) + : CheckTypeForMediaSource( + aConfiguration.mVideo.Value().mContentType); + } + if (aConfiguration.mAudio.WasPassed()) { + audioContainer = CheckAudioConfiguration(aConfiguration.mAudio.Value()); + if (!audioContainer) { + aRv.ThrowTypeError<MSG_INVALID_MEDIA_AUDIO_CONFIGURATION>(); + return nullptr; + } + // We have an audio configuration and it is valid. Check if it is supported. + supported &= + aConfiguration.mType == MediaDecodingType::File + ? CheckTypeForFile(aConfiguration.mAudio.Value().mContentType) + : CheckTypeForMediaSource( + aConfiguration.mAudio.Value().mContentType); + } + + if (!supported) { + auto info = MakeUnique<MediaCapabilitiesInfo>( + false /* supported */, false /* smooth */, false /* power efficient */); + LOG("%s -> %s", MediaDecodingConfigurationToStr(aConfiguration).get(), + MediaCapabilitiesInfoToStr(info.get()).get()); + promise->MaybeResolve(std::move(info)); + return promise.forget(); + } + + nsTArray<UniquePtr<TrackInfo>> tracks; + if (aConfiguration.mVideo.WasPassed()) { + MOZ_ASSERT(videoContainer.isSome(), "configuration is valid and supported"); + auto videoTracks = DecoderTraits::GetTracksInfo(*videoContainer); + // If the MIME type does not imply a codec, the string MUST + // also have one and only one parameter that is named codecs with a value + // describing a single media codec. Otherwise, it MUST contain no + // parameters. + if (videoTracks.Length() != 1) { + promise->MaybeRejectWithTypeError<MSG_NO_CODECS_PARAMETER>( + videoContainer->OriginalString()); + return promise.forget(); + } + MOZ_DIAGNOSTIC_ASSERT(videoTracks.ElementAt(0), + "must contain a valid trackinfo"); + // If the type refers to an audio codec, reject now. + if (videoTracks[0]->GetType() != TrackInfo::kVideoTrack) { + promise + ->MaybeRejectWithTypeError<MSG_INVALID_MEDIA_VIDEO_CONFIGURATION>(); + return promise.forget(); + } + tracks.AppendElements(std::move(videoTracks)); + } + if (aConfiguration.mAudio.WasPassed()) { + MOZ_ASSERT(audioContainer.isSome(), "configuration is valid and supported"); + auto audioTracks = DecoderTraits::GetTracksInfo(*audioContainer); + // If the MIME type does not imply a codec, the string MUST + // also have one and only one parameter that is named codecs with a value + // describing a single media codec. Otherwise, it MUST contain no + // parameters. + if (audioTracks.Length() != 1) { + promise->MaybeRejectWithTypeError<MSG_NO_CODECS_PARAMETER>( + audioContainer->OriginalString()); + return promise.forget(); + } + MOZ_DIAGNOSTIC_ASSERT(audioTracks.ElementAt(0), + "must contain a valid trackinfo"); + // If the type refers to a video codec, reject now. + if (audioTracks[0]->GetType() != TrackInfo::kAudioTrack) { + promise + ->MaybeRejectWithTypeError<MSG_INVALID_MEDIA_AUDIO_CONFIGURATION>(); + return promise.forget(); + } + tracks.AppendElements(std::move(audioTracks)); + } + + using CapabilitiesPromise = MozPromise<MediaCapabilitiesInfo, MediaResult, + /* IsExclusive = */ true>; + nsTArray<RefPtr<CapabilitiesPromise>> promises; + + RefPtr<TaskQueue> taskQueue = + TaskQueue::Create(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER), + "MediaCapabilities::TaskQueue"); + for (auto&& config : tracks) { + TrackInfo::TrackType type = + config->IsVideo() ? TrackInfo::kVideoTrack : TrackInfo::kAudioTrack; + + MOZ_ASSERT(type == TrackInfo::kAudioTrack || + videoContainer->ExtendedType().GetFramerate().isSome(), + "framerate is a required member of VideoConfiguration"); + + if (type == TrackInfo::kAudioTrack) { + // There's no need to create an audio decoder has we only want to know if + // such codec is supported. We do need to call the PDMFactory::Supports + // API outside the main thread to get accurate results. + promises.AppendElement( + InvokeAsync(taskQueue, __func__, [config = std::move(config)]() { + RefPtr<PDMFactory> pdm = new PDMFactory(); + SupportDecoderParams params{*config}; + if (pdm->Supports(params, nullptr /* decoder doctor */) == + media::DecodeSupport::Unsupported) { + return CapabilitiesPromise::CreateAndReject(NS_ERROR_FAILURE, + __func__); + } + return CapabilitiesPromise::CreateAndResolve( + MediaCapabilitiesInfo(true /* supported */, true /* smooth */, + true /* power efficient */), + __func__); + })); + continue; + } + + // On Windows, the MediaDataDecoder expects to be created on a thread + // supporting MTA, which the main thread doesn't. So we use our task queue + // to create such decoder and perform initialization. + + RefPtr<layers::KnowsCompositor> compositor = GetCompositor(); + float frameRate = + static_cast<float>(videoContainer->ExtendedType().GetFramerate().ref()); + const bool shouldResistFingerprinting = + mParent->ShouldResistFingerprinting(RFPTarget::Unknown); + + // clang-format off + promises.AppendElement(InvokeAsync( + taskQueue, __func__, + [taskQueue, frameRate, shouldResistFingerprinting, compositor, + config = std::move(config)]() mutable -> RefPtr<CapabilitiesPromise> { + // MediaDataDecoder keeps a reference to the config object, so we must + // keep it alive until the decoder has been shutdown. + static Atomic<uint32_t> sTrackingIdCounter(0); + TrackingId trackingId(TrackingId::Source::MediaCapabilities, + sTrackingIdCounter++, + TrackingId::TrackAcrossProcesses::Yes); + CreateDecoderParams params{ + *config, compositor, + CreateDecoderParams::VideoFrameRate(frameRate), + TrackInfo::kVideoTrack, Some(std::move(trackingId))}; + // We want to ensure that all decoder's queries are occurring only + // once at a time as it can quickly exhaust the system resources + // otherwise. + static RefPtr<AllocPolicy> sVideoAllocPolicy = [&taskQueue]() { + SchedulerGroup::Dispatch( + TaskCategory::Other, + NS_NewRunnableFunction( + "MediaCapabilities::AllocPolicy:Video", []() { + ClearOnShutdown(&sVideoAllocPolicy, + ShutdownPhase::XPCOMShutdownThreads); + })); + return new SingleAllocPolicy(TrackInfo::TrackType::kVideoTrack, + taskQueue); + }(); + return AllocationWrapper::CreateDecoder(params, sVideoAllocPolicy) + ->Then( + taskQueue, __func__, + [taskQueue, frameRate, shouldResistFingerprinting, + config = std::move(config)]( + AllocationWrapper::AllocateDecoderPromise:: + ResolveOrRejectValue&& aValue) mutable { + if (aValue.IsReject()) { + return CapabilitiesPromise::CreateAndReject( + std::move(aValue.RejectValue()), __func__); + } + RefPtr<MediaDataDecoder> decoder = + std::move(aValue.ResolveValue()); + // We now query the decoder to determine if it's power + // efficient. + RefPtr<CapabilitiesPromise> p = decoder->Init()->Then( + taskQueue, __func__, + [taskQueue, decoder, frameRate, + shouldResistFingerprinting, + config = std::move(config)]( + MediaDataDecoder::InitPromise:: + ResolveOrRejectValue&& aValue) mutable { + RefPtr<CapabilitiesPromise> p; + if (aValue.IsReject()) { + p = CapabilitiesPromise::CreateAndReject( + std::move(aValue.RejectValue()), __func__); + } else if (shouldResistFingerprinting) { + p = CapabilitiesPromise::CreateAndResolve( + MediaCapabilitiesInfo(true /* supported */, + true /* smooth */, false /* power efficient */), + __func__); + } else { + MOZ_ASSERT(config->IsVideo()); + if (StaticPrefs::media_mediacapabilities_from_database()) { + nsAutoCString reason; + bool powerEfficient = + decoder->IsHardwareAccelerated(reason); + + int32_t videoFrameRate = std::clamp<int32_t>(frameRate, 1, INT32_MAX); + + DecoderBenchmarkInfo benchmarkInfo{ + config->mMimeType, + config->GetAsVideoInfo()->mImage.width, + config->GetAsVideoInfo()->mImage.height, + videoFrameRate, 8}; + + p = DecoderBenchmark::Get(benchmarkInfo)->Then( + GetMainThreadSerialEventTarget(), + __func__, + [powerEfficient](int32_t score) { + // score < 0 means no entry found. + bool smooth = score < 0 || score > + StaticPrefs:: + media_mediacapabilities_drop_threshold(); + return CapabilitiesPromise:: + CreateAndResolve( + MediaCapabilitiesInfo( + true, smooth, + powerEfficient), + __func__); + }, + [](nsresult rv) { + return CapabilitiesPromise:: + CreateAndReject(rv, __func__); + }); + } else if (config->GetAsVideoInfo()->mImage.height < 480) { + // Assume that we can do stuff at 480p or less in + // a power efficient manner and smoothly. If + // greater than 480p we assume that if the video + // decoding is hardware accelerated it will be + // smooth and power efficient, otherwise we use + // the benchmark to estimate + p = CapabilitiesPromise::CreateAndResolve( + MediaCapabilitiesInfo(true, true, true), + __func__); + } else { + nsAutoCString reason; + bool smooth = true; + bool powerEfficient = + decoder->IsHardwareAccelerated(reason); + if (!powerEfficient && + VPXDecoder::IsVP9(config->mMimeType)) { + smooth = VP9Benchmark::IsVP9DecodeFast( + true /* default */); + uint32_t fps = + VP9Benchmark::MediaBenchmarkVp9Fps(); + if (!smooth && fps > 0) { + // The VP9 estimizer decode a 1280x720 video. + // Let's adjust the result for the resolution + // and frame rate of what we actually want. If + // the result is twice that we need we assume + // it will be smooth. + const auto& videoConfig = + *config->GetAsVideoInfo(); + double needed = ((1280.0 * 720.0) / + (videoConfig.mImage.width * + videoConfig.mImage.height) * + fps) / + frameRate; + smooth = needed > 2; + } + } + + p = CapabilitiesPromise::CreateAndResolve( + MediaCapabilitiesInfo(true /* supported */, + smooth, powerEfficient), + __func__); + } + } + MOZ_ASSERT(p.get(), "the promise has been created"); + // Let's keep alive the decoder and the config object + // until the decoder has shutdown. + decoder->Shutdown()->Then( + taskQueue, __func__, + [taskQueue, decoder, config = std::move(config)]( + const ShutdownPromise::ResolveOrRejectValue& + aValue) {}); + return p; + }); + return p; + }); + })); + // clang-format on + } + + auto holder = MakeRefPtr< + DOMMozPromiseRequestHolder<CapabilitiesPromise::AllPromiseType>>(mParent); + RefPtr<nsISerialEventTarget> targetThread; + RefPtr<StrongWorkerRef> workerRef; + + if (NS_IsMainThread()) { + targetThread = mParent->AbstractMainThreadFor(TaskCategory::Other); + } else { + WorkerPrivate* wp = GetCurrentThreadWorkerPrivate(); + MOZ_ASSERT(wp, "Must be called from a worker thread"); + targetThread = wp->HybridEventTarget(); + workerRef = StrongWorkerRef::Create( + wp, "MediaCapabilities", [holder, targetThread]() { + MOZ_ASSERT(targetThread->IsOnCurrentThread()); + holder->DisconnectIfExists(); + }); + if (NS_WARN_IF(!workerRef)) { + // The worker is shutting down. + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + } + + MOZ_ASSERT(targetThread); + + // this is only captured for use with the LOG macro. + RefPtr<MediaCapabilities> self = this; + + CapabilitiesPromise::All(targetThread, promises) + ->Then(targetThread, __func__, + [promise, tracks = std::move(tracks), workerRef, holder, + aConfiguration, self, + this](CapabilitiesPromise::AllPromiseType::ResolveOrRejectValue&& + aValue) { + holder->Complete(); + if (aValue.IsReject()) { + auto info = MakeUnique<MediaCapabilitiesInfo>( + false /* supported */, false /* smooth */, + false /* power efficient */); + LOG("%s -> %s", + MediaDecodingConfigurationToStr(aConfiguration).get(), + MediaCapabilitiesInfoToStr(info.get()).get()); + promise->MaybeResolve(std::move(info)); + return; + } + bool powerEfficient = true; + bool smooth = true; + for (auto&& capability : aValue.ResolveValue()) { + smooth &= capability.Smooth(); + powerEfficient &= capability.PowerEfficient(); + } + auto info = MakeUnique<MediaCapabilitiesInfo>( + true /* supported */, smooth, powerEfficient); + LOG("%s -> %s", + MediaDecodingConfigurationToStr(aConfiguration).get(), + MediaCapabilitiesInfoToStr(info.get()).get()); + promise->MaybeResolve(std::move(info)); + }) + ->Track(*holder); + + return promise.forget(); +} + +already_AddRefed<Promise> MediaCapabilities::EncodingInfo( + const MediaEncodingConfiguration& aConfiguration, ErrorResult& aRv) { + RefPtr<Promise> promise = Promise::Create(mParent, aRv); + if (aRv.Failed()) { + return nullptr; + } + + // If configuration is not a valid MediaConfiguration, return a Promise + // rejected with a TypeError. + if (!aConfiguration.mVideo.WasPassed() && + !aConfiguration.mAudio.WasPassed()) { + aRv.ThrowTypeError<MSG_MISSING_REQUIRED_DICTIONARY_MEMBER>( + "'audio' or 'video' member of argument of " + "MediaCapabilities.encodingInfo"); + return nullptr; + } + + bool supported = true; + + // If configuration.video is present and is not a valid video configuration, + // return a Promise rejected with a TypeError. + if (aConfiguration.mVideo.WasPassed()) { + if (!CheckVideoConfiguration(aConfiguration.mVideo.Value())) { + aRv.ThrowTypeError<MSG_INVALID_MEDIA_VIDEO_CONFIGURATION>(); + return nullptr; + } + // We have a video configuration and it is valid. Check if it is supported. + supported &= + CheckTypeForEncoder(aConfiguration.mVideo.Value().mContentType); + } + if (aConfiguration.mAudio.WasPassed()) { + if (!CheckAudioConfiguration(aConfiguration.mAudio.Value())) { + aRv.ThrowTypeError<MSG_INVALID_MEDIA_AUDIO_CONFIGURATION>(); + return nullptr; + } + // We have an audio configuration and it is valid. Check if it is supported. + supported &= + CheckTypeForEncoder(aConfiguration.mAudio.Value().mContentType); + } + + auto info = MakeUnique<MediaCapabilitiesInfo>(supported, supported, false); + promise->MaybeResolve(std::move(info)); + + return promise.forget(); +} + +Maybe<MediaContainerType> MediaCapabilities::CheckVideoConfiguration( + const VideoConfiguration& aConfig) const { + Maybe<MediaExtendedMIMEType> container = MakeMediaExtendedMIMEType(aConfig); + if (!container) { + return Nothing(); + } + // A valid video MIME type is a string that is a valid media MIME type and for + // which the type per [RFC7231] is either video or application. + if (!container->Type().HasVideoMajorType() && + !container->Type().HasApplicationMajorType()) { + return Nothing(); + } + + // If the MIME type does not imply a codec, the string MUST also have one and + // only one parameter that is named codecs with a value describing a single + // media codec. Otherwise, it MUST contain no parameters. + // TODO (nsIMOMEHeaderParam doesn't provide backend to count number of + // parameters) + + return Some(MediaContainerType(std::move(*container))); +} + +Maybe<MediaContainerType> MediaCapabilities::CheckAudioConfiguration( + const AudioConfiguration& aConfig) const { + Maybe<MediaExtendedMIMEType> container = MakeMediaExtendedMIMEType(aConfig); + if (!container) { + return Nothing(); + } + // A valid audio MIME type is a string that is valid media MIME type and for + // which the type per [RFC7231] is either audio or application. + if (!container->Type().HasAudioMajorType() && + !container->Type().HasApplicationMajorType()) { + return Nothing(); + } + + // If the MIME type does not imply a codec, the string MUST also have one and + // only one parameter that is named codecs with a value describing a single + // media codec. Otherwise, it MUST contain no parameters. + // TODO (nsIMOMEHeaderParam doesn't provide backend to count number of + // parameters) + + return Some(MediaContainerType(std::move(*container))); +} + +bool MediaCapabilities::CheckTypeForMediaSource(const nsAString& aType) { + IgnoredErrorResult rv; + MediaSource::IsTypeSupported(aType, nullptr /* DecoderDoctorDiagnostics */, + rv); + + return !rv.Failed(); +} + +bool MediaCapabilities::CheckTypeForFile(const nsAString& aType) { + Maybe<MediaContainerType> containerType = MakeMediaContainerType(aType); + if (!containerType) { + return false; + } + + return DecoderTraits::CanHandleContainerType( + *containerType, nullptr /* DecoderDoctorDiagnostics */) != + CANPLAY_NO; +} + +bool MediaCapabilities::CheckTypeForEncoder(const nsAString& aType) { + return MediaRecorder::IsTypeSupported(aType); +} + +already_AddRefed<layers::KnowsCompositor> MediaCapabilities::GetCompositor() { + nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(GetParentObject()); + if (NS_WARN_IF(!window)) { + return nullptr; + } + + nsCOMPtr<Document> doc = window->GetExtantDoc(); + if (NS_WARN_IF(!doc)) { + return nullptr; + } + WindowRenderer* renderer = nsContentUtils::WindowRendererForDocument(doc); + if (NS_WARN_IF(!renderer)) { + return nullptr; + } + RefPtr<layers::KnowsCompositor> knows = renderer->AsKnowsCompositor(); + if (NS_WARN_IF(!knows)) { + return nullptr; + } + return knows->GetForMedia().forget(); +} + +bool MediaCapabilities::Enabled(JSContext* aCx, JSObject* aGlobal) { + return StaticPrefs::media_media_capabilities_enabled(); +} + +JSObject* MediaCapabilities::WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) { + return MediaCapabilities_Binding::Wrap(aCx, this, aGivenProto); +} + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(MediaCapabilities) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +NS_IMPL_CYCLE_COLLECTING_ADDREF(MediaCapabilities) +NS_IMPL_CYCLE_COLLECTING_RELEASE(MediaCapabilities) + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(MediaCapabilities, mParent) + +// MediaCapabilitiesInfo +bool MediaCapabilitiesInfo::WrapObject( + JSContext* aCx, JS::Handle<JSObject*> aGivenProto, + JS::MutableHandle<JSObject*> aReflector) { + return MediaCapabilitiesInfo_Binding::Wrap(aCx, this, aGivenProto, + aReflector); +} + +} // namespace mozilla::dom diff --git a/dom/media/mediacapabilities/MediaCapabilities.h b/dom/media/mediacapabilities/MediaCapabilities.h new file mode 100644 index 0000000000..b31e12e8ab --- /dev/null +++ b/dom/media/mediacapabilities/MediaCapabilities.h @@ -0,0 +1,104 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_MediaCapabilities_h_ +#define mozilla_dom_MediaCapabilities_h_ + +#include "DDLoggedTypeTraits.h" +#include "js/TypeDecls.h" +#include "mozilla/AlreadyAddRefed.h" +#include "mozilla/Assertions.h" +#include "mozilla/Maybe.h" +#include "mozilla/dom/NonRefcountedDOMObject.h" +#include "nsCOMPtr.h" +#include "nsCycleCollectionParticipant.h" +#include "nsISupports.h" +#include "nsStringFwd.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; + +namespace mozilla { +class ErrorResult; +class MediaContainerType; + +namespace layers { +class KnowsCompositor; +} +namespace dom { +class MediaCapabilities; +} // namespace dom +DDLoggedTypeName(dom::MediaCapabilities); + +namespace dom { + +struct MediaDecodingConfiguration; +struct MediaEncodingConfiguration; +struct AudioConfiguration; +struct VideoConfiguration; +class Promise; + +class MediaCapabilities final : public nsISupports, public nsWrapperCache { + public: + // Ref counting and cycle collection + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(MediaCapabilities) + + // WebIDL Methods + already_AddRefed<Promise> DecodingInfo( + const MediaDecodingConfiguration& aConfiguration, ErrorResult& aRv); + already_AddRefed<Promise> EncodingInfo( + const MediaEncodingConfiguration& aConfiguration, ErrorResult& aRv); + // End WebIDL Methods + + explicit MediaCapabilities(nsIGlobalObject* aParent); + + nsIGlobalObject* GetParentObject() const { return mParent; } + JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override; + + static bool Enabled(JSContext* aCx, JSObject* aGlobal); + + private: + virtual ~MediaCapabilities() = default; + Maybe<MediaContainerType> CheckVideoConfiguration( + const VideoConfiguration& aConfig) const; + Maybe<MediaContainerType> CheckAudioConfiguration( + const AudioConfiguration& aConfig) const; + bool CheckTypeForMediaSource(const nsAString& aType); + bool CheckTypeForFile(const nsAString& aType); + bool CheckTypeForEncoder(const nsAString& aType); + already_AddRefed<layers::KnowsCompositor> GetCompositor(); + nsCOMPtr<nsIGlobalObject> mParent; +}; + +class MediaCapabilitiesInfo final : public NonRefcountedDOMObject { + public: + // WebIDL methods + bool Supported() const { return mSupported; } + bool Smooth() const { return mSmooth; } + bool PowerEfficient() const { return mPowerEfficient; } + // End WebIDL methods + + MediaCapabilitiesInfo(bool aSupported, bool aSmooth, bool aPowerEfficient) + : mSupported(aSupported), + mSmooth(aSmooth), + mPowerEfficient(aPowerEfficient) {} + + bool WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto, + JS::MutableHandle<JSObject*> aReflector); + + private: + bool mSupported; + bool mSmooth; + bool mPowerEfficient; +}; + +} // namespace dom + +} // namespace mozilla + +#endif /* mozilla_dom_MediaCapabilities_h_ */ diff --git a/dom/media/mediacapabilities/PBenchmarkStorage.ipdl b/dom/media/mediacapabilities/PBenchmarkStorage.ipdl new file mode 100644 index 0000000000..220d83f978 --- /dev/null +++ b/dom/media/mediacapabilities/PBenchmarkStorage.ipdl @@ -0,0 +1,23 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include protocol PContent; + +namespace mozilla { + +[ManualDealloc] +async protocol PBenchmarkStorage +{ + manager PContent; + +parent: + async Put(nsCString aDbName, nsCString aKey, int32_t aValue); + async Get(nsCString aDbName, nsCString aKey) returns(int32_t aValue); + async CheckVersion(nsCString aDbName, int32_t aVersion); + async __delete__(); + +}; + +} // namespace mozilla diff --git a/dom/media/mediacapabilities/moz.build b/dom/media/mediacapabilities/moz.build new file mode 100644 index 0000000000..bafd3de59c --- /dev/null +++ b/dom/media/mediacapabilities/moz.build @@ -0,0 +1,32 @@ +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXPORTS.mozilla.dom += [ + "MediaCapabilities.h", +] + +EXPORTS.mozilla += [ + "BenchmarkStorageChild.h", + "BenchmarkStorageParent.h", + "KeyValueStorage.h", +] + +EXPORTS += [ + "DecoderBenchmark.h", +] + +UNIFIED_SOURCES += [ + "BenchmarkStorageChild.cpp", + "BenchmarkStorageParent.cpp", + "DecoderBenchmark.cpp", + "KeyValueStorage.cpp", + "MediaCapabilities.cpp", +] + +IPDL_SOURCES += ["PBenchmarkStorage.ipdl"] + +include("/ipc/chromium/chromium-config.mozbuild") + +FINAL_LIBRARY = "xul" |