summaryrefslogtreecommitdiffstats
path: root/dom/media/webcodecs
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
commit26a029d407be480d791972afb5975cf62c9360a6 (patch)
treef435a8308119effd964b339f76abb83a57c29483 /dom/media/webcodecs
parentInitial commit. (diff)
downloadfirefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz
firefox-26a029d407be480d791972afb5975cf62c9360a6.zip
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/media/webcodecs')
-rw-r--r--dom/media/webcodecs/DecoderAgent.cpp491
-rw-r--r--dom/media/webcodecs/DecoderAgent.h117
-rw-r--r--dom/media/webcodecs/DecoderTemplate.cpp891
-rw-r--r--dom/media/webcodecs/DecoderTemplate.h260
-rw-r--r--dom/media/webcodecs/DecoderTypes.h117
-rw-r--r--dom/media/webcodecs/EncodedVideoChunk.cpp261
-rw-r--r--dom/media/webcodecs/EncodedVideoChunk.h119
-rw-r--r--dom/media/webcodecs/EncoderAgent.cpp441
-rw-r--r--dom/media/webcodecs/EncoderAgent.h116
-rw-r--r--dom/media/webcodecs/EncoderTemplate.cpp1228
-rw-r--r--dom/media/webcodecs/EncoderTemplate.h290
-rw-r--r--dom/media/webcodecs/EncoderTypes.h103
-rw-r--r--dom/media/webcodecs/VideoColorSpace.cpp48
-rw-r--r--dom/media/webcodecs/VideoColorSpace.h64
-rw-r--r--dom/media/webcodecs/VideoDecoder.cpp977
-rw-r--r--dom/media/webcodecs/VideoDecoder.h79
-rw-r--r--dom/media/webcodecs/VideoEncoder.cpp624
-rw-r--r--dom/media/webcodecs/VideoEncoder.h78
-rw-r--r--dom/media/webcodecs/VideoFrame.cpp2417
-rw-r--r--dom/media/webcodecs/VideoFrame.h266
-rw-r--r--dom/media/webcodecs/WebCodecsUtils.cpp578
-rw-r--r--dom/media/webcodecs/WebCodecsUtils.h239
-rw-r--r--dom/media/webcodecs/crashtests/1839270.html13
-rw-r--r--dom/media/webcodecs/crashtests/1848460.html17
-rw-r--r--dom/media/webcodecs/crashtests/1849271.html27
-rw-r--r--dom/media/webcodecs/crashtests/1864475.html14
-rw-r--r--dom/media/webcodecs/crashtests/crashtests.list4
-rw-r--r--dom/media/webcodecs/moz.build54
-rw-r--r--dom/media/webcodecs/test/mochitest.toml6
-rw-r--r--dom/media/webcodecs/test/test_videoFrame_mismatched_codedSize.html30
30 files changed, 9969 insertions, 0 deletions
diff --git a/dom/media/webcodecs/DecoderAgent.cpp b/dom/media/webcodecs/DecoderAgent.cpp
new file mode 100644
index 0000000000..5c63e27d48
--- /dev/null
+++ b/dom/media/webcodecs/DecoderAgent.cpp
@@ -0,0 +1,491 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "DecoderAgent.h"
+
+#include <atomic>
+
+#include "ImageContainer.h"
+#include "MediaDataDecoderProxy.h"
+#include "PDMFactory.h"
+#include "VideoUtils.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/Logging.h"
+#include "mozilla/layers/ImageBridgeChild.h"
+#include "nsThreadUtils.h"
+
+extern mozilla::LazyLogModule gWebCodecsLog;
+
+namespace mozilla {
+
+#ifdef LOG_INTERNAL
+# undef LOG_INTERNAL
+#endif // LOG_INTERNAL
+#define LOG_INTERNAL(level, msg, ...) \
+ MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__))
+
+#ifdef LOG
+# undef LOG
+#endif // LOG
+#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__)
+
+#ifdef LOGW
+# undef LOGW
+#endif // LOGE
+#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__)
+
+#ifdef LOGE
+# undef LOGE
+#endif // LOGE
+#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__)
+
+#ifdef LOGV
+# undef LOGV
+#endif // LOGV
+#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__)
+
+DecoderAgent::DecoderAgent(Id aId, UniquePtr<TrackInfo>&& aInfo)
+ : mId(aId),
+ mInfo(std::move(aInfo)),
+ mOwnerThread(GetCurrentSerialEventTarget()),
+ mPDMFactory(MakeRefPtr<PDMFactory>()),
+ mImageContainer(MakeAndAddRef<layers::ImageContainer>(
+ layers::ImageContainer::ASYNCHRONOUS)),
+ mDecoder(nullptr),
+ mState(State::Unconfigured) {
+ MOZ_ASSERT(mInfo);
+ MOZ_ASSERT(mOwnerThread);
+ MOZ_ASSERT(mPDMFactory);
+ MOZ_ASSERT(mImageContainer);
+ LOG("DecoderAgent #%d (%p) ctor", mId, this);
+}
+
+DecoderAgent::~DecoderAgent() {
+ LOG("DecoderAgent #%d (%p) dtor", mId, this);
+ MOZ_ASSERT(mState == State::Unconfigured, "decoder release in wrong state");
+ MOZ_ASSERT(!mDecoder, "decoder must be shutdown");
+}
+
+RefPtr<DecoderAgent::ConfigurePromise> DecoderAgent::Configure(
+ bool aPreferSoftwareDecoder, bool aLowLatency) {
+ MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
+ MOZ_ASSERT(mState == State::Unconfigured || mState == State::Error);
+ MOZ_ASSERT(mConfigurePromise.IsEmpty());
+ MOZ_ASSERT(!mCreateRequest.Exists());
+ MOZ_ASSERT(!mInitRequest.Exists());
+
+ if (mState == State::Error) {
+ LOGE("DecoderAgent #%d (%p) tried to configure in error state", mId, this);
+ return ConfigurePromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ "Cannot configure in error state"),
+ __func__);
+ }
+
+ MOZ_ASSERT(mState == State::Unconfigured);
+ MOZ_ASSERT(!mDecoder);
+ SetState(State::Configuring);
+
+ RefPtr<layers::KnowsCompositor> knowsCompositor =
+ layers::ImageBridgeChild::GetSingleton();
+ // Bug 1839993: FFmpegDataDecoder ignores all decode errors when draining so
+ // WPT cannot receive error callbacks. Forcibly enable LowLatency for now to
+ // get the decoded results immediately to avoid this.
+
+ auto params = CreateDecoderParams{
+ *mInfo,
+ CreateDecoderParams::OptionSet(
+ CreateDecoderParams::Option::LowLatency,
+ aPreferSoftwareDecoder
+ ? CreateDecoderParams::Option::HardwareDecoderNotAllowed
+ : CreateDecoderParams::Option::Default),
+ mInfo->GetType(), mImageContainer, knowsCompositor};
+
+ LOG("DecoderAgent #%d (%p) is creating a decoder - PreferSW: %s, "
+ "low-latency: %syes",
+ mId, this, aPreferSoftwareDecoder ? "yes" : "no",
+ aLowLatency ? "" : "forcibly ");
+
+ RefPtr<ConfigurePromise> p = mConfigurePromise.Ensure(__func__);
+
+ mPDMFactory->CreateDecoder(params)
+ ->Then(
+ mOwnerThread, __func__,
+ [self = RefPtr{this}](RefPtr<MediaDataDecoder>&& aDecoder) {
+ self->mCreateRequest.Complete();
+
+ // If DecoderAgent has been shut down, shut the created decoder down
+ // and return.
+ if (!self->mShutdownWhileCreationPromise.IsEmpty()) {
+ MOZ_ASSERT(self->mState == State::ShuttingDown);
+ MOZ_ASSERT(self->mConfigurePromise.IsEmpty(),
+ "configuration should have been rejected");
+
+ LOGW(
+ "DecoderAgent #%d (%p) has been shut down. We need to shut "
+ "the newly created decoder down",
+ self->mId, self.get());
+ aDecoder->Shutdown()->Then(
+ self->mOwnerThread, __func__,
+ [self](const ShutdownPromise::ResolveOrRejectValue& aValue) {
+ MOZ_ASSERT(self->mState == State::ShuttingDown);
+
+ LOGW(
+ "DecoderAgent #%d (%p), newly created decoder shutdown "
+ "has been %s",
+ self->mId, self.get(),
+ aValue.IsResolve() ? "resolved" : "rejected");
+
+ self->SetState(State::Unconfigured);
+
+ self->mShutdownWhileCreationPromise.ResolveOrReject(
+ aValue, __func__);
+ });
+ return;
+ }
+
+ self->mDecoder = new MediaDataDecoderProxy(
+ aDecoder.forget(),
+ CreateMediaDecodeTaskQueue("DecoderAgent TaskQueue"));
+ LOG("DecoderAgent #%d (%p) has created a decoder, now initialize "
+ "it",
+ self->mId, self.get());
+ self->mDecoder->Init()
+ ->Then(
+ self->mOwnerThread, __func__,
+ [self](const TrackInfo::TrackType aTrackType) {
+ self->mInitRequest.Complete();
+ LOG("DecoderAgent #%d (%p) has initialized the decoder",
+ self->mId, self.get());
+ MOZ_ASSERT(aTrackType == self->mInfo->GetType());
+ self->SetState(State::Configured);
+ self->mConfigurePromise.Resolve(true, __func__);
+ },
+ [self](const MediaResult& aError) {
+ self->mInitRequest.Complete();
+ LOGE(
+ "DecoderAgent #%d (%p) failed to initialize the "
+ "decoder",
+ self->mId, self.get());
+ self->SetState(State::Error);
+ self->mConfigurePromise.Reject(aError, __func__);
+ })
+ ->Track(self->mInitRequest);
+ },
+ [self = RefPtr{this}](const MediaResult& aError) {
+ self->mCreateRequest.Complete();
+ LOGE("DecoderAgent #%d (%p) failed to create a decoder", self->mId,
+ self.get());
+
+ // If DecoderAgent has been shut down, we need to resolve the
+ // shutdown promise.
+ if (!self->mShutdownWhileCreationPromise.IsEmpty()) {
+ MOZ_ASSERT(self->mState == State::ShuttingDown);
+ MOZ_ASSERT(self->mConfigurePromise.IsEmpty(),
+ "configuration should have been rejected");
+
+ LOGW(
+ "DecoderAgent #%d (%p) has been shut down. Resolve the "
+ "shutdown promise right away since decoder creation failed",
+ self->mId, self.get());
+
+ self->SetState(State::Unconfigured);
+ self->mShutdownWhileCreationPromise.Resolve(true, __func__);
+ return;
+ }
+
+ self->SetState(State::Error);
+ self->mConfigurePromise.Reject(aError, __func__);
+ })
+ ->Track(mCreateRequest);
+
+ return p;
+}
+
+RefPtr<ShutdownPromise> DecoderAgent::Shutdown() {
+ MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
+
+ auto r =
+ MediaResult(NS_ERROR_DOM_MEDIA_CANCELED, "Canceled by decoder shutdown");
+
+ // If the decoder creation has not been completed yet, wait until the decoder
+ // being created has been shut down.
+ if (mCreateRequest.Exists()) {
+ MOZ_ASSERT(!mInitRequest.Exists());
+ MOZ_ASSERT(!mConfigurePromise.IsEmpty());
+ MOZ_ASSERT(!mDecoder);
+ MOZ_ASSERT(mState == State::Configuring);
+ MOZ_ASSERT(mShutdownWhileCreationPromise.IsEmpty());
+
+ LOGW(
+ "DecoderAgent #%d (%p) shutdown while the decoder-creation for "
+ "configuration is in flight. Reject the configuration now and defer "
+ "the shutdown until the created decoder has been shut down",
+ mId, this);
+
+ // Reject the configuration in flight.
+ mConfigurePromise.Reject(r, __func__);
+
+ // Get the promise that will be resolved when the decoder being created has
+ // been destroyed.
+ SetState(State::ShuttingDown);
+ return mShutdownWhileCreationPromise.Ensure(__func__);
+ }
+
+ // If decoder creation has been completed, we must have the decoder now.
+ MOZ_ASSERT(mDecoder);
+
+ // Cancel pending initialization for configuration in flight if any.
+ mInitRequest.DisconnectIfExists();
+ mConfigurePromise.RejectIfExists(r, __func__);
+
+ // Cancel decode in flight if any.
+ mDecodeRequest.DisconnectIfExists();
+ mDecodePromise.RejectIfExists(r, __func__);
+
+ // Cancel flush-out in flight if any.
+ mDrainRequest.DisconnectIfExists();
+ mFlushRequest.DisconnectIfExists();
+ mDryRequest.DisconnectIfExists();
+ mDryPromise.RejectIfExists(r, __func__);
+ mDrainAndFlushPromise.RejectIfExists(r, __func__);
+ mDryData.Clear();
+ mDrainAndFlushData.Clear();
+
+ SetState(State::Unconfigured);
+
+ RefPtr<MediaDataDecoder> decoder = std::move(mDecoder);
+ return decoder->Shutdown();
+}
+
+RefPtr<DecoderAgent::DecodePromise> DecoderAgent::Decode(
+ MediaRawData* aSample) {
+ MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
+ MOZ_ASSERT(aSample);
+ MOZ_ASSERT(mState == State::Configured || mState == State::Error);
+ MOZ_ASSERT(mDecodePromise.IsEmpty());
+ MOZ_ASSERT(!mDecodeRequest.Exists());
+
+ if (mState == State::Error) {
+ LOGE("DecoderAgent #%d (%p) tried to decode in error state", mId, this);
+ return DecodePromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ "Cannot decode in error state"),
+ __func__);
+ }
+
+ MOZ_ASSERT(mState == State::Configured);
+ MOZ_ASSERT(mDecoder);
+ SetState(State::Decoding);
+
+ RefPtr<DecodePromise> p = mDecodePromise.Ensure(__func__);
+
+ mDecoder->Decode(aSample)
+ ->Then(
+ mOwnerThread, __func__,
+ [self = RefPtr{this}](MediaDataDecoder::DecodedData&& aData) {
+ self->mDecodeRequest.Complete();
+ LOGV("DecoderAgent #%d (%p) decode successfully", self->mId,
+ self.get());
+ self->SetState(State::Configured);
+ self->mDecodePromise.Resolve(std::move(aData), __func__);
+ },
+ [self = RefPtr{this}](const MediaResult& aError) {
+ self->mDecodeRequest.Complete();
+ LOGV("DecoderAgent #%d (%p) failed to decode", self->mId,
+ self.get());
+ self->SetState(State::Error);
+ self->mDecodePromise.Reject(aError, __func__);
+ })
+ ->Track(mDecodeRequest);
+
+ return p;
+}
+
+RefPtr<DecoderAgent::DecodePromise> DecoderAgent::DrainAndFlush() {
+ MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
+ MOZ_ASSERT(mState == State::Configured || mState == State::Error);
+ MOZ_ASSERT(mDrainAndFlushPromise.IsEmpty());
+ MOZ_ASSERT(mDrainAndFlushData.IsEmpty());
+ MOZ_ASSERT(!mDryRequest.Exists());
+ MOZ_ASSERT(mDryPromise.IsEmpty());
+ MOZ_ASSERT(mDryData.IsEmpty());
+ MOZ_ASSERT(!mDrainRequest.Exists());
+ MOZ_ASSERT(!mFlushRequest.Exists());
+
+ if (mState == State::Error) {
+ LOGE("DecoderAgent #%d (%p) tried to flush-out in error state", mId, this);
+ return DecodePromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ "Cannot flush in error state"),
+ __func__);
+ }
+
+ MOZ_ASSERT(mState == State::Configured);
+ MOZ_ASSERT(mDecoder);
+ SetState(State::Flushing);
+
+ RefPtr<DecoderAgent::DecodePromise> p =
+ mDrainAndFlushPromise.Ensure(__func__);
+
+ Dry()
+ ->Then(
+ mOwnerThread, __func__,
+ [self = RefPtr{this}](MediaDataDecoder::DecodedData&& aData) {
+ self->mDryRequest.Complete();
+ LOG("DecoderAgent #%d (%p) has dried the decoder. Now flushing the "
+ "decoder",
+ self->mId, self.get());
+ MOZ_ASSERT(self->mDrainAndFlushData.IsEmpty());
+ self->mDrainAndFlushData.AppendElements(std::move(aData));
+ self->mDecoder->Flush()
+ ->Then(
+ self->mOwnerThread, __func__,
+ [self](const bool /* aUnUsed */) {
+ self->mFlushRequest.Complete();
+ LOG("DecoderAgent #%d (%p) has flushed the decoder",
+ self->mId, self.get());
+ self->SetState(State::Configured);
+ self->mDrainAndFlushPromise.Resolve(
+ std::move(self->mDrainAndFlushData), __func__);
+ },
+ [self](const MediaResult& aError) {
+ self->mFlushRequest.Complete();
+ LOGE("DecoderAgent #%d (%p) failed to flush the decoder",
+ self->mId, self.get());
+ self->SetState(State::Error);
+ self->mDrainAndFlushData.Clear();
+ self->mDrainAndFlushPromise.Reject(aError, __func__);
+ })
+ ->Track(self->mFlushRequest);
+ },
+ [self = RefPtr{this}](const MediaResult& aError) {
+ self->mDryRequest.Complete();
+ LOGE("DecoderAgent #%d (%p) failed to dry the decoder", self->mId,
+ self.get());
+ self->SetState(State::Error);
+ self->mDrainAndFlushPromise.Reject(aError, __func__);
+ })
+ ->Track(mDryRequest);
+
+ return p;
+}
+
+RefPtr<DecoderAgent::DecodePromise> DecoderAgent::Dry() {
+ MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
+ MOZ_ASSERT(mState == State::Flushing);
+ MOZ_ASSERT(mDryPromise.IsEmpty());
+ MOZ_ASSERT(!mDryRequest.Exists());
+ MOZ_ASSERT(mDryData.IsEmpty());
+ MOZ_ASSERT(mDecoder);
+
+ RefPtr<DecodePromise> p = mDryPromise.Ensure(__func__);
+ DrainUntilDry();
+ return p;
+}
+
+void DecoderAgent::DrainUntilDry() {
+ MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
+ MOZ_ASSERT(mState == State::Flushing);
+ MOZ_ASSERT(!mDryPromise.IsEmpty());
+ MOZ_ASSERT(!mDrainRequest.Exists());
+ MOZ_ASSERT(mDecoder);
+
+ LOG("DecoderAgent #%d (%p) is drainng the decoder", mId, this);
+ mDecoder->Drain()
+ ->Then(
+ mOwnerThread, __func__,
+ [self = RefPtr{this}](MediaDataDecoder::DecodedData&& aData) {
+ self->mDrainRequest.Complete();
+
+ if (aData.IsEmpty()) {
+ LOG("DecoderAgent #%d (%p) is dry now", self->mId, self.get());
+ self->mDryPromise.Resolve(std::move(self->mDryData), __func__);
+ return;
+ }
+
+ LOG("DecoderAgent #%d (%p) drained %zu decoded data. Keep draining "
+ "until dry",
+ self->mId, self.get(), aData.Length());
+ self->mDryData.AppendElements(std::move(aData));
+ self->DrainUntilDry();
+ },
+ [self = RefPtr{this}](const MediaResult& aError) {
+ self->mDrainRequest.Complete();
+
+ LOGE("DecoderAgent %p failed to drain decoder", self.get());
+ self->mDryData.Clear();
+ self->mDryPromise.Reject(aError, __func__);
+ })
+ ->Track(mDrainRequest);
+}
+
+void DecoderAgent::SetState(State aState) {
+ MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
+
+ auto validateStateTransition = [](State aOldState, State aNewState) {
+ switch (aOldState) {
+ case State::Unconfigured:
+ return aNewState == State::Configuring;
+ case State::Configuring:
+ return aNewState == State::Configured || aNewState == State::Error ||
+ aNewState == State::Unconfigured ||
+ aNewState == State::ShuttingDown;
+ case State::Configured:
+ return aNewState == State::Unconfigured ||
+ aNewState == State::Decoding || aNewState == State::Flushing;
+ case State::Decoding:
+ case State::Flushing:
+ return aNewState == State::Configured || aNewState == State::Error ||
+ aNewState == State::Unconfigured;
+ case State::ShuttingDown:
+ return aNewState == State::Unconfigured;
+ case State::Error:
+ return aNewState == State::Unconfigured;
+ default:
+ break;
+ }
+ MOZ_ASSERT_UNREACHABLE("Unhandled state transition");
+ return false;
+ };
+
+ auto stateToString = [](State aState) -> const char* {
+ switch (aState) {
+ case State::Unconfigured:
+ return "Unconfigured";
+ case State::Configuring:
+ return "Configuring";
+ case State::Configured:
+ return "Configured";
+ case State::Decoding:
+ return "Decoding";
+ case State::Flushing:
+ return "Flushing";
+ case State::ShuttingDown:
+ return "ShuttingDown";
+ case State::Error:
+ return "Error";
+ default:
+ break;
+ }
+ MOZ_ASSERT_UNREACHABLE("Unhandled state type");
+ return "Unknown";
+ };
+
+ DebugOnly<bool> isValid = validateStateTransition(mState, aState);
+ MOZ_ASSERT(isValid);
+ LOG("DecoderAgent #%d (%p) state change: %s -> %s", mId, this,
+ stateToString(mState), stateToString(aState));
+ mState = aState;
+}
+
+#undef LOG
+#undef LOGW
+#undef LOGE
+#undef LOGV
+#undef LOG_INTERNAL
+
+} // namespace mozilla
diff --git a/dom/media/webcodecs/DecoderAgent.h b/dom/media/webcodecs/DecoderAgent.h
new file mode 100644
index 0000000000..f8107b27a6
--- /dev/null
+++ b/dom/media/webcodecs/DecoderAgent.h
@@ -0,0 +1,117 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef DOM_MEDIA_WEBCODECS_DECODERAGENT_H
+#define DOM_MEDIA_WEBCODECS_DECODERAGENT_H
+
+#include "MediaResult.h"
+#include "PlatformDecoderModule.h"
+#include "mozilla/RefPtr.h"
+#include "mozilla/TaskQueue.h"
+#include "mozilla/UniquePtr.h"
+
+class nsISerialEventTarget;
+
+namespace mozilla {
+
+class PDMFactory;
+class TrackInfo;
+
+namespace layers {
+class ImageContainer;
+} // namespace layers
+
+// DecoderAgent is a wrapper that contains a MediaDataDecoder. It adapts the
+// MediaDataDecoder APIs for use in WebCodecs.
+//
+// If Configure() is called, Shutdown() must be called to release the resources
+// gracefully. Except Shutdown(), all the methods can't be called concurrently,
+// meaning a method can only be called when the previous API call has completed.
+// The responsability of arranging the method calls is on the caller.
+//
+// When Shutdown() is called, all the operations in flight are canceled and the
+// MediaDataDecoder is shut down. On the other hand, errors are final. A new
+// DecoderAgent must be created when an error is encountered.
+//
+// All the methods need to be called on the DecoderAgent's owner thread. In
+// WebCodecs, it's either on the main thread or worker thread.
+class DecoderAgent final {
+ public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DecoderAgent);
+
+ using Id = uint32_t;
+ DecoderAgent(Id aId, UniquePtr<TrackInfo>&& aInfo);
+
+ // The following APIs are owner thread only.
+
+ using ConfigurePromise = MozPromise<bool, MediaResult, true /* exclusive */>;
+ RefPtr<ConfigurePromise> Configure(bool aPreferSoftwareDecoder,
+ bool aLowLatency);
+ RefPtr<ShutdownPromise> Shutdown();
+ using DecodePromise = MediaDataDecoder::DecodePromise;
+ RefPtr<DecodePromise> Decode(MediaRawData* aSample);
+ // WebCodecs's flush() flushes out all the pending decoded data in the
+ // MediaDataDecoder so it is a combination of Drain and Flush. To distinguish
+ // the term from MediaDataDecoder's one, we call it DrainAndFlush() here.
+ RefPtr<DecodePromise> DrainAndFlush();
+
+ const Id mId; // A unique id.
+ const UniquePtr<TrackInfo> mInfo;
+
+ private:
+ ~DecoderAgent();
+
+ // Push out all the data in the MediaDataDecoder's pipeline.
+ // TODO: MediaDataDecoder should implement this, instead of asking call site
+ // to run `Drain` multiple times.
+ RefPtr<DecodePromise> Dry();
+ void DrainUntilDry();
+
+ enum class State {
+ Unconfigured,
+ Configuring,
+ Configured,
+ Decoding,
+ Flushing,
+ ShuttingDown,
+ Error,
+ };
+ void SetState(State aState);
+
+ const RefPtr<nsISerialEventTarget> mOwnerThread;
+ const RefPtr<PDMFactory> mPDMFactory;
+ const RefPtr<layers::ImageContainer> mImageContainer;
+ RefPtr<MediaDataDecoder> mDecoder;
+ State mState;
+
+ // Configure
+ MozPromiseHolder<ConfigurePromise> mConfigurePromise;
+ using CreateDecoderPromise = PlatformDecoderModule::CreateDecoderPromise;
+ MozPromiseRequestHolder<CreateDecoderPromise> mCreateRequest;
+ using InitPromise = MediaDataDecoder::InitPromise;
+ MozPromiseRequestHolder<InitPromise> mInitRequest;
+
+ // Shutdown
+ MozPromiseHolder<ShutdownPromise> mShutdownWhileCreationPromise;
+
+ // Decode
+ MozPromiseHolder<DecodePromise> mDecodePromise;
+ MozPromiseRequestHolder<DecodePromise> mDecodeRequest;
+
+ // DrainAndFlush
+ MozPromiseHolder<DecodePromise> mDrainAndFlushPromise;
+ MediaDataDecoder::DecodedData mDrainAndFlushData;
+ MozPromiseRequestHolder<DecodePromise> mDryRequest;
+ MozPromiseHolder<DecodePromise> mDryPromise;
+ MediaDataDecoder::DecodedData mDryData;
+ MozPromiseRequestHolder<DecodePromise> mDrainRequest;
+ using FlushPromise = MediaDataDecoder::FlushPromise;
+ MozPromiseRequestHolder<FlushPromise> mFlushRequest;
+};
+
+} // namespace mozilla
+
+#endif // DOM_MEDIA_WEBCODECS_DECODERAGENT_H
diff --git a/dom/media/webcodecs/DecoderTemplate.cpp b/dom/media/webcodecs/DecoderTemplate.cpp
new file mode 100644
index 0000000000..0fa25a208b
--- /dev/null
+++ b/dom/media/webcodecs/DecoderTemplate.cpp
@@ -0,0 +1,891 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "DecoderTemplate.h"
+
+#include <atomic>
+#include <utility>
+
+#include "DecoderTypes.h"
+#include "MediaInfo.h"
+#include "mozilla/ScopeExit.h"
+#include "mozilla/Try.h"
+#include "mozilla/Unused.h"
+#include "mozilla/dom/DOMException.h"
+#include "mozilla/dom/Event.h"
+#include "mozilla/dom/Promise.h"
+#include "mozilla/dom/VideoDecoderBinding.h"
+#include "mozilla/dom/VideoFrame.h"
+#include "mozilla/dom/WorkerCommon.h"
+#include "nsGkAtoms.h"
+#include "nsString.h"
+#include "nsThreadUtils.h"
+
+mozilla::LazyLogModule gWebCodecsLog("WebCodecs");
+
+namespace mozilla::dom {
+
+#ifdef LOG_INTERNAL
+# undef LOG_INTERNAL
+#endif // LOG_INTERNAL
+#define LOG_INTERNAL(level, msg, ...) \
+ MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__))
+
+#ifdef LOG
+# undef LOG
+#endif // LOG
+#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__)
+
+#ifdef LOGW
+# undef LOGW
+#endif // LOGW
+#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__)
+
+#ifdef LOGE
+# undef LOGE
+#endif // LOGE
+#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__)
+
+#ifdef LOGV
+# undef LOGV
+#endif // LOGV
+#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__)
+
+/*
+ * Below are ControlMessage classes implementations
+ */
+
+template <typename DecoderType>
+DecoderTemplate<DecoderType>::ControlMessage::ControlMessage(
+ const nsACString& aTitle)
+ : mTitle(aTitle) {}
+
+template <typename DecoderType>
+DecoderTemplate<DecoderType>::ConfigureMessage::ConfigureMessage(
+ Id aId, UniquePtr<ConfigTypeInternal>&& aConfig)
+ : ControlMessage(
+ nsPrintfCString("configure #%d (%s)", aId,
+ NS_ConvertUTF16toUTF8(aConfig->mCodec).get())),
+ mId(aId),
+ mConfig(std::move(aConfig)) {}
+
+/* static */
+template <typename DecoderType>
+typename DecoderTemplate<DecoderType>::ConfigureMessage*
+DecoderTemplate<DecoderType>::ConfigureMessage::Create(
+ UniquePtr<ConfigTypeInternal>&& aConfig) {
+ // This needs to be atomic since this can run on the main thread or worker
+ // thread.
+ static std::atomic<Id> sNextId = NoId;
+ return new ConfigureMessage(++sNextId, std::move(aConfig));
+}
+
+template <typename DecoderType>
+DecoderTemplate<DecoderType>::DecodeMessage::DecodeMessage(
+ Id aId, ConfigId aConfigId, UniquePtr<InputTypeInternal>&& aData)
+ : ControlMessage(
+ nsPrintfCString("decode #%zu (config #%d)", aId, aConfigId)),
+ mId(aId),
+ mData(std::move(aData)) {}
+
+template <typename DecoderType>
+DecoderTemplate<DecoderType>::FlushMessage::FlushMessage(Id aId,
+ ConfigId aConfigId,
+ Promise* aPromise)
+ : ControlMessage(
+ nsPrintfCString("flush #%zu (config #%d)", aId, aConfigId)),
+ mId(aId),
+ mPromise(aPromise) {}
+
+template <typename DecoderType>
+void DecoderTemplate<DecoderType>::FlushMessage::RejectPromiseIfAny(
+ const nsresult& aReason) {
+ if (mPromise) {
+ mPromise->MaybeReject(aReason);
+ }
+}
+
+/*
+ * Below are DecoderTemplate implementation
+ */
+
+template <typename DecoderType>
+DecoderTemplate<DecoderType>::DecoderTemplate(
+ nsIGlobalObject* aGlobalObject,
+ RefPtr<WebCodecsErrorCallback>&& aErrorCallback,
+ RefPtr<OutputCallbackType>&& aOutputCallback)
+ : DOMEventTargetHelper(aGlobalObject),
+ mErrorCallback(std::move(aErrorCallback)),
+ mOutputCallback(std::move(aOutputCallback)),
+ mState(CodecState::Unconfigured),
+ mKeyChunkRequired(true),
+ mMessageQueueBlocked(false),
+ mDecodeQueueSize(0),
+ mDequeueEventScheduled(false),
+ mLatestConfigureId(ConfigureMessage::NoId),
+ mDecodeCounter(0),
+ mFlushCounter(0) {}
+
+template <typename DecoderType>
+void DecoderTemplate<DecoderType>::Configure(const ConfigType& aConfig,
+ ErrorResult& aRv) {
+ AssertIsOnOwningThread();
+
+ LOG("%s %p, Configure: codec %s", DecoderType::Name.get(), this,
+ NS_ConvertUTF16toUTF8(aConfig.mCodec).get());
+
+ nsCString errorMessage;
+ if (!DecoderType::Validate(aConfig, errorMessage)) {
+ aRv.ThrowTypeError(
+ nsPrintfCString("config is invalid: %s", errorMessage.get()));
+ return;
+ }
+
+ if (mState == CodecState::Closed) {
+ LOG("Configure: CodecState::Closed, rejecting with InvalidState");
+ aRv.ThrowInvalidStateError("The codec is no longer usable");
+ return;
+ }
+
+ // Clone a ConfigType as the active decoder config.
+ UniquePtr<ConfigTypeInternal> config =
+ DecoderType::CreateConfigInternal(aConfig);
+ if (!config) {
+ aRv.Throw(NS_ERROR_UNEXPECTED); // Invalid description data.
+ return;
+ }
+
+ mState = CodecState::Configured;
+ mKeyChunkRequired = true;
+ mDecodeCounter = 0;
+ mFlushCounter = 0;
+
+ mControlMessageQueue.emplace(
+ UniquePtr<ControlMessage>(ConfigureMessage::Create(std::move(config))));
+ mLatestConfigureId = mControlMessageQueue.back()->AsConfigureMessage()->mId;
+ LOG("%s %p enqueues %s", DecoderType::Name.get(), this,
+ mControlMessageQueue.back()->ToString().get());
+ ProcessControlMessageQueue();
+}
+
+template <typename DecoderType>
+void DecoderTemplate<DecoderType>::Decode(InputType& aInput, ErrorResult& aRv) {
+ AssertIsOnOwningThread();
+
+ LOG("%s %p, Decode", DecoderType::Name.get(), this);
+
+ if (mState != CodecState::Configured) {
+ aRv.ThrowInvalidStateError("Decoder must be configured first");
+ return;
+ }
+
+ if (mKeyChunkRequired) {
+ // TODO: Verify input's data is truly a key chunk
+ if (!DecoderType::IsKeyChunk(aInput)) {
+ aRv.ThrowDataError(
+ nsPrintfCString("%s needs a key chunk", DecoderType::Name.get()));
+ return;
+ }
+ mKeyChunkRequired = false;
+ }
+
+ mDecodeQueueSize += 1;
+ mControlMessageQueue.emplace(UniquePtr<ControlMessage>(
+ new DecodeMessage(++mDecodeCounter, mLatestConfigureId,
+ DecoderType::CreateInputInternal(aInput))));
+ LOGV("%s %p enqueues %s", DecoderType::Name.get(), this,
+ mControlMessageQueue.back()->ToString().get());
+ ProcessControlMessageQueue();
+}
+
+template <typename DecoderType>
+already_AddRefed<Promise> DecoderTemplate<DecoderType>::Flush(
+ ErrorResult& aRv) {
+ AssertIsOnOwningThread();
+
+ LOG("%s %p, Flush", DecoderType::Name.get(), this);
+
+ if (mState != CodecState::Configured) {
+ LOG("%s %p, wrong state!", DecoderType::Name.get(), this);
+ aRv.ThrowInvalidStateError("Decoder must be configured first");
+ return nullptr;
+ }
+
+ RefPtr<Promise> p = Promise::Create(GetParentObject(), aRv);
+ if (NS_WARN_IF(aRv.Failed())) {
+ return p.forget();
+ }
+
+ mKeyChunkRequired = true;
+
+ mControlMessageQueue.emplace(UniquePtr<ControlMessage>(
+ new FlushMessage(++mFlushCounter, mLatestConfigureId, p)));
+ LOG("%s %p enqueues %s", DecoderType::Name.get(), this,
+ mControlMessageQueue.back()->ToString().get());
+ ProcessControlMessageQueue();
+ return p.forget();
+}
+
+template <typename DecoderType>
+void DecoderTemplate<DecoderType>::Reset(ErrorResult& aRv) {
+ AssertIsOnOwningThread();
+
+ LOG("%s %p, Reset", DecoderType::Name.get(), this);
+
+ if (auto r = ResetInternal(NS_ERROR_DOM_ABORT_ERR); r.isErr()) {
+ aRv.Throw(r.unwrapErr());
+ }
+}
+
+template <typename DecoderType>
+void DecoderTemplate<DecoderType>::Close(ErrorResult& aRv) {
+ AssertIsOnOwningThread();
+
+ LOG("%s %p, Close", DecoderType::Name.get(), this);
+
+ if (auto r = CloseInternalWithAbort(); r.isErr()) {
+ aRv.Throw(r.unwrapErr());
+ }
+}
+
+template <typename DecoderType>
+Result<Ok, nsresult> DecoderTemplate<DecoderType>::ResetInternal(
+ const nsresult& aResult) {
+ AssertIsOnOwningThread();
+
+ if (mState == CodecState::Closed) {
+ return Err(NS_ERROR_DOM_INVALID_STATE_ERR);
+ }
+
+ mState = CodecState::Unconfigured;
+ mDecodeCounter = 0;
+ mFlushCounter = 0;
+
+ CancelPendingControlMessages(aResult);
+ DestroyDecoderAgentIfAny();
+
+ if (mDecodeQueueSize > 0) {
+ mDecodeQueueSize = 0;
+ ScheduleDequeueEventIfNeeded();
+ }
+
+ LOG("%s %p now has its message queue unblocked", DecoderType::Name.get(),
+ this);
+ mMessageQueueBlocked = false;
+
+ return Ok();
+}
+template <typename DecoderType>
+Result<Ok, nsresult> DecoderTemplate<DecoderType>::CloseInternalWithAbort() {
+ AssertIsOnOwningThread();
+
+ MOZ_TRY(ResetInternal(NS_ERROR_DOM_ABORT_ERR));
+ mState = CodecState::Closed;
+ return Ok();
+}
+
+template <typename DecoderType>
+void DecoderTemplate<DecoderType>::CloseInternal(const nsresult& aResult) {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(aResult != NS_ERROR_DOM_ABORT_ERR, "Use CloseInternalWithAbort");
+
+ auto r = ResetInternal(aResult);
+ if (r.isErr()) {
+ nsCString name;
+ GetErrorName(r.unwrapErr(), name);
+ LOGE("Error in ResetInternal: %s", name.get());
+ MOZ_CRASH();
+ }
+ mState = CodecState::Closed;
+ nsCString error;
+ GetErrorName(aResult, error);
+ LOGE("%s %p Close on error: %s", DecoderType::Name.get(), this, error.get());
+ ReportError(aResult);
+}
+
+template <typename DecoderType>
+void DecoderTemplate<DecoderType>::ReportError(const nsresult& aResult) {
+ AssertIsOnOwningThread();
+
+ RefPtr<DOMException> e = DOMException::Create(aResult);
+ RefPtr<WebCodecsErrorCallback> cb(mErrorCallback);
+ cb->Call(*e);
+}
+
+template <typename DecoderType>
+void DecoderTemplate<DecoderType>::OutputDecodedData(
+ const nsTArray<RefPtr<MediaData>>&& aData) {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(mState == CodecState::Configured);
+ MOZ_ASSERT(mActiveConfig);
+
+ nsTArray<RefPtr<VideoFrame>> frames = DecodedDataToOutputType(
+ GetParentObject(), std::move(aData), *mActiveConfig);
+ RefPtr<VideoFrameOutputCallback> cb(mOutputCallback);
+ for (RefPtr<VideoFrame>& frame : frames) {
+ LOG("Outputing decoded data: ts: %" PRId64, frame->Timestamp());
+ RefPtr<VideoFrame> f = frame;
+ cb->Call((VideoFrame&)(*f));
+ }
+}
+
+template <typename DecoderType>
+void DecoderTemplate<DecoderType>::ScheduleDequeueEventIfNeeded() {
+ AssertIsOnOwningThread();
+
+ if (mDequeueEventScheduled) {
+ return;
+ }
+ mDequeueEventScheduled = true;
+
+ QueueATask("dequeue event task", [self = RefPtr{this}]() {
+ self->FireEvent(nsGkAtoms::ondequeue, u"dequeue"_ns);
+ self->mDequeueEventScheduled = false;
+ });
+}
+
+template <typename DecoderType>
+nsresult DecoderTemplate<DecoderType>::FireEvent(nsAtom* aTypeWithOn,
+ const nsAString& aEventType) {
+ if (aTypeWithOn && !HasListenersFor(aTypeWithOn)) {
+ LOGV("%s %p has no %s event listener", DecoderType::Name.get(), this,
+ NS_ConvertUTF16toUTF8(aEventType).get());
+ return NS_ERROR_ABORT;
+ }
+
+ LOGV("Dispatch %s event to %s %p", NS_ConvertUTF16toUTF8(aEventType).get(),
+ DecoderType::Name.get(), this);
+ RefPtr<Event> event = new Event(this, nullptr, nullptr);
+ event->InitEvent(aEventType, true, true);
+ event->SetTrusted(true);
+ this->DispatchEvent(*event);
+ return NS_OK;
+}
+
+template <typename DecoderType>
+void DecoderTemplate<DecoderType>::ProcessControlMessageQueue() {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(mState == CodecState::Configured);
+
+ while (!mMessageQueueBlocked && !mControlMessageQueue.empty()) {
+ UniquePtr<ControlMessage>& msg = mControlMessageQueue.front();
+ if (msg->AsConfigureMessage()) {
+ if (ProcessConfigureMessage(msg) ==
+ MessageProcessedResult::NotProcessed) {
+ break;
+ }
+ } else if (msg->AsDecodeMessage()) {
+ if (ProcessDecodeMessage(msg) == MessageProcessedResult::NotProcessed) {
+ break;
+ }
+ } else {
+ MOZ_ASSERT(msg->AsFlushMessage());
+ if (ProcessFlushMessage(msg) == MessageProcessedResult::NotProcessed) {
+ break;
+ }
+ }
+ }
+}
+
+template <typename DecoderType>
+void DecoderTemplate<DecoderType>::CancelPendingControlMessages(
+ const nsresult& aResult) {
+ AssertIsOnOwningThread();
+
+ // Cancel the message that is being processed.
+ if (mProcessingMessage) {
+ LOG("%s %p cancels current %s", DecoderType::Name.get(), this,
+ mProcessingMessage->ToString().get());
+ mProcessingMessage->Cancel();
+
+ if (FlushMessage* flush = mProcessingMessage->AsFlushMessage()) {
+ flush->RejectPromiseIfAny(aResult);
+ }
+
+ mProcessingMessage.reset();
+ }
+
+ // Clear the message queue.
+ while (!mControlMessageQueue.empty()) {
+ LOG("%s %p cancels pending %s", DecoderType::Name.get(), this,
+ mControlMessageQueue.front()->ToString().get());
+
+ MOZ_ASSERT(!mControlMessageQueue.front()->IsProcessing());
+ if (FlushMessage* flush = mControlMessageQueue.front()->AsFlushMessage()) {
+ flush->RejectPromiseIfAny(aResult);
+ }
+
+ mControlMessageQueue.pop();
+ }
+}
+
+template <typename DecoderType>
+template <typename Func>
+void DecoderTemplate<DecoderType>::QueueATask(const char* aName,
+ Func&& aSteps) {
+ AssertIsOnOwningThread();
+ MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(
+ NS_NewRunnableFunction(aName, std::forward<Func>(aSteps))));
+}
+
+template <typename DecoderType>
+MessageProcessedResult DecoderTemplate<DecoderType>::ProcessConfigureMessage(
+ UniquePtr<ControlMessage>& aMessage) {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(mState == CodecState::Configured);
+ MOZ_ASSERT(aMessage->AsConfigureMessage());
+
+ if (mProcessingMessage) {
+ LOG("%s %p is processing %s. Defer %s", DecoderType::Name.get(), this,
+ mProcessingMessage->ToString().get(), aMessage->ToString().get());
+ return MessageProcessedResult::NotProcessed;
+ }
+
+ mProcessingMessage.reset(aMessage.release());
+ mControlMessageQueue.pop();
+
+ ConfigureMessage* msg = mProcessingMessage->AsConfigureMessage();
+ LOG("%s %p starts processing %s", DecoderType::Name.get(), this,
+ msg->ToString().get());
+
+ DestroyDecoderAgentIfAny();
+
+ mMessageQueueBlocked = true;
+
+ nsAutoCString errorMessage;
+ auto i = DecoderType::CreateTrackInfo(msg->Config());
+ if (i.isErr()) {
+ nsCString res;
+ GetErrorName(i.unwrapErr(), res);
+ errorMessage.AppendPrintf("CreateTrackInfo failed: %s", res.get());
+ } else if (!DecoderType::IsSupported(msg->Config())) {
+ errorMessage.Append("Not supported.");
+ } else if (!CreateDecoderAgent(msg->mId, msg->TakeConfig(), i.unwrap())) {
+ errorMessage.Append("DecoderAgent creation failed.");
+ }
+ if (!errorMessage.IsEmpty()) {
+ LOGE("%s %p ProcessConfigureMessage error (sync): %s",
+ DecoderType::Name.get(), this, errorMessage.get());
+
+ mProcessingMessage.reset();
+ QueueATask("Error while configuring decoder",
+ [self = RefPtr{this}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ MOZ_ASSERT(self->mState != CodecState::Closed);
+ self->CloseInternal(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+ });
+ return MessageProcessedResult::Processed;
+ }
+
+ MOZ_ASSERT(mAgent);
+ MOZ_ASSERT(mActiveConfig);
+
+ LOG("%s %p now blocks message-queue-processing", DecoderType::Name.get(),
+ this);
+
+ bool preferSW = mActiveConfig->mHardwareAcceleration ==
+ HardwareAcceleration::Prefer_software;
+ bool lowLatency = mActiveConfig->mOptimizeForLatency.isSome() &&
+ mActiveConfig->mOptimizeForLatency.value();
+ mAgent->Configure(preferSW, lowLatency)
+ ->Then(GetCurrentSerialEventTarget(), __func__,
+ [self = RefPtr{this}, id = mAgent->mId](
+ const DecoderAgent::ConfigurePromise::ResolveOrRejectValue&
+ aResult) {
+ MOZ_ASSERT(self->mProcessingMessage);
+ MOZ_ASSERT(self->mProcessingMessage->AsConfigureMessage());
+ MOZ_ASSERT(self->mState == CodecState::Configured);
+ MOZ_ASSERT(self->mAgent);
+ MOZ_ASSERT(id == self->mAgent->mId);
+ MOZ_ASSERT(self->mActiveConfig);
+
+ ConfigureMessage* msg =
+ self->mProcessingMessage->AsConfigureMessage();
+ LOG("%s %p, DecoderAgent #%d %s has been %s. now unblocks "
+ "message-queue-processing",
+ DecoderType::Name.get(), self.get(), id,
+ msg->ToString().get(),
+ aResult.IsResolve() ? "resolved" : "rejected");
+
+ msg->Complete();
+ self->mProcessingMessage.reset();
+
+ if (aResult.IsReject()) {
+ // The spec asks to close the decoder with an
+ // NotSupportedError so we log the exact error here.
+ const MediaResult& error = aResult.RejectValue();
+ LOGE("%s %p, DecoderAgent #%d failed to configure: %s",
+ DecoderType::Name.get(), self.get(), id,
+ error.Description().get());
+
+ self->QueueATask(
+ "Error during configure",
+ [self = RefPtr{self}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ MOZ_ASSERT(self->mState != CodecState::Closed);
+ self->CloseInternal(
+ NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ });
+ return;
+ }
+
+ self->mMessageQueueBlocked = false;
+ self->ProcessControlMessageQueue();
+ })
+ ->Track(msg->Request());
+
+ return MessageProcessedResult::Processed;
+}
+
+template <typename DecoderType>
+MessageProcessedResult DecoderTemplate<DecoderType>::ProcessDecodeMessage(
+ UniquePtr<ControlMessage>& aMessage) {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(mState == CodecState::Configured);
+ MOZ_ASSERT(aMessage->AsDecodeMessage());
+
+ if (mProcessingMessage) {
+ LOGV("%s %p is processing %s. Defer %s", DecoderType::Name.get(), this,
+ mProcessingMessage->ToString().get(), aMessage->ToString().get());
+ return MessageProcessedResult::NotProcessed;
+ }
+
+ mProcessingMessage.reset(aMessage.release());
+ mControlMessageQueue.pop();
+
+ DecodeMessage* msg = mProcessingMessage->AsDecodeMessage();
+ LOGV("%s %p starts processing %s", DecoderType::Name.get(), this,
+ msg->ToString().get());
+
+ mDecodeQueueSize -= 1;
+ ScheduleDequeueEventIfNeeded();
+
+ // Treat it like decode error if no DecoderAgent is available or the encoded
+ // data is invalid.
+ auto closeOnError = [&]() {
+ mProcessingMessage.reset();
+ QueueATask("Error during decode",
+ [self = RefPtr{this}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ MOZ_ASSERT(self->mState != CodecState::Closed);
+ self->CloseInternal(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ });
+ return MessageProcessedResult::Processed;
+ };
+
+ if (!mAgent) {
+ LOGE("%s %p is not configured", DecoderType::Name.get(), this);
+ return closeOnError();
+ }
+
+ MOZ_ASSERT(mActiveConfig);
+ RefPtr<MediaRawData> data = InputDataToMediaRawData(
+ std::move(msg->mData), *(mAgent->mInfo), *mActiveConfig);
+ if (!data) {
+ LOGE("%s %p, data for %s is empty or invalid", DecoderType::Name.get(),
+ this, msg->ToString().get());
+ return closeOnError();
+ }
+
+ mAgent->Decode(data.get())
+ ->Then(
+ GetCurrentSerialEventTarget(), __func__,
+ [self = RefPtr{this}, id = mAgent->mId](
+ DecoderAgent::DecodePromise::ResolveOrRejectValue&& aResult) {
+ MOZ_ASSERT(self->mProcessingMessage);
+ MOZ_ASSERT(self->mProcessingMessage->AsDecodeMessage());
+ MOZ_ASSERT(self->mState == CodecState::Configured);
+ MOZ_ASSERT(self->mAgent);
+ MOZ_ASSERT(id == self->mAgent->mId);
+ MOZ_ASSERT(self->mActiveConfig);
+
+ DecodeMessage* msg = self->mProcessingMessage->AsDecodeMessage();
+ LOGV("%s %p, DecoderAgent #%d %s has been %s",
+ DecoderType::Name.get(), self.get(), id, msg->ToString().get(),
+ aResult.IsResolve() ? "resolved" : "rejected");
+
+ nsCString msgStr = msg->ToString();
+
+ msg->Complete();
+ self->mProcessingMessage.reset();
+
+ if (aResult.IsReject()) {
+ // The spec asks to queue a task to run close the decoder
+ // with an EncodingError so we log the exact error here.
+ const MediaResult& error = aResult.RejectValue();
+ LOGE("%s %p, DecoderAgent #%d %s failed: %s",
+ DecoderType::Name.get(), self.get(), id, msgStr.get(),
+ error.Description().get());
+ self->QueueATask(
+ "Error during decode runnable",
+ [self = RefPtr{self}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ MOZ_ASSERT(self->mState != CodecState::Closed);
+ self->CloseInternal(
+ NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ });
+ return;
+ }
+
+ MOZ_ASSERT(aResult.IsResolve());
+ nsTArray<RefPtr<MediaData>> data =
+ std::move(aResult.ResolveValue());
+ if (data.IsEmpty()) {
+ LOGV("%s %p got no data for %s", DecoderType::Name.get(),
+ self.get(), msgStr.get());
+ } else {
+ LOGV("%s %p, schedule %zu decoded data output for %s",
+ DecoderType::Name.get(), self.get(), data.Length(),
+ msgStr.get());
+ self->QueueATask("Output Decoded Data",
+ [self = RefPtr{self}, data = std::move(data)]()
+ MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ self->OutputDecodedData(std::move(data));
+ });
+ }
+ self->ProcessControlMessageQueue();
+ })
+ ->Track(msg->Request());
+
+ return MessageProcessedResult::Processed;
+}
+
+template <typename DecoderType>
+MessageProcessedResult DecoderTemplate<DecoderType>::ProcessFlushMessage(
+ UniquePtr<ControlMessage>& aMessage) {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(mState == CodecState::Configured);
+ MOZ_ASSERT(aMessage->AsFlushMessage());
+
+ if (mProcessingMessage) {
+ LOG("%s %p is processing %s. Defer %s", DecoderType::Name.get(), this,
+ mProcessingMessage->ToString().get(), aMessage->ToString().get());
+ return MessageProcessedResult::NotProcessed;
+ }
+
+ mProcessingMessage.reset(aMessage.release());
+ mControlMessageQueue.pop();
+
+ FlushMessage* msg = mProcessingMessage->AsFlushMessage();
+ LOG("%s %p starts processing %s", DecoderType::Name.get(), this,
+ msg->ToString().get());
+
+ // No agent, no thing to do. The promise has been rejected with the
+ // appropriate error in ResetInternal already.
+ if (!mAgent) {
+ LOGE("%s %p no agent, nothing to do", DecoderType::Name.get(), this);
+ mProcessingMessage.reset();
+ return MessageProcessedResult::Processed;
+ }
+
+ mAgent->DrainAndFlush()
+ ->Then(
+ GetCurrentSerialEventTarget(), __func__,
+ [self = RefPtr{this}, id = mAgent->mId,
+ this](DecoderAgent::DecodePromise::ResolveOrRejectValue&& aResult) {
+ MOZ_ASSERT(self->mProcessingMessage);
+ MOZ_ASSERT(self->mProcessingMessage->AsFlushMessage());
+ MOZ_ASSERT(self->mState == CodecState::Configured);
+ MOZ_ASSERT(self->mAgent);
+ MOZ_ASSERT(id == self->mAgent->mId);
+ MOZ_ASSERT(self->mActiveConfig);
+
+ FlushMessage* msg = self->mProcessingMessage->AsFlushMessage();
+ LOG("%s %p, DecoderAgent #%d %s has been %s",
+ DecoderType::Name.get(), self.get(), id, msg->ToString().get(),
+ aResult.IsResolve() ? "resolved" : "rejected");
+
+ nsCString msgStr = msg->ToString();
+
+ msg->Complete();
+
+ // If flush failed, it means decoder fails to decode the data
+ // sent before, so we treat it like decode error. We reject
+ // the promise first and then queue a task to close
+ // VideoDecoder with an EncodingError.
+ if (aResult.IsReject()) {
+ const MediaResult& error = aResult.RejectValue();
+ LOGE("%s %p, DecoderAgent #%d failed to flush: %s",
+ DecoderType::Name.get(), self.get(), id,
+ error.Description().get());
+ RefPtr<Promise> promise = msg->TakePromise();
+ // Reject with an EncodingError instead of the error we got
+ // above.
+ self->QueueATask(
+ "Error during flush runnable",
+ [self = RefPtr{this}, promise]() MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ promise->MaybeReject(
+ NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ self->mProcessingMessage.reset();
+ MOZ_ASSERT(self->mState != CodecState::Closed);
+ self->CloseInternal(
+ NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ });
+ return;
+ }
+
+ nsTArray<RefPtr<MediaData>> data =
+ std::move(aResult.ResolveValue());
+
+ if (data.IsEmpty()) {
+ LOG("%s %p gets no data for %s", DecoderType::Name.get(),
+ self.get(), msgStr.get());
+ } else {
+ LOG("%s %p, schedule %zu decoded data output for %s",
+ DecoderType::Name.get(), self.get(), data.Length(),
+ msgStr.get());
+ }
+
+ RefPtr<Promise> promise = msg->TakePromise();
+ self->QueueATask(
+ "Flush: output decoding data task",
+ [self = RefPtr{self}, promise, data = std::move(data)]()
+ MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ self->OutputDecodedData(std::move(data));
+ promise->MaybeResolveWithUndefined();
+ });
+ self->mProcessingMessage.reset();
+ self->ProcessControlMessageQueue();
+ })
+ ->Track(msg->Request());
+
+ return MessageProcessedResult::Processed;
+}
+
+// CreateDecoderAgent will create an DecoderAgent paired with a xpcom-shutdown
+// blocker and a worker-reference. Besides the needs mentioned in the header
+// file, the blocker and the worker-reference also provides an entry point for
+// us to clean up the resources. Other than the decoder dtor, Reset(), or
+// Close(), the resources should be cleaned up in the following situations:
+// 1. Decoder on window, closing document
+// 2. Decoder on worker, closing document
+// 3. Decoder on worker, terminating worker
+//
+// In case 1, the entry point to clean up is in the mShutdownBlocker's
+// ShutdownpPomise-resolver. In case 2, the entry point is in mWorkerRef's
+// shutting down callback. In case 3, the entry point is in mWorkerRef's
+// shutting down callback.
+
+template <typename DecoderType>
+bool DecoderTemplate<DecoderType>::CreateDecoderAgent(
+ DecoderAgent::Id aId, UniquePtr<ConfigTypeInternal>&& aConfig,
+ UniquePtr<TrackInfo>&& aInfo) {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(mState == CodecState::Configured);
+ MOZ_ASSERT(!mAgent);
+ MOZ_ASSERT(!mActiveConfig);
+ MOZ_ASSERT(!mShutdownBlocker);
+ MOZ_ASSERT_IF(!NS_IsMainThread(), !mWorkerRef);
+
+ auto resetOnFailure = MakeScopeExit([&]() {
+ mAgent = nullptr;
+ mActiveConfig = nullptr;
+ mShutdownBlocker = nullptr;
+ mWorkerRef = nullptr;
+ });
+
+ // If the decoder is on worker, get a worker reference.
+ if (!NS_IsMainThread()) {
+ WorkerPrivate* workerPrivate = GetCurrentThreadWorkerPrivate();
+ if (NS_WARN_IF(!workerPrivate)) {
+ return false;
+ }
+
+ // Clean up all the resources when worker is going away.
+ RefPtr<StrongWorkerRef> workerRef = StrongWorkerRef::Create(
+ workerPrivate, "DecoderTemplate::CreateDecoderAgent",
+ [self = RefPtr{this}]() {
+ LOG("%s %p, worker is going away", DecoderType::Name.get(),
+ self.get());
+ Unused << self->ResetInternal(NS_ERROR_DOM_ABORT_ERR);
+ });
+ if (NS_WARN_IF(!workerRef)) {
+ return false;
+ }
+
+ mWorkerRef = new ThreadSafeWorkerRef(workerRef);
+ }
+
+ mAgent = MakeRefPtr<DecoderAgent>(aId, std::move(aInfo));
+ mActiveConfig = std::move(aConfig);
+
+ // ShutdownBlockingTicket requires an unique name to register its own
+ // nsIAsyncShutdownBlocker since each blocker needs a distinct name.
+ // To do that, we use DecoderAgent's unique id to create a unique name.
+ nsAutoString uniqueName;
+ uniqueName.AppendPrintf(
+ "Blocker for DecoderAgent #%d (codec: %s) @ %p", mAgent->mId,
+ NS_ConvertUTF16toUTF8(mActiveConfig->mCodec).get(), mAgent.get());
+
+ mShutdownBlocker = media::ShutdownBlockingTicket::Create(
+ uniqueName, NS_LITERAL_STRING_FROM_CSTRING(__FILE__), __LINE__);
+ if (!mShutdownBlocker) {
+ LOGE("%s %p failed to create %s", DecoderType::Name.get(), this,
+ NS_ConvertUTF16toUTF8(uniqueName).get());
+ return false;
+ }
+
+ // Clean up all the resources when xpcom-will-shutdown arrives since the page
+ // is going to be closed.
+ mShutdownBlocker->ShutdownPromise()->Then(
+ GetCurrentSerialEventTarget(), __func__,
+ [self = RefPtr{this}, id = mAgent->mId,
+ ref = mWorkerRef](bool /* aUnUsed*/) {
+ LOG("%s %p gets xpcom-will-shutdown notification for DecoderAgent #%d",
+ DecoderType::Name.get(), self.get(), id);
+ Unused << self->ResetInternal(NS_ERROR_DOM_ABORT_ERR);
+ },
+ [self = RefPtr{this}, id = mAgent->mId,
+ ref = mWorkerRef](bool /* aUnUsed*/) {
+ LOG("%s %p removes shutdown-blocker #%d before getting any "
+ "notification. DecoderAgent #%d should have been dropped",
+ DecoderType::Name.get(), self.get(), id, id);
+ MOZ_ASSERT(!self->mAgent || self->mAgent->mId != id);
+ });
+
+ LOG("%s %p creates DecoderAgent #%d @ %p and its shutdown-blocker",
+ DecoderType::Name.get(), this, mAgent->mId, mAgent.get());
+
+ resetOnFailure.release();
+ return true;
+}
+
+template <typename DecoderType>
+void DecoderTemplate<DecoderType>::DestroyDecoderAgentIfAny() {
+ AssertIsOnOwningThread();
+
+ if (!mAgent) {
+ LOG("%s %p has no DecoderAgent to destroy", DecoderType::Name.get(), this);
+ return;
+ }
+
+ MOZ_ASSERT(mActiveConfig);
+ MOZ_ASSERT(mShutdownBlocker);
+ MOZ_ASSERT_IF(!NS_IsMainThread(), mWorkerRef);
+
+ LOG("%s %p destroys DecoderAgent #%d @ %p", DecoderType::Name.get(), this,
+ mAgent->mId, mAgent.get());
+ mActiveConfig = nullptr;
+ RefPtr<DecoderAgent> agent = std::move(mAgent);
+ // mShutdownBlocker should be kept alive until the shutdown is done.
+ // mWorkerRef is used to ensure this task won't be discarded in worker.
+ agent->Shutdown()->Then(
+ GetCurrentSerialEventTarget(), __func__,
+ [self = RefPtr{this}, id = agent->mId, ref = std::move(mWorkerRef),
+ blocker = std::move(mShutdownBlocker)](
+ const ShutdownPromise::ResolveOrRejectValue& aResult) {
+ LOG("%s %p, DecoderAgent #%d's shutdown has been %s. Drop its "
+ "shutdown-blocker now",
+ DecoderType::Name.get(), self.get(), id,
+ aResult.IsResolve() ? "resolved" : "rejected");
+ });
+}
+
+template class DecoderTemplate<VideoDecoderTraits>;
+
+#undef LOG
+#undef LOGW
+#undef LOGE
+#undef LOGV
+#undef LOG_INTERNAL
+
+} // namespace mozilla::dom
diff --git a/dom/media/webcodecs/DecoderTemplate.h b/dom/media/webcodecs/DecoderTemplate.h
new file mode 100644
index 0000000000..fe0cb5baee
--- /dev/null
+++ b/dom/media/webcodecs/DecoderTemplate.h
@@ -0,0 +1,260 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_DecoderTemplate_h
+#define mozilla_dom_DecoderTemplate_h
+
+#include <queue>
+
+#include "mozilla/DOMEventTargetHelper.h"
+#include "mozilla/DecoderAgent.h"
+#include "mozilla/MozPromise.h"
+#include "mozilla/RefPtr.h"
+#include "mozilla/Result.h"
+#include "mozilla/UniquePtr.h"
+#include "mozilla/dom/WorkerRef.h"
+#include "mozilla/media/MediaUtils.h"
+#include "nsStringFwd.h"
+#include "WebCodecsUtils.h"
+
+namespace mozilla {
+
+class TrackInfo;
+
+namespace dom {
+
+class WebCodecsErrorCallback;
+class Promise;
+enum class CodecState : uint8_t;
+
+template <typename DecoderType>
+class DecoderTemplate : public DOMEventTargetHelper {
+ using Self = DecoderTemplate<DecoderType>;
+ using ConfigType = typename DecoderType::ConfigType;
+ using ConfigTypeInternal = typename DecoderType::ConfigTypeInternal;
+ using InputType = typename DecoderType::InputType;
+ using InputTypeInternal = typename DecoderType::InputTypeInternal;
+ using OutputType = typename DecoderType::OutputType;
+ using OutputCallbackType = typename DecoderType::OutputCallbackType;
+
+ /* ControlMessage classes */
+ protected:
+ class ConfigureMessage;
+ class DecodeMessage;
+ class FlushMessage;
+
+ class ControlMessage {
+ public:
+ explicit ControlMessage(const nsACString& aTitle);
+ virtual ~ControlMessage() = default;
+ virtual void Cancel() = 0;
+ virtual bool IsProcessing() = 0;
+
+ virtual const nsCString& ToString() const { return mTitle; }
+ virtual ConfigureMessage* AsConfigureMessage() { return nullptr; }
+ virtual DecodeMessage* AsDecodeMessage() { return nullptr; }
+ virtual FlushMessage* AsFlushMessage() { return nullptr; }
+
+ const nsCString mTitle; // Used to identify the message in the logs.
+ };
+
+ class ConfigureMessage final
+ : public ControlMessage,
+ public MessageRequestHolder<DecoderAgent::ConfigurePromise> {
+ public:
+ using Id = DecoderAgent::Id;
+ static constexpr Id NoId = 0;
+ static ConfigureMessage* Create(UniquePtr<ConfigTypeInternal>&& aConfig);
+
+ ~ConfigureMessage() = default;
+ virtual void Cancel() override { Disconnect(); }
+ virtual bool IsProcessing() override { return Exists(); };
+ virtual ConfigureMessage* AsConfigureMessage() override { return this; }
+ const ConfigTypeInternal& Config() { return *mConfig; }
+ UniquePtr<ConfigTypeInternal> TakeConfig() { return std::move(mConfig); }
+
+ const Id mId; // A unique id shown in log.
+
+ private:
+ ConfigureMessage(Id aId, UniquePtr<ConfigTypeInternal>&& aConfig);
+
+ UniquePtr<ConfigTypeInternal> mConfig;
+ };
+
+ class DecodeMessage final
+ : public ControlMessage,
+ public MessageRequestHolder<DecoderAgent::DecodePromise> {
+ public:
+ using Id = size_t;
+ using ConfigId = typename Self::ConfigureMessage::Id;
+ DecodeMessage(Id aId, ConfigId aConfigId,
+ UniquePtr<InputTypeInternal>&& aData);
+ ~DecodeMessage() = default;
+ virtual void Cancel() override { Disconnect(); }
+ virtual bool IsProcessing() override { return Exists(); };
+ virtual DecodeMessage* AsDecodeMessage() override { return this; }
+ const Id mId; // A unique id shown in log.
+
+ UniquePtr<InputTypeInternal> mData;
+ };
+
+ class FlushMessage final
+ : public ControlMessage,
+ public MessageRequestHolder<DecoderAgent::DecodePromise> {
+ public:
+ using Id = size_t;
+ using ConfigId = typename Self::ConfigureMessage::Id;
+ FlushMessage(Id aId, ConfigId aConfigId, Promise* aPromise);
+ ~FlushMessage() = default;
+ virtual void Cancel() override { Disconnect(); }
+ virtual bool IsProcessing() override { return Exists(); };
+ virtual FlushMessage* AsFlushMessage() override { return this; }
+ already_AddRefed<Promise> TakePromise() { return mPromise.forget(); }
+ void RejectPromiseIfAny(const nsresult& aReason);
+
+ const Id mId; // A unique id shown in log.
+
+ private:
+ RefPtr<Promise> mPromise;
+ };
+
+ protected:
+ DecoderTemplate(nsIGlobalObject* aGlobalObject,
+ RefPtr<WebCodecsErrorCallback>&& aErrorCallback,
+ RefPtr<OutputCallbackType>&& aOutputCallback);
+
+ virtual ~DecoderTemplate() = default;
+
+ /* WebCodecs interfaces */
+ public:
+ IMPL_EVENT_HANDLER(dequeue)
+
+ CodecState State() const { return mState; };
+
+ uint32_t DecodeQueueSize() const { return mDecodeQueueSize; };
+
+ void Configure(const ConfigType& aConfig, ErrorResult& aRv);
+
+ void Decode(InputType& aInput, ErrorResult& aRv);
+
+ already_AddRefed<Promise> Flush(ErrorResult& aRv);
+
+ void Reset(ErrorResult& aRv);
+
+ void Close(ErrorResult& aRv);
+
+ /* Type conversion functions for the Decoder implementation */
+ protected:
+ virtual already_AddRefed<MediaRawData> InputDataToMediaRawData(
+ UniquePtr<InputTypeInternal>&& aData, TrackInfo& aInfo,
+ const ConfigTypeInternal& aConfig) = 0;
+ virtual nsTArray<RefPtr<OutputType>> DecodedDataToOutputType(
+ nsIGlobalObject* aGlobalObject, const nsTArray<RefPtr<MediaData>>&& aData,
+ ConfigTypeInternal& aConfig) = 0;
+
+ protected:
+ // DecoderTemplate can run on either main thread or worker thread.
+ void AssertIsOnOwningThread() const {
+ NS_ASSERT_OWNINGTHREAD(DecoderTemplate);
+ }
+
+ Result<Ok, nsresult> ResetInternal(const nsresult& aResult);
+ // Calling this method calls the error callback synchronously.
+ MOZ_CAN_RUN_SCRIPT
+ void CloseInternal(const nsresult& aResult);
+ // Calling this method doesn't call the error calback.
+ Result<Ok, nsresult> CloseInternalWithAbort();
+
+ MOZ_CAN_RUN_SCRIPT void ReportError(const nsresult& aResult);
+ MOZ_CAN_RUN_SCRIPT void OutputDecodedData(
+ const nsTArray<RefPtr<MediaData>>&& aData);
+
+ void ScheduleDequeueEventIfNeeded();
+ nsresult FireEvent(nsAtom* aTypeWithOn, const nsAString& aEventType);
+
+ void ProcessControlMessageQueue();
+ void CancelPendingControlMessages(const nsresult& aResult);
+
+ // Queue a task to the control thread. This is to be used when a task needs to
+ // perform multiple steps.
+ template <typename Func>
+ void QueueATask(const char* aName, Func&& aSteps);
+
+ MessageProcessedResult ProcessConfigureMessage(
+ UniquePtr<ControlMessage>& aMessage);
+
+ MessageProcessedResult ProcessDecodeMessage(
+ UniquePtr<ControlMessage>& aMessage);
+
+ MessageProcessedResult ProcessFlushMessage(
+ UniquePtr<ControlMessage>& aMessage);
+
+ // Returns true when mAgent can be created.
+ bool CreateDecoderAgent(DecoderAgent::Id aId,
+ UniquePtr<ConfigTypeInternal>&& aConfig,
+ UniquePtr<TrackInfo>&& aInfo);
+ void DestroyDecoderAgentIfAny();
+
+ // Constant in practice, only set in ctor.
+ RefPtr<WebCodecsErrorCallback> mErrorCallback;
+ RefPtr<OutputCallbackType> mOutputCallback;
+
+ CodecState mState;
+ bool mKeyChunkRequired;
+
+ bool mMessageQueueBlocked;
+ std::queue<UniquePtr<ControlMessage>> mControlMessageQueue;
+ UniquePtr<ControlMessage> mProcessingMessage;
+
+ uint32_t mDecodeQueueSize;
+ bool mDequeueEventScheduled;
+
+ // A unique id tracking the ConfigureMessage and will be used as the
+ // DecoderAgent's Id.
+ uint32_t mLatestConfigureId;
+ // Tracking how many decode data has been enqueued and this number will be
+ // used as the DecodeMessage's Id.
+ size_t mDecodeCounter;
+ // Tracking how many flush request has been enqueued and this number will be
+ // used as the FlushMessage's Id.
+ size_t mFlushCounter;
+
+ // DecoderAgent will be created every time "configure" is being processed, and
+ // will be destroyed when "reset" or another "configure" is called (spec
+ // allows calling two "configure" without a "reset" in between).
+ RefPtr<DecoderAgent> mAgent;
+ UniquePtr<ConfigTypeInternal> mActiveConfig;
+
+ // Used to add a nsIAsyncShutdownBlocker on main thread to block
+ // xpcom-shutdown before the underlying MediaDataDecoder is created. The
+ // blocker will be held until the underlying MediaDataDecoder has been shut
+ // down. This blocker guarantees RemoteDecoderManagerChild's thread, where the
+ // underlying RemoteMediaDataDecoder is on, outlives the
+ // RemoteMediaDataDecoder, since the thread releasing, which happens on main
+ // thread when getting a xpcom-shutdown signal, is blocked by the added
+ // blocker. As a result, RemoteMediaDataDecoder can safely work on worker
+ // thread with a holding blocker (otherwise, if RemoteDecoderManagerChild
+ // releases its thread on main thread before RemoteMediaDataDecoder's
+ // Shutdown() task run on worker thread, RemoteMediaDataDecoder has no thread
+ // to run).
+ UniquePtr<media::ShutdownBlockingTicket> mShutdownBlocker;
+
+ // Held to make sure the dispatched tasks can be done before worker is going
+ // away. As long as this worker-ref is held somewhere, the tasks dispatched to
+ // the worker can be executed (otherwise the tasks would be canceled). This
+ // ref should be activated as long as the underlying MediaDataDecoder is
+ // alive, and should keep alive until mShutdownBlocker is dropped, so all
+ // MediaDataDecoder's tasks and mShutdownBlocker-releasing task can be
+ // executed.
+ // TODO: Use StrongWorkerRef instead if this is always used in the same
+ // thread?
+ RefPtr<ThreadSafeWorkerRef> mWorkerRef;
+};
+
+} // namespace dom
+} // namespace mozilla
+
+#endif // mozilla_dom_DecoderTemplate_h
diff --git a/dom/media/webcodecs/DecoderTypes.h b/dom/media/webcodecs/DecoderTypes.h
new file mode 100644
index 0000000000..56aa82046f
--- /dev/null
+++ b/dom/media/webcodecs/DecoderTypes.h
@@ -0,0 +1,117 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_DecoderTypes_h
+#define mozilla_dom_DecoderTypes_h
+
+#include "MediaData.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/dom/EncodedVideoChunk.h"
+#include "mozilla/dom/VideoColorSpaceBinding.h"
+#include "mozilla/dom/VideoDecoderBinding.h"
+#include "mozilla/dom/VideoFrame.h"
+#include "nsStringFwd.h"
+#include "nsTLiteralString.h"
+
+namespace mozilla {
+
+class TrackInfo;
+class MediaByteBuffer;
+
+namespace dom {
+
+struct VideoColorSpaceInternal {
+ explicit VideoColorSpaceInternal(const VideoColorSpaceInit& aColorSpaceInit);
+ VideoColorSpaceInternal() = default;
+ VideoColorSpaceInit ToColorSpaceInit() const;
+
+ bool operator==(const VideoColorSpaceInternal& aOther) const {
+ return mFullRange == aOther.mFullRange && mMatrix == aOther.mMatrix &&
+ mPrimaries == aOther.mPrimaries && mTransfer == aOther.mTransfer;
+ }
+
+ Maybe<bool> mFullRange;
+ Maybe<VideoMatrixCoefficients> mMatrix;
+ Maybe<VideoColorPrimaries> mPrimaries;
+ Maybe<VideoTransferCharacteristics> mTransfer;
+};
+
+class VideoDecoderConfigInternal {
+ public:
+ static UniquePtr<VideoDecoderConfigInternal> Create(
+ const VideoDecoderConfig& aConfig);
+ VideoDecoderConfigInternal(const nsAString& aCodec,
+ Maybe<uint32_t>&& aCodedHeight,
+ Maybe<uint32_t>&& aCodedWidth,
+ Maybe<VideoColorSpaceInternal>&& aColorSpace,
+ Maybe<RefPtr<MediaByteBuffer>>&& aDescription,
+ Maybe<uint32_t>&& aDisplayAspectHeight,
+ Maybe<uint32_t>&& aDisplayAspectWidth,
+ const HardwareAcceleration& aHardwareAcceleration,
+ Maybe<bool>&& aOptimizeForLatency);
+ ~VideoDecoderConfigInternal() = default;
+
+ nsString ToString() const;
+
+ bool Equals(const VideoDecoderConfigInternal& aOther) const {
+ if (mDescription.isSome() != aOther.mDescription.isSome()) {
+ return false;
+ }
+ if (mDescription.isSome() && aOther.mDescription.isSome()) {
+ auto lhs = mDescription.value();
+ auto rhs = aOther.mDescription.value();
+ if (lhs->Length() != rhs->Length()) {
+ return false;
+ }
+ if (!ArrayEqual(lhs->Elements(), rhs->Elements(), lhs->Length())) {
+ return false;
+ }
+ }
+ return mCodec.Equals(aOther.mCodec) &&
+ mCodedHeight == aOther.mCodedHeight &&
+ mCodedWidth == aOther.mCodedWidth &&
+ mColorSpace == aOther.mColorSpace &&
+ mDisplayAspectHeight == aOther.mDisplayAspectWidth &&
+ mHardwareAcceleration == aOther.mHardwareAcceleration &&
+ mOptimizeForLatency == aOther.mOptimizeForLatency;
+ }
+
+ nsString mCodec;
+ Maybe<uint32_t> mCodedHeight;
+ Maybe<uint32_t> mCodedWidth;
+ Maybe<VideoColorSpaceInternal> mColorSpace;
+ Maybe<RefPtr<MediaByteBuffer>> mDescription;
+ Maybe<uint32_t> mDisplayAspectHeight;
+ Maybe<uint32_t> mDisplayAspectWidth;
+ HardwareAcceleration mHardwareAcceleration;
+ Maybe<bool> mOptimizeForLatency;
+};
+
+class VideoDecoderTraits {
+ public:
+ static constexpr nsLiteralCString Name = "VideoDecoder"_ns;
+ using ConfigType = VideoDecoderConfig;
+ using ConfigTypeInternal = VideoDecoderConfigInternal;
+ using InputType = EncodedVideoChunk;
+ using InputTypeInternal = EncodedVideoChunkData;
+ using OutputType = VideoFrame;
+ using OutputCallbackType = VideoFrameOutputCallback;
+
+ static bool IsSupported(const ConfigTypeInternal& aConfig);
+ static Result<UniquePtr<TrackInfo>, nsresult> CreateTrackInfo(
+ const ConfigTypeInternal& aConfig);
+ static bool Validate(const ConfigType& aConfig, nsCString& aErrorMessage);
+ static UniquePtr<ConfigTypeInternal> CreateConfigInternal(
+ const ConfigType& aConfig);
+ static bool IsKeyChunk(const InputType& aInput);
+ static UniquePtr<InputTypeInternal> CreateInputInternal(
+ const InputType& aInput);
+};
+
+} // namespace dom
+} // namespace mozilla
+
+#endif // mozilla_dom_DecoderTypes_h
diff --git a/dom/media/webcodecs/EncodedVideoChunk.cpp b/dom/media/webcodecs/EncodedVideoChunk.cpp
new file mode 100644
index 0000000000..c231740d05
--- /dev/null
+++ b/dom/media/webcodecs/EncodedVideoChunk.cpp
@@ -0,0 +1,261 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/dom/EncodedVideoChunk.h"
+#include <utility>
+#include "mozilla/dom/EncodedVideoChunkBinding.h"
+
+#include "MediaData.h"
+#include "TimeUnits.h"
+#include "mozilla/CheckedInt.h"
+#include "mozilla/Logging.h"
+#include "mozilla/PodOperations.h"
+#include "mozilla/dom/StructuredCloneHolder.h"
+#include "mozilla/dom/StructuredCloneTags.h"
+#include "mozilla/dom/WebCodecsUtils.h"
+
+extern mozilla::LazyLogModule gWebCodecsLog;
+using mozilla::media::TimeUnit;
+
+namespace mozilla::dom {
+
+#ifdef LOG_INTERNAL
+# undef LOG_INTERNAL
+#endif // LOG_INTERNAL
+#define LOG_INTERNAL(level, msg, ...) \
+ MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__))
+
+#ifdef LOGW
+# undef LOGW
+#endif // LOGW
+#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__)
+
+#ifdef LOGE
+# undef LOGE
+#endif // LOGE
+#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__)
+
+// Only needed for refcounted objects.
+NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(EncodedVideoChunk, mParent)
+NS_IMPL_CYCLE_COLLECTING_ADDREF(EncodedVideoChunk)
+NS_IMPL_CYCLE_COLLECTING_RELEASE(EncodedVideoChunk)
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(EncodedVideoChunk)
+ NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY
+ NS_INTERFACE_MAP_ENTRY(nsISupports)
+NS_INTERFACE_MAP_END
+
+EncodedVideoChunkData::EncodedVideoChunkData(
+ already_AddRefed<MediaAlignedByteBuffer> aBuffer,
+ const EncodedVideoChunkType& aType, int64_t aTimestamp,
+ Maybe<uint64_t>&& aDuration)
+ : mBuffer(aBuffer),
+ mType(aType),
+ mTimestamp(aTimestamp),
+ mDuration(aDuration) {
+ MOZ_ASSERT(mBuffer);
+ MOZ_ASSERT(mBuffer->Length() == mBuffer->Size());
+ MOZ_ASSERT(mBuffer->Length() <=
+ static_cast<size_t>(std::numeric_limits<uint32_t>::max()));
+}
+
+EncodedVideoChunkData::~EncodedVideoChunkData() = default;
+
+UniquePtr<EncodedVideoChunkData> EncodedVideoChunkData::Clone() const {
+ if (!mBuffer) {
+ LOGE("No buffer in EncodedVideoChunkData %p to clone!", this);
+ return nullptr;
+ }
+
+ // Since EncodedVideoChunkData can be zero-sized, cloning a zero-sized chunk
+ // is allowed.
+ if (mBuffer->Size() == 0) {
+ LOGW("Cloning an empty EncodedVideoChunkData %p", this);
+ }
+
+ auto buffer =
+ MakeRefPtr<MediaAlignedByteBuffer>(mBuffer->Data(), mBuffer->Length());
+ if (!buffer || buffer->Size() != mBuffer->Size()) {
+ LOGE("OOM to copy EncodedVideoChunkData %p", this);
+ return nullptr;
+ }
+
+ return MakeUnique<EncodedVideoChunkData>(buffer.forget(), mType, mTimestamp,
+ Maybe<uint64_t>(mDuration));
+}
+
+already_AddRefed<MediaRawData> EncodedVideoChunkData::TakeData() {
+ if (!mBuffer || !(*mBuffer)) {
+ LOGE("EncodedVideoChunkData %p has no data!", this);
+ return nullptr;
+ }
+
+ RefPtr<MediaRawData> sample(new MediaRawData(std::move(*mBuffer)));
+ sample->mKeyframe = mType == EncodedVideoChunkType::Key;
+ sample->mTime = TimeUnit::FromMicroseconds(mTimestamp);
+ sample->mTimecode = TimeUnit::FromMicroseconds(mTimestamp);
+
+ if (mDuration) {
+ CheckedInt64 duration(*mDuration);
+ if (!duration.isValid()) {
+ LOGE("EncodedVideoChunkData %p 's duration exceeds TimeUnit's limit",
+ this);
+ return nullptr;
+ }
+ sample->mDuration = TimeUnit::FromMicroseconds(duration.value());
+ }
+
+ return sample.forget();
+}
+
+EncodedVideoChunk::EncodedVideoChunk(
+ nsIGlobalObject* aParent, already_AddRefed<MediaAlignedByteBuffer> aBuffer,
+ const EncodedVideoChunkType& aType, int64_t aTimestamp,
+ Maybe<uint64_t>&& aDuration)
+ : EncodedVideoChunkData(std::move(aBuffer), aType, aTimestamp,
+ std::move(aDuration)),
+ mParent(aParent) {}
+
+EncodedVideoChunk::EncodedVideoChunk(nsIGlobalObject* aParent,
+ const EncodedVideoChunkData& aData)
+ : EncodedVideoChunkData(aData), mParent(aParent) {}
+
+nsIGlobalObject* EncodedVideoChunk::GetParentObject() const {
+ AssertIsOnOwningThread();
+
+ return mParent.get();
+}
+
+JSObject* EncodedVideoChunk::WrapObject(JSContext* aCx,
+ JS::Handle<JSObject*> aGivenProto) {
+ AssertIsOnOwningThread();
+
+ return EncodedVideoChunk_Binding::Wrap(aCx, this, aGivenProto);
+}
+
+// https://w3c.github.io/webcodecs/#encodedvideochunk-constructors
+/* static */
+already_AddRefed<EncodedVideoChunk> EncodedVideoChunk::Constructor(
+ const GlobalObject& aGlobal, const EncodedVideoChunkInit& aInit,
+ ErrorResult& aRv) {
+ nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+ if (!global) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return nullptr;
+ }
+
+ auto buffer = ProcessTypedArrays(
+ aInit.mData,
+ [&](const Span<uint8_t>& aData,
+ JS::AutoCheckCannotGC&&) -> RefPtr<MediaAlignedByteBuffer> {
+ // Make sure it's in uint32_t's range.
+ CheckedUint32 byteLength(aData.Length());
+ if (!byteLength.isValid()) {
+ aRv.Throw(NS_ERROR_INVALID_ARG);
+ return nullptr;
+ }
+ if (aData.Length() == 0) {
+ LOGW("Buffer for constructing EncodedVideoChunk is empty!");
+ }
+ RefPtr<MediaAlignedByteBuffer> buf = MakeRefPtr<MediaAlignedByteBuffer>(
+ aData.Elements(), aData.Length());
+
+ // Instead of checking *buf, size comparision is used to allow
+ // constructing a zero-sized EncodedVideoChunk.
+ if (!buf || buf->Size() != aData.Length()) {
+ aRv.Throw(NS_ERROR_OUT_OF_MEMORY);
+ return nullptr;
+ }
+ return buf;
+ });
+
+ RefPtr<EncodedVideoChunk> chunk(new EncodedVideoChunk(
+ global, buffer.forget(), aInit.mType, aInit.mTimestamp,
+ OptionalToMaybe(aInit.mDuration)));
+ return aRv.Failed() ? nullptr : chunk.forget();
+}
+
+EncodedVideoChunkType EncodedVideoChunk::Type() const {
+ AssertIsOnOwningThread();
+
+ return mType;
+}
+
+int64_t EncodedVideoChunk::Timestamp() const {
+ AssertIsOnOwningThread();
+
+ return mTimestamp;
+}
+
+Nullable<uint64_t> EncodedVideoChunk::GetDuration() const {
+ AssertIsOnOwningThread();
+ return MaybeToNullable(mDuration);
+}
+
+uint32_t EncodedVideoChunk::ByteLength() const {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(mBuffer);
+
+ return static_cast<uint32_t>(mBuffer->Length());
+}
+
+// https://w3c.github.io/webcodecs/#dom-encodedvideochunk-copyto
+void EncodedVideoChunk::CopyTo(
+ const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination,
+ ErrorResult& aRv) {
+ AssertIsOnOwningThread();
+
+ ProcessTypedArraysFixed(aDestination, [&](const Span<uint8_t>& aData) {
+ if (mBuffer->Size() > aData.size_bytes()) {
+ aRv.ThrowTypeError(
+ "Destination ArrayBuffer smaller than source EncodedVideoChunk");
+ return;
+ }
+
+ PodCopy(aData.data(), mBuffer->Data(), mBuffer->Size());
+ });
+}
+
+// https://w3c.github.io/webcodecs/#ref-for-deserialization-steps%E2%91%A0
+/* static */
+JSObject* EncodedVideoChunk::ReadStructuredClone(
+ JSContext* aCx, nsIGlobalObject* aGlobal, JSStructuredCloneReader* aReader,
+ const EncodedVideoChunkData& aData) {
+ JS::Rooted<JS::Value> value(aCx, JS::NullValue());
+ // To avoid a rooting hazard error from returning a raw JSObject* before
+ // running the RefPtr destructor, RefPtr needs to be destructed before
+ // returning the raw JSObject*, which is why the RefPtr<EncodedVideoChunk> is
+ // created in the scope below. Otherwise, the static analysis infers the
+ // RefPtr cannot be safely destructed while the unrooted return JSObject* is
+ // on the stack.
+ {
+ auto frame = MakeRefPtr<EncodedVideoChunk>(aGlobal, aData);
+ if (!GetOrCreateDOMReflector(aCx, frame, &value) || !value.isObject()) {
+ return nullptr;
+ }
+ }
+ return value.toObjectOrNull();
+}
+
+// https://w3c.github.io/webcodecs/#ref-for-serialization-steps%E2%91%A0
+bool EncodedVideoChunk::WriteStructuredClone(
+ JSStructuredCloneWriter* aWriter, StructuredCloneHolder* aHolder) const {
+ AssertIsOnOwningThread();
+
+ // Indexing the chunk and send the index to the receiver.
+ const uint32_t index =
+ static_cast<uint32_t>(aHolder->EncodedVideoChunks().Length());
+ // The serialization is limited to the same process scope so it's ok to
+ // serialize a reference instead of a copy.
+ aHolder->EncodedVideoChunks().AppendElement(EncodedVideoChunkData(*this));
+ return !NS_WARN_IF(
+ !JS_WriteUint32Pair(aWriter, SCTAG_DOM_ENCODEDVIDEOCHUNK, index));
+}
+
+#undef LOGW
+#undef LOGE
+#undef LOG_INTERNAL
+
+} // namespace mozilla::dom
diff --git a/dom/media/webcodecs/EncodedVideoChunk.h b/dom/media/webcodecs/EncodedVideoChunk.h
new file mode 100644
index 0000000000..3a9fb1e368
--- /dev/null
+++ b/dom/media/webcodecs/EncodedVideoChunk.h
@@ -0,0 +1,119 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_EncodedVideoChunk_h
+#define mozilla_dom_EncodedVideoChunk_h
+
+#include "js/TypeDecls.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/Buffer.h"
+#include "mozilla/ErrorResult.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/dom/BindingDeclarations.h"
+#include "nsCycleCollectionParticipant.h"
+#include "nsWrapperCache.h"
+
+class nsIGlobalObject;
+
+namespace mozilla {
+
+class MediaAlignedByteBuffer;
+class MediaRawData;
+
+namespace dom {
+
+class MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer;
+class OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer;
+class StructuredCloneHolder;
+
+enum class EncodedVideoChunkType : uint8_t;
+struct EncodedVideoChunkInit;
+
+} // namespace dom
+} // namespace mozilla
+
+namespace mozilla::dom {
+
+class EncodedVideoChunkData {
+ public:
+ EncodedVideoChunkData(already_AddRefed<MediaAlignedByteBuffer> aBuffer,
+ const EncodedVideoChunkType& aType, int64_t aTimestamp,
+ Maybe<uint64_t>&& aDuration);
+ EncodedVideoChunkData(const EncodedVideoChunkData& aData) = default;
+ ~EncodedVideoChunkData();
+
+ UniquePtr<EncodedVideoChunkData> Clone() const;
+ already_AddRefed<MediaRawData> TakeData();
+
+ protected:
+ // mBuffer's byte length is guaranteed to be smaller than UINT32_MAX.
+ RefPtr<MediaAlignedByteBuffer> mBuffer;
+ EncodedVideoChunkType mType;
+ int64_t mTimestamp;
+ Maybe<uint64_t> mDuration;
+};
+
+class EncodedVideoChunk final : public EncodedVideoChunkData,
+ public nsISupports,
+ public nsWrapperCache {
+ public:
+ NS_DECL_CYCLE_COLLECTING_ISUPPORTS
+ NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(EncodedVideoChunk)
+
+ public:
+ EncodedVideoChunk(nsIGlobalObject* aParent,
+ already_AddRefed<MediaAlignedByteBuffer> aBuffer,
+ const EncodedVideoChunkType& aType, int64_t aTimestamp,
+ Maybe<uint64_t>&& aDuration);
+
+ EncodedVideoChunk(nsIGlobalObject* aParent,
+ const EncodedVideoChunkData& aData);
+
+ protected:
+ ~EncodedVideoChunk() = default;
+
+ public:
+ nsIGlobalObject* GetParentObject() const;
+
+ JSObject* WrapObject(JSContext* aCx,
+ JS::Handle<JSObject*> aGivenProto) override;
+
+ static already_AddRefed<EncodedVideoChunk> Constructor(
+ const GlobalObject& aGlobal, const EncodedVideoChunkInit& aInit,
+ ErrorResult& aRv);
+
+ EncodedVideoChunkType Type() const;
+
+ int64_t Timestamp() const;
+
+ Nullable<uint64_t> GetDuration() const;
+
+ uint32_t ByteLength() const;
+
+ void CopyTo(
+ const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination,
+ ErrorResult& aRv);
+
+ // [Serializable] implementations: {Read, Write}StructuredClone
+ static JSObject* ReadStructuredClone(JSContext* aCx, nsIGlobalObject* aGlobal,
+ JSStructuredCloneReader* aReader,
+ const EncodedVideoChunkData& aData);
+
+ bool WriteStructuredClone(JSStructuredCloneWriter* aWriter,
+ StructuredCloneHolder* aHolder) const;
+
+ private:
+ // EncodedVideoChunk can run on either main thread or worker thread.
+ void AssertIsOnOwningThread() const {
+ NS_ASSERT_OWNINGTHREAD(EncodedVideoChunk);
+ }
+
+ nsCOMPtr<nsIGlobalObject> mParent;
+};
+
+} // namespace mozilla::dom
+
+#endif // mozilla_dom_EncodedVideoChunk_h
diff --git a/dom/media/webcodecs/EncoderAgent.cpp b/dom/media/webcodecs/EncoderAgent.cpp
new file mode 100644
index 0000000000..e6af17a0be
--- /dev/null
+++ b/dom/media/webcodecs/EncoderAgent.cpp
@@ -0,0 +1,441 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "EncoderAgent.h"
+
+#include "PDMFactory.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/Logging.h"
+#include "nsThreadUtils.h"
+
+extern mozilla::LazyLogModule gWebCodecsLog;
+
+namespace mozilla {
+
+#ifdef LOG_INTERNAL
+# undef LOG_INTERNAL
+#endif // LOG_INTERNAL
+#define LOG_INTERNAL(level, msg, ...) \
+ MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__))
+
+#ifdef LOG
+# undef LOG
+#endif // LOG
+#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__)
+
+#ifdef LOGW
+# undef LOGW
+#endif // LOGE
+#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__)
+
+#ifdef LOGE
+# undef LOGE
+#endif // LOGE
+#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__)
+
+#ifdef LOGV
+# undef LOGV
+#endif // LOGV
+#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__)
+
+EncoderAgent::EncoderAgent(WebCodecsId aId)
+ : mId(aId),
+ mOwnerThread(GetCurrentSerialEventTarget()),
+ mPEMFactory(MakeRefPtr<PEMFactory>()),
+ mEncoder(nullptr),
+ mState(State::Unconfigured) {
+ MOZ_ASSERT(mOwnerThread);
+ MOZ_ASSERT(mPEMFactory);
+ LOG("EncoderAgent #%zu (%p) ctor", mId, this);
+}
+
+EncoderAgent::~EncoderAgent() {
+ LOG("EncoderAgent #%zu (%p) dtor", mId, this);
+ MOZ_ASSERT(mState == State::Unconfigured, "encoder released in wrong state");
+ MOZ_ASSERT(!mEncoder, "encoder must be shutdown");
+}
+
+RefPtr<EncoderAgent::ConfigurePromise> EncoderAgent::Configure(
+ const EncoderConfig& aConfig) {
+ MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
+ MOZ_ASSERT(mState == State::Unconfigured || mState == State::Error);
+ MOZ_ASSERT(mConfigurePromise.IsEmpty());
+ MOZ_ASSERT(!mCreateRequest.Exists());
+ MOZ_ASSERT(!mInitRequest.Exists());
+
+ if (mState == State::Error) {
+ LOGE("EncoderAgent #%zu (%p) tried to configure in error state", mId, this);
+ return ConfigurePromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ "Cannot configure in error state"),
+ __func__);
+ }
+
+ MOZ_ASSERT(mState == State::Unconfigured);
+ MOZ_ASSERT(!mEncoder);
+ SetState(State::Configuring);
+
+ LOG("EncoderAgent #%zu (%p) is creating an encoder (%s)", mId, this,
+ GetCodecTypeString(aConfig.mCodec));
+
+ RefPtr<ConfigurePromise> p = mConfigurePromise.Ensure(__func__);
+
+ mPEMFactory->CreateEncoderAsync(aConfig, dom::GetWebCodecsEncoderTaskQueue())
+ ->Then(
+ mOwnerThread, __func__,
+ [self = RefPtr{this}](RefPtr<MediaDataEncoder>&& aEncoder) {
+ self->mCreateRequest.Complete();
+
+ // If EncoderAgent has been shut down, shut the created encoder down
+ // and return.
+ if (!self->mShutdownWhileCreationPromise.IsEmpty()) {
+ MOZ_ASSERT(self->mState == State::ShuttingDown);
+ MOZ_ASSERT(self->mConfigurePromise.IsEmpty(),
+ "configuration should have been rejected");
+
+ LOGW(
+ "EncoderAgent #%zu (%p) has been shut down. We need to shut "
+ "the newly created encoder down",
+ self->mId, self.get());
+ aEncoder->Shutdown()->Then(
+ self->mOwnerThread, __func__,
+ [self](const ShutdownPromise::ResolveOrRejectValue& aValue) {
+ MOZ_ASSERT(self->mState == State::ShuttingDown);
+
+ LOGW(
+ "EncoderAgent #%zu (%p), newly created encoder "
+ "shutdown "
+ "has been %s",
+ self->mId, self.get(),
+ aValue.IsResolve() ? "resolved" : "rejected");
+
+ self->SetState(State::Unconfigured);
+
+ self->mShutdownWhileCreationPromise.ResolveOrReject(
+ aValue, __func__);
+ });
+ return;
+ }
+
+ self->mEncoder = aEncoder.forget();
+ LOG("EncoderAgent #%zu (%p) has created a encoder, now initialize "
+ "it",
+ self->mId, self.get());
+ self->mEncoder->Init()
+ ->Then(
+ self->mOwnerThread, __func__,
+ [self]() {
+ self->mInitRequest.Complete();
+ LOG("EncoderAgent #%zu (%p) has initialized the encoder",
+ self->mId, self.get());
+ self->SetState(State::Configured);
+ self->mConfigurePromise.Resolve(true, __func__);
+ },
+ [self](const MediaResult& aError) {
+ self->mInitRequest.Complete();
+ LOGE(
+ "EncoderAgent #%zu (%p) failed to initialize the "
+ "encoder",
+ self->mId, self.get());
+ self->SetState(State::Error);
+ self->mConfigurePromise.Reject(aError, __func__);
+ })
+ ->Track(self->mInitRequest);
+ },
+ [self = RefPtr{this}](const MediaResult& aError) {
+ self->mCreateRequest.Complete();
+ LOGE("EncoderAgent #%zu (%p) failed to create a encoder", self->mId,
+ self.get());
+
+ // If EncoderAgent has been shut down, we need to resolve the
+ // shutdown promise.
+ if (!self->mShutdownWhileCreationPromise.IsEmpty()) {
+ MOZ_ASSERT(self->mState == State::ShuttingDown);
+ MOZ_ASSERT(self->mConfigurePromise.IsEmpty(),
+ "configuration should have been rejected");
+
+ LOGW(
+ "EncoderAgent #%zu (%p) has been shut down. Resolve the "
+ "shutdown promise right away since encoder creation failed",
+ self->mId, self.get());
+
+ self->SetState(State::Unconfigured);
+ self->mShutdownWhileCreationPromise.Resolve(true, __func__);
+ return;
+ }
+
+ self->SetState(State::Error);
+ self->mConfigurePromise.Reject(aError, __func__);
+ })
+ ->Track(mCreateRequest);
+
+ return p;
+}
+
+RefPtr<EncoderAgent::ReconfigurationPromise> EncoderAgent::Reconfigure(
+ const RefPtr<const EncoderConfigurationChangeList>& aConfigChanges) {
+ MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
+ MOZ_ASSERT(mState == State::Configured || mState == State::Error);
+ MOZ_ASSERT(mReconfigurationPromise.IsEmpty());
+
+ if (mState == State::Error) {
+ LOGE("EncoderAgent #%zu (%p) tried to reconfigure in error state", mId,
+ this);
+ return ReconfigurationPromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ "Cannot reconfigure in error state"),
+ __func__);
+ }
+
+ MOZ_ASSERT(mEncoder);
+ SetState(State::Configuring);
+
+ LOG("EncoderAgent #%zu (%p) is reconfiguring its encoder (%s)", mId, this,
+ NS_ConvertUTF16toUTF8(aConfigChanges->ToString().get()).get());
+
+ RefPtr<ReconfigurationPromise> p = mReconfigurationPromise.Ensure(__func__);
+
+ mEncoder->Reconfigure(aConfigChanges)
+ ->Then(
+ mOwnerThread, __func__,
+ [self = RefPtr{this}](bool) {
+ self->mReconfigurationRequest.Complete();
+ LOGE("EncoderAgent #%zu (%p) reconfigure success", self->mId,
+ self.get());
+ self->mReconfigurationPromise.Resolve(true, __func__);
+ },
+ [self = RefPtr{this}](const MediaResult& aError) {
+ self->mReconfigurationRequest.Complete();
+ LOGE("EncoderAgent #%zu (%p) reconfigure failure", self->mId,
+ self.get());
+ // Not a a fatal error per se, the owner will deal with it.
+ self->mReconfigurationPromise.Reject(aError, __func__);
+ })
+ ->Track(mReconfigurationRequest);
+
+ return p;
+}
+
+RefPtr<ShutdownPromise> EncoderAgent::Shutdown() {
+ MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
+
+ auto r =
+ MediaResult(NS_ERROR_DOM_MEDIA_CANCELED, "Canceled by encoder shutdown");
+
+ // If the encoder creation has not been completed yet, wait until the encoder
+ // being created has been shut down.
+ if (mCreateRequest.Exists()) {
+ MOZ_ASSERT(!mInitRequest.Exists());
+ MOZ_ASSERT(!mConfigurePromise.IsEmpty());
+ MOZ_ASSERT(!mEncoder);
+ MOZ_ASSERT(mState == State::Configuring);
+ MOZ_ASSERT(mShutdownWhileCreationPromise.IsEmpty());
+
+ LOGW(
+ "EncoderAgent #%zu (%p) shutdown while the encoder creation for "
+ "configuration is in flight. Reject the configuration now and defer "
+ "the shutdown until the created encoder has been shut down",
+ mId, this);
+
+ // Reject the configuration in flight.
+ mConfigurePromise.Reject(r, __func__);
+
+ // Get the promise that will be resolved when the encoder being created has
+ // been destroyed.
+ SetState(State::ShuttingDown);
+ return mShutdownWhileCreationPromise.Ensure(__func__);
+ }
+
+ // If encoder creation has been completed, we must have the encoder now.
+ MOZ_ASSERT(mEncoder);
+
+ // Cancel pending initialization for configuration in flight if any.
+ mInitRequest.DisconnectIfExists();
+ mConfigurePromise.RejectIfExists(r, __func__);
+
+ mReconfigurationRequest.DisconnectIfExists();
+ mReconfigurationPromise.RejectIfExists(r, __func__);
+
+ // Cancel encoder in flight if any.
+ mEncodeRequest.DisconnectIfExists();
+ mEncodePromise.RejectIfExists(r, __func__);
+
+ // Cancel flush-out in flight if any.
+ mDrainRequest.DisconnectIfExists();
+ mEncodeRequest.DisconnectIfExists();
+
+ mDrainRequest.DisconnectIfExists();
+ mDrainPromise.RejectIfExists(r, __func__);
+
+ SetState(State::Unconfigured);
+
+ RefPtr<MediaDataEncoder> encoder = std::move(mEncoder);
+ return encoder->Shutdown();
+}
+
+RefPtr<EncoderAgent::EncodePromise> EncoderAgent::Encode(MediaData* aInput) {
+ MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
+ MOZ_ASSERT(aInput);
+ MOZ_ASSERT(mState == State::Configured || mState == State::Error);
+ MOZ_ASSERT(mEncodePromise.IsEmpty());
+ MOZ_ASSERT(!mEncodeRequest.Exists());
+
+ if (mState == State::Error) {
+ LOGE("EncoderAgent #%zu (%p) tried to encoder in error state", mId, this);
+ return EncodePromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ "Cannot encoder in error state"),
+ __func__);
+ }
+
+ MOZ_ASSERT(mState == State::Configured);
+ MOZ_ASSERT(mEncoder);
+ SetState(State::Encoding);
+
+ RefPtr<EncodePromise> p = mEncodePromise.Ensure(__func__);
+
+ mEncoder->Encode(aInput)
+ ->Then(
+ mOwnerThread, __func__,
+ [self = RefPtr{this}](MediaDataEncoder::EncodedData&& aData) {
+ self->mEncodeRequest.Complete();
+ LOGV("EncoderAgent #%zu (%p) encode successful", self->mId,
+ self.get());
+ self->SetState(State::Configured);
+ self->mEncodePromise.Resolve(std::move(aData), __func__);
+ },
+ [self = RefPtr{this}](const MediaResult& aError) {
+ self->mEncodeRequest.Complete();
+ LOGV("EncoderAgent #%zu (%p) failed to encode", self->mId,
+ self.get());
+ self->SetState(State::Error);
+ self->mEncodePromise.Reject(aError, __func__);
+ })
+ ->Track(mEncodeRequest);
+
+ return p;
+}
+
+RefPtr<EncoderAgent::EncodePromise> EncoderAgent::Drain() {
+ MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
+ // This can be called when reconfiguring the encoder.
+ MOZ_ASSERT(mState == State::Configured || mState == State::Configuring);
+ MOZ_ASSERT(mDrainPromise.IsEmpty());
+ MOZ_ASSERT(mEncoder);
+
+ SetState(State::Flushing);
+
+ RefPtr<EncodePromise> p = mDrainPromise.Ensure(__func__);
+ DryUntilDrain();
+ return p;
+}
+
+void EncoderAgent::DryUntilDrain() {
+ MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
+ MOZ_ASSERT(mState == State::Flushing);
+ MOZ_ASSERT(!mDrainRequest.Exists());
+ MOZ_ASSERT(mEncoder);
+
+ LOG("EncoderAgent #%zu (%p) is draining the encoder", mId, this);
+ mEncoder->Drain()
+ ->Then(
+ mOwnerThread, __func__,
+ [self = RefPtr{this}](MediaDataEncoder::EncodedData&& aData) {
+ self->mDrainRequest.Complete();
+
+ if (aData.IsEmpty()) {
+ LOG("EncoderAgent #%zu (%p) is dry now", self->mId, self.get());
+ self->SetState(State::Configured);
+ self->mDrainPromise.Resolve(std::move(self->mDrainData),
+ __func__);
+ return;
+ }
+
+ LOG("EncoderAgent #%zu (%p) drained %zu encoder data. Keep "
+ "draining "
+ "until dry",
+ self->mId, self.get(), aData.Length());
+ self->mDrainData.AppendElements(std::move(aData));
+ self->DryUntilDrain();
+ },
+ [self = RefPtr{this}](const MediaResult& aError) {
+ self->mDrainRequest.Complete();
+
+ LOGE("EncoderAgent %p failed to drain encoder", self.get());
+ self->mDrainData.Clear();
+ self->mDrainPromise.Reject(aError, __func__);
+ })
+ ->Track(mDrainRequest);
+}
+
+void EncoderAgent::SetState(State aState) {
+ MOZ_ASSERT(mOwnerThread->IsOnCurrentThread());
+
+ auto validateStateTransition = [](State aOldState, State aNewState) {
+ switch (aOldState) {
+ case State::Unconfigured:
+ return aNewState == State::Configuring;
+ case State::Configuring:
+ return aNewState == State::Configured || aNewState == State::Error ||
+ aNewState == State::Flushing ||
+ aNewState == State::Unconfigured ||
+ aNewState == State::ShuttingDown;
+ case State::Configured:
+ return aNewState == State::Unconfigured ||
+ aNewState == State::Configuring ||
+ aNewState == State::Encoding || aNewState == State::Flushing;
+ case State::Encoding:
+ case State::Flushing:
+ return aNewState == State::Configured || aNewState == State::Error ||
+ aNewState == State::Unconfigured;
+ case State::ShuttingDown:
+ return aNewState == State::Unconfigured;
+ case State::Error:
+ return aNewState == State::Unconfigured;
+ default:
+ break;
+ }
+ MOZ_ASSERT_UNREACHABLE("Unhandled state transition");
+ return false;
+ };
+
+ auto stateToString = [](State aState) -> const char* {
+ switch (aState) {
+ case State::Unconfigured:
+ return "Unconfigured";
+ case State::Configuring:
+ return "Configuring";
+ case State::Configured:
+ return "Configured";
+ case State::Encoding:
+ return "Encoding";
+ case State::Flushing:
+ return "Flushing";
+ case State::ShuttingDown:
+ return "ShuttingDown";
+ case State::Error:
+ return "Error";
+ default:
+ break;
+ }
+ MOZ_ASSERT_UNREACHABLE("Unhandled state type");
+ return "Unknown";
+ };
+
+ DebugOnly<bool> isValid = validateStateTransition(mState, aState);
+ LOGV("EncoderAgent #%zu (%p) state change: %s -> %s", mId, this,
+ stateToString(mState), stateToString(aState));
+ MOZ_ASSERT(isValid);
+ mState = aState;
+}
+
+#undef LOG
+#undef LOGW
+#undef LOGE
+#undef LOGV
+#undef LOG_INTERNAL
+
+} // namespace mozilla
diff --git a/dom/media/webcodecs/EncoderAgent.h b/dom/media/webcodecs/EncoderAgent.h
new file mode 100644
index 0000000000..387f218e29
--- /dev/null
+++ b/dom/media/webcodecs/EncoderAgent.h
@@ -0,0 +1,116 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef DOM_MEDIA_WEBCODECS_EncoderAgent_H
+#define DOM_MEDIA_WEBCODECS_EncoderAgent_H
+
+#include "MediaResult.h"
+#include "PlatformEncoderModule.h"
+#include "PEMFactory.h"
+#include "mozilla/RefPtr.h"
+#include "mozilla/TaskQueue.h"
+#include "WebCodecsUtils.h"
+
+class nsISerialEventTarget;
+
+namespace mozilla {
+
+class PDMFactory;
+class TrackInfo;
+
+namespace layers {
+class ImageContainer;
+} // namespace layers
+
+// EncoderAgent is a wrapper that contains a MediaDataEncoder. It adapts the
+// MediaDataEncoder APIs for use in WebCodecs.
+//
+// If Configure() is called, Shutdown() must be called to release the resources
+// gracefully. Except Shutdown(), all the methods can't be called concurrently,
+// meaning a method can only be called when the previous API call has completed.
+// The responsability of arranging the method calls is on the caller.
+//
+// When Shutdown() is called, all the operations in flight are canceled and the
+// MediaDataEncoder is shut down. On the other hand, errors are final. A new
+// EncoderAgent must be created when an error is encountered.
+//
+// All the methods need to be called on the EncoderAgent's owner thread. In
+// WebCodecs, it's either on the main thread or worker thread.
+class EncoderAgent final {
+ public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(EncoderAgent);
+
+ explicit EncoderAgent(WebCodecsId aId);
+
+ // The following APIs are owner thread only.
+
+ using ConfigurePromise = MozPromise<bool, MediaResult, true /* exclusive */>;
+ using ReconfigurationPromise = MediaDataEncoder::ReconfigurationPromise;
+ RefPtr<ConfigurePromise> Configure(const EncoderConfig& aConfig);
+ RefPtr<ReconfigurationPromise> Reconfigure(
+ const RefPtr<const EncoderConfigurationChangeList>& aConfigChange);
+ RefPtr<ShutdownPromise> Shutdown();
+ using EncodePromise = MediaDataEncoder::EncodePromise;
+ RefPtr<EncodePromise> Encode(MediaData* aInput);
+ // WebCodecs's flush() flushes out all the pending encoded data in the
+ // encoder. It's called Drain internally.
+ RefPtr<EncodePromise> Drain();
+
+ const WebCodecsId mId;
+
+ private:
+ ~EncoderAgent();
+
+ // Push out all the data in the MediaDataEncoder's pipeline.
+ // TODO: MediaDataEncoder should implement this, instead of asking call site
+ // to run `Drain` multiple times.
+ RefPtr<EncodePromise> Dry();
+ void DryUntilDrain();
+
+ enum class State {
+ Unconfigured,
+ Configuring,
+ Configured,
+ Encoding,
+ Flushing,
+ ShuttingDown,
+ Error,
+ };
+ void SetState(State aState);
+
+ const RefPtr<nsISerialEventTarget> mOwnerThread;
+ const RefPtr<PEMFactory> mPEMFactory;
+ RefPtr<MediaDataEncoder> mEncoder;
+ State mState;
+
+ // Configure
+ MozPromiseHolder<ConfigurePromise> mConfigurePromise;
+ using CreateEncoderPromise = PlatformEncoderModule::CreateEncoderPromise;
+ MozPromiseRequestHolder<CreateEncoderPromise> mCreateRequest;
+ using InitPromise = MediaDataEncoder::InitPromise;
+ MozPromiseRequestHolder<InitPromise> mInitRequest;
+
+ // Reconfigure
+ MozPromiseHolder<ReconfigurationPromise> mReconfigurationPromise;
+ using ReconfigureEncoderRequest = ReconfigurationPromise;
+ MozPromiseRequestHolder<ReconfigureEncoderRequest> mReconfigurationRequest;
+
+ // Shutdown
+ MozPromiseHolder<ShutdownPromise> mShutdownWhileCreationPromise;
+
+ // Encoding
+ MozPromiseHolder<EncodePromise> mEncodePromise;
+ MozPromiseRequestHolder<EncodePromise> mEncodeRequest;
+
+ // Drain
+ MozPromiseRequestHolder<EncodePromise> mDrainRequest;
+ MozPromiseHolder<EncodePromise> mDrainPromise;
+ MediaDataEncoder::EncodedData mDrainData;
+};
+
+} // namespace mozilla
+
+#endif // DOM_MEDIA_WEBCODECS_EncoderAgent_H
diff --git a/dom/media/webcodecs/EncoderTemplate.cpp b/dom/media/webcodecs/EncoderTemplate.cpp
new file mode 100644
index 0000000000..35c8feb3f8
--- /dev/null
+++ b/dom/media/webcodecs/EncoderTemplate.cpp
@@ -0,0 +1,1228 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "EncoderTemplate.h"
+
+#include "EncoderTypes.h"
+#include "mozilla/ScopeExit.h"
+#include "mozilla/Try.h"
+#include "mozilla/Unused.h"
+#include "mozilla/dom/DOMException.h"
+#include "mozilla/dom/Event.h"
+#include "mozilla/dom/Promise.h"
+#include "mozilla/dom/VideoFrame.h"
+#include "mozilla/dom/WorkerCommon.h"
+#include "nsGkAtoms.h"
+#include "nsString.h"
+#include "nsThreadUtils.h"
+
+extern mozilla::LazyLogModule gWebCodecsLog;
+
+namespace mozilla::dom {
+
+#ifdef LOG_INTERNAL
+# undef LOG_INTERNAL
+#endif // LOG_INTERNAL
+#define LOG_INTERNAL(level, msg, ...) \
+ MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__))
+
+#ifdef LOG
+# undef LOG
+#endif // LOG
+#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__)
+
+#ifdef LOGW
+# undef LOGW
+#endif // LOGW
+#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__)
+
+#ifdef LOGE
+# undef LOGE
+#endif // LOGE
+#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__)
+
+#ifdef LOGV
+# undef LOGV
+#endif // LOGV
+#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__)
+
+/*
+ * Below are ControlMessage classes implementations
+ */
+
+template <typename EncoderType>
+EncoderTemplate<EncoderType>::ControlMessage::ControlMessage(
+ WebCodecsId aConfigureId)
+ : mConfigureId(aConfigureId), mMessageId(sNextId++) {}
+
+template <typename EncoderType>
+EncoderTemplate<EncoderType>::ConfigureMessage::ConfigureMessage(
+ WebCodecsId aConfigureId, const RefPtr<ConfigTypeInternal>& aConfig)
+ : ControlMessage(aConfigureId), mConfig(aConfig) {}
+
+template <typename EncoderType>
+EncoderTemplate<EncoderType>::EncodeMessage::EncodeMessage(
+ WebCodecsId aConfigureId, RefPtr<InputTypeInternal>&& aData,
+ Maybe<VideoEncoderEncodeOptions>&& aOptions)
+ : ControlMessage(aConfigureId), mData(aData) {}
+
+template <typename EncoderType>
+EncoderTemplate<EncoderType>::FlushMessage::FlushMessage(
+ WebCodecsId aConfigureId, Promise* aPromise)
+ : ControlMessage(aConfigureId), mPromise(aPromise) {}
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::FlushMessage::RejectPromiseIfAny(
+ const nsresult& aReason) {
+ if (mPromise) {
+ mPromise->MaybeReject(aReason);
+ }
+}
+
+/*
+ * Below are EncoderTemplate implementation
+ */
+
+template <typename EncoderType>
+EncoderTemplate<EncoderType>::EncoderTemplate(
+ nsIGlobalObject* aGlobalObject,
+ RefPtr<WebCodecsErrorCallback>&& aErrorCallback,
+ RefPtr<OutputCallbackType>&& aOutputCallback)
+ : DOMEventTargetHelper(aGlobalObject),
+ mErrorCallback(std::move(aErrorCallback)),
+ mOutputCallback(std::move(aOutputCallback)),
+ mState(CodecState::Unconfigured),
+ mMessageQueueBlocked(false),
+ mEncodeQueueSize(0),
+ mDequeueEventScheduled(false),
+ mLatestConfigureId(0),
+ mEncodeCounter(0),
+ mFlushCounter(0) {}
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::Configure(const ConfigType& aConfig,
+ ErrorResult& aRv) {
+ AssertIsOnOwningThread();
+
+ LOG("%s::Configure %p codec %s", EncoderType::Name.get(), this,
+ NS_ConvertUTF16toUTF8(aConfig.mCodec).get());
+
+ nsCString errorMessage;
+ if (!EncoderType::Validate(aConfig, errorMessage)) {
+ LOG("Configure: Validate error: %s", errorMessage.get());
+ aRv.ThrowTypeError(errorMessage);
+ return;
+ }
+
+ if (mState == CodecState::Closed) {
+ LOG("Configure: CodecState::Closed, rejecting with InvalidState");
+ aRv.ThrowInvalidStateError("The codec is no longer usable");
+ return;
+ }
+
+ // Clone a ConfigType as the active Encode config.
+ RefPtr<ConfigTypeInternal> config =
+ EncoderType::CreateConfigInternal(aConfig);
+ if (!config) {
+ aRv.Throw(NS_ERROR_UNEXPECTED); // Invalid description data.
+ return;
+ }
+
+ mState = CodecState::Configured;
+ mEncodeCounter = 0;
+ mFlushCounter = 0;
+
+ mControlMessageQueue.push(MakeRefPtr<ConfigureMessage>(sNextId++, config));
+ mLatestConfigureId = mControlMessageQueue.back()->mMessageId;
+ LOG("%s %p enqueues %s", EncoderType::Name.get(), this,
+ mControlMessageQueue.back()->ToString().get());
+ ProcessControlMessageQueue();
+}
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::EncodeAudioData(InputType& aInput,
+ ErrorResult& aRv) {
+ AssertIsOnOwningThread();
+
+ LOG("%s %p, EncodeAudioData", EncoderType::Name.get(), this);
+
+ if (mState != CodecState::Configured) {
+ aRv.ThrowInvalidStateError("Encoder must be configured first");
+ return;
+ }
+
+ if (aInput.IsClosed()) {
+ aRv.ThrowTypeError("input AudioData has been closed");
+ return;
+ }
+
+ mEncodeQueueSize += 1;
+ // Dummy options here as a shortcut
+ mControlMessageQueue.push(MakeRefPtr<EncodeMessage>(
+ mLatestConfigureId,
+ EncoderType::CreateInputInternal(aInput, VideoEncoderEncodeOptions())));
+ LOGV("%s %p enqueues %s", EncoderType::Name.get(), this,
+ mControlMessageQueue.back()->ToString().get());
+ ProcessControlMessageQueue();
+}
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::EncodeVideoFrame(
+ InputType& aInput, const VideoEncoderEncodeOptions& aOptions,
+ ErrorResult& aRv) {
+ AssertIsOnOwningThread();
+
+ LOG("%s::Encode %p %s", EncoderType::Name.get(), this,
+ aInput.ToString().get());
+
+ if (mState != CodecState::Configured) {
+ aRv.ThrowInvalidStateError("Encoder must be configured first");
+ return;
+ }
+
+ if (aInput.IsClosed()) {
+ aRv.ThrowTypeError("input VideoFrame has been closed");
+ return;
+ }
+
+ mEncodeQueueSize += 1;
+ mControlMessageQueue.push(MakeRefPtr<EncodeMessage>(
+ mLatestConfigureId, EncoderType::CreateInputInternal(aInput, aOptions),
+ Some(aOptions)));
+ LOGV("%s %p enqueues %s", EncoderType::Name.get(), this,
+ mControlMessageQueue.back()->ToString().get());
+ ProcessControlMessageQueue();
+}
+
+template <typename EncoderType>
+already_AddRefed<Promise> EncoderTemplate<EncoderType>::Flush(
+ ErrorResult& aRv) {
+ AssertIsOnOwningThread();
+
+ LOG("%s::Flush %p", EncoderType::Name.get(), this);
+
+ if (mState != CodecState::Configured) {
+ LOG("%s %p, wrong state!", EncoderType::Name.get(), this);
+ aRv.ThrowInvalidStateError("Encoder must be configured first");
+ return nullptr;
+ }
+
+ RefPtr<Promise> p = Promise::Create(GetParentObject(), aRv);
+ if (NS_WARN_IF(aRv.Failed())) {
+ return p.forget();
+ }
+
+ mControlMessageQueue.push(MakeRefPtr<FlushMessage>(mLatestConfigureId, p));
+ LOG("%s %p enqueues %s", EncoderType::Name.get(), this,
+ mControlMessageQueue.back()->ToString().get());
+ ProcessControlMessageQueue();
+ return p.forget();
+}
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::Reset(ErrorResult& aRv) {
+ AssertIsOnOwningThread();
+
+ LOG("%s %p, Reset", EncoderType::Name.get(), this);
+
+ if (auto r = ResetInternal(NS_ERROR_DOM_ABORT_ERR); r.isErr()) {
+ aRv.Throw(r.unwrapErr());
+ }
+}
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::Close(ErrorResult& aRv) {
+ AssertIsOnOwningThread();
+
+ LOG("%s::Close %p", EncoderType::Name.get(), this);
+
+ if (auto r = CloseInternal(NS_ERROR_DOM_ABORT_ERR); r.isErr()) {
+ aRv.Throw(r.unwrapErr());
+ }
+}
+
+template <typename EncoderType>
+Result<Ok, nsresult> EncoderTemplate<EncoderType>::ResetInternal(
+ const nsresult& aResult) {
+ AssertIsOnOwningThread();
+
+ LOG("%s::Reset %p", EncoderType::Name.get(), this);
+
+ if (mState == CodecState::Closed) {
+ return Err(NS_ERROR_DOM_INVALID_STATE_ERR);
+ }
+
+ mState = CodecState::Unconfigured;
+ mEncodeCounter = 0;
+ mFlushCounter = 0;
+
+ CancelPendingControlMessages(aResult);
+ DestroyEncoderAgentIfAny();
+
+ if (mEncodeQueueSize > 0) {
+ mEncodeQueueSize = 0;
+ ScheduleDequeueEvent();
+ }
+
+ StopBlockingMessageQueue();
+
+ return Ok();
+}
+
+template <typename EncoderType>
+Result<Ok, nsresult> EncoderTemplate<EncoderType>::CloseInternal(
+ const nsresult& aResult) {
+ AssertIsOnOwningThread();
+
+ MOZ_TRY(ResetInternal(aResult));
+ mState = CodecState::Closed;
+ if (aResult != NS_ERROR_DOM_ABORT_ERR) {
+ nsCString error;
+ GetErrorName(aResult, error);
+ LOGE("%s %p Close on error: %s", EncoderType::Name.get(), this,
+ error.get());
+ ReportError(aResult);
+ }
+ return Ok();
+}
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::ReportError(const nsresult& aResult) {
+ AssertIsOnOwningThread();
+
+ RefPtr<DOMException> e = DOMException::Create(aResult);
+ RefPtr<WebCodecsErrorCallback> cb(mErrorCallback);
+ cb->Call(*e);
+}
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::OutputEncodedData(
+ nsTArray<RefPtr<MediaRawData>>&& aData) {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(mState == CodecState::Configured);
+ MOZ_ASSERT(mActiveConfig);
+
+ // Get JSContext for RootedDictionary.
+ // The EncoderType::MetadataType, VideoDecoderConfig, and VideoColorSpaceInit
+ // below are rooted to work around the JS hazard issues.
+ AutoJSAPI jsapi;
+ DebugOnly<bool> ok =
+ jsapi.Init(GetParentObject()); // TODO: check returned value?
+ JSContext* cx = jsapi.cx();
+
+ RefPtr<typename EncoderType::OutputCallbackType> cb(mOutputCallback);
+ for (auto& data : aData) {
+ // It's possible to have reset() called in between this task having been
+ // dispatched, and running -- no output callback should happen when that's
+ // the case.
+ // This is imprecise in the spec, but discussed in
+ // https://github.com/w3c/webcodecs/issues/755 and agreed upon.
+ if (!mActiveConfig) {
+ return;
+ }
+ RefPtr<typename EncoderType::OutputType> encodedData =
+ EncodedDataToOutputType(GetParentObject(), data);
+
+ RootedDictionary<typename EncoderType::MetadataType> metadata(cx);
+ if (mOutputNewDecoderConfig) {
+ VideoDecoderConfigInternal decoderConfigInternal =
+ EncoderConfigToDecoderConfig(GetParentObject(), data, *mActiveConfig);
+
+ // Convert VideoDecoderConfigInternal to VideoDecoderConfig
+ RootedDictionary<VideoDecoderConfig> decoderConfig(cx);
+ decoderConfig.mCodec = decoderConfigInternal.mCodec;
+ if (decoderConfigInternal.mCodedHeight) {
+ decoderConfig.mCodedHeight.Construct(
+ decoderConfigInternal.mCodedHeight.value());
+ }
+ if (decoderConfigInternal.mCodedWidth) {
+ decoderConfig.mCodedWidth.Construct(
+ decoderConfigInternal.mCodedWidth.value());
+ }
+ if (decoderConfigInternal.mColorSpace) {
+ RootedDictionary<VideoColorSpaceInit> colorSpace(cx);
+ colorSpace.mFullRange =
+ MaybeToNullable(decoderConfigInternal.mColorSpace->mFullRange);
+ colorSpace.mMatrix =
+ MaybeToNullable(decoderConfigInternal.mColorSpace->mMatrix);
+ colorSpace.mPrimaries =
+ MaybeToNullable(decoderConfigInternal.mColorSpace->mPrimaries);
+ colorSpace.mTransfer =
+ MaybeToNullable(decoderConfigInternal.mColorSpace->mTransfer);
+ decoderConfig.mColorSpace.Construct(std::move(colorSpace));
+ }
+ if (decoderConfigInternal.mDescription &&
+ !decoderConfigInternal.mDescription.value()->IsEmpty()) {
+ auto& abov = decoderConfig.mDescription.Construct();
+ AutoEntryScript aes(GetParentObject(), "EncoderConfigToDecoderConfig");
+ size_t lengthBytes =
+ decoderConfigInternal.mDescription.value()->Length();
+ UniquePtr<uint8_t[], JS::FreePolicy> extradata(
+ new uint8_t[lengthBytes]);
+ PodCopy(extradata.get(),
+ decoderConfigInternal.mDescription.value()->Elements(),
+ lengthBytes);
+ JS::Rooted<JSObject*> description(
+ aes.cx(), JS::NewArrayBufferWithContents(aes.cx(), lengthBytes,
+ std::move(extradata)));
+ JS::Rooted<JS::Value> value(aes.cx(), JS::ObjectValue(*description));
+ DebugOnly<bool> rv = abov.Init(aes.cx(), value);
+ }
+ if (decoderConfigInternal.mDisplayAspectHeight) {
+ decoderConfig.mDisplayAspectHeight.Construct(
+ decoderConfigInternal.mDisplayAspectHeight.value());
+ }
+ if (decoderConfigInternal.mDisplayAspectWidth) {
+ decoderConfig.mDisplayAspectWidth.Construct(
+ decoderConfigInternal.mDisplayAspectWidth.value());
+ }
+ if (decoderConfigInternal.mOptimizeForLatency) {
+ decoderConfig.mOptimizeForLatency.Construct(
+ decoderConfigInternal.mOptimizeForLatency.value());
+ }
+
+ metadata.mDecoderConfig.Construct(std::move(decoderConfig));
+ mOutputNewDecoderConfig = false;
+ LOGE("New config passed to output callback: %s",
+ NS_ConvertUTF16toUTF8(decoderConfigInternal.ToString()).get());
+ }
+
+ nsAutoCString metadataInfo;
+
+ if (data->mTemporalLayerId) {
+ RootedDictionary<SvcOutputMetadata> svc(cx);
+ svc.mTemporalLayerId.Construct(data->mTemporalLayerId.value());
+ metadata.mSvc.Construct(std::move(svc));
+ metadataInfo.Append(
+ nsPrintfCString(", temporal layer id %d",
+ metadata.mSvc.Value().mTemporalLayerId.Value()));
+ }
+
+ if (metadata.mDecoderConfig.WasPassed()) {
+ metadataInfo.Append(", new decoder config");
+ }
+
+ LOG("EncoderTemplate:: output callback (ts: % " PRId64 ")%s",
+ encodedData->Timestamp(), metadataInfo.get());
+ cb->Call((typename EncoderType::OutputType&)(*encodedData), metadata);
+ }
+}
+
+template <typename EncoderType>
+class EncoderTemplate<EncoderType>::ErrorRunnable final
+ : public DiscardableRunnable {
+ public:
+ ErrorRunnable(Self* aEncoder, const nsresult& aError)
+ : DiscardableRunnable("Decoder ErrorRunnable"),
+ mEncoder(aEncoder),
+ mError(aError) {
+ MOZ_ASSERT(mEncoder);
+ }
+ ~ErrorRunnable() = default;
+
+ // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT.
+ // See bug 1535398.
+ MOZ_CAN_RUN_SCRIPT_BOUNDARY NS_IMETHOD Run() override {
+ nsCString error;
+ GetErrorName(mError, error);
+ LOGE("%s %p report error: %s", EncoderType::Name.get(), mEncoder.get(),
+ error.get());
+ RefPtr<Self> d = std::move(mEncoder);
+ d->ReportError(mError);
+ return NS_OK;
+ }
+
+ private:
+ RefPtr<Self> mEncoder;
+ const nsresult mError;
+};
+
+template <typename EncoderType>
+class EncoderTemplate<EncoderType>::OutputRunnable final
+ : public DiscardableRunnable {
+ public:
+ OutputRunnable(Self* aEncoder, WebCodecsId aConfigureId,
+ const nsACString& aLabel,
+ nsTArray<RefPtr<MediaRawData>>&& aData)
+ : DiscardableRunnable("Decoder OutputRunnable"),
+ mEncoder(aEncoder),
+ mConfigureId(aConfigureId),
+ mLabel(aLabel),
+ mData(std::move(aData)) {
+ MOZ_ASSERT(mEncoder);
+ }
+ ~OutputRunnable() = default;
+
+ // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT.
+ // See bug 1535398.
+ MOZ_CAN_RUN_SCRIPT_BOUNDARY NS_IMETHOD Run() override {
+ if (mEncoder->mState != CodecState::Configured) {
+ LOGV("%s %p has been %s. Discard %s-result for EncoderAgent #%zu",
+ EncoderType::Name.get(), mEncoder.get(),
+ mEncoder->mState == CodecState::Closed ? "closed" : "reset",
+ mLabel.get(), mConfigureId);
+ return NS_OK;
+ }
+
+ MOZ_ASSERT(mEncoder->mAgent);
+ if (mConfigureId != mEncoder->mAgent->mId) {
+ LOGW(
+ "%s %p has been re-configured. Still yield %s-result for "
+ "EncoderAgent #%zu",
+ EncoderType::Name.get(), mEncoder.get(), mLabel.get(), mConfigureId);
+ }
+
+ LOGV("%s %p, yields %s-result for EncoderAgent #%zu",
+ EncoderType::Name.get(), mEncoder.get(), mLabel.get(), mConfigureId);
+ RefPtr<Self> d = std::move(mEncoder);
+ d->OutputEncodedData(std::move(mData));
+
+ return NS_OK;
+ }
+
+ private:
+ RefPtr<Self> mEncoder;
+ const WebCodecsId mConfigureId;
+ const nsCString mLabel;
+ nsTArray<RefPtr<MediaRawData>> mData;
+};
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::ScheduleOutputEncodedData(
+ nsTArray<RefPtr<MediaRawData>>&& aData, const nsACString& aLabel) {
+ MOZ_ASSERT(mState == CodecState::Configured);
+ MOZ_ASSERT(mAgent);
+
+ MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(MakeAndAddRef<OutputRunnable>(
+ this, mAgent->mId, aLabel, std::move(aData))));
+}
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::ScheduleClose(const nsresult& aResult) {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(mState == CodecState::Configured);
+
+ auto task = [self = RefPtr{this}, result = aResult] {
+ if (self->mState == CodecState::Closed) {
+ nsCString error;
+ GetErrorName(result, error);
+ LOGW("%s %p has been closed. Ignore close with %s",
+ EncoderType::Name.get(), self.get(), error.get());
+ return;
+ }
+ DebugOnly<Result<Ok, nsresult>> r = self->CloseInternal(result);
+ MOZ_ASSERT(r.value.isOk());
+ };
+ nsISerialEventTarget* target = GetCurrentSerialEventTarget();
+
+ if (NS_IsMainThread()) {
+ MOZ_ALWAYS_SUCCEEDS(target->Dispatch(
+ NS_NewRunnableFunction("ScheduleClose Runnable (main)", task)));
+ return;
+ }
+
+ MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewCancelableRunnableFunction(
+ "ScheduleClose Runnable (worker)", task)));
+}
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::ScheduleDequeueEvent() {
+ AssertIsOnOwningThread();
+
+ if (mDequeueEventScheduled) {
+ return;
+ }
+ mDequeueEventScheduled = true;
+
+ auto dispatcher = [self = RefPtr{this}] {
+ self->FireEvent(nsGkAtoms::ondequeue, u"dequeue"_ns);
+ self->mDequeueEventScheduled = false;
+ };
+ nsISerialEventTarget* target = GetCurrentSerialEventTarget();
+
+ if (NS_IsMainThread()) {
+ MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewRunnableFunction(
+ "ScheduleDequeueEvent Runnable (main)", dispatcher)));
+ return;
+ }
+
+ MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewCancelableRunnableFunction(
+ "ScheduleDequeueEvent Runnable (worker)", dispatcher)));
+}
+
+template <typename EncoderType>
+nsresult EncoderTemplate<EncoderType>::FireEvent(nsAtom* aTypeWithOn,
+ const nsAString& aEventType) {
+ if (aTypeWithOn && !HasListenersFor(aTypeWithOn)) {
+ return NS_ERROR_ABORT;
+ }
+
+ LOGV("Dispatching %s event to %s %p", NS_ConvertUTF16toUTF8(aEventType).get(),
+ EncoderType::Name.get(), this);
+ RefPtr<Event> event = new Event(this, nullptr, nullptr);
+ event->InitEvent(aEventType, true, true);
+ event->SetTrusted(true);
+ this->DispatchEvent(*event);
+ return NS_OK;
+}
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::SchedulePromiseResolveOrReject(
+ already_AddRefed<Promise> aPromise, const nsresult& aResult) {
+ AssertIsOnOwningThread();
+
+ RefPtr<Promise> p = aPromise;
+ auto resolver = [p, result = aResult] {
+ if (NS_FAILED(result)) {
+ p->MaybeReject(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ return;
+ }
+ p->MaybeResolveWithUndefined();
+ };
+ nsISerialEventTarget* target = GetCurrentSerialEventTarget();
+
+ if (NS_IsMainThread()) {
+ MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewRunnableFunction(
+ "SchedulePromiseResolveOrReject Runnable (main)", resolver)));
+ return;
+ }
+
+ MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewCancelableRunnableFunction(
+ "SchedulePromiseResolveOrReject Runnable (worker)", resolver)));
+}
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::ProcessControlMessageQueue() {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(mState == CodecState::Configured);
+
+ while (!mMessageQueueBlocked && !mControlMessageQueue.empty()) {
+ RefPtr<ControlMessage>& msg = mControlMessageQueue.front();
+ if (msg->AsConfigureMessage()) {
+ if (ProcessConfigureMessage(msg->AsConfigureMessage()) ==
+ MessageProcessedResult::NotProcessed) {
+ break;
+ }
+ } else if (msg->AsEncodeMessage()) {
+ if (ProcessEncodeMessage(msg->AsEncodeMessage()) ==
+ MessageProcessedResult::NotProcessed) {
+ break;
+ }
+ } else {
+ MOZ_ASSERT(msg->AsFlushMessage());
+ if (ProcessFlushMessage(msg->AsFlushMessage()) ==
+ MessageProcessedResult::NotProcessed) {
+ break;
+ }
+ }
+ }
+}
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::CancelPendingControlMessages(
+ const nsresult& aResult) {
+ AssertIsOnOwningThread();
+
+ // Cancel the message that is being processed.
+ if (mProcessingMessage) {
+ LOG("%s %p cancels current %s", EncoderType::Name.get(), this,
+ mProcessingMessage->ToString().get());
+ mProcessingMessage->Cancel();
+
+ if (RefPtr<FlushMessage> flush = mProcessingMessage->AsFlushMessage()) {
+ flush->RejectPromiseIfAny(aResult);
+ }
+
+ mProcessingMessage = nullptr;
+ }
+
+ // Clear the message queue.
+ while (!mControlMessageQueue.empty()) {
+ LOG("%s %p cancels pending %s", EncoderType::Name.get(), this,
+ mControlMessageQueue.front()->ToString().get());
+
+ MOZ_ASSERT(!mControlMessageQueue.front()->IsProcessing());
+ if (RefPtr<FlushMessage> flush =
+ mControlMessageQueue.front()->AsFlushMessage()) {
+ flush->RejectPromiseIfAny(aResult);
+ }
+
+ mControlMessageQueue.pop();
+ }
+}
+
+template <typename EncoderType>
+MessageProcessedResult EncoderTemplate<EncoderType>::ProcessConfigureMessage(
+ RefPtr<ConfigureMessage> aMessage) {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(mState == CodecState::Configured);
+ MOZ_ASSERT(aMessage->AsConfigureMessage());
+
+ if (mProcessingMessage) {
+ return MessageProcessedResult::NotProcessed;
+ }
+
+ mProcessingMessage = aMessage;
+ mControlMessageQueue.pop();
+ LOG("%s %p Configuring, message queue processing blocked(%s)",
+ EncoderType::Name.get(), this, aMessage->ToString().get());
+ StartBlockingMessageQueue();
+
+ bool supported = EncoderType::IsSupported(*aMessage->Config());
+
+ if (!supported) {
+ LOGE("%s %p ProcessConfigureMessage error (sync): Not supported",
+ EncoderType::Name.get(), this);
+ mProcessingMessage = nullptr;
+ NS_DispatchToCurrentThread(NS_NewRunnableFunction(
+ "ProcessConfigureMessage (async): not supported",
+ [self = RefPtr(this)]() MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ LOGE("%s %p ProcessConfigureMessage (async close): Not supported",
+ EncoderType::Name.get(), self.get());
+ DebugOnly<Result<Ok, nsresult>> r =
+ self->CloseInternal(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+ MOZ_ASSERT(r.value.isOk());
+ }));
+ return MessageProcessedResult::Processed;
+ }
+
+ if (mAgent) {
+ Reconfigure(aMessage);
+ } else {
+ Configure(aMessage);
+ }
+
+ return MessageProcessedResult::Processed;
+}
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::StartBlockingMessageQueue() {
+ LOG("=== Message queue blocked");
+ mMessageQueueBlocked = true;
+}
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::StopBlockingMessageQueue() {
+ LOG("=== Message queue unblocked");
+ mMessageQueueBlocked = false;
+}
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::Reconfigure(
+ RefPtr<ConfigureMessage> aMessage) {
+ MOZ_ASSERT(mAgent);
+
+ LOG("Reconfiguring encoder: %s",
+ NS_ConvertUTF16toUTF8(aMessage->Config()->ToString()).get());
+
+ RefPtr<ConfigTypeInternal> config = aMessage->Config();
+ RefPtr<WebCodecsConfigurationChangeList> configDiff =
+ config->Diff(*mActiveConfig);
+
+ // Nothing to do, return now
+ if (configDiff->Empty()) {
+ LOG("Reconfigure with identical config, returning.");
+ mProcessingMessage = nullptr;
+ StopBlockingMessageQueue();
+ return;
+ }
+
+ LOG("Attempting to reconfigure encoder: old: %s new: %s, diff: %s",
+ NS_ConvertUTF16toUTF8(mActiveConfig->ToString()).get(),
+ NS_ConvertUTF16toUTF8(config->ToString()).get(),
+ NS_ConvertUTF16toUTF8(configDiff->ToString()).get());
+
+ RefPtr<EncoderConfigurationChangeList> changeList =
+ configDiff->ToPEMChangeList();
+
+ // Attempt to reconfigure the encoder, if the config is similar enough.
+ // Otherwise, or if reconfiguring on the fly didn't work, flush the encoder
+ // and recreate a new one.
+
+ mAgent->Reconfigure(changeList)
+ ->Then(
+ GetCurrentSerialEventTarget(), __func__,
+ [self = RefPtr{this}, id = mAgent->mId,
+ message = std::move(aMessage)](
+ const EncoderAgent::ReconfigurationPromise::ResolveOrRejectValue&
+ aResult) {
+ MOZ_ASSERT(self->mProcessingMessage);
+ MOZ_ASSERT(self->mProcessingMessage->AsConfigureMessage());
+ MOZ_ASSERT(self->mState == CodecState::Configured);
+ MOZ_ASSERT(self->mAgent);
+ MOZ_ASSERT(id == self->mAgent->mId);
+ MOZ_ASSERT(self->mActiveConfig);
+
+ if (aResult.IsReject()) {
+ LOGE(
+ "Reconfiguring on the fly didn't succeed, flushing and "
+ "configuring a new encoder");
+ self->mAgent->Drain()->Then(
+ GetCurrentSerialEventTarget(), __func__,
+ [self, id,
+ message](EncoderAgent::EncodePromise::ResolveOrRejectValue&&
+ aResult) {
+ if (aResult.IsReject()) {
+ const MediaResult& error = aResult.RejectValue();
+ LOGE(
+ "%s %p, EncoderAgent #%zu failed to flush during "
+ "reconfigure, closing: %s",
+ EncoderType::Name.get(), self.get(), id,
+ error.Description().get());
+
+ self->mProcessingMessage = nullptr;
+ self->ScheduleClose(
+ NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ return;
+ }
+
+ LOG("%s %p flush during reconfiguration succeeded.",
+ EncoderType::Name.get(), self.get());
+
+ // If flush succeeded, schedule to output encoded data
+ // first, destroy the current encoder, and proceed to create
+ // a new one.
+ MOZ_ASSERT(aResult.IsResolve());
+ nsTArray<RefPtr<MediaRawData>> data =
+ std::move(aResult.ResolveValue());
+
+ if (data.IsEmpty()) {
+ LOG("%s %p no data during flush for reconfiguration with "
+ "encoder destruction",
+ EncoderType::Name.get(), self.get());
+ } else {
+ LOG("%s %p Outputing %zu frames during flush "
+ " for reconfiguration with encoder destruction",
+ EncoderType::Name.get(), self.get(), data.Length());
+ self->ScheduleOutputEncodedData(
+ std::move(data),
+ nsLiteralCString("Flush before reconfigure"));
+ }
+
+ NS_DispatchToCurrentThread(NS_NewRunnableFunction(
+ "Destroy + recreate encoder after failed reconfigure",
+ [self = RefPtr(self), message]()
+ MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ // Destroy the agent, and finally create a fresh
+ // encoder with the new configuration.
+ self->DestroyEncoderAgentIfAny();
+ self->Configure(message);
+ }));
+ });
+ return;
+ }
+
+ LOG("%s %p, EncoderAgent #%zu has been reconfigured on the fly to "
+ "%s",
+ EncoderType::Name.get(), self.get(), id,
+ message->ToString().get());
+
+ self->mOutputNewDecoderConfig = true;
+ self->mActiveConfig = message->Config();
+ self->mProcessingMessage = nullptr;
+ self->StopBlockingMessageQueue();
+ self->ProcessControlMessageQueue();
+ });
+}
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::Configure(
+ RefPtr<ConfigureMessage> aMessage) {
+ MOZ_ASSERT(!mAgent);
+
+ LOG("Configuring encoder: %s",
+ NS_ConvertUTF16toUTF8(aMessage->Config()->ToString()).get());
+
+ mOutputNewDecoderConfig = true;
+ mActiveConfig = aMessage->Config();
+
+ bool decoderAgentCreated =
+ CreateEncoderAgent(aMessage->mMessageId, aMessage->Config());
+ if (!decoderAgentCreated) {
+ LOGE(
+ "%s %p ProcessConfigureMessage error (sync): encoder agent "
+ "creation "
+ "failed",
+ EncoderType::Name.get(), this);
+ mProcessingMessage = nullptr;
+ NS_DispatchToCurrentThread(NS_NewRunnableFunction(
+ "ProcessConfigureMessage (async): encoder agent creating failed",
+ [self = RefPtr(this)]() MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ LOGE(
+ "%s %p ProcessConfigureMessage (async close): encoder agent "
+ "creation failed",
+ EncoderType::Name.get(), self.get());
+ DebugOnly<Result<Ok, nsresult>> r =
+ self->CloseInternal(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+ MOZ_ASSERT(r.value.isOk());
+ }));
+ return;
+ }
+
+ MOZ_ASSERT(mAgent);
+ MOZ_ASSERT(mActiveConfig);
+
+ LOG("Real configuration with fresh config: %s",
+ NS_ConvertUTF16toUTF8(mActiveConfig->ToString().get()).get());
+
+ EncoderConfig config = mActiveConfig->ToEncoderConfig();
+ mAgent->Configure(config)
+ ->Then(GetCurrentSerialEventTarget(), __func__,
+ [self = RefPtr{this}, id = mAgent->mId, aMessage](
+ const EncoderAgent::ConfigurePromise::ResolveOrRejectValue&
+ aResult) MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ MOZ_ASSERT(self->mProcessingMessage);
+ MOZ_ASSERT(self->mProcessingMessage->AsConfigureMessage());
+ MOZ_ASSERT(self->mState == CodecState::Configured);
+ MOZ_ASSERT(self->mAgent);
+ MOZ_ASSERT(id == self->mAgent->mId);
+ MOZ_ASSERT(self->mActiveConfig);
+
+ LOG("%s %p, EncoderAgent #%zu %s has been %s. now unblocks "
+ "message-queue-processing",
+ EncoderType::Name.get(), self.get(), id,
+ aMessage->ToString().get(),
+ aResult.IsResolve() ? "resolved" : "rejected");
+
+ aMessage->Complete();
+ self->mProcessingMessage = nullptr;
+
+ if (aResult.IsReject()) {
+ // The spec asks to close the decoder with an
+ // NotSupportedError so we log the exact error here.
+ const MediaResult& error = aResult.RejectValue();
+ LOGE("%s %p, EncoderAgent #%zu failed to configure: %s",
+ EncoderType::Name.get(), self.get(), id,
+ error.Description().get());
+ DebugOnly<Result<Ok, nsresult>> r = self->CloseInternal(
+ NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ MOZ_ASSERT(r.value.isOk());
+ return; // No further process
+ }
+
+ self->StopBlockingMessageQueue();
+ self->ProcessControlMessageQueue();
+ })
+ ->Track(aMessage->Request());
+}
+
+template <typename EncoderType>
+MessageProcessedResult EncoderTemplate<EncoderType>::ProcessEncodeMessage(
+ RefPtr<EncodeMessage> aMessage) {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(mState == CodecState::Configured);
+ MOZ_ASSERT(aMessage->AsEncodeMessage());
+
+ if (mProcessingMessage) {
+ return MessageProcessedResult::NotProcessed;
+ }
+
+ mProcessingMessage = aMessage;
+ mControlMessageQueue.pop();
+
+ LOGV("%s %p processing %s", EncoderType::Name.get(), this,
+ aMessage->ToString().get());
+
+ mEncodeQueueSize -= 1;
+ ScheduleDequeueEvent();
+
+ // Treat it like decode error if no EncoderAgent is available or the encoded
+ // data is invalid.
+ auto closeOnError = [&]() {
+ mProcessingMessage = nullptr;
+ ScheduleClose(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ return MessageProcessedResult::Processed;
+ };
+
+ if (!mAgent) {
+ LOGE("%s %p is not configured", EncoderType::Name.get(), this);
+ return closeOnError();
+ }
+
+ MOZ_ASSERT(mActiveConfig);
+ RefPtr<InputTypeInternal> data = aMessage->mData;
+ if (!data) {
+ LOGE("%s %p, data for %s is empty or invalid", EncoderType::Name.get(),
+ this, aMessage->ToString().get());
+ return closeOnError();
+ }
+
+ mAgent->Encode(data.get())
+ ->Then(GetCurrentSerialEventTarget(), __func__,
+ [self = RefPtr{this}, id = mAgent->mId, aMessage](
+ EncoderAgent::EncodePromise::ResolveOrRejectValue&& aResult) {
+ MOZ_ASSERT(self->mProcessingMessage);
+ MOZ_ASSERT(self->mProcessingMessage->AsEncodeMessage());
+ MOZ_ASSERT(self->mState == CodecState::Configured);
+ MOZ_ASSERT(self->mAgent);
+ MOZ_ASSERT(id == self->mAgent->mId);
+ MOZ_ASSERT(self->mActiveConfig);
+
+ nsCString msgStr = aMessage->ToString();
+
+ aMessage->Complete();
+ self->mProcessingMessage = nullptr;
+
+ if (aResult.IsReject()) {
+ // The spec asks to queue a task to run close the decoder
+ // with an EncodingError so we log the exact error here.
+ const MediaResult& error = aResult.RejectValue();
+ LOGE("%s %p, EncoderAgent #%zu %s failed: %s",
+ EncoderType::Name.get(), self.get(), id, msgStr.get(),
+ error.Description().get());
+ self->ScheduleClose(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ return; // No further process
+ }
+
+ MOZ_ASSERT(aResult.IsResolve());
+ nsTArray<RefPtr<MediaRawData>> data =
+ std::move(aResult.ResolveValue());
+ if (data.IsEmpty()) {
+ LOGV("%s %p got no data for %s", EncoderType::Name.get(),
+ self.get(), msgStr.get());
+ } else {
+ LOGV("%s %p, schedule %zu encoded data output",
+ EncoderType::Name.get(), self.get(), data.Length());
+ self->ScheduleOutputEncodedData(std::move(data), msgStr);
+ }
+
+ self->ProcessControlMessageQueue();
+ })
+ ->Track(aMessage->Request());
+
+ return MessageProcessedResult::Processed;
+}
+
+template <typename EncoderType>
+MessageProcessedResult EncoderTemplate<EncoderType>::ProcessFlushMessage(
+ RefPtr<FlushMessage> aMessage) {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(mState == CodecState::Configured);
+ MOZ_ASSERT(aMessage->AsFlushMessage());
+
+ if (mProcessingMessage) {
+ return MessageProcessedResult::NotProcessed;
+ }
+
+ mProcessingMessage = aMessage;
+ mControlMessageQueue.pop();
+
+ LOG("%s %p starts processing %s", EncoderType::Name.get(), this,
+ aMessage->ToString().get());
+
+ // No agent, no thing to do. The promise has been rejected with the
+ // appropriate error in ResetInternal already.
+ if (!mAgent) {
+ LOGE("%s %p no agent, nothing to do", EncoderType::Name.get(), this);
+ mProcessingMessage = nullptr;
+ return MessageProcessedResult::Processed;
+ }
+
+ mAgent->Drain()
+ ->Then(GetCurrentSerialEventTarget(), __func__,
+ [self = RefPtr{this}, id = mAgent->mId, aMessage](
+ EncoderAgent::EncodePromise::ResolveOrRejectValue&& aResult) {
+ MOZ_ASSERT(self->mProcessingMessage);
+ MOZ_ASSERT(self->mProcessingMessage->AsFlushMessage());
+ MOZ_ASSERT(self->mState == CodecState::Configured);
+ MOZ_ASSERT(self->mAgent);
+ MOZ_ASSERT(id == self->mAgent->mId);
+ MOZ_ASSERT(self->mActiveConfig);
+
+ LOG("%s %p, EncoderAgent #%zu %s has been %s",
+ EncoderType::Name.get(), self.get(), id,
+ aMessage->ToString().get(),
+ aResult.IsResolve() ? "resolved" : "rejected");
+
+ nsCString msgStr = aMessage->ToString();
+
+ aMessage->Complete();
+
+ // If flush failed, it means encoder fails to encode the data
+ // sent before, so we treat it like an encode error. We reject
+ // the promise first and then queue a task to close VideoEncoder
+ // with an EncodingError.
+ if (aResult.IsReject()) {
+ const MediaResult& error = aResult.RejectValue();
+ LOGE("%s %p, EncoderAgent #%zu failed to flush: %s",
+ EncoderType::Name.get(), self.get(), id,
+ error.Description().get());
+
+ // Reject with an EncodingError instead of the error we got
+ // above.
+ self->SchedulePromiseResolveOrReject(
+ aMessage->TakePromise(),
+ NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+
+ self->mProcessingMessage = nullptr;
+
+ self->ScheduleClose(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ return; // No further process
+ }
+
+ // If flush succeeded, schedule to output encoded data first
+ // and then resolve the promise, then keep processing the
+ // control messages.
+ MOZ_ASSERT(aResult.IsResolve());
+ nsTArray<RefPtr<MediaRawData>> data =
+ std::move(aResult.ResolveValue());
+
+ if (data.IsEmpty()) {
+ LOG("%s %p gets no data for %s", EncoderType::Name.get(),
+ self.get(), msgStr.get());
+ } else {
+ LOG("%s %p, schedule %zu encoded data output for %s",
+ EncoderType::Name.get(), self.get(), data.Length(),
+ msgStr.get());
+ self->ScheduleOutputEncodedData(std::move(data), msgStr);
+ }
+
+ self->SchedulePromiseResolveOrReject(aMessage->TakePromise(),
+ NS_OK);
+ self->mProcessingMessage = nullptr;
+ self->ProcessControlMessageQueue();
+ })
+ ->Track(aMessage->Request());
+
+ return MessageProcessedResult::Processed;
+}
+
+// CreateEncoderAgent will create an EncoderAgent paired with a xpcom-shutdown
+// blocker and a worker-reference. Besides the needs mentioned in the header
+// file, the blocker and the worker-reference also provides an entry point for
+// us to clean up the resources. Other than the encoder dtor, Reset(), or
+// Close(), the resources should be cleaned up in the following situations:
+// 1. Encoder on window, closing document
+// 2. Encoder on worker, closing document
+// 3. Encoder on worker, terminating worker
+//
+// In case 1, the entry point to clean up is in the mShutdownBlocker's
+// ShutdownpPomise-resolver. In case 2, the entry point is in mWorkerRef's
+// shutting down callback. In case 3, the entry point is in mWorkerRef's
+// shutting down callback.
+
+template <typename EncoderType>
+bool EncoderTemplate<EncoderType>::CreateEncoderAgent(
+ WebCodecsId aId, RefPtr<ConfigTypeInternal> aConfig) {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(mState == CodecState::Configured);
+ MOZ_ASSERT(!mAgent);
+ MOZ_ASSERT(!mShutdownBlocker);
+ MOZ_ASSERT_IF(!NS_IsMainThread(), !mWorkerRef);
+
+ auto resetOnFailure = MakeScopeExit([&]() {
+ mAgent = nullptr;
+ mActiveConfig = nullptr;
+ mShutdownBlocker = nullptr;
+ mWorkerRef = nullptr;
+ });
+
+ // If the encoder is on worker, get a worker reference.
+ if (!NS_IsMainThread()) {
+ WorkerPrivate* workerPrivate = GetCurrentThreadWorkerPrivate();
+ if (NS_WARN_IF(!workerPrivate)) {
+ return false;
+ }
+
+ // Clean up all the resources when worker is going away.
+ RefPtr<StrongWorkerRef> workerRef = StrongWorkerRef::Create(
+ workerPrivate, "EncoderTemplate::CreateEncoderAgent",
+ [self = RefPtr{this}]() {
+ LOG("%s %p, worker is going away", EncoderType::Name.get(),
+ self.get());
+ Unused << self->ResetInternal(NS_ERROR_DOM_ABORT_ERR);
+ });
+ if (NS_WARN_IF(!workerRef)) {
+ return false;
+ }
+
+ mWorkerRef = new ThreadSafeWorkerRef(workerRef);
+ }
+
+ mAgent = MakeRefPtr<EncoderAgent>(aId);
+
+ // ShutdownBlockingTicket requires an unique name to register its own
+ // nsIAsyncShutdownBlocker since each blocker needs a distinct name.
+ // To do that, we use EncoderAgent's unique id to create a unique name.
+ nsAutoString uniqueName;
+ uniqueName.AppendPrintf(
+ "Blocker for EncoderAgent #%zu (codec: %s) @ %p", mAgent->mId,
+ NS_ConvertUTF16toUTF8(mActiveConfig->mCodec).get(), mAgent.get());
+
+ mShutdownBlocker = media::ShutdownBlockingTicket::Create(
+ uniqueName, NS_LITERAL_STRING_FROM_CSTRING(__FILE__), __LINE__);
+ if (!mShutdownBlocker) {
+ LOGE("%s %p failed to create %s", EncoderType::Name.get(), this,
+ NS_ConvertUTF16toUTF8(uniqueName).get());
+ return false;
+ }
+
+ // Clean up all the resources when xpcom-will-shutdown arrives since the
+ // page is going to be closed.
+ mShutdownBlocker->ShutdownPromise()->Then(
+ GetCurrentSerialEventTarget(), __func__,
+ [self = RefPtr{this}, id = mAgent->mId,
+ ref = mWorkerRef](bool /* aUnUsed*/) {
+ LOG("%s %p gets xpcom-will-shutdown notification for EncoderAgent "
+ "#%zu",
+ EncoderType::Name.get(), self.get(), id);
+ Unused << self->ResetInternal(NS_ERROR_DOM_ABORT_ERR);
+ },
+ [self = RefPtr{this}, id = mAgent->mId,
+ ref = mWorkerRef](bool /* aUnUsed*/) {
+ LOG("%s %p removes shutdown-blocker #%zu before getting any "
+ "notification. EncoderAgent should have been dropped",
+ EncoderType::Name.get(), self.get(), id);
+ MOZ_ASSERT(!self->mAgent || self->mAgent->mId != id);
+ });
+
+ LOG("%s %p creates EncoderAgent #%zu @ %p and its shutdown-blocker",
+ EncoderType::Name.get(), this, mAgent->mId, mAgent.get());
+
+ resetOnFailure.release();
+ return true;
+}
+
+template <typename EncoderType>
+void EncoderTemplate<EncoderType>::DestroyEncoderAgentIfAny() {
+ AssertIsOnOwningThread();
+
+ if (!mAgent) {
+ LOG("%s %p has no EncoderAgent to destroy", EncoderType::Name.get(), this);
+ return;
+ }
+
+ MOZ_ASSERT(mActiveConfig);
+ MOZ_ASSERT(mShutdownBlocker);
+ MOZ_ASSERT_IF(!NS_IsMainThread(), mWorkerRef);
+
+ LOG("%s %p destroys EncoderAgent #%zu @ %p", EncoderType::Name.get(), this,
+ mAgent->mId, mAgent.get());
+ mActiveConfig = nullptr;
+ RefPtr<EncoderAgent> agent = std::move(mAgent);
+ // mShutdownBlocker should be kept alive until the shutdown is done.
+ // mWorkerRef is used to ensure this task won't be discarded in worker.
+ agent->Shutdown()->Then(
+ GetCurrentSerialEventTarget(), __func__,
+ [self = RefPtr{this}, id = agent->mId, ref = std::move(mWorkerRef),
+ blocker = std::move(mShutdownBlocker)](
+ const ShutdownPromise::ResolveOrRejectValue& aResult) {
+ LOG("%s %p, EncoderAgent #%zu's shutdown has been %s. Drop its "
+ "shutdown-blocker now",
+ EncoderType::Name.get(), self.get(), id,
+ aResult.IsResolve() ? "resolved" : "rejected");
+ });
+}
+
+template class EncoderTemplate<VideoEncoderTraits>;
+
+#undef LOG
+#undef LOGW
+#undef LOGE
+#undef LOGV
+#undef LOG_INTERNAL
+
+} // namespace mozilla::dom
diff --git a/dom/media/webcodecs/EncoderTemplate.h b/dom/media/webcodecs/EncoderTemplate.h
new file mode 100644
index 0000000000..e53d7166d1
--- /dev/null
+++ b/dom/media/webcodecs/EncoderTemplate.h
@@ -0,0 +1,290 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_EncoderTemplate_h
+#define mozilla_dom_EncoderTemplate_h
+
+#include <queue>
+
+#include "EncoderAgent.h"
+#include "MediaData.h"
+#include "WebCodecsUtils.h"
+#include "mozilla/DOMEventTargetHelper.h"
+#include "mozilla/MozPromise.h"
+#include "mozilla/RefPtr.h"
+#include "mozilla/Result.h"
+#include "mozilla/UniquePtr.h"
+#include "mozilla/dom/VideoEncoderBinding.h"
+#include "mozilla/dom/WorkerRef.h"
+#include "mozilla/media/MediaUtils.h"
+#include "nsStringFwd.h"
+
+namespace mozilla::dom {
+
+class WebCodecsErrorCallback;
+class Promise;
+enum class CodecState : uint8_t;
+
+using Id = size_t;
+
+template <typename EncoderType>
+class EncoderTemplate : public DOMEventTargetHelper {
+ using Self = EncoderTemplate<EncoderType>;
+ using ConfigType = typename EncoderType::ConfigType;
+ using ConfigTypeInternal = typename EncoderType::ConfigTypeInternal;
+ using OutputConfigType = typename EncoderType::OutputConfigType;
+ using InputType = typename EncoderType::InputType;
+ using InputTypeInternal = typename EncoderType::InputTypeInternal;
+ using OutputType = typename EncoderType::OutputType;
+ using OutputCallbackType = typename EncoderType::OutputCallbackType;
+
+ /* ControlMessage classes */
+ protected:
+ class ConfigureMessage;
+ class EncodeMessage;
+ class FlushMessage;
+
+ class ControlMessage {
+ public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ControlMessage)
+ explicit ControlMessage(Id aConfigureId);
+ virtual void Cancel() = 0;
+ virtual bool IsProcessing() = 0;
+
+ virtual nsCString ToString() const = 0;
+ virtual RefPtr<ConfigureMessage> AsConfigureMessage() { return nullptr; }
+ virtual RefPtr<EncodeMessage> AsEncodeMessage() { return nullptr; }
+ virtual RefPtr<FlushMessage> AsFlushMessage() { return nullptr; }
+
+ // For logging purposes
+ const WebCodecsId mConfigureId;
+ const WebCodecsId mMessageId;
+
+ protected:
+ virtual ~ControlMessage() = default;
+ };
+
+ class ConfigureMessage final
+ : public ControlMessage,
+ public MessageRequestHolder<EncoderAgent::ConfigurePromise> {
+ public:
+ ConfigureMessage(Id aConfigureId,
+ const RefPtr<ConfigTypeInternal>& aConfig);
+ virtual void Cancel() override { Disconnect(); }
+ virtual bool IsProcessing() override { return Exists(); };
+ virtual RefPtr<ConfigureMessage> AsConfigureMessage() override {
+ return this;
+ }
+ RefPtr<ConfigTypeInternal> Config() { return mConfig; }
+ nsCString ToString() const override {
+ nsCString rv;
+ rv.AppendPrintf(
+ "ConfigureMessage(#%zu): %s", this->mMessageId,
+ mConfig ? NS_ConvertUTF16toUTF8(mConfig->ToString().get()).get()
+ : "null cfg");
+ return rv;
+ }
+
+ private:
+ const RefPtr<ConfigTypeInternal> mConfig;
+ };
+
+ class EncodeMessage final
+ : public ControlMessage,
+ public MessageRequestHolder<EncoderAgent::EncodePromise> {
+ public:
+ EncodeMessage(WebCodecsId aConfigureId, RefPtr<InputTypeInternal>&& aData,
+ Maybe<VideoEncoderEncodeOptions>&& aOptions = Nothing());
+ nsCString ToString() const override {
+ nsCString rv;
+ bool isKeyFrame = mOptions.isSome() && mOptions.ref().mKeyFrame;
+ rv.AppendPrintf("EncodeMessage(#%zu,#%zu): %s (%s)", this->mConfigureId,
+ this->mMessageId, mData->ToString().get(),
+ isKeyFrame ? "kf" : "");
+ return rv;
+ }
+ virtual void Cancel() override { Disconnect(); }
+ virtual bool IsProcessing() override { return Exists(); };
+ virtual RefPtr<EncodeMessage> AsEncodeMessage() override { return this; }
+ RefPtr<InputTypeInternal> mData;
+ Maybe<VideoEncoderEncodeOptions> mOptions;
+ };
+
+ class FlushMessage final
+ : public ControlMessage,
+ public MessageRequestHolder<EncoderAgent::EncodePromise> {
+ public:
+ FlushMessage(WebCodecsId aConfigureId, Promise* aPromise);
+ virtual void Cancel() override { Disconnect(); }
+ virtual bool IsProcessing() override { return Exists(); };
+ virtual RefPtr<FlushMessage> AsFlushMessage() override { return this; }
+ already_AddRefed<Promise> TakePromise() { return mPromise.forget(); }
+ void RejectPromiseIfAny(const nsresult& aReason);
+
+ nsCString ToString() const override {
+ nsCString rv;
+ rv.AppendPrintf("FlushMessage(#%zu,#%zu)", this->mConfigureId,
+ this->mMessageId);
+ return rv;
+ }
+
+ private:
+ RefPtr<Promise> mPromise;
+ };
+
+ protected:
+ EncoderTemplate(nsIGlobalObject* aGlobalObject,
+ RefPtr<WebCodecsErrorCallback>&& aErrorCallback,
+ RefPtr<OutputCallbackType>&& aOutputCallback);
+
+ virtual ~EncoderTemplate() = default;
+
+ /* WebCodecs interfaces */
+ public:
+ IMPL_EVENT_HANDLER(dequeue)
+
+ void StartBlockingMessageQueue();
+ void StopBlockingMessageQueue();
+
+ CodecState State() const { return mState; };
+
+ uint32_t EncodeQueueSize() const { return mEncodeQueueSize; };
+
+ void Configure(const ConfigType& aConfig, ErrorResult& aRv);
+
+ void EncodeAudioData(InputType& aInput, ErrorResult& aRv);
+ void EncodeVideoFrame(InputType& aInput,
+ const VideoEncoderEncodeOptions& aOptions,
+ ErrorResult& aRv);
+
+ already_AddRefed<Promise> Flush(ErrorResult& aRv);
+
+ void Reset(ErrorResult& aRv);
+
+ MOZ_CAN_RUN_SCRIPT
+ void Close(ErrorResult& aRv);
+
+ /* Type conversion functions for the Encoder implementation */
+ protected:
+ virtual RefPtr<OutputType> EncodedDataToOutputType(
+ nsIGlobalObject* aGlobalObject, RefPtr<MediaRawData>& aData) = 0;
+ virtual OutputConfigType EncoderConfigToDecoderConfig(
+ nsIGlobalObject* aGlobalObject, const RefPtr<MediaRawData>& aData,
+ const ConfigTypeInternal& aOutputConfig) const = 0;
+ /* Internal member variables and functions */
+ protected:
+ // EncoderTemplate can run on either main thread or worker thread.
+ void AssertIsOnOwningThread() const {
+ NS_ASSERT_OWNINGTHREAD(EncoderTemplate);
+ }
+
+ Result<Ok, nsresult> ResetInternal(const nsresult& aResult);
+ MOZ_CAN_RUN_SCRIPT_BOUNDARY
+ Result<Ok, nsresult> CloseInternal(const nsresult& aResult);
+
+ MOZ_CAN_RUN_SCRIPT void ReportError(const nsresult& aResult);
+ MOZ_CAN_RUN_SCRIPT void OutputEncodedData(
+ nsTArray<RefPtr<MediaRawData>>&& aData);
+
+ class ErrorRunnable;
+ void ScheduleReportError(const nsresult& aResult);
+
+ class OutputRunnable;
+ void ScheduleOutputEncodedData(nsTArray<RefPtr<MediaRawData>>&& aData,
+ const nsACString& aLabel);
+
+ void ScheduleClose(const nsresult& aResult);
+
+ void ScheduleDequeueEvent();
+ nsresult FireEvent(nsAtom* aTypeWithOn, const nsAString& aEventType);
+
+ void SchedulePromiseResolveOrReject(already_AddRefed<Promise> aPromise,
+ const nsresult& aResult);
+
+ void ProcessControlMessageQueue();
+ void CancelPendingControlMessages(const nsresult& aResult);
+
+ MessageProcessedResult ProcessConfigureMessage(
+ RefPtr<ConfigureMessage> aMessage);
+
+ MessageProcessedResult ProcessEncodeMessage(RefPtr<EncodeMessage> aMessage);
+
+ MessageProcessedResult ProcessFlushMessage(RefPtr<FlushMessage> aMessage);
+
+ void Configure(RefPtr<ConfigureMessage> aMessage);
+ void Reconfigure(RefPtr<ConfigureMessage> aMessage);
+
+ // Returns true when mAgent can be created.
+ bool CreateEncoderAgent(WebCodecsId aId, RefPtr<ConfigTypeInternal> aConfig);
+ void DestroyEncoderAgentIfAny();
+
+ // Constant in practice, only set in ctor.
+ RefPtr<WebCodecsErrorCallback> mErrorCallback;
+ RefPtr<OutputCallbackType> mOutputCallback;
+
+ CodecState mState;
+
+ bool mMessageQueueBlocked;
+ std::queue<RefPtr<ControlMessage>> mControlMessageQueue;
+ RefPtr<ControlMessage> mProcessingMessage;
+
+ uint32_t mEncodeQueueSize;
+ bool mDequeueEventScheduled;
+
+ // A unique id tracking the ConfigureMessage and will be used as the
+ // EncoderAgent's Id.
+ uint32_t mLatestConfigureId;
+ // Tracking how many encoded data has been enqueued and this number will be
+ // used as the EncodeMessage's Id.
+ size_t mEncodeCounter;
+ // Tracking how many flush request has been enqueued and this number will be
+ // used as the FlushMessage's Id.
+ size_t mFlushCounter;
+
+ // EncoderAgent will be created the first time "configure" is being processed,
+ // and will be destroyed when "reset" is called. If another "configure" is
+ // called, either it's possible to reconfigure the underlying encoder without
+ // tearing eveyrthing down (e.g. a bitrate change), or it's not possible, and
+ // the current encoder will be destroyed and a new one create.
+ // In both cases, the encoder is implicitely flushed before the configuration
+ // change.
+ // See CanReconfigure on the {Audio,Video}EncoderConfigInternal
+ RefPtr<EncoderAgent> mAgent;
+ RefPtr<ConfigTypeInternal> mActiveConfig;
+ // This is true when a configure call has just been processed, and it's
+ // necessary to pass the new decoding configuration when the callback is
+ // called. Read and modified on owner thread only.
+ bool mOutputNewDecoderConfig = false;
+
+ // Used to add a nsIAsyncShutdownBlocker on main thread to block
+ // xpcom-shutdown before the underlying MediaDataEncoder is created. The
+ // blocker will be held until the underlying MediaDataEncoder has been shut
+ // down. This blocker guarantees RemoteEncoderManagerChild's thread, where
+ // the underlying RemoteMediaDataEncoder is on, outlives the
+ // RemoteMediaDataEncoder since the thread releasing, which happens on main
+ // thread when getting a xpcom-shutdown signal, is blocked by the added
+ // blocker. As a result, RemoteMediaDataEncoder can safely work on worker
+ // thread with a holding blocker (otherwise, if RemoteEncoderManagerChild
+ // releases its thread on main thread before RemoteMediaDataEncoder's
+ // Shutdown() task run on worker thread, RemoteMediaDataEncoder has no
+ // thread to run).
+ UniquePtr<media::ShutdownBlockingTicket> mShutdownBlocker;
+
+ // Held to make sure the dispatched tasks can be done before worker is going
+ // away. As long as this worker-ref is held somewhere, the tasks dispatched
+ // to the worker can be executed (otherwise the tasks would be canceled).
+ // This ref should be activated as long as the underlying MediaDataEncoder
+ // is alive, and should keep alive until mShutdownBlocker is dropped, so all
+ // MediaDataEncoder's tasks and mShutdownBlocker-releasing task can be
+ // executed.
+ // TODO: Use StrongWorkerRef instead if this is always used in the same
+ // thread?
+ RefPtr<ThreadSafeWorkerRef> mWorkerRef;
+};
+
+} // namespace mozilla::dom
+
+#endif // mozilla_dom_EncoderTemplate_h
diff --git a/dom/media/webcodecs/EncoderTypes.h b/dom/media/webcodecs/EncoderTypes.h
new file mode 100644
index 0000000000..d58d7c54c8
--- /dev/null
+++ b/dom/media/webcodecs/EncoderTypes.h
@@ -0,0 +1,103 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_EncoderTypes_h
+#define mozilla_dom_EncoderTypes_h
+
+#include "mozilla/Maybe.h"
+#include "mozilla/dom/EncodedVideoChunk.h"
+#include "mozilla/dom/VideoEncoderBinding.h"
+#include "mozilla/dom/VideoFrame.h"
+#include "mozilla/dom/VideoFrameBinding.h"
+#include "nsStringFwd.h"
+#include "nsTLiteralString.h"
+#include "VideoDecoder.h"
+#include "PlatformEncoderModule.h"
+
+namespace mozilla {
+
+class TrackInfo;
+class MediaByteBuffer;
+
+namespace dom {
+
+class VideoEncoderConfigInternal {
+ public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoEncoderConfigInternal);
+ explicit VideoEncoderConfigInternal(const VideoEncoderConfig& aConfig);
+ explicit VideoEncoderConfigInternal(
+ const VideoEncoderConfigInternal& aConfig);
+
+ // Returns an EncoderConfig struct with as many filled members as
+ // possible.
+ // TODO: handle codec specific things
+ EncoderConfig ToEncoderConfig() const;
+
+ bool Equals(const VideoEncoderConfigInternal& aOther) const;
+ bool CanReconfigure(const VideoEncoderConfigInternal& aOther) const;
+ already_AddRefed<WebCodecsConfigurationChangeList> Diff(
+ const VideoEncoderConfigInternal& aOther) const;
+ nsString ToString() const;
+
+ nsString mCodec;
+ uint32_t mWidth;
+ uint32_t mHeight;
+ Maybe<uint32_t> mDisplayWidth;
+ Maybe<uint32_t> mDisplayHeight;
+ Maybe<uint32_t> mBitrate;
+ Maybe<double> mFramerate;
+ HardwareAcceleration mHardwareAcceleration;
+ AlphaOption mAlpha;
+ Maybe<nsString> mScalabilityMode;
+ VideoEncoderBitrateMode mBitrateMode;
+ LatencyMode mLatencyMode;
+ Maybe<nsString> mContentHint;
+ Maybe<AvcEncoderConfig> mAvc;
+
+ private:
+ VideoEncoderConfigInternal(
+ const nsAString& aCodec, uint32_t aWidth, uint32_t aHeight,
+ Maybe<uint32_t>&& aDisplayWidth, Maybe<uint32_t>&& aDisplayHeight,
+ Maybe<uint32_t>&& aBitrate, Maybe<double>&& aFramerate,
+ const HardwareAcceleration& aHardwareAcceleration,
+ const AlphaOption& aAlpha, Maybe<nsString>&& aScalabilityMode,
+ const VideoEncoderBitrateMode& aBitrateMode,
+ const LatencyMode& aLatencyMode, Maybe<nsString>&& aContentHint);
+
+ ~VideoEncoderConfigInternal() = default;
+};
+
+class VideoEncoderTraits {
+ public:
+ static constexpr nsLiteralCString Name = "VideoEncoder"_ns;
+ using ConfigType = VideoEncoderConfig;
+ using ConfigTypeInternal = VideoEncoderConfigInternal;
+ using InputType = dom::VideoFrame;
+ using InputTypeInternal = mozilla::VideoData;
+ using OutputConfigType = mozilla::dom::VideoDecoderConfigInternal;
+ using OutputType = EncodedVideoChunk;
+ using OutputCallbackType = EncodedVideoChunkOutputCallback;
+ using MetadataType = EncodedVideoChunkMetadata;
+
+ static bool IsSupported(const ConfigTypeInternal& aConfig);
+ static bool CanEncodeVideo(const ConfigTypeInternal& aConfig);
+ static Result<UniquePtr<TrackInfo>, nsresult> CreateTrackInfo(
+ const ConfigTypeInternal& aConfig);
+ static bool Validate(const ConfigType& aConfig, nsCString& aErrorMessage);
+ static RefPtr<ConfigTypeInternal> CreateConfigInternal(
+ const ConfigType& aConfig);
+ static RefPtr<InputTypeInternal> CreateInputInternal(
+ const InputType& aInput, const VideoEncoderEncodeOptions& aOptions);
+ static already_AddRefed<OutputConfigType> EncoderConfigToDecoderConfig(
+ nsIGlobalObject* aGlobal,
+ const RefPtr<MediaRawData>& aData,
+ const ConfigTypeInternal& mOutputConfig);
+};
+
+} // namespace dom
+} // namespace mozilla
+
+#endif // mozilla_dom_EncoderTypes_h
diff --git a/dom/media/webcodecs/VideoColorSpace.cpp b/dom/media/webcodecs/VideoColorSpace.cpp
new file mode 100644
index 0000000000..5b0dad0f31
--- /dev/null
+++ b/dom/media/webcodecs/VideoColorSpace.cpp
@@ -0,0 +1,48 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/dom/VideoColorSpace.h"
+#include "mozilla/dom/VideoColorSpaceBinding.h"
+#include "nsIGlobalObject.h"
+
+namespace mozilla::dom {
+
+// Only needed for refcounted objects.
+NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(VideoColorSpace, mParent)
+NS_IMPL_CYCLE_COLLECTING_ADDREF(VideoColorSpace)
+NS_IMPL_CYCLE_COLLECTING_RELEASE(VideoColorSpace)
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(VideoColorSpace)
+ NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY
+ NS_INTERFACE_MAP_ENTRY(nsISupports)
+NS_INTERFACE_MAP_END
+
+VideoColorSpace::VideoColorSpace(nsIGlobalObject* aParent,
+ const VideoColorSpaceInit& aInit)
+ : mParent(aParent), mInit(aInit) {
+ MOZ_ASSERT(mParent);
+}
+
+nsIGlobalObject* VideoColorSpace::GetParentObject() const { return mParent; }
+
+JSObject* VideoColorSpace::WrapObject(JSContext* aCx,
+ JS::Handle<JSObject*> aGivenProto) {
+ return VideoColorSpace_Binding::Wrap(aCx, this, aGivenProto);
+}
+
+/* static */
+already_AddRefed<VideoColorSpace> VideoColorSpace::Constructor(
+ const GlobalObject& aGlobal, const VideoColorSpaceInit& aInit,
+ ErrorResult& aRv) {
+ nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+ if (!global) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return nullptr;
+ }
+ RefPtr<VideoColorSpace> videoColorSpace(new VideoColorSpace(global, aInit));
+ return aRv.Failed() ? nullptr : videoColorSpace.forget();
+}
+
+} // namespace mozilla::dom
diff --git a/dom/media/webcodecs/VideoColorSpace.h b/dom/media/webcodecs/VideoColorSpace.h
new file mode 100644
index 0000000000..e6fdc22317
--- /dev/null
+++ b/dom/media/webcodecs/VideoColorSpace.h
@@ -0,0 +1,64 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_VideoColorSpace_h
+#define mozilla_dom_VideoColorSpace_h
+
+#include "js/TypeDecls.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/ErrorResult.h"
+#include "mozilla/dom/BindingDeclarations.h"
+#include "mozilla/dom/VideoColorSpaceBinding.h"
+#include "nsCycleCollectionParticipant.h"
+#include "nsWrapperCache.h"
+
+class nsIGlobalObject;
+
+namespace mozilla::dom {
+
+class VideoColorSpace final : public nsISupports, public nsWrapperCache {
+ public:
+ NS_DECL_CYCLE_COLLECTING_ISUPPORTS
+ NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(VideoColorSpace)
+
+ public:
+ VideoColorSpace(nsIGlobalObject* aParent, const VideoColorSpaceInit& aInit);
+
+ protected:
+ ~VideoColorSpace() = default;
+
+ public:
+ nsIGlobalObject* GetParentObject() const;
+
+ JSObject* WrapObject(JSContext* aCx,
+ JS::Handle<JSObject*> aGivenProto) override;
+
+ static already_AddRefed<VideoColorSpace> Constructor(
+ const GlobalObject& aGlobal, const VideoColorSpaceInit& aInit,
+ ErrorResult& aRv);
+
+ const Nullable<VideoColorPrimaries>& GetPrimaries() const {
+ return mInit.mPrimaries;
+ }
+
+ const Nullable<VideoTransferCharacteristics>& GetTransfer() const {
+ return mInit.mTransfer;
+ }
+
+ const Nullable<VideoMatrixCoefficients>& GetMatrix() const {
+ return mInit.mMatrix;
+ }
+
+ const Nullable<bool>& GetFullRange() const { return mInit.mFullRange; }
+
+ private:
+ nsCOMPtr<nsIGlobalObject> mParent;
+ const VideoColorSpaceInit mInit;
+};
+
+} // namespace mozilla::dom
+
+#endif // mozilla_dom_VideoColorSpace_h
diff --git a/dom/media/webcodecs/VideoDecoder.cpp b/dom/media/webcodecs/VideoDecoder.cpp
new file mode 100644
index 0000000000..47ca5bb459
--- /dev/null
+++ b/dom/media/webcodecs/VideoDecoder.cpp
@@ -0,0 +1,977 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/dom/VideoDecoder.h"
+#include "mozilla/dom/VideoDecoderBinding.h"
+
+#include "DecoderTraits.h"
+#include "GPUVideoImage.h"
+#include "H264.h"
+#include "ImageContainer.h"
+#include "MediaContainerType.h"
+#include "MediaData.h"
+#include "VideoUtils.h"
+#include "mozilla/Assertions.h"
+#include "mozilla/CheckedInt.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/Logging.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/StaticPrefs_dom.h"
+#include "mozilla/Try.h"
+#include "mozilla/Unused.h"
+#include "mozilla/dom/EncodedVideoChunk.h"
+#include "mozilla/dom/EncodedVideoChunkBinding.h"
+#include "mozilla/dom/ImageUtils.h"
+#include "mozilla/dom/Promise.h"
+#include "mozilla/dom/VideoColorSpaceBinding.h"
+#include "mozilla/dom/VideoFrameBinding.h"
+#include "mozilla/dom/WebCodecsUtils.h"
+#include "nsPrintfCString.h"
+#include "nsReadableUtils.h"
+#include "nsThreadUtils.h"
+
+#ifdef XP_MACOSX
+# include "MacIOSurfaceImage.h"
+#elif MOZ_WAYLAND
+# include "mozilla/layers/DMABUFSurfaceImage.h"
+# include "mozilla/widget/DMABufSurface.h"
+#endif
+
+extern mozilla::LazyLogModule gWebCodecsLog;
+
+namespace mozilla::dom {
+
+#ifdef LOG_INTERNAL
+# undef LOG_INTERNAL
+#endif // LOG_INTERNAL
+#define LOG_INTERNAL(level, msg, ...) \
+ MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__))
+
+#ifdef LOG
+# undef LOG
+#endif // LOG
+#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__)
+
+#ifdef LOGW
+# undef LOGW
+#endif // LOGW
+#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__)
+
+#ifdef LOGE
+# undef LOGE
+#endif // LOGE
+#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__)
+
+#ifdef LOGV
+# undef LOGV
+#endif // LOGV
+#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__)
+
+NS_IMPL_CYCLE_COLLECTION_INHERITED(VideoDecoder, DOMEventTargetHelper,
+ mErrorCallback, mOutputCallback)
+NS_IMPL_ADDREF_INHERITED(VideoDecoder, DOMEventTargetHelper)
+NS_IMPL_RELEASE_INHERITED(VideoDecoder, DOMEventTargetHelper)
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(VideoDecoder)
+NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper)
+
+/*
+ * Below are helper classes
+ */
+
+VideoColorSpaceInternal::VideoColorSpaceInternal(
+ const VideoColorSpaceInit& aColorSpaceInit)
+ : mFullRange(NullableToMaybe(aColorSpaceInit.mFullRange)),
+ mMatrix(NullableToMaybe(aColorSpaceInit.mMatrix)),
+ mPrimaries(NullableToMaybe(aColorSpaceInit.mPrimaries)),
+ mTransfer(NullableToMaybe(aColorSpaceInit.mTransfer)) {}
+
+VideoColorSpaceInit VideoColorSpaceInternal::ToColorSpaceInit() const {
+ VideoColorSpaceInit init;
+ init.mFullRange = MaybeToNullable(mFullRange);
+ init.mMatrix = MaybeToNullable(mMatrix);
+ init.mPrimaries = MaybeToNullable(mPrimaries);
+ init.mTransfer = MaybeToNullable(mTransfer);
+ return init;
+};
+
+static Result<RefPtr<MediaByteBuffer>, nsresult> GetExtraData(
+ const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer);
+
+VideoDecoderConfigInternal::VideoDecoderConfigInternal(
+ const nsAString& aCodec, Maybe<uint32_t>&& aCodedHeight,
+ Maybe<uint32_t>&& aCodedWidth, Maybe<VideoColorSpaceInternal>&& aColorSpace,
+ Maybe<RefPtr<MediaByteBuffer>>&& aDescription,
+ Maybe<uint32_t>&& aDisplayAspectHeight,
+ Maybe<uint32_t>&& aDisplayAspectWidth,
+ const HardwareAcceleration& aHardwareAcceleration,
+ Maybe<bool>&& aOptimizeForLatency)
+ : mCodec(aCodec),
+ mCodedHeight(std::move(aCodedHeight)),
+ mCodedWidth(std::move(aCodedWidth)),
+ mColorSpace(std::move(aColorSpace)),
+ mDescription(std::move(aDescription)),
+ mDisplayAspectHeight(std::move(aDisplayAspectHeight)),
+ mDisplayAspectWidth(std::move(aDisplayAspectWidth)),
+ mHardwareAcceleration(aHardwareAcceleration),
+ mOptimizeForLatency(std::move(aOptimizeForLatency)){};
+
+/*static*/
+UniquePtr<VideoDecoderConfigInternal> VideoDecoderConfigInternal::Create(
+ const VideoDecoderConfig& aConfig) {
+ nsCString errorMessage;
+ if (!VideoDecoderTraits::Validate(aConfig, errorMessage)) {
+ LOGE("Failed to create VideoDecoderConfigInternal: %s", errorMessage.get());
+ return nullptr;
+ }
+
+ Maybe<RefPtr<MediaByteBuffer>> description;
+ if (aConfig.mDescription.WasPassed()) {
+ auto rv = GetExtraData(aConfig.mDescription.Value());
+ if (rv.isErr()) { // Invalid description data.
+ LOGE(
+ "Failed to create VideoDecoderConfigInternal due to invalid "
+ "description data. Error: 0x%08" PRIx32,
+ static_cast<uint32_t>(rv.unwrapErr()));
+ return nullptr;
+ }
+ description.emplace(rv.unwrap());
+ }
+
+ Maybe<VideoColorSpaceInternal> colorSpace;
+ if (aConfig.mColorSpace.WasPassed()) {
+ colorSpace.emplace(VideoColorSpaceInternal(aConfig.mColorSpace.Value()));
+ }
+
+ return UniquePtr<VideoDecoderConfigInternal>(new VideoDecoderConfigInternal(
+ aConfig.mCodec, OptionalToMaybe(aConfig.mCodedHeight),
+ OptionalToMaybe(aConfig.mCodedWidth), std::move(colorSpace),
+ std::move(description), OptionalToMaybe(aConfig.mDisplayAspectHeight),
+ OptionalToMaybe(aConfig.mDisplayAspectWidth),
+ aConfig.mHardwareAcceleration,
+ OptionalToMaybe(aConfig.mOptimizeForLatency)));
+}
+
+nsString VideoDecoderConfigInternal::ToString() const {
+ nsString rv;
+
+ rv.Append(mCodec);
+ if (mCodedWidth.isSome()) {
+ rv.AppendPrintf("coded: %dx%d", mCodedWidth.value(), mCodedHeight.value());
+ }
+ if (mDisplayAspectWidth.isSome()) {
+ rv.AppendPrintf("display %dx%d", mDisplayAspectWidth.value(),
+ mDisplayAspectHeight.value());
+ }
+ if (mColorSpace.isSome()) {
+ rv.AppendPrintf("colorspace %s", "todo");
+ }
+ if (mDescription.isSome()) {
+ rv.AppendPrintf("extradata: %zu bytes", mDescription.value()->Length());
+ }
+ rv.AppendPrintf(
+ "hw accel: %s",
+ HardwareAccelerationValues::GetString(mHardwareAcceleration).data());
+ if (mOptimizeForLatency.isSome()) {
+ rv.AppendPrintf("optimize for latency: %s",
+ mOptimizeForLatency.value() ? "true" : "false");
+ }
+
+ return rv;
+}
+
+/*
+ * The followings are helpers for VideoDecoder methods
+ */
+
+struct MIMECreateParam {
+ explicit MIMECreateParam(const VideoDecoderConfigInternal& aConfig)
+ : mParsedCodec(ParseCodecString(aConfig.mCodec).valueOr(EmptyString())),
+ mWidth(aConfig.mCodedWidth),
+ mHeight(aConfig.mCodedHeight) {}
+ explicit MIMECreateParam(const VideoDecoderConfig& aConfig)
+ : mParsedCodec(ParseCodecString(aConfig.mCodec).valueOr(EmptyString())),
+ mWidth(OptionalToMaybe(aConfig.mCodedWidth)),
+ mHeight(OptionalToMaybe(aConfig.mCodedHeight)) {}
+
+ const nsString mParsedCodec;
+ const Maybe<uint32_t> mWidth;
+ const Maybe<uint32_t> mHeight;
+};
+
+static nsTArray<nsCString> GuessMIMETypes(const MIMECreateParam& aParam) {
+ const auto codec = NS_ConvertUTF16toUTF8(aParam.mParsedCodec);
+ nsTArray<nsCString> types;
+ for (const nsCString& container : GuessContainers(aParam.mParsedCodec)) {
+ nsPrintfCString mime("video/%s; codecs=%s", container.get(), codec.get());
+ if (aParam.mWidth) {
+ mime.AppendPrintf("; width=%d", *aParam.mWidth);
+ }
+ if (aParam.mHeight) {
+ mime.AppendPrintf("; height=%d", *aParam.mHeight);
+ }
+ types.AppendElement(mime);
+ }
+ return types;
+}
+
+static bool IsSupportedCodec(const nsAString& aCodec) {
+ // H265 is unsupported.
+ if (!IsAV1CodecString(aCodec) && !IsVP9CodecString(aCodec) &&
+ !IsVP8CodecString(aCodec) && !IsH264CodecString(aCodec)) {
+ return false;
+ }
+
+ // Gecko allows codec string starts with vp9 or av1 but Webcodecs requires to
+ // starts with av01 and vp09.
+ // https://www.w3.org/TR/webcodecs-codec-registry/#video-codec-registry
+ if (StringBeginsWith(aCodec, u"vp9"_ns) ||
+ StringBeginsWith(aCodec, u"av1"_ns)) {
+ return false;
+ }
+
+ return true;
+}
+
+// https://w3c.github.io/webcodecs/#check-configuration-support
+template <typename Config>
+static bool CanDecode(const Config& aConfig) {
+ auto param = MIMECreateParam(aConfig);
+ // TODO: Enable WebCodecs on Android (Bug 1840508)
+ if (IsOnAndroid()) {
+ return false;
+ }
+ if (!IsSupportedCodec(param.mParsedCodec)) {
+ return false;
+ }
+ if (IsOnMacOS() && IsH264CodecString(param.mParsedCodec) &&
+ !StaticPrefs::dom_media_webcodecs_force_osx_h264_enabled()) {
+ // This will be fixed in Bug 1846796.
+ return false;
+ }
+ // TODO: Instead of calling CanHandleContainerType with the guessed the
+ // containers, DecoderTraits should provide an API to tell if a codec is
+ // decodable or not.
+ for (const nsCString& mime : GuessMIMETypes(param)) {
+ if (Maybe<MediaContainerType> containerType =
+ MakeMediaExtendedMIMEType(mime)) {
+ if (DecoderTraits::CanHandleContainerType(
+ *containerType, nullptr /* DecoderDoctorDiagnostics */) !=
+ CANPLAY_NO) {
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+static nsTArray<UniquePtr<TrackInfo>> GetTracksInfo(
+ const VideoDecoderConfigInternal& aConfig) {
+ // TODO: Instead of calling GetTracksInfo with the guessed containers,
+ // DecoderTraits should provide an API to create the TrackInfo directly.
+ for (const nsCString& mime : GuessMIMETypes(MIMECreateParam(aConfig))) {
+ if (Maybe<MediaContainerType> containerType =
+ MakeMediaExtendedMIMEType(mime)) {
+ if (nsTArray<UniquePtr<TrackInfo>> tracks =
+ DecoderTraits::GetTracksInfo(*containerType);
+ !tracks.IsEmpty()) {
+ return tracks;
+ }
+ }
+ }
+ return {};
+}
+
+static Result<RefPtr<MediaByteBuffer>, nsresult> GetExtraData(
+ const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer) {
+ RefPtr<MediaByteBuffer> data = MakeRefPtr<MediaByteBuffer>();
+ if (!AppendTypedArrayDataTo(aBuffer, *data)) {
+ return Err(NS_ERROR_OUT_OF_MEMORY);
+ }
+ return data->Length() > 0 ? data : nullptr;
+}
+
+static Result<Ok, nsresult> CloneConfiguration(
+ RootedDictionary<VideoDecoderConfig>& aDest, JSContext* aCx,
+ const VideoDecoderConfig& aConfig) {
+ DebugOnly<nsCString> str;
+ MOZ_ASSERT(VideoDecoderTraits::Validate(aConfig, str));
+
+ aDest.mCodec = aConfig.mCodec;
+ if (aConfig.mCodedHeight.WasPassed()) {
+ aDest.mCodedHeight.Construct(aConfig.mCodedHeight.Value());
+ }
+ if (aConfig.mCodedWidth.WasPassed()) {
+ aDest.mCodedWidth.Construct(aConfig.mCodedWidth.Value());
+ }
+ if (aConfig.mColorSpace.WasPassed()) {
+ aDest.mColorSpace.Construct(aConfig.mColorSpace.Value());
+ }
+ if (aConfig.mDescription.WasPassed()) {
+ aDest.mDescription.Construct();
+ MOZ_TRY(CloneBuffer(aCx, aDest.mDescription.Value(),
+ aConfig.mDescription.Value()));
+ }
+ if (aConfig.mDisplayAspectHeight.WasPassed()) {
+ aDest.mDisplayAspectHeight.Construct(aConfig.mDisplayAspectHeight.Value());
+ }
+ if (aConfig.mDisplayAspectWidth.WasPassed()) {
+ aDest.mDisplayAspectWidth.Construct(aConfig.mDisplayAspectWidth.Value());
+ }
+ aDest.mHardwareAcceleration = aConfig.mHardwareAcceleration;
+ if (aConfig.mOptimizeForLatency.WasPassed()) {
+ aDest.mOptimizeForLatency.Construct(aConfig.mOptimizeForLatency.Value());
+ }
+
+ return Ok();
+}
+
+static Maybe<VideoPixelFormat> GuessPixelFormat(layers::Image* aImage) {
+ if (aImage) {
+ // TODO: Implement ImageUtils::Impl for MacIOSurfaceImage and
+ // DMABUFSurfaceImage?
+ if (aImage->AsPlanarYCbCrImage() || aImage->AsNVImage()) {
+ const ImageUtils imageUtils(aImage);
+ Maybe<VideoPixelFormat> f =
+ ImageBitmapFormatToVideoPixelFormat(imageUtils.GetFormat());
+
+ // ImageBitmapFormat cannot distinguish YUV420 or YUV420A.
+ bool hasAlpha = aImage->AsPlanarYCbCrImage() &&
+ aImage->AsPlanarYCbCrImage()->GetData() &&
+ aImage->AsPlanarYCbCrImage()->GetData()->mAlpha;
+ if (f && *f == VideoPixelFormat::I420 && hasAlpha) {
+ return Some(VideoPixelFormat::I420A);
+ }
+ return f;
+ }
+ if (layers::GPUVideoImage* image = aImage->AsGPUVideoImage()) {
+ RefPtr<layers::ImageBridgeChild> imageBridge =
+ layers::ImageBridgeChild::GetSingleton();
+ layers::TextureClient* texture = image->GetTextureClient(imageBridge);
+ if (NS_WARN_IF(!texture)) {
+ return Nothing();
+ }
+ return SurfaceFormatToVideoPixelFormat(texture->GetFormat());
+ }
+#ifdef XP_MACOSX
+ if (layers::MacIOSurfaceImage* image = aImage->AsMacIOSurfaceImage()) {
+ MOZ_ASSERT(image->GetSurface());
+ return SurfaceFormatToVideoPixelFormat(image->GetSurface()->GetFormat());
+ }
+#endif
+#ifdef MOZ_WAYLAND
+ if (layers::DMABUFSurfaceImage* image = aImage->AsDMABUFSurfaceImage()) {
+ MOZ_ASSERT(image->GetSurface());
+ return SurfaceFormatToVideoPixelFormat(image->GetSurface()->GetFormat());
+ }
+#endif
+ }
+ LOGW("Failed to get pixel format from layers::Image");
+ return Nothing();
+}
+
+static VideoColorSpaceInternal GuessColorSpace(
+ const layers::PlanarYCbCrData* aData) {
+ if (!aData) {
+ LOGE("nullptr in GuessColorSpace");
+ return {};
+ }
+
+ VideoColorSpaceInternal colorSpace;
+ colorSpace.mFullRange = Some(ToFullRange(aData->mColorRange));
+ if (Maybe<VideoMatrixCoefficients> m =
+ ToMatrixCoefficients(aData->mYUVColorSpace)) {
+ colorSpace.mMatrix = ToMatrixCoefficients(aData->mYUVColorSpace);
+ colorSpace.mPrimaries = ToPrimaries(aData->mColorPrimaries);
+ }
+ if (!colorSpace.mPrimaries) {
+ LOG("Missing primaries, guessing from colorspace");
+ // Make an educated guess based on the coefficients.
+ colorSpace.mPrimaries = colorSpace.mMatrix.map([](const auto& aMatrix) {
+ switch (aMatrix) {
+ case VideoMatrixCoefficients::EndGuard_:
+ MOZ_CRASH("This should not happen");
+ case VideoMatrixCoefficients::Bt2020_ncl:
+ return VideoColorPrimaries::Bt2020;
+ case VideoMatrixCoefficients::Rgb:
+ case VideoMatrixCoefficients::Bt470bg:
+ case VideoMatrixCoefficients::Smpte170m:
+ LOGW(
+ "Warning: Falling back to BT709 when attempting to determine the "
+ "primaries function of a YCbCr buffer");
+ [[fallthrough]];
+ case VideoMatrixCoefficients::Bt709:
+ return VideoColorPrimaries::Bt709;
+ }
+ MOZ_ASSERT_UNREACHABLE("Unexpected matrix coefficients");
+ LOGW(
+ "Warning: Falling back to BT709 due to unexpected matrix "
+ "coefficients "
+ "when attempting to determine the primaries function of a YCbCr "
+ "buffer");
+ return VideoColorPrimaries::Bt709;
+ });
+ }
+
+ if (Maybe<VideoTransferCharacteristics> c =
+ ToTransferCharacteristics(aData->mTransferFunction)) {
+ colorSpace.mTransfer = Some(*c);
+ }
+ if (!colorSpace.mTransfer) {
+ LOG("Missing transfer characteristics, guessing from colorspace");
+ colorSpace.mTransfer = Some(([&] {
+ switch (aData->mYUVColorSpace) {
+ case gfx::YUVColorSpace::Identity:
+ return VideoTransferCharacteristics::Iec61966_2_1;
+ case gfx::YUVColorSpace::BT2020:
+ return VideoTransferCharacteristics::Pq;
+ case gfx::YUVColorSpace::BT601:
+ LOGW(
+ "Warning: Falling back to BT709 when attempting to determine the "
+ "transfer function of a MacIOSurface");
+ [[fallthrough]];
+ case gfx::YUVColorSpace::BT709:
+ return VideoTransferCharacteristics::Bt709;
+ }
+ MOZ_ASSERT_UNREACHABLE("Unexpected color space");
+ LOGW(
+ "Warning: Falling back to BT709 due to unexpected color space "
+ "when attempting to determine the transfer function of a "
+ "MacIOSurface");
+ return VideoTransferCharacteristics::Bt709;
+ })());
+ }
+
+ return colorSpace;
+}
+
+#ifdef XP_MACOSX
+static VideoColorSpaceInternal GuessColorSpace(const MacIOSurface* aSurface) {
+ if (!aSurface) {
+ return {};
+ }
+ VideoColorSpaceInternal colorSpace;
+ colorSpace.mFullRange = Some(aSurface->IsFullRange());
+ if (Maybe<dom::VideoMatrixCoefficients> m =
+ ToMatrixCoefficients(aSurface->GetYUVColorSpace())) {
+ colorSpace.mMatrix = Some(*m);
+ }
+ if (Maybe<VideoColorPrimaries> p = ToPrimaries(aSurface->mColorPrimaries)) {
+ colorSpace.mPrimaries = Some(*p);
+ }
+ // Make an educated guess based on the coefficients.
+ if (aSurface->GetYUVColorSpace() == gfx::YUVColorSpace::Identity) {
+ colorSpace.mTransfer = Some(VideoTransferCharacteristics::Iec61966_2_1);
+ } else if (aSurface->GetYUVColorSpace() == gfx::YUVColorSpace::BT709) {
+ colorSpace.mTransfer = Some(VideoTransferCharacteristics::Bt709);
+ } else if (aSurface->GetYUVColorSpace() == gfx::YUVColorSpace::BT2020) {
+ colorSpace.mTransfer = Some(VideoTransferCharacteristics::Pq);
+ } else {
+ LOGW(
+ "Warning: Falling back to BT709 when attempting to determine the "
+ "transfer function of a MacIOSurface");
+ colorSpace.mTransfer = Some(VideoTransferCharacteristics::Bt709);
+ }
+
+ return colorSpace;
+}
+#endif
+#ifdef MOZ_WAYLAND
+// TODO: Set DMABufSurface::IsFullRange() to const so aSurface can be const.
+static VideoColorSpaceInternal GuessColorSpace(DMABufSurface* aSurface) {
+ if (!aSurface) {
+ return {};
+ }
+ VideoColorSpaceInternal colorSpace;
+ colorSpace.mFullRange = Some(aSurface->IsFullRange());
+ if (Maybe<dom::VideoMatrixCoefficients> m =
+ ToMatrixCoefficients(aSurface->GetYUVColorSpace())) {
+ colorSpace.mMatrix = Some(*m);
+ }
+ // No other color space information.
+ return colorSpace;
+}
+#endif
+
+static VideoColorSpaceInternal GuessColorSpace(layers::Image* aImage) {
+ if (aImage) {
+ if (layers::PlanarYCbCrImage* image = aImage->AsPlanarYCbCrImage()) {
+ return GuessColorSpace(image->GetData());
+ }
+ if (layers::NVImage* image = aImage->AsNVImage()) {
+ return GuessColorSpace(image->GetData());
+ }
+ if (layers::GPUVideoImage* image = aImage->AsGPUVideoImage()) {
+ VideoColorSpaceInternal colorSpace;
+ colorSpace.mFullRange =
+ Some(image->GetColorRange() != gfx::ColorRange::LIMITED);
+ colorSpace.mMatrix = ToMatrixCoefficients(image->GetYUVColorSpace());
+ colorSpace.mPrimaries = ToPrimaries(image->GetColorPrimaries());
+ colorSpace.mTransfer =
+ ToTransferCharacteristics(image->GetTransferFunction());
+ // In some circumstances, e.g. on Linux software decoding when using
+ // VPXDecoder and RDD, the primaries aren't set correctly. Make a good
+ // guess based on the other params. Fixing this is tracked in
+ // https://bugzilla.mozilla.org/show_bug.cgi?id=1869825
+ if (!colorSpace.mPrimaries) {
+ if (colorSpace.mMatrix.isSome()) {
+ switch (colorSpace.mMatrix.value()) {
+ case VideoMatrixCoefficients::Rgb:
+ case VideoMatrixCoefficients::Bt709:
+ colorSpace.mPrimaries = Some(VideoColorPrimaries::Bt709);
+ break;
+ case VideoMatrixCoefficients::Bt470bg:
+ case VideoMatrixCoefficients::Smpte170m:
+ colorSpace.mPrimaries = Some(VideoColorPrimaries::Bt470bg);
+ break;
+ case VideoMatrixCoefficients::Bt2020_ncl:
+ colorSpace.mPrimaries = Some(VideoColorPrimaries::Bt2020);
+ break;
+ case VideoMatrixCoefficients::EndGuard_:
+ MOZ_ASSERT_UNREACHABLE("bad enum value");
+ break;
+ };
+ }
+ }
+ return colorSpace;
+ }
+#ifdef XP_MACOSX
+ // TODO: Make sure VideoFrame can interpret its internal data in different
+ // formats.
+ if (layers::MacIOSurfaceImage* image = aImage->AsMacIOSurfaceImage()) {
+ return GuessColorSpace(image->GetSurface());
+ }
+#endif
+#ifdef MOZ_WAYLAND
+ // TODO: Make sure VideoFrame can interpret its internal data in different
+ // formats.
+ if (layers::DMABUFSurfaceImage* image = aImage->AsDMABUFSurfaceImage()) {
+ return GuessColorSpace(image->GetSurface());
+ }
+#endif
+ }
+ LOGW("Failed to get color space from layers::Image");
+ return {};
+}
+
+static Result<gfx::IntSize, nsresult> AdjustDisplaySize(
+ const uint32_t aDisplayAspectWidth, const uint32_t aDisplayAspectHeight,
+ const gfx::IntSize& aDisplaySize) {
+ if (aDisplayAspectHeight == 0) {
+ return Err(NS_ERROR_ILLEGAL_VALUE);
+ }
+
+ const double aspectRatio =
+ static_cast<double>(aDisplayAspectWidth) / aDisplayAspectHeight;
+
+ double w = aDisplaySize.width;
+ double h = aDisplaySize.height;
+
+ if (aspectRatio >= w / h) {
+ // Adjust w to match the aspect ratio
+ w = aspectRatio * h;
+ } else {
+ // Adjust h to match the aspect ratio
+ h = w / aspectRatio;
+ }
+
+ w = std::round(w);
+ h = std::round(h);
+ constexpr double MAX = static_cast<double>(
+ std::numeric_limits<decltype(gfx::IntSize::width)>::max());
+ if (w > MAX || h > MAX || w < 1.0 || h < 1.0) {
+ return Err(NS_ERROR_ILLEGAL_VALUE);
+ }
+ return gfx::IntSize(static_cast<decltype(gfx::IntSize::width)>(w),
+ static_cast<decltype(gfx::IntSize::height)>(h));
+}
+
+// https://w3c.github.io/webcodecs/#create-a-videoframe
+static RefPtr<VideoFrame> CreateVideoFrame(
+ nsIGlobalObject* aGlobalObject, const VideoData* aData, int64_t aTimestamp,
+ uint64_t aDuration, const Maybe<uint32_t> aDisplayAspectWidth,
+ const Maybe<uint32_t> aDisplayAspectHeight,
+ const VideoColorSpaceInternal& aColorSpace) {
+ MOZ_ASSERT(aGlobalObject);
+ MOZ_ASSERT(aData);
+ MOZ_ASSERT((!!aDisplayAspectWidth) == (!!aDisplayAspectHeight));
+
+ Maybe<VideoPixelFormat> format = GuessPixelFormat(aData->mImage.get());
+ gfx::IntSize displaySize = aData->mDisplay;
+ if (aDisplayAspectWidth && aDisplayAspectHeight) {
+ auto r = AdjustDisplaySize(*aDisplayAspectWidth, *aDisplayAspectHeight,
+ displaySize);
+ if (r.isOk()) {
+ displaySize = r.unwrap();
+ }
+ }
+
+ return MakeRefPtr<VideoFrame>(
+ aGlobalObject, aData->mImage, format, aData->mImage->GetSize(),
+ aData->mImage->GetPictureRect(), displaySize, Some(aDuration), aTimestamp,
+ aColorSpace.ToColorSpaceInit());
+}
+
+/* static */
+bool VideoDecoderTraits::IsSupported(
+ const VideoDecoderConfigInternal& aConfig) {
+ return CanDecode(aConfig);
+}
+
+/* static */
+Result<UniquePtr<TrackInfo>, nsresult> VideoDecoderTraits::CreateTrackInfo(
+ const VideoDecoderConfigInternal& aConfig) {
+ LOG("Create a VideoInfo from %s config",
+ NS_ConvertUTF16toUTF8(aConfig.ToString()).get());
+
+ nsTArray<UniquePtr<TrackInfo>> tracks = GetTracksInfo(aConfig);
+ if (tracks.Length() != 1 || tracks[0]->GetType() != TrackInfo::kVideoTrack) {
+ LOGE("Failed to get TrackInfo");
+ return Err(NS_ERROR_INVALID_ARG);
+ }
+
+ UniquePtr<TrackInfo> track(std::move(tracks[0]));
+ VideoInfo* vi = track->GetAsVideoInfo();
+ if (!vi) {
+ LOGE("Failed to get VideoInfo");
+ return Err(NS_ERROR_INVALID_ARG);
+ }
+
+ constexpr uint32_t MAX = static_cast<uint32_t>(
+ std::numeric_limits<decltype(gfx::IntSize::width)>::max());
+ if (aConfig.mCodedHeight.isSome()) {
+ if (aConfig.mCodedHeight.value() > MAX) {
+ LOGE("codedHeight overflows");
+ return Err(NS_ERROR_INVALID_ARG);
+ }
+ vi->mImage.height = static_cast<decltype(gfx::IntSize::height)>(
+ aConfig.mCodedHeight.value());
+ }
+ if (aConfig.mCodedWidth.isSome()) {
+ if (aConfig.mCodedWidth.value() > MAX) {
+ LOGE("codedWidth overflows");
+ return Err(NS_ERROR_INVALID_ARG);
+ }
+ vi->mImage.width =
+ static_cast<decltype(gfx::IntSize::width)>(aConfig.mCodedWidth.value());
+ }
+
+ if (aConfig.mDisplayAspectHeight.isSome()) {
+ if (aConfig.mDisplayAspectHeight.value() > MAX) {
+ LOGE("displayAspectHeight overflows");
+ return Err(NS_ERROR_INVALID_ARG);
+ }
+ vi->mDisplay.height = static_cast<decltype(gfx::IntSize::height)>(
+ aConfig.mDisplayAspectHeight.value());
+ }
+ if (aConfig.mDisplayAspectWidth.isSome()) {
+ if (aConfig.mDisplayAspectWidth.value() > MAX) {
+ LOGE("displayAspectWidth overflows");
+ return Err(NS_ERROR_INVALID_ARG);
+ }
+ vi->mDisplay.width = static_cast<decltype(gfx::IntSize::width)>(
+ aConfig.mDisplayAspectWidth.value());
+ }
+
+ if (aConfig.mColorSpace.isSome()) {
+ const VideoColorSpaceInternal& colorSpace(aConfig.mColorSpace.value());
+ if (colorSpace.mFullRange.isSome()) {
+ vi->mColorRange = ToColorRange(colorSpace.mFullRange.value());
+ }
+ if (colorSpace.mMatrix.isSome()) {
+ vi->mColorSpace.emplace(ToColorSpace(colorSpace.mMatrix.value()));
+ }
+ // Some decoders get their primaries and transfer function from the codec
+ // string, and it's already set here. This is the case for VP9 decoders.
+ if (colorSpace.mPrimaries.isSome()) {
+ auto primaries = ToPrimaries(colorSpace.mPrimaries.value());
+ if (vi->mColorPrimaries.isSome()) {
+ if (vi->mColorPrimaries.value() != primaries) {
+ LOG("Conflict between decoder config and codec string, keeping codec "
+ "string primaries of %d",
+ static_cast<int>(primaries));
+ }
+ } else {
+ vi->mColorPrimaries.emplace(primaries);
+ }
+ }
+ if (colorSpace.mTransfer.isSome()) {
+ auto primaries = ToTransferFunction(colorSpace.mTransfer.value());
+ if (vi->mTransferFunction.isSome()) {
+ if (vi->mTransferFunction.value() != primaries) {
+ LOG("Conflict between decoder config and codec string, keeping codec "
+ "string transfer function of %d",
+ static_cast<int>(vi->mTransferFunction.value()));
+ }
+ } else {
+ vi->mTransferFunction.emplace(
+ ToTransferFunction(colorSpace.mTransfer.value()));
+ }
+ }
+ }
+
+ if (aConfig.mDescription.isSome()) {
+ RefPtr<MediaByteBuffer> buf;
+ buf = aConfig.mDescription.value();
+ if (buf) {
+ LOG("The given config has %zu bytes of description data", buf->Length());
+ if (vi->mExtraData) {
+ LOGW("The default extra data is overwritten");
+ }
+ vi->mExtraData = buf;
+ }
+
+ // TODO: Make this utility and replace the similar one in MP4Demuxer.cpp.
+ if (vi->mExtraData && !vi->mExtraData->IsEmpty() &&
+ IsH264CodecString(aConfig.mCodec)) {
+ SPSData spsdata;
+ if (H264::DecodeSPSFromExtraData(vi->mExtraData.get(), spsdata) &&
+ spsdata.pic_width > 0 && spsdata.pic_height > 0 &&
+ H264::EnsureSPSIsSane(spsdata)) {
+ LOG("H264 sps data - pic size: %d x %d, display size: %d x %d",
+ spsdata.pic_width, spsdata.pic_height, spsdata.display_width,
+ spsdata.display_height);
+
+ if (spsdata.pic_width > MAX || spsdata.pic_height > MAX ||
+ spsdata.display_width > MAX || spsdata.display_height > MAX) {
+ LOGE("H264 width or height in sps data overflows");
+ return Err(NS_ERROR_INVALID_ARG);
+ }
+
+ vi->mImage.width =
+ static_cast<decltype(gfx::IntSize::width)>(spsdata.pic_width);
+ vi->mImage.height =
+ static_cast<decltype(gfx::IntSize::height)>(spsdata.pic_height);
+ vi->mDisplay.width =
+ static_cast<decltype(gfx::IntSize::width)>(spsdata.display_width);
+ vi->mDisplay.height =
+ static_cast<decltype(gfx::IntSize::height)>(spsdata.display_height);
+ }
+ }
+ } else {
+ vi->mExtraData = new MediaByteBuffer();
+ }
+
+ LOG("Created a VideoInfo for decoder - %s",
+ NS_ConvertUTF16toUTF8(vi->ToString()).get());
+
+ return track;
+}
+
+// https://w3c.github.io/webcodecs/#valid-videodecoderconfig
+/* static */
+bool VideoDecoderTraits::Validate(const VideoDecoderConfig& aConfig,
+ nsCString& aErrorMessage) {
+ Maybe<nsString> codec = ParseCodecString(aConfig.mCodec);
+ if (!codec || codec->IsEmpty()) {
+ LOGE("Invalid codec string");
+ return false;
+ }
+
+ if (aConfig.mCodedWidth.WasPassed() != aConfig.mCodedHeight.WasPassed()) {
+ LOGE("Missing coded %s",
+ aConfig.mCodedWidth.WasPassed() ? "height" : "width");
+ return false;
+ }
+ if (aConfig.mCodedWidth.WasPassed() &&
+ (aConfig.mCodedWidth.Value() == 0 || aConfig.mCodedHeight.Value() == 0)) {
+ LOGE("codedWidth and/or codedHeight can't be zero");
+ return false;
+ }
+
+ if (aConfig.mDisplayAspectWidth.WasPassed() !=
+ aConfig.mDisplayAspectHeight.WasPassed()) {
+ LOGE("Missing display aspect %s",
+ aConfig.mDisplayAspectWidth.WasPassed() ? "height" : "width");
+ return false;
+ }
+ if (aConfig.mDisplayAspectWidth.WasPassed() &&
+ (aConfig.mDisplayAspectWidth.Value() == 0 ||
+ aConfig.mDisplayAspectHeight.Value() == 0)) {
+ LOGE("display aspect width and height cannot be zero");
+ return false;
+ }
+
+ return true;
+}
+
+/* static */
+UniquePtr<VideoDecoderConfigInternal> VideoDecoderTraits::CreateConfigInternal(
+ const VideoDecoderConfig& aConfig) {
+ return VideoDecoderConfigInternal::Create(aConfig);
+}
+
+/* static */
+bool VideoDecoderTraits::IsKeyChunk(const EncodedVideoChunk& aInput) {
+ return aInput.Type() == EncodedVideoChunkType::Key;
+}
+
+/* static */
+UniquePtr<EncodedVideoChunkData> VideoDecoderTraits::CreateInputInternal(
+ const EncodedVideoChunk& aInput) {
+ return aInput.Clone();
+}
+
+/*
+ * Below are VideoDecoder implementation
+ */
+
+VideoDecoder::VideoDecoder(nsIGlobalObject* aParent,
+ RefPtr<WebCodecsErrorCallback>&& aErrorCallback,
+ RefPtr<VideoFrameOutputCallback>&& aOutputCallback)
+ : DecoderTemplate(aParent, std::move(aErrorCallback),
+ std::move(aOutputCallback)) {
+ MOZ_ASSERT(mErrorCallback);
+ MOZ_ASSERT(mOutputCallback);
+ LOG("VideoDecoder %p ctor", this);
+}
+
+VideoDecoder::~VideoDecoder() {
+ LOG("VideoDecoder %p dtor", this);
+}
+
+JSObject* VideoDecoder::WrapObject(JSContext* aCx,
+ JS::Handle<JSObject*> aGivenProto) {
+ AssertIsOnOwningThread();
+
+ return VideoDecoder_Binding::Wrap(aCx, this, aGivenProto);
+}
+
+// https://w3c.github.io/webcodecs/#dom-videodecoder-videodecoder
+/* static */
+already_AddRefed<VideoDecoder> VideoDecoder::Constructor(
+ const GlobalObject& aGlobal, const VideoDecoderInit& aInit,
+ ErrorResult& aRv) {
+ nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+ if (!global) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return nullptr;
+ }
+
+ return MakeAndAddRef<VideoDecoder>(
+ global.get(), RefPtr<WebCodecsErrorCallback>(aInit.mError),
+ RefPtr<VideoFrameOutputCallback>(aInit.mOutput));
+}
+
+// https://w3c.github.io/webcodecs/#dom-videodecoder-isconfigsupported
+/* static */
+already_AddRefed<Promise> VideoDecoder::IsConfigSupported(
+ const GlobalObject& aGlobal, const VideoDecoderConfig& aConfig,
+ ErrorResult& aRv) {
+ LOG("VideoDecoder::IsConfigSupported, config: %s",
+ NS_ConvertUTF16toUTF8(aConfig.mCodec).get());
+
+ nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+ if (!global) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return nullptr;
+ }
+
+ RefPtr<Promise> p = Promise::Create(global.get(), aRv);
+ if (NS_WARN_IF(aRv.Failed())) {
+ return p.forget();
+ }
+
+ nsCString errorMessage;
+ if (!VideoDecoderTraits::Validate(aConfig, errorMessage)) {
+ p->MaybeRejectWithTypeError(nsPrintfCString(
+ "VideoDecoderConfig is invalid: %s", errorMessage.get()));
+ return p.forget();
+ }
+
+ // TODO: Move the following works to another thread to unblock the current
+ // thread, as what spec suggests.
+
+ RootedDictionary<VideoDecoderConfig> config(aGlobal.Context());
+ auto r = CloneConfiguration(config, aGlobal.Context(), aConfig);
+ if (r.isErr()) {
+ nsresult e = r.unwrapErr();
+ LOGE("Failed to clone VideoDecoderConfig. Error: 0x%08" PRIx32,
+ static_cast<uint32_t>(e));
+ p->MaybeRejectWithTypeError("Failed to clone VideoDecoderConfig");
+ aRv.Throw(e);
+ return p.forget();
+ }
+
+ bool canDecode = CanDecode(config);
+ RootedDictionary<VideoDecoderSupport> s(aGlobal.Context());
+ s.mConfig.Construct(std::move(config));
+ s.mSupported.Construct(canDecode);
+
+ p->MaybeResolve(s);
+ return p.forget();
+}
+
+already_AddRefed<MediaRawData> VideoDecoder::InputDataToMediaRawData(
+ UniquePtr<EncodedVideoChunkData>&& aData, TrackInfo& aInfo,
+ const VideoDecoderConfigInternal& aConfig) {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(aInfo.GetAsVideoInfo());
+
+ if (!aData) {
+ LOGE("No data for conversion");
+ return nullptr;
+ }
+
+ RefPtr<MediaRawData> sample = aData->TakeData();
+ if (!sample) {
+ LOGE("Take no data for conversion");
+ return nullptr;
+ }
+
+ // aExtraData is either provided by Configure() or a default one created for
+ // the decoder creation. If it's created for decoder creation only, we don't
+ // set it to sample.
+ if (aConfig.mDescription && aInfo.GetAsVideoInfo()->mExtraData) {
+ sample->mExtraData = aInfo.GetAsVideoInfo()->mExtraData;
+ }
+
+ LOGV(
+ "EncodedVideoChunkData %p converted to %zu-byte MediaRawData - time: "
+ "%" PRIi64 "us, timecode: %" PRIi64 "us, duration: %" PRIi64
+ "us, key-frame: %s, has extra data: %s",
+ aData.get(), sample->Size(), sample->mTime.ToMicroseconds(),
+ sample->mTimecode.ToMicroseconds(), sample->mDuration.ToMicroseconds(),
+ sample->mKeyframe ? "yes" : "no", sample->mExtraData ? "yes" : "no");
+
+ return sample.forget();
+}
+
+nsTArray<RefPtr<VideoFrame>> VideoDecoder::DecodedDataToOutputType(
+ nsIGlobalObject* aGlobalObject, const nsTArray<RefPtr<MediaData>>&& aData,
+ VideoDecoderConfigInternal& aConfig) {
+ AssertIsOnOwningThread();
+
+ nsTArray<RefPtr<VideoFrame>> frames;
+ for (const RefPtr<MediaData>& data : aData) {
+ MOZ_RELEASE_ASSERT(data->mType == MediaData::Type::VIDEO_DATA);
+ RefPtr<const VideoData> d(data->As<const VideoData>());
+ VideoColorSpaceInternal colorSpace;
+ // Determine which color space to use: prefer the color space as configured
+ // at the decoder level, if it has one, otherwise look at the underlying
+ // image and make a guess.
+ if (aConfig.mColorSpace.isSome() &&
+ aConfig.mColorSpace->mPrimaries.isSome() &&
+ aConfig.mColorSpace->mTransfer.isSome() &&
+ aConfig.mColorSpace->mMatrix.isSome()) {
+ colorSpace = aConfig.mColorSpace.value();
+ } else {
+ colorSpace = GuessColorSpace(d->mImage.get());
+ }
+ frames.AppendElement(CreateVideoFrame(
+ aGlobalObject, d.get(), d->mTime.ToMicroseconds(),
+ static_cast<uint64_t>(d->mDuration.ToMicroseconds()),
+ aConfig.mDisplayAspectWidth, aConfig.mDisplayAspectHeight, colorSpace));
+ }
+ return frames;
+}
+
+#undef LOG
+#undef LOGW
+#undef LOGE
+#undef LOGV
+#undef LOG_INTERNAL
+
+} // namespace mozilla::dom
diff --git a/dom/media/webcodecs/VideoDecoder.h b/dom/media/webcodecs/VideoDecoder.h
new file mode 100644
index 0000000000..f4b3fe10ed
--- /dev/null
+++ b/dom/media/webcodecs/VideoDecoder.h
@@ -0,0 +1,79 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_VideoDecoder_h
+#define mozilla_dom_VideoDecoder_h
+
+#include "js/TypeDecls.h"
+#include "mozilla/ErrorResult.h"
+#include "mozilla/RefPtr.h"
+#include "mozilla/UniquePtr.h"
+#include "mozilla/dom/BindingDeclarations.h"
+#include "mozilla/dom/DecoderTemplate.h"
+#include "mozilla/dom/DecoderTypes.h"
+#include "mozilla/dom/VideoFrame.h"
+#include "nsCycleCollectionParticipant.h"
+
+class nsIGlobalObject;
+
+namespace mozilla {
+
+namespace dom {
+
+class EncodedVideoChunk;
+class EncodedVideoChunkData;
+class EventHandlerNonNull;
+class GlobalObject;
+class Promise;
+class VideoFrameOutputCallback;
+class WebCodecsErrorCallback;
+struct VideoDecoderConfig;
+struct VideoDecoderInit;
+
+} // namespace dom
+
+} // namespace mozilla
+
+namespace mozilla::dom {
+
+class VideoDecoder final : public DecoderTemplate<VideoDecoderTraits> {
+ public:
+ NS_DECL_ISUPPORTS_INHERITED
+ NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(VideoDecoder, DOMEventTargetHelper)
+
+ public:
+ VideoDecoder(nsIGlobalObject* aParent,
+ RefPtr<WebCodecsErrorCallback>&& aErrorCallback,
+ RefPtr<VideoFrameOutputCallback>&& aOutputCallback);
+
+ protected:
+ ~VideoDecoder();
+
+ public:
+ JSObject* WrapObject(JSContext* aCx,
+ JS::Handle<JSObject*> aGivenProto) override;
+
+ static already_AddRefed<VideoDecoder> Constructor(
+ const GlobalObject& aGlobal, const VideoDecoderInit& aInit,
+ ErrorResult& aRv);
+
+ static already_AddRefed<Promise> IsConfigSupported(
+ const GlobalObject& aGlobal, const VideoDecoderConfig& aConfig,
+ ErrorResult& aRv);
+
+ protected:
+ virtual already_AddRefed<MediaRawData> InputDataToMediaRawData(
+ UniquePtr<EncodedVideoChunkData>&& aData, TrackInfo& aInfo,
+ const VideoDecoderConfigInternal& aConfig) override;
+
+ virtual nsTArray<RefPtr<VideoFrame>> DecodedDataToOutputType(
+ nsIGlobalObject* aGlobalObject, const nsTArray<RefPtr<MediaData>>&& aData,
+ VideoDecoderConfigInternal& aConfig) override;
+};
+
+} // namespace mozilla::dom
+
+#endif // mozilla_dom_VideoDecoder_h
diff --git a/dom/media/webcodecs/VideoEncoder.cpp b/dom/media/webcodecs/VideoEncoder.cpp
new file mode 100644
index 0000000000..0e71417cb0
--- /dev/null
+++ b/dom/media/webcodecs/VideoEncoder.cpp
@@ -0,0 +1,624 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/dom/VideoEncoder.h"
+#include "mozilla/dom/VideoEncoderBinding.h"
+#include "mozilla/dom/VideoColorSpaceBinding.h"
+#include "mozilla/dom/VideoColorSpace.h"
+#include "mozilla/dom/VideoFrame.h"
+
+#include "EncoderTraits.h"
+#include "ImageContainer.h"
+#include "VideoUtils.h"
+#include "mozilla/Assertions.h"
+#include "mozilla/Logging.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/Unused.h"
+#include "mozilla/dom/EncodedVideoChunk.h"
+#include "mozilla/dom/EncodedVideoChunkBinding.h"
+#include "mozilla/dom/ImageUtils.h"
+#include "mozilla/dom/Promise.h"
+#include "mozilla/dom/VideoColorSpaceBinding.h"
+#include "mozilla/dom/VideoFrameBinding.h"
+#include "mozilla/dom/WebCodecsUtils.h"
+#include "nsPrintfCString.h"
+#include "nsReadableUtils.h"
+
+extern mozilla::LazyLogModule gWebCodecsLog;
+
+namespace mozilla::dom {
+
+#ifdef LOG_INTERNAL
+# undef LOG_INTERNAL
+#endif // LOG_INTERNAL
+#define LOG_INTERNAL(level, msg, ...) \
+ MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__))
+
+#ifdef LOG
+# undef LOG
+#endif // LOG
+#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__)
+
+#ifdef LOGW
+# undef LOGW
+#endif // LOGW
+#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__)
+
+#ifdef LOGE
+# undef LOGE
+#endif // LOGE
+#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__)
+
+#ifdef LOGV
+# undef LOGV
+#endif // LOGV
+#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__)
+
+NS_IMPL_CYCLE_COLLECTION_INHERITED(VideoEncoder, DOMEventTargetHelper,
+ mErrorCallback, mOutputCallback)
+NS_IMPL_ADDREF_INHERITED(VideoEncoder, DOMEventTargetHelper)
+NS_IMPL_RELEASE_INHERITED(VideoEncoder, DOMEventTargetHelper)
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(VideoEncoder)
+NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper)
+
+VideoEncoderConfigInternal::VideoEncoderConfigInternal(
+ const nsAString& aCodec, uint32_t aWidth, uint32_t aHeight,
+ Maybe<uint32_t>&& aDisplayWidth, Maybe<uint32_t>&& aDisplayHeight,
+ Maybe<uint32_t>&& aBitrate, Maybe<double>&& aFramerate,
+ const HardwareAcceleration& aHardwareAcceleration,
+ const AlphaOption& aAlpha, Maybe<nsString>&& aScalabilityMode,
+ const VideoEncoderBitrateMode& aBitrateMode,
+ const LatencyMode& aLatencyMode, Maybe<nsString>&& aContentHint)
+ : mCodec(aCodec),
+ mWidth(aWidth),
+ mHeight(aHeight),
+ mDisplayWidth(std::move(aDisplayWidth)),
+ mDisplayHeight(std::move(aDisplayHeight)),
+ mBitrate(std::move(aBitrate)),
+ mFramerate(std::move(aFramerate)),
+ mHardwareAcceleration(aHardwareAcceleration),
+ mAlpha(aAlpha),
+ mScalabilityMode(std::move(aScalabilityMode)),
+ mBitrateMode(aBitrateMode),
+ mLatencyMode(aLatencyMode),
+ mContentHint(std::move(aContentHint)) {}
+
+VideoEncoderConfigInternal::VideoEncoderConfigInternal(
+ const VideoEncoderConfigInternal& aConfig)
+ : mCodec(aConfig.mCodec),
+ mWidth(aConfig.mWidth),
+ mHeight(aConfig.mHeight),
+ mDisplayWidth(aConfig.mDisplayWidth),
+ mDisplayHeight(aConfig.mDisplayHeight),
+ mBitrate(aConfig.mBitrate),
+ mFramerate(aConfig.mFramerate),
+ mHardwareAcceleration(aConfig.mHardwareAcceleration),
+ mAlpha(aConfig.mAlpha),
+ mScalabilityMode(aConfig.mScalabilityMode),
+ mBitrateMode(aConfig.mBitrateMode),
+ mLatencyMode(aConfig.mLatencyMode),
+ mContentHint(aConfig.mContentHint),
+ mAvc(aConfig.mAvc) {
+}
+
+VideoEncoderConfigInternal::VideoEncoderConfigInternal(
+ const VideoEncoderConfig& aConfig)
+ : mCodec(aConfig.mCodec),
+ mWidth(aConfig.mWidth),
+ mHeight(aConfig.mHeight),
+ mDisplayWidth(OptionalToMaybe(aConfig.mDisplayWidth)),
+ mDisplayHeight(OptionalToMaybe(aConfig.mDisplayHeight)),
+ mBitrate(OptionalToMaybe(aConfig.mBitrate)),
+ mFramerate(OptionalToMaybe(aConfig.mFramerate)),
+ mHardwareAcceleration(aConfig.mHardwareAcceleration),
+ mAlpha(aConfig.mAlpha),
+ mScalabilityMode(OptionalToMaybe(aConfig.mScalabilityMode)),
+ mBitrateMode(aConfig.mBitrateMode),
+ mLatencyMode(aConfig.mLatencyMode),
+ mContentHint(OptionalToMaybe(aConfig.mContentHint)),
+ mAvc(OptionalToMaybe(aConfig.mAvc)) {
+}
+
+nsString VideoEncoderConfigInternal::ToString() const {
+ nsString rv;
+
+ rv.AppendPrintf("Codec: %s, [%" PRIu32 "x%" PRIu32 "],",
+ NS_ConvertUTF16toUTF8(mCodec).get(), mWidth, mHeight);
+ if (mDisplayWidth.isSome()) {
+ rv.AppendPrintf(", display[%" PRIu32 "x%" PRIu32 "]", mDisplayWidth.value(),
+ mDisplayHeight.value());
+ }
+ if (mBitrate.isSome()) {
+ rv.AppendPrintf(", %" PRIu32 "kbps", mBitrate.value());
+ }
+ if (mFramerate.isSome()) {
+ rv.AppendPrintf(", %lfHz", mFramerate.value());
+ }
+ rv.AppendPrintf(
+ " hw: %s",
+ HardwareAccelerationValues::GetString(mHardwareAcceleration).data());
+ rv.AppendPrintf(", alpha: %s", AlphaOptionValues::GetString(mAlpha).data());
+ if (mScalabilityMode.isSome()) {
+ rv.AppendPrintf(", scalability mode: %s",
+ NS_ConvertUTF16toUTF8(mScalabilityMode.value()).get());
+ }
+ rv.AppendPrintf(
+ ", bitrate mode: %s",
+ VideoEncoderBitrateModeValues::GetString(mBitrateMode).data());
+ rv.AppendPrintf(", latency mode: %s",
+ LatencyModeValues::GetString(mLatencyMode).data());
+ if (mContentHint.isSome()) {
+ rv.AppendPrintf(", content hint: %s",
+ NS_ConvertUTF16toUTF8(mContentHint.value()).get());
+ }
+ if (mAvc.isSome()) {
+ rv.AppendPrintf(", avc-specific: %s",
+ AvcBitstreamFormatValues::GetString(mAvc->mFormat).data());
+ }
+
+ return rv;
+}
+
+template <typename T>
+bool MaybeAreEqual(const Maybe<T>& aLHS, const Maybe<T> aRHS) {
+ if (aLHS.isSome() && aRHS.isSome()) {
+ return aLHS.value() == aRHS.value();
+ }
+ if (aLHS.isNothing() && aRHS.isNothing()) {
+ return true;
+ }
+ return false;
+}
+
+bool VideoEncoderConfigInternal::Equals(
+ const VideoEncoderConfigInternal& aOther) const {
+ bool sameCodecSpecific = true;
+ if ((mAvc.isSome() && aOther.mAvc.isSome() &&
+ mAvc->mFormat != aOther.mAvc->mFormat) ||
+ mAvc.isSome() != aOther.mAvc.isSome()) {
+ sameCodecSpecific = false;
+ }
+ return mCodec.Equals(aOther.mCodec) && mWidth == aOther.mWidth &&
+ mHeight == aOther.mHeight &&
+ MaybeAreEqual(mDisplayWidth, aOther.mDisplayWidth) &&
+ MaybeAreEqual(mDisplayHeight, aOther.mDisplayHeight) &&
+ MaybeAreEqual(mBitrate, aOther.mBitrate) &&
+ MaybeAreEqual(mFramerate, aOther.mFramerate) &&
+ mHardwareAcceleration == aOther.mHardwareAcceleration &&
+ mAlpha == aOther.mAlpha &&
+ MaybeAreEqual(mScalabilityMode, aOther.mScalabilityMode) &&
+ mBitrateMode == aOther.mBitrateMode &&
+ mLatencyMode == aOther.mLatencyMode &&
+ MaybeAreEqual(mContentHint, aOther.mContentHint) && sameCodecSpecific;
+}
+
+bool VideoEncoderConfigInternal::CanReconfigure(
+ const VideoEncoderConfigInternal& aOther) const {
+ return mCodec.Equals(aOther.mCodec) &&
+ mHardwareAcceleration == aOther.mHardwareAcceleration;
+}
+
+EncoderConfig VideoEncoderConfigInternal::ToEncoderConfig() const {
+ MediaDataEncoder::Usage usage;
+ if (mLatencyMode == LatencyMode::Quality) {
+ usage = MediaDataEncoder::Usage::Record;
+ } else {
+ usage = MediaDataEncoder::Usage::Realtime;
+ }
+ MediaDataEncoder::HardwarePreference hwPref =
+ MediaDataEncoder::HardwarePreference::None;
+ if (mHardwareAcceleration ==
+ mozilla::dom::HardwareAcceleration::Prefer_hardware) {
+ hwPref = MediaDataEncoder::HardwarePreference::RequireHardware;
+ } else if (mHardwareAcceleration ==
+ mozilla::dom::HardwareAcceleration::Prefer_software) {
+ hwPref = MediaDataEncoder::HardwarePreference::RequireSoftware;
+ }
+ CodecType codecType;
+ auto maybeCodecType = CodecStringToCodecType(mCodec);
+ if (maybeCodecType.isSome()) {
+ codecType = maybeCodecType.value();
+ } else {
+ MOZ_CRASH("The string should always contain a valid codec at this point.");
+ }
+ Maybe<EncoderConfig::CodecSpecific> specific;
+ if (codecType == CodecType::H264) {
+ uint8_t profile, constraints, level;
+ H264BitStreamFormat format;
+ if (mAvc) {
+ format = mAvc->mFormat == AvcBitstreamFormat::Annexb
+ ? H264BitStreamFormat::ANNEXB
+ : H264BitStreamFormat::AVC;
+ } else {
+ format = H264BitStreamFormat::AVC;
+ }
+ if (ExtractH264CodecDetails(mCodec, profile, constraints, level)) {
+ if (profile == H264_PROFILE_BASE || profile == H264_PROFILE_MAIN ||
+ profile == H264_PROFILE_EXTENDED || profile == H264_PROFILE_HIGH) {
+ specific.emplace(
+ H264Specific(static_cast<H264_PROFILE>(profile), static_cast<H264_LEVEL>(level), format));
+ }
+ }
+ }
+ // Only for vp9, not vp8
+ if (codecType == CodecType::VP9) {
+ uint8_t profile, level, bitdepth, chromasubsampling;
+ mozilla::VideoColorSpace colorspace;
+ DebugOnly<bool> rv = ExtractVPXCodecDetails(
+ mCodec, profile, level, bitdepth, chromasubsampling, colorspace);
+#ifdef DEBUG
+ if (!rv) {
+ LOGE("Error extracting VPX codec details, non fatal");
+ }
+#endif
+ specific.emplace(VP9Specific());
+ }
+ MediaDataEncoder::ScalabilityMode scalabilityMode;
+ if (mScalabilityMode) {
+ if (mScalabilityMode->EqualsLiteral("L1T2")) {
+ scalabilityMode = MediaDataEncoder::ScalabilityMode::L1T2;
+ } else if (mScalabilityMode->EqualsLiteral("L1T3")) {
+ scalabilityMode = MediaDataEncoder::ScalabilityMode::L1T3;
+ } else {
+ scalabilityMode = MediaDataEncoder::ScalabilityMode::None;
+ }
+ } else {
+ scalabilityMode = MediaDataEncoder::ScalabilityMode::None;
+ }
+ return EncoderConfig(
+ codecType, {mWidth, mHeight}, usage, ImageBitmapFormat::RGBA32, ImageBitmapFormat::RGBA32,
+ AssertedCast<uint8_t>(mFramerate.refOr(0.f)), 0, mBitrate.refOr(0),
+ mBitrateMode == VideoEncoderBitrateMode::Constant
+ ? MediaDataEncoder::BitrateMode::Constant
+ : MediaDataEncoder::BitrateMode::Variable,
+ hwPref, scalabilityMode, specific);
+}
+already_AddRefed<WebCodecsConfigurationChangeList>
+VideoEncoderConfigInternal::Diff(
+ const VideoEncoderConfigInternal& aOther) const {
+ auto list = MakeRefPtr<WebCodecsConfigurationChangeList>();
+ if (!mCodec.Equals(aOther.mCodec)) {
+ list->Push(CodecChange{aOther.mCodec});
+ }
+ // Both must always be present, when a `VideoEncoderConfig` is passed to
+ // `configure`.
+ if (mWidth != aOther.mWidth || mHeight != aOther.mHeight) {
+ list->Push(DimensionsChange{gfx::IntSize{aOther.mWidth, aOther.mHeight}});
+ }
+ // Similarly, both must always be present, when a `VideoEncoderConfig` is
+ // passed to `configure`.
+ if (!MaybeAreEqual(mDisplayWidth, aOther.mDisplayWidth) ||
+ !MaybeAreEqual(mDisplayHeight, aOther.mDisplayHeight)) {
+ Maybe<gfx::IntSize> displaySize =
+ aOther.mDisplayWidth.isSome()
+ ? Some(gfx::IntSize{aOther.mDisplayWidth.value(),
+ aOther.mDisplayHeight.value()})
+ : Nothing();
+ list->Push(DisplayDimensionsChange{displaySize});
+ }
+ if (!MaybeAreEqual(mBitrate, aOther.mBitrate)) {
+ list->Push(BitrateChange{aOther.mBitrate});
+ }
+ if (!MaybeAreEqual(mFramerate, aOther.mFramerate)) {
+ list->Push(FramerateChange{aOther.mFramerate});
+ }
+ if (mHardwareAcceleration != aOther.mHardwareAcceleration) {
+ list->Push(HardwareAccelerationChange{aOther.mHardwareAcceleration});
+ }
+ if (mAlpha != aOther.mAlpha) {
+ list->Push(AlphaChange{aOther.mAlpha});
+ }
+ if (!MaybeAreEqual(mScalabilityMode, aOther.mScalabilityMode)) {
+ list->Push(ScalabilityModeChange{aOther.mScalabilityMode});
+ }
+ if (mBitrateMode != aOther.mBitrateMode) {
+ list->Push(BitrateModeChange{aOther.mBitrateMode});
+ }
+ if (mLatencyMode != aOther.mLatencyMode) {
+ list->Push(LatencyModeChange{aOther.mLatencyMode});
+ }
+ if (!MaybeAreEqual(mContentHint, aOther.mContentHint)) {
+ list->Push(ContentHintChange{aOther.mContentHint});
+ }
+ return list.forget();
+}
+
+/*
+ * The followings are helpers for VideoEncoder methods
+ */
+static bool IsEncodeSupportedCodec(const nsAString& aCodec) {
+ LOG("IsEncodeSupported: %s", NS_ConvertUTF16toUTF8(aCodec).get());
+ if (!IsVP9CodecString(aCodec) && !IsVP8CodecString(aCodec) &&
+ !IsH264CodecString(aCodec) && !IsAV1CodecString(aCodec)) {
+ return false;
+ }
+
+ // Gecko allows codec string starts with vp9 or av1 but Webcodecs requires to
+ // starts with av01 and vp09.
+ // https://www.w3.org/TR/webcodecs-codec-registry/#video-codec-registry
+ if (StringBeginsWith(aCodec, u"vp9"_ns) ||
+ StringBeginsWith(aCodec, u"av1"_ns)) {
+ return false;
+ }
+
+ return true;
+}
+
+// https://w3c.github.io/webcodecs/#check-configuration-support
+static bool CanEncode(const RefPtr<VideoEncoderConfigInternal>& aConfig) {
+ auto parsedCodecString =
+ ParseCodecString(aConfig->mCodec).valueOr(EmptyString());
+ // TODO: Enable WebCodecs on Android (Bug 1840508)
+ if (IsOnAndroid()) {
+ return false;
+ }
+ if (!IsEncodeSupportedCodec(parsedCodecString)) {
+ return false;
+ }
+
+ // TODO (bug 1872879, bug 1872880): Support this on Windows and Mac.
+ if (aConfig->mScalabilityMode.isSome()) {
+ // We only support L1T2 and L1T3 ScalabilityMode in VP8 and VP9 encoders on
+ // Linux.
+ bool supported = IsOnLinux() && (IsVP8CodecString(parsedCodecString) ||
+ IsVP9CodecString(parsedCodecString))
+ ? aConfig->mScalabilityMode->EqualsLiteral("L1T2") ||
+ aConfig->mScalabilityMode->EqualsLiteral("L1T3")
+ : false;
+
+ if (!supported) {
+ LOGE("Scalability mode %s not supported for codec: %s",
+ NS_ConvertUTF16toUTF8(aConfig->mScalabilityMode.value()).get(),
+ NS_ConvertUTF16toUTF8(parsedCodecString).get());
+ return false;
+ }
+ }
+
+ return EncoderSupport::Supports(aConfig);
+}
+
+static Result<Ok, nsresult> CloneConfiguration(
+ VideoEncoderConfig& aDest, JSContext* aCx,
+ const VideoEncoderConfig& aConfig) {
+ nsCString errorMessage;
+ MOZ_ASSERT(VideoEncoderTraits::Validate(aConfig, errorMessage));
+
+ aDest.mCodec = aConfig.mCodec;
+ aDest.mWidth = aConfig.mWidth;
+ aDest.mHeight = aConfig.mHeight;
+ aDest.mAlpha = aConfig.mAlpha;
+ if (aConfig.mBitrate.WasPassed()) {
+ aDest.mBitrate.Construct(aConfig.mBitrate.Value());
+ }
+ aDest.mBitrateMode = aConfig.mBitrateMode;
+ if (aConfig.mContentHint.WasPassed()) {
+ aDest.mContentHint.Construct(aConfig.mContentHint.Value());
+ }
+ if (aConfig.mDisplayWidth.WasPassed()) {
+ aDest.mDisplayWidth.Construct(aConfig.mDisplayWidth.Value());
+ }
+ if (aConfig.mDisplayHeight.WasPassed()) {
+ aDest.mDisplayHeight.Construct(aConfig.mDisplayHeight.Value());
+ }
+ if (aConfig.mFramerate.WasPassed()) {
+ aDest.mFramerate.Construct(aConfig.mFramerate.Value());
+ }
+ aDest.mHardwareAcceleration = aConfig.mHardwareAcceleration;
+ aDest.mLatencyMode = aConfig.mLatencyMode;
+ if (aConfig.mScalabilityMode.WasPassed()) {
+ aDest.mScalabilityMode.Construct(aConfig.mScalabilityMode.Value());
+ }
+
+ // AVC specific
+ if (aConfig.mAvc.WasPassed()) {
+ aDest.mAvc.Construct(aConfig.mAvc.Value());
+ }
+
+ return Ok();
+}
+
+/* static */
+bool VideoEncoderTraits::IsSupported(
+ const VideoEncoderConfigInternal& aConfig) {
+ return CanEncode(MakeRefPtr<VideoEncoderConfigInternal>(aConfig));
+}
+
+// https://w3c.github.io/webcodecs/#valid-videoencoderconfig
+/* static */
+bool VideoEncoderTraits::Validate(const VideoEncoderConfig& aConfig,
+ nsCString& aErrorMessage) {
+ Maybe<nsString> codec = ParseCodecString(aConfig.mCodec);
+ // 1.
+ if (!codec || codec->IsEmpty()) {
+ LOGE("Invalid VideoEncoderConfig: invalid codec string");
+ return false;
+ }
+
+ // 2.
+ if (aConfig.mWidth == 0 || aConfig.mHeight == 0) {
+ LOGE("Invalid VideoEncoderConfig: %s equal to 0",
+ aConfig.mWidth == 0 ? "width" : "height");
+ return false;
+ }
+
+ // 3.
+ if ((aConfig.mDisplayWidth.WasPassed() &&
+ aConfig.mDisplayWidth.Value() == 0)) {
+ LOGE("Invalid VideoEncoderConfig: displayWidth equal to 0");
+ return false;
+ }
+ if ((aConfig.mDisplayHeight.WasPassed() &&
+ aConfig.mDisplayHeight.Value() == 0)) {
+ LOGE("Invalid VideoEncoderConfig: displayHeight equal to 0");
+ return false;
+ }
+
+ return true;
+}
+
+/* static */
+RefPtr<VideoEncoderConfigInternal> VideoEncoderTraits::CreateConfigInternal(
+ const VideoEncoderConfig& aConfig) {
+ return MakeRefPtr<VideoEncoderConfigInternal>(aConfig);
+}
+
+/* static */
+RefPtr<mozilla::VideoData> VideoEncoderTraits::CreateInputInternal(
+ const dom::VideoFrame& aInput,
+ const dom::VideoEncoderEncodeOptions& aOptions) {
+ media::TimeUnit duration =
+ aInput.GetDuration().IsNull()
+ ? media::TimeUnit::Zero()
+ : media::TimeUnit::FromMicroseconds(
+ AssertedCast<int64_t>(aInput.GetDuration().Value()));
+ media::TimeUnit pts = media::TimeUnit::FromMicroseconds(aInput.Timestamp());
+ return VideoData::CreateFromImage(
+ gfx::IntSize{aInput.DisplayWidth(), aInput.DisplayHeight()},
+ 0 /* bytestream offset */, pts, duration, aInput.GetImage(),
+ aOptions.mKeyFrame, pts);
+}
+
+/*
+ * Below are VideoEncoder implementation
+ */
+
+VideoEncoder::VideoEncoder(
+ nsIGlobalObject* aParent, RefPtr<WebCodecsErrorCallback>&& aErrorCallback,
+ RefPtr<EncodedVideoChunkOutputCallback>&& aOutputCallback)
+ : EncoderTemplate(aParent, std::move(aErrorCallback),
+ std::move(aOutputCallback)) {
+ MOZ_ASSERT(mErrorCallback);
+ MOZ_ASSERT(mOutputCallback);
+ LOG("VideoEncoder %p ctor", this);
+}
+
+VideoEncoder::~VideoEncoder() {
+ LOG("VideoEncoder %p dtor", this);
+ Unused << ResetInternal(NS_ERROR_DOM_ABORT_ERR);
+}
+
+JSObject* VideoEncoder::WrapObject(JSContext* aCx,
+ JS::Handle<JSObject*> aGivenProto) {
+ AssertIsOnOwningThread();
+
+ return VideoEncoder_Binding::Wrap(aCx, this, aGivenProto);
+}
+
+// https://w3c.github.io/webcodecs/#dom-videoencoder-videoencoder
+/* static */
+already_AddRefed<VideoEncoder> VideoEncoder::Constructor(
+ const GlobalObject& aGlobal, const VideoEncoderInit& aInit,
+ ErrorResult& aRv) {
+ nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+ if (!global) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return nullptr;
+ }
+
+ return MakeAndAddRef<VideoEncoder>(
+ global.get(), RefPtr<WebCodecsErrorCallback>(aInit.mError),
+ RefPtr<EncodedVideoChunkOutputCallback>(aInit.mOutput));
+}
+
+// https://w3c.github.io/webcodecs/#dom-videoencoder-isconfigsupported
+/* static */
+already_AddRefed<Promise> VideoEncoder::IsConfigSupported(
+ const GlobalObject& aGlobal, const VideoEncoderConfig& aConfig,
+ ErrorResult& aRv) {
+ LOG("VideoEncoder::IsConfigSupported, config: %s",
+ NS_ConvertUTF16toUTF8(aConfig.mCodec).get());
+
+ nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+ if (!global) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return nullptr;
+ }
+
+ RefPtr<Promise> p = Promise::Create(global.get(), aRv);
+ if (NS_WARN_IF(aRv.Failed())) {
+ return p.forget();
+ }
+
+ nsCString errorMessage;
+ if (!VideoEncoderTraits::Validate(aConfig, errorMessage)) {
+ p->MaybeRejectWithTypeError(nsPrintfCString(
+ "IsConfigSupported: config is invalid: %s", errorMessage.get()));
+ return p.forget();
+ }
+
+ // TODO: Move the following works to another thread to unblock the current
+ // thread, as what spec suggests.
+
+ VideoEncoderConfig config;
+ auto r = CloneConfiguration(config, aGlobal.Context(), aConfig);
+ if (r.isErr()) {
+ nsresult e = r.unwrapErr();
+ LOGE("Failed to clone VideoEncoderConfig. Error: 0x%08" PRIx32,
+ static_cast<uint32_t>(e));
+ p->MaybeRejectWithTypeError("Failed to clone VideoEncoderConfig");
+ aRv.Throw(e);
+ return p.forget();
+ }
+
+ bool canEncode = CanEncode(MakeRefPtr<VideoEncoderConfigInternal>(config));
+ VideoEncoderSupport s;
+ s.mConfig.Construct(std::move(config));
+ s.mSupported.Construct(canEncode);
+
+ p->MaybeResolve(s);
+ return p.forget();
+}
+
+RefPtr<EncodedVideoChunk> VideoEncoder::EncodedDataToOutputType(
+ nsIGlobalObject* aGlobalObject, RefPtr<MediaRawData>& aData) {
+ AssertIsOnOwningThread();
+
+ MOZ_RELEASE_ASSERT(aData->mType == MediaData::Type::RAW_DATA);
+ // Package into an EncodedVideoChunk
+ auto buffer =
+ MakeRefPtr<MediaAlignedByteBuffer>(aData->Data(), aData->Size());
+ auto encodedVideoChunk = MakeRefPtr<EncodedVideoChunk>(
+ aGlobalObject, buffer.forget(),
+ aData->mKeyframe ? EncodedVideoChunkType::Key
+ : EncodedVideoChunkType::Delta,
+ aData->mTime.ToMicroseconds(),
+ aData->mDuration.IsZero() ? Nothing()
+ : Some(aData->mDuration.ToMicroseconds()));
+ return encodedVideoChunk;
+}
+
+VideoDecoderConfigInternal VideoEncoder::EncoderConfigToDecoderConfig(
+ nsIGlobalObject* aGlobal, const RefPtr<MediaRawData>& aRawData,
+ const VideoEncoderConfigInternal& mOutputConfig) const {
+ // Colorspace is mandatory when outputing a decoder config after encode
+ VideoColorSpaceInternal init;
+ init.mFullRange.emplace(false);
+ init.mMatrix.emplace(VideoMatrixCoefficients::Bt709);
+ init.mPrimaries.emplace(VideoColorPrimaries::Bt709);
+ init.mTransfer.emplace(VideoTransferCharacteristics::Bt709);
+
+ return VideoDecoderConfigInternal(
+ mOutputConfig.mCodec, /* aCodec */
+ Some(mOutputConfig.mHeight), /* aCodedHeight */
+ Some(mOutputConfig.mWidth), /* aCodedWidth */
+ Some(init), /* aColorSpace */
+ aRawData->mExtraData && !aRawData->mExtraData->IsEmpty()
+ ? Some(aRawData->mExtraData)
+ : Nothing(), /* aDescription*/
+ Maybe<uint32_t>(mOutputConfig.mDisplayHeight), /* aDisplayAspectHeight*/
+ Maybe<uint32_t>(mOutputConfig.mDisplayWidth), /* aDisplayAspectWidth */
+ mOutputConfig.mHardwareAcceleration, /* aHardwareAcceleration */
+ Nothing() /* aOptimizeForLatency */
+ );
+}
+
+#undef LOG
+#undef LOGW
+#undef LOGE
+#undef LOGV
+#undef LOG_INTERNAL
+
+} // namespace mozilla::dom
diff --git a/dom/media/webcodecs/VideoEncoder.h b/dom/media/webcodecs/VideoEncoder.h
new file mode 100644
index 0000000000..9251b5023a
--- /dev/null
+++ b/dom/media/webcodecs/VideoEncoder.h
@@ -0,0 +1,78 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_VideoEncoder_h
+#define mozilla_dom_VideoEncoder_h
+
+#include "js/TypeDecls.h"
+#include "mozilla/ErrorResult.h"
+#include "mozilla/RefPtr.h"
+#include "mozilla/dom/BindingDeclarations.h"
+#include "mozilla/dom/EncoderTemplate.h"
+#include "mozilla/dom/EncoderTypes.h"
+#include "mozilla/dom/VideoFrame.h"
+#include "nsCycleCollectionParticipant.h"
+
+class nsIGlobalObject;
+
+namespace mozilla {
+
+namespace dom {
+
+class EncodedVideoChunk;
+class EncodedVideoChunkData;
+class EventHandlerNonNull;
+class GlobalObject;
+class Promise;
+class VideoFrameOutputCallback;
+class WebCodecsErrorCallback;
+struct VideoEncoderConfig;
+struct VideoEncoderInit;
+
+} // namespace dom
+
+} // namespace mozilla
+
+namespace mozilla::dom {
+
+class VideoEncoder final : public EncoderTemplate<VideoEncoderTraits> {
+ public:
+ NS_DECL_ISUPPORTS_INHERITED
+ NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(VideoEncoder, DOMEventTargetHelper)
+
+ public:
+ VideoEncoder(nsIGlobalObject* aParent,
+ RefPtr<WebCodecsErrorCallback>&& aErrorCallback,
+ RefPtr<EncodedVideoChunkOutputCallback>&& aOutputCallback);
+
+ protected:
+ ~VideoEncoder();
+
+ public:
+ JSObject* WrapObject(JSContext* aCx,
+ JS::Handle<JSObject*> aGivenProto) override;
+
+ static already_AddRefed<VideoEncoder> Constructor(
+ const GlobalObject& aGlobal, const VideoEncoderInit& aInit,
+ ErrorResult& aRv);
+
+ static already_AddRefed<Promise> IsConfigSupported(
+ const GlobalObject& aGlobal, const VideoEncoderConfig& aConfig,
+ ErrorResult& aRv);
+
+ protected:
+ virtual RefPtr<EncodedVideoChunk> EncodedDataToOutputType(
+ nsIGlobalObject* aGlobal, RefPtr<MediaRawData>& aData) override;
+
+ virtual VideoDecoderConfigInternal EncoderConfigToDecoderConfig(
+ nsIGlobalObject* aGlobal /* TODO: delete */,
+ const RefPtr<MediaRawData>& aRawData,
+ const VideoEncoderConfigInternal& mOutputConfig) const override;
+};
+
+} // namespace mozilla::dom
+
+#endif // mozilla_dom_VideoEncoder_h
diff --git a/dom/media/webcodecs/VideoFrame.cpp b/dom/media/webcodecs/VideoFrame.cpp
new file mode 100644
index 0000000000..602bc95c29
--- /dev/null
+++ b/dom/media/webcodecs/VideoFrame.cpp
@@ -0,0 +1,2417 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/dom/VideoFrame.h"
+#include "mozilla/dom/VideoFrameBinding.h"
+
+#include <math.h>
+#include <limits>
+#include <utility>
+
+#include "ImageContainer.h"
+#include "VideoColorSpace.h"
+#include "WebCodecsUtils.h"
+#include "js/StructuredClone.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/ResultVariant.h"
+#include "mozilla/ScopeExit.h"
+#include "mozilla/Try.h"
+
+#include "mozilla/UniquePtr.h"
+#include "mozilla/dom/CanvasUtils.h"
+#include "mozilla/dom/DOMRect.h"
+#include "mozilla/dom/HTMLCanvasElement.h"
+#include "mozilla/dom/HTMLImageElement.h"
+#include "mozilla/dom/HTMLVideoElement.h"
+#include "mozilla/dom/ImageBitmap.h"
+#include "mozilla/dom/ImageUtils.h"
+#include "mozilla/dom/OffscreenCanvas.h"
+#include "mozilla/dom/Promise.h"
+#include "mozilla/dom/SVGImageElement.h"
+#include "mozilla/dom/StructuredCloneHolder.h"
+#include "mozilla/dom/StructuredCloneTags.h"
+#include "mozilla/dom/UnionTypes.h"
+#include "mozilla/gfx/2D.h"
+#include "mozilla/gfx/Swizzle.h"
+#include "mozilla/layers/LayersSurfaces.h"
+#include "nsLayoutUtils.h"
+#include "nsIPrincipal.h"
+#include "nsIURI.h"
+
+extern mozilla::LazyLogModule gWebCodecsLog;
+
+namespace mozilla::dom {
+
+#ifdef LOG_INTERNAL
+# undef LOG_INTERNAL
+#endif // LOG_INTERNAL
+#define LOG_INTERNAL(level, msg, ...) \
+ MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__))
+
+#ifdef LOG
+# undef LOG
+#endif // LOG
+#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__)
+
+#ifdef LOGW
+# undef LOGW
+#endif // LOGW
+#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__)
+
+NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(VideoFrame)
+NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(VideoFrame)
+ tmp->CloseIfNeeded();
+ NS_IMPL_CYCLE_COLLECTION_UNLINK(mParent)
+ NS_IMPL_CYCLE_COLLECTION_UNLINK_PRESERVED_WRAPPER
+NS_IMPL_CYCLE_COLLECTION_UNLINK_END
+NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(VideoFrame)
+ NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mParent)
+NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
+
+NS_IMPL_CYCLE_COLLECTING_ADDREF(VideoFrame)
+// VideoFrame should be released as soon as its refcount drops to zero,
+// without waiting for async deletion by the cycle collector, since it may hold
+// a large-size image.
+NS_IMPL_CYCLE_COLLECTING_RELEASE_WITH_LAST_RELEASE(VideoFrame, CloseIfNeeded())
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(VideoFrame)
+ NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY
+ NS_INTERFACE_MAP_ENTRY(nsISupports)
+NS_INTERFACE_MAP_END
+
+/*
+ * The following are helpers to read the image data from the given buffer and
+ * the format. The data layout is illustrated in the comments for
+ * `VideoFrame::Format` below.
+ */
+
+static int32_t CeilingOfHalf(int32_t aValue) {
+ MOZ_ASSERT(aValue >= 0);
+ return aValue / 2 + (aValue % 2);
+}
+
+class YUVBufferReaderBase {
+ public:
+ YUVBufferReaderBase(const Span<uint8_t>& aBuffer, int32_t aWidth,
+ int32_t aHeight)
+ : mWidth(aWidth), mHeight(aHeight), mStrideY(aWidth), mBuffer(aBuffer) {}
+ virtual ~YUVBufferReaderBase() = default;
+
+ const uint8_t* DataY() const { return mBuffer.data(); }
+ const int32_t mWidth;
+ const int32_t mHeight;
+ const int32_t mStrideY;
+
+ protected:
+ CheckedInt<size_t> YByteSize() const {
+ return CheckedInt<size_t>(mStrideY) * mHeight;
+ }
+
+ const Span<uint8_t> mBuffer;
+};
+
+class I420ABufferReader;
+class I420BufferReader : public YUVBufferReaderBase {
+ public:
+ I420BufferReader(const Span<uint8_t>& aBuffer, int32_t aWidth,
+ int32_t aHeight)
+ : YUVBufferReaderBase(aBuffer, aWidth, aHeight),
+ mStrideU(CeilingOfHalf(aWidth)),
+ mStrideV(CeilingOfHalf(aWidth)) {}
+ virtual ~I420BufferReader() = default;
+
+ const uint8_t* DataU() const { return &mBuffer[YByteSize().value()]; }
+ const uint8_t* DataV() const {
+ return &mBuffer[YByteSize().value() + UByteSize().value()];
+ }
+ virtual I420ABufferReader* AsI420ABufferReader() { return nullptr; }
+
+ const int32_t mStrideU;
+ const int32_t mStrideV;
+
+ protected:
+ CheckedInt<size_t> UByteSize() const {
+ return CheckedInt<size_t>(CeilingOfHalf(mHeight)) * mStrideU;
+ }
+
+ CheckedInt<size_t> VSize() const {
+ return CheckedInt<size_t>(CeilingOfHalf(mHeight)) * mStrideV;
+ }
+};
+
+class I420ABufferReader final : public I420BufferReader {
+ public:
+ I420ABufferReader(const Span<uint8_t>& aBuffer, int32_t aWidth,
+ int32_t aHeight)
+ : I420BufferReader(aBuffer, aWidth, aHeight), mStrideA(aWidth) {
+ MOZ_ASSERT(mStrideA == mStrideY);
+ }
+ virtual ~I420ABufferReader() = default;
+
+ const uint8_t* DataA() const {
+ return &mBuffer[YByteSize().value() + UByteSize().value() +
+ VSize().value()];
+ }
+
+ virtual I420ABufferReader* AsI420ABufferReader() override { return this; }
+
+ const int32_t mStrideA;
+};
+
+class NV12BufferReader final : public YUVBufferReaderBase {
+ public:
+ NV12BufferReader(const Span<uint8_t>& aBuffer, int32_t aWidth,
+ int32_t aHeight)
+ : YUVBufferReaderBase(aBuffer, aWidth, aHeight),
+ mStrideUV(aWidth + aWidth % 2) {}
+ virtual ~NV12BufferReader() = default;
+
+ const uint8_t* DataUV() const { return &mBuffer[YByteSize().value()]; }
+
+ const int32_t mStrideUV;
+};
+
+/*
+ * The followings are helpers defined in
+ * https://w3c.github.io/webcodecs/#videoframe-algorithms
+ */
+
+static bool IsSameOrigin(nsIGlobalObject* aGlobal, const VideoFrame& aFrame) {
+ MOZ_ASSERT(aGlobal);
+ MOZ_ASSERT(aFrame.GetParentObject());
+
+ nsIPrincipal* principalX = aGlobal->PrincipalOrNull();
+ nsIPrincipal* principalY = aFrame.GetParentObject()->PrincipalOrNull();
+
+ // If both of VideoFrames are created in worker, they are in the same origin
+ // domain.
+ if (!principalX) {
+ return !principalY;
+ }
+ // Otherwise, check their domains.
+ return principalX->Equals(principalY);
+}
+
+static bool IsSameOrigin(nsIGlobalObject* aGlobal,
+ HTMLVideoElement& aVideoElement) {
+ MOZ_ASSERT(aGlobal);
+
+ // If CORS is in use, consider the video source is same-origin.
+ if (aVideoElement.GetCORSMode() != CORS_NONE) {
+ return true;
+ }
+
+ // Otherwise, check if video source has cross-origin redirect or not.
+ if (aVideoElement.HadCrossOriginRedirects()) {
+ return false;
+ }
+
+ // Finally, compare the VideoFrame's domain and video's one.
+ nsIPrincipal* principal = aGlobal->PrincipalOrNull();
+ nsCOMPtr<nsIPrincipal> elementPrincipal =
+ aVideoElement.GetCurrentVideoPrincipal();
+ // <video> cannot be created in worker, so it should have a valid principal.
+ if (NS_WARN_IF(!elementPrincipal) || !principal) {
+ return false;
+ }
+ return principal->Subsumes(elementPrincipal);
+}
+
+// A sub-helper to convert DOMRectInit to gfx::IntRect.
+static Result<gfx::IntRect, nsCString> ToIntRect(const DOMRectInit& aRectInit) {
+ auto EQ = [](const double& a, const double& b) {
+ constexpr double e = std::numeric_limits<double>::epsilon();
+ return std::fabs(a - b) <= e;
+ };
+ auto GT = [&](const double& a, const double& b) {
+ return !EQ(a, b) && a > b;
+ };
+
+ // Make sure the double values are in the gfx::IntRect's valid range, before
+ // checking the spec's valid range. The double's infinity value is larger than
+ // gfx::IntRect's max value so it will be filtered out here.
+ constexpr double MAX = static_cast<double>(
+ std::numeric_limits<decltype(gfx::IntRect::x)>::max());
+ constexpr double MIN = static_cast<double>(
+ std::numeric_limits<decltype(gfx::IntRect::x)>::min());
+ if (GT(aRectInit.mX, MAX) || GT(MIN, aRectInit.mX)) {
+ return Err(nsCString("x is out of the valid range"));
+ }
+ if (GT(aRectInit.mY, MAX) || GT(MIN, aRectInit.mY)) {
+ return Err(nsCString("y is out of the valid range"));
+ }
+ if (GT(aRectInit.mWidth, MAX) || GT(MIN, aRectInit.mWidth)) {
+ return Err(nsCString("width is out of the valid range"));
+ }
+ if (GT(aRectInit.mHeight, MAX) || GT(MIN, aRectInit.mHeight)) {
+ return Err(nsCString("height is out of the valid range"));
+ }
+
+ gfx::IntRect rect(
+ static_cast<decltype(gfx::IntRect::x)>(aRectInit.mX),
+ static_cast<decltype(gfx::IntRect::y)>(aRectInit.mY),
+ static_cast<decltype(gfx::IntRect::width)>(aRectInit.mWidth),
+ static_cast<decltype(gfx::IntRect::height)>(aRectInit.mHeight));
+ // Check the spec's valid range.
+ if (rect.X() < 0) {
+ return Err(nsCString("x must be non-negative"));
+ }
+ if (rect.Y() < 0) {
+ return Err(nsCString("y must be non-negative"));
+ }
+ if (rect.Width() <= 0) {
+ return Err(nsCString("width must be positive"));
+ }
+ if (rect.Height() <= 0) {
+ return Err(nsCString("height must be positive"));
+ }
+
+ return rect;
+}
+
+// A sub-helper to convert a (width, height) pair to gfx::IntRect.
+static Result<gfx::IntSize, nsCString> ToIntSize(const uint32_t& aWidth,
+ const uint32_t& aHeight) {
+ // Make sure the given values are in the gfx::IntSize's valid range, before
+ // checking the spec's valid range.
+ constexpr uint32_t MAX = static_cast<uint32_t>(
+ std::numeric_limits<decltype(gfx::IntRect::width)>::max());
+ if (aWidth > MAX) {
+ return Err(nsCString("Width exceeds the implementation's range"));
+ }
+ if (aHeight > MAX) {
+ return Err(nsCString("Height exceeds the implementation's range"));
+ }
+
+ gfx::IntSize size(static_cast<decltype(gfx::IntRect::width)>(aWidth),
+ static_cast<decltype(gfx::IntRect::height)>(aHeight));
+ // Check the spec's valid range.
+ if (size.Width() <= 0) {
+ return Err(nsCString("Width must be positive"));
+ }
+ if (size.Height() <= 0) {
+ return Err(nsCString("Height must be positive"));
+ }
+ return size;
+}
+
+// A sub-helper to make sure visible range is in the picture.
+static Result<Ok, nsCString> ValidateVisibility(
+ const gfx::IntRect& aVisibleRect, const gfx::IntSize& aPicSize) {
+ MOZ_ASSERT(aVisibleRect.X() >= 0);
+ MOZ_ASSERT(aVisibleRect.Y() >= 0);
+ MOZ_ASSERT(aVisibleRect.Width() > 0);
+ MOZ_ASSERT(aVisibleRect.Height() > 0);
+
+ const auto w = CheckedInt<uint32_t>(aVisibleRect.Width()) + aVisibleRect.X();
+ if (w.value() > static_cast<uint32_t>(aPicSize.Width())) {
+ return Err(nsCString(
+ "Sum of visible rectangle's x and width exceeds the picture's width"));
+ }
+
+ const auto h = CheckedInt<uint32_t>(aVisibleRect.Height()) + aVisibleRect.Y();
+ if (h.value() > static_cast<uint32_t>(aPicSize.Height())) {
+ return Err(
+ nsCString("Sum of visible rectangle's y and height exceeds the "
+ "picture's height"));
+ }
+
+ return Ok();
+}
+
+// A sub-helper to check and get display{Width, Height} in
+// VideoFrame(Buffer)Init.
+template <class T>
+static Result<Maybe<gfx::IntSize>, nsCString> MaybeGetDisplaySize(
+ const T& aInit) {
+ if (aInit.mDisplayWidth.WasPassed() != aInit.mDisplayHeight.WasPassed()) {
+ return Err(nsCString(
+ "displayWidth and displayHeight cannot be set without the other"));
+ }
+
+ Maybe<gfx::IntSize> displaySize;
+ if (aInit.mDisplayWidth.WasPassed() && aInit.mDisplayHeight.WasPassed()) {
+ displaySize.emplace();
+ MOZ_TRY_VAR(displaySize.ref(), ToIntSize(aInit.mDisplayWidth.Value(),
+ aInit.mDisplayHeight.Value())
+ .mapErr([](nsCString error) {
+ error.Insert("display", 0);
+ return error;
+ }));
+ }
+ return displaySize;
+}
+
+// https://w3c.github.io/webcodecs/#valid-videoframebufferinit
+static Result<
+ std::tuple<gfx::IntSize, Maybe<gfx::IntRect>, Maybe<gfx::IntSize>>,
+ nsCString>
+ValidateVideoFrameBufferInit(const VideoFrameBufferInit& aInit) {
+ gfx::IntSize codedSize;
+ MOZ_TRY_VAR(codedSize, ToIntSize(aInit.mCodedWidth, aInit.mCodedHeight)
+ .mapErr([](nsCString error) {
+ error.Insert("coded", 0);
+ return error;
+ }));
+
+ Maybe<gfx::IntRect> visibleRect;
+ if (aInit.mVisibleRect.WasPassed()) {
+ visibleRect.emplace();
+ MOZ_TRY_VAR(
+ visibleRect.ref(),
+ ToIntRect(aInit.mVisibleRect.Value()).mapErr([](nsCString error) {
+ error.Insert("visibleRect's ", 0);
+ return error;
+ }));
+ MOZ_TRY(ValidateVisibility(visibleRect.ref(), codedSize));
+ }
+
+ Maybe<gfx::IntSize> displaySize;
+ MOZ_TRY_VAR(displaySize, MaybeGetDisplaySize(aInit));
+
+ return std::make_tuple(codedSize, visibleRect, displaySize);
+}
+
+// https://w3c.github.io/webcodecs/#videoframe-verify-rect-offset-alignment
+static Result<Ok, nsCString> VerifyRectOffsetAlignment(
+ const Maybe<VideoFrame::Format>& aFormat, const gfx::IntRect& aRect) {
+ if (!aFormat) {
+ return Ok();
+ }
+ for (const VideoFrame::Format::Plane& p : aFormat->Planes()) {
+ const gfx::IntSize sample = aFormat->SampleSize(p);
+ if (aRect.X() % sample.Width() != 0) {
+ return Err(nsCString("Mismatch between format and given left offset"));
+ }
+
+ if (aRect.Y() % sample.Height() != 0) {
+ return Err(nsCString("Mismatch between format and given top offset"));
+ }
+ }
+ return Ok();
+}
+
+// https://w3c.github.io/webcodecs/#videoframe-parse-visible-rect
+static Result<gfx::IntRect, nsCString> ParseVisibleRect(
+ const gfx::IntRect& aDefaultRect, const Maybe<gfx::IntRect>& aOverrideRect,
+ const gfx::IntSize& aCodedSize, const VideoFrame::Format& aFormat) {
+ MOZ_ASSERT(ValidateVisibility(aDefaultRect, aCodedSize).isOk());
+
+ gfx::IntRect rect = aDefaultRect;
+ if (aOverrideRect) {
+ // Skip checking overrideRect's width and height here. They should be
+ // checked before reaching here, and ValidateVisibility will assert it.
+
+ MOZ_TRY(ValidateVisibility(aOverrideRect.ref(), aCodedSize));
+ rect = *aOverrideRect;
+ }
+
+ MOZ_TRY(VerifyRectOffsetAlignment(Some(aFormat), rect));
+
+ return rect;
+}
+
+// https://w3c.github.io/webcodecs/#computed-plane-layout
+struct ComputedPlaneLayout {
+ // The offset from the beginning of the buffer in one plane.
+ uint32_t mDestinationOffset = 0;
+ // The stride of the image data in one plane.
+ uint32_t mDestinationStride = 0;
+ // Sample count of picture's top offset (a.k.a samples of y).
+ uint32_t mSourceTop = 0;
+ // Sample count of the picture's height.
+ uint32_t mSourceHeight = 0;
+ // Byte count of the picture's left offset (a.k.a bytes of x).
+ uint32_t mSourceLeftBytes = 0;
+ // Byte count of the picture's width.
+ uint32_t mSourceWidthBytes = 0;
+};
+
+// https://w3c.github.io/webcodecs/#combined-buffer-layout
+struct CombinedBufferLayout {
+ CombinedBufferLayout() : mAllocationSize(0) {}
+ CombinedBufferLayout(uint32_t aAllocationSize,
+ nsTArray<ComputedPlaneLayout>&& aLayout)
+ : mAllocationSize(aAllocationSize),
+ mComputedLayouts(std::move(aLayout)) {}
+ uint32_t mAllocationSize = 0;
+ nsTArray<ComputedPlaneLayout> mComputedLayouts;
+};
+
+// https://w3c.github.io/webcodecs/#videoframe-compute-layout-and-allocation-size
+static Result<CombinedBufferLayout, nsCString> ComputeLayoutAndAllocationSize(
+ const gfx::IntRect& aRect, const VideoFrame::Format& aFormat,
+ const Sequence<PlaneLayout>* aPlaneLayouts) {
+ nsTArray<VideoFrame::Format::Plane> planes = aFormat.Planes();
+
+ if (aPlaneLayouts && aPlaneLayouts->Length() != planes.Length()) {
+ return Err(nsCString("Mismatch between format and layout"));
+ }
+
+ uint32_t minAllocationSize = 0;
+ nsTArray<ComputedPlaneLayout> layouts;
+ nsTArray<uint32_t> endOffsets;
+
+ for (size_t i = 0; i < planes.Length(); ++i) {
+ const VideoFrame::Format::Plane& p = planes[i];
+ const gfx::IntSize sampleSize = aFormat.SampleSize(p);
+ MOZ_RELEASE_ASSERT(!sampleSize.IsEmpty());
+
+ // aRect's x, y, width, and height are int32_t, and sampleSize's width and
+ // height >= 1, so (aRect.* / sampleSize.*) must be in int32_t range.
+
+ CheckedUint32 sourceTop(aRect.Y());
+ sourceTop /= sampleSize.Height();
+ MOZ_RELEASE_ASSERT(sourceTop.isValid());
+
+ CheckedUint32 sourceHeight(aRect.Height());
+ sourceHeight /= sampleSize.Height();
+ MOZ_RELEASE_ASSERT(sourceHeight.isValid());
+
+ CheckedUint32 sourceLeftBytes(aRect.X());
+ sourceLeftBytes /= sampleSize.Width();
+ MOZ_RELEASE_ASSERT(sourceLeftBytes.isValid());
+ sourceLeftBytes *= aFormat.SampleBytes(p);
+ if (!sourceLeftBytes.isValid()) {
+ return Err(nsPrintfCString(
+ "The parsed-rect's x-offset is too large for %s plane",
+ aFormat.PlaneName(p)));
+ }
+
+ CheckedUint32 sourceWidthBytes(aRect.Width());
+ sourceWidthBytes /= sampleSize.Width();
+ MOZ_RELEASE_ASSERT(sourceWidthBytes.isValid());
+ sourceWidthBytes *= aFormat.SampleBytes(p);
+ if (!sourceWidthBytes.isValid()) {
+ return Err(
+ nsPrintfCString("The parsed-rect's width is too large for %s plane",
+ aFormat.PlaneName(p)));
+ }
+
+ // TODO: Spec here is wrong so we do differently:
+ // https://github.com/w3c/webcodecs/issues/511
+ // This comment should be removed once the issue is resolved.
+ ComputedPlaneLayout layout{.mDestinationOffset = 0,
+ .mDestinationStride = 0,
+ .mSourceTop = sourceTop.value(),
+ .mSourceHeight = sourceHeight.value(),
+ .mSourceLeftBytes = sourceLeftBytes.value(),
+ .mSourceWidthBytes = sourceWidthBytes.value()};
+ if (aPlaneLayouts) {
+ const PlaneLayout& planeLayout = aPlaneLayouts->ElementAt(i);
+ if (planeLayout.mStride < layout.mSourceWidthBytes) {
+ return Err(nsPrintfCString("The stride in %s plane is too small",
+ aFormat.PlaneName(p)));
+ }
+ layout.mDestinationOffset = planeLayout.mOffset;
+ layout.mDestinationStride = planeLayout.mStride;
+ } else {
+ layout.mDestinationOffset = minAllocationSize;
+ layout.mDestinationStride = layout.mSourceWidthBytes;
+ }
+
+ const CheckedInt<uint32_t> planeSize =
+ CheckedInt<uint32_t>(layout.mDestinationStride) * layout.mSourceHeight;
+ if (!planeSize.isValid()) {
+ return Err(nsCString("Invalid layout with an over-sized plane"));
+ }
+ const CheckedInt<uint32_t> planeEnd = planeSize + layout.mDestinationOffset;
+ if (!planeEnd.isValid()) {
+ return Err(nsCString("Invalid layout with the out-out-bound offset"));
+ }
+ endOffsets.AppendElement(planeEnd.value());
+
+ minAllocationSize = std::max(minAllocationSize, planeEnd.value());
+
+ for (size_t j = 0; j < i; ++j) {
+ const ComputedPlaneLayout& earlier = layouts[j];
+ // If the current data's end is smaller or equal to the previous one's
+ // head, or if the previous data's end is smaller or equal to the current
+ // one's head, then they do not overlap. Otherwise, they do.
+ if (endOffsets[i] > earlier.mDestinationOffset &&
+ endOffsets[j] > layout.mDestinationOffset) {
+ return Err(nsCString("Invalid layout with the overlapped planes"));
+ }
+ }
+ layouts.AppendElement(layout);
+ }
+
+ return CombinedBufferLayout(minAllocationSize, std::move(layouts));
+}
+
+// https://w3c.github.io/webcodecs/#videoframe-verify-rect-size-alignment
+static Result<Ok, nsCString> VerifyRectSizeAlignment(
+ const VideoFrame::Format& aFormat, const gfx::IntRect& aRect) {
+ for (const VideoFrame::Format::Plane& p : aFormat.Planes()) {
+ const gfx::IntSize sample = aFormat.SampleSize(p);
+ if (aRect.Width() % sample.Width() != 0) {
+ return Err(nsCString("Mismatch between format and given rect's width"));
+ }
+
+ if (aRect.Height() % sample.Height() != 0) {
+ return Err(nsCString("Mismatch between format and given rect's height"));
+ }
+ }
+ return Ok();
+}
+
+// https://w3c.github.io/webcodecs/#videoframe-parse-videoframecopytooptions
+static Result<CombinedBufferLayout, nsCString> ParseVideoFrameCopyToOptions(
+ const VideoFrameCopyToOptions& aOptions, const gfx::IntRect& aVisibleRect,
+ const gfx::IntSize& aCodedSize, const VideoFrame::Format& aFormat) {
+ Maybe<gfx::IntRect> overrideRect;
+ if (aOptions.mRect.WasPassed()) {
+ // TODO: We handle some edge cases that spec misses:
+ // https://github.com/w3c/webcodecs/issues/513
+ // This comment should be removed once the issue is resolved.
+ overrideRect.emplace();
+ MOZ_TRY_VAR(overrideRect.ref(),
+ ToIntRect(aOptions.mRect.Value()).mapErr([](nsCString error) {
+ error.Insert("rect's ", 0);
+ return error;
+ }));
+
+ MOZ_TRY(VerifyRectSizeAlignment(aFormat, overrideRect.ref()));
+ }
+
+ gfx::IntRect parsedRect;
+ MOZ_TRY_VAR(parsedRect, ParseVisibleRect(aVisibleRect, overrideRect,
+ aCodedSize, aFormat));
+
+ const Sequence<PlaneLayout>* optLayout = OptionalToPointer(aOptions.mLayout);
+
+ return ComputeLayoutAndAllocationSize(parsedRect, aFormat, optLayout);
+}
+
+static bool IsYUVFormat(const VideoPixelFormat& aFormat) {
+ switch (aFormat) {
+ case VideoPixelFormat::I420:
+ case VideoPixelFormat::I420A:
+ case VideoPixelFormat::I422:
+ case VideoPixelFormat::I444:
+ case VideoPixelFormat::NV12:
+ return true;
+ case VideoPixelFormat::RGBA:
+ case VideoPixelFormat::RGBX:
+ case VideoPixelFormat::BGRA:
+ case VideoPixelFormat::BGRX:
+ return false;
+ case VideoPixelFormat::EndGuard_:
+ MOZ_ASSERT_UNREACHABLE("unsupported format");
+ }
+ return false;
+}
+
+// https://w3c.github.io/webcodecs/#videoframe-pick-color-space
+static VideoColorSpaceInit PickColorSpace(
+ const VideoColorSpaceInit* aInitColorSpace,
+ const VideoPixelFormat& aFormat) {
+ VideoColorSpaceInit colorSpace;
+ if (aInitColorSpace) {
+ colorSpace = *aInitColorSpace;
+ // By spec, we MAY replace null members of aInitColorSpace with guessed
+ // values so we can always use these in CreateYUVImageFromBuffer.
+ if (IsYUVFormat(aFormat) && colorSpace.mMatrix.IsNull()) {
+ colorSpace.mMatrix.SetValue(VideoMatrixCoefficients::Bt709);
+ }
+ return colorSpace;
+ }
+
+ switch (aFormat) {
+ case VideoPixelFormat::I420:
+ case VideoPixelFormat::I420A:
+ case VideoPixelFormat::I422:
+ case VideoPixelFormat::I444:
+ case VideoPixelFormat::NV12:
+ // https://w3c.github.io/webcodecs/#rec709-color-space
+ colorSpace.mFullRange.SetValue(false);
+ colorSpace.mMatrix.SetValue(VideoMatrixCoefficients::Bt709);
+ colorSpace.mPrimaries.SetValue(VideoColorPrimaries::Bt709);
+ colorSpace.mTransfer.SetValue(VideoTransferCharacteristics::Bt709);
+ break;
+ case VideoPixelFormat::RGBA:
+ case VideoPixelFormat::RGBX:
+ case VideoPixelFormat::BGRA:
+ case VideoPixelFormat::BGRX:
+ // https://w3c.github.io/webcodecs/#srgb-color-space
+ colorSpace.mFullRange.SetValue(true);
+ colorSpace.mMatrix.SetValue(VideoMatrixCoefficients::Rgb);
+ colorSpace.mPrimaries.SetValue(VideoColorPrimaries::Bt709);
+ colorSpace.mTransfer.SetValue(VideoTransferCharacteristics::Iec61966_2_1);
+ break;
+ case VideoPixelFormat::EndGuard_:
+ MOZ_ASSERT_UNREACHABLE("unsupported format");
+ }
+
+ return colorSpace;
+}
+
+// https://w3c.github.io/webcodecs/#validate-videoframeinit
+static Result<std::pair<Maybe<gfx::IntRect>, Maybe<gfx::IntSize>>, nsCString>
+ValidateVideoFrameInit(const VideoFrameInit& aInit,
+ const Maybe<VideoFrame::Format>& aFormat,
+ const gfx::IntSize& aCodedSize) {
+ if (aCodedSize.Width() <= 0 || aCodedSize.Height() <= 0) {
+ return Err(nsCString("codedWidth and codedHeight must be positive"));
+ }
+
+ Maybe<gfx::IntRect> visibleRect;
+ if (aInit.mVisibleRect.WasPassed()) {
+ visibleRect.emplace();
+ MOZ_TRY_VAR(
+ visibleRect.ref(),
+ ToIntRect(aInit.mVisibleRect.Value()).mapErr([](nsCString error) {
+ error.Insert("visibleRect's ", 0);
+ return error;
+ }));
+ MOZ_TRY(ValidateVisibility(visibleRect.ref(), aCodedSize));
+
+ MOZ_TRY(VerifyRectOffsetAlignment(aFormat, visibleRect.ref()));
+ }
+
+ Maybe<gfx::IntSize> displaySize;
+ MOZ_TRY_VAR(displaySize, MaybeGetDisplaySize(aInit));
+
+ return std::make_pair(visibleRect, displaySize);
+}
+
+/*
+ * The followings are helpers to create a VideoFrame from a given buffer
+ */
+
+static Result<RefPtr<gfx::DataSourceSurface>, nsCString> AllocateBGRASurface(
+ gfx::DataSourceSurface* aSurface) {
+ MOZ_ASSERT(aSurface);
+
+ // Memory allocation relies on CreateDataSourceSurfaceWithStride so we still
+ // need to do this even if the format is SurfaceFormat::BGR{A, X}.
+
+ gfx::DataSourceSurface::ScopedMap surfaceMap(aSurface,
+ gfx::DataSourceSurface::READ);
+ if (!surfaceMap.IsMapped()) {
+ return Err(nsCString("The source surface is not readable"));
+ }
+
+ RefPtr<gfx::DataSourceSurface> bgraSurface =
+ gfx::Factory::CreateDataSourceSurfaceWithStride(
+ aSurface->GetSize(), gfx::SurfaceFormat::B8G8R8A8,
+ surfaceMap.GetStride());
+ if (!bgraSurface) {
+ return Err(nsCString("Failed to allocate a BGRA surface"));
+ }
+
+ gfx::DataSourceSurface::ScopedMap bgraMap(bgraSurface,
+ gfx::DataSourceSurface::WRITE);
+ if (!bgraMap.IsMapped()) {
+ return Err(nsCString("The allocated BGRA surface is not writable"));
+ }
+
+ gfx::SwizzleData(surfaceMap.GetData(), surfaceMap.GetStride(),
+ aSurface->GetFormat(), bgraMap.GetData(),
+ bgraMap.GetStride(), bgraSurface->GetFormat(),
+ bgraSurface->GetSize());
+
+ return bgraSurface;
+}
+
+static Result<RefPtr<layers::Image>, nsCString> CreateImageFromRawData(
+ const gfx::IntSize& aSize, int32_t aStride, gfx::SurfaceFormat aFormat,
+ const Span<uint8_t>& aBuffer) {
+ MOZ_ASSERT(!aSize.IsEmpty());
+
+ // Wrap the source buffer into a DataSourceSurface.
+ RefPtr<gfx::DataSourceSurface> surface =
+ gfx::Factory::CreateWrappingDataSourceSurface(aBuffer.data(), aStride,
+ aSize, aFormat);
+ if (!surface) {
+ return Err(nsCString("Failed to wrap the raw data into a surface"));
+ }
+
+ // Gecko favors BGRA so we convert surface into BGRA format first.
+ RefPtr<gfx::DataSourceSurface> bgraSurface;
+ MOZ_TRY_VAR(bgraSurface, AllocateBGRASurface(surface));
+ MOZ_ASSERT(bgraSurface);
+
+ return RefPtr<layers::Image>(
+ new layers::SourceSurfaceImage(bgraSurface.get()));
+}
+
+static Result<RefPtr<layers::Image>, nsCString> CreateRGBAImageFromBuffer(
+ const VideoFrame::Format& aFormat, const gfx::IntSize& aSize,
+ const Span<uint8_t>& aBuffer) {
+ const gfx::SurfaceFormat format = aFormat.ToSurfaceFormat();
+ MOZ_ASSERT(format == gfx::SurfaceFormat::R8G8B8A8 ||
+ format == gfx::SurfaceFormat::R8G8B8X8 ||
+ format == gfx::SurfaceFormat::B8G8R8A8 ||
+ format == gfx::SurfaceFormat::B8G8R8X8);
+ // TODO: Use aFormat.SampleBytes() instead?
+ CheckedInt<int32_t> stride(BytesPerPixel(format));
+ stride *= aSize.Width();
+ if (!stride.isValid()) {
+ return Err(nsCString("Image size exceeds implementation's limit"));
+ }
+ return CreateImageFromRawData(aSize, stride.value(), format, aBuffer);
+}
+
+static Result<RefPtr<layers::Image>, nsCString> CreateYUVImageFromBuffer(
+ const VideoFrame::Format& aFormat, const VideoColorSpaceInit& aColorSpace,
+ const gfx::IntSize& aSize, const Span<uint8_t>& aBuffer) {
+ if (aFormat.PixelFormat() == VideoPixelFormat::I420 ||
+ aFormat.PixelFormat() == VideoPixelFormat::I420A) {
+ UniquePtr<I420BufferReader> reader;
+ if (aFormat.PixelFormat() == VideoPixelFormat::I420) {
+ reader.reset(
+ new I420BufferReader(aBuffer, aSize.Width(), aSize.Height()));
+ } else {
+ reader.reset(
+ new I420ABufferReader(aBuffer, aSize.Width(), aSize.Height()));
+ }
+
+ layers::PlanarYCbCrData data;
+ data.mPictureRect = gfx::IntRect(0, 0, reader->mWidth, reader->mHeight);
+
+ // Y plane.
+ data.mYChannel = const_cast<uint8_t*>(reader->DataY());
+ data.mYStride = reader->mStrideY;
+ data.mYSkip = 0;
+ // Cb plane.
+ data.mCbChannel = const_cast<uint8_t*>(reader->DataU());
+ data.mCbSkip = 0;
+ // Cr plane.
+ data.mCrChannel = const_cast<uint8_t*>(reader->DataV());
+ data.mCbSkip = 0;
+ // A plane.
+ if (aFormat.PixelFormat() == VideoPixelFormat::I420A) {
+ data.mAlpha.emplace();
+ data.mAlpha->mChannel =
+ const_cast<uint8_t*>(reader->AsI420ABufferReader()->DataA());
+ data.mAlpha->mSize = data.mPictureRect.Size();
+ // No values for mDepth and mPremultiplied.
+ }
+
+ // CbCr plane vector.
+ MOZ_RELEASE_ASSERT(reader->mStrideU == reader->mStrideV);
+ data.mCbCrStride = reader->mStrideU;
+ data.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT;
+ // Color settings.
+ if (!aColorSpace.mFullRange.IsNull()) {
+ data.mColorRange = ToColorRange(aColorSpace.mFullRange.Value());
+ }
+ MOZ_RELEASE_ASSERT(!aColorSpace.mMatrix.IsNull());
+ data.mYUVColorSpace = ToColorSpace(aColorSpace.mMatrix.Value());
+ if (!aColorSpace.mTransfer.IsNull()) {
+ data.mTransferFunction =
+ ToTransferFunction(aColorSpace.mTransfer.Value());
+ }
+ if (!aColorSpace.mPrimaries.IsNull()) {
+ data.mColorPrimaries = ToPrimaries(aColorSpace.mPrimaries.Value());
+ }
+
+ RefPtr<layers::PlanarYCbCrImage> image =
+ new layers::RecyclingPlanarYCbCrImage(new layers::BufferRecycleBin());
+ if (NS_FAILED(image->CopyData(data))) {
+ return Err(nsPrintfCString(
+ "Failed to create I420%s image",
+ (aFormat.PixelFormat() == VideoPixelFormat::I420A ? "A" : "")));
+ }
+ // Manually cast type to make Result work.
+ return RefPtr<layers::Image>(image.forget());
+ }
+
+ if (aFormat.PixelFormat() == VideoPixelFormat::NV12) {
+ NV12BufferReader reader(aBuffer, aSize.Width(), aSize.Height());
+
+ layers::PlanarYCbCrData data;
+ data.mPictureRect = gfx::IntRect(0, 0, reader.mWidth, reader.mHeight);
+
+ // Y plane.
+ data.mYChannel = const_cast<uint8_t*>(reader.DataY());
+ data.mYStride = reader.mStrideY;
+ data.mYSkip = 0;
+ // Cb plane.
+ data.mCbChannel = const_cast<uint8_t*>(reader.DataUV());
+ data.mCbSkip = 1;
+ // Cr plane.
+ data.mCrChannel = data.mCbChannel + 1;
+ data.mCrSkip = 1;
+ // CbCr plane vector.
+ data.mCbCrStride = reader.mStrideUV;
+ data.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT;
+ // Color settings.
+ if (!aColorSpace.mFullRange.IsNull()) {
+ data.mColorRange = ToColorRange(aColorSpace.mFullRange.Value());
+ }
+ MOZ_RELEASE_ASSERT(!aColorSpace.mMatrix.IsNull());
+ data.mYUVColorSpace = ToColorSpace(aColorSpace.mMatrix.Value());
+ if (!aColorSpace.mTransfer.IsNull()) {
+ data.mTransferFunction =
+ ToTransferFunction(aColorSpace.mTransfer.Value());
+ }
+ if (!aColorSpace.mPrimaries.IsNull()) {
+ data.mColorPrimaries = ToPrimaries(aColorSpace.mPrimaries.Value());
+ }
+
+ RefPtr<layers::NVImage> image = new layers::NVImage();
+ if (!image->SetData(data)) {
+ return Err(nsCString("Failed to create NV12 image"));
+ }
+ // Manually cast type to make Result work.
+ return RefPtr<layers::Image>(image.forget());
+ }
+
+ return Err(nsCString("Unsupported image format"));
+}
+
+static Result<RefPtr<layers::Image>, nsCString> CreateImageFromBuffer(
+ const VideoFrame::Format& aFormat, const VideoColorSpaceInit& aColorSpace,
+ const gfx::IntSize& aSize, const Span<uint8_t>& aBuffer) {
+ switch (aFormat.PixelFormat()) {
+ case VideoPixelFormat::I420:
+ case VideoPixelFormat::I420A:
+ case VideoPixelFormat::NV12:
+ return CreateYUVImageFromBuffer(aFormat, aColorSpace, aSize, aBuffer);
+ case VideoPixelFormat::I422:
+ case VideoPixelFormat::I444:
+ // Not yet support for now.
+ break;
+ case VideoPixelFormat::RGBA:
+ case VideoPixelFormat::RGBX:
+ case VideoPixelFormat::BGRA:
+ case VideoPixelFormat::BGRX:
+ return CreateRGBAImageFromBuffer(aFormat, aSize, aBuffer);
+ case VideoPixelFormat::EndGuard_:
+ MOZ_ASSERT_UNREACHABLE("unsupported format");
+ }
+ return Err(nsCString("Invalid image format"));
+}
+
+// https://w3c.github.io/webcodecs/#dom-videoframe-videoframe-data-init
+template <class T>
+static Result<RefPtr<VideoFrame>, nsCString> CreateVideoFrameFromBuffer(
+ nsIGlobalObject* aGlobal, const T& aBuffer,
+ const VideoFrameBufferInit& aInit) {
+ if (aInit.mColorSpace.WasPassed() &&
+ !aInit.mColorSpace.Value().mTransfer.IsNull() &&
+ aInit.mColorSpace.Value().mTransfer.Value() ==
+ VideoTransferCharacteristics::Linear) {
+ return Err(nsCString("linear RGB is not supported"));
+ }
+
+ std::tuple<gfx::IntSize, Maybe<gfx::IntRect>, Maybe<gfx::IntSize>> init;
+ MOZ_TRY_VAR(init, ValidateVideoFrameBufferInit(aInit));
+ gfx::IntSize codedSize = std::get<0>(init);
+ Maybe<gfx::IntRect> visibleRect = std::get<1>(init);
+ Maybe<gfx::IntSize> displaySize = std::get<2>(init);
+
+ VideoFrame::Format format(aInit.mFormat);
+ // TODO: Spec doesn't ask for this in ctor but Pixel Format does. See
+ // https://github.com/w3c/webcodecs/issues/512
+ // This comment should be removed once the issue is resolved.
+ if (!format.IsValidSize(codedSize)) {
+ return Err(nsCString("coded width and/or height is invalid"));
+ }
+
+ gfx::IntRect parsedRect;
+ MOZ_TRY_VAR(parsedRect, ParseVisibleRect(gfx::IntRect({0, 0}, codedSize),
+ visibleRect, codedSize, format));
+
+ const Sequence<PlaneLayout>* optLayout = OptionalToPointer(aInit.mLayout);
+
+ CombinedBufferLayout combinedLayout;
+ MOZ_TRY_VAR(combinedLayout,
+ ComputeLayoutAndAllocationSize(parsedRect, format, optLayout));
+
+ Maybe<uint64_t> duration = OptionalToMaybe(aInit.mDuration);
+
+ VideoColorSpaceInit colorSpace =
+ PickColorSpace(OptionalToPointer(aInit.mColorSpace), aInit.mFormat);
+
+ RefPtr<layers::Image> data;
+ MOZ_TRY_VAR(
+ data,
+ aBuffer.ProcessFixedData([&](const Span<uint8_t>& aData)
+ -> Result<RefPtr<layers::Image>, nsCString> {
+ if (aData.Length() <
+ static_cast<size_t>(combinedLayout.mAllocationSize)) {
+ return Err(nsCString("data is too small"));
+ }
+
+ // TODO: If codedSize is (3, 3) and visibleRect is (0, 0, 1, 1) but the
+ // data is 2 x 2 RGBA buffer (2 x 2 x 4 bytes), it pass the above check.
+ // In this case, we can crop it to a 1 x 1-codedSize image (Bug
+ // 1782128).
+ if (aData.Length() < format.SampleCount(codedSize)) { // 1 byte/sample
+ return Err(nsCString("data is too small"));
+ }
+
+ return CreateImageFromBuffer(format, colorSpace, codedSize, aData);
+ }));
+
+ MOZ_ASSERT(data);
+ MOZ_ASSERT(data->GetSize() == codedSize);
+
+ // By spec, we should set visible* here. But if we don't change the image,
+ // visible* is same as parsedRect here. The display{Width, Height} is
+ // visible{Width, Height} if it's not set.
+
+ // TODO: Spec should assign aInit.mFormat to inner format value:
+ // https://github.com/w3c/webcodecs/issues/509.
+ // This comment should be removed once the issue is resolved.
+ return MakeRefPtr<VideoFrame>(aGlobal, data, Some(aInit.mFormat), codedSize,
+ parsedRect,
+ displaySize ? *displaySize : parsedRect.Size(),
+ duration, aInit.mTimestamp, colorSpace);
+}
+
+template <class T>
+static already_AddRefed<VideoFrame> CreateVideoFrameFromBuffer(
+ const GlobalObject& aGlobal, const T& aBuffer,
+ const VideoFrameBufferInit& aInit, ErrorResult& aRv) {
+ nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+ if (!global) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return nullptr;
+ }
+
+ auto r = CreateVideoFrameFromBuffer(global, aBuffer, aInit);
+ if (r.isErr()) {
+ aRv.ThrowTypeError(r.unwrapErr());
+ return nullptr;
+ }
+ return r.unwrap().forget();
+}
+
+// https://w3c.github.io/webcodecs/#videoframe-initialize-visible-rect-and-display-size
+static void InitializeVisibleRectAndDisplaySize(
+ Maybe<gfx::IntRect>& aVisibleRect, Maybe<gfx::IntSize>& aDisplaySize,
+ gfx::IntRect aDefaultVisibleRect, gfx::IntSize aDefaultDisplaySize) {
+ if (!aVisibleRect) {
+ aVisibleRect.emplace(aDefaultVisibleRect);
+ }
+ if (!aDisplaySize) {
+ double wScale = static_cast<double>(aDefaultDisplaySize.Width()) /
+ aDefaultVisibleRect.Width();
+ double hScale = static_cast<double>(aDefaultDisplaySize.Height()) /
+ aDefaultVisibleRect.Height();
+ double w = wScale * aVisibleRect->Width();
+ double h = hScale * aVisibleRect->Height();
+ aDisplaySize.emplace(gfx::IntSize(static_cast<uint32_t>(round(w)),
+ static_cast<uint32_t>(round(h))));
+ }
+}
+
+// https://w3c.github.io/webcodecs/#videoframe-initialize-frame-with-resource-and-size
+static Result<already_AddRefed<VideoFrame>, nsCString>
+InitializeFrameWithResourceAndSize(
+ nsIGlobalObject* aGlobal, const VideoFrameInit& aInit,
+ already_AddRefed<layers::SourceSurfaceImage> aImage) {
+ MOZ_ASSERT(aInit.mTimestamp.WasPassed());
+
+ RefPtr<layers::SourceSurfaceImage> image(aImage);
+ MOZ_ASSERT(image);
+
+ RefPtr<gfx::SourceSurface> surface = image->GetAsSourceSurface();
+ Maybe<VideoFrame::Format> format =
+ SurfaceFormatToVideoPixelFormat(surface->GetFormat())
+ .map([](const VideoPixelFormat& aFormat) {
+ return VideoFrame::Format(aFormat);
+ });
+
+ std::pair<Maybe<gfx::IntRect>, Maybe<gfx::IntSize>> init;
+ MOZ_TRY_VAR(init, ValidateVideoFrameInit(aInit, format, image->GetSize()));
+ Maybe<gfx::IntRect> visibleRect = init.first;
+ Maybe<gfx::IntSize> displaySize = init.second;
+
+ if (format && aInit.mAlpha == AlphaOption::Discard) {
+ format->MakeOpaque();
+ // Keep the alpha data in image for now until it's being rendered.
+ // TODO: The alpha will still be rendered if the format is unrecognized
+ // since no additional flag keeping this request. Should spec address what
+ // to do in this case?
+ }
+
+ InitializeVisibleRectAndDisplaySize(visibleRect, displaySize,
+ gfx::IntRect({0, 0}, image->GetSize()),
+ image->GetSize());
+
+ Maybe<uint64_t> duration = OptionalToMaybe(aInit.mDuration);
+
+ VideoColorSpaceInit colorSpace{};
+ if (IsYUVFormat(
+ SurfaceFormatToVideoPixelFormat(surface->GetFormat()).ref())) {
+ colorSpace = FallbackColorSpaceForVideoContent();
+ } else {
+ colorSpace = FallbackColorSpaceForWebContent();
+ }
+ return MakeAndAddRef<VideoFrame>(
+ aGlobal, image, format ? Some(format->PixelFormat()) : Nothing(),
+ image->GetSize(), visibleRect.value(), displaySize.value(), duration,
+ aInit.mTimestamp.Value(), colorSpace);
+}
+
+// https://w3c.github.io/webcodecs/#videoframe-initialize-frame-from-other-frame
+static Result<already_AddRefed<VideoFrame>, nsCString>
+InitializeFrameFromOtherFrame(nsIGlobalObject* aGlobal, VideoFrameData&& aData,
+ const VideoFrameInit& aInit) {
+ MOZ_ASSERT(aGlobal);
+ MOZ_ASSERT(aData.mImage);
+
+ Maybe<VideoFrame::Format> format =
+ aData.mFormat ? Some(VideoFrame::Format(*aData.mFormat)) : Nothing();
+ if (format && aInit.mAlpha == AlphaOption::Discard) {
+ format->MakeOpaque();
+ // Keep the alpha data in image for now until it's being rendered.
+ // TODO: The alpha will still be rendered if the format is unrecognized
+ // since no additional flag keeping this request. Should spec address what
+ // to do in this case?
+ }
+
+ std::pair<Maybe<gfx::IntRect>, Maybe<gfx::IntSize>> init;
+ MOZ_TRY_VAR(init,
+ ValidateVideoFrameInit(aInit, format, aData.mImage->GetSize()));
+ Maybe<gfx::IntRect> visibleRect = init.first;
+ Maybe<gfx::IntSize> displaySize = init.second;
+
+ InitializeVisibleRectAndDisplaySize(visibleRect, displaySize,
+ aData.mVisibleRect, aData.mDisplaySize);
+
+ Maybe<uint64_t> duration = OptionalToMaybe(aInit.mDuration);
+
+ int64_t timestamp = aInit.mTimestamp.WasPassed() ? aInit.mTimestamp.Value()
+ : aData.mTimestamp;
+
+ return MakeAndAddRef<VideoFrame>(
+ aGlobal, aData.mImage, format ? Some(format->PixelFormat()) : Nothing(),
+ aData.mImage->GetSize(), *visibleRect, *displaySize, duration, timestamp,
+ aData.mColorSpace);
+}
+
+/*
+ * Helper classes carrying VideoFrame data
+ */
+
+VideoFrameData::VideoFrameData(layers::Image* aImage,
+ const Maybe<VideoPixelFormat>& aFormat,
+ gfx::IntRect aVisibleRect,
+ gfx::IntSize aDisplaySize,
+ Maybe<uint64_t> aDuration, int64_t aTimestamp,
+ const VideoColorSpaceInit& aColorSpace)
+ : mImage(aImage),
+ mFormat(aFormat),
+ mVisibleRect(aVisibleRect),
+ mDisplaySize(aDisplaySize),
+ mDuration(aDuration),
+ mTimestamp(aTimestamp),
+ mColorSpace(aColorSpace) {}
+
+VideoFrameSerializedData::VideoFrameSerializedData(const VideoFrameData& aData,
+ gfx::IntSize aCodedSize)
+ : VideoFrameData(aData), mCodedSize(aCodedSize) {}
+
+/*
+ * W3C Webcodecs VideoFrame implementation
+ */
+
+VideoFrame::VideoFrame(nsIGlobalObject* aParent,
+ const RefPtr<layers::Image>& aImage,
+ const Maybe<VideoPixelFormat>& aFormat,
+ gfx::IntSize aCodedSize, gfx::IntRect aVisibleRect,
+ gfx::IntSize aDisplaySize,
+ const Maybe<uint64_t>& aDuration, int64_t aTimestamp,
+ const VideoColorSpaceInit& aColorSpace)
+ : mParent(aParent),
+ mCodedSize(aCodedSize),
+ mVisibleRect(aVisibleRect),
+ mDisplaySize(aDisplaySize),
+ mDuration(aDuration),
+ mTimestamp(aTimestamp),
+ mColorSpace(aColorSpace) {
+ MOZ_ASSERT(mParent);
+ LOG("VideoFrame %p ctor", this);
+ mResource.emplace(
+ Resource(aImage, aFormat.map([](const VideoPixelFormat& aPixelFormat) {
+ return VideoFrame::Format(aPixelFormat);
+ })));
+ if (!mResource->mFormat) {
+ LOGW("Create a VideoFrame with an unrecognized image format");
+ }
+ StartAutoClose();
+}
+
+VideoFrame::VideoFrame(nsIGlobalObject* aParent,
+ const VideoFrameSerializedData& aData)
+ : mParent(aParent),
+ mCodedSize(aData.mCodedSize),
+ mVisibleRect(aData.mVisibleRect),
+ mDisplaySize(aData.mDisplaySize),
+ mDuration(aData.mDuration),
+ mTimestamp(aData.mTimestamp),
+ mColorSpace(aData.mColorSpace) {
+ MOZ_ASSERT(mParent);
+ LOG("VideoFrame %p ctor", this);
+ mResource.emplace(Resource(
+ aData.mImage, aData.mFormat.map([](const VideoPixelFormat& aPixelFormat) {
+ return VideoFrame::Format(aPixelFormat);
+ })));
+ if (!mResource->mFormat) {
+ LOGW("Create a VideoFrame with an unrecognized image format");
+ }
+ StartAutoClose();
+}
+
+VideoFrame::VideoFrame(const VideoFrame& aOther)
+ : mParent(aOther.mParent),
+ mResource(aOther.mResource),
+ mCodedSize(aOther.mCodedSize),
+ mVisibleRect(aOther.mVisibleRect),
+ mDisplaySize(aOther.mDisplaySize),
+ mDuration(aOther.mDuration),
+ mTimestamp(aOther.mTimestamp),
+ mColorSpace(aOther.mColorSpace) {
+ MOZ_ASSERT(mParent);
+ LOG("VideoFrame %p ctor", this);
+ StartAutoClose();
+}
+
+VideoFrame::~VideoFrame() {
+ MOZ_ASSERT(IsClosed());
+ LOG("VideoFrame %p dtor", this);
+}
+
+nsIGlobalObject* VideoFrame::GetParentObject() const {
+ AssertIsOnOwningThread();
+
+ return mParent.get();
+}
+
+JSObject* VideoFrame::WrapObject(JSContext* aCx,
+ JS::Handle<JSObject*> aGivenProto) {
+ AssertIsOnOwningThread();
+
+ return VideoFrame_Binding::Wrap(aCx, this, aGivenProto);
+}
+
+// The following constructors are defined in
+// https://w3c.github.io/webcodecs/#dom-videoframe-videoframe
+
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::Constructor(
+ const GlobalObject& aGlobal, HTMLImageElement& aImageElement,
+ const VideoFrameInit& aInit, ErrorResult& aRv) {
+ nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+ if (!global) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return nullptr;
+ }
+
+ // Check the usability.
+ if (aImageElement.State().HasState(ElementState::BROKEN)) {
+ aRv.ThrowInvalidStateError("The image's state is broken");
+ return nullptr;
+ }
+ if (!aImageElement.Complete()) {
+ aRv.ThrowInvalidStateError("The image is not completely loaded yet");
+ return nullptr;
+ }
+ if (aImageElement.NaturalWidth() == 0) {
+ aRv.ThrowInvalidStateError("The image has a width of 0");
+ return nullptr;
+ }
+ if (aImageElement.NaturalHeight() == 0) {
+ aRv.ThrowInvalidStateError("The image has a height of 0");
+ return nullptr;
+ }
+
+ // If the origin of HTMLImageElement's image data is not same origin with the
+ // entry settings object's origin, then throw a SecurityError DOMException.
+ SurfaceFromElementResult res = nsLayoutUtils::SurfaceFromElement(
+ &aImageElement, nsLayoutUtils::SFE_WANT_FIRST_FRAME_IF_IMAGE);
+ if (res.mIsWriteOnly) {
+ // Being write-only implies its image is cross-origin w/out CORS headers.
+ aRv.ThrowSecurityError("The image is not same-origin");
+ return nullptr;
+ }
+
+ RefPtr<gfx::SourceSurface> surface = res.GetSourceSurface();
+ if (NS_WARN_IF(!surface)) {
+ aRv.ThrowInvalidStateError("The image's surface acquisition failed");
+ return nullptr;
+ }
+
+ if (!aInit.mTimestamp.WasPassed()) {
+ aRv.ThrowTypeError("Missing timestamp");
+ return nullptr;
+ }
+
+ RefPtr<layers::SourceSurfaceImage> image =
+ new layers::SourceSurfaceImage(surface.get());
+ auto r = InitializeFrameWithResourceAndSize(global, aInit, image.forget());
+ if (r.isErr()) {
+ aRv.ThrowTypeError(r.unwrapErr());
+ return nullptr;
+ }
+ return r.unwrap();
+}
+
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::Constructor(
+ const GlobalObject& aGlobal, SVGImageElement& aSVGImageElement,
+ const VideoFrameInit& aInit, ErrorResult& aRv) {
+ nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+ if (!global) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return nullptr;
+ }
+
+ // Check the usability.
+ if (aSVGImageElement.State().HasState(ElementState::BROKEN)) {
+ aRv.ThrowInvalidStateError("The SVG's state is broken");
+ return nullptr;
+ }
+
+ // If the origin of SVGImageElement's image data is not same origin with the
+ // entry settings object's origin, then throw a SecurityError DOMException.
+ SurfaceFromElementResult res = nsLayoutUtils::SurfaceFromElement(
+ &aSVGImageElement, nsLayoutUtils::SFE_WANT_FIRST_FRAME_IF_IMAGE);
+ if (res.mIsWriteOnly) {
+ // Being write-only implies its image is cross-origin w/out CORS headers.
+ aRv.ThrowSecurityError("The SVG is not same-origin");
+ return nullptr;
+ }
+
+ RefPtr<gfx::SourceSurface> surface = res.GetSourceSurface();
+ if (NS_WARN_IF(!surface)) {
+ aRv.ThrowInvalidStateError("The SVG's surface acquisition failed");
+ return nullptr;
+ }
+
+ if (!aInit.mTimestamp.WasPassed()) {
+ aRv.ThrowTypeError("Missing timestamp");
+ return nullptr;
+ }
+
+ RefPtr<layers::SourceSurfaceImage> image =
+ new layers::SourceSurfaceImage(surface.get());
+ auto r = InitializeFrameWithResourceAndSize(global, aInit, image.forget());
+ if (r.isErr()) {
+ aRv.ThrowTypeError(r.unwrapErr());
+ return nullptr;
+ }
+ return r.unwrap();
+}
+
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::Constructor(
+ const GlobalObject& aGlobal, HTMLCanvasElement& aCanvasElement,
+ const VideoFrameInit& aInit, ErrorResult& aRv) {
+ nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+ if (!global) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return nullptr;
+ }
+
+ // Check the usability.
+ if (aCanvasElement.Width() == 0) {
+ aRv.ThrowInvalidStateError("The canvas has a width of 0");
+ return nullptr;
+ }
+
+ if (aCanvasElement.Height() == 0) {
+ aRv.ThrowInvalidStateError("The canvas has a height of 0");
+ return nullptr;
+ }
+
+ // If the origin of HTMLCanvasElement's image data is not same origin with the
+ // entry settings object's origin, then throw a SecurityError DOMException.
+ SurfaceFromElementResult res = nsLayoutUtils::SurfaceFromElement(
+ &aCanvasElement, nsLayoutUtils::SFE_WANT_FIRST_FRAME_IF_IMAGE);
+ if (res.mIsWriteOnly) {
+ // Being write-only implies its image is cross-origin w/out CORS headers.
+ aRv.ThrowSecurityError("The canvas is not same-origin");
+ return nullptr;
+ }
+
+ RefPtr<gfx::SourceSurface> surface = res.GetSourceSurface();
+ if (NS_WARN_IF(!surface)) {
+ aRv.ThrowInvalidStateError("The canvas' surface acquisition failed");
+ return nullptr;
+ }
+
+ if (!aInit.mTimestamp.WasPassed()) {
+ aRv.ThrowTypeError("Missing timestamp");
+ return nullptr;
+ }
+
+ RefPtr<layers::SourceSurfaceImage> image =
+ new layers::SourceSurfaceImage(surface.get());
+ auto r = InitializeFrameWithResourceAndSize(global, aInit, image.forget());
+ if (r.isErr()) {
+ aRv.ThrowTypeError(r.unwrapErr());
+ return nullptr;
+ }
+ return r.unwrap();
+}
+
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::Constructor(
+ const GlobalObject& aGlobal, HTMLVideoElement& aVideoElement,
+ const VideoFrameInit& aInit, ErrorResult& aRv) {
+ nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+ if (!global) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return nullptr;
+ }
+
+ aVideoElement.LogVisibility(
+ mozilla::dom::HTMLVideoElement::CallerAPI::CREATE_VIDEOFRAME);
+
+ // Check the usability.
+ if (aVideoElement.NetworkState() == HTMLMediaElement_Binding::NETWORK_EMPTY) {
+ aRv.ThrowInvalidStateError("The video has not been initialized yet");
+ return nullptr;
+ }
+ if (aVideoElement.ReadyState() <= HTMLMediaElement_Binding::HAVE_METADATA) {
+ aRv.ThrowInvalidStateError("The video is not ready yet");
+ return nullptr;
+ }
+ RefPtr<layers::Image> image = aVideoElement.GetCurrentImage();
+ if (!image) {
+ aRv.ThrowInvalidStateError("The video doesn't have any image yet");
+ return nullptr;
+ }
+
+ // If the origin of HTMLVideoElement's image data is not same origin with the
+ // entry settings object's origin, then throw a SecurityError DOMException.
+ if (!IsSameOrigin(global.get(), aVideoElement)) {
+ aRv.ThrowSecurityError("The video is not same-origin");
+ return nullptr;
+ }
+
+ const ImageUtils imageUtils(image);
+ Maybe<VideoPixelFormat> format =
+ ImageBitmapFormatToVideoPixelFormat(imageUtils.GetFormat());
+
+ // TODO: Retrive/infer the duration, and colorspace.
+ auto r = InitializeFrameFromOtherFrame(
+ global.get(),
+ VideoFrameData(image.get(), format, image->GetPictureRect(),
+ image->GetSize(), Nothing(),
+ static_cast<int64_t>(aVideoElement.CurrentTime()), {}),
+ aInit);
+ if (r.isErr()) {
+ aRv.ThrowTypeError(r.unwrapErr());
+ return nullptr;
+ }
+ return r.unwrap();
+}
+
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::Constructor(
+ const GlobalObject& aGlobal, OffscreenCanvas& aOffscreenCanvas,
+ const VideoFrameInit& aInit, ErrorResult& aRv) {
+ nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+ if (!global) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return nullptr;
+ }
+
+ // Check the usability.
+ if (aOffscreenCanvas.Width() == 0) {
+ aRv.ThrowInvalidStateError("The canvas has a width of 0");
+ return nullptr;
+ }
+ if (aOffscreenCanvas.Height() == 0) {
+ aRv.ThrowInvalidStateError("The canvas has a height of 0");
+ return nullptr;
+ }
+
+ // If the origin of the OffscreenCanvas's image data is not same origin with
+ // the entry settings object's origin, then throw a SecurityError
+ // DOMException.
+ SurfaceFromElementResult res = nsLayoutUtils::SurfaceFromOffscreenCanvas(
+ &aOffscreenCanvas, nsLayoutUtils::SFE_WANT_FIRST_FRAME_IF_IMAGE);
+ if (res.mIsWriteOnly) {
+ // Being write-only implies its image is cross-origin w/out CORS headers.
+ aRv.ThrowSecurityError("The canvas is not same-origin");
+ return nullptr;
+ }
+
+ RefPtr<gfx::SourceSurface> surface = res.GetSourceSurface();
+ if (NS_WARN_IF(!surface)) {
+ aRv.ThrowInvalidStateError("The canvas' surface acquisition failed");
+ return nullptr;
+ }
+
+ if (!aInit.mTimestamp.WasPassed()) {
+ aRv.ThrowTypeError("Missing timestamp");
+ return nullptr;
+ }
+
+ RefPtr<layers::SourceSurfaceImage> image =
+ new layers::SourceSurfaceImage(surface.get());
+ auto r = InitializeFrameWithResourceAndSize(global, aInit, image.forget());
+ if (r.isErr()) {
+ aRv.ThrowTypeError(r.unwrapErr());
+ return nullptr;
+ }
+ return r.unwrap();
+}
+
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::Constructor(
+ const GlobalObject& aGlobal, ImageBitmap& aImageBitmap,
+ const VideoFrameInit& aInit, ErrorResult& aRv) {
+ nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+ if (!global) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return nullptr;
+ }
+
+ // Check the usability.
+ UniquePtr<ImageBitmapCloneData> data = aImageBitmap.ToCloneData();
+ if (!data || !data->mSurface) {
+ aRv.ThrowInvalidStateError(
+ "The ImageBitmap is closed or its surface acquisition failed");
+ return nullptr;
+ }
+
+ // If the origin of the ImageBitmap's image data is not same origin with the
+ // entry settings object's origin, then throw a SecurityError DOMException.
+ if (data->mWriteOnly) {
+ // Being write-only implies its image is cross-origin w/out CORS headers.
+ aRv.ThrowSecurityError("The ImageBitmap is not same-origin");
+ return nullptr;
+ }
+
+ if (!aInit.mTimestamp.WasPassed()) {
+ aRv.ThrowTypeError("Missing timestamp");
+ return nullptr;
+ }
+
+ RefPtr<layers::SourceSurfaceImage> image =
+ new layers::SourceSurfaceImage(data->mSurface.get());
+ // TODO: Take care of data->mAlphaType
+ auto r = InitializeFrameWithResourceAndSize(global, aInit, image.forget());
+ if (r.isErr()) {
+ aRv.ThrowTypeError(r.unwrapErr());
+ return nullptr;
+ }
+ return r.unwrap();
+}
+
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::Constructor(
+ const GlobalObject& aGlobal, VideoFrame& aVideoFrame,
+ const VideoFrameInit& aInit, ErrorResult& aRv) {
+ nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+ if (!global) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return nullptr;
+ }
+
+ // Check the usability.
+ if (!aVideoFrame.mResource) {
+ aRv.ThrowInvalidStateError(
+ "The VideoFrame is closed or no image found there");
+ return nullptr;
+ }
+ MOZ_ASSERT(aVideoFrame.mResource->mImage->GetSize() ==
+ aVideoFrame.mCodedSize);
+ MOZ_ASSERT(!aVideoFrame.mCodedSize.IsEmpty());
+ MOZ_ASSERT(!aVideoFrame.mVisibleRect.IsEmpty());
+ MOZ_ASSERT(!aVideoFrame.mDisplaySize.IsEmpty());
+
+ // If the origin of the VideoFrame is not same origin with the entry settings
+ // object's origin, then throw a SecurityError DOMException.
+ if (!IsSameOrigin(global.get(), aVideoFrame)) {
+ aRv.ThrowSecurityError("The VideoFrame is not same-origin");
+ return nullptr;
+ }
+
+ auto r = InitializeFrameFromOtherFrame(
+ global.get(), aVideoFrame.GetVideoFrameData(), aInit);
+ if (r.isErr()) {
+ aRv.ThrowTypeError(r.unwrapErr());
+ return nullptr;
+ }
+ return r.unwrap();
+}
+
+// The following constructors are defined in
+// https://w3c.github.io/webcodecs/#dom-videoframe-videoframe-data-init
+
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::Constructor(
+ const GlobalObject& aGlobal, const ArrayBufferView& aBufferView,
+ const VideoFrameBufferInit& aInit, ErrorResult& aRv) {
+ return CreateVideoFrameFromBuffer(aGlobal, aBufferView, aInit, aRv);
+}
+
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::Constructor(
+ const GlobalObject& aGlobal, const ArrayBuffer& aBuffer,
+ const VideoFrameBufferInit& aInit, ErrorResult& aRv) {
+ return CreateVideoFrameFromBuffer(aGlobal, aBuffer, aInit, aRv);
+}
+
+// https://w3c.github.io/webcodecs/#dom-videoframe-format
+Nullable<VideoPixelFormat> VideoFrame::GetFormat() const {
+ AssertIsOnOwningThread();
+
+ return mResource ? MaybeToNullable(mResource->TryPixelFormat())
+ : Nullable<VideoPixelFormat>();
+}
+
+// https://w3c.github.io/webcodecs/#dom-videoframe-codedwidth
+uint32_t VideoFrame::CodedWidth() const {
+ AssertIsOnOwningThread();
+
+ return static_cast<uint32_t>(mCodedSize.Width());
+}
+
+// https://w3c.github.io/webcodecs/#dom-videoframe-codedheight
+uint32_t VideoFrame::CodedHeight() const {
+ AssertIsOnOwningThread();
+
+ return static_cast<uint32_t>(mCodedSize.Height());
+}
+
+// https://w3c.github.io/webcodecs/#dom-videoframe-codedrect
+already_AddRefed<DOMRectReadOnly> VideoFrame::GetCodedRect() const {
+ AssertIsOnOwningThread();
+
+ return mResource
+ ? MakeAndAddRef<DOMRectReadOnly>(
+ mParent, 0.0f, 0.0f, static_cast<double>(mCodedSize.Width()),
+ static_cast<double>(mCodedSize.Height()))
+ : nullptr;
+}
+
+// https://w3c.github.io/webcodecs/#dom-videoframe-visiblerect
+already_AddRefed<DOMRectReadOnly> VideoFrame::GetVisibleRect() const {
+ AssertIsOnOwningThread();
+
+ return mResource ? MakeAndAddRef<DOMRectReadOnly>(
+ mParent, static_cast<double>(mVisibleRect.X()),
+ static_cast<double>(mVisibleRect.Y()),
+ static_cast<double>(mVisibleRect.Width()),
+ static_cast<double>(mVisibleRect.Height()))
+ : nullptr;
+}
+
+// https://w3c.github.io/webcodecs/#dom-videoframe-displaywidth
+uint32_t VideoFrame::DisplayWidth() const {
+ AssertIsOnOwningThread();
+
+ return static_cast<uint32_t>(mDisplaySize.Width());
+}
+
+// https://w3c.github.io/webcodecs/#dom-videoframe-displayheight
+uint32_t VideoFrame::DisplayHeight() const {
+ AssertIsOnOwningThread();
+
+ return static_cast<uint32_t>(mDisplaySize.Height());
+}
+
+// https://w3c.github.io/webcodecs/#dom-videoframe-duration
+Nullable<uint64_t> VideoFrame::GetDuration() const {
+ AssertIsOnOwningThread();
+ return MaybeToNullable(mDuration);
+}
+
+// https://w3c.github.io/webcodecs/#dom-videoframe-timestamp
+int64_t VideoFrame::Timestamp() const {
+ AssertIsOnOwningThread();
+
+ return mTimestamp;
+}
+
+// https://w3c.github.io/webcodecs/#dom-videoframe-colorspace
+already_AddRefed<VideoColorSpace> VideoFrame::ColorSpace() const {
+ AssertIsOnOwningThread();
+
+ return MakeAndAddRef<VideoColorSpace>(mParent, mColorSpace);
+}
+
+// https://w3c.github.io/webcodecs/#dom-videoframe-allocationsize
+uint32_t VideoFrame::AllocationSize(const VideoFrameCopyToOptions& aOptions,
+ ErrorResult& aRv) {
+ AssertIsOnOwningThread();
+
+ if (!mResource) {
+ aRv.ThrowInvalidStateError("No media resource in VideoFrame");
+ return 0;
+ }
+
+ if (!mResource->mFormat) {
+ aRv.ThrowAbortError("The VideoFrame image format is not VideoPixelFormat");
+ return 0;
+ }
+
+ auto r = ParseVideoFrameCopyToOptions(aOptions, mVisibleRect, mCodedSize,
+ mResource->mFormat.ref());
+ if (r.isErr()) {
+ // TODO: Should throw layout.
+ aRv.ThrowTypeError(r.unwrapErr());
+ return 0;
+ }
+ CombinedBufferLayout layout = r.unwrap();
+
+ return layout.mAllocationSize;
+}
+
+// https://w3c.github.io/webcodecs/#dom-videoframe-copyto
+already_AddRefed<Promise> VideoFrame::CopyTo(
+ const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination,
+ const VideoFrameCopyToOptions& aOptions, ErrorResult& aRv) {
+ AssertIsOnOwningThread();
+
+ if (!mResource) {
+ aRv.ThrowInvalidStateError("No media resource in VideoFrame");
+ return nullptr;
+ }
+
+ if (!mResource->mFormat) {
+ aRv.ThrowNotSupportedError("VideoFrame's image format is unrecognized");
+ return nullptr;
+ }
+
+ RefPtr<Promise> p = Promise::Create(mParent.get(), aRv);
+ if (NS_WARN_IF(aRv.Failed())) {
+ return p.forget();
+ }
+
+ CombinedBufferLayout layout;
+ auto r1 = ParseVideoFrameCopyToOptions(aOptions, mVisibleRect, mCodedSize,
+ mResource->mFormat.ref());
+ if (r1.isErr()) {
+ // TODO: Should reject with layout.
+ p->MaybeRejectWithTypeError(r1.unwrapErr());
+ return p.forget();
+ }
+ layout = r1.unwrap();
+
+ return ProcessTypedArraysFixed(aDestination, [&](const Span<uint8_t>& aData) {
+ if (aData.size_bytes() < layout.mAllocationSize) {
+ p->MaybeRejectWithTypeError("Destination buffer is too small");
+ return p.forget();
+ }
+
+ Sequence<PlaneLayout> planeLayouts;
+
+ nsTArray<Format::Plane> planes = mResource->mFormat->Planes();
+ MOZ_ASSERT(layout.mComputedLayouts.Length() == planes.Length());
+
+ // TODO: These jobs can be run in a thread pool (bug 1780656) to unblock
+ // the current thread.
+ for (size_t i = 0; i < layout.mComputedLayouts.Length(); ++i) {
+ ComputedPlaneLayout& l = layout.mComputedLayouts[i];
+ uint32_t destinationOffset = l.mDestinationOffset;
+
+ PlaneLayout* pl = planeLayouts.AppendElement(fallible);
+ if (!pl) {
+ p->MaybeRejectWithTypeError("Out of memory");
+ return p.forget();
+ }
+ pl->mOffset = l.mDestinationOffset;
+ pl->mStride = l.mDestinationStride;
+
+ // Copy pixels of `size` starting from `origin` on planes[i] to
+ // `aDestination`.
+ gfx::IntPoint origin(
+ l.mSourceLeftBytes / mResource->mFormat->SampleBytes(planes[i]),
+ l.mSourceTop);
+ gfx::IntSize size(
+ l.mSourceWidthBytes / mResource->mFormat->SampleBytes(planes[i]),
+ l.mSourceHeight);
+ if (!mResource->CopyTo(planes[i], {origin, size},
+ aData.From(destinationOffset),
+ static_cast<size_t>(l.mDestinationStride))) {
+ p->MaybeRejectWithTypeError(
+ nsPrintfCString("Failed to copy image data in %s plane",
+ mResource->mFormat->PlaneName(planes[i])));
+ return p.forget();
+ }
+ }
+
+ MOZ_ASSERT(layout.mComputedLayouts.Length() == planes.Length());
+ // TODO: Spec doesn't resolve with a value. See
+ // https://github.com/w3c/webcodecs/issues/510 This comment should be
+ // removed once the issue is resolved.
+ p->MaybeResolve(planeLayouts);
+ return p.forget();
+ });
+}
+
+// https://w3c.github.io/webcodecs/#dom-videoframe-clone
+already_AddRefed<VideoFrame> VideoFrame::Clone(ErrorResult& aRv) const {
+ AssertIsOnOwningThread();
+
+ if (!mResource) {
+ aRv.ThrowInvalidStateError("No media resource in the VideoFrame now");
+ return nullptr;
+ }
+ // The VideoFrame's data must be shared instead of copied:
+ // https://w3c.github.io/webcodecs/#raw-media-memory-model-reference-counting
+ return MakeAndAddRef<VideoFrame>(*this);
+}
+
+// https://w3c.github.io/webcodecs/#close-videoframe
+void VideoFrame::Close() {
+ AssertIsOnOwningThread();
+ LOG("VideoFrame %p is closed", this);
+
+ mResource.reset();
+ mCodedSize = gfx::IntSize();
+ mVisibleRect = gfx::IntRect();
+ mDisplaySize = gfx::IntSize();
+ mColorSpace = VideoColorSpaceInit();
+
+ StopAutoClose();
+}
+
+bool VideoFrame::IsClosed() const { return !mResource; }
+
+already_AddRefed<layers::Image> VideoFrame::GetImage() const {
+ if (!mResource) {
+ return nullptr;
+ }
+ return do_AddRef(mResource->mImage);
+}
+
+nsCString VideoFrame::ToString() const {
+ nsCString rv;
+
+ if (IsClosed()) {
+ rv.AppendPrintf("VideoFrame (closed)");
+ return rv;
+ }
+
+ rv.AppendPrintf(
+ "VideoFrame ts: %" PRId64
+ ", %s, coded[%dx%d] visible[%dx%d], display[%dx%d] color: %s",
+ mTimestamp,
+ dom::VideoPixelFormatValues::GetString(mResource->mFormat->PixelFormat())
+ .data(),
+ mCodedSize.width, mCodedSize.height, mVisibleRect.width,
+ mVisibleRect.height, mDisplaySize.width, mDisplaySize.height,
+ ColorSpaceInitToString(mColorSpace).get());
+
+ if (mDuration) {
+ rv.AppendPrintf(" dur: %" PRId64, mDuration.value());
+ }
+
+ return rv;
+}
+
+// https://w3c.github.io/webcodecs/#ref-for-deserialization-steps%E2%91%A0
+/* static */
+JSObject* VideoFrame::ReadStructuredClone(
+ JSContext* aCx, nsIGlobalObject* aGlobal, JSStructuredCloneReader* aReader,
+ const VideoFrameSerializedData& aData) {
+ JS::Rooted<JS::Value> value(aCx, JS::NullValue());
+ // To avoid a rooting hazard error from returning a raw JSObject* before
+ // running the RefPtr destructor, RefPtr needs to be destructed before
+ // returning the raw JSObject*, which is why the RefPtr<VideoFrame> is created
+ // in the scope below. Otherwise, the static analysis infers the RefPtr cannot
+ // be safely destructed while the unrooted return JSObject* is on the stack.
+ {
+ RefPtr<VideoFrame> frame = MakeAndAddRef<VideoFrame>(aGlobal, aData);
+ if (!GetOrCreateDOMReflector(aCx, frame, &value) || !value.isObject()) {
+ return nullptr;
+ }
+ }
+ return value.toObjectOrNull();
+}
+
+// https://w3c.github.io/webcodecs/#ref-for-serialization-steps%E2%91%A0
+bool VideoFrame::WriteStructuredClone(JSStructuredCloneWriter* aWriter,
+ StructuredCloneHolder* aHolder) const {
+ AssertIsOnOwningThread();
+
+ if (!mResource) {
+ return false;
+ }
+
+ // Indexing the image and send the index to the receiver.
+ const uint32_t index = aHolder->VideoFrames().Length();
+ // The serialization is limited to the same process scope so it's ok to
+ // serialize a reference instead of a copy.
+ aHolder->VideoFrames().AppendElement(
+ VideoFrameSerializedData(GetVideoFrameData(), mCodedSize));
+
+ return !NS_WARN_IF(!JS_WriteUint32Pair(aWriter, SCTAG_DOM_VIDEOFRAME, index));
+}
+
+// https://w3c.github.io/webcodecs/#ref-for-transfer-steps%E2%91%A0
+UniquePtr<VideoFrame::TransferredData> VideoFrame::Transfer() {
+ AssertIsOnOwningThread();
+
+ if (!mResource) {
+ return nullptr;
+ }
+
+ auto frame = MakeUnique<TransferredData>(GetVideoFrameData(), mCodedSize);
+ Close();
+ return frame;
+}
+
+// https://w3c.github.io/webcodecs/#ref-for-transfer-receiving-steps%E2%91%A0
+/* static */
+already_AddRefed<VideoFrame> VideoFrame::FromTransferred(
+ nsIGlobalObject* aGlobal, TransferredData* aData) {
+ MOZ_ASSERT(aData);
+
+ return MakeAndAddRef<VideoFrame>(aGlobal, *aData);
+}
+
+VideoFrameData VideoFrame::GetVideoFrameData() const {
+ return VideoFrameData(mResource->mImage.get(), mResource->TryPixelFormat(),
+ mVisibleRect, mDisplaySize, mDuration, mTimestamp,
+ mColorSpace);
+}
+
+void VideoFrame::StartAutoClose() {
+ AssertIsOnOwningThread();
+
+ LOG("VideoFrame %p, start monitoring resource release", this);
+
+ if (NS_IsMainThread()) {
+ mShutdownBlocker = media::ShutdownBlockingTicket::Create(
+ u"VideoFrame::mShutdownBlocker"_ns,
+ NS_LITERAL_STRING_FROM_CSTRING(__FILE__), __LINE__);
+ if (mShutdownBlocker) {
+ mShutdownBlocker->ShutdownPromise()->Then(
+ GetCurrentSerialEventTarget(), __func__,
+ [self = RefPtr{this}](bool /* aUnUsed*/) {
+ LOG("VideoFrame %p gets shutdown notification", self.get());
+ self->CloseIfNeeded();
+ },
+ [self = RefPtr{this}](bool /* aUnUsed*/) {
+ LOG("VideoFrame %p removes shutdown-blocker before getting "
+ "shutdown "
+ "notification",
+ self.get());
+ });
+ }
+ } else if (WorkerPrivate* workerPrivate = GetCurrentThreadWorkerPrivate()) {
+ // Clean up all the resources when the worker is going away.
+ mWorkerRef = WeakWorkerRef::Create(workerPrivate, [self = RefPtr{this}]() {
+ LOG("VideoFrame %p, worker is going away", self.get());
+ self->CloseIfNeeded();
+ });
+ }
+}
+
+void VideoFrame::StopAutoClose() {
+ AssertIsOnOwningThread();
+
+ LOG("VideoFrame %p, stop monitoring resource release", this);
+
+ mShutdownBlocker = nullptr;
+ mWorkerRef = nullptr;
+}
+
+void VideoFrame::CloseIfNeeded() {
+ AssertIsOnOwningThread();
+
+ LOG("VideoFrame %p, needs to close itself? %s", this,
+ IsClosed() ? "no" : "yes");
+ if (!IsClosed()) {
+ LOG("Close VideoFrame %p obligatorily", this);
+ Close();
+ }
+}
+
+/*
+ * VideoFrame::Format
+ *
+ * This class wraps a VideoPixelFormat defined in [1] and provides some
+ * utilities for the VideoFrame's functions. Each sample in the format is 8
+ * bits. The pixel layouts for a 4 x 2 image in the spec are illustrated below:
+ * [1] https://w3c.github.io/webcodecs/#pixel-format
+ *
+ * I420 - 3 planes: Y, U, V
+ * ------
+ * <- width ->
+ * Y: Y1 Y2 Y3 Y4 ^ height
+ * Y5 Y6 Y7 Y8 v
+ * U: U1 U2 => 1/2 Y's width, 1/2 Y's height
+ * V: V1 V2 => 1/2 Y's width, 1/2 Y's height
+ *
+ * I420A - 4 planes: Y, U, V, A
+ * ------
+ * <- width ->
+ * Y: Y1 Y2 Y3 Y4 ^ height
+ * Y5 Y6 Y7 Y8 v
+ * U: U1 U2 => 1/2 Y's width, 1/2 Y's height
+ * V: V1 V2 => 1/2 Y's width, 1/2 Y's height
+ * A: A1 A2 A3 A4 => Y's width, Y's height
+ * A5 A6 A7 A8
+ *
+ * I422 - 3 planes: Y, U, V
+ * ------
+ * <- width ->
+ * Y: Y1 Y2 Y3 Y4 ^ height
+ * Y5 Y6 Y7 Y8 v
+ * U: U1 U2 U3 U4 => Y's width, 1/2 Y's height
+ * V: V1 V2 V3 V4 => Y's width, 1/2 Y's height
+ *
+ * I444 - 3 planes: Y, U, V
+ * ------
+ * <- width ->
+ * Y: Y1 Y2 Y3 Y4 ^ height
+ * Y5 Y6 Y7 Y8 v
+ * U: U1 U2 U3 U4 => Y's width, Y's height
+ * U5 U6 U7 U8
+ * V: V1 V2 V3 V4 => Y's width, Y's height
+ * V5 V6 V7 B8
+ *
+ * NV12 - 2 planes: Y, UV
+ * ------
+ * <- width ->
+ * Y: Y1 Y2 Y3 Y4 ^ height
+ * Y5 Y6 Y7 Y8 v
+ * UV: U1 V1 U2 V2 => Y's width, 1/2 Y's height
+ *
+ * RGBA - 1 plane encoding 3 colors: Red, Green, Blue, and an Alpha value
+ * ------
+ * <---------------------- width ---------------------->
+ * R1 G1 B1 A1 | R2 G2 B2 A2 | R3 G3 B3 A3 | R4 G4 B4 A4 ^ height
+ * R5 G5 B5 A5 | R6 G6 B6 A6 | R7 G7 B7 A7 | R8 G8 B8 A8 v
+ *
+ * RGBX - 1 plane encoding 3 colors: Red, Green, Blue, and an padding value
+ * This is the opaque version of RGBA
+ * ------
+ * <---------------------- width ---------------------->
+ * R1 G1 B1 X1 | R2 G2 B2 X2 | R3 G3 B3 X3 | R4 G4 B4 X4 ^ height
+ * R5 G5 B5 X5 | R6 G6 B6 X6 | R7 G7 B7 X7 | R8 G8 B8 X8 v
+ *
+ * BGRA - 1 plane encoding 3 colors: Blue, Green, Red, and an Alpha value
+ * ------
+ * <---------------------- width ---------------------->
+ * B1 G1 R1 A1 | B2 G2 R2 A2 | B3 G3 R3 A3 | B4 G4 R4 A4 ^ height
+ * B5 G5 R5 A5 | B6 G6 R6 A6 | B7 G7 R7 A7 | B8 G8 R8 A8 v
+ *
+ * BGRX - 1 plane encoding 3 colors: Blue, Green, Red, and an padding value
+ * This is the opaque version of BGRA
+ * ------
+ * <---------------------- width ---------------------->
+ * B1 G1 R1 X1 | B2 G2 R2 X2 | B3 G3 R3 X3 | B4 G4 R4 X4 ^ height
+ * B5 G5 R5 X5 | B6 G6 R6 X6 | B7 G7 R7 X7 | B8 G8 R8 X8 v
+ */
+
+VideoFrame::Format::Format(const VideoPixelFormat& aFormat)
+ : mFormat(aFormat) {}
+
+const VideoPixelFormat& VideoFrame::Format::PixelFormat() const {
+ return mFormat;
+}
+
+gfx::SurfaceFormat VideoFrame::Format::ToSurfaceFormat() const {
+ gfx::SurfaceFormat format = gfx::SurfaceFormat::UNKNOWN;
+ switch (mFormat) {
+ case VideoPixelFormat::I420:
+ case VideoPixelFormat::I420A:
+ case VideoPixelFormat::I422:
+ case VideoPixelFormat::I444:
+ case VideoPixelFormat::NV12:
+ // Not yet support for now.
+ break;
+ case VideoPixelFormat::RGBA:
+ format = gfx::SurfaceFormat::R8G8B8A8;
+ break;
+ case VideoPixelFormat::RGBX:
+ format = gfx::SurfaceFormat::R8G8B8X8;
+ break;
+ case VideoPixelFormat::BGRA:
+ format = gfx::SurfaceFormat::B8G8R8A8;
+ break;
+ case VideoPixelFormat::BGRX:
+ format = gfx::SurfaceFormat::B8G8R8X8;
+ break;
+ case VideoPixelFormat::EndGuard_:
+ MOZ_ASSERT_UNREACHABLE("unsupported format");
+ }
+ return format;
+}
+
+void VideoFrame::Format::MakeOpaque() {
+ switch (mFormat) {
+ case VideoPixelFormat::I420A:
+ mFormat = VideoPixelFormat::I420;
+ return;
+ case VideoPixelFormat::RGBA:
+ mFormat = VideoPixelFormat::RGBX;
+ return;
+ case VideoPixelFormat::BGRA:
+ mFormat = VideoPixelFormat::BGRX;
+ return;
+ case VideoPixelFormat::I420:
+ case VideoPixelFormat::I422:
+ case VideoPixelFormat::I444:
+ case VideoPixelFormat::NV12:
+ case VideoPixelFormat::RGBX:
+ case VideoPixelFormat::BGRX:
+ return;
+ case VideoPixelFormat::EndGuard_:
+ break;
+ }
+ MOZ_ASSERT_UNREACHABLE("unsupported format");
+}
+
+nsTArray<VideoFrame::Format::Plane> VideoFrame::Format::Planes() const {
+ switch (mFormat) {
+ case VideoPixelFormat::I420:
+ case VideoPixelFormat::I422:
+ case VideoPixelFormat::I444:
+ return {Plane::Y, Plane::U, Plane::V};
+ case VideoPixelFormat::I420A:
+ return {Plane::Y, Plane::U, Plane::V, Plane::A};
+ case VideoPixelFormat::NV12:
+ return {Plane::Y, Plane::UV};
+ case VideoPixelFormat::RGBA:
+ case VideoPixelFormat::RGBX:
+ case VideoPixelFormat::BGRA:
+ case VideoPixelFormat::BGRX:
+ return {Plane::RGBA};
+ case VideoPixelFormat::EndGuard_:
+ break;
+ }
+ MOZ_ASSERT_UNREACHABLE("unsupported format");
+ return {};
+}
+
+const char* VideoFrame::Format::PlaneName(const Plane& aPlane) const {
+ switch (aPlane) {
+ case Format::Plane::Y: // and RGBA
+ return IsYUV() ? "Y" : "RGBA";
+ case Format::Plane::U: // and UV
+ MOZ_ASSERT(IsYUV());
+ return mFormat == VideoPixelFormat::NV12 ? "UV" : "U";
+ case Format::Plane::V:
+ MOZ_ASSERT(IsYUV());
+ return "V";
+ case Format::Plane::A:
+ MOZ_ASSERT(IsYUV());
+ return "A";
+ }
+ MOZ_ASSERT_UNREACHABLE("invalid plane");
+ return "Unknown";
+}
+
+uint32_t VideoFrame::Format::SampleBytes(const Plane& aPlane) const {
+ switch (mFormat) {
+ case VideoPixelFormat::I420:
+ case VideoPixelFormat::I420A:
+ case VideoPixelFormat::I422:
+ case VideoPixelFormat::I444:
+ return 1; // 8 bits/sample on the Y, U, V, A plane.
+ case VideoPixelFormat::NV12:
+ switch (aPlane) {
+ case Plane::Y:
+ return 1; // 8 bits/sample on the Y plane
+ case Plane::UV:
+ return 2; // Interleaved U and V values on the UV plane.
+ case Plane::V:
+ case Plane::A:
+ MOZ_ASSERT_UNREACHABLE("invalid plane");
+ }
+ return 0;
+ case VideoPixelFormat::RGBA:
+ case VideoPixelFormat::RGBX:
+ case VideoPixelFormat::BGRA:
+ case VideoPixelFormat::BGRX:
+ return 4; // 8 bits/sample, 32 bits/pixel
+ case VideoPixelFormat::EndGuard_:
+ break;
+ }
+ MOZ_ASSERT_UNREACHABLE("unsupported format");
+ return 0;
+}
+
+gfx::IntSize VideoFrame::Format::SampleSize(const Plane& aPlane) const {
+ // The sample width and height refers to
+ // https://w3c.github.io/webcodecs/#sub-sampling-factor
+ switch (aPlane) {
+ case Plane::Y: // and RGBA
+ case Plane::A:
+ return gfx::IntSize(1, 1);
+ case Plane::U: // and UV
+ case Plane::V:
+ switch (mFormat) {
+ case VideoPixelFormat::I420:
+ case VideoPixelFormat::I420A:
+ case VideoPixelFormat::NV12:
+ return gfx::IntSize(2, 2);
+ case VideoPixelFormat::I422:
+ return gfx::IntSize(2, 1);
+ case VideoPixelFormat::I444:
+ return gfx::IntSize(1, 1);
+ case VideoPixelFormat::RGBA:
+ case VideoPixelFormat::RGBX:
+ case VideoPixelFormat::BGRA:
+ case VideoPixelFormat::BGRX:
+ case VideoPixelFormat::EndGuard_:
+ MOZ_ASSERT_UNREACHABLE("invalid format");
+ return {0, 0};
+ }
+ }
+ MOZ_ASSERT_UNREACHABLE("invalid plane");
+ return {0, 0};
+}
+
+bool VideoFrame::Format::IsValidSize(const gfx::IntSize& aSize) const {
+ switch (mFormat) {
+ case VideoPixelFormat::I420:
+ case VideoPixelFormat::I420A:
+ case VideoPixelFormat::NV12:
+ return (aSize.Width() % 2 == 0) && (aSize.Height() % 2 == 0);
+ case VideoPixelFormat::I422:
+ return aSize.Height() % 2 == 0;
+ case VideoPixelFormat::I444:
+ case VideoPixelFormat::RGBA:
+ case VideoPixelFormat::RGBX:
+ case VideoPixelFormat::BGRA:
+ case VideoPixelFormat::BGRX:
+ return true;
+ case VideoPixelFormat::EndGuard_:
+ break;
+ }
+ MOZ_ASSERT_UNREACHABLE("unsupported format");
+ return false;
+}
+
+size_t VideoFrame::Format::SampleCount(const gfx::IntSize& aSize) const {
+ MOZ_ASSERT(IsValidSize(aSize));
+
+ CheckedInt<size_t> count(aSize.Width());
+ count *= aSize.Height();
+
+ switch (mFormat) {
+ case VideoPixelFormat::I420:
+ case VideoPixelFormat::NV12:
+ return (count + count / 2).value();
+ case VideoPixelFormat::I420A:
+ return (count * 2 + count / 2).value();
+ case VideoPixelFormat::I422:
+ return (count * 2).value();
+ case VideoPixelFormat::I444:
+ return (count * 3).value();
+ case VideoPixelFormat::RGBA:
+ case VideoPixelFormat::RGBX:
+ case VideoPixelFormat::BGRA:
+ case VideoPixelFormat::BGRX:
+ return (count * 4).value();
+ case VideoPixelFormat::EndGuard_:
+ break;
+ }
+
+ MOZ_ASSERT_UNREACHABLE("unsupported format");
+ return 0;
+}
+
+bool VideoFrame::Format::IsYUV() const { return IsYUVFormat(mFormat); }
+
+/*
+ * VideoFrame::Resource
+ */
+
+VideoFrame::Resource::Resource(const RefPtr<layers::Image>& aImage,
+ Maybe<class Format>&& aFormat)
+ : mImage(aImage), mFormat(aFormat) {
+ MOZ_ASSERT(mImage);
+}
+
+VideoFrame::Resource::Resource(const Resource& aOther)
+ : mImage(aOther.mImage), mFormat(aOther.mFormat) {
+ MOZ_ASSERT(mImage);
+}
+
+Maybe<VideoPixelFormat> VideoFrame::Resource::TryPixelFormat() const {
+ return mFormat ? Some(mFormat->PixelFormat()) : Nothing();
+}
+
+uint32_t VideoFrame::Resource::Stride(const Format::Plane& aPlane) const {
+ MOZ_RELEASE_ASSERT(mFormat);
+
+ CheckedInt<uint32_t> width(mImage->GetSize().Width());
+ switch (aPlane) {
+ case Format::Plane::Y: // and RGBA
+ case Format::Plane::A:
+ switch (mFormat->PixelFormat()) {
+ case VideoPixelFormat::I420:
+ case VideoPixelFormat::I420A:
+ case VideoPixelFormat::I422:
+ case VideoPixelFormat::I444:
+ case VideoPixelFormat::NV12:
+ case VideoPixelFormat::RGBA:
+ case VideoPixelFormat::RGBX:
+ case VideoPixelFormat::BGRA:
+ case VideoPixelFormat::BGRX:
+ return (width * mFormat->SampleBytes(aPlane)).value();
+ case VideoPixelFormat::EndGuard_:
+ MOZ_ASSERT_UNREACHABLE("invalid format");
+ }
+ return 0;
+ case Format::Plane::U: // and UV
+ case Format::Plane::V:
+ switch (mFormat->PixelFormat()) {
+ case VideoPixelFormat::I420:
+ case VideoPixelFormat::I420A:
+ case VideoPixelFormat::I422:
+ case VideoPixelFormat::I444:
+ case VideoPixelFormat::NV12:
+ return (((width + 1) / 2) * mFormat->SampleBytes(aPlane)).value();
+ case VideoPixelFormat::RGBA:
+ case VideoPixelFormat::RGBX:
+ case VideoPixelFormat::BGRA:
+ case VideoPixelFormat::BGRX:
+ case VideoPixelFormat::EndGuard_:
+ MOZ_ASSERT_UNREACHABLE("invalid format");
+ }
+ return 0;
+ }
+ MOZ_ASSERT_UNREACHABLE("invalid plane");
+ return 0;
+}
+
+bool VideoFrame::Resource::CopyTo(const Format::Plane& aPlane,
+ const gfx::IntRect& aRect,
+ Span<uint8_t>&& aPlaneDest,
+ size_t aDestinationStride) const {
+ if (!mFormat) {
+ return false;
+ }
+
+ auto copyPlane = [&](const uint8_t* aPlaneData) {
+ MOZ_ASSERT(aPlaneData);
+
+ CheckedInt<size_t> offset(aRect.Y());
+ offset *= Stride(aPlane);
+ offset += aRect.X() * mFormat->SampleBytes(aPlane);
+ if (!offset.isValid()) {
+ return false;
+ }
+
+ CheckedInt<size_t> elementsBytes(aRect.Width());
+ elementsBytes *= mFormat->SampleBytes(aPlane);
+ if (!elementsBytes.isValid()) {
+ return false;
+ }
+
+ aPlaneData += offset.value();
+ for (int32_t row = 0; row < aRect.Height(); ++row) {
+ PodCopy(aPlaneDest.data(), aPlaneData, elementsBytes.value());
+ aPlaneData += Stride(aPlane);
+ // Spec asks to move `aDestinationStride` bytes instead of
+ // `Stride(aPlane)` forward.
+ aPlaneDest = aPlaneDest.From(aDestinationStride);
+ }
+ return true;
+ };
+
+ if (mImage->GetFormat() == ImageFormat::MOZ2D_SURFACE) {
+ RefPtr<gfx::SourceSurface> surface = mImage->GetAsSourceSurface();
+ if (NS_WARN_IF(!surface)) {
+ return false;
+ }
+
+ RefPtr<gfx::DataSourceSurface> dataSurface = surface->GetDataSurface();
+ if (NS_WARN_IF(!dataSurface)) {
+ return false;
+ }
+
+ gfx::DataSourceSurface::ScopedMap map(dataSurface,
+ gfx::DataSourceSurface::READ);
+ if (NS_WARN_IF(!map.IsMapped())) {
+ return false;
+ }
+
+ const gfx::SurfaceFormat format = dataSurface->GetFormat();
+
+ if (format == gfx::SurfaceFormat::R8G8B8A8 ||
+ format == gfx::SurfaceFormat::R8G8B8X8 ||
+ format == gfx::SurfaceFormat::B8G8R8A8 ||
+ format == gfx::SurfaceFormat::B8G8R8X8) {
+ MOZ_ASSERT(aPlane == Format::Plane::RGBA);
+
+ // The mImage's format can be different from mFormat (since Gecko prefers
+ // BGRA). To get the data in the matched format, we create a temp buffer
+ // holding the image data in that format and then copy them to
+ // `aDestination`.
+ const gfx::SurfaceFormat f = mFormat->ToSurfaceFormat();
+ MOZ_ASSERT(f == gfx::SurfaceFormat::R8G8B8A8 ||
+ f == gfx::SurfaceFormat::R8G8B8X8 ||
+ f == gfx::SurfaceFormat::B8G8R8A8 ||
+ f == gfx::SurfaceFormat::B8G8R8X8);
+
+ // TODO: We could use Factory::CreateWrappingDataSourceSurface to wrap
+ // `aDestination` to avoid extra copy.
+ RefPtr<gfx::DataSourceSurface> tempSurface =
+ gfx::Factory::CreateDataSourceSurfaceWithStride(
+ dataSurface->GetSize(), f, map.GetStride());
+ if (NS_WARN_IF(!tempSurface)) {
+ return false;
+ }
+
+ gfx::DataSourceSurface::ScopedMap tempMap(tempSurface,
+ gfx::DataSourceSurface::WRITE);
+ if (NS_WARN_IF(!tempMap.IsMapped())) {
+ return false;
+ }
+
+ if (!gfx::SwizzleData(map.GetData(), map.GetStride(),
+ dataSurface->GetFormat(), tempMap.GetData(),
+ tempMap.GetStride(), tempSurface->GetFormat(),
+ tempSurface->GetSize())) {
+ return false;
+ }
+
+ return copyPlane(tempMap.GetData());
+ }
+
+ return false;
+ }
+
+ if (mImage->GetFormat() == ImageFormat::PLANAR_YCBCR) {
+ switch (aPlane) {
+ case Format::Plane::Y:
+ return copyPlane(mImage->AsPlanarYCbCrImage()->GetData()->mYChannel);
+ case Format::Plane::U:
+ return copyPlane(mImage->AsPlanarYCbCrImage()->GetData()->mCbChannel);
+ case Format::Plane::V:
+ return copyPlane(mImage->AsPlanarYCbCrImage()->GetData()->mCrChannel);
+ case Format::Plane::A:
+ MOZ_ASSERT(mFormat->PixelFormat() == VideoPixelFormat::I420A);
+ MOZ_ASSERT(mImage->AsPlanarYCbCrImage()->GetData()->mAlpha);
+ return copyPlane(
+ mImage->AsPlanarYCbCrImage()->GetData()->mAlpha->mChannel);
+ }
+ MOZ_ASSERT_UNREACHABLE("invalid plane");
+ }
+
+ if (mImage->GetFormat() == ImageFormat::NV_IMAGE) {
+ switch (aPlane) {
+ case Format::Plane::Y:
+ return copyPlane(mImage->AsNVImage()->GetData()->mYChannel);
+ case Format::Plane::UV:
+ return copyPlane(mImage->AsNVImage()->GetData()->mCbChannel);
+ case Format::Plane::V:
+ case Format::Plane::A:
+ MOZ_ASSERT_UNREACHABLE("invalid plane");
+ }
+ }
+
+ // TODO: ImageFormat::MAC_IOSURFACE or ImageFormat::DMABUF
+ LOGW("Cannot copy image data of an unrecognized format");
+
+ return false;
+}
+
+#undef LOGW
+#undef LOG_INTERNAL
+
+} // namespace mozilla::dom
diff --git a/dom/media/webcodecs/VideoFrame.h b/dom/media/webcodecs/VideoFrame.h
new file mode 100644
index 0000000000..0bef496b79
--- /dev/null
+++ b/dom/media/webcodecs/VideoFrame.h
@@ -0,0 +1,266 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_VideoFrame_h
+#define mozilla_dom_VideoFrame_h
+
+#include "js/TypeDecls.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/ErrorResult.h"
+#include "mozilla/NotNull.h"
+#include "mozilla/Span.h"
+#include "mozilla/dom/BindingDeclarations.h"
+#include "mozilla/dom/TypedArray.h"
+#include "mozilla/dom/VideoColorSpaceBinding.h"
+#include "mozilla/dom/WorkerRef.h"
+#include "mozilla/gfx/Point.h"
+#include "mozilla/gfx/Rect.h"
+#include "mozilla/media/MediaUtils.h"
+#include "nsCycleCollectionParticipant.h"
+#include "nsWrapperCache.h"
+
+class nsIGlobalObject;
+class nsIURI;
+
+namespace mozilla {
+
+namespace layers {
+class Image;
+} // namespace layers
+
+namespace dom {
+
+class DOMRectReadOnly;
+class HTMLCanvasElement;
+class HTMLImageElement;
+class HTMLVideoElement;
+class ImageBitmap;
+class MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer;
+class OffscreenCanvas;
+class OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer;
+class Promise;
+class SVGImageElement;
+class StructuredCloneHolder;
+class VideoColorSpace;
+class VideoFrame;
+enum class VideoPixelFormat : uint8_t;
+struct VideoFrameBufferInit;
+struct VideoFrameInit;
+struct VideoFrameCopyToOptions;
+
+} // namespace dom
+} // namespace mozilla
+
+namespace mozilla::dom {
+
+struct VideoFrameData {
+ VideoFrameData(layers::Image* aImage, const Maybe<VideoPixelFormat>& aFormat,
+ gfx::IntRect aVisibleRect, gfx::IntSize aDisplaySize,
+ Maybe<uint64_t> aDuration, int64_t aTimestamp,
+ const VideoColorSpaceInit& aColorSpace);
+ VideoFrameData(const VideoFrameData& aData) = default;
+
+ const RefPtr<layers::Image> mImage;
+ const Maybe<VideoPixelFormat> mFormat;
+ const gfx::IntRect mVisibleRect;
+ const gfx::IntSize mDisplaySize;
+ const Maybe<uint64_t> mDuration;
+ const int64_t mTimestamp;
+ const VideoColorSpaceInit mColorSpace;
+};
+
+struct VideoFrameSerializedData : VideoFrameData {
+ VideoFrameSerializedData(const VideoFrameData& aData,
+ gfx::IntSize aCodedSize);
+
+ const gfx::IntSize mCodedSize;
+};
+
+class VideoFrame final : public nsISupports, public nsWrapperCache {
+ public:
+ NS_DECL_CYCLE_COLLECTING_ISUPPORTS
+ NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(VideoFrame)
+
+ public:
+ VideoFrame(nsIGlobalObject* aParent, const RefPtr<layers::Image>& aImage,
+ const Maybe<VideoPixelFormat>& aFormat, gfx::IntSize aCodedSize,
+ gfx::IntRect aVisibleRect, gfx::IntSize aDisplaySize,
+ const Maybe<uint64_t>& aDuration, int64_t aTimestamp,
+ const VideoColorSpaceInit& aColorSpace);
+ VideoFrame(nsIGlobalObject* aParent, const VideoFrameSerializedData& aData);
+ VideoFrame(const VideoFrame& aOther);
+
+ protected:
+ ~VideoFrame();
+
+ public:
+ nsIGlobalObject* GetParentObject() const;
+
+ JSObject* WrapObject(JSContext* aCx,
+ JS::Handle<JSObject*> aGivenProto) override;
+
+ static already_AddRefed<VideoFrame> Constructor(
+ const GlobalObject& aGlobal, HTMLImageElement& aImageElement,
+ const VideoFrameInit& aInit, ErrorResult& aRv);
+ static already_AddRefed<VideoFrame> Constructor(
+ const GlobalObject& aGlobal, SVGImageElement& aSVGImageElement,
+ const VideoFrameInit& aInit, ErrorResult& aRv);
+ static already_AddRefed<VideoFrame> Constructor(
+ const GlobalObject& aGlobal, HTMLCanvasElement& aCanvasElement,
+ const VideoFrameInit& aInit, ErrorResult& aRv);
+ static already_AddRefed<VideoFrame> Constructor(
+ const GlobalObject& aGlobal, HTMLVideoElement& aVideoElement,
+ const VideoFrameInit& aInit, ErrorResult& aRv);
+ static already_AddRefed<VideoFrame> Constructor(
+ const GlobalObject& aGlobal, OffscreenCanvas& aOffscreenCanvas,
+ const VideoFrameInit& aInit, ErrorResult& aRv);
+ static already_AddRefed<VideoFrame> Constructor(const GlobalObject& aGlobal,
+ ImageBitmap& aImageBitmap,
+ const VideoFrameInit& aInit,
+ ErrorResult& aRv);
+ static already_AddRefed<VideoFrame> Constructor(const GlobalObject& aGlobal,
+ VideoFrame& aVideoFrame,
+ const VideoFrameInit& aInit,
+ ErrorResult& aRv);
+ static already_AddRefed<VideoFrame> Constructor(
+ const GlobalObject& aGlobal, const ArrayBufferView& aBufferView,
+ const VideoFrameBufferInit& aInit, ErrorResult& aRv);
+ static already_AddRefed<VideoFrame> Constructor(
+ const GlobalObject& aGlobal, const ArrayBuffer& aBuffer,
+ const VideoFrameBufferInit& aInit, ErrorResult& aRv);
+
+ Nullable<VideoPixelFormat> GetFormat() const;
+
+ uint32_t CodedWidth() const;
+
+ uint32_t CodedHeight() const;
+
+ already_AddRefed<DOMRectReadOnly> GetCodedRect() const;
+
+ already_AddRefed<DOMRectReadOnly> GetVisibleRect() const;
+
+ uint32_t DisplayWidth() const;
+
+ uint32_t DisplayHeight() const;
+
+ Nullable<uint64_t> GetDuration() const;
+
+ int64_t Timestamp() const;
+
+ already_AddRefed<VideoColorSpace> ColorSpace() const;
+
+ uint32_t AllocationSize(const VideoFrameCopyToOptions& aOptions,
+ ErrorResult& aRv);
+
+ already_AddRefed<Promise> CopyTo(
+ const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination,
+ const VideoFrameCopyToOptions& aOptions, ErrorResult& aRv);
+
+ already_AddRefed<VideoFrame> Clone(ErrorResult& aRv) const;
+
+ void Close();
+ bool IsClosed() const;
+
+ // [Serializable] implementations: {Read, Write}StructuredClone
+ static JSObject* ReadStructuredClone(JSContext* aCx, nsIGlobalObject* aGlobal,
+ JSStructuredCloneReader* aReader,
+ const VideoFrameSerializedData& aData);
+
+ bool WriteStructuredClone(JSStructuredCloneWriter* aWriter,
+ StructuredCloneHolder* aHolder) const;
+
+ // [Transferable] implementations: Transfer, FromTransferred
+ using TransferredData = VideoFrameSerializedData;
+
+ UniquePtr<TransferredData> Transfer();
+
+ static already_AddRefed<VideoFrame> FromTransferred(nsIGlobalObject* aGlobal,
+ TransferredData* aData);
+
+ // Native only methods.
+ const gfx::IntSize& NativeCodedSize() const { return mCodedSize; }
+ const gfx::IntSize& NativeDisplaySize() const { return mDisplaySize; }
+ const gfx::IntRect& NativeVisibleRect() const { return mVisibleRect; }
+ already_AddRefed<layers::Image> GetImage() const;
+
+ nsCString ToString() const;
+
+ public:
+ // A VideoPixelFormat wrapper providing utilities for VideoFrame.
+ class Format final {
+ public:
+ explicit Format(const VideoPixelFormat& aFormat);
+ ~Format() = default;
+ const VideoPixelFormat& PixelFormat() const;
+ gfx::SurfaceFormat ToSurfaceFormat() const;
+ void MakeOpaque();
+
+ // TODO: Assign unique value for each plane?
+ // The value indicates the order of the plane in format.
+ enum class Plane : uint8_t { Y = 0, RGBA = Y, U = 1, UV = U, V = 2, A = 3 };
+ nsTArray<Plane> Planes() const;
+ const char* PlaneName(const Plane& aPlane) const;
+ uint32_t SampleBytes(const Plane& aPlane) const;
+ gfx::IntSize SampleSize(const Plane& aPlane) const;
+ bool IsValidSize(const gfx::IntSize& aSize) const;
+ size_t SampleCount(const gfx::IntSize& aSize) const;
+
+ private:
+ bool IsYUV() const;
+ VideoPixelFormat mFormat;
+ };
+
+ private:
+ // VideoFrame can run on either main thread or worker thread.
+ void AssertIsOnOwningThread() const { NS_ASSERT_OWNINGTHREAD(VideoFrame); }
+
+ VideoFrameData GetVideoFrameData() const;
+
+ // Below helpers are used to automatically release the holding Resource if
+ // VideoFrame is never Close()d by the users.
+ void StartAutoClose();
+ void StopAutoClose();
+ void CloseIfNeeded();
+
+ // A class representing the VideoFrame's data.
+ class Resource final {
+ public:
+ Resource(const RefPtr<layers::Image>& aImage, Maybe<Format>&& aFormat);
+ Resource(const Resource& aOther);
+ ~Resource() = default;
+ Maybe<VideoPixelFormat> TryPixelFormat() const;
+ uint32_t Stride(const Format::Plane& aPlane) const;
+ bool CopyTo(const Format::Plane& aPlane, const gfx::IntRect& aRect,
+ Span<uint8_t>&& aPlaneDest, size_t aDestinationStride) const;
+
+ const RefPtr<layers::Image> mImage;
+ // Nothing() if mImage is not in VideoPixelFormat
+ const Maybe<Format> mFormat;
+ };
+
+ nsCOMPtr<nsIGlobalObject> mParent;
+
+ // Use Maybe instead of UniquePtr to allow copy ctor.
+ // The mResource's existence is used as the [[Detached]] for [Transferable].
+ Maybe<const Resource> mResource; // Nothing() after `Close()`d
+
+ // TODO: Replace this by mResource->mImage->GetSize()?
+ gfx::IntSize mCodedSize;
+ gfx::IntRect mVisibleRect;
+ gfx::IntSize mDisplaySize;
+
+ Maybe<uint64_t> mDuration;
+ int64_t mTimestamp;
+ VideoColorSpaceInit mColorSpace;
+
+ // The following are used to help monitoring mResource release.
+ UniquePtr<media::ShutdownBlockingTicket> mShutdownBlocker = nullptr;
+ RefPtr<WeakWorkerRef> mWorkerRef = nullptr;
+};
+
+} // namespace mozilla::dom
+
+#endif // mozilla_dom_VideoFrame_h
diff --git a/dom/media/webcodecs/WebCodecsUtils.cpp b/dom/media/webcodecs/WebCodecsUtils.cpp
new file mode 100644
index 0000000000..1e03f616db
--- /dev/null
+++ b/dom/media/webcodecs/WebCodecsUtils.cpp
@@ -0,0 +1,578 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "WebCodecsUtils.h"
+
+#include "DecoderTypes.h"
+#include "VideoUtils.h"
+#include "js/experimental/TypedData.h"
+#include "mozilla/Assertions.h"
+#include "mozilla/dom/ImageBitmapBinding.h"
+#include "mozilla/dom/VideoColorSpaceBinding.h"
+#include "mozilla/dom/VideoFrameBinding.h"
+#include "mozilla/gfx/Types.h"
+#include "nsDebug.h"
+#include "PlatformEncoderModule.h"
+#include "PlatformEncoderModule.h"
+
+extern mozilla::LazyLogModule gWebCodecsLog;
+
+#ifdef LOG_INTERNAL
+# undef LOG_INTERNAL
+#endif // LOG_INTERNAL
+#define LOG_INTERNAL(level, msg, ...) \
+ MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__))
+#ifdef LOG
+# undef LOG
+#endif // LOG
+#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__)
+
+namespace mozilla {
+std::atomic<WebCodecsId> sNextId = 0;
+};
+
+namespace mozilla::dom {
+
+/*
+ * The followings are helpers for VideoDecoder methods
+ */
+
+nsTArray<nsCString> GuessContainers(const nsAString& aCodec) {
+ if (IsAV1CodecString(aCodec)) {
+ return {"mp4"_ns, "webm"_ns};
+ }
+
+ if (IsVP9CodecString(aCodec)) {
+ return {"mp4"_ns, "webm"_ns, "ogg"_ns};
+ }
+
+ if (IsVP8CodecString(aCodec)) {
+ return {"webm"_ns, "ogg"_ns, "3gpp"_ns, "3gpp2"_ns, "3gp2"_ns};
+ }
+
+ if (IsH264CodecString(aCodec)) {
+ return {"mp4"_ns, "3gpp"_ns, "3gpp2"_ns, "3gp2"_ns};
+ }
+
+ return {};
+}
+
+Maybe<nsString> ParseCodecString(const nsAString& aCodec) {
+ // Trim the spaces on each end.
+ nsString str(aCodec);
+ str.Trim(" ");
+ nsTArray<nsString> codecs;
+ if (!ParseCodecsString(str, codecs) || codecs.Length() != 1 ||
+ codecs[0] != str) {
+ return Nothing();
+ }
+ return Some(codecs[0]);
+}
+
+/*
+ * The below are helpers to operate ArrayBuffer or ArrayBufferView.
+ */
+
+static std::tuple<JS::ArrayBufferOrView, size_t, size_t> GetArrayBufferInfo(
+ JSContext* aCx,
+ const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer) {
+ if (aBuffer.IsArrayBuffer()) {
+ const ArrayBuffer& buffer = aBuffer.GetAsArrayBuffer();
+ size_t length;
+ {
+ bool isShared;
+ uint8_t* data;
+ JS::GetArrayBufferMaybeSharedLengthAndData(buffer.Obj(), &length,
+ &isShared, &data);
+ }
+ return std::make_tuple(JS::ArrayBufferOrView::fromObject(buffer.Obj()),
+ (size_t)0, length);
+ }
+
+ MOZ_ASSERT(aBuffer.IsArrayBufferView());
+ const ArrayBufferView& view = aBuffer.GetAsArrayBufferView();
+ bool isSharedMemory;
+ JS::Rooted<JSObject*> obj(aCx, view.Obj());
+ return std::make_tuple(
+ JS::ArrayBufferOrView::fromObject(
+ JS_GetArrayBufferViewBuffer(aCx, obj, &isSharedMemory)),
+ JS_GetArrayBufferViewByteOffset(obj),
+ JS_GetArrayBufferViewByteLength(obj));
+}
+
+Result<Ok, nsresult> CloneBuffer(
+ JSContext* aCx,
+ OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDest,
+ const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aSrc) {
+ std::tuple<JS::ArrayBufferOrView, size_t, size_t> info =
+ GetArrayBufferInfo(aCx, aSrc);
+ JS::Rooted<JS::ArrayBufferOrView> abov(aCx);
+ abov.set(std::get<0>(info));
+ size_t offset = std::get<1>(info);
+ size_t len = std::get<2>(info);
+ if (NS_WARN_IF(!abov)) {
+ return Err(NS_ERROR_UNEXPECTED);
+ }
+
+ JS::Rooted<JSObject*> obj(aCx, abov.asObject());
+ JS::Rooted<JSObject*> cloned(aCx,
+ JS::ArrayBufferClone(aCx, obj, offset, len));
+ if (NS_WARN_IF(!cloned)) {
+ return Err(NS_ERROR_OUT_OF_MEMORY);
+ }
+
+ JS::Rooted<JS::Value> value(aCx, JS::ObjectValue(*cloned));
+ if (NS_WARN_IF(!aDest.Init(aCx, value))) {
+ return Err(NS_ERROR_UNEXPECTED);
+ }
+ return Ok();
+}
+
+/*
+ * The following are utilities to convert between VideoColorSpace values to
+ * gfx's values.
+ */
+
+gfx::ColorRange ToColorRange(bool aIsFullRange) {
+ return aIsFullRange ? gfx::ColorRange::FULL : gfx::ColorRange::LIMITED;
+}
+
+gfx::YUVColorSpace ToColorSpace(VideoMatrixCoefficients aMatrix) {
+ switch (aMatrix) {
+ case VideoMatrixCoefficients::Rgb:
+ return gfx::YUVColorSpace::Identity;
+ case VideoMatrixCoefficients::Bt709:
+ case VideoMatrixCoefficients::Bt470bg:
+ return gfx::YUVColorSpace::BT709;
+ case VideoMatrixCoefficients::Smpte170m:
+ return gfx::YUVColorSpace::BT601;
+ case VideoMatrixCoefficients::Bt2020_ncl:
+ return gfx::YUVColorSpace::BT2020;
+ case VideoMatrixCoefficients::EndGuard_:
+ break;
+ }
+ MOZ_ASSERT_UNREACHABLE("unsupported VideoMatrixCoefficients");
+ return gfx::YUVColorSpace::Default;
+}
+
+gfx::TransferFunction ToTransferFunction(
+ VideoTransferCharacteristics aTransfer) {
+ switch (aTransfer) {
+ case VideoTransferCharacteristics::Bt709:
+ case VideoTransferCharacteristics::Smpte170m:
+ return gfx::TransferFunction::BT709;
+ case VideoTransferCharacteristics::Iec61966_2_1:
+ return gfx::TransferFunction::SRGB;
+ case VideoTransferCharacteristics::Pq:
+ return gfx::TransferFunction::PQ;
+ case VideoTransferCharacteristics::Hlg:
+ return gfx::TransferFunction::HLG;
+ case VideoTransferCharacteristics::Linear:
+ case VideoTransferCharacteristics::EndGuard_:
+ break;
+ }
+ MOZ_ASSERT_UNREACHABLE("unsupported VideoTransferCharacteristics");
+ return gfx::TransferFunction::Default;
+}
+
+gfx::ColorSpace2 ToPrimaries(VideoColorPrimaries aPrimaries) {
+ switch (aPrimaries) {
+ case VideoColorPrimaries::Bt709:
+ return gfx::ColorSpace2::BT709;
+ case VideoColorPrimaries::Bt470bg:
+ return gfx::ColorSpace2::BT601_625;
+ case VideoColorPrimaries::Smpte170m:
+ return gfx::ColorSpace2::BT601_525;
+ case VideoColorPrimaries::Bt2020:
+ return gfx::ColorSpace2::BT2020;
+ case VideoColorPrimaries::Smpte432:
+ return gfx::ColorSpace2::DISPLAY_P3;
+ case VideoColorPrimaries::EndGuard_:
+ break;
+ }
+ MOZ_ASSERT_UNREACHABLE("unsupported VideoTransferCharacteristics");
+ return gfx::ColorSpace2::UNKNOWN;
+}
+
+bool ToFullRange(const gfx::ColorRange& aColorRange) {
+ return aColorRange == gfx::ColorRange::FULL;
+}
+
+Maybe<VideoMatrixCoefficients> ToMatrixCoefficients(
+ const gfx::YUVColorSpace& aColorSpace) {
+ switch (aColorSpace) {
+ case gfx::YUVColorSpace::BT601:
+ return Some(VideoMatrixCoefficients::Smpte170m);
+ case gfx::YUVColorSpace::BT709:
+ return Some(VideoMatrixCoefficients::Bt709);
+ case gfx::YUVColorSpace::BT2020:
+ return Some(VideoMatrixCoefficients::Bt2020_ncl);
+ case gfx::YUVColorSpace::Identity:
+ return Some(VideoMatrixCoefficients::Rgb);
+ }
+ MOZ_ASSERT_UNREACHABLE("unsupported gfx::YUVColorSpace");
+ return Nothing();
+}
+
+Maybe<VideoTransferCharacteristics> ToTransferCharacteristics(
+ const gfx::TransferFunction& aTransferFunction) {
+ switch (aTransferFunction) {
+ case gfx::TransferFunction::BT709:
+ return Some(VideoTransferCharacteristics::Bt709);
+ case gfx::TransferFunction::SRGB:
+ return Some(VideoTransferCharacteristics::Iec61966_2_1);
+ case gfx::TransferFunction::PQ:
+ return Some(VideoTransferCharacteristics::Pq);
+ case gfx::TransferFunction::HLG:
+ return Some(VideoTransferCharacteristics::Hlg);
+ }
+ MOZ_ASSERT_UNREACHABLE("unsupported gfx::TransferFunction");
+ return Nothing();
+}
+
+Maybe<VideoColorPrimaries> ToPrimaries(const gfx::ColorSpace2& aColorSpace) {
+ switch (aColorSpace) {
+ case gfx::ColorSpace2::UNKNOWN:
+ return Nothing();
+ case gfx::ColorSpace2::DISPLAY_P3:
+ return Some(VideoColorPrimaries::Smpte432);
+ case gfx::ColorSpace2::BT601_525:
+ return Some(VideoColorPrimaries::Smpte170m);
+ case gfx::ColorSpace2::SRGB:
+ case gfx::ColorSpace2::BT709:
+ return Some(VideoColorPrimaries::Bt709);
+ case gfx::ColorSpace2::BT2020:
+ return Some(VideoColorPrimaries::Bt2020);
+ }
+ MOZ_ASSERT_UNREACHABLE("unsupported gfx::ColorSpace2");
+ return Nothing();
+}
+
+/*
+ * The following are utilities to convert from gfx's formats to
+ * VideoPixelFormats.
+ */
+
+Maybe<VideoPixelFormat> SurfaceFormatToVideoPixelFormat(
+ gfx::SurfaceFormat aFormat) {
+ switch (aFormat) {
+ case gfx::SurfaceFormat::B8G8R8A8:
+ return Some(VideoPixelFormat::BGRA);
+ case gfx::SurfaceFormat::B8G8R8X8:
+ return Some(VideoPixelFormat::BGRX);
+ case gfx::SurfaceFormat::R8G8B8A8:
+ return Some(VideoPixelFormat::RGBA);
+ case gfx::SurfaceFormat::R8G8B8X8:
+ return Some(VideoPixelFormat::RGBX);
+ case gfx::SurfaceFormat::YUV:
+ return Some(VideoPixelFormat::I420);
+ case gfx::SurfaceFormat::NV12:
+ return Some(VideoPixelFormat::NV12);
+ case gfx::SurfaceFormat::YUV422:
+ return Some(VideoPixelFormat::I422);
+ default:
+ break;
+ }
+ return Nothing();
+}
+
+Maybe<VideoPixelFormat> ImageBitmapFormatToVideoPixelFormat(
+ ImageBitmapFormat aFormat) {
+ switch (aFormat) {
+ case ImageBitmapFormat::RGBA32:
+ return Some(VideoPixelFormat::RGBA);
+ case ImageBitmapFormat::BGRA32:
+ return Some(VideoPixelFormat::BGRA);
+ case ImageBitmapFormat::YUV444P:
+ return Some(VideoPixelFormat::I444);
+ case ImageBitmapFormat::YUV422P:
+ return Some(VideoPixelFormat::I422);
+ case ImageBitmapFormat::YUV420P:
+ return Some(VideoPixelFormat::I420);
+ case ImageBitmapFormat::YUV420SP_NV12:
+ return Some(VideoPixelFormat::NV12);
+ default:
+ break;
+ }
+ return Nothing();
+}
+
+Result<RefPtr<MediaByteBuffer>, nsresult> GetExtraDataFromArrayBuffer(
+ const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer) {
+ RefPtr<MediaByteBuffer> data = MakeRefPtr<MediaByteBuffer>();
+ Unused << AppendTypedArrayDataTo(aBuffer, *data);
+ return data->Length() > 0 ? data : nullptr;
+}
+
+bool IsOnAndroid() {
+#if defined(ANDROID)
+ return true;
+#else
+ return false;
+#endif
+}
+
+bool IsOnMacOS() {
+#if defined(XP_MACOSX)
+ return true;
+#else
+ return false;
+#endif
+}
+
+bool IsOnLinux() {
+#if defined(XP_LINUX)
+ return true;
+#else
+ return false;
+#endif
+}
+
+template <typename T>
+nsCString MaybeToString(const Maybe<T>& aMaybe) {
+ return nsPrintfCString(
+ "%s", aMaybe.isSome() ? ToString(aMaybe.value()).c_str() : "nothing");
+}
+
+struct ConfigurationChangeToString {
+ nsCString operator()(const CodecChange& aCodecChange) {
+ return nsPrintfCString("Codec: %s",
+ NS_ConvertUTF16toUTF8(aCodecChange.get()).get());
+ }
+ nsCString operator()(const DimensionsChange& aDimensionChange) {
+ return nsPrintfCString("Dimensions: %dx%d", aDimensionChange.get().width,
+ aDimensionChange.get().height);
+ }
+ nsCString operator()(const DisplayDimensionsChange& aDisplayDimensionChange) {
+ if (aDisplayDimensionChange.get().isNothing()) {
+ return nsPrintfCString("Display dimensions: nothing");
+ }
+ gfx::IntSize displayDimensions = aDisplayDimensionChange.get().value();
+ return nsPrintfCString("Dimensions: %dx%d", displayDimensions.width,
+ displayDimensions.height);
+ }
+ nsCString operator()(const BitrateChange& aBitrateChange) {
+ return nsPrintfCString("Bitrate: %skbps",
+ MaybeToString(aBitrateChange.get()).get());
+ }
+ nsCString operator()(const FramerateChange& aFramerateChange) {
+ return nsPrintfCString("Framerate: %sHz",
+ MaybeToString(aFramerateChange.get()).get());
+ }
+ nsCString operator()(
+ const HardwareAccelerationChange& aHardwareAccelerationChange) {
+ return nsPrintfCString("HW acceleration: %s",
+ dom::HardwareAccelerationValues::GetString(
+ aHardwareAccelerationChange.get())
+ .data());
+ }
+ nsCString operator()(const AlphaChange& aAlphaChange) {
+ return nsPrintfCString(
+ "Alpha: %s",
+ dom::AlphaOptionValues::GetString(aAlphaChange.get()).data());
+ }
+ nsCString operator()(const ScalabilityModeChange& aScalabilityModeChange) {
+ if (aScalabilityModeChange.get().isNothing()) {
+ return nsCString("Scalability mode: nothing");
+ }
+ return nsPrintfCString(
+ "Scalability mode: %s",
+ NS_ConvertUTF16toUTF8(aScalabilityModeChange.get().value()).get());
+ }
+ nsCString operator()(const BitrateModeChange& aBitrateModeChange) {
+ return nsPrintfCString(
+ "Bitrate mode: %s",
+ dom::VideoEncoderBitrateModeValues::GetString(aBitrateModeChange.get())
+ .data());
+ }
+ nsCString operator()(const LatencyModeChange& aLatencyModeChange) {
+ return nsPrintfCString(
+ "Latency mode: %s",
+ dom::LatencyModeValues::GetString(aLatencyModeChange.get()).data());
+ }
+ nsCString operator()(const ContentHintChange& aContentHintChange) {
+ return nsPrintfCString("Content hint: %s",
+ MaybeToString(aContentHintChange.get()).get());
+ }
+ template <typename T>
+ nsCString operator()(const T& aNewBitrate) {
+ return nsPrintfCString("Not implemented");
+ }
+};
+
+nsString WebCodecsConfigurationChangeList::ToString() const {
+ nsString rv;
+ for (const WebCodecsEncoderConfigurationItem& change : mChanges) {
+ nsCString str = change.match(ConfigurationChangeToString());
+ rv.AppendPrintf("- %s\n", str.get());
+ }
+ return rv;
+}
+
+using CodecChange = StrongTypedef<nsString, struct CodecChangeTypeWebCodecs>;
+using DimensionsChange =
+ StrongTypedef<gfx::IntSize, struct DimensionsChangeTypeWebCodecs>;
+using DisplayDimensionsChange =
+ StrongTypedef<Maybe<gfx::IntSize>,
+ struct DisplayDimensionsChangeTypeWebCodecs>;
+using BitrateChange =
+ StrongTypedef<Maybe<uint32_t>, struct BitrateChangeTypeWebCodecs>;
+using FramerateChange =
+ StrongTypedef<Maybe<double>, struct FramerateChangeTypeWebCodecs>;
+using HardwareAccelerationChange =
+ StrongTypedef<dom::HardwareAcceleration,
+ struct HardwareAccelerationChangeTypeWebCodecs>;
+using AlphaChange =
+ StrongTypedef<dom::AlphaOption, struct AlphaChangeTypeWebCodecs>;
+using ScalabilityModeChange =
+ StrongTypedef<Maybe<nsString>, struct ScalabilityModeChangeTypeWebCodecs>;
+using BitrateModeChange = StrongTypedef<dom::VideoEncoderBitrateMode,
+ struct BitrateModeChangeTypeWebCodecs>;
+using LatencyModeChange =
+ StrongTypedef<dom::LatencyMode, struct LatencyModeTypeChangeTypeWebCodecs>;
+using ContentHintChange =
+ StrongTypedef<Maybe<nsString>, struct ContentHintTypeTypeWebCodecs>;
+
+bool WebCodecsConfigurationChangeList::CanAttemptReconfigure() const {
+ for (const auto& change : mChanges) {
+ if (change.is<CodecChange>() || change.is<HardwareAccelerationChange>() ||
+ change.is<AlphaChange>() || change.is<ScalabilityModeChange>()) {
+ return false;
+ }
+ }
+ return true;
+}
+
+RefPtr<EncoderConfigurationChangeList>
+WebCodecsConfigurationChangeList::ToPEMChangeList() const {
+ auto rv = MakeRefPtr<EncoderConfigurationChangeList>();
+ MOZ_ASSERT(CanAttemptReconfigure());
+ for (const auto& change : mChanges) {
+ if (change.is<dom::DimensionsChange>()) {
+ rv->Push(mozilla::DimensionsChange(change.as<DimensionsChange>().get()));
+ } else if (change.is<dom::DisplayDimensionsChange>()) {
+ rv->Push(mozilla::DisplayDimensionsChange(
+ change.as<DisplayDimensionsChange>().get()));
+ } else if (change.is<dom::BitrateChange>()) {
+ rv->Push(mozilla::BitrateChange(change.as<BitrateChange>().get()));
+ } else if (change.is<FramerateChange>()) {
+ rv->Push(mozilla::FramerateChange(change.as<FramerateChange>().get()));
+ } else if (change.is<dom::BitrateModeChange>()) {
+ MediaDataEncoder::BitrateMode mode;
+ if (change.as<dom::BitrateModeChange>().get() ==
+ dom::VideoEncoderBitrateMode::Constant) {
+ mode = MediaDataEncoder::BitrateMode::Constant;
+ } else if (change.as<BitrateModeChange>().get() ==
+ dom::VideoEncoderBitrateMode::Variable) {
+ mode = MediaDataEncoder::BitrateMode::Variable;
+ } else {
+ // Quantizer, not underlying support yet.
+ mode = MediaDataEncoder::BitrateMode::Variable;
+ }
+ rv->Push(mozilla::BitrateModeChange(mode));
+ } else if (change.is<LatencyModeChange>()) {
+ MediaDataEncoder::Usage usage;
+ if (change.as<LatencyModeChange>().get() == dom::LatencyMode::Quality) {
+ usage = MediaDataEncoder::Usage::Record;
+ } else {
+ usage = MediaDataEncoder::Usage::Realtime;
+ }
+ rv->Push(UsageChange(usage));
+ } else if (change.is<ContentHintChange>()) {
+ rv->Push(
+ mozilla::ContentHintChange(change.as<ContentHintChange>().get()));
+ }
+ }
+ return rv.forget();
+}
+
+#define ENUM_TO_STRING(enumType, enumValue) \
+ enumType##Values::GetString(enumValue).data()
+
+nsCString ColorSpaceInitToString(
+ const dom::VideoColorSpaceInit& aColorSpaceInit) {
+ nsCString rv("VideoColorSpace");
+
+ if (!aColorSpaceInit.mFullRange.IsNull()) {
+ rv.AppendPrintf(" range: %s",
+ aColorSpaceInit.mFullRange.Value() ? "true" : "false");
+ }
+ if (!aColorSpaceInit.mMatrix.IsNull()) {
+ rv.AppendPrintf(" matrix: %s",
+ ENUM_TO_STRING(dom::VideoMatrixCoefficients,
+ aColorSpaceInit.mMatrix.Value()));
+ }
+ if (!aColorSpaceInit.mTransfer.IsNull()) {
+ rv.AppendPrintf(" transfer: %s",
+ ENUM_TO_STRING(dom::VideoTransferCharacteristics,
+ aColorSpaceInit.mTransfer.Value()));
+ }
+ if (!aColorSpaceInit.mPrimaries.IsNull()) {
+ rv.AppendPrintf(" primaries: %s",
+ ENUM_TO_STRING(dom::VideoColorPrimaries,
+ aColorSpaceInit.mPrimaries.Value()));
+ }
+
+ return rv;
+}
+
+RefPtr<TaskQueue> GetWebCodecsEncoderTaskQueue() {
+ return TaskQueue::Create(
+ GetMediaThreadPool(MediaThreadType::PLATFORM_ENCODER),
+ "WebCodecs encoding", false);
+}
+
+VideoColorSpaceInit FallbackColorSpaceForVideoContent() {
+ // If we're unable to determine the color space, but we think this is video
+ // content (e.g. because it's in YUV or NV12 or something like that,
+ // consider it's in BT709).
+ // This is step 3 of
+ // https://w3c.github.io/webcodecs/#videoframe-pick-color-space
+ VideoColorSpaceInit colorSpace;
+ colorSpace.mFullRange = false;
+ colorSpace.mMatrix = VideoMatrixCoefficients::Bt709;
+ colorSpace.mTransfer = VideoTransferCharacteristics::Bt709;
+ colorSpace.mPrimaries = VideoColorPrimaries::Bt709;
+ return colorSpace;
+}
+VideoColorSpaceInit FallbackColorSpaceForWebContent() {
+ // If we're unable to determine the color space, but we think this is from
+ // Web content (canvas, image, svg, etc.), consider it's in sRGB.
+ // This is step 2 of
+ // https://w3c.github.io/webcodecs/#videoframe-pick-color-space
+ VideoColorSpaceInit colorSpace;
+ colorSpace.mFullRange = true;
+ colorSpace.mMatrix = VideoMatrixCoefficients::Rgb;
+ colorSpace.mTransfer = VideoTransferCharacteristics::Iec61966_2_1;
+ colorSpace.mPrimaries = VideoColorPrimaries::Bt709;
+ return colorSpace;
+}
+
+Maybe<CodecType> CodecStringToCodecType(const nsAString& aCodecString) {
+ if (StringBeginsWith(aCodecString, u"av01"_ns)) {
+ return Some(CodecType::AV1);
+ }
+ if (StringBeginsWith(aCodecString, u"vp8"_ns)) {
+ return Some(CodecType::VP8);
+ }
+ if (StringBeginsWith(aCodecString, u"vp09"_ns)) {
+ return Some(CodecType::VP9);
+ }
+ if (StringBeginsWith(aCodecString, u"avc1"_ns)) {
+ return Some(CodecType::H264);
+ }
+ return Nothing();
+}
+
+nsString ConfigToString(const VideoDecoderConfig& aConfig) {
+ nsString rv;
+
+ auto internal = VideoDecoderConfigInternal::Create(aConfig);
+
+ return internal->ToString();
+}
+
+}; // namespace mozilla::dom
diff --git a/dom/media/webcodecs/WebCodecsUtils.h b/dom/media/webcodecs/WebCodecsUtils.h
new file mode 100644
index 0000000000..7c0e6b6bbc
--- /dev/null
+++ b/dom/media/webcodecs/WebCodecsUtils.h
@@ -0,0 +1,239 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MOZILLA_DOM_WEBCODECS_WEBCODECSUTILS_H
+#define MOZILLA_DOM_WEBCODECS_WEBCODECSUTILS_H
+
+#include "ErrorList.h"
+#include "js/TypeDecls.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/MozPromise.h"
+#include "mozilla/Result.h"
+#include "mozilla/TaskQueue.h"
+#include "mozilla/dom/BindingDeclarations.h"
+#include "mozilla/dom/Nullable.h"
+#include "mozilla/dom/UnionTypes.h"
+#include "mozilla/dom/VideoEncoderBinding.h"
+#include "mozilla/dom/VideoFrameBinding.h"
+#include "PlatformEncoderModule.h"
+
+namespace mozilla {
+
+namespace gfx {
+enum class ColorRange : uint8_t;
+enum class ColorSpace2 : uint8_t;
+enum class SurfaceFormat : int8_t;
+enum class TransferFunction : uint8_t;
+enum class YUVColorSpace : uint8_t;
+} // namespace gfx
+
+using WebCodecsId = size_t;
+
+extern std::atomic<WebCodecsId> sNextId;
+
+struct EncoderConfigurationChangeList;
+
+namespace dom {
+
+/*
+ * The followings are helpers for WebCodecs methods.
+ */
+
+nsTArray<nsCString> GuessContainers(const nsAString& aCodec);
+
+Maybe<nsString> ParseCodecString(const nsAString& aCodec);
+
+/*
+ * Below are helpers for conversion among Maybe, Optional, and Nullable.
+ */
+
+template <typename T>
+Maybe<T> OptionalToMaybe(const Optional<T>& aOptional) {
+ if (aOptional.WasPassed()) {
+ return Some(aOptional.Value());
+ }
+ return Nothing();
+}
+
+template <typename T>
+const T* OptionalToPointer(const Optional<T>& aOptional) {
+ return aOptional.WasPassed() ? &aOptional.Value() : nullptr;
+}
+
+template <typename T>
+Maybe<T> NullableToMaybe(const Nullable<T>& aNullable) {
+ if (!aNullable.IsNull()) {
+ return Some(aNullable.Value());
+ }
+ return Nothing();
+}
+
+template <typename T>
+Nullable<T> MaybeToNullable(const Maybe<T>& aOptional) {
+ if (aOptional.isSome()) {
+ return Nullable<T>(aOptional.value());
+ }
+ return Nullable<T>();
+}
+
+/*
+ * Below are helpers to operate ArrayBuffer or ArrayBufferView.
+ */
+
+Result<Ok, nsresult> CloneBuffer(
+ JSContext* aCx,
+ OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDest,
+ const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aSrc);
+
+/*
+ * The following are utilities to convert between VideoColorSpace values to
+ * gfx's values.
+ */
+
+enum class VideoColorPrimaries : uint8_t;
+enum class VideoMatrixCoefficients : uint8_t;
+enum class VideoTransferCharacteristics : uint8_t;
+
+gfx::ColorRange ToColorRange(bool aIsFullRange);
+
+gfx::YUVColorSpace ToColorSpace(VideoMatrixCoefficients aMatrix);
+
+gfx::TransferFunction ToTransferFunction(
+ VideoTransferCharacteristics aTransfer);
+
+gfx::ColorSpace2 ToPrimaries(VideoColorPrimaries aPrimaries);
+
+bool ToFullRange(const gfx::ColorRange& aColorRange);
+
+Maybe<VideoMatrixCoefficients> ToMatrixCoefficients(
+ const gfx::YUVColorSpace& aColorSpace);
+
+Maybe<VideoTransferCharacteristics> ToTransferCharacteristics(
+ const gfx::TransferFunction& aTransferFunction);
+
+Maybe<VideoColorPrimaries> ToPrimaries(const gfx::ColorSpace2& aColorSpace);
+
+/*
+ * The following are utilities to convert from gfx's formats to
+ * VideoPixelFormats.
+ */
+
+enum class ImageBitmapFormat : uint8_t;
+enum class VideoPixelFormat : uint8_t;
+
+Maybe<VideoPixelFormat> SurfaceFormatToVideoPixelFormat(
+ gfx::SurfaceFormat aFormat);
+
+Maybe<VideoPixelFormat> ImageBitmapFormatToVideoPixelFormat(
+ ImageBitmapFormat aFormat);
+
+template <typename T>
+class MessageRequestHolder {
+ public:
+ MessageRequestHolder() = default;
+ ~MessageRequestHolder() = default;
+
+ MozPromiseRequestHolder<T>& Request() { return mRequest; }
+ void Disconnect() { mRequest.DisconnectIfExists(); }
+ void Complete() { mRequest.Complete(); }
+ bool Exists() const { return mRequest.Exists(); }
+
+ protected:
+ MozPromiseRequestHolder<T> mRequest{};
+};
+
+enum class MessageProcessedResult { NotProcessed, Processed };
+
+bool IsOnAndroid();
+bool IsOnMacOS();
+bool IsOnLinux();
+
+// Wrap a type to make it unique. This allows using ergonomically in the Variant
+// below. Simply aliasing with `using` isn't enough, because typedefs in C++
+// don't produce strong types, so two integer variants result in
+// the same type, making it ambiguous to the Variant code.
+// T is the type to be wrapped. Phantom is a type that is only used to
+// disambiguate and should be unique in the program.
+template <typename T, typename Phantom>
+class StrongTypedef {
+ public:
+ explicit StrongTypedef(T const& value) : mValue(value) {}
+ explicit StrongTypedef(T&& value) : mValue(std::move(value)) {}
+ T& get() { return mValue; }
+ T const& get() const { return mValue; }
+
+ private:
+ T mValue;
+};
+
+using CodecChange = StrongTypedef<nsString, struct CodecChangeTypeWebCodecs>;
+using DimensionsChange =
+ StrongTypedef<gfx::IntSize, struct DimensionsChangeTypeWebCodecs>;
+using DisplayDimensionsChange =
+ StrongTypedef<Maybe<gfx::IntSize>,
+ struct DisplayDimensionsChangeTypeWebCodecs>;
+using BitrateChange =
+ StrongTypedef<Maybe<uint32_t>, struct BitrateChangeTypeWebCodecs>;
+using FramerateChange =
+ StrongTypedef<Maybe<double>, struct FramerateChangeTypeWebCodecs>;
+using HardwareAccelerationChange =
+ StrongTypedef<dom::HardwareAcceleration,
+ struct HardwareAccelerationChangeTypeWebCodecs>;
+using AlphaChange =
+ StrongTypedef<dom::AlphaOption, struct AlphaChangeTypeWebCodecs>;
+using ScalabilityModeChange =
+ StrongTypedef<Maybe<nsString>, struct ScalabilityModeChangeTypeWebCodecs>;
+using BitrateModeChange = StrongTypedef<dom::VideoEncoderBitrateMode,
+ struct BitrateModeChangeTypeWebCodecs>;
+using LatencyModeChange =
+ StrongTypedef<dom::LatencyMode, struct LatencyModeTypeChangeTypeWebCodecs>;
+using ContentHintChange =
+ StrongTypedef<Maybe<nsString>, struct ContentHintTypeTypeWebCodecs>;
+
+using WebCodecsEncoderConfigurationItem =
+ Variant<CodecChange, DimensionsChange, DisplayDimensionsChange,
+ BitrateModeChange, BitrateChange, FramerateChange,
+ HardwareAccelerationChange, AlphaChange, ScalabilityModeChange,
+ LatencyModeChange, ContentHintChange>;
+
+struct WebCodecsConfigurationChangeList {
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(WebCodecsConfigurationChangeList)
+ bool Empty() const { return mChanges.IsEmpty(); }
+ template <typename T>
+ void Push(const T& aItem) {
+ mChanges.AppendElement(aItem);
+ }
+ // This returns true if it should be possible to attempt to reconfigure the
+ // encoder on the fly. It can fail, in which case the encoder will be flushed
+ // and a new one will be created with the new set of parameters.
+ bool CanAttemptReconfigure() const;
+
+ // Convert this to the format the underlying PEM can understand
+ RefPtr<EncoderConfigurationChangeList> ToPEMChangeList() const;
+ nsString ToString() const;
+
+ nsTArray<WebCodecsEncoderConfigurationItem> mChanges;
+
+ private:
+ ~WebCodecsConfigurationChangeList() = default;
+};
+
+nsCString ColorSpaceInitToString(
+ const dom::VideoColorSpaceInit& aColorSpaceInit);
+
+RefPtr<TaskQueue> GetWebCodecsEncoderTaskQueue();
+VideoColorSpaceInit FallbackColorSpaceForVideoContent();
+VideoColorSpaceInit FallbackColorSpaceForWebContent();
+
+Maybe<CodecType> CodecStringToCodecType(const nsAString& aCodecString);
+
+nsString ConfigToString(const VideoDecoderConfig& aConfig);
+
+} // namespace dom
+
+} // namespace mozilla
+
+#endif // MOZILLA_DOM_WEBCODECS_WEBCODECSUTILS_H
diff --git a/dom/media/webcodecs/crashtests/1839270.html b/dom/media/webcodecs/crashtests/1839270.html
new file mode 100644
index 0000000000..3b8f7908e5
--- /dev/null
+++ b/dom/media/webcodecs/crashtests/1839270.html
@@ -0,0 +1,13 @@
+<script>
+window.addEventListener("load", async () => {
+ let _ = new Response("", {"headers": []})
+ let a = new ArrayBuffer(60005)
+ let v = new VideoFrame(a, {
+ "format": "RGBA",
+ "codedWidth": 1458585599,
+ "codedHeight": 84,
+ "timestamp": 0.541,
+ "layout": [{"offset": 168, "stride": 198}],
+ })
+})
+</script>
diff --git a/dom/media/webcodecs/crashtests/1848460.html b/dom/media/webcodecs/crashtests/1848460.html
new file mode 100644
index 0000000000..2f6f8b930e
--- /dev/null
+++ b/dom/media/webcodecs/crashtests/1848460.html
@@ -0,0 +1,17 @@
+<script id="worker" type="javascript/worker">
+self.onmessage = async function(e) {
+ let a = new ArrayBuffer(12583)
+ let b = new DataView(a)
+ await VideoDecoder.isConfigSupported({
+ "codec": "7󠎢ﷺ۹.9",
+ "description": b,
+ })
+}
+</script>
+<script>
+window.addEventListener("load", async () => {
+ const blob = new Blob([document.querySelector('#worker').textContent], { type: "text/javascript" })
+ const worker = new Worker(window.URL.createObjectURL(blob))
+ worker.postMessage([], [])
+})
+</script>
diff --git a/dom/media/webcodecs/crashtests/1849271.html b/dom/media/webcodecs/crashtests/1849271.html
new file mode 100644
index 0000000000..67e170d8bf
--- /dev/null
+++ b/dom/media/webcodecs/crashtests/1849271.html
@@ -0,0 +1,27 @@
+<html class="reftest-wait">
+ <script>
+ var cfg = {
+ codec: "vp8",
+ colorSpace: { primaries: "bt709" },
+ };
+ var decoder = new VideoDecoder({
+ output: () => {},
+ error: e => {
+ document.documentElement.removeAttribute("class");
+ },
+ });
+ decoder.configure(cfg);
+ try {
+ decoder.decode(
+ new EncodedVideoChunk({
+ type: "key",
+ timestamp: 0,
+ duration: 10,
+ data: new Uint8Array(10),
+ })
+ );
+ } catch (e) {
+ document.documentElement.removeAttribute("class");
+ }
+ </script>
+</html>
diff --git a/dom/media/webcodecs/crashtests/1864475.html b/dom/media/webcodecs/crashtests/1864475.html
new file mode 100644
index 0000000000..d7164b2854
--- /dev/null
+++ b/dom/media/webcodecs/crashtests/1864475.html
@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<script>
+ document.addEventListener('DOMContentLoaded', async () => {
+ const buffer = new ArrayBuffer(26838)
+ const array = new Uint8ClampedArray(buffer)
+ const frame = new VideoFrame(array, {
+ 'format': 'I420A',
+ 'codedWidth': 192,
+ 'codedHeight': 2,
+ 'timestamp': 14.19024535832334,
+ })
+ await frame.copyTo(buffer, {})
+ })
+</script>
diff --git a/dom/media/webcodecs/crashtests/crashtests.list b/dom/media/webcodecs/crashtests/crashtests.list
new file mode 100644
index 0000000000..cea5139fe9
--- /dev/null
+++ b/dom/media/webcodecs/crashtests/crashtests.list
@@ -0,0 +1,4 @@
+skip-if(Android) pref(dom.media.webcodecs.enabled,true) load 1839270.html
+skip-if(Android) pref(dom.media.webcodecs.enabled,true) load 1848460.html
+skip-if(Android) pref(dom.media.webcodecs.enabled,true) load 1849271.html
+skip-if(Android) pref(dom.media.webcodecs.enabled,true) load 1864475.html \ No newline at end of file
diff --git a/dom/media/webcodecs/moz.build b/dom/media/webcodecs/moz.build
new file mode 100644
index 0000000000..267a822286
--- /dev/null
+++ b/dom/media/webcodecs/moz.build
@@ -0,0 +1,54 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+with Files("*"):
+ BUG_COMPONENT = ("Core", "Audio/Video: Web Codecs")
+
+MOCHITEST_MANIFESTS += ["test/mochitest.toml"]
+CRASHTEST_MANIFESTS += ["crashtests/crashtests.list"]
+
+# For mozilla/layers/ImageBridgeChild.h
+LOCAL_INCLUDES += [
+ "!/ipc/ipdl/_ipdlheaders",
+ "/ipc/chromium/src/",
+]
+
+EXPORTS.mozilla += [
+ "DecoderAgent.h",
+]
+
+EXPORTS.mozilla.dom += [
+ "DecoderTemplate.h",
+ "DecoderTypes.h",
+ "EncodedVideoChunk.h",
+ "EncoderAgent.h",
+ "EncoderTemplate.h",
+ "EncoderTypes.h",
+ "VideoColorSpace.h",
+ "VideoDecoder.h",
+ "VideoEncoder.h",
+ "VideoFrame.h",
+ "WebCodecsUtils.h",
+]
+
+UNIFIED_SOURCES += [
+ "DecoderAgent.cpp",
+ "DecoderTemplate.cpp",
+ "EncodedVideoChunk.cpp",
+ "EncoderAgent.cpp",
+ "EncoderTemplate.cpp",
+ "VideoColorSpace.cpp",
+ "VideoDecoder.cpp",
+ "VideoEncoder.cpp",
+ "VideoFrame.cpp",
+ "WebCodecsUtils.cpp",
+]
+
+if CONFIG["MOZ_WAYLAND"]:
+ CXXFLAGS += CONFIG["MOZ_WAYLAND_CFLAGS"]
+ CFLAGS += CONFIG["MOZ_WAYLAND_CFLAGS"]
+
+FINAL_LIBRARY = "xul"
diff --git a/dom/media/webcodecs/test/mochitest.toml b/dom/media/webcodecs/test/mochitest.toml
new file mode 100644
index 0000000000..061bfd12fb
--- /dev/null
+++ b/dom/media/webcodecs/test/mochitest.toml
@@ -0,0 +1,6 @@
+[DEFAULT]
+subsuite = "media"
+tags = "webcodecs"
+prefs = ["dom.media.webcodecs.enabled=true"]
+
+["test_videoFrame_mismatched_codedSize.html"]
diff --git a/dom/media/webcodecs/test/test_videoFrame_mismatched_codedSize.html b/dom/media/webcodecs/test/test_videoFrame_mismatched_codedSize.html
new file mode 100644
index 0000000000..52e26f7854
--- /dev/null
+++ b/dom/media/webcodecs/test/test_videoFrame_mismatched_codedSize.html
@@ -0,0 +1,30 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+ <title></title>
+ <script src="/tests/SimpleTest/SimpleTest.js"></script>
+ <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+</head>
+<body>
+<script>
+let data = new Uint8Array([
+ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+]);
+// TODO: Crop the image instead of returning errors (Bug 1782128).
+try {
+ // Bug 1793814: Remove eslint-disable-line below
+ let frame = new VideoFrame(data, { // eslint-disable-line no-undef
+ timestamp: 10,
+ codedWidth: 3,
+ codedHeight: 3,
+ visibleRect: { x: 0, y: 0, width: 1, height: 1 },
+ format: "RGBA",
+ });
+ frame.close();
+ ok(false, "Should not create a VideoFrame from a mismatched-size buffer");
+} catch (e) {
+ ok(e instanceof TypeError, "Should throw a TypeError");
+}
+</script>
+</body>
+</html>