From 26a029d407be480d791972afb5975cf62c9360a6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 02:47:55 +0200 Subject: Adding upstream version 124.0.1. Signed-off-by: Daniel Baumann --- dom/media/webcodecs/DecoderAgent.cpp | 491 ++++ dom/media/webcodecs/DecoderAgent.h | 117 + dom/media/webcodecs/DecoderTemplate.cpp | 891 ++++++++ dom/media/webcodecs/DecoderTemplate.h | 260 +++ dom/media/webcodecs/DecoderTypes.h | 117 + dom/media/webcodecs/EncodedVideoChunk.cpp | 261 +++ dom/media/webcodecs/EncodedVideoChunk.h | 119 + dom/media/webcodecs/EncoderAgent.cpp | 441 ++++ dom/media/webcodecs/EncoderAgent.h | 116 + dom/media/webcodecs/EncoderTemplate.cpp | 1228 ++++++++++ dom/media/webcodecs/EncoderTemplate.h | 290 +++ dom/media/webcodecs/EncoderTypes.h | 103 + dom/media/webcodecs/VideoColorSpace.cpp | 48 + dom/media/webcodecs/VideoColorSpace.h | 64 + dom/media/webcodecs/VideoDecoder.cpp | 977 ++++++++ dom/media/webcodecs/VideoDecoder.h | 79 + dom/media/webcodecs/VideoEncoder.cpp | 624 +++++ dom/media/webcodecs/VideoEncoder.h | 78 + dom/media/webcodecs/VideoFrame.cpp | 2417 ++++++++++++++++++++ dom/media/webcodecs/VideoFrame.h | 266 +++ dom/media/webcodecs/WebCodecsUtils.cpp | 578 +++++ dom/media/webcodecs/WebCodecsUtils.h | 239 ++ dom/media/webcodecs/crashtests/1839270.html | 13 + dom/media/webcodecs/crashtests/1848460.html | 17 + dom/media/webcodecs/crashtests/1849271.html | 27 + dom/media/webcodecs/crashtests/1864475.html | 14 + dom/media/webcodecs/crashtests/crashtests.list | 4 + dom/media/webcodecs/moz.build | 54 + dom/media/webcodecs/test/mochitest.toml | 6 + .../test/test_videoFrame_mismatched_codedSize.html | 30 + 30 files changed, 9969 insertions(+) create mode 100644 dom/media/webcodecs/DecoderAgent.cpp create mode 100644 dom/media/webcodecs/DecoderAgent.h create mode 100644 dom/media/webcodecs/DecoderTemplate.cpp create mode 100644 dom/media/webcodecs/DecoderTemplate.h create mode 100644 dom/media/webcodecs/DecoderTypes.h create mode 100644 dom/media/webcodecs/EncodedVideoChunk.cpp create mode 100644 dom/media/webcodecs/EncodedVideoChunk.h create mode 100644 dom/media/webcodecs/EncoderAgent.cpp create mode 100644 dom/media/webcodecs/EncoderAgent.h create mode 100644 dom/media/webcodecs/EncoderTemplate.cpp create mode 100644 dom/media/webcodecs/EncoderTemplate.h create mode 100644 dom/media/webcodecs/EncoderTypes.h create mode 100644 dom/media/webcodecs/VideoColorSpace.cpp create mode 100644 dom/media/webcodecs/VideoColorSpace.h create mode 100644 dom/media/webcodecs/VideoDecoder.cpp create mode 100644 dom/media/webcodecs/VideoDecoder.h create mode 100644 dom/media/webcodecs/VideoEncoder.cpp create mode 100644 dom/media/webcodecs/VideoEncoder.h create mode 100644 dom/media/webcodecs/VideoFrame.cpp create mode 100644 dom/media/webcodecs/VideoFrame.h create mode 100644 dom/media/webcodecs/WebCodecsUtils.cpp create mode 100644 dom/media/webcodecs/WebCodecsUtils.h create mode 100644 dom/media/webcodecs/crashtests/1839270.html create mode 100644 dom/media/webcodecs/crashtests/1848460.html create mode 100644 dom/media/webcodecs/crashtests/1849271.html create mode 100644 dom/media/webcodecs/crashtests/1864475.html create mode 100644 dom/media/webcodecs/crashtests/crashtests.list create mode 100644 dom/media/webcodecs/moz.build create mode 100644 dom/media/webcodecs/test/mochitest.toml create mode 100644 dom/media/webcodecs/test/test_videoFrame_mismatched_codedSize.html (limited to 'dom/media/webcodecs') diff --git a/dom/media/webcodecs/DecoderAgent.cpp b/dom/media/webcodecs/DecoderAgent.cpp new file mode 100644 index 0000000000..5c63e27d48 --- /dev/null +++ b/dom/media/webcodecs/DecoderAgent.cpp @@ -0,0 +1,491 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "DecoderAgent.h" + +#include + +#include "ImageContainer.h" +#include "MediaDataDecoderProxy.h" +#include "PDMFactory.h" +#include "VideoUtils.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/Logging.h" +#include "mozilla/layers/ImageBridgeChild.h" +#include "nsThreadUtils.h" + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOG +# undef LOG +#endif // LOG +#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +#ifdef LOGW +# undef LOGW +#endif // LOGE +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +#ifdef LOGV +# undef LOGV +#endif // LOGV +#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__) + +DecoderAgent::DecoderAgent(Id aId, UniquePtr&& aInfo) + : mId(aId), + mInfo(std::move(aInfo)), + mOwnerThread(GetCurrentSerialEventTarget()), + mPDMFactory(MakeRefPtr()), + mImageContainer(MakeAndAddRef( + layers::ImageContainer::ASYNCHRONOUS)), + mDecoder(nullptr), + mState(State::Unconfigured) { + MOZ_ASSERT(mInfo); + MOZ_ASSERT(mOwnerThread); + MOZ_ASSERT(mPDMFactory); + MOZ_ASSERT(mImageContainer); + LOG("DecoderAgent #%d (%p) ctor", mId, this); +} + +DecoderAgent::~DecoderAgent() { + LOG("DecoderAgent #%d (%p) dtor", mId, this); + MOZ_ASSERT(mState == State::Unconfigured, "decoder release in wrong state"); + MOZ_ASSERT(!mDecoder, "decoder must be shutdown"); +} + +RefPtr DecoderAgent::Configure( + bool aPreferSoftwareDecoder, bool aLowLatency) { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + MOZ_ASSERT(mState == State::Unconfigured || mState == State::Error); + MOZ_ASSERT(mConfigurePromise.IsEmpty()); + MOZ_ASSERT(!mCreateRequest.Exists()); + MOZ_ASSERT(!mInitRequest.Exists()); + + if (mState == State::Error) { + LOGE("DecoderAgent #%d (%p) tried to configure in error state", mId, this); + return ConfigurePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "Cannot configure in error state"), + __func__); + } + + MOZ_ASSERT(mState == State::Unconfigured); + MOZ_ASSERT(!mDecoder); + SetState(State::Configuring); + + RefPtr knowsCompositor = + layers::ImageBridgeChild::GetSingleton(); + // Bug 1839993: FFmpegDataDecoder ignores all decode errors when draining so + // WPT cannot receive error callbacks. Forcibly enable LowLatency for now to + // get the decoded results immediately to avoid this. + + auto params = CreateDecoderParams{ + *mInfo, + CreateDecoderParams::OptionSet( + CreateDecoderParams::Option::LowLatency, + aPreferSoftwareDecoder + ? CreateDecoderParams::Option::HardwareDecoderNotAllowed + : CreateDecoderParams::Option::Default), + mInfo->GetType(), mImageContainer, knowsCompositor}; + + LOG("DecoderAgent #%d (%p) is creating a decoder - PreferSW: %s, " + "low-latency: %syes", + mId, this, aPreferSoftwareDecoder ? "yes" : "no", + aLowLatency ? "" : "forcibly "); + + RefPtr p = mConfigurePromise.Ensure(__func__); + + mPDMFactory->CreateDecoder(params) + ->Then( + mOwnerThread, __func__, + [self = RefPtr{this}](RefPtr&& aDecoder) { + self->mCreateRequest.Complete(); + + // If DecoderAgent has been shut down, shut the created decoder down + // and return. + if (!self->mShutdownWhileCreationPromise.IsEmpty()) { + MOZ_ASSERT(self->mState == State::ShuttingDown); + MOZ_ASSERT(self->mConfigurePromise.IsEmpty(), + "configuration should have been rejected"); + + LOGW( + "DecoderAgent #%d (%p) has been shut down. We need to shut " + "the newly created decoder down", + self->mId, self.get()); + aDecoder->Shutdown()->Then( + self->mOwnerThread, __func__, + [self](const ShutdownPromise::ResolveOrRejectValue& aValue) { + MOZ_ASSERT(self->mState == State::ShuttingDown); + + LOGW( + "DecoderAgent #%d (%p), newly created decoder shutdown " + "has been %s", + self->mId, self.get(), + aValue.IsResolve() ? "resolved" : "rejected"); + + self->SetState(State::Unconfigured); + + self->mShutdownWhileCreationPromise.ResolveOrReject( + aValue, __func__); + }); + return; + } + + self->mDecoder = new MediaDataDecoderProxy( + aDecoder.forget(), + CreateMediaDecodeTaskQueue("DecoderAgent TaskQueue")); + LOG("DecoderAgent #%d (%p) has created a decoder, now initialize " + "it", + self->mId, self.get()); + self->mDecoder->Init() + ->Then( + self->mOwnerThread, __func__, + [self](const TrackInfo::TrackType aTrackType) { + self->mInitRequest.Complete(); + LOG("DecoderAgent #%d (%p) has initialized the decoder", + self->mId, self.get()); + MOZ_ASSERT(aTrackType == self->mInfo->GetType()); + self->SetState(State::Configured); + self->mConfigurePromise.Resolve(true, __func__); + }, + [self](const MediaResult& aError) { + self->mInitRequest.Complete(); + LOGE( + "DecoderAgent #%d (%p) failed to initialize the " + "decoder", + self->mId, self.get()); + self->SetState(State::Error); + self->mConfigurePromise.Reject(aError, __func__); + }) + ->Track(self->mInitRequest); + }, + [self = RefPtr{this}](const MediaResult& aError) { + self->mCreateRequest.Complete(); + LOGE("DecoderAgent #%d (%p) failed to create a decoder", self->mId, + self.get()); + + // If DecoderAgent has been shut down, we need to resolve the + // shutdown promise. + if (!self->mShutdownWhileCreationPromise.IsEmpty()) { + MOZ_ASSERT(self->mState == State::ShuttingDown); + MOZ_ASSERT(self->mConfigurePromise.IsEmpty(), + "configuration should have been rejected"); + + LOGW( + "DecoderAgent #%d (%p) has been shut down. Resolve the " + "shutdown promise right away since decoder creation failed", + self->mId, self.get()); + + self->SetState(State::Unconfigured); + self->mShutdownWhileCreationPromise.Resolve(true, __func__); + return; + } + + self->SetState(State::Error); + self->mConfigurePromise.Reject(aError, __func__); + }) + ->Track(mCreateRequest); + + return p; +} + +RefPtr DecoderAgent::Shutdown() { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + + auto r = + MediaResult(NS_ERROR_DOM_MEDIA_CANCELED, "Canceled by decoder shutdown"); + + // If the decoder creation has not been completed yet, wait until the decoder + // being created has been shut down. + if (mCreateRequest.Exists()) { + MOZ_ASSERT(!mInitRequest.Exists()); + MOZ_ASSERT(!mConfigurePromise.IsEmpty()); + MOZ_ASSERT(!mDecoder); + MOZ_ASSERT(mState == State::Configuring); + MOZ_ASSERT(mShutdownWhileCreationPromise.IsEmpty()); + + LOGW( + "DecoderAgent #%d (%p) shutdown while the decoder-creation for " + "configuration is in flight. Reject the configuration now and defer " + "the shutdown until the created decoder has been shut down", + mId, this); + + // Reject the configuration in flight. + mConfigurePromise.Reject(r, __func__); + + // Get the promise that will be resolved when the decoder being created has + // been destroyed. + SetState(State::ShuttingDown); + return mShutdownWhileCreationPromise.Ensure(__func__); + } + + // If decoder creation has been completed, we must have the decoder now. + MOZ_ASSERT(mDecoder); + + // Cancel pending initialization for configuration in flight if any. + mInitRequest.DisconnectIfExists(); + mConfigurePromise.RejectIfExists(r, __func__); + + // Cancel decode in flight if any. + mDecodeRequest.DisconnectIfExists(); + mDecodePromise.RejectIfExists(r, __func__); + + // Cancel flush-out in flight if any. + mDrainRequest.DisconnectIfExists(); + mFlushRequest.DisconnectIfExists(); + mDryRequest.DisconnectIfExists(); + mDryPromise.RejectIfExists(r, __func__); + mDrainAndFlushPromise.RejectIfExists(r, __func__); + mDryData.Clear(); + mDrainAndFlushData.Clear(); + + SetState(State::Unconfigured); + + RefPtr decoder = std::move(mDecoder); + return decoder->Shutdown(); +} + +RefPtr DecoderAgent::Decode( + MediaRawData* aSample) { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + MOZ_ASSERT(aSample); + MOZ_ASSERT(mState == State::Configured || mState == State::Error); + MOZ_ASSERT(mDecodePromise.IsEmpty()); + MOZ_ASSERT(!mDecodeRequest.Exists()); + + if (mState == State::Error) { + LOGE("DecoderAgent #%d (%p) tried to decode in error state", mId, this); + return DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "Cannot decode in error state"), + __func__); + } + + MOZ_ASSERT(mState == State::Configured); + MOZ_ASSERT(mDecoder); + SetState(State::Decoding); + + RefPtr p = mDecodePromise.Ensure(__func__); + + mDecoder->Decode(aSample) + ->Then( + mOwnerThread, __func__, + [self = RefPtr{this}](MediaDataDecoder::DecodedData&& aData) { + self->mDecodeRequest.Complete(); + LOGV("DecoderAgent #%d (%p) decode successfully", self->mId, + self.get()); + self->SetState(State::Configured); + self->mDecodePromise.Resolve(std::move(aData), __func__); + }, + [self = RefPtr{this}](const MediaResult& aError) { + self->mDecodeRequest.Complete(); + LOGV("DecoderAgent #%d (%p) failed to decode", self->mId, + self.get()); + self->SetState(State::Error); + self->mDecodePromise.Reject(aError, __func__); + }) + ->Track(mDecodeRequest); + + return p; +} + +RefPtr DecoderAgent::DrainAndFlush() { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + MOZ_ASSERT(mState == State::Configured || mState == State::Error); + MOZ_ASSERT(mDrainAndFlushPromise.IsEmpty()); + MOZ_ASSERT(mDrainAndFlushData.IsEmpty()); + MOZ_ASSERT(!mDryRequest.Exists()); + MOZ_ASSERT(mDryPromise.IsEmpty()); + MOZ_ASSERT(mDryData.IsEmpty()); + MOZ_ASSERT(!mDrainRequest.Exists()); + MOZ_ASSERT(!mFlushRequest.Exists()); + + if (mState == State::Error) { + LOGE("DecoderAgent #%d (%p) tried to flush-out in error state", mId, this); + return DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "Cannot flush in error state"), + __func__); + } + + MOZ_ASSERT(mState == State::Configured); + MOZ_ASSERT(mDecoder); + SetState(State::Flushing); + + RefPtr p = + mDrainAndFlushPromise.Ensure(__func__); + + Dry() + ->Then( + mOwnerThread, __func__, + [self = RefPtr{this}](MediaDataDecoder::DecodedData&& aData) { + self->mDryRequest.Complete(); + LOG("DecoderAgent #%d (%p) has dried the decoder. Now flushing the " + "decoder", + self->mId, self.get()); + MOZ_ASSERT(self->mDrainAndFlushData.IsEmpty()); + self->mDrainAndFlushData.AppendElements(std::move(aData)); + self->mDecoder->Flush() + ->Then( + self->mOwnerThread, __func__, + [self](const bool /* aUnUsed */) { + self->mFlushRequest.Complete(); + LOG("DecoderAgent #%d (%p) has flushed the decoder", + self->mId, self.get()); + self->SetState(State::Configured); + self->mDrainAndFlushPromise.Resolve( + std::move(self->mDrainAndFlushData), __func__); + }, + [self](const MediaResult& aError) { + self->mFlushRequest.Complete(); + LOGE("DecoderAgent #%d (%p) failed to flush the decoder", + self->mId, self.get()); + self->SetState(State::Error); + self->mDrainAndFlushData.Clear(); + self->mDrainAndFlushPromise.Reject(aError, __func__); + }) + ->Track(self->mFlushRequest); + }, + [self = RefPtr{this}](const MediaResult& aError) { + self->mDryRequest.Complete(); + LOGE("DecoderAgent #%d (%p) failed to dry the decoder", self->mId, + self.get()); + self->SetState(State::Error); + self->mDrainAndFlushPromise.Reject(aError, __func__); + }) + ->Track(mDryRequest); + + return p; +} + +RefPtr DecoderAgent::Dry() { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + MOZ_ASSERT(mState == State::Flushing); + MOZ_ASSERT(mDryPromise.IsEmpty()); + MOZ_ASSERT(!mDryRequest.Exists()); + MOZ_ASSERT(mDryData.IsEmpty()); + MOZ_ASSERT(mDecoder); + + RefPtr p = mDryPromise.Ensure(__func__); + DrainUntilDry(); + return p; +} + +void DecoderAgent::DrainUntilDry() { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + MOZ_ASSERT(mState == State::Flushing); + MOZ_ASSERT(!mDryPromise.IsEmpty()); + MOZ_ASSERT(!mDrainRequest.Exists()); + MOZ_ASSERT(mDecoder); + + LOG("DecoderAgent #%d (%p) is drainng the decoder", mId, this); + mDecoder->Drain() + ->Then( + mOwnerThread, __func__, + [self = RefPtr{this}](MediaDataDecoder::DecodedData&& aData) { + self->mDrainRequest.Complete(); + + if (aData.IsEmpty()) { + LOG("DecoderAgent #%d (%p) is dry now", self->mId, self.get()); + self->mDryPromise.Resolve(std::move(self->mDryData), __func__); + return; + } + + LOG("DecoderAgent #%d (%p) drained %zu decoded data. Keep draining " + "until dry", + self->mId, self.get(), aData.Length()); + self->mDryData.AppendElements(std::move(aData)); + self->DrainUntilDry(); + }, + [self = RefPtr{this}](const MediaResult& aError) { + self->mDrainRequest.Complete(); + + LOGE("DecoderAgent %p failed to drain decoder", self.get()); + self->mDryData.Clear(); + self->mDryPromise.Reject(aError, __func__); + }) + ->Track(mDrainRequest); +} + +void DecoderAgent::SetState(State aState) { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + + auto validateStateTransition = [](State aOldState, State aNewState) { + switch (aOldState) { + case State::Unconfigured: + return aNewState == State::Configuring; + case State::Configuring: + return aNewState == State::Configured || aNewState == State::Error || + aNewState == State::Unconfigured || + aNewState == State::ShuttingDown; + case State::Configured: + return aNewState == State::Unconfigured || + aNewState == State::Decoding || aNewState == State::Flushing; + case State::Decoding: + case State::Flushing: + return aNewState == State::Configured || aNewState == State::Error || + aNewState == State::Unconfigured; + case State::ShuttingDown: + return aNewState == State::Unconfigured; + case State::Error: + return aNewState == State::Unconfigured; + default: + break; + } + MOZ_ASSERT_UNREACHABLE("Unhandled state transition"); + return false; + }; + + auto stateToString = [](State aState) -> const char* { + switch (aState) { + case State::Unconfigured: + return "Unconfigured"; + case State::Configuring: + return "Configuring"; + case State::Configured: + return "Configured"; + case State::Decoding: + return "Decoding"; + case State::Flushing: + return "Flushing"; + case State::ShuttingDown: + return "ShuttingDown"; + case State::Error: + return "Error"; + default: + break; + } + MOZ_ASSERT_UNREACHABLE("Unhandled state type"); + return "Unknown"; + }; + + DebugOnly isValid = validateStateTransition(mState, aState); + MOZ_ASSERT(isValid); + LOG("DecoderAgent #%d (%p) state change: %s -> %s", mId, this, + stateToString(mState), stateToString(aState)); + mState = aState; +} + +#undef LOG +#undef LOGW +#undef LOGE +#undef LOGV +#undef LOG_INTERNAL + +} // namespace mozilla diff --git a/dom/media/webcodecs/DecoderAgent.h b/dom/media/webcodecs/DecoderAgent.h new file mode 100644 index 0000000000..f8107b27a6 --- /dev/null +++ b/dom/media/webcodecs/DecoderAgent.h @@ -0,0 +1,117 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_WEBCODECS_DECODERAGENT_H +#define DOM_MEDIA_WEBCODECS_DECODERAGENT_H + +#include "MediaResult.h" +#include "PlatformDecoderModule.h" +#include "mozilla/RefPtr.h" +#include "mozilla/TaskQueue.h" +#include "mozilla/UniquePtr.h" + +class nsISerialEventTarget; + +namespace mozilla { + +class PDMFactory; +class TrackInfo; + +namespace layers { +class ImageContainer; +} // namespace layers + +// DecoderAgent is a wrapper that contains a MediaDataDecoder. It adapts the +// MediaDataDecoder APIs for use in WebCodecs. +// +// If Configure() is called, Shutdown() must be called to release the resources +// gracefully. Except Shutdown(), all the methods can't be called concurrently, +// meaning a method can only be called when the previous API call has completed. +// The responsability of arranging the method calls is on the caller. +// +// When Shutdown() is called, all the operations in flight are canceled and the +// MediaDataDecoder is shut down. On the other hand, errors are final. A new +// DecoderAgent must be created when an error is encountered. +// +// All the methods need to be called on the DecoderAgent's owner thread. In +// WebCodecs, it's either on the main thread or worker thread. +class DecoderAgent final { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DecoderAgent); + + using Id = uint32_t; + DecoderAgent(Id aId, UniquePtr&& aInfo); + + // The following APIs are owner thread only. + + using ConfigurePromise = MozPromise; + RefPtr Configure(bool aPreferSoftwareDecoder, + bool aLowLatency); + RefPtr Shutdown(); + using DecodePromise = MediaDataDecoder::DecodePromise; + RefPtr Decode(MediaRawData* aSample); + // WebCodecs's flush() flushes out all the pending decoded data in the + // MediaDataDecoder so it is a combination of Drain and Flush. To distinguish + // the term from MediaDataDecoder's one, we call it DrainAndFlush() here. + RefPtr DrainAndFlush(); + + const Id mId; // A unique id. + const UniquePtr mInfo; + + private: + ~DecoderAgent(); + + // Push out all the data in the MediaDataDecoder's pipeline. + // TODO: MediaDataDecoder should implement this, instead of asking call site + // to run `Drain` multiple times. + RefPtr Dry(); + void DrainUntilDry(); + + enum class State { + Unconfigured, + Configuring, + Configured, + Decoding, + Flushing, + ShuttingDown, + Error, + }; + void SetState(State aState); + + const RefPtr mOwnerThread; + const RefPtr mPDMFactory; + const RefPtr mImageContainer; + RefPtr mDecoder; + State mState; + + // Configure + MozPromiseHolder mConfigurePromise; + using CreateDecoderPromise = PlatformDecoderModule::CreateDecoderPromise; + MozPromiseRequestHolder mCreateRequest; + using InitPromise = MediaDataDecoder::InitPromise; + MozPromiseRequestHolder mInitRequest; + + // Shutdown + MozPromiseHolder mShutdownWhileCreationPromise; + + // Decode + MozPromiseHolder mDecodePromise; + MozPromiseRequestHolder mDecodeRequest; + + // DrainAndFlush + MozPromiseHolder mDrainAndFlushPromise; + MediaDataDecoder::DecodedData mDrainAndFlushData; + MozPromiseRequestHolder mDryRequest; + MozPromiseHolder mDryPromise; + MediaDataDecoder::DecodedData mDryData; + MozPromiseRequestHolder mDrainRequest; + using FlushPromise = MediaDataDecoder::FlushPromise; + MozPromiseRequestHolder mFlushRequest; +}; + +} // namespace mozilla + +#endif // DOM_MEDIA_WEBCODECS_DECODERAGENT_H diff --git a/dom/media/webcodecs/DecoderTemplate.cpp b/dom/media/webcodecs/DecoderTemplate.cpp new file mode 100644 index 0000000000..0fa25a208b --- /dev/null +++ b/dom/media/webcodecs/DecoderTemplate.cpp @@ -0,0 +1,891 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "DecoderTemplate.h" + +#include +#include + +#include "DecoderTypes.h" +#include "MediaInfo.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/Try.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/DOMException.h" +#include "mozilla/dom/Event.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/VideoDecoderBinding.h" +#include "mozilla/dom/VideoFrame.h" +#include "mozilla/dom/WorkerCommon.h" +#include "nsGkAtoms.h" +#include "nsString.h" +#include "nsThreadUtils.h" + +mozilla::LazyLogModule gWebCodecsLog("WebCodecs"); + +namespace mozilla::dom { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOG +# undef LOG +#endif // LOG +#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +#ifdef LOGW +# undef LOGW +#endif // LOGW +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +#ifdef LOGV +# undef LOGV +#endif // LOGV +#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__) + +/* + * Below are ControlMessage classes implementations + */ + +template +DecoderTemplate::ControlMessage::ControlMessage( + const nsACString& aTitle) + : mTitle(aTitle) {} + +template +DecoderTemplate::ConfigureMessage::ConfigureMessage( + Id aId, UniquePtr&& aConfig) + : ControlMessage( + nsPrintfCString("configure #%d (%s)", aId, + NS_ConvertUTF16toUTF8(aConfig->mCodec).get())), + mId(aId), + mConfig(std::move(aConfig)) {} + +/* static */ +template +typename DecoderTemplate::ConfigureMessage* +DecoderTemplate::ConfigureMessage::Create( + UniquePtr&& aConfig) { + // This needs to be atomic since this can run on the main thread or worker + // thread. + static std::atomic sNextId = NoId; + return new ConfigureMessage(++sNextId, std::move(aConfig)); +} + +template +DecoderTemplate::DecodeMessage::DecodeMessage( + Id aId, ConfigId aConfigId, UniquePtr&& aData) + : ControlMessage( + nsPrintfCString("decode #%zu (config #%d)", aId, aConfigId)), + mId(aId), + mData(std::move(aData)) {} + +template +DecoderTemplate::FlushMessage::FlushMessage(Id aId, + ConfigId aConfigId, + Promise* aPromise) + : ControlMessage( + nsPrintfCString("flush #%zu (config #%d)", aId, aConfigId)), + mId(aId), + mPromise(aPromise) {} + +template +void DecoderTemplate::FlushMessage::RejectPromiseIfAny( + const nsresult& aReason) { + if (mPromise) { + mPromise->MaybeReject(aReason); + } +} + +/* + * Below are DecoderTemplate implementation + */ + +template +DecoderTemplate::DecoderTemplate( + nsIGlobalObject* aGlobalObject, + RefPtr&& aErrorCallback, + RefPtr&& aOutputCallback) + : DOMEventTargetHelper(aGlobalObject), + mErrorCallback(std::move(aErrorCallback)), + mOutputCallback(std::move(aOutputCallback)), + mState(CodecState::Unconfigured), + mKeyChunkRequired(true), + mMessageQueueBlocked(false), + mDecodeQueueSize(0), + mDequeueEventScheduled(false), + mLatestConfigureId(ConfigureMessage::NoId), + mDecodeCounter(0), + mFlushCounter(0) {} + +template +void DecoderTemplate::Configure(const ConfigType& aConfig, + ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s %p, Configure: codec %s", DecoderType::Name.get(), this, + NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); + + nsCString errorMessage; + if (!DecoderType::Validate(aConfig, errorMessage)) { + aRv.ThrowTypeError( + nsPrintfCString("config is invalid: %s", errorMessage.get())); + return; + } + + if (mState == CodecState::Closed) { + LOG("Configure: CodecState::Closed, rejecting with InvalidState"); + aRv.ThrowInvalidStateError("The codec is no longer usable"); + return; + } + + // Clone a ConfigType as the active decoder config. + UniquePtr config = + DecoderType::CreateConfigInternal(aConfig); + if (!config) { + aRv.Throw(NS_ERROR_UNEXPECTED); // Invalid description data. + return; + } + + mState = CodecState::Configured; + mKeyChunkRequired = true; + mDecodeCounter = 0; + mFlushCounter = 0; + + mControlMessageQueue.emplace( + UniquePtr(ConfigureMessage::Create(std::move(config)))); + mLatestConfigureId = mControlMessageQueue.back()->AsConfigureMessage()->mId; + LOG("%s %p enqueues %s", DecoderType::Name.get(), this, + mControlMessageQueue.back()->ToString().get()); + ProcessControlMessageQueue(); +} + +template +void DecoderTemplate::Decode(InputType& aInput, ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s %p, Decode", DecoderType::Name.get(), this); + + if (mState != CodecState::Configured) { + aRv.ThrowInvalidStateError("Decoder must be configured first"); + return; + } + + if (mKeyChunkRequired) { + // TODO: Verify input's data is truly a key chunk + if (!DecoderType::IsKeyChunk(aInput)) { + aRv.ThrowDataError( + nsPrintfCString("%s needs a key chunk", DecoderType::Name.get())); + return; + } + mKeyChunkRequired = false; + } + + mDecodeQueueSize += 1; + mControlMessageQueue.emplace(UniquePtr( + new DecodeMessage(++mDecodeCounter, mLatestConfigureId, + DecoderType::CreateInputInternal(aInput)))); + LOGV("%s %p enqueues %s", DecoderType::Name.get(), this, + mControlMessageQueue.back()->ToString().get()); + ProcessControlMessageQueue(); +} + +template +already_AddRefed DecoderTemplate::Flush( + ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s %p, Flush", DecoderType::Name.get(), this); + + if (mState != CodecState::Configured) { + LOG("%s %p, wrong state!", DecoderType::Name.get(), this); + aRv.ThrowInvalidStateError("Decoder must be configured first"); + return nullptr; + } + + RefPtr p = Promise::Create(GetParentObject(), aRv); + if (NS_WARN_IF(aRv.Failed())) { + return p.forget(); + } + + mKeyChunkRequired = true; + + mControlMessageQueue.emplace(UniquePtr( + new FlushMessage(++mFlushCounter, mLatestConfigureId, p))); + LOG("%s %p enqueues %s", DecoderType::Name.get(), this, + mControlMessageQueue.back()->ToString().get()); + ProcessControlMessageQueue(); + return p.forget(); +} + +template +void DecoderTemplate::Reset(ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s %p, Reset", DecoderType::Name.get(), this); + + if (auto r = ResetInternal(NS_ERROR_DOM_ABORT_ERR); r.isErr()) { + aRv.Throw(r.unwrapErr()); + } +} + +template +void DecoderTemplate::Close(ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s %p, Close", DecoderType::Name.get(), this); + + if (auto r = CloseInternalWithAbort(); r.isErr()) { + aRv.Throw(r.unwrapErr()); + } +} + +template +Result DecoderTemplate::ResetInternal( + const nsresult& aResult) { + AssertIsOnOwningThread(); + + if (mState == CodecState::Closed) { + return Err(NS_ERROR_DOM_INVALID_STATE_ERR); + } + + mState = CodecState::Unconfigured; + mDecodeCounter = 0; + mFlushCounter = 0; + + CancelPendingControlMessages(aResult); + DestroyDecoderAgentIfAny(); + + if (mDecodeQueueSize > 0) { + mDecodeQueueSize = 0; + ScheduleDequeueEventIfNeeded(); + } + + LOG("%s %p now has its message queue unblocked", DecoderType::Name.get(), + this); + mMessageQueueBlocked = false; + + return Ok(); +} +template +Result DecoderTemplate::CloseInternalWithAbort() { + AssertIsOnOwningThread(); + + MOZ_TRY(ResetInternal(NS_ERROR_DOM_ABORT_ERR)); + mState = CodecState::Closed; + return Ok(); +} + +template +void DecoderTemplate::CloseInternal(const nsresult& aResult) { + AssertIsOnOwningThread(); + MOZ_ASSERT(aResult != NS_ERROR_DOM_ABORT_ERR, "Use CloseInternalWithAbort"); + + auto r = ResetInternal(aResult); + if (r.isErr()) { + nsCString name; + GetErrorName(r.unwrapErr(), name); + LOGE("Error in ResetInternal: %s", name.get()); + MOZ_CRASH(); + } + mState = CodecState::Closed; + nsCString error; + GetErrorName(aResult, error); + LOGE("%s %p Close on error: %s", DecoderType::Name.get(), this, error.get()); + ReportError(aResult); +} + +template +void DecoderTemplate::ReportError(const nsresult& aResult) { + AssertIsOnOwningThread(); + + RefPtr e = DOMException::Create(aResult); + RefPtr cb(mErrorCallback); + cb->Call(*e); +} + +template +void DecoderTemplate::OutputDecodedData( + const nsTArray>&& aData) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(mActiveConfig); + + nsTArray> frames = DecodedDataToOutputType( + GetParentObject(), std::move(aData), *mActiveConfig); + RefPtr cb(mOutputCallback); + for (RefPtr& frame : frames) { + LOG("Outputing decoded data: ts: %" PRId64, frame->Timestamp()); + RefPtr f = frame; + cb->Call((VideoFrame&)(*f)); + } +} + +template +void DecoderTemplate::ScheduleDequeueEventIfNeeded() { + AssertIsOnOwningThread(); + + if (mDequeueEventScheduled) { + return; + } + mDequeueEventScheduled = true; + + QueueATask("dequeue event task", [self = RefPtr{this}]() { + self->FireEvent(nsGkAtoms::ondequeue, u"dequeue"_ns); + self->mDequeueEventScheduled = false; + }); +} + +template +nsresult DecoderTemplate::FireEvent(nsAtom* aTypeWithOn, + const nsAString& aEventType) { + if (aTypeWithOn && !HasListenersFor(aTypeWithOn)) { + LOGV("%s %p has no %s event listener", DecoderType::Name.get(), this, + NS_ConvertUTF16toUTF8(aEventType).get()); + return NS_ERROR_ABORT; + } + + LOGV("Dispatch %s event to %s %p", NS_ConvertUTF16toUTF8(aEventType).get(), + DecoderType::Name.get(), this); + RefPtr event = new Event(this, nullptr, nullptr); + event->InitEvent(aEventType, true, true); + event->SetTrusted(true); + this->DispatchEvent(*event); + return NS_OK; +} + +template +void DecoderTemplate::ProcessControlMessageQueue() { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + + while (!mMessageQueueBlocked && !mControlMessageQueue.empty()) { + UniquePtr& msg = mControlMessageQueue.front(); + if (msg->AsConfigureMessage()) { + if (ProcessConfigureMessage(msg) == + MessageProcessedResult::NotProcessed) { + break; + } + } else if (msg->AsDecodeMessage()) { + if (ProcessDecodeMessage(msg) == MessageProcessedResult::NotProcessed) { + break; + } + } else { + MOZ_ASSERT(msg->AsFlushMessage()); + if (ProcessFlushMessage(msg) == MessageProcessedResult::NotProcessed) { + break; + } + } + } +} + +template +void DecoderTemplate::CancelPendingControlMessages( + const nsresult& aResult) { + AssertIsOnOwningThread(); + + // Cancel the message that is being processed. + if (mProcessingMessage) { + LOG("%s %p cancels current %s", DecoderType::Name.get(), this, + mProcessingMessage->ToString().get()); + mProcessingMessage->Cancel(); + + if (FlushMessage* flush = mProcessingMessage->AsFlushMessage()) { + flush->RejectPromiseIfAny(aResult); + } + + mProcessingMessage.reset(); + } + + // Clear the message queue. + while (!mControlMessageQueue.empty()) { + LOG("%s %p cancels pending %s", DecoderType::Name.get(), this, + mControlMessageQueue.front()->ToString().get()); + + MOZ_ASSERT(!mControlMessageQueue.front()->IsProcessing()); + if (FlushMessage* flush = mControlMessageQueue.front()->AsFlushMessage()) { + flush->RejectPromiseIfAny(aResult); + } + + mControlMessageQueue.pop(); + } +} + +template +template +void DecoderTemplate::QueueATask(const char* aName, + Func&& aSteps) { + AssertIsOnOwningThread(); + MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread( + NS_NewRunnableFunction(aName, std::forward(aSteps)))); +} + +template +MessageProcessedResult DecoderTemplate::ProcessConfigureMessage( + UniquePtr& aMessage) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(aMessage->AsConfigureMessage()); + + if (mProcessingMessage) { + LOG("%s %p is processing %s. Defer %s", DecoderType::Name.get(), this, + mProcessingMessage->ToString().get(), aMessage->ToString().get()); + return MessageProcessedResult::NotProcessed; + } + + mProcessingMessage.reset(aMessage.release()); + mControlMessageQueue.pop(); + + ConfigureMessage* msg = mProcessingMessage->AsConfigureMessage(); + LOG("%s %p starts processing %s", DecoderType::Name.get(), this, + msg->ToString().get()); + + DestroyDecoderAgentIfAny(); + + mMessageQueueBlocked = true; + + nsAutoCString errorMessage; + auto i = DecoderType::CreateTrackInfo(msg->Config()); + if (i.isErr()) { + nsCString res; + GetErrorName(i.unwrapErr(), res); + errorMessage.AppendPrintf("CreateTrackInfo failed: %s", res.get()); + } else if (!DecoderType::IsSupported(msg->Config())) { + errorMessage.Append("Not supported."); + } else if (!CreateDecoderAgent(msg->mId, msg->TakeConfig(), i.unwrap())) { + errorMessage.Append("DecoderAgent creation failed."); + } + if (!errorMessage.IsEmpty()) { + LOGE("%s %p ProcessConfigureMessage error (sync): %s", + DecoderType::Name.get(), this, errorMessage.get()); + + mProcessingMessage.reset(); + QueueATask("Error while configuring decoder", + [self = RefPtr{this}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { + MOZ_ASSERT(self->mState != CodecState::Closed); + self->CloseInternal(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + }); + return MessageProcessedResult::Processed; + } + + MOZ_ASSERT(mAgent); + MOZ_ASSERT(mActiveConfig); + + LOG("%s %p now blocks message-queue-processing", DecoderType::Name.get(), + this); + + bool preferSW = mActiveConfig->mHardwareAcceleration == + HardwareAcceleration::Prefer_software; + bool lowLatency = mActiveConfig->mOptimizeForLatency.isSome() && + mActiveConfig->mOptimizeForLatency.value(); + mAgent->Configure(preferSW, lowLatency) + ->Then(GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = mAgent->mId]( + const DecoderAgent::ConfigurePromise::ResolveOrRejectValue& + aResult) { + MOZ_ASSERT(self->mProcessingMessage); + MOZ_ASSERT(self->mProcessingMessage->AsConfigureMessage()); + MOZ_ASSERT(self->mState == CodecState::Configured); + MOZ_ASSERT(self->mAgent); + MOZ_ASSERT(id == self->mAgent->mId); + MOZ_ASSERT(self->mActiveConfig); + + ConfigureMessage* msg = + self->mProcessingMessage->AsConfigureMessage(); + LOG("%s %p, DecoderAgent #%d %s has been %s. now unblocks " + "message-queue-processing", + DecoderType::Name.get(), self.get(), id, + msg->ToString().get(), + aResult.IsResolve() ? "resolved" : "rejected"); + + msg->Complete(); + self->mProcessingMessage.reset(); + + if (aResult.IsReject()) { + // The spec asks to close the decoder with an + // NotSupportedError so we log the exact error here. + const MediaResult& error = aResult.RejectValue(); + LOGE("%s %p, DecoderAgent #%d failed to configure: %s", + DecoderType::Name.get(), self.get(), id, + error.Description().get()); + + self->QueueATask( + "Error during configure", + [self = RefPtr{self}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { + MOZ_ASSERT(self->mState != CodecState::Closed); + self->CloseInternal( + NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + }); + return; + } + + self->mMessageQueueBlocked = false; + self->ProcessControlMessageQueue(); + }) + ->Track(msg->Request()); + + return MessageProcessedResult::Processed; +} + +template +MessageProcessedResult DecoderTemplate::ProcessDecodeMessage( + UniquePtr& aMessage) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(aMessage->AsDecodeMessage()); + + if (mProcessingMessage) { + LOGV("%s %p is processing %s. Defer %s", DecoderType::Name.get(), this, + mProcessingMessage->ToString().get(), aMessage->ToString().get()); + return MessageProcessedResult::NotProcessed; + } + + mProcessingMessage.reset(aMessage.release()); + mControlMessageQueue.pop(); + + DecodeMessage* msg = mProcessingMessage->AsDecodeMessage(); + LOGV("%s %p starts processing %s", DecoderType::Name.get(), this, + msg->ToString().get()); + + mDecodeQueueSize -= 1; + ScheduleDequeueEventIfNeeded(); + + // Treat it like decode error if no DecoderAgent is available or the encoded + // data is invalid. + auto closeOnError = [&]() { + mProcessingMessage.reset(); + QueueATask("Error during decode", + [self = RefPtr{this}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { + MOZ_ASSERT(self->mState != CodecState::Closed); + self->CloseInternal(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + }); + return MessageProcessedResult::Processed; + }; + + if (!mAgent) { + LOGE("%s %p is not configured", DecoderType::Name.get(), this); + return closeOnError(); + } + + MOZ_ASSERT(mActiveConfig); + RefPtr data = InputDataToMediaRawData( + std::move(msg->mData), *(mAgent->mInfo), *mActiveConfig); + if (!data) { + LOGE("%s %p, data for %s is empty or invalid", DecoderType::Name.get(), + this, msg->ToString().get()); + return closeOnError(); + } + + mAgent->Decode(data.get()) + ->Then( + GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = mAgent->mId]( + DecoderAgent::DecodePromise::ResolveOrRejectValue&& aResult) { + MOZ_ASSERT(self->mProcessingMessage); + MOZ_ASSERT(self->mProcessingMessage->AsDecodeMessage()); + MOZ_ASSERT(self->mState == CodecState::Configured); + MOZ_ASSERT(self->mAgent); + MOZ_ASSERT(id == self->mAgent->mId); + MOZ_ASSERT(self->mActiveConfig); + + DecodeMessage* msg = self->mProcessingMessage->AsDecodeMessage(); + LOGV("%s %p, DecoderAgent #%d %s has been %s", + DecoderType::Name.get(), self.get(), id, msg->ToString().get(), + aResult.IsResolve() ? "resolved" : "rejected"); + + nsCString msgStr = msg->ToString(); + + msg->Complete(); + self->mProcessingMessage.reset(); + + if (aResult.IsReject()) { + // The spec asks to queue a task to run close the decoder + // with an EncodingError so we log the exact error here. + const MediaResult& error = aResult.RejectValue(); + LOGE("%s %p, DecoderAgent #%d %s failed: %s", + DecoderType::Name.get(), self.get(), id, msgStr.get(), + error.Description().get()); + self->QueueATask( + "Error during decode runnable", + [self = RefPtr{self}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { + MOZ_ASSERT(self->mState != CodecState::Closed); + self->CloseInternal( + NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + }); + return; + } + + MOZ_ASSERT(aResult.IsResolve()); + nsTArray> data = + std::move(aResult.ResolveValue()); + if (data.IsEmpty()) { + LOGV("%s %p got no data for %s", DecoderType::Name.get(), + self.get(), msgStr.get()); + } else { + LOGV("%s %p, schedule %zu decoded data output for %s", + DecoderType::Name.get(), self.get(), data.Length(), + msgStr.get()); + self->QueueATask("Output Decoded Data", + [self = RefPtr{self}, data = std::move(data)]() + MOZ_CAN_RUN_SCRIPT_BOUNDARY { + self->OutputDecodedData(std::move(data)); + }); + } + self->ProcessControlMessageQueue(); + }) + ->Track(msg->Request()); + + return MessageProcessedResult::Processed; +} + +template +MessageProcessedResult DecoderTemplate::ProcessFlushMessage( + UniquePtr& aMessage) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(aMessage->AsFlushMessage()); + + if (mProcessingMessage) { + LOG("%s %p is processing %s. Defer %s", DecoderType::Name.get(), this, + mProcessingMessage->ToString().get(), aMessage->ToString().get()); + return MessageProcessedResult::NotProcessed; + } + + mProcessingMessage.reset(aMessage.release()); + mControlMessageQueue.pop(); + + FlushMessage* msg = mProcessingMessage->AsFlushMessage(); + LOG("%s %p starts processing %s", DecoderType::Name.get(), this, + msg->ToString().get()); + + // No agent, no thing to do. The promise has been rejected with the + // appropriate error in ResetInternal already. + if (!mAgent) { + LOGE("%s %p no agent, nothing to do", DecoderType::Name.get(), this); + mProcessingMessage.reset(); + return MessageProcessedResult::Processed; + } + + mAgent->DrainAndFlush() + ->Then( + GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = mAgent->mId, + this](DecoderAgent::DecodePromise::ResolveOrRejectValue&& aResult) { + MOZ_ASSERT(self->mProcessingMessage); + MOZ_ASSERT(self->mProcessingMessage->AsFlushMessage()); + MOZ_ASSERT(self->mState == CodecState::Configured); + MOZ_ASSERT(self->mAgent); + MOZ_ASSERT(id == self->mAgent->mId); + MOZ_ASSERT(self->mActiveConfig); + + FlushMessage* msg = self->mProcessingMessage->AsFlushMessage(); + LOG("%s %p, DecoderAgent #%d %s has been %s", + DecoderType::Name.get(), self.get(), id, msg->ToString().get(), + aResult.IsResolve() ? "resolved" : "rejected"); + + nsCString msgStr = msg->ToString(); + + msg->Complete(); + + // If flush failed, it means decoder fails to decode the data + // sent before, so we treat it like decode error. We reject + // the promise first and then queue a task to close + // VideoDecoder with an EncodingError. + if (aResult.IsReject()) { + const MediaResult& error = aResult.RejectValue(); + LOGE("%s %p, DecoderAgent #%d failed to flush: %s", + DecoderType::Name.get(), self.get(), id, + error.Description().get()); + RefPtr promise = msg->TakePromise(); + // Reject with an EncodingError instead of the error we got + // above. + self->QueueATask( + "Error during flush runnable", + [self = RefPtr{this}, promise]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { + promise->MaybeReject( + NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + self->mProcessingMessage.reset(); + MOZ_ASSERT(self->mState != CodecState::Closed); + self->CloseInternal( + NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + }); + return; + } + + nsTArray> data = + std::move(aResult.ResolveValue()); + + if (data.IsEmpty()) { + LOG("%s %p gets no data for %s", DecoderType::Name.get(), + self.get(), msgStr.get()); + } else { + LOG("%s %p, schedule %zu decoded data output for %s", + DecoderType::Name.get(), self.get(), data.Length(), + msgStr.get()); + } + + RefPtr promise = msg->TakePromise(); + self->QueueATask( + "Flush: output decoding data task", + [self = RefPtr{self}, promise, data = std::move(data)]() + MOZ_CAN_RUN_SCRIPT_BOUNDARY { + self->OutputDecodedData(std::move(data)); + promise->MaybeResolveWithUndefined(); + }); + self->mProcessingMessage.reset(); + self->ProcessControlMessageQueue(); + }) + ->Track(msg->Request()); + + return MessageProcessedResult::Processed; +} + +// CreateDecoderAgent will create an DecoderAgent paired with a xpcom-shutdown +// blocker and a worker-reference. Besides the needs mentioned in the header +// file, the blocker and the worker-reference also provides an entry point for +// us to clean up the resources. Other than the decoder dtor, Reset(), or +// Close(), the resources should be cleaned up in the following situations: +// 1. Decoder on window, closing document +// 2. Decoder on worker, closing document +// 3. Decoder on worker, terminating worker +// +// In case 1, the entry point to clean up is in the mShutdownBlocker's +// ShutdownpPomise-resolver. In case 2, the entry point is in mWorkerRef's +// shutting down callback. In case 3, the entry point is in mWorkerRef's +// shutting down callback. + +template +bool DecoderTemplate::CreateDecoderAgent( + DecoderAgent::Id aId, UniquePtr&& aConfig, + UniquePtr&& aInfo) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(!mAgent); + MOZ_ASSERT(!mActiveConfig); + MOZ_ASSERT(!mShutdownBlocker); + MOZ_ASSERT_IF(!NS_IsMainThread(), !mWorkerRef); + + auto resetOnFailure = MakeScopeExit([&]() { + mAgent = nullptr; + mActiveConfig = nullptr; + mShutdownBlocker = nullptr; + mWorkerRef = nullptr; + }); + + // If the decoder is on worker, get a worker reference. + if (!NS_IsMainThread()) { + WorkerPrivate* workerPrivate = GetCurrentThreadWorkerPrivate(); + if (NS_WARN_IF(!workerPrivate)) { + return false; + } + + // Clean up all the resources when worker is going away. + RefPtr workerRef = StrongWorkerRef::Create( + workerPrivate, "DecoderTemplate::CreateDecoderAgent", + [self = RefPtr{this}]() { + LOG("%s %p, worker is going away", DecoderType::Name.get(), + self.get()); + Unused << self->ResetInternal(NS_ERROR_DOM_ABORT_ERR); + }); + if (NS_WARN_IF(!workerRef)) { + return false; + } + + mWorkerRef = new ThreadSafeWorkerRef(workerRef); + } + + mAgent = MakeRefPtr(aId, std::move(aInfo)); + mActiveConfig = std::move(aConfig); + + // ShutdownBlockingTicket requires an unique name to register its own + // nsIAsyncShutdownBlocker since each blocker needs a distinct name. + // To do that, we use DecoderAgent's unique id to create a unique name. + nsAutoString uniqueName; + uniqueName.AppendPrintf( + "Blocker for DecoderAgent #%d (codec: %s) @ %p", mAgent->mId, + NS_ConvertUTF16toUTF8(mActiveConfig->mCodec).get(), mAgent.get()); + + mShutdownBlocker = media::ShutdownBlockingTicket::Create( + uniqueName, NS_LITERAL_STRING_FROM_CSTRING(__FILE__), __LINE__); + if (!mShutdownBlocker) { + LOGE("%s %p failed to create %s", DecoderType::Name.get(), this, + NS_ConvertUTF16toUTF8(uniqueName).get()); + return false; + } + + // Clean up all the resources when xpcom-will-shutdown arrives since the page + // is going to be closed. + mShutdownBlocker->ShutdownPromise()->Then( + GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = mAgent->mId, + ref = mWorkerRef](bool /* aUnUsed*/) { + LOG("%s %p gets xpcom-will-shutdown notification for DecoderAgent #%d", + DecoderType::Name.get(), self.get(), id); + Unused << self->ResetInternal(NS_ERROR_DOM_ABORT_ERR); + }, + [self = RefPtr{this}, id = mAgent->mId, + ref = mWorkerRef](bool /* aUnUsed*/) { + LOG("%s %p removes shutdown-blocker #%d before getting any " + "notification. DecoderAgent #%d should have been dropped", + DecoderType::Name.get(), self.get(), id, id); + MOZ_ASSERT(!self->mAgent || self->mAgent->mId != id); + }); + + LOG("%s %p creates DecoderAgent #%d @ %p and its shutdown-blocker", + DecoderType::Name.get(), this, mAgent->mId, mAgent.get()); + + resetOnFailure.release(); + return true; +} + +template +void DecoderTemplate::DestroyDecoderAgentIfAny() { + AssertIsOnOwningThread(); + + if (!mAgent) { + LOG("%s %p has no DecoderAgent to destroy", DecoderType::Name.get(), this); + return; + } + + MOZ_ASSERT(mActiveConfig); + MOZ_ASSERT(mShutdownBlocker); + MOZ_ASSERT_IF(!NS_IsMainThread(), mWorkerRef); + + LOG("%s %p destroys DecoderAgent #%d @ %p", DecoderType::Name.get(), this, + mAgent->mId, mAgent.get()); + mActiveConfig = nullptr; + RefPtr agent = std::move(mAgent); + // mShutdownBlocker should be kept alive until the shutdown is done. + // mWorkerRef is used to ensure this task won't be discarded in worker. + agent->Shutdown()->Then( + GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = agent->mId, ref = std::move(mWorkerRef), + blocker = std::move(mShutdownBlocker)]( + const ShutdownPromise::ResolveOrRejectValue& aResult) { + LOG("%s %p, DecoderAgent #%d's shutdown has been %s. Drop its " + "shutdown-blocker now", + DecoderType::Name.get(), self.get(), id, + aResult.IsResolve() ? "resolved" : "rejected"); + }); +} + +template class DecoderTemplate; + +#undef LOG +#undef LOGW +#undef LOGE +#undef LOGV +#undef LOG_INTERNAL + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/DecoderTemplate.h b/dom/media/webcodecs/DecoderTemplate.h new file mode 100644 index 0000000000..fe0cb5baee --- /dev/null +++ b/dom/media/webcodecs/DecoderTemplate.h @@ -0,0 +1,260 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_DecoderTemplate_h +#define mozilla_dom_DecoderTemplate_h + +#include + +#include "mozilla/DOMEventTargetHelper.h" +#include "mozilla/DecoderAgent.h" +#include "mozilla/MozPromise.h" +#include "mozilla/RefPtr.h" +#include "mozilla/Result.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/dom/WorkerRef.h" +#include "mozilla/media/MediaUtils.h" +#include "nsStringFwd.h" +#include "WebCodecsUtils.h" + +namespace mozilla { + +class TrackInfo; + +namespace dom { + +class WebCodecsErrorCallback; +class Promise; +enum class CodecState : uint8_t; + +template +class DecoderTemplate : public DOMEventTargetHelper { + using Self = DecoderTemplate; + using ConfigType = typename DecoderType::ConfigType; + using ConfigTypeInternal = typename DecoderType::ConfigTypeInternal; + using InputType = typename DecoderType::InputType; + using InputTypeInternal = typename DecoderType::InputTypeInternal; + using OutputType = typename DecoderType::OutputType; + using OutputCallbackType = typename DecoderType::OutputCallbackType; + + /* ControlMessage classes */ + protected: + class ConfigureMessage; + class DecodeMessage; + class FlushMessage; + + class ControlMessage { + public: + explicit ControlMessage(const nsACString& aTitle); + virtual ~ControlMessage() = default; + virtual void Cancel() = 0; + virtual bool IsProcessing() = 0; + + virtual const nsCString& ToString() const { return mTitle; } + virtual ConfigureMessage* AsConfigureMessage() { return nullptr; } + virtual DecodeMessage* AsDecodeMessage() { return nullptr; } + virtual FlushMessage* AsFlushMessage() { return nullptr; } + + const nsCString mTitle; // Used to identify the message in the logs. + }; + + class ConfigureMessage final + : public ControlMessage, + public MessageRequestHolder { + public: + using Id = DecoderAgent::Id; + static constexpr Id NoId = 0; + static ConfigureMessage* Create(UniquePtr&& aConfig); + + ~ConfigureMessage() = default; + virtual void Cancel() override { Disconnect(); } + virtual bool IsProcessing() override { return Exists(); }; + virtual ConfigureMessage* AsConfigureMessage() override { return this; } + const ConfigTypeInternal& Config() { return *mConfig; } + UniquePtr TakeConfig() { return std::move(mConfig); } + + const Id mId; // A unique id shown in log. + + private: + ConfigureMessage(Id aId, UniquePtr&& aConfig); + + UniquePtr mConfig; + }; + + class DecodeMessage final + : public ControlMessage, + public MessageRequestHolder { + public: + using Id = size_t; + using ConfigId = typename Self::ConfigureMessage::Id; + DecodeMessage(Id aId, ConfigId aConfigId, + UniquePtr&& aData); + ~DecodeMessage() = default; + virtual void Cancel() override { Disconnect(); } + virtual bool IsProcessing() override { return Exists(); }; + virtual DecodeMessage* AsDecodeMessage() override { return this; } + const Id mId; // A unique id shown in log. + + UniquePtr mData; + }; + + class FlushMessage final + : public ControlMessage, + public MessageRequestHolder { + public: + using Id = size_t; + using ConfigId = typename Self::ConfigureMessage::Id; + FlushMessage(Id aId, ConfigId aConfigId, Promise* aPromise); + ~FlushMessage() = default; + virtual void Cancel() override { Disconnect(); } + virtual bool IsProcessing() override { return Exists(); }; + virtual FlushMessage* AsFlushMessage() override { return this; } + already_AddRefed TakePromise() { return mPromise.forget(); } + void RejectPromiseIfAny(const nsresult& aReason); + + const Id mId; // A unique id shown in log. + + private: + RefPtr mPromise; + }; + + protected: + DecoderTemplate(nsIGlobalObject* aGlobalObject, + RefPtr&& aErrorCallback, + RefPtr&& aOutputCallback); + + virtual ~DecoderTemplate() = default; + + /* WebCodecs interfaces */ + public: + IMPL_EVENT_HANDLER(dequeue) + + CodecState State() const { return mState; }; + + uint32_t DecodeQueueSize() const { return mDecodeQueueSize; }; + + void Configure(const ConfigType& aConfig, ErrorResult& aRv); + + void Decode(InputType& aInput, ErrorResult& aRv); + + already_AddRefed Flush(ErrorResult& aRv); + + void Reset(ErrorResult& aRv); + + void Close(ErrorResult& aRv); + + /* Type conversion functions for the Decoder implementation */ + protected: + virtual already_AddRefed InputDataToMediaRawData( + UniquePtr&& aData, TrackInfo& aInfo, + const ConfigTypeInternal& aConfig) = 0; + virtual nsTArray> DecodedDataToOutputType( + nsIGlobalObject* aGlobalObject, const nsTArray>&& aData, + ConfigTypeInternal& aConfig) = 0; + + protected: + // DecoderTemplate can run on either main thread or worker thread. + void AssertIsOnOwningThread() const { + NS_ASSERT_OWNINGTHREAD(DecoderTemplate); + } + + Result ResetInternal(const nsresult& aResult); + // Calling this method calls the error callback synchronously. + MOZ_CAN_RUN_SCRIPT + void CloseInternal(const nsresult& aResult); + // Calling this method doesn't call the error calback. + Result CloseInternalWithAbort(); + + MOZ_CAN_RUN_SCRIPT void ReportError(const nsresult& aResult); + MOZ_CAN_RUN_SCRIPT void OutputDecodedData( + const nsTArray>&& aData); + + void ScheduleDequeueEventIfNeeded(); + nsresult FireEvent(nsAtom* aTypeWithOn, const nsAString& aEventType); + + void ProcessControlMessageQueue(); + void CancelPendingControlMessages(const nsresult& aResult); + + // Queue a task to the control thread. This is to be used when a task needs to + // perform multiple steps. + template + void QueueATask(const char* aName, Func&& aSteps); + + MessageProcessedResult ProcessConfigureMessage( + UniquePtr& aMessage); + + MessageProcessedResult ProcessDecodeMessage( + UniquePtr& aMessage); + + MessageProcessedResult ProcessFlushMessage( + UniquePtr& aMessage); + + // Returns true when mAgent can be created. + bool CreateDecoderAgent(DecoderAgent::Id aId, + UniquePtr&& aConfig, + UniquePtr&& aInfo); + void DestroyDecoderAgentIfAny(); + + // Constant in practice, only set in ctor. + RefPtr mErrorCallback; + RefPtr mOutputCallback; + + CodecState mState; + bool mKeyChunkRequired; + + bool mMessageQueueBlocked; + std::queue> mControlMessageQueue; + UniquePtr mProcessingMessage; + + uint32_t mDecodeQueueSize; + bool mDequeueEventScheduled; + + // A unique id tracking the ConfigureMessage and will be used as the + // DecoderAgent's Id. + uint32_t mLatestConfigureId; + // Tracking how many decode data has been enqueued and this number will be + // used as the DecodeMessage's Id. + size_t mDecodeCounter; + // Tracking how many flush request has been enqueued and this number will be + // used as the FlushMessage's Id. + size_t mFlushCounter; + + // DecoderAgent will be created every time "configure" is being processed, and + // will be destroyed when "reset" or another "configure" is called (spec + // allows calling two "configure" without a "reset" in between). + RefPtr mAgent; + UniquePtr mActiveConfig; + + // Used to add a nsIAsyncShutdownBlocker on main thread to block + // xpcom-shutdown before the underlying MediaDataDecoder is created. The + // blocker will be held until the underlying MediaDataDecoder has been shut + // down. This blocker guarantees RemoteDecoderManagerChild's thread, where the + // underlying RemoteMediaDataDecoder is on, outlives the + // RemoteMediaDataDecoder, since the thread releasing, which happens on main + // thread when getting a xpcom-shutdown signal, is blocked by the added + // blocker. As a result, RemoteMediaDataDecoder can safely work on worker + // thread with a holding blocker (otherwise, if RemoteDecoderManagerChild + // releases its thread on main thread before RemoteMediaDataDecoder's + // Shutdown() task run on worker thread, RemoteMediaDataDecoder has no thread + // to run). + UniquePtr mShutdownBlocker; + + // Held to make sure the dispatched tasks can be done before worker is going + // away. As long as this worker-ref is held somewhere, the tasks dispatched to + // the worker can be executed (otherwise the tasks would be canceled). This + // ref should be activated as long as the underlying MediaDataDecoder is + // alive, and should keep alive until mShutdownBlocker is dropped, so all + // MediaDataDecoder's tasks and mShutdownBlocker-releasing task can be + // executed. + // TODO: Use StrongWorkerRef instead if this is always used in the same + // thread? + RefPtr mWorkerRef; +}; + +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_DecoderTemplate_h diff --git a/dom/media/webcodecs/DecoderTypes.h b/dom/media/webcodecs/DecoderTypes.h new file mode 100644 index 0000000000..56aa82046f --- /dev/null +++ b/dom/media/webcodecs/DecoderTypes.h @@ -0,0 +1,117 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_DecoderTypes_h +#define mozilla_dom_DecoderTypes_h + +#include "MediaData.h" +#include "mozilla/Maybe.h" +#include "mozilla/dom/EncodedVideoChunk.h" +#include "mozilla/dom/VideoColorSpaceBinding.h" +#include "mozilla/dom/VideoDecoderBinding.h" +#include "mozilla/dom/VideoFrame.h" +#include "nsStringFwd.h" +#include "nsTLiteralString.h" + +namespace mozilla { + +class TrackInfo; +class MediaByteBuffer; + +namespace dom { + +struct VideoColorSpaceInternal { + explicit VideoColorSpaceInternal(const VideoColorSpaceInit& aColorSpaceInit); + VideoColorSpaceInternal() = default; + VideoColorSpaceInit ToColorSpaceInit() const; + + bool operator==(const VideoColorSpaceInternal& aOther) const { + return mFullRange == aOther.mFullRange && mMatrix == aOther.mMatrix && + mPrimaries == aOther.mPrimaries && mTransfer == aOther.mTransfer; + } + + Maybe mFullRange; + Maybe mMatrix; + Maybe mPrimaries; + Maybe mTransfer; +}; + +class VideoDecoderConfigInternal { + public: + static UniquePtr Create( + const VideoDecoderConfig& aConfig); + VideoDecoderConfigInternal(const nsAString& aCodec, + Maybe&& aCodedHeight, + Maybe&& aCodedWidth, + Maybe&& aColorSpace, + Maybe>&& aDescription, + Maybe&& aDisplayAspectHeight, + Maybe&& aDisplayAspectWidth, + const HardwareAcceleration& aHardwareAcceleration, + Maybe&& aOptimizeForLatency); + ~VideoDecoderConfigInternal() = default; + + nsString ToString() const; + + bool Equals(const VideoDecoderConfigInternal& aOther) const { + if (mDescription.isSome() != aOther.mDescription.isSome()) { + return false; + } + if (mDescription.isSome() && aOther.mDescription.isSome()) { + auto lhs = mDescription.value(); + auto rhs = aOther.mDescription.value(); + if (lhs->Length() != rhs->Length()) { + return false; + } + if (!ArrayEqual(lhs->Elements(), rhs->Elements(), lhs->Length())) { + return false; + } + } + return mCodec.Equals(aOther.mCodec) && + mCodedHeight == aOther.mCodedHeight && + mCodedWidth == aOther.mCodedWidth && + mColorSpace == aOther.mColorSpace && + mDisplayAspectHeight == aOther.mDisplayAspectWidth && + mHardwareAcceleration == aOther.mHardwareAcceleration && + mOptimizeForLatency == aOther.mOptimizeForLatency; + } + + nsString mCodec; + Maybe mCodedHeight; + Maybe mCodedWidth; + Maybe mColorSpace; + Maybe> mDescription; + Maybe mDisplayAspectHeight; + Maybe mDisplayAspectWidth; + HardwareAcceleration mHardwareAcceleration; + Maybe mOptimizeForLatency; +}; + +class VideoDecoderTraits { + public: + static constexpr nsLiteralCString Name = "VideoDecoder"_ns; + using ConfigType = VideoDecoderConfig; + using ConfigTypeInternal = VideoDecoderConfigInternal; + using InputType = EncodedVideoChunk; + using InputTypeInternal = EncodedVideoChunkData; + using OutputType = VideoFrame; + using OutputCallbackType = VideoFrameOutputCallback; + + static bool IsSupported(const ConfigTypeInternal& aConfig); + static Result, nsresult> CreateTrackInfo( + const ConfigTypeInternal& aConfig); + static bool Validate(const ConfigType& aConfig, nsCString& aErrorMessage); + static UniquePtr CreateConfigInternal( + const ConfigType& aConfig); + static bool IsKeyChunk(const InputType& aInput); + static UniquePtr CreateInputInternal( + const InputType& aInput); +}; + +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_DecoderTypes_h diff --git a/dom/media/webcodecs/EncodedVideoChunk.cpp b/dom/media/webcodecs/EncodedVideoChunk.cpp new file mode 100644 index 0000000000..c231740d05 --- /dev/null +++ b/dom/media/webcodecs/EncodedVideoChunk.cpp @@ -0,0 +1,261 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/EncodedVideoChunk.h" +#include +#include "mozilla/dom/EncodedVideoChunkBinding.h" + +#include "MediaData.h" +#include "TimeUnits.h" +#include "mozilla/CheckedInt.h" +#include "mozilla/Logging.h" +#include "mozilla/PodOperations.h" +#include "mozilla/dom/StructuredCloneHolder.h" +#include "mozilla/dom/StructuredCloneTags.h" +#include "mozilla/dom/WebCodecsUtils.h" + +extern mozilla::LazyLogModule gWebCodecsLog; +using mozilla::media::TimeUnit; + +namespace mozilla::dom { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOGW +# undef LOGW +#endif // LOGW +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +// Only needed for refcounted objects. +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(EncodedVideoChunk, mParent) +NS_IMPL_CYCLE_COLLECTING_ADDREF(EncodedVideoChunk) +NS_IMPL_CYCLE_COLLECTING_RELEASE(EncodedVideoChunk) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(EncodedVideoChunk) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +EncodedVideoChunkData::EncodedVideoChunkData( + already_AddRefed aBuffer, + const EncodedVideoChunkType& aType, int64_t aTimestamp, + Maybe&& aDuration) + : mBuffer(aBuffer), + mType(aType), + mTimestamp(aTimestamp), + mDuration(aDuration) { + MOZ_ASSERT(mBuffer); + MOZ_ASSERT(mBuffer->Length() == mBuffer->Size()); + MOZ_ASSERT(mBuffer->Length() <= + static_cast(std::numeric_limits::max())); +} + +EncodedVideoChunkData::~EncodedVideoChunkData() = default; + +UniquePtr EncodedVideoChunkData::Clone() const { + if (!mBuffer) { + LOGE("No buffer in EncodedVideoChunkData %p to clone!", this); + return nullptr; + } + + // Since EncodedVideoChunkData can be zero-sized, cloning a zero-sized chunk + // is allowed. + if (mBuffer->Size() == 0) { + LOGW("Cloning an empty EncodedVideoChunkData %p", this); + } + + auto buffer = + MakeRefPtr(mBuffer->Data(), mBuffer->Length()); + if (!buffer || buffer->Size() != mBuffer->Size()) { + LOGE("OOM to copy EncodedVideoChunkData %p", this); + return nullptr; + } + + return MakeUnique(buffer.forget(), mType, mTimestamp, + Maybe(mDuration)); +} + +already_AddRefed EncodedVideoChunkData::TakeData() { + if (!mBuffer || !(*mBuffer)) { + LOGE("EncodedVideoChunkData %p has no data!", this); + return nullptr; + } + + RefPtr sample(new MediaRawData(std::move(*mBuffer))); + sample->mKeyframe = mType == EncodedVideoChunkType::Key; + sample->mTime = TimeUnit::FromMicroseconds(mTimestamp); + sample->mTimecode = TimeUnit::FromMicroseconds(mTimestamp); + + if (mDuration) { + CheckedInt64 duration(*mDuration); + if (!duration.isValid()) { + LOGE("EncodedVideoChunkData %p 's duration exceeds TimeUnit's limit", + this); + return nullptr; + } + sample->mDuration = TimeUnit::FromMicroseconds(duration.value()); + } + + return sample.forget(); +} + +EncodedVideoChunk::EncodedVideoChunk( + nsIGlobalObject* aParent, already_AddRefed aBuffer, + const EncodedVideoChunkType& aType, int64_t aTimestamp, + Maybe&& aDuration) + : EncodedVideoChunkData(std::move(aBuffer), aType, aTimestamp, + std::move(aDuration)), + mParent(aParent) {} + +EncodedVideoChunk::EncodedVideoChunk(nsIGlobalObject* aParent, + const EncodedVideoChunkData& aData) + : EncodedVideoChunkData(aData), mParent(aParent) {} + +nsIGlobalObject* EncodedVideoChunk::GetParentObject() const { + AssertIsOnOwningThread(); + + return mParent.get(); +} + +JSObject* EncodedVideoChunk::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + AssertIsOnOwningThread(); + + return EncodedVideoChunk_Binding::Wrap(aCx, this, aGivenProto); +} + +// https://w3c.github.io/webcodecs/#encodedvideochunk-constructors +/* static */ +already_AddRefed EncodedVideoChunk::Constructor( + const GlobalObject& aGlobal, const EncodedVideoChunkInit& aInit, + ErrorResult& aRv) { + nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + auto buffer = ProcessTypedArrays( + aInit.mData, + [&](const Span& aData, + JS::AutoCheckCannotGC&&) -> RefPtr { + // Make sure it's in uint32_t's range. + CheckedUint32 byteLength(aData.Length()); + if (!byteLength.isValid()) { + aRv.Throw(NS_ERROR_INVALID_ARG); + return nullptr; + } + if (aData.Length() == 0) { + LOGW("Buffer for constructing EncodedVideoChunk is empty!"); + } + RefPtr buf = MakeRefPtr( + aData.Elements(), aData.Length()); + + // Instead of checking *buf, size comparision is used to allow + // constructing a zero-sized EncodedVideoChunk. + if (!buf || buf->Size() != aData.Length()) { + aRv.Throw(NS_ERROR_OUT_OF_MEMORY); + return nullptr; + } + return buf; + }); + + RefPtr chunk(new EncodedVideoChunk( + global, buffer.forget(), aInit.mType, aInit.mTimestamp, + OptionalToMaybe(aInit.mDuration))); + return aRv.Failed() ? nullptr : chunk.forget(); +} + +EncodedVideoChunkType EncodedVideoChunk::Type() const { + AssertIsOnOwningThread(); + + return mType; +} + +int64_t EncodedVideoChunk::Timestamp() const { + AssertIsOnOwningThread(); + + return mTimestamp; +} + +Nullable EncodedVideoChunk::GetDuration() const { + AssertIsOnOwningThread(); + return MaybeToNullable(mDuration); +} + +uint32_t EncodedVideoChunk::ByteLength() const { + AssertIsOnOwningThread(); + MOZ_ASSERT(mBuffer); + + return static_cast(mBuffer->Length()); +} + +// https://w3c.github.io/webcodecs/#dom-encodedvideochunk-copyto +void EncodedVideoChunk::CopyTo( + const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination, + ErrorResult& aRv) { + AssertIsOnOwningThread(); + + ProcessTypedArraysFixed(aDestination, [&](const Span& aData) { + if (mBuffer->Size() > aData.size_bytes()) { + aRv.ThrowTypeError( + "Destination ArrayBuffer smaller than source EncodedVideoChunk"); + return; + } + + PodCopy(aData.data(), mBuffer->Data(), mBuffer->Size()); + }); +} + +// https://w3c.github.io/webcodecs/#ref-for-deserialization-steps%E2%91%A0 +/* static */ +JSObject* EncodedVideoChunk::ReadStructuredClone( + JSContext* aCx, nsIGlobalObject* aGlobal, JSStructuredCloneReader* aReader, + const EncodedVideoChunkData& aData) { + JS::Rooted value(aCx, JS::NullValue()); + // To avoid a rooting hazard error from returning a raw JSObject* before + // running the RefPtr destructor, RefPtr needs to be destructed before + // returning the raw JSObject*, which is why the RefPtr is + // created in the scope below. Otherwise, the static analysis infers the + // RefPtr cannot be safely destructed while the unrooted return JSObject* is + // on the stack. + { + auto frame = MakeRefPtr(aGlobal, aData); + if (!GetOrCreateDOMReflector(aCx, frame, &value) || !value.isObject()) { + return nullptr; + } + } + return value.toObjectOrNull(); +} + +// https://w3c.github.io/webcodecs/#ref-for-serialization-steps%E2%91%A0 +bool EncodedVideoChunk::WriteStructuredClone( + JSStructuredCloneWriter* aWriter, StructuredCloneHolder* aHolder) const { + AssertIsOnOwningThread(); + + // Indexing the chunk and send the index to the receiver. + const uint32_t index = + static_cast(aHolder->EncodedVideoChunks().Length()); + // The serialization is limited to the same process scope so it's ok to + // serialize a reference instead of a copy. + aHolder->EncodedVideoChunks().AppendElement(EncodedVideoChunkData(*this)); + return !NS_WARN_IF( + !JS_WriteUint32Pair(aWriter, SCTAG_DOM_ENCODEDVIDEOCHUNK, index)); +} + +#undef LOGW +#undef LOGE +#undef LOG_INTERNAL + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/EncodedVideoChunk.h b/dom/media/webcodecs/EncodedVideoChunk.h new file mode 100644 index 0000000000..3a9fb1e368 --- /dev/null +++ b/dom/media/webcodecs/EncodedVideoChunk.h @@ -0,0 +1,119 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_EncodedVideoChunk_h +#define mozilla_dom_EncodedVideoChunk_h + +#include "js/TypeDecls.h" +#include "mozilla/Attributes.h" +#include "mozilla/Buffer.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/Maybe.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; + +namespace mozilla { + +class MediaAlignedByteBuffer; +class MediaRawData; + +namespace dom { + +class MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer; +class OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer; +class StructuredCloneHolder; + +enum class EncodedVideoChunkType : uint8_t; +struct EncodedVideoChunkInit; + +} // namespace dom +} // namespace mozilla + +namespace mozilla::dom { + +class EncodedVideoChunkData { + public: + EncodedVideoChunkData(already_AddRefed aBuffer, + const EncodedVideoChunkType& aType, int64_t aTimestamp, + Maybe&& aDuration); + EncodedVideoChunkData(const EncodedVideoChunkData& aData) = default; + ~EncodedVideoChunkData(); + + UniquePtr Clone() const; + already_AddRefed TakeData(); + + protected: + // mBuffer's byte length is guaranteed to be smaller than UINT32_MAX. + RefPtr mBuffer; + EncodedVideoChunkType mType; + int64_t mTimestamp; + Maybe mDuration; +}; + +class EncodedVideoChunk final : public EncodedVideoChunkData, + public nsISupports, + public nsWrapperCache { + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(EncodedVideoChunk) + + public: + EncodedVideoChunk(nsIGlobalObject* aParent, + already_AddRefed aBuffer, + const EncodedVideoChunkType& aType, int64_t aTimestamp, + Maybe&& aDuration); + + EncodedVideoChunk(nsIGlobalObject* aParent, + const EncodedVideoChunkData& aData); + + protected: + ~EncodedVideoChunk() = default; + + public: + nsIGlobalObject* GetParentObject() const; + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + static already_AddRefed Constructor( + const GlobalObject& aGlobal, const EncodedVideoChunkInit& aInit, + ErrorResult& aRv); + + EncodedVideoChunkType Type() const; + + int64_t Timestamp() const; + + Nullable GetDuration() const; + + uint32_t ByteLength() const; + + void CopyTo( + const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination, + ErrorResult& aRv); + + // [Serializable] implementations: {Read, Write}StructuredClone + static JSObject* ReadStructuredClone(JSContext* aCx, nsIGlobalObject* aGlobal, + JSStructuredCloneReader* aReader, + const EncodedVideoChunkData& aData); + + bool WriteStructuredClone(JSStructuredCloneWriter* aWriter, + StructuredCloneHolder* aHolder) const; + + private: + // EncodedVideoChunk can run on either main thread or worker thread. + void AssertIsOnOwningThread() const { + NS_ASSERT_OWNINGTHREAD(EncodedVideoChunk); + } + + nsCOMPtr mParent; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_EncodedVideoChunk_h diff --git a/dom/media/webcodecs/EncoderAgent.cpp b/dom/media/webcodecs/EncoderAgent.cpp new file mode 100644 index 0000000000..e6af17a0be --- /dev/null +++ b/dom/media/webcodecs/EncoderAgent.cpp @@ -0,0 +1,441 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "EncoderAgent.h" + +#include "PDMFactory.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/Logging.h" +#include "nsThreadUtils.h" + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOG +# undef LOG +#endif // LOG +#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +#ifdef LOGW +# undef LOGW +#endif // LOGE +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +#ifdef LOGV +# undef LOGV +#endif // LOGV +#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__) + +EncoderAgent::EncoderAgent(WebCodecsId aId) + : mId(aId), + mOwnerThread(GetCurrentSerialEventTarget()), + mPEMFactory(MakeRefPtr()), + mEncoder(nullptr), + mState(State::Unconfigured) { + MOZ_ASSERT(mOwnerThread); + MOZ_ASSERT(mPEMFactory); + LOG("EncoderAgent #%zu (%p) ctor", mId, this); +} + +EncoderAgent::~EncoderAgent() { + LOG("EncoderAgent #%zu (%p) dtor", mId, this); + MOZ_ASSERT(mState == State::Unconfigured, "encoder released in wrong state"); + MOZ_ASSERT(!mEncoder, "encoder must be shutdown"); +} + +RefPtr EncoderAgent::Configure( + const EncoderConfig& aConfig) { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + MOZ_ASSERT(mState == State::Unconfigured || mState == State::Error); + MOZ_ASSERT(mConfigurePromise.IsEmpty()); + MOZ_ASSERT(!mCreateRequest.Exists()); + MOZ_ASSERT(!mInitRequest.Exists()); + + if (mState == State::Error) { + LOGE("EncoderAgent #%zu (%p) tried to configure in error state", mId, this); + return ConfigurePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "Cannot configure in error state"), + __func__); + } + + MOZ_ASSERT(mState == State::Unconfigured); + MOZ_ASSERT(!mEncoder); + SetState(State::Configuring); + + LOG("EncoderAgent #%zu (%p) is creating an encoder (%s)", mId, this, + GetCodecTypeString(aConfig.mCodec)); + + RefPtr p = mConfigurePromise.Ensure(__func__); + + mPEMFactory->CreateEncoderAsync(aConfig, dom::GetWebCodecsEncoderTaskQueue()) + ->Then( + mOwnerThread, __func__, + [self = RefPtr{this}](RefPtr&& aEncoder) { + self->mCreateRequest.Complete(); + + // If EncoderAgent has been shut down, shut the created encoder down + // and return. + if (!self->mShutdownWhileCreationPromise.IsEmpty()) { + MOZ_ASSERT(self->mState == State::ShuttingDown); + MOZ_ASSERT(self->mConfigurePromise.IsEmpty(), + "configuration should have been rejected"); + + LOGW( + "EncoderAgent #%zu (%p) has been shut down. We need to shut " + "the newly created encoder down", + self->mId, self.get()); + aEncoder->Shutdown()->Then( + self->mOwnerThread, __func__, + [self](const ShutdownPromise::ResolveOrRejectValue& aValue) { + MOZ_ASSERT(self->mState == State::ShuttingDown); + + LOGW( + "EncoderAgent #%zu (%p), newly created encoder " + "shutdown " + "has been %s", + self->mId, self.get(), + aValue.IsResolve() ? "resolved" : "rejected"); + + self->SetState(State::Unconfigured); + + self->mShutdownWhileCreationPromise.ResolveOrReject( + aValue, __func__); + }); + return; + } + + self->mEncoder = aEncoder.forget(); + LOG("EncoderAgent #%zu (%p) has created a encoder, now initialize " + "it", + self->mId, self.get()); + self->mEncoder->Init() + ->Then( + self->mOwnerThread, __func__, + [self]() { + self->mInitRequest.Complete(); + LOG("EncoderAgent #%zu (%p) has initialized the encoder", + self->mId, self.get()); + self->SetState(State::Configured); + self->mConfigurePromise.Resolve(true, __func__); + }, + [self](const MediaResult& aError) { + self->mInitRequest.Complete(); + LOGE( + "EncoderAgent #%zu (%p) failed to initialize the " + "encoder", + self->mId, self.get()); + self->SetState(State::Error); + self->mConfigurePromise.Reject(aError, __func__); + }) + ->Track(self->mInitRequest); + }, + [self = RefPtr{this}](const MediaResult& aError) { + self->mCreateRequest.Complete(); + LOGE("EncoderAgent #%zu (%p) failed to create a encoder", self->mId, + self.get()); + + // If EncoderAgent has been shut down, we need to resolve the + // shutdown promise. + if (!self->mShutdownWhileCreationPromise.IsEmpty()) { + MOZ_ASSERT(self->mState == State::ShuttingDown); + MOZ_ASSERT(self->mConfigurePromise.IsEmpty(), + "configuration should have been rejected"); + + LOGW( + "EncoderAgent #%zu (%p) has been shut down. Resolve the " + "shutdown promise right away since encoder creation failed", + self->mId, self.get()); + + self->SetState(State::Unconfigured); + self->mShutdownWhileCreationPromise.Resolve(true, __func__); + return; + } + + self->SetState(State::Error); + self->mConfigurePromise.Reject(aError, __func__); + }) + ->Track(mCreateRequest); + + return p; +} + +RefPtr EncoderAgent::Reconfigure( + const RefPtr& aConfigChanges) { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + MOZ_ASSERT(mState == State::Configured || mState == State::Error); + MOZ_ASSERT(mReconfigurationPromise.IsEmpty()); + + if (mState == State::Error) { + LOGE("EncoderAgent #%zu (%p) tried to reconfigure in error state", mId, + this); + return ReconfigurationPromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "Cannot reconfigure in error state"), + __func__); + } + + MOZ_ASSERT(mEncoder); + SetState(State::Configuring); + + LOG("EncoderAgent #%zu (%p) is reconfiguring its encoder (%s)", mId, this, + NS_ConvertUTF16toUTF8(aConfigChanges->ToString().get()).get()); + + RefPtr p = mReconfigurationPromise.Ensure(__func__); + + mEncoder->Reconfigure(aConfigChanges) + ->Then( + mOwnerThread, __func__, + [self = RefPtr{this}](bool) { + self->mReconfigurationRequest.Complete(); + LOGE("EncoderAgent #%zu (%p) reconfigure success", self->mId, + self.get()); + self->mReconfigurationPromise.Resolve(true, __func__); + }, + [self = RefPtr{this}](const MediaResult& aError) { + self->mReconfigurationRequest.Complete(); + LOGE("EncoderAgent #%zu (%p) reconfigure failure", self->mId, + self.get()); + // Not a a fatal error per se, the owner will deal with it. + self->mReconfigurationPromise.Reject(aError, __func__); + }) + ->Track(mReconfigurationRequest); + + return p; +} + +RefPtr EncoderAgent::Shutdown() { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + + auto r = + MediaResult(NS_ERROR_DOM_MEDIA_CANCELED, "Canceled by encoder shutdown"); + + // If the encoder creation has not been completed yet, wait until the encoder + // being created has been shut down. + if (mCreateRequest.Exists()) { + MOZ_ASSERT(!mInitRequest.Exists()); + MOZ_ASSERT(!mConfigurePromise.IsEmpty()); + MOZ_ASSERT(!mEncoder); + MOZ_ASSERT(mState == State::Configuring); + MOZ_ASSERT(mShutdownWhileCreationPromise.IsEmpty()); + + LOGW( + "EncoderAgent #%zu (%p) shutdown while the encoder creation for " + "configuration is in flight. Reject the configuration now and defer " + "the shutdown until the created encoder has been shut down", + mId, this); + + // Reject the configuration in flight. + mConfigurePromise.Reject(r, __func__); + + // Get the promise that will be resolved when the encoder being created has + // been destroyed. + SetState(State::ShuttingDown); + return mShutdownWhileCreationPromise.Ensure(__func__); + } + + // If encoder creation has been completed, we must have the encoder now. + MOZ_ASSERT(mEncoder); + + // Cancel pending initialization for configuration in flight if any. + mInitRequest.DisconnectIfExists(); + mConfigurePromise.RejectIfExists(r, __func__); + + mReconfigurationRequest.DisconnectIfExists(); + mReconfigurationPromise.RejectIfExists(r, __func__); + + // Cancel encoder in flight if any. + mEncodeRequest.DisconnectIfExists(); + mEncodePromise.RejectIfExists(r, __func__); + + // Cancel flush-out in flight if any. + mDrainRequest.DisconnectIfExists(); + mEncodeRequest.DisconnectIfExists(); + + mDrainRequest.DisconnectIfExists(); + mDrainPromise.RejectIfExists(r, __func__); + + SetState(State::Unconfigured); + + RefPtr encoder = std::move(mEncoder); + return encoder->Shutdown(); +} + +RefPtr EncoderAgent::Encode(MediaData* aInput) { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + MOZ_ASSERT(aInput); + MOZ_ASSERT(mState == State::Configured || mState == State::Error); + MOZ_ASSERT(mEncodePromise.IsEmpty()); + MOZ_ASSERT(!mEncodeRequest.Exists()); + + if (mState == State::Error) { + LOGE("EncoderAgent #%zu (%p) tried to encoder in error state", mId, this); + return EncodePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "Cannot encoder in error state"), + __func__); + } + + MOZ_ASSERT(mState == State::Configured); + MOZ_ASSERT(mEncoder); + SetState(State::Encoding); + + RefPtr p = mEncodePromise.Ensure(__func__); + + mEncoder->Encode(aInput) + ->Then( + mOwnerThread, __func__, + [self = RefPtr{this}](MediaDataEncoder::EncodedData&& aData) { + self->mEncodeRequest.Complete(); + LOGV("EncoderAgent #%zu (%p) encode successful", self->mId, + self.get()); + self->SetState(State::Configured); + self->mEncodePromise.Resolve(std::move(aData), __func__); + }, + [self = RefPtr{this}](const MediaResult& aError) { + self->mEncodeRequest.Complete(); + LOGV("EncoderAgent #%zu (%p) failed to encode", self->mId, + self.get()); + self->SetState(State::Error); + self->mEncodePromise.Reject(aError, __func__); + }) + ->Track(mEncodeRequest); + + return p; +} + +RefPtr EncoderAgent::Drain() { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + // This can be called when reconfiguring the encoder. + MOZ_ASSERT(mState == State::Configured || mState == State::Configuring); + MOZ_ASSERT(mDrainPromise.IsEmpty()); + MOZ_ASSERT(mEncoder); + + SetState(State::Flushing); + + RefPtr p = mDrainPromise.Ensure(__func__); + DryUntilDrain(); + return p; +} + +void EncoderAgent::DryUntilDrain() { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + MOZ_ASSERT(mState == State::Flushing); + MOZ_ASSERT(!mDrainRequest.Exists()); + MOZ_ASSERT(mEncoder); + + LOG("EncoderAgent #%zu (%p) is draining the encoder", mId, this); + mEncoder->Drain() + ->Then( + mOwnerThread, __func__, + [self = RefPtr{this}](MediaDataEncoder::EncodedData&& aData) { + self->mDrainRequest.Complete(); + + if (aData.IsEmpty()) { + LOG("EncoderAgent #%zu (%p) is dry now", self->mId, self.get()); + self->SetState(State::Configured); + self->mDrainPromise.Resolve(std::move(self->mDrainData), + __func__); + return; + } + + LOG("EncoderAgent #%zu (%p) drained %zu encoder data. Keep " + "draining " + "until dry", + self->mId, self.get(), aData.Length()); + self->mDrainData.AppendElements(std::move(aData)); + self->DryUntilDrain(); + }, + [self = RefPtr{this}](const MediaResult& aError) { + self->mDrainRequest.Complete(); + + LOGE("EncoderAgent %p failed to drain encoder", self.get()); + self->mDrainData.Clear(); + self->mDrainPromise.Reject(aError, __func__); + }) + ->Track(mDrainRequest); +} + +void EncoderAgent::SetState(State aState) { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + + auto validateStateTransition = [](State aOldState, State aNewState) { + switch (aOldState) { + case State::Unconfigured: + return aNewState == State::Configuring; + case State::Configuring: + return aNewState == State::Configured || aNewState == State::Error || + aNewState == State::Flushing || + aNewState == State::Unconfigured || + aNewState == State::ShuttingDown; + case State::Configured: + return aNewState == State::Unconfigured || + aNewState == State::Configuring || + aNewState == State::Encoding || aNewState == State::Flushing; + case State::Encoding: + case State::Flushing: + return aNewState == State::Configured || aNewState == State::Error || + aNewState == State::Unconfigured; + case State::ShuttingDown: + return aNewState == State::Unconfigured; + case State::Error: + return aNewState == State::Unconfigured; + default: + break; + } + MOZ_ASSERT_UNREACHABLE("Unhandled state transition"); + return false; + }; + + auto stateToString = [](State aState) -> const char* { + switch (aState) { + case State::Unconfigured: + return "Unconfigured"; + case State::Configuring: + return "Configuring"; + case State::Configured: + return "Configured"; + case State::Encoding: + return "Encoding"; + case State::Flushing: + return "Flushing"; + case State::ShuttingDown: + return "ShuttingDown"; + case State::Error: + return "Error"; + default: + break; + } + MOZ_ASSERT_UNREACHABLE("Unhandled state type"); + return "Unknown"; + }; + + DebugOnly isValid = validateStateTransition(mState, aState); + LOGV("EncoderAgent #%zu (%p) state change: %s -> %s", mId, this, + stateToString(mState), stateToString(aState)); + MOZ_ASSERT(isValid); + mState = aState; +} + +#undef LOG +#undef LOGW +#undef LOGE +#undef LOGV +#undef LOG_INTERNAL + +} // namespace mozilla diff --git a/dom/media/webcodecs/EncoderAgent.h b/dom/media/webcodecs/EncoderAgent.h new file mode 100644 index 0000000000..387f218e29 --- /dev/null +++ b/dom/media/webcodecs/EncoderAgent.h @@ -0,0 +1,116 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_WEBCODECS_EncoderAgent_H +#define DOM_MEDIA_WEBCODECS_EncoderAgent_H + +#include "MediaResult.h" +#include "PlatformEncoderModule.h" +#include "PEMFactory.h" +#include "mozilla/RefPtr.h" +#include "mozilla/TaskQueue.h" +#include "WebCodecsUtils.h" + +class nsISerialEventTarget; + +namespace mozilla { + +class PDMFactory; +class TrackInfo; + +namespace layers { +class ImageContainer; +} // namespace layers + +// EncoderAgent is a wrapper that contains a MediaDataEncoder. It adapts the +// MediaDataEncoder APIs for use in WebCodecs. +// +// If Configure() is called, Shutdown() must be called to release the resources +// gracefully. Except Shutdown(), all the methods can't be called concurrently, +// meaning a method can only be called when the previous API call has completed. +// The responsability of arranging the method calls is on the caller. +// +// When Shutdown() is called, all the operations in flight are canceled and the +// MediaDataEncoder is shut down. On the other hand, errors are final. A new +// EncoderAgent must be created when an error is encountered. +// +// All the methods need to be called on the EncoderAgent's owner thread. In +// WebCodecs, it's either on the main thread or worker thread. +class EncoderAgent final { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(EncoderAgent); + + explicit EncoderAgent(WebCodecsId aId); + + // The following APIs are owner thread only. + + using ConfigurePromise = MozPromise; + using ReconfigurationPromise = MediaDataEncoder::ReconfigurationPromise; + RefPtr Configure(const EncoderConfig& aConfig); + RefPtr Reconfigure( + const RefPtr& aConfigChange); + RefPtr Shutdown(); + using EncodePromise = MediaDataEncoder::EncodePromise; + RefPtr Encode(MediaData* aInput); + // WebCodecs's flush() flushes out all the pending encoded data in the + // encoder. It's called Drain internally. + RefPtr Drain(); + + const WebCodecsId mId; + + private: + ~EncoderAgent(); + + // Push out all the data in the MediaDataEncoder's pipeline. + // TODO: MediaDataEncoder should implement this, instead of asking call site + // to run `Drain` multiple times. + RefPtr Dry(); + void DryUntilDrain(); + + enum class State { + Unconfigured, + Configuring, + Configured, + Encoding, + Flushing, + ShuttingDown, + Error, + }; + void SetState(State aState); + + const RefPtr mOwnerThread; + const RefPtr mPEMFactory; + RefPtr mEncoder; + State mState; + + // Configure + MozPromiseHolder mConfigurePromise; + using CreateEncoderPromise = PlatformEncoderModule::CreateEncoderPromise; + MozPromiseRequestHolder mCreateRequest; + using InitPromise = MediaDataEncoder::InitPromise; + MozPromiseRequestHolder mInitRequest; + + // Reconfigure + MozPromiseHolder mReconfigurationPromise; + using ReconfigureEncoderRequest = ReconfigurationPromise; + MozPromiseRequestHolder mReconfigurationRequest; + + // Shutdown + MozPromiseHolder mShutdownWhileCreationPromise; + + // Encoding + MozPromiseHolder mEncodePromise; + MozPromiseRequestHolder mEncodeRequest; + + // Drain + MozPromiseRequestHolder mDrainRequest; + MozPromiseHolder mDrainPromise; + MediaDataEncoder::EncodedData mDrainData; +}; + +} // namespace mozilla + +#endif // DOM_MEDIA_WEBCODECS_EncoderAgent_H diff --git a/dom/media/webcodecs/EncoderTemplate.cpp b/dom/media/webcodecs/EncoderTemplate.cpp new file mode 100644 index 0000000000..35c8feb3f8 --- /dev/null +++ b/dom/media/webcodecs/EncoderTemplate.cpp @@ -0,0 +1,1228 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "EncoderTemplate.h" + +#include "EncoderTypes.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/Try.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/DOMException.h" +#include "mozilla/dom/Event.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/VideoFrame.h" +#include "mozilla/dom/WorkerCommon.h" +#include "nsGkAtoms.h" +#include "nsString.h" +#include "nsThreadUtils.h" + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla::dom { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOG +# undef LOG +#endif // LOG +#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +#ifdef LOGW +# undef LOGW +#endif // LOGW +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +#ifdef LOGV +# undef LOGV +#endif // LOGV +#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__) + +/* + * Below are ControlMessage classes implementations + */ + +template +EncoderTemplate::ControlMessage::ControlMessage( + WebCodecsId aConfigureId) + : mConfigureId(aConfigureId), mMessageId(sNextId++) {} + +template +EncoderTemplate::ConfigureMessage::ConfigureMessage( + WebCodecsId aConfigureId, const RefPtr& aConfig) + : ControlMessage(aConfigureId), mConfig(aConfig) {} + +template +EncoderTemplate::EncodeMessage::EncodeMessage( + WebCodecsId aConfigureId, RefPtr&& aData, + Maybe&& aOptions) + : ControlMessage(aConfigureId), mData(aData) {} + +template +EncoderTemplate::FlushMessage::FlushMessage( + WebCodecsId aConfigureId, Promise* aPromise) + : ControlMessage(aConfigureId), mPromise(aPromise) {} + +template +void EncoderTemplate::FlushMessage::RejectPromiseIfAny( + const nsresult& aReason) { + if (mPromise) { + mPromise->MaybeReject(aReason); + } +} + +/* + * Below are EncoderTemplate implementation + */ + +template +EncoderTemplate::EncoderTemplate( + nsIGlobalObject* aGlobalObject, + RefPtr&& aErrorCallback, + RefPtr&& aOutputCallback) + : DOMEventTargetHelper(aGlobalObject), + mErrorCallback(std::move(aErrorCallback)), + mOutputCallback(std::move(aOutputCallback)), + mState(CodecState::Unconfigured), + mMessageQueueBlocked(false), + mEncodeQueueSize(0), + mDequeueEventScheduled(false), + mLatestConfigureId(0), + mEncodeCounter(0), + mFlushCounter(0) {} + +template +void EncoderTemplate::Configure(const ConfigType& aConfig, + ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s::Configure %p codec %s", EncoderType::Name.get(), this, + NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); + + nsCString errorMessage; + if (!EncoderType::Validate(aConfig, errorMessage)) { + LOG("Configure: Validate error: %s", errorMessage.get()); + aRv.ThrowTypeError(errorMessage); + return; + } + + if (mState == CodecState::Closed) { + LOG("Configure: CodecState::Closed, rejecting with InvalidState"); + aRv.ThrowInvalidStateError("The codec is no longer usable"); + return; + } + + // Clone a ConfigType as the active Encode config. + RefPtr config = + EncoderType::CreateConfigInternal(aConfig); + if (!config) { + aRv.Throw(NS_ERROR_UNEXPECTED); // Invalid description data. + return; + } + + mState = CodecState::Configured; + mEncodeCounter = 0; + mFlushCounter = 0; + + mControlMessageQueue.push(MakeRefPtr(sNextId++, config)); + mLatestConfigureId = mControlMessageQueue.back()->mMessageId; + LOG("%s %p enqueues %s", EncoderType::Name.get(), this, + mControlMessageQueue.back()->ToString().get()); + ProcessControlMessageQueue(); +} + +template +void EncoderTemplate::EncodeAudioData(InputType& aInput, + ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s %p, EncodeAudioData", EncoderType::Name.get(), this); + + if (mState != CodecState::Configured) { + aRv.ThrowInvalidStateError("Encoder must be configured first"); + return; + } + + if (aInput.IsClosed()) { + aRv.ThrowTypeError("input AudioData has been closed"); + return; + } + + mEncodeQueueSize += 1; + // Dummy options here as a shortcut + mControlMessageQueue.push(MakeRefPtr( + mLatestConfigureId, + EncoderType::CreateInputInternal(aInput, VideoEncoderEncodeOptions()))); + LOGV("%s %p enqueues %s", EncoderType::Name.get(), this, + mControlMessageQueue.back()->ToString().get()); + ProcessControlMessageQueue(); +} + +template +void EncoderTemplate::EncodeVideoFrame( + InputType& aInput, const VideoEncoderEncodeOptions& aOptions, + ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s::Encode %p %s", EncoderType::Name.get(), this, + aInput.ToString().get()); + + if (mState != CodecState::Configured) { + aRv.ThrowInvalidStateError("Encoder must be configured first"); + return; + } + + if (aInput.IsClosed()) { + aRv.ThrowTypeError("input VideoFrame has been closed"); + return; + } + + mEncodeQueueSize += 1; + mControlMessageQueue.push(MakeRefPtr( + mLatestConfigureId, EncoderType::CreateInputInternal(aInput, aOptions), + Some(aOptions))); + LOGV("%s %p enqueues %s", EncoderType::Name.get(), this, + mControlMessageQueue.back()->ToString().get()); + ProcessControlMessageQueue(); +} + +template +already_AddRefed EncoderTemplate::Flush( + ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s::Flush %p", EncoderType::Name.get(), this); + + if (mState != CodecState::Configured) { + LOG("%s %p, wrong state!", EncoderType::Name.get(), this); + aRv.ThrowInvalidStateError("Encoder must be configured first"); + return nullptr; + } + + RefPtr p = Promise::Create(GetParentObject(), aRv); + if (NS_WARN_IF(aRv.Failed())) { + return p.forget(); + } + + mControlMessageQueue.push(MakeRefPtr(mLatestConfigureId, p)); + LOG("%s %p enqueues %s", EncoderType::Name.get(), this, + mControlMessageQueue.back()->ToString().get()); + ProcessControlMessageQueue(); + return p.forget(); +} + +template +void EncoderTemplate::Reset(ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s %p, Reset", EncoderType::Name.get(), this); + + if (auto r = ResetInternal(NS_ERROR_DOM_ABORT_ERR); r.isErr()) { + aRv.Throw(r.unwrapErr()); + } +} + +template +void EncoderTemplate::Close(ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s::Close %p", EncoderType::Name.get(), this); + + if (auto r = CloseInternal(NS_ERROR_DOM_ABORT_ERR); r.isErr()) { + aRv.Throw(r.unwrapErr()); + } +} + +template +Result EncoderTemplate::ResetInternal( + const nsresult& aResult) { + AssertIsOnOwningThread(); + + LOG("%s::Reset %p", EncoderType::Name.get(), this); + + if (mState == CodecState::Closed) { + return Err(NS_ERROR_DOM_INVALID_STATE_ERR); + } + + mState = CodecState::Unconfigured; + mEncodeCounter = 0; + mFlushCounter = 0; + + CancelPendingControlMessages(aResult); + DestroyEncoderAgentIfAny(); + + if (mEncodeQueueSize > 0) { + mEncodeQueueSize = 0; + ScheduleDequeueEvent(); + } + + StopBlockingMessageQueue(); + + return Ok(); +} + +template +Result EncoderTemplate::CloseInternal( + const nsresult& aResult) { + AssertIsOnOwningThread(); + + MOZ_TRY(ResetInternal(aResult)); + mState = CodecState::Closed; + if (aResult != NS_ERROR_DOM_ABORT_ERR) { + nsCString error; + GetErrorName(aResult, error); + LOGE("%s %p Close on error: %s", EncoderType::Name.get(), this, + error.get()); + ReportError(aResult); + } + return Ok(); +} + +template +void EncoderTemplate::ReportError(const nsresult& aResult) { + AssertIsOnOwningThread(); + + RefPtr e = DOMException::Create(aResult); + RefPtr cb(mErrorCallback); + cb->Call(*e); +} + +template +void EncoderTemplate::OutputEncodedData( + nsTArray>&& aData) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(mActiveConfig); + + // Get JSContext for RootedDictionary. + // The EncoderType::MetadataType, VideoDecoderConfig, and VideoColorSpaceInit + // below are rooted to work around the JS hazard issues. + AutoJSAPI jsapi; + DebugOnly ok = + jsapi.Init(GetParentObject()); // TODO: check returned value? + JSContext* cx = jsapi.cx(); + + RefPtr cb(mOutputCallback); + for (auto& data : aData) { + // It's possible to have reset() called in between this task having been + // dispatched, and running -- no output callback should happen when that's + // the case. + // This is imprecise in the spec, but discussed in + // https://github.com/w3c/webcodecs/issues/755 and agreed upon. + if (!mActiveConfig) { + return; + } + RefPtr encodedData = + EncodedDataToOutputType(GetParentObject(), data); + + RootedDictionary metadata(cx); + if (mOutputNewDecoderConfig) { + VideoDecoderConfigInternal decoderConfigInternal = + EncoderConfigToDecoderConfig(GetParentObject(), data, *mActiveConfig); + + // Convert VideoDecoderConfigInternal to VideoDecoderConfig + RootedDictionary decoderConfig(cx); + decoderConfig.mCodec = decoderConfigInternal.mCodec; + if (decoderConfigInternal.mCodedHeight) { + decoderConfig.mCodedHeight.Construct( + decoderConfigInternal.mCodedHeight.value()); + } + if (decoderConfigInternal.mCodedWidth) { + decoderConfig.mCodedWidth.Construct( + decoderConfigInternal.mCodedWidth.value()); + } + if (decoderConfigInternal.mColorSpace) { + RootedDictionary colorSpace(cx); + colorSpace.mFullRange = + MaybeToNullable(decoderConfigInternal.mColorSpace->mFullRange); + colorSpace.mMatrix = + MaybeToNullable(decoderConfigInternal.mColorSpace->mMatrix); + colorSpace.mPrimaries = + MaybeToNullable(decoderConfigInternal.mColorSpace->mPrimaries); + colorSpace.mTransfer = + MaybeToNullable(decoderConfigInternal.mColorSpace->mTransfer); + decoderConfig.mColorSpace.Construct(std::move(colorSpace)); + } + if (decoderConfigInternal.mDescription && + !decoderConfigInternal.mDescription.value()->IsEmpty()) { + auto& abov = decoderConfig.mDescription.Construct(); + AutoEntryScript aes(GetParentObject(), "EncoderConfigToDecoderConfig"); + size_t lengthBytes = + decoderConfigInternal.mDescription.value()->Length(); + UniquePtr extradata( + new uint8_t[lengthBytes]); + PodCopy(extradata.get(), + decoderConfigInternal.mDescription.value()->Elements(), + lengthBytes); + JS::Rooted description( + aes.cx(), JS::NewArrayBufferWithContents(aes.cx(), lengthBytes, + std::move(extradata))); + JS::Rooted value(aes.cx(), JS::ObjectValue(*description)); + DebugOnly rv = abov.Init(aes.cx(), value); + } + if (decoderConfigInternal.mDisplayAspectHeight) { + decoderConfig.mDisplayAspectHeight.Construct( + decoderConfigInternal.mDisplayAspectHeight.value()); + } + if (decoderConfigInternal.mDisplayAspectWidth) { + decoderConfig.mDisplayAspectWidth.Construct( + decoderConfigInternal.mDisplayAspectWidth.value()); + } + if (decoderConfigInternal.mOptimizeForLatency) { + decoderConfig.mOptimizeForLatency.Construct( + decoderConfigInternal.mOptimizeForLatency.value()); + } + + metadata.mDecoderConfig.Construct(std::move(decoderConfig)); + mOutputNewDecoderConfig = false; + LOGE("New config passed to output callback: %s", + NS_ConvertUTF16toUTF8(decoderConfigInternal.ToString()).get()); + } + + nsAutoCString metadataInfo; + + if (data->mTemporalLayerId) { + RootedDictionary svc(cx); + svc.mTemporalLayerId.Construct(data->mTemporalLayerId.value()); + metadata.mSvc.Construct(std::move(svc)); + metadataInfo.Append( + nsPrintfCString(", temporal layer id %d", + metadata.mSvc.Value().mTemporalLayerId.Value())); + } + + if (metadata.mDecoderConfig.WasPassed()) { + metadataInfo.Append(", new decoder config"); + } + + LOG("EncoderTemplate:: output callback (ts: % " PRId64 ")%s", + encodedData->Timestamp(), metadataInfo.get()); + cb->Call((typename EncoderType::OutputType&)(*encodedData), metadata); + } +} + +template +class EncoderTemplate::ErrorRunnable final + : public DiscardableRunnable { + public: + ErrorRunnable(Self* aEncoder, const nsresult& aError) + : DiscardableRunnable("Decoder ErrorRunnable"), + mEncoder(aEncoder), + mError(aError) { + MOZ_ASSERT(mEncoder); + } + ~ErrorRunnable() = default; + + // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT. + // See bug 1535398. + MOZ_CAN_RUN_SCRIPT_BOUNDARY NS_IMETHOD Run() override { + nsCString error; + GetErrorName(mError, error); + LOGE("%s %p report error: %s", EncoderType::Name.get(), mEncoder.get(), + error.get()); + RefPtr d = std::move(mEncoder); + d->ReportError(mError); + return NS_OK; + } + + private: + RefPtr mEncoder; + const nsresult mError; +}; + +template +class EncoderTemplate::OutputRunnable final + : public DiscardableRunnable { + public: + OutputRunnable(Self* aEncoder, WebCodecsId aConfigureId, + const nsACString& aLabel, + nsTArray>&& aData) + : DiscardableRunnable("Decoder OutputRunnable"), + mEncoder(aEncoder), + mConfigureId(aConfigureId), + mLabel(aLabel), + mData(std::move(aData)) { + MOZ_ASSERT(mEncoder); + } + ~OutputRunnable() = default; + + // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT. + // See bug 1535398. + MOZ_CAN_RUN_SCRIPT_BOUNDARY NS_IMETHOD Run() override { + if (mEncoder->mState != CodecState::Configured) { + LOGV("%s %p has been %s. Discard %s-result for EncoderAgent #%zu", + EncoderType::Name.get(), mEncoder.get(), + mEncoder->mState == CodecState::Closed ? "closed" : "reset", + mLabel.get(), mConfigureId); + return NS_OK; + } + + MOZ_ASSERT(mEncoder->mAgent); + if (mConfigureId != mEncoder->mAgent->mId) { + LOGW( + "%s %p has been re-configured. Still yield %s-result for " + "EncoderAgent #%zu", + EncoderType::Name.get(), mEncoder.get(), mLabel.get(), mConfigureId); + } + + LOGV("%s %p, yields %s-result for EncoderAgent #%zu", + EncoderType::Name.get(), mEncoder.get(), mLabel.get(), mConfigureId); + RefPtr d = std::move(mEncoder); + d->OutputEncodedData(std::move(mData)); + + return NS_OK; + } + + private: + RefPtr mEncoder; + const WebCodecsId mConfigureId; + const nsCString mLabel; + nsTArray> mData; +}; + +template +void EncoderTemplate::ScheduleOutputEncodedData( + nsTArray>&& aData, const nsACString& aLabel) { + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(mAgent); + + MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(MakeAndAddRef( + this, mAgent->mId, aLabel, std::move(aData)))); +} + +template +void EncoderTemplate::ScheduleClose(const nsresult& aResult) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + + auto task = [self = RefPtr{this}, result = aResult] { + if (self->mState == CodecState::Closed) { + nsCString error; + GetErrorName(result, error); + LOGW("%s %p has been closed. Ignore close with %s", + EncoderType::Name.get(), self.get(), error.get()); + return; + } + DebugOnly> r = self->CloseInternal(result); + MOZ_ASSERT(r.value.isOk()); + }; + nsISerialEventTarget* target = GetCurrentSerialEventTarget(); + + if (NS_IsMainThread()) { + MOZ_ALWAYS_SUCCEEDS(target->Dispatch( + NS_NewRunnableFunction("ScheduleClose Runnable (main)", task))); + return; + } + + MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewCancelableRunnableFunction( + "ScheduleClose Runnable (worker)", task))); +} + +template +void EncoderTemplate::ScheduleDequeueEvent() { + AssertIsOnOwningThread(); + + if (mDequeueEventScheduled) { + return; + } + mDequeueEventScheduled = true; + + auto dispatcher = [self = RefPtr{this}] { + self->FireEvent(nsGkAtoms::ondequeue, u"dequeue"_ns); + self->mDequeueEventScheduled = false; + }; + nsISerialEventTarget* target = GetCurrentSerialEventTarget(); + + if (NS_IsMainThread()) { + MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewRunnableFunction( + "ScheduleDequeueEvent Runnable (main)", dispatcher))); + return; + } + + MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewCancelableRunnableFunction( + "ScheduleDequeueEvent Runnable (worker)", dispatcher))); +} + +template +nsresult EncoderTemplate::FireEvent(nsAtom* aTypeWithOn, + const nsAString& aEventType) { + if (aTypeWithOn && !HasListenersFor(aTypeWithOn)) { + return NS_ERROR_ABORT; + } + + LOGV("Dispatching %s event to %s %p", NS_ConvertUTF16toUTF8(aEventType).get(), + EncoderType::Name.get(), this); + RefPtr event = new Event(this, nullptr, nullptr); + event->InitEvent(aEventType, true, true); + event->SetTrusted(true); + this->DispatchEvent(*event); + return NS_OK; +} + +template +void EncoderTemplate::SchedulePromiseResolveOrReject( + already_AddRefed aPromise, const nsresult& aResult) { + AssertIsOnOwningThread(); + + RefPtr p = aPromise; + auto resolver = [p, result = aResult] { + if (NS_FAILED(result)) { + p->MaybeReject(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + return; + } + p->MaybeResolveWithUndefined(); + }; + nsISerialEventTarget* target = GetCurrentSerialEventTarget(); + + if (NS_IsMainThread()) { + MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewRunnableFunction( + "SchedulePromiseResolveOrReject Runnable (main)", resolver))); + return; + } + + MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewCancelableRunnableFunction( + "SchedulePromiseResolveOrReject Runnable (worker)", resolver))); +} + +template +void EncoderTemplate::ProcessControlMessageQueue() { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + + while (!mMessageQueueBlocked && !mControlMessageQueue.empty()) { + RefPtr& msg = mControlMessageQueue.front(); + if (msg->AsConfigureMessage()) { + if (ProcessConfigureMessage(msg->AsConfigureMessage()) == + MessageProcessedResult::NotProcessed) { + break; + } + } else if (msg->AsEncodeMessage()) { + if (ProcessEncodeMessage(msg->AsEncodeMessage()) == + MessageProcessedResult::NotProcessed) { + break; + } + } else { + MOZ_ASSERT(msg->AsFlushMessage()); + if (ProcessFlushMessage(msg->AsFlushMessage()) == + MessageProcessedResult::NotProcessed) { + break; + } + } + } +} + +template +void EncoderTemplate::CancelPendingControlMessages( + const nsresult& aResult) { + AssertIsOnOwningThread(); + + // Cancel the message that is being processed. + if (mProcessingMessage) { + LOG("%s %p cancels current %s", EncoderType::Name.get(), this, + mProcessingMessage->ToString().get()); + mProcessingMessage->Cancel(); + + if (RefPtr flush = mProcessingMessage->AsFlushMessage()) { + flush->RejectPromiseIfAny(aResult); + } + + mProcessingMessage = nullptr; + } + + // Clear the message queue. + while (!mControlMessageQueue.empty()) { + LOG("%s %p cancels pending %s", EncoderType::Name.get(), this, + mControlMessageQueue.front()->ToString().get()); + + MOZ_ASSERT(!mControlMessageQueue.front()->IsProcessing()); + if (RefPtr flush = + mControlMessageQueue.front()->AsFlushMessage()) { + flush->RejectPromiseIfAny(aResult); + } + + mControlMessageQueue.pop(); + } +} + +template +MessageProcessedResult EncoderTemplate::ProcessConfigureMessage( + RefPtr aMessage) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(aMessage->AsConfigureMessage()); + + if (mProcessingMessage) { + return MessageProcessedResult::NotProcessed; + } + + mProcessingMessage = aMessage; + mControlMessageQueue.pop(); + LOG("%s %p Configuring, message queue processing blocked(%s)", + EncoderType::Name.get(), this, aMessage->ToString().get()); + StartBlockingMessageQueue(); + + bool supported = EncoderType::IsSupported(*aMessage->Config()); + + if (!supported) { + LOGE("%s %p ProcessConfigureMessage error (sync): Not supported", + EncoderType::Name.get(), this); + mProcessingMessage = nullptr; + NS_DispatchToCurrentThread(NS_NewRunnableFunction( + "ProcessConfigureMessage (async): not supported", + [self = RefPtr(this)]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { + LOGE("%s %p ProcessConfigureMessage (async close): Not supported", + EncoderType::Name.get(), self.get()); + DebugOnly> r = + self->CloseInternal(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + MOZ_ASSERT(r.value.isOk()); + })); + return MessageProcessedResult::Processed; + } + + if (mAgent) { + Reconfigure(aMessage); + } else { + Configure(aMessage); + } + + return MessageProcessedResult::Processed; +} + +template +void EncoderTemplate::StartBlockingMessageQueue() { + LOG("=== Message queue blocked"); + mMessageQueueBlocked = true; +} + +template +void EncoderTemplate::StopBlockingMessageQueue() { + LOG("=== Message queue unblocked"); + mMessageQueueBlocked = false; +} + +template +void EncoderTemplate::Reconfigure( + RefPtr aMessage) { + MOZ_ASSERT(mAgent); + + LOG("Reconfiguring encoder: %s", + NS_ConvertUTF16toUTF8(aMessage->Config()->ToString()).get()); + + RefPtr config = aMessage->Config(); + RefPtr configDiff = + config->Diff(*mActiveConfig); + + // Nothing to do, return now + if (configDiff->Empty()) { + LOG("Reconfigure with identical config, returning."); + mProcessingMessage = nullptr; + StopBlockingMessageQueue(); + return; + } + + LOG("Attempting to reconfigure encoder: old: %s new: %s, diff: %s", + NS_ConvertUTF16toUTF8(mActiveConfig->ToString()).get(), + NS_ConvertUTF16toUTF8(config->ToString()).get(), + NS_ConvertUTF16toUTF8(configDiff->ToString()).get()); + + RefPtr changeList = + configDiff->ToPEMChangeList(); + + // Attempt to reconfigure the encoder, if the config is similar enough. + // Otherwise, or if reconfiguring on the fly didn't work, flush the encoder + // and recreate a new one. + + mAgent->Reconfigure(changeList) + ->Then( + GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = mAgent->mId, + message = std::move(aMessage)]( + const EncoderAgent::ReconfigurationPromise::ResolveOrRejectValue& + aResult) { + MOZ_ASSERT(self->mProcessingMessage); + MOZ_ASSERT(self->mProcessingMessage->AsConfigureMessage()); + MOZ_ASSERT(self->mState == CodecState::Configured); + MOZ_ASSERT(self->mAgent); + MOZ_ASSERT(id == self->mAgent->mId); + MOZ_ASSERT(self->mActiveConfig); + + if (aResult.IsReject()) { + LOGE( + "Reconfiguring on the fly didn't succeed, flushing and " + "configuring a new encoder"); + self->mAgent->Drain()->Then( + GetCurrentSerialEventTarget(), __func__, + [self, id, + message](EncoderAgent::EncodePromise::ResolveOrRejectValue&& + aResult) { + if (aResult.IsReject()) { + const MediaResult& error = aResult.RejectValue(); + LOGE( + "%s %p, EncoderAgent #%zu failed to flush during " + "reconfigure, closing: %s", + EncoderType::Name.get(), self.get(), id, + error.Description().get()); + + self->mProcessingMessage = nullptr; + self->ScheduleClose( + NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + return; + } + + LOG("%s %p flush during reconfiguration succeeded.", + EncoderType::Name.get(), self.get()); + + // If flush succeeded, schedule to output encoded data + // first, destroy the current encoder, and proceed to create + // a new one. + MOZ_ASSERT(aResult.IsResolve()); + nsTArray> data = + std::move(aResult.ResolveValue()); + + if (data.IsEmpty()) { + LOG("%s %p no data during flush for reconfiguration with " + "encoder destruction", + EncoderType::Name.get(), self.get()); + } else { + LOG("%s %p Outputing %zu frames during flush " + " for reconfiguration with encoder destruction", + EncoderType::Name.get(), self.get(), data.Length()); + self->ScheduleOutputEncodedData( + std::move(data), + nsLiteralCString("Flush before reconfigure")); + } + + NS_DispatchToCurrentThread(NS_NewRunnableFunction( + "Destroy + recreate encoder after failed reconfigure", + [self = RefPtr(self), message]() + MOZ_CAN_RUN_SCRIPT_BOUNDARY { + // Destroy the agent, and finally create a fresh + // encoder with the new configuration. + self->DestroyEncoderAgentIfAny(); + self->Configure(message); + })); + }); + return; + } + + LOG("%s %p, EncoderAgent #%zu has been reconfigured on the fly to " + "%s", + EncoderType::Name.get(), self.get(), id, + message->ToString().get()); + + self->mOutputNewDecoderConfig = true; + self->mActiveConfig = message->Config(); + self->mProcessingMessage = nullptr; + self->StopBlockingMessageQueue(); + self->ProcessControlMessageQueue(); + }); +} + +template +void EncoderTemplate::Configure( + RefPtr aMessage) { + MOZ_ASSERT(!mAgent); + + LOG("Configuring encoder: %s", + NS_ConvertUTF16toUTF8(aMessage->Config()->ToString()).get()); + + mOutputNewDecoderConfig = true; + mActiveConfig = aMessage->Config(); + + bool decoderAgentCreated = + CreateEncoderAgent(aMessage->mMessageId, aMessage->Config()); + if (!decoderAgentCreated) { + LOGE( + "%s %p ProcessConfigureMessage error (sync): encoder agent " + "creation " + "failed", + EncoderType::Name.get(), this); + mProcessingMessage = nullptr; + NS_DispatchToCurrentThread(NS_NewRunnableFunction( + "ProcessConfigureMessage (async): encoder agent creating failed", + [self = RefPtr(this)]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { + LOGE( + "%s %p ProcessConfigureMessage (async close): encoder agent " + "creation failed", + EncoderType::Name.get(), self.get()); + DebugOnly> r = + self->CloseInternal(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + MOZ_ASSERT(r.value.isOk()); + })); + return; + } + + MOZ_ASSERT(mAgent); + MOZ_ASSERT(mActiveConfig); + + LOG("Real configuration with fresh config: %s", + NS_ConvertUTF16toUTF8(mActiveConfig->ToString().get()).get()); + + EncoderConfig config = mActiveConfig->ToEncoderConfig(); + mAgent->Configure(config) + ->Then(GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = mAgent->mId, aMessage]( + const EncoderAgent::ConfigurePromise::ResolveOrRejectValue& + aResult) MOZ_CAN_RUN_SCRIPT_BOUNDARY { + MOZ_ASSERT(self->mProcessingMessage); + MOZ_ASSERT(self->mProcessingMessage->AsConfigureMessage()); + MOZ_ASSERT(self->mState == CodecState::Configured); + MOZ_ASSERT(self->mAgent); + MOZ_ASSERT(id == self->mAgent->mId); + MOZ_ASSERT(self->mActiveConfig); + + LOG("%s %p, EncoderAgent #%zu %s has been %s. now unblocks " + "message-queue-processing", + EncoderType::Name.get(), self.get(), id, + aMessage->ToString().get(), + aResult.IsResolve() ? "resolved" : "rejected"); + + aMessage->Complete(); + self->mProcessingMessage = nullptr; + + if (aResult.IsReject()) { + // The spec asks to close the decoder with an + // NotSupportedError so we log the exact error here. + const MediaResult& error = aResult.RejectValue(); + LOGE("%s %p, EncoderAgent #%zu failed to configure: %s", + EncoderType::Name.get(), self.get(), id, + error.Description().get()); + DebugOnly> r = self->CloseInternal( + NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + MOZ_ASSERT(r.value.isOk()); + return; // No further process + } + + self->StopBlockingMessageQueue(); + self->ProcessControlMessageQueue(); + }) + ->Track(aMessage->Request()); +} + +template +MessageProcessedResult EncoderTemplate::ProcessEncodeMessage( + RefPtr aMessage) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(aMessage->AsEncodeMessage()); + + if (mProcessingMessage) { + return MessageProcessedResult::NotProcessed; + } + + mProcessingMessage = aMessage; + mControlMessageQueue.pop(); + + LOGV("%s %p processing %s", EncoderType::Name.get(), this, + aMessage->ToString().get()); + + mEncodeQueueSize -= 1; + ScheduleDequeueEvent(); + + // Treat it like decode error if no EncoderAgent is available or the encoded + // data is invalid. + auto closeOnError = [&]() { + mProcessingMessage = nullptr; + ScheduleClose(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + return MessageProcessedResult::Processed; + }; + + if (!mAgent) { + LOGE("%s %p is not configured", EncoderType::Name.get(), this); + return closeOnError(); + } + + MOZ_ASSERT(mActiveConfig); + RefPtr data = aMessage->mData; + if (!data) { + LOGE("%s %p, data for %s is empty or invalid", EncoderType::Name.get(), + this, aMessage->ToString().get()); + return closeOnError(); + } + + mAgent->Encode(data.get()) + ->Then(GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = mAgent->mId, aMessage]( + EncoderAgent::EncodePromise::ResolveOrRejectValue&& aResult) { + MOZ_ASSERT(self->mProcessingMessage); + MOZ_ASSERT(self->mProcessingMessage->AsEncodeMessage()); + MOZ_ASSERT(self->mState == CodecState::Configured); + MOZ_ASSERT(self->mAgent); + MOZ_ASSERT(id == self->mAgent->mId); + MOZ_ASSERT(self->mActiveConfig); + + nsCString msgStr = aMessage->ToString(); + + aMessage->Complete(); + self->mProcessingMessage = nullptr; + + if (aResult.IsReject()) { + // The spec asks to queue a task to run close the decoder + // with an EncodingError so we log the exact error here. + const MediaResult& error = aResult.RejectValue(); + LOGE("%s %p, EncoderAgent #%zu %s failed: %s", + EncoderType::Name.get(), self.get(), id, msgStr.get(), + error.Description().get()); + self->ScheduleClose(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + return; // No further process + } + + MOZ_ASSERT(aResult.IsResolve()); + nsTArray> data = + std::move(aResult.ResolveValue()); + if (data.IsEmpty()) { + LOGV("%s %p got no data for %s", EncoderType::Name.get(), + self.get(), msgStr.get()); + } else { + LOGV("%s %p, schedule %zu encoded data output", + EncoderType::Name.get(), self.get(), data.Length()); + self->ScheduleOutputEncodedData(std::move(data), msgStr); + } + + self->ProcessControlMessageQueue(); + }) + ->Track(aMessage->Request()); + + return MessageProcessedResult::Processed; +} + +template +MessageProcessedResult EncoderTemplate::ProcessFlushMessage( + RefPtr aMessage) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(aMessage->AsFlushMessage()); + + if (mProcessingMessage) { + return MessageProcessedResult::NotProcessed; + } + + mProcessingMessage = aMessage; + mControlMessageQueue.pop(); + + LOG("%s %p starts processing %s", EncoderType::Name.get(), this, + aMessage->ToString().get()); + + // No agent, no thing to do. The promise has been rejected with the + // appropriate error in ResetInternal already. + if (!mAgent) { + LOGE("%s %p no agent, nothing to do", EncoderType::Name.get(), this); + mProcessingMessage = nullptr; + return MessageProcessedResult::Processed; + } + + mAgent->Drain() + ->Then(GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = mAgent->mId, aMessage]( + EncoderAgent::EncodePromise::ResolveOrRejectValue&& aResult) { + MOZ_ASSERT(self->mProcessingMessage); + MOZ_ASSERT(self->mProcessingMessage->AsFlushMessage()); + MOZ_ASSERT(self->mState == CodecState::Configured); + MOZ_ASSERT(self->mAgent); + MOZ_ASSERT(id == self->mAgent->mId); + MOZ_ASSERT(self->mActiveConfig); + + LOG("%s %p, EncoderAgent #%zu %s has been %s", + EncoderType::Name.get(), self.get(), id, + aMessage->ToString().get(), + aResult.IsResolve() ? "resolved" : "rejected"); + + nsCString msgStr = aMessage->ToString(); + + aMessage->Complete(); + + // If flush failed, it means encoder fails to encode the data + // sent before, so we treat it like an encode error. We reject + // the promise first and then queue a task to close VideoEncoder + // with an EncodingError. + if (aResult.IsReject()) { + const MediaResult& error = aResult.RejectValue(); + LOGE("%s %p, EncoderAgent #%zu failed to flush: %s", + EncoderType::Name.get(), self.get(), id, + error.Description().get()); + + // Reject with an EncodingError instead of the error we got + // above. + self->SchedulePromiseResolveOrReject( + aMessage->TakePromise(), + NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + + self->mProcessingMessage = nullptr; + + self->ScheduleClose(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + return; // No further process + } + + // If flush succeeded, schedule to output encoded data first + // and then resolve the promise, then keep processing the + // control messages. + MOZ_ASSERT(aResult.IsResolve()); + nsTArray> data = + std::move(aResult.ResolveValue()); + + if (data.IsEmpty()) { + LOG("%s %p gets no data for %s", EncoderType::Name.get(), + self.get(), msgStr.get()); + } else { + LOG("%s %p, schedule %zu encoded data output for %s", + EncoderType::Name.get(), self.get(), data.Length(), + msgStr.get()); + self->ScheduleOutputEncodedData(std::move(data), msgStr); + } + + self->SchedulePromiseResolveOrReject(aMessage->TakePromise(), + NS_OK); + self->mProcessingMessage = nullptr; + self->ProcessControlMessageQueue(); + }) + ->Track(aMessage->Request()); + + return MessageProcessedResult::Processed; +} + +// CreateEncoderAgent will create an EncoderAgent paired with a xpcom-shutdown +// blocker and a worker-reference. Besides the needs mentioned in the header +// file, the blocker and the worker-reference also provides an entry point for +// us to clean up the resources. Other than the encoder dtor, Reset(), or +// Close(), the resources should be cleaned up in the following situations: +// 1. Encoder on window, closing document +// 2. Encoder on worker, closing document +// 3. Encoder on worker, terminating worker +// +// In case 1, the entry point to clean up is in the mShutdownBlocker's +// ShutdownpPomise-resolver. In case 2, the entry point is in mWorkerRef's +// shutting down callback. In case 3, the entry point is in mWorkerRef's +// shutting down callback. + +template +bool EncoderTemplate::CreateEncoderAgent( + WebCodecsId aId, RefPtr aConfig) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(!mAgent); + MOZ_ASSERT(!mShutdownBlocker); + MOZ_ASSERT_IF(!NS_IsMainThread(), !mWorkerRef); + + auto resetOnFailure = MakeScopeExit([&]() { + mAgent = nullptr; + mActiveConfig = nullptr; + mShutdownBlocker = nullptr; + mWorkerRef = nullptr; + }); + + // If the encoder is on worker, get a worker reference. + if (!NS_IsMainThread()) { + WorkerPrivate* workerPrivate = GetCurrentThreadWorkerPrivate(); + if (NS_WARN_IF(!workerPrivate)) { + return false; + } + + // Clean up all the resources when worker is going away. + RefPtr workerRef = StrongWorkerRef::Create( + workerPrivate, "EncoderTemplate::CreateEncoderAgent", + [self = RefPtr{this}]() { + LOG("%s %p, worker is going away", EncoderType::Name.get(), + self.get()); + Unused << self->ResetInternal(NS_ERROR_DOM_ABORT_ERR); + }); + if (NS_WARN_IF(!workerRef)) { + return false; + } + + mWorkerRef = new ThreadSafeWorkerRef(workerRef); + } + + mAgent = MakeRefPtr(aId); + + // ShutdownBlockingTicket requires an unique name to register its own + // nsIAsyncShutdownBlocker since each blocker needs a distinct name. + // To do that, we use EncoderAgent's unique id to create a unique name. + nsAutoString uniqueName; + uniqueName.AppendPrintf( + "Blocker for EncoderAgent #%zu (codec: %s) @ %p", mAgent->mId, + NS_ConvertUTF16toUTF8(mActiveConfig->mCodec).get(), mAgent.get()); + + mShutdownBlocker = media::ShutdownBlockingTicket::Create( + uniqueName, NS_LITERAL_STRING_FROM_CSTRING(__FILE__), __LINE__); + if (!mShutdownBlocker) { + LOGE("%s %p failed to create %s", EncoderType::Name.get(), this, + NS_ConvertUTF16toUTF8(uniqueName).get()); + return false; + } + + // Clean up all the resources when xpcom-will-shutdown arrives since the + // page is going to be closed. + mShutdownBlocker->ShutdownPromise()->Then( + GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = mAgent->mId, + ref = mWorkerRef](bool /* aUnUsed*/) { + LOG("%s %p gets xpcom-will-shutdown notification for EncoderAgent " + "#%zu", + EncoderType::Name.get(), self.get(), id); + Unused << self->ResetInternal(NS_ERROR_DOM_ABORT_ERR); + }, + [self = RefPtr{this}, id = mAgent->mId, + ref = mWorkerRef](bool /* aUnUsed*/) { + LOG("%s %p removes shutdown-blocker #%zu before getting any " + "notification. EncoderAgent should have been dropped", + EncoderType::Name.get(), self.get(), id); + MOZ_ASSERT(!self->mAgent || self->mAgent->mId != id); + }); + + LOG("%s %p creates EncoderAgent #%zu @ %p and its shutdown-blocker", + EncoderType::Name.get(), this, mAgent->mId, mAgent.get()); + + resetOnFailure.release(); + return true; +} + +template +void EncoderTemplate::DestroyEncoderAgentIfAny() { + AssertIsOnOwningThread(); + + if (!mAgent) { + LOG("%s %p has no EncoderAgent to destroy", EncoderType::Name.get(), this); + return; + } + + MOZ_ASSERT(mActiveConfig); + MOZ_ASSERT(mShutdownBlocker); + MOZ_ASSERT_IF(!NS_IsMainThread(), mWorkerRef); + + LOG("%s %p destroys EncoderAgent #%zu @ %p", EncoderType::Name.get(), this, + mAgent->mId, mAgent.get()); + mActiveConfig = nullptr; + RefPtr agent = std::move(mAgent); + // mShutdownBlocker should be kept alive until the shutdown is done. + // mWorkerRef is used to ensure this task won't be discarded in worker. + agent->Shutdown()->Then( + GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = agent->mId, ref = std::move(mWorkerRef), + blocker = std::move(mShutdownBlocker)]( + const ShutdownPromise::ResolveOrRejectValue& aResult) { + LOG("%s %p, EncoderAgent #%zu's shutdown has been %s. Drop its " + "shutdown-blocker now", + EncoderType::Name.get(), self.get(), id, + aResult.IsResolve() ? "resolved" : "rejected"); + }); +} + +template class EncoderTemplate; + +#undef LOG +#undef LOGW +#undef LOGE +#undef LOGV +#undef LOG_INTERNAL + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/EncoderTemplate.h b/dom/media/webcodecs/EncoderTemplate.h new file mode 100644 index 0000000000..e53d7166d1 --- /dev/null +++ b/dom/media/webcodecs/EncoderTemplate.h @@ -0,0 +1,290 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_EncoderTemplate_h +#define mozilla_dom_EncoderTemplate_h + +#include + +#include "EncoderAgent.h" +#include "MediaData.h" +#include "WebCodecsUtils.h" +#include "mozilla/DOMEventTargetHelper.h" +#include "mozilla/MozPromise.h" +#include "mozilla/RefPtr.h" +#include "mozilla/Result.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/dom/VideoEncoderBinding.h" +#include "mozilla/dom/WorkerRef.h" +#include "mozilla/media/MediaUtils.h" +#include "nsStringFwd.h" + +namespace mozilla::dom { + +class WebCodecsErrorCallback; +class Promise; +enum class CodecState : uint8_t; + +using Id = size_t; + +template +class EncoderTemplate : public DOMEventTargetHelper { + using Self = EncoderTemplate; + using ConfigType = typename EncoderType::ConfigType; + using ConfigTypeInternal = typename EncoderType::ConfigTypeInternal; + using OutputConfigType = typename EncoderType::OutputConfigType; + using InputType = typename EncoderType::InputType; + using InputTypeInternal = typename EncoderType::InputTypeInternal; + using OutputType = typename EncoderType::OutputType; + using OutputCallbackType = typename EncoderType::OutputCallbackType; + + /* ControlMessage classes */ + protected: + class ConfigureMessage; + class EncodeMessage; + class FlushMessage; + + class ControlMessage { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ControlMessage) + explicit ControlMessage(Id aConfigureId); + virtual void Cancel() = 0; + virtual bool IsProcessing() = 0; + + virtual nsCString ToString() const = 0; + virtual RefPtr AsConfigureMessage() { return nullptr; } + virtual RefPtr AsEncodeMessage() { return nullptr; } + virtual RefPtr AsFlushMessage() { return nullptr; } + + // For logging purposes + const WebCodecsId mConfigureId; + const WebCodecsId mMessageId; + + protected: + virtual ~ControlMessage() = default; + }; + + class ConfigureMessage final + : public ControlMessage, + public MessageRequestHolder { + public: + ConfigureMessage(Id aConfigureId, + const RefPtr& aConfig); + virtual void Cancel() override { Disconnect(); } + virtual bool IsProcessing() override { return Exists(); }; + virtual RefPtr AsConfigureMessage() override { + return this; + } + RefPtr Config() { return mConfig; } + nsCString ToString() const override { + nsCString rv; + rv.AppendPrintf( + "ConfigureMessage(#%zu): %s", this->mMessageId, + mConfig ? NS_ConvertUTF16toUTF8(mConfig->ToString().get()).get() + : "null cfg"); + return rv; + } + + private: + const RefPtr mConfig; + }; + + class EncodeMessage final + : public ControlMessage, + public MessageRequestHolder { + public: + EncodeMessage(WebCodecsId aConfigureId, RefPtr&& aData, + Maybe&& aOptions = Nothing()); + nsCString ToString() const override { + nsCString rv; + bool isKeyFrame = mOptions.isSome() && mOptions.ref().mKeyFrame; + rv.AppendPrintf("EncodeMessage(#%zu,#%zu): %s (%s)", this->mConfigureId, + this->mMessageId, mData->ToString().get(), + isKeyFrame ? "kf" : ""); + return rv; + } + virtual void Cancel() override { Disconnect(); } + virtual bool IsProcessing() override { return Exists(); }; + virtual RefPtr AsEncodeMessage() override { return this; } + RefPtr mData; + Maybe mOptions; + }; + + class FlushMessage final + : public ControlMessage, + public MessageRequestHolder { + public: + FlushMessage(WebCodecsId aConfigureId, Promise* aPromise); + virtual void Cancel() override { Disconnect(); } + virtual bool IsProcessing() override { return Exists(); }; + virtual RefPtr AsFlushMessage() override { return this; } + already_AddRefed TakePromise() { return mPromise.forget(); } + void RejectPromiseIfAny(const nsresult& aReason); + + nsCString ToString() const override { + nsCString rv; + rv.AppendPrintf("FlushMessage(#%zu,#%zu)", this->mConfigureId, + this->mMessageId); + return rv; + } + + private: + RefPtr mPromise; + }; + + protected: + EncoderTemplate(nsIGlobalObject* aGlobalObject, + RefPtr&& aErrorCallback, + RefPtr&& aOutputCallback); + + virtual ~EncoderTemplate() = default; + + /* WebCodecs interfaces */ + public: + IMPL_EVENT_HANDLER(dequeue) + + void StartBlockingMessageQueue(); + void StopBlockingMessageQueue(); + + CodecState State() const { return mState; }; + + uint32_t EncodeQueueSize() const { return mEncodeQueueSize; }; + + void Configure(const ConfigType& aConfig, ErrorResult& aRv); + + void EncodeAudioData(InputType& aInput, ErrorResult& aRv); + void EncodeVideoFrame(InputType& aInput, + const VideoEncoderEncodeOptions& aOptions, + ErrorResult& aRv); + + already_AddRefed Flush(ErrorResult& aRv); + + void Reset(ErrorResult& aRv); + + MOZ_CAN_RUN_SCRIPT + void Close(ErrorResult& aRv); + + /* Type conversion functions for the Encoder implementation */ + protected: + virtual RefPtr EncodedDataToOutputType( + nsIGlobalObject* aGlobalObject, RefPtr& aData) = 0; + virtual OutputConfigType EncoderConfigToDecoderConfig( + nsIGlobalObject* aGlobalObject, const RefPtr& aData, + const ConfigTypeInternal& aOutputConfig) const = 0; + /* Internal member variables and functions */ + protected: + // EncoderTemplate can run on either main thread or worker thread. + void AssertIsOnOwningThread() const { + NS_ASSERT_OWNINGTHREAD(EncoderTemplate); + } + + Result ResetInternal(const nsresult& aResult); + MOZ_CAN_RUN_SCRIPT_BOUNDARY + Result CloseInternal(const nsresult& aResult); + + MOZ_CAN_RUN_SCRIPT void ReportError(const nsresult& aResult); + MOZ_CAN_RUN_SCRIPT void OutputEncodedData( + nsTArray>&& aData); + + class ErrorRunnable; + void ScheduleReportError(const nsresult& aResult); + + class OutputRunnable; + void ScheduleOutputEncodedData(nsTArray>&& aData, + const nsACString& aLabel); + + void ScheduleClose(const nsresult& aResult); + + void ScheduleDequeueEvent(); + nsresult FireEvent(nsAtom* aTypeWithOn, const nsAString& aEventType); + + void SchedulePromiseResolveOrReject(already_AddRefed aPromise, + const nsresult& aResult); + + void ProcessControlMessageQueue(); + void CancelPendingControlMessages(const nsresult& aResult); + + MessageProcessedResult ProcessConfigureMessage( + RefPtr aMessage); + + MessageProcessedResult ProcessEncodeMessage(RefPtr aMessage); + + MessageProcessedResult ProcessFlushMessage(RefPtr aMessage); + + void Configure(RefPtr aMessage); + void Reconfigure(RefPtr aMessage); + + // Returns true when mAgent can be created. + bool CreateEncoderAgent(WebCodecsId aId, RefPtr aConfig); + void DestroyEncoderAgentIfAny(); + + // Constant in practice, only set in ctor. + RefPtr mErrorCallback; + RefPtr mOutputCallback; + + CodecState mState; + + bool mMessageQueueBlocked; + std::queue> mControlMessageQueue; + RefPtr mProcessingMessage; + + uint32_t mEncodeQueueSize; + bool mDequeueEventScheduled; + + // A unique id tracking the ConfigureMessage and will be used as the + // EncoderAgent's Id. + uint32_t mLatestConfigureId; + // Tracking how many encoded data has been enqueued and this number will be + // used as the EncodeMessage's Id. + size_t mEncodeCounter; + // Tracking how many flush request has been enqueued and this number will be + // used as the FlushMessage's Id. + size_t mFlushCounter; + + // EncoderAgent will be created the first time "configure" is being processed, + // and will be destroyed when "reset" is called. If another "configure" is + // called, either it's possible to reconfigure the underlying encoder without + // tearing eveyrthing down (e.g. a bitrate change), or it's not possible, and + // the current encoder will be destroyed and a new one create. + // In both cases, the encoder is implicitely flushed before the configuration + // change. + // See CanReconfigure on the {Audio,Video}EncoderConfigInternal + RefPtr mAgent; + RefPtr mActiveConfig; + // This is true when a configure call has just been processed, and it's + // necessary to pass the new decoding configuration when the callback is + // called. Read and modified on owner thread only. + bool mOutputNewDecoderConfig = false; + + // Used to add a nsIAsyncShutdownBlocker on main thread to block + // xpcom-shutdown before the underlying MediaDataEncoder is created. The + // blocker will be held until the underlying MediaDataEncoder has been shut + // down. This blocker guarantees RemoteEncoderManagerChild's thread, where + // the underlying RemoteMediaDataEncoder is on, outlives the + // RemoteMediaDataEncoder since the thread releasing, which happens on main + // thread when getting a xpcom-shutdown signal, is blocked by the added + // blocker. As a result, RemoteMediaDataEncoder can safely work on worker + // thread with a holding blocker (otherwise, if RemoteEncoderManagerChild + // releases its thread on main thread before RemoteMediaDataEncoder's + // Shutdown() task run on worker thread, RemoteMediaDataEncoder has no + // thread to run). + UniquePtr mShutdownBlocker; + + // Held to make sure the dispatched tasks can be done before worker is going + // away. As long as this worker-ref is held somewhere, the tasks dispatched + // to the worker can be executed (otherwise the tasks would be canceled). + // This ref should be activated as long as the underlying MediaDataEncoder + // is alive, and should keep alive until mShutdownBlocker is dropped, so all + // MediaDataEncoder's tasks and mShutdownBlocker-releasing task can be + // executed. + // TODO: Use StrongWorkerRef instead if this is always used in the same + // thread? + RefPtr mWorkerRef; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_EncoderTemplate_h diff --git a/dom/media/webcodecs/EncoderTypes.h b/dom/media/webcodecs/EncoderTypes.h new file mode 100644 index 0000000000..d58d7c54c8 --- /dev/null +++ b/dom/media/webcodecs/EncoderTypes.h @@ -0,0 +1,103 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_EncoderTypes_h +#define mozilla_dom_EncoderTypes_h + +#include "mozilla/Maybe.h" +#include "mozilla/dom/EncodedVideoChunk.h" +#include "mozilla/dom/VideoEncoderBinding.h" +#include "mozilla/dom/VideoFrame.h" +#include "mozilla/dom/VideoFrameBinding.h" +#include "nsStringFwd.h" +#include "nsTLiteralString.h" +#include "VideoDecoder.h" +#include "PlatformEncoderModule.h" + +namespace mozilla { + +class TrackInfo; +class MediaByteBuffer; + +namespace dom { + +class VideoEncoderConfigInternal { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoEncoderConfigInternal); + explicit VideoEncoderConfigInternal(const VideoEncoderConfig& aConfig); + explicit VideoEncoderConfigInternal( + const VideoEncoderConfigInternal& aConfig); + + // Returns an EncoderConfig struct with as many filled members as + // possible. + // TODO: handle codec specific things + EncoderConfig ToEncoderConfig() const; + + bool Equals(const VideoEncoderConfigInternal& aOther) const; + bool CanReconfigure(const VideoEncoderConfigInternal& aOther) const; + already_AddRefed Diff( + const VideoEncoderConfigInternal& aOther) const; + nsString ToString() const; + + nsString mCodec; + uint32_t mWidth; + uint32_t mHeight; + Maybe mDisplayWidth; + Maybe mDisplayHeight; + Maybe mBitrate; + Maybe mFramerate; + HardwareAcceleration mHardwareAcceleration; + AlphaOption mAlpha; + Maybe mScalabilityMode; + VideoEncoderBitrateMode mBitrateMode; + LatencyMode mLatencyMode; + Maybe mContentHint; + Maybe mAvc; + + private: + VideoEncoderConfigInternal( + const nsAString& aCodec, uint32_t aWidth, uint32_t aHeight, + Maybe&& aDisplayWidth, Maybe&& aDisplayHeight, + Maybe&& aBitrate, Maybe&& aFramerate, + const HardwareAcceleration& aHardwareAcceleration, + const AlphaOption& aAlpha, Maybe&& aScalabilityMode, + const VideoEncoderBitrateMode& aBitrateMode, + const LatencyMode& aLatencyMode, Maybe&& aContentHint); + + ~VideoEncoderConfigInternal() = default; +}; + +class VideoEncoderTraits { + public: + static constexpr nsLiteralCString Name = "VideoEncoder"_ns; + using ConfigType = VideoEncoderConfig; + using ConfigTypeInternal = VideoEncoderConfigInternal; + using InputType = dom::VideoFrame; + using InputTypeInternal = mozilla::VideoData; + using OutputConfigType = mozilla::dom::VideoDecoderConfigInternal; + using OutputType = EncodedVideoChunk; + using OutputCallbackType = EncodedVideoChunkOutputCallback; + using MetadataType = EncodedVideoChunkMetadata; + + static bool IsSupported(const ConfigTypeInternal& aConfig); + static bool CanEncodeVideo(const ConfigTypeInternal& aConfig); + static Result, nsresult> CreateTrackInfo( + const ConfigTypeInternal& aConfig); + static bool Validate(const ConfigType& aConfig, nsCString& aErrorMessage); + static RefPtr CreateConfigInternal( + const ConfigType& aConfig); + static RefPtr CreateInputInternal( + const InputType& aInput, const VideoEncoderEncodeOptions& aOptions); + static already_AddRefed EncoderConfigToDecoderConfig( + nsIGlobalObject* aGlobal, + const RefPtr& aData, + const ConfigTypeInternal& mOutputConfig); +}; + +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_EncoderTypes_h diff --git a/dom/media/webcodecs/VideoColorSpace.cpp b/dom/media/webcodecs/VideoColorSpace.cpp new file mode 100644 index 0000000000..5b0dad0f31 --- /dev/null +++ b/dom/media/webcodecs/VideoColorSpace.cpp @@ -0,0 +1,48 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/VideoColorSpace.h" +#include "mozilla/dom/VideoColorSpaceBinding.h" +#include "nsIGlobalObject.h" + +namespace mozilla::dom { + +// Only needed for refcounted objects. +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(VideoColorSpace, mParent) +NS_IMPL_CYCLE_COLLECTING_ADDREF(VideoColorSpace) +NS_IMPL_CYCLE_COLLECTING_RELEASE(VideoColorSpace) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(VideoColorSpace) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +VideoColorSpace::VideoColorSpace(nsIGlobalObject* aParent, + const VideoColorSpaceInit& aInit) + : mParent(aParent), mInit(aInit) { + MOZ_ASSERT(mParent); +} + +nsIGlobalObject* VideoColorSpace::GetParentObject() const { return mParent; } + +JSObject* VideoColorSpace::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + return VideoColorSpace_Binding::Wrap(aCx, this, aGivenProto); +} + +/* static */ +already_AddRefed VideoColorSpace::Constructor( + const GlobalObject& aGlobal, const VideoColorSpaceInit& aInit, + ErrorResult& aRv) { + nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + RefPtr videoColorSpace(new VideoColorSpace(global, aInit)); + return aRv.Failed() ? nullptr : videoColorSpace.forget(); +} + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/VideoColorSpace.h b/dom/media/webcodecs/VideoColorSpace.h new file mode 100644 index 0000000000..e6fdc22317 --- /dev/null +++ b/dom/media/webcodecs/VideoColorSpace.h @@ -0,0 +1,64 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_VideoColorSpace_h +#define mozilla_dom_VideoColorSpace_h + +#include "js/TypeDecls.h" +#include "mozilla/Attributes.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/VideoColorSpaceBinding.h" +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; + +namespace mozilla::dom { + +class VideoColorSpace final : public nsISupports, public nsWrapperCache { + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(VideoColorSpace) + + public: + VideoColorSpace(nsIGlobalObject* aParent, const VideoColorSpaceInit& aInit); + + protected: + ~VideoColorSpace() = default; + + public: + nsIGlobalObject* GetParentObject() const; + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + static already_AddRefed Constructor( + const GlobalObject& aGlobal, const VideoColorSpaceInit& aInit, + ErrorResult& aRv); + + const Nullable& GetPrimaries() const { + return mInit.mPrimaries; + } + + const Nullable& GetTransfer() const { + return mInit.mTransfer; + } + + const Nullable& GetMatrix() const { + return mInit.mMatrix; + } + + const Nullable& GetFullRange() const { return mInit.mFullRange; } + + private: + nsCOMPtr mParent; + const VideoColorSpaceInit mInit; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_VideoColorSpace_h diff --git a/dom/media/webcodecs/VideoDecoder.cpp b/dom/media/webcodecs/VideoDecoder.cpp new file mode 100644 index 0000000000..47ca5bb459 --- /dev/null +++ b/dom/media/webcodecs/VideoDecoder.cpp @@ -0,0 +1,977 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/VideoDecoder.h" +#include "mozilla/dom/VideoDecoderBinding.h" + +#include "DecoderTraits.h" +#include "GPUVideoImage.h" +#include "H264.h" +#include "ImageContainer.h" +#include "MediaContainerType.h" +#include "MediaData.h" +#include "VideoUtils.h" +#include "mozilla/Assertions.h" +#include "mozilla/CheckedInt.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/Logging.h" +#include "mozilla/Maybe.h" +#include "mozilla/StaticPrefs_dom.h" +#include "mozilla/Try.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/EncodedVideoChunk.h" +#include "mozilla/dom/EncodedVideoChunkBinding.h" +#include "mozilla/dom/ImageUtils.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/VideoColorSpaceBinding.h" +#include "mozilla/dom/VideoFrameBinding.h" +#include "mozilla/dom/WebCodecsUtils.h" +#include "nsPrintfCString.h" +#include "nsReadableUtils.h" +#include "nsThreadUtils.h" + +#ifdef XP_MACOSX +# include "MacIOSurfaceImage.h" +#elif MOZ_WAYLAND +# include "mozilla/layers/DMABUFSurfaceImage.h" +# include "mozilla/widget/DMABufSurface.h" +#endif + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla::dom { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOG +# undef LOG +#endif // LOG +#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +#ifdef LOGW +# undef LOGW +#endif // LOGW +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +#ifdef LOGV +# undef LOGV +#endif // LOGV +#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__) + +NS_IMPL_CYCLE_COLLECTION_INHERITED(VideoDecoder, DOMEventTargetHelper, + mErrorCallback, mOutputCallback) +NS_IMPL_ADDREF_INHERITED(VideoDecoder, DOMEventTargetHelper) +NS_IMPL_RELEASE_INHERITED(VideoDecoder, DOMEventTargetHelper) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(VideoDecoder) +NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper) + +/* + * Below are helper classes + */ + +VideoColorSpaceInternal::VideoColorSpaceInternal( + const VideoColorSpaceInit& aColorSpaceInit) + : mFullRange(NullableToMaybe(aColorSpaceInit.mFullRange)), + mMatrix(NullableToMaybe(aColorSpaceInit.mMatrix)), + mPrimaries(NullableToMaybe(aColorSpaceInit.mPrimaries)), + mTransfer(NullableToMaybe(aColorSpaceInit.mTransfer)) {} + +VideoColorSpaceInit VideoColorSpaceInternal::ToColorSpaceInit() const { + VideoColorSpaceInit init; + init.mFullRange = MaybeToNullable(mFullRange); + init.mMatrix = MaybeToNullable(mMatrix); + init.mPrimaries = MaybeToNullable(mPrimaries); + init.mTransfer = MaybeToNullable(mTransfer); + return init; +}; + +static Result, nsresult> GetExtraData( + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer); + +VideoDecoderConfigInternal::VideoDecoderConfigInternal( + const nsAString& aCodec, Maybe&& aCodedHeight, + Maybe&& aCodedWidth, Maybe&& aColorSpace, + Maybe>&& aDescription, + Maybe&& aDisplayAspectHeight, + Maybe&& aDisplayAspectWidth, + const HardwareAcceleration& aHardwareAcceleration, + Maybe&& aOptimizeForLatency) + : mCodec(aCodec), + mCodedHeight(std::move(aCodedHeight)), + mCodedWidth(std::move(aCodedWidth)), + mColorSpace(std::move(aColorSpace)), + mDescription(std::move(aDescription)), + mDisplayAspectHeight(std::move(aDisplayAspectHeight)), + mDisplayAspectWidth(std::move(aDisplayAspectWidth)), + mHardwareAcceleration(aHardwareAcceleration), + mOptimizeForLatency(std::move(aOptimizeForLatency)){}; + +/*static*/ +UniquePtr VideoDecoderConfigInternal::Create( + const VideoDecoderConfig& aConfig) { + nsCString errorMessage; + if (!VideoDecoderTraits::Validate(aConfig, errorMessage)) { + LOGE("Failed to create VideoDecoderConfigInternal: %s", errorMessage.get()); + return nullptr; + } + + Maybe> description; + if (aConfig.mDescription.WasPassed()) { + auto rv = GetExtraData(aConfig.mDescription.Value()); + if (rv.isErr()) { // Invalid description data. + LOGE( + "Failed to create VideoDecoderConfigInternal due to invalid " + "description data. Error: 0x%08" PRIx32, + static_cast(rv.unwrapErr())); + return nullptr; + } + description.emplace(rv.unwrap()); + } + + Maybe colorSpace; + if (aConfig.mColorSpace.WasPassed()) { + colorSpace.emplace(VideoColorSpaceInternal(aConfig.mColorSpace.Value())); + } + + return UniquePtr(new VideoDecoderConfigInternal( + aConfig.mCodec, OptionalToMaybe(aConfig.mCodedHeight), + OptionalToMaybe(aConfig.mCodedWidth), std::move(colorSpace), + std::move(description), OptionalToMaybe(aConfig.mDisplayAspectHeight), + OptionalToMaybe(aConfig.mDisplayAspectWidth), + aConfig.mHardwareAcceleration, + OptionalToMaybe(aConfig.mOptimizeForLatency))); +} + +nsString VideoDecoderConfigInternal::ToString() const { + nsString rv; + + rv.Append(mCodec); + if (mCodedWidth.isSome()) { + rv.AppendPrintf("coded: %dx%d", mCodedWidth.value(), mCodedHeight.value()); + } + if (mDisplayAspectWidth.isSome()) { + rv.AppendPrintf("display %dx%d", mDisplayAspectWidth.value(), + mDisplayAspectHeight.value()); + } + if (mColorSpace.isSome()) { + rv.AppendPrintf("colorspace %s", "todo"); + } + if (mDescription.isSome()) { + rv.AppendPrintf("extradata: %zu bytes", mDescription.value()->Length()); + } + rv.AppendPrintf( + "hw accel: %s", + HardwareAccelerationValues::GetString(mHardwareAcceleration).data()); + if (mOptimizeForLatency.isSome()) { + rv.AppendPrintf("optimize for latency: %s", + mOptimizeForLatency.value() ? "true" : "false"); + } + + return rv; +} + +/* + * The followings are helpers for VideoDecoder methods + */ + +struct MIMECreateParam { + explicit MIMECreateParam(const VideoDecoderConfigInternal& aConfig) + : mParsedCodec(ParseCodecString(aConfig.mCodec).valueOr(EmptyString())), + mWidth(aConfig.mCodedWidth), + mHeight(aConfig.mCodedHeight) {} + explicit MIMECreateParam(const VideoDecoderConfig& aConfig) + : mParsedCodec(ParseCodecString(aConfig.mCodec).valueOr(EmptyString())), + mWidth(OptionalToMaybe(aConfig.mCodedWidth)), + mHeight(OptionalToMaybe(aConfig.mCodedHeight)) {} + + const nsString mParsedCodec; + const Maybe mWidth; + const Maybe mHeight; +}; + +static nsTArray GuessMIMETypes(const MIMECreateParam& aParam) { + const auto codec = NS_ConvertUTF16toUTF8(aParam.mParsedCodec); + nsTArray types; + for (const nsCString& container : GuessContainers(aParam.mParsedCodec)) { + nsPrintfCString mime("video/%s; codecs=%s", container.get(), codec.get()); + if (aParam.mWidth) { + mime.AppendPrintf("; width=%d", *aParam.mWidth); + } + if (aParam.mHeight) { + mime.AppendPrintf("; height=%d", *aParam.mHeight); + } + types.AppendElement(mime); + } + return types; +} + +static bool IsSupportedCodec(const nsAString& aCodec) { + // H265 is unsupported. + if (!IsAV1CodecString(aCodec) && !IsVP9CodecString(aCodec) && + !IsVP8CodecString(aCodec) && !IsH264CodecString(aCodec)) { + return false; + } + + // Gecko allows codec string starts with vp9 or av1 but Webcodecs requires to + // starts with av01 and vp09. + // https://www.w3.org/TR/webcodecs-codec-registry/#video-codec-registry + if (StringBeginsWith(aCodec, u"vp9"_ns) || + StringBeginsWith(aCodec, u"av1"_ns)) { + return false; + } + + return true; +} + +// https://w3c.github.io/webcodecs/#check-configuration-support +template +static bool CanDecode(const Config& aConfig) { + auto param = MIMECreateParam(aConfig); + // TODO: Enable WebCodecs on Android (Bug 1840508) + if (IsOnAndroid()) { + return false; + } + if (!IsSupportedCodec(param.mParsedCodec)) { + return false; + } + if (IsOnMacOS() && IsH264CodecString(param.mParsedCodec) && + !StaticPrefs::dom_media_webcodecs_force_osx_h264_enabled()) { + // This will be fixed in Bug 1846796. + return false; + } + // TODO: Instead of calling CanHandleContainerType with the guessed the + // containers, DecoderTraits should provide an API to tell if a codec is + // decodable or not. + for (const nsCString& mime : GuessMIMETypes(param)) { + if (Maybe containerType = + MakeMediaExtendedMIMEType(mime)) { + if (DecoderTraits::CanHandleContainerType( + *containerType, nullptr /* DecoderDoctorDiagnostics */) != + CANPLAY_NO) { + return true; + } + } + } + return false; +} + +static nsTArray> GetTracksInfo( + const VideoDecoderConfigInternal& aConfig) { + // TODO: Instead of calling GetTracksInfo with the guessed containers, + // DecoderTraits should provide an API to create the TrackInfo directly. + for (const nsCString& mime : GuessMIMETypes(MIMECreateParam(aConfig))) { + if (Maybe containerType = + MakeMediaExtendedMIMEType(mime)) { + if (nsTArray> tracks = + DecoderTraits::GetTracksInfo(*containerType); + !tracks.IsEmpty()) { + return tracks; + } + } + } + return {}; +} + +static Result, nsresult> GetExtraData( + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer) { + RefPtr data = MakeRefPtr(); + if (!AppendTypedArrayDataTo(aBuffer, *data)) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + return data->Length() > 0 ? data : nullptr; +} + +static Result CloneConfiguration( + RootedDictionary& aDest, JSContext* aCx, + const VideoDecoderConfig& aConfig) { + DebugOnly str; + MOZ_ASSERT(VideoDecoderTraits::Validate(aConfig, str)); + + aDest.mCodec = aConfig.mCodec; + if (aConfig.mCodedHeight.WasPassed()) { + aDest.mCodedHeight.Construct(aConfig.mCodedHeight.Value()); + } + if (aConfig.mCodedWidth.WasPassed()) { + aDest.mCodedWidth.Construct(aConfig.mCodedWidth.Value()); + } + if (aConfig.mColorSpace.WasPassed()) { + aDest.mColorSpace.Construct(aConfig.mColorSpace.Value()); + } + if (aConfig.mDescription.WasPassed()) { + aDest.mDescription.Construct(); + MOZ_TRY(CloneBuffer(aCx, aDest.mDescription.Value(), + aConfig.mDescription.Value())); + } + if (aConfig.mDisplayAspectHeight.WasPassed()) { + aDest.mDisplayAspectHeight.Construct(aConfig.mDisplayAspectHeight.Value()); + } + if (aConfig.mDisplayAspectWidth.WasPassed()) { + aDest.mDisplayAspectWidth.Construct(aConfig.mDisplayAspectWidth.Value()); + } + aDest.mHardwareAcceleration = aConfig.mHardwareAcceleration; + if (aConfig.mOptimizeForLatency.WasPassed()) { + aDest.mOptimizeForLatency.Construct(aConfig.mOptimizeForLatency.Value()); + } + + return Ok(); +} + +static Maybe GuessPixelFormat(layers::Image* aImage) { + if (aImage) { + // TODO: Implement ImageUtils::Impl for MacIOSurfaceImage and + // DMABUFSurfaceImage? + if (aImage->AsPlanarYCbCrImage() || aImage->AsNVImage()) { + const ImageUtils imageUtils(aImage); + Maybe f = + ImageBitmapFormatToVideoPixelFormat(imageUtils.GetFormat()); + + // ImageBitmapFormat cannot distinguish YUV420 or YUV420A. + bool hasAlpha = aImage->AsPlanarYCbCrImage() && + aImage->AsPlanarYCbCrImage()->GetData() && + aImage->AsPlanarYCbCrImage()->GetData()->mAlpha; + if (f && *f == VideoPixelFormat::I420 && hasAlpha) { + return Some(VideoPixelFormat::I420A); + } + return f; + } + if (layers::GPUVideoImage* image = aImage->AsGPUVideoImage()) { + RefPtr imageBridge = + layers::ImageBridgeChild::GetSingleton(); + layers::TextureClient* texture = image->GetTextureClient(imageBridge); + if (NS_WARN_IF(!texture)) { + return Nothing(); + } + return SurfaceFormatToVideoPixelFormat(texture->GetFormat()); + } +#ifdef XP_MACOSX + if (layers::MacIOSurfaceImage* image = aImage->AsMacIOSurfaceImage()) { + MOZ_ASSERT(image->GetSurface()); + return SurfaceFormatToVideoPixelFormat(image->GetSurface()->GetFormat()); + } +#endif +#ifdef MOZ_WAYLAND + if (layers::DMABUFSurfaceImage* image = aImage->AsDMABUFSurfaceImage()) { + MOZ_ASSERT(image->GetSurface()); + return SurfaceFormatToVideoPixelFormat(image->GetSurface()->GetFormat()); + } +#endif + } + LOGW("Failed to get pixel format from layers::Image"); + return Nothing(); +} + +static VideoColorSpaceInternal GuessColorSpace( + const layers::PlanarYCbCrData* aData) { + if (!aData) { + LOGE("nullptr in GuessColorSpace"); + return {}; + } + + VideoColorSpaceInternal colorSpace; + colorSpace.mFullRange = Some(ToFullRange(aData->mColorRange)); + if (Maybe m = + ToMatrixCoefficients(aData->mYUVColorSpace)) { + colorSpace.mMatrix = ToMatrixCoefficients(aData->mYUVColorSpace); + colorSpace.mPrimaries = ToPrimaries(aData->mColorPrimaries); + } + if (!colorSpace.mPrimaries) { + LOG("Missing primaries, guessing from colorspace"); + // Make an educated guess based on the coefficients. + colorSpace.mPrimaries = colorSpace.mMatrix.map([](const auto& aMatrix) { + switch (aMatrix) { + case VideoMatrixCoefficients::EndGuard_: + MOZ_CRASH("This should not happen"); + case VideoMatrixCoefficients::Bt2020_ncl: + return VideoColorPrimaries::Bt2020; + case VideoMatrixCoefficients::Rgb: + case VideoMatrixCoefficients::Bt470bg: + case VideoMatrixCoefficients::Smpte170m: + LOGW( + "Warning: Falling back to BT709 when attempting to determine the " + "primaries function of a YCbCr buffer"); + [[fallthrough]]; + case VideoMatrixCoefficients::Bt709: + return VideoColorPrimaries::Bt709; + } + MOZ_ASSERT_UNREACHABLE("Unexpected matrix coefficients"); + LOGW( + "Warning: Falling back to BT709 due to unexpected matrix " + "coefficients " + "when attempting to determine the primaries function of a YCbCr " + "buffer"); + return VideoColorPrimaries::Bt709; + }); + } + + if (Maybe c = + ToTransferCharacteristics(aData->mTransferFunction)) { + colorSpace.mTransfer = Some(*c); + } + if (!colorSpace.mTransfer) { + LOG("Missing transfer characteristics, guessing from colorspace"); + colorSpace.mTransfer = Some(([&] { + switch (aData->mYUVColorSpace) { + case gfx::YUVColorSpace::Identity: + return VideoTransferCharacteristics::Iec61966_2_1; + case gfx::YUVColorSpace::BT2020: + return VideoTransferCharacteristics::Pq; + case gfx::YUVColorSpace::BT601: + LOGW( + "Warning: Falling back to BT709 when attempting to determine the " + "transfer function of a MacIOSurface"); + [[fallthrough]]; + case gfx::YUVColorSpace::BT709: + return VideoTransferCharacteristics::Bt709; + } + MOZ_ASSERT_UNREACHABLE("Unexpected color space"); + LOGW( + "Warning: Falling back to BT709 due to unexpected color space " + "when attempting to determine the transfer function of a " + "MacIOSurface"); + return VideoTransferCharacteristics::Bt709; + })()); + } + + return colorSpace; +} + +#ifdef XP_MACOSX +static VideoColorSpaceInternal GuessColorSpace(const MacIOSurface* aSurface) { + if (!aSurface) { + return {}; + } + VideoColorSpaceInternal colorSpace; + colorSpace.mFullRange = Some(aSurface->IsFullRange()); + if (Maybe m = + ToMatrixCoefficients(aSurface->GetYUVColorSpace())) { + colorSpace.mMatrix = Some(*m); + } + if (Maybe p = ToPrimaries(aSurface->mColorPrimaries)) { + colorSpace.mPrimaries = Some(*p); + } + // Make an educated guess based on the coefficients. + if (aSurface->GetYUVColorSpace() == gfx::YUVColorSpace::Identity) { + colorSpace.mTransfer = Some(VideoTransferCharacteristics::Iec61966_2_1); + } else if (aSurface->GetYUVColorSpace() == gfx::YUVColorSpace::BT709) { + colorSpace.mTransfer = Some(VideoTransferCharacteristics::Bt709); + } else if (aSurface->GetYUVColorSpace() == gfx::YUVColorSpace::BT2020) { + colorSpace.mTransfer = Some(VideoTransferCharacteristics::Pq); + } else { + LOGW( + "Warning: Falling back to BT709 when attempting to determine the " + "transfer function of a MacIOSurface"); + colorSpace.mTransfer = Some(VideoTransferCharacteristics::Bt709); + } + + return colorSpace; +} +#endif +#ifdef MOZ_WAYLAND +// TODO: Set DMABufSurface::IsFullRange() to const so aSurface can be const. +static VideoColorSpaceInternal GuessColorSpace(DMABufSurface* aSurface) { + if (!aSurface) { + return {}; + } + VideoColorSpaceInternal colorSpace; + colorSpace.mFullRange = Some(aSurface->IsFullRange()); + if (Maybe m = + ToMatrixCoefficients(aSurface->GetYUVColorSpace())) { + colorSpace.mMatrix = Some(*m); + } + // No other color space information. + return colorSpace; +} +#endif + +static VideoColorSpaceInternal GuessColorSpace(layers::Image* aImage) { + if (aImage) { + if (layers::PlanarYCbCrImage* image = aImage->AsPlanarYCbCrImage()) { + return GuessColorSpace(image->GetData()); + } + if (layers::NVImage* image = aImage->AsNVImage()) { + return GuessColorSpace(image->GetData()); + } + if (layers::GPUVideoImage* image = aImage->AsGPUVideoImage()) { + VideoColorSpaceInternal colorSpace; + colorSpace.mFullRange = + Some(image->GetColorRange() != gfx::ColorRange::LIMITED); + colorSpace.mMatrix = ToMatrixCoefficients(image->GetYUVColorSpace()); + colorSpace.mPrimaries = ToPrimaries(image->GetColorPrimaries()); + colorSpace.mTransfer = + ToTransferCharacteristics(image->GetTransferFunction()); + // In some circumstances, e.g. on Linux software decoding when using + // VPXDecoder and RDD, the primaries aren't set correctly. Make a good + // guess based on the other params. Fixing this is tracked in + // https://bugzilla.mozilla.org/show_bug.cgi?id=1869825 + if (!colorSpace.mPrimaries) { + if (colorSpace.mMatrix.isSome()) { + switch (colorSpace.mMatrix.value()) { + case VideoMatrixCoefficients::Rgb: + case VideoMatrixCoefficients::Bt709: + colorSpace.mPrimaries = Some(VideoColorPrimaries::Bt709); + break; + case VideoMatrixCoefficients::Bt470bg: + case VideoMatrixCoefficients::Smpte170m: + colorSpace.mPrimaries = Some(VideoColorPrimaries::Bt470bg); + break; + case VideoMatrixCoefficients::Bt2020_ncl: + colorSpace.mPrimaries = Some(VideoColorPrimaries::Bt2020); + break; + case VideoMatrixCoefficients::EndGuard_: + MOZ_ASSERT_UNREACHABLE("bad enum value"); + break; + }; + } + } + return colorSpace; + } +#ifdef XP_MACOSX + // TODO: Make sure VideoFrame can interpret its internal data in different + // formats. + if (layers::MacIOSurfaceImage* image = aImage->AsMacIOSurfaceImage()) { + return GuessColorSpace(image->GetSurface()); + } +#endif +#ifdef MOZ_WAYLAND + // TODO: Make sure VideoFrame can interpret its internal data in different + // formats. + if (layers::DMABUFSurfaceImage* image = aImage->AsDMABUFSurfaceImage()) { + return GuessColorSpace(image->GetSurface()); + } +#endif + } + LOGW("Failed to get color space from layers::Image"); + return {}; +} + +static Result AdjustDisplaySize( + const uint32_t aDisplayAspectWidth, const uint32_t aDisplayAspectHeight, + const gfx::IntSize& aDisplaySize) { + if (aDisplayAspectHeight == 0) { + return Err(NS_ERROR_ILLEGAL_VALUE); + } + + const double aspectRatio = + static_cast(aDisplayAspectWidth) / aDisplayAspectHeight; + + double w = aDisplaySize.width; + double h = aDisplaySize.height; + + if (aspectRatio >= w / h) { + // Adjust w to match the aspect ratio + w = aspectRatio * h; + } else { + // Adjust h to match the aspect ratio + h = w / aspectRatio; + } + + w = std::round(w); + h = std::round(h); + constexpr double MAX = static_cast( + std::numeric_limits::max()); + if (w > MAX || h > MAX || w < 1.0 || h < 1.0) { + return Err(NS_ERROR_ILLEGAL_VALUE); + } + return gfx::IntSize(static_cast(w), + static_cast(h)); +} + +// https://w3c.github.io/webcodecs/#create-a-videoframe +static RefPtr CreateVideoFrame( + nsIGlobalObject* aGlobalObject, const VideoData* aData, int64_t aTimestamp, + uint64_t aDuration, const Maybe aDisplayAspectWidth, + const Maybe aDisplayAspectHeight, + const VideoColorSpaceInternal& aColorSpace) { + MOZ_ASSERT(aGlobalObject); + MOZ_ASSERT(aData); + MOZ_ASSERT((!!aDisplayAspectWidth) == (!!aDisplayAspectHeight)); + + Maybe format = GuessPixelFormat(aData->mImage.get()); + gfx::IntSize displaySize = aData->mDisplay; + if (aDisplayAspectWidth && aDisplayAspectHeight) { + auto r = AdjustDisplaySize(*aDisplayAspectWidth, *aDisplayAspectHeight, + displaySize); + if (r.isOk()) { + displaySize = r.unwrap(); + } + } + + return MakeRefPtr( + aGlobalObject, aData->mImage, format, aData->mImage->GetSize(), + aData->mImage->GetPictureRect(), displaySize, Some(aDuration), aTimestamp, + aColorSpace.ToColorSpaceInit()); +} + +/* static */ +bool VideoDecoderTraits::IsSupported( + const VideoDecoderConfigInternal& aConfig) { + return CanDecode(aConfig); +} + +/* static */ +Result, nsresult> VideoDecoderTraits::CreateTrackInfo( + const VideoDecoderConfigInternal& aConfig) { + LOG("Create a VideoInfo from %s config", + NS_ConvertUTF16toUTF8(aConfig.ToString()).get()); + + nsTArray> tracks = GetTracksInfo(aConfig); + if (tracks.Length() != 1 || tracks[0]->GetType() != TrackInfo::kVideoTrack) { + LOGE("Failed to get TrackInfo"); + return Err(NS_ERROR_INVALID_ARG); + } + + UniquePtr track(std::move(tracks[0])); + VideoInfo* vi = track->GetAsVideoInfo(); + if (!vi) { + LOGE("Failed to get VideoInfo"); + return Err(NS_ERROR_INVALID_ARG); + } + + constexpr uint32_t MAX = static_cast( + std::numeric_limits::max()); + if (aConfig.mCodedHeight.isSome()) { + if (aConfig.mCodedHeight.value() > MAX) { + LOGE("codedHeight overflows"); + return Err(NS_ERROR_INVALID_ARG); + } + vi->mImage.height = static_cast( + aConfig.mCodedHeight.value()); + } + if (aConfig.mCodedWidth.isSome()) { + if (aConfig.mCodedWidth.value() > MAX) { + LOGE("codedWidth overflows"); + return Err(NS_ERROR_INVALID_ARG); + } + vi->mImage.width = + static_cast(aConfig.mCodedWidth.value()); + } + + if (aConfig.mDisplayAspectHeight.isSome()) { + if (aConfig.mDisplayAspectHeight.value() > MAX) { + LOGE("displayAspectHeight overflows"); + return Err(NS_ERROR_INVALID_ARG); + } + vi->mDisplay.height = static_cast( + aConfig.mDisplayAspectHeight.value()); + } + if (aConfig.mDisplayAspectWidth.isSome()) { + if (aConfig.mDisplayAspectWidth.value() > MAX) { + LOGE("displayAspectWidth overflows"); + return Err(NS_ERROR_INVALID_ARG); + } + vi->mDisplay.width = static_cast( + aConfig.mDisplayAspectWidth.value()); + } + + if (aConfig.mColorSpace.isSome()) { + const VideoColorSpaceInternal& colorSpace(aConfig.mColorSpace.value()); + if (colorSpace.mFullRange.isSome()) { + vi->mColorRange = ToColorRange(colorSpace.mFullRange.value()); + } + if (colorSpace.mMatrix.isSome()) { + vi->mColorSpace.emplace(ToColorSpace(colorSpace.mMatrix.value())); + } + // Some decoders get their primaries and transfer function from the codec + // string, and it's already set here. This is the case for VP9 decoders. + if (colorSpace.mPrimaries.isSome()) { + auto primaries = ToPrimaries(colorSpace.mPrimaries.value()); + if (vi->mColorPrimaries.isSome()) { + if (vi->mColorPrimaries.value() != primaries) { + LOG("Conflict between decoder config and codec string, keeping codec " + "string primaries of %d", + static_cast(primaries)); + } + } else { + vi->mColorPrimaries.emplace(primaries); + } + } + if (colorSpace.mTransfer.isSome()) { + auto primaries = ToTransferFunction(colorSpace.mTransfer.value()); + if (vi->mTransferFunction.isSome()) { + if (vi->mTransferFunction.value() != primaries) { + LOG("Conflict between decoder config and codec string, keeping codec " + "string transfer function of %d", + static_cast(vi->mTransferFunction.value())); + } + } else { + vi->mTransferFunction.emplace( + ToTransferFunction(colorSpace.mTransfer.value())); + } + } + } + + if (aConfig.mDescription.isSome()) { + RefPtr buf; + buf = aConfig.mDescription.value(); + if (buf) { + LOG("The given config has %zu bytes of description data", buf->Length()); + if (vi->mExtraData) { + LOGW("The default extra data is overwritten"); + } + vi->mExtraData = buf; + } + + // TODO: Make this utility and replace the similar one in MP4Demuxer.cpp. + if (vi->mExtraData && !vi->mExtraData->IsEmpty() && + IsH264CodecString(aConfig.mCodec)) { + SPSData spsdata; + if (H264::DecodeSPSFromExtraData(vi->mExtraData.get(), spsdata) && + spsdata.pic_width > 0 && spsdata.pic_height > 0 && + H264::EnsureSPSIsSane(spsdata)) { + LOG("H264 sps data - pic size: %d x %d, display size: %d x %d", + spsdata.pic_width, spsdata.pic_height, spsdata.display_width, + spsdata.display_height); + + if (spsdata.pic_width > MAX || spsdata.pic_height > MAX || + spsdata.display_width > MAX || spsdata.display_height > MAX) { + LOGE("H264 width or height in sps data overflows"); + return Err(NS_ERROR_INVALID_ARG); + } + + vi->mImage.width = + static_cast(spsdata.pic_width); + vi->mImage.height = + static_cast(spsdata.pic_height); + vi->mDisplay.width = + static_cast(spsdata.display_width); + vi->mDisplay.height = + static_cast(spsdata.display_height); + } + } + } else { + vi->mExtraData = new MediaByteBuffer(); + } + + LOG("Created a VideoInfo for decoder - %s", + NS_ConvertUTF16toUTF8(vi->ToString()).get()); + + return track; +} + +// https://w3c.github.io/webcodecs/#valid-videodecoderconfig +/* static */ +bool VideoDecoderTraits::Validate(const VideoDecoderConfig& aConfig, + nsCString& aErrorMessage) { + Maybe codec = ParseCodecString(aConfig.mCodec); + if (!codec || codec->IsEmpty()) { + LOGE("Invalid codec string"); + return false; + } + + if (aConfig.mCodedWidth.WasPassed() != aConfig.mCodedHeight.WasPassed()) { + LOGE("Missing coded %s", + aConfig.mCodedWidth.WasPassed() ? "height" : "width"); + return false; + } + if (aConfig.mCodedWidth.WasPassed() && + (aConfig.mCodedWidth.Value() == 0 || aConfig.mCodedHeight.Value() == 0)) { + LOGE("codedWidth and/or codedHeight can't be zero"); + return false; + } + + if (aConfig.mDisplayAspectWidth.WasPassed() != + aConfig.mDisplayAspectHeight.WasPassed()) { + LOGE("Missing display aspect %s", + aConfig.mDisplayAspectWidth.WasPassed() ? "height" : "width"); + return false; + } + if (aConfig.mDisplayAspectWidth.WasPassed() && + (aConfig.mDisplayAspectWidth.Value() == 0 || + aConfig.mDisplayAspectHeight.Value() == 0)) { + LOGE("display aspect width and height cannot be zero"); + return false; + } + + return true; +} + +/* static */ +UniquePtr VideoDecoderTraits::CreateConfigInternal( + const VideoDecoderConfig& aConfig) { + return VideoDecoderConfigInternal::Create(aConfig); +} + +/* static */ +bool VideoDecoderTraits::IsKeyChunk(const EncodedVideoChunk& aInput) { + return aInput.Type() == EncodedVideoChunkType::Key; +} + +/* static */ +UniquePtr VideoDecoderTraits::CreateInputInternal( + const EncodedVideoChunk& aInput) { + return aInput.Clone(); +} + +/* + * Below are VideoDecoder implementation + */ + +VideoDecoder::VideoDecoder(nsIGlobalObject* aParent, + RefPtr&& aErrorCallback, + RefPtr&& aOutputCallback) + : DecoderTemplate(aParent, std::move(aErrorCallback), + std::move(aOutputCallback)) { + MOZ_ASSERT(mErrorCallback); + MOZ_ASSERT(mOutputCallback); + LOG("VideoDecoder %p ctor", this); +} + +VideoDecoder::~VideoDecoder() { + LOG("VideoDecoder %p dtor", this); +} + +JSObject* VideoDecoder::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + AssertIsOnOwningThread(); + + return VideoDecoder_Binding::Wrap(aCx, this, aGivenProto); +} + +// https://w3c.github.io/webcodecs/#dom-videodecoder-videodecoder +/* static */ +already_AddRefed VideoDecoder::Constructor( + const GlobalObject& aGlobal, const VideoDecoderInit& aInit, + ErrorResult& aRv) { + nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + return MakeAndAddRef( + global.get(), RefPtr(aInit.mError), + RefPtr(aInit.mOutput)); +} + +// https://w3c.github.io/webcodecs/#dom-videodecoder-isconfigsupported +/* static */ +already_AddRefed VideoDecoder::IsConfigSupported( + const GlobalObject& aGlobal, const VideoDecoderConfig& aConfig, + ErrorResult& aRv) { + LOG("VideoDecoder::IsConfigSupported, config: %s", + NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); + + nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + RefPtr p = Promise::Create(global.get(), aRv); + if (NS_WARN_IF(aRv.Failed())) { + return p.forget(); + } + + nsCString errorMessage; + if (!VideoDecoderTraits::Validate(aConfig, errorMessage)) { + p->MaybeRejectWithTypeError(nsPrintfCString( + "VideoDecoderConfig is invalid: %s", errorMessage.get())); + return p.forget(); + } + + // TODO: Move the following works to another thread to unblock the current + // thread, as what spec suggests. + + RootedDictionary config(aGlobal.Context()); + auto r = CloneConfiguration(config, aGlobal.Context(), aConfig); + if (r.isErr()) { + nsresult e = r.unwrapErr(); + LOGE("Failed to clone VideoDecoderConfig. Error: 0x%08" PRIx32, + static_cast(e)); + p->MaybeRejectWithTypeError("Failed to clone VideoDecoderConfig"); + aRv.Throw(e); + return p.forget(); + } + + bool canDecode = CanDecode(config); + RootedDictionary s(aGlobal.Context()); + s.mConfig.Construct(std::move(config)); + s.mSupported.Construct(canDecode); + + p->MaybeResolve(s); + return p.forget(); +} + +already_AddRefed VideoDecoder::InputDataToMediaRawData( + UniquePtr&& aData, TrackInfo& aInfo, + const VideoDecoderConfigInternal& aConfig) { + AssertIsOnOwningThread(); + MOZ_ASSERT(aInfo.GetAsVideoInfo()); + + if (!aData) { + LOGE("No data for conversion"); + return nullptr; + } + + RefPtr sample = aData->TakeData(); + if (!sample) { + LOGE("Take no data for conversion"); + return nullptr; + } + + // aExtraData is either provided by Configure() or a default one created for + // the decoder creation. If it's created for decoder creation only, we don't + // set it to sample. + if (aConfig.mDescription && aInfo.GetAsVideoInfo()->mExtraData) { + sample->mExtraData = aInfo.GetAsVideoInfo()->mExtraData; + } + + LOGV( + "EncodedVideoChunkData %p converted to %zu-byte MediaRawData - time: " + "%" PRIi64 "us, timecode: %" PRIi64 "us, duration: %" PRIi64 + "us, key-frame: %s, has extra data: %s", + aData.get(), sample->Size(), sample->mTime.ToMicroseconds(), + sample->mTimecode.ToMicroseconds(), sample->mDuration.ToMicroseconds(), + sample->mKeyframe ? "yes" : "no", sample->mExtraData ? "yes" : "no"); + + return sample.forget(); +} + +nsTArray> VideoDecoder::DecodedDataToOutputType( + nsIGlobalObject* aGlobalObject, const nsTArray>&& aData, + VideoDecoderConfigInternal& aConfig) { + AssertIsOnOwningThread(); + + nsTArray> frames; + for (const RefPtr& data : aData) { + MOZ_RELEASE_ASSERT(data->mType == MediaData::Type::VIDEO_DATA); + RefPtr d(data->As()); + VideoColorSpaceInternal colorSpace; + // Determine which color space to use: prefer the color space as configured + // at the decoder level, if it has one, otherwise look at the underlying + // image and make a guess. + if (aConfig.mColorSpace.isSome() && + aConfig.mColorSpace->mPrimaries.isSome() && + aConfig.mColorSpace->mTransfer.isSome() && + aConfig.mColorSpace->mMatrix.isSome()) { + colorSpace = aConfig.mColorSpace.value(); + } else { + colorSpace = GuessColorSpace(d->mImage.get()); + } + frames.AppendElement(CreateVideoFrame( + aGlobalObject, d.get(), d->mTime.ToMicroseconds(), + static_cast(d->mDuration.ToMicroseconds()), + aConfig.mDisplayAspectWidth, aConfig.mDisplayAspectHeight, colorSpace)); + } + return frames; +} + +#undef LOG +#undef LOGW +#undef LOGE +#undef LOGV +#undef LOG_INTERNAL + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/VideoDecoder.h b/dom/media/webcodecs/VideoDecoder.h new file mode 100644 index 0000000000..f4b3fe10ed --- /dev/null +++ b/dom/media/webcodecs/VideoDecoder.h @@ -0,0 +1,79 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_VideoDecoder_h +#define mozilla_dom_VideoDecoder_h + +#include "js/TypeDecls.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/RefPtr.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/DecoderTemplate.h" +#include "mozilla/dom/DecoderTypes.h" +#include "mozilla/dom/VideoFrame.h" +#include "nsCycleCollectionParticipant.h" + +class nsIGlobalObject; + +namespace mozilla { + +namespace dom { + +class EncodedVideoChunk; +class EncodedVideoChunkData; +class EventHandlerNonNull; +class GlobalObject; +class Promise; +class VideoFrameOutputCallback; +class WebCodecsErrorCallback; +struct VideoDecoderConfig; +struct VideoDecoderInit; + +} // namespace dom + +} // namespace mozilla + +namespace mozilla::dom { + +class VideoDecoder final : public DecoderTemplate { + public: + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(VideoDecoder, DOMEventTargetHelper) + + public: + VideoDecoder(nsIGlobalObject* aParent, + RefPtr&& aErrorCallback, + RefPtr&& aOutputCallback); + + protected: + ~VideoDecoder(); + + public: + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + static already_AddRefed Constructor( + const GlobalObject& aGlobal, const VideoDecoderInit& aInit, + ErrorResult& aRv); + + static already_AddRefed IsConfigSupported( + const GlobalObject& aGlobal, const VideoDecoderConfig& aConfig, + ErrorResult& aRv); + + protected: + virtual already_AddRefed InputDataToMediaRawData( + UniquePtr&& aData, TrackInfo& aInfo, + const VideoDecoderConfigInternal& aConfig) override; + + virtual nsTArray> DecodedDataToOutputType( + nsIGlobalObject* aGlobalObject, const nsTArray>&& aData, + VideoDecoderConfigInternal& aConfig) override; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_VideoDecoder_h diff --git a/dom/media/webcodecs/VideoEncoder.cpp b/dom/media/webcodecs/VideoEncoder.cpp new file mode 100644 index 0000000000..0e71417cb0 --- /dev/null +++ b/dom/media/webcodecs/VideoEncoder.cpp @@ -0,0 +1,624 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/VideoEncoder.h" +#include "mozilla/dom/VideoEncoderBinding.h" +#include "mozilla/dom/VideoColorSpaceBinding.h" +#include "mozilla/dom/VideoColorSpace.h" +#include "mozilla/dom/VideoFrame.h" + +#include "EncoderTraits.h" +#include "ImageContainer.h" +#include "VideoUtils.h" +#include "mozilla/Assertions.h" +#include "mozilla/Logging.h" +#include "mozilla/Maybe.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/EncodedVideoChunk.h" +#include "mozilla/dom/EncodedVideoChunkBinding.h" +#include "mozilla/dom/ImageUtils.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/VideoColorSpaceBinding.h" +#include "mozilla/dom/VideoFrameBinding.h" +#include "mozilla/dom/WebCodecsUtils.h" +#include "nsPrintfCString.h" +#include "nsReadableUtils.h" + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla::dom { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOG +# undef LOG +#endif // LOG +#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +#ifdef LOGW +# undef LOGW +#endif // LOGW +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +#ifdef LOGV +# undef LOGV +#endif // LOGV +#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__) + +NS_IMPL_CYCLE_COLLECTION_INHERITED(VideoEncoder, DOMEventTargetHelper, + mErrorCallback, mOutputCallback) +NS_IMPL_ADDREF_INHERITED(VideoEncoder, DOMEventTargetHelper) +NS_IMPL_RELEASE_INHERITED(VideoEncoder, DOMEventTargetHelper) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(VideoEncoder) +NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper) + +VideoEncoderConfigInternal::VideoEncoderConfigInternal( + const nsAString& aCodec, uint32_t aWidth, uint32_t aHeight, + Maybe&& aDisplayWidth, Maybe&& aDisplayHeight, + Maybe&& aBitrate, Maybe&& aFramerate, + const HardwareAcceleration& aHardwareAcceleration, + const AlphaOption& aAlpha, Maybe&& aScalabilityMode, + const VideoEncoderBitrateMode& aBitrateMode, + const LatencyMode& aLatencyMode, Maybe&& aContentHint) + : mCodec(aCodec), + mWidth(aWidth), + mHeight(aHeight), + mDisplayWidth(std::move(aDisplayWidth)), + mDisplayHeight(std::move(aDisplayHeight)), + mBitrate(std::move(aBitrate)), + mFramerate(std::move(aFramerate)), + mHardwareAcceleration(aHardwareAcceleration), + mAlpha(aAlpha), + mScalabilityMode(std::move(aScalabilityMode)), + mBitrateMode(aBitrateMode), + mLatencyMode(aLatencyMode), + mContentHint(std::move(aContentHint)) {} + +VideoEncoderConfigInternal::VideoEncoderConfigInternal( + const VideoEncoderConfigInternal& aConfig) + : mCodec(aConfig.mCodec), + mWidth(aConfig.mWidth), + mHeight(aConfig.mHeight), + mDisplayWidth(aConfig.mDisplayWidth), + mDisplayHeight(aConfig.mDisplayHeight), + mBitrate(aConfig.mBitrate), + mFramerate(aConfig.mFramerate), + mHardwareAcceleration(aConfig.mHardwareAcceleration), + mAlpha(aConfig.mAlpha), + mScalabilityMode(aConfig.mScalabilityMode), + mBitrateMode(aConfig.mBitrateMode), + mLatencyMode(aConfig.mLatencyMode), + mContentHint(aConfig.mContentHint), + mAvc(aConfig.mAvc) { +} + +VideoEncoderConfigInternal::VideoEncoderConfigInternal( + const VideoEncoderConfig& aConfig) + : mCodec(aConfig.mCodec), + mWidth(aConfig.mWidth), + mHeight(aConfig.mHeight), + mDisplayWidth(OptionalToMaybe(aConfig.mDisplayWidth)), + mDisplayHeight(OptionalToMaybe(aConfig.mDisplayHeight)), + mBitrate(OptionalToMaybe(aConfig.mBitrate)), + mFramerate(OptionalToMaybe(aConfig.mFramerate)), + mHardwareAcceleration(aConfig.mHardwareAcceleration), + mAlpha(aConfig.mAlpha), + mScalabilityMode(OptionalToMaybe(aConfig.mScalabilityMode)), + mBitrateMode(aConfig.mBitrateMode), + mLatencyMode(aConfig.mLatencyMode), + mContentHint(OptionalToMaybe(aConfig.mContentHint)), + mAvc(OptionalToMaybe(aConfig.mAvc)) { +} + +nsString VideoEncoderConfigInternal::ToString() const { + nsString rv; + + rv.AppendPrintf("Codec: %s, [%" PRIu32 "x%" PRIu32 "],", + NS_ConvertUTF16toUTF8(mCodec).get(), mWidth, mHeight); + if (mDisplayWidth.isSome()) { + rv.AppendPrintf(", display[%" PRIu32 "x%" PRIu32 "]", mDisplayWidth.value(), + mDisplayHeight.value()); + } + if (mBitrate.isSome()) { + rv.AppendPrintf(", %" PRIu32 "kbps", mBitrate.value()); + } + if (mFramerate.isSome()) { + rv.AppendPrintf(", %lfHz", mFramerate.value()); + } + rv.AppendPrintf( + " hw: %s", + HardwareAccelerationValues::GetString(mHardwareAcceleration).data()); + rv.AppendPrintf(", alpha: %s", AlphaOptionValues::GetString(mAlpha).data()); + if (mScalabilityMode.isSome()) { + rv.AppendPrintf(", scalability mode: %s", + NS_ConvertUTF16toUTF8(mScalabilityMode.value()).get()); + } + rv.AppendPrintf( + ", bitrate mode: %s", + VideoEncoderBitrateModeValues::GetString(mBitrateMode).data()); + rv.AppendPrintf(", latency mode: %s", + LatencyModeValues::GetString(mLatencyMode).data()); + if (mContentHint.isSome()) { + rv.AppendPrintf(", content hint: %s", + NS_ConvertUTF16toUTF8(mContentHint.value()).get()); + } + if (mAvc.isSome()) { + rv.AppendPrintf(", avc-specific: %s", + AvcBitstreamFormatValues::GetString(mAvc->mFormat).data()); + } + + return rv; +} + +template +bool MaybeAreEqual(const Maybe& aLHS, const Maybe aRHS) { + if (aLHS.isSome() && aRHS.isSome()) { + return aLHS.value() == aRHS.value(); + } + if (aLHS.isNothing() && aRHS.isNothing()) { + return true; + } + return false; +} + +bool VideoEncoderConfigInternal::Equals( + const VideoEncoderConfigInternal& aOther) const { + bool sameCodecSpecific = true; + if ((mAvc.isSome() && aOther.mAvc.isSome() && + mAvc->mFormat != aOther.mAvc->mFormat) || + mAvc.isSome() != aOther.mAvc.isSome()) { + sameCodecSpecific = false; + } + return mCodec.Equals(aOther.mCodec) && mWidth == aOther.mWidth && + mHeight == aOther.mHeight && + MaybeAreEqual(mDisplayWidth, aOther.mDisplayWidth) && + MaybeAreEqual(mDisplayHeight, aOther.mDisplayHeight) && + MaybeAreEqual(mBitrate, aOther.mBitrate) && + MaybeAreEqual(mFramerate, aOther.mFramerate) && + mHardwareAcceleration == aOther.mHardwareAcceleration && + mAlpha == aOther.mAlpha && + MaybeAreEqual(mScalabilityMode, aOther.mScalabilityMode) && + mBitrateMode == aOther.mBitrateMode && + mLatencyMode == aOther.mLatencyMode && + MaybeAreEqual(mContentHint, aOther.mContentHint) && sameCodecSpecific; +} + +bool VideoEncoderConfigInternal::CanReconfigure( + const VideoEncoderConfigInternal& aOther) const { + return mCodec.Equals(aOther.mCodec) && + mHardwareAcceleration == aOther.mHardwareAcceleration; +} + +EncoderConfig VideoEncoderConfigInternal::ToEncoderConfig() const { + MediaDataEncoder::Usage usage; + if (mLatencyMode == LatencyMode::Quality) { + usage = MediaDataEncoder::Usage::Record; + } else { + usage = MediaDataEncoder::Usage::Realtime; + } + MediaDataEncoder::HardwarePreference hwPref = + MediaDataEncoder::HardwarePreference::None; + if (mHardwareAcceleration == + mozilla::dom::HardwareAcceleration::Prefer_hardware) { + hwPref = MediaDataEncoder::HardwarePreference::RequireHardware; + } else if (mHardwareAcceleration == + mozilla::dom::HardwareAcceleration::Prefer_software) { + hwPref = MediaDataEncoder::HardwarePreference::RequireSoftware; + } + CodecType codecType; + auto maybeCodecType = CodecStringToCodecType(mCodec); + if (maybeCodecType.isSome()) { + codecType = maybeCodecType.value(); + } else { + MOZ_CRASH("The string should always contain a valid codec at this point."); + } + Maybe specific; + if (codecType == CodecType::H264) { + uint8_t profile, constraints, level; + H264BitStreamFormat format; + if (mAvc) { + format = mAvc->mFormat == AvcBitstreamFormat::Annexb + ? H264BitStreamFormat::ANNEXB + : H264BitStreamFormat::AVC; + } else { + format = H264BitStreamFormat::AVC; + } + if (ExtractH264CodecDetails(mCodec, profile, constraints, level)) { + if (profile == H264_PROFILE_BASE || profile == H264_PROFILE_MAIN || + profile == H264_PROFILE_EXTENDED || profile == H264_PROFILE_HIGH) { + specific.emplace( + H264Specific(static_cast(profile), static_cast(level), format)); + } + } + } + // Only for vp9, not vp8 + if (codecType == CodecType::VP9) { + uint8_t profile, level, bitdepth, chromasubsampling; + mozilla::VideoColorSpace colorspace; + DebugOnly rv = ExtractVPXCodecDetails( + mCodec, profile, level, bitdepth, chromasubsampling, colorspace); +#ifdef DEBUG + if (!rv) { + LOGE("Error extracting VPX codec details, non fatal"); + } +#endif + specific.emplace(VP9Specific()); + } + MediaDataEncoder::ScalabilityMode scalabilityMode; + if (mScalabilityMode) { + if (mScalabilityMode->EqualsLiteral("L1T2")) { + scalabilityMode = MediaDataEncoder::ScalabilityMode::L1T2; + } else if (mScalabilityMode->EqualsLiteral("L1T3")) { + scalabilityMode = MediaDataEncoder::ScalabilityMode::L1T3; + } else { + scalabilityMode = MediaDataEncoder::ScalabilityMode::None; + } + } else { + scalabilityMode = MediaDataEncoder::ScalabilityMode::None; + } + return EncoderConfig( + codecType, {mWidth, mHeight}, usage, ImageBitmapFormat::RGBA32, ImageBitmapFormat::RGBA32, + AssertedCast(mFramerate.refOr(0.f)), 0, mBitrate.refOr(0), + mBitrateMode == VideoEncoderBitrateMode::Constant + ? MediaDataEncoder::BitrateMode::Constant + : MediaDataEncoder::BitrateMode::Variable, + hwPref, scalabilityMode, specific); +} +already_AddRefed +VideoEncoderConfigInternal::Diff( + const VideoEncoderConfigInternal& aOther) const { + auto list = MakeRefPtr(); + if (!mCodec.Equals(aOther.mCodec)) { + list->Push(CodecChange{aOther.mCodec}); + } + // Both must always be present, when a `VideoEncoderConfig` is passed to + // `configure`. + if (mWidth != aOther.mWidth || mHeight != aOther.mHeight) { + list->Push(DimensionsChange{gfx::IntSize{aOther.mWidth, aOther.mHeight}}); + } + // Similarly, both must always be present, when a `VideoEncoderConfig` is + // passed to `configure`. + if (!MaybeAreEqual(mDisplayWidth, aOther.mDisplayWidth) || + !MaybeAreEqual(mDisplayHeight, aOther.mDisplayHeight)) { + Maybe displaySize = + aOther.mDisplayWidth.isSome() + ? Some(gfx::IntSize{aOther.mDisplayWidth.value(), + aOther.mDisplayHeight.value()}) + : Nothing(); + list->Push(DisplayDimensionsChange{displaySize}); + } + if (!MaybeAreEqual(mBitrate, aOther.mBitrate)) { + list->Push(BitrateChange{aOther.mBitrate}); + } + if (!MaybeAreEqual(mFramerate, aOther.mFramerate)) { + list->Push(FramerateChange{aOther.mFramerate}); + } + if (mHardwareAcceleration != aOther.mHardwareAcceleration) { + list->Push(HardwareAccelerationChange{aOther.mHardwareAcceleration}); + } + if (mAlpha != aOther.mAlpha) { + list->Push(AlphaChange{aOther.mAlpha}); + } + if (!MaybeAreEqual(mScalabilityMode, aOther.mScalabilityMode)) { + list->Push(ScalabilityModeChange{aOther.mScalabilityMode}); + } + if (mBitrateMode != aOther.mBitrateMode) { + list->Push(BitrateModeChange{aOther.mBitrateMode}); + } + if (mLatencyMode != aOther.mLatencyMode) { + list->Push(LatencyModeChange{aOther.mLatencyMode}); + } + if (!MaybeAreEqual(mContentHint, aOther.mContentHint)) { + list->Push(ContentHintChange{aOther.mContentHint}); + } + return list.forget(); +} + +/* + * The followings are helpers for VideoEncoder methods + */ +static bool IsEncodeSupportedCodec(const nsAString& aCodec) { + LOG("IsEncodeSupported: %s", NS_ConvertUTF16toUTF8(aCodec).get()); + if (!IsVP9CodecString(aCodec) && !IsVP8CodecString(aCodec) && + !IsH264CodecString(aCodec) && !IsAV1CodecString(aCodec)) { + return false; + } + + // Gecko allows codec string starts with vp9 or av1 but Webcodecs requires to + // starts with av01 and vp09. + // https://www.w3.org/TR/webcodecs-codec-registry/#video-codec-registry + if (StringBeginsWith(aCodec, u"vp9"_ns) || + StringBeginsWith(aCodec, u"av1"_ns)) { + return false; + } + + return true; +} + +// https://w3c.github.io/webcodecs/#check-configuration-support +static bool CanEncode(const RefPtr& aConfig) { + auto parsedCodecString = + ParseCodecString(aConfig->mCodec).valueOr(EmptyString()); + // TODO: Enable WebCodecs on Android (Bug 1840508) + if (IsOnAndroid()) { + return false; + } + if (!IsEncodeSupportedCodec(parsedCodecString)) { + return false; + } + + // TODO (bug 1872879, bug 1872880): Support this on Windows and Mac. + if (aConfig->mScalabilityMode.isSome()) { + // We only support L1T2 and L1T3 ScalabilityMode in VP8 and VP9 encoders on + // Linux. + bool supported = IsOnLinux() && (IsVP8CodecString(parsedCodecString) || + IsVP9CodecString(parsedCodecString)) + ? aConfig->mScalabilityMode->EqualsLiteral("L1T2") || + aConfig->mScalabilityMode->EqualsLiteral("L1T3") + : false; + + if (!supported) { + LOGE("Scalability mode %s not supported for codec: %s", + NS_ConvertUTF16toUTF8(aConfig->mScalabilityMode.value()).get(), + NS_ConvertUTF16toUTF8(parsedCodecString).get()); + return false; + } + } + + return EncoderSupport::Supports(aConfig); +} + +static Result CloneConfiguration( + VideoEncoderConfig& aDest, JSContext* aCx, + const VideoEncoderConfig& aConfig) { + nsCString errorMessage; + MOZ_ASSERT(VideoEncoderTraits::Validate(aConfig, errorMessage)); + + aDest.mCodec = aConfig.mCodec; + aDest.mWidth = aConfig.mWidth; + aDest.mHeight = aConfig.mHeight; + aDest.mAlpha = aConfig.mAlpha; + if (aConfig.mBitrate.WasPassed()) { + aDest.mBitrate.Construct(aConfig.mBitrate.Value()); + } + aDest.mBitrateMode = aConfig.mBitrateMode; + if (aConfig.mContentHint.WasPassed()) { + aDest.mContentHint.Construct(aConfig.mContentHint.Value()); + } + if (aConfig.mDisplayWidth.WasPassed()) { + aDest.mDisplayWidth.Construct(aConfig.mDisplayWidth.Value()); + } + if (aConfig.mDisplayHeight.WasPassed()) { + aDest.mDisplayHeight.Construct(aConfig.mDisplayHeight.Value()); + } + if (aConfig.mFramerate.WasPassed()) { + aDest.mFramerate.Construct(aConfig.mFramerate.Value()); + } + aDest.mHardwareAcceleration = aConfig.mHardwareAcceleration; + aDest.mLatencyMode = aConfig.mLatencyMode; + if (aConfig.mScalabilityMode.WasPassed()) { + aDest.mScalabilityMode.Construct(aConfig.mScalabilityMode.Value()); + } + + // AVC specific + if (aConfig.mAvc.WasPassed()) { + aDest.mAvc.Construct(aConfig.mAvc.Value()); + } + + return Ok(); +} + +/* static */ +bool VideoEncoderTraits::IsSupported( + const VideoEncoderConfigInternal& aConfig) { + return CanEncode(MakeRefPtr(aConfig)); +} + +// https://w3c.github.io/webcodecs/#valid-videoencoderconfig +/* static */ +bool VideoEncoderTraits::Validate(const VideoEncoderConfig& aConfig, + nsCString& aErrorMessage) { + Maybe codec = ParseCodecString(aConfig.mCodec); + // 1. + if (!codec || codec->IsEmpty()) { + LOGE("Invalid VideoEncoderConfig: invalid codec string"); + return false; + } + + // 2. + if (aConfig.mWidth == 0 || aConfig.mHeight == 0) { + LOGE("Invalid VideoEncoderConfig: %s equal to 0", + aConfig.mWidth == 0 ? "width" : "height"); + return false; + } + + // 3. + if ((aConfig.mDisplayWidth.WasPassed() && + aConfig.mDisplayWidth.Value() == 0)) { + LOGE("Invalid VideoEncoderConfig: displayWidth equal to 0"); + return false; + } + if ((aConfig.mDisplayHeight.WasPassed() && + aConfig.mDisplayHeight.Value() == 0)) { + LOGE("Invalid VideoEncoderConfig: displayHeight equal to 0"); + return false; + } + + return true; +} + +/* static */ +RefPtr VideoEncoderTraits::CreateConfigInternal( + const VideoEncoderConfig& aConfig) { + return MakeRefPtr(aConfig); +} + +/* static */ +RefPtr VideoEncoderTraits::CreateInputInternal( + const dom::VideoFrame& aInput, + const dom::VideoEncoderEncodeOptions& aOptions) { + media::TimeUnit duration = + aInput.GetDuration().IsNull() + ? media::TimeUnit::Zero() + : media::TimeUnit::FromMicroseconds( + AssertedCast(aInput.GetDuration().Value())); + media::TimeUnit pts = media::TimeUnit::FromMicroseconds(aInput.Timestamp()); + return VideoData::CreateFromImage( + gfx::IntSize{aInput.DisplayWidth(), aInput.DisplayHeight()}, + 0 /* bytestream offset */, pts, duration, aInput.GetImage(), + aOptions.mKeyFrame, pts); +} + +/* + * Below are VideoEncoder implementation + */ + +VideoEncoder::VideoEncoder( + nsIGlobalObject* aParent, RefPtr&& aErrorCallback, + RefPtr&& aOutputCallback) + : EncoderTemplate(aParent, std::move(aErrorCallback), + std::move(aOutputCallback)) { + MOZ_ASSERT(mErrorCallback); + MOZ_ASSERT(mOutputCallback); + LOG("VideoEncoder %p ctor", this); +} + +VideoEncoder::~VideoEncoder() { + LOG("VideoEncoder %p dtor", this); + Unused << ResetInternal(NS_ERROR_DOM_ABORT_ERR); +} + +JSObject* VideoEncoder::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + AssertIsOnOwningThread(); + + return VideoEncoder_Binding::Wrap(aCx, this, aGivenProto); +} + +// https://w3c.github.io/webcodecs/#dom-videoencoder-videoencoder +/* static */ +already_AddRefed VideoEncoder::Constructor( + const GlobalObject& aGlobal, const VideoEncoderInit& aInit, + ErrorResult& aRv) { + nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + return MakeAndAddRef( + global.get(), RefPtr(aInit.mError), + RefPtr(aInit.mOutput)); +} + +// https://w3c.github.io/webcodecs/#dom-videoencoder-isconfigsupported +/* static */ +already_AddRefed VideoEncoder::IsConfigSupported( + const GlobalObject& aGlobal, const VideoEncoderConfig& aConfig, + ErrorResult& aRv) { + LOG("VideoEncoder::IsConfigSupported, config: %s", + NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); + + nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + RefPtr p = Promise::Create(global.get(), aRv); + if (NS_WARN_IF(aRv.Failed())) { + return p.forget(); + } + + nsCString errorMessage; + if (!VideoEncoderTraits::Validate(aConfig, errorMessage)) { + p->MaybeRejectWithTypeError(nsPrintfCString( + "IsConfigSupported: config is invalid: %s", errorMessage.get())); + return p.forget(); + } + + // TODO: Move the following works to another thread to unblock the current + // thread, as what spec suggests. + + VideoEncoderConfig config; + auto r = CloneConfiguration(config, aGlobal.Context(), aConfig); + if (r.isErr()) { + nsresult e = r.unwrapErr(); + LOGE("Failed to clone VideoEncoderConfig. Error: 0x%08" PRIx32, + static_cast(e)); + p->MaybeRejectWithTypeError("Failed to clone VideoEncoderConfig"); + aRv.Throw(e); + return p.forget(); + } + + bool canEncode = CanEncode(MakeRefPtr(config)); + VideoEncoderSupport s; + s.mConfig.Construct(std::move(config)); + s.mSupported.Construct(canEncode); + + p->MaybeResolve(s); + return p.forget(); +} + +RefPtr VideoEncoder::EncodedDataToOutputType( + nsIGlobalObject* aGlobalObject, RefPtr& aData) { + AssertIsOnOwningThread(); + + MOZ_RELEASE_ASSERT(aData->mType == MediaData::Type::RAW_DATA); + // Package into an EncodedVideoChunk + auto buffer = + MakeRefPtr(aData->Data(), aData->Size()); + auto encodedVideoChunk = MakeRefPtr( + aGlobalObject, buffer.forget(), + aData->mKeyframe ? EncodedVideoChunkType::Key + : EncodedVideoChunkType::Delta, + aData->mTime.ToMicroseconds(), + aData->mDuration.IsZero() ? Nothing() + : Some(aData->mDuration.ToMicroseconds())); + return encodedVideoChunk; +} + +VideoDecoderConfigInternal VideoEncoder::EncoderConfigToDecoderConfig( + nsIGlobalObject* aGlobal, const RefPtr& aRawData, + const VideoEncoderConfigInternal& mOutputConfig) const { + // Colorspace is mandatory when outputing a decoder config after encode + VideoColorSpaceInternal init; + init.mFullRange.emplace(false); + init.mMatrix.emplace(VideoMatrixCoefficients::Bt709); + init.mPrimaries.emplace(VideoColorPrimaries::Bt709); + init.mTransfer.emplace(VideoTransferCharacteristics::Bt709); + + return VideoDecoderConfigInternal( + mOutputConfig.mCodec, /* aCodec */ + Some(mOutputConfig.mHeight), /* aCodedHeight */ + Some(mOutputConfig.mWidth), /* aCodedWidth */ + Some(init), /* aColorSpace */ + aRawData->mExtraData && !aRawData->mExtraData->IsEmpty() + ? Some(aRawData->mExtraData) + : Nothing(), /* aDescription*/ + Maybe(mOutputConfig.mDisplayHeight), /* aDisplayAspectHeight*/ + Maybe(mOutputConfig.mDisplayWidth), /* aDisplayAspectWidth */ + mOutputConfig.mHardwareAcceleration, /* aHardwareAcceleration */ + Nothing() /* aOptimizeForLatency */ + ); +} + +#undef LOG +#undef LOGW +#undef LOGE +#undef LOGV +#undef LOG_INTERNAL + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/VideoEncoder.h b/dom/media/webcodecs/VideoEncoder.h new file mode 100644 index 0000000000..9251b5023a --- /dev/null +++ b/dom/media/webcodecs/VideoEncoder.h @@ -0,0 +1,78 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_VideoEncoder_h +#define mozilla_dom_VideoEncoder_h + +#include "js/TypeDecls.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/RefPtr.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/EncoderTemplate.h" +#include "mozilla/dom/EncoderTypes.h" +#include "mozilla/dom/VideoFrame.h" +#include "nsCycleCollectionParticipant.h" + +class nsIGlobalObject; + +namespace mozilla { + +namespace dom { + +class EncodedVideoChunk; +class EncodedVideoChunkData; +class EventHandlerNonNull; +class GlobalObject; +class Promise; +class VideoFrameOutputCallback; +class WebCodecsErrorCallback; +struct VideoEncoderConfig; +struct VideoEncoderInit; + +} // namespace dom + +} // namespace mozilla + +namespace mozilla::dom { + +class VideoEncoder final : public EncoderTemplate { + public: + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(VideoEncoder, DOMEventTargetHelper) + + public: + VideoEncoder(nsIGlobalObject* aParent, + RefPtr&& aErrorCallback, + RefPtr&& aOutputCallback); + + protected: + ~VideoEncoder(); + + public: + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + static already_AddRefed Constructor( + const GlobalObject& aGlobal, const VideoEncoderInit& aInit, + ErrorResult& aRv); + + static already_AddRefed IsConfigSupported( + const GlobalObject& aGlobal, const VideoEncoderConfig& aConfig, + ErrorResult& aRv); + + protected: + virtual RefPtr EncodedDataToOutputType( + nsIGlobalObject* aGlobal, RefPtr& aData) override; + + virtual VideoDecoderConfigInternal EncoderConfigToDecoderConfig( + nsIGlobalObject* aGlobal /* TODO: delete */, + const RefPtr& aRawData, + const VideoEncoderConfigInternal& mOutputConfig) const override; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_VideoEncoder_h diff --git a/dom/media/webcodecs/VideoFrame.cpp b/dom/media/webcodecs/VideoFrame.cpp new file mode 100644 index 0000000000..602bc95c29 --- /dev/null +++ b/dom/media/webcodecs/VideoFrame.cpp @@ -0,0 +1,2417 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/VideoFrame.h" +#include "mozilla/dom/VideoFrameBinding.h" + +#include +#include +#include + +#include "ImageContainer.h" +#include "VideoColorSpace.h" +#include "WebCodecsUtils.h" +#include "js/StructuredClone.h" +#include "mozilla/Maybe.h" +#include "mozilla/ResultVariant.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/Try.h" + +#include "mozilla/UniquePtr.h" +#include "mozilla/dom/CanvasUtils.h" +#include "mozilla/dom/DOMRect.h" +#include "mozilla/dom/HTMLCanvasElement.h" +#include "mozilla/dom/HTMLImageElement.h" +#include "mozilla/dom/HTMLVideoElement.h" +#include "mozilla/dom/ImageBitmap.h" +#include "mozilla/dom/ImageUtils.h" +#include "mozilla/dom/OffscreenCanvas.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/SVGImageElement.h" +#include "mozilla/dom/StructuredCloneHolder.h" +#include "mozilla/dom/StructuredCloneTags.h" +#include "mozilla/dom/UnionTypes.h" +#include "mozilla/gfx/2D.h" +#include "mozilla/gfx/Swizzle.h" +#include "mozilla/layers/LayersSurfaces.h" +#include "nsLayoutUtils.h" +#include "nsIPrincipal.h" +#include "nsIURI.h" + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla::dom { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOG +# undef LOG +#endif // LOG +#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +#ifdef LOGW +# undef LOGW +#endif // LOGW +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(VideoFrame) +NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(VideoFrame) + tmp->CloseIfNeeded(); + NS_IMPL_CYCLE_COLLECTION_UNLINK(mParent) + NS_IMPL_CYCLE_COLLECTION_UNLINK_PRESERVED_WRAPPER +NS_IMPL_CYCLE_COLLECTION_UNLINK_END +NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(VideoFrame) + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mParent) +NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END + +NS_IMPL_CYCLE_COLLECTING_ADDREF(VideoFrame) +// VideoFrame should be released as soon as its refcount drops to zero, +// without waiting for async deletion by the cycle collector, since it may hold +// a large-size image. +NS_IMPL_CYCLE_COLLECTING_RELEASE_WITH_LAST_RELEASE(VideoFrame, CloseIfNeeded()) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(VideoFrame) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +/* + * The following are helpers to read the image data from the given buffer and + * the format. The data layout is illustrated in the comments for + * `VideoFrame::Format` below. + */ + +static int32_t CeilingOfHalf(int32_t aValue) { + MOZ_ASSERT(aValue >= 0); + return aValue / 2 + (aValue % 2); +} + +class YUVBufferReaderBase { + public: + YUVBufferReaderBase(const Span& aBuffer, int32_t aWidth, + int32_t aHeight) + : mWidth(aWidth), mHeight(aHeight), mStrideY(aWidth), mBuffer(aBuffer) {} + virtual ~YUVBufferReaderBase() = default; + + const uint8_t* DataY() const { return mBuffer.data(); } + const int32_t mWidth; + const int32_t mHeight; + const int32_t mStrideY; + + protected: + CheckedInt YByteSize() const { + return CheckedInt(mStrideY) * mHeight; + } + + const Span mBuffer; +}; + +class I420ABufferReader; +class I420BufferReader : public YUVBufferReaderBase { + public: + I420BufferReader(const Span& aBuffer, int32_t aWidth, + int32_t aHeight) + : YUVBufferReaderBase(aBuffer, aWidth, aHeight), + mStrideU(CeilingOfHalf(aWidth)), + mStrideV(CeilingOfHalf(aWidth)) {} + virtual ~I420BufferReader() = default; + + const uint8_t* DataU() const { return &mBuffer[YByteSize().value()]; } + const uint8_t* DataV() const { + return &mBuffer[YByteSize().value() + UByteSize().value()]; + } + virtual I420ABufferReader* AsI420ABufferReader() { return nullptr; } + + const int32_t mStrideU; + const int32_t mStrideV; + + protected: + CheckedInt UByteSize() const { + return CheckedInt(CeilingOfHalf(mHeight)) * mStrideU; + } + + CheckedInt VSize() const { + return CheckedInt(CeilingOfHalf(mHeight)) * mStrideV; + } +}; + +class I420ABufferReader final : public I420BufferReader { + public: + I420ABufferReader(const Span& aBuffer, int32_t aWidth, + int32_t aHeight) + : I420BufferReader(aBuffer, aWidth, aHeight), mStrideA(aWidth) { + MOZ_ASSERT(mStrideA == mStrideY); + } + virtual ~I420ABufferReader() = default; + + const uint8_t* DataA() const { + return &mBuffer[YByteSize().value() + UByteSize().value() + + VSize().value()]; + } + + virtual I420ABufferReader* AsI420ABufferReader() override { return this; } + + const int32_t mStrideA; +}; + +class NV12BufferReader final : public YUVBufferReaderBase { + public: + NV12BufferReader(const Span& aBuffer, int32_t aWidth, + int32_t aHeight) + : YUVBufferReaderBase(aBuffer, aWidth, aHeight), + mStrideUV(aWidth + aWidth % 2) {} + virtual ~NV12BufferReader() = default; + + const uint8_t* DataUV() const { return &mBuffer[YByteSize().value()]; } + + const int32_t mStrideUV; +}; + +/* + * The followings are helpers defined in + * https://w3c.github.io/webcodecs/#videoframe-algorithms + */ + +static bool IsSameOrigin(nsIGlobalObject* aGlobal, const VideoFrame& aFrame) { + MOZ_ASSERT(aGlobal); + MOZ_ASSERT(aFrame.GetParentObject()); + + nsIPrincipal* principalX = aGlobal->PrincipalOrNull(); + nsIPrincipal* principalY = aFrame.GetParentObject()->PrincipalOrNull(); + + // If both of VideoFrames are created in worker, they are in the same origin + // domain. + if (!principalX) { + return !principalY; + } + // Otherwise, check their domains. + return principalX->Equals(principalY); +} + +static bool IsSameOrigin(nsIGlobalObject* aGlobal, + HTMLVideoElement& aVideoElement) { + MOZ_ASSERT(aGlobal); + + // If CORS is in use, consider the video source is same-origin. + if (aVideoElement.GetCORSMode() != CORS_NONE) { + return true; + } + + // Otherwise, check if video source has cross-origin redirect or not. + if (aVideoElement.HadCrossOriginRedirects()) { + return false; + } + + // Finally, compare the VideoFrame's domain and video's one. + nsIPrincipal* principal = aGlobal->PrincipalOrNull(); + nsCOMPtr elementPrincipal = + aVideoElement.GetCurrentVideoPrincipal(); + //