diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
commit | 26a029d407be480d791972afb5975cf62c9360a6 (patch) | |
tree | f435a8308119effd964b339f76abb83a57c29483 /dom/media/webcodecs | |
parent | Initial commit. (diff) | |
download | firefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz firefox-26a029d407be480d791972afb5975cf62c9360a6.zip |
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/media/webcodecs')
30 files changed, 9969 insertions, 0 deletions
diff --git a/dom/media/webcodecs/DecoderAgent.cpp b/dom/media/webcodecs/DecoderAgent.cpp new file mode 100644 index 0000000000..5c63e27d48 --- /dev/null +++ b/dom/media/webcodecs/DecoderAgent.cpp @@ -0,0 +1,491 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "DecoderAgent.h" + +#include <atomic> + +#include "ImageContainer.h" +#include "MediaDataDecoderProxy.h" +#include "PDMFactory.h" +#include "VideoUtils.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/Logging.h" +#include "mozilla/layers/ImageBridgeChild.h" +#include "nsThreadUtils.h" + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOG +# undef LOG +#endif // LOG +#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +#ifdef LOGW +# undef LOGW +#endif // LOGE +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +#ifdef LOGV +# undef LOGV +#endif // LOGV +#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__) + +DecoderAgent::DecoderAgent(Id aId, UniquePtr<TrackInfo>&& aInfo) + : mId(aId), + mInfo(std::move(aInfo)), + mOwnerThread(GetCurrentSerialEventTarget()), + mPDMFactory(MakeRefPtr<PDMFactory>()), + mImageContainer(MakeAndAddRef<layers::ImageContainer>( + layers::ImageContainer::ASYNCHRONOUS)), + mDecoder(nullptr), + mState(State::Unconfigured) { + MOZ_ASSERT(mInfo); + MOZ_ASSERT(mOwnerThread); + MOZ_ASSERT(mPDMFactory); + MOZ_ASSERT(mImageContainer); + LOG("DecoderAgent #%d (%p) ctor", mId, this); +} + +DecoderAgent::~DecoderAgent() { + LOG("DecoderAgent #%d (%p) dtor", mId, this); + MOZ_ASSERT(mState == State::Unconfigured, "decoder release in wrong state"); + MOZ_ASSERT(!mDecoder, "decoder must be shutdown"); +} + +RefPtr<DecoderAgent::ConfigurePromise> DecoderAgent::Configure( + bool aPreferSoftwareDecoder, bool aLowLatency) { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + MOZ_ASSERT(mState == State::Unconfigured || mState == State::Error); + MOZ_ASSERT(mConfigurePromise.IsEmpty()); + MOZ_ASSERT(!mCreateRequest.Exists()); + MOZ_ASSERT(!mInitRequest.Exists()); + + if (mState == State::Error) { + LOGE("DecoderAgent #%d (%p) tried to configure in error state", mId, this); + return ConfigurePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "Cannot configure in error state"), + __func__); + } + + MOZ_ASSERT(mState == State::Unconfigured); + MOZ_ASSERT(!mDecoder); + SetState(State::Configuring); + + RefPtr<layers::KnowsCompositor> knowsCompositor = + layers::ImageBridgeChild::GetSingleton(); + // Bug 1839993: FFmpegDataDecoder ignores all decode errors when draining so + // WPT cannot receive error callbacks. Forcibly enable LowLatency for now to + // get the decoded results immediately to avoid this. + + auto params = CreateDecoderParams{ + *mInfo, + CreateDecoderParams::OptionSet( + CreateDecoderParams::Option::LowLatency, + aPreferSoftwareDecoder + ? CreateDecoderParams::Option::HardwareDecoderNotAllowed + : CreateDecoderParams::Option::Default), + mInfo->GetType(), mImageContainer, knowsCompositor}; + + LOG("DecoderAgent #%d (%p) is creating a decoder - PreferSW: %s, " + "low-latency: %syes", + mId, this, aPreferSoftwareDecoder ? "yes" : "no", + aLowLatency ? "" : "forcibly "); + + RefPtr<ConfigurePromise> p = mConfigurePromise.Ensure(__func__); + + mPDMFactory->CreateDecoder(params) + ->Then( + mOwnerThread, __func__, + [self = RefPtr{this}](RefPtr<MediaDataDecoder>&& aDecoder) { + self->mCreateRequest.Complete(); + + // If DecoderAgent has been shut down, shut the created decoder down + // and return. + if (!self->mShutdownWhileCreationPromise.IsEmpty()) { + MOZ_ASSERT(self->mState == State::ShuttingDown); + MOZ_ASSERT(self->mConfigurePromise.IsEmpty(), + "configuration should have been rejected"); + + LOGW( + "DecoderAgent #%d (%p) has been shut down. We need to shut " + "the newly created decoder down", + self->mId, self.get()); + aDecoder->Shutdown()->Then( + self->mOwnerThread, __func__, + [self](const ShutdownPromise::ResolveOrRejectValue& aValue) { + MOZ_ASSERT(self->mState == State::ShuttingDown); + + LOGW( + "DecoderAgent #%d (%p), newly created decoder shutdown " + "has been %s", + self->mId, self.get(), + aValue.IsResolve() ? "resolved" : "rejected"); + + self->SetState(State::Unconfigured); + + self->mShutdownWhileCreationPromise.ResolveOrReject( + aValue, __func__); + }); + return; + } + + self->mDecoder = new MediaDataDecoderProxy( + aDecoder.forget(), + CreateMediaDecodeTaskQueue("DecoderAgent TaskQueue")); + LOG("DecoderAgent #%d (%p) has created a decoder, now initialize " + "it", + self->mId, self.get()); + self->mDecoder->Init() + ->Then( + self->mOwnerThread, __func__, + [self](const TrackInfo::TrackType aTrackType) { + self->mInitRequest.Complete(); + LOG("DecoderAgent #%d (%p) has initialized the decoder", + self->mId, self.get()); + MOZ_ASSERT(aTrackType == self->mInfo->GetType()); + self->SetState(State::Configured); + self->mConfigurePromise.Resolve(true, __func__); + }, + [self](const MediaResult& aError) { + self->mInitRequest.Complete(); + LOGE( + "DecoderAgent #%d (%p) failed to initialize the " + "decoder", + self->mId, self.get()); + self->SetState(State::Error); + self->mConfigurePromise.Reject(aError, __func__); + }) + ->Track(self->mInitRequest); + }, + [self = RefPtr{this}](const MediaResult& aError) { + self->mCreateRequest.Complete(); + LOGE("DecoderAgent #%d (%p) failed to create a decoder", self->mId, + self.get()); + + // If DecoderAgent has been shut down, we need to resolve the + // shutdown promise. + if (!self->mShutdownWhileCreationPromise.IsEmpty()) { + MOZ_ASSERT(self->mState == State::ShuttingDown); + MOZ_ASSERT(self->mConfigurePromise.IsEmpty(), + "configuration should have been rejected"); + + LOGW( + "DecoderAgent #%d (%p) has been shut down. Resolve the " + "shutdown promise right away since decoder creation failed", + self->mId, self.get()); + + self->SetState(State::Unconfigured); + self->mShutdownWhileCreationPromise.Resolve(true, __func__); + return; + } + + self->SetState(State::Error); + self->mConfigurePromise.Reject(aError, __func__); + }) + ->Track(mCreateRequest); + + return p; +} + +RefPtr<ShutdownPromise> DecoderAgent::Shutdown() { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + + auto r = + MediaResult(NS_ERROR_DOM_MEDIA_CANCELED, "Canceled by decoder shutdown"); + + // If the decoder creation has not been completed yet, wait until the decoder + // being created has been shut down. + if (mCreateRequest.Exists()) { + MOZ_ASSERT(!mInitRequest.Exists()); + MOZ_ASSERT(!mConfigurePromise.IsEmpty()); + MOZ_ASSERT(!mDecoder); + MOZ_ASSERT(mState == State::Configuring); + MOZ_ASSERT(mShutdownWhileCreationPromise.IsEmpty()); + + LOGW( + "DecoderAgent #%d (%p) shutdown while the decoder-creation for " + "configuration is in flight. Reject the configuration now and defer " + "the shutdown until the created decoder has been shut down", + mId, this); + + // Reject the configuration in flight. + mConfigurePromise.Reject(r, __func__); + + // Get the promise that will be resolved when the decoder being created has + // been destroyed. + SetState(State::ShuttingDown); + return mShutdownWhileCreationPromise.Ensure(__func__); + } + + // If decoder creation has been completed, we must have the decoder now. + MOZ_ASSERT(mDecoder); + + // Cancel pending initialization for configuration in flight if any. + mInitRequest.DisconnectIfExists(); + mConfigurePromise.RejectIfExists(r, __func__); + + // Cancel decode in flight if any. + mDecodeRequest.DisconnectIfExists(); + mDecodePromise.RejectIfExists(r, __func__); + + // Cancel flush-out in flight if any. + mDrainRequest.DisconnectIfExists(); + mFlushRequest.DisconnectIfExists(); + mDryRequest.DisconnectIfExists(); + mDryPromise.RejectIfExists(r, __func__); + mDrainAndFlushPromise.RejectIfExists(r, __func__); + mDryData.Clear(); + mDrainAndFlushData.Clear(); + + SetState(State::Unconfigured); + + RefPtr<MediaDataDecoder> decoder = std::move(mDecoder); + return decoder->Shutdown(); +} + +RefPtr<DecoderAgent::DecodePromise> DecoderAgent::Decode( + MediaRawData* aSample) { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + MOZ_ASSERT(aSample); + MOZ_ASSERT(mState == State::Configured || mState == State::Error); + MOZ_ASSERT(mDecodePromise.IsEmpty()); + MOZ_ASSERT(!mDecodeRequest.Exists()); + + if (mState == State::Error) { + LOGE("DecoderAgent #%d (%p) tried to decode in error state", mId, this); + return DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "Cannot decode in error state"), + __func__); + } + + MOZ_ASSERT(mState == State::Configured); + MOZ_ASSERT(mDecoder); + SetState(State::Decoding); + + RefPtr<DecodePromise> p = mDecodePromise.Ensure(__func__); + + mDecoder->Decode(aSample) + ->Then( + mOwnerThread, __func__, + [self = RefPtr{this}](MediaDataDecoder::DecodedData&& aData) { + self->mDecodeRequest.Complete(); + LOGV("DecoderAgent #%d (%p) decode successfully", self->mId, + self.get()); + self->SetState(State::Configured); + self->mDecodePromise.Resolve(std::move(aData), __func__); + }, + [self = RefPtr{this}](const MediaResult& aError) { + self->mDecodeRequest.Complete(); + LOGV("DecoderAgent #%d (%p) failed to decode", self->mId, + self.get()); + self->SetState(State::Error); + self->mDecodePromise.Reject(aError, __func__); + }) + ->Track(mDecodeRequest); + + return p; +} + +RefPtr<DecoderAgent::DecodePromise> DecoderAgent::DrainAndFlush() { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + MOZ_ASSERT(mState == State::Configured || mState == State::Error); + MOZ_ASSERT(mDrainAndFlushPromise.IsEmpty()); + MOZ_ASSERT(mDrainAndFlushData.IsEmpty()); + MOZ_ASSERT(!mDryRequest.Exists()); + MOZ_ASSERT(mDryPromise.IsEmpty()); + MOZ_ASSERT(mDryData.IsEmpty()); + MOZ_ASSERT(!mDrainRequest.Exists()); + MOZ_ASSERT(!mFlushRequest.Exists()); + + if (mState == State::Error) { + LOGE("DecoderAgent #%d (%p) tried to flush-out in error state", mId, this); + return DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "Cannot flush in error state"), + __func__); + } + + MOZ_ASSERT(mState == State::Configured); + MOZ_ASSERT(mDecoder); + SetState(State::Flushing); + + RefPtr<DecoderAgent::DecodePromise> p = + mDrainAndFlushPromise.Ensure(__func__); + + Dry() + ->Then( + mOwnerThread, __func__, + [self = RefPtr{this}](MediaDataDecoder::DecodedData&& aData) { + self->mDryRequest.Complete(); + LOG("DecoderAgent #%d (%p) has dried the decoder. Now flushing the " + "decoder", + self->mId, self.get()); + MOZ_ASSERT(self->mDrainAndFlushData.IsEmpty()); + self->mDrainAndFlushData.AppendElements(std::move(aData)); + self->mDecoder->Flush() + ->Then( + self->mOwnerThread, __func__, + [self](const bool /* aUnUsed */) { + self->mFlushRequest.Complete(); + LOG("DecoderAgent #%d (%p) has flushed the decoder", + self->mId, self.get()); + self->SetState(State::Configured); + self->mDrainAndFlushPromise.Resolve( + std::move(self->mDrainAndFlushData), __func__); + }, + [self](const MediaResult& aError) { + self->mFlushRequest.Complete(); + LOGE("DecoderAgent #%d (%p) failed to flush the decoder", + self->mId, self.get()); + self->SetState(State::Error); + self->mDrainAndFlushData.Clear(); + self->mDrainAndFlushPromise.Reject(aError, __func__); + }) + ->Track(self->mFlushRequest); + }, + [self = RefPtr{this}](const MediaResult& aError) { + self->mDryRequest.Complete(); + LOGE("DecoderAgent #%d (%p) failed to dry the decoder", self->mId, + self.get()); + self->SetState(State::Error); + self->mDrainAndFlushPromise.Reject(aError, __func__); + }) + ->Track(mDryRequest); + + return p; +} + +RefPtr<DecoderAgent::DecodePromise> DecoderAgent::Dry() { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + MOZ_ASSERT(mState == State::Flushing); + MOZ_ASSERT(mDryPromise.IsEmpty()); + MOZ_ASSERT(!mDryRequest.Exists()); + MOZ_ASSERT(mDryData.IsEmpty()); + MOZ_ASSERT(mDecoder); + + RefPtr<DecodePromise> p = mDryPromise.Ensure(__func__); + DrainUntilDry(); + return p; +} + +void DecoderAgent::DrainUntilDry() { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + MOZ_ASSERT(mState == State::Flushing); + MOZ_ASSERT(!mDryPromise.IsEmpty()); + MOZ_ASSERT(!mDrainRequest.Exists()); + MOZ_ASSERT(mDecoder); + + LOG("DecoderAgent #%d (%p) is drainng the decoder", mId, this); + mDecoder->Drain() + ->Then( + mOwnerThread, __func__, + [self = RefPtr{this}](MediaDataDecoder::DecodedData&& aData) { + self->mDrainRequest.Complete(); + + if (aData.IsEmpty()) { + LOG("DecoderAgent #%d (%p) is dry now", self->mId, self.get()); + self->mDryPromise.Resolve(std::move(self->mDryData), __func__); + return; + } + + LOG("DecoderAgent #%d (%p) drained %zu decoded data. Keep draining " + "until dry", + self->mId, self.get(), aData.Length()); + self->mDryData.AppendElements(std::move(aData)); + self->DrainUntilDry(); + }, + [self = RefPtr{this}](const MediaResult& aError) { + self->mDrainRequest.Complete(); + + LOGE("DecoderAgent %p failed to drain decoder", self.get()); + self->mDryData.Clear(); + self->mDryPromise.Reject(aError, __func__); + }) + ->Track(mDrainRequest); +} + +void DecoderAgent::SetState(State aState) { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + + auto validateStateTransition = [](State aOldState, State aNewState) { + switch (aOldState) { + case State::Unconfigured: + return aNewState == State::Configuring; + case State::Configuring: + return aNewState == State::Configured || aNewState == State::Error || + aNewState == State::Unconfigured || + aNewState == State::ShuttingDown; + case State::Configured: + return aNewState == State::Unconfigured || + aNewState == State::Decoding || aNewState == State::Flushing; + case State::Decoding: + case State::Flushing: + return aNewState == State::Configured || aNewState == State::Error || + aNewState == State::Unconfigured; + case State::ShuttingDown: + return aNewState == State::Unconfigured; + case State::Error: + return aNewState == State::Unconfigured; + default: + break; + } + MOZ_ASSERT_UNREACHABLE("Unhandled state transition"); + return false; + }; + + auto stateToString = [](State aState) -> const char* { + switch (aState) { + case State::Unconfigured: + return "Unconfigured"; + case State::Configuring: + return "Configuring"; + case State::Configured: + return "Configured"; + case State::Decoding: + return "Decoding"; + case State::Flushing: + return "Flushing"; + case State::ShuttingDown: + return "ShuttingDown"; + case State::Error: + return "Error"; + default: + break; + } + MOZ_ASSERT_UNREACHABLE("Unhandled state type"); + return "Unknown"; + }; + + DebugOnly<bool> isValid = validateStateTransition(mState, aState); + MOZ_ASSERT(isValid); + LOG("DecoderAgent #%d (%p) state change: %s -> %s", mId, this, + stateToString(mState), stateToString(aState)); + mState = aState; +} + +#undef LOG +#undef LOGW +#undef LOGE +#undef LOGV +#undef LOG_INTERNAL + +} // namespace mozilla diff --git a/dom/media/webcodecs/DecoderAgent.h b/dom/media/webcodecs/DecoderAgent.h new file mode 100644 index 0000000000..f8107b27a6 --- /dev/null +++ b/dom/media/webcodecs/DecoderAgent.h @@ -0,0 +1,117 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_WEBCODECS_DECODERAGENT_H +#define DOM_MEDIA_WEBCODECS_DECODERAGENT_H + +#include "MediaResult.h" +#include "PlatformDecoderModule.h" +#include "mozilla/RefPtr.h" +#include "mozilla/TaskQueue.h" +#include "mozilla/UniquePtr.h" + +class nsISerialEventTarget; + +namespace mozilla { + +class PDMFactory; +class TrackInfo; + +namespace layers { +class ImageContainer; +} // namespace layers + +// DecoderAgent is a wrapper that contains a MediaDataDecoder. It adapts the +// MediaDataDecoder APIs for use in WebCodecs. +// +// If Configure() is called, Shutdown() must be called to release the resources +// gracefully. Except Shutdown(), all the methods can't be called concurrently, +// meaning a method can only be called when the previous API call has completed. +// The responsability of arranging the method calls is on the caller. +// +// When Shutdown() is called, all the operations in flight are canceled and the +// MediaDataDecoder is shut down. On the other hand, errors are final. A new +// DecoderAgent must be created when an error is encountered. +// +// All the methods need to be called on the DecoderAgent's owner thread. In +// WebCodecs, it's either on the main thread or worker thread. +class DecoderAgent final { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DecoderAgent); + + using Id = uint32_t; + DecoderAgent(Id aId, UniquePtr<TrackInfo>&& aInfo); + + // The following APIs are owner thread only. + + using ConfigurePromise = MozPromise<bool, MediaResult, true /* exclusive */>; + RefPtr<ConfigurePromise> Configure(bool aPreferSoftwareDecoder, + bool aLowLatency); + RefPtr<ShutdownPromise> Shutdown(); + using DecodePromise = MediaDataDecoder::DecodePromise; + RefPtr<DecodePromise> Decode(MediaRawData* aSample); + // WebCodecs's flush() flushes out all the pending decoded data in the + // MediaDataDecoder so it is a combination of Drain and Flush. To distinguish + // the term from MediaDataDecoder's one, we call it DrainAndFlush() here. + RefPtr<DecodePromise> DrainAndFlush(); + + const Id mId; // A unique id. + const UniquePtr<TrackInfo> mInfo; + + private: + ~DecoderAgent(); + + // Push out all the data in the MediaDataDecoder's pipeline. + // TODO: MediaDataDecoder should implement this, instead of asking call site + // to run `Drain` multiple times. + RefPtr<DecodePromise> Dry(); + void DrainUntilDry(); + + enum class State { + Unconfigured, + Configuring, + Configured, + Decoding, + Flushing, + ShuttingDown, + Error, + }; + void SetState(State aState); + + const RefPtr<nsISerialEventTarget> mOwnerThread; + const RefPtr<PDMFactory> mPDMFactory; + const RefPtr<layers::ImageContainer> mImageContainer; + RefPtr<MediaDataDecoder> mDecoder; + State mState; + + // Configure + MozPromiseHolder<ConfigurePromise> mConfigurePromise; + using CreateDecoderPromise = PlatformDecoderModule::CreateDecoderPromise; + MozPromiseRequestHolder<CreateDecoderPromise> mCreateRequest; + using InitPromise = MediaDataDecoder::InitPromise; + MozPromiseRequestHolder<InitPromise> mInitRequest; + + // Shutdown + MozPromiseHolder<ShutdownPromise> mShutdownWhileCreationPromise; + + // Decode + MozPromiseHolder<DecodePromise> mDecodePromise; + MozPromiseRequestHolder<DecodePromise> mDecodeRequest; + + // DrainAndFlush + MozPromiseHolder<DecodePromise> mDrainAndFlushPromise; + MediaDataDecoder::DecodedData mDrainAndFlushData; + MozPromiseRequestHolder<DecodePromise> mDryRequest; + MozPromiseHolder<DecodePromise> mDryPromise; + MediaDataDecoder::DecodedData mDryData; + MozPromiseRequestHolder<DecodePromise> mDrainRequest; + using FlushPromise = MediaDataDecoder::FlushPromise; + MozPromiseRequestHolder<FlushPromise> mFlushRequest; +}; + +} // namespace mozilla + +#endif // DOM_MEDIA_WEBCODECS_DECODERAGENT_H diff --git a/dom/media/webcodecs/DecoderTemplate.cpp b/dom/media/webcodecs/DecoderTemplate.cpp new file mode 100644 index 0000000000..0fa25a208b --- /dev/null +++ b/dom/media/webcodecs/DecoderTemplate.cpp @@ -0,0 +1,891 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "DecoderTemplate.h" + +#include <atomic> +#include <utility> + +#include "DecoderTypes.h" +#include "MediaInfo.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/Try.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/DOMException.h" +#include "mozilla/dom/Event.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/VideoDecoderBinding.h" +#include "mozilla/dom/VideoFrame.h" +#include "mozilla/dom/WorkerCommon.h" +#include "nsGkAtoms.h" +#include "nsString.h" +#include "nsThreadUtils.h" + +mozilla::LazyLogModule gWebCodecsLog("WebCodecs"); + +namespace mozilla::dom { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOG +# undef LOG +#endif // LOG +#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +#ifdef LOGW +# undef LOGW +#endif // LOGW +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +#ifdef LOGV +# undef LOGV +#endif // LOGV +#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__) + +/* + * Below are ControlMessage classes implementations + */ + +template <typename DecoderType> +DecoderTemplate<DecoderType>::ControlMessage::ControlMessage( + const nsACString& aTitle) + : mTitle(aTitle) {} + +template <typename DecoderType> +DecoderTemplate<DecoderType>::ConfigureMessage::ConfigureMessage( + Id aId, UniquePtr<ConfigTypeInternal>&& aConfig) + : ControlMessage( + nsPrintfCString("configure #%d (%s)", aId, + NS_ConvertUTF16toUTF8(aConfig->mCodec).get())), + mId(aId), + mConfig(std::move(aConfig)) {} + +/* static */ +template <typename DecoderType> +typename DecoderTemplate<DecoderType>::ConfigureMessage* +DecoderTemplate<DecoderType>::ConfigureMessage::Create( + UniquePtr<ConfigTypeInternal>&& aConfig) { + // This needs to be atomic since this can run on the main thread or worker + // thread. + static std::atomic<Id> sNextId = NoId; + return new ConfigureMessage(++sNextId, std::move(aConfig)); +} + +template <typename DecoderType> +DecoderTemplate<DecoderType>::DecodeMessage::DecodeMessage( + Id aId, ConfigId aConfigId, UniquePtr<InputTypeInternal>&& aData) + : ControlMessage( + nsPrintfCString("decode #%zu (config #%d)", aId, aConfigId)), + mId(aId), + mData(std::move(aData)) {} + +template <typename DecoderType> +DecoderTemplate<DecoderType>::FlushMessage::FlushMessage(Id aId, + ConfigId aConfigId, + Promise* aPromise) + : ControlMessage( + nsPrintfCString("flush #%zu (config #%d)", aId, aConfigId)), + mId(aId), + mPromise(aPromise) {} + +template <typename DecoderType> +void DecoderTemplate<DecoderType>::FlushMessage::RejectPromiseIfAny( + const nsresult& aReason) { + if (mPromise) { + mPromise->MaybeReject(aReason); + } +} + +/* + * Below are DecoderTemplate implementation + */ + +template <typename DecoderType> +DecoderTemplate<DecoderType>::DecoderTemplate( + nsIGlobalObject* aGlobalObject, + RefPtr<WebCodecsErrorCallback>&& aErrorCallback, + RefPtr<OutputCallbackType>&& aOutputCallback) + : DOMEventTargetHelper(aGlobalObject), + mErrorCallback(std::move(aErrorCallback)), + mOutputCallback(std::move(aOutputCallback)), + mState(CodecState::Unconfigured), + mKeyChunkRequired(true), + mMessageQueueBlocked(false), + mDecodeQueueSize(0), + mDequeueEventScheduled(false), + mLatestConfigureId(ConfigureMessage::NoId), + mDecodeCounter(0), + mFlushCounter(0) {} + +template <typename DecoderType> +void DecoderTemplate<DecoderType>::Configure(const ConfigType& aConfig, + ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s %p, Configure: codec %s", DecoderType::Name.get(), this, + NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); + + nsCString errorMessage; + if (!DecoderType::Validate(aConfig, errorMessage)) { + aRv.ThrowTypeError( + nsPrintfCString("config is invalid: %s", errorMessage.get())); + return; + } + + if (mState == CodecState::Closed) { + LOG("Configure: CodecState::Closed, rejecting with InvalidState"); + aRv.ThrowInvalidStateError("The codec is no longer usable"); + return; + } + + // Clone a ConfigType as the active decoder config. + UniquePtr<ConfigTypeInternal> config = + DecoderType::CreateConfigInternal(aConfig); + if (!config) { + aRv.Throw(NS_ERROR_UNEXPECTED); // Invalid description data. + return; + } + + mState = CodecState::Configured; + mKeyChunkRequired = true; + mDecodeCounter = 0; + mFlushCounter = 0; + + mControlMessageQueue.emplace( + UniquePtr<ControlMessage>(ConfigureMessage::Create(std::move(config)))); + mLatestConfigureId = mControlMessageQueue.back()->AsConfigureMessage()->mId; + LOG("%s %p enqueues %s", DecoderType::Name.get(), this, + mControlMessageQueue.back()->ToString().get()); + ProcessControlMessageQueue(); +} + +template <typename DecoderType> +void DecoderTemplate<DecoderType>::Decode(InputType& aInput, ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s %p, Decode", DecoderType::Name.get(), this); + + if (mState != CodecState::Configured) { + aRv.ThrowInvalidStateError("Decoder must be configured first"); + return; + } + + if (mKeyChunkRequired) { + // TODO: Verify input's data is truly a key chunk + if (!DecoderType::IsKeyChunk(aInput)) { + aRv.ThrowDataError( + nsPrintfCString("%s needs a key chunk", DecoderType::Name.get())); + return; + } + mKeyChunkRequired = false; + } + + mDecodeQueueSize += 1; + mControlMessageQueue.emplace(UniquePtr<ControlMessage>( + new DecodeMessage(++mDecodeCounter, mLatestConfigureId, + DecoderType::CreateInputInternal(aInput)))); + LOGV("%s %p enqueues %s", DecoderType::Name.get(), this, + mControlMessageQueue.back()->ToString().get()); + ProcessControlMessageQueue(); +} + +template <typename DecoderType> +already_AddRefed<Promise> DecoderTemplate<DecoderType>::Flush( + ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s %p, Flush", DecoderType::Name.get(), this); + + if (mState != CodecState::Configured) { + LOG("%s %p, wrong state!", DecoderType::Name.get(), this); + aRv.ThrowInvalidStateError("Decoder must be configured first"); + return nullptr; + } + + RefPtr<Promise> p = Promise::Create(GetParentObject(), aRv); + if (NS_WARN_IF(aRv.Failed())) { + return p.forget(); + } + + mKeyChunkRequired = true; + + mControlMessageQueue.emplace(UniquePtr<ControlMessage>( + new FlushMessage(++mFlushCounter, mLatestConfigureId, p))); + LOG("%s %p enqueues %s", DecoderType::Name.get(), this, + mControlMessageQueue.back()->ToString().get()); + ProcessControlMessageQueue(); + return p.forget(); +} + +template <typename DecoderType> +void DecoderTemplate<DecoderType>::Reset(ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s %p, Reset", DecoderType::Name.get(), this); + + if (auto r = ResetInternal(NS_ERROR_DOM_ABORT_ERR); r.isErr()) { + aRv.Throw(r.unwrapErr()); + } +} + +template <typename DecoderType> +void DecoderTemplate<DecoderType>::Close(ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s %p, Close", DecoderType::Name.get(), this); + + if (auto r = CloseInternalWithAbort(); r.isErr()) { + aRv.Throw(r.unwrapErr()); + } +} + +template <typename DecoderType> +Result<Ok, nsresult> DecoderTemplate<DecoderType>::ResetInternal( + const nsresult& aResult) { + AssertIsOnOwningThread(); + + if (mState == CodecState::Closed) { + return Err(NS_ERROR_DOM_INVALID_STATE_ERR); + } + + mState = CodecState::Unconfigured; + mDecodeCounter = 0; + mFlushCounter = 0; + + CancelPendingControlMessages(aResult); + DestroyDecoderAgentIfAny(); + + if (mDecodeQueueSize > 0) { + mDecodeQueueSize = 0; + ScheduleDequeueEventIfNeeded(); + } + + LOG("%s %p now has its message queue unblocked", DecoderType::Name.get(), + this); + mMessageQueueBlocked = false; + + return Ok(); +} +template <typename DecoderType> +Result<Ok, nsresult> DecoderTemplate<DecoderType>::CloseInternalWithAbort() { + AssertIsOnOwningThread(); + + MOZ_TRY(ResetInternal(NS_ERROR_DOM_ABORT_ERR)); + mState = CodecState::Closed; + return Ok(); +} + +template <typename DecoderType> +void DecoderTemplate<DecoderType>::CloseInternal(const nsresult& aResult) { + AssertIsOnOwningThread(); + MOZ_ASSERT(aResult != NS_ERROR_DOM_ABORT_ERR, "Use CloseInternalWithAbort"); + + auto r = ResetInternal(aResult); + if (r.isErr()) { + nsCString name; + GetErrorName(r.unwrapErr(), name); + LOGE("Error in ResetInternal: %s", name.get()); + MOZ_CRASH(); + } + mState = CodecState::Closed; + nsCString error; + GetErrorName(aResult, error); + LOGE("%s %p Close on error: %s", DecoderType::Name.get(), this, error.get()); + ReportError(aResult); +} + +template <typename DecoderType> +void DecoderTemplate<DecoderType>::ReportError(const nsresult& aResult) { + AssertIsOnOwningThread(); + + RefPtr<DOMException> e = DOMException::Create(aResult); + RefPtr<WebCodecsErrorCallback> cb(mErrorCallback); + cb->Call(*e); +} + +template <typename DecoderType> +void DecoderTemplate<DecoderType>::OutputDecodedData( + const nsTArray<RefPtr<MediaData>>&& aData) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(mActiveConfig); + + nsTArray<RefPtr<VideoFrame>> frames = DecodedDataToOutputType( + GetParentObject(), std::move(aData), *mActiveConfig); + RefPtr<VideoFrameOutputCallback> cb(mOutputCallback); + for (RefPtr<VideoFrame>& frame : frames) { + LOG("Outputing decoded data: ts: %" PRId64, frame->Timestamp()); + RefPtr<VideoFrame> f = frame; + cb->Call((VideoFrame&)(*f)); + } +} + +template <typename DecoderType> +void DecoderTemplate<DecoderType>::ScheduleDequeueEventIfNeeded() { + AssertIsOnOwningThread(); + + if (mDequeueEventScheduled) { + return; + } + mDequeueEventScheduled = true; + + QueueATask("dequeue event task", [self = RefPtr{this}]() { + self->FireEvent(nsGkAtoms::ondequeue, u"dequeue"_ns); + self->mDequeueEventScheduled = false; + }); +} + +template <typename DecoderType> +nsresult DecoderTemplate<DecoderType>::FireEvent(nsAtom* aTypeWithOn, + const nsAString& aEventType) { + if (aTypeWithOn && !HasListenersFor(aTypeWithOn)) { + LOGV("%s %p has no %s event listener", DecoderType::Name.get(), this, + NS_ConvertUTF16toUTF8(aEventType).get()); + return NS_ERROR_ABORT; + } + + LOGV("Dispatch %s event to %s %p", NS_ConvertUTF16toUTF8(aEventType).get(), + DecoderType::Name.get(), this); + RefPtr<Event> event = new Event(this, nullptr, nullptr); + event->InitEvent(aEventType, true, true); + event->SetTrusted(true); + this->DispatchEvent(*event); + return NS_OK; +} + +template <typename DecoderType> +void DecoderTemplate<DecoderType>::ProcessControlMessageQueue() { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + + while (!mMessageQueueBlocked && !mControlMessageQueue.empty()) { + UniquePtr<ControlMessage>& msg = mControlMessageQueue.front(); + if (msg->AsConfigureMessage()) { + if (ProcessConfigureMessage(msg) == + MessageProcessedResult::NotProcessed) { + break; + } + } else if (msg->AsDecodeMessage()) { + if (ProcessDecodeMessage(msg) == MessageProcessedResult::NotProcessed) { + break; + } + } else { + MOZ_ASSERT(msg->AsFlushMessage()); + if (ProcessFlushMessage(msg) == MessageProcessedResult::NotProcessed) { + break; + } + } + } +} + +template <typename DecoderType> +void DecoderTemplate<DecoderType>::CancelPendingControlMessages( + const nsresult& aResult) { + AssertIsOnOwningThread(); + + // Cancel the message that is being processed. + if (mProcessingMessage) { + LOG("%s %p cancels current %s", DecoderType::Name.get(), this, + mProcessingMessage->ToString().get()); + mProcessingMessage->Cancel(); + + if (FlushMessage* flush = mProcessingMessage->AsFlushMessage()) { + flush->RejectPromiseIfAny(aResult); + } + + mProcessingMessage.reset(); + } + + // Clear the message queue. + while (!mControlMessageQueue.empty()) { + LOG("%s %p cancels pending %s", DecoderType::Name.get(), this, + mControlMessageQueue.front()->ToString().get()); + + MOZ_ASSERT(!mControlMessageQueue.front()->IsProcessing()); + if (FlushMessage* flush = mControlMessageQueue.front()->AsFlushMessage()) { + flush->RejectPromiseIfAny(aResult); + } + + mControlMessageQueue.pop(); + } +} + +template <typename DecoderType> +template <typename Func> +void DecoderTemplate<DecoderType>::QueueATask(const char* aName, + Func&& aSteps) { + AssertIsOnOwningThread(); + MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread( + NS_NewRunnableFunction(aName, std::forward<Func>(aSteps)))); +} + +template <typename DecoderType> +MessageProcessedResult DecoderTemplate<DecoderType>::ProcessConfigureMessage( + UniquePtr<ControlMessage>& aMessage) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(aMessage->AsConfigureMessage()); + + if (mProcessingMessage) { + LOG("%s %p is processing %s. Defer %s", DecoderType::Name.get(), this, + mProcessingMessage->ToString().get(), aMessage->ToString().get()); + return MessageProcessedResult::NotProcessed; + } + + mProcessingMessage.reset(aMessage.release()); + mControlMessageQueue.pop(); + + ConfigureMessage* msg = mProcessingMessage->AsConfigureMessage(); + LOG("%s %p starts processing %s", DecoderType::Name.get(), this, + msg->ToString().get()); + + DestroyDecoderAgentIfAny(); + + mMessageQueueBlocked = true; + + nsAutoCString errorMessage; + auto i = DecoderType::CreateTrackInfo(msg->Config()); + if (i.isErr()) { + nsCString res; + GetErrorName(i.unwrapErr(), res); + errorMessage.AppendPrintf("CreateTrackInfo failed: %s", res.get()); + } else if (!DecoderType::IsSupported(msg->Config())) { + errorMessage.Append("Not supported."); + } else if (!CreateDecoderAgent(msg->mId, msg->TakeConfig(), i.unwrap())) { + errorMessage.Append("DecoderAgent creation failed."); + } + if (!errorMessage.IsEmpty()) { + LOGE("%s %p ProcessConfigureMessage error (sync): %s", + DecoderType::Name.get(), this, errorMessage.get()); + + mProcessingMessage.reset(); + QueueATask("Error while configuring decoder", + [self = RefPtr{this}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { + MOZ_ASSERT(self->mState != CodecState::Closed); + self->CloseInternal(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + }); + return MessageProcessedResult::Processed; + } + + MOZ_ASSERT(mAgent); + MOZ_ASSERT(mActiveConfig); + + LOG("%s %p now blocks message-queue-processing", DecoderType::Name.get(), + this); + + bool preferSW = mActiveConfig->mHardwareAcceleration == + HardwareAcceleration::Prefer_software; + bool lowLatency = mActiveConfig->mOptimizeForLatency.isSome() && + mActiveConfig->mOptimizeForLatency.value(); + mAgent->Configure(preferSW, lowLatency) + ->Then(GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = mAgent->mId]( + const DecoderAgent::ConfigurePromise::ResolveOrRejectValue& + aResult) { + MOZ_ASSERT(self->mProcessingMessage); + MOZ_ASSERT(self->mProcessingMessage->AsConfigureMessage()); + MOZ_ASSERT(self->mState == CodecState::Configured); + MOZ_ASSERT(self->mAgent); + MOZ_ASSERT(id == self->mAgent->mId); + MOZ_ASSERT(self->mActiveConfig); + + ConfigureMessage* msg = + self->mProcessingMessage->AsConfigureMessage(); + LOG("%s %p, DecoderAgent #%d %s has been %s. now unblocks " + "message-queue-processing", + DecoderType::Name.get(), self.get(), id, + msg->ToString().get(), + aResult.IsResolve() ? "resolved" : "rejected"); + + msg->Complete(); + self->mProcessingMessage.reset(); + + if (aResult.IsReject()) { + // The spec asks to close the decoder with an + // NotSupportedError so we log the exact error here. + const MediaResult& error = aResult.RejectValue(); + LOGE("%s %p, DecoderAgent #%d failed to configure: %s", + DecoderType::Name.get(), self.get(), id, + error.Description().get()); + + self->QueueATask( + "Error during configure", + [self = RefPtr{self}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { + MOZ_ASSERT(self->mState != CodecState::Closed); + self->CloseInternal( + NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + }); + return; + } + + self->mMessageQueueBlocked = false; + self->ProcessControlMessageQueue(); + }) + ->Track(msg->Request()); + + return MessageProcessedResult::Processed; +} + +template <typename DecoderType> +MessageProcessedResult DecoderTemplate<DecoderType>::ProcessDecodeMessage( + UniquePtr<ControlMessage>& aMessage) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(aMessage->AsDecodeMessage()); + + if (mProcessingMessage) { + LOGV("%s %p is processing %s. Defer %s", DecoderType::Name.get(), this, + mProcessingMessage->ToString().get(), aMessage->ToString().get()); + return MessageProcessedResult::NotProcessed; + } + + mProcessingMessage.reset(aMessage.release()); + mControlMessageQueue.pop(); + + DecodeMessage* msg = mProcessingMessage->AsDecodeMessage(); + LOGV("%s %p starts processing %s", DecoderType::Name.get(), this, + msg->ToString().get()); + + mDecodeQueueSize -= 1; + ScheduleDequeueEventIfNeeded(); + + // Treat it like decode error if no DecoderAgent is available or the encoded + // data is invalid. + auto closeOnError = [&]() { + mProcessingMessage.reset(); + QueueATask("Error during decode", + [self = RefPtr{this}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { + MOZ_ASSERT(self->mState != CodecState::Closed); + self->CloseInternal(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + }); + return MessageProcessedResult::Processed; + }; + + if (!mAgent) { + LOGE("%s %p is not configured", DecoderType::Name.get(), this); + return closeOnError(); + } + + MOZ_ASSERT(mActiveConfig); + RefPtr<MediaRawData> data = InputDataToMediaRawData( + std::move(msg->mData), *(mAgent->mInfo), *mActiveConfig); + if (!data) { + LOGE("%s %p, data for %s is empty or invalid", DecoderType::Name.get(), + this, msg->ToString().get()); + return closeOnError(); + } + + mAgent->Decode(data.get()) + ->Then( + GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = mAgent->mId]( + DecoderAgent::DecodePromise::ResolveOrRejectValue&& aResult) { + MOZ_ASSERT(self->mProcessingMessage); + MOZ_ASSERT(self->mProcessingMessage->AsDecodeMessage()); + MOZ_ASSERT(self->mState == CodecState::Configured); + MOZ_ASSERT(self->mAgent); + MOZ_ASSERT(id == self->mAgent->mId); + MOZ_ASSERT(self->mActiveConfig); + + DecodeMessage* msg = self->mProcessingMessage->AsDecodeMessage(); + LOGV("%s %p, DecoderAgent #%d %s has been %s", + DecoderType::Name.get(), self.get(), id, msg->ToString().get(), + aResult.IsResolve() ? "resolved" : "rejected"); + + nsCString msgStr = msg->ToString(); + + msg->Complete(); + self->mProcessingMessage.reset(); + + if (aResult.IsReject()) { + // The spec asks to queue a task to run close the decoder + // with an EncodingError so we log the exact error here. + const MediaResult& error = aResult.RejectValue(); + LOGE("%s %p, DecoderAgent #%d %s failed: %s", + DecoderType::Name.get(), self.get(), id, msgStr.get(), + error.Description().get()); + self->QueueATask( + "Error during decode runnable", + [self = RefPtr{self}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { + MOZ_ASSERT(self->mState != CodecState::Closed); + self->CloseInternal( + NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + }); + return; + } + + MOZ_ASSERT(aResult.IsResolve()); + nsTArray<RefPtr<MediaData>> data = + std::move(aResult.ResolveValue()); + if (data.IsEmpty()) { + LOGV("%s %p got no data for %s", DecoderType::Name.get(), + self.get(), msgStr.get()); + } else { + LOGV("%s %p, schedule %zu decoded data output for %s", + DecoderType::Name.get(), self.get(), data.Length(), + msgStr.get()); + self->QueueATask("Output Decoded Data", + [self = RefPtr{self}, data = std::move(data)]() + MOZ_CAN_RUN_SCRIPT_BOUNDARY { + self->OutputDecodedData(std::move(data)); + }); + } + self->ProcessControlMessageQueue(); + }) + ->Track(msg->Request()); + + return MessageProcessedResult::Processed; +} + +template <typename DecoderType> +MessageProcessedResult DecoderTemplate<DecoderType>::ProcessFlushMessage( + UniquePtr<ControlMessage>& aMessage) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(aMessage->AsFlushMessage()); + + if (mProcessingMessage) { + LOG("%s %p is processing %s. Defer %s", DecoderType::Name.get(), this, + mProcessingMessage->ToString().get(), aMessage->ToString().get()); + return MessageProcessedResult::NotProcessed; + } + + mProcessingMessage.reset(aMessage.release()); + mControlMessageQueue.pop(); + + FlushMessage* msg = mProcessingMessage->AsFlushMessage(); + LOG("%s %p starts processing %s", DecoderType::Name.get(), this, + msg->ToString().get()); + + // No agent, no thing to do. The promise has been rejected with the + // appropriate error in ResetInternal already. + if (!mAgent) { + LOGE("%s %p no agent, nothing to do", DecoderType::Name.get(), this); + mProcessingMessage.reset(); + return MessageProcessedResult::Processed; + } + + mAgent->DrainAndFlush() + ->Then( + GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = mAgent->mId, + this](DecoderAgent::DecodePromise::ResolveOrRejectValue&& aResult) { + MOZ_ASSERT(self->mProcessingMessage); + MOZ_ASSERT(self->mProcessingMessage->AsFlushMessage()); + MOZ_ASSERT(self->mState == CodecState::Configured); + MOZ_ASSERT(self->mAgent); + MOZ_ASSERT(id == self->mAgent->mId); + MOZ_ASSERT(self->mActiveConfig); + + FlushMessage* msg = self->mProcessingMessage->AsFlushMessage(); + LOG("%s %p, DecoderAgent #%d %s has been %s", + DecoderType::Name.get(), self.get(), id, msg->ToString().get(), + aResult.IsResolve() ? "resolved" : "rejected"); + + nsCString msgStr = msg->ToString(); + + msg->Complete(); + + // If flush failed, it means decoder fails to decode the data + // sent before, so we treat it like decode error. We reject + // the promise first and then queue a task to close + // VideoDecoder with an EncodingError. + if (aResult.IsReject()) { + const MediaResult& error = aResult.RejectValue(); + LOGE("%s %p, DecoderAgent #%d failed to flush: %s", + DecoderType::Name.get(), self.get(), id, + error.Description().get()); + RefPtr<Promise> promise = msg->TakePromise(); + // Reject with an EncodingError instead of the error we got + // above. + self->QueueATask( + "Error during flush runnable", + [self = RefPtr{this}, promise]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { + promise->MaybeReject( + NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + self->mProcessingMessage.reset(); + MOZ_ASSERT(self->mState != CodecState::Closed); + self->CloseInternal( + NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + }); + return; + } + + nsTArray<RefPtr<MediaData>> data = + std::move(aResult.ResolveValue()); + + if (data.IsEmpty()) { + LOG("%s %p gets no data for %s", DecoderType::Name.get(), + self.get(), msgStr.get()); + } else { + LOG("%s %p, schedule %zu decoded data output for %s", + DecoderType::Name.get(), self.get(), data.Length(), + msgStr.get()); + } + + RefPtr<Promise> promise = msg->TakePromise(); + self->QueueATask( + "Flush: output decoding data task", + [self = RefPtr{self}, promise, data = std::move(data)]() + MOZ_CAN_RUN_SCRIPT_BOUNDARY { + self->OutputDecodedData(std::move(data)); + promise->MaybeResolveWithUndefined(); + }); + self->mProcessingMessage.reset(); + self->ProcessControlMessageQueue(); + }) + ->Track(msg->Request()); + + return MessageProcessedResult::Processed; +} + +// CreateDecoderAgent will create an DecoderAgent paired with a xpcom-shutdown +// blocker and a worker-reference. Besides the needs mentioned in the header +// file, the blocker and the worker-reference also provides an entry point for +// us to clean up the resources. Other than the decoder dtor, Reset(), or +// Close(), the resources should be cleaned up in the following situations: +// 1. Decoder on window, closing document +// 2. Decoder on worker, closing document +// 3. Decoder on worker, terminating worker +// +// In case 1, the entry point to clean up is in the mShutdownBlocker's +// ShutdownpPomise-resolver. In case 2, the entry point is in mWorkerRef's +// shutting down callback. In case 3, the entry point is in mWorkerRef's +// shutting down callback. + +template <typename DecoderType> +bool DecoderTemplate<DecoderType>::CreateDecoderAgent( + DecoderAgent::Id aId, UniquePtr<ConfigTypeInternal>&& aConfig, + UniquePtr<TrackInfo>&& aInfo) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(!mAgent); + MOZ_ASSERT(!mActiveConfig); + MOZ_ASSERT(!mShutdownBlocker); + MOZ_ASSERT_IF(!NS_IsMainThread(), !mWorkerRef); + + auto resetOnFailure = MakeScopeExit([&]() { + mAgent = nullptr; + mActiveConfig = nullptr; + mShutdownBlocker = nullptr; + mWorkerRef = nullptr; + }); + + // If the decoder is on worker, get a worker reference. + if (!NS_IsMainThread()) { + WorkerPrivate* workerPrivate = GetCurrentThreadWorkerPrivate(); + if (NS_WARN_IF(!workerPrivate)) { + return false; + } + + // Clean up all the resources when worker is going away. + RefPtr<StrongWorkerRef> workerRef = StrongWorkerRef::Create( + workerPrivate, "DecoderTemplate::CreateDecoderAgent", + [self = RefPtr{this}]() { + LOG("%s %p, worker is going away", DecoderType::Name.get(), + self.get()); + Unused << self->ResetInternal(NS_ERROR_DOM_ABORT_ERR); + }); + if (NS_WARN_IF(!workerRef)) { + return false; + } + + mWorkerRef = new ThreadSafeWorkerRef(workerRef); + } + + mAgent = MakeRefPtr<DecoderAgent>(aId, std::move(aInfo)); + mActiveConfig = std::move(aConfig); + + // ShutdownBlockingTicket requires an unique name to register its own + // nsIAsyncShutdownBlocker since each blocker needs a distinct name. + // To do that, we use DecoderAgent's unique id to create a unique name. + nsAutoString uniqueName; + uniqueName.AppendPrintf( + "Blocker for DecoderAgent #%d (codec: %s) @ %p", mAgent->mId, + NS_ConvertUTF16toUTF8(mActiveConfig->mCodec).get(), mAgent.get()); + + mShutdownBlocker = media::ShutdownBlockingTicket::Create( + uniqueName, NS_LITERAL_STRING_FROM_CSTRING(__FILE__), __LINE__); + if (!mShutdownBlocker) { + LOGE("%s %p failed to create %s", DecoderType::Name.get(), this, + NS_ConvertUTF16toUTF8(uniqueName).get()); + return false; + } + + // Clean up all the resources when xpcom-will-shutdown arrives since the page + // is going to be closed. + mShutdownBlocker->ShutdownPromise()->Then( + GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = mAgent->mId, + ref = mWorkerRef](bool /* aUnUsed*/) { + LOG("%s %p gets xpcom-will-shutdown notification for DecoderAgent #%d", + DecoderType::Name.get(), self.get(), id); + Unused << self->ResetInternal(NS_ERROR_DOM_ABORT_ERR); + }, + [self = RefPtr{this}, id = mAgent->mId, + ref = mWorkerRef](bool /* aUnUsed*/) { + LOG("%s %p removes shutdown-blocker #%d before getting any " + "notification. DecoderAgent #%d should have been dropped", + DecoderType::Name.get(), self.get(), id, id); + MOZ_ASSERT(!self->mAgent || self->mAgent->mId != id); + }); + + LOG("%s %p creates DecoderAgent #%d @ %p and its shutdown-blocker", + DecoderType::Name.get(), this, mAgent->mId, mAgent.get()); + + resetOnFailure.release(); + return true; +} + +template <typename DecoderType> +void DecoderTemplate<DecoderType>::DestroyDecoderAgentIfAny() { + AssertIsOnOwningThread(); + + if (!mAgent) { + LOG("%s %p has no DecoderAgent to destroy", DecoderType::Name.get(), this); + return; + } + + MOZ_ASSERT(mActiveConfig); + MOZ_ASSERT(mShutdownBlocker); + MOZ_ASSERT_IF(!NS_IsMainThread(), mWorkerRef); + + LOG("%s %p destroys DecoderAgent #%d @ %p", DecoderType::Name.get(), this, + mAgent->mId, mAgent.get()); + mActiveConfig = nullptr; + RefPtr<DecoderAgent> agent = std::move(mAgent); + // mShutdownBlocker should be kept alive until the shutdown is done. + // mWorkerRef is used to ensure this task won't be discarded in worker. + agent->Shutdown()->Then( + GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = agent->mId, ref = std::move(mWorkerRef), + blocker = std::move(mShutdownBlocker)]( + const ShutdownPromise::ResolveOrRejectValue& aResult) { + LOG("%s %p, DecoderAgent #%d's shutdown has been %s. Drop its " + "shutdown-blocker now", + DecoderType::Name.get(), self.get(), id, + aResult.IsResolve() ? "resolved" : "rejected"); + }); +} + +template class DecoderTemplate<VideoDecoderTraits>; + +#undef LOG +#undef LOGW +#undef LOGE +#undef LOGV +#undef LOG_INTERNAL + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/DecoderTemplate.h b/dom/media/webcodecs/DecoderTemplate.h new file mode 100644 index 0000000000..fe0cb5baee --- /dev/null +++ b/dom/media/webcodecs/DecoderTemplate.h @@ -0,0 +1,260 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_DecoderTemplate_h +#define mozilla_dom_DecoderTemplate_h + +#include <queue> + +#include "mozilla/DOMEventTargetHelper.h" +#include "mozilla/DecoderAgent.h" +#include "mozilla/MozPromise.h" +#include "mozilla/RefPtr.h" +#include "mozilla/Result.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/dom/WorkerRef.h" +#include "mozilla/media/MediaUtils.h" +#include "nsStringFwd.h" +#include "WebCodecsUtils.h" + +namespace mozilla { + +class TrackInfo; + +namespace dom { + +class WebCodecsErrorCallback; +class Promise; +enum class CodecState : uint8_t; + +template <typename DecoderType> +class DecoderTemplate : public DOMEventTargetHelper { + using Self = DecoderTemplate<DecoderType>; + using ConfigType = typename DecoderType::ConfigType; + using ConfigTypeInternal = typename DecoderType::ConfigTypeInternal; + using InputType = typename DecoderType::InputType; + using InputTypeInternal = typename DecoderType::InputTypeInternal; + using OutputType = typename DecoderType::OutputType; + using OutputCallbackType = typename DecoderType::OutputCallbackType; + + /* ControlMessage classes */ + protected: + class ConfigureMessage; + class DecodeMessage; + class FlushMessage; + + class ControlMessage { + public: + explicit ControlMessage(const nsACString& aTitle); + virtual ~ControlMessage() = default; + virtual void Cancel() = 0; + virtual bool IsProcessing() = 0; + + virtual const nsCString& ToString() const { return mTitle; } + virtual ConfigureMessage* AsConfigureMessage() { return nullptr; } + virtual DecodeMessage* AsDecodeMessage() { return nullptr; } + virtual FlushMessage* AsFlushMessage() { return nullptr; } + + const nsCString mTitle; // Used to identify the message in the logs. + }; + + class ConfigureMessage final + : public ControlMessage, + public MessageRequestHolder<DecoderAgent::ConfigurePromise> { + public: + using Id = DecoderAgent::Id; + static constexpr Id NoId = 0; + static ConfigureMessage* Create(UniquePtr<ConfigTypeInternal>&& aConfig); + + ~ConfigureMessage() = default; + virtual void Cancel() override { Disconnect(); } + virtual bool IsProcessing() override { return Exists(); }; + virtual ConfigureMessage* AsConfigureMessage() override { return this; } + const ConfigTypeInternal& Config() { return *mConfig; } + UniquePtr<ConfigTypeInternal> TakeConfig() { return std::move(mConfig); } + + const Id mId; // A unique id shown in log. + + private: + ConfigureMessage(Id aId, UniquePtr<ConfigTypeInternal>&& aConfig); + + UniquePtr<ConfigTypeInternal> mConfig; + }; + + class DecodeMessage final + : public ControlMessage, + public MessageRequestHolder<DecoderAgent::DecodePromise> { + public: + using Id = size_t; + using ConfigId = typename Self::ConfigureMessage::Id; + DecodeMessage(Id aId, ConfigId aConfigId, + UniquePtr<InputTypeInternal>&& aData); + ~DecodeMessage() = default; + virtual void Cancel() override { Disconnect(); } + virtual bool IsProcessing() override { return Exists(); }; + virtual DecodeMessage* AsDecodeMessage() override { return this; } + const Id mId; // A unique id shown in log. + + UniquePtr<InputTypeInternal> mData; + }; + + class FlushMessage final + : public ControlMessage, + public MessageRequestHolder<DecoderAgent::DecodePromise> { + public: + using Id = size_t; + using ConfigId = typename Self::ConfigureMessage::Id; + FlushMessage(Id aId, ConfigId aConfigId, Promise* aPromise); + ~FlushMessage() = default; + virtual void Cancel() override { Disconnect(); } + virtual bool IsProcessing() override { return Exists(); }; + virtual FlushMessage* AsFlushMessage() override { return this; } + already_AddRefed<Promise> TakePromise() { return mPromise.forget(); } + void RejectPromiseIfAny(const nsresult& aReason); + + const Id mId; // A unique id shown in log. + + private: + RefPtr<Promise> mPromise; + }; + + protected: + DecoderTemplate(nsIGlobalObject* aGlobalObject, + RefPtr<WebCodecsErrorCallback>&& aErrorCallback, + RefPtr<OutputCallbackType>&& aOutputCallback); + + virtual ~DecoderTemplate() = default; + + /* WebCodecs interfaces */ + public: + IMPL_EVENT_HANDLER(dequeue) + + CodecState State() const { return mState; }; + + uint32_t DecodeQueueSize() const { return mDecodeQueueSize; }; + + void Configure(const ConfigType& aConfig, ErrorResult& aRv); + + void Decode(InputType& aInput, ErrorResult& aRv); + + already_AddRefed<Promise> Flush(ErrorResult& aRv); + + void Reset(ErrorResult& aRv); + + void Close(ErrorResult& aRv); + + /* Type conversion functions for the Decoder implementation */ + protected: + virtual already_AddRefed<MediaRawData> InputDataToMediaRawData( + UniquePtr<InputTypeInternal>&& aData, TrackInfo& aInfo, + const ConfigTypeInternal& aConfig) = 0; + virtual nsTArray<RefPtr<OutputType>> DecodedDataToOutputType( + nsIGlobalObject* aGlobalObject, const nsTArray<RefPtr<MediaData>>&& aData, + ConfigTypeInternal& aConfig) = 0; + + protected: + // DecoderTemplate can run on either main thread or worker thread. + void AssertIsOnOwningThread() const { + NS_ASSERT_OWNINGTHREAD(DecoderTemplate); + } + + Result<Ok, nsresult> ResetInternal(const nsresult& aResult); + // Calling this method calls the error callback synchronously. + MOZ_CAN_RUN_SCRIPT + void CloseInternal(const nsresult& aResult); + // Calling this method doesn't call the error calback. + Result<Ok, nsresult> CloseInternalWithAbort(); + + MOZ_CAN_RUN_SCRIPT void ReportError(const nsresult& aResult); + MOZ_CAN_RUN_SCRIPT void OutputDecodedData( + const nsTArray<RefPtr<MediaData>>&& aData); + + void ScheduleDequeueEventIfNeeded(); + nsresult FireEvent(nsAtom* aTypeWithOn, const nsAString& aEventType); + + void ProcessControlMessageQueue(); + void CancelPendingControlMessages(const nsresult& aResult); + + // Queue a task to the control thread. This is to be used when a task needs to + // perform multiple steps. + template <typename Func> + void QueueATask(const char* aName, Func&& aSteps); + + MessageProcessedResult ProcessConfigureMessage( + UniquePtr<ControlMessage>& aMessage); + + MessageProcessedResult ProcessDecodeMessage( + UniquePtr<ControlMessage>& aMessage); + + MessageProcessedResult ProcessFlushMessage( + UniquePtr<ControlMessage>& aMessage); + + // Returns true when mAgent can be created. + bool CreateDecoderAgent(DecoderAgent::Id aId, + UniquePtr<ConfigTypeInternal>&& aConfig, + UniquePtr<TrackInfo>&& aInfo); + void DestroyDecoderAgentIfAny(); + + // Constant in practice, only set in ctor. + RefPtr<WebCodecsErrorCallback> mErrorCallback; + RefPtr<OutputCallbackType> mOutputCallback; + + CodecState mState; + bool mKeyChunkRequired; + + bool mMessageQueueBlocked; + std::queue<UniquePtr<ControlMessage>> mControlMessageQueue; + UniquePtr<ControlMessage> mProcessingMessage; + + uint32_t mDecodeQueueSize; + bool mDequeueEventScheduled; + + // A unique id tracking the ConfigureMessage and will be used as the + // DecoderAgent's Id. + uint32_t mLatestConfigureId; + // Tracking how many decode data has been enqueued and this number will be + // used as the DecodeMessage's Id. + size_t mDecodeCounter; + // Tracking how many flush request has been enqueued and this number will be + // used as the FlushMessage's Id. + size_t mFlushCounter; + + // DecoderAgent will be created every time "configure" is being processed, and + // will be destroyed when "reset" or another "configure" is called (spec + // allows calling two "configure" without a "reset" in between). + RefPtr<DecoderAgent> mAgent; + UniquePtr<ConfigTypeInternal> mActiveConfig; + + // Used to add a nsIAsyncShutdownBlocker on main thread to block + // xpcom-shutdown before the underlying MediaDataDecoder is created. The + // blocker will be held until the underlying MediaDataDecoder has been shut + // down. This blocker guarantees RemoteDecoderManagerChild's thread, where the + // underlying RemoteMediaDataDecoder is on, outlives the + // RemoteMediaDataDecoder, since the thread releasing, which happens on main + // thread when getting a xpcom-shutdown signal, is blocked by the added + // blocker. As a result, RemoteMediaDataDecoder can safely work on worker + // thread with a holding blocker (otherwise, if RemoteDecoderManagerChild + // releases its thread on main thread before RemoteMediaDataDecoder's + // Shutdown() task run on worker thread, RemoteMediaDataDecoder has no thread + // to run). + UniquePtr<media::ShutdownBlockingTicket> mShutdownBlocker; + + // Held to make sure the dispatched tasks can be done before worker is going + // away. As long as this worker-ref is held somewhere, the tasks dispatched to + // the worker can be executed (otherwise the tasks would be canceled). This + // ref should be activated as long as the underlying MediaDataDecoder is + // alive, and should keep alive until mShutdownBlocker is dropped, so all + // MediaDataDecoder's tasks and mShutdownBlocker-releasing task can be + // executed. + // TODO: Use StrongWorkerRef instead if this is always used in the same + // thread? + RefPtr<ThreadSafeWorkerRef> mWorkerRef; +}; + +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_DecoderTemplate_h diff --git a/dom/media/webcodecs/DecoderTypes.h b/dom/media/webcodecs/DecoderTypes.h new file mode 100644 index 0000000000..56aa82046f --- /dev/null +++ b/dom/media/webcodecs/DecoderTypes.h @@ -0,0 +1,117 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_DecoderTypes_h +#define mozilla_dom_DecoderTypes_h + +#include "MediaData.h" +#include "mozilla/Maybe.h" +#include "mozilla/dom/EncodedVideoChunk.h" +#include "mozilla/dom/VideoColorSpaceBinding.h" +#include "mozilla/dom/VideoDecoderBinding.h" +#include "mozilla/dom/VideoFrame.h" +#include "nsStringFwd.h" +#include "nsTLiteralString.h" + +namespace mozilla { + +class TrackInfo; +class MediaByteBuffer; + +namespace dom { + +struct VideoColorSpaceInternal { + explicit VideoColorSpaceInternal(const VideoColorSpaceInit& aColorSpaceInit); + VideoColorSpaceInternal() = default; + VideoColorSpaceInit ToColorSpaceInit() const; + + bool operator==(const VideoColorSpaceInternal& aOther) const { + return mFullRange == aOther.mFullRange && mMatrix == aOther.mMatrix && + mPrimaries == aOther.mPrimaries && mTransfer == aOther.mTransfer; + } + + Maybe<bool> mFullRange; + Maybe<VideoMatrixCoefficients> mMatrix; + Maybe<VideoColorPrimaries> mPrimaries; + Maybe<VideoTransferCharacteristics> mTransfer; +}; + +class VideoDecoderConfigInternal { + public: + static UniquePtr<VideoDecoderConfigInternal> Create( + const VideoDecoderConfig& aConfig); + VideoDecoderConfigInternal(const nsAString& aCodec, + Maybe<uint32_t>&& aCodedHeight, + Maybe<uint32_t>&& aCodedWidth, + Maybe<VideoColorSpaceInternal>&& aColorSpace, + Maybe<RefPtr<MediaByteBuffer>>&& aDescription, + Maybe<uint32_t>&& aDisplayAspectHeight, + Maybe<uint32_t>&& aDisplayAspectWidth, + const HardwareAcceleration& aHardwareAcceleration, + Maybe<bool>&& aOptimizeForLatency); + ~VideoDecoderConfigInternal() = default; + + nsString ToString() const; + + bool Equals(const VideoDecoderConfigInternal& aOther) const { + if (mDescription.isSome() != aOther.mDescription.isSome()) { + return false; + } + if (mDescription.isSome() && aOther.mDescription.isSome()) { + auto lhs = mDescription.value(); + auto rhs = aOther.mDescription.value(); + if (lhs->Length() != rhs->Length()) { + return false; + } + if (!ArrayEqual(lhs->Elements(), rhs->Elements(), lhs->Length())) { + return false; + } + } + return mCodec.Equals(aOther.mCodec) && + mCodedHeight == aOther.mCodedHeight && + mCodedWidth == aOther.mCodedWidth && + mColorSpace == aOther.mColorSpace && + mDisplayAspectHeight == aOther.mDisplayAspectWidth && + mHardwareAcceleration == aOther.mHardwareAcceleration && + mOptimizeForLatency == aOther.mOptimizeForLatency; + } + + nsString mCodec; + Maybe<uint32_t> mCodedHeight; + Maybe<uint32_t> mCodedWidth; + Maybe<VideoColorSpaceInternal> mColorSpace; + Maybe<RefPtr<MediaByteBuffer>> mDescription; + Maybe<uint32_t> mDisplayAspectHeight; + Maybe<uint32_t> mDisplayAspectWidth; + HardwareAcceleration mHardwareAcceleration; + Maybe<bool> mOptimizeForLatency; +}; + +class VideoDecoderTraits { + public: + static constexpr nsLiteralCString Name = "VideoDecoder"_ns; + using ConfigType = VideoDecoderConfig; + using ConfigTypeInternal = VideoDecoderConfigInternal; + using InputType = EncodedVideoChunk; + using InputTypeInternal = EncodedVideoChunkData; + using OutputType = VideoFrame; + using OutputCallbackType = VideoFrameOutputCallback; + + static bool IsSupported(const ConfigTypeInternal& aConfig); + static Result<UniquePtr<TrackInfo>, nsresult> CreateTrackInfo( + const ConfigTypeInternal& aConfig); + static bool Validate(const ConfigType& aConfig, nsCString& aErrorMessage); + static UniquePtr<ConfigTypeInternal> CreateConfigInternal( + const ConfigType& aConfig); + static bool IsKeyChunk(const InputType& aInput); + static UniquePtr<InputTypeInternal> CreateInputInternal( + const InputType& aInput); +}; + +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_DecoderTypes_h diff --git a/dom/media/webcodecs/EncodedVideoChunk.cpp b/dom/media/webcodecs/EncodedVideoChunk.cpp new file mode 100644 index 0000000000..c231740d05 --- /dev/null +++ b/dom/media/webcodecs/EncodedVideoChunk.cpp @@ -0,0 +1,261 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/EncodedVideoChunk.h" +#include <utility> +#include "mozilla/dom/EncodedVideoChunkBinding.h" + +#include "MediaData.h" +#include "TimeUnits.h" +#include "mozilla/CheckedInt.h" +#include "mozilla/Logging.h" +#include "mozilla/PodOperations.h" +#include "mozilla/dom/StructuredCloneHolder.h" +#include "mozilla/dom/StructuredCloneTags.h" +#include "mozilla/dom/WebCodecsUtils.h" + +extern mozilla::LazyLogModule gWebCodecsLog; +using mozilla::media::TimeUnit; + +namespace mozilla::dom { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOGW +# undef LOGW +#endif // LOGW +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +// Only needed for refcounted objects. +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(EncodedVideoChunk, mParent) +NS_IMPL_CYCLE_COLLECTING_ADDREF(EncodedVideoChunk) +NS_IMPL_CYCLE_COLLECTING_RELEASE(EncodedVideoChunk) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(EncodedVideoChunk) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +EncodedVideoChunkData::EncodedVideoChunkData( + already_AddRefed<MediaAlignedByteBuffer> aBuffer, + const EncodedVideoChunkType& aType, int64_t aTimestamp, + Maybe<uint64_t>&& aDuration) + : mBuffer(aBuffer), + mType(aType), + mTimestamp(aTimestamp), + mDuration(aDuration) { + MOZ_ASSERT(mBuffer); + MOZ_ASSERT(mBuffer->Length() == mBuffer->Size()); + MOZ_ASSERT(mBuffer->Length() <= + static_cast<size_t>(std::numeric_limits<uint32_t>::max())); +} + +EncodedVideoChunkData::~EncodedVideoChunkData() = default; + +UniquePtr<EncodedVideoChunkData> EncodedVideoChunkData::Clone() const { + if (!mBuffer) { + LOGE("No buffer in EncodedVideoChunkData %p to clone!", this); + return nullptr; + } + + // Since EncodedVideoChunkData can be zero-sized, cloning a zero-sized chunk + // is allowed. + if (mBuffer->Size() == 0) { + LOGW("Cloning an empty EncodedVideoChunkData %p", this); + } + + auto buffer = + MakeRefPtr<MediaAlignedByteBuffer>(mBuffer->Data(), mBuffer->Length()); + if (!buffer || buffer->Size() != mBuffer->Size()) { + LOGE("OOM to copy EncodedVideoChunkData %p", this); + return nullptr; + } + + return MakeUnique<EncodedVideoChunkData>(buffer.forget(), mType, mTimestamp, + Maybe<uint64_t>(mDuration)); +} + +already_AddRefed<MediaRawData> EncodedVideoChunkData::TakeData() { + if (!mBuffer || !(*mBuffer)) { + LOGE("EncodedVideoChunkData %p has no data!", this); + return nullptr; + } + + RefPtr<MediaRawData> sample(new MediaRawData(std::move(*mBuffer))); + sample->mKeyframe = mType == EncodedVideoChunkType::Key; + sample->mTime = TimeUnit::FromMicroseconds(mTimestamp); + sample->mTimecode = TimeUnit::FromMicroseconds(mTimestamp); + + if (mDuration) { + CheckedInt64 duration(*mDuration); + if (!duration.isValid()) { + LOGE("EncodedVideoChunkData %p 's duration exceeds TimeUnit's limit", + this); + return nullptr; + } + sample->mDuration = TimeUnit::FromMicroseconds(duration.value()); + } + + return sample.forget(); +} + +EncodedVideoChunk::EncodedVideoChunk( + nsIGlobalObject* aParent, already_AddRefed<MediaAlignedByteBuffer> aBuffer, + const EncodedVideoChunkType& aType, int64_t aTimestamp, + Maybe<uint64_t>&& aDuration) + : EncodedVideoChunkData(std::move(aBuffer), aType, aTimestamp, + std::move(aDuration)), + mParent(aParent) {} + +EncodedVideoChunk::EncodedVideoChunk(nsIGlobalObject* aParent, + const EncodedVideoChunkData& aData) + : EncodedVideoChunkData(aData), mParent(aParent) {} + +nsIGlobalObject* EncodedVideoChunk::GetParentObject() const { + AssertIsOnOwningThread(); + + return mParent.get(); +} + +JSObject* EncodedVideoChunk::WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) { + AssertIsOnOwningThread(); + + return EncodedVideoChunk_Binding::Wrap(aCx, this, aGivenProto); +} + +// https://w3c.github.io/webcodecs/#encodedvideochunk-constructors +/* static */ +already_AddRefed<EncodedVideoChunk> EncodedVideoChunk::Constructor( + const GlobalObject& aGlobal, const EncodedVideoChunkInit& aInit, + ErrorResult& aRv) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + auto buffer = ProcessTypedArrays( + aInit.mData, + [&](const Span<uint8_t>& aData, + JS::AutoCheckCannotGC&&) -> RefPtr<MediaAlignedByteBuffer> { + // Make sure it's in uint32_t's range. + CheckedUint32 byteLength(aData.Length()); + if (!byteLength.isValid()) { + aRv.Throw(NS_ERROR_INVALID_ARG); + return nullptr; + } + if (aData.Length() == 0) { + LOGW("Buffer for constructing EncodedVideoChunk is empty!"); + } + RefPtr<MediaAlignedByteBuffer> buf = MakeRefPtr<MediaAlignedByteBuffer>( + aData.Elements(), aData.Length()); + + // Instead of checking *buf, size comparision is used to allow + // constructing a zero-sized EncodedVideoChunk. + if (!buf || buf->Size() != aData.Length()) { + aRv.Throw(NS_ERROR_OUT_OF_MEMORY); + return nullptr; + } + return buf; + }); + + RefPtr<EncodedVideoChunk> chunk(new EncodedVideoChunk( + global, buffer.forget(), aInit.mType, aInit.mTimestamp, + OptionalToMaybe(aInit.mDuration))); + return aRv.Failed() ? nullptr : chunk.forget(); +} + +EncodedVideoChunkType EncodedVideoChunk::Type() const { + AssertIsOnOwningThread(); + + return mType; +} + +int64_t EncodedVideoChunk::Timestamp() const { + AssertIsOnOwningThread(); + + return mTimestamp; +} + +Nullable<uint64_t> EncodedVideoChunk::GetDuration() const { + AssertIsOnOwningThread(); + return MaybeToNullable(mDuration); +} + +uint32_t EncodedVideoChunk::ByteLength() const { + AssertIsOnOwningThread(); + MOZ_ASSERT(mBuffer); + + return static_cast<uint32_t>(mBuffer->Length()); +} + +// https://w3c.github.io/webcodecs/#dom-encodedvideochunk-copyto +void EncodedVideoChunk::CopyTo( + const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination, + ErrorResult& aRv) { + AssertIsOnOwningThread(); + + ProcessTypedArraysFixed(aDestination, [&](const Span<uint8_t>& aData) { + if (mBuffer->Size() > aData.size_bytes()) { + aRv.ThrowTypeError( + "Destination ArrayBuffer smaller than source EncodedVideoChunk"); + return; + } + + PodCopy(aData.data(), mBuffer->Data(), mBuffer->Size()); + }); +} + +// https://w3c.github.io/webcodecs/#ref-for-deserialization-steps%E2%91%A0 +/* static */ +JSObject* EncodedVideoChunk::ReadStructuredClone( + JSContext* aCx, nsIGlobalObject* aGlobal, JSStructuredCloneReader* aReader, + const EncodedVideoChunkData& aData) { + JS::Rooted<JS::Value> value(aCx, JS::NullValue()); + // To avoid a rooting hazard error from returning a raw JSObject* before + // running the RefPtr destructor, RefPtr needs to be destructed before + // returning the raw JSObject*, which is why the RefPtr<EncodedVideoChunk> is + // created in the scope below. Otherwise, the static analysis infers the + // RefPtr cannot be safely destructed while the unrooted return JSObject* is + // on the stack. + { + auto frame = MakeRefPtr<EncodedVideoChunk>(aGlobal, aData); + if (!GetOrCreateDOMReflector(aCx, frame, &value) || !value.isObject()) { + return nullptr; + } + } + return value.toObjectOrNull(); +} + +// https://w3c.github.io/webcodecs/#ref-for-serialization-steps%E2%91%A0 +bool EncodedVideoChunk::WriteStructuredClone( + JSStructuredCloneWriter* aWriter, StructuredCloneHolder* aHolder) const { + AssertIsOnOwningThread(); + + // Indexing the chunk and send the index to the receiver. + const uint32_t index = + static_cast<uint32_t>(aHolder->EncodedVideoChunks().Length()); + // The serialization is limited to the same process scope so it's ok to + // serialize a reference instead of a copy. + aHolder->EncodedVideoChunks().AppendElement(EncodedVideoChunkData(*this)); + return !NS_WARN_IF( + !JS_WriteUint32Pair(aWriter, SCTAG_DOM_ENCODEDVIDEOCHUNK, index)); +} + +#undef LOGW +#undef LOGE +#undef LOG_INTERNAL + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/EncodedVideoChunk.h b/dom/media/webcodecs/EncodedVideoChunk.h new file mode 100644 index 0000000000..3a9fb1e368 --- /dev/null +++ b/dom/media/webcodecs/EncodedVideoChunk.h @@ -0,0 +1,119 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_EncodedVideoChunk_h +#define mozilla_dom_EncodedVideoChunk_h + +#include "js/TypeDecls.h" +#include "mozilla/Attributes.h" +#include "mozilla/Buffer.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/Maybe.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; + +namespace mozilla { + +class MediaAlignedByteBuffer; +class MediaRawData; + +namespace dom { + +class MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer; +class OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer; +class StructuredCloneHolder; + +enum class EncodedVideoChunkType : uint8_t; +struct EncodedVideoChunkInit; + +} // namespace dom +} // namespace mozilla + +namespace mozilla::dom { + +class EncodedVideoChunkData { + public: + EncodedVideoChunkData(already_AddRefed<MediaAlignedByteBuffer> aBuffer, + const EncodedVideoChunkType& aType, int64_t aTimestamp, + Maybe<uint64_t>&& aDuration); + EncodedVideoChunkData(const EncodedVideoChunkData& aData) = default; + ~EncodedVideoChunkData(); + + UniquePtr<EncodedVideoChunkData> Clone() const; + already_AddRefed<MediaRawData> TakeData(); + + protected: + // mBuffer's byte length is guaranteed to be smaller than UINT32_MAX. + RefPtr<MediaAlignedByteBuffer> mBuffer; + EncodedVideoChunkType mType; + int64_t mTimestamp; + Maybe<uint64_t> mDuration; +}; + +class EncodedVideoChunk final : public EncodedVideoChunkData, + public nsISupports, + public nsWrapperCache { + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(EncodedVideoChunk) + + public: + EncodedVideoChunk(nsIGlobalObject* aParent, + already_AddRefed<MediaAlignedByteBuffer> aBuffer, + const EncodedVideoChunkType& aType, int64_t aTimestamp, + Maybe<uint64_t>&& aDuration); + + EncodedVideoChunk(nsIGlobalObject* aParent, + const EncodedVideoChunkData& aData); + + protected: + ~EncodedVideoChunk() = default; + + public: + nsIGlobalObject* GetParentObject() const; + + JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override; + + static already_AddRefed<EncodedVideoChunk> Constructor( + const GlobalObject& aGlobal, const EncodedVideoChunkInit& aInit, + ErrorResult& aRv); + + EncodedVideoChunkType Type() const; + + int64_t Timestamp() const; + + Nullable<uint64_t> GetDuration() const; + + uint32_t ByteLength() const; + + void CopyTo( + const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination, + ErrorResult& aRv); + + // [Serializable] implementations: {Read, Write}StructuredClone + static JSObject* ReadStructuredClone(JSContext* aCx, nsIGlobalObject* aGlobal, + JSStructuredCloneReader* aReader, + const EncodedVideoChunkData& aData); + + bool WriteStructuredClone(JSStructuredCloneWriter* aWriter, + StructuredCloneHolder* aHolder) const; + + private: + // EncodedVideoChunk can run on either main thread or worker thread. + void AssertIsOnOwningThread() const { + NS_ASSERT_OWNINGTHREAD(EncodedVideoChunk); + } + + nsCOMPtr<nsIGlobalObject> mParent; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_EncodedVideoChunk_h diff --git a/dom/media/webcodecs/EncoderAgent.cpp b/dom/media/webcodecs/EncoderAgent.cpp new file mode 100644 index 0000000000..e6af17a0be --- /dev/null +++ b/dom/media/webcodecs/EncoderAgent.cpp @@ -0,0 +1,441 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "EncoderAgent.h" + +#include "PDMFactory.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/Logging.h" +#include "nsThreadUtils.h" + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOG +# undef LOG +#endif // LOG +#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +#ifdef LOGW +# undef LOGW +#endif // LOGE +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +#ifdef LOGV +# undef LOGV +#endif // LOGV +#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__) + +EncoderAgent::EncoderAgent(WebCodecsId aId) + : mId(aId), + mOwnerThread(GetCurrentSerialEventTarget()), + mPEMFactory(MakeRefPtr<PEMFactory>()), + mEncoder(nullptr), + mState(State::Unconfigured) { + MOZ_ASSERT(mOwnerThread); + MOZ_ASSERT(mPEMFactory); + LOG("EncoderAgent #%zu (%p) ctor", mId, this); +} + +EncoderAgent::~EncoderAgent() { + LOG("EncoderAgent #%zu (%p) dtor", mId, this); + MOZ_ASSERT(mState == State::Unconfigured, "encoder released in wrong state"); + MOZ_ASSERT(!mEncoder, "encoder must be shutdown"); +} + +RefPtr<EncoderAgent::ConfigurePromise> EncoderAgent::Configure( + const EncoderConfig& aConfig) { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + MOZ_ASSERT(mState == State::Unconfigured || mState == State::Error); + MOZ_ASSERT(mConfigurePromise.IsEmpty()); + MOZ_ASSERT(!mCreateRequest.Exists()); + MOZ_ASSERT(!mInitRequest.Exists()); + + if (mState == State::Error) { + LOGE("EncoderAgent #%zu (%p) tried to configure in error state", mId, this); + return ConfigurePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "Cannot configure in error state"), + __func__); + } + + MOZ_ASSERT(mState == State::Unconfigured); + MOZ_ASSERT(!mEncoder); + SetState(State::Configuring); + + LOG("EncoderAgent #%zu (%p) is creating an encoder (%s)", mId, this, + GetCodecTypeString(aConfig.mCodec)); + + RefPtr<ConfigurePromise> p = mConfigurePromise.Ensure(__func__); + + mPEMFactory->CreateEncoderAsync(aConfig, dom::GetWebCodecsEncoderTaskQueue()) + ->Then( + mOwnerThread, __func__, + [self = RefPtr{this}](RefPtr<MediaDataEncoder>&& aEncoder) { + self->mCreateRequest.Complete(); + + // If EncoderAgent has been shut down, shut the created encoder down + // and return. + if (!self->mShutdownWhileCreationPromise.IsEmpty()) { + MOZ_ASSERT(self->mState == State::ShuttingDown); + MOZ_ASSERT(self->mConfigurePromise.IsEmpty(), + "configuration should have been rejected"); + + LOGW( + "EncoderAgent #%zu (%p) has been shut down. We need to shut " + "the newly created encoder down", + self->mId, self.get()); + aEncoder->Shutdown()->Then( + self->mOwnerThread, __func__, + [self](const ShutdownPromise::ResolveOrRejectValue& aValue) { + MOZ_ASSERT(self->mState == State::ShuttingDown); + + LOGW( + "EncoderAgent #%zu (%p), newly created encoder " + "shutdown " + "has been %s", + self->mId, self.get(), + aValue.IsResolve() ? "resolved" : "rejected"); + + self->SetState(State::Unconfigured); + + self->mShutdownWhileCreationPromise.ResolveOrReject( + aValue, __func__); + }); + return; + } + + self->mEncoder = aEncoder.forget(); + LOG("EncoderAgent #%zu (%p) has created a encoder, now initialize " + "it", + self->mId, self.get()); + self->mEncoder->Init() + ->Then( + self->mOwnerThread, __func__, + [self]() { + self->mInitRequest.Complete(); + LOG("EncoderAgent #%zu (%p) has initialized the encoder", + self->mId, self.get()); + self->SetState(State::Configured); + self->mConfigurePromise.Resolve(true, __func__); + }, + [self](const MediaResult& aError) { + self->mInitRequest.Complete(); + LOGE( + "EncoderAgent #%zu (%p) failed to initialize the " + "encoder", + self->mId, self.get()); + self->SetState(State::Error); + self->mConfigurePromise.Reject(aError, __func__); + }) + ->Track(self->mInitRequest); + }, + [self = RefPtr{this}](const MediaResult& aError) { + self->mCreateRequest.Complete(); + LOGE("EncoderAgent #%zu (%p) failed to create a encoder", self->mId, + self.get()); + + // If EncoderAgent has been shut down, we need to resolve the + // shutdown promise. + if (!self->mShutdownWhileCreationPromise.IsEmpty()) { + MOZ_ASSERT(self->mState == State::ShuttingDown); + MOZ_ASSERT(self->mConfigurePromise.IsEmpty(), + "configuration should have been rejected"); + + LOGW( + "EncoderAgent #%zu (%p) has been shut down. Resolve the " + "shutdown promise right away since encoder creation failed", + self->mId, self.get()); + + self->SetState(State::Unconfigured); + self->mShutdownWhileCreationPromise.Resolve(true, __func__); + return; + } + + self->SetState(State::Error); + self->mConfigurePromise.Reject(aError, __func__); + }) + ->Track(mCreateRequest); + + return p; +} + +RefPtr<EncoderAgent::ReconfigurationPromise> EncoderAgent::Reconfigure( + const RefPtr<const EncoderConfigurationChangeList>& aConfigChanges) { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + MOZ_ASSERT(mState == State::Configured || mState == State::Error); + MOZ_ASSERT(mReconfigurationPromise.IsEmpty()); + + if (mState == State::Error) { + LOGE("EncoderAgent #%zu (%p) tried to reconfigure in error state", mId, + this); + return ReconfigurationPromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "Cannot reconfigure in error state"), + __func__); + } + + MOZ_ASSERT(mEncoder); + SetState(State::Configuring); + + LOG("EncoderAgent #%zu (%p) is reconfiguring its encoder (%s)", mId, this, + NS_ConvertUTF16toUTF8(aConfigChanges->ToString().get()).get()); + + RefPtr<ReconfigurationPromise> p = mReconfigurationPromise.Ensure(__func__); + + mEncoder->Reconfigure(aConfigChanges) + ->Then( + mOwnerThread, __func__, + [self = RefPtr{this}](bool) { + self->mReconfigurationRequest.Complete(); + LOGE("EncoderAgent #%zu (%p) reconfigure success", self->mId, + self.get()); + self->mReconfigurationPromise.Resolve(true, __func__); + }, + [self = RefPtr{this}](const MediaResult& aError) { + self->mReconfigurationRequest.Complete(); + LOGE("EncoderAgent #%zu (%p) reconfigure failure", self->mId, + self.get()); + // Not a a fatal error per se, the owner will deal with it. + self->mReconfigurationPromise.Reject(aError, __func__); + }) + ->Track(mReconfigurationRequest); + + return p; +} + +RefPtr<ShutdownPromise> EncoderAgent::Shutdown() { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + + auto r = + MediaResult(NS_ERROR_DOM_MEDIA_CANCELED, "Canceled by encoder shutdown"); + + // If the encoder creation has not been completed yet, wait until the encoder + // being created has been shut down. + if (mCreateRequest.Exists()) { + MOZ_ASSERT(!mInitRequest.Exists()); + MOZ_ASSERT(!mConfigurePromise.IsEmpty()); + MOZ_ASSERT(!mEncoder); + MOZ_ASSERT(mState == State::Configuring); + MOZ_ASSERT(mShutdownWhileCreationPromise.IsEmpty()); + + LOGW( + "EncoderAgent #%zu (%p) shutdown while the encoder creation for " + "configuration is in flight. Reject the configuration now and defer " + "the shutdown until the created encoder has been shut down", + mId, this); + + // Reject the configuration in flight. + mConfigurePromise.Reject(r, __func__); + + // Get the promise that will be resolved when the encoder being created has + // been destroyed. + SetState(State::ShuttingDown); + return mShutdownWhileCreationPromise.Ensure(__func__); + } + + // If encoder creation has been completed, we must have the encoder now. + MOZ_ASSERT(mEncoder); + + // Cancel pending initialization for configuration in flight if any. + mInitRequest.DisconnectIfExists(); + mConfigurePromise.RejectIfExists(r, __func__); + + mReconfigurationRequest.DisconnectIfExists(); + mReconfigurationPromise.RejectIfExists(r, __func__); + + // Cancel encoder in flight if any. + mEncodeRequest.DisconnectIfExists(); + mEncodePromise.RejectIfExists(r, __func__); + + // Cancel flush-out in flight if any. + mDrainRequest.DisconnectIfExists(); + mEncodeRequest.DisconnectIfExists(); + + mDrainRequest.DisconnectIfExists(); + mDrainPromise.RejectIfExists(r, __func__); + + SetState(State::Unconfigured); + + RefPtr<MediaDataEncoder> encoder = std::move(mEncoder); + return encoder->Shutdown(); +} + +RefPtr<EncoderAgent::EncodePromise> EncoderAgent::Encode(MediaData* aInput) { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + MOZ_ASSERT(aInput); + MOZ_ASSERT(mState == State::Configured || mState == State::Error); + MOZ_ASSERT(mEncodePromise.IsEmpty()); + MOZ_ASSERT(!mEncodeRequest.Exists()); + + if (mState == State::Error) { + LOGE("EncoderAgent #%zu (%p) tried to encoder in error state", mId, this); + return EncodePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "Cannot encoder in error state"), + __func__); + } + + MOZ_ASSERT(mState == State::Configured); + MOZ_ASSERT(mEncoder); + SetState(State::Encoding); + + RefPtr<EncodePromise> p = mEncodePromise.Ensure(__func__); + + mEncoder->Encode(aInput) + ->Then( + mOwnerThread, __func__, + [self = RefPtr{this}](MediaDataEncoder::EncodedData&& aData) { + self->mEncodeRequest.Complete(); + LOGV("EncoderAgent #%zu (%p) encode successful", self->mId, + self.get()); + self->SetState(State::Configured); + self->mEncodePromise.Resolve(std::move(aData), __func__); + }, + [self = RefPtr{this}](const MediaResult& aError) { + self->mEncodeRequest.Complete(); + LOGV("EncoderAgent #%zu (%p) failed to encode", self->mId, + self.get()); + self->SetState(State::Error); + self->mEncodePromise.Reject(aError, __func__); + }) + ->Track(mEncodeRequest); + + return p; +} + +RefPtr<EncoderAgent::EncodePromise> EncoderAgent::Drain() { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + // This can be called when reconfiguring the encoder. + MOZ_ASSERT(mState == State::Configured || mState == State::Configuring); + MOZ_ASSERT(mDrainPromise.IsEmpty()); + MOZ_ASSERT(mEncoder); + + SetState(State::Flushing); + + RefPtr<EncodePromise> p = mDrainPromise.Ensure(__func__); + DryUntilDrain(); + return p; +} + +void EncoderAgent::DryUntilDrain() { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + MOZ_ASSERT(mState == State::Flushing); + MOZ_ASSERT(!mDrainRequest.Exists()); + MOZ_ASSERT(mEncoder); + + LOG("EncoderAgent #%zu (%p) is draining the encoder", mId, this); + mEncoder->Drain() + ->Then( + mOwnerThread, __func__, + [self = RefPtr{this}](MediaDataEncoder::EncodedData&& aData) { + self->mDrainRequest.Complete(); + + if (aData.IsEmpty()) { + LOG("EncoderAgent #%zu (%p) is dry now", self->mId, self.get()); + self->SetState(State::Configured); + self->mDrainPromise.Resolve(std::move(self->mDrainData), + __func__); + return; + } + + LOG("EncoderAgent #%zu (%p) drained %zu encoder data. Keep " + "draining " + "until dry", + self->mId, self.get(), aData.Length()); + self->mDrainData.AppendElements(std::move(aData)); + self->DryUntilDrain(); + }, + [self = RefPtr{this}](const MediaResult& aError) { + self->mDrainRequest.Complete(); + + LOGE("EncoderAgent %p failed to drain encoder", self.get()); + self->mDrainData.Clear(); + self->mDrainPromise.Reject(aError, __func__); + }) + ->Track(mDrainRequest); +} + +void EncoderAgent::SetState(State aState) { + MOZ_ASSERT(mOwnerThread->IsOnCurrentThread()); + + auto validateStateTransition = [](State aOldState, State aNewState) { + switch (aOldState) { + case State::Unconfigured: + return aNewState == State::Configuring; + case State::Configuring: + return aNewState == State::Configured || aNewState == State::Error || + aNewState == State::Flushing || + aNewState == State::Unconfigured || + aNewState == State::ShuttingDown; + case State::Configured: + return aNewState == State::Unconfigured || + aNewState == State::Configuring || + aNewState == State::Encoding || aNewState == State::Flushing; + case State::Encoding: + case State::Flushing: + return aNewState == State::Configured || aNewState == State::Error || + aNewState == State::Unconfigured; + case State::ShuttingDown: + return aNewState == State::Unconfigured; + case State::Error: + return aNewState == State::Unconfigured; + default: + break; + } + MOZ_ASSERT_UNREACHABLE("Unhandled state transition"); + return false; + }; + + auto stateToString = [](State aState) -> const char* { + switch (aState) { + case State::Unconfigured: + return "Unconfigured"; + case State::Configuring: + return "Configuring"; + case State::Configured: + return "Configured"; + case State::Encoding: + return "Encoding"; + case State::Flushing: + return "Flushing"; + case State::ShuttingDown: + return "ShuttingDown"; + case State::Error: + return "Error"; + default: + break; + } + MOZ_ASSERT_UNREACHABLE("Unhandled state type"); + return "Unknown"; + }; + + DebugOnly<bool> isValid = validateStateTransition(mState, aState); + LOGV("EncoderAgent #%zu (%p) state change: %s -> %s", mId, this, + stateToString(mState), stateToString(aState)); + MOZ_ASSERT(isValid); + mState = aState; +} + +#undef LOG +#undef LOGW +#undef LOGE +#undef LOGV +#undef LOG_INTERNAL + +} // namespace mozilla diff --git a/dom/media/webcodecs/EncoderAgent.h b/dom/media/webcodecs/EncoderAgent.h new file mode 100644 index 0000000000..387f218e29 --- /dev/null +++ b/dom/media/webcodecs/EncoderAgent.h @@ -0,0 +1,116 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_WEBCODECS_EncoderAgent_H +#define DOM_MEDIA_WEBCODECS_EncoderAgent_H + +#include "MediaResult.h" +#include "PlatformEncoderModule.h" +#include "PEMFactory.h" +#include "mozilla/RefPtr.h" +#include "mozilla/TaskQueue.h" +#include "WebCodecsUtils.h" + +class nsISerialEventTarget; + +namespace mozilla { + +class PDMFactory; +class TrackInfo; + +namespace layers { +class ImageContainer; +} // namespace layers + +// EncoderAgent is a wrapper that contains a MediaDataEncoder. It adapts the +// MediaDataEncoder APIs for use in WebCodecs. +// +// If Configure() is called, Shutdown() must be called to release the resources +// gracefully. Except Shutdown(), all the methods can't be called concurrently, +// meaning a method can only be called when the previous API call has completed. +// The responsability of arranging the method calls is on the caller. +// +// When Shutdown() is called, all the operations in flight are canceled and the +// MediaDataEncoder is shut down. On the other hand, errors are final. A new +// EncoderAgent must be created when an error is encountered. +// +// All the methods need to be called on the EncoderAgent's owner thread. In +// WebCodecs, it's either on the main thread or worker thread. +class EncoderAgent final { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(EncoderAgent); + + explicit EncoderAgent(WebCodecsId aId); + + // The following APIs are owner thread only. + + using ConfigurePromise = MozPromise<bool, MediaResult, true /* exclusive */>; + using ReconfigurationPromise = MediaDataEncoder::ReconfigurationPromise; + RefPtr<ConfigurePromise> Configure(const EncoderConfig& aConfig); + RefPtr<ReconfigurationPromise> Reconfigure( + const RefPtr<const EncoderConfigurationChangeList>& aConfigChange); + RefPtr<ShutdownPromise> Shutdown(); + using EncodePromise = MediaDataEncoder::EncodePromise; + RefPtr<EncodePromise> Encode(MediaData* aInput); + // WebCodecs's flush() flushes out all the pending encoded data in the + // encoder. It's called Drain internally. + RefPtr<EncodePromise> Drain(); + + const WebCodecsId mId; + + private: + ~EncoderAgent(); + + // Push out all the data in the MediaDataEncoder's pipeline. + // TODO: MediaDataEncoder should implement this, instead of asking call site + // to run `Drain` multiple times. + RefPtr<EncodePromise> Dry(); + void DryUntilDrain(); + + enum class State { + Unconfigured, + Configuring, + Configured, + Encoding, + Flushing, + ShuttingDown, + Error, + }; + void SetState(State aState); + + const RefPtr<nsISerialEventTarget> mOwnerThread; + const RefPtr<PEMFactory> mPEMFactory; + RefPtr<MediaDataEncoder> mEncoder; + State mState; + + // Configure + MozPromiseHolder<ConfigurePromise> mConfigurePromise; + using CreateEncoderPromise = PlatformEncoderModule::CreateEncoderPromise; + MozPromiseRequestHolder<CreateEncoderPromise> mCreateRequest; + using InitPromise = MediaDataEncoder::InitPromise; + MozPromiseRequestHolder<InitPromise> mInitRequest; + + // Reconfigure + MozPromiseHolder<ReconfigurationPromise> mReconfigurationPromise; + using ReconfigureEncoderRequest = ReconfigurationPromise; + MozPromiseRequestHolder<ReconfigureEncoderRequest> mReconfigurationRequest; + + // Shutdown + MozPromiseHolder<ShutdownPromise> mShutdownWhileCreationPromise; + + // Encoding + MozPromiseHolder<EncodePromise> mEncodePromise; + MozPromiseRequestHolder<EncodePromise> mEncodeRequest; + + // Drain + MozPromiseRequestHolder<EncodePromise> mDrainRequest; + MozPromiseHolder<EncodePromise> mDrainPromise; + MediaDataEncoder::EncodedData mDrainData; +}; + +} // namespace mozilla + +#endif // DOM_MEDIA_WEBCODECS_EncoderAgent_H diff --git a/dom/media/webcodecs/EncoderTemplate.cpp b/dom/media/webcodecs/EncoderTemplate.cpp new file mode 100644 index 0000000000..35c8feb3f8 --- /dev/null +++ b/dom/media/webcodecs/EncoderTemplate.cpp @@ -0,0 +1,1228 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "EncoderTemplate.h" + +#include "EncoderTypes.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/Try.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/DOMException.h" +#include "mozilla/dom/Event.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/VideoFrame.h" +#include "mozilla/dom/WorkerCommon.h" +#include "nsGkAtoms.h" +#include "nsString.h" +#include "nsThreadUtils.h" + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla::dom { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOG +# undef LOG +#endif // LOG +#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +#ifdef LOGW +# undef LOGW +#endif // LOGW +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +#ifdef LOGV +# undef LOGV +#endif // LOGV +#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__) + +/* + * Below are ControlMessage classes implementations + */ + +template <typename EncoderType> +EncoderTemplate<EncoderType>::ControlMessage::ControlMessage( + WebCodecsId aConfigureId) + : mConfigureId(aConfigureId), mMessageId(sNextId++) {} + +template <typename EncoderType> +EncoderTemplate<EncoderType>::ConfigureMessage::ConfigureMessage( + WebCodecsId aConfigureId, const RefPtr<ConfigTypeInternal>& aConfig) + : ControlMessage(aConfigureId), mConfig(aConfig) {} + +template <typename EncoderType> +EncoderTemplate<EncoderType>::EncodeMessage::EncodeMessage( + WebCodecsId aConfigureId, RefPtr<InputTypeInternal>&& aData, + Maybe<VideoEncoderEncodeOptions>&& aOptions) + : ControlMessage(aConfigureId), mData(aData) {} + +template <typename EncoderType> +EncoderTemplate<EncoderType>::FlushMessage::FlushMessage( + WebCodecsId aConfigureId, Promise* aPromise) + : ControlMessage(aConfigureId), mPromise(aPromise) {} + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::FlushMessage::RejectPromiseIfAny( + const nsresult& aReason) { + if (mPromise) { + mPromise->MaybeReject(aReason); + } +} + +/* + * Below are EncoderTemplate implementation + */ + +template <typename EncoderType> +EncoderTemplate<EncoderType>::EncoderTemplate( + nsIGlobalObject* aGlobalObject, + RefPtr<WebCodecsErrorCallback>&& aErrorCallback, + RefPtr<OutputCallbackType>&& aOutputCallback) + : DOMEventTargetHelper(aGlobalObject), + mErrorCallback(std::move(aErrorCallback)), + mOutputCallback(std::move(aOutputCallback)), + mState(CodecState::Unconfigured), + mMessageQueueBlocked(false), + mEncodeQueueSize(0), + mDequeueEventScheduled(false), + mLatestConfigureId(0), + mEncodeCounter(0), + mFlushCounter(0) {} + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::Configure(const ConfigType& aConfig, + ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s::Configure %p codec %s", EncoderType::Name.get(), this, + NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); + + nsCString errorMessage; + if (!EncoderType::Validate(aConfig, errorMessage)) { + LOG("Configure: Validate error: %s", errorMessage.get()); + aRv.ThrowTypeError(errorMessage); + return; + } + + if (mState == CodecState::Closed) { + LOG("Configure: CodecState::Closed, rejecting with InvalidState"); + aRv.ThrowInvalidStateError("The codec is no longer usable"); + return; + } + + // Clone a ConfigType as the active Encode config. + RefPtr<ConfigTypeInternal> config = + EncoderType::CreateConfigInternal(aConfig); + if (!config) { + aRv.Throw(NS_ERROR_UNEXPECTED); // Invalid description data. + return; + } + + mState = CodecState::Configured; + mEncodeCounter = 0; + mFlushCounter = 0; + + mControlMessageQueue.push(MakeRefPtr<ConfigureMessage>(sNextId++, config)); + mLatestConfigureId = mControlMessageQueue.back()->mMessageId; + LOG("%s %p enqueues %s", EncoderType::Name.get(), this, + mControlMessageQueue.back()->ToString().get()); + ProcessControlMessageQueue(); +} + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::EncodeAudioData(InputType& aInput, + ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s %p, EncodeAudioData", EncoderType::Name.get(), this); + + if (mState != CodecState::Configured) { + aRv.ThrowInvalidStateError("Encoder must be configured first"); + return; + } + + if (aInput.IsClosed()) { + aRv.ThrowTypeError("input AudioData has been closed"); + return; + } + + mEncodeQueueSize += 1; + // Dummy options here as a shortcut + mControlMessageQueue.push(MakeRefPtr<EncodeMessage>( + mLatestConfigureId, + EncoderType::CreateInputInternal(aInput, VideoEncoderEncodeOptions()))); + LOGV("%s %p enqueues %s", EncoderType::Name.get(), this, + mControlMessageQueue.back()->ToString().get()); + ProcessControlMessageQueue(); +} + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::EncodeVideoFrame( + InputType& aInput, const VideoEncoderEncodeOptions& aOptions, + ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s::Encode %p %s", EncoderType::Name.get(), this, + aInput.ToString().get()); + + if (mState != CodecState::Configured) { + aRv.ThrowInvalidStateError("Encoder must be configured first"); + return; + } + + if (aInput.IsClosed()) { + aRv.ThrowTypeError("input VideoFrame has been closed"); + return; + } + + mEncodeQueueSize += 1; + mControlMessageQueue.push(MakeRefPtr<EncodeMessage>( + mLatestConfigureId, EncoderType::CreateInputInternal(aInput, aOptions), + Some(aOptions))); + LOGV("%s %p enqueues %s", EncoderType::Name.get(), this, + mControlMessageQueue.back()->ToString().get()); + ProcessControlMessageQueue(); +} + +template <typename EncoderType> +already_AddRefed<Promise> EncoderTemplate<EncoderType>::Flush( + ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s::Flush %p", EncoderType::Name.get(), this); + + if (mState != CodecState::Configured) { + LOG("%s %p, wrong state!", EncoderType::Name.get(), this); + aRv.ThrowInvalidStateError("Encoder must be configured first"); + return nullptr; + } + + RefPtr<Promise> p = Promise::Create(GetParentObject(), aRv); + if (NS_WARN_IF(aRv.Failed())) { + return p.forget(); + } + + mControlMessageQueue.push(MakeRefPtr<FlushMessage>(mLatestConfigureId, p)); + LOG("%s %p enqueues %s", EncoderType::Name.get(), this, + mControlMessageQueue.back()->ToString().get()); + ProcessControlMessageQueue(); + return p.forget(); +} + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::Reset(ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s %p, Reset", EncoderType::Name.get(), this); + + if (auto r = ResetInternal(NS_ERROR_DOM_ABORT_ERR); r.isErr()) { + aRv.Throw(r.unwrapErr()); + } +} + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::Close(ErrorResult& aRv) { + AssertIsOnOwningThread(); + + LOG("%s::Close %p", EncoderType::Name.get(), this); + + if (auto r = CloseInternal(NS_ERROR_DOM_ABORT_ERR); r.isErr()) { + aRv.Throw(r.unwrapErr()); + } +} + +template <typename EncoderType> +Result<Ok, nsresult> EncoderTemplate<EncoderType>::ResetInternal( + const nsresult& aResult) { + AssertIsOnOwningThread(); + + LOG("%s::Reset %p", EncoderType::Name.get(), this); + + if (mState == CodecState::Closed) { + return Err(NS_ERROR_DOM_INVALID_STATE_ERR); + } + + mState = CodecState::Unconfigured; + mEncodeCounter = 0; + mFlushCounter = 0; + + CancelPendingControlMessages(aResult); + DestroyEncoderAgentIfAny(); + + if (mEncodeQueueSize > 0) { + mEncodeQueueSize = 0; + ScheduleDequeueEvent(); + } + + StopBlockingMessageQueue(); + + return Ok(); +} + +template <typename EncoderType> +Result<Ok, nsresult> EncoderTemplate<EncoderType>::CloseInternal( + const nsresult& aResult) { + AssertIsOnOwningThread(); + + MOZ_TRY(ResetInternal(aResult)); + mState = CodecState::Closed; + if (aResult != NS_ERROR_DOM_ABORT_ERR) { + nsCString error; + GetErrorName(aResult, error); + LOGE("%s %p Close on error: %s", EncoderType::Name.get(), this, + error.get()); + ReportError(aResult); + } + return Ok(); +} + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::ReportError(const nsresult& aResult) { + AssertIsOnOwningThread(); + + RefPtr<DOMException> e = DOMException::Create(aResult); + RefPtr<WebCodecsErrorCallback> cb(mErrorCallback); + cb->Call(*e); +} + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::OutputEncodedData( + nsTArray<RefPtr<MediaRawData>>&& aData) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(mActiveConfig); + + // Get JSContext for RootedDictionary. + // The EncoderType::MetadataType, VideoDecoderConfig, and VideoColorSpaceInit + // below are rooted to work around the JS hazard issues. + AutoJSAPI jsapi; + DebugOnly<bool> ok = + jsapi.Init(GetParentObject()); // TODO: check returned value? + JSContext* cx = jsapi.cx(); + + RefPtr<typename EncoderType::OutputCallbackType> cb(mOutputCallback); + for (auto& data : aData) { + // It's possible to have reset() called in between this task having been + // dispatched, and running -- no output callback should happen when that's + // the case. + // This is imprecise in the spec, but discussed in + // https://github.com/w3c/webcodecs/issues/755 and agreed upon. + if (!mActiveConfig) { + return; + } + RefPtr<typename EncoderType::OutputType> encodedData = + EncodedDataToOutputType(GetParentObject(), data); + + RootedDictionary<typename EncoderType::MetadataType> metadata(cx); + if (mOutputNewDecoderConfig) { + VideoDecoderConfigInternal decoderConfigInternal = + EncoderConfigToDecoderConfig(GetParentObject(), data, *mActiveConfig); + + // Convert VideoDecoderConfigInternal to VideoDecoderConfig + RootedDictionary<VideoDecoderConfig> decoderConfig(cx); + decoderConfig.mCodec = decoderConfigInternal.mCodec; + if (decoderConfigInternal.mCodedHeight) { + decoderConfig.mCodedHeight.Construct( + decoderConfigInternal.mCodedHeight.value()); + } + if (decoderConfigInternal.mCodedWidth) { + decoderConfig.mCodedWidth.Construct( + decoderConfigInternal.mCodedWidth.value()); + } + if (decoderConfigInternal.mColorSpace) { + RootedDictionary<VideoColorSpaceInit> colorSpace(cx); + colorSpace.mFullRange = + MaybeToNullable(decoderConfigInternal.mColorSpace->mFullRange); + colorSpace.mMatrix = + MaybeToNullable(decoderConfigInternal.mColorSpace->mMatrix); + colorSpace.mPrimaries = + MaybeToNullable(decoderConfigInternal.mColorSpace->mPrimaries); + colorSpace.mTransfer = + MaybeToNullable(decoderConfigInternal.mColorSpace->mTransfer); + decoderConfig.mColorSpace.Construct(std::move(colorSpace)); + } + if (decoderConfigInternal.mDescription && + !decoderConfigInternal.mDescription.value()->IsEmpty()) { + auto& abov = decoderConfig.mDescription.Construct(); + AutoEntryScript aes(GetParentObject(), "EncoderConfigToDecoderConfig"); + size_t lengthBytes = + decoderConfigInternal.mDescription.value()->Length(); + UniquePtr<uint8_t[], JS::FreePolicy> extradata( + new uint8_t[lengthBytes]); + PodCopy(extradata.get(), + decoderConfigInternal.mDescription.value()->Elements(), + lengthBytes); + JS::Rooted<JSObject*> description( + aes.cx(), JS::NewArrayBufferWithContents(aes.cx(), lengthBytes, + std::move(extradata))); + JS::Rooted<JS::Value> value(aes.cx(), JS::ObjectValue(*description)); + DebugOnly<bool> rv = abov.Init(aes.cx(), value); + } + if (decoderConfigInternal.mDisplayAspectHeight) { + decoderConfig.mDisplayAspectHeight.Construct( + decoderConfigInternal.mDisplayAspectHeight.value()); + } + if (decoderConfigInternal.mDisplayAspectWidth) { + decoderConfig.mDisplayAspectWidth.Construct( + decoderConfigInternal.mDisplayAspectWidth.value()); + } + if (decoderConfigInternal.mOptimizeForLatency) { + decoderConfig.mOptimizeForLatency.Construct( + decoderConfigInternal.mOptimizeForLatency.value()); + } + + metadata.mDecoderConfig.Construct(std::move(decoderConfig)); + mOutputNewDecoderConfig = false; + LOGE("New config passed to output callback: %s", + NS_ConvertUTF16toUTF8(decoderConfigInternal.ToString()).get()); + } + + nsAutoCString metadataInfo; + + if (data->mTemporalLayerId) { + RootedDictionary<SvcOutputMetadata> svc(cx); + svc.mTemporalLayerId.Construct(data->mTemporalLayerId.value()); + metadata.mSvc.Construct(std::move(svc)); + metadataInfo.Append( + nsPrintfCString(", temporal layer id %d", + metadata.mSvc.Value().mTemporalLayerId.Value())); + } + + if (metadata.mDecoderConfig.WasPassed()) { + metadataInfo.Append(", new decoder config"); + } + + LOG("EncoderTemplate:: output callback (ts: % " PRId64 ")%s", + encodedData->Timestamp(), metadataInfo.get()); + cb->Call((typename EncoderType::OutputType&)(*encodedData), metadata); + } +} + +template <typename EncoderType> +class EncoderTemplate<EncoderType>::ErrorRunnable final + : public DiscardableRunnable { + public: + ErrorRunnable(Self* aEncoder, const nsresult& aError) + : DiscardableRunnable("Decoder ErrorRunnable"), + mEncoder(aEncoder), + mError(aError) { + MOZ_ASSERT(mEncoder); + } + ~ErrorRunnable() = default; + + // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT. + // See bug 1535398. + MOZ_CAN_RUN_SCRIPT_BOUNDARY NS_IMETHOD Run() override { + nsCString error; + GetErrorName(mError, error); + LOGE("%s %p report error: %s", EncoderType::Name.get(), mEncoder.get(), + error.get()); + RefPtr<Self> d = std::move(mEncoder); + d->ReportError(mError); + return NS_OK; + } + + private: + RefPtr<Self> mEncoder; + const nsresult mError; +}; + +template <typename EncoderType> +class EncoderTemplate<EncoderType>::OutputRunnable final + : public DiscardableRunnable { + public: + OutputRunnable(Self* aEncoder, WebCodecsId aConfigureId, + const nsACString& aLabel, + nsTArray<RefPtr<MediaRawData>>&& aData) + : DiscardableRunnable("Decoder OutputRunnable"), + mEncoder(aEncoder), + mConfigureId(aConfigureId), + mLabel(aLabel), + mData(std::move(aData)) { + MOZ_ASSERT(mEncoder); + } + ~OutputRunnable() = default; + + // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT. + // See bug 1535398. + MOZ_CAN_RUN_SCRIPT_BOUNDARY NS_IMETHOD Run() override { + if (mEncoder->mState != CodecState::Configured) { + LOGV("%s %p has been %s. Discard %s-result for EncoderAgent #%zu", + EncoderType::Name.get(), mEncoder.get(), + mEncoder->mState == CodecState::Closed ? "closed" : "reset", + mLabel.get(), mConfigureId); + return NS_OK; + } + + MOZ_ASSERT(mEncoder->mAgent); + if (mConfigureId != mEncoder->mAgent->mId) { + LOGW( + "%s %p has been re-configured. Still yield %s-result for " + "EncoderAgent #%zu", + EncoderType::Name.get(), mEncoder.get(), mLabel.get(), mConfigureId); + } + + LOGV("%s %p, yields %s-result for EncoderAgent #%zu", + EncoderType::Name.get(), mEncoder.get(), mLabel.get(), mConfigureId); + RefPtr<Self> d = std::move(mEncoder); + d->OutputEncodedData(std::move(mData)); + + return NS_OK; + } + + private: + RefPtr<Self> mEncoder; + const WebCodecsId mConfigureId; + const nsCString mLabel; + nsTArray<RefPtr<MediaRawData>> mData; +}; + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::ScheduleOutputEncodedData( + nsTArray<RefPtr<MediaRawData>>&& aData, const nsACString& aLabel) { + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(mAgent); + + MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(MakeAndAddRef<OutputRunnable>( + this, mAgent->mId, aLabel, std::move(aData)))); +} + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::ScheduleClose(const nsresult& aResult) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + + auto task = [self = RefPtr{this}, result = aResult] { + if (self->mState == CodecState::Closed) { + nsCString error; + GetErrorName(result, error); + LOGW("%s %p has been closed. Ignore close with %s", + EncoderType::Name.get(), self.get(), error.get()); + return; + } + DebugOnly<Result<Ok, nsresult>> r = self->CloseInternal(result); + MOZ_ASSERT(r.value.isOk()); + }; + nsISerialEventTarget* target = GetCurrentSerialEventTarget(); + + if (NS_IsMainThread()) { + MOZ_ALWAYS_SUCCEEDS(target->Dispatch( + NS_NewRunnableFunction("ScheduleClose Runnable (main)", task))); + return; + } + + MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewCancelableRunnableFunction( + "ScheduleClose Runnable (worker)", task))); +} + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::ScheduleDequeueEvent() { + AssertIsOnOwningThread(); + + if (mDequeueEventScheduled) { + return; + } + mDequeueEventScheduled = true; + + auto dispatcher = [self = RefPtr{this}] { + self->FireEvent(nsGkAtoms::ondequeue, u"dequeue"_ns); + self->mDequeueEventScheduled = false; + }; + nsISerialEventTarget* target = GetCurrentSerialEventTarget(); + + if (NS_IsMainThread()) { + MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewRunnableFunction( + "ScheduleDequeueEvent Runnable (main)", dispatcher))); + return; + } + + MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewCancelableRunnableFunction( + "ScheduleDequeueEvent Runnable (worker)", dispatcher))); +} + +template <typename EncoderType> +nsresult EncoderTemplate<EncoderType>::FireEvent(nsAtom* aTypeWithOn, + const nsAString& aEventType) { + if (aTypeWithOn && !HasListenersFor(aTypeWithOn)) { + return NS_ERROR_ABORT; + } + + LOGV("Dispatching %s event to %s %p", NS_ConvertUTF16toUTF8(aEventType).get(), + EncoderType::Name.get(), this); + RefPtr<Event> event = new Event(this, nullptr, nullptr); + event->InitEvent(aEventType, true, true); + event->SetTrusted(true); + this->DispatchEvent(*event); + return NS_OK; +} + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::SchedulePromiseResolveOrReject( + already_AddRefed<Promise> aPromise, const nsresult& aResult) { + AssertIsOnOwningThread(); + + RefPtr<Promise> p = aPromise; + auto resolver = [p, result = aResult] { + if (NS_FAILED(result)) { + p->MaybeReject(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + return; + } + p->MaybeResolveWithUndefined(); + }; + nsISerialEventTarget* target = GetCurrentSerialEventTarget(); + + if (NS_IsMainThread()) { + MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewRunnableFunction( + "SchedulePromiseResolveOrReject Runnable (main)", resolver))); + return; + } + + MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewCancelableRunnableFunction( + "SchedulePromiseResolveOrReject Runnable (worker)", resolver))); +} + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::ProcessControlMessageQueue() { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + + while (!mMessageQueueBlocked && !mControlMessageQueue.empty()) { + RefPtr<ControlMessage>& msg = mControlMessageQueue.front(); + if (msg->AsConfigureMessage()) { + if (ProcessConfigureMessage(msg->AsConfigureMessage()) == + MessageProcessedResult::NotProcessed) { + break; + } + } else if (msg->AsEncodeMessage()) { + if (ProcessEncodeMessage(msg->AsEncodeMessage()) == + MessageProcessedResult::NotProcessed) { + break; + } + } else { + MOZ_ASSERT(msg->AsFlushMessage()); + if (ProcessFlushMessage(msg->AsFlushMessage()) == + MessageProcessedResult::NotProcessed) { + break; + } + } + } +} + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::CancelPendingControlMessages( + const nsresult& aResult) { + AssertIsOnOwningThread(); + + // Cancel the message that is being processed. + if (mProcessingMessage) { + LOG("%s %p cancels current %s", EncoderType::Name.get(), this, + mProcessingMessage->ToString().get()); + mProcessingMessage->Cancel(); + + if (RefPtr<FlushMessage> flush = mProcessingMessage->AsFlushMessage()) { + flush->RejectPromiseIfAny(aResult); + } + + mProcessingMessage = nullptr; + } + + // Clear the message queue. + while (!mControlMessageQueue.empty()) { + LOG("%s %p cancels pending %s", EncoderType::Name.get(), this, + mControlMessageQueue.front()->ToString().get()); + + MOZ_ASSERT(!mControlMessageQueue.front()->IsProcessing()); + if (RefPtr<FlushMessage> flush = + mControlMessageQueue.front()->AsFlushMessage()) { + flush->RejectPromiseIfAny(aResult); + } + + mControlMessageQueue.pop(); + } +} + +template <typename EncoderType> +MessageProcessedResult EncoderTemplate<EncoderType>::ProcessConfigureMessage( + RefPtr<ConfigureMessage> aMessage) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(aMessage->AsConfigureMessage()); + + if (mProcessingMessage) { + return MessageProcessedResult::NotProcessed; + } + + mProcessingMessage = aMessage; + mControlMessageQueue.pop(); + LOG("%s %p Configuring, message queue processing blocked(%s)", + EncoderType::Name.get(), this, aMessage->ToString().get()); + StartBlockingMessageQueue(); + + bool supported = EncoderType::IsSupported(*aMessage->Config()); + + if (!supported) { + LOGE("%s %p ProcessConfigureMessage error (sync): Not supported", + EncoderType::Name.get(), this); + mProcessingMessage = nullptr; + NS_DispatchToCurrentThread(NS_NewRunnableFunction( + "ProcessConfigureMessage (async): not supported", + [self = RefPtr(this)]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { + LOGE("%s %p ProcessConfigureMessage (async close): Not supported", + EncoderType::Name.get(), self.get()); + DebugOnly<Result<Ok, nsresult>> r = + self->CloseInternal(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + MOZ_ASSERT(r.value.isOk()); + })); + return MessageProcessedResult::Processed; + } + + if (mAgent) { + Reconfigure(aMessage); + } else { + Configure(aMessage); + } + + return MessageProcessedResult::Processed; +} + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::StartBlockingMessageQueue() { + LOG("=== Message queue blocked"); + mMessageQueueBlocked = true; +} + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::StopBlockingMessageQueue() { + LOG("=== Message queue unblocked"); + mMessageQueueBlocked = false; +} + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::Reconfigure( + RefPtr<ConfigureMessage> aMessage) { + MOZ_ASSERT(mAgent); + + LOG("Reconfiguring encoder: %s", + NS_ConvertUTF16toUTF8(aMessage->Config()->ToString()).get()); + + RefPtr<ConfigTypeInternal> config = aMessage->Config(); + RefPtr<WebCodecsConfigurationChangeList> configDiff = + config->Diff(*mActiveConfig); + + // Nothing to do, return now + if (configDiff->Empty()) { + LOG("Reconfigure with identical config, returning."); + mProcessingMessage = nullptr; + StopBlockingMessageQueue(); + return; + } + + LOG("Attempting to reconfigure encoder: old: %s new: %s, diff: %s", + NS_ConvertUTF16toUTF8(mActiveConfig->ToString()).get(), + NS_ConvertUTF16toUTF8(config->ToString()).get(), + NS_ConvertUTF16toUTF8(configDiff->ToString()).get()); + + RefPtr<EncoderConfigurationChangeList> changeList = + configDiff->ToPEMChangeList(); + + // Attempt to reconfigure the encoder, if the config is similar enough. + // Otherwise, or if reconfiguring on the fly didn't work, flush the encoder + // and recreate a new one. + + mAgent->Reconfigure(changeList) + ->Then( + GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = mAgent->mId, + message = std::move(aMessage)]( + const EncoderAgent::ReconfigurationPromise::ResolveOrRejectValue& + aResult) { + MOZ_ASSERT(self->mProcessingMessage); + MOZ_ASSERT(self->mProcessingMessage->AsConfigureMessage()); + MOZ_ASSERT(self->mState == CodecState::Configured); + MOZ_ASSERT(self->mAgent); + MOZ_ASSERT(id == self->mAgent->mId); + MOZ_ASSERT(self->mActiveConfig); + + if (aResult.IsReject()) { + LOGE( + "Reconfiguring on the fly didn't succeed, flushing and " + "configuring a new encoder"); + self->mAgent->Drain()->Then( + GetCurrentSerialEventTarget(), __func__, + [self, id, + message](EncoderAgent::EncodePromise::ResolveOrRejectValue&& + aResult) { + if (aResult.IsReject()) { + const MediaResult& error = aResult.RejectValue(); + LOGE( + "%s %p, EncoderAgent #%zu failed to flush during " + "reconfigure, closing: %s", + EncoderType::Name.get(), self.get(), id, + error.Description().get()); + + self->mProcessingMessage = nullptr; + self->ScheduleClose( + NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + return; + } + + LOG("%s %p flush during reconfiguration succeeded.", + EncoderType::Name.get(), self.get()); + + // If flush succeeded, schedule to output encoded data + // first, destroy the current encoder, and proceed to create + // a new one. + MOZ_ASSERT(aResult.IsResolve()); + nsTArray<RefPtr<MediaRawData>> data = + std::move(aResult.ResolveValue()); + + if (data.IsEmpty()) { + LOG("%s %p no data during flush for reconfiguration with " + "encoder destruction", + EncoderType::Name.get(), self.get()); + } else { + LOG("%s %p Outputing %zu frames during flush " + " for reconfiguration with encoder destruction", + EncoderType::Name.get(), self.get(), data.Length()); + self->ScheduleOutputEncodedData( + std::move(data), + nsLiteralCString("Flush before reconfigure")); + } + + NS_DispatchToCurrentThread(NS_NewRunnableFunction( + "Destroy + recreate encoder after failed reconfigure", + [self = RefPtr(self), message]() + MOZ_CAN_RUN_SCRIPT_BOUNDARY { + // Destroy the agent, and finally create a fresh + // encoder with the new configuration. + self->DestroyEncoderAgentIfAny(); + self->Configure(message); + })); + }); + return; + } + + LOG("%s %p, EncoderAgent #%zu has been reconfigured on the fly to " + "%s", + EncoderType::Name.get(), self.get(), id, + message->ToString().get()); + + self->mOutputNewDecoderConfig = true; + self->mActiveConfig = message->Config(); + self->mProcessingMessage = nullptr; + self->StopBlockingMessageQueue(); + self->ProcessControlMessageQueue(); + }); +} + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::Configure( + RefPtr<ConfigureMessage> aMessage) { + MOZ_ASSERT(!mAgent); + + LOG("Configuring encoder: %s", + NS_ConvertUTF16toUTF8(aMessage->Config()->ToString()).get()); + + mOutputNewDecoderConfig = true; + mActiveConfig = aMessage->Config(); + + bool decoderAgentCreated = + CreateEncoderAgent(aMessage->mMessageId, aMessage->Config()); + if (!decoderAgentCreated) { + LOGE( + "%s %p ProcessConfigureMessage error (sync): encoder agent " + "creation " + "failed", + EncoderType::Name.get(), this); + mProcessingMessage = nullptr; + NS_DispatchToCurrentThread(NS_NewRunnableFunction( + "ProcessConfigureMessage (async): encoder agent creating failed", + [self = RefPtr(this)]() MOZ_CAN_RUN_SCRIPT_BOUNDARY { + LOGE( + "%s %p ProcessConfigureMessage (async close): encoder agent " + "creation failed", + EncoderType::Name.get(), self.get()); + DebugOnly<Result<Ok, nsresult>> r = + self->CloseInternal(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + MOZ_ASSERT(r.value.isOk()); + })); + return; + } + + MOZ_ASSERT(mAgent); + MOZ_ASSERT(mActiveConfig); + + LOG("Real configuration with fresh config: %s", + NS_ConvertUTF16toUTF8(mActiveConfig->ToString().get()).get()); + + EncoderConfig config = mActiveConfig->ToEncoderConfig(); + mAgent->Configure(config) + ->Then(GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = mAgent->mId, aMessage]( + const EncoderAgent::ConfigurePromise::ResolveOrRejectValue& + aResult) MOZ_CAN_RUN_SCRIPT_BOUNDARY { + MOZ_ASSERT(self->mProcessingMessage); + MOZ_ASSERT(self->mProcessingMessage->AsConfigureMessage()); + MOZ_ASSERT(self->mState == CodecState::Configured); + MOZ_ASSERT(self->mAgent); + MOZ_ASSERT(id == self->mAgent->mId); + MOZ_ASSERT(self->mActiveConfig); + + LOG("%s %p, EncoderAgent #%zu %s has been %s. now unblocks " + "message-queue-processing", + EncoderType::Name.get(), self.get(), id, + aMessage->ToString().get(), + aResult.IsResolve() ? "resolved" : "rejected"); + + aMessage->Complete(); + self->mProcessingMessage = nullptr; + + if (aResult.IsReject()) { + // The spec asks to close the decoder with an + // NotSupportedError so we log the exact error here. + const MediaResult& error = aResult.RejectValue(); + LOGE("%s %p, EncoderAgent #%zu failed to configure: %s", + EncoderType::Name.get(), self.get(), id, + error.Description().get()); + DebugOnly<Result<Ok, nsresult>> r = self->CloseInternal( + NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + MOZ_ASSERT(r.value.isOk()); + return; // No further process + } + + self->StopBlockingMessageQueue(); + self->ProcessControlMessageQueue(); + }) + ->Track(aMessage->Request()); +} + +template <typename EncoderType> +MessageProcessedResult EncoderTemplate<EncoderType>::ProcessEncodeMessage( + RefPtr<EncodeMessage> aMessage) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(aMessage->AsEncodeMessage()); + + if (mProcessingMessage) { + return MessageProcessedResult::NotProcessed; + } + + mProcessingMessage = aMessage; + mControlMessageQueue.pop(); + + LOGV("%s %p processing %s", EncoderType::Name.get(), this, + aMessage->ToString().get()); + + mEncodeQueueSize -= 1; + ScheduleDequeueEvent(); + + // Treat it like decode error if no EncoderAgent is available or the encoded + // data is invalid. + auto closeOnError = [&]() { + mProcessingMessage = nullptr; + ScheduleClose(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + return MessageProcessedResult::Processed; + }; + + if (!mAgent) { + LOGE("%s %p is not configured", EncoderType::Name.get(), this); + return closeOnError(); + } + + MOZ_ASSERT(mActiveConfig); + RefPtr<InputTypeInternal> data = aMessage->mData; + if (!data) { + LOGE("%s %p, data for %s is empty or invalid", EncoderType::Name.get(), + this, aMessage->ToString().get()); + return closeOnError(); + } + + mAgent->Encode(data.get()) + ->Then(GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = mAgent->mId, aMessage]( + EncoderAgent::EncodePromise::ResolveOrRejectValue&& aResult) { + MOZ_ASSERT(self->mProcessingMessage); + MOZ_ASSERT(self->mProcessingMessage->AsEncodeMessage()); + MOZ_ASSERT(self->mState == CodecState::Configured); + MOZ_ASSERT(self->mAgent); + MOZ_ASSERT(id == self->mAgent->mId); + MOZ_ASSERT(self->mActiveConfig); + + nsCString msgStr = aMessage->ToString(); + + aMessage->Complete(); + self->mProcessingMessage = nullptr; + + if (aResult.IsReject()) { + // The spec asks to queue a task to run close the decoder + // with an EncodingError so we log the exact error here. + const MediaResult& error = aResult.RejectValue(); + LOGE("%s %p, EncoderAgent #%zu %s failed: %s", + EncoderType::Name.get(), self.get(), id, msgStr.get(), + error.Description().get()); + self->ScheduleClose(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + return; // No further process + } + + MOZ_ASSERT(aResult.IsResolve()); + nsTArray<RefPtr<MediaRawData>> data = + std::move(aResult.ResolveValue()); + if (data.IsEmpty()) { + LOGV("%s %p got no data for %s", EncoderType::Name.get(), + self.get(), msgStr.get()); + } else { + LOGV("%s %p, schedule %zu encoded data output", + EncoderType::Name.get(), self.get(), data.Length()); + self->ScheduleOutputEncodedData(std::move(data), msgStr); + } + + self->ProcessControlMessageQueue(); + }) + ->Track(aMessage->Request()); + + return MessageProcessedResult::Processed; +} + +template <typename EncoderType> +MessageProcessedResult EncoderTemplate<EncoderType>::ProcessFlushMessage( + RefPtr<FlushMessage> aMessage) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(aMessage->AsFlushMessage()); + + if (mProcessingMessage) { + return MessageProcessedResult::NotProcessed; + } + + mProcessingMessage = aMessage; + mControlMessageQueue.pop(); + + LOG("%s %p starts processing %s", EncoderType::Name.get(), this, + aMessage->ToString().get()); + + // No agent, no thing to do. The promise has been rejected with the + // appropriate error in ResetInternal already. + if (!mAgent) { + LOGE("%s %p no agent, nothing to do", EncoderType::Name.get(), this); + mProcessingMessage = nullptr; + return MessageProcessedResult::Processed; + } + + mAgent->Drain() + ->Then(GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = mAgent->mId, aMessage]( + EncoderAgent::EncodePromise::ResolveOrRejectValue&& aResult) { + MOZ_ASSERT(self->mProcessingMessage); + MOZ_ASSERT(self->mProcessingMessage->AsFlushMessage()); + MOZ_ASSERT(self->mState == CodecState::Configured); + MOZ_ASSERT(self->mAgent); + MOZ_ASSERT(id == self->mAgent->mId); + MOZ_ASSERT(self->mActiveConfig); + + LOG("%s %p, EncoderAgent #%zu %s has been %s", + EncoderType::Name.get(), self.get(), id, + aMessage->ToString().get(), + aResult.IsResolve() ? "resolved" : "rejected"); + + nsCString msgStr = aMessage->ToString(); + + aMessage->Complete(); + + // If flush failed, it means encoder fails to encode the data + // sent before, so we treat it like an encode error. We reject + // the promise first and then queue a task to close VideoEncoder + // with an EncodingError. + if (aResult.IsReject()) { + const MediaResult& error = aResult.RejectValue(); + LOGE("%s %p, EncoderAgent #%zu failed to flush: %s", + EncoderType::Name.get(), self.get(), id, + error.Description().get()); + + // Reject with an EncodingError instead of the error we got + // above. + self->SchedulePromiseResolveOrReject( + aMessage->TakePromise(), + NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + + self->mProcessingMessage = nullptr; + + self->ScheduleClose(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR); + return; // No further process + } + + // If flush succeeded, schedule to output encoded data first + // and then resolve the promise, then keep processing the + // control messages. + MOZ_ASSERT(aResult.IsResolve()); + nsTArray<RefPtr<MediaRawData>> data = + std::move(aResult.ResolveValue()); + + if (data.IsEmpty()) { + LOG("%s %p gets no data for %s", EncoderType::Name.get(), + self.get(), msgStr.get()); + } else { + LOG("%s %p, schedule %zu encoded data output for %s", + EncoderType::Name.get(), self.get(), data.Length(), + msgStr.get()); + self->ScheduleOutputEncodedData(std::move(data), msgStr); + } + + self->SchedulePromiseResolveOrReject(aMessage->TakePromise(), + NS_OK); + self->mProcessingMessage = nullptr; + self->ProcessControlMessageQueue(); + }) + ->Track(aMessage->Request()); + + return MessageProcessedResult::Processed; +} + +// CreateEncoderAgent will create an EncoderAgent paired with a xpcom-shutdown +// blocker and a worker-reference. Besides the needs mentioned in the header +// file, the blocker and the worker-reference also provides an entry point for +// us to clean up the resources. Other than the encoder dtor, Reset(), or +// Close(), the resources should be cleaned up in the following situations: +// 1. Encoder on window, closing document +// 2. Encoder on worker, closing document +// 3. Encoder on worker, terminating worker +// +// In case 1, the entry point to clean up is in the mShutdownBlocker's +// ShutdownpPomise-resolver. In case 2, the entry point is in mWorkerRef's +// shutting down callback. In case 3, the entry point is in mWorkerRef's +// shutting down callback. + +template <typename EncoderType> +bool EncoderTemplate<EncoderType>::CreateEncoderAgent( + WebCodecsId aId, RefPtr<ConfigTypeInternal> aConfig) { + AssertIsOnOwningThread(); + MOZ_ASSERT(mState == CodecState::Configured); + MOZ_ASSERT(!mAgent); + MOZ_ASSERT(!mShutdownBlocker); + MOZ_ASSERT_IF(!NS_IsMainThread(), !mWorkerRef); + + auto resetOnFailure = MakeScopeExit([&]() { + mAgent = nullptr; + mActiveConfig = nullptr; + mShutdownBlocker = nullptr; + mWorkerRef = nullptr; + }); + + // If the encoder is on worker, get a worker reference. + if (!NS_IsMainThread()) { + WorkerPrivate* workerPrivate = GetCurrentThreadWorkerPrivate(); + if (NS_WARN_IF(!workerPrivate)) { + return false; + } + + // Clean up all the resources when worker is going away. + RefPtr<StrongWorkerRef> workerRef = StrongWorkerRef::Create( + workerPrivate, "EncoderTemplate::CreateEncoderAgent", + [self = RefPtr{this}]() { + LOG("%s %p, worker is going away", EncoderType::Name.get(), + self.get()); + Unused << self->ResetInternal(NS_ERROR_DOM_ABORT_ERR); + }); + if (NS_WARN_IF(!workerRef)) { + return false; + } + + mWorkerRef = new ThreadSafeWorkerRef(workerRef); + } + + mAgent = MakeRefPtr<EncoderAgent>(aId); + + // ShutdownBlockingTicket requires an unique name to register its own + // nsIAsyncShutdownBlocker since each blocker needs a distinct name. + // To do that, we use EncoderAgent's unique id to create a unique name. + nsAutoString uniqueName; + uniqueName.AppendPrintf( + "Blocker for EncoderAgent #%zu (codec: %s) @ %p", mAgent->mId, + NS_ConvertUTF16toUTF8(mActiveConfig->mCodec).get(), mAgent.get()); + + mShutdownBlocker = media::ShutdownBlockingTicket::Create( + uniqueName, NS_LITERAL_STRING_FROM_CSTRING(__FILE__), __LINE__); + if (!mShutdownBlocker) { + LOGE("%s %p failed to create %s", EncoderType::Name.get(), this, + NS_ConvertUTF16toUTF8(uniqueName).get()); + return false; + } + + // Clean up all the resources when xpcom-will-shutdown arrives since the + // page is going to be closed. + mShutdownBlocker->ShutdownPromise()->Then( + GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = mAgent->mId, + ref = mWorkerRef](bool /* aUnUsed*/) { + LOG("%s %p gets xpcom-will-shutdown notification for EncoderAgent " + "#%zu", + EncoderType::Name.get(), self.get(), id); + Unused << self->ResetInternal(NS_ERROR_DOM_ABORT_ERR); + }, + [self = RefPtr{this}, id = mAgent->mId, + ref = mWorkerRef](bool /* aUnUsed*/) { + LOG("%s %p removes shutdown-blocker #%zu before getting any " + "notification. EncoderAgent should have been dropped", + EncoderType::Name.get(), self.get(), id); + MOZ_ASSERT(!self->mAgent || self->mAgent->mId != id); + }); + + LOG("%s %p creates EncoderAgent #%zu @ %p and its shutdown-blocker", + EncoderType::Name.get(), this, mAgent->mId, mAgent.get()); + + resetOnFailure.release(); + return true; +} + +template <typename EncoderType> +void EncoderTemplate<EncoderType>::DestroyEncoderAgentIfAny() { + AssertIsOnOwningThread(); + + if (!mAgent) { + LOG("%s %p has no EncoderAgent to destroy", EncoderType::Name.get(), this); + return; + } + + MOZ_ASSERT(mActiveConfig); + MOZ_ASSERT(mShutdownBlocker); + MOZ_ASSERT_IF(!NS_IsMainThread(), mWorkerRef); + + LOG("%s %p destroys EncoderAgent #%zu @ %p", EncoderType::Name.get(), this, + mAgent->mId, mAgent.get()); + mActiveConfig = nullptr; + RefPtr<EncoderAgent> agent = std::move(mAgent); + // mShutdownBlocker should be kept alive until the shutdown is done. + // mWorkerRef is used to ensure this task won't be discarded in worker. + agent->Shutdown()->Then( + GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, id = agent->mId, ref = std::move(mWorkerRef), + blocker = std::move(mShutdownBlocker)]( + const ShutdownPromise::ResolveOrRejectValue& aResult) { + LOG("%s %p, EncoderAgent #%zu's shutdown has been %s. Drop its " + "shutdown-blocker now", + EncoderType::Name.get(), self.get(), id, + aResult.IsResolve() ? "resolved" : "rejected"); + }); +} + +template class EncoderTemplate<VideoEncoderTraits>; + +#undef LOG +#undef LOGW +#undef LOGE +#undef LOGV +#undef LOG_INTERNAL + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/EncoderTemplate.h b/dom/media/webcodecs/EncoderTemplate.h new file mode 100644 index 0000000000..e53d7166d1 --- /dev/null +++ b/dom/media/webcodecs/EncoderTemplate.h @@ -0,0 +1,290 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_EncoderTemplate_h +#define mozilla_dom_EncoderTemplate_h + +#include <queue> + +#include "EncoderAgent.h" +#include "MediaData.h" +#include "WebCodecsUtils.h" +#include "mozilla/DOMEventTargetHelper.h" +#include "mozilla/MozPromise.h" +#include "mozilla/RefPtr.h" +#include "mozilla/Result.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/dom/VideoEncoderBinding.h" +#include "mozilla/dom/WorkerRef.h" +#include "mozilla/media/MediaUtils.h" +#include "nsStringFwd.h" + +namespace mozilla::dom { + +class WebCodecsErrorCallback; +class Promise; +enum class CodecState : uint8_t; + +using Id = size_t; + +template <typename EncoderType> +class EncoderTemplate : public DOMEventTargetHelper { + using Self = EncoderTemplate<EncoderType>; + using ConfigType = typename EncoderType::ConfigType; + using ConfigTypeInternal = typename EncoderType::ConfigTypeInternal; + using OutputConfigType = typename EncoderType::OutputConfigType; + using InputType = typename EncoderType::InputType; + using InputTypeInternal = typename EncoderType::InputTypeInternal; + using OutputType = typename EncoderType::OutputType; + using OutputCallbackType = typename EncoderType::OutputCallbackType; + + /* ControlMessage classes */ + protected: + class ConfigureMessage; + class EncodeMessage; + class FlushMessage; + + class ControlMessage { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ControlMessage) + explicit ControlMessage(Id aConfigureId); + virtual void Cancel() = 0; + virtual bool IsProcessing() = 0; + + virtual nsCString ToString() const = 0; + virtual RefPtr<ConfigureMessage> AsConfigureMessage() { return nullptr; } + virtual RefPtr<EncodeMessage> AsEncodeMessage() { return nullptr; } + virtual RefPtr<FlushMessage> AsFlushMessage() { return nullptr; } + + // For logging purposes + const WebCodecsId mConfigureId; + const WebCodecsId mMessageId; + + protected: + virtual ~ControlMessage() = default; + }; + + class ConfigureMessage final + : public ControlMessage, + public MessageRequestHolder<EncoderAgent::ConfigurePromise> { + public: + ConfigureMessage(Id aConfigureId, + const RefPtr<ConfigTypeInternal>& aConfig); + virtual void Cancel() override { Disconnect(); } + virtual bool IsProcessing() override { return Exists(); }; + virtual RefPtr<ConfigureMessage> AsConfigureMessage() override { + return this; + } + RefPtr<ConfigTypeInternal> Config() { return mConfig; } + nsCString ToString() const override { + nsCString rv; + rv.AppendPrintf( + "ConfigureMessage(#%zu): %s", this->mMessageId, + mConfig ? NS_ConvertUTF16toUTF8(mConfig->ToString().get()).get() + : "null cfg"); + return rv; + } + + private: + const RefPtr<ConfigTypeInternal> mConfig; + }; + + class EncodeMessage final + : public ControlMessage, + public MessageRequestHolder<EncoderAgent::EncodePromise> { + public: + EncodeMessage(WebCodecsId aConfigureId, RefPtr<InputTypeInternal>&& aData, + Maybe<VideoEncoderEncodeOptions>&& aOptions = Nothing()); + nsCString ToString() const override { + nsCString rv; + bool isKeyFrame = mOptions.isSome() && mOptions.ref().mKeyFrame; + rv.AppendPrintf("EncodeMessage(#%zu,#%zu): %s (%s)", this->mConfigureId, + this->mMessageId, mData->ToString().get(), + isKeyFrame ? "kf" : ""); + return rv; + } + virtual void Cancel() override { Disconnect(); } + virtual bool IsProcessing() override { return Exists(); }; + virtual RefPtr<EncodeMessage> AsEncodeMessage() override { return this; } + RefPtr<InputTypeInternal> mData; + Maybe<VideoEncoderEncodeOptions> mOptions; + }; + + class FlushMessage final + : public ControlMessage, + public MessageRequestHolder<EncoderAgent::EncodePromise> { + public: + FlushMessage(WebCodecsId aConfigureId, Promise* aPromise); + virtual void Cancel() override { Disconnect(); } + virtual bool IsProcessing() override { return Exists(); }; + virtual RefPtr<FlushMessage> AsFlushMessage() override { return this; } + already_AddRefed<Promise> TakePromise() { return mPromise.forget(); } + void RejectPromiseIfAny(const nsresult& aReason); + + nsCString ToString() const override { + nsCString rv; + rv.AppendPrintf("FlushMessage(#%zu,#%zu)", this->mConfigureId, + this->mMessageId); + return rv; + } + + private: + RefPtr<Promise> mPromise; + }; + + protected: + EncoderTemplate(nsIGlobalObject* aGlobalObject, + RefPtr<WebCodecsErrorCallback>&& aErrorCallback, + RefPtr<OutputCallbackType>&& aOutputCallback); + + virtual ~EncoderTemplate() = default; + + /* WebCodecs interfaces */ + public: + IMPL_EVENT_HANDLER(dequeue) + + void StartBlockingMessageQueue(); + void StopBlockingMessageQueue(); + + CodecState State() const { return mState; }; + + uint32_t EncodeQueueSize() const { return mEncodeQueueSize; }; + + void Configure(const ConfigType& aConfig, ErrorResult& aRv); + + void EncodeAudioData(InputType& aInput, ErrorResult& aRv); + void EncodeVideoFrame(InputType& aInput, + const VideoEncoderEncodeOptions& aOptions, + ErrorResult& aRv); + + already_AddRefed<Promise> Flush(ErrorResult& aRv); + + void Reset(ErrorResult& aRv); + + MOZ_CAN_RUN_SCRIPT + void Close(ErrorResult& aRv); + + /* Type conversion functions for the Encoder implementation */ + protected: + virtual RefPtr<OutputType> EncodedDataToOutputType( + nsIGlobalObject* aGlobalObject, RefPtr<MediaRawData>& aData) = 0; + virtual OutputConfigType EncoderConfigToDecoderConfig( + nsIGlobalObject* aGlobalObject, const RefPtr<MediaRawData>& aData, + const ConfigTypeInternal& aOutputConfig) const = 0; + /* Internal member variables and functions */ + protected: + // EncoderTemplate can run on either main thread or worker thread. + void AssertIsOnOwningThread() const { + NS_ASSERT_OWNINGTHREAD(EncoderTemplate); + } + + Result<Ok, nsresult> ResetInternal(const nsresult& aResult); + MOZ_CAN_RUN_SCRIPT_BOUNDARY + Result<Ok, nsresult> CloseInternal(const nsresult& aResult); + + MOZ_CAN_RUN_SCRIPT void ReportError(const nsresult& aResult); + MOZ_CAN_RUN_SCRIPT void OutputEncodedData( + nsTArray<RefPtr<MediaRawData>>&& aData); + + class ErrorRunnable; + void ScheduleReportError(const nsresult& aResult); + + class OutputRunnable; + void ScheduleOutputEncodedData(nsTArray<RefPtr<MediaRawData>>&& aData, + const nsACString& aLabel); + + void ScheduleClose(const nsresult& aResult); + + void ScheduleDequeueEvent(); + nsresult FireEvent(nsAtom* aTypeWithOn, const nsAString& aEventType); + + void SchedulePromiseResolveOrReject(already_AddRefed<Promise> aPromise, + const nsresult& aResult); + + void ProcessControlMessageQueue(); + void CancelPendingControlMessages(const nsresult& aResult); + + MessageProcessedResult ProcessConfigureMessage( + RefPtr<ConfigureMessage> aMessage); + + MessageProcessedResult ProcessEncodeMessage(RefPtr<EncodeMessage> aMessage); + + MessageProcessedResult ProcessFlushMessage(RefPtr<FlushMessage> aMessage); + + void Configure(RefPtr<ConfigureMessage> aMessage); + void Reconfigure(RefPtr<ConfigureMessage> aMessage); + + // Returns true when mAgent can be created. + bool CreateEncoderAgent(WebCodecsId aId, RefPtr<ConfigTypeInternal> aConfig); + void DestroyEncoderAgentIfAny(); + + // Constant in practice, only set in ctor. + RefPtr<WebCodecsErrorCallback> mErrorCallback; + RefPtr<OutputCallbackType> mOutputCallback; + + CodecState mState; + + bool mMessageQueueBlocked; + std::queue<RefPtr<ControlMessage>> mControlMessageQueue; + RefPtr<ControlMessage> mProcessingMessage; + + uint32_t mEncodeQueueSize; + bool mDequeueEventScheduled; + + // A unique id tracking the ConfigureMessage and will be used as the + // EncoderAgent's Id. + uint32_t mLatestConfigureId; + // Tracking how many encoded data has been enqueued and this number will be + // used as the EncodeMessage's Id. + size_t mEncodeCounter; + // Tracking how many flush request has been enqueued and this number will be + // used as the FlushMessage's Id. + size_t mFlushCounter; + + // EncoderAgent will be created the first time "configure" is being processed, + // and will be destroyed when "reset" is called. If another "configure" is + // called, either it's possible to reconfigure the underlying encoder without + // tearing eveyrthing down (e.g. a bitrate change), or it's not possible, and + // the current encoder will be destroyed and a new one create. + // In both cases, the encoder is implicitely flushed before the configuration + // change. + // See CanReconfigure on the {Audio,Video}EncoderConfigInternal + RefPtr<EncoderAgent> mAgent; + RefPtr<ConfigTypeInternal> mActiveConfig; + // This is true when a configure call has just been processed, and it's + // necessary to pass the new decoding configuration when the callback is + // called. Read and modified on owner thread only. + bool mOutputNewDecoderConfig = false; + + // Used to add a nsIAsyncShutdownBlocker on main thread to block + // xpcom-shutdown before the underlying MediaDataEncoder is created. The + // blocker will be held until the underlying MediaDataEncoder has been shut + // down. This blocker guarantees RemoteEncoderManagerChild's thread, where + // the underlying RemoteMediaDataEncoder is on, outlives the + // RemoteMediaDataEncoder since the thread releasing, which happens on main + // thread when getting a xpcom-shutdown signal, is blocked by the added + // blocker. As a result, RemoteMediaDataEncoder can safely work on worker + // thread with a holding blocker (otherwise, if RemoteEncoderManagerChild + // releases its thread on main thread before RemoteMediaDataEncoder's + // Shutdown() task run on worker thread, RemoteMediaDataEncoder has no + // thread to run). + UniquePtr<media::ShutdownBlockingTicket> mShutdownBlocker; + + // Held to make sure the dispatched tasks can be done before worker is going + // away. As long as this worker-ref is held somewhere, the tasks dispatched + // to the worker can be executed (otherwise the tasks would be canceled). + // This ref should be activated as long as the underlying MediaDataEncoder + // is alive, and should keep alive until mShutdownBlocker is dropped, so all + // MediaDataEncoder's tasks and mShutdownBlocker-releasing task can be + // executed. + // TODO: Use StrongWorkerRef instead if this is always used in the same + // thread? + RefPtr<ThreadSafeWorkerRef> mWorkerRef; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_EncoderTemplate_h diff --git a/dom/media/webcodecs/EncoderTypes.h b/dom/media/webcodecs/EncoderTypes.h new file mode 100644 index 0000000000..d58d7c54c8 --- /dev/null +++ b/dom/media/webcodecs/EncoderTypes.h @@ -0,0 +1,103 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_EncoderTypes_h +#define mozilla_dom_EncoderTypes_h + +#include "mozilla/Maybe.h" +#include "mozilla/dom/EncodedVideoChunk.h" +#include "mozilla/dom/VideoEncoderBinding.h" +#include "mozilla/dom/VideoFrame.h" +#include "mozilla/dom/VideoFrameBinding.h" +#include "nsStringFwd.h" +#include "nsTLiteralString.h" +#include "VideoDecoder.h" +#include "PlatformEncoderModule.h" + +namespace mozilla { + +class TrackInfo; +class MediaByteBuffer; + +namespace dom { + +class VideoEncoderConfigInternal { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoEncoderConfigInternal); + explicit VideoEncoderConfigInternal(const VideoEncoderConfig& aConfig); + explicit VideoEncoderConfigInternal( + const VideoEncoderConfigInternal& aConfig); + + // Returns an EncoderConfig struct with as many filled members as + // possible. + // TODO: handle codec specific things + EncoderConfig ToEncoderConfig() const; + + bool Equals(const VideoEncoderConfigInternal& aOther) const; + bool CanReconfigure(const VideoEncoderConfigInternal& aOther) const; + already_AddRefed<WebCodecsConfigurationChangeList> Diff( + const VideoEncoderConfigInternal& aOther) const; + nsString ToString() const; + + nsString mCodec; + uint32_t mWidth; + uint32_t mHeight; + Maybe<uint32_t> mDisplayWidth; + Maybe<uint32_t> mDisplayHeight; + Maybe<uint32_t> mBitrate; + Maybe<double> mFramerate; + HardwareAcceleration mHardwareAcceleration; + AlphaOption mAlpha; + Maybe<nsString> mScalabilityMode; + VideoEncoderBitrateMode mBitrateMode; + LatencyMode mLatencyMode; + Maybe<nsString> mContentHint; + Maybe<AvcEncoderConfig> mAvc; + + private: + VideoEncoderConfigInternal( + const nsAString& aCodec, uint32_t aWidth, uint32_t aHeight, + Maybe<uint32_t>&& aDisplayWidth, Maybe<uint32_t>&& aDisplayHeight, + Maybe<uint32_t>&& aBitrate, Maybe<double>&& aFramerate, + const HardwareAcceleration& aHardwareAcceleration, + const AlphaOption& aAlpha, Maybe<nsString>&& aScalabilityMode, + const VideoEncoderBitrateMode& aBitrateMode, + const LatencyMode& aLatencyMode, Maybe<nsString>&& aContentHint); + + ~VideoEncoderConfigInternal() = default; +}; + +class VideoEncoderTraits { + public: + static constexpr nsLiteralCString Name = "VideoEncoder"_ns; + using ConfigType = VideoEncoderConfig; + using ConfigTypeInternal = VideoEncoderConfigInternal; + using InputType = dom::VideoFrame; + using InputTypeInternal = mozilla::VideoData; + using OutputConfigType = mozilla::dom::VideoDecoderConfigInternal; + using OutputType = EncodedVideoChunk; + using OutputCallbackType = EncodedVideoChunkOutputCallback; + using MetadataType = EncodedVideoChunkMetadata; + + static bool IsSupported(const ConfigTypeInternal& aConfig); + static bool CanEncodeVideo(const ConfigTypeInternal& aConfig); + static Result<UniquePtr<TrackInfo>, nsresult> CreateTrackInfo( + const ConfigTypeInternal& aConfig); + static bool Validate(const ConfigType& aConfig, nsCString& aErrorMessage); + static RefPtr<ConfigTypeInternal> CreateConfigInternal( + const ConfigType& aConfig); + static RefPtr<InputTypeInternal> CreateInputInternal( + const InputType& aInput, const VideoEncoderEncodeOptions& aOptions); + static already_AddRefed<OutputConfigType> EncoderConfigToDecoderConfig( + nsIGlobalObject* aGlobal, + const RefPtr<MediaRawData>& aData, + const ConfigTypeInternal& mOutputConfig); +}; + +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_EncoderTypes_h diff --git a/dom/media/webcodecs/VideoColorSpace.cpp b/dom/media/webcodecs/VideoColorSpace.cpp new file mode 100644 index 0000000000..5b0dad0f31 --- /dev/null +++ b/dom/media/webcodecs/VideoColorSpace.cpp @@ -0,0 +1,48 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/VideoColorSpace.h" +#include "mozilla/dom/VideoColorSpaceBinding.h" +#include "nsIGlobalObject.h" + +namespace mozilla::dom { + +// Only needed for refcounted objects. +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(VideoColorSpace, mParent) +NS_IMPL_CYCLE_COLLECTING_ADDREF(VideoColorSpace) +NS_IMPL_CYCLE_COLLECTING_RELEASE(VideoColorSpace) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(VideoColorSpace) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +VideoColorSpace::VideoColorSpace(nsIGlobalObject* aParent, + const VideoColorSpaceInit& aInit) + : mParent(aParent), mInit(aInit) { + MOZ_ASSERT(mParent); +} + +nsIGlobalObject* VideoColorSpace::GetParentObject() const { return mParent; } + +JSObject* VideoColorSpace::WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) { + return VideoColorSpace_Binding::Wrap(aCx, this, aGivenProto); +} + +/* static */ +already_AddRefed<VideoColorSpace> VideoColorSpace::Constructor( + const GlobalObject& aGlobal, const VideoColorSpaceInit& aInit, + ErrorResult& aRv) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + RefPtr<VideoColorSpace> videoColorSpace(new VideoColorSpace(global, aInit)); + return aRv.Failed() ? nullptr : videoColorSpace.forget(); +} + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/VideoColorSpace.h b/dom/media/webcodecs/VideoColorSpace.h new file mode 100644 index 0000000000..e6fdc22317 --- /dev/null +++ b/dom/media/webcodecs/VideoColorSpace.h @@ -0,0 +1,64 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_VideoColorSpace_h +#define mozilla_dom_VideoColorSpace_h + +#include "js/TypeDecls.h" +#include "mozilla/Attributes.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/VideoColorSpaceBinding.h" +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; + +namespace mozilla::dom { + +class VideoColorSpace final : public nsISupports, public nsWrapperCache { + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(VideoColorSpace) + + public: + VideoColorSpace(nsIGlobalObject* aParent, const VideoColorSpaceInit& aInit); + + protected: + ~VideoColorSpace() = default; + + public: + nsIGlobalObject* GetParentObject() const; + + JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override; + + static already_AddRefed<VideoColorSpace> Constructor( + const GlobalObject& aGlobal, const VideoColorSpaceInit& aInit, + ErrorResult& aRv); + + const Nullable<VideoColorPrimaries>& GetPrimaries() const { + return mInit.mPrimaries; + } + + const Nullable<VideoTransferCharacteristics>& GetTransfer() const { + return mInit.mTransfer; + } + + const Nullable<VideoMatrixCoefficients>& GetMatrix() const { + return mInit.mMatrix; + } + + const Nullable<bool>& GetFullRange() const { return mInit.mFullRange; } + + private: + nsCOMPtr<nsIGlobalObject> mParent; + const VideoColorSpaceInit mInit; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_VideoColorSpace_h diff --git a/dom/media/webcodecs/VideoDecoder.cpp b/dom/media/webcodecs/VideoDecoder.cpp new file mode 100644 index 0000000000..47ca5bb459 --- /dev/null +++ b/dom/media/webcodecs/VideoDecoder.cpp @@ -0,0 +1,977 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/VideoDecoder.h" +#include "mozilla/dom/VideoDecoderBinding.h" + +#include "DecoderTraits.h" +#include "GPUVideoImage.h" +#include "H264.h" +#include "ImageContainer.h" +#include "MediaContainerType.h" +#include "MediaData.h" +#include "VideoUtils.h" +#include "mozilla/Assertions.h" +#include "mozilla/CheckedInt.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/Logging.h" +#include "mozilla/Maybe.h" +#include "mozilla/StaticPrefs_dom.h" +#include "mozilla/Try.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/EncodedVideoChunk.h" +#include "mozilla/dom/EncodedVideoChunkBinding.h" +#include "mozilla/dom/ImageUtils.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/VideoColorSpaceBinding.h" +#include "mozilla/dom/VideoFrameBinding.h" +#include "mozilla/dom/WebCodecsUtils.h" +#include "nsPrintfCString.h" +#include "nsReadableUtils.h" +#include "nsThreadUtils.h" + +#ifdef XP_MACOSX +# include "MacIOSurfaceImage.h" +#elif MOZ_WAYLAND +# include "mozilla/layers/DMABUFSurfaceImage.h" +# include "mozilla/widget/DMABufSurface.h" +#endif + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla::dom { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOG +# undef LOG +#endif // LOG +#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +#ifdef LOGW +# undef LOGW +#endif // LOGW +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +#ifdef LOGV +# undef LOGV +#endif // LOGV +#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__) + +NS_IMPL_CYCLE_COLLECTION_INHERITED(VideoDecoder, DOMEventTargetHelper, + mErrorCallback, mOutputCallback) +NS_IMPL_ADDREF_INHERITED(VideoDecoder, DOMEventTargetHelper) +NS_IMPL_RELEASE_INHERITED(VideoDecoder, DOMEventTargetHelper) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(VideoDecoder) +NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper) + +/* + * Below are helper classes + */ + +VideoColorSpaceInternal::VideoColorSpaceInternal( + const VideoColorSpaceInit& aColorSpaceInit) + : mFullRange(NullableToMaybe(aColorSpaceInit.mFullRange)), + mMatrix(NullableToMaybe(aColorSpaceInit.mMatrix)), + mPrimaries(NullableToMaybe(aColorSpaceInit.mPrimaries)), + mTransfer(NullableToMaybe(aColorSpaceInit.mTransfer)) {} + +VideoColorSpaceInit VideoColorSpaceInternal::ToColorSpaceInit() const { + VideoColorSpaceInit init; + init.mFullRange = MaybeToNullable(mFullRange); + init.mMatrix = MaybeToNullable(mMatrix); + init.mPrimaries = MaybeToNullable(mPrimaries); + init.mTransfer = MaybeToNullable(mTransfer); + return init; +}; + +static Result<RefPtr<MediaByteBuffer>, nsresult> GetExtraData( + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer); + +VideoDecoderConfigInternal::VideoDecoderConfigInternal( + const nsAString& aCodec, Maybe<uint32_t>&& aCodedHeight, + Maybe<uint32_t>&& aCodedWidth, Maybe<VideoColorSpaceInternal>&& aColorSpace, + Maybe<RefPtr<MediaByteBuffer>>&& aDescription, + Maybe<uint32_t>&& aDisplayAspectHeight, + Maybe<uint32_t>&& aDisplayAspectWidth, + const HardwareAcceleration& aHardwareAcceleration, + Maybe<bool>&& aOptimizeForLatency) + : mCodec(aCodec), + mCodedHeight(std::move(aCodedHeight)), + mCodedWidth(std::move(aCodedWidth)), + mColorSpace(std::move(aColorSpace)), + mDescription(std::move(aDescription)), + mDisplayAspectHeight(std::move(aDisplayAspectHeight)), + mDisplayAspectWidth(std::move(aDisplayAspectWidth)), + mHardwareAcceleration(aHardwareAcceleration), + mOptimizeForLatency(std::move(aOptimizeForLatency)){}; + +/*static*/ +UniquePtr<VideoDecoderConfigInternal> VideoDecoderConfigInternal::Create( + const VideoDecoderConfig& aConfig) { + nsCString errorMessage; + if (!VideoDecoderTraits::Validate(aConfig, errorMessage)) { + LOGE("Failed to create VideoDecoderConfigInternal: %s", errorMessage.get()); + return nullptr; + } + + Maybe<RefPtr<MediaByteBuffer>> description; + if (aConfig.mDescription.WasPassed()) { + auto rv = GetExtraData(aConfig.mDescription.Value()); + if (rv.isErr()) { // Invalid description data. + LOGE( + "Failed to create VideoDecoderConfigInternal due to invalid " + "description data. Error: 0x%08" PRIx32, + static_cast<uint32_t>(rv.unwrapErr())); + return nullptr; + } + description.emplace(rv.unwrap()); + } + + Maybe<VideoColorSpaceInternal> colorSpace; + if (aConfig.mColorSpace.WasPassed()) { + colorSpace.emplace(VideoColorSpaceInternal(aConfig.mColorSpace.Value())); + } + + return UniquePtr<VideoDecoderConfigInternal>(new VideoDecoderConfigInternal( + aConfig.mCodec, OptionalToMaybe(aConfig.mCodedHeight), + OptionalToMaybe(aConfig.mCodedWidth), std::move(colorSpace), + std::move(description), OptionalToMaybe(aConfig.mDisplayAspectHeight), + OptionalToMaybe(aConfig.mDisplayAspectWidth), + aConfig.mHardwareAcceleration, + OptionalToMaybe(aConfig.mOptimizeForLatency))); +} + +nsString VideoDecoderConfigInternal::ToString() const { + nsString rv; + + rv.Append(mCodec); + if (mCodedWidth.isSome()) { + rv.AppendPrintf("coded: %dx%d", mCodedWidth.value(), mCodedHeight.value()); + } + if (mDisplayAspectWidth.isSome()) { + rv.AppendPrintf("display %dx%d", mDisplayAspectWidth.value(), + mDisplayAspectHeight.value()); + } + if (mColorSpace.isSome()) { + rv.AppendPrintf("colorspace %s", "todo"); + } + if (mDescription.isSome()) { + rv.AppendPrintf("extradata: %zu bytes", mDescription.value()->Length()); + } + rv.AppendPrintf( + "hw accel: %s", + HardwareAccelerationValues::GetString(mHardwareAcceleration).data()); + if (mOptimizeForLatency.isSome()) { + rv.AppendPrintf("optimize for latency: %s", + mOptimizeForLatency.value() ? "true" : "false"); + } + + return rv; +} + +/* + * The followings are helpers for VideoDecoder methods + */ + +struct MIMECreateParam { + explicit MIMECreateParam(const VideoDecoderConfigInternal& aConfig) + : mParsedCodec(ParseCodecString(aConfig.mCodec).valueOr(EmptyString())), + mWidth(aConfig.mCodedWidth), + mHeight(aConfig.mCodedHeight) {} + explicit MIMECreateParam(const VideoDecoderConfig& aConfig) + : mParsedCodec(ParseCodecString(aConfig.mCodec).valueOr(EmptyString())), + mWidth(OptionalToMaybe(aConfig.mCodedWidth)), + mHeight(OptionalToMaybe(aConfig.mCodedHeight)) {} + + const nsString mParsedCodec; + const Maybe<uint32_t> mWidth; + const Maybe<uint32_t> mHeight; +}; + +static nsTArray<nsCString> GuessMIMETypes(const MIMECreateParam& aParam) { + const auto codec = NS_ConvertUTF16toUTF8(aParam.mParsedCodec); + nsTArray<nsCString> types; + for (const nsCString& container : GuessContainers(aParam.mParsedCodec)) { + nsPrintfCString mime("video/%s; codecs=%s", container.get(), codec.get()); + if (aParam.mWidth) { + mime.AppendPrintf("; width=%d", *aParam.mWidth); + } + if (aParam.mHeight) { + mime.AppendPrintf("; height=%d", *aParam.mHeight); + } + types.AppendElement(mime); + } + return types; +} + +static bool IsSupportedCodec(const nsAString& aCodec) { + // H265 is unsupported. + if (!IsAV1CodecString(aCodec) && !IsVP9CodecString(aCodec) && + !IsVP8CodecString(aCodec) && !IsH264CodecString(aCodec)) { + return false; + } + + // Gecko allows codec string starts with vp9 or av1 but Webcodecs requires to + // starts with av01 and vp09. + // https://www.w3.org/TR/webcodecs-codec-registry/#video-codec-registry + if (StringBeginsWith(aCodec, u"vp9"_ns) || + StringBeginsWith(aCodec, u"av1"_ns)) { + return false; + } + + return true; +} + +// https://w3c.github.io/webcodecs/#check-configuration-support +template <typename Config> +static bool CanDecode(const Config& aConfig) { + auto param = MIMECreateParam(aConfig); + // TODO: Enable WebCodecs on Android (Bug 1840508) + if (IsOnAndroid()) { + return false; + } + if (!IsSupportedCodec(param.mParsedCodec)) { + return false; + } + if (IsOnMacOS() && IsH264CodecString(param.mParsedCodec) && + !StaticPrefs::dom_media_webcodecs_force_osx_h264_enabled()) { + // This will be fixed in Bug 1846796. + return false; + } + // TODO: Instead of calling CanHandleContainerType with the guessed the + // containers, DecoderTraits should provide an API to tell if a codec is + // decodable or not. + for (const nsCString& mime : GuessMIMETypes(param)) { + if (Maybe<MediaContainerType> containerType = + MakeMediaExtendedMIMEType(mime)) { + if (DecoderTraits::CanHandleContainerType( + *containerType, nullptr /* DecoderDoctorDiagnostics */) != + CANPLAY_NO) { + return true; + } + } + } + return false; +} + +static nsTArray<UniquePtr<TrackInfo>> GetTracksInfo( + const VideoDecoderConfigInternal& aConfig) { + // TODO: Instead of calling GetTracksInfo with the guessed containers, + // DecoderTraits should provide an API to create the TrackInfo directly. + for (const nsCString& mime : GuessMIMETypes(MIMECreateParam(aConfig))) { + if (Maybe<MediaContainerType> containerType = + MakeMediaExtendedMIMEType(mime)) { + if (nsTArray<UniquePtr<TrackInfo>> tracks = + DecoderTraits::GetTracksInfo(*containerType); + !tracks.IsEmpty()) { + return tracks; + } + } + } + return {}; +} + +static Result<RefPtr<MediaByteBuffer>, nsresult> GetExtraData( + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer) { + RefPtr<MediaByteBuffer> data = MakeRefPtr<MediaByteBuffer>(); + if (!AppendTypedArrayDataTo(aBuffer, *data)) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + return data->Length() > 0 ? data : nullptr; +} + +static Result<Ok, nsresult> CloneConfiguration( + RootedDictionary<VideoDecoderConfig>& aDest, JSContext* aCx, + const VideoDecoderConfig& aConfig) { + DebugOnly<nsCString> str; + MOZ_ASSERT(VideoDecoderTraits::Validate(aConfig, str)); + + aDest.mCodec = aConfig.mCodec; + if (aConfig.mCodedHeight.WasPassed()) { + aDest.mCodedHeight.Construct(aConfig.mCodedHeight.Value()); + } + if (aConfig.mCodedWidth.WasPassed()) { + aDest.mCodedWidth.Construct(aConfig.mCodedWidth.Value()); + } + if (aConfig.mColorSpace.WasPassed()) { + aDest.mColorSpace.Construct(aConfig.mColorSpace.Value()); + } + if (aConfig.mDescription.WasPassed()) { + aDest.mDescription.Construct(); + MOZ_TRY(CloneBuffer(aCx, aDest.mDescription.Value(), + aConfig.mDescription.Value())); + } + if (aConfig.mDisplayAspectHeight.WasPassed()) { + aDest.mDisplayAspectHeight.Construct(aConfig.mDisplayAspectHeight.Value()); + } + if (aConfig.mDisplayAspectWidth.WasPassed()) { + aDest.mDisplayAspectWidth.Construct(aConfig.mDisplayAspectWidth.Value()); + } + aDest.mHardwareAcceleration = aConfig.mHardwareAcceleration; + if (aConfig.mOptimizeForLatency.WasPassed()) { + aDest.mOptimizeForLatency.Construct(aConfig.mOptimizeForLatency.Value()); + } + + return Ok(); +} + +static Maybe<VideoPixelFormat> GuessPixelFormat(layers::Image* aImage) { + if (aImage) { + // TODO: Implement ImageUtils::Impl for MacIOSurfaceImage and + // DMABUFSurfaceImage? + if (aImage->AsPlanarYCbCrImage() || aImage->AsNVImage()) { + const ImageUtils imageUtils(aImage); + Maybe<VideoPixelFormat> f = + ImageBitmapFormatToVideoPixelFormat(imageUtils.GetFormat()); + + // ImageBitmapFormat cannot distinguish YUV420 or YUV420A. + bool hasAlpha = aImage->AsPlanarYCbCrImage() && + aImage->AsPlanarYCbCrImage()->GetData() && + aImage->AsPlanarYCbCrImage()->GetData()->mAlpha; + if (f && *f == VideoPixelFormat::I420 && hasAlpha) { + return Some(VideoPixelFormat::I420A); + } + return f; + } + if (layers::GPUVideoImage* image = aImage->AsGPUVideoImage()) { + RefPtr<layers::ImageBridgeChild> imageBridge = + layers::ImageBridgeChild::GetSingleton(); + layers::TextureClient* texture = image->GetTextureClient(imageBridge); + if (NS_WARN_IF(!texture)) { + return Nothing(); + } + return SurfaceFormatToVideoPixelFormat(texture->GetFormat()); + } +#ifdef XP_MACOSX + if (layers::MacIOSurfaceImage* image = aImage->AsMacIOSurfaceImage()) { + MOZ_ASSERT(image->GetSurface()); + return SurfaceFormatToVideoPixelFormat(image->GetSurface()->GetFormat()); + } +#endif +#ifdef MOZ_WAYLAND + if (layers::DMABUFSurfaceImage* image = aImage->AsDMABUFSurfaceImage()) { + MOZ_ASSERT(image->GetSurface()); + return SurfaceFormatToVideoPixelFormat(image->GetSurface()->GetFormat()); + } +#endif + } + LOGW("Failed to get pixel format from layers::Image"); + return Nothing(); +} + +static VideoColorSpaceInternal GuessColorSpace( + const layers::PlanarYCbCrData* aData) { + if (!aData) { + LOGE("nullptr in GuessColorSpace"); + return {}; + } + + VideoColorSpaceInternal colorSpace; + colorSpace.mFullRange = Some(ToFullRange(aData->mColorRange)); + if (Maybe<VideoMatrixCoefficients> m = + ToMatrixCoefficients(aData->mYUVColorSpace)) { + colorSpace.mMatrix = ToMatrixCoefficients(aData->mYUVColorSpace); + colorSpace.mPrimaries = ToPrimaries(aData->mColorPrimaries); + } + if (!colorSpace.mPrimaries) { + LOG("Missing primaries, guessing from colorspace"); + // Make an educated guess based on the coefficients. + colorSpace.mPrimaries = colorSpace.mMatrix.map([](const auto& aMatrix) { + switch (aMatrix) { + case VideoMatrixCoefficients::EndGuard_: + MOZ_CRASH("This should not happen"); + case VideoMatrixCoefficients::Bt2020_ncl: + return VideoColorPrimaries::Bt2020; + case VideoMatrixCoefficients::Rgb: + case VideoMatrixCoefficients::Bt470bg: + case VideoMatrixCoefficients::Smpte170m: + LOGW( + "Warning: Falling back to BT709 when attempting to determine the " + "primaries function of a YCbCr buffer"); + [[fallthrough]]; + case VideoMatrixCoefficients::Bt709: + return VideoColorPrimaries::Bt709; + } + MOZ_ASSERT_UNREACHABLE("Unexpected matrix coefficients"); + LOGW( + "Warning: Falling back to BT709 due to unexpected matrix " + "coefficients " + "when attempting to determine the primaries function of a YCbCr " + "buffer"); + return VideoColorPrimaries::Bt709; + }); + } + + if (Maybe<VideoTransferCharacteristics> c = + ToTransferCharacteristics(aData->mTransferFunction)) { + colorSpace.mTransfer = Some(*c); + } + if (!colorSpace.mTransfer) { + LOG("Missing transfer characteristics, guessing from colorspace"); + colorSpace.mTransfer = Some(([&] { + switch (aData->mYUVColorSpace) { + case gfx::YUVColorSpace::Identity: + return VideoTransferCharacteristics::Iec61966_2_1; + case gfx::YUVColorSpace::BT2020: + return VideoTransferCharacteristics::Pq; + case gfx::YUVColorSpace::BT601: + LOGW( + "Warning: Falling back to BT709 when attempting to determine the " + "transfer function of a MacIOSurface"); + [[fallthrough]]; + case gfx::YUVColorSpace::BT709: + return VideoTransferCharacteristics::Bt709; + } + MOZ_ASSERT_UNREACHABLE("Unexpected color space"); + LOGW( + "Warning: Falling back to BT709 due to unexpected color space " + "when attempting to determine the transfer function of a " + "MacIOSurface"); + return VideoTransferCharacteristics::Bt709; + })()); + } + + return colorSpace; +} + +#ifdef XP_MACOSX +static VideoColorSpaceInternal GuessColorSpace(const MacIOSurface* aSurface) { + if (!aSurface) { + return {}; + } + VideoColorSpaceInternal colorSpace; + colorSpace.mFullRange = Some(aSurface->IsFullRange()); + if (Maybe<dom::VideoMatrixCoefficients> m = + ToMatrixCoefficients(aSurface->GetYUVColorSpace())) { + colorSpace.mMatrix = Some(*m); + } + if (Maybe<VideoColorPrimaries> p = ToPrimaries(aSurface->mColorPrimaries)) { + colorSpace.mPrimaries = Some(*p); + } + // Make an educated guess based on the coefficients. + if (aSurface->GetYUVColorSpace() == gfx::YUVColorSpace::Identity) { + colorSpace.mTransfer = Some(VideoTransferCharacteristics::Iec61966_2_1); + } else if (aSurface->GetYUVColorSpace() == gfx::YUVColorSpace::BT709) { + colorSpace.mTransfer = Some(VideoTransferCharacteristics::Bt709); + } else if (aSurface->GetYUVColorSpace() == gfx::YUVColorSpace::BT2020) { + colorSpace.mTransfer = Some(VideoTransferCharacteristics::Pq); + } else { + LOGW( + "Warning: Falling back to BT709 when attempting to determine the " + "transfer function of a MacIOSurface"); + colorSpace.mTransfer = Some(VideoTransferCharacteristics::Bt709); + } + + return colorSpace; +} +#endif +#ifdef MOZ_WAYLAND +// TODO: Set DMABufSurface::IsFullRange() to const so aSurface can be const. +static VideoColorSpaceInternal GuessColorSpace(DMABufSurface* aSurface) { + if (!aSurface) { + return {}; + } + VideoColorSpaceInternal colorSpace; + colorSpace.mFullRange = Some(aSurface->IsFullRange()); + if (Maybe<dom::VideoMatrixCoefficients> m = + ToMatrixCoefficients(aSurface->GetYUVColorSpace())) { + colorSpace.mMatrix = Some(*m); + } + // No other color space information. + return colorSpace; +} +#endif + +static VideoColorSpaceInternal GuessColorSpace(layers::Image* aImage) { + if (aImage) { + if (layers::PlanarYCbCrImage* image = aImage->AsPlanarYCbCrImage()) { + return GuessColorSpace(image->GetData()); + } + if (layers::NVImage* image = aImage->AsNVImage()) { + return GuessColorSpace(image->GetData()); + } + if (layers::GPUVideoImage* image = aImage->AsGPUVideoImage()) { + VideoColorSpaceInternal colorSpace; + colorSpace.mFullRange = + Some(image->GetColorRange() != gfx::ColorRange::LIMITED); + colorSpace.mMatrix = ToMatrixCoefficients(image->GetYUVColorSpace()); + colorSpace.mPrimaries = ToPrimaries(image->GetColorPrimaries()); + colorSpace.mTransfer = + ToTransferCharacteristics(image->GetTransferFunction()); + // In some circumstances, e.g. on Linux software decoding when using + // VPXDecoder and RDD, the primaries aren't set correctly. Make a good + // guess based on the other params. Fixing this is tracked in + // https://bugzilla.mozilla.org/show_bug.cgi?id=1869825 + if (!colorSpace.mPrimaries) { + if (colorSpace.mMatrix.isSome()) { + switch (colorSpace.mMatrix.value()) { + case VideoMatrixCoefficients::Rgb: + case VideoMatrixCoefficients::Bt709: + colorSpace.mPrimaries = Some(VideoColorPrimaries::Bt709); + break; + case VideoMatrixCoefficients::Bt470bg: + case VideoMatrixCoefficients::Smpte170m: + colorSpace.mPrimaries = Some(VideoColorPrimaries::Bt470bg); + break; + case VideoMatrixCoefficients::Bt2020_ncl: + colorSpace.mPrimaries = Some(VideoColorPrimaries::Bt2020); + break; + case VideoMatrixCoefficients::EndGuard_: + MOZ_ASSERT_UNREACHABLE("bad enum value"); + break; + }; + } + } + return colorSpace; + } +#ifdef XP_MACOSX + // TODO: Make sure VideoFrame can interpret its internal data in different + // formats. + if (layers::MacIOSurfaceImage* image = aImage->AsMacIOSurfaceImage()) { + return GuessColorSpace(image->GetSurface()); + } +#endif +#ifdef MOZ_WAYLAND + // TODO: Make sure VideoFrame can interpret its internal data in different + // formats. + if (layers::DMABUFSurfaceImage* image = aImage->AsDMABUFSurfaceImage()) { + return GuessColorSpace(image->GetSurface()); + } +#endif + } + LOGW("Failed to get color space from layers::Image"); + return {}; +} + +static Result<gfx::IntSize, nsresult> AdjustDisplaySize( + const uint32_t aDisplayAspectWidth, const uint32_t aDisplayAspectHeight, + const gfx::IntSize& aDisplaySize) { + if (aDisplayAspectHeight == 0) { + return Err(NS_ERROR_ILLEGAL_VALUE); + } + + const double aspectRatio = + static_cast<double>(aDisplayAspectWidth) / aDisplayAspectHeight; + + double w = aDisplaySize.width; + double h = aDisplaySize.height; + + if (aspectRatio >= w / h) { + // Adjust w to match the aspect ratio + w = aspectRatio * h; + } else { + // Adjust h to match the aspect ratio + h = w / aspectRatio; + } + + w = std::round(w); + h = std::round(h); + constexpr double MAX = static_cast<double>( + std::numeric_limits<decltype(gfx::IntSize::width)>::max()); + if (w > MAX || h > MAX || w < 1.0 || h < 1.0) { + return Err(NS_ERROR_ILLEGAL_VALUE); + } + return gfx::IntSize(static_cast<decltype(gfx::IntSize::width)>(w), + static_cast<decltype(gfx::IntSize::height)>(h)); +} + +// https://w3c.github.io/webcodecs/#create-a-videoframe +static RefPtr<VideoFrame> CreateVideoFrame( + nsIGlobalObject* aGlobalObject, const VideoData* aData, int64_t aTimestamp, + uint64_t aDuration, const Maybe<uint32_t> aDisplayAspectWidth, + const Maybe<uint32_t> aDisplayAspectHeight, + const VideoColorSpaceInternal& aColorSpace) { + MOZ_ASSERT(aGlobalObject); + MOZ_ASSERT(aData); + MOZ_ASSERT((!!aDisplayAspectWidth) == (!!aDisplayAspectHeight)); + + Maybe<VideoPixelFormat> format = GuessPixelFormat(aData->mImage.get()); + gfx::IntSize displaySize = aData->mDisplay; + if (aDisplayAspectWidth && aDisplayAspectHeight) { + auto r = AdjustDisplaySize(*aDisplayAspectWidth, *aDisplayAspectHeight, + displaySize); + if (r.isOk()) { + displaySize = r.unwrap(); + } + } + + return MakeRefPtr<VideoFrame>( + aGlobalObject, aData->mImage, format, aData->mImage->GetSize(), + aData->mImage->GetPictureRect(), displaySize, Some(aDuration), aTimestamp, + aColorSpace.ToColorSpaceInit()); +} + +/* static */ +bool VideoDecoderTraits::IsSupported( + const VideoDecoderConfigInternal& aConfig) { + return CanDecode(aConfig); +} + +/* static */ +Result<UniquePtr<TrackInfo>, nsresult> VideoDecoderTraits::CreateTrackInfo( + const VideoDecoderConfigInternal& aConfig) { + LOG("Create a VideoInfo from %s config", + NS_ConvertUTF16toUTF8(aConfig.ToString()).get()); + + nsTArray<UniquePtr<TrackInfo>> tracks = GetTracksInfo(aConfig); + if (tracks.Length() != 1 || tracks[0]->GetType() != TrackInfo::kVideoTrack) { + LOGE("Failed to get TrackInfo"); + return Err(NS_ERROR_INVALID_ARG); + } + + UniquePtr<TrackInfo> track(std::move(tracks[0])); + VideoInfo* vi = track->GetAsVideoInfo(); + if (!vi) { + LOGE("Failed to get VideoInfo"); + return Err(NS_ERROR_INVALID_ARG); + } + + constexpr uint32_t MAX = static_cast<uint32_t>( + std::numeric_limits<decltype(gfx::IntSize::width)>::max()); + if (aConfig.mCodedHeight.isSome()) { + if (aConfig.mCodedHeight.value() > MAX) { + LOGE("codedHeight overflows"); + return Err(NS_ERROR_INVALID_ARG); + } + vi->mImage.height = static_cast<decltype(gfx::IntSize::height)>( + aConfig.mCodedHeight.value()); + } + if (aConfig.mCodedWidth.isSome()) { + if (aConfig.mCodedWidth.value() > MAX) { + LOGE("codedWidth overflows"); + return Err(NS_ERROR_INVALID_ARG); + } + vi->mImage.width = + static_cast<decltype(gfx::IntSize::width)>(aConfig.mCodedWidth.value()); + } + + if (aConfig.mDisplayAspectHeight.isSome()) { + if (aConfig.mDisplayAspectHeight.value() > MAX) { + LOGE("displayAspectHeight overflows"); + return Err(NS_ERROR_INVALID_ARG); + } + vi->mDisplay.height = static_cast<decltype(gfx::IntSize::height)>( + aConfig.mDisplayAspectHeight.value()); + } + if (aConfig.mDisplayAspectWidth.isSome()) { + if (aConfig.mDisplayAspectWidth.value() > MAX) { + LOGE("displayAspectWidth overflows"); + return Err(NS_ERROR_INVALID_ARG); + } + vi->mDisplay.width = static_cast<decltype(gfx::IntSize::width)>( + aConfig.mDisplayAspectWidth.value()); + } + + if (aConfig.mColorSpace.isSome()) { + const VideoColorSpaceInternal& colorSpace(aConfig.mColorSpace.value()); + if (colorSpace.mFullRange.isSome()) { + vi->mColorRange = ToColorRange(colorSpace.mFullRange.value()); + } + if (colorSpace.mMatrix.isSome()) { + vi->mColorSpace.emplace(ToColorSpace(colorSpace.mMatrix.value())); + } + // Some decoders get their primaries and transfer function from the codec + // string, and it's already set here. This is the case for VP9 decoders. + if (colorSpace.mPrimaries.isSome()) { + auto primaries = ToPrimaries(colorSpace.mPrimaries.value()); + if (vi->mColorPrimaries.isSome()) { + if (vi->mColorPrimaries.value() != primaries) { + LOG("Conflict between decoder config and codec string, keeping codec " + "string primaries of %d", + static_cast<int>(primaries)); + } + } else { + vi->mColorPrimaries.emplace(primaries); + } + } + if (colorSpace.mTransfer.isSome()) { + auto primaries = ToTransferFunction(colorSpace.mTransfer.value()); + if (vi->mTransferFunction.isSome()) { + if (vi->mTransferFunction.value() != primaries) { + LOG("Conflict between decoder config and codec string, keeping codec " + "string transfer function of %d", + static_cast<int>(vi->mTransferFunction.value())); + } + } else { + vi->mTransferFunction.emplace( + ToTransferFunction(colorSpace.mTransfer.value())); + } + } + } + + if (aConfig.mDescription.isSome()) { + RefPtr<MediaByteBuffer> buf; + buf = aConfig.mDescription.value(); + if (buf) { + LOG("The given config has %zu bytes of description data", buf->Length()); + if (vi->mExtraData) { + LOGW("The default extra data is overwritten"); + } + vi->mExtraData = buf; + } + + // TODO: Make this utility and replace the similar one in MP4Demuxer.cpp. + if (vi->mExtraData && !vi->mExtraData->IsEmpty() && + IsH264CodecString(aConfig.mCodec)) { + SPSData spsdata; + if (H264::DecodeSPSFromExtraData(vi->mExtraData.get(), spsdata) && + spsdata.pic_width > 0 && spsdata.pic_height > 0 && + H264::EnsureSPSIsSane(spsdata)) { + LOG("H264 sps data - pic size: %d x %d, display size: %d x %d", + spsdata.pic_width, spsdata.pic_height, spsdata.display_width, + spsdata.display_height); + + if (spsdata.pic_width > MAX || spsdata.pic_height > MAX || + spsdata.display_width > MAX || spsdata.display_height > MAX) { + LOGE("H264 width or height in sps data overflows"); + return Err(NS_ERROR_INVALID_ARG); + } + + vi->mImage.width = + static_cast<decltype(gfx::IntSize::width)>(spsdata.pic_width); + vi->mImage.height = + static_cast<decltype(gfx::IntSize::height)>(spsdata.pic_height); + vi->mDisplay.width = + static_cast<decltype(gfx::IntSize::width)>(spsdata.display_width); + vi->mDisplay.height = + static_cast<decltype(gfx::IntSize::height)>(spsdata.display_height); + } + } + } else { + vi->mExtraData = new MediaByteBuffer(); + } + + LOG("Created a VideoInfo for decoder - %s", + NS_ConvertUTF16toUTF8(vi->ToString()).get()); + + return track; +} + +// https://w3c.github.io/webcodecs/#valid-videodecoderconfig +/* static */ +bool VideoDecoderTraits::Validate(const VideoDecoderConfig& aConfig, + nsCString& aErrorMessage) { + Maybe<nsString> codec = ParseCodecString(aConfig.mCodec); + if (!codec || codec->IsEmpty()) { + LOGE("Invalid codec string"); + return false; + } + + if (aConfig.mCodedWidth.WasPassed() != aConfig.mCodedHeight.WasPassed()) { + LOGE("Missing coded %s", + aConfig.mCodedWidth.WasPassed() ? "height" : "width"); + return false; + } + if (aConfig.mCodedWidth.WasPassed() && + (aConfig.mCodedWidth.Value() == 0 || aConfig.mCodedHeight.Value() == 0)) { + LOGE("codedWidth and/or codedHeight can't be zero"); + return false; + } + + if (aConfig.mDisplayAspectWidth.WasPassed() != + aConfig.mDisplayAspectHeight.WasPassed()) { + LOGE("Missing display aspect %s", + aConfig.mDisplayAspectWidth.WasPassed() ? "height" : "width"); + return false; + } + if (aConfig.mDisplayAspectWidth.WasPassed() && + (aConfig.mDisplayAspectWidth.Value() == 0 || + aConfig.mDisplayAspectHeight.Value() == 0)) { + LOGE("display aspect width and height cannot be zero"); + return false; + } + + return true; +} + +/* static */ +UniquePtr<VideoDecoderConfigInternal> VideoDecoderTraits::CreateConfigInternal( + const VideoDecoderConfig& aConfig) { + return VideoDecoderConfigInternal::Create(aConfig); +} + +/* static */ +bool VideoDecoderTraits::IsKeyChunk(const EncodedVideoChunk& aInput) { + return aInput.Type() == EncodedVideoChunkType::Key; +} + +/* static */ +UniquePtr<EncodedVideoChunkData> VideoDecoderTraits::CreateInputInternal( + const EncodedVideoChunk& aInput) { + return aInput.Clone(); +} + +/* + * Below are VideoDecoder implementation + */ + +VideoDecoder::VideoDecoder(nsIGlobalObject* aParent, + RefPtr<WebCodecsErrorCallback>&& aErrorCallback, + RefPtr<VideoFrameOutputCallback>&& aOutputCallback) + : DecoderTemplate(aParent, std::move(aErrorCallback), + std::move(aOutputCallback)) { + MOZ_ASSERT(mErrorCallback); + MOZ_ASSERT(mOutputCallback); + LOG("VideoDecoder %p ctor", this); +} + +VideoDecoder::~VideoDecoder() { + LOG("VideoDecoder %p dtor", this); +} + +JSObject* VideoDecoder::WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) { + AssertIsOnOwningThread(); + + return VideoDecoder_Binding::Wrap(aCx, this, aGivenProto); +} + +// https://w3c.github.io/webcodecs/#dom-videodecoder-videodecoder +/* static */ +already_AddRefed<VideoDecoder> VideoDecoder::Constructor( + const GlobalObject& aGlobal, const VideoDecoderInit& aInit, + ErrorResult& aRv) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + return MakeAndAddRef<VideoDecoder>( + global.get(), RefPtr<WebCodecsErrorCallback>(aInit.mError), + RefPtr<VideoFrameOutputCallback>(aInit.mOutput)); +} + +// https://w3c.github.io/webcodecs/#dom-videodecoder-isconfigsupported +/* static */ +already_AddRefed<Promise> VideoDecoder::IsConfigSupported( + const GlobalObject& aGlobal, const VideoDecoderConfig& aConfig, + ErrorResult& aRv) { + LOG("VideoDecoder::IsConfigSupported, config: %s", + NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); + + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + RefPtr<Promise> p = Promise::Create(global.get(), aRv); + if (NS_WARN_IF(aRv.Failed())) { + return p.forget(); + } + + nsCString errorMessage; + if (!VideoDecoderTraits::Validate(aConfig, errorMessage)) { + p->MaybeRejectWithTypeError(nsPrintfCString( + "VideoDecoderConfig is invalid: %s", errorMessage.get())); + return p.forget(); + } + + // TODO: Move the following works to another thread to unblock the current + // thread, as what spec suggests. + + RootedDictionary<VideoDecoderConfig> config(aGlobal.Context()); + auto r = CloneConfiguration(config, aGlobal.Context(), aConfig); + if (r.isErr()) { + nsresult e = r.unwrapErr(); + LOGE("Failed to clone VideoDecoderConfig. Error: 0x%08" PRIx32, + static_cast<uint32_t>(e)); + p->MaybeRejectWithTypeError("Failed to clone VideoDecoderConfig"); + aRv.Throw(e); + return p.forget(); + } + + bool canDecode = CanDecode(config); + RootedDictionary<VideoDecoderSupport> s(aGlobal.Context()); + s.mConfig.Construct(std::move(config)); + s.mSupported.Construct(canDecode); + + p->MaybeResolve(s); + return p.forget(); +} + +already_AddRefed<MediaRawData> VideoDecoder::InputDataToMediaRawData( + UniquePtr<EncodedVideoChunkData>&& aData, TrackInfo& aInfo, + const VideoDecoderConfigInternal& aConfig) { + AssertIsOnOwningThread(); + MOZ_ASSERT(aInfo.GetAsVideoInfo()); + + if (!aData) { + LOGE("No data for conversion"); + return nullptr; + } + + RefPtr<MediaRawData> sample = aData->TakeData(); + if (!sample) { + LOGE("Take no data for conversion"); + return nullptr; + } + + // aExtraData is either provided by Configure() or a default one created for + // the decoder creation. If it's created for decoder creation only, we don't + // set it to sample. + if (aConfig.mDescription && aInfo.GetAsVideoInfo()->mExtraData) { + sample->mExtraData = aInfo.GetAsVideoInfo()->mExtraData; + } + + LOGV( + "EncodedVideoChunkData %p converted to %zu-byte MediaRawData - time: " + "%" PRIi64 "us, timecode: %" PRIi64 "us, duration: %" PRIi64 + "us, key-frame: %s, has extra data: %s", + aData.get(), sample->Size(), sample->mTime.ToMicroseconds(), + sample->mTimecode.ToMicroseconds(), sample->mDuration.ToMicroseconds(), + sample->mKeyframe ? "yes" : "no", sample->mExtraData ? "yes" : "no"); + + return sample.forget(); +} + +nsTArray<RefPtr<VideoFrame>> VideoDecoder::DecodedDataToOutputType( + nsIGlobalObject* aGlobalObject, const nsTArray<RefPtr<MediaData>>&& aData, + VideoDecoderConfigInternal& aConfig) { + AssertIsOnOwningThread(); + + nsTArray<RefPtr<VideoFrame>> frames; + for (const RefPtr<MediaData>& data : aData) { + MOZ_RELEASE_ASSERT(data->mType == MediaData::Type::VIDEO_DATA); + RefPtr<const VideoData> d(data->As<const VideoData>()); + VideoColorSpaceInternal colorSpace; + // Determine which color space to use: prefer the color space as configured + // at the decoder level, if it has one, otherwise look at the underlying + // image and make a guess. + if (aConfig.mColorSpace.isSome() && + aConfig.mColorSpace->mPrimaries.isSome() && + aConfig.mColorSpace->mTransfer.isSome() && + aConfig.mColorSpace->mMatrix.isSome()) { + colorSpace = aConfig.mColorSpace.value(); + } else { + colorSpace = GuessColorSpace(d->mImage.get()); + } + frames.AppendElement(CreateVideoFrame( + aGlobalObject, d.get(), d->mTime.ToMicroseconds(), + static_cast<uint64_t>(d->mDuration.ToMicroseconds()), + aConfig.mDisplayAspectWidth, aConfig.mDisplayAspectHeight, colorSpace)); + } + return frames; +} + +#undef LOG +#undef LOGW +#undef LOGE +#undef LOGV +#undef LOG_INTERNAL + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/VideoDecoder.h b/dom/media/webcodecs/VideoDecoder.h new file mode 100644 index 0000000000..f4b3fe10ed --- /dev/null +++ b/dom/media/webcodecs/VideoDecoder.h @@ -0,0 +1,79 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_VideoDecoder_h +#define mozilla_dom_VideoDecoder_h + +#include "js/TypeDecls.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/RefPtr.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/DecoderTemplate.h" +#include "mozilla/dom/DecoderTypes.h" +#include "mozilla/dom/VideoFrame.h" +#include "nsCycleCollectionParticipant.h" + +class nsIGlobalObject; + +namespace mozilla { + +namespace dom { + +class EncodedVideoChunk; +class EncodedVideoChunkData; +class EventHandlerNonNull; +class GlobalObject; +class Promise; +class VideoFrameOutputCallback; +class WebCodecsErrorCallback; +struct VideoDecoderConfig; +struct VideoDecoderInit; + +} // namespace dom + +} // namespace mozilla + +namespace mozilla::dom { + +class VideoDecoder final : public DecoderTemplate<VideoDecoderTraits> { + public: + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(VideoDecoder, DOMEventTargetHelper) + + public: + VideoDecoder(nsIGlobalObject* aParent, + RefPtr<WebCodecsErrorCallback>&& aErrorCallback, + RefPtr<VideoFrameOutputCallback>&& aOutputCallback); + + protected: + ~VideoDecoder(); + + public: + JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override; + + static already_AddRefed<VideoDecoder> Constructor( + const GlobalObject& aGlobal, const VideoDecoderInit& aInit, + ErrorResult& aRv); + + static already_AddRefed<Promise> IsConfigSupported( + const GlobalObject& aGlobal, const VideoDecoderConfig& aConfig, + ErrorResult& aRv); + + protected: + virtual already_AddRefed<MediaRawData> InputDataToMediaRawData( + UniquePtr<EncodedVideoChunkData>&& aData, TrackInfo& aInfo, + const VideoDecoderConfigInternal& aConfig) override; + + virtual nsTArray<RefPtr<VideoFrame>> DecodedDataToOutputType( + nsIGlobalObject* aGlobalObject, const nsTArray<RefPtr<MediaData>>&& aData, + VideoDecoderConfigInternal& aConfig) override; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_VideoDecoder_h diff --git a/dom/media/webcodecs/VideoEncoder.cpp b/dom/media/webcodecs/VideoEncoder.cpp new file mode 100644 index 0000000000..0e71417cb0 --- /dev/null +++ b/dom/media/webcodecs/VideoEncoder.cpp @@ -0,0 +1,624 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/VideoEncoder.h" +#include "mozilla/dom/VideoEncoderBinding.h" +#include "mozilla/dom/VideoColorSpaceBinding.h" +#include "mozilla/dom/VideoColorSpace.h" +#include "mozilla/dom/VideoFrame.h" + +#include "EncoderTraits.h" +#include "ImageContainer.h" +#include "VideoUtils.h" +#include "mozilla/Assertions.h" +#include "mozilla/Logging.h" +#include "mozilla/Maybe.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/EncodedVideoChunk.h" +#include "mozilla/dom/EncodedVideoChunkBinding.h" +#include "mozilla/dom/ImageUtils.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/VideoColorSpaceBinding.h" +#include "mozilla/dom/VideoFrameBinding.h" +#include "mozilla/dom/WebCodecsUtils.h" +#include "nsPrintfCString.h" +#include "nsReadableUtils.h" + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla::dom { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOG +# undef LOG +#endif // LOG +#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +#ifdef LOGW +# undef LOGW +#endif // LOGW +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +#ifdef LOGV +# undef LOGV +#endif // LOGV +#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__) + +NS_IMPL_CYCLE_COLLECTION_INHERITED(VideoEncoder, DOMEventTargetHelper, + mErrorCallback, mOutputCallback) +NS_IMPL_ADDREF_INHERITED(VideoEncoder, DOMEventTargetHelper) +NS_IMPL_RELEASE_INHERITED(VideoEncoder, DOMEventTargetHelper) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(VideoEncoder) +NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper) + +VideoEncoderConfigInternal::VideoEncoderConfigInternal( + const nsAString& aCodec, uint32_t aWidth, uint32_t aHeight, + Maybe<uint32_t>&& aDisplayWidth, Maybe<uint32_t>&& aDisplayHeight, + Maybe<uint32_t>&& aBitrate, Maybe<double>&& aFramerate, + const HardwareAcceleration& aHardwareAcceleration, + const AlphaOption& aAlpha, Maybe<nsString>&& aScalabilityMode, + const VideoEncoderBitrateMode& aBitrateMode, + const LatencyMode& aLatencyMode, Maybe<nsString>&& aContentHint) + : mCodec(aCodec), + mWidth(aWidth), + mHeight(aHeight), + mDisplayWidth(std::move(aDisplayWidth)), + mDisplayHeight(std::move(aDisplayHeight)), + mBitrate(std::move(aBitrate)), + mFramerate(std::move(aFramerate)), + mHardwareAcceleration(aHardwareAcceleration), + mAlpha(aAlpha), + mScalabilityMode(std::move(aScalabilityMode)), + mBitrateMode(aBitrateMode), + mLatencyMode(aLatencyMode), + mContentHint(std::move(aContentHint)) {} + +VideoEncoderConfigInternal::VideoEncoderConfigInternal( + const VideoEncoderConfigInternal& aConfig) + : mCodec(aConfig.mCodec), + mWidth(aConfig.mWidth), + mHeight(aConfig.mHeight), + mDisplayWidth(aConfig.mDisplayWidth), + mDisplayHeight(aConfig.mDisplayHeight), + mBitrate(aConfig.mBitrate), + mFramerate(aConfig.mFramerate), + mHardwareAcceleration(aConfig.mHardwareAcceleration), + mAlpha(aConfig.mAlpha), + mScalabilityMode(aConfig.mScalabilityMode), + mBitrateMode(aConfig.mBitrateMode), + mLatencyMode(aConfig.mLatencyMode), + mContentHint(aConfig.mContentHint), + mAvc(aConfig.mAvc) { +} + +VideoEncoderConfigInternal::VideoEncoderConfigInternal( + const VideoEncoderConfig& aConfig) + : mCodec(aConfig.mCodec), + mWidth(aConfig.mWidth), + mHeight(aConfig.mHeight), + mDisplayWidth(OptionalToMaybe(aConfig.mDisplayWidth)), + mDisplayHeight(OptionalToMaybe(aConfig.mDisplayHeight)), + mBitrate(OptionalToMaybe(aConfig.mBitrate)), + mFramerate(OptionalToMaybe(aConfig.mFramerate)), + mHardwareAcceleration(aConfig.mHardwareAcceleration), + mAlpha(aConfig.mAlpha), + mScalabilityMode(OptionalToMaybe(aConfig.mScalabilityMode)), + mBitrateMode(aConfig.mBitrateMode), + mLatencyMode(aConfig.mLatencyMode), + mContentHint(OptionalToMaybe(aConfig.mContentHint)), + mAvc(OptionalToMaybe(aConfig.mAvc)) { +} + +nsString VideoEncoderConfigInternal::ToString() const { + nsString rv; + + rv.AppendPrintf("Codec: %s, [%" PRIu32 "x%" PRIu32 "],", + NS_ConvertUTF16toUTF8(mCodec).get(), mWidth, mHeight); + if (mDisplayWidth.isSome()) { + rv.AppendPrintf(", display[%" PRIu32 "x%" PRIu32 "]", mDisplayWidth.value(), + mDisplayHeight.value()); + } + if (mBitrate.isSome()) { + rv.AppendPrintf(", %" PRIu32 "kbps", mBitrate.value()); + } + if (mFramerate.isSome()) { + rv.AppendPrintf(", %lfHz", mFramerate.value()); + } + rv.AppendPrintf( + " hw: %s", + HardwareAccelerationValues::GetString(mHardwareAcceleration).data()); + rv.AppendPrintf(", alpha: %s", AlphaOptionValues::GetString(mAlpha).data()); + if (mScalabilityMode.isSome()) { + rv.AppendPrintf(", scalability mode: %s", + NS_ConvertUTF16toUTF8(mScalabilityMode.value()).get()); + } + rv.AppendPrintf( + ", bitrate mode: %s", + VideoEncoderBitrateModeValues::GetString(mBitrateMode).data()); + rv.AppendPrintf(", latency mode: %s", + LatencyModeValues::GetString(mLatencyMode).data()); + if (mContentHint.isSome()) { + rv.AppendPrintf(", content hint: %s", + NS_ConvertUTF16toUTF8(mContentHint.value()).get()); + } + if (mAvc.isSome()) { + rv.AppendPrintf(", avc-specific: %s", + AvcBitstreamFormatValues::GetString(mAvc->mFormat).data()); + } + + return rv; +} + +template <typename T> +bool MaybeAreEqual(const Maybe<T>& aLHS, const Maybe<T> aRHS) { + if (aLHS.isSome() && aRHS.isSome()) { + return aLHS.value() == aRHS.value(); + } + if (aLHS.isNothing() && aRHS.isNothing()) { + return true; + } + return false; +} + +bool VideoEncoderConfigInternal::Equals( + const VideoEncoderConfigInternal& aOther) const { + bool sameCodecSpecific = true; + if ((mAvc.isSome() && aOther.mAvc.isSome() && + mAvc->mFormat != aOther.mAvc->mFormat) || + mAvc.isSome() != aOther.mAvc.isSome()) { + sameCodecSpecific = false; + } + return mCodec.Equals(aOther.mCodec) && mWidth == aOther.mWidth && + mHeight == aOther.mHeight && + MaybeAreEqual(mDisplayWidth, aOther.mDisplayWidth) && + MaybeAreEqual(mDisplayHeight, aOther.mDisplayHeight) && + MaybeAreEqual(mBitrate, aOther.mBitrate) && + MaybeAreEqual(mFramerate, aOther.mFramerate) && + mHardwareAcceleration == aOther.mHardwareAcceleration && + mAlpha == aOther.mAlpha && + MaybeAreEqual(mScalabilityMode, aOther.mScalabilityMode) && + mBitrateMode == aOther.mBitrateMode && + mLatencyMode == aOther.mLatencyMode && + MaybeAreEqual(mContentHint, aOther.mContentHint) && sameCodecSpecific; +} + +bool VideoEncoderConfigInternal::CanReconfigure( + const VideoEncoderConfigInternal& aOther) const { + return mCodec.Equals(aOther.mCodec) && + mHardwareAcceleration == aOther.mHardwareAcceleration; +} + +EncoderConfig VideoEncoderConfigInternal::ToEncoderConfig() const { + MediaDataEncoder::Usage usage; + if (mLatencyMode == LatencyMode::Quality) { + usage = MediaDataEncoder::Usage::Record; + } else { + usage = MediaDataEncoder::Usage::Realtime; + } + MediaDataEncoder::HardwarePreference hwPref = + MediaDataEncoder::HardwarePreference::None; + if (mHardwareAcceleration == + mozilla::dom::HardwareAcceleration::Prefer_hardware) { + hwPref = MediaDataEncoder::HardwarePreference::RequireHardware; + } else if (mHardwareAcceleration == + mozilla::dom::HardwareAcceleration::Prefer_software) { + hwPref = MediaDataEncoder::HardwarePreference::RequireSoftware; + } + CodecType codecType; + auto maybeCodecType = CodecStringToCodecType(mCodec); + if (maybeCodecType.isSome()) { + codecType = maybeCodecType.value(); + } else { + MOZ_CRASH("The string should always contain a valid codec at this point."); + } + Maybe<EncoderConfig::CodecSpecific> specific; + if (codecType == CodecType::H264) { + uint8_t profile, constraints, level; + H264BitStreamFormat format; + if (mAvc) { + format = mAvc->mFormat == AvcBitstreamFormat::Annexb + ? H264BitStreamFormat::ANNEXB + : H264BitStreamFormat::AVC; + } else { + format = H264BitStreamFormat::AVC; + } + if (ExtractH264CodecDetails(mCodec, profile, constraints, level)) { + if (profile == H264_PROFILE_BASE || profile == H264_PROFILE_MAIN || + profile == H264_PROFILE_EXTENDED || profile == H264_PROFILE_HIGH) { + specific.emplace( + H264Specific(static_cast<H264_PROFILE>(profile), static_cast<H264_LEVEL>(level), format)); + } + } + } + // Only for vp9, not vp8 + if (codecType == CodecType::VP9) { + uint8_t profile, level, bitdepth, chromasubsampling; + mozilla::VideoColorSpace colorspace; + DebugOnly<bool> rv = ExtractVPXCodecDetails( + mCodec, profile, level, bitdepth, chromasubsampling, colorspace); +#ifdef DEBUG + if (!rv) { + LOGE("Error extracting VPX codec details, non fatal"); + } +#endif + specific.emplace(VP9Specific()); + } + MediaDataEncoder::ScalabilityMode scalabilityMode; + if (mScalabilityMode) { + if (mScalabilityMode->EqualsLiteral("L1T2")) { + scalabilityMode = MediaDataEncoder::ScalabilityMode::L1T2; + } else if (mScalabilityMode->EqualsLiteral("L1T3")) { + scalabilityMode = MediaDataEncoder::ScalabilityMode::L1T3; + } else { + scalabilityMode = MediaDataEncoder::ScalabilityMode::None; + } + } else { + scalabilityMode = MediaDataEncoder::ScalabilityMode::None; + } + return EncoderConfig( + codecType, {mWidth, mHeight}, usage, ImageBitmapFormat::RGBA32, ImageBitmapFormat::RGBA32, + AssertedCast<uint8_t>(mFramerate.refOr(0.f)), 0, mBitrate.refOr(0), + mBitrateMode == VideoEncoderBitrateMode::Constant + ? MediaDataEncoder::BitrateMode::Constant + : MediaDataEncoder::BitrateMode::Variable, + hwPref, scalabilityMode, specific); +} +already_AddRefed<WebCodecsConfigurationChangeList> +VideoEncoderConfigInternal::Diff( + const VideoEncoderConfigInternal& aOther) const { + auto list = MakeRefPtr<WebCodecsConfigurationChangeList>(); + if (!mCodec.Equals(aOther.mCodec)) { + list->Push(CodecChange{aOther.mCodec}); + } + // Both must always be present, when a `VideoEncoderConfig` is passed to + // `configure`. + if (mWidth != aOther.mWidth || mHeight != aOther.mHeight) { + list->Push(DimensionsChange{gfx::IntSize{aOther.mWidth, aOther.mHeight}}); + } + // Similarly, both must always be present, when a `VideoEncoderConfig` is + // passed to `configure`. + if (!MaybeAreEqual(mDisplayWidth, aOther.mDisplayWidth) || + !MaybeAreEqual(mDisplayHeight, aOther.mDisplayHeight)) { + Maybe<gfx::IntSize> displaySize = + aOther.mDisplayWidth.isSome() + ? Some(gfx::IntSize{aOther.mDisplayWidth.value(), + aOther.mDisplayHeight.value()}) + : Nothing(); + list->Push(DisplayDimensionsChange{displaySize}); + } + if (!MaybeAreEqual(mBitrate, aOther.mBitrate)) { + list->Push(BitrateChange{aOther.mBitrate}); + } + if (!MaybeAreEqual(mFramerate, aOther.mFramerate)) { + list->Push(FramerateChange{aOther.mFramerate}); + } + if (mHardwareAcceleration != aOther.mHardwareAcceleration) { + list->Push(HardwareAccelerationChange{aOther.mHardwareAcceleration}); + } + if (mAlpha != aOther.mAlpha) { + list->Push(AlphaChange{aOther.mAlpha}); + } + if (!MaybeAreEqual(mScalabilityMode, aOther.mScalabilityMode)) { + list->Push(ScalabilityModeChange{aOther.mScalabilityMode}); + } + if (mBitrateMode != aOther.mBitrateMode) { + list->Push(BitrateModeChange{aOther.mBitrateMode}); + } + if (mLatencyMode != aOther.mLatencyMode) { + list->Push(LatencyModeChange{aOther.mLatencyMode}); + } + if (!MaybeAreEqual(mContentHint, aOther.mContentHint)) { + list->Push(ContentHintChange{aOther.mContentHint}); + } + return list.forget(); +} + +/* + * The followings are helpers for VideoEncoder methods + */ +static bool IsEncodeSupportedCodec(const nsAString& aCodec) { + LOG("IsEncodeSupported: %s", NS_ConvertUTF16toUTF8(aCodec).get()); + if (!IsVP9CodecString(aCodec) && !IsVP8CodecString(aCodec) && + !IsH264CodecString(aCodec) && !IsAV1CodecString(aCodec)) { + return false; + } + + // Gecko allows codec string starts with vp9 or av1 but Webcodecs requires to + // starts with av01 and vp09. + // https://www.w3.org/TR/webcodecs-codec-registry/#video-codec-registry + if (StringBeginsWith(aCodec, u"vp9"_ns) || + StringBeginsWith(aCodec, u"av1"_ns)) { + return false; + } + + return true; +} + +// https://w3c.github.io/webcodecs/#check-configuration-support +static bool CanEncode(const RefPtr<VideoEncoderConfigInternal>& aConfig) { + auto parsedCodecString = + ParseCodecString(aConfig->mCodec).valueOr(EmptyString()); + // TODO: Enable WebCodecs on Android (Bug 1840508) + if (IsOnAndroid()) { + return false; + } + if (!IsEncodeSupportedCodec(parsedCodecString)) { + return false; + } + + // TODO (bug 1872879, bug 1872880): Support this on Windows and Mac. + if (aConfig->mScalabilityMode.isSome()) { + // We only support L1T2 and L1T3 ScalabilityMode in VP8 and VP9 encoders on + // Linux. + bool supported = IsOnLinux() && (IsVP8CodecString(parsedCodecString) || + IsVP9CodecString(parsedCodecString)) + ? aConfig->mScalabilityMode->EqualsLiteral("L1T2") || + aConfig->mScalabilityMode->EqualsLiteral("L1T3") + : false; + + if (!supported) { + LOGE("Scalability mode %s not supported for codec: %s", + NS_ConvertUTF16toUTF8(aConfig->mScalabilityMode.value()).get(), + NS_ConvertUTF16toUTF8(parsedCodecString).get()); + return false; + } + } + + return EncoderSupport::Supports(aConfig); +} + +static Result<Ok, nsresult> CloneConfiguration( + VideoEncoderConfig& aDest, JSContext* aCx, + const VideoEncoderConfig& aConfig) { + nsCString errorMessage; + MOZ_ASSERT(VideoEncoderTraits::Validate(aConfig, errorMessage)); + + aDest.mCodec = aConfig.mCodec; + aDest.mWidth = aConfig.mWidth; + aDest.mHeight = aConfig.mHeight; + aDest.mAlpha = aConfig.mAlpha; + if (aConfig.mBitrate.WasPassed()) { + aDest.mBitrate.Construct(aConfig.mBitrate.Value()); + } + aDest.mBitrateMode = aConfig.mBitrateMode; + if (aConfig.mContentHint.WasPassed()) { + aDest.mContentHint.Construct(aConfig.mContentHint.Value()); + } + if (aConfig.mDisplayWidth.WasPassed()) { + aDest.mDisplayWidth.Construct(aConfig.mDisplayWidth.Value()); + } + if (aConfig.mDisplayHeight.WasPassed()) { + aDest.mDisplayHeight.Construct(aConfig.mDisplayHeight.Value()); + } + if (aConfig.mFramerate.WasPassed()) { + aDest.mFramerate.Construct(aConfig.mFramerate.Value()); + } + aDest.mHardwareAcceleration = aConfig.mHardwareAcceleration; + aDest.mLatencyMode = aConfig.mLatencyMode; + if (aConfig.mScalabilityMode.WasPassed()) { + aDest.mScalabilityMode.Construct(aConfig.mScalabilityMode.Value()); + } + + // AVC specific + if (aConfig.mAvc.WasPassed()) { + aDest.mAvc.Construct(aConfig.mAvc.Value()); + } + + return Ok(); +} + +/* static */ +bool VideoEncoderTraits::IsSupported( + const VideoEncoderConfigInternal& aConfig) { + return CanEncode(MakeRefPtr<VideoEncoderConfigInternal>(aConfig)); +} + +// https://w3c.github.io/webcodecs/#valid-videoencoderconfig +/* static */ +bool VideoEncoderTraits::Validate(const VideoEncoderConfig& aConfig, + nsCString& aErrorMessage) { + Maybe<nsString> codec = ParseCodecString(aConfig.mCodec); + // 1. + if (!codec || codec->IsEmpty()) { + LOGE("Invalid VideoEncoderConfig: invalid codec string"); + return false; + } + + // 2. + if (aConfig.mWidth == 0 || aConfig.mHeight == 0) { + LOGE("Invalid VideoEncoderConfig: %s equal to 0", + aConfig.mWidth == 0 ? "width" : "height"); + return false; + } + + // 3. + if ((aConfig.mDisplayWidth.WasPassed() && + aConfig.mDisplayWidth.Value() == 0)) { + LOGE("Invalid VideoEncoderConfig: displayWidth equal to 0"); + return false; + } + if ((aConfig.mDisplayHeight.WasPassed() && + aConfig.mDisplayHeight.Value() == 0)) { + LOGE("Invalid VideoEncoderConfig: displayHeight equal to 0"); + return false; + } + + return true; +} + +/* static */ +RefPtr<VideoEncoderConfigInternal> VideoEncoderTraits::CreateConfigInternal( + const VideoEncoderConfig& aConfig) { + return MakeRefPtr<VideoEncoderConfigInternal>(aConfig); +} + +/* static */ +RefPtr<mozilla::VideoData> VideoEncoderTraits::CreateInputInternal( + const dom::VideoFrame& aInput, + const dom::VideoEncoderEncodeOptions& aOptions) { + media::TimeUnit duration = + aInput.GetDuration().IsNull() + ? media::TimeUnit::Zero() + : media::TimeUnit::FromMicroseconds( + AssertedCast<int64_t>(aInput.GetDuration().Value())); + media::TimeUnit pts = media::TimeUnit::FromMicroseconds(aInput.Timestamp()); + return VideoData::CreateFromImage( + gfx::IntSize{aInput.DisplayWidth(), aInput.DisplayHeight()}, + 0 /* bytestream offset */, pts, duration, aInput.GetImage(), + aOptions.mKeyFrame, pts); +} + +/* + * Below are VideoEncoder implementation + */ + +VideoEncoder::VideoEncoder( + nsIGlobalObject* aParent, RefPtr<WebCodecsErrorCallback>&& aErrorCallback, + RefPtr<EncodedVideoChunkOutputCallback>&& aOutputCallback) + : EncoderTemplate(aParent, std::move(aErrorCallback), + std::move(aOutputCallback)) { + MOZ_ASSERT(mErrorCallback); + MOZ_ASSERT(mOutputCallback); + LOG("VideoEncoder %p ctor", this); +} + +VideoEncoder::~VideoEncoder() { + LOG("VideoEncoder %p dtor", this); + Unused << ResetInternal(NS_ERROR_DOM_ABORT_ERR); +} + +JSObject* VideoEncoder::WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) { + AssertIsOnOwningThread(); + + return VideoEncoder_Binding::Wrap(aCx, this, aGivenProto); +} + +// https://w3c.github.io/webcodecs/#dom-videoencoder-videoencoder +/* static */ +already_AddRefed<VideoEncoder> VideoEncoder::Constructor( + const GlobalObject& aGlobal, const VideoEncoderInit& aInit, + ErrorResult& aRv) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + return MakeAndAddRef<VideoEncoder>( + global.get(), RefPtr<WebCodecsErrorCallback>(aInit.mError), + RefPtr<EncodedVideoChunkOutputCallback>(aInit.mOutput)); +} + +// https://w3c.github.io/webcodecs/#dom-videoencoder-isconfigsupported +/* static */ +already_AddRefed<Promise> VideoEncoder::IsConfigSupported( + const GlobalObject& aGlobal, const VideoEncoderConfig& aConfig, + ErrorResult& aRv) { + LOG("VideoEncoder::IsConfigSupported, config: %s", + NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); + + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + RefPtr<Promise> p = Promise::Create(global.get(), aRv); + if (NS_WARN_IF(aRv.Failed())) { + return p.forget(); + } + + nsCString errorMessage; + if (!VideoEncoderTraits::Validate(aConfig, errorMessage)) { + p->MaybeRejectWithTypeError(nsPrintfCString( + "IsConfigSupported: config is invalid: %s", errorMessage.get())); + return p.forget(); + } + + // TODO: Move the following works to another thread to unblock the current + // thread, as what spec suggests. + + VideoEncoderConfig config; + auto r = CloneConfiguration(config, aGlobal.Context(), aConfig); + if (r.isErr()) { + nsresult e = r.unwrapErr(); + LOGE("Failed to clone VideoEncoderConfig. Error: 0x%08" PRIx32, + static_cast<uint32_t>(e)); + p->MaybeRejectWithTypeError("Failed to clone VideoEncoderConfig"); + aRv.Throw(e); + return p.forget(); + } + + bool canEncode = CanEncode(MakeRefPtr<VideoEncoderConfigInternal>(config)); + VideoEncoderSupport s; + s.mConfig.Construct(std::move(config)); + s.mSupported.Construct(canEncode); + + p->MaybeResolve(s); + return p.forget(); +} + +RefPtr<EncodedVideoChunk> VideoEncoder::EncodedDataToOutputType( + nsIGlobalObject* aGlobalObject, RefPtr<MediaRawData>& aData) { + AssertIsOnOwningThread(); + + MOZ_RELEASE_ASSERT(aData->mType == MediaData::Type::RAW_DATA); + // Package into an EncodedVideoChunk + auto buffer = + MakeRefPtr<MediaAlignedByteBuffer>(aData->Data(), aData->Size()); + auto encodedVideoChunk = MakeRefPtr<EncodedVideoChunk>( + aGlobalObject, buffer.forget(), + aData->mKeyframe ? EncodedVideoChunkType::Key + : EncodedVideoChunkType::Delta, + aData->mTime.ToMicroseconds(), + aData->mDuration.IsZero() ? Nothing() + : Some(aData->mDuration.ToMicroseconds())); + return encodedVideoChunk; +} + +VideoDecoderConfigInternal VideoEncoder::EncoderConfigToDecoderConfig( + nsIGlobalObject* aGlobal, const RefPtr<MediaRawData>& aRawData, + const VideoEncoderConfigInternal& mOutputConfig) const { + // Colorspace is mandatory when outputing a decoder config after encode + VideoColorSpaceInternal init; + init.mFullRange.emplace(false); + init.mMatrix.emplace(VideoMatrixCoefficients::Bt709); + init.mPrimaries.emplace(VideoColorPrimaries::Bt709); + init.mTransfer.emplace(VideoTransferCharacteristics::Bt709); + + return VideoDecoderConfigInternal( + mOutputConfig.mCodec, /* aCodec */ + Some(mOutputConfig.mHeight), /* aCodedHeight */ + Some(mOutputConfig.mWidth), /* aCodedWidth */ + Some(init), /* aColorSpace */ + aRawData->mExtraData && !aRawData->mExtraData->IsEmpty() + ? Some(aRawData->mExtraData) + : Nothing(), /* aDescription*/ + Maybe<uint32_t>(mOutputConfig.mDisplayHeight), /* aDisplayAspectHeight*/ + Maybe<uint32_t>(mOutputConfig.mDisplayWidth), /* aDisplayAspectWidth */ + mOutputConfig.mHardwareAcceleration, /* aHardwareAcceleration */ + Nothing() /* aOptimizeForLatency */ + ); +} + +#undef LOG +#undef LOGW +#undef LOGE +#undef LOGV +#undef LOG_INTERNAL + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/VideoEncoder.h b/dom/media/webcodecs/VideoEncoder.h new file mode 100644 index 0000000000..9251b5023a --- /dev/null +++ b/dom/media/webcodecs/VideoEncoder.h @@ -0,0 +1,78 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_VideoEncoder_h +#define mozilla_dom_VideoEncoder_h + +#include "js/TypeDecls.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/RefPtr.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/EncoderTemplate.h" +#include "mozilla/dom/EncoderTypes.h" +#include "mozilla/dom/VideoFrame.h" +#include "nsCycleCollectionParticipant.h" + +class nsIGlobalObject; + +namespace mozilla { + +namespace dom { + +class EncodedVideoChunk; +class EncodedVideoChunkData; +class EventHandlerNonNull; +class GlobalObject; +class Promise; +class VideoFrameOutputCallback; +class WebCodecsErrorCallback; +struct VideoEncoderConfig; +struct VideoEncoderInit; + +} // namespace dom + +} // namespace mozilla + +namespace mozilla::dom { + +class VideoEncoder final : public EncoderTemplate<VideoEncoderTraits> { + public: + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(VideoEncoder, DOMEventTargetHelper) + + public: + VideoEncoder(nsIGlobalObject* aParent, + RefPtr<WebCodecsErrorCallback>&& aErrorCallback, + RefPtr<EncodedVideoChunkOutputCallback>&& aOutputCallback); + + protected: + ~VideoEncoder(); + + public: + JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override; + + static already_AddRefed<VideoEncoder> Constructor( + const GlobalObject& aGlobal, const VideoEncoderInit& aInit, + ErrorResult& aRv); + + static already_AddRefed<Promise> IsConfigSupported( + const GlobalObject& aGlobal, const VideoEncoderConfig& aConfig, + ErrorResult& aRv); + + protected: + virtual RefPtr<EncodedVideoChunk> EncodedDataToOutputType( + nsIGlobalObject* aGlobal, RefPtr<MediaRawData>& aData) override; + + virtual VideoDecoderConfigInternal EncoderConfigToDecoderConfig( + nsIGlobalObject* aGlobal /* TODO: delete */, + const RefPtr<MediaRawData>& aRawData, + const VideoEncoderConfigInternal& mOutputConfig) const override; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_VideoEncoder_h diff --git a/dom/media/webcodecs/VideoFrame.cpp b/dom/media/webcodecs/VideoFrame.cpp new file mode 100644 index 0000000000..602bc95c29 --- /dev/null +++ b/dom/media/webcodecs/VideoFrame.cpp @@ -0,0 +1,2417 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/VideoFrame.h" +#include "mozilla/dom/VideoFrameBinding.h" + +#include <math.h> +#include <limits> +#include <utility> + +#include "ImageContainer.h" +#include "VideoColorSpace.h" +#include "WebCodecsUtils.h" +#include "js/StructuredClone.h" +#include "mozilla/Maybe.h" +#include "mozilla/ResultVariant.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/Try.h" + +#include "mozilla/UniquePtr.h" +#include "mozilla/dom/CanvasUtils.h" +#include "mozilla/dom/DOMRect.h" +#include "mozilla/dom/HTMLCanvasElement.h" +#include "mozilla/dom/HTMLImageElement.h" +#include "mozilla/dom/HTMLVideoElement.h" +#include "mozilla/dom/ImageBitmap.h" +#include "mozilla/dom/ImageUtils.h" +#include "mozilla/dom/OffscreenCanvas.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/SVGImageElement.h" +#include "mozilla/dom/StructuredCloneHolder.h" +#include "mozilla/dom/StructuredCloneTags.h" +#include "mozilla/dom/UnionTypes.h" +#include "mozilla/gfx/2D.h" +#include "mozilla/gfx/Swizzle.h" +#include "mozilla/layers/LayersSurfaces.h" +#include "nsLayoutUtils.h" +#include "nsIPrincipal.h" +#include "nsIURI.h" + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla::dom { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOG +# undef LOG +#endif // LOG +#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +#ifdef LOGW +# undef LOGW +#endif // LOGW +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(VideoFrame) +NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(VideoFrame) + tmp->CloseIfNeeded(); + NS_IMPL_CYCLE_COLLECTION_UNLINK(mParent) + NS_IMPL_CYCLE_COLLECTION_UNLINK_PRESERVED_WRAPPER +NS_IMPL_CYCLE_COLLECTION_UNLINK_END +NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(VideoFrame) + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mParent) +NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END + +NS_IMPL_CYCLE_COLLECTING_ADDREF(VideoFrame) +// VideoFrame should be released as soon as its refcount drops to zero, +// without waiting for async deletion by the cycle collector, since it may hold +// a large-size image. +NS_IMPL_CYCLE_COLLECTING_RELEASE_WITH_LAST_RELEASE(VideoFrame, CloseIfNeeded()) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(VideoFrame) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +/* + * The following are helpers to read the image data from the given buffer and + * the format. The data layout is illustrated in the comments for + * `VideoFrame::Format` below. + */ + +static int32_t CeilingOfHalf(int32_t aValue) { + MOZ_ASSERT(aValue >= 0); + return aValue / 2 + (aValue % 2); +} + +class YUVBufferReaderBase { + public: + YUVBufferReaderBase(const Span<uint8_t>& aBuffer, int32_t aWidth, + int32_t aHeight) + : mWidth(aWidth), mHeight(aHeight), mStrideY(aWidth), mBuffer(aBuffer) {} + virtual ~YUVBufferReaderBase() = default; + + const uint8_t* DataY() const { return mBuffer.data(); } + const int32_t mWidth; + const int32_t mHeight; + const int32_t mStrideY; + + protected: + CheckedInt<size_t> YByteSize() const { + return CheckedInt<size_t>(mStrideY) * mHeight; + } + + const Span<uint8_t> mBuffer; +}; + +class I420ABufferReader; +class I420BufferReader : public YUVBufferReaderBase { + public: + I420BufferReader(const Span<uint8_t>& aBuffer, int32_t aWidth, + int32_t aHeight) + : YUVBufferReaderBase(aBuffer, aWidth, aHeight), + mStrideU(CeilingOfHalf(aWidth)), + mStrideV(CeilingOfHalf(aWidth)) {} + virtual ~I420BufferReader() = default; + + const uint8_t* DataU() const { return &mBuffer[YByteSize().value()]; } + const uint8_t* DataV() const { + return &mBuffer[YByteSize().value() + UByteSize().value()]; + } + virtual I420ABufferReader* AsI420ABufferReader() { return nullptr; } + + const int32_t mStrideU; + const int32_t mStrideV; + + protected: + CheckedInt<size_t> UByteSize() const { + return CheckedInt<size_t>(CeilingOfHalf(mHeight)) * mStrideU; + } + + CheckedInt<size_t> VSize() const { + return CheckedInt<size_t>(CeilingOfHalf(mHeight)) * mStrideV; + } +}; + +class I420ABufferReader final : public I420BufferReader { + public: + I420ABufferReader(const Span<uint8_t>& aBuffer, int32_t aWidth, + int32_t aHeight) + : I420BufferReader(aBuffer, aWidth, aHeight), mStrideA(aWidth) { + MOZ_ASSERT(mStrideA == mStrideY); + } + virtual ~I420ABufferReader() = default; + + const uint8_t* DataA() const { + return &mBuffer[YByteSize().value() + UByteSize().value() + + VSize().value()]; + } + + virtual I420ABufferReader* AsI420ABufferReader() override { return this; } + + const int32_t mStrideA; +}; + +class NV12BufferReader final : public YUVBufferReaderBase { + public: + NV12BufferReader(const Span<uint8_t>& aBuffer, int32_t aWidth, + int32_t aHeight) + : YUVBufferReaderBase(aBuffer, aWidth, aHeight), + mStrideUV(aWidth + aWidth % 2) {} + virtual ~NV12BufferReader() = default; + + const uint8_t* DataUV() const { return &mBuffer[YByteSize().value()]; } + + const int32_t mStrideUV; +}; + +/* + * The followings are helpers defined in + * https://w3c.github.io/webcodecs/#videoframe-algorithms + */ + +static bool IsSameOrigin(nsIGlobalObject* aGlobal, const VideoFrame& aFrame) { + MOZ_ASSERT(aGlobal); + MOZ_ASSERT(aFrame.GetParentObject()); + + nsIPrincipal* principalX = aGlobal->PrincipalOrNull(); + nsIPrincipal* principalY = aFrame.GetParentObject()->PrincipalOrNull(); + + // If both of VideoFrames are created in worker, they are in the same origin + // domain. + if (!principalX) { + return !principalY; + } + // Otherwise, check their domains. + return principalX->Equals(principalY); +} + +static bool IsSameOrigin(nsIGlobalObject* aGlobal, + HTMLVideoElement& aVideoElement) { + MOZ_ASSERT(aGlobal); + + // If CORS is in use, consider the video source is same-origin. + if (aVideoElement.GetCORSMode() != CORS_NONE) { + return true; + } + + // Otherwise, check if video source has cross-origin redirect or not. + if (aVideoElement.HadCrossOriginRedirects()) { + return false; + } + + // Finally, compare the VideoFrame's domain and video's one. + nsIPrincipal* principal = aGlobal->PrincipalOrNull(); + nsCOMPtr<nsIPrincipal> elementPrincipal = + aVideoElement.GetCurrentVideoPrincipal(); + // <video> cannot be created in worker, so it should have a valid principal. + if (NS_WARN_IF(!elementPrincipal) || !principal) { + return false; + } + return principal->Subsumes(elementPrincipal); +} + +// A sub-helper to convert DOMRectInit to gfx::IntRect. +static Result<gfx::IntRect, nsCString> ToIntRect(const DOMRectInit& aRectInit) { + auto EQ = [](const double& a, const double& b) { + constexpr double e = std::numeric_limits<double>::epsilon(); + return std::fabs(a - b) <= e; + }; + auto GT = [&](const double& a, const double& b) { + return !EQ(a, b) && a > b; + }; + + // Make sure the double values are in the gfx::IntRect's valid range, before + // checking the spec's valid range. The double's infinity value is larger than + // gfx::IntRect's max value so it will be filtered out here. + constexpr double MAX = static_cast<double>( + std::numeric_limits<decltype(gfx::IntRect::x)>::max()); + constexpr double MIN = static_cast<double>( + std::numeric_limits<decltype(gfx::IntRect::x)>::min()); + if (GT(aRectInit.mX, MAX) || GT(MIN, aRectInit.mX)) { + return Err(nsCString("x is out of the valid range")); + } + if (GT(aRectInit.mY, MAX) || GT(MIN, aRectInit.mY)) { + return Err(nsCString("y is out of the valid range")); + } + if (GT(aRectInit.mWidth, MAX) || GT(MIN, aRectInit.mWidth)) { + return Err(nsCString("width is out of the valid range")); + } + if (GT(aRectInit.mHeight, MAX) || GT(MIN, aRectInit.mHeight)) { + return Err(nsCString("height is out of the valid range")); + } + + gfx::IntRect rect( + static_cast<decltype(gfx::IntRect::x)>(aRectInit.mX), + static_cast<decltype(gfx::IntRect::y)>(aRectInit.mY), + static_cast<decltype(gfx::IntRect::width)>(aRectInit.mWidth), + static_cast<decltype(gfx::IntRect::height)>(aRectInit.mHeight)); + // Check the spec's valid range. + if (rect.X() < 0) { + return Err(nsCString("x must be non-negative")); + } + if (rect.Y() < 0) { + return Err(nsCString("y must be non-negative")); + } + if (rect.Width() <= 0) { + return Err(nsCString("width must be positive")); + } + if (rect.Height() <= 0) { + return Err(nsCString("height must be positive")); + } + + return rect; +} + +// A sub-helper to convert a (width, height) pair to gfx::IntRect. +static Result<gfx::IntSize, nsCString> ToIntSize(const uint32_t& aWidth, + const uint32_t& aHeight) { + // Make sure the given values are in the gfx::IntSize's valid range, before + // checking the spec's valid range. + constexpr uint32_t MAX = static_cast<uint32_t>( + std::numeric_limits<decltype(gfx::IntRect::width)>::max()); + if (aWidth > MAX) { + return Err(nsCString("Width exceeds the implementation's range")); + } + if (aHeight > MAX) { + return Err(nsCString("Height exceeds the implementation's range")); + } + + gfx::IntSize size(static_cast<decltype(gfx::IntRect::width)>(aWidth), + static_cast<decltype(gfx::IntRect::height)>(aHeight)); + // Check the spec's valid range. + if (size.Width() <= 0) { + return Err(nsCString("Width must be positive")); + } + if (size.Height() <= 0) { + return Err(nsCString("Height must be positive")); + } + return size; +} + +// A sub-helper to make sure visible range is in the picture. +static Result<Ok, nsCString> ValidateVisibility( + const gfx::IntRect& aVisibleRect, const gfx::IntSize& aPicSize) { + MOZ_ASSERT(aVisibleRect.X() >= 0); + MOZ_ASSERT(aVisibleRect.Y() >= 0); + MOZ_ASSERT(aVisibleRect.Width() > 0); + MOZ_ASSERT(aVisibleRect.Height() > 0); + + const auto w = CheckedInt<uint32_t>(aVisibleRect.Width()) + aVisibleRect.X(); + if (w.value() > static_cast<uint32_t>(aPicSize.Width())) { + return Err(nsCString( + "Sum of visible rectangle's x and width exceeds the picture's width")); + } + + const auto h = CheckedInt<uint32_t>(aVisibleRect.Height()) + aVisibleRect.Y(); + if (h.value() > static_cast<uint32_t>(aPicSize.Height())) { + return Err( + nsCString("Sum of visible rectangle's y and height exceeds the " + "picture's height")); + } + + return Ok(); +} + +// A sub-helper to check and get display{Width, Height} in +// VideoFrame(Buffer)Init. +template <class T> +static Result<Maybe<gfx::IntSize>, nsCString> MaybeGetDisplaySize( + const T& aInit) { + if (aInit.mDisplayWidth.WasPassed() != aInit.mDisplayHeight.WasPassed()) { + return Err(nsCString( + "displayWidth and displayHeight cannot be set without the other")); + } + + Maybe<gfx::IntSize> displaySize; + if (aInit.mDisplayWidth.WasPassed() && aInit.mDisplayHeight.WasPassed()) { + displaySize.emplace(); + MOZ_TRY_VAR(displaySize.ref(), ToIntSize(aInit.mDisplayWidth.Value(), + aInit.mDisplayHeight.Value()) + .mapErr([](nsCString error) { + error.Insert("display", 0); + return error; + })); + } + return displaySize; +} + +// https://w3c.github.io/webcodecs/#valid-videoframebufferinit +static Result< + std::tuple<gfx::IntSize, Maybe<gfx::IntRect>, Maybe<gfx::IntSize>>, + nsCString> +ValidateVideoFrameBufferInit(const VideoFrameBufferInit& aInit) { + gfx::IntSize codedSize; + MOZ_TRY_VAR(codedSize, ToIntSize(aInit.mCodedWidth, aInit.mCodedHeight) + .mapErr([](nsCString error) { + error.Insert("coded", 0); + return error; + })); + + Maybe<gfx::IntRect> visibleRect; + if (aInit.mVisibleRect.WasPassed()) { + visibleRect.emplace(); + MOZ_TRY_VAR( + visibleRect.ref(), + ToIntRect(aInit.mVisibleRect.Value()).mapErr([](nsCString error) { + error.Insert("visibleRect's ", 0); + return error; + })); + MOZ_TRY(ValidateVisibility(visibleRect.ref(), codedSize)); + } + + Maybe<gfx::IntSize> displaySize; + MOZ_TRY_VAR(displaySize, MaybeGetDisplaySize(aInit)); + + return std::make_tuple(codedSize, visibleRect, displaySize); +} + +// https://w3c.github.io/webcodecs/#videoframe-verify-rect-offset-alignment +static Result<Ok, nsCString> VerifyRectOffsetAlignment( + const Maybe<VideoFrame::Format>& aFormat, const gfx::IntRect& aRect) { + if (!aFormat) { + return Ok(); + } + for (const VideoFrame::Format::Plane& p : aFormat->Planes()) { + const gfx::IntSize sample = aFormat->SampleSize(p); + if (aRect.X() % sample.Width() != 0) { + return Err(nsCString("Mismatch between format and given left offset")); + } + + if (aRect.Y() % sample.Height() != 0) { + return Err(nsCString("Mismatch between format and given top offset")); + } + } + return Ok(); +} + +// https://w3c.github.io/webcodecs/#videoframe-parse-visible-rect +static Result<gfx::IntRect, nsCString> ParseVisibleRect( + const gfx::IntRect& aDefaultRect, const Maybe<gfx::IntRect>& aOverrideRect, + const gfx::IntSize& aCodedSize, const VideoFrame::Format& aFormat) { + MOZ_ASSERT(ValidateVisibility(aDefaultRect, aCodedSize).isOk()); + + gfx::IntRect rect = aDefaultRect; + if (aOverrideRect) { + // Skip checking overrideRect's width and height here. They should be + // checked before reaching here, and ValidateVisibility will assert it. + + MOZ_TRY(ValidateVisibility(aOverrideRect.ref(), aCodedSize)); + rect = *aOverrideRect; + } + + MOZ_TRY(VerifyRectOffsetAlignment(Some(aFormat), rect)); + + return rect; +} + +// https://w3c.github.io/webcodecs/#computed-plane-layout +struct ComputedPlaneLayout { + // The offset from the beginning of the buffer in one plane. + uint32_t mDestinationOffset = 0; + // The stride of the image data in one plane. + uint32_t mDestinationStride = 0; + // Sample count of picture's top offset (a.k.a samples of y). + uint32_t mSourceTop = 0; + // Sample count of the picture's height. + uint32_t mSourceHeight = 0; + // Byte count of the picture's left offset (a.k.a bytes of x). + uint32_t mSourceLeftBytes = 0; + // Byte count of the picture's width. + uint32_t mSourceWidthBytes = 0; +}; + +// https://w3c.github.io/webcodecs/#combined-buffer-layout +struct CombinedBufferLayout { + CombinedBufferLayout() : mAllocationSize(0) {} + CombinedBufferLayout(uint32_t aAllocationSize, + nsTArray<ComputedPlaneLayout>&& aLayout) + : mAllocationSize(aAllocationSize), + mComputedLayouts(std::move(aLayout)) {} + uint32_t mAllocationSize = 0; + nsTArray<ComputedPlaneLayout> mComputedLayouts; +}; + +// https://w3c.github.io/webcodecs/#videoframe-compute-layout-and-allocation-size +static Result<CombinedBufferLayout, nsCString> ComputeLayoutAndAllocationSize( + const gfx::IntRect& aRect, const VideoFrame::Format& aFormat, + const Sequence<PlaneLayout>* aPlaneLayouts) { + nsTArray<VideoFrame::Format::Plane> planes = aFormat.Planes(); + + if (aPlaneLayouts && aPlaneLayouts->Length() != planes.Length()) { + return Err(nsCString("Mismatch between format and layout")); + } + + uint32_t minAllocationSize = 0; + nsTArray<ComputedPlaneLayout> layouts; + nsTArray<uint32_t> endOffsets; + + for (size_t i = 0; i < planes.Length(); ++i) { + const VideoFrame::Format::Plane& p = planes[i]; + const gfx::IntSize sampleSize = aFormat.SampleSize(p); + MOZ_RELEASE_ASSERT(!sampleSize.IsEmpty()); + + // aRect's x, y, width, and height are int32_t, and sampleSize's width and + // height >= 1, so (aRect.* / sampleSize.*) must be in int32_t range. + + CheckedUint32 sourceTop(aRect.Y()); + sourceTop /= sampleSize.Height(); + MOZ_RELEASE_ASSERT(sourceTop.isValid()); + + CheckedUint32 sourceHeight(aRect.Height()); + sourceHeight /= sampleSize.Height(); + MOZ_RELEASE_ASSERT(sourceHeight.isValid()); + + CheckedUint32 sourceLeftBytes(aRect.X()); + sourceLeftBytes /= sampleSize.Width(); + MOZ_RELEASE_ASSERT(sourceLeftBytes.isValid()); + sourceLeftBytes *= aFormat.SampleBytes(p); + if (!sourceLeftBytes.isValid()) { + return Err(nsPrintfCString( + "The parsed-rect's x-offset is too large for %s plane", + aFormat.PlaneName(p))); + } + + CheckedUint32 sourceWidthBytes(aRect.Width()); + sourceWidthBytes /= sampleSize.Width(); + MOZ_RELEASE_ASSERT(sourceWidthBytes.isValid()); + sourceWidthBytes *= aFormat.SampleBytes(p); + if (!sourceWidthBytes.isValid()) { + return Err( + nsPrintfCString("The parsed-rect's width is too large for %s plane", + aFormat.PlaneName(p))); + } + + // TODO: Spec here is wrong so we do differently: + // https://github.com/w3c/webcodecs/issues/511 + // This comment should be removed once the issue is resolved. + ComputedPlaneLayout layout{.mDestinationOffset = 0, + .mDestinationStride = 0, + .mSourceTop = sourceTop.value(), + .mSourceHeight = sourceHeight.value(), + .mSourceLeftBytes = sourceLeftBytes.value(), + .mSourceWidthBytes = sourceWidthBytes.value()}; + if (aPlaneLayouts) { + const PlaneLayout& planeLayout = aPlaneLayouts->ElementAt(i); + if (planeLayout.mStride < layout.mSourceWidthBytes) { + return Err(nsPrintfCString("The stride in %s plane is too small", + aFormat.PlaneName(p))); + } + layout.mDestinationOffset = planeLayout.mOffset; + layout.mDestinationStride = planeLayout.mStride; + } else { + layout.mDestinationOffset = minAllocationSize; + layout.mDestinationStride = layout.mSourceWidthBytes; + } + + const CheckedInt<uint32_t> planeSize = + CheckedInt<uint32_t>(layout.mDestinationStride) * layout.mSourceHeight; + if (!planeSize.isValid()) { + return Err(nsCString("Invalid layout with an over-sized plane")); + } + const CheckedInt<uint32_t> planeEnd = planeSize + layout.mDestinationOffset; + if (!planeEnd.isValid()) { + return Err(nsCString("Invalid layout with the out-out-bound offset")); + } + endOffsets.AppendElement(planeEnd.value()); + + minAllocationSize = std::max(minAllocationSize, planeEnd.value()); + + for (size_t j = 0; j < i; ++j) { + const ComputedPlaneLayout& earlier = layouts[j]; + // If the current data's end is smaller or equal to the previous one's + // head, or if the previous data's end is smaller or equal to the current + // one's head, then they do not overlap. Otherwise, they do. + if (endOffsets[i] > earlier.mDestinationOffset && + endOffsets[j] > layout.mDestinationOffset) { + return Err(nsCString("Invalid layout with the overlapped planes")); + } + } + layouts.AppendElement(layout); + } + + return CombinedBufferLayout(minAllocationSize, std::move(layouts)); +} + +// https://w3c.github.io/webcodecs/#videoframe-verify-rect-size-alignment +static Result<Ok, nsCString> VerifyRectSizeAlignment( + const VideoFrame::Format& aFormat, const gfx::IntRect& aRect) { + for (const VideoFrame::Format::Plane& p : aFormat.Planes()) { + const gfx::IntSize sample = aFormat.SampleSize(p); + if (aRect.Width() % sample.Width() != 0) { + return Err(nsCString("Mismatch between format and given rect's width")); + } + + if (aRect.Height() % sample.Height() != 0) { + return Err(nsCString("Mismatch between format and given rect's height")); + } + } + return Ok(); +} + +// https://w3c.github.io/webcodecs/#videoframe-parse-videoframecopytooptions +static Result<CombinedBufferLayout, nsCString> ParseVideoFrameCopyToOptions( + const VideoFrameCopyToOptions& aOptions, const gfx::IntRect& aVisibleRect, + const gfx::IntSize& aCodedSize, const VideoFrame::Format& aFormat) { + Maybe<gfx::IntRect> overrideRect; + if (aOptions.mRect.WasPassed()) { + // TODO: We handle some edge cases that spec misses: + // https://github.com/w3c/webcodecs/issues/513 + // This comment should be removed once the issue is resolved. + overrideRect.emplace(); + MOZ_TRY_VAR(overrideRect.ref(), + ToIntRect(aOptions.mRect.Value()).mapErr([](nsCString error) { + error.Insert("rect's ", 0); + return error; + })); + + MOZ_TRY(VerifyRectSizeAlignment(aFormat, overrideRect.ref())); + } + + gfx::IntRect parsedRect; + MOZ_TRY_VAR(parsedRect, ParseVisibleRect(aVisibleRect, overrideRect, + aCodedSize, aFormat)); + + const Sequence<PlaneLayout>* optLayout = OptionalToPointer(aOptions.mLayout); + + return ComputeLayoutAndAllocationSize(parsedRect, aFormat, optLayout); +} + +static bool IsYUVFormat(const VideoPixelFormat& aFormat) { + switch (aFormat) { + case VideoPixelFormat::I420: + case VideoPixelFormat::I420A: + case VideoPixelFormat::I422: + case VideoPixelFormat::I444: + case VideoPixelFormat::NV12: + return true; + case VideoPixelFormat::RGBA: + case VideoPixelFormat::RGBX: + case VideoPixelFormat::BGRA: + case VideoPixelFormat::BGRX: + return false; + case VideoPixelFormat::EndGuard_: + MOZ_ASSERT_UNREACHABLE("unsupported format"); + } + return false; +} + +// https://w3c.github.io/webcodecs/#videoframe-pick-color-space +static VideoColorSpaceInit PickColorSpace( + const VideoColorSpaceInit* aInitColorSpace, + const VideoPixelFormat& aFormat) { + VideoColorSpaceInit colorSpace; + if (aInitColorSpace) { + colorSpace = *aInitColorSpace; + // By spec, we MAY replace null members of aInitColorSpace with guessed + // values so we can always use these in CreateYUVImageFromBuffer. + if (IsYUVFormat(aFormat) && colorSpace.mMatrix.IsNull()) { + colorSpace.mMatrix.SetValue(VideoMatrixCoefficients::Bt709); + } + return colorSpace; + } + + switch (aFormat) { + case VideoPixelFormat::I420: + case VideoPixelFormat::I420A: + case VideoPixelFormat::I422: + case VideoPixelFormat::I444: + case VideoPixelFormat::NV12: + // https://w3c.github.io/webcodecs/#rec709-color-space + colorSpace.mFullRange.SetValue(false); + colorSpace.mMatrix.SetValue(VideoMatrixCoefficients::Bt709); + colorSpace.mPrimaries.SetValue(VideoColorPrimaries::Bt709); + colorSpace.mTransfer.SetValue(VideoTransferCharacteristics::Bt709); + break; + case VideoPixelFormat::RGBA: + case VideoPixelFormat::RGBX: + case VideoPixelFormat::BGRA: + case VideoPixelFormat::BGRX: + // https://w3c.github.io/webcodecs/#srgb-color-space + colorSpace.mFullRange.SetValue(true); + colorSpace.mMatrix.SetValue(VideoMatrixCoefficients::Rgb); + colorSpace.mPrimaries.SetValue(VideoColorPrimaries::Bt709); + colorSpace.mTransfer.SetValue(VideoTransferCharacteristics::Iec61966_2_1); + break; + case VideoPixelFormat::EndGuard_: + MOZ_ASSERT_UNREACHABLE("unsupported format"); + } + + return colorSpace; +} + +// https://w3c.github.io/webcodecs/#validate-videoframeinit +static Result<std::pair<Maybe<gfx::IntRect>, Maybe<gfx::IntSize>>, nsCString> +ValidateVideoFrameInit(const VideoFrameInit& aInit, + const Maybe<VideoFrame::Format>& aFormat, + const gfx::IntSize& aCodedSize) { + if (aCodedSize.Width() <= 0 || aCodedSize.Height() <= 0) { + return Err(nsCString("codedWidth and codedHeight must be positive")); + } + + Maybe<gfx::IntRect> visibleRect; + if (aInit.mVisibleRect.WasPassed()) { + visibleRect.emplace(); + MOZ_TRY_VAR( + visibleRect.ref(), + ToIntRect(aInit.mVisibleRect.Value()).mapErr([](nsCString error) { + error.Insert("visibleRect's ", 0); + return error; + })); + MOZ_TRY(ValidateVisibility(visibleRect.ref(), aCodedSize)); + + MOZ_TRY(VerifyRectOffsetAlignment(aFormat, visibleRect.ref())); + } + + Maybe<gfx::IntSize> displaySize; + MOZ_TRY_VAR(displaySize, MaybeGetDisplaySize(aInit)); + + return std::make_pair(visibleRect, displaySize); +} + +/* + * The followings are helpers to create a VideoFrame from a given buffer + */ + +static Result<RefPtr<gfx::DataSourceSurface>, nsCString> AllocateBGRASurface( + gfx::DataSourceSurface* aSurface) { + MOZ_ASSERT(aSurface); + + // Memory allocation relies on CreateDataSourceSurfaceWithStride so we still + // need to do this even if the format is SurfaceFormat::BGR{A, X}. + + gfx::DataSourceSurface::ScopedMap surfaceMap(aSurface, + gfx::DataSourceSurface::READ); + if (!surfaceMap.IsMapped()) { + return Err(nsCString("The source surface is not readable")); + } + + RefPtr<gfx::DataSourceSurface> bgraSurface = + gfx::Factory::CreateDataSourceSurfaceWithStride( + aSurface->GetSize(), gfx::SurfaceFormat::B8G8R8A8, + surfaceMap.GetStride()); + if (!bgraSurface) { + return Err(nsCString("Failed to allocate a BGRA surface")); + } + + gfx::DataSourceSurface::ScopedMap bgraMap(bgraSurface, + gfx::DataSourceSurface::WRITE); + if (!bgraMap.IsMapped()) { + return Err(nsCString("The allocated BGRA surface is not writable")); + } + + gfx::SwizzleData(surfaceMap.GetData(), surfaceMap.GetStride(), + aSurface->GetFormat(), bgraMap.GetData(), + bgraMap.GetStride(), bgraSurface->GetFormat(), + bgraSurface->GetSize()); + + return bgraSurface; +} + +static Result<RefPtr<layers::Image>, nsCString> CreateImageFromRawData( + const gfx::IntSize& aSize, int32_t aStride, gfx::SurfaceFormat aFormat, + const Span<uint8_t>& aBuffer) { + MOZ_ASSERT(!aSize.IsEmpty()); + + // Wrap the source buffer into a DataSourceSurface. + RefPtr<gfx::DataSourceSurface> surface = + gfx::Factory::CreateWrappingDataSourceSurface(aBuffer.data(), aStride, + aSize, aFormat); + if (!surface) { + return Err(nsCString("Failed to wrap the raw data into a surface")); + } + + // Gecko favors BGRA so we convert surface into BGRA format first. + RefPtr<gfx::DataSourceSurface> bgraSurface; + MOZ_TRY_VAR(bgraSurface, AllocateBGRASurface(surface)); + MOZ_ASSERT(bgraSurface); + + return RefPtr<layers::Image>( + new layers::SourceSurfaceImage(bgraSurface.get())); +} + +static Result<RefPtr<layers::Image>, nsCString> CreateRGBAImageFromBuffer( + const VideoFrame::Format& aFormat, const gfx::IntSize& aSize, + const Span<uint8_t>& aBuffer) { + const gfx::SurfaceFormat format = aFormat.ToSurfaceFormat(); + MOZ_ASSERT(format == gfx::SurfaceFormat::R8G8B8A8 || + format == gfx::SurfaceFormat::R8G8B8X8 || + format == gfx::SurfaceFormat::B8G8R8A8 || + format == gfx::SurfaceFormat::B8G8R8X8); + // TODO: Use aFormat.SampleBytes() instead? + CheckedInt<int32_t> stride(BytesPerPixel(format)); + stride *= aSize.Width(); + if (!stride.isValid()) { + return Err(nsCString("Image size exceeds implementation's limit")); + } + return CreateImageFromRawData(aSize, stride.value(), format, aBuffer); +} + +static Result<RefPtr<layers::Image>, nsCString> CreateYUVImageFromBuffer( + const VideoFrame::Format& aFormat, const VideoColorSpaceInit& aColorSpace, + const gfx::IntSize& aSize, const Span<uint8_t>& aBuffer) { + if (aFormat.PixelFormat() == VideoPixelFormat::I420 || + aFormat.PixelFormat() == VideoPixelFormat::I420A) { + UniquePtr<I420BufferReader> reader; + if (aFormat.PixelFormat() == VideoPixelFormat::I420) { + reader.reset( + new I420BufferReader(aBuffer, aSize.Width(), aSize.Height())); + } else { + reader.reset( + new I420ABufferReader(aBuffer, aSize.Width(), aSize.Height())); + } + + layers::PlanarYCbCrData data; + data.mPictureRect = gfx::IntRect(0, 0, reader->mWidth, reader->mHeight); + + // Y plane. + data.mYChannel = const_cast<uint8_t*>(reader->DataY()); + data.mYStride = reader->mStrideY; + data.mYSkip = 0; + // Cb plane. + data.mCbChannel = const_cast<uint8_t*>(reader->DataU()); + data.mCbSkip = 0; + // Cr plane. + data.mCrChannel = const_cast<uint8_t*>(reader->DataV()); + data.mCbSkip = 0; + // A plane. + if (aFormat.PixelFormat() == VideoPixelFormat::I420A) { + data.mAlpha.emplace(); + data.mAlpha->mChannel = + const_cast<uint8_t*>(reader->AsI420ABufferReader()->DataA()); + data.mAlpha->mSize = data.mPictureRect.Size(); + // No values for mDepth and mPremultiplied. + } + + // CbCr plane vector. + MOZ_RELEASE_ASSERT(reader->mStrideU == reader->mStrideV); + data.mCbCrStride = reader->mStrideU; + data.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT; + // Color settings. + if (!aColorSpace.mFullRange.IsNull()) { + data.mColorRange = ToColorRange(aColorSpace.mFullRange.Value()); + } + MOZ_RELEASE_ASSERT(!aColorSpace.mMatrix.IsNull()); + data.mYUVColorSpace = ToColorSpace(aColorSpace.mMatrix.Value()); + if (!aColorSpace.mTransfer.IsNull()) { + data.mTransferFunction = + ToTransferFunction(aColorSpace.mTransfer.Value()); + } + if (!aColorSpace.mPrimaries.IsNull()) { + data.mColorPrimaries = ToPrimaries(aColorSpace.mPrimaries.Value()); + } + + RefPtr<layers::PlanarYCbCrImage> image = + new layers::RecyclingPlanarYCbCrImage(new layers::BufferRecycleBin()); + if (NS_FAILED(image->CopyData(data))) { + return Err(nsPrintfCString( + "Failed to create I420%s image", + (aFormat.PixelFormat() == VideoPixelFormat::I420A ? "A" : ""))); + } + // Manually cast type to make Result work. + return RefPtr<layers::Image>(image.forget()); + } + + if (aFormat.PixelFormat() == VideoPixelFormat::NV12) { + NV12BufferReader reader(aBuffer, aSize.Width(), aSize.Height()); + + layers::PlanarYCbCrData data; + data.mPictureRect = gfx::IntRect(0, 0, reader.mWidth, reader.mHeight); + + // Y plane. + data.mYChannel = const_cast<uint8_t*>(reader.DataY()); + data.mYStride = reader.mStrideY; + data.mYSkip = 0; + // Cb plane. + data.mCbChannel = const_cast<uint8_t*>(reader.DataUV()); + data.mCbSkip = 1; + // Cr plane. + data.mCrChannel = data.mCbChannel + 1; + data.mCrSkip = 1; + // CbCr plane vector. + data.mCbCrStride = reader.mStrideUV; + data.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT; + // Color settings. + if (!aColorSpace.mFullRange.IsNull()) { + data.mColorRange = ToColorRange(aColorSpace.mFullRange.Value()); + } + MOZ_RELEASE_ASSERT(!aColorSpace.mMatrix.IsNull()); + data.mYUVColorSpace = ToColorSpace(aColorSpace.mMatrix.Value()); + if (!aColorSpace.mTransfer.IsNull()) { + data.mTransferFunction = + ToTransferFunction(aColorSpace.mTransfer.Value()); + } + if (!aColorSpace.mPrimaries.IsNull()) { + data.mColorPrimaries = ToPrimaries(aColorSpace.mPrimaries.Value()); + } + + RefPtr<layers::NVImage> image = new layers::NVImage(); + if (!image->SetData(data)) { + return Err(nsCString("Failed to create NV12 image")); + } + // Manually cast type to make Result work. + return RefPtr<layers::Image>(image.forget()); + } + + return Err(nsCString("Unsupported image format")); +} + +static Result<RefPtr<layers::Image>, nsCString> CreateImageFromBuffer( + const VideoFrame::Format& aFormat, const VideoColorSpaceInit& aColorSpace, + const gfx::IntSize& aSize, const Span<uint8_t>& aBuffer) { + switch (aFormat.PixelFormat()) { + case VideoPixelFormat::I420: + case VideoPixelFormat::I420A: + case VideoPixelFormat::NV12: + return CreateYUVImageFromBuffer(aFormat, aColorSpace, aSize, aBuffer); + case VideoPixelFormat::I422: + case VideoPixelFormat::I444: + // Not yet support for now. + break; + case VideoPixelFormat::RGBA: + case VideoPixelFormat::RGBX: + case VideoPixelFormat::BGRA: + case VideoPixelFormat::BGRX: + return CreateRGBAImageFromBuffer(aFormat, aSize, aBuffer); + case VideoPixelFormat::EndGuard_: + MOZ_ASSERT_UNREACHABLE("unsupported format"); + } + return Err(nsCString("Invalid image format")); +} + +// https://w3c.github.io/webcodecs/#dom-videoframe-videoframe-data-init +template <class T> +static Result<RefPtr<VideoFrame>, nsCString> CreateVideoFrameFromBuffer( + nsIGlobalObject* aGlobal, const T& aBuffer, + const VideoFrameBufferInit& aInit) { + if (aInit.mColorSpace.WasPassed() && + !aInit.mColorSpace.Value().mTransfer.IsNull() && + aInit.mColorSpace.Value().mTransfer.Value() == + VideoTransferCharacteristics::Linear) { + return Err(nsCString("linear RGB is not supported")); + } + + std::tuple<gfx::IntSize, Maybe<gfx::IntRect>, Maybe<gfx::IntSize>> init; + MOZ_TRY_VAR(init, ValidateVideoFrameBufferInit(aInit)); + gfx::IntSize codedSize = std::get<0>(init); + Maybe<gfx::IntRect> visibleRect = std::get<1>(init); + Maybe<gfx::IntSize> displaySize = std::get<2>(init); + + VideoFrame::Format format(aInit.mFormat); + // TODO: Spec doesn't ask for this in ctor but Pixel Format does. See + // https://github.com/w3c/webcodecs/issues/512 + // This comment should be removed once the issue is resolved. + if (!format.IsValidSize(codedSize)) { + return Err(nsCString("coded width and/or height is invalid")); + } + + gfx::IntRect parsedRect; + MOZ_TRY_VAR(parsedRect, ParseVisibleRect(gfx::IntRect({0, 0}, codedSize), + visibleRect, codedSize, format)); + + const Sequence<PlaneLayout>* optLayout = OptionalToPointer(aInit.mLayout); + + CombinedBufferLayout combinedLayout; + MOZ_TRY_VAR(combinedLayout, + ComputeLayoutAndAllocationSize(parsedRect, format, optLayout)); + + Maybe<uint64_t> duration = OptionalToMaybe(aInit.mDuration); + + VideoColorSpaceInit colorSpace = + PickColorSpace(OptionalToPointer(aInit.mColorSpace), aInit.mFormat); + + RefPtr<layers::Image> data; + MOZ_TRY_VAR( + data, + aBuffer.ProcessFixedData([&](const Span<uint8_t>& aData) + -> Result<RefPtr<layers::Image>, nsCString> { + if (aData.Length() < + static_cast<size_t>(combinedLayout.mAllocationSize)) { + return Err(nsCString("data is too small")); + } + + // TODO: If codedSize is (3, 3) and visibleRect is (0, 0, 1, 1) but the + // data is 2 x 2 RGBA buffer (2 x 2 x 4 bytes), it pass the above check. + // In this case, we can crop it to a 1 x 1-codedSize image (Bug + // 1782128). + if (aData.Length() < format.SampleCount(codedSize)) { // 1 byte/sample + return Err(nsCString("data is too small")); + } + + return CreateImageFromBuffer(format, colorSpace, codedSize, aData); + })); + + MOZ_ASSERT(data); + MOZ_ASSERT(data->GetSize() == codedSize); + + // By spec, we should set visible* here. But if we don't change the image, + // visible* is same as parsedRect here. The display{Width, Height} is + // visible{Width, Height} if it's not set. + + // TODO: Spec should assign aInit.mFormat to inner format value: + // https://github.com/w3c/webcodecs/issues/509. + // This comment should be removed once the issue is resolved. + return MakeRefPtr<VideoFrame>(aGlobal, data, Some(aInit.mFormat), codedSize, + parsedRect, + displaySize ? *displaySize : parsedRect.Size(), + duration, aInit.mTimestamp, colorSpace); +} + +template <class T> +static already_AddRefed<VideoFrame> CreateVideoFrameFromBuffer( + const GlobalObject& aGlobal, const T& aBuffer, + const VideoFrameBufferInit& aInit, ErrorResult& aRv) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + auto r = CreateVideoFrameFromBuffer(global, aBuffer, aInit); + if (r.isErr()) { + aRv.ThrowTypeError(r.unwrapErr()); + return nullptr; + } + return r.unwrap().forget(); +} + +// https://w3c.github.io/webcodecs/#videoframe-initialize-visible-rect-and-display-size +static void InitializeVisibleRectAndDisplaySize( + Maybe<gfx::IntRect>& aVisibleRect, Maybe<gfx::IntSize>& aDisplaySize, + gfx::IntRect aDefaultVisibleRect, gfx::IntSize aDefaultDisplaySize) { + if (!aVisibleRect) { + aVisibleRect.emplace(aDefaultVisibleRect); + } + if (!aDisplaySize) { + double wScale = static_cast<double>(aDefaultDisplaySize.Width()) / + aDefaultVisibleRect.Width(); + double hScale = static_cast<double>(aDefaultDisplaySize.Height()) / + aDefaultVisibleRect.Height(); + double w = wScale * aVisibleRect->Width(); + double h = hScale * aVisibleRect->Height(); + aDisplaySize.emplace(gfx::IntSize(static_cast<uint32_t>(round(w)), + static_cast<uint32_t>(round(h)))); + } +} + +// https://w3c.github.io/webcodecs/#videoframe-initialize-frame-with-resource-and-size +static Result<already_AddRefed<VideoFrame>, nsCString> +InitializeFrameWithResourceAndSize( + nsIGlobalObject* aGlobal, const VideoFrameInit& aInit, + already_AddRefed<layers::SourceSurfaceImage> aImage) { + MOZ_ASSERT(aInit.mTimestamp.WasPassed()); + + RefPtr<layers::SourceSurfaceImage> image(aImage); + MOZ_ASSERT(image); + + RefPtr<gfx::SourceSurface> surface = image->GetAsSourceSurface(); + Maybe<VideoFrame::Format> format = + SurfaceFormatToVideoPixelFormat(surface->GetFormat()) + .map([](const VideoPixelFormat& aFormat) { + return VideoFrame::Format(aFormat); + }); + + std::pair<Maybe<gfx::IntRect>, Maybe<gfx::IntSize>> init; + MOZ_TRY_VAR(init, ValidateVideoFrameInit(aInit, format, image->GetSize())); + Maybe<gfx::IntRect> visibleRect = init.first; + Maybe<gfx::IntSize> displaySize = init.second; + + if (format && aInit.mAlpha == AlphaOption::Discard) { + format->MakeOpaque(); + // Keep the alpha data in image for now until it's being rendered. + // TODO: The alpha will still be rendered if the format is unrecognized + // since no additional flag keeping this request. Should spec address what + // to do in this case? + } + + InitializeVisibleRectAndDisplaySize(visibleRect, displaySize, + gfx::IntRect({0, 0}, image->GetSize()), + image->GetSize()); + + Maybe<uint64_t> duration = OptionalToMaybe(aInit.mDuration); + + VideoColorSpaceInit colorSpace{}; + if (IsYUVFormat( + SurfaceFormatToVideoPixelFormat(surface->GetFormat()).ref())) { + colorSpace = FallbackColorSpaceForVideoContent(); + } else { + colorSpace = FallbackColorSpaceForWebContent(); + } + return MakeAndAddRef<VideoFrame>( + aGlobal, image, format ? Some(format->PixelFormat()) : Nothing(), + image->GetSize(), visibleRect.value(), displaySize.value(), duration, + aInit.mTimestamp.Value(), colorSpace); +} + +// https://w3c.github.io/webcodecs/#videoframe-initialize-frame-from-other-frame +static Result<already_AddRefed<VideoFrame>, nsCString> +InitializeFrameFromOtherFrame(nsIGlobalObject* aGlobal, VideoFrameData&& aData, + const VideoFrameInit& aInit) { + MOZ_ASSERT(aGlobal); + MOZ_ASSERT(aData.mImage); + + Maybe<VideoFrame::Format> format = + aData.mFormat ? Some(VideoFrame::Format(*aData.mFormat)) : Nothing(); + if (format && aInit.mAlpha == AlphaOption::Discard) { + format->MakeOpaque(); + // Keep the alpha data in image for now until it's being rendered. + // TODO: The alpha will still be rendered if the format is unrecognized + // since no additional flag keeping this request. Should spec address what + // to do in this case? + } + + std::pair<Maybe<gfx::IntRect>, Maybe<gfx::IntSize>> init; + MOZ_TRY_VAR(init, + ValidateVideoFrameInit(aInit, format, aData.mImage->GetSize())); + Maybe<gfx::IntRect> visibleRect = init.first; + Maybe<gfx::IntSize> displaySize = init.second; + + InitializeVisibleRectAndDisplaySize(visibleRect, displaySize, + aData.mVisibleRect, aData.mDisplaySize); + + Maybe<uint64_t> duration = OptionalToMaybe(aInit.mDuration); + + int64_t timestamp = aInit.mTimestamp.WasPassed() ? aInit.mTimestamp.Value() + : aData.mTimestamp; + + return MakeAndAddRef<VideoFrame>( + aGlobal, aData.mImage, format ? Some(format->PixelFormat()) : Nothing(), + aData.mImage->GetSize(), *visibleRect, *displaySize, duration, timestamp, + aData.mColorSpace); +} + +/* + * Helper classes carrying VideoFrame data + */ + +VideoFrameData::VideoFrameData(layers::Image* aImage, + const Maybe<VideoPixelFormat>& aFormat, + gfx::IntRect aVisibleRect, + gfx::IntSize aDisplaySize, + Maybe<uint64_t> aDuration, int64_t aTimestamp, + const VideoColorSpaceInit& aColorSpace) + : mImage(aImage), + mFormat(aFormat), + mVisibleRect(aVisibleRect), + mDisplaySize(aDisplaySize), + mDuration(aDuration), + mTimestamp(aTimestamp), + mColorSpace(aColorSpace) {} + +VideoFrameSerializedData::VideoFrameSerializedData(const VideoFrameData& aData, + gfx::IntSize aCodedSize) + : VideoFrameData(aData), mCodedSize(aCodedSize) {} + +/* + * W3C Webcodecs VideoFrame implementation + */ + +VideoFrame::VideoFrame(nsIGlobalObject* aParent, + const RefPtr<layers::Image>& aImage, + const Maybe<VideoPixelFormat>& aFormat, + gfx::IntSize aCodedSize, gfx::IntRect aVisibleRect, + gfx::IntSize aDisplaySize, + const Maybe<uint64_t>& aDuration, int64_t aTimestamp, + const VideoColorSpaceInit& aColorSpace) + : mParent(aParent), + mCodedSize(aCodedSize), + mVisibleRect(aVisibleRect), + mDisplaySize(aDisplaySize), + mDuration(aDuration), + mTimestamp(aTimestamp), + mColorSpace(aColorSpace) { + MOZ_ASSERT(mParent); + LOG("VideoFrame %p ctor", this); + mResource.emplace( + Resource(aImage, aFormat.map([](const VideoPixelFormat& aPixelFormat) { + return VideoFrame::Format(aPixelFormat); + }))); + if (!mResource->mFormat) { + LOGW("Create a VideoFrame with an unrecognized image format"); + } + StartAutoClose(); +} + +VideoFrame::VideoFrame(nsIGlobalObject* aParent, + const VideoFrameSerializedData& aData) + : mParent(aParent), + mCodedSize(aData.mCodedSize), + mVisibleRect(aData.mVisibleRect), + mDisplaySize(aData.mDisplaySize), + mDuration(aData.mDuration), + mTimestamp(aData.mTimestamp), + mColorSpace(aData.mColorSpace) { + MOZ_ASSERT(mParent); + LOG("VideoFrame %p ctor", this); + mResource.emplace(Resource( + aData.mImage, aData.mFormat.map([](const VideoPixelFormat& aPixelFormat) { + return VideoFrame::Format(aPixelFormat); + }))); + if (!mResource->mFormat) { + LOGW("Create a VideoFrame with an unrecognized image format"); + } + StartAutoClose(); +} + +VideoFrame::VideoFrame(const VideoFrame& aOther) + : mParent(aOther.mParent), + mResource(aOther.mResource), + mCodedSize(aOther.mCodedSize), + mVisibleRect(aOther.mVisibleRect), + mDisplaySize(aOther.mDisplaySize), + mDuration(aOther.mDuration), + mTimestamp(aOther.mTimestamp), + mColorSpace(aOther.mColorSpace) { + MOZ_ASSERT(mParent); + LOG("VideoFrame %p ctor", this); + StartAutoClose(); +} + +VideoFrame::~VideoFrame() { + MOZ_ASSERT(IsClosed()); + LOG("VideoFrame %p dtor", this); +} + +nsIGlobalObject* VideoFrame::GetParentObject() const { + AssertIsOnOwningThread(); + + return mParent.get(); +} + +JSObject* VideoFrame::WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) { + AssertIsOnOwningThread(); + + return VideoFrame_Binding::Wrap(aCx, this, aGivenProto); +} + +// The following constructors are defined in +// https://w3c.github.io/webcodecs/#dom-videoframe-videoframe + +/* static */ +already_AddRefed<VideoFrame> VideoFrame::Constructor( + const GlobalObject& aGlobal, HTMLImageElement& aImageElement, + const VideoFrameInit& aInit, ErrorResult& aRv) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + // Check the usability. + if (aImageElement.State().HasState(ElementState::BROKEN)) { + aRv.ThrowInvalidStateError("The image's state is broken"); + return nullptr; + } + if (!aImageElement.Complete()) { + aRv.ThrowInvalidStateError("The image is not completely loaded yet"); + return nullptr; + } + if (aImageElement.NaturalWidth() == 0) { + aRv.ThrowInvalidStateError("The image has a width of 0"); + return nullptr; + } + if (aImageElement.NaturalHeight() == 0) { + aRv.ThrowInvalidStateError("The image has a height of 0"); + return nullptr; + } + + // If the origin of HTMLImageElement's image data is not same origin with the + // entry settings object's origin, then throw a SecurityError DOMException. + SurfaceFromElementResult res = nsLayoutUtils::SurfaceFromElement( + &aImageElement, nsLayoutUtils::SFE_WANT_FIRST_FRAME_IF_IMAGE); + if (res.mIsWriteOnly) { + // Being write-only implies its image is cross-origin w/out CORS headers. + aRv.ThrowSecurityError("The image is not same-origin"); + return nullptr; + } + + RefPtr<gfx::SourceSurface> surface = res.GetSourceSurface(); + if (NS_WARN_IF(!surface)) { + aRv.ThrowInvalidStateError("The image's surface acquisition failed"); + return nullptr; + } + + if (!aInit.mTimestamp.WasPassed()) { + aRv.ThrowTypeError("Missing timestamp"); + return nullptr; + } + + RefPtr<layers::SourceSurfaceImage> image = + new layers::SourceSurfaceImage(surface.get()); + auto r = InitializeFrameWithResourceAndSize(global, aInit, image.forget()); + if (r.isErr()) { + aRv.ThrowTypeError(r.unwrapErr()); + return nullptr; + } + return r.unwrap(); +} + +/* static */ +already_AddRefed<VideoFrame> VideoFrame::Constructor( + const GlobalObject& aGlobal, SVGImageElement& aSVGImageElement, + const VideoFrameInit& aInit, ErrorResult& aRv) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + // Check the usability. + if (aSVGImageElement.State().HasState(ElementState::BROKEN)) { + aRv.ThrowInvalidStateError("The SVG's state is broken"); + return nullptr; + } + + // If the origin of SVGImageElement's image data is not same origin with the + // entry settings object's origin, then throw a SecurityError DOMException. + SurfaceFromElementResult res = nsLayoutUtils::SurfaceFromElement( + &aSVGImageElement, nsLayoutUtils::SFE_WANT_FIRST_FRAME_IF_IMAGE); + if (res.mIsWriteOnly) { + // Being write-only implies its image is cross-origin w/out CORS headers. + aRv.ThrowSecurityError("The SVG is not same-origin"); + return nullptr; + } + + RefPtr<gfx::SourceSurface> surface = res.GetSourceSurface(); + if (NS_WARN_IF(!surface)) { + aRv.ThrowInvalidStateError("The SVG's surface acquisition failed"); + return nullptr; + } + + if (!aInit.mTimestamp.WasPassed()) { + aRv.ThrowTypeError("Missing timestamp"); + return nullptr; + } + + RefPtr<layers::SourceSurfaceImage> image = + new layers::SourceSurfaceImage(surface.get()); + auto r = InitializeFrameWithResourceAndSize(global, aInit, image.forget()); + if (r.isErr()) { + aRv.ThrowTypeError(r.unwrapErr()); + return nullptr; + } + return r.unwrap(); +} + +/* static */ +already_AddRefed<VideoFrame> VideoFrame::Constructor( + const GlobalObject& aGlobal, HTMLCanvasElement& aCanvasElement, + const VideoFrameInit& aInit, ErrorResult& aRv) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + // Check the usability. + if (aCanvasElement.Width() == 0) { + aRv.ThrowInvalidStateError("The canvas has a width of 0"); + return nullptr; + } + + if (aCanvasElement.Height() == 0) { + aRv.ThrowInvalidStateError("The canvas has a height of 0"); + return nullptr; + } + + // If the origin of HTMLCanvasElement's image data is not same origin with the + // entry settings object's origin, then throw a SecurityError DOMException. + SurfaceFromElementResult res = nsLayoutUtils::SurfaceFromElement( + &aCanvasElement, nsLayoutUtils::SFE_WANT_FIRST_FRAME_IF_IMAGE); + if (res.mIsWriteOnly) { + // Being write-only implies its image is cross-origin w/out CORS headers. + aRv.ThrowSecurityError("The canvas is not same-origin"); + return nullptr; + } + + RefPtr<gfx::SourceSurface> surface = res.GetSourceSurface(); + if (NS_WARN_IF(!surface)) { + aRv.ThrowInvalidStateError("The canvas' surface acquisition failed"); + return nullptr; + } + + if (!aInit.mTimestamp.WasPassed()) { + aRv.ThrowTypeError("Missing timestamp"); + return nullptr; + } + + RefPtr<layers::SourceSurfaceImage> image = + new layers::SourceSurfaceImage(surface.get()); + auto r = InitializeFrameWithResourceAndSize(global, aInit, image.forget()); + if (r.isErr()) { + aRv.ThrowTypeError(r.unwrapErr()); + return nullptr; + } + return r.unwrap(); +} + +/* static */ +already_AddRefed<VideoFrame> VideoFrame::Constructor( + const GlobalObject& aGlobal, HTMLVideoElement& aVideoElement, + const VideoFrameInit& aInit, ErrorResult& aRv) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + aVideoElement.LogVisibility( + mozilla::dom::HTMLVideoElement::CallerAPI::CREATE_VIDEOFRAME); + + // Check the usability. + if (aVideoElement.NetworkState() == HTMLMediaElement_Binding::NETWORK_EMPTY) { + aRv.ThrowInvalidStateError("The video has not been initialized yet"); + return nullptr; + } + if (aVideoElement.ReadyState() <= HTMLMediaElement_Binding::HAVE_METADATA) { + aRv.ThrowInvalidStateError("The video is not ready yet"); + return nullptr; + } + RefPtr<layers::Image> image = aVideoElement.GetCurrentImage(); + if (!image) { + aRv.ThrowInvalidStateError("The video doesn't have any image yet"); + return nullptr; + } + + // If the origin of HTMLVideoElement's image data is not same origin with the + // entry settings object's origin, then throw a SecurityError DOMException. + if (!IsSameOrigin(global.get(), aVideoElement)) { + aRv.ThrowSecurityError("The video is not same-origin"); + return nullptr; + } + + const ImageUtils imageUtils(image); + Maybe<VideoPixelFormat> format = + ImageBitmapFormatToVideoPixelFormat(imageUtils.GetFormat()); + + // TODO: Retrive/infer the duration, and colorspace. + auto r = InitializeFrameFromOtherFrame( + global.get(), + VideoFrameData(image.get(), format, image->GetPictureRect(), + image->GetSize(), Nothing(), + static_cast<int64_t>(aVideoElement.CurrentTime()), {}), + aInit); + if (r.isErr()) { + aRv.ThrowTypeError(r.unwrapErr()); + return nullptr; + } + return r.unwrap(); +} + +/* static */ +already_AddRefed<VideoFrame> VideoFrame::Constructor( + const GlobalObject& aGlobal, OffscreenCanvas& aOffscreenCanvas, + const VideoFrameInit& aInit, ErrorResult& aRv) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + // Check the usability. + if (aOffscreenCanvas.Width() == 0) { + aRv.ThrowInvalidStateError("The canvas has a width of 0"); + return nullptr; + } + if (aOffscreenCanvas.Height() == 0) { + aRv.ThrowInvalidStateError("The canvas has a height of 0"); + return nullptr; + } + + // If the origin of the OffscreenCanvas's image data is not same origin with + // the entry settings object's origin, then throw a SecurityError + // DOMException. + SurfaceFromElementResult res = nsLayoutUtils::SurfaceFromOffscreenCanvas( + &aOffscreenCanvas, nsLayoutUtils::SFE_WANT_FIRST_FRAME_IF_IMAGE); + if (res.mIsWriteOnly) { + // Being write-only implies its image is cross-origin w/out CORS headers. + aRv.ThrowSecurityError("The canvas is not same-origin"); + return nullptr; + } + + RefPtr<gfx::SourceSurface> surface = res.GetSourceSurface(); + if (NS_WARN_IF(!surface)) { + aRv.ThrowInvalidStateError("The canvas' surface acquisition failed"); + return nullptr; + } + + if (!aInit.mTimestamp.WasPassed()) { + aRv.ThrowTypeError("Missing timestamp"); + return nullptr; + } + + RefPtr<layers::SourceSurfaceImage> image = + new layers::SourceSurfaceImage(surface.get()); + auto r = InitializeFrameWithResourceAndSize(global, aInit, image.forget()); + if (r.isErr()) { + aRv.ThrowTypeError(r.unwrapErr()); + return nullptr; + } + return r.unwrap(); +} + +/* static */ +already_AddRefed<VideoFrame> VideoFrame::Constructor( + const GlobalObject& aGlobal, ImageBitmap& aImageBitmap, + const VideoFrameInit& aInit, ErrorResult& aRv) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + // Check the usability. + UniquePtr<ImageBitmapCloneData> data = aImageBitmap.ToCloneData(); + if (!data || !data->mSurface) { + aRv.ThrowInvalidStateError( + "The ImageBitmap is closed or its surface acquisition failed"); + return nullptr; + } + + // If the origin of the ImageBitmap's image data is not same origin with the + // entry settings object's origin, then throw a SecurityError DOMException. + if (data->mWriteOnly) { + // Being write-only implies its image is cross-origin w/out CORS headers. + aRv.ThrowSecurityError("The ImageBitmap is not same-origin"); + return nullptr; + } + + if (!aInit.mTimestamp.WasPassed()) { + aRv.ThrowTypeError("Missing timestamp"); + return nullptr; + } + + RefPtr<layers::SourceSurfaceImage> image = + new layers::SourceSurfaceImage(data->mSurface.get()); + // TODO: Take care of data->mAlphaType + auto r = InitializeFrameWithResourceAndSize(global, aInit, image.forget()); + if (r.isErr()) { + aRv.ThrowTypeError(r.unwrapErr()); + return nullptr; + } + return r.unwrap(); +} + +/* static */ +already_AddRefed<VideoFrame> VideoFrame::Constructor( + const GlobalObject& aGlobal, VideoFrame& aVideoFrame, + const VideoFrameInit& aInit, ErrorResult& aRv) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + // Check the usability. + if (!aVideoFrame.mResource) { + aRv.ThrowInvalidStateError( + "The VideoFrame is closed or no image found there"); + return nullptr; + } + MOZ_ASSERT(aVideoFrame.mResource->mImage->GetSize() == + aVideoFrame.mCodedSize); + MOZ_ASSERT(!aVideoFrame.mCodedSize.IsEmpty()); + MOZ_ASSERT(!aVideoFrame.mVisibleRect.IsEmpty()); + MOZ_ASSERT(!aVideoFrame.mDisplaySize.IsEmpty()); + + // If the origin of the VideoFrame is not same origin with the entry settings + // object's origin, then throw a SecurityError DOMException. + if (!IsSameOrigin(global.get(), aVideoFrame)) { + aRv.ThrowSecurityError("The VideoFrame is not same-origin"); + return nullptr; + } + + auto r = InitializeFrameFromOtherFrame( + global.get(), aVideoFrame.GetVideoFrameData(), aInit); + if (r.isErr()) { + aRv.ThrowTypeError(r.unwrapErr()); + return nullptr; + } + return r.unwrap(); +} + +// The following constructors are defined in +// https://w3c.github.io/webcodecs/#dom-videoframe-videoframe-data-init + +/* static */ +already_AddRefed<VideoFrame> VideoFrame::Constructor( + const GlobalObject& aGlobal, const ArrayBufferView& aBufferView, + const VideoFrameBufferInit& aInit, ErrorResult& aRv) { + return CreateVideoFrameFromBuffer(aGlobal, aBufferView, aInit, aRv); +} + +/* static */ +already_AddRefed<VideoFrame> VideoFrame::Constructor( + const GlobalObject& aGlobal, const ArrayBuffer& aBuffer, + const VideoFrameBufferInit& aInit, ErrorResult& aRv) { + return CreateVideoFrameFromBuffer(aGlobal, aBuffer, aInit, aRv); +} + +// https://w3c.github.io/webcodecs/#dom-videoframe-format +Nullable<VideoPixelFormat> VideoFrame::GetFormat() const { + AssertIsOnOwningThread(); + + return mResource ? MaybeToNullable(mResource->TryPixelFormat()) + : Nullable<VideoPixelFormat>(); +} + +// https://w3c.github.io/webcodecs/#dom-videoframe-codedwidth +uint32_t VideoFrame::CodedWidth() const { + AssertIsOnOwningThread(); + + return static_cast<uint32_t>(mCodedSize.Width()); +} + +// https://w3c.github.io/webcodecs/#dom-videoframe-codedheight +uint32_t VideoFrame::CodedHeight() const { + AssertIsOnOwningThread(); + + return static_cast<uint32_t>(mCodedSize.Height()); +} + +// https://w3c.github.io/webcodecs/#dom-videoframe-codedrect +already_AddRefed<DOMRectReadOnly> VideoFrame::GetCodedRect() const { + AssertIsOnOwningThread(); + + return mResource + ? MakeAndAddRef<DOMRectReadOnly>( + mParent, 0.0f, 0.0f, static_cast<double>(mCodedSize.Width()), + static_cast<double>(mCodedSize.Height())) + : nullptr; +} + +// https://w3c.github.io/webcodecs/#dom-videoframe-visiblerect +already_AddRefed<DOMRectReadOnly> VideoFrame::GetVisibleRect() const { + AssertIsOnOwningThread(); + + return mResource ? MakeAndAddRef<DOMRectReadOnly>( + mParent, static_cast<double>(mVisibleRect.X()), + static_cast<double>(mVisibleRect.Y()), + static_cast<double>(mVisibleRect.Width()), + static_cast<double>(mVisibleRect.Height())) + : nullptr; +} + +// https://w3c.github.io/webcodecs/#dom-videoframe-displaywidth +uint32_t VideoFrame::DisplayWidth() const { + AssertIsOnOwningThread(); + + return static_cast<uint32_t>(mDisplaySize.Width()); +} + +// https://w3c.github.io/webcodecs/#dom-videoframe-displayheight +uint32_t VideoFrame::DisplayHeight() const { + AssertIsOnOwningThread(); + + return static_cast<uint32_t>(mDisplaySize.Height()); +} + +// https://w3c.github.io/webcodecs/#dom-videoframe-duration +Nullable<uint64_t> VideoFrame::GetDuration() const { + AssertIsOnOwningThread(); + return MaybeToNullable(mDuration); +} + +// https://w3c.github.io/webcodecs/#dom-videoframe-timestamp +int64_t VideoFrame::Timestamp() const { + AssertIsOnOwningThread(); + + return mTimestamp; +} + +// https://w3c.github.io/webcodecs/#dom-videoframe-colorspace +already_AddRefed<VideoColorSpace> VideoFrame::ColorSpace() const { + AssertIsOnOwningThread(); + + return MakeAndAddRef<VideoColorSpace>(mParent, mColorSpace); +} + +// https://w3c.github.io/webcodecs/#dom-videoframe-allocationsize +uint32_t VideoFrame::AllocationSize(const VideoFrameCopyToOptions& aOptions, + ErrorResult& aRv) { + AssertIsOnOwningThread(); + + if (!mResource) { + aRv.ThrowInvalidStateError("No media resource in VideoFrame"); + return 0; + } + + if (!mResource->mFormat) { + aRv.ThrowAbortError("The VideoFrame image format is not VideoPixelFormat"); + return 0; + } + + auto r = ParseVideoFrameCopyToOptions(aOptions, mVisibleRect, mCodedSize, + mResource->mFormat.ref()); + if (r.isErr()) { + // TODO: Should throw layout. + aRv.ThrowTypeError(r.unwrapErr()); + return 0; + } + CombinedBufferLayout layout = r.unwrap(); + + return layout.mAllocationSize; +} + +// https://w3c.github.io/webcodecs/#dom-videoframe-copyto +already_AddRefed<Promise> VideoFrame::CopyTo( + const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination, + const VideoFrameCopyToOptions& aOptions, ErrorResult& aRv) { + AssertIsOnOwningThread(); + + if (!mResource) { + aRv.ThrowInvalidStateError("No media resource in VideoFrame"); + return nullptr; + } + + if (!mResource->mFormat) { + aRv.ThrowNotSupportedError("VideoFrame's image format is unrecognized"); + return nullptr; + } + + RefPtr<Promise> p = Promise::Create(mParent.get(), aRv); + if (NS_WARN_IF(aRv.Failed())) { + return p.forget(); + } + + CombinedBufferLayout layout; + auto r1 = ParseVideoFrameCopyToOptions(aOptions, mVisibleRect, mCodedSize, + mResource->mFormat.ref()); + if (r1.isErr()) { + // TODO: Should reject with layout. + p->MaybeRejectWithTypeError(r1.unwrapErr()); + return p.forget(); + } + layout = r1.unwrap(); + + return ProcessTypedArraysFixed(aDestination, [&](const Span<uint8_t>& aData) { + if (aData.size_bytes() < layout.mAllocationSize) { + p->MaybeRejectWithTypeError("Destination buffer is too small"); + return p.forget(); + } + + Sequence<PlaneLayout> planeLayouts; + + nsTArray<Format::Plane> planes = mResource->mFormat->Planes(); + MOZ_ASSERT(layout.mComputedLayouts.Length() == planes.Length()); + + // TODO: These jobs can be run in a thread pool (bug 1780656) to unblock + // the current thread. + for (size_t i = 0; i < layout.mComputedLayouts.Length(); ++i) { + ComputedPlaneLayout& l = layout.mComputedLayouts[i]; + uint32_t destinationOffset = l.mDestinationOffset; + + PlaneLayout* pl = planeLayouts.AppendElement(fallible); + if (!pl) { + p->MaybeRejectWithTypeError("Out of memory"); + return p.forget(); + } + pl->mOffset = l.mDestinationOffset; + pl->mStride = l.mDestinationStride; + + // Copy pixels of `size` starting from `origin` on planes[i] to + // `aDestination`. + gfx::IntPoint origin( + l.mSourceLeftBytes / mResource->mFormat->SampleBytes(planes[i]), + l.mSourceTop); + gfx::IntSize size( + l.mSourceWidthBytes / mResource->mFormat->SampleBytes(planes[i]), + l.mSourceHeight); + if (!mResource->CopyTo(planes[i], {origin, size}, + aData.From(destinationOffset), + static_cast<size_t>(l.mDestinationStride))) { + p->MaybeRejectWithTypeError( + nsPrintfCString("Failed to copy image data in %s plane", + mResource->mFormat->PlaneName(planes[i]))); + return p.forget(); + } + } + + MOZ_ASSERT(layout.mComputedLayouts.Length() == planes.Length()); + // TODO: Spec doesn't resolve with a value. See + // https://github.com/w3c/webcodecs/issues/510 This comment should be + // removed once the issue is resolved. + p->MaybeResolve(planeLayouts); + return p.forget(); + }); +} + +// https://w3c.github.io/webcodecs/#dom-videoframe-clone +already_AddRefed<VideoFrame> VideoFrame::Clone(ErrorResult& aRv) const { + AssertIsOnOwningThread(); + + if (!mResource) { + aRv.ThrowInvalidStateError("No media resource in the VideoFrame now"); + return nullptr; + } + // The VideoFrame's data must be shared instead of copied: + // https://w3c.github.io/webcodecs/#raw-media-memory-model-reference-counting + return MakeAndAddRef<VideoFrame>(*this); +} + +// https://w3c.github.io/webcodecs/#close-videoframe +void VideoFrame::Close() { + AssertIsOnOwningThread(); + LOG("VideoFrame %p is closed", this); + + mResource.reset(); + mCodedSize = gfx::IntSize(); + mVisibleRect = gfx::IntRect(); + mDisplaySize = gfx::IntSize(); + mColorSpace = VideoColorSpaceInit(); + + StopAutoClose(); +} + +bool VideoFrame::IsClosed() const { return !mResource; } + +already_AddRefed<layers::Image> VideoFrame::GetImage() const { + if (!mResource) { + return nullptr; + } + return do_AddRef(mResource->mImage); +} + +nsCString VideoFrame::ToString() const { + nsCString rv; + + if (IsClosed()) { + rv.AppendPrintf("VideoFrame (closed)"); + return rv; + } + + rv.AppendPrintf( + "VideoFrame ts: %" PRId64 + ", %s, coded[%dx%d] visible[%dx%d], display[%dx%d] color: %s", + mTimestamp, + dom::VideoPixelFormatValues::GetString(mResource->mFormat->PixelFormat()) + .data(), + mCodedSize.width, mCodedSize.height, mVisibleRect.width, + mVisibleRect.height, mDisplaySize.width, mDisplaySize.height, + ColorSpaceInitToString(mColorSpace).get()); + + if (mDuration) { + rv.AppendPrintf(" dur: %" PRId64, mDuration.value()); + } + + return rv; +} + +// https://w3c.github.io/webcodecs/#ref-for-deserialization-steps%E2%91%A0 +/* static */ +JSObject* VideoFrame::ReadStructuredClone( + JSContext* aCx, nsIGlobalObject* aGlobal, JSStructuredCloneReader* aReader, + const VideoFrameSerializedData& aData) { + JS::Rooted<JS::Value> value(aCx, JS::NullValue()); + // To avoid a rooting hazard error from returning a raw JSObject* before + // running the RefPtr destructor, RefPtr needs to be destructed before + // returning the raw JSObject*, which is why the RefPtr<VideoFrame> is created + // in the scope below. Otherwise, the static analysis infers the RefPtr cannot + // be safely destructed while the unrooted return JSObject* is on the stack. + { + RefPtr<VideoFrame> frame = MakeAndAddRef<VideoFrame>(aGlobal, aData); + if (!GetOrCreateDOMReflector(aCx, frame, &value) || !value.isObject()) { + return nullptr; + } + } + return value.toObjectOrNull(); +} + +// https://w3c.github.io/webcodecs/#ref-for-serialization-steps%E2%91%A0 +bool VideoFrame::WriteStructuredClone(JSStructuredCloneWriter* aWriter, + StructuredCloneHolder* aHolder) const { + AssertIsOnOwningThread(); + + if (!mResource) { + return false; + } + + // Indexing the image and send the index to the receiver. + const uint32_t index = aHolder->VideoFrames().Length(); + // The serialization is limited to the same process scope so it's ok to + // serialize a reference instead of a copy. + aHolder->VideoFrames().AppendElement( + VideoFrameSerializedData(GetVideoFrameData(), mCodedSize)); + + return !NS_WARN_IF(!JS_WriteUint32Pair(aWriter, SCTAG_DOM_VIDEOFRAME, index)); +} + +// https://w3c.github.io/webcodecs/#ref-for-transfer-steps%E2%91%A0 +UniquePtr<VideoFrame::TransferredData> VideoFrame::Transfer() { + AssertIsOnOwningThread(); + + if (!mResource) { + return nullptr; + } + + auto frame = MakeUnique<TransferredData>(GetVideoFrameData(), mCodedSize); + Close(); + return frame; +} + +// https://w3c.github.io/webcodecs/#ref-for-transfer-receiving-steps%E2%91%A0 +/* static */ +already_AddRefed<VideoFrame> VideoFrame::FromTransferred( + nsIGlobalObject* aGlobal, TransferredData* aData) { + MOZ_ASSERT(aData); + + return MakeAndAddRef<VideoFrame>(aGlobal, *aData); +} + +VideoFrameData VideoFrame::GetVideoFrameData() const { + return VideoFrameData(mResource->mImage.get(), mResource->TryPixelFormat(), + mVisibleRect, mDisplaySize, mDuration, mTimestamp, + mColorSpace); +} + +void VideoFrame::StartAutoClose() { + AssertIsOnOwningThread(); + + LOG("VideoFrame %p, start monitoring resource release", this); + + if (NS_IsMainThread()) { + mShutdownBlocker = media::ShutdownBlockingTicket::Create( + u"VideoFrame::mShutdownBlocker"_ns, + NS_LITERAL_STRING_FROM_CSTRING(__FILE__), __LINE__); + if (mShutdownBlocker) { + mShutdownBlocker->ShutdownPromise()->Then( + GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}](bool /* aUnUsed*/) { + LOG("VideoFrame %p gets shutdown notification", self.get()); + self->CloseIfNeeded(); + }, + [self = RefPtr{this}](bool /* aUnUsed*/) { + LOG("VideoFrame %p removes shutdown-blocker before getting " + "shutdown " + "notification", + self.get()); + }); + } + } else if (WorkerPrivate* workerPrivate = GetCurrentThreadWorkerPrivate()) { + // Clean up all the resources when the worker is going away. + mWorkerRef = WeakWorkerRef::Create(workerPrivate, [self = RefPtr{this}]() { + LOG("VideoFrame %p, worker is going away", self.get()); + self->CloseIfNeeded(); + }); + } +} + +void VideoFrame::StopAutoClose() { + AssertIsOnOwningThread(); + + LOG("VideoFrame %p, stop monitoring resource release", this); + + mShutdownBlocker = nullptr; + mWorkerRef = nullptr; +} + +void VideoFrame::CloseIfNeeded() { + AssertIsOnOwningThread(); + + LOG("VideoFrame %p, needs to close itself? %s", this, + IsClosed() ? "no" : "yes"); + if (!IsClosed()) { + LOG("Close VideoFrame %p obligatorily", this); + Close(); + } +} + +/* + * VideoFrame::Format + * + * This class wraps a VideoPixelFormat defined in [1] and provides some + * utilities for the VideoFrame's functions. Each sample in the format is 8 + * bits. The pixel layouts for a 4 x 2 image in the spec are illustrated below: + * [1] https://w3c.github.io/webcodecs/#pixel-format + * + * I420 - 3 planes: Y, U, V + * ------ + * <- width -> + * Y: Y1 Y2 Y3 Y4 ^ height + * Y5 Y6 Y7 Y8 v + * U: U1 U2 => 1/2 Y's width, 1/2 Y's height + * V: V1 V2 => 1/2 Y's width, 1/2 Y's height + * + * I420A - 4 planes: Y, U, V, A + * ------ + * <- width -> + * Y: Y1 Y2 Y3 Y4 ^ height + * Y5 Y6 Y7 Y8 v + * U: U1 U2 => 1/2 Y's width, 1/2 Y's height + * V: V1 V2 => 1/2 Y's width, 1/2 Y's height + * A: A1 A2 A3 A4 => Y's width, Y's height + * A5 A6 A7 A8 + * + * I422 - 3 planes: Y, U, V + * ------ + * <- width -> + * Y: Y1 Y2 Y3 Y4 ^ height + * Y5 Y6 Y7 Y8 v + * U: U1 U2 U3 U4 => Y's width, 1/2 Y's height + * V: V1 V2 V3 V4 => Y's width, 1/2 Y's height + * + * I444 - 3 planes: Y, U, V + * ------ + * <- width -> + * Y: Y1 Y2 Y3 Y4 ^ height + * Y5 Y6 Y7 Y8 v + * U: U1 U2 U3 U4 => Y's width, Y's height + * U5 U6 U7 U8 + * V: V1 V2 V3 V4 => Y's width, Y's height + * V5 V6 V7 B8 + * + * NV12 - 2 planes: Y, UV + * ------ + * <- width -> + * Y: Y1 Y2 Y3 Y4 ^ height + * Y5 Y6 Y7 Y8 v + * UV: U1 V1 U2 V2 => Y's width, 1/2 Y's height + * + * RGBA - 1 plane encoding 3 colors: Red, Green, Blue, and an Alpha value + * ------ + * <---------------------- width ----------------------> + * R1 G1 B1 A1 | R2 G2 B2 A2 | R3 G3 B3 A3 | R4 G4 B4 A4 ^ height + * R5 G5 B5 A5 | R6 G6 B6 A6 | R7 G7 B7 A7 | R8 G8 B8 A8 v + * + * RGBX - 1 plane encoding 3 colors: Red, Green, Blue, and an padding value + * This is the opaque version of RGBA + * ------ + * <---------------------- width ----------------------> + * R1 G1 B1 X1 | R2 G2 B2 X2 | R3 G3 B3 X3 | R4 G4 B4 X4 ^ height + * R5 G5 B5 X5 | R6 G6 B6 X6 | R7 G7 B7 X7 | R8 G8 B8 X8 v + * + * BGRA - 1 plane encoding 3 colors: Blue, Green, Red, and an Alpha value + * ------ + * <---------------------- width ----------------------> + * B1 G1 R1 A1 | B2 G2 R2 A2 | B3 G3 R3 A3 | B4 G4 R4 A4 ^ height + * B5 G5 R5 A5 | B6 G6 R6 A6 | B7 G7 R7 A7 | B8 G8 R8 A8 v + * + * BGRX - 1 plane encoding 3 colors: Blue, Green, Red, and an padding value + * This is the opaque version of BGRA + * ------ + * <---------------------- width ----------------------> + * B1 G1 R1 X1 | B2 G2 R2 X2 | B3 G3 R3 X3 | B4 G4 R4 X4 ^ height + * B5 G5 R5 X5 | B6 G6 R6 X6 | B7 G7 R7 X7 | B8 G8 R8 X8 v + */ + +VideoFrame::Format::Format(const VideoPixelFormat& aFormat) + : mFormat(aFormat) {} + +const VideoPixelFormat& VideoFrame::Format::PixelFormat() const { + return mFormat; +} + +gfx::SurfaceFormat VideoFrame::Format::ToSurfaceFormat() const { + gfx::SurfaceFormat format = gfx::SurfaceFormat::UNKNOWN; + switch (mFormat) { + case VideoPixelFormat::I420: + case VideoPixelFormat::I420A: + case VideoPixelFormat::I422: + case VideoPixelFormat::I444: + case VideoPixelFormat::NV12: + // Not yet support for now. + break; + case VideoPixelFormat::RGBA: + format = gfx::SurfaceFormat::R8G8B8A8; + break; + case VideoPixelFormat::RGBX: + format = gfx::SurfaceFormat::R8G8B8X8; + break; + case VideoPixelFormat::BGRA: + format = gfx::SurfaceFormat::B8G8R8A8; + break; + case VideoPixelFormat::BGRX: + format = gfx::SurfaceFormat::B8G8R8X8; + break; + case VideoPixelFormat::EndGuard_: + MOZ_ASSERT_UNREACHABLE("unsupported format"); + } + return format; +} + +void VideoFrame::Format::MakeOpaque() { + switch (mFormat) { + case VideoPixelFormat::I420A: + mFormat = VideoPixelFormat::I420; + return; + case VideoPixelFormat::RGBA: + mFormat = VideoPixelFormat::RGBX; + return; + case VideoPixelFormat::BGRA: + mFormat = VideoPixelFormat::BGRX; + return; + case VideoPixelFormat::I420: + case VideoPixelFormat::I422: + case VideoPixelFormat::I444: + case VideoPixelFormat::NV12: + case VideoPixelFormat::RGBX: + case VideoPixelFormat::BGRX: + return; + case VideoPixelFormat::EndGuard_: + break; + } + MOZ_ASSERT_UNREACHABLE("unsupported format"); +} + +nsTArray<VideoFrame::Format::Plane> VideoFrame::Format::Planes() const { + switch (mFormat) { + case VideoPixelFormat::I420: + case VideoPixelFormat::I422: + case VideoPixelFormat::I444: + return {Plane::Y, Plane::U, Plane::V}; + case VideoPixelFormat::I420A: + return {Plane::Y, Plane::U, Plane::V, Plane::A}; + case VideoPixelFormat::NV12: + return {Plane::Y, Plane::UV}; + case VideoPixelFormat::RGBA: + case VideoPixelFormat::RGBX: + case VideoPixelFormat::BGRA: + case VideoPixelFormat::BGRX: + return {Plane::RGBA}; + case VideoPixelFormat::EndGuard_: + break; + } + MOZ_ASSERT_UNREACHABLE("unsupported format"); + return {}; +} + +const char* VideoFrame::Format::PlaneName(const Plane& aPlane) const { + switch (aPlane) { + case Format::Plane::Y: // and RGBA + return IsYUV() ? "Y" : "RGBA"; + case Format::Plane::U: // and UV + MOZ_ASSERT(IsYUV()); + return mFormat == VideoPixelFormat::NV12 ? "UV" : "U"; + case Format::Plane::V: + MOZ_ASSERT(IsYUV()); + return "V"; + case Format::Plane::A: + MOZ_ASSERT(IsYUV()); + return "A"; + } + MOZ_ASSERT_UNREACHABLE("invalid plane"); + return "Unknown"; +} + +uint32_t VideoFrame::Format::SampleBytes(const Plane& aPlane) const { + switch (mFormat) { + case VideoPixelFormat::I420: + case VideoPixelFormat::I420A: + case VideoPixelFormat::I422: + case VideoPixelFormat::I444: + return 1; // 8 bits/sample on the Y, U, V, A plane. + case VideoPixelFormat::NV12: + switch (aPlane) { + case Plane::Y: + return 1; // 8 bits/sample on the Y plane + case Plane::UV: + return 2; // Interleaved U and V values on the UV plane. + case Plane::V: + case Plane::A: + MOZ_ASSERT_UNREACHABLE("invalid plane"); + } + return 0; + case VideoPixelFormat::RGBA: + case VideoPixelFormat::RGBX: + case VideoPixelFormat::BGRA: + case VideoPixelFormat::BGRX: + return 4; // 8 bits/sample, 32 bits/pixel + case VideoPixelFormat::EndGuard_: + break; + } + MOZ_ASSERT_UNREACHABLE("unsupported format"); + return 0; +} + +gfx::IntSize VideoFrame::Format::SampleSize(const Plane& aPlane) const { + // The sample width and height refers to + // https://w3c.github.io/webcodecs/#sub-sampling-factor + switch (aPlane) { + case Plane::Y: // and RGBA + case Plane::A: + return gfx::IntSize(1, 1); + case Plane::U: // and UV + case Plane::V: + switch (mFormat) { + case VideoPixelFormat::I420: + case VideoPixelFormat::I420A: + case VideoPixelFormat::NV12: + return gfx::IntSize(2, 2); + case VideoPixelFormat::I422: + return gfx::IntSize(2, 1); + case VideoPixelFormat::I444: + return gfx::IntSize(1, 1); + case VideoPixelFormat::RGBA: + case VideoPixelFormat::RGBX: + case VideoPixelFormat::BGRA: + case VideoPixelFormat::BGRX: + case VideoPixelFormat::EndGuard_: + MOZ_ASSERT_UNREACHABLE("invalid format"); + return {0, 0}; + } + } + MOZ_ASSERT_UNREACHABLE("invalid plane"); + return {0, 0}; +} + +bool VideoFrame::Format::IsValidSize(const gfx::IntSize& aSize) const { + switch (mFormat) { + case VideoPixelFormat::I420: + case VideoPixelFormat::I420A: + case VideoPixelFormat::NV12: + return (aSize.Width() % 2 == 0) && (aSize.Height() % 2 == 0); + case VideoPixelFormat::I422: + return aSize.Height() % 2 == 0; + case VideoPixelFormat::I444: + case VideoPixelFormat::RGBA: + case VideoPixelFormat::RGBX: + case VideoPixelFormat::BGRA: + case VideoPixelFormat::BGRX: + return true; + case VideoPixelFormat::EndGuard_: + break; + } + MOZ_ASSERT_UNREACHABLE("unsupported format"); + return false; +} + +size_t VideoFrame::Format::SampleCount(const gfx::IntSize& aSize) const { + MOZ_ASSERT(IsValidSize(aSize)); + + CheckedInt<size_t> count(aSize.Width()); + count *= aSize.Height(); + + switch (mFormat) { + case VideoPixelFormat::I420: + case VideoPixelFormat::NV12: + return (count + count / 2).value(); + case VideoPixelFormat::I420A: + return (count * 2 + count / 2).value(); + case VideoPixelFormat::I422: + return (count * 2).value(); + case VideoPixelFormat::I444: + return (count * 3).value(); + case VideoPixelFormat::RGBA: + case VideoPixelFormat::RGBX: + case VideoPixelFormat::BGRA: + case VideoPixelFormat::BGRX: + return (count * 4).value(); + case VideoPixelFormat::EndGuard_: + break; + } + + MOZ_ASSERT_UNREACHABLE("unsupported format"); + return 0; +} + +bool VideoFrame::Format::IsYUV() const { return IsYUVFormat(mFormat); } + +/* + * VideoFrame::Resource + */ + +VideoFrame::Resource::Resource(const RefPtr<layers::Image>& aImage, + Maybe<class Format>&& aFormat) + : mImage(aImage), mFormat(aFormat) { + MOZ_ASSERT(mImage); +} + +VideoFrame::Resource::Resource(const Resource& aOther) + : mImage(aOther.mImage), mFormat(aOther.mFormat) { + MOZ_ASSERT(mImage); +} + +Maybe<VideoPixelFormat> VideoFrame::Resource::TryPixelFormat() const { + return mFormat ? Some(mFormat->PixelFormat()) : Nothing(); +} + +uint32_t VideoFrame::Resource::Stride(const Format::Plane& aPlane) const { + MOZ_RELEASE_ASSERT(mFormat); + + CheckedInt<uint32_t> width(mImage->GetSize().Width()); + switch (aPlane) { + case Format::Plane::Y: // and RGBA + case Format::Plane::A: + switch (mFormat->PixelFormat()) { + case VideoPixelFormat::I420: + case VideoPixelFormat::I420A: + case VideoPixelFormat::I422: + case VideoPixelFormat::I444: + case VideoPixelFormat::NV12: + case VideoPixelFormat::RGBA: + case VideoPixelFormat::RGBX: + case VideoPixelFormat::BGRA: + case VideoPixelFormat::BGRX: + return (width * mFormat->SampleBytes(aPlane)).value(); + case VideoPixelFormat::EndGuard_: + MOZ_ASSERT_UNREACHABLE("invalid format"); + } + return 0; + case Format::Plane::U: // and UV + case Format::Plane::V: + switch (mFormat->PixelFormat()) { + case VideoPixelFormat::I420: + case VideoPixelFormat::I420A: + case VideoPixelFormat::I422: + case VideoPixelFormat::I444: + case VideoPixelFormat::NV12: + return (((width + 1) / 2) * mFormat->SampleBytes(aPlane)).value(); + case VideoPixelFormat::RGBA: + case VideoPixelFormat::RGBX: + case VideoPixelFormat::BGRA: + case VideoPixelFormat::BGRX: + case VideoPixelFormat::EndGuard_: + MOZ_ASSERT_UNREACHABLE("invalid format"); + } + return 0; + } + MOZ_ASSERT_UNREACHABLE("invalid plane"); + return 0; +} + +bool VideoFrame::Resource::CopyTo(const Format::Plane& aPlane, + const gfx::IntRect& aRect, + Span<uint8_t>&& aPlaneDest, + size_t aDestinationStride) const { + if (!mFormat) { + return false; + } + + auto copyPlane = [&](const uint8_t* aPlaneData) { + MOZ_ASSERT(aPlaneData); + + CheckedInt<size_t> offset(aRect.Y()); + offset *= Stride(aPlane); + offset += aRect.X() * mFormat->SampleBytes(aPlane); + if (!offset.isValid()) { + return false; + } + + CheckedInt<size_t> elementsBytes(aRect.Width()); + elementsBytes *= mFormat->SampleBytes(aPlane); + if (!elementsBytes.isValid()) { + return false; + } + + aPlaneData += offset.value(); + for (int32_t row = 0; row < aRect.Height(); ++row) { + PodCopy(aPlaneDest.data(), aPlaneData, elementsBytes.value()); + aPlaneData += Stride(aPlane); + // Spec asks to move `aDestinationStride` bytes instead of + // `Stride(aPlane)` forward. + aPlaneDest = aPlaneDest.From(aDestinationStride); + } + return true; + }; + + if (mImage->GetFormat() == ImageFormat::MOZ2D_SURFACE) { + RefPtr<gfx::SourceSurface> surface = mImage->GetAsSourceSurface(); + if (NS_WARN_IF(!surface)) { + return false; + } + + RefPtr<gfx::DataSourceSurface> dataSurface = surface->GetDataSurface(); + if (NS_WARN_IF(!dataSurface)) { + return false; + } + + gfx::DataSourceSurface::ScopedMap map(dataSurface, + gfx::DataSourceSurface::READ); + if (NS_WARN_IF(!map.IsMapped())) { + return false; + } + + const gfx::SurfaceFormat format = dataSurface->GetFormat(); + + if (format == gfx::SurfaceFormat::R8G8B8A8 || + format == gfx::SurfaceFormat::R8G8B8X8 || + format == gfx::SurfaceFormat::B8G8R8A8 || + format == gfx::SurfaceFormat::B8G8R8X8) { + MOZ_ASSERT(aPlane == Format::Plane::RGBA); + + // The mImage's format can be different from mFormat (since Gecko prefers + // BGRA). To get the data in the matched format, we create a temp buffer + // holding the image data in that format and then copy them to + // `aDestination`. + const gfx::SurfaceFormat f = mFormat->ToSurfaceFormat(); + MOZ_ASSERT(f == gfx::SurfaceFormat::R8G8B8A8 || + f == gfx::SurfaceFormat::R8G8B8X8 || + f == gfx::SurfaceFormat::B8G8R8A8 || + f == gfx::SurfaceFormat::B8G8R8X8); + + // TODO: We could use Factory::CreateWrappingDataSourceSurface to wrap + // `aDestination` to avoid extra copy. + RefPtr<gfx::DataSourceSurface> tempSurface = + gfx::Factory::CreateDataSourceSurfaceWithStride( + dataSurface->GetSize(), f, map.GetStride()); + if (NS_WARN_IF(!tempSurface)) { + return false; + } + + gfx::DataSourceSurface::ScopedMap tempMap(tempSurface, + gfx::DataSourceSurface::WRITE); + if (NS_WARN_IF(!tempMap.IsMapped())) { + return false; + } + + if (!gfx::SwizzleData(map.GetData(), map.GetStride(), + dataSurface->GetFormat(), tempMap.GetData(), + tempMap.GetStride(), tempSurface->GetFormat(), + tempSurface->GetSize())) { + return false; + } + + return copyPlane(tempMap.GetData()); + } + + return false; + } + + if (mImage->GetFormat() == ImageFormat::PLANAR_YCBCR) { + switch (aPlane) { + case Format::Plane::Y: + return copyPlane(mImage->AsPlanarYCbCrImage()->GetData()->mYChannel); + case Format::Plane::U: + return copyPlane(mImage->AsPlanarYCbCrImage()->GetData()->mCbChannel); + case Format::Plane::V: + return copyPlane(mImage->AsPlanarYCbCrImage()->GetData()->mCrChannel); + case Format::Plane::A: + MOZ_ASSERT(mFormat->PixelFormat() == VideoPixelFormat::I420A); + MOZ_ASSERT(mImage->AsPlanarYCbCrImage()->GetData()->mAlpha); + return copyPlane( + mImage->AsPlanarYCbCrImage()->GetData()->mAlpha->mChannel); + } + MOZ_ASSERT_UNREACHABLE("invalid plane"); + } + + if (mImage->GetFormat() == ImageFormat::NV_IMAGE) { + switch (aPlane) { + case Format::Plane::Y: + return copyPlane(mImage->AsNVImage()->GetData()->mYChannel); + case Format::Plane::UV: + return copyPlane(mImage->AsNVImage()->GetData()->mCbChannel); + case Format::Plane::V: + case Format::Plane::A: + MOZ_ASSERT_UNREACHABLE("invalid plane"); + } + } + + // TODO: ImageFormat::MAC_IOSURFACE or ImageFormat::DMABUF + LOGW("Cannot copy image data of an unrecognized format"); + + return false; +} + +#undef LOGW +#undef LOG_INTERNAL + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/VideoFrame.h b/dom/media/webcodecs/VideoFrame.h new file mode 100644 index 0000000000..0bef496b79 --- /dev/null +++ b/dom/media/webcodecs/VideoFrame.h @@ -0,0 +1,266 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_VideoFrame_h +#define mozilla_dom_VideoFrame_h + +#include "js/TypeDecls.h" +#include "mozilla/Attributes.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/NotNull.h" +#include "mozilla/Span.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/TypedArray.h" +#include "mozilla/dom/VideoColorSpaceBinding.h" +#include "mozilla/dom/WorkerRef.h" +#include "mozilla/gfx/Point.h" +#include "mozilla/gfx/Rect.h" +#include "mozilla/media/MediaUtils.h" +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; +class nsIURI; + +namespace mozilla { + +namespace layers { +class Image; +} // namespace layers + +namespace dom { + +class DOMRectReadOnly; +class HTMLCanvasElement; +class HTMLImageElement; +class HTMLVideoElement; +class ImageBitmap; +class MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer; +class OffscreenCanvas; +class OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer; +class Promise; +class SVGImageElement; +class StructuredCloneHolder; +class VideoColorSpace; +class VideoFrame; +enum class VideoPixelFormat : uint8_t; +struct VideoFrameBufferInit; +struct VideoFrameInit; +struct VideoFrameCopyToOptions; + +} // namespace dom +} // namespace mozilla + +namespace mozilla::dom { + +struct VideoFrameData { + VideoFrameData(layers::Image* aImage, const Maybe<VideoPixelFormat>& aFormat, + gfx::IntRect aVisibleRect, gfx::IntSize aDisplaySize, + Maybe<uint64_t> aDuration, int64_t aTimestamp, + const VideoColorSpaceInit& aColorSpace); + VideoFrameData(const VideoFrameData& aData) = default; + + const RefPtr<layers::Image> mImage; + const Maybe<VideoPixelFormat> mFormat; + const gfx::IntRect mVisibleRect; + const gfx::IntSize mDisplaySize; + const Maybe<uint64_t> mDuration; + const int64_t mTimestamp; + const VideoColorSpaceInit mColorSpace; +}; + +struct VideoFrameSerializedData : VideoFrameData { + VideoFrameSerializedData(const VideoFrameData& aData, + gfx::IntSize aCodedSize); + + const gfx::IntSize mCodedSize; +}; + +class VideoFrame final : public nsISupports, public nsWrapperCache { + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(VideoFrame) + + public: + VideoFrame(nsIGlobalObject* aParent, const RefPtr<layers::Image>& aImage, + const Maybe<VideoPixelFormat>& aFormat, gfx::IntSize aCodedSize, + gfx::IntRect aVisibleRect, gfx::IntSize aDisplaySize, + const Maybe<uint64_t>& aDuration, int64_t aTimestamp, + const VideoColorSpaceInit& aColorSpace); + VideoFrame(nsIGlobalObject* aParent, const VideoFrameSerializedData& aData); + VideoFrame(const VideoFrame& aOther); + + protected: + ~VideoFrame(); + + public: + nsIGlobalObject* GetParentObject() const; + + JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override; + + static already_AddRefed<VideoFrame> Constructor( + const GlobalObject& aGlobal, HTMLImageElement& aImageElement, + const VideoFrameInit& aInit, ErrorResult& aRv); + static already_AddRefed<VideoFrame> Constructor( + const GlobalObject& aGlobal, SVGImageElement& aSVGImageElement, + const VideoFrameInit& aInit, ErrorResult& aRv); + static already_AddRefed<VideoFrame> Constructor( + const GlobalObject& aGlobal, HTMLCanvasElement& aCanvasElement, + const VideoFrameInit& aInit, ErrorResult& aRv); + static already_AddRefed<VideoFrame> Constructor( + const GlobalObject& aGlobal, HTMLVideoElement& aVideoElement, + const VideoFrameInit& aInit, ErrorResult& aRv); + static already_AddRefed<VideoFrame> Constructor( + const GlobalObject& aGlobal, OffscreenCanvas& aOffscreenCanvas, + const VideoFrameInit& aInit, ErrorResult& aRv); + static already_AddRefed<VideoFrame> Constructor(const GlobalObject& aGlobal, + ImageBitmap& aImageBitmap, + const VideoFrameInit& aInit, + ErrorResult& aRv); + static already_AddRefed<VideoFrame> Constructor(const GlobalObject& aGlobal, + VideoFrame& aVideoFrame, + const VideoFrameInit& aInit, + ErrorResult& aRv); + static already_AddRefed<VideoFrame> Constructor( + const GlobalObject& aGlobal, const ArrayBufferView& aBufferView, + const VideoFrameBufferInit& aInit, ErrorResult& aRv); + static already_AddRefed<VideoFrame> Constructor( + const GlobalObject& aGlobal, const ArrayBuffer& aBuffer, + const VideoFrameBufferInit& aInit, ErrorResult& aRv); + + Nullable<VideoPixelFormat> GetFormat() const; + + uint32_t CodedWidth() const; + + uint32_t CodedHeight() const; + + already_AddRefed<DOMRectReadOnly> GetCodedRect() const; + + already_AddRefed<DOMRectReadOnly> GetVisibleRect() const; + + uint32_t DisplayWidth() const; + + uint32_t DisplayHeight() const; + + Nullable<uint64_t> GetDuration() const; + + int64_t Timestamp() const; + + already_AddRefed<VideoColorSpace> ColorSpace() const; + + uint32_t AllocationSize(const VideoFrameCopyToOptions& aOptions, + ErrorResult& aRv); + + already_AddRefed<Promise> CopyTo( + const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination, + const VideoFrameCopyToOptions& aOptions, ErrorResult& aRv); + + already_AddRefed<VideoFrame> Clone(ErrorResult& aRv) const; + + void Close(); + bool IsClosed() const; + + // [Serializable] implementations: {Read, Write}StructuredClone + static JSObject* ReadStructuredClone(JSContext* aCx, nsIGlobalObject* aGlobal, + JSStructuredCloneReader* aReader, + const VideoFrameSerializedData& aData); + + bool WriteStructuredClone(JSStructuredCloneWriter* aWriter, + StructuredCloneHolder* aHolder) const; + + // [Transferable] implementations: Transfer, FromTransferred + using TransferredData = VideoFrameSerializedData; + + UniquePtr<TransferredData> Transfer(); + + static already_AddRefed<VideoFrame> FromTransferred(nsIGlobalObject* aGlobal, + TransferredData* aData); + + // Native only methods. + const gfx::IntSize& NativeCodedSize() const { return mCodedSize; } + const gfx::IntSize& NativeDisplaySize() const { return mDisplaySize; } + const gfx::IntRect& NativeVisibleRect() const { return mVisibleRect; } + already_AddRefed<layers::Image> GetImage() const; + + nsCString ToString() const; + + public: + // A VideoPixelFormat wrapper providing utilities for VideoFrame. + class Format final { + public: + explicit Format(const VideoPixelFormat& aFormat); + ~Format() = default; + const VideoPixelFormat& PixelFormat() const; + gfx::SurfaceFormat ToSurfaceFormat() const; + void MakeOpaque(); + + // TODO: Assign unique value for each plane? + // The value indicates the order of the plane in format. + enum class Plane : uint8_t { Y = 0, RGBA = Y, U = 1, UV = U, V = 2, A = 3 }; + nsTArray<Plane> Planes() const; + const char* PlaneName(const Plane& aPlane) const; + uint32_t SampleBytes(const Plane& aPlane) const; + gfx::IntSize SampleSize(const Plane& aPlane) const; + bool IsValidSize(const gfx::IntSize& aSize) const; + size_t SampleCount(const gfx::IntSize& aSize) const; + + private: + bool IsYUV() const; + VideoPixelFormat mFormat; + }; + + private: + // VideoFrame can run on either main thread or worker thread. + void AssertIsOnOwningThread() const { NS_ASSERT_OWNINGTHREAD(VideoFrame); } + + VideoFrameData GetVideoFrameData() const; + + // Below helpers are used to automatically release the holding Resource if + // VideoFrame is never Close()d by the users. + void StartAutoClose(); + void StopAutoClose(); + void CloseIfNeeded(); + + // A class representing the VideoFrame's data. + class Resource final { + public: + Resource(const RefPtr<layers::Image>& aImage, Maybe<Format>&& aFormat); + Resource(const Resource& aOther); + ~Resource() = default; + Maybe<VideoPixelFormat> TryPixelFormat() const; + uint32_t Stride(const Format::Plane& aPlane) const; + bool CopyTo(const Format::Plane& aPlane, const gfx::IntRect& aRect, + Span<uint8_t>&& aPlaneDest, size_t aDestinationStride) const; + + const RefPtr<layers::Image> mImage; + // Nothing() if mImage is not in VideoPixelFormat + const Maybe<Format> mFormat; + }; + + nsCOMPtr<nsIGlobalObject> mParent; + + // Use Maybe instead of UniquePtr to allow copy ctor. + // The mResource's existence is used as the [[Detached]] for [Transferable]. + Maybe<const Resource> mResource; // Nothing() after `Close()`d + + // TODO: Replace this by mResource->mImage->GetSize()? + gfx::IntSize mCodedSize; + gfx::IntRect mVisibleRect; + gfx::IntSize mDisplaySize; + + Maybe<uint64_t> mDuration; + int64_t mTimestamp; + VideoColorSpaceInit mColorSpace; + + // The following are used to help monitoring mResource release. + UniquePtr<media::ShutdownBlockingTicket> mShutdownBlocker = nullptr; + RefPtr<WeakWorkerRef> mWorkerRef = nullptr; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_VideoFrame_h diff --git a/dom/media/webcodecs/WebCodecsUtils.cpp b/dom/media/webcodecs/WebCodecsUtils.cpp new file mode 100644 index 0000000000..1e03f616db --- /dev/null +++ b/dom/media/webcodecs/WebCodecsUtils.cpp @@ -0,0 +1,578 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "WebCodecsUtils.h" + +#include "DecoderTypes.h" +#include "VideoUtils.h" +#include "js/experimental/TypedData.h" +#include "mozilla/Assertions.h" +#include "mozilla/dom/ImageBitmapBinding.h" +#include "mozilla/dom/VideoColorSpaceBinding.h" +#include "mozilla/dom/VideoFrameBinding.h" +#include "mozilla/gfx/Types.h" +#include "nsDebug.h" +#include "PlatformEncoderModule.h" +#include "PlatformEncoderModule.h" + +extern mozilla::LazyLogModule gWebCodecsLog; + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) +#ifdef LOG +# undef LOG +#endif // LOG +#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +namespace mozilla { +std::atomic<WebCodecsId> sNextId = 0; +}; + +namespace mozilla::dom { + +/* + * The followings are helpers for VideoDecoder methods + */ + +nsTArray<nsCString> GuessContainers(const nsAString& aCodec) { + if (IsAV1CodecString(aCodec)) { + return {"mp4"_ns, "webm"_ns}; + } + + if (IsVP9CodecString(aCodec)) { + return {"mp4"_ns, "webm"_ns, "ogg"_ns}; + } + + if (IsVP8CodecString(aCodec)) { + return {"webm"_ns, "ogg"_ns, "3gpp"_ns, "3gpp2"_ns, "3gp2"_ns}; + } + + if (IsH264CodecString(aCodec)) { + return {"mp4"_ns, "3gpp"_ns, "3gpp2"_ns, "3gp2"_ns}; + } + + return {}; +} + +Maybe<nsString> ParseCodecString(const nsAString& aCodec) { + // Trim the spaces on each end. + nsString str(aCodec); + str.Trim(" "); + nsTArray<nsString> codecs; + if (!ParseCodecsString(str, codecs) || codecs.Length() != 1 || + codecs[0] != str) { + return Nothing(); + } + return Some(codecs[0]); +} + +/* + * The below are helpers to operate ArrayBuffer or ArrayBufferView. + */ + +static std::tuple<JS::ArrayBufferOrView, size_t, size_t> GetArrayBufferInfo( + JSContext* aCx, + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer) { + if (aBuffer.IsArrayBuffer()) { + const ArrayBuffer& buffer = aBuffer.GetAsArrayBuffer(); + size_t length; + { + bool isShared; + uint8_t* data; + JS::GetArrayBufferMaybeSharedLengthAndData(buffer.Obj(), &length, + &isShared, &data); + } + return std::make_tuple(JS::ArrayBufferOrView::fromObject(buffer.Obj()), + (size_t)0, length); + } + + MOZ_ASSERT(aBuffer.IsArrayBufferView()); + const ArrayBufferView& view = aBuffer.GetAsArrayBufferView(); + bool isSharedMemory; + JS::Rooted<JSObject*> obj(aCx, view.Obj()); + return std::make_tuple( + JS::ArrayBufferOrView::fromObject( + JS_GetArrayBufferViewBuffer(aCx, obj, &isSharedMemory)), + JS_GetArrayBufferViewByteOffset(obj), + JS_GetArrayBufferViewByteLength(obj)); +} + +Result<Ok, nsresult> CloneBuffer( + JSContext* aCx, + OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDest, + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aSrc) { + std::tuple<JS::ArrayBufferOrView, size_t, size_t> info = + GetArrayBufferInfo(aCx, aSrc); + JS::Rooted<JS::ArrayBufferOrView> abov(aCx); + abov.set(std::get<0>(info)); + size_t offset = std::get<1>(info); + size_t len = std::get<2>(info); + if (NS_WARN_IF(!abov)) { + return Err(NS_ERROR_UNEXPECTED); + } + + JS::Rooted<JSObject*> obj(aCx, abov.asObject()); + JS::Rooted<JSObject*> cloned(aCx, + JS::ArrayBufferClone(aCx, obj, offset, len)); + if (NS_WARN_IF(!cloned)) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + + JS::Rooted<JS::Value> value(aCx, JS::ObjectValue(*cloned)); + if (NS_WARN_IF(!aDest.Init(aCx, value))) { + return Err(NS_ERROR_UNEXPECTED); + } + return Ok(); +} + +/* + * The following are utilities to convert between VideoColorSpace values to + * gfx's values. + */ + +gfx::ColorRange ToColorRange(bool aIsFullRange) { + return aIsFullRange ? gfx::ColorRange::FULL : gfx::ColorRange::LIMITED; +} + +gfx::YUVColorSpace ToColorSpace(VideoMatrixCoefficients aMatrix) { + switch (aMatrix) { + case VideoMatrixCoefficients::Rgb: + return gfx::YUVColorSpace::Identity; + case VideoMatrixCoefficients::Bt709: + case VideoMatrixCoefficients::Bt470bg: + return gfx::YUVColorSpace::BT709; + case VideoMatrixCoefficients::Smpte170m: + return gfx::YUVColorSpace::BT601; + case VideoMatrixCoefficients::Bt2020_ncl: + return gfx::YUVColorSpace::BT2020; + case VideoMatrixCoefficients::EndGuard_: + break; + } + MOZ_ASSERT_UNREACHABLE("unsupported VideoMatrixCoefficients"); + return gfx::YUVColorSpace::Default; +} + +gfx::TransferFunction ToTransferFunction( + VideoTransferCharacteristics aTransfer) { + switch (aTransfer) { + case VideoTransferCharacteristics::Bt709: + case VideoTransferCharacteristics::Smpte170m: + return gfx::TransferFunction::BT709; + case VideoTransferCharacteristics::Iec61966_2_1: + return gfx::TransferFunction::SRGB; + case VideoTransferCharacteristics::Pq: + return gfx::TransferFunction::PQ; + case VideoTransferCharacteristics::Hlg: + return gfx::TransferFunction::HLG; + case VideoTransferCharacteristics::Linear: + case VideoTransferCharacteristics::EndGuard_: + break; + } + MOZ_ASSERT_UNREACHABLE("unsupported VideoTransferCharacteristics"); + return gfx::TransferFunction::Default; +} + +gfx::ColorSpace2 ToPrimaries(VideoColorPrimaries aPrimaries) { + switch (aPrimaries) { + case VideoColorPrimaries::Bt709: + return gfx::ColorSpace2::BT709; + case VideoColorPrimaries::Bt470bg: + return gfx::ColorSpace2::BT601_625; + case VideoColorPrimaries::Smpte170m: + return gfx::ColorSpace2::BT601_525; + case VideoColorPrimaries::Bt2020: + return gfx::ColorSpace2::BT2020; + case VideoColorPrimaries::Smpte432: + return gfx::ColorSpace2::DISPLAY_P3; + case VideoColorPrimaries::EndGuard_: + break; + } + MOZ_ASSERT_UNREACHABLE("unsupported VideoTransferCharacteristics"); + return gfx::ColorSpace2::UNKNOWN; +} + +bool ToFullRange(const gfx::ColorRange& aColorRange) { + return aColorRange == gfx::ColorRange::FULL; +} + +Maybe<VideoMatrixCoefficients> ToMatrixCoefficients( + const gfx::YUVColorSpace& aColorSpace) { + switch (aColorSpace) { + case gfx::YUVColorSpace::BT601: + return Some(VideoMatrixCoefficients::Smpte170m); + case gfx::YUVColorSpace::BT709: + return Some(VideoMatrixCoefficients::Bt709); + case gfx::YUVColorSpace::BT2020: + return Some(VideoMatrixCoefficients::Bt2020_ncl); + case gfx::YUVColorSpace::Identity: + return Some(VideoMatrixCoefficients::Rgb); + } + MOZ_ASSERT_UNREACHABLE("unsupported gfx::YUVColorSpace"); + return Nothing(); +} + +Maybe<VideoTransferCharacteristics> ToTransferCharacteristics( + const gfx::TransferFunction& aTransferFunction) { + switch (aTransferFunction) { + case gfx::TransferFunction::BT709: + return Some(VideoTransferCharacteristics::Bt709); + case gfx::TransferFunction::SRGB: + return Some(VideoTransferCharacteristics::Iec61966_2_1); + case gfx::TransferFunction::PQ: + return Some(VideoTransferCharacteristics::Pq); + case gfx::TransferFunction::HLG: + return Some(VideoTransferCharacteristics::Hlg); + } + MOZ_ASSERT_UNREACHABLE("unsupported gfx::TransferFunction"); + return Nothing(); +} + +Maybe<VideoColorPrimaries> ToPrimaries(const gfx::ColorSpace2& aColorSpace) { + switch (aColorSpace) { + case gfx::ColorSpace2::UNKNOWN: + return Nothing(); + case gfx::ColorSpace2::DISPLAY_P3: + return Some(VideoColorPrimaries::Smpte432); + case gfx::ColorSpace2::BT601_525: + return Some(VideoColorPrimaries::Smpte170m); + case gfx::ColorSpace2::SRGB: + case gfx::ColorSpace2::BT709: + return Some(VideoColorPrimaries::Bt709); + case gfx::ColorSpace2::BT2020: + return Some(VideoColorPrimaries::Bt2020); + } + MOZ_ASSERT_UNREACHABLE("unsupported gfx::ColorSpace2"); + return Nothing(); +} + +/* + * The following are utilities to convert from gfx's formats to + * VideoPixelFormats. + */ + +Maybe<VideoPixelFormat> SurfaceFormatToVideoPixelFormat( + gfx::SurfaceFormat aFormat) { + switch (aFormat) { + case gfx::SurfaceFormat::B8G8R8A8: + return Some(VideoPixelFormat::BGRA); + case gfx::SurfaceFormat::B8G8R8X8: + return Some(VideoPixelFormat::BGRX); + case gfx::SurfaceFormat::R8G8B8A8: + return Some(VideoPixelFormat::RGBA); + case gfx::SurfaceFormat::R8G8B8X8: + return Some(VideoPixelFormat::RGBX); + case gfx::SurfaceFormat::YUV: + return Some(VideoPixelFormat::I420); + case gfx::SurfaceFormat::NV12: + return Some(VideoPixelFormat::NV12); + case gfx::SurfaceFormat::YUV422: + return Some(VideoPixelFormat::I422); + default: + break; + } + return Nothing(); +} + +Maybe<VideoPixelFormat> ImageBitmapFormatToVideoPixelFormat( + ImageBitmapFormat aFormat) { + switch (aFormat) { + case ImageBitmapFormat::RGBA32: + return Some(VideoPixelFormat::RGBA); + case ImageBitmapFormat::BGRA32: + return Some(VideoPixelFormat::BGRA); + case ImageBitmapFormat::YUV444P: + return Some(VideoPixelFormat::I444); + case ImageBitmapFormat::YUV422P: + return Some(VideoPixelFormat::I422); + case ImageBitmapFormat::YUV420P: + return Some(VideoPixelFormat::I420); + case ImageBitmapFormat::YUV420SP_NV12: + return Some(VideoPixelFormat::NV12); + default: + break; + } + return Nothing(); +} + +Result<RefPtr<MediaByteBuffer>, nsresult> GetExtraDataFromArrayBuffer( + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer) { + RefPtr<MediaByteBuffer> data = MakeRefPtr<MediaByteBuffer>(); + Unused << AppendTypedArrayDataTo(aBuffer, *data); + return data->Length() > 0 ? data : nullptr; +} + +bool IsOnAndroid() { +#if defined(ANDROID) + return true; +#else + return false; +#endif +} + +bool IsOnMacOS() { +#if defined(XP_MACOSX) + return true; +#else + return false; +#endif +} + +bool IsOnLinux() { +#if defined(XP_LINUX) + return true; +#else + return false; +#endif +} + +template <typename T> +nsCString MaybeToString(const Maybe<T>& aMaybe) { + return nsPrintfCString( + "%s", aMaybe.isSome() ? ToString(aMaybe.value()).c_str() : "nothing"); +} + +struct ConfigurationChangeToString { + nsCString operator()(const CodecChange& aCodecChange) { + return nsPrintfCString("Codec: %s", + NS_ConvertUTF16toUTF8(aCodecChange.get()).get()); + } + nsCString operator()(const DimensionsChange& aDimensionChange) { + return nsPrintfCString("Dimensions: %dx%d", aDimensionChange.get().width, + aDimensionChange.get().height); + } + nsCString operator()(const DisplayDimensionsChange& aDisplayDimensionChange) { + if (aDisplayDimensionChange.get().isNothing()) { + return nsPrintfCString("Display dimensions: nothing"); + } + gfx::IntSize displayDimensions = aDisplayDimensionChange.get().value(); + return nsPrintfCString("Dimensions: %dx%d", displayDimensions.width, + displayDimensions.height); + } + nsCString operator()(const BitrateChange& aBitrateChange) { + return nsPrintfCString("Bitrate: %skbps", + MaybeToString(aBitrateChange.get()).get()); + } + nsCString operator()(const FramerateChange& aFramerateChange) { + return nsPrintfCString("Framerate: %sHz", + MaybeToString(aFramerateChange.get()).get()); + } + nsCString operator()( + const HardwareAccelerationChange& aHardwareAccelerationChange) { + return nsPrintfCString("HW acceleration: %s", + dom::HardwareAccelerationValues::GetString( + aHardwareAccelerationChange.get()) + .data()); + } + nsCString operator()(const AlphaChange& aAlphaChange) { + return nsPrintfCString( + "Alpha: %s", + dom::AlphaOptionValues::GetString(aAlphaChange.get()).data()); + } + nsCString operator()(const ScalabilityModeChange& aScalabilityModeChange) { + if (aScalabilityModeChange.get().isNothing()) { + return nsCString("Scalability mode: nothing"); + } + return nsPrintfCString( + "Scalability mode: %s", + NS_ConvertUTF16toUTF8(aScalabilityModeChange.get().value()).get()); + } + nsCString operator()(const BitrateModeChange& aBitrateModeChange) { + return nsPrintfCString( + "Bitrate mode: %s", + dom::VideoEncoderBitrateModeValues::GetString(aBitrateModeChange.get()) + .data()); + } + nsCString operator()(const LatencyModeChange& aLatencyModeChange) { + return nsPrintfCString( + "Latency mode: %s", + dom::LatencyModeValues::GetString(aLatencyModeChange.get()).data()); + } + nsCString operator()(const ContentHintChange& aContentHintChange) { + return nsPrintfCString("Content hint: %s", + MaybeToString(aContentHintChange.get()).get()); + } + template <typename T> + nsCString operator()(const T& aNewBitrate) { + return nsPrintfCString("Not implemented"); + } +}; + +nsString WebCodecsConfigurationChangeList::ToString() const { + nsString rv; + for (const WebCodecsEncoderConfigurationItem& change : mChanges) { + nsCString str = change.match(ConfigurationChangeToString()); + rv.AppendPrintf("- %s\n", str.get()); + } + return rv; +} + +using CodecChange = StrongTypedef<nsString, struct CodecChangeTypeWebCodecs>; +using DimensionsChange = + StrongTypedef<gfx::IntSize, struct DimensionsChangeTypeWebCodecs>; +using DisplayDimensionsChange = + StrongTypedef<Maybe<gfx::IntSize>, + struct DisplayDimensionsChangeTypeWebCodecs>; +using BitrateChange = + StrongTypedef<Maybe<uint32_t>, struct BitrateChangeTypeWebCodecs>; +using FramerateChange = + StrongTypedef<Maybe<double>, struct FramerateChangeTypeWebCodecs>; +using HardwareAccelerationChange = + StrongTypedef<dom::HardwareAcceleration, + struct HardwareAccelerationChangeTypeWebCodecs>; +using AlphaChange = + StrongTypedef<dom::AlphaOption, struct AlphaChangeTypeWebCodecs>; +using ScalabilityModeChange = + StrongTypedef<Maybe<nsString>, struct ScalabilityModeChangeTypeWebCodecs>; +using BitrateModeChange = StrongTypedef<dom::VideoEncoderBitrateMode, + struct BitrateModeChangeTypeWebCodecs>; +using LatencyModeChange = + StrongTypedef<dom::LatencyMode, struct LatencyModeTypeChangeTypeWebCodecs>; +using ContentHintChange = + StrongTypedef<Maybe<nsString>, struct ContentHintTypeTypeWebCodecs>; + +bool WebCodecsConfigurationChangeList::CanAttemptReconfigure() const { + for (const auto& change : mChanges) { + if (change.is<CodecChange>() || change.is<HardwareAccelerationChange>() || + change.is<AlphaChange>() || change.is<ScalabilityModeChange>()) { + return false; + } + } + return true; +} + +RefPtr<EncoderConfigurationChangeList> +WebCodecsConfigurationChangeList::ToPEMChangeList() const { + auto rv = MakeRefPtr<EncoderConfigurationChangeList>(); + MOZ_ASSERT(CanAttemptReconfigure()); + for (const auto& change : mChanges) { + if (change.is<dom::DimensionsChange>()) { + rv->Push(mozilla::DimensionsChange(change.as<DimensionsChange>().get())); + } else if (change.is<dom::DisplayDimensionsChange>()) { + rv->Push(mozilla::DisplayDimensionsChange( + change.as<DisplayDimensionsChange>().get())); + } else if (change.is<dom::BitrateChange>()) { + rv->Push(mozilla::BitrateChange(change.as<BitrateChange>().get())); + } else if (change.is<FramerateChange>()) { + rv->Push(mozilla::FramerateChange(change.as<FramerateChange>().get())); + } else if (change.is<dom::BitrateModeChange>()) { + MediaDataEncoder::BitrateMode mode; + if (change.as<dom::BitrateModeChange>().get() == + dom::VideoEncoderBitrateMode::Constant) { + mode = MediaDataEncoder::BitrateMode::Constant; + } else if (change.as<BitrateModeChange>().get() == + dom::VideoEncoderBitrateMode::Variable) { + mode = MediaDataEncoder::BitrateMode::Variable; + } else { + // Quantizer, not underlying support yet. + mode = MediaDataEncoder::BitrateMode::Variable; + } + rv->Push(mozilla::BitrateModeChange(mode)); + } else if (change.is<LatencyModeChange>()) { + MediaDataEncoder::Usage usage; + if (change.as<LatencyModeChange>().get() == dom::LatencyMode::Quality) { + usage = MediaDataEncoder::Usage::Record; + } else { + usage = MediaDataEncoder::Usage::Realtime; + } + rv->Push(UsageChange(usage)); + } else if (change.is<ContentHintChange>()) { + rv->Push( + mozilla::ContentHintChange(change.as<ContentHintChange>().get())); + } + } + return rv.forget(); +} + +#define ENUM_TO_STRING(enumType, enumValue) \ + enumType##Values::GetString(enumValue).data() + +nsCString ColorSpaceInitToString( + const dom::VideoColorSpaceInit& aColorSpaceInit) { + nsCString rv("VideoColorSpace"); + + if (!aColorSpaceInit.mFullRange.IsNull()) { + rv.AppendPrintf(" range: %s", + aColorSpaceInit.mFullRange.Value() ? "true" : "false"); + } + if (!aColorSpaceInit.mMatrix.IsNull()) { + rv.AppendPrintf(" matrix: %s", + ENUM_TO_STRING(dom::VideoMatrixCoefficients, + aColorSpaceInit.mMatrix.Value())); + } + if (!aColorSpaceInit.mTransfer.IsNull()) { + rv.AppendPrintf(" transfer: %s", + ENUM_TO_STRING(dom::VideoTransferCharacteristics, + aColorSpaceInit.mTransfer.Value())); + } + if (!aColorSpaceInit.mPrimaries.IsNull()) { + rv.AppendPrintf(" primaries: %s", + ENUM_TO_STRING(dom::VideoColorPrimaries, + aColorSpaceInit.mPrimaries.Value())); + } + + return rv; +} + +RefPtr<TaskQueue> GetWebCodecsEncoderTaskQueue() { + return TaskQueue::Create( + GetMediaThreadPool(MediaThreadType::PLATFORM_ENCODER), + "WebCodecs encoding", false); +} + +VideoColorSpaceInit FallbackColorSpaceForVideoContent() { + // If we're unable to determine the color space, but we think this is video + // content (e.g. because it's in YUV or NV12 or something like that, + // consider it's in BT709). + // This is step 3 of + // https://w3c.github.io/webcodecs/#videoframe-pick-color-space + VideoColorSpaceInit colorSpace; + colorSpace.mFullRange = false; + colorSpace.mMatrix = VideoMatrixCoefficients::Bt709; + colorSpace.mTransfer = VideoTransferCharacteristics::Bt709; + colorSpace.mPrimaries = VideoColorPrimaries::Bt709; + return colorSpace; +} +VideoColorSpaceInit FallbackColorSpaceForWebContent() { + // If we're unable to determine the color space, but we think this is from + // Web content (canvas, image, svg, etc.), consider it's in sRGB. + // This is step 2 of + // https://w3c.github.io/webcodecs/#videoframe-pick-color-space + VideoColorSpaceInit colorSpace; + colorSpace.mFullRange = true; + colorSpace.mMatrix = VideoMatrixCoefficients::Rgb; + colorSpace.mTransfer = VideoTransferCharacteristics::Iec61966_2_1; + colorSpace.mPrimaries = VideoColorPrimaries::Bt709; + return colorSpace; +} + +Maybe<CodecType> CodecStringToCodecType(const nsAString& aCodecString) { + if (StringBeginsWith(aCodecString, u"av01"_ns)) { + return Some(CodecType::AV1); + } + if (StringBeginsWith(aCodecString, u"vp8"_ns)) { + return Some(CodecType::VP8); + } + if (StringBeginsWith(aCodecString, u"vp09"_ns)) { + return Some(CodecType::VP9); + } + if (StringBeginsWith(aCodecString, u"avc1"_ns)) { + return Some(CodecType::H264); + } + return Nothing(); +} + +nsString ConfigToString(const VideoDecoderConfig& aConfig) { + nsString rv; + + auto internal = VideoDecoderConfigInternal::Create(aConfig); + + return internal->ToString(); +} + +}; // namespace mozilla::dom diff --git a/dom/media/webcodecs/WebCodecsUtils.h b/dom/media/webcodecs/WebCodecsUtils.h new file mode 100644 index 0000000000..7c0e6b6bbc --- /dev/null +++ b/dom/media/webcodecs/WebCodecsUtils.h @@ -0,0 +1,239 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_DOM_WEBCODECS_WEBCODECSUTILS_H +#define MOZILLA_DOM_WEBCODECS_WEBCODECSUTILS_H + +#include "ErrorList.h" +#include "js/TypeDecls.h" +#include "mozilla/Maybe.h" +#include "mozilla/MozPromise.h" +#include "mozilla/Result.h" +#include "mozilla/TaskQueue.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/Nullable.h" +#include "mozilla/dom/UnionTypes.h" +#include "mozilla/dom/VideoEncoderBinding.h" +#include "mozilla/dom/VideoFrameBinding.h" +#include "PlatformEncoderModule.h" + +namespace mozilla { + +namespace gfx { +enum class ColorRange : uint8_t; +enum class ColorSpace2 : uint8_t; +enum class SurfaceFormat : int8_t; +enum class TransferFunction : uint8_t; +enum class YUVColorSpace : uint8_t; +} // namespace gfx + +using WebCodecsId = size_t; + +extern std::atomic<WebCodecsId> sNextId; + +struct EncoderConfigurationChangeList; + +namespace dom { + +/* + * The followings are helpers for WebCodecs methods. + */ + +nsTArray<nsCString> GuessContainers(const nsAString& aCodec); + +Maybe<nsString> ParseCodecString(const nsAString& aCodec); + +/* + * Below are helpers for conversion among Maybe, Optional, and Nullable. + */ + +template <typename T> +Maybe<T> OptionalToMaybe(const Optional<T>& aOptional) { + if (aOptional.WasPassed()) { + return Some(aOptional.Value()); + } + return Nothing(); +} + +template <typename T> +const T* OptionalToPointer(const Optional<T>& aOptional) { + return aOptional.WasPassed() ? &aOptional.Value() : nullptr; +} + +template <typename T> +Maybe<T> NullableToMaybe(const Nullable<T>& aNullable) { + if (!aNullable.IsNull()) { + return Some(aNullable.Value()); + } + return Nothing(); +} + +template <typename T> +Nullable<T> MaybeToNullable(const Maybe<T>& aOptional) { + if (aOptional.isSome()) { + return Nullable<T>(aOptional.value()); + } + return Nullable<T>(); +} + +/* + * Below are helpers to operate ArrayBuffer or ArrayBufferView. + */ + +Result<Ok, nsresult> CloneBuffer( + JSContext* aCx, + OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDest, + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aSrc); + +/* + * The following are utilities to convert between VideoColorSpace values to + * gfx's values. + */ + +enum class VideoColorPrimaries : uint8_t; +enum class VideoMatrixCoefficients : uint8_t; +enum class VideoTransferCharacteristics : uint8_t; + +gfx::ColorRange ToColorRange(bool aIsFullRange); + +gfx::YUVColorSpace ToColorSpace(VideoMatrixCoefficients aMatrix); + +gfx::TransferFunction ToTransferFunction( + VideoTransferCharacteristics aTransfer); + +gfx::ColorSpace2 ToPrimaries(VideoColorPrimaries aPrimaries); + +bool ToFullRange(const gfx::ColorRange& aColorRange); + +Maybe<VideoMatrixCoefficients> ToMatrixCoefficients( + const gfx::YUVColorSpace& aColorSpace); + +Maybe<VideoTransferCharacteristics> ToTransferCharacteristics( + const gfx::TransferFunction& aTransferFunction); + +Maybe<VideoColorPrimaries> ToPrimaries(const gfx::ColorSpace2& aColorSpace); + +/* + * The following are utilities to convert from gfx's formats to + * VideoPixelFormats. + */ + +enum class ImageBitmapFormat : uint8_t; +enum class VideoPixelFormat : uint8_t; + +Maybe<VideoPixelFormat> SurfaceFormatToVideoPixelFormat( + gfx::SurfaceFormat aFormat); + +Maybe<VideoPixelFormat> ImageBitmapFormatToVideoPixelFormat( + ImageBitmapFormat aFormat); + +template <typename T> +class MessageRequestHolder { + public: + MessageRequestHolder() = default; + ~MessageRequestHolder() = default; + + MozPromiseRequestHolder<T>& Request() { return mRequest; } + void Disconnect() { mRequest.DisconnectIfExists(); } + void Complete() { mRequest.Complete(); } + bool Exists() const { return mRequest.Exists(); } + + protected: + MozPromiseRequestHolder<T> mRequest{}; +}; + +enum class MessageProcessedResult { NotProcessed, Processed }; + +bool IsOnAndroid(); +bool IsOnMacOS(); +bool IsOnLinux(); + +// Wrap a type to make it unique. This allows using ergonomically in the Variant +// below. Simply aliasing with `using` isn't enough, because typedefs in C++ +// don't produce strong types, so two integer variants result in +// the same type, making it ambiguous to the Variant code. +// T is the type to be wrapped. Phantom is a type that is only used to +// disambiguate and should be unique in the program. +template <typename T, typename Phantom> +class StrongTypedef { + public: + explicit StrongTypedef(T const& value) : mValue(value) {} + explicit StrongTypedef(T&& value) : mValue(std::move(value)) {} + T& get() { return mValue; } + T const& get() const { return mValue; } + + private: + T mValue; +}; + +using CodecChange = StrongTypedef<nsString, struct CodecChangeTypeWebCodecs>; +using DimensionsChange = + StrongTypedef<gfx::IntSize, struct DimensionsChangeTypeWebCodecs>; +using DisplayDimensionsChange = + StrongTypedef<Maybe<gfx::IntSize>, + struct DisplayDimensionsChangeTypeWebCodecs>; +using BitrateChange = + StrongTypedef<Maybe<uint32_t>, struct BitrateChangeTypeWebCodecs>; +using FramerateChange = + StrongTypedef<Maybe<double>, struct FramerateChangeTypeWebCodecs>; +using HardwareAccelerationChange = + StrongTypedef<dom::HardwareAcceleration, + struct HardwareAccelerationChangeTypeWebCodecs>; +using AlphaChange = + StrongTypedef<dom::AlphaOption, struct AlphaChangeTypeWebCodecs>; +using ScalabilityModeChange = + StrongTypedef<Maybe<nsString>, struct ScalabilityModeChangeTypeWebCodecs>; +using BitrateModeChange = StrongTypedef<dom::VideoEncoderBitrateMode, + struct BitrateModeChangeTypeWebCodecs>; +using LatencyModeChange = + StrongTypedef<dom::LatencyMode, struct LatencyModeTypeChangeTypeWebCodecs>; +using ContentHintChange = + StrongTypedef<Maybe<nsString>, struct ContentHintTypeTypeWebCodecs>; + +using WebCodecsEncoderConfigurationItem = + Variant<CodecChange, DimensionsChange, DisplayDimensionsChange, + BitrateModeChange, BitrateChange, FramerateChange, + HardwareAccelerationChange, AlphaChange, ScalabilityModeChange, + LatencyModeChange, ContentHintChange>; + +struct WebCodecsConfigurationChangeList { + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(WebCodecsConfigurationChangeList) + bool Empty() const { return mChanges.IsEmpty(); } + template <typename T> + void Push(const T& aItem) { + mChanges.AppendElement(aItem); + } + // This returns true if it should be possible to attempt to reconfigure the + // encoder on the fly. It can fail, in which case the encoder will be flushed + // and a new one will be created with the new set of parameters. + bool CanAttemptReconfigure() const; + + // Convert this to the format the underlying PEM can understand + RefPtr<EncoderConfigurationChangeList> ToPEMChangeList() const; + nsString ToString() const; + + nsTArray<WebCodecsEncoderConfigurationItem> mChanges; + + private: + ~WebCodecsConfigurationChangeList() = default; +}; + +nsCString ColorSpaceInitToString( + const dom::VideoColorSpaceInit& aColorSpaceInit); + +RefPtr<TaskQueue> GetWebCodecsEncoderTaskQueue(); +VideoColorSpaceInit FallbackColorSpaceForVideoContent(); +VideoColorSpaceInit FallbackColorSpaceForWebContent(); + +Maybe<CodecType> CodecStringToCodecType(const nsAString& aCodecString); + +nsString ConfigToString(const VideoDecoderConfig& aConfig); + +} // namespace dom + +} // namespace mozilla + +#endif // MOZILLA_DOM_WEBCODECS_WEBCODECSUTILS_H diff --git a/dom/media/webcodecs/crashtests/1839270.html b/dom/media/webcodecs/crashtests/1839270.html new file mode 100644 index 0000000000..3b8f7908e5 --- /dev/null +++ b/dom/media/webcodecs/crashtests/1839270.html @@ -0,0 +1,13 @@ +<script> +window.addEventListener("load", async () => { + let _ = new Response("", {"headers": []}) + let a = new ArrayBuffer(60005) + let v = new VideoFrame(a, { + "format": "RGBA", + "codedWidth": 1458585599, + "codedHeight": 84, + "timestamp": 0.541, + "layout": [{"offset": 168, "stride": 198}], + }) +}) +</script> diff --git a/dom/media/webcodecs/crashtests/1848460.html b/dom/media/webcodecs/crashtests/1848460.html new file mode 100644 index 0000000000..2f6f8b930e --- /dev/null +++ b/dom/media/webcodecs/crashtests/1848460.html @@ -0,0 +1,17 @@ +<script id="worker" type="javascript/worker"> +self.onmessage = async function(e) { + let a = new ArrayBuffer(12583) + let b = new DataView(a) + await VideoDecoder.isConfigSupported({ + "codec": "7ﷺ۹.9", + "description": b, + }) +} +</script> +<script> +window.addEventListener("load", async () => { + const blob = new Blob([document.querySelector('#worker').textContent], { type: "text/javascript" }) + const worker = new Worker(window.URL.createObjectURL(blob)) + worker.postMessage([], []) +}) +</script> diff --git a/dom/media/webcodecs/crashtests/1849271.html b/dom/media/webcodecs/crashtests/1849271.html new file mode 100644 index 0000000000..67e170d8bf --- /dev/null +++ b/dom/media/webcodecs/crashtests/1849271.html @@ -0,0 +1,27 @@ +<html class="reftest-wait"> + <script> + var cfg = { + codec: "vp8", + colorSpace: { primaries: "bt709" }, + }; + var decoder = new VideoDecoder({ + output: () => {}, + error: e => { + document.documentElement.removeAttribute("class"); + }, + }); + decoder.configure(cfg); + try { + decoder.decode( + new EncodedVideoChunk({ + type: "key", + timestamp: 0, + duration: 10, + data: new Uint8Array(10), + }) + ); + } catch (e) { + document.documentElement.removeAttribute("class"); + } + </script> +</html> diff --git a/dom/media/webcodecs/crashtests/1864475.html b/dom/media/webcodecs/crashtests/1864475.html new file mode 100644 index 0000000000..d7164b2854 --- /dev/null +++ b/dom/media/webcodecs/crashtests/1864475.html @@ -0,0 +1,14 @@ +<!DOCTYPE html> +<script> + document.addEventListener('DOMContentLoaded', async () => { + const buffer = new ArrayBuffer(26838) + const array = new Uint8ClampedArray(buffer) + const frame = new VideoFrame(array, { + 'format': 'I420A', + 'codedWidth': 192, + 'codedHeight': 2, + 'timestamp': 14.19024535832334, + }) + await frame.copyTo(buffer, {}) + }) +</script> diff --git a/dom/media/webcodecs/crashtests/crashtests.list b/dom/media/webcodecs/crashtests/crashtests.list new file mode 100644 index 0000000000..cea5139fe9 --- /dev/null +++ b/dom/media/webcodecs/crashtests/crashtests.list @@ -0,0 +1,4 @@ +skip-if(Android) pref(dom.media.webcodecs.enabled,true) load 1839270.html +skip-if(Android) pref(dom.media.webcodecs.enabled,true) load 1848460.html +skip-if(Android) pref(dom.media.webcodecs.enabled,true) load 1849271.html +skip-if(Android) pref(dom.media.webcodecs.enabled,true) load 1864475.html
\ No newline at end of file diff --git a/dom/media/webcodecs/moz.build b/dom/media/webcodecs/moz.build new file mode 100644 index 0000000000..267a822286 --- /dev/null +++ b/dom/media/webcodecs/moz.build @@ -0,0 +1,54 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("*"): + BUG_COMPONENT = ("Core", "Audio/Video: Web Codecs") + +MOCHITEST_MANIFESTS += ["test/mochitest.toml"] +CRASHTEST_MANIFESTS += ["crashtests/crashtests.list"] + +# For mozilla/layers/ImageBridgeChild.h +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src/", +] + +EXPORTS.mozilla += [ + "DecoderAgent.h", +] + +EXPORTS.mozilla.dom += [ + "DecoderTemplate.h", + "DecoderTypes.h", + "EncodedVideoChunk.h", + "EncoderAgent.h", + "EncoderTemplate.h", + "EncoderTypes.h", + "VideoColorSpace.h", + "VideoDecoder.h", + "VideoEncoder.h", + "VideoFrame.h", + "WebCodecsUtils.h", +] + +UNIFIED_SOURCES += [ + "DecoderAgent.cpp", + "DecoderTemplate.cpp", + "EncodedVideoChunk.cpp", + "EncoderAgent.cpp", + "EncoderTemplate.cpp", + "VideoColorSpace.cpp", + "VideoDecoder.cpp", + "VideoEncoder.cpp", + "VideoFrame.cpp", + "WebCodecsUtils.cpp", +] + +if CONFIG["MOZ_WAYLAND"]: + CXXFLAGS += CONFIG["MOZ_WAYLAND_CFLAGS"] + CFLAGS += CONFIG["MOZ_WAYLAND_CFLAGS"] + +FINAL_LIBRARY = "xul" diff --git a/dom/media/webcodecs/test/mochitest.toml b/dom/media/webcodecs/test/mochitest.toml new file mode 100644 index 0000000000..061bfd12fb --- /dev/null +++ b/dom/media/webcodecs/test/mochitest.toml @@ -0,0 +1,6 @@ +[DEFAULT] +subsuite = "media" +tags = "webcodecs" +prefs = ["dom.media.webcodecs.enabled=true"] + +["test_videoFrame_mismatched_codedSize.html"] diff --git a/dom/media/webcodecs/test/test_videoFrame_mismatched_codedSize.html b/dom/media/webcodecs/test/test_videoFrame_mismatched_codedSize.html new file mode 100644 index 0000000000..52e26f7854 --- /dev/null +++ b/dom/media/webcodecs/test/test_videoFrame_mismatched_codedSize.html @@ -0,0 +1,30 @@ +<!DOCTYPE HTML> +<html> +<head> + <title></title> + <script src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> +<body> +<script> +let data = new Uint8Array([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, +]); +// TODO: Crop the image instead of returning errors (Bug 1782128). +try { + // Bug 1793814: Remove eslint-disable-line below + let frame = new VideoFrame(data, { // eslint-disable-line no-undef + timestamp: 10, + codedWidth: 3, + codedHeight: 3, + visibleRect: { x: 0, y: 0, width: 1, height: 1 }, + format: "RGBA", + }); + frame.close(); + ok(false, "Should not create a VideoFrame from a mismatched-size buffer"); +} catch (e) { + ok(e instanceof TypeError, "Should throw a TypeError"); +} +</script> +</body> +</html> |