From 26a029d407be480d791972afb5975cf62c9360a6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 02:47:55 +0200 Subject: Adding upstream version 124.0.1. Signed-off-by: Daniel Baumann --- dom/media/platforms/PlatformEncoderModule.h | 385 ++++++++++++++++++++++++++++ 1 file changed, 385 insertions(+) create mode 100644 dom/media/platforms/PlatformEncoderModule.h (limited to 'dom/media/platforms/PlatformEncoderModule.h') diff --git a/dom/media/platforms/PlatformEncoderModule.h b/dom/media/platforms/PlatformEncoderModule.h new file mode 100644 index 0000000000..72dad430e6 --- /dev/null +++ b/dom/media/platforms/PlatformEncoderModule.h @@ -0,0 +1,385 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(PlatformEncoderModule_h_) +# define PlatformEncoderModule_h_ + +# include "MP4Decoder.h" +# include "MediaData.h" +# include "MediaInfo.h" +# include "MediaResult.h" +# include "VPXDecoder.h" +# include "mozilla/Attributes.h" +# include "mozilla/Maybe.h" +# include "mozilla/MozPromise.h" +# include "mozilla/RefPtr.h" +# include "mozilla/TaskQueue.h" +# include "mozilla/dom/ImageBitmapBinding.h" +# include "nsISupportsImpl.h" +# include "VideoUtils.h" + +namespace mozilla { + +class MediaDataEncoder; +class EncoderConfig; +struct EncoderConfigurationChangeList; + +enum class CodecType { + _BeginVideo_, + H264, + VP8, + VP9, + AV1, + _EndVideo_, + _BeginAudio_ = _EndVideo_, + Opus, + G722, + _EndAudio_, + Unknown, +}; + +// TODO: Automatically generate this (Bug 1865896) +const char* GetCodecTypeString(const CodecType& aCodecType); + +enum class H264BitStreamFormat { AVC, ANNEXB }; + +struct H264Specific final { + const H264_PROFILE mProfile; + const H264_LEVEL mLevel; + const H264BitStreamFormat mFormat; + + H264Specific(H264_PROFILE aProfile, H264_LEVEL aLevel, + H264BitStreamFormat aFormat) + : mProfile(aProfile), mLevel(aLevel), mFormat(aFormat) {} +}; + +struct OpusSpecific final { + enum class Application { Voip, Audio, RestricedLowDelay }; + + const Application mApplication; + const uint8_t mComplexity; // from 0-10 + + OpusSpecific(const Application aApplication, const uint8_t aComplexity) + : mApplication(aApplication), mComplexity(aComplexity) { + MOZ_ASSERT(mComplexity <= 10); + } +}; + +enum class VPXComplexity { Normal, High, Higher, Max }; +struct VP8Specific { + VP8Specific() = default; + // Ignore webrtc::VideoCodecVP8::errorConcealmentOn, + // for it's always false in the codebase (except libwebrtc test cases). + VP8Specific(const VPXComplexity aComplexity, const bool aResilience, + const uint8_t aNumTemporalLayers, const bool aDenoising, + const bool aAutoResize, const bool aFrameDropping) + : mComplexity(aComplexity), + mResilience(aResilience), + mNumTemporalLayers(aNumTemporalLayers), + mDenoising(aDenoising), + mAutoResize(aAutoResize), + mFrameDropping(aFrameDropping) {} + const VPXComplexity mComplexity{VPXComplexity::Normal}; + const bool mResilience{true}; + const uint8_t mNumTemporalLayers{1}; + const bool mDenoising{true}; + const bool mAutoResize{false}; + const bool mFrameDropping{false}; +}; + +struct VP9Specific : public VP8Specific { + VP9Specific() = default; + VP9Specific(const VPXComplexity aComplexity, const bool aResilience, + const uint8_t aNumTemporalLayers, const bool aDenoising, + const bool aAutoResize, const bool aFrameDropping, + const bool aAdaptiveQp, const uint8_t aNumSpatialLayers, + const bool aFlexible) + : VP8Specific(aComplexity, aResilience, aNumTemporalLayers, aDenoising, + aAutoResize, aFrameDropping), + mAdaptiveQp(aAdaptiveQp), + mNumSpatialLayers(aNumSpatialLayers), + mFlexible(aFlexible) {} + const bool mAdaptiveQp{true}; + const uint8_t mNumSpatialLayers{1}; + const bool mFlexible{false}; +}; + +class PlatformEncoderModule { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(PlatformEncoderModule) + + virtual already_AddRefed CreateVideoEncoder( + const EncoderConfig& aConfig, const RefPtr& aTaskQueue) const { + return nullptr; + }; + + virtual already_AddRefed CreateAudioEncoder( + const EncoderConfig& aConfig, const RefPtr& aTaskQueue) const { + return nullptr; + }; + + using CreateEncoderPromise = MozPromise, MediaResult, + /* IsExclusive = */ true>; + + // Indicates if the PlatformDecoderModule supports encoding of a codec. + virtual bool Supports(const EncoderConfig& aConfig) const = 0; + virtual bool SupportsCodec(CodecType aCodecType) const = 0; + + // Returns a readable name for this Platform Encoder Module + virtual const char* GetName() const = 0; + + // Asychronously create an encoder + RefPtr AsyncCreateEncoder( + const EncoderConfig& aEncoderConfig, const RefPtr& aTaskQueue); + + protected: + PlatformEncoderModule() = default; + virtual ~PlatformEncoderModule() = default; +}; + +class MediaDataEncoder { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaDataEncoder) + + enum class Usage { + Realtime, // Low latency prefered + Record + }; + using PixelFormat = dom::ImageBitmapFormat; + enum class BitrateMode { Constant, Variable }; + // Scalable Video Coding (SVC) settings for WebCodecs: + // https://www.w3.org/TR/webrtc-svc/ + enum class ScalabilityMode { None, L1T2, L1T3 }; + + enum class HardwarePreference { RequireHardware, RequireSoftware, None }; + + static bool IsVideo(const CodecType aCodec) { + return aCodec > CodecType::_BeginVideo_ && aCodec < CodecType::_EndVideo_; + } + static bool IsAudio(const CodecType aCodec) { + return aCodec > CodecType::_BeginAudio_ && aCodec < CodecType::_EndAudio_; + } + + using InitPromise = + MozPromise; + using EncodedData = nsTArray>; + using EncodePromise = + MozPromise; + using ReconfigurationPromise = + MozPromise; + + // Initialize the encoder. It should be ready to encode once the returned + // promise resolves. The encoder should do any initialization here, rather + // than in its constructor or PlatformEncoderModule::Create*Encoder(), + // so that if the client needs to shutdown during initialization, + // it can call Shutdown() to cancel this operation. Any initialization + // that requires blocking the calling thread in this function *must* + // be done here so that it can be canceled by calling Shutdown()! + virtual RefPtr Init() = 0; + + // Inserts a sample into the encoder's encode pipeline. The EncodePromise it + // returns will be resolved with already encoded MediaRawData at the moment, + // or empty when there is none available yet. + virtual RefPtr Encode(const MediaData* aSample) = 0; + + // Attempt to reconfigure the encoder on the fly. This can fail if the + // underlying PEM doesn't support this type of reconfiguration. + virtual RefPtr Reconfigure( + const RefPtr& + aConfigurationChanges) = 0; + + // Causes all complete samples in the pipeline that can be encoded to be + // output. It indicates that there is no more input sample to insert. + // This function is asynchronous. + // The MediaDataEncoder shall resolve the pending EncodePromise with drained + // samples. Drain will be called multiple times until the resolved + // EncodePromise is empty which indicates that there are no more samples to + // drain. + virtual RefPtr Drain() = 0; + + // Cancels all init/encode/drain operations, and shuts down the encoder. The + // platform encoder should clean up any resources it's using and release + // memory etc. The shutdown promise will be resolved once the encoder has + // completed shutdown. The client will delete the decoder once the promise is + // resolved. + // The ShutdownPromise must only ever be resolved. + virtual RefPtr Shutdown() = 0; + + virtual RefPtr SetBitrate(uint32_t aBitsPerSec) { + return GenericPromise::CreateAndResolve(true, __func__); + } + + // Decoder needs to decide whether or not hardware acceleration is supported + // after creating. It doesn't need to call Init() before calling this + // function. + virtual bool IsHardwareAccelerated(nsACString& aFailureReason) const { + return false; + } + + // Return the name of the MediaDataEncoder, only used for encoding. + // May be accessed in a non thread-safe fashion. + virtual nsCString GetDescriptionName() const = 0; + + friend class PlatformEncoderModule; + + protected: + virtual ~MediaDataEncoder() = default; +}; + +class EncoderConfig final { + public: + using CodecSpecific = + Variant; + + EncoderConfig(const EncoderConfig& aConfig) + : mCodec(aConfig.mCodec), + mSize(aConfig.mSize), + mUsage(aConfig.mUsage), + mHardwarePreference(aConfig.mHardwarePreference), + mPixelFormat(aConfig.mPixelFormat), + mSourcePixelFormat(aConfig.mSourcePixelFormat), + mScalabilityMode(aConfig.mScalabilityMode), + mFramerate(aConfig.mFramerate), + mKeyframeInterval(aConfig.mKeyframeInterval), + mBitrate(aConfig.mBitrate), + mBitrateMode(aConfig.mBitrateMode), + mCodecSpecific(aConfig.mCodecSpecific) {} + + template + EncoderConfig(const CodecType aCodecType, gfx::IntSize aSize, + const MediaDataEncoder::Usage aUsage, + const MediaDataEncoder::PixelFormat aPixelFormat, + const MediaDataEncoder::PixelFormat aSourcePixelFormat, + const uint8_t aFramerate, const size_t aKeyframeInterval, + const uint32_t aBitrate, + const MediaDataEncoder::BitrateMode aBitrateMode, + const MediaDataEncoder::HardwarePreference aHardwarePreference, + const MediaDataEncoder::ScalabilityMode aScalabilityMode, + const Maybe& aCodecSpecific) + : mCodec(aCodecType), + mSize(aSize), + mUsage(aUsage), + mHardwarePreference(aHardwarePreference), + mPixelFormat(aPixelFormat), + mSourcePixelFormat(aSourcePixelFormat), + mScalabilityMode(aScalabilityMode), + mFramerate(aFramerate), + mKeyframeInterval(aKeyframeInterval), + mBitrate(aBitrate), + mBitrateMode(aBitrateMode), + mCodecSpecific(aCodecSpecific) {} + + static CodecType CodecTypeForMime(const nsACString& aMimeType) { + if (MP4Decoder::IsH264(aMimeType)) { + return CodecType::H264; + } + if (VPXDecoder::IsVPX(aMimeType, VPXDecoder::VP8)) { + return CodecType::VP8; + } + if (VPXDecoder::IsVPX(aMimeType, VPXDecoder::VP9)) { + return CodecType::VP9; + } + MOZ_ASSERT_UNREACHABLE("Unsupported Mimetype"); + return CodecType::Unknown; + } + + bool IsVideo() const { + return mCodec > CodecType::_BeginVideo_ && mCodec < CodecType::_EndVideo_; + } + + bool IsAudio() const { + return mCodec > CodecType::_BeginAudio_ && mCodec < CodecType::_EndAudio_; + } + + CodecType mCodec; + gfx::IntSize mSize; + MediaDataEncoder::Usage mUsage; + MediaDataEncoder::HardwarePreference mHardwarePreference; + MediaDataEncoder::PixelFormat mPixelFormat; + MediaDataEncoder::PixelFormat mSourcePixelFormat; + MediaDataEncoder::ScalabilityMode mScalabilityMode; + uint8_t mFramerate{}; + size_t mKeyframeInterval{}; + uint32_t mBitrate{}; + MediaDataEncoder::BitrateMode mBitrateMode{}; + Maybe mCodecSpecific; +}; + +// Wrap a type to make it unique. This allows using ergonomically in the Variant +// below. Simply aliasing with `using` isn't enough, because typedefs in C++ +// don't produce strong types, so two integer variants result in +// the same type, making it ambiguous to the Variant code. +// T is the type to be wrapped. Phantom is a type that is only used to +// disambiguate and should be unique in the program. +template +class StrongTypedef { + public: + explicit StrongTypedef(T const& value) : mValue(value) {} + explicit StrongTypedef(T&& value) : mValue(std::move(value)) {} + T& get() { return mValue; } + T const& get() const { return mValue; } + + private: + T mValue; +}; + +// Dimensions of the video frames +using DimensionsChange = + StrongTypedef; +// Expected display size of the encoded frames, can influence encoding +using DisplayDimensionsChange = + StrongTypedef, struct DisplayDimensionsChangeType>; +// If present, the bitrate in kbps of the encoded stream. If absent, let the +// platform decide. +using BitrateChange = StrongTypedef, struct BitrateChangeType>; +// If present, the expected framerate of the output video stream. If absent, +// infer from the input frames timestamp. +using FramerateChange = + StrongTypedef, struct FramerateChangeType>; +// The bitrate mode (variable, constant) of the encoding +using BitrateModeChange = + StrongTypedef; +// The usage for the encoded stream, this influence latency, ordering, etc. +using UsageChange = + StrongTypedef; +// If present, the expected content of the video frames (screen, movie, etc.). +// The value the string can have isn't decided just yet. When absent, the +// encoder uses generic settings. +using ContentHintChange = + StrongTypedef, struct ContentHintTypeType>; + +// A change to a parameter of an encoder instance. +using EncoderConfigurationItem = + Variant; + +// A list of changes to an encoder configuration, that _might_ be able to change +// on the fly. Not all encoder modules can adjust their configuration on the +// fly. +struct EncoderConfigurationChangeList { + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(EncoderConfigurationChangeList) + bool Empty() const { return mChanges.IsEmpty(); } + template + void Push(const T& aItem) { + mChanges.AppendElement(aItem); + } + nsString ToString() const; + + nsTArray mChanges; + + private: + ~EncoderConfigurationChangeList() = default; +}; + +// Just by inspecting the configuration and before asking the PEM, it's +// sometimes possible to know that a media won't be able to be encoded. For +// example, VP8 encodes the frame size on 14 bits, so a resolution of more than +// 16383x16383 pixels cannot work. +bool CanLikelyEncode(const EncoderConfig& aConfig); + +} // namespace mozilla + +#endif /* PlatformEncoderModule_h_ */ -- cgit v1.2.3