/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim:set ts=2 sw=2 sts=2 et cindent: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "WebCodecsUtils.h" #include "DecoderTypes.h" #include "VideoUtils.h" #include "js/experimental/TypedData.h" #include "mozilla/Assertions.h" #include "mozilla/dom/ImageBitmapBinding.h" #include "mozilla/dom/VideoColorSpaceBinding.h" #include "mozilla/dom/VideoFrameBinding.h" #include "mozilla/gfx/Types.h" #include "nsDebug.h" #include "PlatformEncoderModule.h" #include "PlatformEncoderModule.h" extern mozilla::LazyLogModule gWebCodecsLog; #ifdef LOG_INTERNAL # undef LOG_INTERNAL #endif // LOG_INTERNAL #define LOG_INTERNAL(level, msg, ...) \ MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) #ifdef LOG # undef LOG #endif // LOG #define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) namespace mozilla { std::atomic sNextId = 0; }; namespace mozilla::dom { /* * The followings are helpers for VideoDecoder methods */ nsTArray GuessContainers(const nsAString& aCodec) { if (IsAV1CodecString(aCodec)) { return {"mp4"_ns, "webm"_ns}; } if (IsVP9CodecString(aCodec)) { return {"mp4"_ns, "webm"_ns, "ogg"_ns}; } if (IsVP8CodecString(aCodec)) { return {"webm"_ns, "ogg"_ns, "3gpp"_ns, "3gpp2"_ns, "3gp2"_ns}; } if (IsH264CodecString(aCodec)) { return {"mp4"_ns, "3gpp"_ns, "3gpp2"_ns, "3gp2"_ns}; } return {}; } Maybe ParseCodecString(const nsAString& aCodec) { // Trim the spaces on each end. nsString str(aCodec); str.Trim(" "); nsTArray codecs; if (!ParseCodecsString(str, codecs) || codecs.Length() != 1 || codecs[0] != str) { return Nothing(); } return Some(codecs[0]); } /* * The below are helpers to operate ArrayBuffer or ArrayBufferView. */ static std::tuple GetArrayBufferInfo( JSContext* aCx, const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer) { if (aBuffer.IsArrayBuffer()) { const ArrayBuffer& buffer = aBuffer.GetAsArrayBuffer(); size_t length; { bool isShared; uint8_t* data; JS::GetArrayBufferMaybeSharedLengthAndData(buffer.Obj(), &length, &isShared, &data); } return std::make_tuple(JS::ArrayBufferOrView::fromObject(buffer.Obj()), (size_t)0, length); } MOZ_ASSERT(aBuffer.IsArrayBufferView()); const ArrayBufferView& view = aBuffer.GetAsArrayBufferView(); bool isSharedMemory; JS::Rooted obj(aCx, view.Obj()); return std::make_tuple( JS::ArrayBufferOrView::fromObject( JS_GetArrayBufferViewBuffer(aCx, obj, &isSharedMemory)), JS_GetArrayBufferViewByteOffset(obj), JS_GetArrayBufferViewByteLength(obj)); } Result CloneBuffer( JSContext* aCx, OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDest, const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aSrc) { std::tuple info = GetArrayBufferInfo(aCx, aSrc); JS::Rooted abov(aCx); abov.set(std::get<0>(info)); size_t offset = std::get<1>(info); size_t len = std::get<2>(info); if (NS_WARN_IF(!abov)) { return Err(NS_ERROR_UNEXPECTED); } JS::Rooted obj(aCx, abov.asObject()); JS::Rooted cloned(aCx, JS::ArrayBufferClone(aCx, obj, offset, len)); if (NS_WARN_IF(!cloned)) { return Err(NS_ERROR_OUT_OF_MEMORY); } JS::Rooted value(aCx, JS::ObjectValue(*cloned)); if (NS_WARN_IF(!aDest.Init(aCx, value))) { return Err(NS_ERROR_UNEXPECTED); } return Ok(); } /* * The following are utilities to convert between VideoColorSpace values to * gfx's values. */ gfx::ColorRange ToColorRange(bool aIsFullRange) { return aIsFullRange ? gfx::ColorRange::FULL : gfx::ColorRange::LIMITED; } gfx::YUVColorSpace ToColorSpace(VideoMatrixCoefficients aMatrix) { switch (aMatrix) { case VideoMatrixCoefficients::Rgb: return gfx::YUVColorSpace::Identity; case VideoMatrixCoefficients::Bt709: case VideoMatrixCoefficients::Bt470bg: return gfx::YUVColorSpace::BT709; case VideoMatrixCoefficients::Smpte170m: return gfx::YUVColorSpace::BT601; case VideoMatrixCoefficients::Bt2020_ncl: return gfx::YUVColorSpace::BT2020; case VideoMatrixCoefficients::EndGuard_: break; } MOZ_ASSERT_UNREACHABLE("unsupported VideoMatrixCoefficients"); return gfx::YUVColorSpace::Default; } gfx::TransferFunction ToTransferFunction( VideoTransferCharacteristics aTransfer) { switch (aTransfer) { case VideoTransferCharacteristics::Bt709: case VideoTransferCharacteristics::Smpte170m: return gfx::TransferFunction::BT709; case VideoTransferCharacteristics::Iec61966_2_1: return gfx::TransferFunction::SRGB; case VideoTransferCharacteristics::Pq: return gfx::TransferFunction::PQ; case VideoTransferCharacteristics::Hlg: return gfx::TransferFunction::HLG; case VideoTransferCharacteristics::Linear: case VideoTransferCharacteristics::EndGuard_: break; } MOZ_ASSERT_UNREACHABLE("unsupported VideoTransferCharacteristics"); return gfx::TransferFunction::Default; } gfx::ColorSpace2 ToPrimaries(VideoColorPrimaries aPrimaries) { switch (aPrimaries) { case VideoColorPrimaries::Bt709: return gfx::ColorSpace2::BT709; case VideoColorPrimaries::Bt470bg: return gfx::ColorSpace2::BT601_625; case VideoColorPrimaries::Smpte170m: return gfx::ColorSpace2::BT601_525; case VideoColorPrimaries::Bt2020: return gfx::ColorSpace2::BT2020; case VideoColorPrimaries::Smpte432: return gfx::ColorSpace2::DISPLAY_P3; case VideoColorPrimaries::EndGuard_: break; } MOZ_ASSERT_UNREACHABLE("unsupported VideoTransferCharacteristics"); return gfx::ColorSpace2::UNKNOWN; } bool ToFullRange(const gfx::ColorRange& aColorRange) { return aColorRange == gfx::ColorRange::FULL; } Maybe ToMatrixCoefficients( const gfx::YUVColorSpace& aColorSpace) { switch (aColorSpace) { case gfx::YUVColorSpace::BT601: return Some(VideoMatrixCoefficients::Smpte170m); case gfx::YUVColorSpace::BT709: return Some(VideoMatrixCoefficients::Bt709); case gfx::YUVColorSpace::BT2020: return Some(VideoMatrixCoefficients::Bt2020_ncl); case gfx::YUVColorSpace::Identity: return Some(VideoMatrixCoefficients::Rgb); } MOZ_ASSERT_UNREACHABLE("unsupported gfx::YUVColorSpace"); return Nothing(); } Maybe ToTransferCharacteristics( const gfx::TransferFunction& aTransferFunction) { switch (aTransferFunction) { case gfx::TransferFunction::BT709: return Some(VideoTransferCharacteristics::Bt709); case gfx::TransferFunction::SRGB: return Some(VideoTransferCharacteristics::Iec61966_2_1); case gfx::TransferFunction::PQ: return Some(VideoTransferCharacteristics::Pq); case gfx::TransferFunction::HLG: return Some(VideoTransferCharacteristics::Hlg); } MOZ_ASSERT_UNREACHABLE("unsupported gfx::TransferFunction"); return Nothing(); } Maybe ToPrimaries(const gfx::ColorSpace2& aColorSpace) { switch (aColorSpace) { case gfx::ColorSpace2::UNKNOWN: return Nothing(); case gfx::ColorSpace2::DISPLAY_P3: return Some(VideoColorPrimaries::Smpte432); case gfx::ColorSpace2::BT601_525: return Some(VideoColorPrimaries::Smpte170m); case gfx::ColorSpace2::SRGB: case gfx::ColorSpace2::BT709: return Some(VideoColorPrimaries::Bt709); case gfx::ColorSpace2::BT2020: return Some(VideoColorPrimaries::Bt2020); } MOZ_ASSERT_UNREACHABLE("unsupported gfx::ColorSpace2"); return Nothing(); } /* * The following are utilities to convert from gfx's formats to * VideoPixelFormats. */ Maybe SurfaceFormatToVideoPixelFormat( gfx::SurfaceFormat aFormat) { switch (aFormat) { case gfx::SurfaceFormat::B8G8R8A8: return Some(VideoPixelFormat::BGRA); case gfx::SurfaceFormat::B8G8R8X8: return Some(VideoPixelFormat::BGRX); case gfx::SurfaceFormat::R8G8B8A8: return Some(VideoPixelFormat::RGBA); case gfx::SurfaceFormat::R8G8B8X8: return Some(VideoPixelFormat::RGBX); case gfx::SurfaceFormat::YUV: return Some(VideoPixelFormat::I420); case gfx::SurfaceFormat::NV12: return Some(VideoPixelFormat::NV12); case gfx::SurfaceFormat::YUV422: return Some(VideoPixelFormat::I422); default: break; } return Nothing(); } Maybe ImageBitmapFormatToVideoPixelFormat( ImageBitmapFormat aFormat) { switch (aFormat) { case ImageBitmapFormat::RGBA32: return Some(VideoPixelFormat::RGBA); case ImageBitmapFormat::BGRA32: return Some(VideoPixelFormat::BGRA); case ImageBitmapFormat::YUV444P: return Some(VideoPixelFormat::I444); case ImageBitmapFormat::YUV422P: return Some(VideoPixelFormat::I422); case ImageBitmapFormat::YUV420P: return Some(VideoPixelFormat::I420); case ImageBitmapFormat::YUV420SP_NV12: return Some(VideoPixelFormat::NV12); default: break; } return Nothing(); } Result, nsresult> GetExtraDataFromArrayBuffer( const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer) { RefPtr data = MakeRefPtr(); Unused << AppendTypedArrayDataTo(aBuffer, *data); return data->Length() > 0 ? data : nullptr; } bool IsOnAndroid() { #if defined(ANDROID) return true; #else return false; #endif } bool IsOnMacOS() { #if defined(XP_MACOSX) return true; #else return false; #endif } bool IsOnLinux() { #if defined(XP_LINUX) return true; #else return false; #endif } template nsCString MaybeToString(const Maybe& aMaybe) { return nsPrintfCString( "%s", aMaybe.isSome() ? ToString(aMaybe.value()).c_str() : "nothing"); } struct ConfigurationChangeToString { nsCString operator()(const CodecChange& aCodecChange) { return nsPrintfCString("Codec: %s", NS_ConvertUTF16toUTF8(aCodecChange.get()).get()); } nsCString operator()(const DimensionsChange& aDimensionChange) { return nsPrintfCString("Dimensions: %dx%d", aDimensionChange.get().width, aDimensionChange.get().height); } nsCString operator()(const DisplayDimensionsChange& aDisplayDimensionChange) { if (aDisplayDimensionChange.get().isNothing()) { return nsPrintfCString("Display dimensions: nothing"); } gfx::IntSize displayDimensions = aDisplayDimensionChange.get().value(); return nsPrintfCString("Dimensions: %dx%d", displayDimensions.width, displayDimensions.height); } nsCString operator()(const BitrateChange& aBitrateChange) { return nsPrintfCString("Bitrate: %skbps", MaybeToString(aBitrateChange.get()).get()); } nsCString operator()(const FramerateChange& aFramerateChange) { return nsPrintfCString("Framerate: %sHz", MaybeToString(aFramerateChange.get()).get()); } nsCString operator()( const HardwareAccelerationChange& aHardwareAccelerationChange) { return nsPrintfCString("HW acceleration: %s", dom::HardwareAccelerationValues::GetString( aHardwareAccelerationChange.get()) .data()); } nsCString operator()(const AlphaChange& aAlphaChange) { return nsPrintfCString( "Alpha: %s", dom::AlphaOptionValues::GetString(aAlphaChange.get()).data()); } nsCString operator()(const ScalabilityModeChange& aScalabilityModeChange) { if (aScalabilityModeChange.get().isNothing()) { return nsCString("Scalability mode: nothing"); } return nsPrintfCString( "Scalability mode: %s", NS_ConvertUTF16toUTF8(aScalabilityModeChange.get().value()).get()); } nsCString operator()(const BitrateModeChange& aBitrateModeChange) { return nsPrintfCString( "Bitrate mode: %s", dom::VideoEncoderBitrateModeValues::GetString(aBitrateModeChange.get()) .data()); } nsCString operator()(const LatencyModeChange& aLatencyModeChange) { return nsPrintfCString( "Latency mode: %s", dom::LatencyModeValues::GetString(aLatencyModeChange.get()).data()); } nsCString operator()(const ContentHintChange& aContentHintChange) { return nsPrintfCString("Content hint: %s", MaybeToString(aContentHintChange.get()).get()); } template nsCString operator()(const T& aNewBitrate) { return nsPrintfCString("Not implemented"); } }; nsString WebCodecsConfigurationChangeList::ToString() const { nsString rv; for (const WebCodecsEncoderConfigurationItem& change : mChanges) { nsCString str = change.match(ConfigurationChangeToString()); rv.AppendPrintf("- %s\n", str.get()); } return rv; } using CodecChange = StrongTypedef; using DimensionsChange = StrongTypedef; using DisplayDimensionsChange = StrongTypedef, struct DisplayDimensionsChangeTypeWebCodecs>; using BitrateChange = StrongTypedef, struct BitrateChangeTypeWebCodecs>; using FramerateChange = StrongTypedef, struct FramerateChangeTypeWebCodecs>; using HardwareAccelerationChange = StrongTypedef; using AlphaChange = StrongTypedef; using ScalabilityModeChange = StrongTypedef, struct ScalabilityModeChangeTypeWebCodecs>; using BitrateModeChange = StrongTypedef; using LatencyModeChange = StrongTypedef; using ContentHintChange = StrongTypedef, struct ContentHintTypeTypeWebCodecs>; bool WebCodecsConfigurationChangeList::CanAttemptReconfigure() const { for (const auto& change : mChanges) { if (change.is() || change.is() || change.is() || change.is()) { return false; } } return true; } RefPtr WebCodecsConfigurationChangeList::ToPEMChangeList() const { auto rv = MakeRefPtr(); MOZ_ASSERT(CanAttemptReconfigure()); for (const auto& change : mChanges) { if (change.is()) { rv->Push(mozilla::DimensionsChange(change.as().get())); } else if (change.is()) { rv->Push(mozilla::DisplayDimensionsChange( change.as().get())); } else if (change.is()) { rv->Push(mozilla::BitrateChange(change.as().get())); } else if (change.is()) { rv->Push(mozilla::FramerateChange(change.as().get())); } else if (change.is()) { MediaDataEncoder::BitrateMode mode; if (change.as().get() == dom::VideoEncoderBitrateMode::Constant) { mode = MediaDataEncoder::BitrateMode::Constant; } else if (change.as().get() == dom::VideoEncoderBitrateMode::Variable) { mode = MediaDataEncoder::BitrateMode::Variable; } else { // Quantizer, not underlying support yet. mode = MediaDataEncoder::BitrateMode::Variable; } rv->Push(mozilla::BitrateModeChange(mode)); } else if (change.is()) { MediaDataEncoder::Usage usage; if (change.as().get() == dom::LatencyMode::Quality) { usage = MediaDataEncoder::Usage::Record; } else { usage = MediaDataEncoder::Usage::Realtime; } rv->Push(UsageChange(usage)); } else if (change.is()) { rv->Push( mozilla::ContentHintChange(change.as().get())); } } return rv.forget(); } #define ENUM_TO_STRING(enumType, enumValue) \ enumType##Values::GetString(enumValue).data() nsCString ColorSpaceInitToString( const dom::VideoColorSpaceInit& aColorSpaceInit) { nsCString rv("VideoColorSpace"); if (!aColorSpaceInit.mFullRange.IsNull()) { rv.AppendPrintf(" range: %s", aColorSpaceInit.mFullRange.Value() ? "true" : "false"); } if (!aColorSpaceInit.mMatrix.IsNull()) { rv.AppendPrintf(" matrix: %s", ENUM_TO_STRING(dom::VideoMatrixCoefficients, aColorSpaceInit.mMatrix.Value())); } if (!aColorSpaceInit.mTransfer.IsNull()) { rv.AppendPrintf(" transfer: %s", ENUM_TO_STRING(dom::VideoTransferCharacteristics, aColorSpaceInit.mTransfer.Value())); } if (!aColorSpaceInit.mPrimaries.IsNull()) { rv.AppendPrintf(" primaries: %s", ENUM_TO_STRING(dom::VideoColorPrimaries, aColorSpaceInit.mPrimaries.Value())); } return rv; } RefPtr GetWebCodecsEncoderTaskQueue() { return TaskQueue::Create( GetMediaThreadPool(MediaThreadType::PLATFORM_ENCODER), "WebCodecs encoding", false); } VideoColorSpaceInit FallbackColorSpaceForVideoContent() { // If we're unable to determine the color space, but we think this is video // content (e.g. because it's in YUV or NV12 or something like that, // consider it's in BT709). // This is step 3 of // https://w3c.github.io/webcodecs/#videoframe-pick-color-space VideoColorSpaceInit colorSpace; colorSpace.mFullRange = false; colorSpace.mMatrix = VideoMatrixCoefficients::Bt709; colorSpace.mTransfer = VideoTransferCharacteristics::Bt709; colorSpace.mPrimaries = VideoColorPrimaries::Bt709; return colorSpace; } VideoColorSpaceInit FallbackColorSpaceForWebContent() { // If we're unable to determine the color space, but we think this is from // Web content (canvas, image, svg, etc.), consider it's in sRGB. // This is step 2 of // https://w3c.github.io/webcodecs/#videoframe-pick-color-space VideoColorSpaceInit colorSpace; colorSpace.mFullRange = true; colorSpace.mMatrix = VideoMatrixCoefficients::Rgb; colorSpace.mTransfer = VideoTransferCharacteristics::Iec61966_2_1; colorSpace.mPrimaries = VideoColorPrimaries::Bt709; return colorSpace; } Maybe CodecStringToCodecType(const nsAString& aCodecString) { if (StringBeginsWith(aCodecString, u"av01"_ns)) { return Some(CodecType::AV1); } if (StringBeginsWith(aCodecString, u"vp8"_ns)) { return Some(CodecType::VP8); } if (StringBeginsWith(aCodecString, u"vp09"_ns)) { return Some(CodecType::VP9); } if (StringBeginsWith(aCodecString, u"avc1"_ns)) { return Some(CodecType::H264); } return Nothing(); } nsString ConfigToString(const VideoDecoderConfig& aConfig) { nsString rv; auto internal = VideoDecoderConfigInternal::Create(aConfig); return internal->ToString(); } }; // namespace mozilla::dom