From 40a355a42d4a9444dc753c04c6608dade2f06a23 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 03:13:27 +0200 Subject: Adding upstream version 125.0.1. Signed-off-by: Daniel Baumann --- dom/media/webcodecs/AudioData.cpp | 731 ++++++++++++++++++++++++++++++ dom/media/webcodecs/AudioData.h | 176 +++++++ dom/media/webcodecs/AudioDecoder.cpp | 481 ++++++++++++++++++++ dom/media/webcodecs/AudioDecoder.h | 83 ++++ dom/media/webcodecs/DecoderAgent.cpp | 2 - dom/media/webcodecs/DecoderTemplate.cpp | 15 +- dom/media/webcodecs/DecoderTypes.h | 64 ++- dom/media/webcodecs/EncodedAudioChunk.cpp | 260 +++++++++++ dom/media/webcodecs/EncodedAudioChunk.h | 117 +++++ dom/media/webcodecs/VideoDecoder.cpp | 97 ++-- dom/media/webcodecs/VideoEncoder.cpp | 103 ++--- dom/media/webcodecs/VideoFrame.cpp | 41 +- dom/media/webcodecs/WebCodecsUtils.cpp | 109 +++-- dom/media/webcodecs/WebCodecsUtils.h | 9 +- dom/media/webcodecs/moz.build | 6 + 15 files changed, 2079 insertions(+), 215 deletions(-) create mode 100644 dom/media/webcodecs/AudioData.cpp create mode 100644 dom/media/webcodecs/AudioData.h create mode 100644 dom/media/webcodecs/AudioDecoder.cpp create mode 100644 dom/media/webcodecs/AudioDecoder.h create mode 100644 dom/media/webcodecs/EncodedAudioChunk.cpp create mode 100644 dom/media/webcodecs/EncodedAudioChunk.h (limited to 'dom/media/webcodecs') diff --git a/dom/media/webcodecs/AudioData.cpp b/dom/media/webcodecs/AudioData.cpp new file mode 100644 index 0000000000..0b21798be8 --- /dev/null +++ b/dom/media/webcodecs/AudioData.cpp @@ -0,0 +1,731 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/Assertions.h" +#include "mozilla/Logging.h" +#include "mozilla/dom/AudioData.h" +#include "mozilla/dom/AudioDataBinding.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/StructuredCloneTags.h" +#include "nsStringFwd.h" + +#include + +#include "AudioSampleFormat.h" +#include "WebCodecsUtils.h" +#include "js/StructuredClone.h" +#include "mozilla/Maybe.h" +#include "mozilla/Result.h" + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla::dom { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOGD +# undef LOGD +#endif // LOGD +#define LOGD(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +// Only needed for refcounted objects. +// +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(AudioData) +NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(AudioData) + tmp->CloseIfNeeded(); + NS_IMPL_CYCLE_COLLECTION_UNLINK(mParent) + NS_IMPL_CYCLE_COLLECTION_UNLINK_PRESERVED_WRAPPER +NS_IMPL_CYCLE_COLLECTION_UNLINK_END +NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN(AudioData) + NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mParent) +NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END + +NS_IMPL_CYCLE_COLLECTING_ADDREF(AudioData) +// AudioData should be released as soon as its refcount drops to zero, +// without waiting for async deletion by the cycle collector, since it may hold +// a large-size PCM buffer. +NS_IMPL_CYCLE_COLLECTING_RELEASE_WITH_LAST_RELEASE(AudioData, CloseIfNeeded()) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(AudioData) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +/* + * W3C Webcodecs AudioData implementation + */ + +AudioData::AudioData(nsIGlobalObject* aParent, + const AudioDataSerializedData& aData) + : mParent(aParent), + mTimestamp(aData.mTimestamp), + mNumberOfChannels(aData.mNumberOfChannels), + mNumberOfFrames(aData.mNumberOfFrames), + mSampleRate(aData.mSampleRate), + mAudioSampleFormat(aData.mAudioSampleFormat), + // The resource is not copied, but referenced + mResource(aData.mResource) { + MOZ_ASSERT(mParent); + MOZ_ASSERT(mResource, + "Resource should always be present then receiving a transfer."); +} + +AudioData::AudioData(const AudioData& aOther) + : mParent(aOther.mParent), + mTimestamp(aOther.mTimestamp), + mNumberOfChannels(aOther.mNumberOfChannels), + mNumberOfFrames(aOther.mNumberOfFrames), + mSampleRate(aOther.mSampleRate), + mAudioSampleFormat(aOther.mAudioSampleFormat), + // The resource is not copied, but referenced + mResource(aOther.mResource) { + MOZ_ASSERT(mParent); +} + +Result, nsresult> +AudioDataResource::Construct( + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aInit) { + FallibleTArray copied; + uint8_t* rv = ProcessTypedArraysFixed( + aInit, [&](const Span& aData) -> uint8_t* { + return copied.AppendElements(aData.Elements(), aData.Length(), + fallible); + }); + if (!rv) { + LOGE("AudioDataResource::Ctor: OOM"); + return Err(NS_ERROR_OUT_OF_MEMORY); + } + return MakeAndAddRef(std::move(copied)); +} + +AudioData::AudioData( + nsIGlobalObject* aParent, + already_AddRefed aResource, + const AudioDataInit& aInit) + : mParent(aParent), + mTimestamp(aInit.mTimestamp), + mNumberOfChannels(aInit.mNumberOfChannels), + mNumberOfFrames(aInit.mNumberOfFrames), + mSampleRate(aInit.mSampleRate), + mAudioSampleFormat(Some(aInit.mFormat)), + mResource(std::move(aResource)) { + MOZ_ASSERT(mParent); +} + +AudioData::AudioData( + nsIGlobalObject* aParent, + already_AddRefed aResource, + int64_t aTimestamp, uint32_t aNumberOfChannels, uint32_t aNumberOfFrames, + float aSampleRate, AudioSampleFormat aAudioSampleFormat) + : mParent(aParent), + mTimestamp(aTimestamp), + mNumberOfChannels(aNumberOfChannels), + mNumberOfFrames(aNumberOfFrames), + mSampleRate(aSampleRate), + mAudioSampleFormat(Some(aAudioSampleFormat)), + mResource(aResource) { + MOZ_ASSERT(mParent); +} + +nsIGlobalObject* AudioData::GetParentObject() const { + AssertIsOnOwningThread(); + + return mParent.get(); +} + +JSObject* AudioData::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + AssertIsOnOwningThread(); + + return AudioData_Binding::Wrap(aCx, this, aGivenProto); +} + +uint32_t BytesPerSamples(const mozilla::dom::AudioSampleFormat& aFormat) { + switch (aFormat) { + case AudioSampleFormat::U8: + case AudioSampleFormat::U8_planar: + return sizeof(uint8_t); + case AudioSampleFormat::S16: + case AudioSampleFormat::S16_planar: + return sizeof(int16_t); + case AudioSampleFormat::S32: + case AudioSampleFormat::F32: + case AudioSampleFormat::S32_planar: + case AudioSampleFormat::F32_planar: + return sizeof(float); + default: + MOZ_ASSERT_UNREACHABLE("wrong enum value"); + } + return 0; +} + +Result IsValidAudioDataInit(const AudioDataInit& aInit) { + if (aInit.mSampleRate <= 0.0) { + auto msg = nsLiteralCString("sampleRate must be positive"); + LOGD("%s", msg.get()); + return Err(msg); + } + if (aInit.mNumberOfFrames == 0) { + auto msg = nsLiteralCString("mNumberOfFrames must be positive"); + LOGD("%s", msg.get()); + return Err(msg); + } + if (aInit.mNumberOfChannels == 0) { + auto msg = nsLiteralCString("mNumberOfChannels must be positive"); + LOGD("%s", msg.get()); + return Err(msg); + } + + uint64_t totalSamples = aInit.mNumberOfFrames * aInit.mNumberOfChannels; + uint32_t bytesPerSamples = BytesPerSamples(aInit.mFormat); + uint64_t totalSize = totalSamples * bytesPerSamples; + uint64_t arraySizeBytes = ProcessTypedArraysFixed( + aInit.mData, [&](const Span& aData) -> uint64_t { + return aData.LengthBytes(); + }); + if (arraySizeBytes < totalSize) { + auto msg = + nsPrintfCString("Array of size %" PRIu64 + " not big enough, should be at least %" PRIu64 " bytes", + arraySizeBytes, totalSize); + LOGD("%s", msg.get()); + return Err(msg); + } + return Ok(); +} + +const char* FormatToString(AudioSampleFormat aFormat) { + switch (aFormat) { + case AudioSampleFormat::U8: + return "u8"; + case AudioSampleFormat::S16: + return "s16"; + case AudioSampleFormat::S32: + return "s32"; + case AudioSampleFormat::F32: + return "f32"; + case AudioSampleFormat::U8_planar: + return "u8-planar"; + case AudioSampleFormat::S16_planar: + return "s16-planar"; + case AudioSampleFormat::S32_planar: + return "s32-planar"; + case AudioSampleFormat::F32_planar: + return "f32-planar"; + default: + MOZ_ASSERT_UNREACHABLE("wrong enum value"); + } + return "unsupported"; +} + +/* static */ +already_AddRefed AudioData::Constructor(const GlobalObject& aGlobal, + const AudioDataInit& aInit, + ErrorResult& aRv) { + nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); + LOGD("[%p] AudioData(fmt: %s, rate: %f, ch: %" PRIu32 ", ts: %" PRId64 ")", + global.get(), FormatToString(aInit.mFormat), aInit.mSampleRate, + aInit.mNumberOfChannels, aInit.mTimestamp); + if (!global) { + LOGE("Global unavailable"); + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + nsString errorMessage; + auto rv = IsValidAudioDataInit(aInit); + if (rv.isErr()) { + LOGD("AudioData::Constructor failure (IsValidAudioDataInit)"); + aRv.ThrowTypeError(rv.inspectErr()); + return nullptr; + } + auto resource = AudioDataResource::Construct(aInit.mData); + if (resource.isErr()) { + LOGD("AudioData::Constructor failure (OOM)"); + aRv.Throw(NS_ERROR_OUT_OF_MEMORY); + return nullptr; + } + + return MakeAndAddRef(global, resource.unwrap(), + aInit); +} + +// https://w3c.github.io/webcodecs/#dom-audiodata-format +Nullable AudioData::GetFormat() const { + AssertIsOnOwningThread(); + return MaybeToNullable(mAudioSampleFormat); +} + +// https://w3c.github.io/webcodecs/#dom-audiodata-samplerate +float AudioData::SampleRate() const { + AssertIsOnOwningThread(); + return mSampleRate; +} + +// https://w3c.github.io/webcodecs/#dom-audiodata-numberofframes +uint32_t AudioData::NumberOfFrames() const { + AssertIsOnOwningThread(); + return mNumberOfFrames; +} + +// https://w3c.github.io/webcodecs/#dom-audiodata-numberofchannels +uint32_t AudioData::NumberOfChannels() const { + AssertIsOnOwningThread(); + return mNumberOfChannels; +} + +// https://w3c.github.io/webcodecs/#dom-audiodata-duration +uint64_t AudioData::Duration() const { + AssertIsOnOwningThread(); + // The spec isn't clear in which direction to convert to integer. + // https://github.com/w3c/webcodecs/issues/726 + return static_cast( + static_cast(USECS_PER_S * mNumberOfFrames) / mSampleRate); +} + +// https://w3c.github.io/webcodecs/#dom-audiodata-timestamp +int64_t AudioData::Timestamp() const { + AssertIsOnOwningThread(); + return mTimestamp; +} + +struct CopyToSpec { + CopyToSpec(uint32_t aFrameCount, uint32_t aFrameOffset, uint32_t mPlaneIndex, + AudioSampleFormat aFormat) + : mFrameCount(aFrameCount), + mFrameOffset(aFrameOffset), + mPlaneIndex(mPlaneIndex), + mFormat(aFormat) {} + + const uint32_t mFrameCount; + const uint32_t mFrameOffset; + const uint32_t mPlaneIndex; + const AudioSampleFormat mFormat; +}; + +bool IsInterleaved(const AudioSampleFormat& aFormat) { + switch (aFormat) { + case AudioSampleFormat::U8: + case AudioSampleFormat::S16: + case AudioSampleFormat::S32: + case AudioSampleFormat::F32: + return true; + case AudioSampleFormat::U8_planar: + case AudioSampleFormat::S16_planar: + case AudioSampleFormat::S32_planar: + case AudioSampleFormat::F32_planar: + return false; + }; + MOZ_ASSERT_UNREACHABLE("Invalid enum value"); + return false; +} + +size_t AudioData::ComputeCopyElementCount( + const AudioDataCopyToOptions& aOptions, ErrorResult& aRv) { + // https://w3c.github.io/webcodecs/#compute-copy-element-count + // 1, 2 + auto destFormat = mAudioSampleFormat; + if (aOptions.mFormat.WasPassed()) { + destFormat = OptionalToMaybe(aOptions.mFormat); + } + // 3, 4 + MOZ_ASSERT(destFormat.isSome()); + if (IsInterleaved(destFormat.value())) { + if (aOptions.mPlaneIndex > 0) { + auto msg = "Interleaved format, but plane index > 0"_ns; + LOGD("%s", msg.get()); + aRv.ThrowRangeError(msg); + return 0; + } + } else { + if (aOptions.mPlaneIndex >= mNumberOfChannels) { + auto msg = nsPrintfCString( + "Plane index %" PRIu32 + " greater or equal than the number of channels %" PRIu32, + aOptions.mPlaneIndex, mNumberOfChannels); + LOGD("%s", msg.get()); + aRv.ThrowRangeError(msg); + return 0; + } + } + // 5 -- conversion between all formats supported + // 6 -- all planes have the same number of frames, always + uint64_t frameCount = mNumberOfFrames; + // 7 + if (aOptions.mFrameOffset >= frameCount) { + auto msg = nsPrintfCString("Frame offset of %" PRIu32 + " greater or equal than frame count %" PRIu64, + aOptions.mFrameOffset, frameCount); + LOGD("%s", msg.get()); + aRv.ThrowRangeError(msg); + return 0; + } + // 8, 9 + uint64_t copyFrameCount = frameCount - aOptions.mFrameOffset; + if (aOptions.mFrameCount.WasPassed()) { + if (aOptions.mFrameCount.Value() > copyFrameCount) { + auto msg = nsPrintfCString( + "Passed copy frame count of %" PRIu32 + " greater than available source frames for copy of %" PRIu64, + aOptions.mFrameCount.Value(), copyFrameCount); + LOGD("%s", msg.get()); + aRv.ThrowRangeError(msg); + return 0; + } + copyFrameCount = aOptions.mFrameCount.Value(); + } + // 10, 11 + uint64_t elementCount = copyFrameCount; + if (IsInterleaved(destFormat.value())) { + elementCount *= mNumberOfChannels; + } + return elementCount; +} + +// https://w3c.github.io/webcodecs/#dom-audiodata-allocationsize +// This method returns an int, that can be zero in case of success or error. +// Caller should check aRv to determine success or error. +uint32_t AudioData::AllocationSize(const AudioDataCopyToOptions& aOptions, + ErrorResult& aRv) { + AssertIsOnOwningThread(); + if (!mResource) { + auto msg = "allocationSize called on detached AudioData"_ns; + LOGD("%s", msg.get()); + aRv.ThrowInvalidStateError(msg); + return 0; + } + size_t copyElementCount = ComputeCopyElementCount(aOptions, aRv); + if (aRv.Failed()) { + LOGD("AudioData::AllocationSize failure"); + // ComputeCopyElementCount has set the exception type. + return 0; + } + Maybe destFormat = mAudioSampleFormat; + if (aOptions.mFormat.WasPassed()) { + destFormat = OptionalToMaybe(aOptions.mFormat); + } + if (destFormat.isNothing()) { + auto msg = "AudioData has an unknown format"_ns; + LOGD("%s", msg.get()); + // See https://github.com/w3c/webcodecs/issues/727 -- it isn't clear yet + // what to do here + aRv.ThrowRangeError(msg); + return 0; + } + CheckedInt bytesPerSample = BytesPerSamples(destFormat.ref()); + + auto res = bytesPerSample * copyElementCount; + if (res.isValid()) { + return res.value(); + } + aRv.ThrowRangeError("Allocation size too large"); + return 0; +} + +template +void CopySamples(Span aSource, Span aDest, uint32_t aSourceChannelCount, + const AudioSampleFormat aSourceFormat, + const CopyToSpec& aCopyToSpec) { + if (IsInterleaved(aSourceFormat) && IsInterleaved(aCopyToSpec.mFormat)) { + MOZ_ASSERT(aCopyToSpec.mPlaneIndex == 0); + MOZ_ASSERT(aDest.Length() >= aCopyToSpec.mFrameCount); + MOZ_ASSERT(aSource.Length() - aCopyToSpec.mFrameOffset >= + aCopyToSpec.mFrameCount); + // This turns into a regular memcpy if the types are in fact equal + ConvertAudioSamples(aSource.data() + aCopyToSpec.mFrameOffset, aDest.data(), + aCopyToSpec.mFrameCount * aSourceChannelCount); + return; + } + if (IsInterleaved(aSourceFormat) && !IsInterleaved(aCopyToSpec.mFormat)) { + DebugOnly sourceFrameCount = aSource.Length() / aSourceChannelCount; + MOZ_ASSERT(aDest.Length() >= aCopyToSpec.mFrameCount); + MOZ_ASSERT(aSource.Length() - aCopyToSpec.mFrameOffset >= + aCopyToSpec.mFrameCount); + // Interleaved to planar -- only copy samples of the correct channel to the + // destination + size_t readIndex = aCopyToSpec.mFrameOffset * aSourceChannelCount + + aCopyToSpec.mPlaneIndex; + for (size_t i = 0; i < aCopyToSpec.mFrameCount; i++) { + aDest[i] = ConvertAudioSample(aSource[readIndex]); + readIndex += aSourceChannelCount; + } + return; + } + + if (!IsInterleaved(aSourceFormat) && IsInterleaved(aCopyToSpec.mFormat)) { + MOZ_CRASH("This should never be hit -- current spec doesn't support it"); + // Planar to interleaved -- copy of all channels of the source into the + // destination buffer. + MOZ_ASSERT(aCopyToSpec.mPlaneIndex == 0); + MOZ_ASSERT(aDest.Length() >= aCopyToSpec.mFrameCount * aSourceChannelCount); + MOZ_ASSERT(aSource.Length() - + aCopyToSpec.mFrameOffset * aSourceChannelCount >= + aCopyToSpec.mFrameCount * aSourceChannelCount); + size_t writeIndex = 0; + // Scan the source linearly and put each sample at the right position in the + // destination interleaved buffer. + size_t readIndex = 0; + for (size_t channel = 0; channel < aSourceChannelCount; channel++) { + writeIndex = channel; + for (size_t i = 0; i < aCopyToSpec.mFrameCount; i++) { + aDest[writeIndex] = ConvertAudioSample(aSource[readIndex]); + readIndex++; + writeIndex += aSourceChannelCount; + } + } + return; + } + if (!IsInterleaved(aSourceFormat) && !IsInterleaved(aCopyToSpec.mFormat)) { + // Planar to Planar / convert + copy from the right index in the source. + size_t offset = + aCopyToSpec.mPlaneIndex * aSource.Length() / aSourceChannelCount; + MOZ_ASSERT(aDest.Length() >= aSource.Length() / aSourceChannelCount - + aCopyToSpec.mFrameOffset); + for (uint32_t i = 0; i < aCopyToSpec.mFrameCount; i++) { + aDest[i] = + ConvertAudioSample(aSource[offset + aCopyToSpec.mFrameOffset + i]); + } + } +} + +nsCString AudioData::ToString() const { + if (!mResource) { + return nsCString("AudioData[detached]"); + } + return nsPrintfCString("AudioData[%zu bytes %s %fHz %" PRIu32 "x%" PRIu32 + "ch]", + mResource->Data().LengthBytes(), + FormatToString(mAudioSampleFormat.value()), + mSampleRate, mNumberOfFrames, mNumberOfChannels); +} + +nsCString CopyToToString(size_t aDestBufSize, + const AudioDataCopyToOptions& aOptions) { + return nsPrintfCString( + "AudioDataCopyToOptions[data: %zu bytes %s frame count:%" PRIu32 + " frame offset: %" PRIu32 " plane: %" PRIu32 "]", + aDestBufSize, + aOptions.mFormat.WasPassed() ? FormatToString(aOptions.mFormat.Value()) + : "null", + aOptions.mFrameCount.WasPassed() ? aOptions.mFrameCount.Value() : 0, + aOptions.mFrameOffset, aOptions.mPlaneIndex); +} + +using DataSpanType = + Variant, Span, Span, Span>; + +DataSpanType GetDataSpan(Span aSpan, const AudioSampleFormat aFormat) { + const size_t Length = aSpan.Length() / BytesPerSamples(aFormat); + // TODO: Check size so Span can be reasonably constructed? + switch (aFormat) { + case AudioSampleFormat::U8: + case AudioSampleFormat::U8_planar: + return AsVariant(aSpan); + case AudioSampleFormat::S16: + case AudioSampleFormat::S16_planar: + return AsVariant(Span(reinterpret_cast(aSpan.data()), Length)); + case AudioSampleFormat::S32: + case AudioSampleFormat::S32_planar: + return AsVariant(Span(reinterpret_cast(aSpan.data()), Length)); + case AudioSampleFormat::F32: + case AudioSampleFormat::F32_planar: + return AsVariant(Span(reinterpret_cast(aSpan.data()), Length)); + } + MOZ_ASSERT_UNREACHABLE("Invalid enum value"); + return AsVariant(aSpan); +} + +void CopySamples(DataSpanType& aSource, DataSpanType& aDest, + uint32_t aSourceChannelCount, + const AudioSampleFormat aSourceFormat, + const CopyToSpec& aCopyToSpec) { + aSource.match([&](auto& src) { + aDest.match([&](auto& dst) { + CopySamples(src, dst, aSourceChannelCount, aSourceFormat, aCopyToSpec); + }); + }); +} + +void DoCopy(Span aSource, Span aDest, + const uint32_t aSourceChannelCount, + const AudioSampleFormat aSourceFormat, + const CopyToSpec& aCopyToSpec) { + DataSpanType source = GetDataSpan(aSource, aSourceFormat); + DataSpanType dest = GetDataSpan(aDest, aCopyToSpec.mFormat); + CopySamples(source, dest, aSourceChannelCount, aSourceFormat, aCopyToSpec); +} + +// https://w3c.github.io/webcodecs/#dom-audiodata-copyto +void AudioData::CopyTo( + const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination, + const AudioDataCopyToOptions& aOptions, ErrorResult& aRv) { + AssertIsOnOwningThread(); + + size_t destLength = ProcessTypedArraysFixed( + aDestination, [&](const Span& aData) -> size_t { + return aData.LengthBytes(); + }); + + LOGD("AudioData::CopyTo %s -> %s", ToString().get(), + CopyToToString(destLength, aOptions).get()); + + if (!mResource) { + auto msg = "copyTo called on closed AudioData"_ns; + LOGD("%s", msg.get()); + aRv.ThrowInvalidStateError(msg); + return; + } + + uint64_t copyElementCount = ComputeCopyElementCount(aOptions, aRv); + if (aRv.Failed()) { + LOGD("AudioData::CopyTo failed in ComputeCopyElementCount"); + return; + } + auto destFormat = mAudioSampleFormat; + if (aOptions.mFormat.WasPassed()) { + destFormat = OptionalToMaybe(aOptions.mFormat); + } + + uint32_t bytesPerSample = BytesPerSamples(destFormat.value()); + CheckedInt copyLength = bytesPerSample; + copyLength *= copyElementCount; + if (copyLength.value() > destLength) { + auto msg = nsPrintfCString( + "destination buffer of length %zu too small for copying %" PRIu64 + " elements", + destLength, bytesPerSample * copyElementCount); + LOGD("%s", msg.get()); + aRv.ThrowRangeError(msg); + return; + } + + uint32_t framesToCopy = mNumberOfFrames - aOptions.mFrameOffset; + if (aOptions.mFrameCount.WasPassed()) { + framesToCopy = aOptions.mFrameCount.Value(); + } + + CopyToSpec copyToSpec(framesToCopy, aOptions.mFrameOffset, + aOptions.mPlaneIndex, destFormat.value()); + + // Now a couple layers of macros to type the pointers and perform the actual + // copy. + ProcessTypedArraysFixed(aDestination, [&](const Span& aData) { + DoCopy(mResource->Data(), aData, mNumberOfChannels, + mAudioSampleFormat.value(), copyToSpec); + }); +} + +// https://w3c.github.io/webcodecs/#dom-audiodata-clone +already_AddRefed AudioData::Clone(ErrorResult& aRv) { + AssertIsOnOwningThread(); + + if (!mResource) { + auto msg = "No media resource in the AudioData now"_ns; + LOGD("%s", msg.get()); + aRv.ThrowInvalidStateError(msg); + return nullptr; + } + + return MakeAndAddRef(*this); +} + +// https://w3c.github.io/webcodecs/#close-audiodata +void AudioData::Close() { + AssertIsOnOwningThread(); + + mResource = nullptr; + mSampleRate = 0; + mNumberOfFrames = 0; + mNumberOfChannels = 0; + mAudioSampleFormat = Nothing(); +} + +// https://w3c.github.io/webcodecs/#ref-for-deserialization-steps%E2%91%A1 +/* static */ +JSObject* AudioData::ReadStructuredClone(JSContext* aCx, + nsIGlobalObject* aGlobal, + JSStructuredCloneReader* aReader, + const AudioDataSerializedData& aData) { + JS::Rooted value(aCx, JS::NullValue()); + // To avoid a rooting hazard error from returning a raw JSObject* before + // running the RefPtr destructor, RefPtr needs to be destructed before + // returning the raw JSObject*, which is why the RefPtr is created + // in the scope below. Otherwise, the static analysis infers the RefPtr cannot + // be safely destructed while the unrooted return JSObject* is on the stack. + { + RefPtr frame = MakeAndAddRef(aGlobal, aData); + if (!GetOrCreateDOMReflector(aCx, frame, &value) || !value.isObject()) { + LOGE("GetOrCreateDOMReflect failure"); + return nullptr; + } + } + return value.toObjectOrNull(); +} + +// https://w3c.github.io/webcodecs/#ref-for-audiodata%E2%91%A2%E2%91%A2 +bool AudioData::WriteStructuredClone(JSStructuredCloneWriter* aWriter, + StructuredCloneHolder* aHolder) const { + AssertIsOnOwningThread(); + + // AudioData closed + if (!mResource) { + LOGD("AudioData was already close in WriteStructuredClone"); + return false; + } + const uint32_t index = aHolder->AudioData().Length(); + // https://github.com/w3c/webcodecs/issues/717 + // For now, serialization is only allowed in the same address space, it's OK + // to send a refptr here instead of copying the backing buffer. + aHolder->AudioData().AppendElement(AudioDataSerializedData(*this)); + + return !NS_WARN_IF(!JS_WriteUint32Pair(aWriter, SCTAG_DOM_AUDIODATA, index)); +} + +// https://w3c.github.io/webcodecs/#ref-for-transfer-steps +UniquePtr AudioData::Transfer() { + AssertIsOnOwningThread(); + + if (!mResource) { + // Closed + LOGD("AudioData was already close in Transfer"); + return nullptr; + } + + // This adds a ref to the resource + auto serialized = MakeUnique(*this); + // This removes the ref to the resource, effectively transfering the backing + // storage. + Close(); + return serialized; +} + +// https://w3c.github.io/webcodecs/#ref-for-transfer-receiving-steps +/* static */ +already_AddRefed AudioData::FromTransferred(nsIGlobalObject* aGlobal, + TransferredData* aData) { + MOZ_ASSERT(aData); + + return MakeAndAddRef(aGlobal, *aData); +} + +void AudioData::CloseIfNeeded() { + if (mResource) { + mResource = nullptr; + } +} + +#undef LOGD +#undef LOGE +#undef LOG_INTERNAL + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/AudioData.h b/dom/media/webcodecs/AudioData.h new file mode 100644 index 0000000000..4ae69a225a --- /dev/null +++ b/dom/media/webcodecs/AudioData.h @@ -0,0 +1,176 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_AudioData_h +#define mozilla_dom_AudioData_h + +#include "MediaData.h" +#include "WebCodecsUtils.h" +#include "js/TypeDecls.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/Span.h" +#include "mozilla/dom/AudioDataBinding.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/StructuredCloneHolder.h" +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; +class nsIURI; + +namespace mozilla::dom { + +class MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer; +class OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer; +class Promise; +struct AudioDataBufferInit; +struct AudioDataCopyToOptions; +struct AudioDataInit; + +} // namespace mozilla::dom + +namespace mozilla::dom { + +class AudioData; +class AudioDataResource; +struct AudioDataSerializedData; + +class AudioData final : public nsISupports, public nsWrapperCache { + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(AudioData) + + public: + AudioData(nsIGlobalObject* aParent, const AudioDataSerializedData& aData); + AudioData(nsIGlobalObject* aParent, + already_AddRefed aResource, + const AudioDataInit& aInit); + AudioData(nsIGlobalObject* aParent, + already_AddRefed aResource, + int64_t aTimestamp, uint32_t aNumberOfChannels, + uint32_t aNumberOfFrames, float aSampleRate, + AudioSampleFormat aAudioSampleFormat); + AudioData(const AudioData& aOther); + + protected: + ~AudioData() = default; + + public: + nsIGlobalObject* GetParentObject() const; + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + static already_AddRefed Constructor(const GlobalObject& aGlobal, + const AudioDataInit& aInit, + ErrorResult& aRv); + + Nullable GetFormat() const; + + float SampleRate() const; + + uint32_t NumberOfFrames() const; + + uint32_t NumberOfChannels() const; + + uint64_t Duration() const; // microseconds + + int64_t Timestamp() const; // microseconds + + uint32_t AllocationSize(const AudioDataCopyToOptions& aOptions, + ErrorResult& aRv); + + void CopyTo( + const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination, + const AudioDataCopyToOptions& aOptions, ErrorResult& aRv); + + already_AddRefed Clone(ErrorResult& aRv); + + void Close(); + + // [Serializable] implementations: {Read, Write}StructuredClone + static JSObject* ReadStructuredClone(JSContext* aCx, nsIGlobalObject* aGlobal, + JSStructuredCloneReader* aReader, + const AudioDataSerializedData& aData); + + bool WriteStructuredClone(JSStructuredCloneWriter* aWriter, + StructuredCloneHolder* aHolder) const; + + // [Transferable] implementations: Transfer, FromTransferred + using TransferredData = AudioDataSerializedData; + + UniquePtr Transfer(); + + static already_AddRefed FromTransferred(nsIGlobalObject* aGlobal, + TransferredData* aData); + + private: + size_t ComputeCopyElementCount(const AudioDataCopyToOptions& aOptions, + ErrorResult& aRv); + + nsCString ToString() const; + // AudioData can run on either main thread or worker thread. + void AssertIsOnOwningThread() const { NS_ASSERT_OWNINGTHREAD(AudioData); } + void CloseIfNeeded(); + + nsCOMPtr mParent; + + friend struct AudioDataSerializedData; + + int64_t mTimestamp; + uint32_t mNumberOfChannels; + uint32_t mNumberOfFrames; + float mSampleRate; + Maybe mAudioSampleFormat; + RefPtr mResource; +}; + +class AudioDataResource final { + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AudioDataResource); + explicit AudioDataResource(FallibleTArray&& aData) + : mData(std::move(aData)) {} + + explicit AudioDataResource() : mData() {} + + static AudioDataResource* Create(const Span& aData) { + AudioDataResource* resource = new AudioDataResource(); + if (!resource->mData.AppendElements(aData, mozilla::fallible_t())) { + return nullptr; + } + return resource; + } + + static Result, nsresult> Construct( + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aInit); + + Span Data() { return Span(mData.Elements(), mData.Length()); }; + + private: + ~AudioDataResource() = default; + // It's always possible for the allocation to fail -- the size is + // controled by script. + FallibleTArray mData; +}; + +struct AudioDataSerializedData { + explicit AudioDataSerializedData(const AudioData& aFrom) + : mTimestamp(aFrom.Timestamp()), + mNumberOfChannels(aFrom.NumberOfChannels()), + mNumberOfFrames(aFrom.NumberOfFrames()), + mSampleRate(aFrom.SampleRate()), + mAudioSampleFormat(NullableToMaybe(aFrom.GetFormat())), + mResource(aFrom.mResource) {} + int64_t mTimestamp{}; + uint32_t mNumberOfChannels{}; + uint32_t mNumberOfFrames{}; + float mSampleRate{}; + Maybe mAudioSampleFormat; + RefPtr mResource; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_AudioData_h diff --git a/dom/media/webcodecs/AudioDecoder.cpp b/dom/media/webcodecs/AudioDecoder.cpp new file mode 100644 index 0000000000..6b554dcacf --- /dev/null +++ b/dom/media/webcodecs/AudioDecoder.cpp @@ -0,0 +1,481 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/AudioDecoder.h" +#include "mozilla/dom/AudioDecoderBinding.h" + +#include "DecoderTraits.h" +#include "MediaContainerType.h" +#include "MediaData.h" +#include "VideoUtils.h" +#include "mozilla/Assertions.h" +#include "mozilla/Logging.h" +#include "mozilla/Maybe.h" +#include "mozilla/Try.h" +#include "mozilla/Unused.h" +#include "mozilla/dom/AudioDataBinding.h" +#include "mozilla/dom/EncodedAudioChunk.h" +#include "mozilla/dom/EncodedAudioChunkBinding.h" +#include "mozilla/dom/ImageUtils.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/WebCodecsUtils.h" +#include "nsPrintfCString.h" +#include "nsReadableUtils.h" + +extern mozilla::LazyLogModule gWebCodecsLog; + +namespace mozilla::dom { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOG +# undef LOG +#endif // LOG +#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) + +#ifdef LOGW +# undef LOGW +#endif // LOGW +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +#ifdef LOGV +# undef LOGV +#endif // LOGV +#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__) + +NS_IMPL_CYCLE_COLLECTION_INHERITED(AudioDecoder, DOMEventTargetHelper, + mErrorCallback, mOutputCallback) +NS_IMPL_ADDREF_INHERITED(AudioDecoder, DOMEventTargetHelper) +NS_IMPL_RELEASE_INHERITED(AudioDecoder, DOMEventTargetHelper) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(AudioDecoder) +NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper) + +/* + * Below are helper classes + */ + +AudioDecoderConfigInternal::AudioDecoderConfigInternal( + const nsAString& aCodec, uint32_t aSampleRate, uint32_t aNumberOfChannels, + Maybe>&& aDescription) + : mCodec(aCodec), + mSampleRate(aSampleRate), + mNumberOfChannels(aNumberOfChannels), + mDescription(std::move(aDescription)) {} + +/*static*/ +UniquePtr AudioDecoderConfigInternal::Create( + const AudioDecoderConfig& aConfig) { + nsCString errorMessage; + if (!AudioDecoderTraits::Validate(aConfig, errorMessage)) { + LOGE("Failed to create AudioDecoderConfigInternal: %s", errorMessage.get()); + return nullptr; + } + + Maybe> description; + if (aConfig.mDescription.WasPassed()) { + auto rv = GetExtraDataFromArrayBuffer(aConfig.mDescription.Value()); + if (rv.isErr()) { // Invalid description data. + nsCString error; + GetErrorName(rv.unwrapErr(), error); + LOGE( + "Failed to create AudioDecoderConfigInternal due to invalid " + "description data. Error: %s", + error.get()); + return nullptr; + } + description.emplace(rv.unwrap()); + } + + return UniquePtr(new AudioDecoderConfigInternal( + aConfig.mCodec, aConfig.mSampleRate, aConfig.mNumberOfChannels, + std::move(description))); +} + +/* + * The followings are helpers for AudioDecoder methods + */ + +struct AudioMIMECreateParam { + explicit AudioMIMECreateParam(const AudioDecoderConfigInternal& aConfig) + : mParsedCodec(ParseCodecString(aConfig.mCodec).valueOr(EmptyString())) {} + explicit AudioMIMECreateParam(const AudioDecoderConfig& aConfig) + : mParsedCodec(ParseCodecString(aConfig.mCodec).valueOr(EmptyString())) {} + + const nsString mParsedCodec; +}; + +// Map between WebCodecs pcm types as strings and codec numbers +// All other codecs +nsCString ConvertCodecName(const nsCString& aContainer, + const nsCString& aCodec) { + if (!aContainer.EqualsLiteral("x-wav")) { + return aCodec; + } + if (aCodec.EqualsLiteral("ulaw")) { + return nsCString("7"); + } + if (aCodec.EqualsLiteral("alaw")) { + return nsCString("6"); + } + if (aCodec.Find("f32")) { + return nsCString("3"); + } + // Linear PCM + return nsCString("1"); +} + +static nsTArray GuessMIMETypes(const AudioMIMECreateParam& aParam) { + nsCString codec = NS_ConvertUTF16toUTF8(aParam.mParsedCodec); + nsTArray types; + for (const nsCString& container : GuessContainers(aParam.mParsedCodec)) { + codec = ConvertCodecName(container, codec); + nsPrintfCString mime("audio/%s; codecs=%s", container.get(), codec.get()); + types.AppendElement(mime); + } + return types; +} + +static bool IsSupportedAudioCodec(const nsAString& aCodec) { + LOG("IsSupportedAudioCodec: %s", NS_ConvertUTF16toUTF8(aCodec).get()); + return aCodec.EqualsLiteral("flac") || aCodec.EqualsLiteral("mp3") || + IsAACCodecString(aCodec) || aCodec.EqualsLiteral("opus") || + aCodec.EqualsLiteral("ulaw") || aCodec.EqualsLiteral("alaw") || + aCodec.EqualsLiteral("pcm-u8") || aCodec.EqualsLiteral("pcm-s16") || + aCodec.EqualsLiteral("pcm-s24") || aCodec.EqualsLiteral("pcm-s32") || + aCodec.EqualsLiteral("pcm-f32"); +} + +// https://w3c.github.io/webcodecs/#check-configuration-support +template +static bool CanDecodeAudio(const Config& aConfig) { + auto param = AudioMIMECreateParam(aConfig); + if (!IsSupportedAudioCodec(param.mParsedCodec)) { + return false; + } + if (IsOnAndroid() && IsAACCodecString(param.mParsedCodec)) { + return false; + } + // TODO: Instead of calling CanHandleContainerType with the guessed the + // containers, DecoderTraits should provide an API to tell if a codec is + // decodable or not. + for (const nsCString& mime : GuessMIMETypes(param)) { + if (Maybe containerType = + MakeMediaExtendedMIMEType(mime)) { + if (DecoderTraits::CanHandleContainerType( + *containerType, nullptr /* DecoderDoctorDiagnostics */) != + CANPLAY_NO) { + return true; + } + } + } + return false; +} + +static nsTArray> GetTracksInfo( + const AudioDecoderConfigInternal& aConfig) { + // TODO: Instead of calling GetTracksInfo with the guessed containers, + // DecoderTraits should provide an API to create the TrackInfo directly. + for (const nsCString& mime : GuessMIMETypes(AudioMIMECreateParam(aConfig))) { + if (Maybe containerType = + MakeMediaExtendedMIMEType(mime)) { + if (nsTArray> tracks = + DecoderTraits::GetTracksInfo(*containerType); + !tracks.IsEmpty()) { + return tracks; + } + } + } + return {}; +} + +static Result CloneConfiguration( + RootedDictionary& aDest, JSContext* aCx, + const AudioDecoderConfig& aConfig, ErrorResult& aRv) { + aDest.mCodec = aConfig.mCodec; + if (aConfig.mDescription.WasPassed()) { + aDest.mDescription.Construct(); + MOZ_TRY(CloneBuffer(aCx, aDest.mDescription.Value(), + aConfig.mDescription.Value(), aRv)); + } + + aDest.mNumberOfChannels = aConfig.mNumberOfChannels; + aDest.mSampleRate = aConfig.mSampleRate; + + return Ok(); +} + +// https://w3c.github.io/webcodecs/#create-a-audiodata +static RefPtr CreateAudioData(nsIGlobalObject* aGlobalObject, + mozilla::AudioData* aData) { + MOZ_ASSERT(aGlobalObject); + MOZ_ASSERT(aData); + + auto buf = aData->MoveableData(); + // TODO: Ensure buf.Length() is a multiple of aData->mChannels and put it into + // AssertedCast (sinze return type of buf.Length() is size_t). + uint32_t frames = buf.Length() / aData->mChannels; + RefPtr resource = AudioDataResource::Create(Span{ + reinterpret_cast(buf.Data()), buf.Length() * sizeof(float)}); + return MakeRefPtr(aGlobalObject, resource.forget(), + aData->mTime.ToMicroseconds(), aData->mChannels, + frames, AssertedCast(aData->mRate), + mozilla::dom::AudioSampleFormat::F32); +} + +/* static */ +bool AudioDecoderTraits::IsSupported( + const AudioDecoderConfigInternal& aConfig) { + return CanDecodeAudio(aConfig); +} + +/* static */ +Result, nsresult> AudioDecoderTraits::CreateTrackInfo( + const AudioDecoderConfigInternal& aConfig) { + LOG("Create a AudioInfo from %s config", + NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); + + nsTArray> tracks = GetTracksInfo(aConfig); + if (tracks.Length() != 1 || tracks[0]->GetType() != TrackInfo::kAudioTrack) { + LOGE("Failed to get TrackInfo"); + return Err(NS_ERROR_INVALID_ARG); + } + + UniquePtr track(std::move(tracks[0])); + AudioInfo* ai = track->GetAsAudioInfo(); + if (!ai) { + LOGE("Failed to get AudioInfo"); + return Err(NS_ERROR_INVALID_ARG); + } + + if (aConfig.mDescription.isSome()) { + RefPtr buf; + buf = aConfig.mDescription.value(); + if (buf) { + LOG("The given config has %zu bytes of description data", buf->Length()); + ai->mCodecSpecificConfig = + AudioCodecSpecificVariant{AudioCodecSpecificBinaryBlob{buf}}; + } + } + + ai->mChannels = aConfig.mNumberOfChannels; + ai->mRate = aConfig.mSampleRate; + + LOG("Created AudioInfo %s (%" PRIu32 "ch %" PRIu32 + "Hz - with extra-data: %s)", + NS_ConvertUTF16toUTF8(aConfig.mCodec).get(), ai->mChannels, ai->mChannels, + aConfig.mDescription.isSome() ? "yes" : "no"); + + return track; +} + +// https://w3c.github.io/webcodecs/#valid-audiodecoderconfig +/* static */ +bool AudioDecoderTraits::Validate(const AudioDecoderConfig& aConfig, + nsCString& aErrorMessage) { + Maybe codec = ParseCodecString(aConfig.mCodec); + if (!codec || codec->IsEmpty()) { + LOGE("Validating AudioDecoderConfig: invalid codec string"); + + aErrorMessage.AppendPrintf("Invalid codec string %s", + NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); + return false; + } + + LOG("Validating AudioDecoderConfig: codec: %s %uch %uHz %s extradata", + NS_ConvertUTF16toUTF8(codec.value()).get(), aConfig.mNumberOfChannels, + aConfig.mSampleRate, aConfig.mDescription.WasPassed() ? "w/" : "no"); + + if (aConfig.mNumberOfChannels == 0) { + aErrorMessage.AppendPrintf("Invalid number of channels of %u", + aConfig.mNumberOfChannels); + return false; + } + + if (aConfig.mSampleRate == 0) { + aErrorMessage.AppendPrintf("Invalid sample-rate of %u", + aConfig.mNumberOfChannels); + return false; + } + + bool detached = + aConfig.mDescription.WasPassed() && + (aConfig.mDescription.Value().IsArrayBuffer() + ? JS::ArrayBuffer::fromObject( + aConfig.mDescription.Value().GetAsArrayBuffer().Obj()) + .isDetached() + : JS::ArrayBufferView::fromObject( + aConfig.mDescription.Value().GetAsArrayBufferView().Obj()) + .isDetached()); + + if (detached) { + LOGE("description is detached."); + return false; + } + + return true; +} + +/* static */ +UniquePtr AudioDecoderTraits::CreateConfigInternal( + const AudioDecoderConfig& aConfig) { + return AudioDecoderConfigInternal::Create(aConfig); +} + +/* static */ +bool AudioDecoderTraits::IsKeyChunk(const EncodedAudioChunk& aInput) { + return aInput.Type() == EncodedAudioChunkType::Key; +} + +/* static */ +UniquePtr AudioDecoderTraits::CreateInputInternal( + const EncodedAudioChunk& aInput) { + return aInput.Clone(); +} + +/* + * Below are AudioDecoder implementation + */ + +AudioDecoder::AudioDecoder(nsIGlobalObject* aParent, + RefPtr&& aErrorCallback, + RefPtr&& aOutputCallback) + : DecoderTemplate(aParent, std::move(aErrorCallback), + std::move(aOutputCallback)) { + MOZ_ASSERT(mErrorCallback); + MOZ_ASSERT(mOutputCallback); + LOG("AudioDecoder %p ctor", this); +} + +AudioDecoder::~AudioDecoder() { + LOG("AudioDecoder %p dtor", this); + Unused << ResetInternal(NS_ERROR_DOM_ABORT_ERR); +} + +JSObject* AudioDecoder::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + AssertIsOnOwningThread(); + + return AudioDecoder_Binding::Wrap(aCx, this, aGivenProto); +} + +// https://w3c.github.io/webcodecs/#dom-audiodecoder-audiodecoder +/* static */ +already_AddRefed AudioDecoder::Constructor( + const GlobalObject& aGlobal, const AudioDecoderInit& aInit, + ErrorResult& aRv) { + nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + return MakeAndAddRef( + global.get(), RefPtr(aInit.mError), + RefPtr(aInit.mOutput)); +} + +// https://w3c.github.io/webcodecs/#dom-audiodecoder-isconfigsupported +/* static */ +already_AddRefed AudioDecoder::IsConfigSupported( + const GlobalObject& aGlobal, const AudioDecoderConfig& aConfig, + ErrorResult& aRv) { + LOG("AudioDecoder::IsConfigSupported, config: %s", + NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); + + nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + RefPtr p = Promise::Create(global.get(), aRv); + if (NS_WARN_IF(aRv.Failed())) { + return p.forget(); + } + + nsCString errorMessage; + if (!AudioDecoderTraits::Validate(aConfig, errorMessage)) { + p->MaybeRejectWithTypeError(errorMessage); + return p.forget(); + } + + RootedDictionary config(aGlobal.Context()); + auto r = CloneConfiguration(config, aGlobal.Context(), aConfig, aRv); + if (r.isErr()) { + // This can only be an OOM: all members to clone are known to be valid + // because this is check by ::Validate above. + MOZ_ASSERT(r.inspectErr() == NS_ERROR_OUT_OF_MEMORY && + aRv.ErrorCodeIs(NS_ERROR_OUT_OF_MEMORY)); + return p.forget(); + } + + bool canDecode = CanDecodeAudio(config); + RootedDictionary s(aGlobal.Context()); + s.mConfig.Construct(std::move(config)); + s.mSupported.Construct(canDecode); + + p->MaybeResolve(s); + return p.forget(); +} + +already_AddRefed AudioDecoder::InputDataToMediaRawData( + UniquePtr&& aData, TrackInfo& aInfo, + const AudioDecoderConfigInternal& aConfig) { + AssertIsOnOwningThread(); + MOZ_ASSERT(aInfo.GetAsAudioInfo()); + + if (!aData) { + LOGE("No data for conversion"); + return nullptr; + } + + RefPtr sample = aData->TakeData(); + if (!sample) { + LOGE("Take no data for conversion"); + return nullptr; + } + + LOGV( + "EncodedAudioChunkData %p converted to %zu-byte MediaRawData - time: " + "%" PRIi64 "us, timecode: %" PRIi64 "us, duration: %" PRIi64 + "us, key-frame: %s", + aData.get(), sample->Size(), sample->mTime.ToMicroseconds(), + sample->mTimecode.ToMicroseconds(), sample->mDuration.ToMicroseconds(), + sample->mKeyframe ? "yes" : "no"); + + return sample.forget(); +} + +nsTArray> AudioDecoder::DecodedDataToOutputType( + nsIGlobalObject* aGlobalObject, const nsTArray>&& aData, + AudioDecoderConfigInternal& aConfig) { + AssertIsOnOwningThread(); + + nsTArray> frames; + for (const RefPtr& data : aData) { + MOZ_RELEASE_ASSERT(data->mType == MediaData::Type::AUDIO_DATA); + RefPtr d(data->As()); + frames.AppendElement(CreateAudioData(aGlobalObject, d.get())); + } + return frames; +} + +#undef LOG +#undef LOGW +#undef LOGE +#undef LOGV +#undef LOG_INTERNAL + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/AudioDecoder.h b/dom/media/webcodecs/AudioDecoder.h new file mode 100644 index 0000000000..54fad68f55 --- /dev/null +++ b/dom/media/webcodecs/AudioDecoder.h @@ -0,0 +1,83 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_AudioDecoder_h +#define mozilla_dom_AudioDecoder_h + +#include "js/TypeDecls.h" +#include "mozilla/Attributes.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/Maybe.h" +#include "mozilla/RefPtr.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/dom/AudioData.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/DecoderTemplate.h" +#include "mozilla/dom/DecoderTypes.h" +#include "mozilla/dom/RootedDictionary.h" +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; + +namespace mozilla { + +namespace dom { + +class AudioDataOutputCallback; +class EncodedAudioChunk; +class EncodedAudioChunkData; +class EventHandlerNonNull; +class GlobalObject; +class Promise; +class WebCodecsErrorCallback; +struct AudioDecoderConfig; +struct AudioDecoderInit; + +} // namespace dom + +} // namespace mozilla + +namespace mozilla::dom { + +class AudioDecoder final : public DecoderTemplate { + public: + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(AudioDecoder, DOMEventTargetHelper) + + public: + AudioDecoder(nsIGlobalObject* aParent, + RefPtr&& aErrorCallback, + RefPtr&& aOutputCallback); + + protected: + ~AudioDecoder(); + + public: + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + static already_AddRefed Constructor( + const GlobalObject& aGlobal, const AudioDecoderInit& aInit, + ErrorResult& aRv); + + static already_AddRefed IsConfigSupported( + const GlobalObject& aGlobal, const AudioDecoderConfig& aConfig, + ErrorResult& aRv); + + protected: + virtual already_AddRefed InputDataToMediaRawData( + UniquePtr&& aData, TrackInfo& aInfo, + const AudioDecoderConfigInternal& aConfig) override; + + virtual nsTArray> DecodedDataToOutputType( + nsIGlobalObject* aGlobalObject, const nsTArray>&& aData, + AudioDecoderConfigInternal& aConfig) override; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_AudioDecoder_h diff --git a/dom/media/webcodecs/DecoderAgent.cpp b/dom/media/webcodecs/DecoderAgent.cpp index 5c63e27d48..095852c01d 100644 --- a/dom/media/webcodecs/DecoderAgent.cpp +++ b/dom/media/webcodecs/DecoderAgent.cpp @@ -6,8 +6,6 @@ #include "DecoderAgent.h" -#include - #include "ImageContainer.h" #include "MediaDataDecoderProxy.h" #include "PDMFactory.h" diff --git a/dom/media/webcodecs/DecoderTemplate.cpp b/dom/media/webcodecs/DecoderTemplate.cpp index 0fa25a208b..4d1c310737 100644 --- a/dom/media/webcodecs/DecoderTemplate.cpp +++ b/dom/media/webcodecs/DecoderTemplate.cpp @@ -139,8 +139,8 @@ void DecoderTemplate::Configure(const ConfigType& aConfig, nsCString errorMessage; if (!DecoderType::Validate(aConfig, errorMessage)) { - aRv.ThrowTypeError( - nsPrintfCString("config is invalid: %s", errorMessage.get())); + LOG("Configure: Validate error: %s", errorMessage.get()); + aRv.ThrowTypeError(errorMessage); return; } @@ -322,13 +322,13 @@ void DecoderTemplate::OutputDecodedData( MOZ_ASSERT(mState == CodecState::Configured); MOZ_ASSERT(mActiveConfig); - nsTArray> frames = DecodedDataToOutputType( + nsTArray> frames = DecodedDataToOutputType( GetParentObject(), std::move(aData), *mActiveConfig); - RefPtr cb(mOutputCallback); - for (RefPtr& frame : frames) { + RefPtr cb(mOutputCallback); + for (RefPtr& frame : frames) { LOG("Outputing decoded data: ts: %" PRId64, frame->Timestamp()); - RefPtr f = frame; - cb->Call((VideoFrame&)(*f)); + RefPtr f = frame; + cb->Call((OutputType&)(*f)); } } @@ -881,6 +881,7 @@ void DecoderTemplate::DestroyDecoderAgentIfAny() { } template class DecoderTemplate; +template class DecoderTemplate; #undef LOG #undef LOGW diff --git a/dom/media/webcodecs/DecoderTypes.h b/dom/media/webcodecs/DecoderTypes.h index 56aa82046f..339a164f70 100644 --- a/dom/media/webcodecs/DecoderTypes.h +++ b/dom/media/webcodecs/DecoderTypes.h @@ -9,6 +9,9 @@ #include "MediaData.h" #include "mozilla/Maybe.h" +#include "mozilla/dom/AudioData.h" +#include "mozilla/dom/AudioDecoderBinding.h" +#include "mozilla/dom/EncodedAudioChunk.h" #include "mozilla/dom/EncodedVideoChunk.h" #include "mozilla/dom/VideoColorSpaceBinding.h" #include "mozilla/dom/VideoDecoderBinding.h" @@ -58,17 +61,17 @@ class VideoDecoderConfigInternal { bool Equals(const VideoDecoderConfigInternal& aOther) const { if (mDescription.isSome() != aOther.mDescription.isSome()) { - return false; + return false; } if (mDescription.isSome() && aOther.mDescription.isSome()) { - auto lhs = mDescription.value(); - auto rhs = aOther.mDescription.value(); - if (lhs->Length() != rhs->Length()) { - return false; - } - if (!ArrayEqual(lhs->Elements(), rhs->Elements(), lhs->Length())) { - return false; - } + auto lhs = mDescription.value(); + auto rhs = aOther.mDescription.value(); + if (lhs->Length() != rhs->Length()) { + return false; + } + if (!ArrayEqual(lhs->Elements(), rhs->Elements(), lhs->Length())) { + return false; + } } return mCodec.Equals(aOther.mCodec) && mCodedHeight == aOther.mCodedHeight && @@ -111,6 +114,49 @@ class VideoDecoderTraits { const InputType& aInput); }; +class AudioDecoderConfigInternal { + public: + static UniquePtr Create( + const AudioDecoderConfig& aConfig); + ~AudioDecoderConfigInternal() = default; + + nsString mCodec; + uint32_t mSampleRate; + uint32_t mNumberOfChannels; + Maybe> mDescription; + // Compilation fix, should be abstracted by DecoderAgent since those are not + // supported + HardwareAcceleration mHardwareAcceleration = + HardwareAcceleration::No_preference; + Maybe mOptimizeForLatency; + + private: + AudioDecoderConfigInternal(const nsAString& aCodec, uint32_t aSampleRate, + uint32_t aNumberOfChannels, + Maybe>&& aDescription); +}; + +class AudioDecoderTraits { + public: + static constexpr nsLiteralCString Name = "AudioDecoder"_ns; + using ConfigType = AudioDecoderConfig; + using ConfigTypeInternal = AudioDecoderConfigInternal; + using InputType = EncodedAudioChunk; + using InputTypeInternal = EncodedAudioChunkData; + using OutputType = AudioData; + using OutputCallbackType = AudioDataOutputCallback; + + static bool IsSupported(const ConfigTypeInternal& aConfig); + static Result, nsresult> CreateTrackInfo( + const ConfigTypeInternal& aConfig); + static bool Validate(const ConfigType& aConfig, nsCString& aErrorMessage); + static UniquePtr CreateConfigInternal( + const ConfigType& aConfig); + static bool IsKeyChunk(const InputType& aInput); + static UniquePtr CreateInputInternal( + const InputType& aInput); +}; + } // namespace dom } // namespace mozilla diff --git a/dom/media/webcodecs/EncodedAudioChunk.cpp b/dom/media/webcodecs/EncodedAudioChunk.cpp new file mode 100644 index 0000000000..1cbb2ff9bd --- /dev/null +++ b/dom/media/webcodecs/EncodedAudioChunk.cpp @@ -0,0 +1,260 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/EncodedAudioChunk.h" +#include "mozilla/dom/EncodedAudioChunkBinding.h" + +#include + +#include "MediaData.h" +#include "TimeUnits.h" +#include "mozilla/CheckedInt.h" +#include "mozilla/Logging.h" +#include "mozilla/PodOperations.h" +#include "mozilla/dom/StructuredCloneHolder.h" +#include "mozilla/dom/StructuredCloneTags.h" +#include "mozilla/dom/WebCodecsUtils.h" + +extern mozilla::LazyLogModule gWebCodecsLog; +using mozilla::media::TimeUnit; + +namespace mozilla::dom { + +#ifdef LOG_INTERNAL +# undef LOG_INTERNAL +#endif // LOG_INTERNAL +#define LOG_INTERNAL(level, msg, ...) \ + MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) + +#ifdef LOGW +# undef LOGW +#endif // LOGW +#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) + +#ifdef LOGE +# undef LOGE +#endif // LOGE +#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) + +// Only needed for refcounted objects. +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(EncodedAudioChunk, mParent) +NS_IMPL_CYCLE_COLLECTING_ADDREF(EncodedAudioChunk) +NS_IMPL_CYCLE_COLLECTING_RELEASE(EncodedAudioChunk) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(EncodedAudioChunk) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +EncodedAudioChunkData::EncodedAudioChunkData( + already_AddRefed aBuffer, + const EncodedAudioChunkType& aType, int64_t aTimestamp, + Maybe&& aDuration) + : mBuffer(aBuffer), + mType(aType), + mTimestamp(aTimestamp), + mDuration(aDuration) { + MOZ_ASSERT(mBuffer); + MOZ_ASSERT(mBuffer->Length() == mBuffer->Size()); + MOZ_ASSERT(mBuffer->Length() <= + static_cast(std::numeric_limits::max())); +} + +UniquePtr EncodedAudioChunkData::Clone() const { + if (!mBuffer) { + LOGE("No buffer in EncodedAudioChunkData %p to clone!", this); + return nullptr; + } + + // Since EncodedAudioChunkData can be zero-sized, cloning a zero-sized chunk + // is allowed. + if (mBuffer->Size() == 0) { + LOGW("Cloning an empty EncodedAudioChunkData %p", this); + } + + auto buffer = + MakeRefPtr(mBuffer->Data(), mBuffer->Length()); + if (!buffer || buffer->Size() != mBuffer->Size()) { + LOGE("OOM to copy EncodedAudioChunkData %p", this); + return nullptr; + } + + return MakeUnique(buffer.forget(), mType, mTimestamp, + Maybe(mDuration)); +} + +already_AddRefed EncodedAudioChunkData::TakeData() { + if (!mBuffer || !(*mBuffer)) { + LOGE("EncodedAudioChunkData %p has no data!", this); + return nullptr; + } + + RefPtr sample(new MediaRawData(std::move(*mBuffer))); + sample->mKeyframe = mType == EncodedAudioChunkType::Key; + sample->mTime = TimeUnit::FromMicroseconds(mTimestamp); + sample->mTimecode = TimeUnit::FromMicroseconds(mTimestamp); + + if (mDuration) { + CheckedInt64 duration(*mDuration); + if (!duration.isValid()) { + LOGE("EncodedAudioChunkData %p 's duration exceeds TimeUnit's limit", + this); + return nullptr; + } + sample->mDuration = TimeUnit::FromMicroseconds(duration.value()); + } + + return sample.forget(); +} + +EncodedAudioChunk::EncodedAudioChunk( + nsIGlobalObject* aParent, already_AddRefed aBuffer, + const EncodedAudioChunkType& aType, int64_t aTimestamp, + Maybe&& aDuration) + : EncodedAudioChunkData(std::move(aBuffer), aType, aTimestamp, + std::move(aDuration)), + mParent(aParent) {} + +EncodedAudioChunk::EncodedAudioChunk(nsIGlobalObject* aParent, + const EncodedAudioChunkData& aData) + : EncodedAudioChunkData(aData), mParent(aParent) {} + +nsIGlobalObject* EncodedAudioChunk::GetParentObject() const { + AssertIsOnOwningThread(); + + return mParent.get(); +} + +JSObject* EncodedAudioChunk::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + AssertIsOnOwningThread(); + + return EncodedAudioChunk_Binding::Wrap(aCx, this, aGivenProto); +} + +// https://w3c.github.io/webcodecs/#encodedaudiochunk-constructors +/* static */ +already_AddRefed EncodedAudioChunk::Constructor( + const GlobalObject& aGlobal, const EncodedAudioChunkInit& aInit, + ErrorResult& aRv) { + nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + auto buffer = ProcessTypedArrays( + aInit.mData, + [&](const Span& aData, + JS::AutoCheckCannotGC&&) -> RefPtr { + // Make sure it's in uint32_t's range. + CheckedUint32 byteLength(aData.Length()); + if (!byteLength.isValid()) { + aRv.Throw(NS_ERROR_INVALID_ARG); + return nullptr; + } + if (aData.Length() == 0) { + LOGW("Buffer for constructing EncodedAudioChunk is empty!"); + } + RefPtr buf = MakeRefPtr( + aData.Elements(), aData.Length()); + + // Instead of checking *buf, size comparision is used to allow + // constructing a zero-sized EncodedAudioChunk. + if (!buf || buf->Size() != aData.Length()) { + aRv.Throw(NS_ERROR_OUT_OF_MEMORY); + return nullptr; + } + return buf; + }); + + RefPtr chunk(new EncodedAudioChunk( + global, buffer.forget(), aInit.mType, aInit.mTimestamp, + OptionalToMaybe(aInit.mDuration))); + return aRv.Failed() ? nullptr : chunk.forget(); +} + +EncodedAudioChunkType EncodedAudioChunk::Type() const { + AssertIsOnOwningThread(); + + return mType; +} + +int64_t EncodedAudioChunk::Timestamp() const { + AssertIsOnOwningThread(); + + return mTimestamp; +} + +Nullable EncodedAudioChunk::GetDuration() const { + AssertIsOnOwningThread(); + return MaybeToNullable(mDuration); +} + +uint32_t EncodedAudioChunk::ByteLength() const { + AssertIsOnOwningThread(); + MOZ_ASSERT(mBuffer); + + return static_cast(mBuffer->Length()); +} + +// https://w3c.github.io/webcodecs/#dom-encodedaudiochunk-copyto +void EncodedAudioChunk::CopyTo( + const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination, + ErrorResult& aRv) { + AssertIsOnOwningThread(); + + ProcessTypedArraysFixed(aDestination, [&](const Span& aData) { + if (mBuffer->Size() > aData.size_bytes()) { + aRv.ThrowTypeError( + "Destination ArrayBuffer smaller than source EncodedAudioChunk"); + return; + } + + PodCopy(aData.data(), mBuffer->Data(), mBuffer->Size()); + }); +} + +// https://w3c.github.io/webcodecs/#ref-for-deserialization-steps +/* static */ +JSObject* EncodedAudioChunk::ReadStructuredClone( + JSContext* aCx, nsIGlobalObject* aGlobal, JSStructuredCloneReader* aReader, + const EncodedAudioChunkData& aData) { + JS::Rooted value(aCx, JS::NullValue()); + // To avoid a rooting hazard error from returning a raw JSObject* before + // running the RefPtr destructor, RefPtr needs to be destructed before + // returning the raw JSObject*, which is why the RefPtr is + // created in the scope below. Otherwise, the static analysis infers the + // RefPtr cannot be safely destructed while the unrooted return JSObject* is + // on the stack. + { + auto frame = MakeRefPtr(aGlobal, aData); + if (!GetOrCreateDOMReflector(aCx, frame, &value) || !value.isObject()) { + return nullptr; + } + } + return value.toObjectOrNull(); +} + +// https://w3c.github.io/webcodecs/#ref-for-serialization-steps +bool EncodedAudioChunk::WriteStructuredClone( + JSStructuredCloneWriter* aWriter, StructuredCloneHolder* aHolder) const { + AssertIsOnOwningThread(); + + // Indexing the chunk and send the index to the receiver. + const uint32_t index = + static_cast(aHolder->EncodedAudioChunks().Length()); + // The serialization is limited to the same process scope so it's ok to + // serialize a reference instead of a copy. + aHolder->EncodedAudioChunks().AppendElement(EncodedAudioChunkData(*this)); + return !NS_WARN_IF( + !JS_WriteUint32Pair(aWriter, SCTAG_DOM_ENCODEDAUDIOCHUNK, index)); +} + +#undef LOGW +#undef LOGE +#undef LOG_INTERNAL + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/EncodedAudioChunk.h b/dom/media/webcodecs/EncodedAudioChunk.h new file mode 100644 index 0000000000..53c059495a --- /dev/null +++ b/dom/media/webcodecs/EncodedAudioChunk.h @@ -0,0 +1,117 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_EncodedAudioChunk_h +#define mozilla_dom_EncodedAudioChunk_h + +#include "js/TypeDecls.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/Maybe.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" + +class nsIGlobalObject; + +namespace mozilla { + +class MediaAlignedByteBuffer; +class MediaRawData; + +namespace dom { + +class MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer; +class OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer; +class StructuredCloneHolder; + +enum class EncodedAudioChunkType : uint8_t; +struct EncodedAudioChunkInit; + +} // namespace dom +} // namespace mozilla + +namespace mozilla::dom { + +class EncodedAudioChunkData { + public: + EncodedAudioChunkData(already_AddRefed aBuffer, + const EncodedAudioChunkType& aType, int64_t aTimestamp, + Maybe&& aDuration); + EncodedAudioChunkData(const EncodedAudioChunkData& aData) = default; + ~EncodedAudioChunkData() = default; + + UniquePtr Clone() const; + already_AddRefed TakeData(); + + protected: + // mBuffer's byte length is guaranteed to be smaller than UINT32_MAX. + RefPtr mBuffer; + EncodedAudioChunkType mType; + int64_t mTimestamp; + Maybe mDuration; +}; + +class EncodedAudioChunk final : public EncodedAudioChunkData, + public nsISupports, + public nsWrapperCache { + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(EncodedAudioChunk) + + public: + EncodedAudioChunk(nsIGlobalObject* aParent, + already_AddRefed aBuffer, + const EncodedAudioChunkType& aType, int64_t aTimestamp, + Maybe&& aDuration); + + EncodedAudioChunk(nsIGlobalObject* aParent, + const EncodedAudioChunkData& aData); + + protected: + ~EncodedAudioChunk() = default; + + public: + nsIGlobalObject* GetParentObject() const; + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + static already_AddRefed Constructor( + const GlobalObject& aGlobal, const EncodedAudioChunkInit& aInit, + ErrorResult& aRv); + + EncodedAudioChunkType Type() const; + + int64_t Timestamp() const; + + Nullable GetDuration() const; + + uint32_t ByteLength() const; + + void CopyTo( + const MaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDestination, + ErrorResult& aRv); + + // [Serializable] implementations: {Read, Write}StructuredClone + static JSObject* ReadStructuredClone(JSContext* aCx, nsIGlobalObject* aGlobal, + JSStructuredCloneReader* aReader, + const EncodedAudioChunkData& aData); + + bool WriteStructuredClone(JSStructuredCloneWriter* aWriter, + StructuredCloneHolder* aHolder) const; + + private: + // EncodedAudioChunk can run on either main thread or worker thread. + void AssertIsOnOwningThread() const { + NS_ASSERT_OWNINGTHREAD(EncodedAudioChunk); + } + + nsCOMPtr mParent; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_EncodedAudioChunk_h diff --git a/dom/media/webcodecs/VideoDecoder.cpp b/dom/media/webcodecs/VideoDecoder.cpp index 47ca5bb459..18855e5cea 100644 --- a/dom/media/webcodecs/VideoDecoder.cpp +++ b/dom/media/webcodecs/VideoDecoder.cpp @@ -15,11 +15,8 @@ #include "MediaData.h" #include "VideoUtils.h" #include "mozilla/Assertions.h" -#include "mozilla/CheckedInt.h" -#include "mozilla/DebugOnly.h" #include "mozilla/Logging.h" #include "mozilla/Maybe.h" -#include "mozilla/StaticPrefs_dom.h" #include "mozilla/Try.h" #include "mozilla/Unused.h" #include "mozilla/dom/EncodedVideoChunk.h" @@ -31,7 +28,6 @@ #include "mozilla/dom/WebCodecsUtils.h" #include "nsPrintfCString.h" #include "nsReadableUtils.h" -#include "nsThreadUtils.h" #ifdef XP_MACOSX # include "MacIOSurfaceImage.h" @@ -97,9 +93,6 @@ VideoColorSpaceInit VideoColorSpaceInternal::ToColorSpaceInit() const { return init; }; -static Result, nsresult> GetExtraData( - const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer); - VideoDecoderConfigInternal::VideoDecoderConfigInternal( const nsAString& aCodec, Maybe&& aCodedHeight, Maybe&& aCodedWidth, Maybe&& aColorSpace, @@ -129,7 +122,7 @@ UniquePtr VideoDecoderConfigInternal::Create( Maybe> description; if (aConfig.mDescription.WasPassed()) { - auto rv = GetExtraData(aConfig.mDescription.Value()); + auto rv = GetExtraDataFromArrayBuffer(aConfig.mDescription.Value()); if (rv.isErr()) { // Invalid description data. LOGE( "Failed to create VideoDecoderConfigInternal due to invalid " @@ -168,12 +161,10 @@ nsString VideoDecoderConfigInternal::ToString() const { if (mColorSpace.isSome()) { rv.AppendPrintf("colorspace %s", "todo"); } - if (mDescription.isSome()) { + if (mDescription.isSome() && mDescription.value()) { rv.AppendPrintf("extradata: %zu bytes", mDescription.value()->Length()); } - rv.AppendPrintf( - "hw accel: %s", - HardwareAccelerationValues::GetString(mHardwareAcceleration).data()); + rv.AppendPrintf("hw accel: %s", GetEnumString(mHardwareAcceleration).get()); if (mOptimizeForLatency.isSome()) { rv.AppendPrintf("optimize for latency: %s", mOptimizeForLatency.value() ? "true" : "false"); @@ -217,24 +208,6 @@ static nsTArray GuessMIMETypes(const MIMECreateParam& aParam) { return types; } -static bool IsSupportedCodec(const nsAString& aCodec) { - // H265 is unsupported. - if (!IsAV1CodecString(aCodec) && !IsVP9CodecString(aCodec) && - !IsVP8CodecString(aCodec) && !IsH264CodecString(aCodec)) { - return false; - } - - // Gecko allows codec string starts with vp9 or av1 but Webcodecs requires to - // starts with av01 and vp09. - // https://www.w3.org/TR/webcodecs-codec-registry/#video-codec-registry - if (StringBeginsWith(aCodec, u"vp9"_ns) || - StringBeginsWith(aCodec, u"av1"_ns)) { - return false; - } - - return true; -} - // https://w3c.github.io/webcodecs/#check-configuration-support template static bool CanDecode(const Config& aConfig) { @@ -243,12 +216,7 @@ static bool CanDecode(const Config& aConfig) { if (IsOnAndroid()) { return false; } - if (!IsSupportedCodec(param.mParsedCodec)) { - return false; - } - if (IsOnMacOS() && IsH264CodecString(param.mParsedCodec) && - !StaticPrefs::dom_media_webcodecs_force_osx_h264_enabled()) { - // This will be fixed in Bug 1846796. + if (!IsSupportedVideoCodec(param.mParsedCodec)) { return false; } // TODO: Instead of calling CanHandleContainerType with the guessed the @@ -284,18 +252,9 @@ static nsTArray> GetTracksInfo( return {}; } -static Result, nsresult> GetExtraData( - const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer) { - RefPtr data = MakeRefPtr(); - if (!AppendTypedArrayDataTo(aBuffer, *data)) { - return Err(NS_ERROR_OUT_OF_MEMORY); - } - return data->Length() > 0 ? data : nullptr; -} - static Result CloneConfiguration( RootedDictionary& aDest, JSContext* aCx, - const VideoDecoderConfig& aConfig) { + const VideoDecoderConfig& aConfig, ErrorResult& aRv) { DebugOnly str; MOZ_ASSERT(VideoDecoderTraits::Validate(aConfig, str)); @@ -312,7 +271,7 @@ static Result CloneConfiguration( if (aConfig.mDescription.WasPassed()) { aDest.mDescription.Construct(); MOZ_TRY(CloneBuffer(aCx, aDest.mDescription.Value(), - aConfig.mDescription.Value())); + aConfig.mDescription.Value(), aRv)); } if (aConfig.mDisplayAspectHeight.WasPassed()) { aDest.mDisplayAspectHeight.Construct(aConfig.mDisplayAspectHeight.Value()); @@ -334,8 +293,10 @@ static Maybe GuessPixelFormat(layers::Image* aImage) { // DMABUFSurfaceImage? if (aImage->AsPlanarYCbCrImage() || aImage->AsNVImage()) { const ImageUtils imageUtils(aImage); + Maybe format = imageUtils.GetFormat(); Maybe f = - ImageBitmapFormatToVideoPixelFormat(imageUtils.GetFormat()); + format.isSome() ? ImageBitmapFormatToVideoPixelFormat(format.value()) + : Nothing(); // ImageBitmapFormat cannot distinguish YUV420 or YUV420A. bool hasAlpha = aImage->AsPlanarYCbCrImage() && @@ -391,8 +352,6 @@ static VideoColorSpaceInternal GuessColorSpace( // Make an educated guess based on the coefficients. colorSpace.mPrimaries = colorSpace.mMatrix.map([](const auto& aMatrix) { switch (aMatrix) { - case VideoMatrixCoefficients::EndGuard_: - MOZ_CRASH("This should not happen"); case VideoMatrixCoefficients::Bt2020_ncl: return VideoColorPrimaries::Bt2020; case VideoMatrixCoefficients::Rgb: @@ -529,9 +488,6 @@ static VideoColorSpaceInternal GuessColorSpace(layers::Image* aImage) { case VideoMatrixCoefficients::Bt2020_ncl: colorSpace.mPrimaries = Some(VideoColorPrimaries::Bt2020); break; - case VideoMatrixCoefficients::EndGuard_: - MOZ_ASSERT_UNREACHABLE("bad enum value"); - break; }; } } @@ -794,6 +750,21 @@ bool VideoDecoderTraits::Validate(const VideoDecoderConfig& aConfig, return false; } + bool detached = + aConfig.mDescription.WasPassed() && + (aConfig.mDescription.Value().IsArrayBuffer() + ? JS::ArrayBuffer::fromObject( + aConfig.mDescription.Value().GetAsArrayBuffer().Obj()) + .isDetached() + : JS::ArrayBufferView::fromObject( + aConfig.mDescription.Value().GetAsArrayBufferView().Obj()) + .isDetached()); + + if (detached) { + LOGE("description is detached."); + return false; + } + return true; } @@ -828,9 +799,7 @@ VideoDecoder::VideoDecoder(nsIGlobalObject* aParent, LOG("VideoDecoder %p ctor", this); } -VideoDecoder::~VideoDecoder() { - LOG("VideoDecoder %p dtor", this); -} +VideoDecoder::~VideoDecoder() { LOG("VideoDecoder %p dtor", this); } JSObject* VideoDecoder::WrapObject(JSContext* aCx, JS::Handle aGivenProto) { @@ -877,21 +846,17 @@ already_AddRefed VideoDecoder::IsConfigSupported( nsCString errorMessage; if (!VideoDecoderTraits::Validate(aConfig, errorMessage)) { p->MaybeRejectWithTypeError(nsPrintfCString( - "VideoDecoderConfig is invalid: %s", errorMessage.get())); + "IsConfigSupported: config is invalid: %s", errorMessage.get())); return p.forget(); } - // TODO: Move the following works to another thread to unblock the current - // thread, as what spec suggests. - RootedDictionary config(aGlobal.Context()); - auto r = CloneConfiguration(config, aGlobal.Context(), aConfig); + auto r = CloneConfiguration(config, aGlobal.Context(), aConfig, aRv); if (r.isErr()) { - nsresult e = r.unwrapErr(); - LOGE("Failed to clone VideoDecoderConfig. Error: 0x%08" PRIx32, - static_cast(e)); - p->MaybeRejectWithTypeError("Failed to clone VideoDecoderConfig"); - aRv.Throw(e); + // This can only be an OOM: all members to clone are known to be valid + // because this is check by ::Validate above. + MOZ_ASSERT(r.inspectErr() == NS_ERROR_OUT_OF_MEMORY && + aRv.ErrorCodeIs(NS_ERROR_OUT_OF_MEMORY)); return p.forget(); } diff --git a/dom/media/webcodecs/VideoEncoder.cpp b/dom/media/webcodecs/VideoEncoder.cpp index 0e71417cb0..f593f70c77 100644 --- a/dom/media/webcodecs/VideoEncoder.cpp +++ b/dom/media/webcodecs/VideoEncoder.cpp @@ -101,8 +101,7 @@ VideoEncoderConfigInternal::VideoEncoderConfigInternal( mBitrateMode(aConfig.mBitrateMode), mLatencyMode(aConfig.mLatencyMode), mContentHint(aConfig.mContentHint), - mAvc(aConfig.mAvc) { -} + mAvc(aConfig.mAvc) {} VideoEncoderConfigInternal::VideoEncoderConfigInternal( const VideoEncoderConfig& aConfig) @@ -119,8 +118,7 @@ VideoEncoderConfigInternal::VideoEncoderConfigInternal( mBitrateMode(aConfig.mBitrateMode), mLatencyMode(aConfig.mLatencyMode), mContentHint(OptionalToMaybe(aConfig.mContentHint)), - mAvc(OptionalToMaybe(aConfig.mAvc)) { -} + mAvc(OptionalToMaybe(aConfig.mAvc)) {} nsString VideoEncoderConfigInternal::ToString() const { nsString rv; @@ -137,26 +135,20 @@ nsString VideoEncoderConfigInternal::ToString() const { if (mFramerate.isSome()) { rv.AppendPrintf(", %lfHz", mFramerate.value()); } - rv.AppendPrintf( - " hw: %s", - HardwareAccelerationValues::GetString(mHardwareAcceleration).data()); - rv.AppendPrintf(", alpha: %s", AlphaOptionValues::GetString(mAlpha).data()); + rv.AppendPrintf(" hw: %s", GetEnumString(mHardwareAcceleration).get()); + rv.AppendPrintf(", alpha: %s", GetEnumString(mAlpha).get()); if (mScalabilityMode.isSome()) { rv.AppendPrintf(", scalability mode: %s", NS_ConvertUTF16toUTF8(mScalabilityMode.value()).get()); } - rv.AppendPrintf( - ", bitrate mode: %s", - VideoEncoderBitrateModeValues::GetString(mBitrateMode).data()); - rv.AppendPrintf(", latency mode: %s", - LatencyModeValues::GetString(mLatencyMode).data()); + rv.AppendPrintf(", bitrate mode: %s", GetEnumString(mBitrateMode).get()); + rv.AppendPrintf(", latency mode: %s", GetEnumString(mLatencyMode).get()); if (mContentHint.isSome()) { rv.AppendPrintf(", content hint: %s", NS_ConvertUTF16toUTF8(mContentHint.value()).get()); } if (mAvc.isSome()) { - rv.AppendPrintf(", avc-specific: %s", - AvcBitstreamFormatValues::GetString(mAvc->mFormat).data()); + rv.AppendPrintf(", avc-specific: %s", GetEnumString(mAvc->mFormat).get()); } return rv; @@ -238,43 +230,57 @@ EncoderConfig VideoEncoderConfigInternal::ToEncoderConfig() const { if (ExtractH264CodecDetails(mCodec, profile, constraints, level)) { if (profile == H264_PROFILE_BASE || profile == H264_PROFILE_MAIN || profile == H264_PROFILE_EXTENDED || profile == H264_PROFILE_HIGH) { - specific.emplace( - H264Specific(static_cast(profile), static_cast(level), format)); + specific.emplace(H264Specific(static_cast(profile), + static_cast(level), format)); } } } - // Only for vp9, not vp8 - if (codecType == CodecType::VP9) { - uint8_t profile, level, bitdepth, chromasubsampling; - mozilla::VideoColorSpace colorspace; - DebugOnly rv = ExtractVPXCodecDetails( - mCodec, profile, level, bitdepth, chromasubsampling, colorspace); -#ifdef DEBUG - if (!rv) { - LOGE("Error extracting VPX codec details, non fatal"); - } -#endif - specific.emplace(VP9Specific()); - } + uint8_t numTemporalLayers = 1; MediaDataEncoder::ScalabilityMode scalabilityMode; if (mScalabilityMode) { if (mScalabilityMode->EqualsLiteral("L1T2")) { scalabilityMode = MediaDataEncoder::ScalabilityMode::L1T2; + numTemporalLayers = 2; } else if (mScalabilityMode->EqualsLiteral("L1T3")) { scalabilityMode = MediaDataEncoder::ScalabilityMode::L1T3; + numTemporalLayers = 3; } else { scalabilityMode = MediaDataEncoder::ScalabilityMode::None; } } else { scalabilityMode = MediaDataEncoder::ScalabilityMode::None; } - return EncoderConfig( - codecType, {mWidth, mHeight}, usage, ImageBitmapFormat::RGBA32, ImageBitmapFormat::RGBA32, - AssertedCast(mFramerate.refOr(0.f)), 0, mBitrate.refOr(0), - mBitrateMode == VideoEncoderBitrateMode::Constant - ? MediaDataEncoder::BitrateMode::Constant - : MediaDataEncoder::BitrateMode::Variable, - hwPref, scalabilityMode, specific); + // Only for vp9, not vp8 + if (codecType == CodecType::VP9) { + uint8_t profile, level, bitdepth, chromasubsampling; + mozilla::VideoColorSpace colorspace; + DebugOnly rv = ExtractVPXCodecDetails( + mCodec, profile, level, bitdepth, chromasubsampling, colorspace); +#ifdef DEBUG + if (!rv) { + LOGE("Error extracting VPX codec details, non fatal"); + } +#endif + specific.emplace(VP9Specific( + VPXComplexity::Normal, /* Complexity */ + true, /* Resilience */ + numTemporalLayers, /* Number of temporal layers */ + true, /* Denoising */ + false, /* Auto resize */ + false, /* Frame dropping */ + true, /* Adaptive Qp */ + 1, /* Number of spatial layers */ + false /* Flexible */ + )); + } + return EncoderConfig(codecType, {mWidth, mHeight}, usage, + ImageBitmapFormat::RGBA32, ImageBitmapFormat::RGBA32, + AssertedCast(mFramerate.refOr(0.f)), 0, + mBitrate.refOr(0), + mBitrateMode == VideoEncoderBitrateMode::Constant + ? MediaDataEncoder::BitrateMode::Constant + : MediaDataEncoder::BitrateMode::Variable, + hwPref, scalabilityMode, specific); } already_AddRefed VideoEncoderConfigInternal::Diff( @@ -326,27 +332,6 @@ VideoEncoderConfigInternal::Diff( return list.forget(); } -/* - * The followings are helpers for VideoEncoder methods - */ -static bool IsEncodeSupportedCodec(const nsAString& aCodec) { - LOG("IsEncodeSupported: %s", NS_ConvertUTF16toUTF8(aCodec).get()); - if (!IsVP9CodecString(aCodec) && !IsVP8CodecString(aCodec) && - !IsH264CodecString(aCodec) && !IsAV1CodecString(aCodec)) { - return false; - } - - // Gecko allows codec string starts with vp9 or av1 but Webcodecs requires to - // starts with av01 and vp09. - // https://www.w3.org/TR/webcodecs-codec-registry/#video-codec-registry - if (StringBeginsWith(aCodec, u"vp9"_ns) || - StringBeginsWith(aCodec, u"av1"_ns)) { - return false; - } - - return true; -} - // https://w3c.github.io/webcodecs/#check-configuration-support static bool CanEncode(const RefPtr& aConfig) { auto parsedCodecString = @@ -355,7 +340,7 @@ static bool CanEncode(const RefPtr& aConfig) { if (IsOnAndroid()) { return false; } - if (!IsEncodeSupportedCodec(parsedCodecString)) { + if (!IsSupportedVideoCodec(parsedCodecString)) { return false; } diff --git a/dom/media/webcodecs/VideoFrame.cpp b/dom/media/webcodecs/VideoFrame.cpp index 602bc95c29..bea5611e39 100644 --- a/dom/media/webcodecs/VideoFrame.cpp +++ b/dom/media/webcodecs/VideoFrame.cpp @@ -598,8 +598,6 @@ static bool IsYUVFormat(const VideoPixelFormat& aFormat) { case VideoPixelFormat::BGRA: case VideoPixelFormat::BGRX: return false; - case VideoPixelFormat::EndGuard_: - MOZ_ASSERT_UNREACHABLE("unsupported format"); } return false; } @@ -641,8 +639,6 @@ static VideoColorSpaceInit PickColorSpace( colorSpace.mPrimaries.SetValue(VideoColorPrimaries::Bt709); colorSpace.mTransfer.SetValue(VideoTransferCharacteristics::Iec61966_2_1); break; - case VideoPixelFormat::EndGuard_: - MOZ_ASSERT_UNREACHABLE("unsupported format"); } return colorSpace; @@ -881,8 +877,6 @@ static Result, nsCString> CreateImageFromBuffer( case VideoPixelFormat::BGRA: case VideoPixelFormat::BGRX: return CreateRGBAImageFromBuffer(aFormat, aSize, aBuffer); - case VideoPixelFormat::EndGuard_: - MOZ_ASSERT_UNREACHABLE("unsupported format"); } return Err(nsCString("Invalid image format")); } @@ -1390,8 +1384,9 @@ already_AddRefed VideoFrame::Constructor( } const ImageUtils imageUtils(image); + Maybe f = imageUtils.GetFormat(); Maybe format = - ImageBitmapFormatToVideoPixelFormat(imageUtils.GetFormat()); + f.isSome() ? ImageBitmapFormatToVideoPixelFormat(f.value()) : Nothing(); // TODO: Retrive/infer the duration, and colorspace. auto r = InitializeFrameFromOtherFrame( @@ -1789,15 +1784,13 @@ nsCString VideoFrame::ToString() const { return rv; } - rv.AppendPrintf( - "VideoFrame ts: %" PRId64 - ", %s, coded[%dx%d] visible[%dx%d], display[%dx%d] color: %s", - mTimestamp, - dom::VideoPixelFormatValues::GetString(mResource->mFormat->PixelFormat()) - .data(), - mCodedSize.width, mCodedSize.height, mVisibleRect.width, - mVisibleRect.height, mDisplaySize.width, mDisplaySize.height, - ColorSpaceInitToString(mColorSpace).get()); + rv.AppendPrintf("VideoFrame ts: %" PRId64 + ", %s, coded[%dx%d] visible[%dx%d], display[%dx%d] color: %s", + mTimestamp, + dom::GetEnumString(mResource->mFormat->PixelFormat()).get(), + mCodedSize.width, mCodedSize.height, mVisibleRect.width, + mVisibleRect.height, mDisplaySize.width, mDisplaySize.height, + ColorSpaceInitToString(mColorSpace).get()); if (mDuration) { rv.AppendPrintf(" dur: %" PRId64, mDuration.value()); @@ -2032,8 +2025,6 @@ gfx::SurfaceFormat VideoFrame::Format::ToSurfaceFormat() const { case VideoPixelFormat::BGRX: format = gfx::SurfaceFormat::B8G8R8X8; break; - case VideoPixelFormat::EndGuard_: - MOZ_ASSERT_UNREACHABLE("unsupported format"); } return format; } @@ -2056,8 +2047,6 @@ void VideoFrame::Format::MakeOpaque() { case VideoPixelFormat::RGBX: case VideoPixelFormat::BGRX: return; - case VideoPixelFormat::EndGuard_: - break; } MOZ_ASSERT_UNREACHABLE("unsupported format"); } @@ -2077,8 +2066,6 @@ nsTArray VideoFrame::Format::Planes() const { case VideoPixelFormat::BGRA: case VideoPixelFormat::BGRX: return {Plane::RGBA}; - case VideoPixelFormat::EndGuard_: - break; } MOZ_ASSERT_UNREACHABLE("unsupported format"); return {}; @@ -2125,8 +2112,6 @@ uint32_t VideoFrame::Format::SampleBytes(const Plane& aPlane) const { case VideoPixelFormat::BGRA: case VideoPixelFormat::BGRX: return 4; // 8 bits/sample, 32 bits/pixel - case VideoPixelFormat::EndGuard_: - break; } MOZ_ASSERT_UNREACHABLE("unsupported format"); return 0; @@ -2154,7 +2139,6 @@ gfx::IntSize VideoFrame::Format::SampleSize(const Plane& aPlane) const { case VideoPixelFormat::RGBX: case VideoPixelFormat::BGRA: case VideoPixelFormat::BGRX: - case VideoPixelFormat::EndGuard_: MOZ_ASSERT_UNREACHABLE("invalid format"); return {0, 0}; } @@ -2177,8 +2161,6 @@ bool VideoFrame::Format::IsValidSize(const gfx::IntSize& aSize) const { case VideoPixelFormat::BGRA: case VideoPixelFormat::BGRX: return true; - case VideoPixelFormat::EndGuard_: - break; } MOZ_ASSERT_UNREACHABLE("unsupported format"); return false; @@ -2205,8 +2187,6 @@ size_t VideoFrame::Format::SampleCount(const gfx::IntSize& aSize) const { case VideoPixelFormat::BGRA: case VideoPixelFormat::BGRX: return (count * 4).value(); - case VideoPixelFormat::EndGuard_: - break; } MOZ_ASSERT_UNREACHABLE("unsupported format"); @@ -2252,8 +2232,6 @@ uint32_t VideoFrame::Resource::Stride(const Format::Plane& aPlane) const { case VideoPixelFormat::BGRA: case VideoPixelFormat::BGRX: return (width * mFormat->SampleBytes(aPlane)).value(); - case VideoPixelFormat::EndGuard_: - MOZ_ASSERT_UNREACHABLE("invalid format"); } return 0; case Format::Plane::U: // and UV @@ -2269,7 +2247,6 @@ uint32_t VideoFrame::Resource::Stride(const Format::Plane& aPlane) const { case VideoPixelFormat::RGBX: case VideoPixelFormat::BGRA: case VideoPixelFormat::BGRX: - case VideoPixelFormat::EndGuard_: MOZ_ASSERT_UNREACHABLE("invalid format"); } return 0; diff --git a/dom/media/webcodecs/WebCodecsUtils.cpp b/dom/media/webcodecs/WebCodecsUtils.cpp index 1e03f616db..3507aba440 100644 --- a/dom/media/webcodecs/WebCodecsUtils.cpp +++ b/dom/media/webcodecs/WebCodecsUtils.cpp @@ -37,7 +37,7 @@ std::atomic sNextId = 0; namespace mozilla::dom { /* - * The followings are helpers for VideoDecoder methods + * The followings are helpers for AudioDecoder and VideoDecoder methods */ nsTArray GuessContainers(const nsAString& aCodec) { @@ -57,6 +57,29 @@ nsTArray GuessContainers(const nsAString& aCodec) { return {"mp4"_ns, "3gpp"_ns, "3gpp2"_ns, "3gp2"_ns}; } + if (IsAACCodecString(aCodec)) { + return {"adts"_ns, "mp4"_ns}; + } + + if (aCodec.EqualsLiteral("vorbis") || aCodec.EqualsLiteral("opus")) { + return {"ogg"_ns}; + } + + if (aCodec.EqualsLiteral("flac")) { + return {"flac"_ns}; + } + + if (aCodec.EqualsLiteral("mp3")) { + return {"mp3"_ns}; + } + + if (aCodec.EqualsLiteral("ulaw") || aCodec.EqualsLiteral("alaw") || + aCodec.EqualsLiteral("pcm-u8") || aCodec.EqualsLiteral("pcm-s16") || + aCodec.EqualsLiteral("pcm-s24") || aCodec.EqualsLiteral("pcm-s32") || + aCodec.EqualsLiteral("pcm-f32")) { + return {"x-wav"_ns}; + } + return {}; } @@ -106,7 +129,8 @@ static std::tuple GetArrayBufferInfo( Result CloneBuffer( JSContext* aCx, OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDest, - const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aSrc) { + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aSrc, + ErrorResult& aRv) { std::tuple info = GetArrayBufferInfo(aCx, aSrc); JS::Rooted abov(aCx); @@ -121,6 +145,8 @@ Result CloneBuffer( JS::Rooted cloned(aCx, JS::ArrayBufferClone(aCx, obj, offset, len)); if (NS_WARN_IF(!cloned)) { + aRv.MightThrowJSException(); + aRv.StealExceptionFromJSContext(aCx); return Err(NS_ERROR_OUT_OF_MEMORY); } @@ -151,8 +177,6 @@ gfx::YUVColorSpace ToColorSpace(VideoMatrixCoefficients aMatrix) { return gfx::YUVColorSpace::BT601; case VideoMatrixCoefficients::Bt2020_ncl: return gfx::YUVColorSpace::BT2020; - case VideoMatrixCoefficients::EndGuard_: - break; } MOZ_ASSERT_UNREACHABLE("unsupported VideoMatrixCoefficients"); return gfx::YUVColorSpace::Default; @@ -171,8 +195,7 @@ gfx::TransferFunction ToTransferFunction( case VideoTransferCharacteristics::Hlg: return gfx::TransferFunction::HLG; case VideoTransferCharacteristics::Linear: - case VideoTransferCharacteristics::EndGuard_: - break; + return gfx::TransferFunction::Default; } MOZ_ASSERT_UNREACHABLE("unsupported VideoTransferCharacteristics"); return gfx::TransferFunction::Default; @@ -190,8 +213,6 @@ gfx::ColorSpace2 ToPrimaries(VideoColorPrimaries aPrimaries) { return gfx::ColorSpace2::BT2020; case VideoColorPrimaries::Smpte432: return gfx::ColorSpace2::DISPLAY_P3; - case VideoColorPrimaries::EndGuard_: - break; } MOZ_ASSERT_UNREACHABLE("unsupported VideoTransferCharacteristics"); return gfx::ColorSpace2::UNKNOWN; @@ -300,13 +321,6 @@ Maybe ImageBitmapFormatToVideoPixelFormat( return Nothing(); } -Result, nsresult> GetExtraDataFromArrayBuffer( - const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer) { - RefPtr data = MakeRefPtr(); - Unused << AppendTypedArrayDataTo(aBuffer, *data); - return data->Length() > 0 ? data : nullptr; -} - bool IsOnAndroid() { #if defined(ANDROID) return true; @@ -364,15 +378,13 @@ struct ConfigurationChangeToString { } nsCString operator()( const HardwareAccelerationChange& aHardwareAccelerationChange) { - return nsPrintfCString("HW acceleration: %s", - dom::HardwareAccelerationValues::GetString( - aHardwareAccelerationChange.get()) - .data()); + return nsPrintfCString( + "HW acceleration: %s", + dom::GetEnumString(aHardwareAccelerationChange.get()).get()); } nsCString operator()(const AlphaChange& aAlphaChange) { - return nsPrintfCString( - "Alpha: %s", - dom::AlphaOptionValues::GetString(aAlphaChange.get()).data()); + return nsPrintfCString("Alpha: %s", + dom::GetEnumString(aAlphaChange.get()).get()); } nsCString operator()(const ScalabilityModeChange& aScalabilityModeChange) { if (aScalabilityModeChange.get().isNothing()) { @@ -383,15 +395,12 @@ struct ConfigurationChangeToString { NS_ConvertUTF16toUTF8(aScalabilityModeChange.get().value()).get()); } nsCString operator()(const BitrateModeChange& aBitrateModeChange) { - return nsPrintfCString( - "Bitrate mode: %s", - dom::VideoEncoderBitrateModeValues::GetString(aBitrateModeChange.get()) - .data()); + return nsPrintfCString("Bitrate mode: %s", + dom::GetEnumString(aBitrateModeChange.get()).get()); } nsCString operator()(const LatencyModeChange& aLatencyModeChange) { - return nsPrintfCString( - "Latency mode: %s", - dom::LatencyModeValues::GetString(aLatencyModeChange.get()).data()); + return nsPrintfCString("Latency mode: %s", + dom::GetEnumString(aLatencyModeChange.get()).get()); } nsCString operator()(const ContentHintChange& aContentHintChange) { return nsPrintfCString("Content hint: %s", @@ -489,9 +498,6 @@ WebCodecsConfigurationChangeList::ToPEMChangeList() const { return rv.forget(); } -#define ENUM_TO_STRING(enumType, enumValue) \ - enumType##Values::GetString(enumValue).data() - nsCString ColorSpaceInitToString( const dom::VideoColorSpaceInit& aColorSpaceInit) { nsCString rv("VideoColorSpace"); @@ -502,18 +508,15 @@ nsCString ColorSpaceInitToString( } if (!aColorSpaceInit.mMatrix.IsNull()) { rv.AppendPrintf(" matrix: %s", - ENUM_TO_STRING(dom::VideoMatrixCoefficients, - aColorSpaceInit.mMatrix.Value())); + GetEnumString(aColorSpaceInit.mMatrix.Value()).get()); } if (!aColorSpaceInit.mTransfer.IsNull()) { rv.AppendPrintf(" transfer: %s", - ENUM_TO_STRING(dom::VideoTransferCharacteristics, - aColorSpaceInit.mTransfer.Value())); + GetEnumString(aColorSpaceInit.mTransfer.Value()).get()); } if (!aColorSpaceInit.mPrimaries.IsNull()) { rv.AppendPrintf(" primaries: %s", - ENUM_TO_STRING(dom::VideoColorPrimaries, - aColorSpaceInit.mPrimaries.Value())); + GetEnumString(aColorSpaceInit.mPrimaries.Value()).get()); } return rv; @@ -575,4 +578,32 @@ nsString ConfigToString(const VideoDecoderConfig& aConfig) { return internal->ToString(); } -}; // namespace mozilla::dom +Result, nsresult> GetExtraDataFromArrayBuffer( + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer) { + RefPtr data = MakeRefPtr(); + if (!AppendTypedArrayDataTo(aBuffer, *data)) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + return data->Length() > 0 ? data : nullptr; +} + +bool IsSupportedVideoCodec(const nsAString& aCodec) { + LOG("IsSupportedVideoCodec: %s", NS_ConvertUTF16toUTF8(aCodec).get()); + // The only codec string accepted for vp8 is "vp8" + if (!IsVP9CodecString(aCodec) && !IsH264CodecString(aCodec) && + !IsAV1CodecString(aCodec) && !aCodec.EqualsLiteral("vp8")) { + return false; + } + + // Gecko allows codec string starts with vp9 or av1 but Webcodecs requires to + // starts with av01 and vp09. + // https://w3c.github.io/webcodecs/codec_registry.html. + if (StringBeginsWith(aCodec, u"vp9"_ns) || + StringBeginsWith(aCodec, u"av1"_ns)) { + return false; + } + + return true; +} + +} // namespace mozilla::dom diff --git a/dom/media/webcodecs/WebCodecsUtils.h b/dom/media/webcodecs/WebCodecsUtils.h index 7c0e6b6bbc..196c57421d 100644 --- a/dom/media/webcodecs/WebCodecsUtils.h +++ b/dom/media/webcodecs/WebCodecsUtils.h @@ -8,6 +8,7 @@ #define MOZILLA_DOM_WEBCODECS_WEBCODECSUTILS_H #include "ErrorList.h" +#include "MediaData.h" #include "js/TypeDecls.h" #include "mozilla/Maybe.h" #include "mozilla/MozPromise.h" @@ -86,7 +87,11 @@ Nullable MaybeToNullable(const Maybe& aOptional) { Result CloneBuffer( JSContext* aCx, OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aDest, - const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aSrc); + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aSrc, + ErrorResult& aRv); + +Result, nsresult> GetExtraDataFromArrayBuffer( + const OwningMaybeSharedArrayBufferViewOrMaybeSharedArrayBuffer& aBuffer); /* * The following are utilities to convert between VideoColorSpace values to @@ -232,6 +237,8 @@ Maybe CodecStringToCodecType(const nsAString& aCodecString); nsString ConfigToString(const VideoDecoderConfig& aConfig); +bool IsSupportedVideoCodec(const nsAString& aCodec); + } // namespace dom } // namespace mozilla diff --git a/dom/media/webcodecs/moz.build b/dom/media/webcodecs/moz.build index 267a822286..ddb5aad5cb 100644 --- a/dom/media/webcodecs/moz.build +++ b/dom/media/webcodecs/moz.build @@ -21,8 +21,11 @@ EXPORTS.mozilla += [ ] EXPORTS.mozilla.dom += [ + "AudioData.h", + "AudioDecoder.h", "DecoderTemplate.h", "DecoderTypes.h", + "EncodedAudioChunk.h", "EncodedVideoChunk.h", "EncoderAgent.h", "EncoderTemplate.h", @@ -35,8 +38,11 @@ EXPORTS.mozilla.dom += [ ] UNIFIED_SOURCES += [ + "AudioData.cpp", + "AudioDecoder.cpp", "DecoderAgent.cpp", "DecoderTemplate.cpp", + "EncodedAudioChunk.cpp", "EncodedVideoChunk.cpp", "EncoderAgent.cpp", "EncoderTemplate.cpp", -- cgit v1.2.3