summaryrefslogtreecommitdiffstats
path: root/dom/media/webcodecs/AudioDecoder.cpp
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 01:13:27 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 01:13:27 +0000
commit40a355a42d4a9444dc753c04c6608dade2f06a23 (patch)
tree871fc667d2de662f171103ce5ec067014ef85e61 /dom/media/webcodecs/AudioDecoder.cpp
parentAdding upstream version 124.0.1. (diff)
downloadfirefox-40a355a42d4a9444dc753c04c6608dade2f06a23.tar.xz
firefox-40a355a42d4a9444dc753c04c6608dade2f06a23.zip
Adding upstream version 125.0.1.upstream/125.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/media/webcodecs/AudioDecoder.cpp')
-rw-r--r--dom/media/webcodecs/AudioDecoder.cpp481
1 files changed, 481 insertions, 0 deletions
diff --git a/dom/media/webcodecs/AudioDecoder.cpp b/dom/media/webcodecs/AudioDecoder.cpp
new file mode 100644
index 0000000000..6b554dcacf
--- /dev/null
+++ b/dom/media/webcodecs/AudioDecoder.cpp
@@ -0,0 +1,481 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/dom/AudioDecoder.h"
+#include "mozilla/dom/AudioDecoderBinding.h"
+
+#include "DecoderTraits.h"
+#include "MediaContainerType.h"
+#include "MediaData.h"
+#include "VideoUtils.h"
+#include "mozilla/Assertions.h"
+#include "mozilla/Logging.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/Try.h"
+#include "mozilla/Unused.h"
+#include "mozilla/dom/AudioDataBinding.h"
+#include "mozilla/dom/EncodedAudioChunk.h"
+#include "mozilla/dom/EncodedAudioChunkBinding.h"
+#include "mozilla/dom/ImageUtils.h"
+#include "mozilla/dom/Promise.h"
+#include "mozilla/dom/WebCodecsUtils.h"
+#include "nsPrintfCString.h"
+#include "nsReadableUtils.h"
+
+extern mozilla::LazyLogModule gWebCodecsLog;
+
+namespace mozilla::dom {
+
+#ifdef LOG_INTERNAL
+# undef LOG_INTERNAL
+#endif // LOG_INTERNAL
+#define LOG_INTERNAL(level, msg, ...) \
+ MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__))
+
+#ifdef LOG
+# undef LOG
+#endif // LOG
+#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__)
+
+#ifdef LOGW
+# undef LOGW
+#endif // LOGW
+#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__)
+
+#ifdef LOGE
+# undef LOGE
+#endif // LOGE
+#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__)
+
+#ifdef LOGV
+# undef LOGV
+#endif // LOGV
+#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__)
+
+NS_IMPL_CYCLE_COLLECTION_INHERITED(AudioDecoder, DOMEventTargetHelper,
+ mErrorCallback, mOutputCallback)
+NS_IMPL_ADDREF_INHERITED(AudioDecoder, DOMEventTargetHelper)
+NS_IMPL_RELEASE_INHERITED(AudioDecoder, DOMEventTargetHelper)
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(AudioDecoder)
+NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper)
+
+/*
+ * Below are helper classes
+ */
+
+AudioDecoderConfigInternal::AudioDecoderConfigInternal(
+ const nsAString& aCodec, uint32_t aSampleRate, uint32_t aNumberOfChannels,
+ Maybe<RefPtr<MediaByteBuffer>>&& aDescription)
+ : mCodec(aCodec),
+ mSampleRate(aSampleRate),
+ mNumberOfChannels(aNumberOfChannels),
+ mDescription(std::move(aDescription)) {}
+
+/*static*/
+UniquePtr<AudioDecoderConfigInternal> AudioDecoderConfigInternal::Create(
+ const AudioDecoderConfig& aConfig) {
+ nsCString errorMessage;
+ if (!AudioDecoderTraits::Validate(aConfig, errorMessage)) {
+ LOGE("Failed to create AudioDecoderConfigInternal: %s", errorMessage.get());
+ return nullptr;
+ }
+
+ Maybe<RefPtr<MediaByteBuffer>> description;
+ if (aConfig.mDescription.WasPassed()) {
+ auto rv = GetExtraDataFromArrayBuffer(aConfig.mDescription.Value());
+ if (rv.isErr()) { // Invalid description data.
+ nsCString error;
+ GetErrorName(rv.unwrapErr(), error);
+ LOGE(
+ "Failed to create AudioDecoderConfigInternal due to invalid "
+ "description data. Error: %s",
+ error.get());
+ return nullptr;
+ }
+ description.emplace(rv.unwrap());
+ }
+
+ return UniquePtr<AudioDecoderConfigInternal>(new AudioDecoderConfigInternal(
+ aConfig.mCodec, aConfig.mSampleRate, aConfig.mNumberOfChannels,
+ std::move(description)));
+}
+
+/*
+ * The followings are helpers for AudioDecoder methods
+ */
+
+struct AudioMIMECreateParam {
+ explicit AudioMIMECreateParam(const AudioDecoderConfigInternal& aConfig)
+ : mParsedCodec(ParseCodecString(aConfig.mCodec).valueOr(EmptyString())) {}
+ explicit AudioMIMECreateParam(const AudioDecoderConfig& aConfig)
+ : mParsedCodec(ParseCodecString(aConfig.mCodec).valueOr(EmptyString())) {}
+
+ const nsString mParsedCodec;
+};
+
+// Map between WebCodecs pcm types as strings and codec numbers
+// All other codecs
+nsCString ConvertCodecName(const nsCString& aContainer,
+ const nsCString& aCodec) {
+ if (!aContainer.EqualsLiteral("x-wav")) {
+ return aCodec;
+ }
+ if (aCodec.EqualsLiteral("ulaw")) {
+ return nsCString("7");
+ }
+ if (aCodec.EqualsLiteral("alaw")) {
+ return nsCString("6");
+ }
+ if (aCodec.Find("f32")) {
+ return nsCString("3");
+ }
+ // Linear PCM
+ return nsCString("1");
+}
+
+static nsTArray<nsCString> GuessMIMETypes(const AudioMIMECreateParam& aParam) {
+ nsCString codec = NS_ConvertUTF16toUTF8(aParam.mParsedCodec);
+ nsTArray<nsCString> types;
+ for (const nsCString& container : GuessContainers(aParam.mParsedCodec)) {
+ codec = ConvertCodecName(container, codec);
+ nsPrintfCString mime("audio/%s; codecs=%s", container.get(), codec.get());
+ types.AppendElement(mime);
+ }
+ return types;
+}
+
+static bool IsSupportedAudioCodec(const nsAString& aCodec) {
+ LOG("IsSupportedAudioCodec: %s", NS_ConvertUTF16toUTF8(aCodec).get());
+ return aCodec.EqualsLiteral("flac") || aCodec.EqualsLiteral("mp3") ||
+ IsAACCodecString(aCodec) || aCodec.EqualsLiteral("opus") ||
+ aCodec.EqualsLiteral("ulaw") || aCodec.EqualsLiteral("alaw") ||
+ aCodec.EqualsLiteral("pcm-u8") || aCodec.EqualsLiteral("pcm-s16") ||
+ aCodec.EqualsLiteral("pcm-s24") || aCodec.EqualsLiteral("pcm-s32") ||
+ aCodec.EqualsLiteral("pcm-f32");
+}
+
+// https://w3c.github.io/webcodecs/#check-configuration-support
+template <typename Config>
+static bool CanDecodeAudio(const Config& aConfig) {
+ auto param = AudioMIMECreateParam(aConfig);
+ if (!IsSupportedAudioCodec(param.mParsedCodec)) {
+ return false;
+ }
+ if (IsOnAndroid() && IsAACCodecString(param.mParsedCodec)) {
+ return false;
+ }
+ // TODO: Instead of calling CanHandleContainerType with the guessed the
+ // containers, DecoderTraits should provide an API to tell if a codec is
+ // decodable or not.
+ for (const nsCString& mime : GuessMIMETypes(param)) {
+ if (Maybe<MediaContainerType> containerType =
+ MakeMediaExtendedMIMEType(mime)) {
+ if (DecoderTraits::CanHandleContainerType(
+ *containerType, nullptr /* DecoderDoctorDiagnostics */) !=
+ CANPLAY_NO) {
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+static nsTArray<UniquePtr<TrackInfo>> GetTracksInfo(
+ const AudioDecoderConfigInternal& aConfig) {
+ // TODO: Instead of calling GetTracksInfo with the guessed containers,
+ // DecoderTraits should provide an API to create the TrackInfo directly.
+ for (const nsCString& mime : GuessMIMETypes(AudioMIMECreateParam(aConfig))) {
+ if (Maybe<MediaContainerType> containerType =
+ MakeMediaExtendedMIMEType(mime)) {
+ if (nsTArray<UniquePtr<TrackInfo>> tracks =
+ DecoderTraits::GetTracksInfo(*containerType);
+ !tracks.IsEmpty()) {
+ return tracks;
+ }
+ }
+ }
+ return {};
+}
+
+static Result<Ok, nsresult> CloneConfiguration(
+ RootedDictionary<AudioDecoderConfig>& aDest, JSContext* aCx,
+ const AudioDecoderConfig& aConfig, ErrorResult& aRv) {
+ aDest.mCodec = aConfig.mCodec;
+ if (aConfig.mDescription.WasPassed()) {
+ aDest.mDescription.Construct();
+ MOZ_TRY(CloneBuffer(aCx, aDest.mDescription.Value(),
+ aConfig.mDescription.Value(), aRv));
+ }
+
+ aDest.mNumberOfChannels = aConfig.mNumberOfChannels;
+ aDest.mSampleRate = aConfig.mSampleRate;
+
+ return Ok();
+}
+
+// https://w3c.github.io/webcodecs/#create-a-audiodata
+static RefPtr<AudioData> CreateAudioData(nsIGlobalObject* aGlobalObject,
+ mozilla::AudioData* aData) {
+ MOZ_ASSERT(aGlobalObject);
+ MOZ_ASSERT(aData);
+
+ auto buf = aData->MoveableData();
+ // TODO: Ensure buf.Length() is a multiple of aData->mChannels and put it into
+ // AssertedCast<uint32_t> (sinze return type of buf.Length() is size_t).
+ uint32_t frames = buf.Length() / aData->mChannels;
+ RefPtr<AudioDataResource> resource = AudioDataResource::Create(Span{
+ reinterpret_cast<uint8_t*>(buf.Data()), buf.Length() * sizeof(float)});
+ return MakeRefPtr<AudioData>(aGlobalObject, resource.forget(),
+ aData->mTime.ToMicroseconds(), aData->mChannels,
+ frames, AssertedCast<float>(aData->mRate),
+ mozilla::dom::AudioSampleFormat::F32);
+}
+
+/* static */
+bool AudioDecoderTraits::IsSupported(
+ const AudioDecoderConfigInternal& aConfig) {
+ return CanDecodeAudio(aConfig);
+}
+
+/* static */
+Result<UniquePtr<TrackInfo>, nsresult> AudioDecoderTraits::CreateTrackInfo(
+ const AudioDecoderConfigInternal& aConfig) {
+ LOG("Create a AudioInfo from %s config",
+ NS_ConvertUTF16toUTF8(aConfig.mCodec).get());
+
+ nsTArray<UniquePtr<TrackInfo>> tracks = GetTracksInfo(aConfig);
+ if (tracks.Length() != 1 || tracks[0]->GetType() != TrackInfo::kAudioTrack) {
+ LOGE("Failed to get TrackInfo");
+ return Err(NS_ERROR_INVALID_ARG);
+ }
+
+ UniquePtr<TrackInfo> track(std::move(tracks[0]));
+ AudioInfo* ai = track->GetAsAudioInfo();
+ if (!ai) {
+ LOGE("Failed to get AudioInfo");
+ return Err(NS_ERROR_INVALID_ARG);
+ }
+
+ if (aConfig.mDescription.isSome()) {
+ RefPtr<MediaByteBuffer> buf;
+ buf = aConfig.mDescription.value();
+ if (buf) {
+ LOG("The given config has %zu bytes of description data", buf->Length());
+ ai->mCodecSpecificConfig =
+ AudioCodecSpecificVariant{AudioCodecSpecificBinaryBlob{buf}};
+ }
+ }
+
+ ai->mChannels = aConfig.mNumberOfChannels;
+ ai->mRate = aConfig.mSampleRate;
+
+ LOG("Created AudioInfo %s (%" PRIu32 "ch %" PRIu32
+ "Hz - with extra-data: %s)",
+ NS_ConvertUTF16toUTF8(aConfig.mCodec).get(), ai->mChannels, ai->mChannels,
+ aConfig.mDescription.isSome() ? "yes" : "no");
+
+ return track;
+}
+
+// https://w3c.github.io/webcodecs/#valid-audiodecoderconfig
+/* static */
+bool AudioDecoderTraits::Validate(const AudioDecoderConfig& aConfig,
+ nsCString& aErrorMessage) {
+ Maybe<nsString> codec = ParseCodecString(aConfig.mCodec);
+ if (!codec || codec->IsEmpty()) {
+ LOGE("Validating AudioDecoderConfig: invalid codec string");
+
+ aErrorMessage.AppendPrintf("Invalid codec string %s",
+ NS_ConvertUTF16toUTF8(aConfig.mCodec).get());
+ return false;
+ }
+
+ LOG("Validating AudioDecoderConfig: codec: %s %uch %uHz %s extradata",
+ NS_ConvertUTF16toUTF8(codec.value()).get(), aConfig.mNumberOfChannels,
+ aConfig.mSampleRate, aConfig.mDescription.WasPassed() ? "w/" : "no");
+
+ if (aConfig.mNumberOfChannels == 0) {
+ aErrorMessage.AppendPrintf("Invalid number of channels of %u",
+ aConfig.mNumberOfChannels);
+ return false;
+ }
+
+ if (aConfig.mSampleRate == 0) {
+ aErrorMessage.AppendPrintf("Invalid sample-rate of %u",
+ aConfig.mNumberOfChannels);
+ return false;
+ }
+
+ bool detached =
+ aConfig.mDescription.WasPassed() &&
+ (aConfig.mDescription.Value().IsArrayBuffer()
+ ? JS::ArrayBuffer::fromObject(
+ aConfig.mDescription.Value().GetAsArrayBuffer().Obj())
+ .isDetached()
+ : JS::ArrayBufferView::fromObject(
+ aConfig.mDescription.Value().GetAsArrayBufferView().Obj())
+ .isDetached());
+
+ if (detached) {
+ LOGE("description is detached.");
+ return false;
+ }
+
+ return true;
+}
+
+/* static */
+UniquePtr<AudioDecoderConfigInternal> AudioDecoderTraits::CreateConfigInternal(
+ const AudioDecoderConfig& aConfig) {
+ return AudioDecoderConfigInternal::Create(aConfig);
+}
+
+/* static */
+bool AudioDecoderTraits::IsKeyChunk(const EncodedAudioChunk& aInput) {
+ return aInput.Type() == EncodedAudioChunkType::Key;
+}
+
+/* static */
+UniquePtr<EncodedAudioChunkData> AudioDecoderTraits::CreateInputInternal(
+ const EncodedAudioChunk& aInput) {
+ return aInput.Clone();
+}
+
+/*
+ * Below are AudioDecoder implementation
+ */
+
+AudioDecoder::AudioDecoder(nsIGlobalObject* aParent,
+ RefPtr<WebCodecsErrorCallback>&& aErrorCallback,
+ RefPtr<AudioDataOutputCallback>&& aOutputCallback)
+ : DecoderTemplate(aParent, std::move(aErrorCallback),
+ std::move(aOutputCallback)) {
+ MOZ_ASSERT(mErrorCallback);
+ MOZ_ASSERT(mOutputCallback);
+ LOG("AudioDecoder %p ctor", this);
+}
+
+AudioDecoder::~AudioDecoder() {
+ LOG("AudioDecoder %p dtor", this);
+ Unused << ResetInternal(NS_ERROR_DOM_ABORT_ERR);
+}
+
+JSObject* AudioDecoder::WrapObject(JSContext* aCx,
+ JS::Handle<JSObject*> aGivenProto) {
+ AssertIsOnOwningThread();
+
+ return AudioDecoder_Binding::Wrap(aCx, this, aGivenProto);
+}
+
+// https://w3c.github.io/webcodecs/#dom-audiodecoder-audiodecoder
+/* static */
+already_AddRefed<AudioDecoder> AudioDecoder::Constructor(
+ const GlobalObject& aGlobal, const AudioDecoderInit& aInit,
+ ErrorResult& aRv) {
+ nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+ if (!global) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return nullptr;
+ }
+
+ return MakeAndAddRef<AudioDecoder>(
+ global.get(), RefPtr<WebCodecsErrorCallback>(aInit.mError),
+ RefPtr<AudioDataOutputCallback>(aInit.mOutput));
+}
+
+// https://w3c.github.io/webcodecs/#dom-audiodecoder-isconfigsupported
+/* static */
+already_AddRefed<Promise> AudioDecoder::IsConfigSupported(
+ const GlobalObject& aGlobal, const AudioDecoderConfig& aConfig,
+ ErrorResult& aRv) {
+ LOG("AudioDecoder::IsConfigSupported, config: %s",
+ NS_ConvertUTF16toUTF8(aConfig.mCodec).get());
+
+ nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+ if (!global) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return nullptr;
+ }
+
+ RefPtr<Promise> p = Promise::Create(global.get(), aRv);
+ if (NS_WARN_IF(aRv.Failed())) {
+ return p.forget();
+ }
+
+ nsCString errorMessage;
+ if (!AudioDecoderTraits::Validate(aConfig, errorMessage)) {
+ p->MaybeRejectWithTypeError(errorMessage);
+ return p.forget();
+ }
+
+ RootedDictionary<AudioDecoderConfig> config(aGlobal.Context());
+ auto r = CloneConfiguration(config, aGlobal.Context(), aConfig, aRv);
+ if (r.isErr()) {
+ // This can only be an OOM: all members to clone are known to be valid
+ // because this is check by ::Validate above.
+ MOZ_ASSERT(r.inspectErr() == NS_ERROR_OUT_OF_MEMORY &&
+ aRv.ErrorCodeIs(NS_ERROR_OUT_OF_MEMORY));
+ return p.forget();
+ }
+
+ bool canDecode = CanDecodeAudio(config);
+ RootedDictionary<AudioDecoderSupport> s(aGlobal.Context());
+ s.mConfig.Construct(std::move(config));
+ s.mSupported.Construct(canDecode);
+
+ p->MaybeResolve(s);
+ return p.forget();
+}
+
+already_AddRefed<MediaRawData> AudioDecoder::InputDataToMediaRawData(
+ UniquePtr<EncodedAudioChunkData>&& aData, TrackInfo& aInfo,
+ const AudioDecoderConfigInternal& aConfig) {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(aInfo.GetAsAudioInfo());
+
+ if (!aData) {
+ LOGE("No data for conversion");
+ return nullptr;
+ }
+
+ RefPtr<MediaRawData> sample = aData->TakeData();
+ if (!sample) {
+ LOGE("Take no data for conversion");
+ return nullptr;
+ }
+
+ LOGV(
+ "EncodedAudioChunkData %p converted to %zu-byte MediaRawData - time: "
+ "%" PRIi64 "us, timecode: %" PRIi64 "us, duration: %" PRIi64
+ "us, key-frame: %s",
+ aData.get(), sample->Size(), sample->mTime.ToMicroseconds(),
+ sample->mTimecode.ToMicroseconds(), sample->mDuration.ToMicroseconds(),
+ sample->mKeyframe ? "yes" : "no");
+
+ return sample.forget();
+}
+
+nsTArray<RefPtr<AudioData>> AudioDecoder::DecodedDataToOutputType(
+ nsIGlobalObject* aGlobalObject, const nsTArray<RefPtr<MediaData>>&& aData,
+ AudioDecoderConfigInternal& aConfig) {
+ AssertIsOnOwningThread();
+
+ nsTArray<RefPtr<AudioData>> frames;
+ for (const RefPtr<MediaData>& data : aData) {
+ MOZ_RELEASE_ASSERT(data->mType == MediaData::Type::AUDIO_DATA);
+ RefPtr<mozilla::AudioData> d(data->As<mozilla::AudioData>());
+ frames.AppendElement(CreateAudioData(aGlobalObject, d.get()));
+ }
+ return frames;
+}
+
+#undef LOG
+#undef LOGW
+#undef LOGE
+#undef LOGV
+#undef LOG_INTERNAL
+
+} // namespace mozilla::dom