/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim:set ts=2 sw=2 sts=2 et cindent: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "mozilla/dom/AudioDecoder.h" #include "mozilla/dom/AudioDecoderBinding.h" #include "DecoderTraits.h" #include "MediaContainerType.h" #include "MediaData.h" #include "VideoUtils.h" #include "mozilla/Assertions.h" #include "mozilla/Logging.h" #include "mozilla/Maybe.h" #include "mozilla/Try.h" #include "mozilla/Unused.h" #include "mozilla/dom/AudioDataBinding.h" #include "mozilla/dom/EncodedAudioChunk.h" #include "mozilla/dom/EncodedAudioChunkBinding.h" #include "mozilla/dom/ImageUtils.h" #include "mozilla/dom/Promise.h" #include "mozilla/dom/WebCodecsUtils.h" #include "nsPrintfCString.h" #include "nsReadableUtils.h" extern mozilla::LazyLogModule gWebCodecsLog; namespace mozilla::dom { #ifdef LOG_INTERNAL # undef LOG_INTERNAL #endif // LOG_INTERNAL #define LOG_INTERNAL(level, msg, ...) \ MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) #ifdef LOG # undef LOG #endif // LOG #define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) #ifdef LOGW # undef LOGW #endif // LOGW #define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) #ifdef LOGE # undef LOGE #endif // LOGE #define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) #ifdef LOGV # undef LOGV #endif // LOGV #define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__) NS_IMPL_CYCLE_COLLECTION_INHERITED(AudioDecoder, DOMEventTargetHelper, mErrorCallback, mOutputCallback) NS_IMPL_ADDREF_INHERITED(AudioDecoder, DOMEventTargetHelper) NS_IMPL_RELEASE_INHERITED(AudioDecoder, DOMEventTargetHelper) NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(AudioDecoder) NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper) /* * Below are helper classes */ AudioDecoderConfigInternal::AudioDecoderConfigInternal( const nsAString& aCodec, uint32_t aSampleRate, uint32_t aNumberOfChannels, Maybe>&& aDescription) : mCodec(aCodec), mSampleRate(aSampleRate), mNumberOfChannels(aNumberOfChannels), mDescription(std::move(aDescription)) {} /*static*/ UniquePtr AudioDecoderConfigInternal::Create( const AudioDecoderConfig& aConfig) { nsCString errorMessage; if (!AudioDecoderTraits::Validate(aConfig, errorMessage)) { LOGE("Failed to create AudioDecoderConfigInternal: %s", errorMessage.get()); return nullptr; } Maybe> description; if (aConfig.mDescription.WasPassed()) { auto rv = GetExtraDataFromArrayBuffer(aConfig.mDescription.Value()); if (rv.isErr()) { // Invalid description data. nsCString error; GetErrorName(rv.unwrapErr(), error); LOGE( "Failed to create AudioDecoderConfigInternal due to invalid " "description data. Error: %s", error.get()); return nullptr; } description.emplace(rv.unwrap()); } return UniquePtr(new AudioDecoderConfigInternal( aConfig.mCodec, aConfig.mSampleRate, aConfig.mNumberOfChannels, std::move(description))); } /* * The followings are helpers for AudioDecoder methods */ struct AudioMIMECreateParam { explicit AudioMIMECreateParam(const AudioDecoderConfigInternal& aConfig) : mParsedCodec(ParseCodecString(aConfig.mCodec).valueOr(EmptyString())) {} explicit AudioMIMECreateParam(const AudioDecoderConfig& aConfig) : mParsedCodec(ParseCodecString(aConfig.mCodec).valueOr(EmptyString())) {} const nsString mParsedCodec; }; // Map between WebCodecs pcm types as strings and codec numbers // All other codecs nsCString ConvertCodecName(const nsCString& aContainer, const nsCString& aCodec) { if (!aContainer.EqualsLiteral("x-wav")) { return aCodec; } if (aCodec.EqualsLiteral("ulaw")) { return nsCString("7"); } if (aCodec.EqualsLiteral("alaw")) { return nsCString("6"); } if (aCodec.Find("f32")) { return nsCString("3"); } // Linear PCM return nsCString("1"); } static nsTArray GuessMIMETypes(const AudioMIMECreateParam& aParam) { nsCString codec = NS_ConvertUTF16toUTF8(aParam.mParsedCodec); nsTArray types; for (const nsCString& container : GuessContainers(aParam.mParsedCodec)) { codec = ConvertCodecName(container, codec); nsPrintfCString mime("audio/%s; codecs=%s", container.get(), codec.get()); types.AppendElement(mime); } return types; } static bool IsSupportedAudioCodec(const nsAString& aCodec) { LOG("IsSupportedAudioCodec: %s", NS_ConvertUTF16toUTF8(aCodec).get()); return aCodec.EqualsLiteral("flac") || aCodec.EqualsLiteral("mp3") || IsAACCodecString(aCodec) || aCodec.EqualsLiteral("opus") || aCodec.EqualsLiteral("ulaw") || aCodec.EqualsLiteral("alaw") || aCodec.EqualsLiteral("pcm-u8") || aCodec.EqualsLiteral("pcm-s16") || aCodec.EqualsLiteral("pcm-s24") || aCodec.EqualsLiteral("pcm-s32") || aCodec.EqualsLiteral("pcm-f32"); } // https://w3c.github.io/webcodecs/#check-configuration-support template static bool CanDecodeAudio(const Config& aConfig) { auto param = AudioMIMECreateParam(aConfig); if (!IsSupportedAudioCodec(param.mParsedCodec)) { return false; } if (IsOnAndroid() && IsAACCodecString(param.mParsedCodec)) { return false; } // TODO: Instead of calling CanHandleContainerType with the guessed the // containers, DecoderTraits should provide an API to tell if a codec is // decodable or not. for (const nsCString& mime : GuessMIMETypes(param)) { if (Maybe containerType = MakeMediaExtendedMIMEType(mime)) { if (DecoderTraits::CanHandleContainerType( *containerType, nullptr /* DecoderDoctorDiagnostics */) != CANPLAY_NO) { return true; } } } return false; } static nsTArray> GetTracksInfo( const AudioDecoderConfigInternal& aConfig) { // TODO: Instead of calling GetTracksInfo with the guessed containers, // DecoderTraits should provide an API to create the TrackInfo directly. for (const nsCString& mime : GuessMIMETypes(AudioMIMECreateParam(aConfig))) { if (Maybe containerType = MakeMediaExtendedMIMEType(mime)) { if (nsTArray> tracks = DecoderTraits::GetTracksInfo(*containerType); !tracks.IsEmpty()) { return tracks; } } } return {}; } static Result CloneConfiguration( RootedDictionary& aDest, JSContext* aCx, const AudioDecoderConfig& aConfig, ErrorResult& aRv) { aDest.mCodec = aConfig.mCodec; if (aConfig.mDescription.WasPassed()) { aDest.mDescription.Construct(); MOZ_TRY(CloneBuffer(aCx, aDest.mDescription.Value(), aConfig.mDescription.Value(), aRv)); } aDest.mNumberOfChannels = aConfig.mNumberOfChannels; aDest.mSampleRate = aConfig.mSampleRate; return Ok(); } // https://w3c.github.io/webcodecs/#create-a-audiodata static RefPtr CreateAudioData(nsIGlobalObject* aGlobalObject, mozilla::AudioData* aData) { MOZ_ASSERT(aGlobalObject); MOZ_ASSERT(aData); auto buf = aData->MoveableData(); // TODO: Ensure buf.Length() is a multiple of aData->mChannels and put it into // AssertedCast (sinze return type of buf.Length() is size_t). uint32_t frames = buf.Length() / aData->mChannels; RefPtr resource = AudioDataResource::Create(Span{ reinterpret_cast(buf.Data()), buf.Length() * sizeof(float)}); return MakeRefPtr(aGlobalObject, resource.forget(), aData->mTime.ToMicroseconds(), aData->mChannels, frames, AssertedCast(aData->mRate), mozilla::dom::AudioSampleFormat::F32); } /* static */ bool AudioDecoderTraits::IsSupported( const AudioDecoderConfigInternal& aConfig) { return CanDecodeAudio(aConfig); } /* static */ Result, nsresult> AudioDecoderTraits::CreateTrackInfo( const AudioDecoderConfigInternal& aConfig) { LOG("Create a AudioInfo from %s config", NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); nsTArray> tracks = GetTracksInfo(aConfig); if (tracks.Length() != 1 || tracks[0]->GetType() != TrackInfo::kAudioTrack) { LOGE("Failed to get TrackInfo"); return Err(NS_ERROR_INVALID_ARG); } UniquePtr track(std::move(tracks[0])); AudioInfo* ai = track->GetAsAudioInfo(); if (!ai) { LOGE("Failed to get AudioInfo"); return Err(NS_ERROR_INVALID_ARG); } if (aConfig.mDescription.isSome()) { RefPtr buf; buf = aConfig.mDescription.value(); if (buf) { LOG("The given config has %zu bytes of description data", buf->Length()); ai->mCodecSpecificConfig = AudioCodecSpecificVariant{AudioCodecSpecificBinaryBlob{buf}}; } } ai->mChannels = aConfig.mNumberOfChannels; ai->mRate = aConfig.mSampleRate; LOG("Created AudioInfo %s (%" PRIu32 "ch %" PRIu32 "Hz - with extra-data: %s)", NS_ConvertUTF16toUTF8(aConfig.mCodec).get(), ai->mChannels, ai->mChannels, aConfig.mDescription.isSome() ? "yes" : "no"); return track; } // https://w3c.github.io/webcodecs/#valid-audiodecoderconfig /* static */ bool AudioDecoderTraits::Validate(const AudioDecoderConfig& aConfig, nsCString& aErrorMessage) { Maybe codec = ParseCodecString(aConfig.mCodec); if (!codec || codec->IsEmpty()) { LOGE("Validating AudioDecoderConfig: invalid codec string"); aErrorMessage.AppendPrintf("Invalid codec string %s", NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); return false; } LOG("Validating AudioDecoderConfig: codec: %s %uch %uHz %s extradata", NS_ConvertUTF16toUTF8(codec.value()).get(), aConfig.mNumberOfChannels, aConfig.mSampleRate, aConfig.mDescription.WasPassed() ? "w/" : "no"); if (aConfig.mNumberOfChannels == 0) { aErrorMessage.AppendPrintf("Invalid number of channels of %u", aConfig.mNumberOfChannels); return false; } if (aConfig.mSampleRate == 0) { aErrorMessage.AppendPrintf("Invalid sample-rate of %u", aConfig.mNumberOfChannels); return false; } bool detached = aConfig.mDescription.WasPassed() && (aConfig.mDescription.Value().IsArrayBuffer() ? JS::ArrayBuffer::fromObject( aConfig.mDescription.Value().GetAsArrayBuffer().Obj()) .isDetached() : JS::ArrayBufferView::fromObject( aConfig.mDescription.Value().GetAsArrayBufferView().Obj()) .isDetached()); if (detached) { LOGE("description is detached."); return false; } return true; } /* static */ UniquePtr AudioDecoderTraits::CreateConfigInternal( const AudioDecoderConfig& aConfig) { return AudioDecoderConfigInternal::Create(aConfig); } /* static */ bool AudioDecoderTraits::IsKeyChunk(const EncodedAudioChunk& aInput) { return aInput.Type() == EncodedAudioChunkType::Key; } /* static */ UniquePtr AudioDecoderTraits::CreateInputInternal( const EncodedAudioChunk& aInput) { return aInput.Clone(); } /* * Below are AudioDecoder implementation */ AudioDecoder::AudioDecoder(nsIGlobalObject* aParent, RefPtr&& aErrorCallback, RefPtr&& aOutputCallback) : DecoderTemplate(aParent, std::move(aErrorCallback), std::move(aOutputCallback)) { MOZ_ASSERT(mErrorCallback); MOZ_ASSERT(mOutputCallback); LOG("AudioDecoder %p ctor", this); } AudioDecoder::~AudioDecoder() { LOG("AudioDecoder %p dtor", this); Unused << ResetInternal(NS_ERROR_DOM_ABORT_ERR); } JSObject* AudioDecoder::WrapObject(JSContext* aCx, JS::Handle aGivenProto) { AssertIsOnOwningThread(); return AudioDecoder_Binding::Wrap(aCx, this, aGivenProto); } // https://w3c.github.io/webcodecs/#dom-audiodecoder-audiodecoder /* static */ already_AddRefed AudioDecoder::Constructor( const GlobalObject& aGlobal, const AudioDecoderInit& aInit, ErrorResult& aRv) { nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); if (!global) { aRv.Throw(NS_ERROR_FAILURE); return nullptr; } return MakeAndAddRef( global.get(), RefPtr(aInit.mError), RefPtr(aInit.mOutput)); } // https://w3c.github.io/webcodecs/#dom-audiodecoder-isconfigsupported /* static */ already_AddRefed AudioDecoder::IsConfigSupported( const GlobalObject& aGlobal, const AudioDecoderConfig& aConfig, ErrorResult& aRv) { LOG("AudioDecoder::IsConfigSupported, config: %s", NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); if (!global) { aRv.Throw(NS_ERROR_FAILURE); return nullptr; } RefPtr p = Promise::Create(global.get(), aRv); if (NS_WARN_IF(aRv.Failed())) { return p.forget(); } nsCString errorMessage; if (!AudioDecoderTraits::Validate(aConfig, errorMessage)) { p->MaybeRejectWithTypeError(errorMessage); return p.forget(); } RootedDictionary config(aGlobal.Context()); auto r = CloneConfiguration(config, aGlobal.Context(), aConfig, aRv); if (r.isErr()) { // This can only be an OOM: all members to clone are known to be valid // because this is check by ::Validate above. MOZ_ASSERT(r.inspectErr() == NS_ERROR_OUT_OF_MEMORY && aRv.ErrorCodeIs(NS_ERROR_OUT_OF_MEMORY)); return p.forget(); } bool canDecode = CanDecodeAudio(config); RootedDictionary s(aGlobal.Context()); s.mConfig.Construct(std::move(config)); s.mSupported.Construct(canDecode); p->MaybeResolve(s); return p.forget(); } already_AddRefed AudioDecoder::InputDataToMediaRawData( UniquePtr&& aData, TrackInfo& aInfo, const AudioDecoderConfigInternal& aConfig) { AssertIsOnOwningThread(); MOZ_ASSERT(aInfo.GetAsAudioInfo()); if (!aData) { LOGE("No data for conversion"); return nullptr; } RefPtr sample = aData->TakeData(); if (!sample) { LOGE("Take no data for conversion"); return nullptr; } LOGV( "EncodedAudioChunkData %p converted to %zu-byte MediaRawData - time: " "%" PRIi64 "us, timecode: %" PRIi64 "us, duration: %" PRIi64 "us, key-frame: %s", aData.get(), sample->Size(), sample->mTime.ToMicroseconds(), sample->mTimecode.ToMicroseconds(), sample->mDuration.ToMicroseconds(), sample->mKeyframe ? "yes" : "no"); return sample.forget(); } nsTArray> AudioDecoder::DecodedDataToOutputType( nsIGlobalObject* aGlobalObject, const nsTArray>&& aData, AudioDecoderConfigInternal& aConfig) { AssertIsOnOwningThread(); nsTArray> frames; for (const RefPtr& data : aData) { MOZ_RELEASE_ASSERT(data->mType == MediaData::Type::AUDIO_DATA); RefPtr d(data->As()); frames.AppendElement(CreateAudioData(aGlobalObject, d.get())); } return frames; } #undef LOG #undef LOGW #undef LOGE #undef LOGV #undef LOG_INTERNAL } // namespace mozilla::dom