/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim:set ts=2 sw=2 sts=2 et cindent: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "mozilla/dom/AudioEncoder.h" #include "EncoderTraits.h" #include "mozilla/dom/AudioEncoderBinding.h" #include "EncoderConfig.h" #include "EncoderTypes.h" #include "MediaData.h" #include "mozilla/Assertions.h" #include "mozilla/Logging.h" #include "mozilla/Maybe.h" #include "mozilla/dom/AudioDataBinding.h" #include "mozilla/dom/EncodedAudioChunk.h" #include "mozilla/dom/EncodedAudioChunkBinding.h" #include "mozilla/dom/Promise.h" #include "mozilla/dom/WebCodecsUtils.h" #include "EncoderConfig.h" extern mozilla::LazyLogModule gWebCodecsLog; namespace mozilla::dom { #ifdef LOG_INTERNAL # undef LOG_INTERNAL #endif // LOG_INTERNAL #define LOG_INTERNAL(level, msg, ...) \ MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__)) #ifdef LOG # undef LOG #endif // LOG #define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__) #ifdef LOGW # undef LOGW #endif // LOGW #define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__) #ifdef LOGE # undef LOGE #endif // LOGE #define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__) #ifdef LOGV # undef LOGV #endif // LOGV #define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__) NS_IMPL_CYCLE_COLLECTION_INHERITED(AudioEncoder, DOMEventTargetHelper, mErrorCallback, mOutputCallback) NS_IMPL_ADDREF_INHERITED(AudioEncoder, DOMEventTargetHelper) NS_IMPL_RELEASE_INHERITED(AudioEncoder, DOMEventTargetHelper) NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(AudioEncoder) NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper) /* * Below are helper classes */ AudioEncoderConfigInternal::AudioEncoderConfigInternal( const nsAString& aCodec, Maybe aSampleRate, Maybe aNumberOfChannels, Maybe aBitrate, BitrateMode aBitrateMode) : mCodec(aCodec), mSampleRate(aSampleRate), mNumberOfChannels(aNumberOfChannels), mBitrate(aBitrate), mBitrateMode(aBitrateMode) {} AudioEncoderConfigInternal::AudioEncoderConfigInternal( const AudioEncoderConfig& aConfig) : AudioEncoderConfigInternal( aConfig.mCodec, OptionalToMaybe(aConfig.mSampleRate), OptionalToMaybe(aConfig.mNumberOfChannels), OptionalToMaybe(aConfig.mBitrate), aConfig.mBitrateMode) { DebugOnly errorMessage; if (aConfig.mCodec.EqualsLiteral("opus") && aConfig.mOpus.WasPassed()) { // All values are in range at this point, the config is known valid. OpusSpecific specific; if (aConfig.mOpus.Value().mComplexity.WasPassed()) { specific.mComplexity = aConfig.mOpus.Value().mComplexity.Value(); } else { // https://w3c.github.io/webcodecs/opus_codec_registration.html#dom-opusencoderconfig-complexity // If no value is specificied, the default value is platform-specific: // User Agents SHOULD set a default of 5 for mobile platforms, and a // default of 9 for all other platforms. if (IsOnAndroid()) { specific.mComplexity = 5; } else { specific.mComplexity = 9; } } specific.mApplication = OpusSpecific::Application::Unspecified; specific.mFrameDuration = aConfig.mOpus.Value().mFrameDuration; specific.mPacketLossPerc = aConfig.mOpus.Value().mPacketlossperc; specific.mUseDTX = aConfig.mOpus.Value().mUsedtx; specific.mUseInBandFEC = aConfig.mOpus.Value().mUseinbandfec; mSpecific.emplace(specific); } MOZ_ASSERT(AudioEncoderTraits::Validate(aConfig, errorMessage)); } AudioEncoderConfigInternal::AudioEncoderConfigInternal( const AudioEncoderConfigInternal& aConfig) : AudioEncoderConfigInternal(aConfig.mCodec, aConfig.mSampleRate, aConfig.mNumberOfChannels, aConfig.mBitrate, aConfig.mBitrateMode) {} void AudioEncoderConfigInternal::SetSpecific( const EncoderConfig::CodecSpecific& aSpecific) { mSpecific.emplace(aSpecific); } /* * The followings are helpers for AudioEncoder methods */ static void CloneConfiguration(RootedDictionary& aDest, JSContext* aCx, const AudioEncoderConfig& aConfig) { aDest.mCodec = aConfig.mCodec; if (aConfig.mNumberOfChannels.WasPassed()) { aDest.mNumberOfChannels.Construct(aConfig.mNumberOfChannels.Value()); } if (aConfig.mSampleRate.WasPassed()) { aDest.mSampleRate.Construct(aConfig.mSampleRate.Value()); } if (aConfig.mBitrate.WasPassed()) { aDest.mBitrate.Construct(aConfig.mBitrate.Value()); } if (aConfig.mOpus.WasPassed()) { aDest.mOpus.Construct(aConfig.mOpus.Value()); // Handle the default value manually since it's different on mobile if (!aConfig.mOpus.Value().mComplexity.WasPassed()) { if (IsOnAndroid()) { aDest.mOpus.Value().mComplexity.Construct(5); } else { aDest.mOpus.Value().mComplexity.Construct(9); } } } aDest.mBitrateMode = aConfig.mBitrateMode; } static bool IsAudioEncodeSupported(const nsAString& aCodec) { LOG("IsEncodeSupported: %s", NS_ConvertUTF16toUTF8(aCodec).get()); return aCodec.EqualsLiteral("opus") || aCodec.EqualsLiteral("vorbis"); } static bool CanEncode(const RefPtr& aConfig, nsCString& aErrorMessage) { auto parsedCodecString = ParseCodecString(aConfig->mCodec).valueOr(EmptyString()); // TODO: Enable WebCodecs on Android (Bug 1840508) if (IsOnAndroid()) { return false; } if (!IsAudioEncodeSupported(parsedCodecString)) { return false; } if (aConfig->mNumberOfChannels.value() > 256) { aErrorMessage.AppendPrintf( "Invalid number of channels, supported range is between 1 and 256"); return false; } // Somewhat arbitrarily chosen, but reflects real-life and what the rest of // Gecko does. if (aConfig->mSampleRate.value() < 3000 || aConfig->mSampleRate.value() > 384000) { aErrorMessage.AppendPrintf( "Invalid sample-rate of %d, supported range is 3000Hz to 384000Hz", aConfig->mSampleRate.value()); return false; } return EncoderSupport::Supports(aConfig); } nsCString AudioEncoderConfigInternal::ToString() const { nsCString rv; rv.AppendPrintf("AudioEncoderConfigInternal: %s", NS_ConvertUTF16toUTF8(mCodec).get()); if (mSampleRate) { rv.AppendPrintf(" %" PRIu32 "Hz", mSampleRate.value()); } if (mNumberOfChannels) { rv.AppendPrintf(" %" PRIu32 "ch", mNumberOfChannels.value()); } if (mBitrate) { rv.AppendPrintf(" %" PRIu32 "bps", mBitrate.value()); } rv.AppendPrintf(" (%s)", mBitrateMode == mozilla::dom::BitrateMode::Constant ? "CRB" : "VBR"); return rv; } EncoderConfig AudioEncoderConfigInternal::ToEncoderConfig() const { const mozilla::BitrateMode bitrateMode = mBitrateMode == mozilla::dom::BitrateMode::Constant ? mozilla::BitrateMode::Constant : mozilla::BitrateMode::Variable; CodecType type = CodecType::Opus; Maybe specific; if (mCodec.EqualsLiteral("opus")) { type = CodecType::Opus; MOZ_ASSERT(mSpecific.isNothing() || mSpecific->is()); specific = mSpecific; } else if (mCodec.EqualsLiteral("vorbis")) { type = CodecType::Vorbis; } else if (mCodec.EqualsLiteral("flac")) { type = CodecType::Flac; } else if (StringBeginsWith(mCodec, u"pcm-"_ns)) { type = CodecType::PCM; } else if (mCodec.EqualsLiteral("ulaw")) { type = CodecType::PCM; } else if (mCodec.EqualsLiteral("alaw")) { type = CodecType::PCM; } else if (StringBeginsWith(mCodec, u"mp4a."_ns)) { type = CodecType::AAC; } // This should have been checked ahead of time -- we can't encode without // knowing the sample-rate or the channel count at the very least. MOZ_ASSERT(mSampleRate.value()); MOZ_ASSERT(mNumberOfChannels.value()); return EncoderConfig(type, mNumberOfChannels.value(), bitrateMode, AssertedCast(mSampleRate.value()), mBitrate.valueOr(0), specific); } bool AudioEncoderConfigInternal::Equals( const AudioEncoderConfigInternal& aOther) const { return false; } bool AudioEncoderConfigInternal::CanReconfigure( const AudioEncoderConfigInternal& aOther) const { return false; } already_AddRefed AudioEncoderConfigInternal::Diff( const AudioEncoderConfigInternal& aOther) const { return MakeRefPtr().forget(); } /* static */ bool AudioEncoderTraits::IsSupported( const AudioEncoderConfigInternal& aConfig) { nsCString errorMessage; bool canEncode = CanEncode(MakeRefPtr(aConfig), errorMessage); if (!canEncode) { LOGE("Can't encode configuration %s: %s", aConfig.ToString().get(), errorMessage.get()); } return canEncode; } // https://w3c.github.io/webcodecs/#valid-audioencoderconfig /* static */ bool AudioEncoderTraits::Validate(const AudioEncoderConfig& aConfig, nsCString& aErrorMessage) { Maybe codec = ParseCodecString(aConfig.mCodec); if (!codec || codec->IsEmpty()) { LOGE("Validating AudioEncoderConfig: invalid codec string"); return false; } if (!aConfig.mNumberOfChannels.WasPassed()) { aErrorMessage.AppendPrintf("Channel count required"); return false; } if (aConfig.mNumberOfChannels.Value() == 0) { aErrorMessage.AppendPrintf( "Invalid number of channels, supported range is between 1 and 256"); return false; } if (!aConfig.mSampleRate.WasPassed()) { aErrorMessage.AppendPrintf("Sample-rate required"); return false; } if (aConfig.mSampleRate.Value() == 0) { aErrorMessage.AppendPrintf("Invalid sample-rate of 0"); return false; } if (aConfig.mBitrate.WasPassed() && aConfig.mBitrate.Value() > std::numeric_limits::max()) { aErrorMessage.AppendPrintf("Invalid config: bitrate value too large"); return false; } if (codec->EqualsLiteral("opus")) { // This comes from // https://w3c.github.io/webcodecs/opus_codec_registration.html#opus-encoder-config if (aConfig.mBitrate.WasPassed() && (aConfig.mBitrate.Value() < 6000 || aConfig.mBitrate.Value() > 510000)) { aErrorMessage.AppendPrintf( "Invalid config: bitrate value outside of [6k, 510k] for opus"); return false; } if (aConfig.mOpus.WasPassed()) { // Verify value ranges const std::array validFrameDurationUs = {2500, 5000, 10000, 20000, 40000, 60000}; if (std::find(validFrameDurationUs.begin(), validFrameDurationUs.end(), aConfig.mOpus.Value().mFrameDuration) == validFrameDurationUs.end()) { aErrorMessage.AppendPrintf("Invalid config: invalid frame duration"); return false; } if (aConfig.mOpus.Value().mComplexity.WasPassed() && aConfig.mOpus.Value().mComplexity.Value() > 10) { aErrorMessage.AppendPrintf( "Invalid config: Opus complexity greater than 10"); return false; } if (aConfig.mOpus.Value().mPacketlossperc > 100) { aErrorMessage.AppendPrintf( "Invalid config: Opus packet loss percentage greater than 100"); return false; } } } return true; } /* static */ RefPtr AudioEncoderTraits::CreateConfigInternal( const AudioEncoderConfig& aConfig) { nsCString errorMessage; if (!AudioEncoderTraits::Validate(aConfig, errorMessage)) { return nullptr; } return MakeRefPtr(aConfig); } /* static */ RefPtr AudioEncoderTraits::CreateInputInternal( const dom::AudioData& aInput, const dom::VideoEncoderEncodeOptions& /* unused */) { return aInput.ToAudioData(); } /* * Below are AudioEncoder implementation */ AudioEncoder::AudioEncoder( nsIGlobalObject* aParent, RefPtr&& aErrorCallback, RefPtr&& aOutputCallback) : EncoderTemplate(aParent, std::move(aErrorCallback), std::move(aOutputCallback)) { MOZ_ASSERT(mErrorCallback); MOZ_ASSERT(mOutputCallback); LOG("AudioEncoder %p ctor", this); } AudioEncoder::~AudioEncoder() { LOG("AudioEncoder %p dtor", this); Unused << ResetInternal(NS_ERROR_DOM_ABORT_ERR); } JSObject* AudioEncoder::WrapObject(JSContext* aCx, JS::Handle aGivenProto) { AssertIsOnOwningThread(); return AudioEncoder_Binding::Wrap(aCx, this, aGivenProto); } // https://w3c.github.io/webcodecs/#dom-audioencoder-audioencoder /* static */ already_AddRefed AudioEncoder::Constructor( const GlobalObject& aGlobal, const AudioEncoderInit& aInit, ErrorResult& aRv) { nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); if (!global) { aRv.Throw(NS_ERROR_FAILURE); return nullptr; } return MakeAndAddRef( global.get(), RefPtr(aInit.mError), RefPtr(aInit.mOutput)); } // https://w3c.github.io/webcodecs/#dom-audioencoder-isconfigsupported /* static */ already_AddRefed AudioEncoder::IsConfigSupported( const GlobalObject& aGlobal, const AudioEncoderConfig& aConfig, ErrorResult& aRv) { LOG("AudioEncoder::IsConfigSupported, config: %s", NS_ConvertUTF16toUTF8(aConfig.mCodec).get()); nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); if (!global) { aRv.Throw(NS_ERROR_FAILURE); return nullptr; } RefPtr p = Promise::Create(global.get(), aRv); if (NS_WARN_IF(aRv.Failed())) { return p.forget(); } nsCString errorMessage; if (!AudioEncoderTraits::Validate(aConfig, errorMessage)) { p->MaybeRejectWithTypeError(errorMessage); return p.forget(); } // TODO: Move the following works to another thread to unblock the current // thread, as what spec suggests. RootedDictionary config(aGlobal.Context()); CloneConfiguration(config, aGlobal.Context(), aConfig); bool supportedAudioCodec = IsSupportedAudioCodec(aConfig.mCodec); auto configInternal = MakeRefPtr(aConfig); bool canEncode = CanEncode(configInternal, errorMessage); if (!canEncode) { LOG("CanEncode failed: %s", errorMessage.get()); } RootedDictionary s(aGlobal.Context()); s.mConfig.Construct(std::move(config)); s.mSupported.Construct(supportedAudioCodec && canEncode); p->MaybeResolve(s); return p.forget(); } RefPtr AudioEncoder::EncodedDataToOutputType( nsIGlobalObject* aGlobalObject, const RefPtr& aData) { AssertIsOnOwningThread(); // Package into an EncodedAudioChunk auto buffer = MakeRefPtr(aData->Data(), aData->Size()); auto encoded = MakeRefPtr( aGlobalObject, buffer.forget(), EncodedAudioChunkType::Key, aData->mTime.ToMicroseconds(), aData->mDuration.IsZero() ? Nothing() : Some(aData->mDuration.ToMicroseconds())); return encoded; } AudioDecoderConfigInternal AudioEncoder::EncoderConfigToDecoderConfig( nsIGlobalObject* aGlobal, const RefPtr& aRawData, const AudioEncoderConfigInternal& aOutputConfig) const { MOZ_ASSERT(aOutputConfig.mSampleRate.isSome()); MOZ_ASSERT(aOutputConfig.mNumberOfChannels.isSome()); uint32_t sampleRate = aOutputConfig.mSampleRate.value(); uint32_t channelCount = aOutputConfig.mNumberOfChannels.value(); // Check if the encoder had to modify the settings because of codec // constraints. e.g. FFmpegAudioEncoder can encode any sample-rate, but if the // codec is Opus, then it will resample the audio one of the specific rates // supported by the encoder. if (aRawData->mConfig) { sampleRate = aRawData->mConfig->mSampleRate; channelCount = aRawData->mConfig->mNumberOfChannels; } return AudioDecoderConfigInternal(aOutputConfig.mCodec, sampleRate, channelCount, do_AddRef(aRawData->mExtraData)); } #undef LOG #undef LOGW #undef LOGE #undef LOGV #undef LOG_INTERNAL } // namespace mozilla::dom