summaryrefslogtreecommitdiffstats
path: root/dom/media/webcodecs
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--dom/media/webcodecs/AudioData.cpp84
-rw-r--r--dom/media/webcodecs/AudioData.h7
-rw-r--r--dom/media/webcodecs/AudioDecoder.cpp69
-rw-r--r--dom/media/webcodecs/AudioEncoder.cpp488
-rw-r--r--dom/media/webcodecs/AudioEncoder.h76
-rw-r--r--dom/media/webcodecs/DecoderTemplate.cpp4
-rw-r--r--dom/media/webcodecs/DecoderTypes.h44
-rw-r--r--dom/media/webcodecs/EncoderTemplate.cpp522
-rw-r--r--dom/media/webcodecs/EncoderTemplate.h56
-rw-r--r--dom/media/webcodecs/EncoderTypes.h74
-rw-r--r--dom/media/webcodecs/VideoDecoder.cpp34
-rw-r--r--dom/media/webcodecs/VideoEncoder.cpp42
-rw-r--r--dom/media/webcodecs/VideoEncoder.h2
-rw-r--r--dom/media/webcodecs/WebCodecsUtils.cpp71
-rw-r--r--dom/media/webcodecs/WebCodecsUtils.h17
-rw-r--r--dom/media/webcodecs/crashtests/1881079.html35
-rw-r--r--dom/media/webcodecs/crashtests/crashtests.list4
-rw-r--r--dom/media/webcodecs/moz.build2
18 files changed, 1178 insertions, 453 deletions
diff --git a/dom/media/webcodecs/AudioData.cpp b/dom/media/webcodecs/AudioData.cpp
index 0b21798be8..aae58fb32c 100644
--- a/dom/media/webcodecs/AudioData.cpp
+++ b/dom/media/webcodecs/AudioData.cpp
@@ -151,25 +151,6 @@ JSObject* AudioData::WrapObject(JSContext* aCx,
return AudioData_Binding::Wrap(aCx, this, aGivenProto);
}
-uint32_t BytesPerSamples(const mozilla::dom::AudioSampleFormat& aFormat) {
- switch (aFormat) {
- case AudioSampleFormat::U8:
- case AudioSampleFormat::U8_planar:
- return sizeof(uint8_t);
- case AudioSampleFormat::S16:
- case AudioSampleFormat::S16_planar:
- return sizeof(int16_t);
- case AudioSampleFormat::S32:
- case AudioSampleFormat::F32:
- case AudioSampleFormat::S32_planar:
- case AudioSampleFormat::F32_planar:
- return sizeof(float);
- default:
- MOZ_ASSERT_UNREACHABLE("wrong enum value");
- }
- return 0;
-}
-
Result<Ok, nsCString> IsValidAudioDataInit(const AudioDataInit& aInit) {
if (aInit.mSampleRate <= 0.0) {
auto msg = nsLiteralCString("sampleRate must be positive");
@@ -205,37 +186,13 @@ Result<Ok, nsCString> IsValidAudioDataInit(const AudioDataInit& aInit) {
return Ok();
}
-const char* FormatToString(AudioSampleFormat aFormat) {
- switch (aFormat) {
- case AudioSampleFormat::U8:
- return "u8";
- case AudioSampleFormat::S16:
- return "s16";
- case AudioSampleFormat::S32:
- return "s32";
- case AudioSampleFormat::F32:
- return "f32";
- case AudioSampleFormat::U8_planar:
- return "u8-planar";
- case AudioSampleFormat::S16_planar:
- return "s16-planar";
- case AudioSampleFormat::S32_planar:
- return "s32-planar";
- case AudioSampleFormat::F32_planar:
- return "f32-planar";
- default:
- MOZ_ASSERT_UNREACHABLE("wrong enum value");
- }
- return "unsupported";
-}
-
/* static */
already_AddRefed<AudioData> AudioData::Constructor(const GlobalObject& aGlobal,
const AudioDataInit& aInit,
ErrorResult& aRv) {
nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
LOGD("[%p] AudioData(fmt: %s, rate: %f, ch: %" PRIu32 ", ts: %" PRId64 ")",
- global.get(), FormatToString(aInit.mFormat), aInit.mSampleRate,
+ global.get(), GetEnumString(aInit.mFormat).get(), aInit.mSampleRate,
aInit.mNumberOfChannels, aInit.mTimestamp);
if (!global) {
LOGE("Global unavailable");
@@ -311,6 +268,9 @@ struct CopyToSpec {
const uint32_t mFrameOffset;
const uint32_t mPlaneIndex;
const AudioSampleFormat mFormat;
+ // False if this is used internally, and this copy call doesn't come from
+ // script.
+ DebugOnly<bool> mFromScript = true;
};
bool IsInterleaved(const AudioSampleFormat& aFormat) {
@@ -463,7 +423,7 @@ void CopySamples(Span<S> aSource, Span<D> aDest, uint32_t aSourceChannelCount,
}
if (!IsInterleaved(aSourceFormat) && IsInterleaved(aCopyToSpec.mFormat)) {
- MOZ_CRASH("This should never be hit -- current spec doesn't support it");
+ MOZ_ASSERT(!aCopyToSpec.mFromScript);
// Planar to interleaved -- copy of all channels of the source into the
// destination buffer.
MOZ_ASSERT(aCopyToSpec.mPlaneIndex == 0);
@@ -505,7 +465,7 @@ nsCString AudioData::ToString() const {
return nsPrintfCString("AudioData[%zu bytes %s %fHz %" PRIu32 "x%" PRIu32
"ch]",
mResource->Data().LengthBytes(),
- FormatToString(mAudioSampleFormat.value()),
+ GetEnumString(mAudioSampleFormat.value()).get(),
mSampleRate, mNumberOfFrames, mNumberOfChannels);
}
@@ -515,8 +475,9 @@ nsCString CopyToToString(size_t aDestBufSize,
"AudioDataCopyToOptions[data: %zu bytes %s frame count:%" PRIu32
" frame offset: %" PRIu32 " plane: %" PRIu32 "]",
aDestBufSize,
- aOptions.mFormat.WasPassed() ? FormatToString(aOptions.mFormat.Value())
- : "null",
+ aOptions.mFormat.WasPassed()
+ ? GetEnumString(aOptions.mFormat.Value()).get()
+ : "null",
aOptions.mFrameCount.WasPassed() ? aOptions.mFrameCount.Value() : 0,
aOptions.mFrameOffset, aOptions.mPlaneIndex);
}
@@ -650,6 +611,8 @@ void AudioData::Close() {
mAudioSampleFormat = Nothing();
}
+bool AudioData::IsClosed() const { return !mResource; }
+
// https://w3c.github.io/webcodecs/#ref-for-deserialization-steps%E2%91%A1
/* static */
JSObject* AudioData::ReadStructuredClone(JSContext* aCx,
@@ -724,6 +687,31 @@ void AudioData::CloseIfNeeded() {
}
}
+RefPtr<mozilla::AudioData> AudioData::ToAudioData() const {
+ // Always convert to f32 interleaved for now, as this Gecko's prefered
+ // internal audio representation for encoding and decoding.
+ Span<uint8_t> data = mResource->Data();
+ DebugOnly<uint32_t> frames = mNumberOfFrames;
+ uint32_t bytesPerSample = BytesPerSamples(mAudioSampleFormat.value());
+ uint32_t samples = data.Length() / bytesPerSample;
+ DebugOnly<uint32_t> computedFrames = samples / mNumberOfChannels;
+ MOZ_ASSERT(frames == computedFrames);
+ AlignedAudioBuffer buf(samples);
+ Span<uint8_t> storage(reinterpret_cast<uint8_t*>(buf.Data()),
+ samples * sizeof(float));
+
+ CopyToSpec spec(mNumberOfFrames, 0, 0, AudioSampleFormat::F32);
+#ifdef DEBUG
+ spec.mFromScript = false;
+#endif
+
+ DoCopy(data, storage, mNumberOfChannels, mAudioSampleFormat.value(), spec);
+
+ return MakeRefPtr<mozilla::AudioData>(
+ 0, media::TimeUnit::FromMicroseconds(mTimestamp), std::move(buf),
+ mNumberOfChannels, mSampleRate);
+}
+
#undef LOGD
#undef LOGE
#undef LOG_INTERNAL
diff --git a/dom/media/webcodecs/AudioData.h b/dom/media/webcodecs/AudioData.h
index 4ae69a225a..43af638d11 100644
--- a/dom/media/webcodecs/AudioData.h
+++ b/dom/media/webcodecs/AudioData.h
@@ -90,6 +90,7 @@ class AudioData final : public nsISupports, public nsWrapperCache {
already_AddRefed<AudioData> Clone(ErrorResult& aRv);
void Close();
+ bool IsClosed() const;
// [Serializable] implementations: {Read, Write}StructuredClone
static JSObject* ReadStructuredClone(JSContext* aCx, nsIGlobalObject* aGlobal,
@@ -107,11 +108,13 @@ class AudioData final : public nsISupports, public nsWrapperCache {
static already_AddRefed<AudioData> FromTransferred(nsIGlobalObject* aGlobal,
TransferredData* aData);
+ nsCString ToString() const;
+
+ RefPtr<mozilla::AudioData> ToAudioData() const;
+
private:
size_t ComputeCopyElementCount(const AudioDataCopyToOptions& aOptions,
ErrorResult& aRv);
-
- nsCString ToString() const;
// AudioData can run on either main thread or worker thread.
void AssertIsOnOwningThread() const { NS_ASSERT_OWNINGTHREAD(AudioData); }
void CloseIfNeeded();
diff --git a/dom/media/webcodecs/AudioDecoder.cpp b/dom/media/webcodecs/AudioDecoder.cpp
index 6b554dcacf..ef2acd4eae 100644
--- a/dom/media/webcodecs/AudioDecoder.cpp
+++ b/dom/media/webcodecs/AudioDecoder.cpp
@@ -68,11 +68,11 @@ NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper)
AudioDecoderConfigInternal::AudioDecoderConfigInternal(
const nsAString& aCodec, uint32_t aSampleRate, uint32_t aNumberOfChannels,
- Maybe<RefPtr<MediaByteBuffer>>&& aDescription)
+ already_AddRefed<MediaByteBuffer> aDescription)
: mCodec(aCodec),
mSampleRate(aSampleRate),
mNumberOfChannels(aNumberOfChannels),
- mDescription(std::move(aDescription)) {}
+ mDescription(aDescription) {}
/*static*/
UniquePtr<AudioDecoderConfigInternal> AudioDecoderConfigInternal::Create(
@@ -83,7 +83,7 @@ UniquePtr<AudioDecoderConfigInternal> AudioDecoderConfigInternal::Create(
return nullptr;
}
- Maybe<RefPtr<MediaByteBuffer>> description;
+ RefPtr<MediaByteBuffer> description;
if (aConfig.mDescription.WasPassed()) {
auto rv = GetExtraDataFromArrayBuffer(aConfig.mDescription.Value());
if (rv.isErr()) { // Invalid description data.
@@ -95,12 +95,28 @@ UniquePtr<AudioDecoderConfigInternal> AudioDecoderConfigInternal::Create(
error.get());
return nullptr;
}
- description.emplace(rv.unwrap());
+ description = rv.unwrap();
}
return UniquePtr<AudioDecoderConfigInternal>(new AudioDecoderConfigInternal(
aConfig.mCodec, aConfig.mSampleRate, aConfig.mNumberOfChannels,
- std::move(description)));
+ description.forget()));
+}
+
+nsCString AudioDecoderConfigInternal::ToString() const {
+ nsCString rv;
+
+ rv.AppendLiteral("AudioDecoderConfigInternal: ");
+ rv.AppendPrintf("%s %" PRIu32 "Hz %" PRIu32 " ch",
+ NS_ConvertUTF16toUTF8(mCodec).get(), mSampleRate,
+ mNumberOfChannels);
+ if (mDescription) {
+ rv.AppendPrintf("(%zu bytes of extradata)", mDescription->Length());
+ } else {
+ rv.AppendLiteral("(no extradata)");
+ }
+
+ return rv;
}
/*
@@ -118,24 +134,6 @@ struct AudioMIMECreateParam {
// Map between WebCodecs pcm types as strings and codec numbers
// All other codecs
-nsCString ConvertCodecName(const nsCString& aContainer,
- const nsCString& aCodec) {
- if (!aContainer.EqualsLiteral("x-wav")) {
- return aCodec;
- }
- if (aCodec.EqualsLiteral("ulaw")) {
- return nsCString("7");
- }
- if (aCodec.EqualsLiteral("alaw")) {
- return nsCString("6");
- }
- if (aCodec.Find("f32")) {
- return nsCString("3");
- }
- // Linear PCM
- return nsCString("1");
-}
-
static nsTArray<nsCString> GuessMIMETypes(const AudioMIMECreateParam& aParam) {
nsCString codec = NS_ConvertUTF16toUTF8(aParam.mParsedCodec);
nsTArray<nsCString> types;
@@ -147,16 +145,6 @@ static nsTArray<nsCString> GuessMIMETypes(const AudioMIMECreateParam& aParam) {
return types;
}
-static bool IsSupportedAudioCodec(const nsAString& aCodec) {
- LOG("IsSupportedAudioCodec: %s", NS_ConvertUTF16toUTF8(aCodec).get());
- return aCodec.EqualsLiteral("flac") || aCodec.EqualsLiteral("mp3") ||
- IsAACCodecString(aCodec) || aCodec.EqualsLiteral("opus") ||
- aCodec.EqualsLiteral("ulaw") || aCodec.EqualsLiteral("alaw") ||
- aCodec.EqualsLiteral("pcm-u8") || aCodec.EqualsLiteral("pcm-s16") ||
- aCodec.EqualsLiteral("pcm-s24") || aCodec.EqualsLiteral("pcm-s32") ||
- aCodec.EqualsLiteral("pcm-f32");
-}
-
// https://w3c.github.io/webcodecs/#check-configuration-support
template <typename Config>
static bool CanDecodeAudio(const Config& aConfig) {
@@ -259,13 +247,12 @@ Result<UniquePtr<TrackInfo>, nsresult> AudioDecoderTraits::CreateTrackInfo(
return Err(NS_ERROR_INVALID_ARG);
}
- if (aConfig.mDescription.isSome()) {
- RefPtr<MediaByteBuffer> buf;
- buf = aConfig.mDescription.value();
- if (buf) {
- LOG("The given config has %zu bytes of description data", buf->Length());
- ai->mCodecSpecificConfig =
- AudioCodecSpecificVariant{AudioCodecSpecificBinaryBlob{buf}};
+ if (aConfig.mDescription) {
+ if (!aConfig.mDescription->IsEmpty()) {
+ LOG("The given config has %zu bytes of description data",
+ aConfig.mDescription->Length());
+ ai->mCodecSpecificConfig = AudioCodecSpecificVariant{
+ AudioCodecSpecificBinaryBlob{aConfig.mDescription}};
}
}
@@ -275,7 +262,7 @@ Result<UniquePtr<TrackInfo>, nsresult> AudioDecoderTraits::CreateTrackInfo(
LOG("Created AudioInfo %s (%" PRIu32 "ch %" PRIu32
"Hz - with extra-data: %s)",
NS_ConvertUTF16toUTF8(aConfig.mCodec).get(), ai->mChannels, ai->mChannels,
- aConfig.mDescription.isSome() ? "yes" : "no");
+ aConfig.mDescription && !aConfig.mDescription->IsEmpty() ? "yes" : "no");
return track;
}
diff --git a/dom/media/webcodecs/AudioEncoder.cpp b/dom/media/webcodecs/AudioEncoder.cpp
new file mode 100644
index 0000000000..7204a13200
--- /dev/null
+++ b/dom/media/webcodecs/AudioEncoder.cpp
@@ -0,0 +1,488 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/dom/AudioEncoder.h"
+#include "EncoderTraits.h"
+#include "mozilla/dom/AudioEncoderBinding.h"
+
+#include "EncoderConfig.h"
+#include "EncoderTypes.h"
+#include "MediaData.h"
+#include "mozilla/Assertions.h"
+#include "mozilla/Logging.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/dom/AudioDataBinding.h"
+#include "mozilla/dom/EncodedAudioChunk.h"
+#include "mozilla/dom/EncodedAudioChunkBinding.h"
+#include "mozilla/dom/Promise.h"
+#include "mozilla/dom/WebCodecsUtils.h"
+#include "EncoderConfig.h"
+
+extern mozilla::LazyLogModule gWebCodecsLog;
+
+namespace mozilla::dom {
+
+#ifdef LOG_INTERNAL
+# undef LOG_INTERNAL
+#endif // LOG_INTERNAL
+#define LOG_INTERNAL(level, msg, ...) \
+ MOZ_LOG(gWebCodecsLog, LogLevel::level, (msg, ##__VA_ARGS__))
+
+#ifdef LOG
+# undef LOG
+#endif // LOG
+#define LOG(msg, ...) LOG_INTERNAL(Debug, msg, ##__VA_ARGS__)
+
+#ifdef LOGW
+# undef LOGW
+#endif // LOGW
+#define LOGW(msg, ...) LOG_INTERNAL(Warning, msg, ##__VA_ARGS__)
+
+#ifdef LOGE
+# undef LOGE
+#endif // LOGE
+#define LOGE(msg, ...) LOG_INTERNAL(Error, msg, ##__VA_ARGS__)
+
+#ifdef LOGV
+# undef LOGV
+#endif // LOGV
+#define LOGV(msg, ...) LOG_INTERNAL(Verbose, msg, ##__VA_ARGS__)
+
+NS_IMPL_CYCLE_COLLECTION_INHERITED(AudioEncoder, DOMEventTargetHelper,
+ mErrorCallback, mOutputCallback)
+NS_IMPL_ADDREF_INHERITED(AudioEncoder, DOMEventTargetHelper)
+NS_IMPL_RELEASE_INHERITED(AudioEncoder, DOMEventTargetHelper)
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(AudioEncoder)
+NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper)
+
+/*
+ * Below are helper classes
+ */
+AudioEncoderConfigInternal::AudioEncoderConfigInternal(
+ const nsAString& aCodec, Maybe<uint32_t> aSampleRate,
+ Maybe<uint32_t> aNumberOfChannels, Maybe<uint32_t> aBitrate,
+ BitrateMode aBitrateMode)
+ : mCodec(aCodec),
+ mSampleRate(aSampleRate),
+ mNumberOfChannels(aNumberOfChannels),
+ mBitrate(aBitrate),
+ mBitrateMode(aBitrateMode) {}
+
+AudioEncoderConfigInternal::AudioEncoderConfigInternal(
+ const AudioEncoderConfig& aConfig)
+ : AudioEncoderConfigInternal(
+ aConfig.mCodec, OptionalToMaybe(aConfig.mSampleRate),
+ OptionalToMaybe(aConfig.mNumberOfChannels),
+ OptionalToMaybe(aConfig.mBitrate), aConfig.mBitrateMode) {
+ DebugOnly<nsCString> errorMessage;
+ if (aConfig.mCodec.EqualsLiteral("opus") && aConfig.mOpus.WasPassed()) {
+ // All values are in range at this point, the config is known valid.
+ OpusSpecific specific;
+ if (aConfig.mOpus.Value().mComplexity.WasPassed()) {
+ specific.mComplexity = aConfig.mOpus.Value().mComplexity.Value();
+ } else {
+ // https://w3c.github.io/webcodecs/opus_codec_registration.html#dom-opusencoderconfig-complexity
+ // If no value is specificied, the default value is platform-specific:
+ // User Agents SHOULD set a default of 5 for mobile platforms, and a
+ // default of 9 for all other platforms.
+ if (IsOnAndroid()) {
+ specific.mComplexity = 5;
+ } else {
+ specific.mComplexity = 9;
+ }
+ }
+ specific.mApplication = OpusSpecific::Application::Unspecified;
+ specific.mFrameDuration = aConfig.mOpus.Value().mFrameDuration;
+ specific.mPacketLossPerc = aConfig.mOpus.Value().mPacketlossperc;
+ specific.mUseDTX = aConfig.mOpus.Value().mUsedtx;
+ specific.mUseInBandFEC = aConfig.mOpus.Value().mUseinbandfec;
+ mSpecific.emplace(specific);
+ }
+ MOZ_ASSERT(AudioEncoderTraits::Validate(aConfig, errorMessage));
+}
+
+AudioEncoderConfigInternal::AudioEncoderConfigInternal(
+ const AudioEncoderConfigInternal& aConfig)
+ : AudioEncoderConfigInternal(aConfig.mCodec, aConfig.mSampleRate,
+ aConfig.mNumberOfChannels, aConfig.mBitrate,
+ aConfig.mBitrateMode) {}
+
+void AudioEncoderConfigInternal::SetSpecific(
+ const EncoderConfig::CodecSpecific& aSpecific) {
+ mSpecific.emplace(aSpecific);
+}
+
+/*
+ * The followings are helpers for AudioEncoder methods
+ */
+
+static void CloneConfiguration(RootedDictionary<AudioEncoderConfig>& aDest,
+ JSContext* aCx,
+ const AudioEncoderConfig& aConfig) {
+ aDest.mCodec = aConfig.mCodec;
+
+ if (aConfig.mNumberOfChannels.WasPassed()) {
+ aDest.mNumberOfChannels.Construct(aConfig.mNumberOfChannels.Value());
+ }
+ if (aConfig.mSampleRate.WasPassed()) {
+ aDest.mSampleRate.Construct(aConfig.mSampleRate.Value());
+ }
+ if (aConfig.mBitrate.WasPassed()) {
+ aDest.mBitrate.Construct(aConfig.mBitrate.Value());
+ }
+ if (aConfig.mOpus.WasPassed()) {
+ aDest.mOpus.Construct(aConfig.mOpus.Value());
+ // Handle the default value manually since it's different on mobile
+ if (!aConfig.mOpus.Value().mComplexity.WasPassed()) {
+ if (IsOnAndroid()) {
+ aDest.mOpus.Value().mComplexity.Construct(5);
+ } else {
+ aDest.mOpus.Value().mComplexity.Construct(9);
+ }
+ }
+ }
+ aDest.mBitrateMode = aConfig.mBitrateMode;
+}
+
+static bool IsAudioEncodeSupported(const nsAString& aCodec) {
+ LOG("IsEncodeSupported: %s", NS_ConvertUTF16toUTF8(aCodec).get());
+
+ return aCodec.EqualsLiteral("opus") || aCodec.EqualsLiteral("vorbis");
+}
+
+static bool CanEncode(const RefPtr<AudioEncoderConfigInternal>& aConfig,
+ nsCString& aErrorMessage) {
+ auto parsedCodecString =
+ ParseCodecString(aConfig->mCodec).valueOr(EmptyString());
+ // TODO: Enable WebCodecs on Android (Bug 1840508)
+ if (IsOnAndroid()) {
+ return false;
+ }
+ if (!IsAudioEncodeSupported(parsedCodecString)) {
+ return false;
+ }
+
+ if (aConfig->mNumberOfChannels.value() > 256) {
+ aErrorMessage.AppendPrintf(
+ "Invalid number of channels, supported range is between 1 and 256");
+ return false;
+ }
+
+ // Somewhat arbitrarily chosen, but reflects real-life and what the rest of
+ // Gecko does.
+ if (aConfig->mSampleRate.value() < 3000 ||
+ aConfig->mSampleRate.value() > 384000) {
+ aErrorMessage.AppendPrintf(
+ "Invalid sample-rate of %d, supported range is 3000Hz to 384000Hz",
+ aConfig->mSampleRate.value());
+ return false;
+ }
+
+ return EncoderSupport::Supports(aConfig);
+}
+
+nsCString AudioEncoderConfigInternal::ToString() const {
+ nsCString rv;
+
+ rv.AppendPrintf("AudioEncoderConfigInternal: %s",
+ NS_ConvertUTF16toUTF8(mCodec).get());
+ if (mSampleRate) {
+ rv.AppendPrintf(" %" PRIu32 "Hz", mSampleRate.value());
+ }
+ if (mNumberOfChannels) {
+ rv.AppendPrintf(" %" PRIu32 "ch", mNumberOfChannels.value());
+ }
+ if (mBitrate) {
+ rv.AppendPrintf(" %" PRIu32 "bps", mBitrate.value());
+ }
+ rv.AppendPrintf(" (%s)", mBitrateMode == mozilla::dom::BitrateMode::Constant
+ ? "CRB"
+ : "VBR");
+
+ return rv;
+}
+
+EncoderConfig AudioEncoderConfigInternal::ToEncoderConfig() const {
+ const mozilla::BitrateMode bitrateMode =
+ mBitrateMode == mozilla::dom::BitrateMode::Constant
+ ? mozilla::BitrateMode::Constant
+ : mozilla::BitrateMode::Variable;
+
+ CodecType type = CodecType::Opus;
+ Maybe<EncoderConfig::CodecSpecific> specific;
+ if (mCodec.EqualsLiteral("opus")) {
+ type = CodecType::Opus;
+ MOZ_ASSERT(mSpecific.isNothing() || mSpecific->is<OpusSpecific>());
+ specific = mSpecific;
+ } else if (mCodec.EqualsLiteral("vorbis")) {
+ type = CodecType::Vorbis;
+ } else if (mCodec.EqualsLiteral("flac")) {
+ type = CodecType::Flac;
+ } else if (StringBeginsWith(mCodec, u"pcm-"_ns)) {
+ type = CodecType::PCM;
+ } else if (mCodec.EqualsLiteral("ulaw")) {
+ type = CodecType::PCM;
+ } else if (mCodec.EqualsLiteral("alaw")) {
+ type = CodecType::PCM;
+ } else if (StringBeginsWith(mCodec, u"mp4a."_ns)) {
+ type = CodecType::AAC;
+ }
+
+ // This should have been checked ahead of time -- we can't encode without
+ // knowing the sample-rate or the channel count at the very least.
+ MOZ_ASSERT(mSampleRate.value());
+ MOZ_ASSERT(mNumberOfChannels.value());
+
+ return EncoderConfig(type, mNumberOfChannels.value(), bitrateMode,
+ AssertedCast<uint32_t>(mSampleRate.value()),
+ mBitrate.valueOr(0), specific);
+}
+
+bool AudioEncoderConfigInternal::Equals(
+ const AudioEncoderConfigInternal& aOther) const {
+ return false;
+}
+
+bool AudioEncoderConfigInternal::CanReconfigure(
+ const AudioEncoderConfigInternal& aOther) const {
+ return false;
+}
+
+already_AddRefed<WebCodecsConfigurationChangeList>
+AudioEncoderConfigInternal::Diff(
+ const AudioEncoderConfigInternal& aOther) const {
+ return MakeRefPtr<WebCodecsConfigurationChangeList>().forget();
+}
+
+/* static */
+bool AudioEncoderTraits::IsSupported(
+ const AudioEncoderConfigInternal& aConfig) {
+ nsCString errorMessage;
+ bool canEncode =
+ CanEncode(MakeRefPtr<AudioEncoderConfigInternal>(aConfig), errorMessage);
+ if (!canEncode) {
+ LOGE("Can't encode configuration %s: %s", aConfig.ToString().get(),
+ errorMessage.get());
+ }
+ return canEncode;
+}
+
+// https://w3c.github.io/webcodecs/#valid-audioencoderconfig
+/* static */
+bool AudioEncoderTraits::Validate(const AudioEncoderConfig& aConfig,
+ nsCString& aErrorMessage) {
+ Maybe<nsString> codec = ParseCodecString(aConfig.mCodec);
+ if (!codec || codec->IsEmpty()) {
+ LOGE("Validating AudioEncoderConfig: invalid codec string");
+ return false;
+ }
+
+ if (!aConfig.mNumberOfChannels.WasPassed()) {
+ aErrorMessage.AppendPrintf("Channel count required");
+ return false;
+ }
+ if (aConfig.mNumberOfChannels.Value() == 0) {
+ aErrorMessage.AppendPrintf(
+ "Invalid number of channels, supported range is between 1 and 256");
+ return false;
+ }
+ if (!aConfig.mSampleRate.WasPassed()) {
+ aErrorMessage.AppendPrintf("Sample-rate required");
+ return false;
+ }
+ if (aConfig.mSampleRate.Value() == 0) {
+ aErrorMessage.AppendPrintf("Invalid sample-rate of 0");
+ return false;
+ }
+
+ if (aConfig.mBitrate.WasPassed() &&
+ aConfig.mBitrate.Value() > std::numeric_limits<int>::max()) {
+ aErrorMessage.AppendPrintf("Invalid config: bitrate value too large");
+ return false;
+ }
+
+ if (codec->EqualsLiteral("opus")) {
+ // This comes from
+ // https://w3c.github.io/webcodecs/opus_codec_registration.html#opus-encoder-config
+ if (aConfig.mBitrate.WasPassed() && (aConfig.mBitrate.Value() < 6000 ||
+ aConfig.mBitrate.Value() > 510000)) {
+ aErrorMessage.AppendPrintf(
+ "Invalid config: bitrate value outside of [6k, 510k] for opus");
+ return false;
+ }
+ if (aConfig.mOpus.WasPassed()) {
+ // Verify value ranges
+ const std::array validFrameDurationUs = {2500, 5000, 10000,
+ 20000, 40000, 60000};
+ if (std::find(validFrameDurationUs.begin(), validFrameDurationUs.end(),
+ aConfig.mOpus.Value().mFrameDuration) ==
+ validFrameDurationUs.end()) {
+ aErrorMessage.AppendPrintf("Invalid config: invalid frame duration");
+ return false;
+ }
+ if (aConfig.mOpus.Value().mComplexity.WasPassed() &&
+ aConfig.mOpus.Value().mComplexity.Value() > 10) {
+ aErrorMessage.AppendPrintf(
+ "Invalid config: Opus complexity greater than 10");
+ return false;
+ }
+ if (aConfig.mOpus.Value().mPacketlossperc > 100) {
+ aErrorMessage.AppendPrintf(
+ "Invalid config: Opus packet loss percentage greater than 100");
+ return false;
+ }
+ }
+ }
+
+ return true;
+}
+
+/* static */
+RefPtr<AudioEncoderConfigInternal> AudioEncoderTraits::CreateConfigInternal(
+ const AudioEncoderConfig& aConfig) {
+ nsCString errorMessage;
+ if (!AudioEncoderTraits::Validate(aConfig, errorMessage)) {
+ return nullptr;
+ }
+ return MakeRefPtr<AudioEncoderConfigInternal>(aConfig);
+}
+
+/* static */
+RefPtr<mozilla::AudioData> AudioEncoderTraits::CreateInputInternal(
+ const dom::AudioData& aInput,
+ const dom::VideoEncoderEncodeOptions& /* unused */) {
+ return aInput.ToAudioData();
+}
+
+/*
+ * Below are AudioEncoder implementation
+ */
+
+AudioEncoder::AudioEncoder(
+ nsIGlobalObject* aParent, RefPtr<WebCodecsErrorCallback>&& aErrorCallback,
+ RefPtr<EncodedAudioChunkOutputCallback>&& aOutputCallback)
+ : EncoderTemplate(aParent, std::move(aErrorCallback),
+ std::move(aOutputCallback)) {
+ MOZ_ASSERT(mErrorCallback);
+ MOZ_ASSERT(mOutputCallback);
+ LOG("AudioEncoder %p ctor", this);
+}
+
+AudioEncoder::~AudioEncoder() {
+ LOG("AudioEncoder %p dtor", this);
+ Unused << ResetInternal(NS_ERROR_DOM_ABORT_ERR);
+}
+
+JSObject* AudioEncoder::WrapObject(JSContext* aCx,
+ JS::Handle<JSObject*> aGivenProto) {
+ AssertIsOnOwningThread();
+
+ return AudioEncoder_Binding::Wrap(aCx, this, aGivenProto);
+}
+
+// https://w3c.github.io/webcodecs/#dom-audioencoder-audioencoder
+/* static */
+already_AddRefed<AudioEncoder> AudioEncoder::Constructor(
+ const GlobalObject& aGlobal, const AudioEncoderInit& aInit,
+ ErrorResult& aRv) {
+ nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+ if (!global) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return nullptr;
+ }
+
+ return MakeAndAddRef<AudioEncoder>(
+ global.get(), RefPtr<WebCodecsErrorCallback>(aInit.mError),
+ RefPtr<EncodedAudioChunkOutputCallback>(aInit.mOutput));
+}
+
+// https://w3c.github.io/webcodecs/#dom-audioencoder-isconfigsupported
+/* static */
+already_AddRefed<Promise> AudioEncoder::IsConfigSupported(
+ const GlobalObject& aGlobal, const AudioEncoderConfig& aConfig,
+ ErrorResult& aRv) {
+ LOG("AudioEncoder::IsConfigSupported, config: %s",
+ NS_ConvertUTF16toUTF8(aConfig.mCodec).get());
+
+ nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports());
+ if (!global) {
+ aRv.Throw(NS_ERROR_FAILURE);
+ return nullptr;
+ }
+
+ RefPtr<Promise> p = Promise::Create(global.get(), aRv);
+ if (NS_WARN_IF(aRv.Failed())) {
+ return p.forget();
+ }
+
+ nsCString errorMessage;
+ if (!AudioEncoderTraits::Validate(aConfig, errorMessage)) {
+ p->MaybeRejectWithTypeError(errorMessage);
+ return p.forget();
+ }
+
+ // TODO: Move the following works to another thread to unblock the current
+ // thread, as what spec suggests.
+
+ RootedDictionary<AudioEncoderConfig> config(aGlobal.Context());
+ CloneConfiguration(config, aGlobal.Context(), aConfig);
+
+ bool supportedAudioCodec = IsSupportedAudioCodec(aConfig.mCodec);
+ auto configInternal = MakeRefPtr<AudioEncoderConfigInternal>(aConfig);
+ bool canEncode = CanEncode(configInternal, errorMessage);
+ if (!canEncode) {
+ LOG("CanEncode failed: %s", errorMessage.get());
+ }
+ RootedDictionary<AudioEncoderSupport> s(aGlobal.Context());
+ s.mConfig.Construct(std::move(config));
+ s.mSupported.Construct(supportedAudioCodec && canEncode);
+
+ p->MaybeResolve(s);
+ return p.forget();
+}
+
+RefPtr<EncodedAudioChunk> AudioEncoder::EncodedDataToOutputType(
+ nsIGlobalObject* aGlobalObject, const RefPtr<MediaRawData>& aData) {
+ AssertIsOnOwningThread();
+
+ // Package into an EncodedAudioChunk
+ auto buffer =
+ MakeRefPtr<MediaAlignedByteBuffer>(aData->Data(), aData->Size());
+ auto encoded = MakeRefPtr<EncodedAudioChunk>(
+ aGlobalObject, buffer.forget(), EncodedAudioChunkType::Key,
+ aData->mTime.ToMicroseconds(),
+ aData->mDuration.IsZero() ? Nothing()
+ : Some(aData->mDuration.ToMicroseconds()));
+ return encoded;
+}
+
+AudioDecoderConfigInternal AudioEncoder::EncoderConfigToDecoderConfig(
+ nsIGlobalObject* aGlobal, const RefPtr<MediaRawData>& aRawData,
+ const AudioEncoderConfigInternal& aOutputConfig) const {
+ MOZ_ASSERT(aOutputConfig.mSampleRate.isSome());
+ MOZ_ASSERT(aOutputConfig.mNumberOfChannels.isSome());
+ uint32_t sampleRate = aOutputConfig.mSampleRate.value();
+ uint32_t channelCount = aOutputConfig.mNumberOfChannels.value();
+ // Check if the encoder had to modify the settings because of codec
+ // constraints. e.g. FFmpegAudioEncoder can encode any sample-rate, but if the
+ // codec is Opus, then it will resample the audio one of the specific rates
+ // supported by the encoder.
+ if (aRawData->mConfig) {
+ sampleRate = aRawData->mConfig->mSampleRate;
+ channelCount = aRawData->mConfig->mNumberOfChannels;
+ }
+ return AudioDecoderConfigInternal(aOutputConfig.mCodec, sampleRate,
+ channelCount,
+ do_AddRef(aRawData->mExtraData));
+}
+
+#undef LOG
+#undef LOGW
+#undef LOGE
+#undef LOGV
+#undef LOG_INTERNAL
+
+} // namespace mozilla::dom
diff --git a/dom/media/webcodecs/AudioEncoder.h b/dom/media/webcodecs/AudioEncoder.h
new file mode 100644
index 0000000000..0df6cd23d6
--- /dev/null
+++ b/dom/media/webcodecs/AudioEncoder.h
@@ -0,0 +1,76 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_AudioEncoder_h
+#define mozilla_dom_AudioEncoder_h
+
+#include "js/TypeDecls.h"
+#include "mozilla/ErrorResult.h"
+#include "mozilla/RefPtr.h"
+#include "mozilla/UniquePtr.h"
+#include "mozilla/dom/BindingDeclarations.h"
+#include "mozilla/dom/EncoderTemplate.h"
+#include "mozilla/dom/AudioData.h"
+#include "nsCycleCollectionParticipant.h"
+#include "EncoderTypes.h"
+#include "EncoderAgent.h"
+
+class nsIGlobalObject;
+
+namespace mozilla::dom {
+
+class AudioDataOutputCallback;
+class EncodedAudioChunk;
+class EncodedAudioChunkData;
+class EventHandlerNonNull;
+class GlobalObject;
+class Promise;
+class WebCodecsErrorCallback;
+struct AudioEncoderConfig;
+struct AudioEncoderInit;
+
+} // namespace mozilla::dom
+
+namespace mozilla::dom {
+
+class AudioEncoder final : public EncoderTemplate<AudioEncoderTraits> {
+ public:
+ NS_DECL_ISUPPORTS_INHERITED
+ NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(AudioEncoder, DOMEventTargetHelper)
+
+ public:
+ AudioEncoder(nsIGlobalObject* aParent,
+ RefPtr<WebCodecsErrorCallback>&& aErrorCallback,
+ RefPtr<EncodedAudioChunkOutputCallback>&& aOutputCallback);
+
+ protected:
+ ~AudioEncoder();
+
+ public:
+ JSObject* WrapObject(JSContext* aCx,
+ JS::Handle<JSObject*> aGivenProto) override;
+
+ static already_AddRefed<AudioEncoder> Constructor(
+ const GlobalObject& aGlobal, const AudioEncoderInit& aInit,
+ ErrorResult& aRv);
+
+ static already_AddRefed<Promise> IsConfigSupported(
+ const GlobalObject& aGlobal, const AudioEncoderConfig& aConfig,
+ ErrorResult& aRv);
+
+ protected:
+ virtual RefPtr<EncodedAudioChunk> EncodedDataToOutputType(
+ nsIGlobalObject* aGlobalObject,
+ const RefPtr<MediaRawData>& aData) override;
+
+ virtual AudioDecoderConfigInternal EncoderConfigToDecoderConfig(
+ nsIGlobalObject* aGlobal, const RefPtr<MediaRawData>& aRawData,
+ const AudioEncoderConfigInternal& aOutputConfig) const override;
+};
+
+} // namespace mozilla::dom
+
+#endif // mozilla_dom_AudioEncoder_h
diff --git a/dom/media/webcodecs/DecoderTemplate.cpp b/dom/media/webcodecs/DecoderTemplate.cpp
index 4d1c310737..2fc2471a24 100644
--- a/dom/media/webcodecs/DecoderTemplate.cpp
+++ b/dom/media/webcodecs/DecoderTemplate.cpp
@@ -296,8 +296,7 @@ void DecoderTemplate<DecoderType>::CloseInternal(const nsresult& aResult) {
if (r.isErr()) {
nsCString name;
GetErrorName(r.unwrapErr(), name);
- LOGE("Error in ResetInternal: %s", name.get());
- MOZ_CRASH();
+ LOGE("Error in ResetInternal during CloseInternal: %s", name.get());
}
mState = CodecState::Closed;
nsCString error;
@@ -473,7 +472,6 @@ MessageProcessedResult DecoderTemplate<DecoderType>::ProcessConfigureMessage(
mProcessingMessage.reset();
QueueATask("Error while configuring decoder",
[self = RefPtr{this}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY {
- MOZ_ASSERT(self->mState != CodecState::Closed);
self->CloseInternal(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
});
return MessageProcessedResult::Processed;
diff --git a/dom/media/webcodecs/DecoderTypes.h b/dom/media/webcodecs/DecoderTypes.h
index 339a164f70..4817a66f17 100644
--- a/dom/media/webcodecs/DecoderTypes.h
+++ b/dom/media/webcodecs/DecoderTypes.h
@@ -50,22 +50,22 @@ class VideoDecoderConfigInternal {
Maybe<uint32_t>&& aCodedHeight,
Maybe<uint32_t>&& aCodedWidth,
Maybe<VideoColorSpaceInternal>&& aColorSpace,
- Maybe<RefPtr<MediaByteBuffer>>&& aDescription,
+ already_AddRefed<MediaByteBuffer> aDescription,
Maybe<uint32_t>&& aDisplayAspectHeight,
Maybe<uint32_t>&& aDisplayAspectWidth,
const HardwareAcceleration& aHardwareAcceleration,
Maybe<bool>&& aOptimizeForLatency);
~VideoDecoderConfigInternal() = default;
- nsString ToString() const;
+ nsCString ToString() const;
bool Equals(const VideoDecoderConfigInternal& aOther) const {
- if (mDescription.isSome() != aOther.mDescription.isSome()) {
+ if (mDescription != aOther.mDescription) {
return false;
}
- if (mDescription.isSome() && aOther.mDescription.isSome()) {
- auto lhs = mDescription.value();
- auto rhs = aOther.mDescription.value();
+ if (mDescription && aOther.mDescription) {
+ auto lhs = mDescription;
+ auto rhs = aOther.mDescription;
if (lhs->Length() != rhs->Length()) {
return false;
}
@@ -86,7 +86,7 @@ class VideoDecoderConfigInternal {
Maybe<uint32_t> mCodedHeight;
Maybe<uint32_t> mCodedWidth;
Maybe<VideoColorSpaceInternal> mColorSpace;
- Maybe<RefPtr<MediaByteBuffer>> mDescription;
+ RefPtr<MediaByteBuffer> mDescription;
Maybe<uint32_t> mDisplayAspectHeight;
Maybe<uint32_t> mDisplayAspectWidth;
HardwareAcceleration mHardwareAcceleration;
@@ -116,24 +116,42 @@ class VideoDecoderTraits {
class AudioDecoderConfigInternal {
public:
+ AudioDecoderConfigInternal(const nsAString& aCodec, uint32_t aSampleRate,
+ uint32_t aNumberOfChannels,
+ already_AddRefed<MediaByteBuffer> aDescription);
static UniquePtr<AudioDecoderConfigInternal> Create(
const AudioDecoderConfig& aConfig);
~AudioDecoderConfigInternal() = default;
+ bool Equals(const AudioDecoderConfigInternal& aOther) const {
+ if (mDescription != aOther.mDescription) {
+ return false;
+ }
+ if (mDescription && aOther.mDescription) {
+ auto lhs = mDescription;
+ auto rhs = aOther.mDescription;
+ if (lhs->Length() != rhs->Length()) {
+ return false;
+ }
+ if (!ArrayEqual(lhs->Elements(), rhs->Elements(), lhs->Length())) {
+ return false;
+ }
+ }
+ return mCodec.Equals(aOther.mCodec) && mSampleRate == aOther.mSampleRate &&
+ mNumberOfChannels == aOther.mNumberOfChannels &&
+ mOptimizeForLatency == aOther.mOptimizeForLatency;
+ }
+ nsCString ToString() const;
+
nsString mCodec;
uint32_t mSampleRate;
uint32_t mNumberOfChannels;
- Maybe<RefPtr<MediaByteBuffer>> mDescription;
+ RefPtr<MediaByteBuffer> mDescription;
// Compilation fix, should be abstracted by DecoderAgent since those are not
// supported
HardwareAcceleration mHardwareAcceleration =
HardwareAcceleration::No_preference;
Maybe<bool> mOptimizeForLatency;
-
- private:
- AudioDecoderConfigInternal(const nsAString& aCodec, uint32_t aSampleRate,
- uint32_t aNumberOfChannels,
- Maybe<RefPtr<MediaByteBuffer>>&& aDescription);
};
class AudioDecoderTraits {
diff --git a/dom/media/webcodecs/EncoderTemplate.cpp b/dom/media/webcodecs/EncoderTemplate.cpp
index 35c8feb3f8..34edfae822 100644
--- a/dom/media/webcodecs/EncoderTemplate.cpp
+++ b/dom/media/webcodecs/EncoderTemplate.cpp
@@ -127,7 +127,7 @@ void EncoderTemplate<EncoderType>::Configure(const ConfigType& aConfig,
RefPtr<ConfigTypeInternal> config =
EncoderType::CreateConfigInternal(aConfig);
if (!config) {
- aRv.Throw(NS_ERROR_UNEXPECTED); // Invalid description data.
+ CloseInternal(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
return;
}
@@ -237,9 +237,9 @@ template <typename EncoderType>
void EncoderTemplate<EncoderType>::Close(ErrorResult& aRv) {
AssertIsOnOwningThread();
- LOG("%s::Close %p", EncoderType::Name.get(), this);
+ LOG("%s %p, Close", EncoderType::Name.get(), this);
- if (auto r = CloseInternal(NS_ERROR_DOM_ABORT_ERR); r.isErr()) {
+ if (auto r = CloseInternalWithAbort(); r.isErr()) {
aRv.Throw(r.unwrapErr());
}
}
@@ -273,23 +273,33 @@ Result<Ok, nsresult> EncoderTemplate<EncoderType>::ResetInternal(
}
template <typename EncoderType>
-Result<Ok, nsresult> EncoderTemplate<EncoderType>::CloseInternal(
- const nsresult& aResult) {
+Result<Ok, nsresult> EncoderTemplate<EncoderType>::CloseInternalWithAbort() {
AssertIsOnOwningThread();
- MOZ_TRY(ResetInternal(aResult));
+ MOZ_TRY(ResetInternal(NS_ERROR_DOM_ABORT_ERR));
mState = CodecState::Closed;
- if (aResult != NS_ERROR_DOM_ABORT_ERR) {
- nsCString error;
- GetErrorName(aResult, error);
- LOGE("%s %p Close on error: %s", EncoderType::Name.get(), this,
- error.get());
- ReportError(aResult);
- }
return Ok();
}
template <typename EncoderType>
+void EncoderTemplate<EncoderType>::CloseInternal(const nsresult& aResult) {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(aResult != NS_ERROR_DOM_ABORT_ERR, "Use CloseInternalWithAbort");
+
+ auto r = ResetInternal(aResult);
+ if (r.isErr()) {
+ nsCString name;
+ GetErrorName(r.unwrapErr(), name);
+ LOGE("Error during ResetInternal during CloseInternal: %s", name.get());
+ }
+ mState = CodecState::Closed;
+ nsCString error;
+ GetErrorName(aResult, error);
+ LOGE("%s %p Close on error: %s", EncoderType::Name.get(), this, error.get());
+ ReportError(aResult);
+}
+
+template <typename EncoderType>
void EncoderTemplate<EncoderType>::ReportError(const nsresult& aResult) {
AssertIsOnOwningThread();
@@ -299,8 +309,28 @@ void EncoderTemplate<EncoderType>::ReportError(const nsresult& aResult) {
}
template <typename EncoderType>
-void EncoderTemplate<EncoderType>::OutputEncodedData(
- nsTArray<RefPtr<MediaRawData>>&& aData) {
+template <typename T, typename U>
+void EncoderTemplate<EncoderType>::CopyExtradataToDescriptionIfNeeded(
+ nsIGlobalObject* aGlobal, const T& aConfigInternal, U& aConfig) {
+ if (aConfigInternal.mDescription &&
+ !aConfigInternal.mDescription->IsEmpty()) {
+ auto& abov = aConfig.mDescription.Construct();
+ AutoEntryScript aes(aGlobal, "EncoderConfigToaConfigConfig");
+ size_t lengthBytes = aConfigInternal.mDescription->Length();
+ UniquePtr<uint8_t[], JS::FreePolicy> extradata(new uint8_t[lengthBytes]);
+ PodCopy(extradata.get(), aConfigInternal.mDescription->Elements(),
+ lengthBytes);
+ JS::Rooted<JSObject*> description(
+ aes.cx(), JS::NewArrayBufferWithContents(aes.cx(), lengthBytes,
+ std::move(extradata)));
+ JS::Rooted<JS::Value> value(aes.cx(), JS::ObjectValue(*description));
+ DebugOnly<bool> rv = abov.Init(aes.cx(), value);
+ }
+}
+
+template <>
+void EncoderTemplate<VideoEncoderTraits>::OutputEncodedVideoData(
+ const nsTArray<RefPtr<MediaRawData>>&& aData) {
AssertIsOnOwningThread();
MOZ_ASSERT(mState == CodecState::Configured);
MOZ_ASSERT(mActiveConfig);
@@ -313,7 +343,7 @@ void EncoderTemplate<EncoderType>::OutputEncodedData(
jsapi.Init(GetParentObject()); // TODO: check returned value?
JSContext* cx = jsapi.cx();
- RefPtr<typename EncoderType::OutputCallbackType> cb(mOutputCallback);
+ RefPtr<EncodedVideoChunkOutputCallback> cb(mOutputCallback);
for (auto& data : aData) {
// It's possible to have reset() called in between this task having been
// dispatched, and running -- no output callback should happen when that's
@@ -323,10 +353,10 @@ void EncoderTemplate<EncoderType>::OutputEncodedData(
if (!mActiveConfig) {
return;
}
- RefPtr<typename EncoderType::OutputType> encodedData =
+ RefPtr<EncodedVideoChunk> encodedData =
EncodedDataToOutputType(GetParentObject(), data);
- RootedDictionary<typename EncoderType::MetadataType> metadata(cx);
+ RootedDictionary<EncodedVideoChunkMetadata> metadata(cx);
if (mOutputNewDecoderConfig) {
VideoDecoderConfigInternal decoderConfigInternal =
EncoderConfigToDecoderConfig(GetParentObject(), data, *mActiveConfig);
@@ -354,23 +384,10 @@ void EncoderTemplate<EncoderType>::OutputEncodedData(
MaybeToNullable(decoderConfigInternal.mColorSpace->mTransfer);
decoderConfig.mColorSpace.Construct(std::move(colorSpace));
}
- if (decoderConfigInternal.mDescription &&
- !decoderConfigInternal.mDescription.value()->IsEmpty()) {
- auto& abov = decoderConfig.mDescription.Construct();
- AutoEntryScript aes(GetParentObject(), "EncoderConfigToDecoderConfig");
- size_t lengthBytes =
- decoderConfigInternal.mDescription.value()->Length();
- UniquePtr<uint8_t[], JS::FreePolicy> extradata(
- new uint8_t[lengthBytes]);
- PodCopy(extradata.get(),
- decoderConfigInternal.mDescription.value()->Elements(),
- lengthBytes);
- JS::Rooted<JSObject*> description(
- aes.cx(), JS::NewArrayBufferWithContents(aes.cx(), lengthBytes,
- std::move(extradata)));
- JS::Rooted<JS::Value> value(aes.cx(), JS::ObjectValue(*description));
- DebugOnly<bool> rv = abov.Init(aes.cx(), value);
- }
+
+ CopyExtradataToDescriptionIfNeeded(GetParentObject(),
+ decoderConfigInternal, decoderConfig);
+
if (decoderConfigInternal.mDisplayAspectHeight) {
decoderConfig.mDisplayAspectHeight.Construct(
decoderConfigInternal.mDisplayAspectHeight.value());
@@ -387,7 +404,7 @@ void EncoderTemplate<EncoderType>::OutputEncodedData(
metadata.mDecoderConfig.Construct(std::move(decoderConfig));
mOutputNewDecoderConfig = false;
LOGE("New config passed to output callback: %s",
- NS_ConvertUTF16toUTF8(decoderConfigInternal.ToString()).get());
+ decoderConfigInternal.ToString().get());
}
nsAutoCString metadataInfo;
@@ -407,125 +424,74 @@ void EncoderTemplate<EncoderType>::OutputEncodedData(
LOG("EncoderTemplate:: output callback (ts: % " PRId64 ")%s",
encodedData->Timestamp(), metadataInfo.get());
- cb->Call((typename EncoderType::OutputType&)(*encodedData), metadata);
+ cb->Call((EncodedVideoChunk&)(*encodedData), metadata);
}
}
-template <typename EncoderType>
-class EncoderTemplate<EncoderType>::ErrorRunnable final
- : public DiscardableRunnable {
- public:
- ErrorRunnable(Self* aEncoder, const nsresult& aError)
- : DiscardableRunnable("Decoder ErrorRunnable"),
- mEncoder(aEncoder),
- mError(aError) {
- MOZ_ASSERT(mEncoder);
- }
- ~ErrorRunnable() = default;
-
- // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT.
- // See bug 1535398.
- MOZ_CAN_RUN_SCRIPT_BOUNDARY NS_IMETHOD Run() override {
- nsCString error;
- GetErrorName(mError, error);
- LOGE("%s %p report error: %s", EncoderType::Name.get(), mEncoder.get(),
- error.get());
- RefPtr<Self> d = std::move(mEncoder);
- d->ReportError(mError);
- return NS_OK;
- }
-
- private:
- RefPtr<Self> mEncoder;
- const nsresult mError;
-};
+template <>
+void EncoderTemplate<AudioEncoderTraits>::OutputEncodedAudioData(
+ const nsTArray<RefPtr<MediaRawData>>&& aData) {
+ AssertIsOnOwningThread();
+ MOZ_ASSERT(mState == CodecState::Configured);
+ MOZ_ASSERT(mActiveConfig);
-template <typename EncoderType>
-class EncoderTemplate<EncoderType>::OutputRunnable final
- : public DiscardableRunnable {
- public:
- OutputRunnable(Self* aEncoder, WebCodecsId aConfigureId,
- const nsACString& aLabel,
- nsTArray<RefPtr<MediaRawData>>&& aData)
- : DiscardableRunnable("Decoder OutputRunnable"),
- mEncoder(aEncoder),
- mConfigureId(aConfigureId),
- mLabel(aLabel),
- mData(std::move(aData)) {
- MOZ_ASSERT(mEncoder);
- }
- ~OutputRunnable() = default;
-
- // MOZ_CAN_RUN_SCRIPT_BOUNDARY until Runnable::Run is MOZ_CAN_RUN_SCRIPT.
- // See bug 1535398.
- MOZ_CAN_RUN_SCRIPT_BOUNDARY NS_IMETHOD Run() override {
- if (mEncoder->mState != CodecState::Configured) {
- LOGV("%s %p has been %s. Discard %s-result for EncoderAgent #%zu",
- EncoderType::Name.get(), mEncoder.get(),
- mEncoder->mState == CodecState::Closed ? "closed" : "reset",
- mLabel.get(), mConfigureId);
- return NS_OK;
- }
+ // Get JSContext for RootedDictionary.
+ // The EncoderType::MetadataType, AudioDecoderConfig
+ // below are rooted to work around the JS hazard issues.
+ AutoJSAPI jsapi;
+ DebugOnly<bool> ok =
+ jsapi.Init(GetParentObject()); // TODO: check returned value?
+ JSContext* cx = jsapi.cx();
- MOZ_ASSERT(mEncoder->mAgent);
- if (mConfigureId != mEncoder->mAgent->mId) {
- LOGW(
- "%s %p has been re-configured. Still yield %s-result for "
- "EncoderAgent #%zu",
- EncoderType::Name.get(), mEncoder.get(), mLabel.get(), mConfigureId);
+ RefPtr<EncodedAudioChunkOutputCallback> cb(mOutputCallback);
+ for (auto& data : aData) {
+ // It's possible to have reset() called in between this task having been
+ // dispatched, and running -- no output callback should happen when that's
+ // the case.
+ // This is imprecise in the spec, but discussed in
+ // https://github.com/w3c/webcodecs/issues/755 and agreed upon.
+ if (!mActiveConfig) {
+ return;
}
+ RefPtr<EncodedAudioChunk> encodedData =
+ EncodedDataToOutputType(GetParentObject(), data);
- LOGV("%s %p, yields %s-result for EncoderAgent #%zu",
- EncoderType::Name.get(), mEncoder.get(), mLabel.get(), mConfigureId);
- RefPtr<Self> d = std::move(mEncoder);
- d->OutputEncodedData(std::move(mData));
-
- return NS_OK;
- }
+ RootedDictionary<EncodedAudioChunkMetadata> metadata(cx);
+ if (mOutputNewDecoderConfig) {
+ AudioDecoderConfigInternal decoderConfigInternal =
+ this->EncoderConfigToDecoderConfig(GetParentObject(), data,
+ *mActiveConfig);
- private:
- RefPtr<Self> mEncoder;
- const WebCodecsId mConfigureId;
- const nsCString mLabel;
- nsTArray<RefPtr<MediaRawData>> mData;
-};
+ // Convert VideoDecoderConfigInternal to VideoDecoderConfig
+ RootedDictionary<AudioDecoderConfig> decoderConfig(cx);
+ decoderConfig.mCodec = decoderConfigInternal.mCodec;
+ decoderConfig.mNumberOfChannels = decoderConfigInternal.mNumberOfChannels;
+ decoderConfig.mSampleRate = decoderConfigInternal.mSampleRate;
-template <typename EncoderType>
-void EncoderTemplate<EncoderType>::ScheduleOutputEncodedData(
- nsTArray<RefPtr<MediaRawData>>&& aData, const nsACString& aLabel) {
- MOZ_ASSERT(mState == CodecState::Configured);
- MOZ_ASSERT(mAgent);
+ CopyExtradataToDescriptionIfNeeded(GetParentObject(),
+ decoderConfigInternal, decoderConfig);
- MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(MakeAndAddRef<OutputRunnable>(
- this, mAgent->mId, aLabel, std::move(aData))));
-}
+ metadata.mDecoderConfig.Construct(std::move(decoderConfig));
+ mOutputNewDecoderConfig = false;
+ LOGE("New config passed to output callback: %s",
+ decoderConfigInternal.ToString().get());
+ }
-template <typename EncoderType>
-void EncoderTemplate<EncoderType>::ScheduleClose(const nsresult& aResult) {
- AssertIsOnOwningThread();
- MOZ_ASSERT(mState == CodecState::Configured);
+ nsAutoCString metadataInfo;
- auto task = [self = RefPtr{this}, result = aResult] {
- if (self->mState == CodecState::Closed) {
- nsCString error;
- GetErrorName(result, error);
- LOGW("%s %p has been closed. Ignore close with %s",
- EncoderType::Name.get(), self.get(), error.get());
- return;
+ if (metadata.mDecoderConfig.WasPassed()) {
+ metadataInfo.Append(", new decoder config");
}
- DebugOnly<Result<Ok, nsresult>> r = self->CloseInternal(result);
- MOZ_ASSERT(r.value.isOk());
- };
- nsISerialEventTarget* target = GetCurrentSerialEventTarget();
- if (NS_IsMainThread()) {
- MOZ_ALWAYS_SUCCEEDS(target->Dispatch(
- NS_NewRunnableFunction("ScheduleClose Runnable (main)", task)));
- return;
+ LOG("EncoderTemplate:: output callback (ts: % " PRId64
+ ", duration: % " PRId64 ", %zu bytes, %" PRIu64 " so far)",
+ encodedData->Timestamp(),
+ !encodedData->GetDuration().IsNull()
+ ? encodedData->GetDuration().Value()
+ : 0,
+ data->Size(), mPacketsOutput++);
+ cb->Call((EncodedAudioChunk&)(*encodedData), metadata);
}
-
- MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewCancelableRunnableFunction(
- "ScheduleClose Runnable (worker)", task)));
}
template <typename EncoderType>
@@ -537,20 +503,10 @@ void EncoderTemplate<EncoderType>::ScheduleDequeueEvent() {
}
mDequeueEventScheduled = true;
- auto dispatcher = [self = RefPtr{this}] {
+ QueueATask("dequeue event task", [self = RefPtr{this}]() {
self->FireEvent(nsGkAtoms::ondequeue, u"dequeue"_ns);
self->mDequeueEventScheduled = false;
- };
- nsISerialEventTarget* target = GetCurrentSerialEventTarget();
-
- if (NS_IsMainThread()) {
- MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewRunnableFunction(
- "ScheduleDequeueEvent Runnable (main)", dispatcher)));
- return;
- }
-
- MOZ_ALWAYS_SUCCEEDS(target->Dispatch(NS_NewCancelableRunnableFunction(
- "ScheduleDequeueEvent Runnable (worker)", dispatcher)));
+ });
}
template <typename EncoderType>
@@ -655,6 +611,15 @@ void EncoderTemplate<EncoderType>::CancelPendingControlMessages(
}
template <typename EncoderType>
+template <typename Func>
+void EncoderTemplate<EncoderType>::QueueATask(const char* aName,
+ Func&& aSteps) {
+ AssertIsOnOwningThread();
+ MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(
+ NS_NewRunnableFunction(aName, std::forward<Func>(aSteps))));
+}
+
+template <typename EncoderType>
MessageProcessedResult EncoderTemplate<EncoderType>::ProcessConfigureMessage(
RefPtr<ConfigureMessage> aMessage) {
AssertIsOnOwningThread();
@@ -677,15 +642,13 @@ MessageProcessedResult EncoderTemplate<EncoderType>::ProcessConfigureMessage(
LOGE("%s %p ProcessConfigureMessage error (sync): Not supported",
EncoderType::Name.get(), this);
mProcessingMessage = nullptr;
- NS_DispatchToCurrentThread(NS_NewRunnableFunction(
- "ProcessConfigureMessage (async): not supported",
+ QueueATask(
+ "Error while configuring encoder",
[self = RefPtr(this)]() MOZ_CAN_RUN_SCRIPT_BOUNDARY {
LOGE("%s %p ProcessConfigureMessage (async close): Not supported",
EncoderType::Name.get(), self.get());
- DebugOnly<Result<Ok, nsresult>> r =
- self->CloseInternal(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
- MOZ_ASSERT(r.value.isOk());
- }));
+ self->CloseInternal(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+ });
return MessageProcessedResult::Processed;
}
@@ -711,19 +674,30 @@ void EncoderTemplate<EncoderType>::StopBlockingMessageQueue() {
}
template <typename EncoderType>
+void EncoderTemplate<EncoderType>::OutputEncodedData(
+ const nsTArray<RefPtr<MediaRawData>>&& aData) {
+ if constexpr (std::is_same_v<EncoderType, VideoEncoderTraits>) {
+ OutputEncodedVideoData(std::move(aData));
+ } else {
+ OutputEncodedAudioData(std::move(aData));
+ }
+}
+
+template <typename EncoderType>
void EncoderTemplate<EncoderType>::Reconfigure(
RefPtr<ConfigureMessage> aMessage) {
MOZ_ASSERT(mAgent);
- LOG("Reconfiguring encoder: %s",
- NS_ConvertUTF16toUTF8(aMessage->Config()->ToString()).get());
+ LOG("Reconfiguring encoder: %s", aMessage->Config()->ToString().get());
RefPtr<ConfigTypeInternal> config = aMessage->Config();
RefPtr<WebCodecsConfigurationChangeList> configDiff =
config->Diff(*mActiveConfig);
- // Nothing to do, return now
+ // Nothing to do, return now, but per spec the config
+ // must be output next time a packet is output.
if (configDiff->Empty()) {
+ mOutputNewDecoderConfig = true;
LOG("Reconfigure with identical config, returning.");
mProcessingMessage = nullptr;
StopBlockingMessageQueue();
@@ -731,9 +705,8 @@ void EncoderTemplate<EncoderType>::Reconfigure(
}
LOG("Attempting to reconfigure encoder: old: %s new: %s, diff: %s",
- NS_ConvertUTF16toUTF8(mActiveConfig->ToString()).get(),
- NS_ConvertUTF16toUTF8(config->ToString()).get(),
- NS_ConvertUTF16toUTF8(configDiff->ToString()).get());
+ mActiveConfig->ToString().get(), config->ToString().get(),
+ configDiff->ToString().get());
RefPtr<EncoderConfigurationChangeList> changeList =
configDiff->ToPEMChangeList();
@@ -766,16 +739,20 @@ void EncoderTemplate<EncoderType>::Reconfigure(
message](EncoderAgent::EncodePromise::ResolveOrRejectValue&&
aResult) {
if (aResult.IsReject()) {
+ // The spec asks to close the encoder with an
+ // NotSupportedError so we log the exact error here.
const MediaResult& error = aResult.RejectValue();
- LOGE(
- "%s %p, EncoderAgent #%zu failed to flush during "
- "reconfigure, closing: %s",
- EncoderType::Name.get(), self.get(), id,
- error.Description().get());
-
- self->mProcessingMessage = nullptr;
- self->ScheduleClose(
- NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ LOGE("%s %p, EncoderAgent #%zu failed to configure: %s",
+ EncoderType::Name.get(), self.get(), id,
+ error.Description().get());
+
+ self->QueueATask(
+ "Error during drain during reconfigure",
+ [self = RefPtr{self}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ MOZ_ASSERT(self->mState != CodecState::Closed);
+ self->CloseInternal(
+ NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ });
return;
}
@@ -797,12 +774,15 @@ void EncoderTemplate<EncoderType>::Reconfigure(
LOG("%s %p Outputing %zu frames during flush "
" for reconfiguration with encoder destruction",
EncoderType::Name.get(), self.get(), data.Length());
- self->ScheduleOutputEncodedData(
- std::move(data),
- nsLiteralCString("Flush before reconfigure"));
+ self->QueueATask(
+ "Output encoded Data",
+ [self = RefPtr{self}, data = std::move(data)]()
+ MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ self->OutputEncodedData(std::move(data));
+ });
}
- NS_DispatchToCurrentThread(NS_NewRunnableFunction(
+ self->QueueATask(
"Destroy + recreate encoder after failed reconfigure",
[self = RefPtr(self), message]()
MOZ_CAN_RUN_SCRIPT_BOUNDARY {
@@ -810,7 +790,7 @@ void EncoderTemplate<EncoderType>::Reconfigure(
// encoder with the new configuration.
self->DestroyEncoderAgentIfAny();
self->Configure(message);
- }));
+ });
});
return;
}
@@ -833,32 +813,30 @@ void EncoderTemplate<EncoderType>::Configure(
RefPtr<ConfigureMessage> aMessage) {
MOZ_ASSERT(!mAgent);
- LOG("Configuring encoder: %s",
- NS_ConvertUTF16toUTF8(aMessage->Config()->ToString()).get());
+ LOG("Configuring encoder: %s", aMessage->Config()->ToString().get());
mOutputNewDecoderConfig = true;
mActiveConfig = aMessage->Config();
- bool decoderAgentCreated =
+ bool encoderAgentCreated =
CreateEncoderAgent(aMessage->mMessageId, aMessage->Config());
- if (!decoderAgentCreated) {
+ if (!encoderAgentCreated) {
LOGE(
"%s %p ProcessConfigureMessage error (sync): encoder agent "
"creation "
"failed",
EncoderType::Name.get(), this);
mProcessingMessage = nullptr;
- NS_DispatchToCurrentThread(NS_NewRunnableFunction(
- "ProcessConfigureMessage (async): encoder agent creating failed",
+ QueueATask(
+ "Error when configuring encoder (encoder agent creation failed)",
[self = RefPtr(this)]() MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ MOZ_ASSERT(self->mState != CodecState::Closed);
LOGE(
"%s %p ProcessConfigureMessage (async close): encoder agent "
"creation failed",
EncoderType::Name.get(), self.get());
- DebugOnly<Result<Ok, nsresult>> r =
- self->CloseInternal(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
- MOZ_ASSERT(r.value.isOk());
- }));
+ self->CloseInternal(NS_ERROR_DOM_NOT_SUPPORTED_ERR);
+ });
return;
}
@@ -866,7 +844,7 @@ void EncoderTemplate<EncoderType>::Configure(
MOZ_ASSERT(mActiveConfig);
LOG("Real configuration with fresh config: %s",
- NS_ConvertUTF16toUTF8(mActiveConfig->ToString().get()).get());
+ mActiveConfig->ToString().get());
EncoderConfig config = mActiveConfig->ToEncoderConfig();
mAgent->Configure(config)
@@ -897,10 +875,15 @@ void EncoderTemplate<EncoderType>::Configure(
LOGE("%s %p, EncoderAgent #%zu failed to configure: %s",
EncoderType::Name.get(), self.get(), id,
error.Description().get());
- DebugOnly<Result<Ok, nsresult>> r = self->CloseInternal(
- NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
- MOZ_ASSERT(r.value.isOk());
- return; // No further process
+
+ self->QueueATask(
+ "Error during configure",
+ [self = RefPtr{self}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ MOZ_ASSERT(self->mState != CodecState::Closed);
+ self->CloseInternal(
+ NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ });
+ return;
}
self->StopBlockingMessageQueue();
@@ -933,7 +916,11 @@ MessageProcessedResult EncoderTemplate<EncoderType>::ProcessEncodeMessage(
// data is invalid.
auto closeOnError = [&]() {
mProcessingMessage = nullptr;
- ScheduleClose(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ QueueATask("Error during encode",
+ [self = RefPtr{this}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ MOZ_ASSERT(self->mState != CodecState::Closed);
+ self->CloseInternal(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ });
return MessageProcessedResult::Processed;
};
@@ -973,8 +960,14 @@ MessageProcessedResult EncoderTemplate<EncoderType>::ProcessEncodeMessage(
LOGE("%s %p, EncoderAgent #%zu %s failed: %s",
EncoderType::Name.get(), self.get(), id, msgStr.get(),
error.Description().get());
- self->ScheduleClose(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
- return; // No further process
+ self->QueueATask(
+ "Error during encode runnable",
+ [self = RefPtr{self}]() MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ MOZ_ASSERT(self->mState != CodecState::Closed);
+ self->CloseInternal(
+ NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ });
+ return;
}
MOZ_ASSERT(aResult.IsResolve());
@@ -984,11 +977,16 @@ MessageProcessedResult EncoderTemplate<EncoderType>::ProcessEncodeMessage(
LOGV("%s %p got no data for %s", EncoderType::Name.get(),
self.get(), msgStr.get());
} else {
- LOGV("%s %p, schedule %zu encoded data output",
- EncoderType::Name.get(), self.get(), data.Length());
- self->ScheduleOutputEncodedData(std::move(data), msgStr);
+ LOGV("%s %p, schedule %zu encoded data output for %s",
+ EncoderType::Name.get(), self.get(), data.Length(),
+ msgStr.get());
+ self->QueueATask(
+ "Output encoded Data",
+ [self = RefPtr{self}, data2 = std::move(data)]()
+ MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ self->OutputEncodedData(std::move(data2));
+ });
}
-
self->ProcessControlMessageQueue();
})
->Track(aMessage->Request());
@@ -1022,69 +1020,78 @@ MessageProcessedResult EncoderTemplate<EncoderType>::ProcessFlushMessage(
}
mAgent->Drain()
- ->Then(GetCurrentSerialEventTarget(), __func__,
- [self = RefPtr{this}, id = mAgent->mId, aMessage](
- EncoderAgent::EncodePromise::ResolveOrRejectValue&& aResult) {
- MOZ_ASSERT(self->mProcessingMessage);
- MOZ_ASSERT(self->mProcessingMessage->AsFlushMessage());
- MOZ_ASSERT(self->mState == CodecState::Configured);
- MOZ_ASSERT(self->mAgent);
- MOZ_ASSERT(id == self->mAgent->mId);
- MOZ_ASSERT(self->mActiveConfig);
-
- LOG("%s %p, EncoderAgent #%zu %s has been %s",
- EncoderType::Name.get(), self.get(), id,
- aMessage->ToString().get(),
- aResult.IsResolve() ? "resolved" : "rejected");
-
- nsCString msgStr = aMessage->ToString();
-
- aMessage->Complete();
-
- // If flush failed, it means encoder fails to encode the data
- // sent before, so we treat it like an encode error. We reject
- // the promise first and then queue a task to close VideoEncoder
- // with an EncodingError.
- if (aResult.IsReject()) {
- const MediaResult& error = aResult.RejectValue();
- LOGE("%s %p, EncoderAgent #%zu failed to flush: %s",
- EncoderType::Name.get(), self.get(), id,
- error.Description().get());
+ ->Then(
+ GetCurrentSerialEventTarget(), __func__,
+ [self = RefPtr{this}, id = mAgent->mId, aMessage,
+ this](EncoderAgent::EncodePromise::ResolveOrRejectValue&& aResult) {
+ MOZ_ASSERT(self->mProcessingMessage);
+ MOZ_ASSERT(self->mProcessingMessage->AsFlushMessage());
+ MOZ_ASSERT(self->mState == CodecState::Configured);
+ MOZ_ASSERT(self->mAgent);
+ MOZ_ASSERT(id == self->mAgent->mId);
+ MOZ_ASSERT(self->mActiveConfig);
- // Reject with an EncodingError instead of the error we got
- // above.
- self->SchedulePromiseResolveOrReject(
- aMessage->TakePromise(),
- NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ LOG("%s %p, EncoderAgent #%zu %s has been %s",
+ EncoderType::Name.get(), self.get(), id,
+ aMessage->ToString().get(),
+ aResult.IsResolve() ? "resolved" : "rejected");
- self->mProcessingMessage = nullptr;
+ nsCString msgStr = aMessage->ToString();
- self->ScheduleClose(NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
- return; // No further process
- }
+ aMessage->Complete();
- // If flush succeeded, schedule to output encoded data first
- // and then resolve the promise, then keep processing the
- // control messages.
- MOZ_ASSERT(aResult.IsResolve());
- nsTArray<RefPtr<MediaRawData>> data =
- std::move(aResult.ResolveValue());
+ // If flush failed, it means encoder fails to encode the data
+ // sent before, so we treat it like an encode error. We reject
+ // the promise first and then queue a task to close VideoEncoder
+ // with an EncodingError.
+ if (aResult.IsReject()) {
+ const MediaResult& error = aResult.RejectValue();
+ LOGE("%s %p, EncoderAgent #%zu failed to flush: %s",
+ EncoderType::Name.get(), self.get(), id,
+ error.Description().get());
+ RefPtr<Promise> promise = aMessage->TakePromise();
+ // Reject with an EncodingError instead of the error we got
+ // above.
+ self->QueueATask(
+ "Error during flush runnable",
+ [self = RefPtr{this}, promise]() MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ promise->MaybeReject(
+ NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ self->mProcessingMessage = nullptr;
+ MOZ_ASSERT(self->mState != CodecState::Closed);
+ self->CloseInternal(
+ NS_ERROR_DOM_ENCODING_NOT_SUPPORTED_ERR);
+ });
+ return;
+ }
- if (data.IsEmpty()) {
- LOG("%s %p gets no data for %s", EncoderType::Name.get(),
- self.get(), msgStr.get());
- } else {
- LOG("%s %p, schedule %zu encoded data output for %s",
- EncoderType::Name.get(), self.get(), data.Length(),
- msgStr.get());
- self->ScheduleOutputEncodedData(std::move(data), msgStr);
- }
+ // If flush succeeded, schedule to output encoded data first
+ // and then resolve the promise, then keep processing the
+ // control messages.
+ MOZ_ASSERT(aResult.IsResolve());
+ nsTArray<RefPtr<MediaRawData>> data =
+ std::move(aResult.ResolveValue());
+
+ if (data.IsEmpty()) {
+ LOG("%s %p gets no data for %s", EncoderType::Name.get(),
+ self.get(), msgStr.get());
+ } else {
+ LOG("%s %p, schedule %zu encoded data output for %s",
+ EncoderType::Name.get(), self.get(), data.Length(),
+ msgStr.get());
+ }
- self->SchedulePromiseResolveOrReject(aMessage->TakePromise(),
- NS_OK);
- self->mProcessingMessage = nullptr;
- self->ProcessControlMessageQueue();
- })
+ RefPtr<Promise> promise = aMessage->TakePromise();
+ self->QueueATask(
+ "Flush: output encoded data task",
+ [self = RefPtr{self}, promise, data = std::move(data)]()
+ MOZ_CAN_RUN_SCRIPT_BOUNDARY {
+ self->OutputEncodedData(std::move(data));
+ promise->MaybeResolveWithUndefined();
+ });
+ self->mProcessingMessage = nullptr;
+ self->ProcessControlMessageQueue();
+ })
->Track(aMessage->Request());
return MessageProcessedResult::Processed;
@@ -1218,6 +1225,7 @@ void EncoderTemplate<EncoderType>::DestroyEncoderAgentIfAny() {
}
template class EncoderTemplate<VideoEncoderTraits>;
+template class EncoderTemplate<AudioEncoderTraits>;
#undef LOG
#undef LOGW
diff --git a/dom/media/webcodecs/EncoderTemplate.h b/dom/media/webcodecs/EncoderTemplate.h
index e53d7166d1..bc65edca46 100644
--- a/dom/media/webcodecs/EncoderTemplate.h
+++ b/dom/media/webcodecs/EncoderTemplate.h
@@ -18,6 +18,7 @@
#include "mozilla/Result.h"
#include "mozilla/UniquePtr.h"
#include "mozilla/dom/VideoEncoderBinding.h"
+#include "mozilla/dom/AudioEncoderBinding.h"
#include "mozilla/dom/WorkerRef.h"
#include "mozilla/media/MediaUtils.h"
#include "nsStringFwd.h"
@@ -81,10 +82,8 @@ class EncoderTemplate : public DOMEventTargetHelper {
RefPtr<ConfigTypeInternal> Config() { return mConfig; }
nsCString ToString() const override {
nsCString rv;
- rv.AppendPrintf(
- "ConfigureMessage(#%zu): %s", this->mMessageId,
- mConfig ? NS_ConvertUTF16toUTF8(mConfig->ToString().get()).get()
- : "null cfg");
+ rv.AppendPrintf("ConfigureMessage(#%zu): %s", this->mMessageId,
+ mConfig ? mConfig->ToString().get() : "null cfg");
return rv;
}
@@ -149,10 +148,14 @@ class EncoderTemplate : public DOMEventTargetHelper {
void StartBlockingMessageQueue();
void StopBlockingMessageQueue();
+ MOZ_CAN_RUN_SCRIPT
+ void OutputEncodedData(const nsTArray<RefPtr<MediaRawData>>&& aData);
+
CodecState State() const { return mState; };
uint32_t EncodeQueueSize() const { return mEncodeQueueSize; };
+ MOZ_CAN_RUN_SCRIPT
void Configure(const ConfigType& aConfig, ErrorResult& aRv);
void EncodeAudioData(InputType& aInput, ErrorResult& aRv);
@@ -170,10 +173,13 @@ class EncoderTemplate : public DOMEventTargetHelper {
/* Type conversion functions for the Encoder implementation */
protected:
virtual RefPtr<OutputType> EncodedDataToOutputType(
- nsIGlobalObject* aGlobalObject, RefPtr<MediaRawData>& aData) = 0;
+ nsIGlobalObject* aGlobalObject, const RefPtr<MediaRawData>& aData) = 0;
virtual OutputConfigType EncoderConfigToDecoderConfig(
nsIGlobalObject* aGlobalObject, const RefPtr<MediaRawData>& aData,
const ConfigTypeInternal& aOutputConfig) const = 0;
+ template <typename T, typename U>
+ void CopyExtradataToDescriptionIfNeeded(nsIGlobalObject* aGlobal,
+ const T& aConfigInternal, U& aConfig);
/* Internal member variables and functions */
protected:
// EncoderTemplate can run on either main thread or worker thread.
@@ -182,21 +188,17 @@ class EncoderTemplate : public DOMEventTargetHelper {
}
Result<Ok, nsresult> ResetInternal(const nsresult& aResult);
- MOZ_CAN_RUN_SCRIPT_BOUNDARY
- Result<Ok, nsresult> CloseInternal(const nsresult& aResult);
+ MOZ_CAN_RUN_SCRIPT
+ Result<Ok, nsresult> CloseInternalWithAbort();
+ MOZ_CAN_RUN_SCRIPT
+ void CloseInternal(const nsresult& aResult);
MOZ_CAN_RUN_SCRIPT void ReportError(const nsresult& aResult);
- MOZ_CAN_RUN_SCRIPT void OutputEncodedData(
- nsTArray<RefPtr<MediaRawData>>&& aData);
-
- class ErrorRunnable;
- void ScheduleReportError(const nsresult& aResult);
- class OutputRunnable;
- void ScheduleOutputEncodedData(nsTArray<RefPtr<MediaRawData>>&& aData,
- const nsACString& aLabel);
-
- void ScheduleClose(const nsresult& aResult);
+ MOZ_CAN_RUN_SCRIPT void OutputEncodedVideoData(
+ const nsTArray<RefPtr<MediaRawData>>&& aData);
+ MOZ_CAN_RUN_SCRIPT void OutputEncodedAudioData(
+ const nsTArray<RefPtr<MediaRawData>>&& aData);
void ScheduleDequeueEvent();
nsresult FireEvent(nsAtom* aTypeWithOn, const nsAString& aEventType);
@@ -207,6 +209,9 @@ class EncoderTemplate : public DOMEventTargetHelper {
void ProcessControlMessageQueue();
void CancelPendingControlMessages(const nsresult& aResult);
+ template <typename Func>
+ void QueueATask(const char* aName, Func&& aSteps);
+
MessageProcessedResult ProcessConfigureMessage(
RefPtr<ConfigureMessage> aMessage);
@@ -244,14 +249,14 @@ class EncoderTemplate : public DOMEventTargetHelper {
// used as the FlushMessage's Id.
size_t mFlushCounter;
- // EncoderAgent will be created the first time "configure" is being processed,
- // and will be destroyed when "reset" is called. If another "configure" is
- // called, either it's possible to reconfigure the underlying encoder without
- // tearing eveyrthing down (e.g. a bitrate change), or it's not possible, and
- // the current encoder will be destroyed and a new one create.
- // In both cases, the encoder is implicitely flushed before the configuration
- // change.
- // See CanReconfigure on the {Audio,Video}EncoderConfigInternal
+ // EncoderAgent will be created the first time "configure" is being
+ // processed, and will be destroyed when "reset" is called. If another
+ // "configure" is called, either it's possible to reconfigure the underlying
+ // encoder without tearing everything down (e.g. a bitrate change), or it's
+ // not possible, and the current encoder will be destroyed and a new one
+ // create. In both cases, the encoder is implicitely flushed before the
+ // configuration change. See CanReconfigure on the
+ // {Audio,Video}EncoderConfigInternal
RefPtr<EncoderAgent> mAgent;
RefPtr<ConfigTypeInternal> mActiveConfig;
// This is true when a configure call has just been processed, and it's
@@ -283,6 +288,7 @@ class EncoderTemplate : public DOMEventTargetHelper {
// TODO: Use StrongWorkerRef instead if this is always used in the same
// thread?
RefPtr<ThreadSafeWorkerRef> mWorkerRef;
+ uint64_t mPacketsOutput = 0;
};
} // namespace mozilla::dom
diff --git a/dom/media/webcodecs/EncoderTypes.h b/dom/media/webcodecs/EncoderTypes.h
index d58d7c54c8..39f660203b 100644
--- a/dom/media/webcodecs/EncoderTypes.h
+++ b/dom/media/webcodecs/EncoderTypes.h
@@ -9,13 +9,14 @@
#include "mozilla/Maybe.h"
#include "mozilla/dom/EncodedVideoChunk.h"
+#include "mozilla/dom/MediaRecorderBinding.h"
#include "mozilla/dom/VideoEncoderBinding.h"
+#include "mozilla/dom/AudioEncoderBinding.h"
#include "mozilla/dom/VideoFrame.h"
#include "mozilla/dom/VideoFrameBinding.h"
#include "nsStringFwd.h"
#include "nsTLiteralString.h"
#include "VideoDecoder.h"
-#include "PlatformEncoderModule.h"
namespace mozilla {
@@ -24,6 +25,68 @@ class MediaByteBuffer;
namespace dom {
+class AudioEncoderConfigInternal {
+ public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AudioEncoderConfigInternal);
+ explicit AudioEncoderConfigInternal(const AudioEncoderConfig& aConfig);
+ explicit AudioEncoderConfigInternal(
+ const AudioEncoderConfigInternal& aConfig);
+
+ void SetSpecific(const EncoderConfig::CodecSpecific& aSpecific);
+
+ nsCString ToString() const;
+
+ bool Equals(const AudioEncoderConfigInternal& aOther) const;
+ bool CanReconfigure(const AudioEncoderConfigInternal& aOther) const;
+
+ // Returns an EncoderConfig struct with as many filled members as
+ // possible.
+ EncoderConfig ToEncoderConfig() const;
+
+ already_AddRefed<WebCodecsConfigurationChangeList> Diff(
+ const AudioEncoderConfigInternal& aOther) const;
+
+ nsString mCodec;
+ Maybe<uint32_t> mSampleRate;
+ Maybe<uint32_t> mNumberOfChannels;
+ Maybe<uint32_t> mBitrate;
+ BitrateMode mBitrateMode;
+ Maybe<EncoderConfig::CodecSpecific> mSpecific;
+
+ private:
+ AudioEncoderConfigInternal(const nsAString& aCodec,
+ Maybe<uint32_t> aSampleRate,
+ Maybe<uint32_t> aNumberOfChannels,
+ Maybe<uint32_t> aBitRate,
+ BitrateMode aBitratemode);
+ ~AudioEncoderConfigInternal() = default;
+};
+
+class AudioEncoderTraits {
+ public:
+ static constexpr nsLiteralCString Name = "AudioEncoder"_ns;
+ using ConfigType = AudioEncoderConfig;
+ using ConfigTypeInternal = AudioEncoderConfigInternal;
+ using InputType = dom::AudioData;
+ using OutputConfigType = mozilla::dom::AudioDecoderConfigInternal;
+ using InputTypeInternal = mozilla::AudioData;
+ using OutputType = EncodedAudioChunk;
+ using OutputCallbackType = EncodedAudioChunkOutputCallback;
+ using MetadataType = EncodedAudioChunkMetadata;
+
+ static bool IsSupported(const ConfigTypeInternal& aConfig);
+ static Result<UniquePtr<TrackInfo>, nsresult> CreateTrackInfo(
+ const ConfigTypeInternal& aConfig);
+ static bool Validate(const ConfigType& aConfig, nsCString& aErrorMessage);
+ static RefPtr<ConfigTypeInternal> CreateConfigInternal(
+ const ConfigType& aConfig);
+ static RefPtr<InputTypeInternal> CreateInputInternal(
+ const InputType& aInput, const VideoEncoderEncodeOptions& aOptions);
+ static already_AddRefed<OutputConfigType> EncoderConfigToDecoderConfig(
+ nsIGlobalObject* aGlobal, const RefPtr<MediaRawData>& aData,
+ const ConfigTypeInternal& mOutputConfig);
+};
+
class VideoEncoderConfigInternal {
public:
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoEncoderConfigInternal);
@@ -40,7 +103,7 @@ class VideoEncoderConfigInternal {
bool CanReconfigure(const VideoEncoderConfigInternal& aOther) const;
already_AddRefed<WebCodecsConfigurationChangeList> Diff(
const VideoEncoderConfigInternal& aOther) const;
- nsString ToString() const;
+ nsCString ToString() const;
nsString mCodec;
uint32_t mWidth;
@@ -84,17 +147,14 @@ class VideoEncoderTraits {
static bool IsSupported(const ConfigTypeInternal& aConfig);
static bool CanEncodeVideo(const ConfigTypeInternal& aConfig);
- static Result<UniquePtr<TrackInfo>, nsresult> CreateTrackInfo(
- const ConfigTypeInternal& aConfig);
static bool Validate(const ConfigType& aConfig, nsCString& aErrorMessage);
static RefPtr<ConfigTypeInternal> CreateConfigInternal(
const ConfigType& aConfig);
static RefPtr<InputTypeInternal> CreateInputInternal(
const InputType& aInput, const VideoEncoderEncodeOptions& aOptions);
static already_AddRefed<OutputConfigType> EncoderConfigToDecoderConfig(
- nsIGlobalObject* aGlobal,
- const RefPtr<MediaRawData>& aData,
- const ConfigTypeInternal& mOutputConfig);
+ nsIGlobalObject* aGlobal, const RefPtr<MediaRawData>& aData,
+ const ConfigTypeInternal& mOutputConfig);
};
} // namespace dom
diff --git a/dom/media/webcodecs/VideoDecoder.cpp b/dom/media/webcodecs/VideoDecoder.cpp
index 18855e5cea..dfc1dff093 100644
--- a/dom/media/webcodecs/VideoDecoder.cpp
+++ b/dom/media/webcodecs/VideoDecoder.cpp
@@ -96,7 +96,7 @@ VideoColorSpaceInit VideoColorSpaceInternal::ToColorSpaceInit() const {
VideoDecoderConfigInternal::VideoDecoderConfigInternal(
const nsAString& aCodec, Maybe<uint32_t>&& aCodedHeight,
Maybe<uint32_t>&& aCodedWidth, Maybe<VideoColorSpaceInternal>&& aColorSpace,
- Maybe<RefPtr<MediaByteBuffer>>&& aDescription,
+ already_AddRefed<MediaByteBuffer> aDescription,
Maybe<uint32_t>&& aDisplayAspectHeight,
Maybe<uint32_t>&& aDisplayAspectWidth,
const HardwareAcceleration& aHardwareAcceleration,
@@ -105,7 +105,7 @@ VideoDecoderConfigInternal::VideoDecoderConfigInternal(
mCodedHeight(std::move(aCodedHeight)),
mCodedWidth(std::move(aCodedWidth)),
mColorSpace(std::move(aColorSpace)),
- mDescription(std::move(aDescription)),
+ mDescription(aDescription),
mDisplayAspectHeight(std::move(aDisplayAspectHeight)),
mDisplayAspectWidth(std::move(aDisplayAspectWidth)),
mHardwareAcceleration(aHardwareAcceleration),
@@ -120,7 +120,7 @@ UniquePtr<VideoDecoderConfigInternal> VideoDecoderConfigInternal::Create(
return nullptr;
}
- Maybe<RefPtr<MediaByteBuffer>> description;
+ RefPtr<MediaByteBuffer> description;
if (aConfig.mDescription.WasPassed()) {
auto rv = GetExtraDataFromArrayBuffer(aConfig.mDescription.Value());
if (rv.isErr()) { // Invalid description data.
@@ -130,7 +130,7 @@ UniquePtr<VideoDecoderConfigInternal> VideoDecoderConfigInternal::Create(
static_cast<uint32_t>(rv.unwrapErr()));
return nullptr;
}
- description.emplace(rv.unwrap());
+ description = rv.unwrap();
}
Maybe<VideoColorSpaceInternal> colorSpace;
@@ -141,16 +141,16 @@ UniquePtr<VideoDecoderConfigInternal> VideoDecoderConfigInternal::Create(
return UniquePtr<VideoDecoderConfigInternal>(new VideoDecoderConfigInternal(
aConfig.mCodec, OptionalToMaybe(aConfig.mCodedHeight),
OptionalToMaybe(aConfig.mCodedWidth), std::move(colorSpace),
- std::move(description), OptionalToMaybe(aConfig.mDisplayAspectHeight),
+ description.forget(), OptionalToMaybe(aConfig.mDisplayAspectHeight),
OptionalToMaybe(aConfig.mDisplayAspectWidth),
aConfig.mHardwareAcceleration,
OptionalToMaybe(aConfig.mOptimizeForLatency)));
}
-nsString VideoDecoderConfigInternal::ToString() const {
- nsString rv;
+nsCString VideoDecoderConfigInternal::ToString() const {
+ nsCString rv;
- rv.Append(mCodec);
+ rv.Append(NS_ConvertUTF16toUTF8(mCodec));
if (mCodedWidth.isSome()) {
rv.AppendPrintf("coded: %dx%d", mCodedWidth.value(), mCodedHeight.value());
}
@@ -161,8 +161,8 @@ nsString VideoDecoderConfigInternal::ToString() const {
if (mColorSpace.isSome()) {
rv.AppendPrintf("colorspace %s", "todo");
}
- if (mDescription.isSome() && mDescription.value()) {
- rv.AppendPrintf("extradata: %zu bytes", mDescription.value()->Length());
+ if (mDescription) {
+ rv.AppendPrintf("extradata: %zu bytes", mDescription->Length());
}
rv.AppendPrintf("hw accel: %s", GetEnumString(mHardwareAcceleration).get());
if (mOptimizeForLatency.isSome()) {
@@ -579,8 +579,7 @@ bool VideoDecoderTraits::IsSupported(
/* static */
Result<UniquePtr<TrackInfo>, nsresult> VideoDecoderTraits::CreateTrackInfo(
const VideoDecoderConfigInternal& aConfig) {
- LOG("Create a VideoInfo from %s config",
- NS_ConvertUTF16toUTF8(aConfig.ToString()).get());
+ LOG("Create a VideoInfo from %s config", aConfig.ToString().get());
nsTArray<UniquePtr<TrackInfo>> tracks = GetTracksInfo(aConfig);
if (tracks.Length() != 1 || tracks[0]->GetType() != TrackInfo::kVideoTrack) {
@@ -668,15 +667,14 @@ Result<UniquePtr<TrackInfo>, nsresult> VideoDecoderTraits::CreateTrackInfo(
}
}
- if (aConfig.mDescription.isSome()) {
- RefPtr<MediaByteBuffer> buf;
- buf = aConfig.mDescription.value();
- if (buf) {
- LOG("The given config has %zu bytes of description data", buf->Length());
+ if (aConfig.mDescription) {
+ if (!aConfig.mDescription->IsEmpty()) {
+ LOG("The given config has %zu bytes of description data",
+ aConfig.mDescription->Length());
if (vi->mExtraData) {
LOGW("The default extra data is overwritten");
}
- vi->mExtraData = buf;
+ vi->mExtraData = aConfig.mDescription;
}
// TODO: Make this utility and replace the similar one in MP4Demuxer.cpp.
diff --git a/dom/media/webcodecs/VideoEncoder.cpp b/dom/media/webcodecs/VideoEncoder.cpp
index f593f70c77..5407e917b6 100644
--- a/dom/media/webcodecs/VideoEncoder.cpp
+++ b/dom/media/webcodecs/VideoEncoder.cpp
@@ -120,11 +120,12 @@ VideoEncoderConfigInternal::VideoEncoderConfigInternal(
mContentHint(OptionalToMaybe(aConfig.mContentHint)),
mAvc(OptionalToMaybe(aConfig.mAvc)) {}
-nsString VideoEncoderConfigInternal::ToString() const {
- nsString rv;
+nsCString VideoEncoderConfigInternal::ToString() const {
+ nsCString rv;
- rv.AppendPrintf("Codec: %s, [%" PRIu32 "x%" PRIu32 "],",
- NS_ConvertUTF16toUTF8(mCodec).get(), mWidth, mHeight);
+ rv.AppendLiteral("Codec: ");
+ rv.Append(NS_ConvertUTF16toUTF8(mCodec));
+ rv.AppendPrintf(" [%" PRIu32 "x%" PRIu32 "],", mWidth, mHeight);
if (mDisplayWidth.isSome()) {
rv.AppendPrintf(", display[%" PRIu32 "x%" PRIu32 "]", mDisplayWidth.value(),
mDisplayHeight.value());
@@ -194,20 +195,19 @@ bool VideoEncoderConfigInternal::CanReconfigure(
}
EncoderConfig VideoEncoderConfigInternal::ToEncoderConfig() const {
- MediaDataEncoder::Usage usage;
+ Usage usage;
if (mLatencyMode == LatencyMode::Quality) {
- usage = MediaDataEncoder::Usage::Record;
+ usage = Usage::Record;
} else {
- usage = MediaDataEncoder::Usage::Realtime;
+ usage = Usage::Realtime;
}
- MediaDataEncoder::HardwarePreference hwPref =
- MediaDataEncoder::HardwarePreference::None;
+ HardwarePreference hwPref = HardwarePreference::None;
if (mHardwareAcceleration ==
mozilla::dom::HardwareAcceleration::Prefer_hardware) {
- hwPref = MediaDataEncoder::HardwarePreference::RequireHardware;
+ hwPref = HardwarePreference::RequireHardware;
} else if (mHardwareAcceleration ==
mozilla::dom::HardwareAcceleration::Prefer_software) {
- hwPref = MediaDataEncoder::HardwarePreference::RequireSoftware;
+ hwPref = HardwarePreference::RequireSoftware;
}
CodecType codecType;
auto maybeCodecType = CodecStringToCodecType(mCodec);
@@ -236,19 +236,19 @@ EncoderConfig VideoEncoderConfigInternal::ToEncoderConfig() const {
}
}
uint8_t numTemporalLayers = 1;
- MediaDataEncoder::ScalabilityMode scalabilityMode;
+ ScalabilityMode scalabilityMode;
if (mScalabilityMode) {
if (mScalabilityMode->EqualsLiteral("L1T2")) {
- scalabilityMode = MediaDataEncoder::ScalabilityMode::L1T2;
+ scalabilityMode = ScalabilityMode::L1T2;
numTemporalLayers = 2;
} else if (mScalabilityMode->EqualsLiteral("L1T3")) {
- scalabilityMode = MediaDataEncoder::ScalabilityMode::L1T3;
+ scalabilityMode = ScalabilityMode::L1T3;
numTemporalLayers = 3;
} else {
- scalabilityMode = MediaDataEncoder::ScalabilityMode::None;
+ scalabilityMode = ScalabilityMode::None;
}
} else {
- scalabilityMode = MediaDataEncoder::ScalabilityMode::None;
+ scalabilityMode = ScalabilityMode::None;
}
// Only for vp9, not vp8
if (codecType == CodecType::VP9) {
@@ -278,8 +278,8 @@ EncoderConfig VideoEncoderConfigInternal::ToEncoderConfig() const {
AssertedCast<uint8_t>(mFramerate.refOr(0.f)), 0,
mBitrate.refOr(0),
mBitrateMode == VideoEncoderBitrateMode::Constant
- ? MediaDataEncoder::BitrateMode::Constant
- : MediaDataEncoder::BitrateMode::Variable,
+ ? mozilla::BitrateMode::Constant
+ : mozilla::BitrateMode::Variable,
hwPref, scalabilityMode, specific);
}
already_AddRefed<WebCodecsConfigurationChangeList>
@@ -558,7 +558,7 @@ already_AddRefed<Promise> VideoEncoder::IsConfigSupported(
}
RefPtr<EncodedVideoChunk> VideoEncoder::EncodedDataToOutputType(
- nsIGlobalObject* aGlobalObject, RefPtr<MediaRawData>& aData) {
+ nsIGlobalObject* aGlobalObject, const RefPtr<MediaRawData>& aData) {
AssertIsOnOwningThread();
MOZ_RELEASE_ASSERT(aData->mType == MediaData::Type::RAW_DATA);
@@ -591,8 +591,8 @@ VideoDecoderConfigInternal VideoEncoder::EncoderConfigToDecoderConfig(
Some(mOutputConfig.mWidth), /* aCodedWidth */
Some(init), /* aColorSpace */
aRawData->mExtraData && !aRawData->mExtraData->IsEmpty()
- ? Some(aRawData->mExtraData)
- : Nothing(), /* aDescription*/
+ ? aRawData->mExtraData.forget()
+ : nullptr, /* aDescription*/
Maybe<uint32_t>(mOutputConfig.mDisplayHeight), /* aDisplayAspectHeight*/
Maybe<uint32_t>(mOutputConfig.mDisplayWidth), /* aDisplayAspectWidth */
mOutputConfig.mHardwareAcceleration, /* aHardwareAcceleration */
diff --git a/dom/media/webcodecs/VideoEncoder.h b/dom/media/webcodecs/VideoEncoder.h
index 9251b5023a..f6d1bfffb7 100644
--- a/dom/media/webcodecs/VideoEncoder.h
+++ b/dom/media/webcodecs/VideoEncoder.h
@@ -65,7 +65,7 @@ class VideoEncoder final : public EncoderTemplate<VideoEncoderTraits> {
protected:
virtual RefPtr<EncodedVideoChunk> EncodedDataToOutputType(
- nsIGlobalObject* aGlobal, RefPtr<MediaRawData>& aData) override;
+ nsIGlobalObject* aGlobal, const RefPtr<MediaRawData>& aData) override;
virtual VideoDecoderConfigInternal EncoderConfigToDecoderConfig(
nsIGlobalObject* aGlobal /* TODO: delete */,
diff --git a/dom/media/webcodecs/WebCodecsUtils.cpp b/dom/media/webcodecs/WebCodecsUtils.cpp
index 3507aba440..db4d79220e 100644
--- a/dom/media/webcodecs/WebCodecsUtils.cpp
+++ b/dom/media/webcodecs/WebCodecsUtils.cpp
@@ -7,6 +7,7 @@
#include "WebCodecsUtils.h"
#include "DecoderTypes.h"
+#include "PlatformEncoderModule.h"
#include "VideoUtils.h"
#include "js/experimental/TypedData.h"
#include "mozilla/Assertions.h"
@@ -15,8 +16,6 @@
#include "mozilla/dom/VideoFrameBinding.h"
#include "mozilla/gfx/Types.h"
#include "nsDebug.h"
-#include "PlatformEncoderModule.h"
-#include "PlatformEncoderModule.h"
extern mozilla::LazyLogModule gWebCodecsLog;
@@ -412,8 +411,8 @@ struct ConfigurationChangeToString {
}
};
-nsString WebCodecsConfigurationChangeList::ToString() const {
- nsString rv;
+nsCString WebCodecsConfigurationChangeList::ToString() const {
+ nsCString rv;
for (const WebCodecsEncoderConfigurationItem& change : mChanges) {
nsCString str = change.match(ConfigurationChangeToString());
rv.AppendPrintf("- %s\n", str.get());
@@ -470,24 +469,24 @@ WebCodecsConfigurationChangeList::ToPEMChangeList() const {
} else if (change.is<FramerateChange>()) {
rv->Push(mozilla::FramerateChange(change.as<FramerateChange>().get()));
} else if (change.is<dom::BitrateModeChange>()) {
- MediaDataEncoder::BitrateMode mode;
+ mozilla::BitrateMode mode;
if (change.as<dom::BitrateModeChange>().get() ==
dom::VideoEncoderBitrateMode::Constant) {
- mode = MediaDataEncoder::BitrateMode::Constant;
+ mode = mozilla::BitrateMode::Constant;
} else if (change.as<BitrateModeChange>().get() ==
dom::VideoEncoderBitrateMode::Variable) {
- mode = MediaDataEncoder::BitrateMode::Variable;
+ mode = mozilla::BitrateMode::Variable;
} else {
// Quantizer, not underlying support yet.
- mode = MediaDataEncoder::BitrateMode::Variable;
+ mode = mozilla::BitrateMode::Variable;
}
rv->Push(mozilla::BitrateModeChange(mode));
} else if (change.is<LatencyModeChange>()) {
- MediaDataEncoder::Usage usage;
+ Usage usage;
if (change.as<LatencyModeChange>().get() == dom::LatencyMode::Quality) {
- usage = MediaDataEncoder::Usage::Record;
+ usage = Usage::Record;
} else {
- usage = MediaDataEncoder::Usage::Realtime;
+ usage = Usage::Realtime;
}
rv->Push(UsageChange(usage));
} else if (change.is<ContentHintChange>()) {
@@ -570,7 +569,7 @@ Maybe<CodecType> CodecStringToCodecType(const nsAString& aCodecString) {
return Nothing();
}
-nsString ConfigToString(const VideoDecoderConfig& aConfig) {
+nsCString ConfigToString(const VideoDecoderConfig& aConfig) {
nsString rv;
auto internal = VideoDecoderConfigInternal::Create(aConfig);
@@ -606,4 +605,52 @@ bool IsSupportedVideoCodec(const nsAString& aCodec) {
return true;
}
+nsCString ConvertCodecName(const nsCString& aContainer,
+ const nsCString& aCodec) {
+ if (!aContainer.EqualsLiteral("x-wav")) {
+ return aCodec;
+ }
+
+ // https://www.rfc-editor.org/rfc/rfc2361.txt
+ if (aCodec.EqualsLiteral("ulaw")) {
+ return nsCString("7");
+ }
+ if (aCodec.EqualsLiteral("alaw")) {
+ return nsCString("6");
+ }
+ if (aCodec.Find("f32")) {
+ return nsCString("3");
+ }
+ // Linear PCM
+ return nsCString("1");
+}
+
+bool IsSupportedAudioCodec(const nsAString& aCodec) {
+ LOG("IsSupportedAudioCodec: %s", NS_ConvertUTF16toUTF8(aCodec).get());
+ return aCodec.EqualsLiteral("flac") || aCodec.EqualsLiteral("mp3") ||
+ IsAACCodecString(aCodec) || aCodec.EqualsLiteral("opus") ||
+ aCodec.EqualsLiteral("ulaw") || aCodec.EqualsLiteral("alaw") ||
+ aCodec.EqualsLiteral("pcm-u8") || aCodec.EqualsLiteral("pcm-s16") ||
+ aCodec.EqualsLiteral("pcm-s24") || aCodec.EqualsLiteral("pcm-s32") ||
+ aCodec.EqualsLiteral("pcm-f32");
+}
+
+uint32_t BytesPerSamples(const mozilla::dom::AudioSampleFormat& aFormat) {
+ switch (aFormat) {
+ case AudioSampleFormat::U8:
+ case AudioSampleFormat::U8_planar:
+ return sizeof(uint8_t);
+ case AudioSampleFormat::S16:
+ case AudioSampleFormat::S16_planar:
+ return sizeof(int16_t);
+ case AudioSampleFormat::S32:
+ case AudioSampleFormat::F32:
+ case AudioSampleFormat::S32_planar:
+ case AudioSampleFormat::F32_planar:
+ return sizeof(float);
+ }
+ MOZ_ASSERT_UNREACHABLE("Invalid enum value");
+ return 0;
+}
+
} // namespace mozilla::dom
diff --git a/dom/media/webcodecs/WebCodecsUtils.h b/dom/media/webcodecs/WebCodecsUtils.h
index 196c57421d..b2a8f17398 100644
--- a/dom/media/webcodecs/WebCodecsUtils.h
+++ b/dom/media/webcodecs/WebCodecsUtils.h
@@ -9,17 +9,18 @@
#include "ErrorList.h"
#include "MediaData.h"
+#include "PlatformEncoderModule.h"
#include "js/TypeDecls.h"
#include "mozilla/Maybe.h"
#include "mozilla/MozPromise.h"
#include "mozilla/Result.h"
#include "mozilla/TaskQueue.h"
+#include "mozilla/dom/AudioDataBinding.h"
#include "mozilla/dom/BindingDeclarations.h"
#include "mozilla/dom/Nullable.h"
#include "mozilla/dom/UnionTypes.h"
#include "mozilla/dom/VideoEncoderBinding.h"
#include "mozilla/dom/VideoFrameBinding.h"
-#include "PlatformEncoderModule.h"
namespace mozilla {
@@ -218,7 +219,7 @@ struct WebCodecsConfigurationChangeList {
// Convert this to the format the underlying PEM can understand
RefPtr<EncoderConfigurationChangeList> ToPEMChangeList() const;
- nsString ToString() const;
+ nsCString ToString() const;
nsTArray<WebCodecsEncoderConfigurationItem> mChanges;
@@ -235,12 +236,20 @@ VideoColorSpaceInit FallbackColorSpaceForWebContent();
Maybe<CodecType> CodecStringToCodecType(const nsAString& aCodecString);
-nsString ConfigToString(const VideoDecoderConfig& aConfig);
+nsCString ConfigToString(const VideoDecoderConfig& aConfig);
+// Returns true if a particular codec is supported by WebCodecs.
bool IsSupportedVideoCodec(const nsAString& aCodec);
+bool IsSupportedAudioCodec(const nsAString& aCodec);
-} // namespace dom
+// Returns the codec string to use in Gecko for a particular container and
+// codec name given by WebCodecs. This maps pcm description to the profile
+// number, and simply returns the codec name for all other codecs.
+nsCString ConvertCodecName(const nsCString& aContainer,
+ const nsCString& aCodec);
+uint32_t BytesPerSamples(const mozilla::dom::AudioSampleFormat& aFormat);
+} // namespace dom
} // namespace mozilla
#endif // MOZILLA_DOM_WEBCODECS_WEBCODECSUTILS_H
diff --git a/dom/media/webcodecs/crashtests/1881079.html b/dom/media/webcodecs/crashtests/1881079.html
new file mode 100644
index 0000000000..15fd26ff74
--- /dev/null
+++ b/dom/media/webcodecs/crashtests/1881079.html
@@ -0,0 +1,35 @@
+<!DOCTYPE html>
+<script>
+document.addEventListener("DOMContentLoaded", async () => {
+ const encoder = new VideoEncoder({
+ 'output': (e) => {},
+ 'error': (e) => {},
+ })
+ encoder.configure({
+ 'codec': '󠋚᩿',
+ 'width': 2147483648,
+ 'height': 60,
+ })
+ encoder.reset()
+ encoder.configure({
+ 'codec': '󠋚᩿',
+ 'width': 4294967295,
+ 'height': 29,
+ })
+ const decoder = new VideoDecoder({
+ 'output': (e) => {},
+ 'error': (e) => {},
+ })
+ decoder.configure({
+ 'codec': '󠋚᩿',
+ 'width': 2147483648,
+ 'height': 60,
+ })
+ decoder.reset()
+ decoder.configure({
+ 'codec': '󠋚᩿',
+ 'width': 4294967295,
+ 'height': 29,
+ })
+})
+</script>
diff --git a/dom/media/webcodecs/crashtests/crashtests.list b/dom/media/webcodecs/crashtests/crashtests.list
index cea5139fe9..16fbd90ff5 100644
--- a/dom/media/webcodecs/crashtests/crashtests.list
+++ b/dom/media/webcodecs/crashtests/crashtests.list
@@ -1,4 +1,6 @@
skip-if(Android) pref(dom.media.webcodecs.enabled,true) load 1839270.html
skip-if(Android) pref(dom.media.webcodecs.enabled,true) load 1848460.html
skip-if(Android) pref(dom.media.webcodecs.enabled,true) load 1849271.html
-skip-if(Android) pref(dom.media.webcodecs.enabled,true) load 1864475.html \ No newline at end of file
+skip-if(Android) pref(dom.media.webcodecs.enabled,true) load 1864475.html
+skip-if(Android) pref(dom.media.webcodecs.enabled,true) load 1881079.html
+
diff --git a/dom/media/webcodecs/moz.build b/dom/media/webcodecs/moz.build
index ddb5aad5cb..1c398439a3 100644
--- a/dom/media/webcodecs/moz.build
+++ b/dom/media/webcodecs/moz.build
@@ -23,6 +23,7 @@ EXPORTS.mozilla += [
EXPORTS.mozilla.dom += [
"AudioData.h",
"AudioDecoder.h",
+ "AudioEncoder.h",
"DecoderTemplate.h",
"DecoderTypes.h",
"EncodedAudioChunk.h",
@@ -40,6 +41,7 @@ EXPORTS.mozilla.dom += [
UNIFIED_SOURCES += [
"AudioData.cpp",
"AudioDecoder.cpp",
+ "AudioEncoder.cpp",
"DecoderAgent.cpp",
"DecoderTemplate.cpp",
"EncodedAudioChunk.cpp",
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-18 19:48:24 +0000