summaryrefslogtreecommitdiffstats
path: root/dom/media/platforms/agnostic
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
commit26a029d407be480d791972afb5975cf62c9360a6 (patch)
treef435a8308119effd964b339f76abb83a57c29483 /dom/media/platforms/agnostic
parentInitial commit. (diff)
downloadfirefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz
firefox-26a029d407be480d791972afb5975cf62c9360a6.zip
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/media/platforms/agnostic')
-rw-r--r--dom/media/platforms/agnostic/AOMDecoder.cpp1073
-rw-r--r--dom/media/platforms/agnostic/AOMDecoder.h287
-rw-r--r--dom/media/platforms/agnostic/AgnosticDecoderModule.cpp185
-rw-r--r--dom/media/platforms/agnostic/AgnosticDecoderModule.h39
-rw-r--r--dom/media/platforms/agnostic/BlankDecoderModule.cpp146
-rw-r--r--dom/media/platforms/agnostic/BlankDecoderModule.h68
-rw-r--r--dom/media/platforms/agnostic/DAV1DDecoder.cpp403
-rw-r--r--dom/media/platforms/agnostic/DAV1DDecoder.h71
-rw-r--r--dom/media/platforms/agnostic/DummyMediaDataDecoder.cpp80
-rw-r--r--dom/media/platforms/agnostic/DummyMediaDataDecoder.h68
-rw-r--r--dom/media/platforms/agnostic/NullDecoderModule.cpp57
-rw-r--r--dom/media/platforms/agnostic/TheoraDecoder.cpp269
-rw-r--r--dom/media/platforms/agnostic/TheoraDecoder.h63
-rw-r--r--dom/media/platforms/agnostic/VPXDecoder.cpp679
-rw-r--r--dom/media/platforms/agnostic/VPXDecoder.h208
-rw-r--r--dom/media/platforms/agnostic/bytestreams/Adts.cpp94
-rw-r--r--dom/media/platforms/agnostic/bytestreams/Adts.h22
-rw-r--r--dom/media/platforms/agnostic/bytestreams/AnnexB.cpp542
-rw-r--r--dom/media/platforms/agnostic/bytestreams/AnnexB.h89
-rw-r--r--dom/media/platforms/agnostic/bytestreams/ByteStreamsUtils.h70
-rw-r--r--dom/media/platforms/agnostic/bytestreams/H264.cpp1333
-rw-r--r--dom/media/platforms/agnostic/bytestreams/H264.h547
-rw-r--r--dom/media/platforms/agnostic/bytestreams/H265.cpp1300
-rw-r--r--dom/media/platforms/agnostic/bytestreams/H265.h356
-rw-r--r--dom/media/platforms/agnostic/bytestreams/gtest/TestByteStreams.cpp787
-rw-r--r--dom/media/platforms/agnostic/bytestreams/gtest/moz.build11
-rw-r--r--dom/media/platforms/agnostic/bytestreams/moz.build38
-rw-r--r--dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp156
-rw-r--r--dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.h54
-rw-r--r--dom/media/platforms/agnostic/eme/DecryptThroughputLimit.h103
-rw-r--r--dom/media/platforms/agnostic/eme/EMEDecoderModule.cpp481
-rw-r--r--dom/media/platforms/agnostic/eme/EMEDecoderModule.h79
-rw-r--r--dom/media/platforms/agnostic/eme/SamplesWaitingForKey.cpp79
-rw-r--r--dom/media/platforms/agnostic/eme/SamplesWaitingForKey.h68
-rw-r--r--dom/media/platforms/agnostic/eme/moz.build22
-rw-r--r--dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp94
-rw-r--r--dom/media/platforms/agnostic/gmp/GMPDecoderModule.h58
-rw-r--r--dom/media/platforms/agnostic/gmp/GMPVideoDecoder.cpp489
-rw-r--r--dom/media/platforms/agnostic/gmp/GMPVideoDecoder.h129
-rw-r--r--dom/media/platforms/agnostic/gmp/moz.build24
40 files changed, 10721 insertions, 0 deletions
diff --git a/dom/media/platforms/agnostic/AOMDecoder.cpp b/dom/media/platforms/agnostic/AOMDecoder.cpp
new file mode 100644
index 0000000000..cb7f784848
--- /dev/null
+++ b/dom/media/platforms/agnostic/AOMDecoder.cpp
@@ -0,0 +1,1073 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "AOMDecoder.h"
+
+#include <algorithm>
+
+#include "BitWriter.h"
+#include "BitReader.h"
+#include "ImageContainer.h"
+#include "MediaResult.h"
+#include "TimeUnits.h"
+#include <aom/aom_image.h>
+#include <aom/aomdx.h>
+#include "gfx2DGlue.h"
+#include "gfxUtils.h"
+#include "mozilla/PodOperations.h"
+#include "mozilla/SyncRunnable.h"
+#include "mozilla/TaskQueue.h"
+#include "nsError.h"
+#include "nsThreadUtils.h"
+#include "prsystem.h"
+#include "VideoUtils.h"
+
+#undef LOG
+#define LOG(arg, ...) \
+ DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, __func__, \
+ ##__VA_ARGS__)
+#define LOG_RESULT(code, message, ...) \
+ DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, "::%s: %s (code %d) " message, \
+ __func__, aom_codec_err_to_string(code), (int)code, ##__VA_ARGS__)
+#define LOGEX_RESULT(_this, code, message, ...) \
+ DDMOZ_LOGEX(_this, sPDMLog, mozilla::LogLevel::Debug, \
+ "::%s: %s (code %d) " message, __func__, \
+ aom_codec_err_to_string(code), (int)code, ##__VA_ARGS__)
+#define LOG_STATIC_RESULT(code, message, ...) \
+ MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, \
+ ("AOMDecoder::%s: %s (code %d) " message, __func__, \
+ aom_codec_err_to_string(code), (int)code, ##__VA_ARGS__))
+
+#define ASSERT_BYTE_ALIGNED(bitIO) MOZ_ASSERT((bitIO).BitCount() % 8 == 0)
+
+namespace mozilla {
+
+using namespace gfx;
+using namespace layers;
+using gfx::CICP::ColourPrimaries;
+using gfx::CICP::MatrixCoefficients;
+using gfx::CICP::TransferCharacteristics;
+
+static MediaResult InitContext(AOMDecoder& aAOMDecoder, aom_codec_ctx_t* aCtx,
+ const VideoInfo& aInfo) {
+ aom_codec_iface_t* dx = aom_codec_av1_dx();
+ if (!dx) {
+ return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ RESULT_DETAIL("Couldn't get AV1 decoder interface."));
+ }
+
+ size_t decode_threads = 2;
+ if (aInfo.mDisplay.width >= 2048) {
+ decode_threads = 8;
+ } else if (aInfo.mDisplay.width >= 1024) {
+ decode_threads = 4;
+ }
+ decode_threads = std::min(decode_threads, GetNumberOfProcessors());
+
+ aom_codec_dec_cfg_t config;
+ PodZero(&config);
+ config.threads = static_cast<unsigned int>(decode_threads);
+ config.w = config.h = 0; // set after decode
+ config.allow_lowbitdepth = true;
+
+ aom_codec_flags_t flags = 0;
+
+ auto res = aom_codec_dec_init(aCtx, dx, &config, flags);
+ if (res != AOM_CODEC_OK) {
+ LOGEX_RESULT(&aAOMDecoder, res, "Codec initialization failed, res=%d",
+ int(res));
+ return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ RESULT_DETAIL("AOM error initializing AV1 decoder: %s",
+ aom_codec_err_to_string(res)));
+ }
+ return NS_OK;
+}
+
+AOMDecoder::AOMDecoder(const CreateDecoderParams& aParams)
+ : mImageContainer(aParams.mImageContainer),
+ mTaskQueue(TaskQueue::Create(
+ GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER), "AOMDecoder")),
+ mInfo(aParams.VideoConfig()),
+ mTrackingId(aParams.mTrackingId) {
+ PodZero(&mCodec);
+}
+
+AOMDecoder::~AOMDecoder() = default;
+
+RefPtr<ShutdownPromise> AOMDecoder::Shutdown() {
+ RefPtr<AOMDecoder> self = this;
+ return InvokeAsync(mTaskQueue, __func__, [self]() {
+ auto res = aom_codec_destroy(&self->mCodec);
+ if (res != AOM_CODEC_OK) {
+ LOGEX_RESULT(self.get(), res, "aom_codec_destroy");
+ }
+ return self->mTaskQueue->BeginShutdown();
+ });
+}
+
+RefPtr<MediaDataDecoder::InitPromise> AOMDecoder::Init() {
+ MediaResult rv = InitContext(*this, &mCodec, mInfo);
+ if (NS_FAILED(rv)) {
+ return AOMDecoder::InitPromise::CreateAndReject(rv, __func__);
+ }
+ return AOMDecoder::InitPromise::CreateAndResolve(TrackInfo::kVideoTrack,
+ __func__);
+}
+
+RefPtr<MediaDataDecoder::FlushPromise> AOMDecoder::Flush() {
+ return InvokeAsync(mTaskQueue, __func__, [this, self = RefPtr(this)]() {
+ mPerformanceRecorder.Record(std::numeric_limits<int64_t>::max());
+ return FlushPromise::CreateAndResolve(true, __func__);
+ });
+}
+
+// UniquePtr dtor wrapper for aom_image_t.
+struct AomImageFree {
+ void operator()(aom_image_t* img) { aom_img_free(img); }
+};
+
+RefPtr<MediaDataDecoder::DecodePromise> AOMDecoder::ProcessDecode(
+ MediaRawData* aSample) {
+ MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
+
+#if defined(DEBUG)
+ NS_ASSERTION(
+ IsKeyframe(*aSample) == aSample->mKeyframe,
+ "AOM Decode Keyframe error sample->mKeyframe and si.si_kf out of sync");
+#endif
+
+ MediaInfoFlag flag = MediaInfoFlag::None;
+ flag |= (aSample->mKeyframe ? MediaInfoFlag::KeyFrame
+ : MediaInfoFlag::NonKeyFrame);
+ flag |= MediaInfoFlag::SoftwareDecoding;
+ flag |= MediaInfoFlag::VIDEO_AV1;
+
+ mTrackingId.apply([&](const auto& aId) {
+ mPerformanceRecorder.Start(aSample->mTimecode.ToMicroseconds(),
+ "AOMDecoder"_ns, aId, flag);
+ });
+
+ if (aom_codec_err_t r = aom_codec_decode(&mCodec, aSample->Data(),
+ aSample->Size(), nullptr)) {
+ LOG_RESULT(r, "Decode error!");
+ return DecodePromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
+ RESULT_DETAIL("AOM error decoding AV1 sample: %s",
+ aom_codec_err_to_string(r))),
+ __func__);
+ }
+
+ aom_codec_iter_t iter = nullptr;
+ aom_image_t* img;
+ UniquePtr<aom_image_t, AomImageFree> img8;
+ DecodedData results;
+
+ while ((img = aom_codec_get_frame(&mCodec, &iter))) {
+ NS_ASSERTION(
+ img->fmt == AOM_IMG_FMT_I420 || img->fmt == AOM_IMG_FMT_I42016 ||
+ img->fmt == AOM_IMG_FMT_I444 || img->fmt == AOM_IMG_FMT_I44416,
+ "AV1 image format not I420 or I444");
+
+ // Chroma shifts are rounded down as per the decoding examples in the SDK
+ VideoData::YCbCrBuffer b;
+ b.mPlanes[0].mData = img->planes[0];
+ b.mPlanes[0].mStride = img->stride[0];
+ b.mPlanes[0].mHeight = img->d_h;
+ b.mPlanes[0].mWidth = img->d_w;
+ b.mPlanes[0].mSkip = 0;
+
+ b.mPlanes[1].mData = img->planes[1];
+ b.mPlanes[1].mStride = img->stride[1];
+ b.mPlanes[1].mSkip = 0;
+
+ b.mPlanes[2].mData = img->planes[2];
+ b.mPlanes[2].mStride = img->stride[2];
+ b.mPlanes[2].mSkip = 0;
+
+ if (img->fmt == AOM_IMG_FMT_I420 || img->fmt == AOM_IMG_FMT_I42016) {
+ b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT;
+
+ b.mPlanes[1].mHeight = (img->d_h + 1) >> img->y_chroma_shift;
+ b.mPlanes[1].mWidth = (img->d_w + 1) >> img->x_chroma_shift;
+
+ b.mPlanes[2].mHeight = (img->d_h + 1) >> img->y_chroma_shift;
+ b.mPlanes[2].mWidth = (img->d_w + 1) >> img->x_chroma_shift;
+ } else if (img->fmt == AOM_IMG_FMT_I444 || img->fmt == AOM_IMG_FMT_I44416) {
+ b.mPlanes[1].mHeight = img->d_h;
+ b.mPlanes[1].mWidth = img->d_w;
+
+ b.mPlanes[2].mHeight = img->d_h;
+ b.mPlanes[2].mWidth = img->d_w;
+ } else {
+ LOG("AOM Unknown image format");
+ return DecodePromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
+ RESULT_DETAIL("AOM Unknown image format")),
+ __func__);
+ }
+
+ if (img->bit_depth == 10) {
+ b.mColorDepth = ColorDepth::COLOR_10;
+ } else if (img->bit_depth == 12) {
+ b.mColorDepth = ColorDepth::COLOR_12;
+ }
+
+ switch (img->mc) {
+ case AOM_CICP_MC_BT_601:
+ b.mYUVColorSpace = YUVColorSpace::BT601;
+ break;
+ case AOM_CICP_MC_BT_2020_NCL:
+ case AOM_CICP_MC_BT_2020_CL:
+ b.mYUVColorSpace = YUVColorSpace::BT2020;
+ break;
+ case AOM_CICP_MC_BT_709:
+ b.mYUVColorSpace = YUVColorSpace::BT709;
+ break;
+ default:
+ b.mYUVColorSpace = DefaultColorSpace({img->d_w, img->d_h});
+ break;
+ }
+ b.mColorRange = img->range == AOM_CR_FULL_RANGE ? ColorRange::FULL
+ : ColorRange::LIMITED;
+
+ switch (img->cp) {
+ case AOM_CICP_CP_BT_709:
+ b.mColorPrimaries = ColorSpace2::BT709;
+ break;
+ case AOM_CICP_CP_BT_2020:
+ b.mColorPrimaries = ColorSpace2::BT2020;
+ break;
+ default:
+ b.mColorPrimaries = ColorSpace2::BT709;
+ break;
+ }
+
+ Result<already_AddRefed<VideoData>, MediaResult> r =
+ VideoData::CreateAndCopyData(
+ mInfo, mImageContainer, aSample->mOffset, aSample->mTime,
+ aSample->mDuration, b, aSample->mKeyframe, aSample->mTimecode,
+ mInfo.ScaledImageRect(img->d_w, img->d_h), nullptr);
+
+ if (r.isErr()) {
+ MediaResult rs = r.unwrapErr();
+ LOG("VideoData::CreateAndCopyData error (source %ux%u display %ux%u "
+ "picture %ux%u) - %s: %s",
+ img->d_w, img->d_h, mInfo.mDisplay.width, mInfo.mDisplay.height,
+ mInfo.mImage.width, mInfo.mImage.height, rs.ErrorName().get(),
+ rs.Message().get());
+
+ return DecodePromise::CreateAndReject(std::move(rs), __func__);
+ }
+
+ RefPtr<VideoData> v = r.unwrap();
+ MOZ_ASSERT(v);
+
+ mPerformanceRecorder.Record(
+ aSample->mTimecode.ToMicroseconds(), [&](DecodeStage& aStage) {
+ aStage.SetResolution(mInfo.mImage.width, mInfo.mImage.height);
+ auto format = [&]() -> Maybe<DecodeStage::ImageFormat> {
+ switch (img->fmt) {
+ case AOM_IMG_FMT_I420:
+ case AOM_IMG_FMT_I42016:
+ return Some(DecodeStage::YUV420P);
+ case AOM_IMG_FMT_I444:
+ case AOM_IMG_FMT_I44416:
+ return Some(DecodeStage::YUV444P);
+ default:
+ return Nothing();
+ }
+ }();
+ format.apply([&](auto& aFmt) { aStage.SetImageFormat(aFmt); });
+ aStage.SetYUVColorSpace(b.mYUVColorSpace);
+ aStage.SetColorRange(b.mColorRange);
+ aStage.SetColorDepth(b.mColorDepth);
+ });
+ results.AppendElement(std::move(v));
+ }
+ return DecodePromise::CreateAndResolve(std::move(results), __func__);
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> AOMDecoder::Decode(
+ MediaRawData* aSample) {
+ return InvokeAsync<MediaRawData*>(mTaskQueue, this, __func__,
+ &AOMDecoder::ProcessDecode, aSample);
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> AOMDecoder::Drain() {
+ return InvokeAsync(mTaskQueue, __func__, [] {
+ return DecodePromise::CreateAndResolve(DecodedData(), __func__);
+ });
+}
+
+/* static */
+bool AOMDecoder::IsAV1(const nsACString& aMimeType) {
+ return aMimeType.EqualsLiteral("video/av1");
+}
+
+/* static */
+bool AOMDecoder::IsKeyframe(Span<const uint8_t> aBuffer) {
+ aom_codec_stream_info_t info;
+ PodZero(&info);
+
+ auto res = aom_codec_peek_stream_info(aom_codec_av1_dx(), aBuffer.Elements(),
+ aBuffer.Length(), &info);
+ if (res != AOM_CODEC_OK) {
+ LOG_STATIC_RESULT(
+ res, "couldn't get keyframe flag with aom_codec_peek_stream_info");
+ return false;
+ }
+
+ return bool(info.is_kf);
+}
+
+/* static */
+gfx::IntSize AOMDecoder::GetFrameSize(Span<const uint8_t> aBuffer) {
+ aom_codec_stream_info_t info;
+ PodZero(&info);
+
+ auto res = aom_codec_peek_stream_info(aom_codec_av1_dx(), aBuffer.Elements(),
+ aBuffer.Length(), &info);
+ if (res != AOM_CODEC_OK) {
+ LOG_STATIC_RESULT(
+ res, "couldn't get frame size with aom_codec_peek_stream_info");
+ }
+
+ return gfx::IntSize(info.w, info.h);
+}
+
+/* static */
+AOMDecoder::OBUIterator AOMDecoder::ReadOBUs(const Span<const uint8_t>& aData) {
+ return OBUIterator(aData);
+}
+
+void AOMDecoder::OBUIterator::UpdateNext() {
+ // If mGoNext is not set, we don't need to load a new OBU.
+ if (!mGoNext) {
+ return;
+ }
+ // Check if we've reached the end of the data. Allow mGoNext to stay true so
+ // that HasNext() will return false.
+ if (mPosition >= mData.Length()) {
+ return;
+ }
+ mGoNext = false;
+
+ // If retrieving the next OBU fails, reset the current OBU and set the
+ // position past the end of the data so that HasNext() returns false.
+ auto resetExit = MakeScopeExit([&]() {
+ mCurrent = OBUInfo();
+ mPosition = mData.Length();
+ });
+
+ auto subspan = mData.Subspan(mPosition, mData.Length() - mPosition);
+ BitReader br(subspan.Elements(), subspan.Length() * 8);
+ OBUInfo temp;
+
+ // AV1 spec available at:
+ // https://aomediacodec.github.io/av1-spec/
+ // or https://aomediacodec.github.io/av1-spec/av1-spec.pdf
+
+ // begin open_bitstream_unit( )
+ // https://aomediacodec.github.io/av1-spec/#general-obu-syntax
+
+ // begin obu_header( )
+ // https://aomediacodec.github.io/av1-spec/#obu-header-syntax
+ br.ReadBit(); // obu_forbidden_bit
+ temp.mType = static_cast<OBUType>(br.ReadBits(4));
+ if (!temp.IsValid()) {
+ // Non-fatal error, unknown OBUs can be skipped as long as the size field
+ // is properly specified.
+ NS_WARNING(nsPrintfCString("Encountered unknown OBU type (%" PRIu8
+ ", OBU may be invalid",
+ static_cast<uint8_t>(temp.mType))
+ .get());
+ }
+ temp.mExtensionFlag = br.ReadBit();
+ bool hasSizeField = br.ReadBit();
+ br.ReadBit(); // obu_reserved_1bit
+
+ // begin obu_extension_header( ) (5.3.3)
+ if (temp.mExtensionFlag) {
+ if (br.BitsLeft() < 8) {
+ mResult = MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
+ "Not enough bits left for an OBU extension header");
+ return;
+ }
+ br.ReadBits(3); // temporal_id
+ br.ReadBits(2); // spatial_id
+ br.ReadBits(3); // extension_header_reserved_3bits
+ }
+ // end obu_extension_header( )
+ // end obu_header( )
+
+ // Get the size of the remaining OBU data attached to the header in
+ // bytes.
+ size_t size;
+ if (hasSizeField) {
+ if (br.BitsLeft() < 8) {
+ mResult = MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
+ "Not enough bits left for an OBU size field");
+ return;
+ }
+ CheckedUint32 checkedSize = br.ReadULEB128().toChecked<uint32_t>();
+ // Spec requires that the value ULEB128 reads is (1 << 32) - 1 or below.
+ // See leb128(): https://aomediacodec.github.io/av1-spec/#leb128
+ if (!checkedSize.isValid()) {
+ mResult =
+ MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, "OBU size was too large");
+ return;
+ }
+ size = checkedSize.value();
+ } else {
+ // This case should rarely happen in practice. To support the Annex B
+ // format in the specification, we would have to parse every header type
+ // to skip over them, but this allows us to at least iterate once to
+ // retrieve the first OBU in the data.
+ size = mData.Length() - 1 - temp.mExtensionFlag;
+ }
+
+ if (br.BitsLeft() / 8 < size) {
+ mResult = MediaResult(
+ NS_ERROR_DOM_MEDIA_DECODE_ERR,
+ nsPrintfCString("Size specified by the OBU header (%zu) is more "
+ "than the actual remaining OBU data (%zu)",
+ size, br.BitsLeft() / 8)
+ .get());
+ return;
+ }
+
+ ASSERT_BYTE_ALIGNED(br);
+
+ size_t bytes = br.BitCount() / 8;
+ temp.mContents = mData.Subspan(mPosition + bytes, size);
+ mCurrent = temp;
+ // end open_bitstream_unit( )
+
+ mPosition += bytes + size;
+ resetExit.release();
+ mResult = NS_OK;
+}
+
+/* static */
+already_AddRefed<MediaByteBuffer> AOMDecoder::CreateOBU(
+ const OBUType aType, const Span<const uint8_t>& aContents) {
+ RefPtr<MediaByteBuffer> buffer = new MediaByteBuffer();
+
+ BitWriter bw(buffer);
+ bw.WriteBits(0, 1); // obu_forbidden_bit
+ bw.WriteBits(static_cast<uint8_t>(aType), 4);
+ bw.WriteBit(false); // obu_extension_flag
+ bw.WriteBit(true); // obu_has_size_field
+ bw.WriteBits(0, 1); // obu_reserved_1bit
+ ASSERT_BYTE_ALIGNED(bw);
+ bw.WriteULEB128(aContents.Length());
+ ASSERT_BYTE_ALIGNED(bw);
+
+ buffer->AppendElements(aContents.Elements(), aContents.Length());
+ return buffer.forget();
+}
+
+/* static */
+MediaResult AOMDecoder::ReadSequenceHeaderInfo(
+ const Span<const uint8_t>& aSample, AV1SequenceInfo& aDestInfo) {
+ // We need to get the last sequence header OBU, the specification does not
+ // limit a temporal unit to one sequence header.
+ OBUIterator iter = ReadOBUs(aSample);
+ OBUInfo seqOBU;
+
+ while (true) {
+ if (!iter.HasNext()) {
+ // Pass along the error from parsing the OBU.
+ MediaResult result = iter.GetResult();
+ if (result.Code() != NS_OK) {
+ return result;
+ }
+ break;
+ }
+ OBUInfo obu = iter.Next();
+ if (obu.mType == OBUType::SequenceHeader) {
+ seqOBU = obu;
+ }
+ }
+
+ if (seqOBU.mType != OBUType::SequenceHeader) {
+ return NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA;
+ }
+
+ // Sequence header syntax is specified here:
+ // https://aomediacodec.github.io/av1-spec/#sequence-header-obu-syntax
+ // Section 5.5: Sequence header OBU syntax
+
+ // See also Section 6.4: Sequence header OBU semantics
+ // https://aomediacodec.github.io/av1-spec/#sequence-header-obu-semantics
+ // This section defines all the fields used in the sequence header.
+ BitReader br(seqOBU.mContents.Elements(), seqOBU.mContents.Length() * 8);
+ AV1SequenceInfo tempInfo;
+
+ // begin sequence_header_obu( )
+ // https://aomediacodec.github.io/av1-spec/#general-sequence-header-obu-syntax
+ tempInfo.mProfile = br.ReadBits(3);
+ const bool stillPicture = br.ReadBit();
+ const bool reducedStillPicture = br.ReadBit();
+ if (!stillPicture && reducedStillPicture) {
+ return MediaResult(
+ NS_ERROR_DOM_MEDIA_DECODE_ERR,
+ "reduced_still_picture is true while still_picture is false");
+ }
+
+ if (reducedStillPicture) {
+ OperatingPoint op;
+ op.mLayers = 0;
+ op.mLevel = br.ReadBits(5); // seq_level_idx[0]
+ op.mTier = 0;
+ tempInfo.mOperatingPoints.SetCapacity(1);
+ tempInfo.mOperatingPoints.AppendElement(op);
+ } else {
+ bool decoderModelInfoPresent;
+ uint8_t operatingPointCountMinusOne;
+
+ if (br.ReadBit()) { // timing_info_present_flag
+ // begin timing_info( )
+ // https://aomediacodec.github.io/av1-spec/#timing-info-syntax
+ br.ReadBits(32); // num_units_in_display_tick
+ br.ReadBits(32); // time_scale
+ if (br.ReadBit()) { // equal_picture_interval
+ br.ReadUE(); // num_ticks_per_picture_minus_1
+ }
+ // end timing_info( )
+
+ decoderModelInfoPresent = br.ReadBit();
+ if (decoderModelInfoPresent) {
+ // begin decoder_model_info( )
+ // https://aomediacodec.github.io/av1-spec/#decoder-model-info-syntax
+ br.ReadBits(5); // buffer_delay_length_minus_1
+ br.ReadBits(32); // num_units_in_decoding_tick
+ br.ReadBits(5); // buffer_removal_time_length_minus_1
+ br.ReadBits(5); // frame_presentation_time_length_minus_1
+ // end decoder_model_info( )
+ }
+ } else {
+ decoderModelInfoPresent = false;
+ }
+
+ bool initialDisplayDelayPresent = br.ReadBit();
+ operatingPointCountMinusOne = br.ReadBits(5);
+ tempInfo.mOperatingPoints.SetCapacity(operatingPointCountMinusOne + 1);
+ for (uint8_t i = 0; i <= operatingPointCountMinusOne; i++) {
+ OperatingPoint op;
+ op.mLayers = br.ReadBits(12); // operating_point_idc[ i ]
+ op.mLevel = br.ReadBits(5); // seq_level_idx[ i ]
+ op.mTier = op.mLevel > 7 ? br.ReadBits(1) : 0;
+ if (decoderModelInfoPresent) {
+ br.ReadBit(); // decoder_model_present_for_this_op[ i ]
+ }
+ if (initialDisplayDelayPresent) {
+ if (br.ReadBit()) { // initial_display_delay_present_for_this_op[ i ]
+ br.ReadBits(4);
+ }
+ }
+ tempInfo.mOperatingPoints.AppendElement(op);
+ }
+ }
+
+ uint8_t frameWidthBits = br.ReadBits(4) + 1;
+ uint8_t frameHeightBits = br.ReadBits(4) + 1;
+ uint32_t maxWidth = br.ReadBits(frameWidthBits) + 1;
+ uint32_t maxHeight = br.ReadBits(frameHeightBits) + 1;
+ tempInfo.mImage = gfx::IntSize(maxWidth, maxHeight);
+
+ if (!reducedStillPicture) {
+ if (br.ReadBit()) { // frame_id_numbers_present_flag
+ br.ReadBits(4); // delta_frame_id_length_minus_2
+ br.ReadBits(3); // additional_frame_id_legnth_minus_1
+ }
+ }
+
+ br.ReadBit(); // use_128x128_superblock
+ br.ReadBit(); // enable_filter_intra
+ br.ReadBit(); // enable_intra_edge_filter
+
+ if (reducedStillPicture) {
+ // enable_interintra_compound = 0
+ // enable_masked_compound = 0
+ // enable_warped_motion = 0
+ // enable_dual_filter = 0
+ // enable_order_hint = 0
+ // enable_jnt_comp = 0
+ // enable_ref_frame_mvs = 0
+ // seq_force_screen_content_tools = SELECT_SCREEN_CONTENT_TOOLS
+ // seq_force_integer_mv = SELECT_INTEGER_MV
+ // OrderHintBits = 0
+ } else {
+ br.ReadBit(); // enable_interintra_compound
+ br.ReadBit(); // enable_masked_compound
+ br.ReadBit(); // enable_warped_motion
+ br.ReadBit(); // enable_dual_filter
+
+ const bool enableOrderHint = br.ReadBit();
+
+ if (enableOrderHint) {
+ br.ReadBit(); // enable_jnt_comp
+ br.ReadBit(); // enable_ref_frame_mvs
+ }
+
+ uint8_t forceScreenContentTools;
+
+ if (br.ReadBit()) { // seq_choose_screen_content_tools
+ forceScreenContentTools = 2; // SELECT_SCREEN_CONTENT_TOOLS
+ } else {
+ forceScreenContentTools = br.ReadBits(1);
+ }
+
+ if (forceScreenContentTools > 0) {
+ if (!br.ReadBit()) { // seq_choose_integer_mv
+ br.ReadBit(); // seq_force_integer_mv
+ }
+ }
+
+ if (enableOrderHint) {
+ br.ReadBits(3); // order_hint_bits_minus_1
+ }
+ }
+
+ br.ReadBit(); // enable_superres
+ br.ReadBit(); // enable_cdef
+ br.ReadBit(); // enable_restoration
+
+ // begin color_config( )
+ // https://aomediacodec.github.io/av1-spec/#color-config-syntax
+ const bool highBitDepth = br.ReadBit();
+ if (tempInfo.mProfile == 2 && highBitDepth) {
+ const bool twelveBit = br.ReadBit();
+ tempInfo.mBitDepth = twelveBit ? 12 : 10;
+ } else {
+ tempInfo.mBitDepth = highBitDepth ? 10 : 8;
+ }
+
+ tempInfo.mMonochrome = tempInfo.mProfile == 1 ? false : br.ReadBit();
+
+ VideoColorSpace* colors = &tempInfo.mColorSpace;
+
+ if (br.ReadBit()) { // color_description_present_flag
+ colors->mPrimaries = static_cast<ColourPrimaries>(br.ReadBits(8));
+ colors->mTransfer = static_cast<TransferCharacteristics>(br.ReadBits(8));
+ colors->mMatrix = static_cast<MatrixCoefficients>(br.ReadBits(8));
+ } else {
+ colors->mPrimaries = ColourPrimaries::CP_UNSPECIFIED;
+ colors->mTransfer = TransferCharacteristics::TC_UNSPECIFIED;
+ colors->mMatrix = MatrixCoefficients::MC_UNSPECIFIED;
+ }
+
+ if (tempInfo.mMonochrome) {
+ colors->mRange = br.ReadBit() ? ColorRange::FULL : ColorRange::LIMITED;
+ tempInfo.mSubsamplingX = true;
+ tempInfo.mSubsamplingY = true;
+ tempInfo.mChromaSamplePosition = ChromaSamplePosition::Unknown;
+ } else if (colors->mPrimaries == ColourPrimaries::CP_BT709 &&
+ colors->mTransfer == TransferCharacteristics::TC_SRGB &&
+ colors->mMatrix == MatrixCoefficients::MC_IDENTITY) {
+ colors->mRange = ColorRange::FULL;
+ tempInfo.mSubsamplingX = false;
+ tempInfo.mSubsamplingY = false;
+ } else {
+ colors->mRange = br.ReadBit() ? ColorRange::FULL : ColorRange::LIMITED;
+ switch (tempInfo.mProfile) {
+ case 0:
+ tempInfo.mSubsamplingX = true;
+ tempInfo.mSubsamplingY = true;
+ break;
+ case 1:
+ tempInfo.mSubsamplingX = false;
+ tempInfo.mSubsamplingY = false;
+ break;
+ case 2:
+ if (tempInfo.mBitDepth == 12) {
+ tempInfo.mSubsamplingX = br.ReadBit();
+ tempInfo.mSubsamplingY =
+ tempInfo.mSubsamplingX ? br.ReadBit() : false;
+ } else {
+ tempInfo.mSubsamplingX = true;
+ tempInfo.mSubsamplingY = false;
+ }
+ break;
+ }
+ tempInfo.mChromaSamplePosition =
+ tempInfo.mSubsamplingX && tempInfo.mSubsamplingY
+ ? static_cast<ChromaSamplePosition>(br.ReadBits(2))
+ : ChromaSamplePosition::Unknown;
+ }
+
+ br.ReadBit(); // separate_uv_delta_q
+ // end color_config( )
+
+ br.ReadBit(); // film_grain_params_present
+ // end sequence_header_obu( )
+
+ // begin trailing_bits( )
+ // https://aomediacodec.github.io/av1-spec/#trailing-bits-syntax
+ if (br.BitsLeft() > 8) {
+ NS_WARNING(
+ "AV1 sequence header finished reading with more than "
+ "a byte of aligning bits, may indicate an error");
+ }
+ // Ensure that data is read correctly by checking trailing bits.
+ bool correct = br.ReadBit();
+ correct &= br.ReadBits(br.BitsLeft() % 8) == 0;
+ while (br.BitsLeft() > 0) {
+ correct &= br.ReadBits(8) == 0;
+ }
+ if (!correct) {
+ return MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
+ "AV1 sequence header was corrupted");
+ }
+ // end trailing_bits( )
+
+ aDestInfo = tempInfo;
+ return NS_OK;
+}
+
+/* static */
+already_AddRefed<MediaByteBuffer> AOMDecoder::CreateSequenceHeader(
+ const AV1SequenceInfo& aInfo, nsresult& aResult) {
+ aResult = NS_ERROR_FAILURE;
+
+ RefPtr<MediaByteBuffer> seqHdrBuffer = new MediaByteBuffer();
+ BitWriter bw(seqHdrBuffer);
+
+ // See 5.5.1: General sequence header OBU syntax
+ // https://aomediacodec.github.io/av1-spec/#general-sequence-header-obu-syntax
+ bw.WriteBits(aInfo.mProfile, 3);
+ bw.WriteBit(false); // still_picture
+ bw.WriteBit(false); // reduced_still_picture_header
+
+ bw.WriteBit(false); // timing_info_present_flag
+ // if ( timing_info_present_flag ) {...}
+ bw.WriteBit(false); // initial_display_delay_present_flag
+
+ size_t opCount = aInfo.mOperatingPoints.Length();
+ bw.WriteBits(opCount - 1, 5); // operating_points_cnt_minus_1
+ for (size_t i = 0; i < opCount; i++) {
+ OperatingPoint op = aInfo.mOperatingPoints[i];
+ bw.WriteBits(op.mLayers, 12); // operating_point_idc[ i ]
+ bw.WriteBits(op.mLevel, 5);
+ if (op.mLevel > 7) {
+ bw.WriteBits(op.mTier, 1);
+ } else {
+ // seq_tier[ i ] = 0
+ if (op.mTier != 0) {
+ NS_WARNING("Operating points cannot specify tier for levels under 8.");
+ return nullptr;
+ }
+ }
+ // if ( decoder_model_info_present_flag ) {...}
+ // else
+ // decoder_model_info_present_for_this_op[ i ] = 0
+ // if ( initial_display_delay_present_flag ) {...}
+ }
+
+ if (aInfo.mImage.IsEmpty()) {
+ NS_WARNING("Sequence header requires a valid image size");
+ return nullptr;
+ }
+ auto getBits = [](int32_t value) {
+ uint8_t bit = 0;
+ do {
+ value >>= 1;
+ bit++;
+ } while (value > 0);
+ return bit;
+ };
+ uint8_t bitsW = getBits(aInfo.mImage.Width());
+ uint8_t bitsH = getBits(aInfo.mImage.Height());
+ bw.WriteBits(bitsW - 1, 4);
+ bw.WriteBits(bitsH - 1, 4);
+ bw.WriteBits(aInfo.mImage.Width() - 1, bitsW);
+ bw.WriteBits(aInfo.mImage.Height() - 1, bitsH);
+
+ // if ( !reduced_still_picture_header )
+ bw.WriteBit(false); // frame_id_numbers_present_flag
+ // if ( frame_id_numbers_present_flag ) {...}
+ // end if ( !reduced_still_picture_header )
+
+ // Values below are derived from a 1080p YouTube AV1 stream.
+ // The values are unused currently for determining the usable
+ // decoder, and are only included to allow successful validation
+ // of the generated sequence header.
+
+ bw.WriteBit(true); // use_128x128_superblock
+ bw.WriteBit(true); // enable_filter_intra
+ bw.WriteBit(true); // enable_intra_edge_filter
+
+ // if ( !reduced_still_picture_header)
+ bw.WriteBit(false); // enable_interintra_compound
+ bw.WriteBit(true); // enable_masked_compound
+ bw.WriteBit(true); // enable_warped_motion
+ bw.WriteBit(false); // enable_dual_filter
+
+ bw.WriteBit(true); // enable_order_hint
+ // if ( enable_order_hint )
+ bw.WriteBit(false); // enable_jnt_comp
+ bw.WriteBit(true); // enable_ref_frame_mvs
+ // end if ( enable_order_hint )
+
+ bw.WriteBit(true); // seq_choose_screen_content_tools
+ // if ( seq_choose_screen_content_tools )
+ // seq_force_screen_content_tools = SELECT_SCREEN_CONTENT_TOOLS (2)
+ // else
+ // seq_force_screen_content_tools = f(1)
+
+ // if ( seq_force_screen_content_tools > 0 )
+ bw.WriteBit(true); // seq_choose_integer_mv
+ // if ( !seq_choose_integer_mv ) {...}
+ // end if ( seq_force_screen_content_tools > 0 )
+
+ // if ( enable_order_hint )
+ bw.WriteBits(6, 3); // order_hint_bits_minus_1
+ // end if ( enable_order_hint )
+ // end if ( !reduced_still_picture_header )
+
+ bw.WriteBit(false); // enable_superres
+ bw.WriteBit(false); // enable_cdef
+ bw.WriteBit(true); // enable_restoration
+
+ // Begin color_config( )
+ // https://aomediacodec.github.io/av1-spec/#color-config-syntax
+ bool highBitDepth = aInfo.mBitDepth >= 10;
+ bw.WriteBit(highBitDepth);
+
+ if (aInfo.mBitDepth == 12 && aInfo.mProfile != 2) {
+ NS_WARNING("Profile must be 2 for 12-bit");
+ return nullptr;
+ }
+ if (aInfo.mProfile == 2 && highBitDepth) {
+ bw.WriteBit(aInfo.mBitDepth == 12); // twelve_bit
+ }
+
+ if (aInfo.mMonochrome && aInfo.mProfile == 1) {
+ NS_WARNING("Profile 1 does not support monochrome");
+ return nullptr;
+ }
+ if (aInfo.mProfile != 1) {
+ bw.WriteBit(aInfo.mMonochrome);
+ }
+
+ const VideoColorSpace colors = aInfo.mColorSpace;
+ bool colorsPresent =
+ colors.mPrimaries != ColourPrimaries::CP_UNSPECIFIED ||
+ colors.mTransfer != TransferCharacteristics::TC_UNSPECIFIED ||
+ colors.mMatrix != MatrixCoefficients::MC_UNSPECIFIED;
+ bw.WriteBit(colorsPresent);
+
+ if (colorsPresent) {
+ bw.WriteBits(static_cast<uint8_t>(colors.mPrimaries), 8);
+ bw.WriteBits(static_cast<uint8_t>(colors.mTransfer), 8);
+ bw.WriteBits(static_cast<uint8_t>(colors.mMatrix), 8);
+ }
+
+ if (aInfo.mMonochrome) {
+ if (!aInfo.mSubsamplingX || !aInfo.mSubsamplingY) {
+ NS_WARNING("Monochrome requires 4:0:0 subsampling");
+ return nullptr;
+ }
+ if (aInfo.mChromaSamplePosition != ChromaSamplePosition::Unknown) {
+ NS_WARNING(
+ "Cannot specify chroma sample position on monochrome sequence");
+ return nullptr;
+ }
+ bw.WriteBit(colors.mRange == ColorRange::FULL);
+ } else if (colors.mPrimaries == ColourPrimaries::CP_BT709 &&
+ colors.mTransfer == TransferCharacteristics::TC_SRGB &&
+ colors.mMatrix == MatrixCoefficients::MC_IDENTITY) {
+ if (aInfo.mSubsamplingX || aInfo.mSubsamplingY ||
+ colors.mRange != ColorRange::FULL ||
+ aInfo.mChromaSamplePosition != ChromaSamplePosition::Unknown) {
+ NS_WARNING("sRGB requires 4:4:4 subsampling with full color range");
+ return nullptr;
+ }
+ } else {
+ bw.WriteBit(colors.mRange == ColorRange::FULL);
+ switch (aInfo.mProfile) {
+ case 0:
+ if (!aInfo.mSubsamplingX || !aInfo.mSubsamplingY) {
+ NS_WARNING("Main Profile requires 4:2:0 subsampling");
+ return nullptr;
+ }
+ break;
+ case 1:
+ if (aInfo.mSubsamplingX || aInfo.mSubsamplingY) {
+ NS_WARNING("High Profile requires 4:4:4 subsampling");
+ return nullptr;
+ }
+ break;
+ case 2:
+ if (aInfo.mBitDepth == 12) {
+ bw.WriteBit(aInfo.mSubsamplingX);
+ if (aInfo.mSubsamplingX) {
+ bw.WriteBit(aInfo.mSubsamplingY);
+ }
+ } else {
+ if (!aInfo.mSubsamplingX || aInfo.mSubsamplingY) {
+ NS_WARNING(
+ "Professional Profile < 12-bit requires 4:2:2 subsampling");
+ return nullptr;
+ }
+ }
+ break;
+ }
+
+ if (aInfo.mSubsamplingX && aInfo.mSubsamplingY) {
+ bw.WriteBits(static_cast<uint8_t>(aInfo.mChromaSamplePosition), 2);
+ } else {
+ if (aInfo.mChromaSamplePosition != ChromaSamplePosition::Unknown) {
+ NS_WARNING("Only 4:2:0 subsampling can specify chroma position");
+ return nullptr;
+ }
+ }
+ }
+
+ bw.WriteBit(false); // separate_uv_delta_q
+ // end color_config( )
+
+ bw.WriteBit(true); // film_grain_params_present
+
+ // trailing_bits( )
+ // https://aomediacodec.github.io/av1-spec/#trailing-bits-syntax
+ size_t numTrailingBits = 8 - (bw.BitCount() % 8);
+ bw.WriteBit(true);
+ bw.WriteBits(0, numTrailingBits - 1);
+ ASSERT_BYTE_ALIGNED(bw);
+
+ Span<const uint8_t> seqHdr(seqHdrBuffer->Elements(), seqHdrBuffer->Length());
+ aResult = NS_OK;
+ return CreateOBU(OBUType::SequenceHeader, seqHdr);
+}
+
+/* static */
+void AOMDecoder::TryReadAV1CBox(const MediaByteBuffer* aBox,
+ AV1SequenceInfo& aDestInfo,
+ MediaResult& aSeqHdrResult) {
+ // See av1C specification:
+ // https://aomediacodec.github.io/av1-isobmff/#av1codecconfigurationbox-section
+ BitReader br(aBox);
+
+ br.ReadBits(8); // marker, version
+
+ aDestInfo.mProfile = br.ReadBits(3);
+
+ OperatingPoint op;
+ op.mLevel = br.ReadBits(5);
+ op.mTier = br.ReadBits(1);
+ aDestInfo.mOperatingPoints.AppendElement(op);
+
+ bool highBitDepth = br.ReadBit();
+ bool twelveBit = br.ReadBit();
+ aDestInfo.mBitDepth = highBitDepth ? twelveBit ? 12 : 10 : 8;
+
+ aDestInfo.mMonochrome = br.ReadBit();
+ aDestInfo.mSubsamplingX = br.ReadBit();
+ aDestInfo.mSubsamplingY = br.ReadBit();
+ aDestInfo.mChromaSamplePosition =
+ static_cast<ChromaSamplePosition>(br.ReadBits(2));
+
+ br.ReadBits(3); // reserved
+ br.ReadBit(); // initial_presentation_delay_present
+ br.ReadBits(4); // initial_presentation_delay_minus_one or reserved
+
+ ASSERT_BYTE_ALIGNED(br);
+
+ size_t skipBytes = br.BitCount() / 8;
+ Span<const uint8_t> obus(aBox->Elements() + skipBytes,
+ aBox->Length() - skipBytes);
+
+ // Minimum possible OBU header size
+ if (obus.Length() < 1) {
+ aSeqHdrResult = NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA;
+ return;
+ }
+
+ // If present, the sequence header will be redundant to some values, but any
+ // values stored in it should be treated as more accurate than av1C.
+ aSeqHdrResult = ReadSequenceHeaderInfo(obus, aDestInfo);
+}
+
+/* static */
+void AOMDecoder::WriteAV1CBox(const AV1SequenceInfo& aInfo,
+ MediaByteBuffer* aDestBox, bool& aHasSeqHdr) {
+ aHasSeqHdr = false;
+
+ BitWriter bw(aDestBox);
+
+ bw.WriteBit(true); // marker
+ bw.WriteBits(1, 7); // version
+
+ bw.WriteBits(aInfo.mProfile, 3);
+
+ MOZ_DIAGNOSTIC_ASSERT(aInfo.mOperatingPoints.Length() > 0);
+ bw.WriteBits(aInfo.mOperatingPoints[0].mLevel, 5);
+ bw.WriteBits(aInfo.mOperatingPoints[0].mTier, 1);
+
+ bw.WriteBit(aInfo.mBitDepth >= 10); // high_bitdepth
+ bw.WriteBit(aInfo.mBitDepth == 12); // twelve_bit
+
+ bw.WriteBit(aInfo.mMonochrome);
+ bw.WriteBit(aInfo.mSubsamplingX);
+ bw.WriteBit(aInfo.mSubsamplingY);
+ bw.WriteBits(static_cast<uint8_t>(aInfo.mChromaSamplePosition), 2);
+
+ bw.WriteBits(0, 3); // reserved
+ bw.WriteBit(false); // initial_presentation_delay_present
+ bw.WriteBits(0, 4); // initial_presentation_delay_minus_one or reserved
+
+ ASSERT_BYTE_ALIGNED(bw);
+
+ nsresult rv;
+ RefPtr<MediaByteBuffer> seqHdrBuffer = CreateSequenceHeader(aInfo, rv);
+
+ if (NS_SUCCEEDED(rv)) {
+ aDestBox->AppendElements(seqHdrBuffer->Elements(), seqHdrBuffer->Length());
+ aHasSeqHdr = true;
+ }
+}
+
+/* static */
+Maybe<AOMDecoder::AV1SequenceInfo> AOMDecoder::CreateSequenceInfoFromCodecs(
+ const nsAString& aCodec) {
+ AV1SequenceInfo info;
+ OperatingPoint op;
+ uint8_t chromaSamplePosition;
+ if (!ExtractAV1CodecDetails(aCodec, info.mProfile, op.mLevel, op.mTier,
+ info.mBitDepth, info.mMonochrome,
+ info.mSubsamplingX, info.mSubsamplingY,
+ chromaSamplePosition, info.mColorSpace)) {
+ return Nothing();
+ }
+ info.mOperatingPoints.AppendElement(op);
+ info.mChromaSamplePosition =
+ static_cast<ChromaSamplePosition>(chromaSamplePosition);
+ return Some(info);
+}
+
+/* static */
+bool AOMDecoder::SetVideoInfo(VideoInfo* aDestInfo, const nsAString& aCodec) {
+ Maybe<AV1SequenceInfo> info = CreateSequenceInfoFromCodecs(aCodec);
+ if (info.isNothing()) {
+ return false;
+ }
+
+ if (!aDestInfo->mImage.IsEmpty()) {
+ info->mImage = aDestInfo->mImage;
+ }
+
+ RefPtr<MediaByteBuffer> extraData = new MediaByteBuffer();
+ bool hasSeqHdr;
+ WriteAV1CBox(info.value(), extraData, hasSeqHdr);
+ aDestInfo->mExtraData = extraData;
+ return true;
+}
+
+} // namespace mozilla
+#undef LOG
+#undef ASSERT_BYTE_ALIGNED
diff --git a/dom/media/platforms/agnostic/AOMDecoder.h b/dom/media/platforms/agnostic/AOMDecoder.h
new file mode 100644
index 0000000000..f9620339e0
--- /dev/null
+++ b/dom/media/platforms/agnostic/AOMDecoder.h
@@ -0,0 +1,287 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+#if !defined(AOMDecoder_h_)
+# define AOMDecoder_h_
+
+# include <stdint.h>
+
+# include "PerformanceRecorder.h"
+# include "PlatformDecoderModule.h"
+# include <aom/aom_decoder.h>
+# include "mozilla/Span.h"
+# include "VideoUtils.h"
+
+namespace mozilla {
+
+DDLoggedTypeDeclNameAndBase(AOMDecoder, MediaDataDecoder);
+
+class AOMDecoder final : public MediaDataDecoder,
+ public DecoderDoctorLifeLogger<AOMDecoder> {
+ public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AOMDecoder, final);
+
+ explicit AOMDecoder(const CreateDecoderParams& aParams);
+
+ RefPtr<InitPromise> Init() override;
+ RefPtr<DecodePromise> Decode(MediaRawData* aSample) override;
+ RefPtr<DecodePromise> Drain() override;
+ RefPtr<FlushPromise> Flush() override;
+ RefPtr<ShutdownPromise> Shutdown() override;
+ nsCString GetDescriptionName() const override {
+ return "av1 libaom video decoder"_ns;
+ }
+ nsCString GetCodecName() const override { return "av1"_ns; }
+
+ // Return true if aMimeType is a one of the strings used
+ // by our demuxers to identify AV1 streams.
+ static bool IsAV1(const nsACString& aMimeType);
+
+ // Return true if a sample is a keyframe.
+ static bool IsKeyframe(Span<const uint8_t> aBuffer);
+
+ // Return the frame dimensions for a sample.
+ static gfx::IntSize GetFrameSize(Span<const uint8_t> aBuffer);
+
+ // obu_type defined at:
+ // https://aomediacodec.github.io/av1-spec/av1-spec.pdf#page=123
+ enum class OBUType : uint8_t {
+ Reserved = 0,
+ SequenceHeader = 1,
+ TemporalDelimiter = 2,
+ FrameHeader = 3,
+ TileGroup = 4,
+ Metadata = 5,
+ Frame = 6,
+ RedundantFrameHeader = 7,
+ TileList = 8,
+ Padding = 15
+ };
+
+ struct OBUInfo {
+ OBUType mType = OBUType::Reserved;
+ bool mExtensionFlag = false;
+ Span<const uint8_t> mContents;
+
+ bool IsValid() const {
+ switch (mType) {
+ case OBUType::SequenceHeader:
+ case OBUType::TemporalDelimiter:
+ case OBUType::FrameHeader:
+ case OBUType::TileGroup:
+ case OBUType::Metadata:
+ case OBUType::Frame:
+ case OBUType::RedundantFrameHeader:
+ case OBUType::TileList:
+ case OBUType::Padding:
+ return true;
+ default:
+ return false;
+ }
+ }
+ };
+
+ struct OBUIterator {
+ public:
+ explicit OBUIterator(const Span<const uint8_t>& aData)
+ : mData(aData), mPosition(0), mGoNext(true), mResult(NS_OK) {}
+ bool HasNext() {
+ UpdateNext();
+ return !mGoNext;
+ }
+ OBUInfo Next() {
+ UpdateNext();
+ mGoNext = true;
+ return mCurrent;
+ }
+ MediaResult GetResult() const { return mResult; }
+
+ private:
+ const Span<const uint8_t>& mData;
+ size_t mPosition;
+ OBUInfo mCurrent;
+ bool mGoNext;
+ MediaResult mResult;
+
+ // Used to fill mCurrent with the next OBU in the iterator.
+ // mGoNext must be set to false if the next OBU is retrieved,
+ // otherwise it will be true so that HasNext() returns false.
+ // When an invalid OBU is read, the iterator will finish and
+ // mCurrent will be reset to default OBUInfo().
+ void UpdateNext();
+ };
+
+ // Create an iterator to parse Open Bitstream Units from a buffer.
+ static OBUIterator ReadOBUs(const Span<const uint8_t>& aData);
+ // Writes an Open Bitstream Unit header type and the contained subheader.
+ // Extension flag is set to 0 and size field is always present.
+ static already_AddRefed<MediaByteBuffer> CreateOBU(
+ const OBUType aType, const Span<const uint8_t>& aContents);
+
+ // chroma_sample_position defined at:
+ // https://aomediacodec.github.io/av1-spec/av1-spec.pdf#page=131
+ enum class ChromaSamplePosition : uint8_t {
+ Unknown = 0,
+ Vertical = 1,
+ Colocated = 2,
+ Reserved = 3
+ };
+
+ struct OperatingPoint {
+ // https://aomediacodec.github.io/av1-spec/av1-spec.pdf#page=125
+ // operating_point_idc[ i ]: A set of bitwise flags determining
+ // the temporal and spatial layers to decode.
+ // A value of 0 indicates that scalability is not being used.
+ uint16_t mLayers = 0;
+ // https://aomediacodec.github.io/av1-spec/av1-spec.pdf#page=650
+ // See A.3: Levels for a definition of the available levels.
+ uint8_t mLevel = 0;
+ // https://aomediacodec.github.io/av1-spec/av1-spec.pdf#page=126
+ // seq_tier[ i ]: The tier for the selected operating point.
+ uint8_t mTier = 0;
+
+ bool operator==(const OperatingPoint& aOther) const {
+ return mLayers == aOther.mLayers && mLevel == aOther.mLevel &&
+ mTier == aOther.mTier;
+ }
+ bool operator!=(const OperatingPoint& aOther) const {
+ return !(*this == aOther);
+ }
+ };
+
+ struct AV1SequenceInfo {
+ AV1SequenceInfo() = default;
+
+ AV1SequenceInfo(const AV1SequenceInfo& aOther) { *this = aOther; }
+
+ // Profiles, levels and tiers defined at:
+ // https://aomediacodec.github.io/av1-spec/av1-spec.pdf#page=650
+ uint8_t mProfile = 0;
+
+ // choose_operating_point( ) defines that the operating points are
+ // specified in order of preference by the encoder. Higher operating
+ // points indices in the header will allow a tradeoff of quality for
+ // performance, dropping some data from the decoding process.
+ // Normally we are only interested in the first operating point.
+ // See: https://aomediacodec.github.io/av1-spec/av1-spec.pdf#page=126
+ nsTArray<OperatingPoint> mOperatingPoints = nsTArray<OperatingPoint>(1);
+
+ gfx::IntSize mImage = {0, 0};
+
+ // Color configs explained at:
+ // https://aomediacodec.github.io/av1-spec/av1-spec.pdf#page=129
+ uint8_t mBitDepth = 8;
+ bool mMonochrome = false;
+ bool mSubsamplingX = true;
+ bool mSubsamplingY = true;
+ ChromaSamplePosition mChromaSamplePosition = ChromaSamplePosition::Unknown;
+
+ VideoColorSpace mColorSpace;
+
+ gfx::ColorDepth ColorDepth() const {
+ return gfx::ColorDepthForBitDepth(mBitDepth);
+ }
+
+ bool operator==(const AV1SequenceInfo& aOther) const {
+ if (mProfile != aOther.mProfile || mImage != aOther.mImage ||
+ mBitDepth != aOther.mBitDepth || mMonochrome != aOther.mMonochrome ||
+ mSubsamplingX != aOther.mSubsamplingX ||
+ mSubsamplingY != aOther.mSubsamplingY ||
+ mChromaSamplePosition != aOther.mChromaSamplePosition ||
+ mColorSpace != aOther.mColorSpace) {
+ return false;
+ }
+
+ size_t opCount = mOperatingPoints.Length();
+ if (opCount != aOther.mOperatingPoints.Length()) {
+ return false;
+ }
+ for (size_t i = 0; i < opCount; i++) {
+ if (mOperatingPoints[i] != aOther.mOperatingPoints[i]) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+ bool operator!=(const AV1SequenceInfo& aOther) const {
+ return !(*this == aOther);
+ }
+ AV1SequenceInfo& operator=(const AV1SequenceInfo& aOther) {
+ mProfile = aOther.mProfile;
+
+ size_t opCount = aOther.mOperatingPoints.Length();
+ mOperatingPoints.ClearAndRetainStorage();
+ mOperatingPoints.SetCapacity(opCount);
+ for (size_t i = 0; i < opCount; i++) {
+ mOperatingPoints.AppendElement(aOther.mOperatingPoints[i]);
+ }
+
+ mImage = aOther.mImage;
+ mBitDepth = aOther.mBitDepth;
+ mMonochrome = aOther.mMonochrome;
+ mSubsamplingX = aOther.mSubsamplingX;
+ mSubsamplingY = aOther.mSubsamplingY;
+ mChromaSamplePosition = aOther.mChromaSamplePosition;
+ mColorSpace = aOther.mColorSpace;
+ return *this;
+ }
+ };
+
+ // Get a sequence header's info from a sample.
+ // Returns a MediaResult with codes:
+ // NS_OK: Sequence header was successfully found and read.
+ // NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA: Sequence header was not present.
+ // Other errors will indicate that the data was corrupt.
+ static MediaResult ReadSequenceHeaderInfo(const Span<const uint8_t>& aSample,
+ AV1SequenceInfo& aDestInfo);
+ // Writes a sequence header OBU to the buffer.
+ static already_AddRefed<MediaByteBuffer> CreateSequenceHeader(
+ const AV1SequenceInfo& aInfo, nsresult& aResult);
+
+ // Reads the raw data of an ISOBMFF-compatible av1 configuration box (av1C),
+ // including any included sequence header.
+ static void TryReadAV1CBox(const MediaByteBuffer* aBox,
+ AV1SequenceInfo& aDestInfo,
+ MediaResult& aSeqHdrResult);
+ // Reads the raw data of an ISOBMFF-compatible av1 configuration box (av1C),
+ // including any included sequence header.
+ // This function should only be called for av1C boxes made by WriteAV1CBox, as
+ // it will assert that the box and its contained OBUs are not corrupted.
+ static void ReadAV1CBox(const MediaByteBuffer* aBox,
+ AV1SequenceInfo& aDestInfo, bool& aHadSeqHdr) {
+ MediaResult seqHdrResult;
+ TryReadAV1CBox(aBox, aDestInfo, seqHdrResult);
+ nsresult code = seqHdrResult.Code();
+ MOZ_ASSERT(code == NS_OK || code == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA);
+ aHadSeqHdr = code == NS_OK;
+ }
+ // Writes an ISOBMFF-compatible av1 configuration box (av1C) to the buffer.
+ static void WriteAV1CBox(const AV1SequenceInfo& aInfo,
+ MediaByteBuffer* aDestBox, bool& aHasSeqHdr);
+
+ // Create sequence info from a MIME codecs string.
+ static Maybe<AV1SequenceInfo> CreateSequenceInfoFromCodecs(
+ const nsAString& aCodec);
+ static bool SetVideoInfo(VideoInfo* aDestInfo, const nsAString& aCodec);
+
+ private:
+ ~AOMDecoder();
+ RefPtr<DecodePromise> ProcessDecode(MediaRawData* aSample);
+
+ const RefPtr<layers::ImageContainer> mImageContainer;
+ const RefPtr<TaskQueue> mTaskQueue;
+
+ // AOM decoder state
+ aom_codec_ctx_t mCodec;
+
+ const VideoInfo mInfo;
+ const Maybe<TrackingId> mTrackingId;
+ PerformanceRecorderMulti<DecodeStage> mPerformanceRecorder;
+};
+
+} // namespace mozilla
+
+#endif // AOMDecoder_h_
diff --git a/dom/media/platforms/agnostic/AgnosticDecoderModule.cpp b/dom/media/platforms/agnostic/AgnosticDecoderModule.cpp
new file mode 100644
index 0000000000..7bdc30b432
--- /dev/null
+++ b/dom/media/platforms/agnostic/AgnosticDecoderModule.cpp
@@ -0,0 +1,185 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "AgnosticDecoderModule.h"
+
+#include "TheoraDecoder.h"
+#include "VPXDecoder.h"
+#include "mozilla/Logging.h"
+#include "mozilla/StaticPrefs_media.h"
+#include "VideoUtils.h"
+
+#ifdef MOZ_AV1
+# include "AOMDecoder.h"
+# include "DAV1DDecoder.h"
+#endif
+
+namespace mozilla {
+
+enum class DecoderType {
+#ifdef MOZ_AV1
+ AV1,
+#endif
+ Opus,
+ Theora,
+ Vorbis,
+ VPX,
+ Wave,
+};
+
+static bool IsAvailableInDefault(DecoderType type) {
+ switch (type) {
+#ifdef MOZ_AV1
+ case DecoderType::AV1:
+ return StaticPrefs::media_av1_enabled();
+#endif
+ case DecoderType::Opus:
+ case DecoderType::Theora:
+ case DecoderType::Vorbis:
+ case DecoderType::VPX:
+ case DecoderType::Wave:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static bool IsAvailableInRdd(DecoderType type) {
+ switch (type) {
+#ifdef MOZ_AV1
+ case DecoderType::AV1:
+ return StaticPrefs::media_av1_enabled();
+#endif
+ case DecoderType::Opus:
+ return StaticPrefs::media_rdd_opus_enabled();
+ case DecoderType::Theora:
+ return StaticPrefs::media_rdd_theora_enabled();
+ case DecoderType::Vorbis:
+#if defined(__MINGW32__)
+ // If this is a MinGW build we need to force AgnosticDecoderModule to
+ // handle the decision to support Vorbis decoding (instead of
+ // RDD/RemoteDecoderModule) because of Bug 1597408 (Vorbis decoding on
+ // RDD causing sandboxing failure on MinGW-clang). Typically this
+ // would be dealt with using defines in StaticPrefList.yaml, but we
+ // must handle it here because of Bug 1598426 (the __MINGW32__ define
+ // isn't supported in StaticPrefList.yaml).
+ return false;
+#else
+ return StaticPrefs::media_rdd_vorbis_enabled();
+#endif
+ case DecoderType::VPX:
+ return StaticPrefs::media_rdd_vpx_enabled();
+ case DecoderType::Wave:
+ return StaticPrefs::media_rdd_wav_enabled();
+ default:
+ return false;
+ }
+}
+
+static bool IsAvailableInUtility(DecoderType type) {
+ switch (type) {
+ case DecoderType::Opus:
+ return StaticPrefs::media_utility_opus_enabled();
+ case DecoderType::Vorbis:
+ return StaticPrefs::media_utility_vorbis_enabled();
+ case DecoderType::Wave:
+ return StaticPrefs::media_utility_wav_enabled();
+ case DecoderType::Theora: // Video codecs, dont take care of them
+ case DecoderType::VPX:
+ default:
+ return false;
+ }
+}
+
+// Checks if decoder is available in the current process
+static bool IsAvailable(DecoderType type) {
+ return XRE_IsRDDProcess() ? IsAvailableInRdd(type)
+ : XRE_IsUtilityProcess() ? IsAvailableInUtility(type)
+ : IsAvailableInDefault(type);
+}
+
+media::DecodeSupportSet AgnosticDecoderModule::SupportsMimeType(
+ const nsACString& aMimeType, DecoderDoctorDiagnostics* aDiagnostics) const {
+ UniquePtr<TrackInfo> trackInfo = CreateTrackInfoWithMIMEType(aMimeType);
+ if (!trackInfo) {
+ return media::DecodeSupportSet{};
+ }
+ return Supports(SupportDecoderParams(*trackInfo), aDiagnostics);
+}
+
+media::DecodeSupportSet AgnosticDecoderModule::Supports(
+ const SupportDecoderParams& aParams,
+ DecoderDoctorDiagnostics* aDiagnostics) const {
+ // This should only be supported by MFMediaEngineDecoderModule.
+ if (aParams.mMediaEngineId) {
+ return media::DecodeSupportSet{};
+ }
+
+ const auto& trackInfo = aParams.mConfig;
+ const nsACString& mimeType = trackInfo.mMimeType;
+
+ bool supports =
+#ifdef MOZ_AV1
+ // We remove support for decoding AV1 here if RDD is enabled so that
+ // decoding on the content process doesn't accidentally happen in case
+ // something goes wrong with launching the RDD process.
+ (AOMDecoder::IsAV1(mimeType) && IsAvailable(DecoderType::AV1)) ||
+#endif
+ (VPXDecoder::IsVPX(mimeType) && IsAvailable(DecoderType::VPX)) ||
+ (TheoraDecoder::IsTheora(mimeType) && IsAvailable(DecoderType::Theora));
+ MOZ_LOG(sPDMLog, LogLevel::Debug,
+ ("Agnostic decoder %s requested type '%s'",
+ supports ? "supports" : "rejects", mimeType.BeginReading()));
+ if (supports) {
+ return media::DecodeSupport::SoftwareDecode;
+ }
+ return media::DecodeSupportSet{};
+}
+
+already_AddRefed<MediaDataDecoder> AgnosticDecoderModule::CreateVideoDecoder(
+ const CreateDecoderParams& aParams) {
+ if (Supports(SupportDecoderParams(aParams), nullptr /* diagnostic */)
+ .isEmpty()) {
+ return nullptr;
+ }
+ RefPtr<MediaDataDecoder> m;
+
+ if (VPXDecoder::IsVPX(aParams.mConfig.mMimeType)) {
+ m = new VPXDecoder(aParams);
+ }
+#ifdef MOZ_AV1
+ // We remove support for decoding AV1 here if RDD is enabled so that
+ // decoding on the content process doesn't accidentally happen in case
+ // something goes wrong with launching the RDD process.
+ if (StaticPrefs::media_av1_enabled() &&
+ (!StaticPrefs::media_rdd_process_enabled() || XRE_IsRDDProcess()) &&
+ AOMDecoder::IsAV1(aParams.mConfig.mMimeType)) {
+ if (StaticPrefs::media_av1_use_dav1d()) {
+ m = new DAV1DDecoder(aParams);
+ } else {
+ m = new AOMDecoder(aParams);
+ }
+ }
+#endif
+ else if (TheoraDecoder::IsTheora(aParams.mConfig.mMimeType)) {
+ m = new TheoraDecoder(aParams);
+ }
+
+ return m.forget();
+}
+
+already_AddRefed<MediaDataDecoder> AgnosticDecoderModule::CreateAudioDecoder(
+ const CreateDecoderParams& aParams) {
+ return nullptr;
+}
+
+/* static */
+already_AddRefed<PlatformDecoderModule> AgnosticDecoderModule::Create() {
+ RefPtr<PlatformDecoderModule> pdm = new AgnosticDecoderModule();
+ return pdm.forget();
+}
+
+} // namespace mozilla
diff --git a/dom/media/platforms/agnostic/AgnosticDecoderModule.h b/dom/media/platforms/agnostic/AgnosticDecoderModule.h
new file mode 100644
index 0000000000..bac5f4bf42
--- /dev/null
+++ b/dom/media/platforms/agnostic/AgnosticDecoderModule.h
@@ -0,0 +1,39 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#if !defined(AgnosticDecoderModule_h_)
+# define AgnosticDecoderModule_h_
+
+# include "PlatformDecoderModule.h"
+
+namespace mozilla {
+
+class AgnosticDecoderModule : public PlatformDecoderModule {
+ public:
+ static already_AddRefed<PlatformDecoderModule> Create();
+
+ media::DecodeSupportSet SupportsMimeType(
+ const nsACString& aMimeType,
+ DecoderDoctorDiagnostics* aDiagnostics) const override;
+ media::DecodeSupportSet Supports(
+ const SupportDecoderParams& aParams,
+ DecoderDoctorDiagnostics* aDiagnostics) const override;
+
+ protected:
+ AgnosticDecoderModule() = default;
+ virtual ~AgnosticDecoderModule() = default;
+ // Decode thread.
+ already_AddRefed<MediaDataDecoder> CreateVideoDecoder(
+ const CreateDecoderParams& aParams) override;
+
+ // Decode thread.
+ already_AddRefed<MediaDataDecoder> CreateAudioDecoder(
+ const CreateDecoderParams& aParams) override;
+};
+
+} // namespace mozilla
+
+#endif /* AgnosticDecoderModule_h_ */
diff --git a/dom/media/platforms/agnostic/BlankDecoderModule.cpp b/dom/media/platforms/agnostic/BlankDecoderModule.cpp
new file mode 100644
index 0000000000..05ce51d0c1
--- /dev/null
+++ b/dom/media/platforms/agnostic/BlankDecoderModule.cpp
@@ -0,0 +1,146 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "BlankDecoderModule.h"
+
+#include "mozilla/CheckedInt.h"
+#include "mozilla/UniquePtrExtensions.h"
+#include "mozilla/RefPtr.h"
+#include "mozilla/gfx/Rect.h"
+#include "mozilla/gfx/Point.h"
+#include "ImageContainer.h"
+#include "MediaData.h"
+#include "MediaInfo.h"
+#include "VideoUtils.h"
+
+namespace mozilla {
+
+BlankVideoDataCreator::BlankVideoDataCreator(
+ uint32_t aFrameWidth, uint32_t aFrameHeight,
+ layers::ImageContainer* aImageContainer)
+ : mFrameWidth(aFrameWidth),
+ mFrameHeight(aFrameHeight),
+ mImageContainer(aImageContainer) {
+ mInfo.mDisplay = gfx::IntSize(mFrameWidth, mFrameHeight);
+ mPicture = gfx::IntRect(0, 0, mFrameWidth, mFrameHeight);
+}
+
+already_AddRefed<MediaData> BlankVideoDataCreator::Create(
+ MediaRawData* aSample) {
+ // Create a fake YUV buffer in a 420 format. That is, an 8bpp Y plane,
+ // with a U and V plane that are half the size of the Y plane, i.e 8 bit,
+ // 2x2 subsampled. Have the data pointer of each frame point to the
+ // first plane, they'll always be zero'd memory anyway.
+ const CheckedUint32 size = CheckedUint32(mFrameWidth) * mFrameHeight;
+ if (!size.isValid()) {
+ // Overflow happened.
+ return nullptr;
+ }
+ auto frame = MakeUniqueFallible<uint8_t[]>(size.value());
+ if (!frame) {
+ return nullptr;
+ }
+ memset(frame.get(), 0, mFrameWidth * mFrameHeight);
+ VideoData::YCbCrBuffer buffer;
+
+ // Y plane.
+ buffer.mPlanes[0].mData = frame.get();
+ buffer.mPlanes[0].mStride = mFrameWidth;
+ buffer.mPlanes[0].mHeight = mFrameHeight;
+ buffer.mPlanes[0].mWidth = mFrameWidth;
+ buffer.mPlanes[0].mSkip = 0;
+
+ // Cb plane.
+ buffer.mPlanes[1].mData = frame.get();
+ buffer.mPlanes[1].mStride = (mFrameWidth + 1) / 2;
+ buffer.mPlanes[1].mHeight = (mFrameHeight + 1) / 2;
+ buffer.mPlanes[1].mWidth = (mFrameWidth + 1) / 2;
+ buffer.mPlanes[1].mSkip = 0;
+
+ // Cr plane.
+ buffer.mPlanes[2].mData = frame.get();
+ buffer.mPlanes[2].mStride = (mFrameWidth + 1) / 2;
+ buffer.mPlanes[2].mHeight = (mFrameHeight + 1) / 2;
+ buffer.mPlanes[2].mWidth = (mFrameWidth + 1) / 2;
+ buffer.mPlanes[2].mSkip = 0;
+
+ buffer.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT;
+ buffer.mYUVColorSpace = gfx::YUVColorSpace::BT601;
+ buffer.mColorPrimaries = gfx::ColorSpace2::BT709;
+
+ Result<already_AddRefed<VideoData>, MediaResult> result =
+ VideoData::CreateAndCopyData(mInfo, mImageContainer, aSample->mOffset,
+ aSample->mTime, aSample->mDuration, buffer,
+ aSample->mKeyframe, aSample->mTime, mPicture,
+ nullptr);
+ return result.unwrapOr(nullptr);
+}
+
+BlankAudioDataCreator::BlankAudioDataCreator(uint32_t aChannelCount,
+ uint32_t aSampleRate)
+ : mFrameSum(0), mChannelCount(aChannelCount), mSampleRate(aSampleRate) {}
+
+already_AddRefed<MediaData> BlankAudioDataCreator::Create(
+ MediaRawData* aSample) {
+ // Convert duration to frames. We add 1 to duration to account for
+ // rounding errors, so we get a consistent tone.
+ CheckedInt64 frames =
+ UsecsToFrames(aSample->mDuration.ToMicroseconds() + 1, mSampleRate);
+ if (!frames.isValid() || !mChannelCount || !mSampleRate ||
+ frames.value() > (UINT32_MAX / mChannelCount)) {
+ return nullptr;
+ }
+ AlignedAudioBuffer samples(frames.value() * mChannelCount);
+ if (!samples) {
+ return nullptr;
+ }
+ // Fill the sound buffer with an A4 tone.
+ static const float pi = 3.14159265f;
+ static const float noteHz = 440.0f;
+ for (int i = 0; i < frames.value(); i++) {
+ float f = sin(2 * pi * noteHz * mFrameSum / mSampleRate);
+ for (unsigned c = 0; c < mChannelCount; c++) {
+ samples[i * mChannelCount + c] = AudioDataValue(f);
+ }
+ mFrameSum++;
+ }
+ RefPtr<AudioData> data(new AudioData(aSample->mOffset, aSample->mTime,
+ std::move(samples), mChannelCount,
+ mSampleRate));
+ return data.forget();
+}
+
+already_AddRefed<MediaDataDecoder> BlankDecoderModule::CreateVideoDecoder(
+ const CreateDecoderParams& aParams) {
+ const VideoInfo& config = aParams.VideoConfig();
+ UniquePtr<DummyDataCreator> creator = MakeUnique<BlankVideoDataCreator>(
+ config.mDisplay.width, config.mDisplay.height, aParams.mImageContainer);
+ RefPtr<MediaDataDecoder> decoder = new DummyMediaDataDecoder(
+ std::move(creator), "blank media data decoder"_ns, aParams);
+ return decoder.forget();
+}
+
+already_AddRefed<MediaDataDecoder> BlankDecoderModule::CreateAudioDecoder(
+ const CreateDecoderParams& aParams) {
+ const AudioInfo& config = aParams.AudioConfig();
+ UniquePtr<DummyDataCreator> creator =
+ MakeUnique<BlankAudioDataCreator>(config.mChannels, config.mRate);
+ RefPtr<MediaDataDecoder> decoder = new DummyMediaDataDecoder(
+ std::move(creator), "blank media data decoder"_ns, aParams);
+ return decoder.forget();
+}
+
+media::DecodeSupportSet BlankDecoderModule::SupportsMimeType(
+ const nsACString& aMimeType, DecoderDoctorDiagnostics* aDiagnostics) const {
+ return media::DecodeSupport::SoftwareDecode;
+}
+
+/* static */
+already_AddRefed<PlatformDecoderModule> BlankDecoderModule::Create() {
+ return MakeAndAddRef<BlankDecoderModule>();
+}
+
+} // namespace mozilla
diff --git a/dom/media/platforms/agnostic/BlankDecoderModule.h b/dom/media/platforms/agnostic/BlankDecoderModule.h
new file mode 100644
index 0000000000..65e5d4479e
--- /dev/null
+++ b/dom/media/platforms/agnostic/BlankDecoderModule.h
@@ -0,0 +1,68 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+#if !defined(BlankDecoderModule_h_)
+# define BlankDecoderModule_h_
+
+# include "DummyMediaDataDecoder.h"
+# include "PlatformDecoderModule.h"
+
+namespace mozilla {
+
+namespace layers {
+class ImageContainer;
+}
+
+class MediaData;
+class MediaRawData;
+
+class BlankVideoDataCreator : public DummyDataCreator {
+ public:
+ BlankVideoDataCreator(uint32_t aFrameWidth, uint32_t aFrameHeight,
+ layers::ImageContainer* aImageContainer);
+
+ already_AddRefed<MediaData> Create(MediaRawData* aSample) override;
+
+ private:
+ VideoInfo mInfo;
+ gfx::IntRect mPicture;
+ uint32_t mFrameWidth;
+ uint32_t mFrameHeight;
+ RefPtr<layers::ImageContainer> mImageContainer;
+};
+
+class BlankAudioDataCreator : public DummyDataCreator {
+ public:
+ BlankAudioDataCreator(uint32_t aChannelCount, uint32_t aSampleRate);
+
+ already_AddRefed<MediaData> Create(MediaRawData* aSample) override;
+
+ private:
+ int64_t mFrameSum;
+ uint32_t mChannelCount;
+ uint32_t mSampleRate;
+};
+
+class BlankDecoderModule : public PlatformDecoderModule {
+ template <typename T, typename... Args>
+ friend already_AddRefed<T> MakeAndAddRef(Args&&...);
+
+ public:
+ static already_AddRefed<PlatformDecoderModule> Create();
+
+ already_AddRefed<MediaDataDecoder> CreateVideoDecoder(
+ const CreateDecoderParams& aParams) override;
+
+ already_AddRefed<MediaDataDecoder> CreateAudioDecoder(
+ const CreateDecoderParams& aParams) override;
+
+ media::DecodeSupportSet SupportsMimeType(
+ const nsACString& aMimeType,
+ DecoderDoctorDiagnostics* aDiagnostics) const override;
+};
+
+} // namespace mozilla
+
+#endif /* BlankDecoderModule_h_ */
diff --git a/dom/media/platforms/agnostic/DAV1DDecoder.cpp b/dom/media/platforms/agnostic/DAV1DDecoder.cpp
new file mode 100644
index 0000000000..e93ceb27a1
--- /dev/null
+++ b/dom/media/platforms/agnostic/DAV1DDecoder.cpp
@@ -0,0 +1,403 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "DAV1DDecoder.h"
+
+#include "gfxUtils.h"
+#include "ImageContainer.h"
+#include "mozilla/StaticPrefs_media.h"
+#include "mozilla/TaskQueue.h"
+#include "nsThreadUtils.h"
+#include "PerformanceRecorder.h"
+#include "VideoUtils.h"
+
+#undef LOG
+#define LOG(arg, ...) \
+ DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, __func__, \
+ ##__VA_ARGS__)
+
+namespace mozilla {
+
+static int GetDecodingThreadCount(uint32_t aCodedHeight) {
+ /**
+ * Based on the result we print out from the dav1decoder [1], the
+ * following information shows the number of tiles for AV1 videos served on
+ * Youtube. Each Tile can be decoded in parallel, so we would like to make
+ * sure we at least use enough threads to match the number of tiles.
+ *
+ * ----------------------------
+ * | resolution row col total |
+ * | 480p 2 1 2 |
+ * | 720p 2 2 4 |
+ * | 1080p 4 2 8 |
+ * | 1440p 4 2 8 |
+ * | 2160p 8 4 32 |
+ * ----------------------------
+ *
+ * Besides the tile thread count, the frame thread count also needs to be
+ * considered. As we didn't find anything about what the best number is for
+ * the count of frame thread, just simply use 2 for parallel jobs, which
+ * is similar with Chromium's implementation. They uses 3 frame threads for
+ * 720p+ but less tile threads, so we will still use more total threads. In
+ * addition, their data is measured on 2019, our data should be closer to the
+ * current real world situation.
+ * [1]
+ * https://searchfox.org/mozilla-central/rev/2f5ed7b7244172d46f538051250b14fb4d8f1a5f/third_party/dav1d/src/decode.c#2940
+ */
+ int tileThreads = 2, frameThreads = 2;
+ if (aCodedHeight >= 2160) {
+ tileThreads = 32;
+ } else if (aCodedHeight >= 1080) {
+ tileThreads = 8;
+ } else if (aCodedHeight >= 720) {
+ tileThreads = 4;
+ }
+ return tileThreads * frameThreads;
+}
+
+DAV1DDecoder::DAV1DDecoder(const CreateDecoderParams& aParams)
+ : mInfo(aParams.VideoConfig()),
+ mTaskQueue(TaskQueue::Create(
+ GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
+ "Dav1dDecoder")),
+ mImageContainer(aParams.mImageContainer),
+ mImageAllocator(aParams.mKnowsCompositor),
+ mTrackingId(aParams.mTrackingId),
+ mLowLatency(
+ aParams.mOptions.contains(CreateDecoderParams::Option::LowLatency)) {}
+
+DAV1DDecoder::~DAV1DDecoder() = default;
+
+RefPtr<MediaDataDecoder::InitPromise> DAV1DDecoder::Init() {
+ Dav1dSettings settings;
+ dav1d_default_settings(&settings);
+ if (mLowLatency) {
+ settings.max_frame_delay = 1;
+ }
+ size_t decoder_threads = 2;
+ if (mInfo.mDisplay.width >= 2048) {
+ decoder_threads = 8;
+ } else if (mInfo.mDisplay.width >= 1024) {
+ decoder_threads = 4;
+ }
+ if (StaticPrefs::media_av1_new_thread_count_strategy()) {
+ decoder_threads = GetDecodingThreadCount(mInfo.mImage.Height());
+ }
+ // Still need to consider the amount of physical cores in order to achieve
+ // best performance.
+ settings.n_threads =
+ static_cast<int>(std::min(decoder_threads, GetNumberOfProcessors()));
+ if (int32_t count = StaticPrefs::media_av1_force_thread_count(); count > 0) {
+ settings.n_threads = count;
+ }
+
+ int res = dav1d_open(&mContext, &settings);
+ if (res < 0) {
+ return DAV1DDecoder::InitPromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ RESULT_DETAIL("Couldn't get dAV1d decoder interface.")),
+ __func__);
+ }
+ return DAV1DDecoder::InitPromise::CreateAndResolve(TrackInfo::kVideoTrack,
+ __func__);
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> DAV1DDecoder::Decode(
+ MediaRawData* aSample) {
+ return InvokeAsync<MediaRawData*>(mTaskQueue, this, __func__,
+ &DAV1DDecoder::InvokeDecode, aSample);
+}
+
+void ReleaseDataBuffer_s(const uint8_t* buf, void* user_data) {
+ MOZ_ASSERT(user_data);
+ MOZ_ASSERT(buf);
+ DAV1DDecoder* d = static_cast<DAV1DDecoder*>(user_data);
+ d->ReleaseDataBuffer(buf);
+}
+
+void DAV1DDecoder::ReleaseDataBuffer(const uint8_t* buf) {
+ // The release callback may be called on a different thread defined by the
+ // third party dav1d execution. In that case post a task into TaskQueue to
+ // ensure that mDecodingBuffers is only ever accessed on the TaskQueue.
+ RefPtr<DAV1DDecoder> self = this;
+ auto releaseBuffer = [self, buf] {
+ MOZ_ASSERT(self->mTaskQueue->IsCurrentThreadIn());
+ DebugOnly<bool> found = self->mDecodingBuffers.Remove(buf);
+ MOZ_ASSERT(found);
+ };
+
+ if (mTaskQueue->IsCurrentThreadIn()) {
+ releaseBuffer();
+ } else {
+ nsresult rv = mTaskQueue->Dispatch(NS_NewRunnableFunction(
+ "DAV1DDecoder::ReleaseDataBuffer", std::move(releaseBuffer)));
+ MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+ Unused << rv;
+ }
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> DAV1DDecoder::InvokeDecode(
+ MediaRawData* aSample) {
+ MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
+ MOZ_ASSERT(aSample);
+
+ MediaInfoFlag flag = MediaInfoFlag::None;
+ flag |= (aSample->mKeyframe ? MediaInfoFlag::KeyFrame
+ : MediaInfoFlag::NonKeyFrame);
+ flag |= MediaInfoFlag::SoftwareDecoding;
+ flag |= MediaInfoFlag::VIDEO_AV1;
+ mTrackingId.apply([&](const auto& aId) {
+ mPerformanceRecorder.Start(aSample->mTimecode.ToMicroseconds(),
+ "DAV1DDecoder"_ns, aId, flag);
+ });
+
+ // Add the buffer to the hashtable in order to increase
+ // the ref counter and keep it alive. When dav1d does not
+ // need it any more will call it's release callback. Remove
+ // the buffer, in there, to reduce the ref counter and eventually
+ // free it. We need a hashtable and not an array because the
+ // release callback are not coming in the same order that the
+ // buffers have been added in the decoder (threading ordering
+ // inside decoder)
+ mDecodingBuffers.InsertOrUpdate(aSample->Data(), RefPtr{aSample});
+ Dav1dData data;
+ int res = dav1d_data_wrap(&data, aSample->Data(), aSample->Size(),
+ ReleaseDataBuffer_s, this);
+ data.m.timestamp = aSample->mTimecode.ToMicroseconds();
+ data.m.duration = aSample->mDuration.ToMicroseconds();
+ data.m.offset = aSample->mOffset;
+
+ if (res < 0) {
+ LOG("Create decoder data error.");
+ return DecodePromise::CreateAndReject(
+ MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__), __func__);
+ }
+ DecodedData results;
+ do {
+ res = dav1d_send_data(mContext, &data);
+ if (res < 0 && res != DAV1D_ERR(EAGAIN)) {
+ LOG("Decode error: %d", res);
+ return DecodePromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, __func__), __func__);
+ }
+ // Alway consume the whole buffer on success.
+ // At this point only DAV1D_ERR(EAGAIN) is expected.
+ MOZ_ASSERT((res == 0 && !data.sz) ||
+ (res == DAV1D_ERR(EAGAIN) && data.sz == aSample->Size()));
+
+ Result<already_AddRefed<VideoData>, MediaResult> r = GetPicture();
+ if (r.isOk()) {
+ results.AppendElement(r.unwrap());
+ } else {
+ MediaResult rs = r.unwrapErr();
+ if (rs.Code() == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) {
+ // No frames ready to return. This is not an error, in some
+ // circumstances, we need to feed it with a certain amount of frames
+ // before we get a picture.
+ continue;
+ }
+ // Skip if rs is NS_OK, which can happen if picture layout is I400.
+ if (NS_FAILED(rs.Code())) {
+ return DecodePromise::CreateAndReject(rs, __func__);
+ }
+ }
+ } while (data.sz > 0);
+
+ return DecodePromise::CreateAndResolve(std::move(results), __func__);
+}
+
+Result<already_AddRefed<VideoData>, MediaResult> DAV1DDecoder::GetPicture() {
+ class Dav1dPictureWrapper {
+ public:
+ Dav1dPicture* operator&() { return &p; }
+ const Dav1dPicture& operator*() const { return p; }
+ ~Dav1dPictureWrapper() { dav1d_picture_unref(&p); }
+
+ private:
+ Dav1dPicture p = Dav1dPicture();
+ };
+ Dav1dPictureWrapper picture;
+
+ int res = dav1d_get_picture(mContext, &picture);
+ if (res < 0) {
+ auto r = MediaResult(res == DAV1D_ERR(EAGAIN)
+ ? NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA
+ : NS_ERROR_DOM_MEDIA_DECODE_ERR,
+ RESULT_DETAIL("dav1d_get_picture: %d", res));
+ LOG("%s", r.Message().get());
+ return Err(r);
+ }
+
+ if ((*picture).p.layout == DAV1D_PIXEL_LAYOUT_I400) {
+ // Use NS_OK to indicate that this picture should be skipped.
+ auto r = MediaResult(
+ NS_OK,
+ RESULT_DETAIL("I400 picture: No chroma data to construct an image"));
+ LOG("%s", r.Message().get());
+ return Err(r);
+ }
+
+ Result<already_AddRefed<VideoData>, MediaResult> r = ConstructImage(*picture);
+ return r.mapErr([&](const MediaResult& aResult) {
+ LOG("ConstructImage (%ux%u display %ux%u picture %ux%u ) error - %s: %s",
+ (*picture).p.w, (*picture).p.h, mInfo.mDisplay.width,
+ mInfo.mDisplay.height, mInfo.mImage.width, mInfo.mImage.height,
+ aResult.ErrorName().get(), aResult.Message().get());
+ return aResult;
+ });
+}
+
+/* static */
+Maybe<gfx::YUVColorSpace> DAV1DDecoder::GetColorSpace(
+ const Dav1dPicture& aPicture, LazyLogModule& aLogger) {
+ // When returning Nothing(), the caller chooses the appropriate default.
+ if (!aPicture.seq_hdr || !aPicture.seq_hdr->color_description_present) {
+ return Nothing();
+ }
+
+ return gfxUtils::CicpToColorSpace(
+ static_cast<gfx::CICP::MatrixCoefficients>(aPicture.seq_hdr->mtrx),
+ static_cast<gfx::CICP::ColourPrimaries>(aPicture.seq_hdr->pri), aLogger);
+}
+
+/* static */
+Maybe<gfx::ColorSpace2> DAV1DDecoder::GetColorPrimaries(
+ const Dav1dPicture& aPicture, LazyLogModule& aLogger) {
+ // When returning Nothing(), the caller chooses the appropriate default.
+ if (!aPicture.seq_hdr || !aPicture.seq_hdr->color_description_present) {
+ return Nothing();
+ }
+
+ return gfxUtils::CicpToColorPrimaries(
+ static_cast<gfx::CICP::ColourPrimaries>(aPicture.seq_hdr->pri), aLogger);
+}
+
+Result<already_AddRefed<VideoData>, MediaResult> DAV1DDecoder::ConstructImage(
+ const Dav1dPicture& aPicture) {
+ VideoData::YCbCrBuffer b;
+ if (aPicture.p.bpc == 10) {
+ b.mColorDepth = gfx::ColorDepth::COLOR_10;
+ } else if (aPicture.p.bpc == 12) {
+ b.mColorDepth = gfx::ColorDepth::COLOR_12;
+ } else {
+ b.mColorDepth = gfx::ColorDepth::COLOR_8;
+ }
+
+ b.mYUVColorSpace =
+ DAV1DDecoder::GetColorSpace(aPicture, sPDMLog)
+ .valueOr(DefaultColorSpace({aPicture.p.w, aPicture.p.h}));
+ b.mColorPrimaries = DAV1DDecoder::GetColorPrimaries(aPicture, sPDMLog)
+ .valueOr(gfx::ColorSpace2::BT709);
+ b.mColorRange = aPicture.seq_hdr->color_range ? gfx::ColorRange::FULL
+ : gfx::ColorRange::LIMITED;
+
+ b.mPlanes[0].mData = static_cast<uint8_t*>(aPicture.data[0]);
+ b.mPlanes[0].mStride = aPicture.stride[0];
+ b.mPlanes[0].mHeight = aPicture.p.h;
+ b.mPlanes[0].mWidth = aPicture.p.w;
+ b.mPlanes[0].mSkip = 0;
+
+ b.mPlanes[1].mData = static_cast<uint8_t*>(aPicture.data[1]);
+ b.mPlanes[1].mStride = aPicture.stride[1];
+ b.mPlanes[1].mSkip = 0;
+
+ b.mPlanes[2].mData = static_cast<uint8_t*>(aPicture.data[2]);
+ b.mPlanes[2].mStride = aPicture.stride[1];
+ b.mPlanes[2].mSkip = 0;
+
+ // https://code.videolan.org/videolan/dav1d/blob/master/tools/output/yuv.c#L67
+ const int ss_ver = aPicture.p.layout == DAV1D_PIXEL_LAYOUT_I420;
+ const int ss_hor = aPicture.p.layout != DAV1D_PIXEL_LAYOUT_I444;
+
+ b.mPlanes[1].mHeight = (aPicture.p.h + ss_ver) >> ss_ver;
+ b.mPlanes[1].mWidth = (aPicture.p.w + ss_hor) >> ss_hor;
+
+ b.mPlanes[2].mHeight = (aPicture.p.h + ss_ver) >> ss_ver;
+ b.mPlanes[2].mWidth = (aPicture.p.w + ss_hor) >> ss_hor;
+
+ if (ss_ver) {
+ b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT;
+ } else if (ss_hor) {
+ b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH;
+ }
+
+ // Timestamp, duration and offset used here are wrong.
+ // We need to take those values from the decoder. Latest
+ // dav1d version allows for that.
+ media::TimeUnit timecode =
+ media::TimeUnit::FromMicroseconds(aPicture.m.timestamp);
+ media::TimeUnit duration =
+ media::TimeUnit::FromMicroseconds(aPicture.m.duration);
+ int64_t offset = aPicture.m.offset;
+ bool keyframe = aPicture.frame_hdr->frame_type == DAV1D_FRAME_TYPE_KEY;
+
+ mPerformanceRecorder.Record(aPicture.m.timestamp, [&](DecodeStage& aStage) {
+ aStage.SetResolution(aPicture.p.w, aPicture.p.h);
+ auto format = [&]() -> Maybe<DecodeStage::ImageFormat> {
+ switch (aPicture.p.layout) {
+ case DAV1D_PIXEL_LAYOUT_I420:
+ return Some(DecodeStage::YUV420P);
+ case DAV1D_PIXEL_LAYOUT_I422:
+ return Some(DecodeStage::YUV422P);
+ case DAV1D_PIXEL_LAYOUT_I444:
+ return Some(DecodeStage::YUV444P);
+ default:
+ return Nothing();
+ }
+ }();
+ format.apply([&](auto& aFmt) { aStage.SetImageFormat(aFmt); });
+ aStage.SetYUVColorSpace(b.mYUVColorSpace);
+ aStage.SetColorRange(b.mColorRange);
+ aStage.SetColorDepth(b.mColorDepth);
+ });
+
+ return VideoData::CreateAndCopyData(
+ mInfo, mImageContainer, offset, timecode, duration, b, keyframe, timecode,
+ mInfo.ScaledImageRect(aPicture.p.w, aPicture.p.h), mImageAllocator);
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> DAV1DDecoder::Drain() {
+ RefPtr<DAV1DDecoder> self = this;
+ return InvokeAsync(mTaskQueue, __func__, [self, this] {
+ DecodedData results;
+ while (true) {
+ Result<already_AddRefed<VideoData>, MediaResult> r = GetPicture();
+ if (r.isOk()) {
+ results.AppendElement(r.unwrap());
+ } else {
+ MediaResult rs = r.unwrapErr();
+ if (rs.Code() == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) {
+ break;
+ }
+ // Skip if rs is NS_OK, which can happen if picture layout is I400.
+ if (NS_FAILED(rs.Code())) {
+ return DecodePromise::CreateAndReject(rs, __func__);
+ }
+ }
+ }
+ return DecodePromise::CreateAndResolve(std::move(results), __func__);
+ });
+}
+
+RefPtr<MediaDataDecoder::FlushPromise> DAV1DDecoder::Flush() {
+ RefPtr<DAV1DDecoder> self = this;
+ return InvokeAsync(mTaskQueue, __func__, [this, self]() {
+ dav1d_flush(self->mContext);
+ mPerformanceRecorder.Record(std::numeric_limits<int64_t>::max());
+ return FlushPromise::CreateAndResolve(true, __func__);
+ });
+}
+
+RefPtr<ShutdownPromise> DAV1DDecoder::Shutdown() {
+ RefPtr<DAV1DDecoder> self = this;
+ return InvokeAsync(mTaskQueue, __func__, [self]() {
+ dav1d_close(&self->mContext);
+ return self->mTaskQueue->BeginShutdown();
+ });
+}
+
+} // namespace mozilla
+#undef LOG
diff --git a/dom/media/platforms/agnostic/DAV1DDecoder.h b/dom/media/platforms/agnostic/DAV1DDecoder.h
new file mode 100644
index 0000000000..d928846e36
--- /dev/null
+++ b/dom/media/platforms/agnostic/DAV1DDecoder.h
@@ -0,0 +1,71 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+#if !defined(DAV1DDecoder_h_)
+# define DAV1DDecoder_h_
+
+# include "PerformanceRecorder.h"
+# include "PlatformDecoderModule.h"
+# include "dav1d/dav1d.h"
+# include "mozilla/Result.h"
+# include "nsRefPtrHashtable.h"
+
+namespace mozilla {
+
+DDLoggedTypeDeclNameAndBase(DAV1DDecoder, MediaDataDecoder);
+
+typedef nsRefPtrHashtable<nsPtrHashKey<const uint8_t>, MediaRawData>
+ MediaRawDataHashtable;
+
+class DAV1DDecoder final : public MediaDataDecoder,
+ public DecoderDoctorLifeLogger<DAV1DDecoder> {
+ public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DAV1DDecoder, final);
+
+ explicit DAV1DDecoder(const CreateDecoderParams& aParams);
+
+ RefPtr<InitPromise> Init() override;
+ RefPtr<DecodePromise> Decode(MediaRawData* aSample) override;
+ RefPtr<DecodePromise> Drain() override;
+ RefPtr<FlushPromise> Flush() override;
+ RefPtr<ShutdownPromise> Shutdown() override;
+ nsCString GetDescriptionName() const override {
+ return "av1 libdav1d video decoder"_ns;
+ }
+ nsCString GetCodecName() const override { return "av1"_ns; }
+
+ void ReleaseDataBuffer(const uint8_t* buf);
+
+ static Maybe<gfx::YUVColorSpace> GetColorSpace(const Dav1dPicture& aPicture,
+ LazyLogModule& aLogger);
+
+ static Maybe<gfx::ColorSpace2> GetColorPrimaries(const Dav1dPicture& aPicture,
+ LazyLogModule& aLogger);
+
+ private:
+ virtual ~DAV1DDecoder();
+ RefPtr<DecodePromise> InvokeDecode(MediaRawData* aSample);
+ Result<already_AddRefed<VideoData>, MediaResult> GetPicture();
+ Result<already_AddRefed<VideoData>, MediaResult> ConstructImage(
+ const Dav1dPicture& aPicture);
+
+ Dav1dContext* mContext = nullptr;
+
+ const VideoInfo mInfo;
+ const RefPtr<TaskQueue> mTaskQueue;
+ const RefPtr<layers::ImageContainer> mImageContainer;
+ const RefPtr<layers::KnowsCompositor> mImageAllocator;
+ const Maybe<TrackingId> mTrackingId;
+ const bool mLowLatency;
+ PerformanceRecorderMulti<DecodeStage> mPerformanceRecorder;
+
+ // Keep the buffers alive until dav1d
+ // does not need them any more.
+ MediaRawDataHashtable mDecodingBuffers;
+};
+
+} // namespace mozilla
+
+#endif // DAV1DDecoder_h_
diff --git a/dom/media/platforms/agnostic/DummyMediaDataDecoder.cpp b/dom/media/platforms/agnostic/DummyMediaDataDecoder.cpp
new file mode 100644
index 0000000000..172c7f3dd9
--- /dev/null
+++ b/dom/media/platforms/agnostic/DummyMediaDataDecoder.cpp
@@ -0,0 +1,80 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "DummyMediaDataDecoder.h"
+#include "AnnexB.h"
+#include "H264.h"
+#include "MP4Decoder.h"
+
+namespace mozilla {
+
+DummyDataCreator::~DummyDataCreator() = default;
+
+DummyMediaDataDecoder::DummyMediaDataDecoder(
+ UniquePtr<DummyDataCreator>&& aCreator, const nsACString& aDescription,
+ const CreateDecoderParams& aParams)
+ : mCreator(std::move(aCreator)),
+ mIsH264(MP4Decoder::IsH264(aParams.mConfig.mMimeType)),
+ mMaxRefFrames(mIsH264 ? H264::HasSPS(aParams.VideoConfig().mExtraData)
+ ? H264::ComputeMaxRefFrames(
+ aParams.VideoConfig().mExtraData)
+ : 16
+ : 0),
+ mType(aParams.mConfig.GetType()),
+ mDescription(aDescription) {}
+
+RefPtr<MediaDataDecoder::InitPromise> DummyMediaDataDecoder::Init() {
+ return InitPromise::CreateAndResolve(mType, __func__);
+}
+
+RefPtr<ShutdownPromise> DummyMediaDataDecoder::Shutdown() {
+ return ShutdownPromise::CreateAndResolve(true, __func__);
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> DummyMediaDataDecoder::Decode(
+ MediaRawData* aSample) {
+ RefPtr<MediaData> data = mCreator->Create(aSample);
+
+ if (!data) {
+ return DecodePromise::CreateAndReject(NS_ERROR_OUT_OF_MEMORY, __func__);
+ }
+
+ // Frames come out in DTS order but we need to output them in PTS order.
+ mReorderQueue.Push(std::move(data));
+
+ if (mReorderQueue.Length() > mMaxRefFrames) {
+ return DecodePromise::CreateAndResolve(DecodedData{mReorderQueue.Pop()},
+ __func__);
+ }
+ return DecodePromise::CreateAndResolve(DecodedData(), __func__);
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> DummyMediaDataDecoder::Drain() {
+ DecodedData samples;
+ while (!mReorderQueue.IsEmpty()) {
+ samples.AppendElement(mReorderQueue.Pop());
+ }
+ return DecodePromise::CreateAndResolve(std::move(samples), __func__);
+}
+
+RefPtr<MediaDataDecoder::FlushPromise> DummyMediaDataDecoder::Flush() {
+ mReorderQueue.Clear();
+ return FlushPromise::CreateAndResolve(true, __func__);
+}
+
+nsCString DummyMediaDataDecoder::GetDescriptionName() const {
+ return "blank media data decoder"_ns;
+}
+
+nsCString DummyMediaDataDecoder::GetCodecName() const { return "unknown"_ns; }
+
+MediaDataDecoder::ConversionRequired DummyMediaDataDecoder::NeedsConversion()
+ const {
+ return mIsH264 ? ConversionRequired::kNeedAVCC
+ : ConversionRequired::kNeedNone;
+}
+
+} // namespace mozilla
diff --git a/dom/media/platforms/agnostic/DummyMediaDataDecoder.h b/dom/media/platforms/agnostic/DummyMediaDataDecoder.h
new file mode 100644
index 0000000000..562d289bd9
--- /dev/null
+++ b/dom/media/platforms/agnostic/DummyMediaDataDecoder.h
@@ -0,0 +1,68 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#if !defined(DummyMediaDataDecoder_h_)
+# define DummyMediaDataDecoder_h_
+
+# include "MediaInfo.h"
+# include "mozilla/UniquePtr.h"
+# include "PlatformDecoderModule.h"
+# include "ReorderQueue.h"
+
+namespace mozilla {
+
+class MediaRawData;
+
+class DummyDataCreator {
+ public:
+ virtual ~DummyDataCreator();
+ virtual already_AddRefed<MediaData> Create(MediaRawData* aSample) = 0;
+};
+
+DDLoggedTypeDeclNameAndBase(DummyMediaDataDecoder, MediaDataDecoder);
+
+// Decoder that uses a passed in object's Create function to create Null
+// MediaData objects.
+class DummyMediaDataDecoder final
+ : public MediaDataDecoder,
+ public DecoderDoctorLifeLogger<DummyMediaDataDecoder> {
+ public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DummyMediaDataDecoder, final);
+
+ DummyMediaDataDecoder(UniquePtr<DummyDataCreator>&& aCreator,
+ const nsACString& aDescription,
+ const CreateDecoderParams& aParams);
+
+ RefPtr<InitPromise> Init() override;
+
+ RefPtr<ShutdownPromise> Shutdown() override;
+
+ RefPtr<DecodePromise> Decode(MediaRawData* aSample) override;
+
+ RefPtr<DecodePromise> Drain() override;
+
+ RefPtr<FlushPromise> Flush() override;
+
+ nsCString GetDescriptionName() const override;
+
+ nsCString GetCodecName() const override;
+
+ ConversionRequired NeedsConversion() const override;
+
+ private:
+ ~DummyMediaDataDecoder() = default;
+
+ UniquePtr<DummyDataCreator> mCreator;
+ const bool mIsH264;
+ const uint32_t mMaxRefFrames;
+ ReorderQueue mReorderQueue;
+ TrackInfo::TrackType mType;
+ nsCString mDescription;
+};
+
+} // namespace mozilla
+
+#endif // !defined(DummyMediaDataDecoder_h_)
diff --git a/dom/media/platforms/agnostic/NullDecoderModule.cpp b/dom/media/platforms/agnostic/NullDecoderModule.cpp
new file mode 100644
index 0000000000..2893a4af67
--- /dev/null
+++ b/dom/media/platforms/agnostic/NullDecoderModule.cpp
@@ -0,0 +1,57 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "DummyMediaDataDecoder.h"
+#include "ImageContainer.h"
+
+namespace mozilla {
+
+class NullVideoDataCreator : public DummyDataCreator {
+ public:
+ NullVideoDataCreator() = default;
+
+ already_AddRefed<MediaData> Create(MediaRawData* aSample) override {
+ // Create a dummy VideoData with an empty image. This gives us something to
+ // send to media streams if necessary.
+ RefPtr<layers::PlanarYCbCrImage> image =
+ new layers::RecyclingPlanarYCbCrImage(new layers::BufferRecycleBin());
+ return VideoData::CreateFromImage(gfx::IntSize(), aSample->mOffset,
+ aSample->mTime, aSample->mDuration, image,
+ aSample->mKeyframe, aSample->mTimecode);
+ }
+};
+
+class NullDecoderModule : public PlatformDecoderModule {
+ public:
+ // Decode thread.
+ already_AddRefed<MediaDataDecoder> CreateVideoDecoder(
+ const CreateDecoderParams& aParams) override {
+ UniquePtr<DummyDataCreator> creator = MakeUnique<NullVideoDataCreator>();
+ RefPtr<MediaDataDecoder> decoder = new DummyMediaDataDecoder(
+ std::move(creator), "null media data decoder"_ns, aParams);
+ return decoder.forget();
+ }
+
+ // Decode thread.
+ already_AddRefed<MediaDataDecoder> CreateAudioDecoder(
+ const CreateDecoderParams& aParams) override {
+ MOZ_ASSERT(false, "Audio decoders are unsupported.");
+ return nullptr;
+ }
+
+ media::DecodeSupportSet SupportsMimeType(
+ const nsACString& aMimeType,
+ DecoderDoctorDiagnostics* aDiagnostics) const override {
+ return media::DecodeSupport::SoftwareDecode;
+ }
+};
+
+already_AddRefed<PlatformDecoderModule> CreateNullDecoderModule() {
+ RefPtr<PlatformDecoderModule> pdm = new NullDecoderModule();
+ return pdm.forget();
+}
+
+} // namespace mozilla
diff --git a/dom/media/platforms/agnostic/TheoraDecoder.cpp b/dom/media/platforms/agnostic/TheoraDecoder.cpp
new file mode 100644
index 0000000000..d60093a204
--- /dev/null
+++ b/dom/media/platforms/agnostic/TheoraDecoder.cpp
@@ -0,0 +1,269 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "TheoraDecoder.h"
+
+#include <algorithm>
+#include <ogg/ogg.h>
+
+#include "ImageContainer.h"
+#include "TimeUnits.h"
+#include "XiphExtradata.h"
+#include "gfx2DGlue.h"
+#include "mozilla/PodOperations.h"
+#include "mozilla/TaskQueue.h"
+#include "nsError.h"
+#include "PerformanceRecorder.h"
+#include "VideoUtils.h"
+
+#undef LOG
+#define LOG(arg, ...) \
+ DDMOZ_LOG(gMediaDecoderLog, mozilla::LogLevel::Debug, "::%s: " arg, \
+ __func__, ##__VA_ARGS__)
+
+namespace mozilla {
+
+using namespace gfx;
+using namespace layers;
+
+extern LazyLogModule gMediaDecoderLog;
+
+ogg_packet InitTheoraPacket(const unsigned char* aData, size_t aLength,
+ bool aBOS, bool aEOS, int64_t aGranulepos,
+ int64_t aPacketNo) {
+ ogg_packet packet;
+ packet.packet = const_cast<unsigned char*>(aData);
+ packet.bytes = aLength;
+ packet.b_o_s = aBOS;
+ packet.e_o_s = aEOS;
+ packet.granulepos = aGranulepos;
+ packet.packetno = aPacketNo;
+ return packet;
+}
+
+TheoraDecoder::TheoraDecoder(const CreateDecoderParams& aParams)
+ : mImageAllocator(aParams.mKnowsCompositor),
+ mImageContainer(aParams.mImageContainer),
+ mTaskQueue(TaskQueue::Create(
+ GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
+ "TheoraDecoder")),
+ mTheoraInfo{},
+ mTheoraComment{},
+ mTheoraSetupInfo(nullptr),
+ mTheoraDecoderContext(nullptr),
+ mPacketCount(0),
+ mInfo(aParams.VideoConfig()),
+ mTrackingId(aParams.mTrackingId) {
+ MOZ_COUNT_CTOR(TheoraDecoder);
+}
+
+TheoraDecoder::~TheoraDecoder() {
+ MOZ_COUNT_DTOR(TheoraDecoder);
+ th_setup_free(mTheoraSetupInfo);
+ th_comment_clear(&mTheoraComment);
+ th_info_clear(&mTheoraInfo);
+}
+
+RefPtr<ShutdownPromise> TheoraDecoder::Shutdown() {
+ RefPtr<TheoraDecoder> self = this;
+ return InvokeAsync(mTaskQueue, __func__, [self, this]() {
+ if (mTheoraDecoderContext) {
+ th_decode_free(mTheoraDecoderContext);
+ mTheoraDecoderContext = nullptr;
+ }
+ return mTaskQueue->BeginShutdown();
+ });
+}
+
+RefPtr<MediaDataDecoder::InitPromise> TheoraDecoder::Init() {
+ th_comment_init(&mTheoraComment);
+ th_info_init(&mTheoraInfo);
+
+ nsTArray<unsigned char*> headers;
+ nsTArray<size_t> headerLens;
+ if (!XiphExtradataToHeaders(headers, headerLens,
+ mInfo.mCodecSpecificConfig->Elements(),
+ mInfo.mCodecSpecificConfig->Length())) {
+ return InitPromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ RESULT_DETAIL("Could not get theora header.")),
+ __func__);
+ }
+ for (size_t i = 0; i < headers.Length(); i++) {
+ if (NS_FAILED(DoDecodeHeader(headers[i], headerLens[i]))) {
+ return InitPromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ RESULT_DETAIL("Could not decode theora header.")),
+ __func__);
+ }
+ }
+ if (mPacketCount != 3) {
+ return InitPromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ RESULT_DETAIL("Packet count is wrong.")),
+ __func__);
+ }
+
+ mTheoraDecoderContext = th_decode_alloc(&mTheoraInfo, mTheoraSetupInfo);
+ if (mTheoraDecoderContext) {
+ return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__);
+ } else {
+ return InitPromise::CreateAndReject(
+ MediaResult(NS_ERROR_OUT_OF_MEMORY,
+ RESULT_DETAIL("Could not allocate theora decoder.")),
+ __func__);
+ }
+}
+
+RefPtr<MediaDataDecoder::FlushPromise> TheoraDecoder::Flush() {
+ return InvokeAsync(mTaskQueue, __func__, []() {
+ return FlushPromise::CreateAndResolve(true, __func__);
+ });
+}
+
+nsresult TheoraDecoder::DoDecodeHeader(const unsigned char* aData,
+ size_t aLength) {
+ bool bos = mPacketCount == 0;
+ ogg_packet pkt =
+ InitTheoraPacket(aData, aLength, bos, false, 0, mPacketCount++);
+
+ int r = th_decode_headerin(&mTheoraInfo, &mTheoraComment, &mTheoraSetupInfo,
+ &pkt);
+ return r > 0 ? NS_OK : NS_ERROR_FAILURE;
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> TheoraDecoder::ProcessDecode(
+ MediaRawData* aSample) {
+ MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
+
+ MediaInfoFlag flag = MediaInfoFlag::None;
+ flag |= (aSample->mKeyframe ? MediaInfoFlag::KeyFrame
+ : MediaInfoFlag::NonKeyFrame);
+ flag |= MediaInfoFlag::SoftwareDecoding;
+ flag |= MediaInfoFlag::VIDEO_THEORA;
+ Maybe<PerformanceRecorder<DecodeStage>> rec =
+ mTrackingId.map([&](const auto& aId) {
+ return PerformanceRecorder<DecodeStage>("TheoraDecoder"_ns, aId, flag);
+ });
+
+ const unsigned char* aData = aSample->Data();
+ size_t aLength = aSample->Size();
+
+ bool bos = mPacketCount == 0;
+ ogg_packet pkt =
+ InitTheoraPacket(aData, aLength, bos, false,
+ aSample->mTimecode.ToMicroseconds(), mPacketCount++);
+
+ int ret = th_decode_packetin(mTheoraDecoderContext, &pkt, nullptr);
+ if (ret == 0 || ret == TH_DUPFRAME) {
+ th_ycbcr_buffer ycbcr;
+ th_decode_ycbcr_out(mTheoraDecoderContext, ycbcr);
+
+ int hdec = !(mTheoraInfo.pixel_fmt & 1);
+ int vdec = !(mTheoraInfo.pixel_fmt & 2);
+
+ VideoData::YCbCrBuffer b;
+ b.mPlanes[0].mData = ycbcr[0].data;
+ b.mPlanes[0].mStride = ycbcr[0].stride;
+ b.mPlanes[0].mHeight = mTheoraInfo.frame_height;
+ b.mPlanes[0].mWidth = mTheoraInfo.frame_width;
+ b.mPlanes[0].mSkip = 0;
+
+ b.mPlanes[1].mData = ycbcr[1].data;
+ b.mPlanes[1].mStride = ycbcr[1].stride;
+ b.mPlanes[1].mHeight = mTheoraInfo.frame_height >> vdec;
+ b.mPlanes[1].mWidth = mTheoraInfo.frame_width >> hdec;
+ b.mPlanes[1].mSkip = 0;
+
+ b.mPlanes[2].mData = ycbcr[2].data;
+ b.mPlanes[2].mStride = ycbcr[2].stride;
+ b.mPlanes[2].mHeight = mTheoraInfo.frame_height >> vdec;
+ b.mPlanes[2].mWidth = mTheoraInfo.frame_width >> hdec;
+ b.mPlanes[2].mSkip = 0;
+
+ if (vdec) {
+ b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT;
+ } else if (hdec) {
+ b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH;
+ }
+
+ b.mYUVColorSpace =
+ DefaultColorSpace({mTheoraInfo.frame_width, mTheoraInfo.frame_height});
+
+ IntRect pictureArea(mTheoraInfo.pic_x, mTheoraInfo.pic_y,
+ mTheoraInfo.pic_width, mTheoraInfo.pic_height);
+
+ VideoInfo info;
+ info.mDisplay = mInfo.mDisplay;
+ Result<already_AddRefed<VideoData>, MediaResult> r =
+ VideoData::CreateAndCopyData(
+ info, mImageContainer, aSample->mOffset, aSample->mTime,
+ aSample->mDuration, b, aSample->mKeyframe, aSample->mTimecode,
+ mInfo.ScaledImageRect(mTheoraInfo.frame_width,
+ mTheoraInfo.frame_height),
+ mImageAllocator);
+ if (r.isErr()) {
+ LOG("Image allocation error source %ux%u display %ux%u picture %ux%u",
+ mTheoraInfo.frame_width, mTheoraInfo.frame_height,
+ mInfo.mDisplay.width, mInfo.mDisplay.height, mInfo.mImage.width,
+ mInfo.mImage.height);
+ return DecodePromise::CreateAndReject(r.unwrapErr(), __func__);
+ }
+
+ RefPtr<VideoData> v = r.unwrap();
+ MOZ_ASSERT(v);
+
+ rec.apply([&](auto& aRec) {
+ aRec.Record([&](DecodeStage& aStage) {
+ aStage.SetResolution(static_cast<int>(mTheoraInfo.frame_width),
+ static_cast<int>(mTheoraInfo.frame_height));
+ auto format = [&]() -> Maybe<DecodeStage::ImageFormat> {
+ switch (mTheoraInfo.pixel_fmt) {
+ case TH_PF_420:
+ return Some(DecodeStage::YUV420P);
+ case TH_PF_422:
+ return Some(DecodeStage::YUV422P);
+ case TH_PF_444:
+ return Some(DecodeStage::YUV444P);
+ default:
+ return Nothing();
+ }
+ }();
+ format.apply([&](auto& aFmt) { aStage.SetImageFormat(aFmt); });
+ aStage.SetYUVColorSpace(b.mYUVColorSpace);
+ aStage.SetColorRange(b.mColorRange);
+ aStage.SetColorDepth(b.mColorDepth);
+ });
+ });
+
+ return DecodePromise::CreateAndResolve(DecodedData{v}, __func__);
+ }
+ LOG("Theora Decode error: %d", ret);
+ return DecodePromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
+ RESULT_DETAIL("Theora decode error:%d", ret)),
+ __func__);
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> TheoraDecoder::Decode(
+ MediaRawData* aSample) {
+ return InvokeAsync<MediaRawData*>(mTaskQueue, this, __func__,
+ &TheoraDecoder::ProcessDecode, aSample);
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> TheoraDecoder::Drain() {
+ return InvokeAsync(mTaskQueue, __func__, [] {
+ return DecodePromise::CreateAndResolve(DecodedData(), __func__);
+ });
+}
+
+/* static */
+bool TheoraDecoder::IsTheora(const nsACString& aMimeType) {
+ return aMimeType.EqualsLiteral("video/theora");
+}
+
+} // namespace mozilla
+#undef LOG
diff --git a/dom/media/platforms/agnostic/TheoraDecoder.h b/dom/media/platforms/agnostic/TheoraDecoder.h
new file mode 100644
index 0000000000..f209bb61ec
--- /dev/null
+++ b/dom/media/platforms/agnostic/TheoraDecoder.h
@@ -0,0 +1,63 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+#if !defined(TheoraDecoder_h_)
+# define TheoraDecoder_h_
+
+# include <stdint.h>
+
+# include "PlatformDecoderModule.h"
+# include <theora/theoradec.h>
+
+namespace mozilla {
+
+DDLoggedTypeDeclNameAndBase(TheoraDecoder, MediaDataDecoder);
+
+class TheoraDecoder final : public MediaDataDecoder,
+ public DecoderDoctorLifeLogger<TheoraDecoder> {
+ public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TheoraDecoder, final);
+
+ explicit TheoraDecoder(const CreateDecoderParams& aParams);
+
+ RefPtr<InitPromise> Init() override;
+ RefPtr<DecodePromise> Decode(MediaRawData* aSample) override;
+ RefPtr<DecodePromise> Drain() override;
+ RefPtr<FlushPromise> Flush() override;
+ RefPtr<ShutdownPromise> Shutdown() override;
+
+ // Return true if mimetype is a Theora codec
+ static bool IsTheora(const nsACString& aMimeType);
+
+ nsCString GetDescriptionName() const override {
+ return "theora video decoder"_ns;
+ }
+
+ nsCString GetCodecName() const override { return "theora"_ns; }
+
+ private:
+ ~TheoraDecoder();
+ nsresult DoDecodeHeader(const unsigned char* aData, size_t aLength);
+
+ RefPtr<DecodePromise> ProcessDecode(MediaRawData* aSample);
+
+ const RefPtr<layers::KnowsCompositor> mImageAllocator;
+ const RefPtr<layers::ImageContainer> mImageContainer;
+ const RefPtr<TaskQueue> mTaskQueue;
+
+ // Theora header & decoder state
+ th_info mTheoraInfo;
+ th_comment mTheoraComment;
+ th_setup_info* mTheoraSetupInfo;
+ th_dec_ctx* mTheoraDecoderContext;
+ int mPacketCount;
+
+ const VideoInfo mInfo;
+ const Maybe<TrackingId> mTrackingId;
+};
+
+} // namespace mozilla
+
+#endif
diff --git a/dom/media/platforms/agnostic/VPXDecoder.cpp b/dom/media/platforms/agnostic/VPXDecoder.cpp
new file mode 100644
index 0000000000..1b07606bd5
--- /dev/null
+++ b/dom/media/platforms/agnostic/VPXDecoder.cpp
@@ -0,0 +1,679 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "VPXDecoder.h"
+
+#include <algorithm>
+#include <vpx/vpx_image.h>
+
+#include "BitReader.h"
+#include "BitWriter.h"
+#include "ImageContainer.h"
+#include "TimeUnits.h"
+#include "gfx2DGlue.h"
+#include "gfxUtils.h"
+#include "mozilla/PodOperations.h"
+#include "mozilla/SyncRunnable.h"
+#include "mozilla/TaskQueue.h"
+#include "mozilla/Unused.h"
+#include "nsError.h"
+#include "PerformanceRecorder.h"
+#include "prsystem.h"
+#include "VideoUtils.h"
+
+#undef LOG
+#define LOG(arg, ...) \
+ DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, __func__, \
+ ##__VA_ARGS__)
+
+namespace mozilla {
+
+using namespace gfx;
+using namespace layers;
+
+static VPXDecoder::Codec MimeTypeToCodec(const nsACString& aMimeType) {
+ if (aMimeType.EqualsLiteral("video/vp8")) {
+ return VPXDecoder::Codec::VP8;
+ } else if (aMimeType.EqualsLiteral("video/vp9")) {
+ return VPXDecoder::Codec::VP9;
+ }
+ return VPXDecoder::Codec::Unknown;
+}
+
+static nsresult InitContext(vpx_codec_ctx_t* aCtx, const VideoInfo& aInfo,
+ const VPXDecoder::Codec aCodec, bool aLowLatency) {
+ int decode_threads = 2;
+
+ vpx_codec_iface_t* dx = nullptr;
+ if (aCodec == VPXDecoder::Codec::VP8) {
+ dx = vpx_codec_vp8_dx();
+ } else if (aCodec == VPXDecoder::Codec::VP9) {
+ dx = vpx_codec_vp9_dx();
+ if (aInfo.mDisplay.width >= 2048) {
+ decode_threads = 8;
+ } else if (aInfo.mDisplay.width >= 1024) {
+ decode_threads = 4;
+ }
+ }
+ decode_threads = std::min(decode_threads, PR_GetNumberOfProcessors());
+
+ vpx_codec_dec_cfg_t config;
+ config.threads = aLowLatency ? 1 : decode_threads;
+ config.w = config.h = 0; // set after decode
+
+ if (!dx || vpx_codec_dec_init(aCtx, dx, &config, 0)) {
+ return NS_ERROR_FAILURE;
+ }
+ return NS_OK;
+}
+
+VPXDecoder::VPXDecoder(const CreateDecoderParams& aParams)
+ : mImageContainer(aParams.mImageContainer),
+ mImageAllocator(aParams.mKnowsCompositor),
+ mTaskQueue(TaskQueue::Create(
+ GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER), "VPXDecoder")),
+ mInfo(aParams.VideoConfig()),
+ mCodec(MimeTypeToCodec(aParams.VideoConfig().mMimeType)),
+ mLowLatency(
+ aParams.mOptions.contains(CreateDecoderParams::Option::LowLatency)),
+ mTrackingId(aParams.mTrackingId) {
+ MOZ_COUNT_CTOR(VPXDecoder);
+ PodZero(&mVPX);
+ PodZero(&mVPXAlpha);
+}
+
+VPXDecoder::~VPXDecoder() { MOZ_COUNT_DTOR(VPXDecoder); }
+
+RefPtr<ShutdownPromise> VPXDecoder::Shutdown() {
+ RefPtr<VPXDecoder> self = this;
+ return InvokeAsync(mTaskQueue, __func__, [self]() {
+ vpx_codec_destroy(&self->mVPX);
+ vpx_codec_destroy(&self->mVPXAlpha);
+ return self->mTaskQueue->BeginShutdown();
+ });
+}
+
+RefPtr<MediaDataDecoder::InitPromise> VPXDecoder::Init() {
+ if (NS_FAILED(InitContext(&mVPX, mInfo, mCodec, mLowLatency))) {
+ return VPXDecoder::InitPromise::CreateAndReject(
+ NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
+ }
+ if (mInfo.HasAlpha()) {
+ if (NS_FAILED(InitContext(&mVPXAlpha, mInfo, mCodec, mLowLatency))) {
+ return VPXDecoder::InitPromise::CreateAndReject(
+ NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
+ }
+ }
+ return VPXDecoder::InitPromise::CreateAndResolve(TrackInfo::kVideoTrack,
+ __func__);
+}
+
+RefPtr<MediaDataDecoder::FlushPromise> VPXDecoder::Flush() {
+ return InvokeAsync(mTaskQueue, __func__, []() {
+ return FlushPromise::CreateAndResolve(true, __func__);
+ });
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> VPXDecoder::ProcessDecode(
+ MediaRawData* aSample) {
+ MOZ_ASSERT(mTaskQueue->IsOnCurrentThread());
+
+ MediaInfoFlag flag = MediaInfoFlag::None;
+ flag |= (aSample->mKeyframe ? MediaInfoFlag::KeyFrame
+ : MediaInfoFlag::NonKeyFrame);
+ flag |= MediaInfoFlag::SoftwareDecoding;
+ switch (mCodec) {
+ case Codec::VP8:
+ flag |= MediaInfoFlag::VIDEO_VP8;
+ break;
+ case Codec::VP9:
+ flag |= MediaInfoFlag::VIDEO_VP9;
+ break;
+ default:
+ break;
+ }
+ flag |= MediaInfoFlag::VIDEO_THEORA;
+ auto rec = mTrackingId.map([&](const auto& aId) {
+ return PerformanceRecorder<DecodeStage>("VPXDecoder"_ns, aId, flag);
+ });
+
+ if (vpx_codec_err_t r = vpx_codec_decode(&mVPX, aSample->Data(),
+ aSample->Size(), nullptr, 0)) {
+ LOG("VPX Decode error: %s", vpx_codec_err_to_string(r));
+ return DecodePromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
+ RESULT_DETAIL("VPX error: %s", vpx_codec_err_to_string(r))),
+ __func__);
+ }
+
+ vpx_codec_iter_t iter = nullptr;
+ vpx_image_t* img;
+ vpx_image_t* img_alpha = nullptr;
+ bool alpha_decoded = false;
+ DecodedData results;
+
+ while ((img = vpx_codec_get_frame(&mVPX, &iter))) {
+ NS_ASSERTION(img->fmt == VPX_IMG_FMT_I420 || img->fmt == VPX_IMG_FMT_I444,
+ "WebM image format not I420 or I444");
+ NS_ASSERTION(!alpha_decoded,
+ "Multiple frames per packet that contains alpha");
+
+ if (aSample->AlphaSize() > 0) {
+ if (!alpha_decoded) {
+ MediaResult rv = DecodeAlpha(&img_alpha, aSample);
+ if (NS_FAILED(rv)) {
+ return DecodePromise::CreateAndReject(rv, __func__);
+ }
+ alpha_decoded = true;
+ }
+ }
+ // Chroma shifts are rounded down as per the decoding examples in the SDK
+ VideoData::YCbCrBuffer b;
+ b.mPlanes[0].mData = img->planes[0];
+ b.mPlanes[0].mStride = img->stride[0];
+ b.mPlanes[0].mHeight = img->d_h;
+ b.mPlanes[0].mWidth = img->d_w;
+ b.mPlanes[0].mSkip = 0;
+
+ b.mPlanes[1].mData = img->planes[1];
+ b.mPlanes[1].mStride = img->stride[1];
+ b.mPlanes[1].mSkip = 0;
+
+ b.mPlanes[2].mData = img->planes[2];
+ b.mPlanes[2].mStride = img->stride[2];
+ b.mPlanes[2].mSkip = 0;
+
+ if (img->fmt == VPX_IMG_FMT_I420) {
+ b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT;
+
+ b.mPlanes[1].mHeight = (img->d_h + 1) >> img->y_chroma_shift;
+ b.mPlanes[1].mWidth = (img->d_w + 1) >> img->x_chroma_shift;
+
+ b.mPlanes[2].mHeight = (img->d_h + 1) >> img->y_chroma_shift;
+ b.mPlanes[2].mWidth = (img->d_w + 1) >> img->x_chroma_shift;
+ } else if (img->fmt == VPX_IMG_FMT_I444) {
+ b.mPlanes[1].mHeight = img->d_h;
+ b.mPlanes[1].mWidth = img->d_w;
+
+ b.mPlanes[2].mHeight = img->d_h;
+ b.mPlanes[2].mWidth = img->d_w;
+ } else {
+ LOG("VPX Unknown image format");
+ return DecodePromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
+ RESULT_DETAIL("VPX Unknown image format")),
+ __func__);
+ }
+ b.mYUVColorSpace = [&]() {
+ switch (img->cs) {
+ case VPX_CS_BT_601:
+ case VPX_CS_SMPTE_170:
+ case VPX_CS_SMPTE_240:
+ return gfx::YUVColorSpace::BT601;
+ case VPX_CS_BT_709:
+ return gfx::YUVColorSpace::BT709;
+ case VPX_CS_BT_2020:
+ return gfx::YUVColorSpace::BT2020;
+ default:
+ return DefaultColorSpace({img->d_w, img->d_h});
+ }
+ }();
+ b.mColorRange = img->range == VPX_CR_FULL_RANGE ? gfx::ColorRange::FULL
+ : gfx::ColorRange::LIMITED;
+
+ RefPtr<VideoData> v;
+ if (!img_alpha) {
+ Result<already_AddRefed<VideoData>, MediaResult> r =
+ VideoData::CreateAndCopyData(
+ mInfo, mImageContainer, aSample->mOffset, aSample->mTime,
+ aSample->mDuration, b, aSample->mKeyframe, aSample->mTimecode,
+ mInfo.ScaledImageRect(img->d_w, img->d_h), mImageAllocator);
+ // TODO: Reject DecodePromise below with r's error return.
+ v = r.unwrapOr(nullptr);
+ } else {
+ VideoData::YCbCrBuffer::Plane alpha_plane;
+ alpha_plane.mData = img_alpha->planes[0];
+ alpha_plane.mStride = img_alpha->stride[0];
+ alpha_plane.mHeight = img_alpha->d_h;
+ alpha_plane.mWidth = img_alpha->d_w;
+ alpha_plane.mSkip = 0;
+ v = VideoData::CreateAndCopyData(
+ mInfo, mImageContainer, aSample->mOffset, aSample->mTime,
+ aSample->mDuration, b, alpha_plane, aSample->mKeyframe,
+ aSample->mTimecode, mInfo.ScaledImageRect(img->d_w, img->d_h));
+ }
+
+ if (!v) {
+ LOG("Image allocation error source %ux%u display %ux%u picture %ux%u",
+ img->d_w, img->d_h, mInfo.mDisplay.width, mInfo.mDisplay.height,
+ mInfo.mImage.width, mInfo.mImage.height);
+ return DecodePromise::CreateAndReject(
+ MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__), __func__);
+ }
+
+ rec.apply([&](auto& aRec) {
+ return aRec.Record([&](DecodeStage& aStage) {
+ aStage.SetResolution(static_cast<int>(img->d_w),
+ static_cast<int>(img->d_h));
+ auto format = [&]() -> Maybe<DecodeStage::ImageFormat> {
+ switch (img->fmt) {
+ case VPX_IMG_FMT_I420:
+ return Some(DecodeStage::YUV420P);
+ case VPX_IMG_FMT_I444:
+ return Some(DecodeStage::YUV444P);
+ default:
+ return Nothing();
+ }
+ }();
+ format.apply([&](auto& aFmt) { aStage.SetImageFormat(aFmt); });
+ aStage.SetYUVColorSpace(b.mYUVColorSpace);
+ aStage.SetColorRange(b.mColorRange);
+ aStage.SetColorDepth(b.mColorDepth);
+ });
+ });
+
+ results.AppendElement(std::move(v));
+ }
+ return DecodePromise::CreateAndResolve(std::move(results), __func__);
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> VPXDecoder::Decode(
+ MediaRawData* aSample) {
+ return InvokeAsync<MediaRawData*>(mTaskQueue, this, __func__,
+ &VPXDecoder::ProcessDecode, aSample);
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> VPXDecoder::Drain() {
+ return InvokeAsync(mTaskQueue, __func__, [] {
+ return DecodePromise::CreateAndResolve(DecodedData(), __func__);
+ });
+}
+
+MediaResult VPXDecoder::DecodeAlpha(vpx_image_t** aImgAlpha,
+ const MediaRawData* aSample) {
+ vpx_codec_err_t r = vpx_codec_decode(&mVPXAlpha, aSample->AlphaData(),
+ aSample->AlphaSize(), nullptr, 0);
+ if (r) {
+ LOG("VPX decode alpha error: %s", vpx_codec_err_to_string(r));
+ return MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
+ RESULT_DETAIL("VPX decode alpha error: %s",
+ vpx_codec_err_to_string(r)));
+ }
+
+ vpx_codec_iter_t iter = nullptr;
+
+ *aImgAlpha = vpx_codec_get_frame(&mVPXAlpha, &iter);
+ NS_ASSERTION((*aImgAlpha)->fmt == VPX_IMG_FMT_I420 ||
+ (*aImgAlpha)->fmt == VPX_IMG_FMT_I444,
+ "WebM image format not I420 or I444");
+
+ return NS_OK;
+}
+
+nsCString VPXDecoder::GetCodecName() const {
+ switch (mCodec) {
+ case Codec::VP8:
+ return "vp8"_ns;
+ case Codec::VP9:
+ return "vp9"_ns;
+ default:
+ return "unknown"_ns;
+ }
+}
+
+/* static */
+bool VPXDecoder::IsVPX(const nsACString& aMimeType, uint8_t aCodecMask) {
+ return ((aCodecMask & VPXDecoder::VP8) &&
+ aMimeType.EqualsLiteral("video/vp8")) ||
+ ((aCodecMask & VPXDecoder::VP9) &&
+ aMimeType.EqualsLiteral("video/vp9"));
+}
+
+/* static */
+bool VPXDecoder::IsVP8(const nsACString& aMimeType) {
+ return IsVPX(aMimeType, VPXDecoder::VP8);
+}
+
+/* static */
+bool VPXDecoder::IsVP9(const nsACString& aMimeType) {
+ return IsVPX(aMimeType, VPXDecoder::VP9);
+}
+
+/* static */
+bool VPXDecoder::IsKeyframe(Span<const uint8_t> aBuffer, Codec aCodec) {
+ VPXStreamInfo info;
+ return GetStreamInfo(aBuffer, info, aCodec) && info.mKeyFrame;
+}
+
+/* static */
+gfx::IntSize VPXDecoder::GetFrameSize(Span<const uint8_t> aBuffer,
+ Codec aCodec) {
+ VPXStreamInfo info;
+ if (!GetStreamInfo(aBuffer, info, aCodec)) {
+ return gfx::IntSize();
+ }
+ return info.mImage;
+}
+
+/* static */
+gfx::IntSize VPXDecoder::GetDisplaySize(Span<const uint8_t> aBuffer,
+ Codec aCodec) {
+ VPXStreamInfo info;
+ if (!GetStreamInfo(aBuffer, info, aCodec)) {
+ return gfx::IntSize();
+ }
+ return info.mDisplay;
+}
+
+/* static */
+int VPXDecoder::GetVP9Profile(Span<const uint8_t> aBuffer) {
+ VPXStreamInfo info;
+ if (!GetStreamInfo(aBuffer, info, Codec::VP9)) {
+ return -1;
+ }
+ return info.mProfile;
+}
+
+/* static */
+bool VPXDecoder::GetStreamInfo(Span<const uint8_t> aBuffer,
+ VPXDecoder::VPXStreamInfo& aInfo, Codec aCodec) {
+ if (aBuffer.IsEmpty()) {
+ // Can't be good.
+ return false;
+ }
+
+ aInfo = VPXStreamInfo();
+
+ if (aCodec == Codec::VP8) {
+ aInfo.mKeyFrame = (aBuffer[0] & 1) ==
+ 0; // frame type (0 for key frames, 1 for interframes)
+ if (!aInfo.mKeyFrame) {
+ // We can't retrieve the required information from interframes.
+ return true;
+ }
+ if (aBuffer.Length() < 10) {
+ return false;
+ }
+ uint8_t version = (aBuffer[0] >> 1) & 0x7;
+ if (version > 3) {
+ return false;
+ }
+ uint8_t start_code_byte_0 = aBuffer[3];
+ uint8_t start_code_byte_1 = aBuffer[4];
+ uint8_t start_code_byte_2 = aBuffer[5];
+ if (start_code_byte_0 != 0x9d || start_code_byte_1 != 0x01 ||
+ start_code_byte_2 != 0x2a) {
+ return false;
+ }
+ uint16_t width = (aBuffer[6] | aBuffer[7] << 8) & 0x3fff;
+ uint16_t height = (aBuffer[8] | aBuffer[9] << 8) & 0x3fff;
+
+ // aspect ratio isn't found in the VP8 frame header.
+ aInfo.mImage = gfx::IntSize(width, height);
+ aInfo.mDisplayAndImageDifferent = false;
+ aInfo.mDisplay = aInfo.mImage;
+ return true;
+ }
+
+ BitReader br(aBuffer.Elements(), aBuffer.Length() * 8);
+ uint32_t frameMarker = br.ReadBits(2); // frame_marker
+ if (frameMarker != 2) {
+ // That's not a valid vp9 header.
+ return false;
+ }
+ uint32_t profile = br.ReadBits(1); // profile_low_bit
+ profile |= br.ReadBits(1) << 1; // profile_high_bit
+ if (profile == 3) {
+ profile += br.ReadBits(1); // reserved_zero
+ if (profile > 3) {
+ // reserved_zero wasn't zero.
+ return false;
+ }
+ }
+
+ aInfo.mProfile = profile;
+
+ bool show_existing_frame = br.ReadBits(1);
+ if (show_existing_frame) {
+ if (profile == 3 && aBuffer.Length() < 2) {
+ return false;
+ }
+ Unused << br.ReadBits(3); // frame_to_show_map_idx
+ return true;
+ }
+
+ if (aBuffer.Length() < 10) {
+ // Header too small;
+ return false;
+ }
+
+ aInfo.mKeyFrame = !br.ReadBits(1);
+ bool show_frame = br.ReadBits(1);
+ bool error_resilient_mode = br.ReadBits(1);
+
+ auto frame_sync_code = [&]() -> bool {
+ uint8_t frame_sync_byte_1 = br.ReadBits(8);
+ uint8_t frame_sync_byte_2 = br.ReadBits(8);
+ uint8_t frame_sync_byte_3 = br.ReadBits(8);
+ return frame_sync_byte_1 == 0x49 && frame_sync_byte_2 == 0x83 &&
+ frame_sync_byte_3 == 0x42;
+ };
+
+ auto color_config = [&]() -> bool {
+ aInfo.mBitDepth = 8;
+ if (profile >= 2) {
+ bool ten_or_twelve_bit = br.ReadBits(1);
+ aInfo.mBitDepth = ten_or_twelve_bit ? 12 : 10;
+ }
+ aInfo.mColorSpace = br.ReadBits(3);
+ if (aInfo.mColorSpace != 7 /* CS_RGB */) {
+ aInfo.mFullRange = br.ReadBits(1);
+ if (profile == 1 || profile == 3) {
+ aInfo.mSubSampling_x = br.ReadBits(1);
+ aInfo.mSubSampling_y = br.ReadBits(1);
+ if (br.ReadBits(1)) { // reserved_zero
+ return false;
+ };
+ } else {
+ aInfo.mSubSampling_x = true;
+ aInfo.mSubSampling_y = true;
+ }
+ } else {
+ aInfo.mFullRange = true;
+ if (profile == 1 || profile == 3) {
+ aInfo.mSubSampling_x = false;
+ aInfo.mSubSampling_y = false;
+ if (br.ReadBits(1)) { // reserved_zero
+ return false;
+ };
+ } else {
+ // sRGB color space is only available with VP9 profile 1.
+ return false;
+ }
+ }
+ return true;
+ };
+
+ auto frame_size = [&]() {
+ int32_t width = static_cast<int32_t>(br.ReadBits(16)) + 1;
+ int32_t height = static_cast<int32_t>(br.ReadBits(16)) + 1;
+ aInfo.mImage = gfx::IntSize(width, height);
+ };
+
+ auto render_size = [&]() {
+ // render_and_frame_size_different
+ aInfo.mDisplayAndImageDifferent = br.ReadBits(1);
+ if (aInfo.mDisplayAndImageDifferent) {
+ int32_t width = static_cast<int32_t>(br.ReadBits(16)) + 1;
+ int32_t height = static_cast<int32_t>(br.ReadBits(16)) + 1;
+ aInfo.mDisplay = gfx::IntSize(width, height);
+ } else {
+ aInfo.mDisplay = aInfo.mImage;
+ }
+ };
+
+ if (aInfo.mKeyFrame) {
+ if (!frame_sync_code()) {
+ return false;
+ }
+ if (!color_config()) {
+ return false;
+ }
+ frame_size();
+ render_size();
+ } else {
+ bool intra_only = show_frame ? false : br.ReadBit();
+ if (!error_resilient_mode) {
+ Unused << br.ReadBits(2); // reset_frame_context
+ }
+ if (intra_only) {
+ if (!frame_sync_code()) {
+ return false;
+ }
+ if (profile > 0) {
+ if (!color_config()) {
+ return false;
+ }
+ } else {
+ aInfo.mColorSpace = 1; // CS_BT_601
+ aInfo.mSubSampling_x = true;
+ aInfo.mSubSampling_y = true;
+ aInfo.mBitDepth = 8;
+ }
+ Unused << br.ReadBits(8); // refresh_frame_flags
+ frame_size();
+ render_size();
+ }
+ }
+ return true;
+}
+
+// Ref: "VP Codec ISO Media File Format Binding, v1.0, 2017-03-31"
+// <https://www.webmproject.org/vp9/mp4/>
+//
+// class VPCodecConfigurationBox extends FullBox('vpcC', version = 1, 0)
+// {
+// VPCodecConfigurationRecord() vpcConfig;
+// }
+//
+// aligned (8) class VPCodecConfigurationRecord {
+// unsigned int (8) profile;
+// unsigned int (8) level;
+// unsigned int (4) bitDepth;
+// unsigned int (3) chromaSubsampling;
+// unsigned int (1) videoFullRangeFlag;
+// unsigned int (8) colourPrimaries;
+// unsigned int (8) transferCharacteristics;
+// unsigned int (8) matrixCoefficients;
+// unsigned int (16) codecIntializationDataSize;
+// unsigned int (8)[] codecIntializationData;
+// }
+
+/* static */
+void VPXDecoder::GetVPCCBox(MediaByteBuffer* aDestBox,
+ const VPXStreamInfo& aInfo) {
+ BitWriter writer(aDestBox);
+
+ int chroma = [&]() {
+ if (aInfo.mSubSampling_x && aInfo.mSubSampling_y) {
+ return 1; // 420 Colocated;
+ }
+ if (aInfo.mSubSampling_x && !aInfo.mSubSampling_y) {
+ return 2; // 422
+ }
+ if (!aInfo.mSubSampling_x && !aInfo.mSubSampling_y) {
+ return 3; // 444
+ }
+ // This indicates 4:4:0 subsampling, which is not expressable in the
+ // 'vpcC' box. Default to 4:2:0.
+ return 1;
+ }();
+
+ writer.WriteU8(1); // version
+ writer.WriteBits(0, 24); // flags
+
+ writer.WriteU8(aInfo.mProfile); // profile
+ writer.WriteU8(10); // level set it to 1.0
+
+ writer.WriteBits(aInfo.mBitDepth, 4); // bitdepth
+ writer.WriteBits(chroma, 3); // chroma
+ writer.WriteBit(aInfo.mFullRange); // full/restricted range
+
+ // See VPXDecoder::VPXStreamInfo enums
+ writer.WriteU8(aInfo.mColorPrimaries); // color primaries
+ writer.WriteU8(aInfo.mTransferFunction); // transfer characteristics
+ writer.WriteU8(2); // matrix coefficients: unspecified
+
+ writer.WriteBits(0,
+ 16); // codecIntializationDataSize (must be 0 for VP8/VP9)
+}
+
+/* static */
+bool VPXDecoder::SetVideoInfo(VideoInfo* aDestInfo, const nsAString& aCodec) {
+ VPXDecoder::VPXStreamInfo info;
+ uint8_t level = 0;
+ uint8_t chroma = 1;
+ VideoColorSpace colorSpace;
+ if (!ExtractVPXCodecDetails(aCodec, info.mProfile, level, info.mBitDepth,
+ chroma, colorSpace)) {
+ return false;
+ }
+
+ aDestInfo->mColorPrimaries =
+ gfxUtils::CicpToColorPrimaries(colorSpace.mPrimaries, sPDMLog);
+ aDestInfo->mTransferFunction =
+ gfxUtils::CicpToTransferFunction(colorSpace.mTransfer);
+ aDestInfo->mColorDepth = gfx::ColorDepthForBitDepth(info.mBitDepth);
+ VPXDecoder::SetChroma(info, chroma);
+ info.mFullRange = colorSpace.mRange == ColorRange::FULL;
+ RefPtr<MediaByteBuffer> extraData = new MediaByteBuffer();
+ VPXDecoder::GetVPCCBox(extraData, info);
+ aDestInfo->mExtraData = extraData;
+ return true;
+}
+
+/* static */
+void VPXDecoder::SetChroma(VPXStreamInfo& aDestInfo, uint8_t chroma) {
+ switch (chroma) {
+ case 0:
+ case 1:
+ aDestInfo.mSubSampling_x = true;
+ aDestInfo.mSubSampling_y = true;
+ break;
+ case 2:
+ aDestInfo.mSubSampling_x = true;
+ aDestInfo.mSubSampling_y = false;
+ break;
+ case 3:
+ aDestInfo.mSubSampling_x = false;
+ aDestInfo.mSubSampling_y = false;
+ break;
+ }
+}
+
+/* static */
+void VPXDecoder::ReadVPCCBox(VPXStreamInfo& aDestInfo, MediaByteBuffer* aBox) {
+ BitReader reader(aBox);
+
+ reader.ReadBits(8); // version
+ reader.ReadBits(24); // flags
+ aDestInfo.mProfile = reader.ReadBits(8);
+ reader.ReadBits(8); // level
+
+ aDestInfo.mBitDepth = reader.ReadBits(4);
+ SetChroma(aDestInfo, reader.ReadBits(3));
+ aDestInfo.mFullRange = reader.ReadBit();
+
+ aDestInfo.mColorPrimaries = reader.ReadBits(8); // color primaries
+ aDestInfo.mTransferFunction = reader.ReadBits(8); // transfer characteristics
+ reader.ReadBits(8); // matrix coefficients
+
+ MOZ_ASSERT(reader.ReadBits(16) ==
+ 0); // codecInitializationDataSize (must be 0 for VP8/VP9)
+}
+
+} // namespace mozilla
+#undef LOG
diff --git a/dom/media/platforms/agnostic/VPXDecoder.h b/dom/media/platforms/agnostic/VPXDecoder.h
new file mode 100644
index 0000000000..ea9073ad6b
--- /dev/null
+++ b/dom/media/platforms/agnostic/VPXDecoder.h
@@ -0,0 +1,208 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+#if !defined(VPXDecoder_h_)
+# define VPXDecoder_h_
+
+# include <stdint.h>
+# include <vpx/vp8dx.h>
+# include <vpx/vpx_codec.h>
+# include <vpx/vpx_decoder.h>
+
+# include "PlatformDecoderModule.h"
+# include "mozilla/Span.h"
+# include "mozilla/gfx/Types.h"
+
+namespace mozilla {
+
+DDLoggedTypeDeclNameAndBase(VPXDecoder, MediaDataDecoder);
+
+class VPXDecoder final : public MediaDataDecoder,
+ public DecoderDoctorLifeLogger<VPXDecoder> {
+ public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VPXDecoder, final);
+
+ explicit VPXDecoder(const CreateDecoderParams& aParams);
+
+ RefPtr<InitPromise> Init() override;
+ RefPtr<DecodePromise> Decode(MediaRawData* aSample) override;
+ RefPtr<DecodePromise> Drain() override;
+ RefPtr<FlushPromise> Flush() override;
+ RefPtr<ShutdownPromise> Shutdown() override;
+ nsCString GetDescriptionName() const override {
+ return "libvpx video decoder"_ns;
+ }
+ nsCString GetCodecName() const override;
+
+ enum Codec : uint8_t {
+ VP8 = 1 << 0,
+ VP9 = 1 << 1,
+ Unknown = 1 << 7,
+ };
+
+ // Return true if aMimeType is a one of the strings used by our demuxers to
+ // identify VPX of the specified type. Does not parse general content type
+ // strings, i.e. white space matters.
+ static bool IsVPX(const nsACString& aMimeType,
+ uint8_t aCodecMask = VP8 | VP9);
+ static bool IsVP8(const nsACString& aMimeType);
+ static bool IsVP9(const nsACString& aMimeType);
+
+ // Return true if a sample is a keyframe for the specified codec.
+ static bool IsKeyframe(Span<const uint8_t> aBuffer, Codec aCodec);
+
+ // Return the frame dimensions for a sample for the specified codec.
+ static gfx::IntSize GetFrameSize(Span<const uint8_t> aBuffer, Codec aCodec);
+ // Return the display dimensions for a sample for the specified codec.
+ static gfx::IntSize GetDisplaySize(Span<const uint8_t> aBuffer, Codec aCodec);
+
+ // Return the VP9 profile as per https://www.webmproject.org/vp9/profiles/
+ // Return negative value if error.
+ static int GetVP9Profile(Span<const uint8_t> aBuffer);
+
+ struct VPXStreamInfo {
+ gfx::IntSize mImage;
+ bool mDisplayAndImageDifferent = false;
+ gfx::IntSize mDisplay;
+ bool mKeyFrame = false;
+
+ uint8_t mProfile = 0;
+ uint8_t mBitDepth = 8;
+ /*
+ 0 CS_UNKNOWN Unknown (in this case the color space must be signaled outside
+ the VP9 bitstream).
+ 1 CS_BT_601 Rec. ITU-R BT.601-7
+ 2 CS_BT_709 Rec. ITU-R BT.709-6
+ 3 CS_SMPTE_170 SMPTE-170
+ 4 CS_SMPTE_240 SMPTE-240
+ 5 CS_BT_2020 Rec. ITU-R BT.2020-2
+ 6 CS_RESERVED Reserved
+ 7 CS_RGB sRGB (IEC 61966-2-1)
+ */
+ int mColorSpace = 1; // CS_BT_601
+
+ gfx::YUVColorSpace ColorSpace() const {
+ switch (mColorSpace) {
+ case 1:
+ case 3:
+ case 4:
+ return gfx::YUVColorSpace::BT601;
+ case 2:
+ return gfx::YUVColorSpace::BT709;
+ case 5:
+ return gfx::YUVColorSpace::BT2020;
+ default:
+ return gfx::YUVColorSpace::Default;
+ }
+ }
+
+ uint8_t mColorPrimaries = gfx::CICP::ColourPrimaries::CP_UNSPECIFIED;
+ gfx::ColorSpace2 ColorPrimaries() const {
+ switch (mColorPrimaries) {
+ case gfx::CICP::ColourPrimaries::CP_BT709:
+ return gfx::ColorSpace2::BT709;
+ case gfx::CICP::ColourPrimaries::CP_UNSPECIFIED:
+ return gfx::ColorSpace2::BT709;
+ case gfx::CICP::ColourPrimaries::CP_BT2020:
+ return gfx::ColorSpace2::BT2020;
+ default:
+ return gfx::ColorSpace2::BT709;
+ }
+ }
+
+ uint8_t mTransferFunction =
+ gfx::CICP::TransferCharacteristics::TC_UNSPECIFIED;
+ gfx::TransferFunction TransferFunction() const {
+ switch (mTransferFunction) {
+ case gfx::CICP::TransferCharacteristics::TC_BT709:
+ return gfx::TransferFunction::BT709;
+ case gfx::CICP::TransferCharacteristics::TC_SRGB:
+ return gfx::TransferFunction::SRGB;
+ case gfx::CICP::TransferCharacteristics::TC_SMPTE2084:
+ return gfx::TransferFunction::PQ;
+ case gfx::CICP::TransferCharacteristics::TC_HLG:
+ return gfx::TransferFunction::HLG;
+ default:
+ return gfx::TransferFunction::BT709;
+ }
+ }
+
+ /*
+ mFullRange == false then:
+ For BitDepth equals 8:
+ Y is between 16 and 235 inclusive.
+ U and V are between 16 and 240 inclusive.
+ For BitDepth equals 10:
+ Y is between 64 and 940 inclusive.
+ U and V are between 64 and 960 inclusive.
+ For BitDepth equals 12:
+ Y is between 256 and 3760.
+ U and V are between 256 and 3840 inclusive.
+ mFullRange == true then:
+ No restriction on Y, U, V values.
+ */
+ bool mFullRange = false;
+
+ gfx::ColorRange ColorRange() const {
+ return mFullRange ? gfx::ColorRange::FULL : gfx::ColorRange::LIMITED;
+ }
+
+ /*
+ Sub-sampling, used only for non sRGB colorspace.
+ subsampling_x subsampling_y Description
+ 0 0 YUV 4:4:4
+ 0 1 YUV 4:4:0
+ 1 0 YUV 4:2:2
+ 1 1 YUV 4:2:0
+ */
+ bool mSubSampling_x = true;
+ bool mSubSampling_y = true;
+
+ bool IsCompatible(const VPXStreamInfo& aOther) const {
+ return mImage == aOther.mImage && mProfile == aOther.mProfile &&
+ mBitDepth == aOther.mBitDepth &&
+ mSubSampling_x == aOther.mSubSampling_x &&
+ mSubSampling_y == aOther.mSubSampling_y &&
+ mColorSpace == aOther.mColorSpace &&
+ mFullRange == aOther.mFullRange;
+ }
+ };
+
+ static bool GetStreamInfo(Span<const uint8_t> aBuffer, VPXStreamInfo& aInfo,
+ Codec aCodec);
+
+ static void GetVPCCBox(MediaByteBuffer* aDestBox, const VPXStreamInfo& aInfo);
+ // Set extradata for a VP8/VP9 track, returning false if the codec was
+ // invalid.
+ static bool SetVideoInfo(VideoInfo* aDestInfo, const nsAString& aCodec);
+
+ static void SetChroma(VPXStreamInfo& aDestInfo, uint8_t chroma);
+ static void ReadVPCCBox(VPXStreamInfo& aDestInfo, MediaByteBuffer* aBox);
+
+ private:
+ ~VPXDecoder();
+ RefPtr<DecodePromise> ProcessDecode(MediaRawData* aSample);
+ MediaResult DecodeAlpha(vpx_image_t** aImgAlpha, const MediaRawData* aSample);
+
+ const RefPtr<layers::ImageContainer> mImageContainer;
+ RefPtr<layers::KnowsCompositor> mImageAllocator;
+ const RefPtr<TaskQueue> mTaskQueue;
+
+ // VPx decoder state
+ vpx_codec_ctx_t mVPX;
+
+ // VPx alpha decoder state
+ vpx_codec_ctx_t mVPXAlpha;
+
+ const VideoInfo mInfo;
+
+ const Codec mCodec;
+ const bool mLowLatency;
+ const Maybe<TrackingId> mTrackingId;
+};
+
+} // namespace mozilla
+
+#endif
diff --git a/dom/media/platforms/agnostic/bytestreams/Adts.cpp b/dom/media/platforms/agnostic/bytestreams/Adts.cpp
new file mode 100644
index 0000000000..5f31904d9c
--- /dev/null
+++ b/dom/media/platforms/agnostic/bytestreams/Adts.cpp
@@ -0,0 +1,94 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "Adts.h"
+#include "MediaData.h"
+#include "mozilla/Array.h"
+#include "mozilla/ArrayUtils.h"
+
+namespace mozilla {
+
+static const int kADTSHeaderSize = 7;
+
+int8_t Adts::GetFrequencyIndex(uint32_t aSamplesPerSecond) {
+ static const uint32_t freq_lookup[] = {96000, 88200, 64000, 48000, 44100,
+ 32000, 24000, 22050, 16000, 12000,
+ 11025, 8000, 7350, 0};
+
+ int8_t i = 0;
+ while (freq_lookup[i] && aSamplesPerSecond < freq_lookup[i]) {
+ i++;
+ }
+
+ if (!freq_lookup[i]) {
+ return -1;
+ }
+
+ return i;
+}
+
+bool Adts::ConvertSample(uint16_t aChannelCount, int8_t aFrequencyIndex,
+ int8_t aProfile, MediaRawData* aSample) {
+ size_t newSize = aSample->Size() + kADTSHeaderSize;
+
+ // ADTS header uses 13 bits for packet size.
+ if (newSize >= (1 << 13) || aChannelCount > 15 || aFrequencyIndex < 0 ||
+ aProfile < 1 || aProfile > 4) {
+ return false;
+ }
+
+ Array<uint8_t, kADTSHeaderSize> header;
+ header[0] = 0xff;
+ header[1] = 0xf1;
+ header[2] =
+ ((aProfile - 1) << 6) + (aFrequencyIndex << 2) + (aChannelCount >> 2);
+ header[3] = ((aChannelCount & 0x3) << 6) + (newSize >> 11);
+ header[4] = (newSize & 0x7ff) >> 3;
+ header[5] = ((newSize & 7) << 5) + 0x1f;
+ header[6] = 0xfc;
+
+ UniquePtr<MediaRawDataWriter> writer(aSample->CreateWriter());
+ if (!writer->Prepend(&header[0], ArrayLength(header))) {
+ return false;
+ }
+
+ if (aSample->mCrypto.IsEncrypted()) {
+ if (aSample->mCrypto.mPlainSizes.Length() == 0) {
+ writer->mCrypto.mPlainSizes.AppendElement(kADTSHeaderSize);
+ writer->mCrypto.mEncryptedSizes.AppendElement(aSample->Size() -
+ kADTSHeaderSize);
+ } else {
+ writer->mCrypto.mPlainSizes[0] += kADTSHeaderSize;
+ }
+ }
+
+ return true;
+}
+
+bool Adts::RevertSample(MediaRawData* aSample) {
+ if (aSample->Size() < kADTSHeaderSize) {
+ return false;
+ }
+
+ {
+ const uint8_t* header = aSample->Data();
+ if (header[0] != 0xff || header[1] != 0xf1 || header[6] != 0xfc) {
+ // Not ADTS.
+ return false;
+ }
+ }
+
+ UniquePtr<MediaRawDataWriter> writer(aSample->CreateWriter());
+ writer->PopFront(kADTSHeaderSize);
+
+ if (aSample->mCrypto.IsEncrypted()) {
+ if (aSample->mCrypto.mPlainSizes.Length() > 0 &&
+ writer->mCrypto.mPlainSizes[0] >= kADTSHeaderSize) {
+ writer->mCrypto.mPlainSizes[0] -= kADTSHeaderSize;
+ }
+ }
+
+ return true;
+}
+} // namespace mozilla
diff --git a/dom/media/platforms/agnostic/bytestreams/Adts.h b/dom/media/platforms/agnostic/bytestreams/Adts.h
new file mode 100644
index 0000000000..c2b6b558b6
--- /dev/null
+++ b/dom/media/platforms/agnostic/bytestreams/Adts.h
@@ -0,0 +1,22 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef ADTS_H_
+#define ADTS_H_
+
+#include <stdint.h>
+
+namespace mozilla {
+class MediaRawData;
+
+class Adts {
+ public:
+ static int8_t GetFrequencyIndex(uint32_t aSamplesPerSecond);
+ static bool ConvertSample(uint16_t aChannelCount, int8_t aFrequencyIndex,
+ int8_t aProfile, mozilla::MediaRawData* aSample);
+ static bool RevertSample(MediaRawData* aSample);
+};
+} // namespace mozilla
+
+#endif
diff --git a/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp b/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp
new file mode 100644
index 0000000000..086936dcc6
--- /dev/null
+++ b/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp
@@ -0,0 +1,542 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/ArrayUtils.h"
+#include "mozilla/EndianUtils.h"
+#include "mozilla/ResultExtensions.h"
+#include "mozilla/Try.h"
+#include "mozilla/Unused.h"
+#include "AnnexB.h"
+#include "BufferReader.h"
+#include "ByteWriter.h"
+#include "H264.h"
+#include "H265.h"
+#include "MediaData.h"
+
+mozilla::LazyLogModule gAnnexB("AnnexB");
+
+#define LOG(msg, ...) MOZ_LOG(gAnnexB, LogLevel::Debug, (msg, ##__VA_ARGS__))
+#define LOGV(msg, ...) MOZ_LOG(gAnnexB, LogLevel::Verbose, (msg, ##__VA_ARGS__))
+
+namespace mozilla {
+
+static const uint8_t kAnnexBDelimiter[] = {0, 0, 0, 1};
+
+/* static */
+Result<Ok, nsresult> AnnexB::ConvertAVCCSampleToAnnexB(
+ mozilla::MediaRawData* aSample, bool aAddSPS) {
+ MOZ_ASSERT(aSample);
+
+ if (!IsAVCC(aSample)) {
+ return Ok();
+ }
+ MOZ_ASSERT(aSample->Data());
+
+ MOZ_TRY(ConvertAVCCTo4BytesAVCC(aSample));
+
+ if (aSample->Size() < 4) {
+ // Nothing to do, it's corrupted anyway.
+ return Ok();
+ }
+
+ BufferReader reader(aSample->Data(), aSample->Size());
+
+ nsTArray<uint8_t> tmp;
+ ByteWriter<BigEndian> writer(tmp);
+
+ while (reader.Remaining() >= 4) {
+ uint32_t nalLen;
+ MOZ_TRY_VAR(nalLen, reader.ReadU32());
+ const uint8_t* p = reader.Read(nalLen);
+
+ if (!writer.Write(kAnnexBDelimiter, ArrayLength(kAnnexBDelimiter))) {
+ return Err(NS_ERROR_OUT_OF_MEMORY);
+ }
+ if (!p) {
+ break;
+ }
+ if (!writer.Write(p, nalLen)) {
+ return Err(NS_ERROR_OUT_OF_MEMORY);
+ }
+ }
+
+ UniquePtr<MediaRawDataWriter> samplewriter(aSample->CreateWriter());
+
+ if (!samplewriter->Replace(tmp.Elements(), tmp.Length())) {
+ return Err(NS_ERROR_OUT_OF_MEMORY);
+ }
+
+ // Prepend the Annex B NAL with SPS and PPS tables to keyframes.
+ if (aAddSPS && aSample->mKeyframe) {
+ RefPtr<MediaByteBuffer> annexB =
+ ConvertAVCCExtraDataToAnnexB(aSample->mExtraData);
+ if (!samplewriter->Prepend(annexB->Elements(), annexB->Length())) {
+ return Err(NS_ERROR_OUT_OF_MEMORY);
+ }
+
+ // Prepending the NAL with SPS/PPS will mess up the encryption subsample
+ // offsets. So we need to account for the extra bytes by increasing
+ // the length of the first clear data subsample. Otherwise decryption
+ // will fail.
+ if (aSample->mCrypto.IsEncrypted()) {
+ if (aSample->mCrypto.mPlainSizes.Length() == 0) {
+ CheckedUint32 plainSize{annexB->Length()};
+ CheckedUint32 encryptedSize{samplewriter->Size()};
+ encryptedSize -= annexB->Length();
+ samplewriter->mCrypto.mPlainSizes.AppendElement(plainSize.value());
+ samplewriter->mCrypto.mEncryptedSizes.AppendElement(
+ encryptedSize.value());
+ } else {
+ CheckedUint32 newSize{samplewriter->mCrypto.mPlainSizes[0]};
+ newSize += annexB->Length();
+ samplewriter->mCrypto.mPlainSizes[0] = newSize.value();
+ }
+ }
+ }
+
+ return Ok();
+}
+
+/* static */
+Result<Ok, nsresult> AnnexB::ConvertHVCCSampleToAnnexB(
+ mozilla::MediaRawData* aSample, bool aAddSPS) {
+ MOZ_ASSERT(aSample);
+ if (!IsHVCC(aSample)) {
+ LOG("Not HVCC?");
+ return Ok();
+ }
+ MOZ_ASSERT(aSample->Data());
+
+ MOZ_TRY(ConvertHVCCTo4BytesHVCC(aSample));
+ if (aSample->Size() < 4) {
+ // Nothing to do, it's corrupted anyway.
+ LOG("Corrupted HVCC sample?");
+ return Ok();
+ }
+
+ BufferReader reader(aSample->Data(), aSample->Size());
+ nsTArray<uint8_t> tmp;
+ ByteWriter<BigEndian> writer(tmp);
+ while (reader.Remaining() >= 4) {
+ uint32_t nalLen;
+ MOZ_TRY_VAR(nalLen, reader.ReadU32());
+ const uint8_t* p = reader.Read(nalLen);
+ if (!writer.Write(kAnnexBDelimiter, ArrayLength(kAnnexBDelimiter))) {
+ LOG("Failed to write kAnnexBDelimiter, OOM?");
+ return Err(NS_ERROR_OUT_OF_MEMORY);
+ }
+ if (!p) {
+ break;
+ }
+ if (!writer.Write(p, nalLen)) {
+ LOG("Failed to write nalu, OOM?");
+ return Err(NS_ERROR_OUT_OF_MEMORY);
+ }
+ }
+
+ UniquePtr<MediaRawDataWriter> samplewriter(aSample->CreateWriter());
+ if (!samplewriter->Replace(tmp.Elements(), tmp.Length())) {
+ LOG("Failed to write sample, OOM?");
+ return Err(NS_ERROR_OUT_OF_MEMORY);
+ }
+
+ // Prepend the Annex B NAL with SPS and PPS tables to keyframes.
+ if (aAddSPS && aSample->mKeyframe) {
+ RefPtr<MediaByteBuffer> annexB =
+ ConvertHVCCExtraDataToAnnexB(aSample->mExtraData);
+ if (!annexB) {
+ LOG("Failed to convert HVCC extradata to AnnexB");
+ return Err(NS_ERROR_FAILURE);
+ }
+ if (!samplewriter->Prepend(annexB->Elements(), annexB->Length())) {
+ LOG("Failed to append annexB extradata");
+ return Err(NS_ERROR_OUT_OF_MEMORY);
+ }
+
+ // Prepending the NAL with SPS/PPS will mess up the encryption subsample
+ // offsets. So we need to account for the extra bytes by increasing
+ // the length of the first clear data subsample. Otherwise decryption
+ // will fail.
+ if (aSample->mCrypto.IsEncrypted()) {
+ if (aSample->mCrypto.mPlainSizes.Length() == 0) {
+ CheckedUint32 plainSize{annexB->Length()};
+ CheckedUint32 encryptedSize{samplewriter->Size()};
+ encryptedSize -= annexB->Length();
+ samplewriter->mCrypto.mPlainSizes.AppendElement(plainSize.value());
+ samplewriter->mCrypto.mEncryptedSizes.AppendElement(
+ encryptedSize.value());
+ } else {
+ CheckedUint32 newSize{samplewriter->mCrypto.mPlainSizes[0]};
+ newSize += annexB->Length();
+ samplewriter->mCrypto.mPlainSizes[0] = newSize.value();
+ }
+ }
+ }
+ return Ok();
+}
+
+already_AddRefed<mozilla::MediaByteBuffer> AnnexB::ConvertAVCCExtraDataToAnnexB(
+ const mozilla::MediaByteBuffer* aExtraData) {
+ // AVCC 6 byte header looks like:
+ // +------+------+------+------+------+------+------+------+
+ // [0] | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 |
+ // +------+------+------+------+------+------+------+------+
+ // [1] | profile |
+ // +------+------+------+------+------+------+------+------+
+ // [2] | compatiblity |
+ // +------+------+------+------+------+------+------+------+
+ // [3] | level |
+ // +------+------+------+------+------+------+------+------+
+ // [4] | unused | nalLenSiz-1 |
+ // +------+------+------+------+------+------+------+------+
+ // [5] | unused | numSps |
+ // +------+------+------+------+------+------+------+------+
+
+ RefPtr<mozilla::MediaByteBuffer> annexB = new mozilla::MediaByteBuffer;
+
+ BufferReader reader(*aExtraData);
+ const uint8_t* ptr = reader.Read(5);
+ if (ptr && ptr[0] == 1) {
+ // Append SPS then PPS
+ Unused << reader.ReadU8().map(
+ [&](uint8_t x) { return ConvertSPSOrPPS(reader, x & 31, annexB); });
+ Unused << reader.ReadU8().map(
+ [&](uint8_t x) { return ConvertSPSOrPPS(reader, x, annexB); });
+ // MP4Box adds extra bytes that we ignore. I don't know what they do.
+ }
+
+ return annexB.forget();
+}
+
+already_AddRefed<mozilla::MediaByteBuffer> AnnexB::ConvertHVCCExtraDataToAnnexB(
+ const mozilla::MediaByteBuffer* aExtraData) {
+ auto rv = HVCCConfig::Parse(aExtraData);
+ if (rv.isErr()) {
+ return nullptr;
+ }
+ const HVCCConfig hvcc = rv.unwrap();
+ RefPtr<mozilla::MediaByteBuffer> annexB = new mozilla::MediaByteBuffer;
+ for (const auto& nalu : hvcc.mNALUs) {
+ annexB->AppendElements(kAnnexBDelimiter, ArrayLength(kAnnexBDelimiter));
+ annexB->AppendElements(nalu.mNALU.Elements(), nalu.mNALU.Length());
+ LOGV("Insert NALU (type=%hhu, size=%zu) to AnnexB (size=%zu)",
+ nalu.mNalUnitType, nalu.mNALU.Length(), annexB->Length());
+ }
+ return annexB.forget();
+}
+
+Result<mozilla::Ok, nsresult> AnnexB::ConvertSPSOrPPS(
+ BufferReader& aReader, uint8_t aCount, mozilla::MediaByteBuffer* aAnnexB) {
+ for (int i = 0; i < aCount; i++) {
+ uint16_t length;
+ MOZ_TRY_VAR(length, aReader.ReadU16());
+
+ const uint8_t* ptr = aReader.Read(length);
+ if (!ptr) {
+ return Err(NS_ERROR_FAILURE);
+ }
+ aAnnexB->AppendElements(kAnnexBDelimiter, ArrayLength(kAnnexBDelimiter));
+ aAnnexB->AppendElements(ptr, length);
+ }
+ return Ok();
+}
+
+static Result<Ok, nsresult> FindStartCodeInternal(BufferReader& aBr) {
+ size_t offset = aBr.Offset();
+
+ for (uint32_t i = 0; i < aBr.Align() && aBr.Remaining() >= 3; i++) {
+ auto res = aBr.PeekU24();
+ if (res.isOk() && (res.unwrap() == 0x000001)) {
+ return Ok();
+ }
+ mozilla::Unused << aBr.Read(1);
+ }
+
+ while (aBr.Remaining() >= 6) {
+ uint32_t x32;
+ MOZ_TRY_VAR(x32, aBr.PeekU32());
+ if ((x32 - 0x01010101) & (~x32) & 0x80808080) {
+ if ((x32 >> 8) == 0x000001) {
+ return Ok();
+ }
+ if (x32 == 0x000001) {
+ mozilla::Unused << aBr.Read(1);
+ return Ok();
+ }
+ if ((x32 & 0xff) == 0) {
+ const uint8_t* p = aBr.Peek(1);
+ if ((x32 & 0xff00) == 0 && p[4] == 1) {
+ mozilla::Unused << aBr.Read(2);
+ return Ok();
+ }
+ if (p[4] == 0 && p[5] == 1) {
+ mozilla::Unused << aBr.Read(3);
+ return Ok();
+ }
+ }
+ }
+ mozilla::Unused << aBr.Read(4);
+ }
+
+ while (aBr.Remaining() >= 3) {
+ uint32_t data;
+ MOZ_TRY_VAR(data, aBr.PeekU24());
+ if (data == 0x000001) {
+ return Ok();
+ }
+ mozilla::Unused << aBr.Read(1);
+ }
+
+ // No start code were found; Go back to the beginning.
+ mozilla::Unused << aBr.Seek(offset);
+ return Err(NS_ERROR_FAILURE);
+}
+
+static Result<Ok, nsresult> FindStartCode(BufferReader& aBr,
+ size_t& aStartSize) {
+ if (FindStartCodeInternal(aBr).isErr()) {
+ aStartSize = 0;
+ return Err(NS_ERROR_FAILURE);
+ }
+
+ aStartSize = 3;
+ if (aBr.Offset()) {
+ // Check if it's 4-bytes start code
+ aBr.Rewind(1);
+ uint8_t data;
+ MOZ_TRY_VAR(data, aBr.ReadU8());
+ if (data == 0) {
+ aStartSize = 4;
+ }
+ }
+ mozilla::Unused << aBr.Read(3);
+ return Ok();
+}
+
+/* static */
+void AnnexB::ParseNALEntries(const Span<const uint8_t>& aSpan,
+ nsTArray<AnnexB::NALEntry>& aEntries) {
+ BufferReader reader(aSpan.data(), aSpan.Length());
+ size_t startSize;
+ auto rv = FindStartCode(reader, startSize);
+ size_t startOffset = reader.Offset();
+ if (rv.isOk()) {
+ while (FindStartCode(reader, startSize).isOk()) {
+ int64_t offset = reader.Offset();
+ int64_t sizeNAL = offset - startOffset - startSize;
+ aEntries.AppendElement(AnnexB::NALEntry(startOffset, sizeNAL));
+ reader.Seek(startOffset);
+ reader.Read(sizeNAL + startSize);
+ startOffset = offset;
+ }
+ }
+ int64_t sizeNAL = reader.Remaining();
+ if (sizeNAL) {
+ aEntries.AppendElement(AnnexB::NALEntry(startOffset, sizeNAL));
+ }
+}
+
+static Result<mozilla::Ok, nsresult> ParseNALUnits(ByteWriter<BigEndian>& aBw,
+ BufferReader& aBr) {
+ size_t startSize;
+
+ auto rv = FindStartCode(aBr, startSize);
+ if (rv.isOk()) {
+ size_t startOffset = aBr.Offset();
+ while (FindStartCode(aBr, startSize).isOk()) {
+ size_t offset = aBr.Offset();
+ size_t sizeNAL = offset - startOffset - startSize;
+ aBr.Seek(startOffset);
+ if (!aBw.WriteU32(sizeNAL) || !aBw.Write(aBr.Read(sizeNAL), sizeNAL)) {
+ return Err(NS_ERROR_OUT_OF_MEMORY);
+ }
+ aBr.Read(startSize);
+ startOffset = offset;
+ }
+ }
+ size_t sizeNAL = aBr.Remaining();
+ if (sizeNAL) {
+ if (!aBw.WriteU32(sizeNAL) || !aBw.Write(aBr.Read(sizeNAL), sizeNAL)) {
+ return Err(NS_ERROR_OUT_OF_MEMORY);
+ }
+ }
+ return Ok();
+}
+
+bool AnnexB::ConvertSampleToAVCC(mozilla::MediaRawData* aSample,
+ const RefPtr<MediaByteBuffer>& aAVCCHeader) {
+ if (IsAVCC(aSample)) {
+ return ConvertAVCCTo4BytesAVCC(aSample).isOk();
+ }
+ if (!IsAnnexB(aSample)) {
+ // Not AnnexB, nothing to convert.
+ return true;
+ }
+
+ nsTArray<uint8_t> nalu;
+ ByteWriter<BigEndian> writer(nalu);
+ BufferReader reader(aSample->Data(), aSample->Size());
+
+ if (ParseNALUnits(writer, reader).isErr()) {
+ return false;
+ }
+ UniquePtr<MediaRawDataWriter> samplewriter(aSample->CreateWriter());
+ if (!samplewriter->Replace(nalu.Elements(), nalu.Length())) {
+ return false;
+ }
+
+ if (aAVCCHeader) {
+ aSample->mExtraData = aAVCCHeader;
+ return true;
+ }
+
+ // Create the AVCC header.
+ auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>();
+ static const uint8_t kFakeExtraData[] = {
+ 1 /* version */,
+ 0x64 /* profile (High) */,
+ 0 /* profile compat (0) */,
+ 40 /* level (40) */,
+ 0xfc | 3 /* nal size - 1 */,
+ 0xe0 /* num SPS (0) */,
+ 0 /* num PPS (0) */
+ };
+ // XXX(Bug 1631371) Check if this should use a fallible operation as it
+ // pretended earlier.
+ extradata->AppendElements(kFakeExtraData, ArrayLength(kFakeExtraData));
+ aSample->mExtraData = std::move(extradata);
+ return true;
+}
+
+/* static */
+Result<mozilla::Ok, nsresult> AnnexB::ConvertSampleToHVCC(
+ mozilla::MediaRawData* aSample) {
+ if (IsHVCC(aSample)) {
+ return ConvertHVCCTo4BytesHVCC(aSample);
+ }
+ if (!IsAnnexB(aSample)) {
+ // Not AnnexB, nothing to convert.
+ return Ok();
+ }
+
+ nsTArray<uint8_t> nalu;
+ ByteWriter<BigEndian> writer(nalu);
+ BufferReader reader(aSample->Data(), aSample->Size());
+ if (auto rv = ParseNALUnits(writer, reader); rv.isErr()) {
+ LOG("Failed fo parse AnnexB NALU for HVCC");
+ return rv;
+ }
+ UniquePtr<MediaRawDataWriter> samplewriter(aSample->CreateWriter());
+ if (!samplewriter->Replace(nalu.Elements(), nalu.Length())) {
+ LOG("Failed fo replace NALU");
+ return Err(NS_ERROR_OUT_OF_MEMORY);
+ }
+ if (aSample->mExtraData && HVCCConfig::Parse(aSample).isErr()) {
+ LOG("Failed to parse invalid hvcc extradata");
+ return Err(NS_ERROR_DOM_MEDIA_METADATA_ERR);
+ }
+ // TODO : currently we don't set the fake header because we expect the sample
+ // already has a valid extradata. (set by the media change monitor) We can
+ // support setting a specific/fake header if we want to support HEVC encoding.
+ return Ok();
+}
+
+/* static */
+Result<mozilla::Ok, nsresult> AnnexB::ConvertAVCCTo4BytesAVCC(
+ mozilla::MediaRawData* aSample) {
+ auto avcc = AVCCConfig::Parse(aSample);
+ MOZ_ASSERT(avcc.isOk());
+ return ConvertNALUTo4BytesNALU(aSample, avcc.unwrap().NALUSize());
+}
+
+/* static */
+Result<mozilla::Ok, nsresult> AnnexB::ConvertHVCCTo4BytesHVCC(
+ mozilla::MediaRawData* aSample) {
+ auto hvcc = HVCCConfig::Parse(aSample);
+ MOZ_ASSERT(hvcc.isOk());
+ return ConvertNALUTo4BytesNALU(aSample, hvcc.unwrap().NALUSize());
+}
+
+/* static */
+bool AnnexB::IsAVCC(const mozilla::MediaRawData* aSample) {
+ return AVCCConfig::Parse(aSample).isOk();
+}
+
+/* static */
+bool AnnexB::IsHVCC(const mozilla::MediaRawData* aSample) {
+ return HVCCConfig::Parse(aSample).isOk();
+}
+
+/* static */
+bool AnnexB::IsAnnexB(const mozilla::MediaRawData* aSample) {
+ if (aSample->Size() < 4) {
+ return false;
+ }
+ uint32_t header = mozilla::BigEndian::readUint32(aSample->Data());
+ return header == 0x00000001 || (header >> 8) == 0x000001;
+}
+
+/* static */ mozilla::Result<mozilla::Ok, nsresult>
+AnnexB::ConvertNALUTo4BytesNALU(mozilla::MediaRawData* aSample,
+ uint8_t aNALUSize) {
+ // NALSize should be between 1 to 4.
+ if (aNALUSize == 0 || aNALUSize > 4) {
+ return Err(NS_ERROR_FAILURE);
+ }
+
+ // If the nalLenSize is already 4, we can only check if the data is corrupt
+ // without replacing data in aSample.
+ bool needConversion = aNALUSize != 4;
+
+ MOZ_ASSERT(aSample);
+ nsTArray<uint8_t> dest;
+ ByteWriter<BigEndian> writer(dest);
+ BufferReader reader(aSample->Data(), aSample->Size());
+ while (reader.Remaining() > aNALUSize) {
+ uint32_t nalLen;
+ switch (aNALUSize) {
+ case 1:
+ MOZ_TRY_VAR(nalLen, reader.ReadU8());
+ break;
+ case 2:
+ MOZ_TRY_VAR(nalLen, reader.ReadU16());
+ break;
+ case 3:
+ MOZ_TRY_VAR(nalLen, reader.ReadU24());
+ break;
+ case 4:
+ MOZ_TRY_VAR(nalLen, reader.ReadU32());
+ break;
+ default:
+ MOZ_ASSERT_UNREACHABLE("Bytes of the NAL body length must be in [1,4]");
+ return Err(NS_ERROR_ILLEGAL_VALUE);
+ }
+ const uint8_t* p = reader.Read(nalLen);
+ if (!p) {
+ // The data may be corrupt.
+ return Err(NS_ERROR_UNEXPECTED);
+ }
+ if (!needConversion) {
+ // We only parse aSample to see if it's corrupt.
+ continue;
+ }
+ if (!writer.WriteU32(nalLen) || !writer.Write(p, nalLen)) {
+ return Err(NS_ERROR_OUT_OF_MEMORY);
+ }
+ }
+ if (!needConversion) {
+ // We've parsed all the data, and it's all good.
+ return Ok();
+ }
+ UniquePtr<MediaRawDataWriter> samplewriter(aSample->CreateWriter());
+ if (!samplewriter->Replace(dest.Elements(), dest.Length())) {
+ return Err(NS_ERROR_OUT_OF_MEMORY);
+ }
+ return Ok();
+}
+
+#undef LOG
+#undef LOGV
+
+} // namespace mozilla
diff --git a/dom/media/platforms/agnostic/bytestreams/AnnexB.h b/dom/media/platforms/agnostic/bytestreams/AnnexB.h
new file mode 100644
index 0000000000..d3b2b8892d
--- /dev/null
+++ b/dom/media/platforms/agnostic/bytestreams/AnnexB.h
@@ -0,0 +1,89 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_ANNEX_B_H_
+#define DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_ANNEX_B_H_
+
+#include "ErrorList.h"
+#include "mozilla/RefPtr.h"
+#include "mozilla/Result.h"
+
+template <class>
+class nsTArray;
+
+namespace mozilla {
+class BufferReader;
+class MediaRawData;
+class MediaByteBuffer;
+
+class AnnexB {
+ public:
+ struct NALEntry {
+ NALEntry(int64_t aOffset, int64_t aSize) : mOffset(aOffset), mSize(aSize) {
+ MOZ_ASSERT(mOffset >= 0);
+ MOZ_ASSERT(mSize >= 0);
+ }
+ // They should be non-negative, so we use int64_t to assert their value when
+ // assigning value to them.
+ int64_t mOffset;
+ int64_t mSize;
+ };
+ // All conversions assume size of NAL length field is 4 bytes.
+ // Convert a sample from AVCC format to Annex B.
+ static mozilla::Result<mozilla::Ok, nsresult> ConvertAVCCSampleToAnnexB(
+ mozilla::MediaRawData* aSample, bool aAddSPS = true);
+ // All conversions assume size of NAL length field is 4 bytes.
+ // Convert a sample from HVCC format to Annex B.
+ static mozilla::Result<mozilla::Ok, nsresult> ConvertHVCCSampleToAnnexB(
+ mozilla::MediaRawData* aSample, bool aAddSPS = true);
+
+ // Convert a sample from Annex B to AVCC.
+ // an AVCC extradata must not be set.
+ static bool ConvertSampleToAVCC(
+ mozilla::MediaRawData* aSample,
+ const RefPtr<mozilla::MediaByteBuffer>& aAVCCHeader = nullptr);
+ // Convert a sample from Annex B to HVCC. An HVCC extradata must not be set.
+ static Result<mozilla::Ok, nsresult> ConvertSampleToHVCC(
+ mozilla::MediaRawData* aSample);
+
+ // Covert sample to 4 bytes NALU byte stream.
+ static mozilla::Result<mozilla::Ok, nsresult> ConvertAVCCTo4BytesAVCC(
+ mozilla::MediaRawData* aSample);
+ static mozilla::Result<mozilla::Ok, nsresult> ConvertHVCCTo4BytesHVCC(
+ mozilla::MediaRawData* aSample);
+
+ // Parse an AVCC extradata and construct the Annex B sample header.
+ static already_AddRefed<mozilla::MediaByteBuffer>
+ ConvertAVCCExtraDataToAnnexB(const mozilla::MediaByteBuffer* aExtraData);
+ // Parse a HVCC extradata and construct the Annex B sample header.
+ static already_AddRefed<mozilla::MediaByteBuffer>
+ ConvertHVCCExtraDataToAnnexB(const mozilla::MediaByteBuffer* aExtraData);
+
+ // Returns true if format is AVCC and sample has valid extradata.
+ static bool IsAVCC(const mozilla::MediaRawData* aSample);
+ // Returns true if format is HVCC and sample has valid extradata.
+ static bool IsHVCC(const mozilla::MediaRawData* aSample);
+ // Returns true if format is AnnexB.
+ static bool IsAnnexB(const mozilla::MediaRawData* aSample);
+
+ // Parse NAL entries from the bytes stream to know the offset and the size of
+ // each NAL in the bytes stream.
+ static void ParseNALEntries(const Span<const uint8_t>& aSpan,
+ nsTArray<AnnexB::NALEntry>& aEntries);
+
+ private:
+ // AVCC box parser helper.
+ static mozilla::Result<mozilla::Ok, nsresult> ConvertSPSOrPPS(
+ mozilla::BufferReader& aReader, uint8_t aCount,
+ mozilla::MediaByteBuffer* aAnnexB);
+
+ // NALU size can vary, this function converts the sample to always 4 bytes
+ // NALU.
+ static mozilla::Result<mozilla::Ok, nsresult> ConvertNALUTo4BytesNALU(
+ mozilla::MediaRawData* aSample, uint8_t aNALUSize);
+};
+
+} // namespace mozilla
+
+#endif // DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_ANNEX_B_H_
diff --git a/dom/media/platforms/agnostic/bytestreams/ByteStreamsUtils.h b/dom/media/platforms/agnostic/bytestreams/ByteStreamsUtils.h
new file mode 100644
index 0000000000..464c35911a
--- /dev/null
+++ b/dom/media/platforms/agnostic/bytestreams/ByteStreamsUtils.h
@@ -0,0 +1,70 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_BYTESTREAMUTILS_H_
+#define DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_BYTESTREAMUTILS_H_
+
+namespace mozilla {
+
+// Described in ISO 23001-8:2016
+// H264 spec Table 2, H265 spec Table E.3
+enum class PrimaryID : uint8_t {
+ INVALID = 0,
+ BT709 = 1,
+ UNSPECIFIED = 2,
+ BT470M = 4,
+ BT470BG = 5,
+ SMPTE170M = 6,
+ SMPTE240M = 7,
+ FILM = 8,
+ BT2020 = 9,
+ SMPTEST428_1 = 10,
+ SMPTEST431_2 = 11,
+ SMPTEST432_1 = 12,
+ EBU_3213_E = 22
+};
+
+// H264 spec Table 3, H265 spec Table E.4
+enum class TransferID : uint8_t {
+ INVALID = 0,
+ BT709 = 1,
+ UNSPECIFIED = 2,
+ GAMMA22 = 4,
+ GAMMA28 = 5,
+ SMPTE170M = 6,
+ SMPTE240M = 7,
+ LINEAR = 8,
+ LOG = 9,
+ LOG_SQRT = 10,
+ IEC61966_2_4 = 11,
+ BT1361_ECG = 12,
+ IEC61966_2_1 = 13,
+ BT2020_10 = 14,
+ BT2020_12 = 15,
+ SMPTEST2084 = 16,
+ SMPTEST428_1 = 17,
+
+ // Not yet standardized
+ ARIB_STD_B67 = 18, // AKA hybrid-log gamma, HLG.
+};
+
+// H264 spec Table 4, H265 spec Table E.5
+enum class MatrixID : uint8_t {
+ RGB = 0,
+ BT709 = 1,
+ UNSPECIFIED = 2,
+ FCC = 4,
+ BT470BG = 5,
+ SMPTE170M = 6,
+ SMPTE240M = 7,
+ YCOCG = 8,
+ BT2020_NCL = 9,
+ BT2020_CL = 10,
+ YDZDX = 11,
+ INVALID = 255,
+};
+
+} // namespace mozilla
+
+#endif // DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_H265_H_
diff --git a/dom/media/platforms/agnostic/bytestreams/H264.cpp b/dom/media/platforms/agnostic/bytestreams/H264.cpp
new file mode 100644
index 0000000000..113be67d0e
--- /dev/null
+++ b/dom/media/platforms/agnostic/bytestreams/H264.cpp
@@ -0,0 +1,1333 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "H264.h"
+#include <limits>
+#include "AnnexB.h"
+#include "BitReader.h"
+#include "BitWriter.h"
+#include "BufferReader.h"
+#include "ByteStreamsUtils.h"
+#include "ByteWriter.h"
+#include "mozilla/PodOperations.h"
+#include "mozilla/ResultExtensions.h"
+#include "mozilla/Try.h"
+
+#define READSE(var, min, max) \
+ { \
+ int32_t val = br.ReadSE(); \
+ if (val < min || val > max) { \
+ return false; \
+ } \
+ aDest.var = val; \
+ }
+
+#define READUE(var, max) \
+ { \
+ uint32_t uval = br.ReadUE(); \
+ if (uval > max) { \
+ return false; \
+ } \
+ aDest.var = uval; \
+ }
+
+mozilla::LazyLogModule gH264("H264");
+
+#define LOG(msg, ...) MOZ_LOG(gH264, LogLevel::Debug, (msg, ##__VA_ARGS__))
+
+namespace mozilla {
+
+// Default scaling lists (per spec).
+// ITU H264:
+// Table 7-2 – Assignment of mnemonic names to scaling list indices and
+// specification of fall-back rule
+static const uint8_t Default_4x4_Intra[16] = {6, 13, 13, 20, 20, 20, 28, 28,
+ 28, 28, 32, 32, 32, 37, 37, 42};
+
+static const uint8_t Default_4x4_Inter[16] = {10, 14, 14, 20, 20, 20, 24, 24,
+ 24, 24, 27, 27, 27, 30, 30, 34};
+
+static const uint8_t Default_8x8_Intra[64] = {
+ 6, 10, 10, 13, 11, 13, 16, 16, 16, 16, 18, 18, 18, 18, 18, 23,
+ 23, 23, 23, 23, 23, 25, 25, 25, 25, 25, 25, 25, 27, 27, 27, 27,
+ 27, 27, 27, 27, 29, 29, 29, 29, 29, 29, 29, 31, 31, 31, 31, 31,
+ 31, 33, 33, 33, 33, 33, 36, 36, 36, 36, 38, 38, 38, 40, 40, 42};
+
+static const uint8_t Default_8x8_Inter[64] = {
+ 9, 13, 13, 15, 13, 15, 17, 17, 17, 17, 19, 19, 19, 19, 19, 21,
+ 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 24, 24, 24, 24,
+ 24, 24, 24, 24, 25, 25, 25, 25, 25, 25, 25, 27, 27, 27, 27, 27,
+ 27, 28, 28, 28, 28, 28, 30, 30, 30, 30, 32, 32, 32, 33, 33, 35};
+
+namespace detail {
+static void scaling_list(BitReader& aBr, uint8_t* aScalingList,
+ int aSizeOfScalingList, const uint8_t* aDefaultList,
+ const uint8_t* aFallbackList) {
+ int32_t lastScale = 8;
+ int32_t nextScale = 8;
+ int32_t deltaScale;
+
+ // (pic|seq)_scaling_list_present_flag[i]
+ if (!aBr.ReadBit()) {
+ if (aFallbackList) {
+ memcpy(aScalingList, aFallbackList, aSizeOfScalingList);
+ }
+ return;
+ }
+
+ for (int i = 0; i < aSizeOfScalingList; i++) {
+ if (nextScale != 0) {
+ deltaScale = aBr.ReadSE();
+ nextScale = (lastScale + deltaScale + 256) % 256;
+ if (!i && !nextScale) {
+ memcpy(aScalingList, aDefaultList, aSizeOfScalingList);
+ return;
+ }
+ }
+ aScalingList[i] = (nextScale == 0) ? lastScale : nextScale;
+ lastScale = aScalingList[i];
+ }
+}
+} // namespace detail.
+
+template <size_t N>
+static void scaling_list(BitReader& aBr, uint8_t (&aScalingList)[N],
+ const uint8_t (&aDefaultList)[N],
+ const uint8_t (&aFallbackList)[N]) {
+ detail::scaling_list(aBr, aScalingList, N, aDefaultList, aFallbackList);
+}
+
+template <size_t N>
+static void scaling_list(BitReader& aBr, uint8_t (&aScalingList)[N],
+ const uint8_t (&aDefaultList)[N]) {
+ detail::scaling_list(aBr, aScalingList, N, aDefaultList, nullptr);
+}
+
+SPSData::SPSData() {
+ PodZero(this);
+ // Default values when they aren't defined as per ITU-T H.264 (2014/02).
+ chroma_format_idc = 1;
+ video_format = 5;
+ colour_primaries = 2;
+ transfer_characteristics = 2;
+ sample_ratio = 1.0;
+ memset(scaling_matrix4x4, 16, sizeof(scaling_matrix4x4));
+ memset(scaling_matrix8x8, 16, sizeof(scaling_matrix8x8));
+}
+
+bool SPSData::operator==(const SPSData& aOther) const {
+ return this->valid && aOther.valid && !memcmp(this, &aOther, sizeof(SPSData));
+}
+
+bool SPSData::operator!=(const SPSData& aOther) const {
+ return !(operator==(aOther));
+}
+
+static PrimaryID GetPrimaryID(int aPrimary) {
+ if (aPrimary < 1 || aPrimary > 22 || aPrimary == 3) {
+ return PrimaryID::INVALID;
+ }
+ if (aPrimary > 12 && aPrimary < 22) {
+ return PrimaryID::INVALID;
+ }
+ return static_cast<PrimaryID>(aPrimary);
+}
+
+static TransferID GetTransferID(int aTransfer) {
+ if (aTransfer < 1 || aTransfer > 18 || aTransfer == 3) {
+ return TransferID::INVALID;
+ }
+ return static_cast<TransferID>(aTransfer);
+}
+
+static MatrixID GetMatrixID(int aMatrix) {
+ if (aMatrix < 0 || aMatrix > 11 || aMatrix == 3) {
+ return MatrixID::INVALID;
+ }
+ return static_cast<MatrixID>(aMatrix);
+}
+
+gfx::YUVColorSpace SPSData::ColorSpace() const {
+ // Bitfield, note that guesses with higher values take precedence over
+ // guesses with lower values.
+ enum Guess {
+ GUESS_BT601 = 1 << 0,
+ GUESS_BT709 = 1 << 1,
+ GUESS_BT2020 = 1 << 2,
+ };
+
+ uint32_t guess = 0;
+
+ switch (GetPrimaryID(colour_primaries)) {
+ case PrimaryID::BT709:
+ guess |= GUESS_BT709;
+ break;
+ case PrimaryID::BT470M:
+ case PrimaryID::BT470BG:
+ case PrimaryID::SMPTE170M:
+ case PrimaryID::SMPTE240M:
+ guess |= GUESS_BT601;
+ break;
+ case PrimaryID::BT2020:
+ guess |= GUESS_BT2020;
+ break;
+ case PrimaryID::FILM:
+ case PrimaryID::SMPTEST428_1:
+ case PrimaryID::SMPTEST431_2:
+ case PrimaryID::SMPTEST432_1:
+ case PrimaryID::EBU_3213_E:
+ case PrimaryID::INVALID:
+ case PrimaryID::UNSPECIFIED:
+ break;
+ }
+
+ switch (GetTransferID(transfer_characteristics)) {
+ case TransferID::BT709:
+ guess |= GUESS_BT709;
+ break;
+ case TransferID::GAMMA22:
+ case TransferID::GAMMA28:
+ case TransferID::SMPTE170M:
+ case TransferID::SMPTE240M:
+ guess |= GUESS_BT601;
+ break;
+ case TransferID::BT2020_10:
+ case TransferID::BT2020_12:
+ guess |= GUESS_BT2020;
+ break;
+ case TransferID::LINEAR:
+ case TransferID::LOG:
+ case TransferID::LOG_SQRT:
+ case TransferID::IEC61966_2_4:
+ case TransferID::BT1361_ECG:
+ case TransferID::IEC61966_2_1:
+ case TransferID::SMPTEST2084:
+ case TransferID::SMPTEST428_1:
+ case TransferID::ARIB_STD_B67:
+ case TransferID::INVALID:
+ case TransferID::UNSPECIFIED:
+ break;
+ }
+
+ switch (GetMatrixID(matrix_coefficients)) {
+ case MatrixID::BT709:
+ guess |= GUESS_BT709;
+ break;
+ case MatrixID::BT470BG:
+ case MatrixID::SMPTE170M:
+ case MatrixID::SMPTE240M:
+ guess |= GUESS_BT601;
+ break;
+ case MatrixID::BT2020_NCL:
+ case MatrixID::BT2020_CL:
+ guess |= GUESS_BT2020;
+ break;
+ case MatrixID::RGB:
+ case MatrixID::FCC:
+ case MatrixID::YCOCG:
+ case MatrixID::YDZDX:
+ case MatrixID::INVALID:
+ case MatrixID::UNSPECIFIED:
+ break;
+ }
+
+ // Removes lowest bit until only a single bit remains.
+ while (guess & (guess - 1)) {
+ guess &= guess - 1;
+ }
+ if (!guess) {
+ // A better default to BT601 which should die a slow death.
+ guess = GUESS_BT709;
+ }
+
+ switch (guess) {
+ case GUESS_BT601:
+ return gfx::YUVColorSpace::BT601;
+ case GUESS_BT709:
+ return gfx::YUVColorSpace::BT709;
+ case GUESS_BT2020:
+ return gfx::YUVColorSpace::BT2020;
+ default:
+ MOZ_CRASH("not possible to get here but makes compiler happy");
+ }
+}
+
+gfx::ColorDepth SPSData::ColorDepth() const {
+ if (bit_depth_luma_minus8 != 0 && bit_depth_luma_minus8 != 2 &&
+ bit_depth_luma_minus8 != 4) {
+ // We don't know what that is, just assume 8 bits to prevent decoding
+ // regressions if we ever encounter those.
+ return gfx::ColorDepth::COLOR_8;
+ }
+ return gfx::ColorDepthForBitDepth(bit_depth_luma_minus8 + 8);
+}
+
+// SPSNAL and SPSNALIterator do not own their data.
+class SPSNAL {
+ public:
+ SPSNAL(const uint8_t* aPtr, size_t aLength) {
+ MOZ_ASSERT(aPtr);
+
+ if (aLength == 0 || (*aPtr & 0x1f) != H264_NAL_SPS) {
+ return;
+ }
+ mDecodedNAL = H264::DecodeNALUnit(aPtr, aLength);
+ if (mDecodedNAL) {
+ mLength = BitReader::GetBitLength(mDecodedNAL);
+ }
+ }
+
+ SPSNAL() = default;
+
+ bool IsValid() const { return mDecodedNAL; }
+
+ bool operator==(const SPSNAL& aOther) const {
+ if (!mDecodedNAL || !aOther.mDecodedNAL) {
+ return false;
+ }
+
+ SPSData decodedSPS1;
+ SPSData decodedSPS2;
+ if (!GetSPSData(decodedSPS1) || !aOther.GetSPSData(decodedSPS2)) {
+ // Couldn't decode one SPS, perform a binary comparison
+ if (mLength != aOther.mLength) {
+ return false;
+ }
+ MOZ_ASSERT(mLength / 8 <= mDecodedNAL->Length());
+
+ if (memcmp(mDecodedNAL->Elements(), aOther.mDecodedNAL->Elements(),
+ mLength / 8) != 0) {
+ return false;
+ }
+
+ uint32_t remaining = mLength - (mLength & ~7);
+
+ BitReader b1(mDecodedNAL->Elements() + mLength / 8, remaining);
+ BitReader b2(aOther.mDecodedNAL->Elements() + mLength / 8, remaining);
+ for (uint32_t i = 0; i < remaining; i++) {
+ if (b1.ReadBit() != b2.ReadBit()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ return decodedSPS1 == decodedSPS2;
+ }
+
+ bool operator!=(const SPSNAL& aOther) const { return !(operator==(aOther)); }
+
+ bool GetSPSData(SPSData& aDest) const {
+ return H264::DecodeSPS(mDecodedNAL, aDest);
+ }
+
+ private:
+ RefPtr<mozilla::MediaByteBuffer> mDecodedNAL;
+ uint32_t mLength = 0;
+};
+
+class SPSNALIterator {
+ public:
+ explicit SPSNALIterator(const mozilla::MediaByteBuffer* aExtraData)
+ : mExtraDataPtr(aExtraData->Elements()), mReader(aExtraData) {
+ if (!mReader.Read(5)) {
+ return;
+ }
+
+ auto res = mReader.ReadU8();
+ mNumSPS = res.isOk() ? res.unwrap() & 0x1f : 0;
+ if (mNumSPS == 0) {
+ return;
+ }
+ mValid = true;
+ }
+
+ SPSNALIterator& operator++() {
+ if (mEOS || !mValid) {
+ return *this;
+ }
+ if (--mNumSPS == 0) {
+ mEOS = true;
+ }
+ auto res = mReader.ReadU16();
+ uint16_t length = res.isOk() ? res.unwrap() : 0;
+ if (length == 0 || !mReader.Read(length)) {
+ mEOS = true;
+ }
+ return *this;
+ }
+
+ explicit operator bool() const { return mValid && !mEOS; }
+
+ SPSNAL operator*() const {
+ MOZ_ASSERT(bool(*this));
+ BufferReader reader(mExtraDataPtr + mReader.Offset(), mReader.Remaining());
+
+ auto res = reader.ReadU16();
+ uint16_t length = res.isOk() ? res.unwrap() : 0;
+ const uint8_t* ptr = reader.Read(length);
+ if (!ptr || !length) {
+ return SPSNAL();
+ }
+ return SPSNAL(ptr, length);
+ }
+
+ private:
+ const uint8_t* mExtraDataPtr;
+ BufferReader mReader;
+ bool mValid = false;
+ bool mEOS = false;
+ uint8_t mNumSPS = 0;
+};
+
+/* static */ already_AddRefed<mozilla::MediaByteBuffer> H264::DecodeNALUnit(
+ const uint8_t* aNAL, size_t aLength) {
+ MOZ_ASSERT(aNAL);
+
+ if (aLength < 4) {
+ return nullptr;
+ }
+
+ RefPtr<mozilla::MediaByteBuffer> rbsp = new mozilla::MediaByteBuffer;
+ BufferReader reader(aNAL, aLength);
+ auto res = reader.ReadU8();
+ if (res.isErr()) {
+ return nullptr;
+ }
+ uint8_t nal_unit_type = res.unwrap() & 0x1f;
+ uint32_t nalUnitHeaderBytes = 1;
+ if (nal_unit_type == H264_NAL_PREFIX || nal_unit_type == H264_NAL_SLICE_EXT ||
+ nal_unit_type == H264_NAL_SLICE_EXT_DVC) {
+ bool svc_extension_flag = false;
+ bool avc_3d_extension_flag = false;
+ if (nal_unit_type != H264_NAL_SLICE_EXT_DVC) {
+ res = reader.PeekU8();
+ if (res.isErr()) {
+ return nullptr;
+ }
+ svc_extension_flag = res.unwrap() & 0x80;
+ } else {
+ res = reader.PeekU8();
+ if (res.isErr()) {
+ return nullptr;
+ }
+ avc_3d_extension_flag = res.unwrap() & 0x80;
+ }
+ if (svc_extension_flag) {
+ nalUnitHeaderBytes += 3;
+ } else if (avc_3d_extension_flag) {
+ nalUnitHeaderBytes += 2;
+ } else {
+ nalUnitHeaderBytes += 3;
+ }
+ }
+ if (!reader.Read(nalUnitHeaderBytes - 1)) {
+ return nullptr;
+ }
+ uint32_t lastbytes = 0xffff;
+ while (reader.Remaining()) {
+ auto res = reader.ReadU8();
+ if (res.isErr()) {
+ return nullptr;
+ }
+ uint8_t byte = res.unwrap();
+ if ((lastbytes & 0xffff) == 0 && byte == 0x03) {
+ // reset last two bytes, to detect the 0x000003 sequence again.
+ lastbytes = 0xffff;
+ } else {
+ rbsp->AppendElement(byte);
+ }
+ lastbytes = (lastbytes << 8) | byte;
+ }
+ return rbsp.forget();
+}
+
+// The reverse of DecodeNALUnit. To allow the distinction between Annex B (that
+// uses 0x000001 as marker) and AVCC, the pattern 0x00 0x00 0x0n (where n is
+// between 0 and 3) can't be found in the bytestream. A 0x03 byte is inserted
+// after the second 0. Eg. 0x00 0x00 0x00 becomes 0x00 0x00 0x03 0x00
+/* static */ already_AddRefed<mozilla::MediaByteBuffer> H264::EncodeNALUnit(
+ const uint8_t* aNAL, size_t aLength) {
+ MOZ_ASSERT(aNAL);
+ RefPtr<MediaByteBuffer> rbsp = new MediaByteBuffer();
+ BufferReader reader(aNAL, aLength);
+
+ auto res = reader.ReadU8();
+ if (res.isErr()) {
+ return rbsp.forget();
+ }
+ rbsp->AppendElement(res.unwrap());
+
+ res = reader.ReadU8();
+ if (res.isErr()) {
+ return rbsp.forget();
+ }
+ rbsp->AppendElement(res.unwrap());
+
+ while ((res = reader.ReadU8()).isOk()) {
+ uint8_t val = res.unwrap();
+ if (val <= 0x03 && rbsp->ElementAt(rbsp->Length() - 2) == 0 &&
+ rbsp->ElementAt(rbsp->Length() - 1) == 0) {
+ rbsp->AppendElement(0x03);
+ }
+ rbsp->AppendElement(val);
+ }
+ return rbsp.forget();
+}
+
+static int32_t ConditionDimension(float aValue) {
+ // This will exclude NaNs and too-big values.
+ if (aValue > 1.0 && aValue <= float(INT32_MAX) / 2) {
+ return int32_t(aValue);
+ }
+ return 0;
+}
+
+/* static */
+bool H264::DecodeSPS(const mozilla::MediaByteBuffer* aSPS, SPSData& aDest) {
+ if (!aSPS) {
+ return false;
+ }
+ BitReader br(aSPS, BitReader::GetBitLength(aSPS));
+
+ aDest.profile_idc = br.ReadBits(8);
+ aDest.constraint_set0_flag = br.ReadBit();
+ aDest.constraint_set1_flag = br.ReadBit();
+ aDest.constraint_set2_flag = br.ReadBit();
+ aDest.constraint_set3_flag = br.ReadBit();
+ aDest.constraint_set4_flag = br.ReadBit();
+ aDest.constraint_set5_flag = br.ReadBit();
+ br.ReadBits(2); // reserved_zero_2bits
+ aDest.level_idc = br.ReadBits(8);
+ READUE(seq_parameter_set_id, MAX_SPS_COUNT - 1);
+
+ if (aDest.profile_idc == 100 || aDest.profile_idc == 110 ||
+ aDest.profile_idc == 122 || aDest.profile_idc == 244 ||
+ aDest.profile_idc == 44 || aDest.profile_idc == 83 ||
+ aDest.profile_idc == 86 || aDest.profile_idc == 118 ||
+ aDest.profile_idc == 128 || aDest.profile_idc == 138 ||
+ aDest.profile_idc == 139 || aDest.profile_idc == 134) {
+ READUE(chroma_format_idc, 3);
+ if (aDest.chroma_format_idc == 3) {
+ aDest.separate_colour_plane_flag = br.ReadBit();
+ }
+ READUE(bit_depth_luma_minus8, 6);
+ READUE(bit_depth_chroma_minus8, 6);
+ br.ReadBit(); // qpprime_y_zero_transform_bypass_flag
+ aDest.seq_scaling_matrix_present_flag = br.ReadBit();
+ if (aDest.seq_scaling_matrix_present_flag) {
+ scaling_list(br, aDest.scaling_matrix4x4[0], Default_4x4_Intra,
+ Default_4x4_Intra);
+ scaling_list(br, aDest.scaling_matrix4x4[1], Default_4x4_Intra,
+ aDest.scaling_matrix4x4[0]);
+ scaling_list(br, aDest.scaling_matrix4x4[2], Default_4x4_Intra,
+ aDest.scaling_matrix4x4[1]);
+ scaling_list(br, aDest.scaling_matrix4x4[3], Default_4x4_Inter,
+ Default_4x4_Inter);
+ scaling_list(br, aDest.scaling_matrix4x4[4], Default_4x4_Inter,
+ aDest.scaling_matrix4x4[3]);
+ scaling_list(br, aDest.scaling_matrix4x4[5], Default_4x4_Inter,
+ aDest.scaling_matrix4x4[4]);
+
+ scaling_list(br, aDest.scaling_matrix8x8[0], Default_8x8_Intra,
+ Default_8x8_Intra);
+ scaling_list(br, aDest.scaling_matrix8x8[1], Default_8x8_Inter,
+ Default_8x8_Inter);
+ if (aDest.chroma_format_idc == 3) {
+ scaling_list(br, aDest.scaling_matrix8x8[2], Default_8x8_Intra,
+ aDest.scaling_matrix8x8[0]);
+ scaling_list(br, aDest.scaling_matrix8x8[3], Default_8x8_Inter,
+ aDest.scaling_matrix8x8[1]);
+ scaling_list(br, aDest.scaling_matrix8x8[4], Default_8x8_Intra,
+ aDest.scaling_matrix8x8[2]);
+ scaling_list(br, aDest.scaling_matrix8x8[5], Default_8x8_Inter,
+ aDest.scaling_matrix8x8[3]);
+ }
+ }
+ } else if (aDest.profile_idc == 183) {
+ aDest.chroma_format_idc = 0;
+ } else {
+ // default value if chroma_format_idc isn't set.
+ aDest.chroma_format_idc = 1;
+ }
+ READUE(log2_max_frame_num, 12);
+ aDest.log2_max_frame_num += 4;
+ READUE(pic_order_cnt_type, 2);
+ if (aDest.pic_order_cnt_type == 0) {
+ READUE(log2_max_pic_order_cnt_lsb, 12);
+ aDest.log2_max_pic_order_cnt_lsb += 4;
+ } else if (aDest.pic_order_cnt_type == 1) {
+ aDest.delta_pic_order_always_zero_flag = br.ReadBit();
+ READSE(offset_for_non_ref_pic, -231, 230);
+ READSE(offset_for_top_to_bottom_field, -231, 230);
+ uint32_t num_ref_frames_in_pic_order_cnt_cycle = br.ReadUE();
+ for (uint32_t i = 0; i < num_ref_frames_in_pic_order_cnt_cycle; i++) {
+ br.ReadSE(); // offset_for_ref_frame[i]
+ }
+ }
+ aDest.max_num_ref_frames = br.ReadUE();
+ aDest.gaps_in_frame_num_allowed_flag = br.ReadBit();
+ aDest.pic_width_in_mbs = br.ReadUE() + 1;
+ aDest.pic_height_in_map_units = br.ReadUE() + 1;
+ aDest.frame_mbs_only_flag = br.ReadBit();
+ if (!aDest.frame_mbs_only_flag) {
+ aDest.pic_height_in_map_units *= 2;
+ aDest.mb_adaptive_frame_field_flag = br.ReadBit();
+ }
+ aDest.direct_8x8_inference_flag = br.ReadBit();
+ aDest.frame_cropping_flag = br.ReadBit();
+ if (aDest.frame_cropping_flag) {
+ aDest.frame_crop_left_offset = br.ReadUE();
+ aDest.frame_crop_right_offset = br.ReadUE();
+ aDest.frame_crop_top_offset = br.ReadUE();
+ aDest.frame_crop_bottom_offset = br.ReadUE();
+ }
+
+ aDest.sample_ratio = 1.0f;
+ aDest.vui_parameters_present_flag = br.ReadBit();
+ if (aDest.vui_parameters_present_flag) {
+ if (!vui_parameters(br, aDest)) {
+ return false;
+ }
+ }
+
+ // Calculate common values.
+
+ uint8_t ChromaArrayType =
+ aDest.separate_colour_plane_flag ? 0 : aDest.chroma_format_idc;
+ // Calculate width.
+ uint32_t CropUnitX = 1;
+ uint32_t SubWidthC = aDest.chroma_format_idc == 3 ? 1 : 2;
+ if (ChromaArrayType != 0) {
+ CropUnitX = SubWidthC;
+ }
+
+ // Calculate Height
+ uint32_t CropUnitY = 2 - aDest.frame_mbs_only_flag;
+ uint32_t SubHeightC = aDest.chroma_format_idc <= 1 ? 2 : 1;
+ if (ChromaArrayType != 0) {
+ CropUnitY *= SubHeightC;
+ }
+
+ uint32_t width = aDest.pic_width_in_mbs * 16;
+ uint32_t height = aDest.pic_height_in_map_units * 16;
+ if (aDest.frame_crop_left_offset <=
+ std::numeric_limits<int32_t>::max() / 4 / CropUnitX &&
+ aDest.frame_crop_right_offset <=
+ std::numeric_limits<int32_t>::max() / 4 / CropUnitX &&
+ aDest.frame_crop_top_offset <=
+ std::numeric_limits<int32_t>::max() / 4 / CropUnitY &&
+ aDest.frame_crop_bottom_offset <=
+ std::numeric_limits<int32_t>::max() / 4 / CropUnitY &&
+ (aDest.frame_crop_left_offset + aDest.frame_crop_right_offset) *
+ CropUnitX <
+ width &&
+ (aDest.frame_crop_top_offset + aDest.frame_crop_bottom_offset) *
+ CropUnitY <
+ height) {
+ aDest.crop_left = aDest.frame_crop_left_offset * CropUnitX;
+ aDest.crop_right = aDest.frame_crop_right_offset * CropUnitX;
+ aDest.crop_top = aDest.frame_crop_top_offset * CropUnitY;
+ aDest.crop_bottom = aDest.frame_crop_bottom_offset * CropUnitY;
+ } else {
+ // Nonsensical value, ignore them.
+ aDest.crop_left = aDest.crop_right = aDest.crop_top = aDest.crop_bottom = 0;
+ }
+
+ aDest.pic_width = width - aDest.crop_left - aDest.crop_right;
+ aDest.pic_height = height - aDest.crop_top - aDest.crop_bottom;
+
+ aDest.interlaced = !aDest.frame_mbs_only_flag;
+
+ // Determine display size.
+ if (aDest.sample_ratio > 1.0) {
+ // Increase the intrinsic width
+ aDest.display_width = ConditionDimension(
+ AssertedCast<float>(aDest.pic_width) * aDest.sample_ratio);
+ aDest.display_height = aDest.pic_height;
+ } else {
+ // Increase the intrinsic height
+ aDest.display_width = aDest.pic_width;
+ aDest.display_height = ConditionDimension(
+ AssertedCast<float>(aDest.pic_height) / aDest.sample_ratio);
+ }
+
+ aDest.valid = true;
+
+ return true;
+}
+
+/* static */
+bool H264::vui_parameters(BitReader& aBr, SPSData& aDest) {
+ aDest.aspect_ratio_info_present_flag = aBr.ReadBit();
+ if (aDest.aspect_ratio_info_present_flag) {
+ aDest.aspect_ratio_idc = aBr.ReadBits(8);
+ aDest.sar_width = aDest.sar_height = 0;
+
+ // From E.2.1 VUI parameters semantics (ITU-T H.264 02/2014)
+ switch (aDest.aspect_ratio_idc) {
+ case 0:
+ // Unspecified
+ break;
+ case 1:
+ /*
+ 1:1
+ 7680x4320 16:9 frame without horizontal overscan
+ 3840x2160 16:9 frame without horizontal overscan
+ 1280x720 16:9 frame without horizontal overscan
+ 1920x1080 16:9 frame without horizontal overscan (cropped from
+ 1920x1088) 640x480 4:3 frame without horizontal overscan
+ */
+ aDest.sample_ratio = 1.0f;
+ break;
+ case 2:
+ /*
+ 12:11
+ 720x576 4:3 frame with horizontal overscan
+ 352x288 4:3 frame without horizontal overscan
+ */
+ aDest.sample_ratio = 12.0 / 11.0;
+ break;
+ case 3:
+ /*
+ 10:11
+ 720x480 4:3 frame with horizontal overscan
+ 352x240 4:3 frame without horizontal overscan
+ */
+ aDest.sample_ratio = 10.0 / 11.0;
+ break;
+ case 4:
+ /*
+ 16:11
+ 720x576 16:9 frame with horizontal overscan
+ 528x576 4:3 frame without horizontal overscan
+ */
+ aDest.sample_ratio = 16.0 / 11.0;
+ break;
+ case 5:
+ /*
+ 40:33
+ 720x480 16:9 frame with horizontal overscan
+ 528x480 4:3 frame without horizontal overscan
+ */
+ aDest.sample_ratio = 40.0 / 33.0;
+ break;
+ case 6:
+ /*
+ 24:11
+ 352x576 4:3 frame without horizontal overscan
+ 480x576 16:9 frame with horizontal overscan
+ */
+ aDest.sample_ratio = 24.0 / 11.0;
+ break;
+ case 7:
+ /*
+ 20:11
+ 352x480 4:3 frame without horizontal overscan
+ 480x480 16:9 frame with horizontal overscan
+ */
+ aDest.sample_ratio = 20.0 / 11.0;
+ break;
+ case 8:
+ /*
+ 32:11
+ 352x576 16:9 frame without horizontal overscan
+ */
+ aDest.sample_ratio = 32.0 / 11.0;
+ break;
+ case 9:
+ /*
+ 80:33
+ 352x480 16:9 frame without horizontal overscan
+ */
+ aDest.sample_ratio = 80.0 / 33.0;
+ break;
+ case 10:
+ /*
+ 18:11
+ 480x576 4:3 frame with horizontal overscan
+ */
+ aDest.sample_ratio = 18.0 / 11.0;
+ break;
+ case 11:
+ /*
+ 15:11
+ 480x480 4:3 frame with horizontal overscan
+ */
+ aDest.sample_ratio = 15.0 / 11.0;
+ break;
+ case 12:
+ /*
+ 64:33
+ 528x576 16:9 frame with horizontal overscan
+ */
+ aDest.sample_ratio = 64.0 / 33.0;
+ break;
+ case 13:
+ /*
+ 160:99
+ 528x480 16:9 frame without horizontal overscan
+ */
+ aDest.sample_ratio = 160.0 / 99.0;
+ break;
+ case 14:
+ /*
+ 4:3
+ 1440x1080 16:9 frame without horizontal overscan
+ */
+ aDest.sample_ratio = 4.0 / 3.0;
+ break;
+ case 15:
+ /*
+ 3:2
+ 1280x1080 16:9 frame without horizontal overscan
+ */
+ aDest.sample_ratio = 3.2 / 2.0;
+ break;
+ case 16:
+ /*
+ 2:1
+ 960x1080 16:9 frame without horizontal overscan
+ */
+ aDest.sample_ratio = 2.0 / 1.0;
+ break;
+ case 255:
+ /* Extended_SAR */
+ aDest.sar_width = aBr.ReadBits(16);
+ aDest.sar_height = aBr.ReadBits(16);
+ if (aDest.sar_width && aDest.sar_height) {
+ aDest.sample_ratio = float(aDest.sar_width) / float(aDest.sar_height);
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ if (aBr.ReadBit()) { // overscan_info_present_flag
+ aDest.overscan_appropriate_flag = aBr.ReadBit();
+ }
+
+ if (aBr.ReadBit()) { // video_signal_type_present_flag
+ aDest.video_format = aBr.ReadBits(3);
+ aDest.video_full_range_flag = aBr.ReadBit();
+ aDest.colour_description_present_flag = aBr.ReadBit();
+ if (aDest.colour_description_present_flag) {
+ aDest.colour_primaries = aBr.ReadBits(8);
+ aDest.transfer_characteristics = aBr.ReadBits(8);
+ aDest.matrix_coefficients = aBr.ReadBits(8);
+ }
+ }
+
+ aDest.chroma_loc_info_present_flag = aBr.ReadBit();
+ if (aDest.chroma_loc_info_present_flag) {
+ BitReader& br = aBr; // so that macro READUE works
+ READUE(chroma_sample_loc_type_top_field, 5);
+ READUE(chroma_sample_loc_type_bottom_field, 5);
+ }
+
+ bool timing_info_present_flag = aBr.ReadBit();
+ if (timing_info_present_flag) {
+ aBr.ReadBits(32); // num_units_in_tick
+ aBr.ReadBits(32); // time_scale
+ aBr.ReadBit(); // fixed_frame_rate_flag
+ }
+ return true;
+}
+
+/* static */
+bool H264::DecodeSPSFromExtraData(const mozilla::MediaByteBuffer* aExtraData,
+ SPSData& aDest) {
+ SPSNALIterator it(aExtraData);
+ if (!it) {
+ return false;
+ }
+ return (*it).GetSPSData(aDest);
+}
+
+/* static */
+bool H264::EnsureSPSIsSane(SPSData& aSPS) {
+ bool valid = true;
+ static const float default_aspect = 4.0f / 3.0f;
+ if (aSPS.sample_ratio <= 0.0f || aSPS.sample_ratio > 6.0f) {
+ if (aSPS.pic_width && aSPS.pic_height) {
+ aSPS.sample_ratio = (float)aSPS.pic_width / (float)aSPS.pic_height;
+ } else {
+ aSPS.sample_ratio = default_aspect;
+ }
+ aSPS.display_width = aSPS.pic_width;
+ aSPS.display_height = aSPS.pic_height;
+ valid = false;
+ }
+ if (aSPS.max_num_ref_frames > 16) {
+ aSPS.max_num_ref_frames = 16;
+ valid = false;
+ }
+ return valid;
+}
+
+/* static */
+uint32_t H264::ComputeMaxRefFrames(const mozilla::MediaByteBuffer* aExtraData) {
+ uint32_t maxRefFrames = 4;
+ // Retrieve video dimensions from H264 SPS NAL.
+ SPSData spsdata;
+ if (DecodeSPSFromExtraData(aExtraData, spsdata)) {
+ // max_num_ref_frames determines the size of the sliding window
+ // we need to queue that many frames in order to guarantee proper
+ // pts frames ordering. Use a minimum of 4 to ensure proper playback of
+ // non compliant videos.
+ maxRefFrames =
+ std::min(std::max(maxRefFrames, spsdata.max_num_ref_frames + 1), 16u);
+ }
+ return maxRefFrames;
+}
+
+/* static */ H264::FrameType H264::GetFrameType(
+ const mozilla::MediaRawData* aSample) {
+ auto avcc = AVCCConfig::Parse(aSample);
+ if (avcc.isErr()) {
+ // We must have a valid AVCC frame with extradata.
+ return FrameType::INVALID;
+ }
+ MOZ_ASSERT(aSample->Data());
+
+ int nalLenSize = avcc.unwrap().NALUSize();
+
+ BufferReader reader(aSample->Data(), aSample->Size());
+
+ while (reader.Remaining() >= nalLenSize) {
+ uint32_t nalLen = 0;
+ switch (nalLenSize) {
+ case 1:
+ nalLen = reader.ReadU8().unwrapOr(0);
+ break;
+ case 2:
+ nalLen = reader.ReadU16().unwrapOr(0);
+ break;
+ case 3:
+ nalLen = reader.ReadU24().unwrapOr(0);
+ break;
+ case 4:
+ nalLen = reader.ReadU32().unwrapOr(0);
+ break;
+ default:
+ MOZ_ASSERT_UNREACHABLE("NAL length is up to 4 bytes");
+ }
+ if (!nalLen) {
+ continue;
+ }
+ const uint8_t* p = reader.Read(nalLen);
+ if (!p) {
+ return FrameType::INVALID;
+ }
+ int8_t nalType = AssertedCast<int8_t>(*p & 0x1f);
+ if (nalType == H264_NAL_IDR_SLICE) {
+ // IDR NAL.
+ return FrameType::I_FRAME;
+ }
+ if (nalType == H264_NAL_SEI) {
+ RefPtr<mozilla::MediaByteBuffer> decodedNAL = DecodeNALUnit(p, nalLen);
+ SEIRecoveryData data;
+ if (DecodeRecoverySEI(decodedNAL, data)) {
+ return FrameType::I_FRAME;
+ }
+ } else if (nalType == H264_NAL_SLICE) {
+ RefPtr<mozilla::MediaByteBuffer> decodedNAL = DecodeNALUnit(p, nalLen);
+ if (DecodeISlice(decodedNAL)) {
+ return FrameType::I_FRAME;
+ }
+ }
+ }
+
+ return FrameType::OTHER;
+}
+
+/* static */ already_AddRefed<mozilla::MediaByteBuffer> H264::ExtractExtraData(
+ const mozilla::MediaRawData* aSample) {
+ auto avcc = AVCCConfig::Parse(aSample);
+ MOZ_ASSERT(avcc.isOk());
+
+ RefPtr<mozilla::MediaByteBuffer> extradata = new mozilla::MediaByteBuffer;
+
+ // SPS content
+ nsTArray<uint8_t> sps;
+ ByteWriter<BigEndian> spsw(sps);
+ int numSps = 0;
+ // PPS content
+ nsTArray<uint8_t> pps;
+ ByteWriter<BigEndian> ppsw(pps);
+ int numPps = 0;
+
+ int nalLenSize = avcc.unwrap().NALUSize();
+
+ size_t sampleSize = aSample->Size();
+ if (aSample->mCrypto.IsEncrypted()) {
+ // The content is encrypted, we can only parse the non-encrypted data.
+ MOZ_ASSERT(aSample->mCrypto.mPlainSizes.Length() > 0);
+ if (aSample->mCrypto.mPlainSizes.Length() == 0 ||
+ aSample->mCrypto.mPlainSizes[0] > sampleSize) {
+ // This is invalid content.
+ return nullptr;
+ }
+ sampleSize = aSample->mCrypto.mPlainSizes[0];
+ }
+
+ BufferReader reader(aSample->Data(), sampleSize);
+
+ nsTArray<SPSData> SPSTable;
+ // If we encounter SPS with the same id but different content, we will stop
+ // attempting to detect duplicates.
+ bool checkDuplicate = true;
+
+ // Find SPS and PPS NALUs in AVCC data
+ while (reader.Remaining() > nalLenSize) {
+ uint32_t nalLen = 0;
+ switch (nalLenSize) {
+ case 1:
+ Unused << reader.ReadU8().map(
+ [&](uint8_t x) mutable { return nalLen = x; });
+ break;
+ case 2:
+ Unused << reader.ReadU16().map(
+ [&](uint16_t x) mutable { return nalLen = x; });
+ break;
+ case 3:
+ Unused << reader.ReadU24().map(
+ [&](uint32_t x) mutable { return nalLen = x; });
+ break;
+ case 4:
+ Unused << reader.ReadU32().map(
+ [&](uint32_t x) mutable { return nalLen = x; });
+ break;
+ default:
+ MOZ_ASSERT_UNREACHABLE("NAL length size is at most 4 bytes");
+ }
+ const uint8_t* p = reader.Read(nalLen);
+ if (!p) {
+ // The read failed, but we may already have some SPS + PPS data so
+ // break out of reading and process what we have, if any.
+ break;
+ }
+ uint8_t nalType = *p & 0x1f;
+
+ if (nalType == H264_NAL_SPS) {
+ RefPtr<mozilla::MediaByteBuffer> sps = DecodeNALUnit(p, nalLen);
+ SPSData data;
+ if (!DecodeSPS(sps, data)) {
+ // Invalid SPS, ignore.
+ continue;
+ }
+ uint8_t spsId = data.seq_parameter_set_id;
+ if (spsId >= SPSTable.Length()) {
+ if (!SPSTable.SetLength(spsId + 1, fallible)) {
+ // OOM.
+ return nullptr;
+ }
+ }
+ if (checkDuplicate && SPSTable[spsId].valid && SPSTable[spsId] == data) {
+ // Duplicate ignore.
+ continue;
+ }
+ if (SPSTable[spsId].valid) {
+ // We already have detected a SPS with this Id. Just to be safe we
+ // disable SPS duplicate detection.
+ checkDuplicate = false;
+ } else {
+ SPSTable[spsId] = data;
+ }
+ numSps++;
+ if (!spsw.WriteU16(nalLen) || !spsw.Write(p, nalLen)) {
+ return extradata.forget();
+ }
+ } else if (nalType == H264_NAL_PPS) {
+ numPps++;
+ if (!ppsw.WriteU16(nalLen) || !ppsw.Write(p, nalLen)) {
+ return extradata.forget();
+ }
+ }
+ }
+
+ // We ignore PPS data if we didn't find a SPS as we would be unable to
+ // decode it anyway.
+ numPps = numSps ? numPps : 0;
+
+ if (numSps && sps.Length() > 5) {
+ extradata->AppendElement(1); // version
+ extradata->AppendElement(sps[3]); // profile
+ extradata->AppendElement(sps[4]); // profile compat
+ extradata->AppendElement(sps[5]); // level
+ extradata->AppendElement(0xfc | 3); // nal size - 1
+ extradata->AppendElement(0xe0 | numSps);
+ extradata->AppendElements(sps.Elements(), sps.Length());
+ extradata->AppendElement(numPps);
+ if (numPps) {
+ extradata->AppendElements(pps.Elements(), pps.Length());
+ }
+ }
+
+ return extradata.forget();
+}
+
+/* static */
+uint8_t H264::NumSPS(const mozilla::MediaByteBuffer* aExtraData) {
+ auto avcc = AVCCConfig::Parse(aExtraData);
+ return avcc.isErr() ? 0 : avcc.unwrap().mNumSPS;
+}
+
+/* static */
+bool H264::HasSPS(const mozilla::MediaByteBuffer* aExtraData) {
+ return H264::NumSPS(aExtraData) > 0;
+}
+
+/* static */
+bool H264::CompareExtraData(const mozilla::MediaByteBuffer* aExtraData1,
+ const mozilla::MediaByteBuffer* aExtraData2) {
+ if (aExtraData1 == aExtraData2) {
+ return true;
+ }
+ uint8_t numSPS = NumSPS(aExtraData1);
+ if (numSPS == 0 || numSPS != NumSPS(aExtraData2)) {
+ return false;
+ }
+
+ // We only compare if the SPS are the same as the various H264 decoders can
+ // deal with in-band change of PPS.
+
+ SPSNALIterator it1(aExtraData1);
+ SPSNALIterator it2(aExtraData2);
+
+ while (it1 && it2) {
+ if (*it1 != *it2) {
+ return false;
+ }
+ ++it1;
+ ++it2;
+ }
+ return true;
+}
+
+static inline Result<Ok, nsresult> ReadSEIInt(BufferReader& aBr,
+ uint32_t& aOutput) {
+ uint8_t tmpByte;
+
+ aOutput = 0;
+ MOZ_TRY_VAR(tmpByte, aBr.ReadU8());
+ while (tmpByte == 0xFF) {
+ aOutput += 255;
+ MOZ_TRY_VAR(tmpByte, aBr.ReadU8());
+ }
+ aOutput += tmpByte; // this is the last byte
+ return Ok();
+}
+
+/* static */
+bool H264::DecodeISlice(const mozilla::MediaByteBuffer* aSlice) {
+ if (!aSlice) {
+ return false;
+ }
+
+ // According to ITU-T Rec H.264 Table 7.3.3, read the slice type from
+ // slice_header, and the slice type 2 and 7 are representing I slice.
+ BitReader br(aSlice);
+ // Skip `first_mb_in_slice`
+ br.ReadUE();
+ // The value of slice type can go from 0 to 9, but the value between 5 to
+ // 9 are actually equal to 0 to 4.
+ const uint32_t sliceType = br.ReadUE() % 5;
+ return sliceType == SLICE_TYPES::I_SLICE || sliceType == SI_SLICE;
+}
+
+/* static */
+bool H264::DecodeRecoverySEI(const mozilla::MediaByteBuffer* aSEI,
+ SEIRecoveryData& aDest) {
+ if (!aSEI) {
+ return false;
+ }
+ // sei_rbsp() as per 7.3.2.3 Supplemental enhancement information RBSP syntax
+ BufferReader br(aSEI);
+
+ do {
+ // sei_message() as per
+ // 7.3.2.3.1 Supplemental enhancement information message syntax
+ uint32_t payloadType = 0;
+ if (ReadSEIInt(br, payloadType).isErr()) {
+ return false;
+ }
+
+ uint32_t payloadSize = 0;
+ if (ReadSEIInt(br, payloadSize).isErr()) {
+ return false;
+ }
+
+ // sei_payload(payloadType, payloadSize) as per
+ // D.1 SEI payload syntax.
+ const uint8_t* p = br.Read(payloadSize);
+ if (!p) {
+ return false;
+ }
+ if (payloadType == 6) { // SEI_RECOVERY_POINT
+ if (payloadSize == 0) {
+ // Invalid content, ignore.
+ continue;
+ }
+ // D.1.7 Recovery point SEI message syntax
+ BitReader br(p, payloadSize * 8);
+ aDest.recovery_frame_cnt = br.ReadUE();
+ aDest.exact_match_flag = br.ReadBit();
+ aDest.broken_link_flag = br.ReadBit();
+ aDest.changing_slice_group_idc = br.ReadBits(2);
+ return true;
+ }
+ } while (br.PeekU8().isOk() &&
+ br.PeekU8().unwrap() !=
+ 0x80); // more_rbsp_data() msg[offset] != 0x80
+ // ignore the trailing bits rbsp_trailing_bits();
+ return false;
+}
+
+/*static */ already_AddRefed<mozilla::MediaByteBuffer> H264::CreateExtraData(
+ uint8_t aProfile, uint8_t aConstraints, uint8_t aLevel,
+ const gfx::IntSize& aSize) {
+ // SPS of a 144p video.
+ const uint8_t originSPS[] = {0x4d, 0x40, 0x0c, 0xe8, 0x80, 0x80, 0x9d,
+ 0x80, 0xb5, 0x01, 0x01, 0x01, 0x40, 0x00,
+ 0x00, 0x00, 0x40, 0x00, 0x00, 0x0f, 0x03,
+ 0xc5, 0x0a, 0x44, 0x80};
+
+ RefPtr<MediaByteBuffer> extraData = new MediaByteBuffer();
+ extraData->AppendElements(originSPS, sizeof(originSPS));
+ BitReader br(extraData, BitReader::GetBitLength(extraData));
+
+ RefPtr<MediaByteBuffer> sps = new MediaByteBuffer();
+ BitWriter bw(sps);
+
+ br.ReadBits(8); // Skip original profile_idc
+ bw.WriteU8(aProfile);
+ br.ReadBits(8); // Skip original constraint flags && reserved_zero_2bits
+ aConstraints =
+ aConstraints & ~0x3; // Ensure reserved_zero_2bits are set to 0
+ bw.WriteBits(aConstraints, 8);
+ br.ReadBits(8); // Skip original level_idc
+ bw.WriteU8(aLevel);
+ bw.WriteUE(br.ReadUE()); // seq_parameter_set_id (0 stored on 1 bit)
+
+ if (aProfile == 100 || aProfile == 110 || aProfile == 122 ||
+ aProfile == 244 || aProfile == 44 || aProfile == 83 || aProfile == 86 ||
+ aProfile == 118 || aProfile == 128 || aProfile == 138 ||
+ aProfile == 139 || aProfile == 134) {
+ bw.WriteUE(1); // chroma_format_idc -> always set to 4:2:0 chroma format
+ bw.WriteUE(0); // bit_depth_luma_minus8 -> always 8 bits here
+ bw.WriteUE(0); // bit_depth_chroma_minus8 -> always 8 bits here
+ bw.WriteBit(false); // qpprime_y_zero_transform_bypass_flag
+ bw.WriteBit(false); // seq_scaling_matrix_present_flag
+ }
+
+ bw.WriteBits(br.ReadBits(11),
+ 11); // log2_max_frame_num to gaps_in_frame_num_allowed_flag
+
+ // skip over original exp-golomb encoded width/height
+ br.ReadUE(); // skip width
+ br.ReadUE(); // skip height
+ uint32_t width = aSize.width;
+ uint32_t widthNeeded = width % 16 != 0 ? (width / 16 + 1) * 16 : width;
+ uint32_t height = aSize.height;
+ uint32_t heightNeeded = height % 16 != 0 ? (height / 16 + 1) * 16 : height;
+ bw.WriteUE(widthNeeded / 16 - 1);
+ bw.WriteUE(heightNeeded / 16 - 1);
+ bw.WriteBit(br.ReadBit()); // write frame_mbs_only_flag
+ bw.WriteBit(br.ReadBit()); // write direct_8x8_inference_flag;
+ if (widthNeeded != width || heightNeeded != height) {
+ // Write cropping value
+ bw.WriteBit(true); // skip frame_cropping_flag
+ bw.WriteUE(0); // frame_crop_left_offset
+ bw.WriteUE((widthNeeded - width) / 2); // frame_crop_right_offset
+ bw.WriteUE(0); // frame_crop_top_offset
+ bw.WriteUE((heightNeeded - height) / 2); // frame_crop_bottom_offset
+ } else {
+ bw.WriteBit(false); // skip frame_cropping_flag
+ }
+ br.ReadBit(); // skip frame_cropping_flag;
+ // Write the remainings of the original sps (vui_parameters which sets an
+ // aspect ration of 1.0)
+ while (br.BitsLeft()) {
+ bw.WriteBit(br.ReadBit());
+ }
+ bw.CloseWithRbspTrailing();
+
+ RefPtr<MediaByteBuffer> encodedSPS =
+ EncodeNALUnit(sps->Elements(), sps->Length());
+ extraData->Clear();
+
+ const uint8_t PPS[] = {0xeb, 0xef, 0x20};
+
+ WriteExtraData(
+ extraData, aProfile, aConstraints, aLevel,
+ Span<const uint8_t>(encodedSPS->Elements(), encodedSPS->Length()),
+ Span<const uint8_t>(PPS, sizeof(PPS)));
+
+ return extraData.forget();
+}
+
+void H264::WriteExtraData(MediaByteBuffer* aDestExtraData,
+ const uint8_t aProfile, const uint8_t aConstraints,
+ const uint8_t aLevel, const Span<const uint8_t> aSPS,
+ const Span<const uint8_t> aPPS) {
+ aDestExtraData->AppendElement(1);
+ aDestExtraData->AppendElement(aProfile);
+ aDestExtraData->AppendElement(aConstraints);
+ aDestExtraData->AppendElement(aLevel);
+ aDestExtraData->AppendElement(3); // nalLENSize-1
+ aDestExtraData->AppendElement(1); // numPPS
+ uint8_t c[2];
+ mozilla::BigEndian::writeUint16(&c[0], aSPS.Length() + 1);
+ aDestExtraData->AppendElements(c, 2);
+ aDestExtraData->AppendElement((0x00 << 7) | (0x3 << 5) | H264_NAL_SPS);
+ aDestExtraData->AppendElements(aSPS.Elements(), aSPS.Length());
+
+ aDestExtraData->AppendElement(1); // numPPS
+ mozilla::BigEndian::writeUint16(&c[0], aPPS.Length() + 1);
+ aDestExtraData->AppendElements(c, 2);
+ aDestExtraData->AppendElement((0x00 << 7) | (0x3 << 5) | H264_NAL_PPS);
+ aDestExtraData->AppendElements(aPPS.Elements(), aPPS.Length());
+}
+
+/* static */ Result<AVCCConfig, nsresult> AVCCConfig::Parse(
+ const mozilla::MediaRawData* aSample) {
+ if (!aSample || aSample->Size() < 3) {
+ return mozilla::Err(NS_ERROR_FAILURE);
+ }
+ if (aSample->mTrackInfo &&
+ !aSample->mTrackInfo->mMimeType.EqualsLiteral("video/avc")) {
+ LOG("Only allow 'video/avc' (mimeType=%s)",
+ aSample->mTrackInfo->mMimeType.get());
+ return mozilla::Err(NS_ERROR_FAILURE);
+ }
+ return AVCCConfig::Parse(aSample->mExtraData);
+}
+
+/* static */ Result<AVCCConfig, nsresult> AVCCConfig::Parse(
+ const mozilla::MediaByteBuffer* aExtraData) {
+ if (!aExtraData || aExtraData->Length() < 7) {
+ return mozilla::Err(NS_ERROR_FAILURE);
+ }
+ const auto& byteBuffer = *aExtraData;
+ if (byteBuffer[0] != 1) {
+ return mozilla::Err(NS_ERROR_FAILURE);
+ }
+ AVCCConfig avcc{};
+ avcc.mConfigurationVersion = byteBuffer[0];
+ avcc.mAVCProfileIndication = byteBuffer[1];
+ avcc.mProfileCompatibility = byteBuffer[2];
+ avcc.mAVCLevelIndication = byteBuffer[3];
+ avcc.mLengthSizeMinusOne = byteBuffer[4] & 0x3;
+ avcc.mNumSPS = byteBuffer[5] & 0x1F;
+ return avcc;
+}
+
+#undef READUE
+#undef READSE
+
+} // namespace mozilla
+
+#undef LOG
diff --git a/dom/media/platforms/agnostic/bytestreams/H264.h b/dom/media/platforms/agnostic/bytestreams/H264.h
new file mode 100644
index 0000000000..c3651d1a0f
--- /dev/null
+++ b/dom/media/platforms/agnostic/bytestreams/H264.h
@@ -0,0 +1,547 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef MP4_DEMUXER_H264_H_
+#define MP4_DEMUXER_H264_H_
+
+#include <stdint.h>
+#include "DecoderData.h"
+#include "mozilla/gfx/Types.h"
+
+namespace mozilla {
+class BitReader;
+
+// Spec 7.4.2.1
+#define MAX_SPS_COUNT 32
+#define MAX_PPS_COUNT 256
+
+// NAL unit types
+enum NAL_TYPES {
+ H264_NAL_SLICE = 1,
+ H264_NAL_DPA = 2,
+ H264_NAL_DPB = 3,
+ H264_NAL_DPC = 4,
+ H264_NAL_IDR_SLICE = 5,
+ H264_NAL_SEI = 6,
+ H264_NAL_SPS = 7,
+ H264_NAL_PPS = 8,
+ H264_NAL_AUD = 9,
+ H264_NAL_END_SEQUENCE = 10,
+ H264_NAL_END_STREAM = 11,
+ H264_NAL_FILLER_DATA = 12,
+ H264_NAL_SPS_EXT = 13,
+ H264_NAL_PREFIX = 14,
+ H264_NAL_AUXILIARY_SLICE = 19,
+ H264_NAL_SLICE_EXT = 20,
+ H264_NAL_SLICE_EXT_DVC = 21,
+};
+
+// According to ITU-T Rec H.264 (2017/04) Table 7.6.
+enum SLICE_TYPES {
+ P_SLICE = 0,
+ B_SLICE = 1,
+ I_SLICE = 2,
+ SP_SLICE = 3,
+ SI_SLICE = 4,
+};
+
+struct SPSData {
+ bool operator==(const SPSData& aOther) const;
+ bool operator!=(const SPSData& aOther) const;
+
+ gfx::YUVColorSpace ColorSpace() const;
+ gfx::ColorDepth ColorDepth() const;
+
+ bool valid = {};
+
+ /* Decoded Members */
+ /*
+ pic_width is the decoded width according to:
+ pic_width = ((pic_width_in_mbs_minus1 + 1) * 16)
+ - (frame_crop_left_offset + frame_crop_right_offset) * 2
+ */
+ uint32_t pic_width = {};
+ /*
+ pic_height is the decoded height according to:
+ pic_height = (2 - frame_mbs_only_flag) * ((pic_height_in_map_units_minus1 +
+ 1) * 16)
+ - (frame_crop_top_offset + frame_crop_bottom_offset) * 2
+ */
+ uint32_t pic_height = {};
+
+ bool interlaced = {};
+
+ /*
+ Displayed size.
+ display_width and display_height are adjusted according to the display
+ sample aspect ratio.
+ */
+ uint32_t display_width = {};
+ uint32_t display_height = {};
+
+ float sample_ratio = {};
+
+ uint32_t crop_left = {};
+ uint32_t crop_right = {};
+ uint32_t crop_top = {};
+ uint32_t crop_bottom = {};
+
+ /*
+ H264 decoding parameters according to ITU-T H.264 (T-REC-H.264-201402-I/en)
+ http://www.itu.int/rec/T-REC-H.264-201402-I/en
+ */
+
+ bool constraint_set0_flag = {};
+ bool constraint_set1_flag = {};
+ bool constraint_set2_flag = {};
+ bool constraint_set3_flag = {};
+ bool constraint_set4_flag = {};
+ bool constraint_set5_flag = {};
+
+ /*
+ profile_idc and level_idc indicate the profile and level to which the coded
+ video sequence conforms when the SVC sequence parameter set is the active
+ SVC sequence parameter set.
+ */
+ uint8_t profile_idc = {};
+ uint8_t level_idc = {};
+
+ /*
+ seq_parameter_set_id identifies the sequence parameter set that is referred
+ to by the picture parameter set. The value of seq_parameter_set_id shall be
+ in the range of 0 to 31, inclusive.
+ */
+ uint8_t seq_parameter_set_id = {};
+
+ /*
+ chroma_format_idc specifies the chroma sampling relative to the luma
+ sampling as specified in clause 6.2. The value of chroma_format_idc shall be
+ in the range of 0 to 3, inclusive. When chroma_format_idc is not present,
+ it shall be inferred to be equal to 1 (4:2:0 chroma format).
+ When profile_idc is equal to 183, chroma_format_idc shall be equal to 0
+ (4:0:0 chroma format).
+ */
+ uint8_t chroma_format_idc = {};
+
+ /*
+ bit_depth_luma_minus8 specifies the bit depth of the samples of the luma
+ array and the value of the luma quantisation parameter range offset
+ QpBdOffset Y , as specified by
+ BitDepth Y = 8 + bit_depth_luma_minus8 (7-3)
+ QpBdOffset Y = 6 * bit_depth_luma_minus8 (7-4)
+ When bit_depth_luma_minus8 is not present, it shall be inferred to be equal
+ to 0. bit_depth_luma_minus8 shall be in the range of 0 to 6, inclusive.
+ */
+ uint8_t bit_depth_luma_minus8 = {};
+
+ /*
+ bit_depth_chroma_minus8 specifies the bit depth of the samples of the chroma
+ arrays and the value of the chroma quantisation parameter range offset
+ QpBdOffset C , as specified by
+ BitDepth C = 8 + bit_depth_chroma_minus8 (7-5)
+ QpBdOffset C = 6 * bit_depth_chroma_minus8 (7-6)
+ When bit_depth_chroma_minus8 is not present, it shall be inferred to be
+ equal to 0. bit_depth_chroma_minus8 shall be in the range of 0 to 6,
+ inclusive.
+ */
+ uint8_t bit_depth_chroma_minus8 = {};
+
+ /*
+ separate_colour_plane_flag equal to 1 specifies that the three colour
+ components of the 4:4:4 chroma format are coded separately.
+ separate_colour_plane_flag equal to 0 specifies that the colour components
+ are not coded separately. When separate_colour_plane_flag is not present,
+ it shall be inferred to be equal to 0. When separate_colour_plane_flag is
+ equal to 1, the primary coded picture consists of three separate components,
+ each of which consists of coded samples of one colour plane (Y, Cb or Cr)
+ that each use the monochrome coding syntax. In this case, each colour plane
+ is associated with a specific colour_plane_id value.
+ */
+ bool separate_colour_plane_flag = {};
+
+ /*
+ seq_scaling_matrix_present_flag equal to 1 specifies that the flags
+ seq_scaling_list_present_flag[ i ] for i = 0..7 or
+ i = 0..11 are present. seq_scaling_matrix_present_flag equal to 0 specifies
+ that these flags are not present and the sequence-level scaling list
+ specified by Flat_4x4_16 shall be inferred for i = 0..5 and the
+ sequence-level scaling list specified by Flat_8x8_16 shall be inferred for
+ i = 6..11. When seq_scaling_matrix_present_flag is not present, it shall be
+ inferred to be equal to 0.
+ */
+ bool seq_scaling_matrix_present_flag = {};
+
+ /*
+ log2_max_frame_num_minus4 specifies the value of the variable
+ MaxFrameNum that is used in frame_num related derivations as
+ follows:
+
+ MaxFrameNum = 2( log2_max_frame_num_minus4 + 4 ). The value of
+ log2_max_frame_num_minus4 shall be in the range of 0 to 12, inclusive.
+ */
+ uint8_t log2_max_frame_num = {};
+
+ /*
+ pic_order_cnt_type specifies the method to decode picture order
+ count (as specified in subclause 8.2.1). The value of
+ pic_order_cnt_type shall be in the range of 0 to 2, inclusive.
+ */
+ uint8_t pic_order_cnt_type = {};
+
+ /*
+ log2_max_pic_order_cnt_lsb_minus4 specifies the value of the
+ variable MaxPicOrderCntLsb that is used in the decoding
+ process for picture order count as specified in subclause
+ 8.2.1 as follows:
+
+ MaxPicOrderCntLsb = 2( log2_max_pic_order_cnt_lsb_minus4 + 4 )
+
+ The value of log2_max_pic_order_cnt_lsb_minus4 shall be in
+ the range of 0 to 12, inclusive.
+ */
+ uint8_t log2_max_pic_order_cnt_lsb = {};
+
+ /*
+ delta_pic_order_always_zero_flag equal to 1 specifies that
+ delta_pic_order_cnt[ 0 ] and delta_pic_order_cnt[ 1 ] are
+ not present in the slice headers of the sequence and shall
+ be inferred to be equal to 0.
+ */
+ bool delta_pic_order_always_zero_flag = {};
+
+ /*
+ offset_for_non_ref_pic is used to calculate the picture
+ order count of a non-reference picture as specified in
+ 8.2.1. The value of offset_for_non_ref_pic shall be in the
+ range of -231 to 231 - 1, inclusive.
+ */
+ int8_t offset_for_non_ref_pic = {};
+
+ /*
+ offset_for_top_to_bottom_field is used to calculate the
+ picture order count of a bottom field as specified in
+ subclause 8.2.1. The value of offset_for_top_to_bottom_field
+ shall be in the range of -231 to 231 - 1, inclusive.
+ */
+ int8_t offset_for_top_to_bottom_field = {};
+
+ /*
+ max_num_ref_frames specifies the maximum number of short-term and
+ long-term reference frames, complementary reference field pairs,
+ and non-paired reference fields that may be used by the decoding
+ process for inter prediction of any picture in the
+ sequence. max_num_ref_frames also determines the size of the sliding
+ window operation as specified in subclause 8.2.5.3. The value of
+ max_num_ref_frames shall be in the range of 0 to MaxDpbFrames (as
+ specified in subclause A.3.1 or A.3.2), inclusive.
+ */
+ uint32_t max_num_ref_frames = {};
+
+ /*
+ gaps_in_frame_num_value_allowed_flag specifies the allowed
+ values of frame_num as specified in subclause 7.4.3 and the
+ decoding process in case of an inferred gap between values of
+ frame_num as specified in subclause 8.2.5.2.
+ */
+ bool gaps_in_frame_num_allowed_flag = {};
+
+ /*
+ pic_width_in_mbs_minus1 plus 1 specifies the width of each
+ decoded picture in units of macroblocks. 16 macroblocks in a row
+ */
+ uint32_t pic_width_in_mbs = {};
+
+ /*
+ pic_height_in_map_units_minus1 plus 1 specifies the height in
+ slice group map units of a decoded frame or field. 16
+ macroblocks in each column.
+ */
+ uint32_t pic_height_in_map_units = {};
+
+ /*
+ frame_mbs_only_flag equal to 0 specifies that coded pictures of
+ the coded video sequence may either be coded fields or coded
+ frames. frame_mbs_only_flag equal to 1 specifies that every
+ coded picture of the coded video sequence is a coded frame
+ containing only frame macroblocks.
+ */
+ bool frame_mbs_only_flag = {};
+
+ /*
+ mb_adaptive_frame_field_flag equal to 0 specifies no
+ switching between frame and field macroblocks within a
+ picture. mb_adaptive_frame_field_flag equal to 1 specifies
+ the possible use of switching between frame and field
+ macroblocks within frames. When mb_adaptive_frame_field_flag
+ is not present, it shall be inferred to be equal to 0.
+ */
+ bool mb_adaptive_frame_field_flag = {};
+
+ /*
+ direct_8x8_inference_flag specifies the method used in the derivation
+ process for luma motion vectors for B_Skip, B_Direct_16x16 and B_Direct_8x8
+ as specified in clause 8.4.1.2. When frame_mbs_only_flag is equal to 0,
+ direct_8x8_inference_flag shall be equal to 1.
+ */
+ bool direct_8x8_inference_flag = {};
+
+ /*
+ frame_cropping_flag equal to 1 specifies that the frame cropping
+ offset parameters follow next in the sequence parameter
+ set. frame_cropping_flag equal to 0 specifies that the frame
+ cropping offset parameters are not present.
+ */
+ bool frame_cropping_flag = {};
+ uint32_t frame_crop_left_offset = {};
+ uint32_t frame_crop_right_offset = {};
+ uint32_t frame_crop_top_offset = {};
+ uint32_t frame_crop_bottom_offset = {};
+
+ // VUI Parameters
+
+ /*
+ vui_parameters_present_flag equal to 1 specifies that the
+ vui_parameters( ) syntax structure as specified in Annex E is
+ present. vui_parameters_present_flag equal to 0 specifies that
+ the vui_parameters( ) syntax structure as specified in Annex E
+ is not present.
+ */
+ bool vui_parameters_present_flag = {};
+
+ /*
+ aspect_ratio_info_present_flag equal to 1 specifies that
+ aspect_ratio_idc is present. aspect_ratio_info_present_flag
+ equal to 0 specifies that aspect_ratio_idc is not present.
+ */
+ bool aspect_ratio_info_present_flag = {};
+
+ /*
+ aspect_ratio_idc specifies the value of the sample aspect
+ ratio of the luma samples. Table E-1 shows the meaning of
+ the code. When aspect_ratio_idc indicates Extended_SAR, the
+ sample aspect ratio is represented by sar_width and
+ sar_height. When the aspect_ratio_idc syntax element is not
+ present, aspect_ratio_idc value shall be inferred to be
+ equal to 0.
+ */
+ uint8_t aspect_ratio_idc = {};
+ uint32_t sar_width = {};
+ uint32_t sar_height = {};
+
+ /*
+ video_signal_type_present_flag equal to 1 specifies that video_format,
+ video_full_range_flag and colour_description_present_flag are present.
+ video_signal_type_present_flag equal to 0, specify that video_format,
+ video_full_range_flag and colour_description_present_flag are not present.
+ */
+ bool video_signal_type_present_flag = {};
+
+ /*
+ overscan_info_present_flag equal to1 specifies that the
+ overscan_appropriate_flag is present. When overscan_info_present_flag is
+ equal to 0 or is not present, the preferred display method for the video
+ signal is unspecified (Unspecified).
+ */
+ bool overscan_info_present_flag = {};
+ /*
+ overscan_appropriate_flag equal to 1 indicates that the cropped decoded
+ pictures output are suitable for display using overscan.
+ overscan_appropriate_flag equal to 0 indicates that the cropped decoded
+ pictures output contain visually important information in the entire region
+ out to the edges of the cropping rectangle of the picture
+ */
+ bool overscan_appropriate_flag = {};
+
+ /*
+ video_format indicates the representation of the pictures as specified in
+ Table E-2, before being coded in accordance with this
+ Recommendation | International Standard. When the video_format syntax
+ element is not present, video_format value shall be inferred to be equal
+ to 5. (Unspecified video format)
+ */
+ uint8_t video_format = {};
+
+ /*
+ video_full_range_flag indicates the black level and range of the luma and
+ chroma signals as derived from E′Y, E′PB, and E′PR or E′R, E′G, and E′B
+ real-valued component signals.
+ When the video_full_range_flag syntax element is not present, the value of
+ video_full_range_flag shall be inferred to be equal to 0.
+ */
+ bool video_full_range_flag = {};
+
+ /*
+ colour_description_present_flag equal to1 specifies that colour_primaries,
+ transfer_characteristics and matrix_coefficients are present.
+ colour_description_present_flag equal to 0 specifies that colour_primaries,
+ transfer_characteristics and matrix_coefficients are not present.
+ */
+ bool colour_description_present_flag = {};
+
+ /*
+ colour_primaries indicates the chromaticity coordinates of the source
+ primaries as specified in Table E-3 in terms of the CIE 1931 definition of
+ x and y as specified by ISO 11664-1.
+ When the colour_primaries syntax element is not present, the value of
+ colour_primaries shall be inferred to be equal to 2 (the chromaticity is
+ unspecified or is determined by the application).
+ */
+ uint8_t colour_primaries = {};
+
+ /*
+ transfer_characteristics indicates the opto-electronic transfer
+ characteristic of the source picture as specified in Table E-4 as a function
+ of a linear optical intensity input Lc with a nominal real-valued range of 0
+ to 1.
+ When the transfer_characteristics syntax element is not present, the value
+ of transfer_characteristics shall be inferred to be equal to 2
+ (the transfer characteristics are unspecified or are determined by the
+ application).
+ */
+ uint8_t transfer_characteristics = {};
+
+ uint8_t matrix_coefficients = {};
+ bool chroma_loc_info_present_flag = {};
+ /*
+ The value of chroma_sample_loc_type_top_field and
+ chroma_sample_loc_type_bottom_field shall be in the range of 0 to 5,
+ inclusive
+ */
+ uint8_t chroma_sample_loc_type_top_field = {};
+ uint8_t chroma_sample_loc_type_bottom_field = {};
+
+ bool scaling_matrix_present = {};
+ uint8_t scaling_matrix4x4[6][16] = {};
+ uint8_t scaling_matrix8x8[6][64] = {};
+
+ SPSData();
+};
+
+struct SEIRecoveryData {
+ /*
+ recovery_frame_cnt specifies the recovery point of output pictures in output
+ order. All decoded pictures in output order are indicated to be correct or
+ approximately correct in content starting at the output order position of
+ the reference picture having the frame_num equal to the frame_num of the VCL
+ NAL units for the current access unit incremented by recovery_frame_cnt in
+ modulo MaxFrameNum arithmetic. recovery_frame_cnt shall be in the range of 0
+ to MaxFrameNum − 1, inclusive.
+ */
+ uint32_t recovery_frame_cnt = 0;
+ /*
+ exact_match_flag indicates whether decoded pictures at and subsequent to the
+ specified recovery point in output order derived by starting the decoding
+ process at the access unit associated with the recovery point SEI message
+ shall be an exact match to the pictures that would be produced by starting
+ the decoding process at the location of a previous IDR access unit in the
+ NAL unit stream. The value 0 indicates that the match need not be exact and
+ the value 1 indicates that the match shall be exact.
+ */
+ bool exact_match_flag = false;
+ /*
+ broken_link_flag indicates the presence or absence of a broken link in the
+ NAL unit stream at the location of the recovery point SEI message */
+ bool broken_link_flag = false;
+ /*
+ changing_slice_group_idc equal to 0 indicates that decoded pictures are
+ correct or approximately correct in content at and subsequent to the
+ recovery point in output order when all macroblocks of the primary coded
+ pictures are decoded within the changing slice group period
+ */
+ uint8_t changing_slice_group_idc = 0;
+};
+
+class H264 {
+ public:
+ /* Check if out of band extradata contains a SPS NAL */
+ static bool HasSPS(const mozilla::MediaByteBuffer* aExtraData);
+ // Extract SPS and PPS NALs from aSample by looking into each NALs.
+ // aSample must be in AVCC format.
+ static already_AddRefed<mozilla::MediaByteBuffer> ExtractExtraData(
+ const mozilla::MediaRawData* aSample);
+ // Return true if both extradata are equal.
+ static bool CompareExtraData(const mozilla::MediaByteBuffer* aExtraData1,
+ const mozilla::MediaByteBuffer* aExtraData2);
+
+ // Ensure that SPS data makes sense, Return true if SPS data was, and false
+ // otherwise. If false, then content will be adjusted accordingly.
+ static bool EnsureSPSIsSane(SPSData& aSPS);
+
+ static bool DecodeSPSFromExtraData(const mozilla::MediaByteBuffer* aExtraData,
+ SPSData& aDest);
+ /* Decode SPS NAL RBSP and fill SPSData structure */
+ static bool DecodeSPS(const mozilla::MediaByteBuffer* aSPS, SPSData& aDest);
+
+ // If the given aExtraData is valid, return the aExtraData.max_num_ref_frames
+ // clamped to be in the range of [4, 16]; otherwise return 4.
+ static uint32_t ComputeMaxRefFrames(
+ const mozilla::MediaByteBuffer* aExtraData);
+
+ enum class FrameType {
+ I_FRAME,
+ OTHER,
+ INVALID,
+ };
+
+ // Returns the frame type. Returns I_FRAME if the sample is an IDR
+ // (Instantaneous Decoding Refresh) Picture.
+ static FrameType GetFrameType(const mozilla::MediaRawData* aSample);
+ // Create a dummy extradata, useful to create a decoder and test the
+ // capabilities of the decoder.
+ static already_AddRefed<mozilla::MediaByteBuffer> CreateExtraData(
+ uint8_t aProfile, uint8_t aConstraints, uint8_t aLevel,
+ const gfx::IntSize& aSize);
+ static void WriteExtraData(mozilla::MediaByteBuffer* aDestExtraData,
+ const uint8_t aProfile, const uint8_t aConstraints,
+ const uint8_t aLevel,
+ const Span<const uint8_t> aSPS,
+ const Span<const uint8_t> aPPS);
+
+ private:
+ friend class SPSNAL;
+ /* Extract RAW BYTE SEQUENCE PAYLOAD from NAL content.
+ Returns nullptr if invalid content.
+ This is compliant to ITU H.264 7.3.1 Syntax in tabular form NAL unit syntax
+ */
+ static already_AddRefed<mozilla::MediaByteBuffer> DecodeNALUnit(
+ const uint8_t* aNAL, size_t aLength);
+ static already_AddRefed<mozilla::MediaByteBuffer> EncodeNALUnit(
+ const uint8_t* aNAL, size_t aLength);
+ static bool vui_parameters(mozilla::BitReader& aBr, SPSData& aDest);
+ // Read HRD parameters, all data is ignored.
+ static void hrd_parameters(mozilla::BitReader& aBr);
+ static uint8_t NumSPS(const mozilla::MediaByteBuffer* aExtraData);
+ // Decode SEI payload and return true if the SEI NAL indicates a recovery
+ // point.
+ static bool DecodeRecoverySEI(const mozilla::MediaByteBuffer* aSEI,
+ SEIRecoveryData& aDest);
+ // Decode NAL Slice payload and return true if its slice type is I slice or SI
+ // slice.
+ static bool DecodeISlice(const mozilla::MediaByteBuffer* aSlice);
+};
+
+// ISO/IEC 14496-15 : avcC. We only parse partial attributes, not all of them.
+struct AVCCConfig final {
+ public:
+ static Result<AVCCConfig, nsresult> Parse(
+ const mozilla::MediaRawData* aSample);
+ static Result<AVCCConfig, nsresult> Parse(
+ const mozilla::MediaByteBuffer* aExtraData);
+
+ uint8_t NALUSize() const { return mLengthSizeMinusOne + 1; }
+
+ uint8_t mConfigurationVersion;
+ uint8_t mAVCProfileIndication;
+ uint8_t mProfileCompatibility;
+ uint8_t mAVCLevelIndication;
+ uint8_t mLengthSizeMinusOne;
+ uint8_t mNumSPS;
+
+ private:
+ AVCCConfig() = default;
+};
+
+} // namespace mozilla
+
+#endif // MP4_DEMUXER_H264_H_
diff --git a/dom/media/platforms/agnostic/bytestreams/H265.cpp b/dom/media/platforms/agnostic/bytestreams/H265.cpp
new file mode 100644
index 0000000000..bf2fe4b6b2
--- /dev/null
+++ b/dom/media/platforms/agnostic/bytestreams/H265.cpp
@@ -0,0 +1,1300 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "H265.h"
+#include <stdint.h>
+
+#include <cmath>
+#include <limits>
+
+#include "AnnexB.h"
+#include "BitReader.h"
+#include "BitWriter.h"
+#include "BufferReader.h"
+#include "ByteStreamsUtils.h"
+#include "ByteWriter.h"
+#include "MediaData.h"
+#include "MediaInfo.h"
+#include "mozilla/ArrayUtils.h"
+#include "mozilla/PodOperations.h"
+#include "mozilla/ResultExtensions.h"
+#include "mozilla/Span.h"
+
+mozilla::LazyLogModule gH265("H265");
+
+#define LOG(msg, ...) MOZ_LOG(gH265, LogLevel::Debug, (msg, ##__VA_ARGS__))
+#define LOGV(msg, ...) MOZ_LOG(gH265, LogLevel::Verbose, (msg, ##__VA_ARGS__))
+
+#define TRUE_OR_RETURN(condition) \
+ do { \
+ if (!(condition)) { \
+ LOG(#condition " should be true!"); \
+ return mozilla::Err(NS_ERROR_FAILURE); \
+ } \
+ } while (0)
+
+#define IN_RANGE_OR_RETURN(val, min, max) \
+ do { \
+ int64_t temp = AssertedCast<int64_t>(val); \
+ if ((temp) < (min) || (max) < (temp)) { \
+ LOG(#val " is not in the range of [" #min "," #max "]"); \
+ return mozilla::Err(NS_ERROR_FAILURE); \
+ } \
+ } while (0)
+
+#define NON_ZERO_OR_RETURN(dest, val) \
+ do { \
+ int64_t temp = AssertedCast<int64_t>(val); \
+ if ((temp) != 0) { \
+ (dest) = (temp); \
+ } else { \
+ LOG(#dest " should be non-zero"); \
+ return mozilla::Err(NS_ERROR_FAILURE); \
+ } \
+ } while (0)
+
+namespace mozilla {
+
+H265NALU::H265NALU(const uint8_t* aData, uint32_t aByteSize)
+ : mNALU(aData, aByteSize) {
+ // Per 7.3.1 NAL unit syntax
+ BitReader reader(aData, aByteSize * 8);
+ Unused << reader.ReadBit(); // forbidden_zero_bit
+ mNalUnitType = reader.ReadBits(6);
+ mNuhLayerId = reader.ReadBits(6);
+ mNuhTemporalIdPlus1 = reader.ReadBits(3);
+ LOGV("Created H265NALU, type=%hhu, size=%u", mNalUnitType, aByteSize);
+}
+
+/* static */ Result<HVCCConfig, nsresult> HVCCConfig::Parse(
+ const mozilla::MediaRawData* aSample) {
+ if (!aSample) {
+ LOG("No sample");
+ return mozilla::Err(NS_ERROR_FAILURE);
+ }
+ if (aSample->Size() < 3) {
+ LOG("Incorrect sample size %zu", aSample->Size());
+ return mozilla::Err(NS_ERROR_FAILURE);
+ }
+ if (aSample->mTrackInfo &&
+ !aSample->mTrackInfo->mMimeType.EqualsLiteral("video/hevc")) {
+ LOG("Only allow 'video/hevc' (mimeType=%s)",
+ aSample->mTrackInfo->mMimeType.get());
+ return mozilla::Err(NS_ERROR_FAILURE);
+ }
+ return HVCCConfig::Parse(aSample->mExtraData);
+}
+
+/* static */
+Result<HVCCConfig, nsresult> HVCCConfig::Parse(
+ const mozilla::MediaByteBuffer* aExtraData) {
+ // From configurationVersion to numOfArrays, total 184 bits (23 bytes)
+ if (!aExtraData) {
+ LOG("No extra-data");
+ return mozilla::Err(NS_ERROR_FAILURE);
+ }
+ if (aExtraData->Length() < 23) {
+ LOG("Incorrect extra-data size %zu", aExtraData->Length());
+ return mozilla::Err(NS_ERROR_FAILURE);
+ }
+ const auto& byteBuffer = *aExtraData;
+ if (byteBuffer[0] != 1) {
+ LOG("Version should always be 1");
+ return mozilla::Err(NS_ERROR_FAILURE);
+ }
+ HVCCConfig hvcc;
+
+ BitReader reader(aExtraData);
+ hvcc.configurationVersion = reader.ReadBits(8);
+ hvcc.general_profile_space = reader.ReadBits(2);
+ hvcc.general_tier_flag = reader.ReadBit();
+ hvcc.general_profile_idc = reader.ReadBits(5);
+ hvcc.general_profile_compatibility_flags = reader.ReadU32();
+
+ uint32_t flagHigh = reader.ReadU32();
+ uint16_t flagLow = reader.ReadBits(16);
+ hvcc.general_constraint_indicator_flags =
+ ((uint64_t)(flagHigh) << 16) | (uint64_t)(flagLow);
+
+ hvcc.general_level_idc = reader.ReadBits(8);
+ Unused << reader.ReadBits(4); // reserved
+ hvcc.min_spatial_segmentation_idc = reader.ReadBits(12);
+ Unused << reader.ReadBits(6); // reserved
+ hvcc.parallelismType = reader.ReadBits(2);
+ Unused << reader.ReadBits(6); // reserved
+ hvcc.chroma_format_idc = reader.ReadBits(2);
+ Unused << reader.ReadBits(5); // reserved
+ hvcc.bit_depth_luma_minus8 = reader.ReadBits(3);
+ Unused << reader.ReadBits(5); // reserved
+ hvcc.bit_depth_chroma_minus8 = reader.ReadBits(3);
+ hvcc.avgFrameRate = reader.ReadBits(16);
+ hvcc.constantFrameRate = reader.ReadBits(2);
+ hvcc.numTemporalLayers = reader.ReadBits(3);
+ hvcc.temporalIdNested = reader.ReadBit();
+ hvcc.lengthSizeMinusOne = reader.ReadBits(2);
+ const uint8_t numOfArrays = reader.ReadBits(8);
+ for (uint8_t idx = 0; idx < numOfArrays; idx++) {
+ Unused << reader.ReadBits(2); // array_completeness + reserved
+ const uint8_t nalUnitType = reader.ReadBits(6);
+ const uint16_t numNalus = reader.ReadBits(16);
+ LOGV("nalu-type=%u, nalu-num=%u", nalUnitType, numNalus);
+ for (uint16_t nIdx = 0; nIdx < numNalus; nIdx++) {
+ const uint16_t nalUnitLength = reader.ReadBits(16);
+ if (reader.BitsLeft() < nalUnitLength * 8) {
+ LOG("Aborting parsing, NALU size (%u bits) is larger than remaining "
+ "(%zu bits)!",
+ nalUnitLength * 8u, reader.BitsLeft());
+ // We return what we've parsed so far and ignore the rest.
+ hvcc.mByteBuffer = aExtraData;
+ return hvcc;
+ }
+ const uint8_t* currentPtr =
+ aExtraData->Elements() + reader.BitCount() / 8;
+ H265NALU nalu(currentPtr, nalUnitLength);
+ // ReadBits can only read at most 32 bits at a time.
+ uint32_t nalSize = nalUnitLength * 8;
+ while (nalSize > 0) {
+ uint32_t readBits = nalSize > 32 ? 32 : nalSize;
+ reader.ReadBits(readBits);
+ nalSize -= readBits;
+ }
+ // Per ISO_IEC-14496-15-2022, 8.3.2.1.3 Semantics, NALU should only be
+ // SPS/PPS/VPS or SEI, ignore all the other types of NALU.
+ if (nalu.IsSPS() || nalu.IsPPS() || nalu.IsVPS() || nalu.IsSEI()) {
+ hvcc.mNALUs.AppendElement(nalu);
+ } else {
+ LOG("Ignore NALU (%u) which is not SPS/PPS/VPS or SEI",
+ nalu.mNalUnitType);
+ }
+ }
+ }
+ hvcc.mByteBuffer = aExtraData;
+ return hvcc;
+}
+
+uint32_t HVCCConfig::NumSPS() const {
+ uint32_t spsCounter = 0;
+ for (const auto& nalu : mNALUs) {
+ if (nalu.IsSPS()) {
+ spsCounter++;
+ }
+ }
+ return spsCounter;
+}
+
+bool HVCCConfig::HasSPS() const {
+ bool hasSPS = false;
+ for (const auto& nalu : mNALUs) {
+ if (nalu.IsSPS()) {
+ hasSPS = true;
+ break;
+ }
+ }
+ return hasSPS;
+}
+
+/* static */
+Result<H265SPS, nsresult> H265::DecodeSPSFromSPSNALU(const H265NALU& aSPSNALU) {
+ MOZ_ASSERT(aSPSNALU.IsSPS());
+ RefPtr<MediaByteBuffer> rbsp = H265::DecodeNALUnit(aSPSNALU.mNALU);
+ if (!rbsp) {
+ LOG("Failed to decode NALU");
+ return Err(NS_ERROR_FAILURE);
+ }
+
+ // H265 spec, 7.3.2.2.1 seq_parameter_set_rbsp
+ H265SPS sps;
+ BitReader reader(rbsp);
+ sps.sps_video_parameter_set_id = reader.ReadBits(4);
+ IN_RANGE_OR_RETURN(sps.sps_video_parameter_set_id, 0, 15);
+ sps.sps_max_sub_layers_minus1 = reader.ReadBits(3);
+ IN_RANGE_OR_RETURN(sps.sps_max_sub_layers_minus1, 0, 6);
+ sps.sps_temporal_id_nesting_flag = reader.ReadBit();
+
+ if (auto rv = ParseProfileTierLevel(
+ reader, true /* aProfilePresentFlag, true per spec*/,
+ sps.sps_max_sub_layers_minus1, sps.profile_tier_level);
+ rv.isErr()) {
+ LOG("Failed to parse the profile tier level.");
+ return Err(NS_ERROR_FAILURE);
+ }
+
+ sps.sps_seq_parameter_set_id = reader.ReadUE();
+ IN_RANGE_OR_RETURN(sps.sps_seq_parameter_set_id, 0, 15);
+ sps.chroma_format_idc = reader.ReadUE();
+ IN_RANGE_OR_RETURN(sps.chroma_format_idc, 0, 3);
+
+ if (sps.chroma_format_idc == 3) {
+ sps.separate_colour_plane_flag = reader.ReadBit();
+ }
+
+ // From Table 6-1.
+ if (sps.chroma_format_idc == 1) {
+ sps.subWidthC = sps.subHeightC = 2;
+ } else if (sps.chroma_format_idc == 2) {
+ sps.subWidthC = 2;
+ sps.subHeightC = 1;
+ } else {
+ sps.subWidthC = sps.subHeightC = 1;
+ }
+
+ NON_ZERO_OR_RETURN(sps.pic_width_in_luma_samples, reader.ReadUE());
+ NON_ZERO_OR_RETURN(sps.pic_height_in_luma_samples, reader.ReadUE());
+ {
+ // (A-2) Calculate maxDpbSize
+ const auto maxLumaPs = sps.profile_tier_level.GetMaxLumaPs();
+ CheckedUint32 picSize = sps.pic_height_in_luma_samples;
+ picSize *= sps.pic_width_in_luma_samples;
+ if (!picSize.isValid()) {
+ LOG("Invalid picture size");
+ return Err(NS_ERROR_FAILURE);
+ }
+ const auto picSizeInSamplesY = picSize.value();
+ const auto maxDpbPicBuf = sps.profile_tier_level.GetDpbMaxPicBuf();
+ if (picSizeInSamplesY <= (maxLumaPs >> 2)) {
+ sps.maxDpbSize = std::min(4 * maxDpbPicBuf, 16u);
+ } else if (picSizeInSamplesY <= (maxLumaPs >> 1)) {
+ sps.maxDpbSize = std::min(2 * maxDpbPicBuf, 16u);
+ } else if (picSizeInSamplesY <= ((3 * maxLumaPs) >> 2)) {
+ sps.maxDpbSize = std::min((4 * maxDpbPicBuf) / 3, 16u);
+ } else {
+ sps.maxDpbSize = maxDpbPicBuf;
+ }
+ }
+
+ sps.conformance_window_flag = reader.ReadBit();
+ if (sps.conformance_window_flag) {
+ sps.conf_win_left_offset = reader.ReadUE();
+ sps.conf_win_right_offset = reader.ReadUE();
+ sps.conf_win_top_offset = reader.ReadUE();
+ sps.conf_win_bottom_offset = reader.ReadUE();
+ }
+ sps.bit_depth_luma_minus8 = reader.ReadUE();
+ IN_RANGE_OR_RETURN(sps.bit_depth_luma_minus8, 0, 8);
+ sps.bit_depth_chroma_minus8 = reader.ReadUE();
+ IN_RANGE_OR_RETURN(sps.bit_depth_chroma_minus8, 0, 8);
+ sps.log2_max_pic_order_cnt_lsb_minus4 = reader.ReadUE();
+ IN_RANGE_OR_RETURN(sps.log2_max_pic_order_cnt_lsb_minus4, 0, 12);
+ sps.sps_sub_layer_ordering_info_present_flag = reader.ReadBit();
+ for (auto i = sps.sps_sub_layer_ordering_info_present_flag
+ ? 0
+ : sps.sps_max_sub_layers_minus1;
+ i <= sps.sps_max_sub_layers_minus1; i++) {
+ sps.sps_max_dec_pic_buffering_minus1[i] = reader.ReadUE();
+ IN_RANGE_OR_RETURN(sps.sps_max_dec_pic_buffering_minus1[i], 0,
+ sps.maxDpbSize - 1);
+ sps.sps_max_num_reorder_pics[i] = reader.ReadUE();
+ IN_RANGE_OR_RETURN(sps.sps_max_num_reorder_pics[i], 0,
+ sps.sps_max_dec_pic_buffering_minus1[i]);
+ // 7.4.3.2.1, see sps_max_dec_pic_buffering_minus1 and
+ // sps_max_num_reorder_pics, "When i is greater than 0, ....".
+ if (i > 0) {
+ TRUE_OR_RETURN(sps.sps_max_dec_pic_buffering_minus1[i] >=
+ sps.sps_max_dec_pic_buffering_minus1[i - 1]);
+ TRUE_OR_RETURN(sps.sps_max_num_reorder_pics[i] >=
+ sps.sps_max_num_reorder_pics[i - 1]);
+ }
+ sps.sps_max_latency_increase_plus1[i] = reader.ReadUE();
+ IN_RANGE_OR_RETURN(sps.sps_max_latency_increase_plus1[i], 0, 0xFFFFFFFE);
+ }
+ sps.log2_min_luma_coding_block_size_minus3 = reader.ReadUE();
+ sps.log2_diff_max_min_luma_coding_block_size = reader.ReadUE();
+ sps.log2_min_luma_transform_block_size_minus2 = reader.ReadUE();
+ sps.log2_diff_max_min_luma_transform_block_size = reader.ReadUE();
+ sps.max_transform_hierarchy_depth_inter = reader.ReadUE();
+ sps.max_transform_hierarchy_depth_intra = reader.ReadUE();
+ const auto scaling_list_enabled_flag = reader.ReadBit();
+ if (scaling_list_enabled_flag) {
+ Unused << reader.ReadBit(); // sps_scaling_list_data_present_flag
+ if (auto rv = ParseAndIgnoreScalingListData(reader); rv.isErr()) {
+ LOG("Failed to parse scaling list data.");
+ return Err(NS_ERROR_FAILURE);
+ }
+ }
+
+ // amp_enabled_flag and sample_adaptive_offset_enabled_flag
+ Unused << reader.ReadBits(2);
+
+ sps.pcm_enabled_flag = reader.ReadBit();
+ if (sps.pcm_enabled_flag) {
+ sps.pcm_sample_bit_depth_luma_minus1 = reader.ReadBits(3);
+ IN_RANGE_OR_RETURN(sps.pcm_sample_bit_depth_luma_minus1, 0,
+ sps.BitDepthLuma());
+ sps.pcm_sample_bit_depth_chroma_minus1 = reader.ReadBits(3);
+ IN_RANGE_OR_RETURN(sps.pcm_sample_bit_depth_chroma_minus1, 0,
+ sps.BitDepthChroma());
+ sps.log2_min_pcm_luma_coding_block_size_minus3 = reader.ReadUE();
+ IN_RANGE_OR_RETURN(sps.log2_min_pcm_luma_coding_block_size_minus3, 0, 2);
+ uint32_t log2MinIpcmCbSizeY{sps.log2_min_pcm_luma_coding_block_size_minus3 +
+ 3};
+ sps.log2_diff_max_min_pcm_luma_coding_block_size = reader.ReadUE();
+ {
+ // Validate value
+ CheckedUint32 log2MaxIpcmCbSizeY{
+ sps.log2_diff_max_min_pcm_luma_coding_block_size};
+ log2MaxIpcmCbSizeY += log2MinIpcmCbSizeY;
+ CheckedUint32 minCbLog2SizeY{sps.log2_min_luma_coding_block_size_minus3};
+ minCbLog2SizeY += 3; // (7-10)
+ CheckedUint32 ctbLog2SizeY{minCbLog2SizeY};
+ ctbLog2SizeY += sps.log2_diff_max_min_luma_coding_block_size; // (7-11)
+ IN_RANGE_OR_RETURN(log2MaxIpcmCbSizeY.value(), 0,
+ std::min(ctbLog2SizeY.value(), uint32_t(5)));
+ }
+ sps.pcm_loop_filter_disabled_flag = reader.ReadBit();
+ }
+
+ sps.num_short_term_ref_pic_sets = reader.ReadUE();
+ IN_RANGE_OR_RETURN(sps.num_short_term_ref_pic_sets, 0,
+ kMaxShortTermRefPicSets);
+ for (auto i = 0; i < sps.num_short_term_ref_pic_sets; i++) {
+ if (auto rv = ParseStRefPicSet(reader, i, sps); rv.isErr()) {
+ LOG("Failed to parse short-term reference picture set.");
+ return Err(NS_ERROR_FAILURE);
+ }
+ }
+ const auto long_term_ref_pics_present_flag = reader.ReadBit();
+ if (long_term_ref_pics_present_flag) {
+ uint32_t num_long_term_ref_pics_sps;
+ num_long_term_ref_pics_sps = reader.ReadUE();
+ IN_RANGE_OR_RETURN(num_long_term_ref_pics_sps, 0, kMaxLongTermRefPicSets);
+ for (auto i = 0; i < num_long_term_ref_pics_sps; i++) {
+ Unused << reader.ReadBits(sps.log2_max_pic_order_cnt_lsb_minus4 +
+ 4); // lt_ref_pic_poc_lsb_sps[i]
+ Unused << reader.ReadBit(); // used_by_curr_pic_lt_sps_flag
+ }
+ }
+ sps.sps_temporal_mvp_enabled_flag = reader.ReadBit();
+ sps.strong_intra_smoothing_enabled_flag = reader.ReadBit();
+ const auto vui_parameters_present_flag = reader.ReadBit();
+ if (vui_parameters_present_flag) {
+ if (auto rv = ParseVuiParameters(reader, sps); rv.isErr()) {
+ LOG("Failed to parse VUI parameter.");
+ return Err(NS_ERROR_FAILURE);
+ }
+ }
+
+ // The rest is extension data we don't care about, so no need to parse them.
+ return sps;
+}
+
+/* static */
+Result<H265SPS, nsresult> H265::DecodeSPSFromHVCCExtraData(
+ const mozilla::MediaByteBuffer* aExtraData) {
+ auto rv = HVCCConfig::Parse(aExtraData);
+ if (rv.isErr()) {
+ LOG("Only support HVCC extra-data");
+ return Err(NS_ERROR_FAILURE);
+ }
+ const auto& hvcc = rv.unwrap();
+ const H265NALU* spsNALU = nullptr;
+ for (const auto& nalu : hvcc.mNALUs) {
+ if (nalu.IsSPS()) {
+ spsNALU = &nalu;
+ break;
+ }
+ }
+ if (!spsNALU) {
+ LOG("No sps found");
+ return Err(NS_ERROR_FAILURE);
+ }
+ return DecodeSPSFromSPSNALU(*spsNALU);
+}
+
+/* static */
+Result<Ok, nsresult> H265::ParseProfileTierLevel(
+ BitReader& aReader, bool aProfilePresentFlag,
+ uint8_t aMaxNumSubLayersMinus1, H265ProfileTierLevel& aProfile) {
+ // H265 spec, 7.3.3 Profile, tier and level syntax
+ if (aProfilePresentFlag) {
+ aProfile.general_profile_space = aReader.ReadBits(2);
+ aProfile.general_tier_flag = aReader.ReadBit();
+ aProfile.general_profile_idc = aReader.ReadBits(5);
+ IN_RANGE_OR_RETURN(aProfile.general_profile_idc, 0, 11);
+ aProfile.general_profile_compatibility_flags = aReader.ReadU32();
+ aProfile.general_progressive_source_flag = aReader.ReadBit();
+ aProfile.general_interlaced_source_flag = aReader.ReadBit();
+ aProfile.general_non_packed_constraint_flag = aReader.ReadBit();
+ aProfile.general_frame_only_constraint_flag = aReader.ReadBit();
+ // ignored attributes, in total general_reserved_zero_43bits
+ Unused << aReader.ReadBits(32);
+ Unused << aReader.ReadBits(11);
+ // general_inbld_flag or general_reserved_zero_bit
+ Unused << aReader.ReadBit();
+ }
+ aProfile.general_level_idc = aReader.ReadBits(8);
+
+ // Following are all ignored attributes.
+ bool sub_layer_profile_present_flag[8];
+ bool sub_layer_level_present_flag[8];
+ for (auto i = 0; i < aMaxNumSubLayersMinus1; i++) {
+ sub_layer_profile_present_flag[i] = aReader.ReadBit();
+ sub_layer_level_present_flag[i] = aReader.ReadBit();
+ }
+ if (aMaxNumSubLayersMinus1 > 0) {
+ for (auto i = aMaxNumSubLayersMinus1; i < 8; i++) {
+ // reserved_zero_2bits
+ Unused << aReader.ReadBits(2);
+ }
+ }
+ for (auto i = 0; i < aMaxNumSubLayersMinus1; i++) {
+ if (sub_layer_profile_present_flag[i]) {
+ // sub_layer_profile_space, sub_layer_tier_flag, sub_layer_profile_idc
+ Unused << aReader.ReadBits(8);
+ // sub_layer_profile_compatibility_flag
+ Unused << aReader.ReadBits(32);
+ // sub_layer_progressive_source_flag, sub_layer_interlaced_source_flag,
+ // sub_layer_non_packed_constraint_flag,
+ // sub_layer_frame_only_constraint_flag
+ Unused << aReader.ReadBits(4);
+ // ignored attributes, in total general_reserved_zero_43bits
+ Unused << aReader.ReadBits(32);
+ Unused << aReader.ReadBits(11);
+ // sub_layer_inbld_flag or reserved_zero_bit
+ Unused << aReader.ReadBit();
+ }
+ if (sub_layer_level_present_flag[i]) {
+ Unused << aReader.ReadBits(8); // sub_layer_level_idc
+ }
+ }
+ return Ok();
+}
+
+uint32_t H265ProfileTierLevel::GetMaxLumaPs() const {
+ // From Table A.8 - General tier and level limits.
+ // "general_level_idc and sub_layer_level_idc[ i ] shall be set equal to a
+ // value of 30 times the level number specified in Table A.8".
+ if (general_level_idc <= 30) { // level 1
+ return 36864;
+ }
+ if (general_level_idc <= 60) { // level 2
+ return 122880;
+ }
+ if (general_level_idc <= 63) { // level 2.1
+ return 245760;
+ }
+ if (general_level_idc <= 90) { // level 3
+ return 552960;
+ }
+ if (general_level_idc <= 93) { // level 3.1
+ return 983040;
+ }
+ if (general_level_idc <= 123) { // level 4, 4.1
+ return 2228224;
+ }
+ if (general_level_idc <= 156) { // level 5, 5.1, 5.2
+ return 8912896;
+ }
+ // level 6, 6.1, 6.2 - beyond that there's no actual limit.
+ return 35651584;
+}
+
+uint32_t H265ProfileTierLevel::GetDpbMaxPicBuf() const {
+ // From A.4.2 - Profile-specific level limits for the video profiles.
+ // "maxDpbPicBuf is equal to 6 for all profiles where the value of
+ // sps_curr_pic_ref_enabled_flag is required to be equal to 0 and 7 for all
+ // profiles where the value of sps_curr_pic_ref_enabled_flag is not required
+ // to be equal to 0." From A.3 Profile, the flag in the main, main still,
+ // range extensions and high throughput is required to be zero.
+ return (general_profile_idc >= H265ProfileIdc::kProfileIdcMain &&
+ general_profile_idc <= H265ProfileIdc::kProfileIdcHighThroughput)
+ ? 6
+ : 7;
+}
+
+/* static */
+Result<Ok, nsresult> H265::ParseAndIgnoreScalingListData(BitReader& aReader) {
+ // H265 spec, 7.3.4 Scaling list data syntax
+ for (auto sizeIdx = 0; sizeIdx < 4; sizeIdx++) {
+ for (auto matrixIdx = 0; matrixIdx < 6;
+ matrixIdx += (sizeIdx == 3) ? 3 : 1) {
+ const auto scaling_list_pred_mode_flag = aReader.ReadBit();
+ if (!scaling_list_pred_mode_flag) {
+ Unused << aReader.ReadUE(); // scaling_list_pred_matrix_id_delta
+ } else {
+ int32_t coefNum = std::min(64, (1 << (4 + (sizeIdx << 1))));
+ if (sizeIdx > 1) {
+ Unused << aReader.ReadSE(); // scaling_list_dc_coef_minus8
+ }
+ for (auto i = 0; i < coefNum; i++) {
+ Unused << aReader.ReadSE(); // scaling_list_delta_coef
+ }
+ }
+ }
+ }
+ return Ok();
+}
+
+/* static */
+Result<Ok, nsresult> H265::ParseStRefPicSet(BitReader& aReader,
+ uint32_t aStRpsIdx, H265SPS& aSPS) {
+ // H265 Spec, 7.3.7 Short-term reference picture set syntax
+ MOZ_ASSERT(aStRpsIdx < kMaxShortTermRefPicSets);
+ bool inter_ref_pic_set_prediction_flag = false;
+ H265StRefPicSet& curStRefPicSet = aSPS.st_ref_pic_set[aStRpsIdx];
+ if (aStRpsIdx != 0) {
+ inter_ref_pic_set_prediction_flag = aReader.ReadBit();
+ }
+ if (inter_ref_pic_set_prediction_flag) {
+ int delta_idx_minus1 = 0;
+ if (aStRpsIdx == aSPS.num_short_term_ref_pic_sets) {
+ delta_idx_minus1 = aReader.ReadUE();
+ IN_RANGE_OR_RETURN(delta_idx_minus1, 0, aStRpsIdx - 1);
+ }
+ const uint32_t RefRpsIdx = aStRpsIdx - (delta_idx_minus1 + 1); // (7-59)
+ const bool delta_rps_sign = aReader.ReadBit();
+ const uint32_t abs_delta_rps_minus1 = aReader.ReadUE();
+ IN_RANGE_OR_RETURN(abs_delta_rps_minus1, 0, 0x7FFF);
+ const int32_t deltaRps =
+ (1 - 2 * delta_rps_sign) *
+ AssertedCast<int32_t>(abs_delta_rps_minus1 + 1); // (7-60)
+
+ bool used_by_curr_pic_flag[kMaxShortTermRefPicSets] = {};
+ bool use_delta_flag[kMaxShortTermRefPicSets] = {};
+ // 7.4.8 - use_delta_flag defaults to 1 if not present.
+ std::fill_n(use_delta_flag, kMaxShortTermRefPicSets, true);
+ const H265StRefPicSet& refSet = aSPS.st_ref_pic_set[RefRpsIdx];
+ for (auto j = 0; j <= refSet.numDeltaPocs; j++) {
+ used_by_curr_pic_flag[j] = aReader.ReadBit();
+ if (!used_by_curr_pic_flag[j]) {
+ use_delta_flag[j] = aReader.ReadBit();
+ }
+ }
+ // Calculate fields (7-61)
+ uint32_t i = 0;
+ for (int64_t j = static_cast<int64_t>(refSet.num_positive_pics) - 1; j >= 0;
+ j--) {
+ MOZ_DIAGNOSTIC_ASSERT(j < kMaxShortTermRefPicSets);
+ int64_t d_poc = refSet.deltaPocS1[j] + deltaRps;
+ if (d_poc < 0 && use_delta_flag[refSet.num_negative_pics + j]) {
+ curStRefPicSet.deltaPocS0[i] = d_poc;
+ curStRefPicSet.usedByCurrPicS0[i++] =
+ used_by_curr_pic_flag[refSet.num_negative_pics + j];
+ }
+ }
+ if (deltaRps < 0 && use_delta_flag[refSet.numDeltaPocs]) {
+ curStRefPicSet.deltaPocS0[i] = deltaRps;
+ curStRefPicSet.usedByCurrPicS0[i++] =
+ used_by_curr_pic_flag[refSet.numDeltaPocs];
+ }
+ for (auto j = 0; j < refSet.num_negative_pics; j++) {
+ MOZ_DIAGNOSTIC_ASSERT(j < kMaxShortTermRefPicSets);
+ int64_t d_poc = refSet.deltaPocS0[j] + deltaRps;
+ if (d_poc < 0 && use_delta_flag[j]) {
+ curStRefPicSet.deltaPocS0[i] = d_poc;
+ curStRefPicSet.usedByCurrPicS0[i++] = used_by_curr_pic_flag[j];
+ }
+ }
+ curStRefPicSet.num_negative_pics = i;
+ // Calculate fields (7-62)
+ i = 0;
+ for (int64_t j = static_cast<int64_t>(refSet.num_negative_pics) - 1; j >= 0;
+ j--) {
+ MOZ_DIAGNOSTIC_ASSERT(j < kMaxShortTermRefPicSets);
+ int64_t d_poc = refSet.deltaPocS0[j] + deltaRps;
+ if (d_poc > 0 && use_delta_flag[j]) {
+ curStRefPicSet.deltaPocS1[i] = d_poc;
+ curStRefPicSet.usedByCurrPicS1[i++] = used_by_curr_pic_flag[j];
+ }
+ }
+ if (deltaRps > 0 && use_delta_flag[refSet.numDeltaPocs]) {
+ curStRefPicSet.deltaPocS1[i] = deltaRps;
+ curStRefPicSet.usedByCurrPicS1[i++] =
+ used_by_curr_pic_flag[refSet.numDeltaPocs];
+ }
+ for (auto j = 0; j < refSet.num_positive_pics; j++) {
+ MOZ_DIAGNOSTIC_ASSERT(j < kMaxShortTermRefPicSets);
+ int64_t d_poc = refSet.deltaPocS1[j] + deltaRps;
+ if (d_poc > 0 && use_delta_flag[refSet.num_negative_pics + j]) {
+ curStRefPicSet.deltaPocS1[i] = d_poc;
+ curStRefPicSet.usedByCurrPicS1[i++] =
+ used_by_curr_pic_flag[refSet.num_negative_pics + j];
+ }
+ }
+ curStRefPicSet.num_positive_pics = i;
+ } else {
+ curStRefPicSet.num_negative_pics = aReader.ReadUE();
+ curStRefPicSet.num_positive_pics = aReader.ReadUE();
+ const uint32_t spsMaxDecPicBufferingMinus1 =
+ aSPS.sps_max_dec_pic_buffering_minus1[aSPS.sps_max_sub_layers_minus1];
+ IN_RANGE_OR_RETURN(curStRefPicSet.num_negative_pics, 0,
+ spsMaxDecPicBufferingMinus1);
+ CheckedUint32 maxPositivePics{spsMaxDecPicBufferingMinus1};
+ maxPositivePics -= curStRefPicSet.num_negative_pics;
+ IN_RANGE_OR_RETURN(curStRefPicSet.num_positive_pics, 0,
+ maxPositivePics.value());
+ for (auto i = 0; i < curStRefPicSet.num_negative_pics; i++) {
+ const uint32_t delta_poc_s0_minus1 = aReader.ReadUE();
+ IN_RANGE_OR_RETURN(delta_poc_s0_minus1, 0, 0x7FFF);
+ if (i == 0) {
+ // (7-67)
+ curStRefPicSet.deltaPocS0[i] = -(delta_poc_s0_minus1 + 1);
+ } else {
+ // (7-69)
+ curStRefPicSet.deltaPocS0[i] =
+ curStRefPicSet.deltaPocS0[i - 1] - (delta_poc_s0_minus1 + 1);
+ }
+ curStRefPicSet.usedByCurrPicS0[i] = aReader.ReadBit();
+ }
+ for (auto i = 0; i < curStRefPicSet.num_positive_pics; i++) {
+ const int delta_poc_s1_minus1 = aReader.ReadUE();
+ IN_RANGE_OR_RETURN(delta_poc_s1_minus1, 0, 0x7FFF);
+ if (i == 0) {
+ // (7-68)
+ curStRefPicSet.deltaPocS1[i] = delta_poc_s1_minus1 + 1;
+ } else {
+ // (7-70)
+ curStRefPicSet.deltaPocS1[i] =
+ curStRefPicSet.deltaPocS1[i - 1] + delta_poc_s1_minus1 + 1;
+ }
+ curStRefPicSet.usedByCurrPicS1[i] = aReader.ReadBit();
+ }
+ }
+ // (7-71)
+ curStRefPicSet.numDeltaPocs =
+ curStRefPicSet.num_negative_pics + curStRefPicSet.num_positive_pics;
+ return Ok();
+}
+
+/* static */
+Result<Ok, nsresult> H265::ParseVuiParameters(BitReader& aReader,
+ H265SPS& aSPS) {
+ // VUI parameters: Table E.1 "Interpretation of sample aspect ratio indicator"
+ static constexpr int kTableSarWidth[] = {0, 1, 12, 10, 16, 40, 24, 20, 32,
+ 80, 18, 15, 64, 160, 4, 3, 2};
+ static constexpr int kTableSarHeight[] = {0, 1, 11, 11, 11, 33, 11, 11, 11,
+ 33, 11, 11, 33, 99, 3, 2, 1};
+ static_assert(std::size(kTableSarWidth) == std::size(kTableSarHeight),
+ "sar tables must have the same size");
+ aSPS.vui_parameters = Some(H265VUIParameters());
+ H265VUIParameters* vui = aSPS.vui_parameters.ptr();
+
+ const auto aspect_ratio_info_present_flag = aReader.ReadBit();
+ if (aspect_ratio_info_present_flag) {
+ const auto aspect_ratio_idc = aReader.ReadBits(8);
+ constexpr int kExtendedSar = 255;
+ if (aspect_ratio_idc == kExtendedSar) {
+ vui->sar_width = aReader.ReadBits(16);
+ vui->sar_height = aReader.ReadBits(16);
+ } else {
+ const auto max_aspect_ratio_idc = std::size(kTableSarWidth) - 1;
+ IN_RANGE_OR_RETURN(aspect_ratio_idc, 0, max_aspect_ratio_idc);
+ vui->sar_width = kTableSarWidth[aspect_ratio_idc];
+ vui->sar_height = kTableSarHeight[aspect_ratio_idc];
+ }
+ }
+
+ const auto overscan_info_present_flag = aReader.ReadBit();
+ if (overscan_info_present_flag) {
+ Unused << aReader.ReadBit(); // overscan_appropriate_flag
+ }
+
+ const auto video_signal_type_present_flag = aReader.ReadBit();
+ if (video_signal_type_present_flag) {
+ Unused << aReader.ReadBits(3); // video_format
+ vui->video_full_range_flag = aReader.ReadBit();
+ const auto colour_description_present_flag = aReader.ReadBit();
+ if (colour_description_present_flag) {
+ vui->colour_primaries.emplace(aReader.ReadBits(8));
+ vui->transfer_characteristics.emplace(aReader.ReadBits(8));
+ vui->matrix_coeffs.emplace(aReader.ReadBits(8));
+ }
+ }
+
+ const auto chroma_loc_info_present_flag = aReader.ReadBit();
+ if (chroma_loc_info_present_flag) {
+ Unused << aReader.ReadUE(); // chroma_sample_loc_type_top_field
+ Unused << aReader.ReadUE(); // chroma_sample_loc_type_bottom_field
+ }
+
+ // Ignore neutral_chroma_indication_flag, field_seq_flag and
+ // frame_field_info_present_flag.
+ Unused << aReader.ReadBits(3);
+
+ const auto default_display_window_flag = aReader.ReadBit();
+ if (default_display_window_flag) {
+ uint32_t def_disp_win_left_offset = aReader.ReadUE();
+ uint32_t def_disp_win_right_offset = aReader.ReadUE();
+ uint32_t def_disp_win_top_offset = aReader.ReadUE();
+ uint32_t def_disp_win_bottom_offset = aReader.ReadUE();
+ // (E-68) + (E-69)
+ aSPS.mDisplayWidth = aSPS.subWidthC;
+ aSPS.mDisplayWidth *=
+ (aSPS.conf_win_left_offset + def_disp_win_left_offset);
+ aSPS.mDisplayWidth *=
+ (aSPS.conf_win_right_offset + def_disp_win_right_offset);
+ if (!aSPS.mDisplayWidth.isValid()) {
+ LOG("mDisplayWidth overflow!");
+ return Err(NS_ERROR_FAILURE);
+ }
+ IN_RANGE_OR_RETURN(aSPS.mDisplayWidth.value(), 0,
+ aSPS.pic_width_in_luma_samples);
+
+ // (E-70) + (E-71)
+ aSPS.mDisplayHeight = aSPS.subHeightC;
+ aSPS.mDisplayHeight *= (aSPS.conf_win_top_offset + def_disp_win_top_offset);
+ aSPS.mDisplayHeight *=
+ (aSPS.conf_win_bottom_offset + def_disp_win_bottom_offset);
+ if (!aSPS.mDisplayHeight.isValid()) {
+ LOG("mDisplayHeight overflow!");
+ return Err(NS_ERROR_FAILURE);
+ }
+ IN_RANGE_OR_RETURN(aSPS.mDisplayHeight.value(), 0,
+ aSPS.pic_height_in_luma_samples);
+ }
+
+ const auto vui_timing_info_present_flag = aReader.ReadBit();
+ if (vui_timing_info_present_flag) {
+ Unused << aReader.ReadU32(); // vui_num_units_in_tick
+ Unused << aReader.ReadU32(); // vui_time_scale
+ const auto vui_poc_proportional_to_timing_flag = aReader.ReadBit();
+ if (vui_poc_proportional_to_timing_flag) {
+ Unused << aReader.ReadUE(); // vui_num_ticks_poc_diff_one_minus1
+ }
+ const auto vui_hrd_parameters_present_flag = aReader.ReadBit();
+ if (vui_hrd_parameters_present_flag) {
+ if (auto rv = ParseAndIgnoreHrdParameters(aReader, true,
+ aSPS.sps_max_sub_layers_minus1);
+ rv.isErr()) {
+ LOG("Failed to parse Hrd parameters");
+ return rv;
+ }
+ }
+ }
+
+ const auto bitstream_restriction_flag = aReader.ReadBit();
+ if (bitstream_restriction_flag) {
+ // Skip tiles_fixed_structure_flag, motion_vectors_over_pic_boundaries_flag
+ // and restricted_ref_pic_lists_flag.
+ Unused << aReader.ReadBits(3);
+ Unused << aReader.ReadUE(); // min_spatial_segmentation_idc
+ Unused << aReader.ReadUE(); // max_bytes_per_pic_denom
+ Unused << aReader.ReadUE(); // max_bits_per_min_cu_denom
+ Unused << aReader.ReadUE(); // log2_max_mv_length_horizontal
+ Unused << aReader.ReadUE(); // log2_max_mv_length_vertical
+ }
+ return Ok();
+}
+
+/* static */
+Result<Ok, nsresult> H265::ParseAndIgnoreHrdParameters(
+ BitReader& aReader, bool aCommonInfPresentFlag,
+ int aMaxNumSubLayersMinus1) {
+ // H265 Spec, E.2.2 HRD parameters syntax
+ bool nal_hrd_parameters_present_flag = false;
+ bool vcl_hrd_parameters_present_flag = false;
+ bool sub_pic_hrd_params_present_flag = false;
+ if (aCommonInfPresentFlag) {
+ nal_hrd_parameters_present_flag = aReader.ReadBit();
+ vcl_hrd_parameters_present_flag = aReader.ReadBit();
+ if (nal_hrd_parameters_present_flag || vcl_hrd_parameters_present_flag) {
+ sub_pic_hrd_params_present_flag = aReader.ReadBit();
+ if (sub_pic_hrd_params_present_flag) {
+ Unused << aReader.ReadBits(8); // tick_divisor_minus2
+ // du_cpb_removal_delay_increment_length_minus1
+ Unused << aReader.ReadBits(5);
+ // sub_pic_cpb_params_in_pic_timing_sei_flag
+ Unused << aReader.ReadBits(1);
+ Unused << aReader.ReadBits(5); // dpb_output_delay_du_length_minus1
+ }
+
+ Unused << aReader.ReadBits(4); // bit_rate_scale
+ Unused << aReader.ReadBits(4); // cpb_size_scale
+ if (sub_pic_hrd_params_present_flag) {
+ Unused << aReader.ReadBits(4); // cpb_size_du_scale
+ }
+ Unused << aReader.ReadBits(5); // initial_cpb_removal_delay_length_minus1
+ Unused << aReader.ReadBits(5); // au_cpb_removal_delay_length_minus1
+ Unused << aReader.ReadBits(5); // dpb_output_delay_length_minus1
+ }
+ }
+ for (int i = 0; i <= aMaxNumSubLayersMinus1; i++) {
+ bool fixed_pic_rate_within_cvs_flag = false;
+ if (auto fixed_pic_rate_general_flag = aReader.ReadBit();
+ !fixed_pic_rate_general_flag) {
+ fixed_pic_rate_within_cvs_flag = aReader.ReadBit();
+ }
+ bool low_delay_hrd_flag = false;
+ if (fixed_pic_rate_within_cvs_flag) {
+ Unused << aReader.ReadUE(); // elemental_duration_in_tc_minus1
+ } else {
+ low_delay_hrd_flag = aReader.ReadBit();
+ }
+ int cpb_cnt_minus1 = 0;
+ if (!low_delay_hrd_flag) {
+ cpb_cnt_minus1 = aReader.ReadUE();
+ IN_RANGE_OR_RETURN(cpb_cnt_minus1, 0, 31);
+ }
+ if (nal_hrd_parameters_present_flag) {
+ if (auto rv = ParseAndIgnoreSubLayerHrdParameters(
+ aReader, cpb_cnt_minus1 + 1, sub_pic_hrd_params_present_flag);
+ rv.isErr()) {
+ LOG("Failed to parse nal Hrd parameters");
+ return rv;
+ };
+ }
+ if (vcl_hrd_parameters_present_flag) {
+ if (auto rv = ParseAndIgnoreSubLayerHrdParameters(
+ aReader, cpb_cnt_minus1 + 1, sub_pic_hrd_params_present_flag);
+ rv.isErr()) {
+ LOG("Failed to parse vcl Hrd parameters");
+ return rv;
+ }
+ }
+ }
+ return Ok();
+}
+
+/* static */
+Result<Ok, nsresult> H265::ParseAndIgnoreSubLayerHrdParameters(
+ BitReader& aReader, int aCpbCnt, bool aSubPicHrdParamsPresentFlag) {
+ // H265 Spec, E.2.3 Sub-layer HRD parameters syntax
+ for (auto i = 0; i < aCpbCnt; i++) {
+ Unused << aReader.ReadUE(); // bit_rate_value_minus1
+ Unused << aReader.ReadUE(); // cpb_size_value_minus1
+ if (aSubPicHrdParamsPresentFlag) {
+ Unused << aReader.ReadUE(); // cpb_size_du_value_minus1
+ Unused << aReader.ReadUE(); // bit_rate_du_value_minus1
+ }
+ Unused << aReader.ReadBit(); // cbr_flag
+ }
+ return Ok();
+}
+
+bool H265SPS::operator==(const H265SPS& aOther) const {
+ return memcmp(this, &aOther, sizeof(H265SPS)) == 0;
+}
+
+bool H265SPS::operator!=(const H265SPS& aOther) const {
+ return !(operator==(aOther));
+}
+
+gfx::IntSize H265SPS::GetImageSize() const {
+ return gfx::IntSize(pic_width_in_luma_samples, pic_height_in_luma_samples);
+}
+
+gfx::IntSize H265SPS::GetDisplaySize() const {
+ if (mDisplayWidth.value() == 0 || mDisplayHeight.value() == 0) {
+ return GetImageSize();
+ }
+ return gfx::IntSize(mDisplayWidth.value(), mDisplayHeight.value());
+}
+
+gfx::ColorDepth H265SPS::ColorDepth() const {
+ if (bit_depth_luma_minus8 != 0 && bit_depth_luma_minus8 != 2 &&
+ bit_depth_luma_minus8 != 4) {
+ // We don't know what that is, just assume 8 bits to prevent decoding
+ // regressions if we ever encounter those.
+ return gfx::ColorDepth::COLOR_8;
+ }
+ return gfx::ColorDepthForBitDepth(BitDepthLuma());
+}
+
+// PrimaryID, TransferID and MatrixID are defined in ByteStreamsUtils.h
+static PrimaryID GetPrimaryID(const Maybe<uint8_t>& aPrimary) {
+ if (!aPrimary || *aPrimary < 1 || *aPrimary > 22 || *aPrimary == 3) {
+ return PrimaryID::INVALID;
+ }
+ if (*aPrimary > 12 && *aPrimary < 22) {
+ return PrimaryID::INVALID;
+ }
+ return static_cast<PrimaryID>(*aPrimary);
+}
+
+static TransferID GetTransferID(const Maybe<uint8_t>& aTransfer) {
+ if (!aTransfer || *aTransfer < 1 || *aTransfer > 18 || *aTransfer == 3) {
+ return TransferID::INVALID;
+ }
+ return static_cast<TransferID>(*aTransfer);
+}
+
+static MatrixID GetMatrixID(const Maybe<uint8_t>& aMatrix) {
+ if (!aMatrix || *aMatrix > 11 || *aMatrix == 3) {
+ return MatrixID::INVALID;
+ }
+ return static_cast<MatrixID>(*aMatrix);
+}
+
+gfx::YUVColorSpace H265SPS::ColorSpace() const {
+ // Bitfield, note that guesses with higher values take precedence over
+ // guesses with lower values.
+ enum Guess {
+ GUESS_BT601 = 1 << 0,
+ GUESS_BT709 = 1 << 1,
+ GUESS_BT2020 = 1 << 2,
+ };
+
+ uint32_t guess = 0;
+ if (vui_parameters) {
+ switch (GetPrimaryID(vui_parameters->colour_primaries)) {
+ case PrimaryID::BT709:
+ guess |= GUESS_BT709;
+ break;
+ case PrimaryID::BT470M:
+ case PrimaryID::BT470BG:
+ case PrimaryID::SMPTE170M:
+ case PrimaryID::SMPTE240M:
+ guess |= GUESS_BT601;
+ break;
+ case PrimaryID::BT2020:
+ guess |= GUESS_BT2020;
+ break;
+ case PrimaryID::FILM:
+ case PrimaryID::SMPTEST428_1:
+ case PrimaryID::SMPTEST431_2:
+ case PrimaryID::SMPTEST432_1:
+ case PrimaryID::EBU_3213_E:
+ case PrimaryID::INVALID:
+ case PrimaryID::UNSPECIFIED:
+ break;
+ }
+
+ switch (GetTransferID(vui_parameters->transfer_characteristics)) {
+ case TransferID::BT709:
+ guess |= GUESS_BT709;
+ break;
+ case TransferID::GAMMA22:
+ case TransferID::GAMMA28:
+ case TransferID::SMPTE170M:
+ case TransferID::SMPTE240M:
+ guess |= GUESS_BT601;
+ break;
+ case TransferID::BT2020_10:
+ case TransferID::BT2020_12:
+ guess |= GUESS_BT2020;
+ break;
+ case TransferID::LINEAR:
+ case TransferID::LOG:
+ case TransferID::LOG_SQRT:
+ case TransferID::IEC61966_2_4:
+ case TransferID::BT1361_ECG:
+ case TransferID::IEC61966_2_1:
+ case TransferID::SMPTEST2084:
+ case TransferID::SMPTEST428_1:
+ case TransferID::ARIB_STD_B67:
+ case TransferID::INVALID:
+ case TransferID::UNSPECIFIED:
+ break;
+ }
+
+ switch (GetMatrixID(vui_parameters->matrix_coeffs)) {
+ case MatrixID::BT709:
+ guess |= GUESS_BT709;
+ break;
+ case MatrixID::BT470BG:
+ case MatrixID::SMPTE170M:
+ case MatrixID::SMPTE240M:
+ guess |= GUESS_BT601;
+ break;
+ case MatrixID::BT2020_NCL:
+ case MatrixID::BT2020_CL:
+ guess |= GUESS_BT2020;
+ break;
+ case MatrixID::RGB:
+ case MatrixID::FCC:
+ case MatrixID::YCOCG:
+ case MatrixID::YDZDX:
+ case MatrixID::INVALID:
+ case MatrixID::UNSPECIFIED:
+ break;
+ }
+ }
+
+ // Removes lowest bit until only a single bit remains.
+ while (guess & (guess - 1)) {
+ guess &= guess - 1;
+ }
+ if (!guess) {
+ // A better default to BT601 which should die a slow death.
+ guess = GUESS_BT709;
+ }
+
+ switch (guess) {
+ case GUESS_BT601:
+ return gfx::YUVColorSpace::BT601;
+ case GUESS_BT709:
+ return gfx::YUVColorSpace::BT709;
+ default:
+ MOZ_DIAGNOSTIC_ASSERT(guess == GUESS_BT2020);
+ return gfx::YUVColorSpace::BT2020;
+ }
+}
+
+bool H265SPS::IsFullColorRange() const {
+ return vui_parameters ? vui_parameters->video_full_range_flag : false;
+}
+
+uint8_t H265SPS::ColorPrimaries() const {
+ // Per H265 spec E.3.1, "When the colour_primaries syntax element is not
+ // present, the value of colour_primaries is inferred to be equal to 2 (the
+ // chromaticity is unspecified or is determined by the application).".
+ if (!vui_parameters || !vui_parameters->colour_primaries) {
+ return 2;
+ }
+ return vui_parameters->colour_primaries.value();
+}
+
+uint8_t H265SPS::TransferFunction() const {
+ // Per H265 spec E.3.1, "When the transfer_characteristics syntax element is
+ // not present, the value of transfer_characteristics is inferred to be equal
+ // to 2 (the transfer characteristics are unspecified or are determined by the
+ // application)."
+ if (!vui_parameters || !vui_parameters->transfer_characteristics) {
+ return 2;
+ }
+ return vui_parameters->transfer_characteristics.value();
+}
+
+/* static */
+already_AddRefed<mozilla::MediaByteBuffer> H265::DecodeNALUnit(
+ const Span<const uint8_t>& aNALU) {
+ RefPtr<mozilla::MediaByteBuffer> rbsp = new mozilla::MediaByteBuffer;
+ BufferReader reader(aNALU.Elements(), aNALU.Length());
+ auto header = reader.ReadU16();
+ if (header.isErr()) {
+ return nullptr;
+ }
+ uint32_t lastbytes = 0xffff;
+ while (reader.Remaining()) {
+ auto res = reader.ReadU8();
+ if (res.isErr()) {
+ return nullptr;
+ }
+ uint8_t byte = res.unwrap();
+ if ((lastbytes & 0xffff) == 0 && byte == 0x03) {
+ // reset last two bytes, to detect the 0x000003 sequence again.
+ lastbytes = 0xffff;
+ } else {
+ rbsp->AppendElement(byte);
+ }
+ lastbytes = (lastbytes << 8) | byte;
+ }
+ return rbsp.forget();
+}
+
+/* static */
+already_AddRefed<mozilla::MediaByteBuffer> H265::ExtractHVCCExtraData(
+ const mozilla::MediaRawData* aSample) {
+ size_t sampleSize = aSample->Size();
+ if (aSample->mCrypto.IsEncrypted()) {
+ // The content is encrypted, we can only parse the non-encrypted data.
+ MOZ_ASSERT(aSample->mCrypto.mPlainSizes.Length() > 0);
+ if (aSample->mCrypto.mPlainSizes.Length() == 0 ||
+ aSample->mCrypto.mPlainSizes[0] > sampleSize) {
+ LOG("Invalid crypto content");
+ return nullptr;
+ }
+ sampleSize = aSample->mCrypto.mPlainSizes[0];
+ }
+
+ auto hvcc = HVCCConfig::Parse(aSample);
+ if (hvcc.isErr()) {
+ LOG("Only support extracting extradata from HVCC");
+ return nullptr;
+ }
+ const auto nalLenSize = hvcc.unwrap().NALUSize();
+ BufferReader reader(aSample->Data(), sampleSize);
+
+ nsTArray<Maybe<H265SPS>> spsRefTable;
+ nsTArray<H265NALU> spsNALUs;
+ // If we encounter SPS with the same id but different content, we will stop
+ // attempting to detect duplicates.
+ bool checkDuplicate = true;
+ const H265SPS* firstSPS = nullptr;
+
+ RefPtr<mozilla::MediaByteBuffer> extradata = new mozilla::MediaByteBuffer;
+ while (reader.Remaining() > nalLenSize) {
+ // ISO/IEC 14496-15, 4.2.3.2 Syntax. (NALUSample) Reading the size of NALU.
+ uint32_t nalLen = 0;
+ switch (nalLenSize) {
+ case 1:
+ Unused << reader.ReadU8().map(
+ [&](uint8_t x) mutable { return nalLen = x; });
+ break;
+ case 2:
+ Unused << reader.ReadU16().map(
+ [&](uint16_t x) mutable { return nalLen = x; });
+ break;
+ case 3:
+ Unused << reader.ReadU24().map(
+ [&](uint32_t x) mutable { return nalLen = x; });
+ break;
+ default:
+ MOZ_DIAGNOSTIC_ASSERT(nalLenSize == 4);
+ Unused << reader.ReadU32().map(
+ [&](uint32_t x) mutable { return nalLen = x; });
+ break;
+ }
+ const uint8_t* p = reader.Read(nalLen);
+ if (!p) {
+ // The read failed, but we may already have some SPS data so break out of
+ // reading and process what we have, if any.
+ break;
+ }
+ const H265NALU nalu(p, nalLen);
+ LOGV("Found NALU, type=%u", nalu.mNalUnitType);
+ if (nalu.IsSPS()) {
+ auto rv = H265::DecodeSPSFromSPSNALU(nalu);
+ if (rv.isErr()) {
+ // Invalid SPS, ignore.
+ LOG("Ignore invalid SPS");
+ continue;
+ }
+ const H265SPS sps = rv.unwrap();
+ const uint8_t spsId = sps.sps_seq_parameter_set_id; // 0~15
+ if (spsId >= spsRefTable.Length()) {
+ if (!spsRefTable.SetLength(spsId + 1, fallible)) {
+ NS_WARNING("OOM while expanding spsRefTable!");
+ return nullptr;
+ }
+ }
+ if (checkDuplicate && spsRefTable[spsId] &&
+ *(spsRefTable[spsId]) == sps) {
+ // Duplicate ignore.
+ continue;
+ }
+ if (spsRefTable[spsId]) {
+ // We already have detected a SPS with this Id. Just to be safe we
+ // disable SPS duplicate detection.
+ checkDuplicate = false;
+ } else {
+ spsRefTable[spsId] = Some(sps);
+ spsNALUs.AppendElement(nalu);
+ if (!firstSPS) {
+ firstSPS = spsRefTable[spsId].ptr();
+ }
+ }
+ }
+ }
+
+ LOGV("Found %zu SPS NALU", spsNALUs.Length());
+ if (!spsNALUs.IsEmpty()) {
+ MOZ_ASSERT(firstSPS);
+ BitWriter writer(extradata);
+
+ // ISO/IEC 14496-15, HEVCDecoderConfigurationRecord. But we only append SPS.
+ writer.WriteBits(1, 8); // version
+ const auto& profile = firstSPS->profile_tier_level;
+ writer.WriteBits(profile.general_profile_space, 2);
+ writer.WriteBits(profile.general_tier_flag, 1);
+ writer.WriteBits(profile.general_profile_idc, 5);
+ writer.WriteU32(profile.general_profile_compatibility_flags);
+
+ // general_constraint_indicator_flags
+ writer.WriteBit(profile.general_progressive_source_flag);
+ writer.WriteBit(profile.general_interlaced_source_flag);
+ writer.WriteBit(profile.general_non_packed_constraint_flag);
+ writer.WriteBit(profile.general_frame_only_constraint_flag);
+ writer.WriteBits(0, 44); /* ignored 44 bits */
+
+ writer.WriteU8(profile.general_level_idc);
+ writer.WriteBits(0, 4); // reserved
+ writer.WriteBits(0, 12); // min_spatial_segmentation_idc
+ writer.WriteBits(0, 6); // reserved
+ writer.WriteBits(0, 2); // parallelismType
+ writer.WriteBits(0, 6); // reserved
+ writer.WriteBits(firstSPS->chroma_format_idc, 2);
+ writer.WriteBits(0, 5); // reserved
+ writer.WriteBits(firstSPS->bit_depth_luma_minus8, 3);
+ writer.WriteBits(0, 5); // reserved
+ writer.WriteBits(firstSPS->bit_depth_chroma_minus8, 3);
+ // avgFrameRate + constantFrameRate + numTemporalLayers + temporalIdNested
+ writer.WriteBits(0, 22);
+ writer.WriteBits(nalLenSize - 1, 2); // lengthSizeMinusOne
+ writer.WriteU8(1); // numOfArrays, only SPS
+ for (auto j = 0; j < 1; j++) {
+ writer.WriteBits(0, 2); // array_completeness + reserved
+ writer.WriteBits(H265NALU::SPS_NUT, 6); // NAL_unit_type
+ writer.WriteBits(spsNALUs.Length(), 16); // numNalus
+ for (auto i = 0; i < spsNALUs.Length(); i++) {
+ writer.WriteBits(spsNALUs[i].mNALU.Length(),
+ 16); // nalUnitLength
+ MOZ_ASSERT(writer.BitCount() % 8 == 0);
+ extradata->AppendElements(spsNALUs[i].mNALU.Elements(),
+ spsNALUs[i].mNALU.Length());
+ writer.AdvanceBytes(spsNALUs[i].mNALU.Length());
+ }
+ }
+ }
+
+ return extradata.forget();
+}
+
+class SPSIterator final {
+ public:
+ explicit SPSIterator(const HVCCConfig& aConfig) : mConfig(aConfig) {}
+
+ SPSIterator& operator++() {
+ size_t idx = 0;
+ for (idx = mNextIdx; idx < mConfig.mNALUs.Length(); idx++) {
+ if (mConfig.mNALUs[idx].IsSPS()) {
+ mSPS = &mConfig.mNALUs[idx];
+ break;
+ }
+ }
+ mNextIdx = idx + 1;
+ return *this;
+ }
+
+ explicit operator bool() const { return mNextIdx < mConfig.mNALUs.Length(); }
+
+ const H265NALU* operator*() const { return mSPS ? mSPS : nullptr; }
+
+ private:
+ size_t mNextIdx = 0;
+ const H265NALU* mSPS = nullptr;
+ const HVCCConfig& mConfig;
+};
+
+/* static */
+bool AreTwoSPSIdentical(const H265NALU& aLhs, const H265NALU& aRhs) {
+ MOZ_ASSERT(aLhs.IsSPS() && aRhs.IsSPS());
+ auto rv1 = H265::DecodeSPSFromSPSNALU(aLhs);
+ auto rv2 = H265::DecodeSPSFromSPSNALU(aRhs);
+ if (rv1.isErr() || rv2.isErr()) {
+ return false;
+ }
+ return rv1.unwrap() == rv2.unwrap();
+}
+
+/* static */
+bool H265::CompareExtraData(const mozilla::MediaByteBuffer* aExtraData1,
+ const mozilla::MediaByteBuffer* aExtraData2) {
+ if (aExtraData1 == aExtraData2) {
+ return true;
+ }
+
+ auto config1 = HVCCConfig::Parse(aExtraData1);
+ auto config2 = HVCCConfig::Parse(aExtraData2);
+ if (config1.isErr() || config2.isErr()) {
+ return false;
+ }
+
+ uint8_t numSPS = config1.unwrap().NumSPS();
+ if (numSPS == 0 || numSPS != config2.unwrap().NumSPS()) {
+ return false;
+ }
+
+ // We only compare if the SPS are the same as the various HEVC decoders can
+ // deal with in-band change of PPS.
+ SPSIterator it1(config1.unwrap());
+ SPSIterator it2(config2.unwrap());
+ while (it1 && it2) {
+ const H265NALU* nalu1 = *it1;
+ const H265NALU* nalu2 = *it2;
+ if (!nalu1 || !nalu2) {
+ return false;
+ }
+ if (!AreTwoSPSIdentical(*nalu1, *nalu2)) {
+ return false;
+ }
+ ++it1;
+ ++it2;
+ }
+ return true;
+}
+
+#undef LOG
+#undef LOGV
+
+} // namespace mozilla
diff --git a/dom/media/platforms/agnostic/bytestreams/H265.h b/dom/media/platforms/agnostic/bytestreams/H265.h
new file mode 100644
index 0000000000..5aca28e5cb
--- /dev/null
+++ b/dom/media/platforms/agnostic/bytestreams/H265.h
@@ -0,0 +1,356 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_H265_H_
+#define DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_H265_H_
+
+#include <stdint.h>
+
+#include "mozilla/CheckedInt.h"
+#include "mozilla/gfx/Point.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/Result.h"
+#include "mozilla/Span.h"
+#include "nsTArray.h"
+
+namespace mozilla {
+
+class BitReader;
+class MediaByteBuffer;
+class MediaRawData;
+
+// Most classes in this file are implemented according to the H265 spec
+// (https://www.itu.int/rec/T-REC-H.265-202108-I/en), except the HVCCConfig,
+// which is in the ISO/IEC 14496-15. To make it easier to read the
+// implementation with the spec, the naming style in this file follows the spec
+// instead of our usual style.
+
+enum {
+ kMaxLongTermRefPicSets = 32, // See num_long_term_ref_pics_sps
+ kMaxShortTermRefPicSets = 64, // See num_short_term_ref_pic_sets
+ kMaxSubLayers = 7, // See [v/s]ps_max_sub_layers_minus1
+};
+
+// Spec 7.3.1 NAL unit syntax
+class H265NALU final {
+ public:
+ H265NALU(const uint8_t* aData, uint32_t aByteSize);
+ H265NALU() = default;
+
+ // Table 7-1
+ enum NAL_TYPES {
+ TRAIL_N = 0,
+ TRAIL_R = 1,
+ TSA_N = 2,
+ TSA_R = 3,
+ STSA_N = 4,
+ STSA_R = 5,
+ RADL_N = 6,
+ RADL_R = 7,
+ RASL_N = 8,
+ RASL_R = 9,
+ RSV_VCL_N10 = 10,
+ RSV_VCL_R11 = 11,
+ RSV_VCL_N12 = 12,
+ RSV_VCL_R13 = 13,
+ RSV_VCL_N14 = 14,
+ RSV_VCL_R15 = 15,
+ BLA_W_LP = 16,
+ BLA_W_RADL = 17,
+ BLA_N_LP = 18,
+ IDR_W_RADL = 19,
+ IDR_N_LP = 20,
+ CRA_NUT = 21,
+ RSV_IRAP_VCL22 = 22,
+ RSV_IRAP_VCL23 = 23,
+ RSV_VCL24 = 24,
+ RSV_VCL25 = 25,
+ RSV_VCL26 = 26,
+ RSV_VCL27 = 27,
+ RSV_VCL28 = 28,
+ RSV_VCL29 = 29,
+ RSV_VCL30 = 30,
+ RSV_VCL31 = 31,
+ VPS_NUT = 32,
+ SPS_NUT = 33,
+ PPS_NUT = 34,
+ AUD_NUT = 35,
+ EOS_NUT = 36,
+ EOB_NUT = 37,
+ FD_NUT = 38,
+ PREFIX_SEI_NUT = 39,
+ SUFFIX_SEI_NUT = 40,
+ RSV_NVCL41 = 41,
+ RSV_NVCL42 = 42,
+ RSV_NVCL43 = 43,
+ RSV_NVCL44 = 44,
+ RSV_NVCL45 = 45,
+ RSV_NVCL46 = 46,
+ RSV_NVCL47 = 47,
+ UNSPEC48 = 48,
+ UNSPEC49 = 49,
+ UNSPEC50 = 50,
+ UNSPEC51 = 51,
+ UNSPEC52 = 52,
+ UNSPEC53 = 53,
+ UNSPEC54 = 54,
+ UNSPEC55 = 55,
+ UNSPEC56 = 56,
+ UNSPEC57 = 57,
+ UNSPEC58 = 58,
+ UNSPEC59 = 59,
+ UNSPEC60 = 60,
+ UNSPEC61 = 61,
+ UNSPEC62 = 62,
+ UNSPEC63 = 63,
+ };
+
+ bool IsIframe() const {
+ return mNalUnitType == NAL_TYPES::IDR_W_RADL ||
+ mNalUnitType == NAL_TYPES::IDR_N_LP;
+ }
+
+ bool IsSPS() const { return mNalUnitType == NAL_TYPES::SPS_NUT; }
+ bool IsVPS() const { return mNalUnitType == NAL_TYPES::VPS_NUT; }
+ bool IsPPS() const { return mNalUnitType == NAL_TYPES::PPS_NUT; }
+ bool IsSEI() const {
+ return mNalUnitType == NAL_TYPES::PREFIX_SEI_NUT ||
+ mNalUnitType == NAL_TYPES::SUFFIX_SEI_NUT;
+ }
+
+ uint8_t mNalUnitType;
+ uint8_t mNuhLayerId;
+ uint8_t mNuhTemporalIdPlus1;
+ // This contain the full content of NALU, which can be used to decode rbsp.
+ const Span<const uint8_t> mNALU;
+};
+
+// H265 spec, 7.3.3 Profile, tier and level syntax
+struct H265ProfileTierLevel final {
+ H265ProfileTierLevel() = default;
+
+ enum H265ProfileIdc {
+ kProfileIdcMain = 1,
+ kProfileIdcMain10 = 2,
+ kProfileIdcMainStill = 3,
+ kProfileIdcRangeExtensions = 4,
+ kProfileIdcHighThroughput = 5,
+ kProfileIdcMultiviewMain = 6,
+ kProfileIdcScalableMain = 7,
+ kProfileIdc3dMain = 8,
+ kProfileIdcScreenContentCoding = 9,
+ kProfileIdcScalableRangeExtensions = 10,
+ kProfileIdcHighThroughputScreenContentCoding = 11,
+ };
+
+ // From Table A.8 - General tier and level limits.
+ uint32_t GetMaxLumaPs() const;
+
+ // From A.4.2 - Profile-specific level limits for the video profiles.
+ uint32_t GetDpbMaxPicBuf() const;
+
+ // Syntax elements.
+ uint8_t general_profile_space = {};
+ bool general_tier_flag = {};
+ uint8_t general_profile_idc = {};
+ uint32_t general_profile_compatibility_flags = {};
+ bool general_progressive_source_flag = {};
+ bool general_interlaced_source_flag = {};
+ bool general_non_packed_constraint_flag = {};
+ bool general_frame_only_constraint_flag = {};
+ uint8_t general_level_idc = {};
+};
+
+// H265 spec, 7.3.7 Short-term reference picture set syntax
+struct H265StRefPicSet final {
+ H265StRefPicSet() = default;
+
+ // Syntax elements.
+ uint32_t num_negative_pics = {};
+ uint32_t num_positive_pics = {};
+
+ // Calculated fields
+ // From the H265 spec 7.4.8
+ bool usedByCurrPicS0[kMaxShortTermRefPicSets] = {}; // (7-65)
+ bool usedByCurrPicS1[kMaxShortTermRefPicSets] = {}; // (7-66)
+ uint32_t deltaPocS0[kMaxShortTermRefPicSets] = {}; // (7-67) + (7-69)
+ uint32_t deltaPocS1[kMaxShortTermRefPicSets] = {}; // (7-68) + (7-70)
+ uint32_t numDeltaPocs = {}; // (7-72)
+};
+
+// H265 spec, E.2.1 VUI parameters syntax
+struct H265VUIParameters {
+ H265VUIParameters() = default;
+
+ // Syntax elements.
+ uint32_t sar_width = {};
+ uint32_t sar_height = {};
+ bool video_full_range_flag = {};
+ Maybe<uint8_t> colour_primaries;
+ Maybe<uint8_t> transfer_characteristics;
+ Maybe<uint8_t> matrix_coeffs;
+};
+
+// H265 spec, 7.3.2.2 Sequence parameter set RBSP syntax
+struct H265SPS final {
+ H265SPS() = default;
+
+ bool operator==(const H265SPS& aOther) const;
+ bool operator!=(const H265SPS& aOther) const;
+
+ // Syntax elements.
+ uint8_t sps_video_parameter_set_id = {};
+ uint8_t sps_max_sub_layers_minus1 = {};
+ bool sps_temporal_id_nesting_flag = {};
+ H265ProfileTierLevel profile_tier_level = {};
+ uint32_t sps_seq_parameter_set_id = {};
+ uint32_t chroma_format_idc = {};
+ bool separate_colour_plane_flag = {};
+ uint32_t pic_width_in_luma_samples = {};
+ uint32_t pic_height_in_luma_samples = {};
+
+ bool conformance_window_flag = {};
+ uint32_t conf_win_left_offset = {};
+ uint32_t conf_win_right_offset = {};
+ uint32_t conf_win_top_offset = {};
+ uint32_t conf_win_bottom_offset = {};
+
+ uint32_t bit_depth_luma_minus8 = {};
+ uint32_t bit_depth_chroma_minus8 = {};
+ uint32_t log2_max_pic_order_cnt_lsb_minus4 = {};
+ bool sps_sub_layer_ordering_info_present_flag = {};
+ uint32_t sps_max_dec_pic_buffering_minus1[kMaxSubLayers] = {};
+ uint32_t sps_max_num_reorder_pics[kMaxSubLayers] = {};
+ uint32_t sps_max_latency_increase_plus1[kMaxSubLayers] = {};
+ uint32_t log2_min_luma_coding_block_size_minus3 = {};
+ uint32_t log2_diff_max_min_luma_coding_block_size = {};
+ uint32_t log2_min_luma_transform_block_size_minus2 = {};
+ uint32_t log2_diff_max_min_luma_transform_block_size = {};
+ uint32_t max_transform_hierarchy_depth_inter = {};
+ uint32_t max_transform_hierarchy_depth_intra = {};
+
+ bool pcm_enabled_flag = {};
+ uint8_t pcm_sample_bit_depth_luma_minus1 = {};
+ uint8_t pcm_sample_bit_depth_chroma_minus1 = {};
+ uint32_t log2_min_pcm_luma_coding_block_size_minus3 = {};
+ uint32_t log2_diff_max_min_pcm_luma_coding_block_size = {};
+ bool pcm_loop_filter_disabled_flag = {};
+
+ uint32_t num_short_term_ref_pic_sets = {};
+ H265StRefPicSet st_ref_pic_set[kMaxShortTermRefPicSets] = {};
+
+ bool sps_temporal_mvp_enabled_flag = {};
+ bool strong_intra_smoothing_enabled_flag = {};
+ Maybe<H265VUIParameters> vui_parameters;
+
+ // Calculated fields
+ uint32_t subWidthC = {}; // From Table 6-1.
+ uint32_t subHeightC = {}; // From Table 6-1.
+ CheckedUint32 mDisplayWidth; // Per (E-68) + (E-69)
+ CheckedUint32 mDisplayHeight; // Per (E-70) + (E-71)
+ uint32_t maxDpbSize = {};
+
+ // Often used information
+ uint32_t BitDepthLuma() const { return bit_depth_luma_minus8 + 8; }
+ uint32_t BitDepthChroma() const { return bit_depth_chroma_minus8 + 8; }
+ gfx::IntSize GetImageSize() const;
+ gfx::IntSize GetDisplaySize() const;
+ gfx::ColorDepth ColorDepth() const;
+ gfx::YUVColorSpace ColorSpace() const;
+ bool IsFullColorRange() const;
+ uint8_t ColorPrimaries() const;
+ uint8_t TransferFunction() const;
+};
+
+// ISO/IEC 14496-15 : hvcC.
+struct HVCCConfig final {
+ public:
+ static Result<HVCCConfig, nsresult> Parse(
+ const mozilla::MediaRawData* aSample);
+ static Result<HVCCConfig, nsresult> Parse(
+ const mozilla::MediaByteBuffer* aExtraData);
+
+ uint8_t NALUSize() const { return lengthSizeMinusOne + 1; }
+ uint32_t NumSPS() const;
+ bool HasSPS() const;
+
+ uint8_t configurationVersion;
+ uint8_t general_profile_space;
+ bool general_tier_flag;
+ uint8_t general_profile_idc;
+ uint32_t general_profile_compatibility_flags;
+ uint64_t general_constraint_indicator_flags;
+ uint8_t general_level_idc;
+ uint16_t min_spatial_segmentation_idc;
+ uint8_t parallelismType;
+ uint8_t chroma_format_idc;
+ uint8_t bit_depth_luma_minus8;
+ uint8_t bit_depth_chroma_minus8;
+ uint16_t avgFrameRate;
+ uint8_t constantFrameRate;
+ uint8_t numTemporalLayers;
+ bool temporalIdNested;
+ uint8_t lengthSizeMinusOne;
+
+ nsTArray<H265NALU> mNALUs;
+
+ // Keep the orginal buffer alive in order to let H265NALU always access to
+ // valid data if there is any NALU.
+ RefPtr<const MediaByteBuffer> mByteBuffer;
+
+ private:
+ HVCCConfig() = default;
+};
+
+class H265 final {
+ public:
+ static Result<H265SPS, nsresult> DecodeSPSFromHVCCExtraData(
+ const mozilla::MediaByteBuffer* aExtraData);
+ static Result<H265SPS, nsresult> DecodeSPSFromSPSNALU(
+ const H265NALU& aSPSNALU);
+
+ // Extract SPS and PPS NALs from aSample by looking into each NALs.
+ static already_AddRefed<mozilla::MediaByteBuffer> ExtractHVCCExtraData(
+ const mozilla::MediaRawData* aSample);
+
+ // Return true if both extradata are equal.
+ static bool CompareExtraData(const mozilla::MediaByteBuffer* aExtraData1,
+ const mozilla::MediaByteBuffer* aExtraData2);
+
+ private:
+ // Return RAW BYTE SEQUENCE PAYLOAD (rbsp) from NAL content.
+ static already_AddRefed<mozilla::MediaByteBuffer> DecodeNALUnit(
+ const Span<const uint8_t>& aNALU);
+
+ // Parse the profile level based on the H265 spec, 7.3.3. MUST use a bit
+ // reader which starts from the position of the first bit of the data.
+ static Result<Ok, nsresult> ParseProfileTierLevel(
+ BitReader& aReader, bool aProfilePresentFlag,
+ uint8_t aMaxNumSubLayersMinus1, H265ProfileTierLevel& aProfile);
+
+ // Parse the short-term reference picture set based on the H265 spec, 7.3.7.
+ // MUST use a bit reader which starts from the position of the first bit of
+ // the data.
+ static Result<Ok, nsresult> ParseStRefPicSet(BitReader& aReader,
+ uint32_t aStRpsIdx,
+ H265SPS& aSPS);
+
+ // Parse the VUI parameters based on the H265 spec, E.2.1. MUST use a bit
+ // reader which starts from the position of the first bit of the data.
+ static Result<Ok, nsresult> ParseVuiParameters(BitReader& aReader,
+ H265SPS& aSPS);
+
+ // Parse and ignore the structure. MUST use a bitreader which starts from the
+ // position of the first bit of the data.
+ static Result<Ok, nsresult> ParseAndIgnoreScalingListData(BitReader& aReader);
+ static Result<Ok, nsresult> ParseAndIgnoreHrdParameters(
+ BitReader& aReader, bool aCommonInfPresentFlag,
+ int aMaxNumSubLayersMinus1);
+ static Result<Ok, nsresult> ParseAndIgnoreSubLayerHrdParameters(
+ BitReader& aReader, int aCpbCnt, bool aSubPicHrdParamsPresentFlag);
+};
+
+} // namespace mozilla
+
+#endif // DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_H265_H_
diff --git a/dom/media/platforms/agnostic/bytestreams/gtest/TestByteStreams.cpp b/dom/media/platforms/agnostic/bytestreams/gtest/TestByteStreams.cpp
new file mode 100644
index 0000000000..911f10f193
--- /dev/null
+++ b/dom/media/platforms/agnostic/bytestreams/gtest/TestByteStreams.cpp
@@ -0,0 +1,787 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+#include "gtest/gtest.h"
+
+#include "AnnexB.h"
+#include "BufferReader.h"
+#include "ByteWriter.h"
+#include "H264.h"
+#include "H265.h"
+#include "mozilla/Types.h"
+
+namespace mozilla {
+
+// Create AVCC style extra data (the contents on an AVCC box). Note
+// NALLengthSize will be 4 so AVCC samples need to set their data up
+// accordingly.
+static already_AddRefed<MediaByteBuffer> GetExtraData() {
+ // Extra data with
+ // - baseline profile(0x42 == 66).
+ // - constraint flags 0 and 1 set(0xc0) -- normal for baseline profile.
+ // - level 4.0 (0x28 == 40).
+ // - 1280 * 720 resolution.
+ return H264::CreateExtraData(0x42, 0xc0, 0x28, {1280, 720});
+}
+
+// Create an AVCC style sample with requested size in bytes. This sample is
+// setup to contain a single NAL (in practice samples can contain many). The
+// sample sets its NAL size to aSampleSize - 4 and stores that size in the first
+// 4 bytes. Aside from the NAL size at the start, the data is uninitialized
+// (beware)! aSampleSize is a uint32_t as samples larger than can be expressed
+// by a uint32_t are not to spec.
+static already_AddRefed<MediaRawData> GetAvccSample(uint32_t aSampleSize) {
+ if (aSampleSize < 4) {
+ // Stop tests asking for insane samples.
+ EXPECT_FALSE(true) << "Samples should be requested with sane sizes";
+ }
+ nsTArray<uint8_t> sampleData;
+
+ // Write the NAL size.
+ ByteWriter<BigEndian> writer(sampleData);
+ EXPECT_TRUE(writer.WriteU32(aSampleSize - 4));
+
+ // Write the 'NAL'. Beware, this data is uninitialized.
+ sampleData.AppendElements(static_cast<size_t>(aSampleSize) - 4);
+ RefPtr<MediaRawData> rawData =
+ new MediaRawData{sampleData.Elements(), sampleData.Length()};
+ EXPECT_NE(rawData->Data(), nullptr);
+
+ // Set extra data.
+ rawData->mExtraData = GetExtraData();
+ return rawData.forget();
+}
+
+static const uint8_t sHvccBytesBuffer[] = {
+ 1 /* version */,
+ 1 /* general_profile_space/general_tier_flag/general_profile_idc */,
+ 0x60 /* general_profile_compatibility_flags 1/4 */,
+ 0 /* general_profile_compatibility_flags 2/4 */,
+ 0 /* general_profile_compatibility_flags 3/4 */,
+ 0 /* general_profile_compatibility_flags 4/4 */,
+ 0x90 /* general_constraint_indicator_flags 1/6 */,
+ 0 /* general_constraint_indicator_flags 2/6 */,
+ 0 /* general_constraint_indicator_flags 3/6 */,
+ 0 /* general_constraint_indicator_flags 4/6 */,
+ 0 /* general_constraint_indicator_flags 5/6 */,
+ 0 /* general_constraint_indicator_flags 6/6 */,
+ 0x5A /* general_level_idc */,
+ 0 /* min_spatial_segmentation_idc 1/2 */,
+ 0 /* min_spatial_segmentation_idc 2/2 */,
+ 0 /* parallelismType */,
+ 1 /* chroma_format_idc */,
+ 0 /* bit_depth_luma_minus8 */,
+ 0 /* bit_depth_chroma_minus8 */,
+ 0 /* avgFrameRate 1/2 */,
+ 0 /* avgFrameRate 2/2 */,
+ 0x0F /* constantFrameRate/numTemporalLayers/temporalIdNested/lengthSizeMinusOne
+ */
+ ,
+ 2 /* numOfArrays */,
+ /* SPS Array */
+ 0x21 /* NAL_unit_type (SPS) */,
+ 0 /* numNalus 1/2 */,
+ 1 /* numNalus 2/2 */,
+
+ /* SPS */
+ 0 /* nalUnitLength 1/2 */,
+ 8 /* nalUnitLength 2/2 (header + rsbp) */,
+ 0x42 /* NALU header 1/2 */,
+ 0 /* NALU header 2/2 */,
+ 0 /* rbsp 1/6 */,
+ 0 /* rbsp 2/6 */,
+ 0 /* rbsp 3/6 */,
+ 0 /* rbsp 4/6 */,
+ 0 /* rbsp 5/6 */,
+ 0 /* rbsp 6/6 */,
+
+ /* PPS Array */
+ 0x22 /* NAL_unit_type (PPS) */,
+ 0 /* numNalus 1/2 */,
+ 1 /* numNalus 2/2 */,
+
+ /* PPS */
+ 0 /* nalUnitLength 1/2 */,
+ 3 /* nalUnitLength 2/2 (header + rsbp) */,
+ 0x44 /* NALU header 1/2 */,
+ 0 /* NALU header 2/2 */,
+ 0 /* rbsp */,
+};
+
+// Create a HVCC sample, which contain fake data, in given size.
+static already_AddRefed<MediaRawData> GetHVCCSample(uint32_t aSampleSize) {
+ if (aSampleSize < 4) {
+ // Stop tests asking for insane samples.
+ EXPECT_FALSE(true) << "Samples should be requested with sane sizes";
+ }
+ auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>();
+ extradata->AppendElements(sHvccBytesBuffer, ArrayLength(sHvccBytesBuffer));
+
+ // Write the NAL size.
+ nsTArray<uint8_t> sampleData;
+ ByteWriter<BigEndian> writer(sampleData);
+ EXPECT_TRUE(writer.WriteU32(aSampleSize - 4)); // Assume it's a 4 bytes NALU
+
+ // Fill fake empty data
+ for (uint32_t idx = 0; idx < aSampleSize - 4; idx++) {
+ sampleData.AppendElement(0);
+ }
+ RefPtr<MediaRawData> rawData =
+ new MediaRawData{sampleData.Elements(), sampleData.Length()};
+ EXPECT_NE(rawData->Data(), nullptr);
+ EXPECT_EQ(rawData->Size(), aSampleSize);
+ rawData->mExtraData = extradata;
+ return rawData.forget();
+}
+
+// Create a HVCC sample by using given data in given size.
+static already_AddRefed<MediaRawData> GetHVCCSample(
+ const uint8_t* aData, const uint32_t aDataLength) {
+ if (aDataLength < 4) {
+ // Stop tests asking for insane samples.
+ EXPECT_FALSE(true) << "Samples should be requested with sane sizes";
+ }
+ auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>();
+ extradata->AppendElements(sHvccBytesBuffer, ArrayLength(sHvccBytesBuffer));
+
+ // Write the NAL size.
+ nsTArray<uint8_t> sampleData;
+ ByteWriter<BigEndian> writer(sampleData);
+ EXPECT_TRUE(writer.WriteU32(aDataLength)); // Assume it's a 4 bytes NALU
+ sampleData.AppendElements(aData, aDataLength);
+
+ RefPtr<MediaRawData> rawData =
+ new MediaRawData{sampleData.Elements(), sampleData.Length()};
+ EXPECT_NE(rawData->Data(), nullptr);
+ EXPECT_EQ(rawData->Size(), aDataLength + 4);
+ rawData->mExtraData = extradata;
+ return rawData.forget();
+}
+
+// Test that conversion from AVCC to AnnexB works as expected.
+TEST(AnnexB, AVCCToAnnexBConversion)
+{
+ RefPtr<MediaRawData> rawData{GetAvccSample(128)};
+
+ {
+ // Test conversion of data when not adding SPS works as expected.
+ RefPtr<MediaRawData> rawDataClone = rawData->Clone();
+ Result<Ok, nsresult> result =
+ AnnexB::ConvertAVCCSampleToAnnexB(rawDataClone, /* aAddSps */ false);
+ EXPECT_TRUE(result.isOk()) << "Conversion should succeed";
+ EXPECT_EQ(rawDataClone->Size(), rawData->Size())
+ << "AnnexB sample should be the same size as the AVCC sample -- the 4 "
+ "byte NAL length data (AVCC) is replaced with 4 bytes of NAL "
+ "separator (AnnexB)";
+ EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone))
+ << "The sample should be AnnexB following conversion";
+ }
+
+ {
+ // Test that the SPS data is not added if the frame is not a keyframe.
+ RefPtr<MediaRawData> rawDataClone = rawData->Clone();
+ rawDataClone->mKeyframe =
+ false; // false is the default, but let's be sure.
+ Result<Ok, nsresult> result =
+ AnnexB::ConvertAVCCSampleToAnnexB(rawDataClone, /* aAddSps */ true);
+ EXPECT_TRUE(result.isOk()) << "Conversion should succeed";
+ EXPECT_EQ(rawDataClone->Size(), rawData->Size())
+ << "AnnexB sample should be the same size as the AVCC sample -- the 4 "
+ "byte NAL length data (AVCC) is replaced with 4 bytes of NAL "
+ "separator (AnnexB) and SPS data is not added as the frame is not a "
+ "keyframe";
+ EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone))
+ << "The sample should be AnnexB following conversion";
+ }
+
+ {
+ // Test that the SPS data is added to keyframes.
+ RefPtr<MediaRawData> rawDataClone = rawData->Clone();
+ rawDataClone->mKeyframe = true;
+ Result<Ok, nsresult> result =
+ AnnexB::ConvertAVCCSampleToAnnexB(rawDataClone, /* aAddSps */ true);
+ EXPECT_TRUE(result.isOk()) << "Conversion should succeed";
+ EXPECT_GT(rawDataClone->Size(), rawData->Size())
+ << "AnnexB sample should be larger than the AVCC sample because we've "
+ "added SPS data";
+ EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone))
+ << "The sample should be AnnexB following conversion";
+ // We could verify the SPS and PPS data we add, but we don't have great
+ // tooling to do so. Consider doing so in future.
+ }
+
+ {
+ // Test conversion involving subsample encryption doesn't overflow vlaues.
+ const uint32_t sampleSize = UINT16_MAX * 2;
+ RefPtr<MediaRawData> rawCryptoData{GetAvccSample(sampleSize)};
+ // Need to be a keyframe to test prepending SPS + PPS to sample.
+ rawCryptoData->mKeyframe = true;
+ UniquePtr<MediaRawDataWriter> rawDataWriter = rawCryptoData->CreateWriter();
+
+ rawDataWriter->mCrypto.mCryptoScheme = CryptoScheme::Cenc;
+
+ // We want to check that the clear size doesn't overflow during conversion.
+ // This size originates in a uint16_t, but since it can grow during AnnexB
+ // we cover it here.
+ const uint16_t clearSize = UINT16_MAX - 10;
+ // Set a clear size very close to uint16_t max value.
+ rawDataWriter->mCrypto.mPlainSizes.AppendElement(clearSize);
+ rawDataWriter->mCrypto.mEncryptedSizes.AppendElement(sampleSize -
+ clearSize);
+
+ RefPtr<MediaRawData> rawCryptoDataClone = rawCryptoData->Clone();
+ Result<Ok, nsresult> result = AnnexB::ConvertAVCCSampleToAnnexB(
+ rawCryptoDataClone, /* aAddSps */ true);
+ EXPECT_TRUE(result.isOk()) << "Conversion should succeed";
+ EXPECT_GT(rawCryptoDataClone->Size(), rawCryptoData->Size())
+ << "AnnexB sample should be larger than the AVCC sample because we've "
+ "added SPS data";
+ EXPECT_GT(rawCryptoDataClone->mCrypto.mPlainSizes[0],
+ rawCryptoData->mCrypto.mPlainSizes[0])
+ << "Conversion should have increased clear data sizes without overflow";
+ EXPECT_EQ(rawCryptoDataClone->mCrypto.mEncryptedSizes[0],
+ rawCryptoData->mCrypto.mEncryptedSizes[0])
+ << "Conversion should not affect encrypted sizes";
+ EXPECT_TRUE(AnnexB::IsAnnexB(rawCryptoDataClone))
+ << "The sample should be AnnexB following conversion";
+ }
+}
+
+TEST(AnnexB, HVCCToAnnexBConversion)
+{
+ RefPtr<MediaRawData> rawData{GetHVCCSample(128)};
+ {
+ // Test conversion of data when not adding SPS works as expected.
+ RefPtr<MediaRawData> rawDataClone = rawData->Clone();
+ Result<Ok, nsresult> result =
+ AnnexB::ConvertHVCCSampleToAnnexB(rawDataClone, /* aAddSps */ false);
+ EXPECT_TRUE(result.isOk()) << "Conversion should succeed";
+ EXPECT_EQ(rawDataClone->Size(), rawData->Size())
+ << "AnnexB sample should be the same size as the HVCC sample -- the 4 "
+ "byte NAL length data (HVCC) is replaced with 4 bytes of NAL "
+ "separator (AnnexB)";
+ EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone))
+ << "The sample should be AnnexB following conversion";
+ }
+ {
+ // Test that the SPS data is not added if the frame is not a keyframe.
+ RefPtr<MediaRawData> rawDataClone = rawData->Clone();
+ rawDataClone->mKeyframe =
+ false; // false is the default, but let's be sure.
+ Result<Ok, nsresult> result =
+ AnnexB::ConvertHVCCSampleToAnnexB(rawDataClone, /* aAddSps */ true);
+ EXPECT_TRUE(result.isOk()) << "Conversion should succeed";
+ EXPECT_EQ(rawDataClone->Size(), rawData->Size())
+ << "AnnexB sample should be the same size as the HVCC sample -- the 4 "
+ "byte NAL length data (HVCC) is replaced with 4 bytes of NAL "
+ "separator (AnnexB) and SPS data is not added as the frame is not a "
+ "keyframe";
+ EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone))
+ << "The sample should be AnnexB following conversion";
+ }
+ {
+ // Test that the SPS data is added to keyframes.
+ RefPtr<MediaRawData> rawDataClone = rawData->Clone();
+ rawDataClone->mKeyframe = true;
+ Result<Ok, nsresult> result =
+ AnnexB::ConvertHVCCSampleToAnnexB(rawDataClone, /* aAddSps */ true);
+ EXPECT_TRUE(result.isOk()) << "Conversion should succeed";
+ EXPECT_GT(rawDataClone->Size(), rawData->Size())
+ << "AnnexB sample should be larger than the HVCC sample because we've "
+ "added SPS data";
+ EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone))
+ << "The sample should be AnnexB following conversion";
+ // We could verify the SPS and PPS data we add, but we don't have great
+ // tooling to do so. Consider doing so in future.
+ }
+ {
+ // Test conversion involving subsample encryption doesn't overflow values.
+ const uint32_t sampleSize = UINT16_MAX * 2;
+ RefPtr<MediaRawData> rawCryptoData{GetHVCCSample(sampleSize)};
+ // Need to be a keyframe to test prepending SPS + PPS to sample.
+ rawCryptoData->mKeyframe = true;
+ UniquePtr<MediaRawDataWriter> rawDataWriter = rawCryptoData->CreateWriter();
+
+ rawDataWriter->mCrypto.mCryptoScheme = CryptoScheme::Cenc;
+
+ // We want to check that the clear size doesn't overflow during conversion.
+ // This size originates in a uint16_t, but since it can grow during AnnexB
+ // we cover it here.
+ const uint16_t clearSize = UINT16_MAX - 10;
+ // Set a clear size very close to uint16_t max value.
+ rawDataWriter->mCrypto.mPlainSizes.AppendElement(clearSize);
+ rawDataWriter->mCrypto.mEncryptedSizes.AppendElement(sampleSize -
+ clearSize);
+
+ RefPtr<MediaRawData> rawCryptoDataClone = rawCryptoData->Clone();
+ Result<Ok, nsresult> result = AnnexB::ConvertHVCCSampleToAnnexB(
+ rawCryptoDataClone, /* aAddSps */ true);
+ EXPECT_TRUE(result.isOk()) << "Conversion should succeed";
+ EXPECT_GT(rawCryptoDataClone->Size(), rawCryptoData->Size())
+ << "AnnexB sample should be larger than the HVCC sample because we've "
+ "added SPS data";
+ EXPECT_GT(rawCryptoDataClone->mCrypto.mPlainSizes[0],
+ rawCryptoData->mCrypto.mPlainSizes[0])
+ << "Conversion should have increased clear data sizes without overflow";
+ EXPECT_EQ(rawCryptoDataClone->mCrypto.mEncryptedSizes[0],
+ rawCryptoData->mCrypto.mEncryptedSizes[0])
+ << "Conversion should not affect encrypted sizes";
+ EXPECT_TRUE(AnnexB::IsAnnexB(rawCryptoDataClone))
+ << "The sample should be AnnexB following conversion";
+ }
+}
+
+TEST(H264, AVCCParsingSuccess)
+{
+ auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>();
+ uint8_t avccBytesBuffer[] = {
+ 1 /* version */,
+ 0x64 /* profile (High) */,
+ 0 /* profile compat (0) */,
+ 40 /* level (40) */,
+ 0xfc | 3 /* nal size - 1 */,
+ 0xe0 /* num SPS (0) */,
+ 0 /* num PPS (0) */
+ };
+ extradata->AppendElements(avccBytesBuffer, ArrayLength(avccBytesBuffer));
+ auto rv = AVCCConfig::Parse(extradata);
+ EXPECT_TRUE(rv.isOk());
+ const auto avcc = rv.unwrap();
+ EXPECT_EQ(avcc.mConfigurationVersion, 1);
+ EXPECT_EQ(avcc.mAVCProfileIndication, 0x64);
+ EXPECT_EQ(avcc.mProfileCompatibility, 0);
+ EXPECT_EQ(avcc.mAVCLevelIndication, 40);
+ EXPECT_EQ(avcc.NALUSize(), 4);
+ EXPECT_EQ(avcc.mNumSPS, 0);
+}
+
+TEST(H264, AVCCParsingFailure)
+{
+ {
+ // Incorrect version
+ auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>();
+ uint8_t avccBytesBuffer[] = {
+ 2 /* version */,
+ 0x64 /* profile (High) */,
+ 0 /* profile compat (0) */,
+ 40 /* level (40) */,
+ 0xfc | 3 /* nal size - 1 */,
+ 0xe0 /* num SPS (0) */,
+ 0 /* num PPS (0) */
+ };
+ extradata->AppendElements(avccBytesBuffer, ArrayLength(avccBytesBuffer));
+ auto avcc = AVCCConfig::Parse(extradata);
+ EXPECT_TRUE(avcc.isErr());
+ }
+ {
+ // Insuffient data (lacking of PPS)
+ auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>();
+ uint8_t avccBytesBuffer[] = {
+ 1 /* version */,
+ 0x64 /* profile (High) */,
+ 0 /* profile compat (0) */,
+ 40 /* level (40) */,
+ 0xfc | 3 /* nal size - 1 */,
+ 0xe0 /* num SPS (0) */,
+ };
+ extradata->AppendElements(avccBytesBuffer, ArrayLength(avccBytesBuffer));
+ auto avcc = AVCCConfig::Parse(extradata);
+ EXPECT_TRUE(avcc.isErr());
+ }
+}
+
+TEST(H265, HVCCParsingSuccess)
+{
+ {
+ auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>();
+ uint8_t hvccBytesBuffer[] = {
+ 1 /* version */,
+ 1 /* general_profile_space/general_tier_flag/general_profile_idc */,
+ 0x60 /* general_profile_compatibility_flags 1/4 */,
+ 0 /* general_profile_compatibility_flags 2/4 */,
+ 0 /* general_profile_compatibility_flags 3/4 */,
+ 0 /* general_profile_compatibility_flags 4/4 */,
+ 0x90 /* general_constraint_indicator_flags 1/6 */,
+ 0 /* general_constraint_indicator_flags 2/6 */,
+ 0 /* general_constraint_indicator_flags 3/6 */,
+ 0 /* general_constraint_indicator_flags 4/6 */,
+ 0 /* general_constraint_indicator_flags 5/6 */,
+ 0 /* general_constraint_indicator_flags 6/6 */,
+ 0x5A /* general_level_idc */,
+ 0 /* min_spatial_segmentation_idc 1/2 */,
+ 0 /* min_spatial_segmentation_idc 2/2 */,
+ 0 /* parallelismType */,
+ 1 /* chroma_format_idc */,
+ 0 /* bit_depth_luma_minus8 */,
+ 0 /* bit_depth_chroma_minus8 */,
+ 0 /* avgFrameRate 1/2 */,
+ 0 /* avgFrameRate 2/2 */,
+ 0x0F /* constantFrameRate/numTemporalLayers/temporalIdNested/lengthSizeMinusOne
+ */
+ ,
+ 0 /* numOfArrays */,
+ };
+ extradata->AppendElements(hvccBytesBuffer, ArrayLength(hvccBytesBuffer));
+ auto rv = HVCCConfig::Parse(extradata);
+ EXPECT_TRUE(rv.isOk());
+ auto hvcc = rv.unwrap();
+ EXPECT_EQ(hvcc.configurationVersion, 1);
+ EXPECT_EQ(hvcc.general_profile_space, 0);
+ EXPECT_EQ(hvcc.general_tier_flag, false);
+ EXPECT_EQ(hvcc.general_profile_idc, 1);
+ EXPECT_EQ(hvcc.general_profile_compatibility_flags, (uint32_t)0x60000000);
+ EXPECT_EQ(hvcc.general_constraint_indicator_flags,
+ (uint64_t)0x900000000000);
+ EXPECT_EQ(hvcc.general_level_idc, 0x5A);
+ EXPECT_EQ(hvcc.min_spatial_segmentation_idc, 0);
+ EXPECT_EQ(hvcc.parallelismType, 0);
+ EXPECT_EQ(hvcc.chroma_format_idc, 1);
+ EXPECT_EQ(hvcc.bit_depth_luma_minus8, 0);
+ EXPECT_EQ(hvcc.bit_depth_chroma_minus8, 0);
+ EXPECT_EQ(hvcc.avgFrameRate, 0);
+ EXPECT_EQ(hvcc.constantFrameRate, 0);
+ EXPECT_EQ(hvcc.numTemporalLayers, 1);
+ EXPECT_EQ(hvcc.temporalIdNested, true);
+ EXPECT_EQ(hvcc.NALUSize(), 4);
+ EXPECT_EQ(hvcc.mNALUs.Length(), uint32_t(0));
+ }
+ {
+ // Multple NALUs
+ auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>();
+ uint8_t hvccBytesBuffer[] = {
+ 1 /* version */,
+ 1 /* general_profile_space/general_tier_flag/general_profile_idc */,
+ 0x60 /* general_profile_compatibility_flags 1/4 */,
+ 0 /* general_profile_compatibility_flags 2/4 */,
+ 0 /* general_profile_compatibility_flags 3/4 */,
+ 0 /* general_profile_compatibility_flags 4/4 */,
+ 0x90 /* general_constraint_indicator_flags 1/6 */,
+ 0 /* general_constraint_indicator_flags 2/6 */,
+ 0 /* general_constraint_indicator_flags 3/6 */,
+ 0 /* general_constraint_indicator_flags 4/6 */,
+ 0 /* general_constraint_indicator_flags 5/6 */,
+ 0 /* general_constraint_indicator_flags 6/6 */,
+ 0x5A /* general_level_idc */,
+ 0 /* min_spatial_segmentation_idc 1/2 */,
+ 0 /* min_spatial_segmentation_idc 2/2 */,
+ 0 /* parallelismType */,
+ 1 /* chroma_format_idc */,
+ 0 /* bit_depth_luma_minus8 */,
+ 0 /* bit_depth_chroma_minus8 */,
+ 0 /* avgFrameRate 1/2 */,
+ 0 /* avgFrameRate 2/2 */,
+ 0x0F /* constantFrameRate/numTemporalLayers/temporalIdNested/lengthSizeMinusOne
+ */
+ ,
+ 2 /* numOfArrays */,
+ /* SPS Array */
+ 0x21 /* NAL_unit_type (SPS) */,
+ 0 /* numNalus 1/2 */,
+ 1 /* numNalus 2/2 */,
+
+ /* SPS */
+ 0 /* nalUnitLength 1/2 */,
+ 8 /* nalUnitLength 2/2 (header + rsbp) */,
+ 0x42 /* NALU header 1/2 */,
+ 0 /* NALU header 2/2 */,
+ 0 /* rbsp 1/6 */,
+ 0 /* rbsp 2/6 */,
+ 0 /* rbsp 3/6 */,
+ 0 /* rbsp 4/6 */,
+ 0 /* rbsp 5/6 */,
+ 0 /* rbsp 6/6 */,
+
+ /* PPS Array */
+ 0x22 /* NAL_unit_type (PPS) */,
+ 0 /* numNalus 1/2 */,
+ 2 /* numNalus 2/2 */,
+
+ /* PPS 1 */
+ 0 /* nalUnitLength 1/2 */,
+ 3 /* nalUnitLength 2/2 (header + rsbp) */,
+ 0x44 /* NALU header 1/2 */,
+ 0 /* NALU header 2/2 */,
+ 0 /* rbsp */,
+
+ /* PPS 2 */
+ 0 /* nalUnitLength 1/2 */,
+ 3 /* nalUnitLength 2/2 (header + rsbp) */,
+ 0x44 /* NALU header 1/2 */,
+ 0 /* NALU header 2/2 */,
+ 0 /* rbsp */,
+ };
+ extradata->AppendElements(hvccBytesBuffer, ArrayLength(hvccBytesBuffer));
+ auto rv = HVCCConfig::Parse(extradata);
+ EXPECT_TRUE(rv.isOk());
+ auto hvcc = rv.unwrap();
+ // Check NALU, it should contain 1 SPS and 2 PPS.
+ EXPECT_EQ(hvcc.mNALUs.Length(), uint32_t(3));
+ EXPECT_EQ(hvcc.mNALUs[0].mNalUnitType, H265NALU::NAL_TYPES::SPS_NUT);
+ EXPECT_EQ(hvcc.mNALUs[0].mNuhLayerId, 0);
+ EXPECT_EQ(hvcc.mNALUs[0].mNuhTemporalIdPlus1, 0);
+ EXPECT_EQ(hvcc.mNALUs[0].IsSPS(), true);
+ EXPECT_EQ(hvcc.mNALUs[0].mNALU.Length(), 8u);
+
+ EXPECT_EQ(hvcc.mNALUs[1].mNalUnitType, H265NALU::NAL_TYPES::PPS_NUT);
+ EXPECT_EQ(hvcc.mNALUs[1].mNuhLayerId, 0);
+ EXPECT_EQ(hvcc.mNALUs[1].mNuhTemporalIdPlus1, 0);
+ EXPECT_EQ(hvcc.mNALUs[1].IsSPS(), false);
+ EXPECT_EQ(hvcc.mNALUs[1].mNALU.Length(), 3u);
+
+ EXPECT_EQ(hvcc.mNALUs[2].mNalUnitType, H265NALU::NAL_TYPES::PPS_NUT);
+ EXPECT_EQ(hvcc.mNALUs[2].mNuhLayerId, 0);
+ EXPECT_EQ(hvcc.mNALUs[2].mNuhTemporalIdPlus1, 0);
+ EXPECT_EQ(hvcc.mNALUs[2].IsSPS(), false);
+ EXPECT_EQ(hvcc.mNALUs[2].mNALU.Length(), 3u);
+ }
+}
+
+TEST(H265, HVCCParsingFailure)
+{
+ {
+ // Incorrect version
+ auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>();
+ uint8_t hvccBytesBuffer[] = {
+ 2 /* version */,
+ 1 /* general_profile_space/general_tier_flag/general_profile_idc */,
+ 0x60 /* general_profile_compatibility_flags 1/4 */,
+ 0 /* general_profile_compatibility_flags 2/4 */,
+ 0 /* general_profile_compatibility_flags 3/4 */,
+ 0 /* general_profile_compatibility_flags 4/4 */,
+ 0x90 /* general_constraint_indicator_flags 1/6 */,
+ 0 /* general_constraint_indicator_flags 2/6 */,
+ 0 /* general_constraint_indicator_flags 3/6 */,
+ 0 /* general_constraint_indicator_flags 4/6 */,
+ 0 /* general_constraint_indicator_flags 5/6 */,
+ 0 /* general_constraint_indicator_flags 6/6 */,
+ 0x5A /* general_level_idc */,
+ 0 /* min_spatial_segmentation_idc 1/2 */,
+ 0 /* min_spatial_segmentation_idc 2/2 */,
+ 0 /* parallelismType */,
+ 1 /* chroma_format_idc */,
+ 0 /* bit_depth_luma_minus8 */,
+ 0 /* bit_depth_chroma_minus8 */,
+ 0 /* avgFrameRate 1/2 */,
+ 0 /* avgFrameRate 2/2 */,
+ 0x0F /* constantFrameRate/numTemporalLayers/temporalIdNested/lengthSizeMinusOne
+ */
+ ,
+ 0 /* numOfArrays */,
+ };
+ extradata->AppendElements(hvccBytesBuffer, ArrayLength(hvccBytesBuffer));
+ auto avcc = HVCCConfig::Parse(extradata);
+ EXPECT_TRUE(avcc.isErr());
+ }
+ {
+ // Insuffient data
+ auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>();
+ uint8_t hvccBytesBuffer[] = {
+ 1 /* version */,
+ 1 /* general_profile_space/general_tier_flag/general_profile_idc */,
+ 0x60 /* general_profile_compatibility_flags 1/4 */,
+ 0 /* general_profile_compatibility_flags 2/4 */,
+ 0 /* general_profile_compatibility_flags 3/4 */,
+ 0 /* general_profile_compatibility_flags 4/4 */,
+ 0x90 /* general_constraint_indicator_flags 1/6 */,
+ 0 /* general_constraint_indicator_flags 2/6 */,
+ 0 /* general_constraint_indicator_flags 3/6 */,
+ 0 /* general_constraint_indicator_flags 4/6 */,
+ 0 /* general_constraint_indicator_flags 5/6 */,
+ 0 /* general_constraint_indicator_flags 6/6 */,
+ 0x5A /* general_level_idc */
+ };
+ extradata->AppendElements(hvccBytesBuffer, ArrayLength(hvccBytesBuffer));
+ auto avcc = HVCCConfig::Parse(extradata);
+ EXPECT_TRUE(avcc.isErr());
+ }
+}
+
+TEST(H265, HVCCToAnnexB)
+{
+ auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>();
+ uint8_t hvccBytesBuffer[] = {
+ 1 /* version */,
+ 1 /* general_profile_space/general_tier_flag/general_profile_idc */,
+ 0x60 /* general_profile_compatibility_flags 1/4 */,
+ 0 /* general_profile_compatibility_flags 2/4 */,
+ 0 /* general_profile_compatibility_flags 3/4 */,
+ 0 /* general_profile_compatibility_flags 4/4 */,
+ 0x90 /* general_constraint_indicator_flags 1/6 */,
+ 0 /* general_constraint_indicator_flags 2/6 */,
+ 0 /* general_constraint_indicator_flags 3/6 */,
+ 0 /* general_constraint_indicator_flags 4/6 */,
+ 0 /* general_constraint_indicator_flags 5/6 */,
+ 0 /* general_constraint_indicator_flags 6/6 */,
+ 0x5A /* general_level_idc */,
+ 0 /* min_spatial_segmentation_idc 1/2 */,
+ 0 /* min_spatial_segmentation_idc 2/2 */,
+ 0 /* parallelismType */,
+ 1 /* chroma_format_idc */,
+ 0 /* bit_depth_luma_minus8 */,
+ 0 /* bit_depth_chroma_minus8 */,
+ 0 /* avgFrameRate 1/2 */,
+ 0 /* avgFrameRate 2/2 */,
+ 0x0F /* constantFrameRate/numTemporalLayers/temporalIdNested/lengthSizeMinusOne
+ */
+ ,
+ 2 /* numOfArrays */,
+ /* SPS Array */
+ 0x21 /* NAL_unit_type (SPS) */,
+ 0 /* numNalus 1/2 */,
+ 1 /* numNalus 2/2 */,
+
+ /* SPS */
+ 0 /* nalUnitLength 1/2 */,
+ 3 /* nalUnitLength 2/2 (header + rsbp) */,
+ 0x42 /* NALU header 1/2 */,
+ 0 /* NALU header 2/2 */,
+ 0 /* rbsp */,
+
+ /* PPS Array */
+ 0x22 /* NAL_unit_type (PPS) */,
+ 0 /* numNalus 1/2 */,
+ 1 /* numNalus 2/2 */,
+
+ /* PPS */
+ 0 /* nalUnitLength 1/2 */,
+ 3 /* nalUnitLength 2/2 (header + rsbp) */,
+ 0x44 /* NALU header 1/2 */,
+ 0 /* NALU header 2/2 */,
+ 0 /* rbsp */,
+ };
+ extradata->AppendElements(hvccBytesBuffer, ArrayLength(hvccBytesBuffer));
+
+ // We convert hvcc extra-data to annexb format, then parse each nalu to see if
+ // they are still correct or not.
+ const size_t naluBytesSize = 3; // NAL size is 3, see nalUnitLength above
+ const size_t delimiterBytesSize = 4; // 0x00000001
+ const size_t naluPlusDelimiterBytesSize = naluBytesSize + delimiterBytesSize;
+ RefPtr<mozilla::MediaByteBuffer> annexBExtraData =
+ AnnexB::ConvertHVCCExtraDataToAnnexB(extradata);
+ // 2 NALU, sps and pps
+ EXPECT_EQ(annexBExtraData->Length(), naluPlusDelimiterBytesSize * 2);
+
+ H265NALU sps(
+ static_cast<uint8_t*>(annexBExtraData->Elements() + delimiterBytesSize),
+ naluBytesSize);
+ EXPECT_EQ(sps.mNalUnitType, H265NALU::NAL_TYPES::SPS_NUT);
+ EXPECT_EQ(sps.mNuhLayerId, 0);
+ EXPECT_EQ(sps.mNuhTemporalIdPlus1, 0);
+ EXPECT_EQ(sps.IsSPS(), true);
+ EXPECT_EQ(sps.mNALU.Length(), 3u);
+
+ H265NALU pps(
+ static_cast<uint8_t*>(annexBExtraData->Elements() +
+ naluPlusDelimiterBytesSize + delimiterBytesSize),
+ naluBytesSize);
+ EXPECT_EQ(pps.mNalUnitType, H265NALU::NAL_TYPES::PPS_NUT);
+ EXPECT_EQ(pps.mNuhLayerId, 0);
+ EXPECT_EQ(pps.mNuhTemporalIdPlus1, 0);
+ EXPECT_EQ(pps.IsSPS(), false);
+ EXPECT_EQ(pps.mNALU.Length(), 3u);
+}
+
+TEST(H265, AnnexBToHVCC)
+{
+ RefPtr<MediaRawData> rawData{GetHVCCSample(128)};
+ RefPtr<MediaRawData> rawDataClone = rawData->Clone();
+ Result<Ok, nsresult> result =
+ AnnexB::ConvertHVCCSampleToAnnexB(rawDataClone, /* aAddSps */ false);
+ EXPECT_TRUE(result.isOk()) << "HVCC to AnnexB Conversion should succeed";
+ EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone))
+ << "The sample should be AnnexB following conversion";
+
+ auto rv = AnnexB::ConvertSampleToHVCC(rawDataClone);
+ EXPECT_TRUE(rv.isOk()) << "AnnexB to HVCC Conversion should succeed";
+ EXPECT_TRUE(AnnexB::IsHVCC(rawDataClone))
+ << "The sample should be HVCC following conversion";
+}
+
+// This is SPS from 'hevc_white_frame.mp4'
+static const uint8_t sSps[] = {
+ 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00,
+ 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x5d, 0xa0, 0x02, 0x00, 0x80,
+ 0x30, 0x16, 0x59, 0x59, 0xa4, 0x93, 0x2b, 0xc0, 0x5a, 0x02, 0x00,
+ 0x00, 0x03, 0x00, 0x02, 0x00, 0x00, 0x03, 0x00, 0x3c, 0x10};
+
+TEST(H265, ExtractHVCCExtraData)
+{
+ RefPtr<MediaRawData> rawData{GetHVCCSample(sSps, ArrayLength(sSps))};
+ RefPtr<MediaByteBuffer> extradata = H265::ExtractHVCCExtraData(rawData);
+ EXPECT_TRUE(extradata);
+ auto rv = HVCCConfig::Parse(extradata);
+ EXPECT_TRUE(rv.isOk());
+ auto hvcc = rv.unwrap();
+ EXPECT_EQ(hvcc.mNALUs.Length(), 1u);
+ EXPECT_EQ(hvcc.mNALUs[0].mNalUnitType, H265NALU::NAL_TYPES::SPS_NUT);
+ EXPECT_EQ(hvcc.mNALUs[0].mNuhLayerId, 0u);
+ EXPECT_EQ(hvcc.mNALUs[0].mNuhTemporalIdPlus1, 1);
+ EXPECT_EQ(hvcc.mNALUs[0].IsSPS(), true);
+ EXPECT_EQ(hvcc.mNALUs[0].mNALU.Length(), 43u);
+}
+
+TEST(H265, DecodeSPSFromSPSNALU)
+{
+ H265NALU nalu{sSps, ArrayLength(sSps)};
+ auto rv = H265::DecodeSPSFromSPSNALU(nalu);
+ EXPECT_TRUE(rv.isOk());
+ auto sps = rv.unwrap();
+ // Examine the value by using HEVCESBrowser.
+ EXPECT_EQ(sps.sps_video_parameter_set_id, 0u);
+ EXPECT_EQ(sps.sps_max_sub_layers_minus1, 0u);
+ EXPECT_EQ(sps.sps_temporal_id_nesting_flag, 1);
+ EXPECT_EQ(sps.profile_tier_level.general_profile_space, 0u);
+ EXPECT_EQ(sps.profile_tier_level.general_tier_flag, false);
+ EXPECT_EQ(sps.profile_tier_level.general_profile_idc, 1u);
+ EXPECT_EQ(sps.profile_tier_level.general_profile_compatibility_flags,
+ 0x60000000u);
+ EXPECT_EQ(sps.profile_tier_level.general_progressive_source_flag, true);
+ EXPECT_EQ(sps.profile_tier_level.general_interlaced_source_flag, false);
+ EXPECT_EQ(sps.profile_tier_level.general_non_packed_constraint_flag, false);
+ EXPECT_EQ(sps.profile_tier_level.general_frame_only_constraint_flag, true);
+ EXPECT_EQ(sps.profile_tier_level.general_level_idc, 93u);
+ EXPECT_EQ(sps.sps_seq_parameter_set_id, 0u);
+ EXPECT_EQ(sps.chroma_format_idc, 1u);
+ EXPECT_EQ(sps.separate_colour_plane_flag, false);
+ EXPECT_EQ(sps.pic_width_in_luma_samples, 1024u);
+ EXPECT_EQ(sps.pic_height_in_luma_samples, 768u);
+ EXPECT_EQ(sps.conformance_window_flag, false);
+ EXPECT_EQ(sps.bit_depth_luma_minus8, 0u);
+ EXPECT_EQ(sps.bit_depth_chroma_minus8, 0u);
+ EXPECT_EQ(sps.log2_max_pic_order_cnt_lsb_minus4, 4u);
+ EXPECT_EQ(sps.sps_sub_layer_ordering_info_present_flag, true);
+ EXPECT_EQ(sps.sps_max_dec_pic_buffering_minus1[0], 4u);
+ EXPECT_EQ(sps.sps_max_num_reorder_pics[0], 2u);
+ EXPECT_EQ(sps.sps_max_latency_increase_plus1[0], 5u);
+ EXPECT_EQ(sps.log2_min_luma_coding_block_size_minus3, 0u);
+ EXPECT_EQ(sps.log2_diff_max_min_luma_coding_block_size, 3u);
+ EXPECT_EQ(sps.log2_min_luma_transform_block_size_minus2, 0u);
+ EXPECT_EQ(sps.log2_diff_max_min_luma_transform_block_size, 3u);
+ EXPECT_EQ(sps.max_transform_hierarchy_depth_inter, 0u);
+ EXPECT_EQ(sps.max_transform_hierarchy_depth_inter, 0u);
+ EXPECT_EQ(sps.pcm_enabled_flag, false);
+ EXPECT_EQ(sps.num_short_term_ref_pic_sets, 0u);
+ EXPECT_EQ(sps.sps_temporal_mvp_enabled_flag, true);
+ EXPECT_EQ(sps.strong_intra_smoothing_enabled_flag, true);
+ EXPECT_TRUE(sps.vui_parameters);
+ EXPECT_EQ(sps.vui_parameters->video_full_range_flag, false);
+
+ // Test public methods
+ EXPECT_EQ(sps.BitDepthLuma(), 8u);
+ EXPECT_EQ(sps.BitDepthChroma(), 8u);
+ const auto imgSize = sps.GetImageSize();
+ EXPECT_EQ(imgSize.Width(), 1024);
+ EXPECT_EQ(imgSize.Height(), 768);
+ const auto disSize = sps.GetDisplaySize();
+ EXPECT_EQ(disSize, imgSize);
+ EXPECT_EQ(sps.ColorDepth(), gfx::ColorDepth::COLOR_8);
+ EXPECT_EQ(sps.ColorSpace(), gfx::YUVColorSpace::BT709);
+ EXPECT_EQ(sps.IsFullColorRange(), false);
+ EXPECT_EQ(sps.ColorPrimaries(), 2u);
+ EXPECT_EQ(sps.TransferFunction(), 2u);
+}
+
+} // namespace mozilla
diff --git a/dom/media/platforms/agnostic/bytestreams/gtest/moz.build b/dom/media/platforms/agnostic/bytestreams/gtest/moz.build
new file mode 100644
index 0000000000..5351becdaf
--- /dev/null
+++ b/dom/media/platforms/agnostic/bytestreams/gtest/moz.build
@@ -0,0 +1,11 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+UNIFIED_SOURCES += [
+ "TestByteStreams.cpp",
+]
+
+FINAL_LIBRARY = "xul-gtest"
diff --git a/dom/media/platforms/agnostic/bytestreams/moz.build b/dom/media/platforms/agnostic/bytestreams/moz.build
new file mode 100644
index 0000000000..0244a62d41
--- /dev/null
+++ b/dom/media/platforms/agnostic/bytestreams/moz.build
@@ -0,0 +1,38 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+with Files("**"):
+ BUG_COMPONENT = ("Core", "Audio/Video: Playback")
+
+TEST_DIRS += [
+ "gtest",
+]
+
+EXPORTS += [
+ "Adts.h",
+ "AnnexB.h",
+ "ByteStreamsUtils.h",
+ "H264.h",
+ "H265.h",
+]
+
+UNIFIED_SOURCES += [
+ "Adts.cpp",
+ "AnnexB.cpp",
+ "H264.cpp",
+ "H265.cpp",
+]
+
+LOCAL_INCLUDES += [
+ "../../../mp4/",
+]
+
+FINAL_LIBRARY = "xul"
+
+# Suppress warnings for now.
+CXXFLAGS += [
+ "-Wno-sign-compare",
+]
diff --git a/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp
new file mode 100644
index 0000000000..e71632e6d3
--- /dev/null
+++ b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp
@@ -0,0 +1,156 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "ChromiumCDMVideoDecoder.h"
+#include "ChromiumCDMProxy.h"
+#include "content_decryption_module.h"
+#include "GMPService.h"
+#include "GMPVideoDecoder.h"
+#include "MP4Decoder.h"
+#include "VPXDecoder.h"
+
+namespace mozilla {
+
+ChromiumCDMVideoDecoder::ChromiumCDMVideoDecoder(
+ const GMPVideoDecoderParams& aParams, CDMProxy* aCDMProxy)
+ : mCDMParent(aCDMProxy->AsChromiumCDMProxy()->GetCDMParent()),
+ mConfig(aParams.mConfig),
+ mCrashHelper(aParams.mCrashHelper),
+ mGMPThread(GetGMPThread()),
+ mImageContainer(aParams.mImageContainer),
+ mKnowsCompositor(aParams.mKnowsCompositor) {}
+
+ChromiumCDMVideoDecoder::~ChromiumCDMVideoDecoder() = default;
+
+static uint32_t ToCDMH264Profile(uint8_t aProfile) {
+ switch (aProfile) {
+ case 66:
+ return cdm::VideoCodecProfile::kH264ProfileBaseline;
+ case 77:
+ return cdm::VideoCodecProfile::kH264ProfileMain;
+ case 88:
+ return cdm::VideoCodecProfile::kH264ProfileExtended;
+ case 100:
+ return cdm::VideoCodecProfile::kH264ProfileHigh;
+ case 110:
+ return cdm::VideoCodecProfile::kH264ProfileHigh10;
+ case 122:
+ return cdm::VideoCodecProfile::kH264ProfileHigh422;
+ case 144:
+ return cdm::VideoCodecProfile::kH264ProfileHigh444Predictive;
+ }
+ return cdm::VideoCodecProfile::kUnknownVideoCodecProfile;
+}
+
+RefPtr<MediaDataDecoder::InitPromise> ChromiumCDMVideoDecoder::Init() {
+ if (!mCDMParent) {
+ // Must have failed to get the CDMParent from the ChromiumCDMProxy
+ // in our constructor; the MediaKeys must have shut down the CDM
+ // before we had a chance to start up the decoder.
+ return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
+ }
+
+ gmp::CDMVideoDecoderConfig config;
+ if (MP4Decoder::IsH264(mConfig.mMimeType)) {
+ config.mCodec() = cdm::VideoCodec::kCodecH264;
+ config.mProfile() =
+ ToCDMH264Profile(mConfig.mExtraData->SafeElementAt(1, 0));
+ config.mExtraData() = mConfig.mExtraData->Clone();
+ mConvertToAnnexB = true;
+ } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) {
+ config.mCodec() = cdm::VideoCodec::kCodecVp8;
+ config.mProfile() = cdm::VideoCodecProfile::kProfileNotNeeded;
+ } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) {
+ config.mCodec() = cdm::VideoCodec::kCodecVp9;
+ config.mProfile() = cdm::VideoCodecProfile::kProfileNotNeeded;
+ } else {
+ return MediaDataDecoder::InitPromise::CreateAndReject(
+ NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
+ }
+ config.mImageWidth() = mConfig.mImage.width;
+ config.mImageHeight() = mConfig.mImage.height;
+ config.mEncryptionScheme() = cdm::EncryptionScheme::kUnencrypted;
+ switch (mConfig.mCrypto.mCryptoScheme) {
+ case CryptoScheme::None:
+ break;
+ case CryptoScheme::Cenc:
+ config.mEncryptionScheme() = cdm::EncryptionScheme::kCenc;
+ break;
+ case CryptoScheme::Cbcs:
+ config.mEncryptionScheme() = cdm::EncryptionScheme::kCbcs;
+ break;
+ default:
+ MOZ_ASSERT_UNREACHABLE("Should not have unrecognized encryption type");
+ break;
+ }
+
+ RefPtr<gmp::ChromiumCDMParent> cdm = mCDMParent;
+ VideoInfo info = mConfig;
+ RefPtr<layers::ImageContainer> imageContainer = mImageContainer;
+ RefPtr<layers::KnowsCompositor> knowsCompositor = mKnowsCompositor;
+ return InvokeAsync(mGMPThread, __func__,
+ [cdm, config, info, imageContainer, knowsCompositor]() {
+ return cdm->InitializeVideoDecoder(
+ config, info, imageContainer, knowsCompositor);
+ });
+}
+
+nsCString ChromiumCDMVideoDecoder::GetDescriptionName() const {
+ return "chromium cdm video decoder"_ns;
+}
+
+nsCString ChromiumCDMVideoDecoder::GetCodecName() const {
+ if (MP4Decoder::IsH264(mConfig.mMimeType)) {
+ return "h264"_ns;
+ } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) {
+ return "vp8"_ns;
+ } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) {
+ return "vp9"_ns;
+ }
+ return "unknown"_ns;
+}
+
+MediaDataDecoder::ConversionRequired ChromiumCDMVideoDecoder::NeedsConversion()
+ const {
+ return mConvertToAnnexB ? ConversionRequired::kNeedAnnexB
+ : ConversionRequired::kNeedNone;
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> ChromiumCDMVideoDecoder::Decode(
+ MediaRawData* aSample) {
+ RefPtr<gmp::ChromiumCDMParent> cdm = mCDMParent;
+ RefPtr<MediaRawData> sample = aSample;
+ return InvokeAsync(mGMPThread, __func__, [cdm, sample]() {
+ return cdm->DecryptAndDecodeFrame(sample);
+ });
+}
+
+RefPtr<MediaDataDecoder::FlushPromise> ChromiumCDMVideoDecoder::Flush() {
+ MOZ_ASSERT(mCDMParent);
+ RefPtr<gmp::ChromiumCDMParent> cdm = mCDMParent;
+ return InvokeAsync(mGMPThread, __func__,
+ [cdm]() { return cdm->FlushVideoDecoder(); });
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> ChromiumCDMVideoDecoder::Drain() {
+ MOZ_ASSERT(mCDMParent);
+ RefPtr<gmp::ChromiumCDMParent> cdm = mCDMParent;
+ return InvokeAsync(mGMPThread, __func__, [cdm]() { return cdm->Drain(); });
+}
+
+RefPtr<ShutdownPromise> ChromiumCDMVideoDecoder::Shutdown() {
+ if (!mCDMParent) {
+ // Must have failed to get the CDMParent from the ChromiumCDMProxy
+ // in our constructor; the MediaKeys must have shut down the CDM
+ // before we had a chance to start up the decoder.
+ return ShutdownPromise::CreateAndResolve(true, __func__);
+ }
+ RefPtr<gmp::ChromiumCDMParent> cdm = mCDMParent;
+ return InvokeAsync(mGMPThread, __func__,
+ [cdm]() { return cdm->ShutdownVideoDecoder(); });
+}
+
+} // namespace mozilla
diff --git a/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.h b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.h
new file mode 100644
index 0000000000..c177bf2e48
--- /dev/null
+++ b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.h
@@ -0,0 +1,54 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef ChromiumCDMVideoDecoder_h_
+#define ChromiumCDMVideoDecoder_h_
+
+#include "ChromiumCDMParent.h"
+#include "PlatformDecoderModule.h"
+#include "mozilla/layers/KnowsCompositor.h"
+
+namespace mozilla {
+
+class CDMProxy;
+struct GMPVideoDecoderParams;
+
+DDLoggedTypeDeclNameAndBase(ChromiumCDMVideoDecoder, MediaDataDecoder);
+
+class ChromiumCDMVideoDecoder final
+ : public MediaDataDecoder,
+ public DecoderDoctorLifeLogger<ChromiumCDMVideoDecoder> {
+ public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ChromiumCDMVideoDecoder, final);
+
+ ChromiumCDMVideoDecoder(const GMPVideoDecoderParams& aParams,
+ CDMProxy* aCDMProxy);
+
+ RefPtr<InitPromise> Init() override;
+ RefPtr<DecodePromise> Decode(MediaRawData* aSample) override;
+ RefPtr<FlushPromise> Flush() override;
+ RefPtr<DecodePromise> Drain() override;
+ RefPtr<ShutdownPromise> Shutdown() override;
+ nsCString GetDescriptionName() const override;
+ nsCString GetCodecName() const override;
+ ConversionRequired NeedsConversion() const override;
+
+ private:
+ ~ChromiumCDMVideoDecoder();
+
+ RefPtr<gmp::ChromiumCDMParent> mCDMParent;
+ const VideoInfo mConfig;
+ RefPtr<GMPCrashHelper> mCrashHelper;
+ nsCOMPtr<nsISerialEventTarget> mGMPThread;
+ RefPtr<layers::ImageContainer> mImageContainer;
+ RefPtr<layers::KnowsCompositor> mKnowsCompositor;
+ MozPromiseHolder<InitPromise> mInitPromise;
+ bool mConvertToAnnexB = false;
+};
+
+} // namespace mozilla
+
+#endif // ChromiumCDMVideoDecoder_h_
diff --git a/dom/media/platforms/agnostic/eme/DecryptThroughputLimit.h b/dom/media/platforms/agnostic/eme/DecryptThroughputLimit.h
new file mode 100644
index 0000000000..bafb387f83
--- /dev/null
+++ b/dom/media/platforms/agnostic/eme/DecryptThroughputLimit.h
@@ -0,0 +1,103 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef DecryptThroughputLimit_h
+#define DecryptThroughputLimit_h
+
+#include <deque>
+
+#include "MediaTimer.h"
+#include "PlatformDecoderModule.h"
+
+namespace mozilla {
+
+// We throttle our decrypt so that we don't decrypt more than a certain
+// duration of samples per second. This is to work around bugs in the
+// Widevine CDM. See bugs 1338924, 1342822, 1718223.
+class DecryptThroughputLimit {
+ public:
+ explicit DecryptThroughputLimit(nsISerialEventTarget* aTargetThread,
+ uint32_t aMaxThroughputMs)
+ : mThrottleScheduler(aTargetThread),
+ mMaxThroughput(aMaxThroughputMs / 1000.0) {}
+
+ typedef MozPromise<RefPtr<MediaRawData>, MediaResult, true> ThrottlePromise;
+
+ // Resolves promise after a delay if necessary in order to reduce the
+ // throughput of samples sent through the CDM for decryption.
+ RefPtr<ThrottlePromise> Throttle(MediaRawData* aSample) {
+ // We should only have one decrypt request being processed at once.
+ MOZ_RELEASE_ASSERT(!mThrottleScheduler.IsScheduled());
+
+ const TimeDuration WindowSize = TimeDuration::FromSeconds(0.1);
+ const TimeDuration MaxThroughput =
+ TimeDuration::FromSeconds(mMaxThroughput);
+
+ // Forget decrypts that happened before the start of our window.
+ const TimeStamp now = TimeStamp::Now();
+ while (!mDecrypts.empty() &&
+ mDecrypts.front().mTimestamp < now - WindowSize) {
+ mDecrypts.pop_front();
+ }
+
+ // How much time duration of the media would we have decrypted inside the
+ // time window if we did decrypt this block?
+ TimeDuration sampleDuration = aSample->mDuration.ToTimeDuration();
+ TimeDuration durationDecrypted = sampleDuration;
+ for (const DecryptedJob& job : mDecrypts) {
+ durationDecrypted += job.mSampleDuration;
+ }
+
+ if (durationDecrypted < MaxThroughput) {
+ // If we decrypted a sample of this duration, we would *not* have
+ // decrypted more than our threshold for max throughput, over the
+ // preceding wall time window. So we're safe to proceed with this
+ // decrypt.
+ mDecrypts.push_back(DecryptedJob({now, sampleDuration}));
+ return ThrottlePromise::CreateAndResolve(aSample, __func__);
+ }
+
+ // Otherwise, we need to delay until decrypting won't exceed our
+ // throughput threshold.
+
+ RefPtr<ThrottlePromise> p = mPromiseHolder.Ensure(__func__);
+
+ TimeDuration delay = durationDecrypted - MaxThroughput;
+ TimeStamp target = now + delay;
+ RefPtr<MediaRawData> sample(aSample);
+ mThrottleScheduler.Ensure(
+ target,
+ [this, sample, sampleDuration]() {
+ mThrottleScheduler.CompleteRequest();
+ mDecrypts.push_back(DecryptedJob({TimeStamp::Now(), sampleDuration}));
+ mPromiseHolder.Resolve(sample, __func__);
+ },
+ []() { MOZ_DIAGNOSTIC_ASSERT(false); });
+
+ return p;
+ }
+
+ void Flush() {
+ mThrottleScheduler.Reset();
+ mPromiseHolder.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ }
+
+ private:
+ DelayedScheduler mThrottleScheduler;
+ MozPromiseHolder<ThrottlePromise> mPromiseHolder;
+
+ double mMaxThroughput;
+
+ struct DecryptedJob {
+ TimeStamp mTimestamp;
+ TimeDuration mSampleDuration;
+ };
+ std::deque<DecryptedJob> mDecrypts;
+};
+
+} // namespace mozilla
+
+#endif // DecryptThroughputLimit_h
diff --git a/dom/media/platforms/agnostic/eme/EMEDecoderModule.cpp b/dom/media/platforms/agnostic/eme/EMEDecoderModule.cpp
new file mode 100644
index 0000000000..c143172073
--- /dev/null
+++ b/dom/media/platforms/agnostic/eme/EMEDecoderModule.cpp
@@ -0,0 +1,481 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "EMEDecoderModule.h"
+
+#include <inttypes.h>
+
+#include "Adts.h"
+#include "BlankDecoderModule.h"
+#include "ChromiumCDMVideoDecoder.h"
+#include "DecryptThroughputLimit.h"
+#include "GMPDecoderModule.h"
+#include "GMPService.h"
+#include "GMPVideoDecoder.h"
+#include "MP4Decoder.h"
+#include "MediaInfo.h"
+#include "PDMFactory.h"
+#include "mozilla/CDMProxy.h"
+#include "mozilla/EMEUtils.h"
+#include "mozilla/StaticPrefs_media.h"
+#include "mozilla/UniquePtr.h"
+#include "mozilla/Unused.h"
+#include "nsClassHashtable.h"
+#include "nsServiceManagerUtils.h"
+
+namespace mozilla {
+
+typedef MozPromiseRequestHolder<DecryptPromise> DecryptPromiseRequestHolder;
+
+DDLoggedTypeDeclNameAndBase(EMEDecryptor, MediaDataDecoder);
+
+class ADTSSampleConverter {
+ public:
+ explicit ADTSSampleConverter(const AudioInfo& aInfo)
+ : mNumChannels(aInfo.mChannels)
+ // Note: we set profile to 2 if we encounter an extended profile (which
+ // set mProfile to 0 and then set mExtendedProfile) such as HE-AACv2
+ // (profile 5). These can then pass through conversion to ADTS and back.
+ // This is done as ADTS only has 2 bits for profile, and the transform
+ // subtracts one from the value. We check if the profile supplied is > 4
+ // for safety. 2 is used as a fallback value, though it seems the CDM
+ // doesn't care what is set.
+ ,
+ mProfile(aInfo.mProfile < 1 || aInfo.mProfile > 4 ? 2 : aInfo.mProfile),
+ mFrequencyIndex(Adts::GetFrequencyIndex(aInfo.mRate)) {
+ EME_LOG("ADTSSampleConvertor(): aInfo.mProfile=%" PRIi8
+ " aInfo.mExtendedProfile=%" PRIi8,
+ aInfo.mProfile, aInfo.mExtendedProfile);
+ if (aInfo.mProfile < 1 || aInfo.mProfile > 4) {
+ EME_LOG(
+ "ADTSSampleConvertor(): Profile not in [1, 4]! Samples will "
+ "their profile set to 2!");
+ }
+ }
+ bool Convert(MediaRawData* aSample) const {
+ return Adts::ConvertSample(mNumChannels, mFrequencyIndex, mProfile,
+ aSample);
+ }
+ bool Revert(MediaRawData* aSample) const {
+ return Adts::RevertSample(aSample);
+ }
+
+ private:
+ const uint32_t mNumChannels;
+ const uint8_t mProfile;
+ const uint8_t mFrequencyIndex;
+};
+
+class EMEDecryptor final : public MediaDataDecoder,
+ public DecoderDoctorLifeLogger<EMEDecryptor> {
+ public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(EMEDecryptor, final);
+
+ EMEDecryptor(MediaDataDecoder* aDecoder, CDMProxy* aProxy,
+ TrackInfo::TrackType aType,
+ const std::function<MediaEventProducer<TrackInfo::TrackType>*()>&
+ aOnWaitingForKey,
+ UniquePtr<ADTSSampleConverter> aConverter = nullptr)
+ : mDecoder(aDecoder),
+ mProxy(aProxy),
+ mSamplesWaitingForKey(
+ new SamplesWaitingForKey(mProxy, aType, aOnWaitingForKey)),
+ mADTSSampleConverter(std::move(aConverter)),
+ mIsShutdown(false) {
+ DDLINKCHILD("decoder", mDecoder.get());
+ }
+
+ RefPtr<InitPromise> Init() override {
+ MOZ_ASSERT(!mIsShutdown);
+ mThread = GetCurrentSerialEventTarget();
+ uint32_t maxThroughputMs = StaticPrefs::media_eme_max_throughput_ms();
+ EME_LOG("EME max-throughput-ms=%" PRIu32, maxThroughputMs);
+ mThroughputLimiter.emplace(mThread, maxThroughputMs);
+
+ return mDecoder->Init();
+ }
+
+ RefPtr<DecodePromise> Decode(MediaRawData* aSample) override {
+ MOZ_ASSERT(mThread->IsOnCurrentThread());
+ MOZ_RELEASE_ASSERT(mDecrypts.Count() == 0,
+ "Can only process one sample at a time");
+ RefPtr<DecodePromise> p = mDecodePromise.Ensure(__func__);
+
+ RefPtr<EMEDecryptor> self = this;
+ mSamplesWaitingForKey->WaitIfKeyNotUsable(aSample)
+ ->Then(
+ mThread, __func__,
+ [self](const RefPtr<MediaRawData>& aSample) {
+ self->mKeyRequest.Complete();
+ self->ThrottleDecode(aSample);
+ },
+ [self]() { self->mKeyRequest.Complete(); })
+ ->Track(mKeyRequest);
+ return p;
+ }
+
+ void ThrottleDecode(MediaRawData* aSample) {
+ MOZ_ASSERT(mThread->IsOnCurrentThread());
+
+ RefPtr<EMEDecryptor> self = this;
+ mThroughputLimiter->Throttle(aSample)
+ ->Then(
+ mThread, __func__,
+ [self](RefPtr<MediaRawData> aSample) {
+ self->mThrottleRequest.Complete();
+ self->AttemptDecode(aSample);
+ },
+ [self]() { self->mThrottleRequest.Complete(); })
+ ->Track(mThrottleRequest);
+ }
+
+ void AttemptDecode(MediaRawData* aSample) {
+ MOZ_ASSERT(mThread->IsOnCurrentThread());
+ if (mIsShutdown) {
+ NS_WARNING("EME encrypted sample arrived after shutdown");
+ mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ return;
+ }
+
+ if (mADTSSampleConverter && !mADTSSampleConverter->Convert(aSample)) {
+ mDecodePromise.RejectIfExists(
+ MediaResult(
+ NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ RESULT_DETAIL("Failed to convert encrypted AAC sample to ADTS")),
+ __func__);
+ return;
+ }
+
+ const auto& decrypt = mDecrypts.InsertOrUpdate(
+ aSample, MakeUnique<DecryptPromiseRequestHolder>());
+ mProxy->Decrypt(aSample)
+ ->Then(mThread, __func__, this, &EMEDecryptor::Decrypted,
+ &EMEDecryptor::Decrypted)
+ ->Track(*decrypt);
+ }
+
+ void Decrypted(const DecryptResult& aDecrypted) {
+ MOZ_ASSERT(mThread->IsOnCurrentThread());
+ MOZ_ASSERT(aDecrypted.mSample);
+
+ UniquePtr<DecryptPromiseRequestHolder> holder;
+ mDecrypts.Remove(aDecrypted.mSample, &holder);
+ if (holder) {
+ holder->Complete();
+ } else {
+ // Decryption is not in the list of decrypt operations waiting
+ // for a result. It must have been flushed or drained. Ignore result.
+ return;
+ }
+
+ if (mADTSSampleConverter &&
+ !mADTSSampleConverter->Revert(aDecrypted.mSample)) {
+ mDecodePromise.RejectIfExists(
+ MediaResult(
+ NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ RESULT_DETAIL("Failed to revert decrypted ADTS sample to AAC")),
+ __func__);
+ return;
+ }
+
+ if (mIsShutdown) {
+ NS_WARNING("EME decrypted sample arrived after shutdown");
+ return;
+ }
+
+ if (aDecrypted.mStatus == eme::NoKeyErr) {
+ // Key became unusable after we sent the sample to CDM to decrypt.
+ // Call Decode() again, so that the sample is enqueued for decryption
+ // if the key becomes usable again.
+ AttemptDecode(aDecrypted.mSample);
+ } else if (aDecrypted.mStatus != eme::Ok) {
+ mDecodePromise.RejectIfExists(
+ MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ RESULT_DETAIL("decrypted.mStatus=%u",
+ uint32_t(aDecrypted.mStatus))),
+ __func__);
+ } else {
+ MOZ_ASSERT(!mIsShutdown);
+ // The sample is no longer encrypted, so clear its crypto metadata.
+ UniquePtr<MediaRawDataWriter> writer(aDecrypted.mSample->CreateWriter());
+ writer->mCrypto = CryptoSample();
+ RefPtr<EMEDecryptor> self = this;
+ mDecoder->Decode(aDecrypted.mSample)
+ ->Then(mThread, __func__,
+ [self](DecodePromise::ResolveOrRejectValue&& aValue) {
+ self->mDecodeRequest.Complete();
+ self->mDecodePromise.ResolveOrReject(std::move(aValue),
+ __func__);
+ })
+ ->Track(mDecodeRequest);
+ }
+ }
+
+ RefPtr<FlushPromise> Flush() override {
+ MOZ_ASSERT(mThread->IsOnCurrentThread());
+ MOZ_ASSERT(!mIsShutdown);
+ mKeyRequest.DisconnectIfExists();
+ mThrottleRequest.DisconnectIfExists();
+ mDecodeRequest.DisconnectIfExists();
+ mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ mThroughputLimiter->Flush();
+ for (auto iter = mDecrypts.Iter(); !iter.Done(); iter.Next()) {
+ auto holder = iter.UserData();
+ holder->DisconnectIfExists();
+ iter.Remove();
+ }
+ RefPtr<SamplesWaitingForKey> k = mSamplesWaitingForKey;
+ return mDecoder->Flush()->Then(mThread, __func__, [k]() {
+ k->Flush();
+ return FlushPromise::CreateAndResolve(true, __func__);
+ });
+ }
+
+ RefPtr<DecodePromise> Drain() override {
+ MOZ_ASSERT(mThread->IsOnCurrentThread());
+ MOZ_ASSERT(!mIsShutdown);
+ MOZ_ASSERT(mDecodePromise.IsEmpty() && !mDecodeRequest.Exists(),
+ "Must wait for decoding to complete");
+ for (auto iter = mDecrypts.Iter(); !iter.Done(); iter.Next()) {
+ auto holder = iter.UserData();
+ holder->DisconnectIfExists();
+ iter.Remove();
+ }
+ return mDecoder->Drain();
+ }
+
+ RefPtr<ShutdownPromise> Shutdown() override {
+ // mThread may not be set if Init hasn't been called first.
+ MOZ_ASSERT(!mThread || mThread->IsOnCurrentThread());
+ MOZ_ASSERT(!mIsShutdown);
+ mIsShutdown = true;
+ mSamplesWaitingForKey->BreakCycles();
+ mSamplesWaitingForKey = nullptr;
+ RefPtr<MediaDataDecoder> decoder = std::move(mDecoder);
+ mProxy = nullptr;
+ return decoder->Shutdown();
+ }
+
+ nsCString GetProcessName() const override {
+ return mDecoder->GetProcessName();
+ }
+
+ nsCString GetDescriptionName() const override {
+ return mDecoder->GetDescriptionName();
+ }
+
+ nsCString GetCodecName() const override { return mDecoder->GetCodecName(); }
+
+ ConversionRequired NeedsConversion() const override {
+ return mDecoder->NeedsConversion();
+ }
+
+ private:
+ ~EMEDecryptor() = default;
+
+ RefPtr<MediaDataDecoder> mDecoder;
+ nsCOMPtr<nsISerialEventTarget> mThread;
+ RefPtr<CDMProxy> mProxy;
+ nsClassHashtable<nsRefPtrHashKey<MediaRawData>, DecryptPromiseRequestHolder>
+ mDecrypts;
+ RefPtr<SamplesWaitingForKey> mSamplesWaitingForKey;
+ MozPromiseRequestHolder<SamplesWaitingForKey::WaitForKeyPromise> mKeyRequest;
+ Maybe<DecryptThroughputLimit> mThroughputLimiter;
+ MozPromiseRequestHolder<DecryptThroughputLimit::ThrottlePromise>
+ mThrottleRequest;
+ MozPromiseHolder<DecodePromise> mDecodePromise;
+ MozPromiseHolder<DecodePromise> mDrainPromise;
+ MozPromiseHolder<FlushPromise> mFlushPromise;
+ MozPromiseRequestHolder<DecodePromise> mDecodeRequest;
+ UniquePtr<ADTSSampleConverter> mADTSSampleConverter;
+ bool mIsShutdown;
+};
+
+EMEMediaDataDecoderProxy::EMEMediaDataDecoderProxy(
+ const CreateDecoderParams& aParams,
+ already_AddRefed<MediaDataDecoder> aProxyDecoder,
+ already_AddRefed<nsISerialEventTarget> aProxyThread, CDMProxy* aProxy)
+ : MediaDataDecoderProxy(std::move(aProxyDecoder), std::move(aProxyThread)),
+ mThread(GetCurrentSerialEventTarget()),
+ mSamplesWaitingForKey(new SamplesWaitingForKey(
+ aProxy, aParams.mType, aParams.mOnWaitingForKeyEvent)),
+ mProxy(aProxy) {}
+
+EMEMediaDataDecoderProxy::EMEMediaDataDecoderProxy(
+ const CreateDecoderParams& aParams,
+ already_AddRefed<MediaDataDecoder> aProxyDecoder, CDMProxy* aProxy)
+ : MediaDataDecoderProxy(std::move(aProxyDecoder),
+ do_AddRef(GetCurrentSerialEventTarget())),
+ mThread(GetCurrentSerialEventTarget()),
+ mSamplesWaitingForKey(new SamplesWaitingForKey(
+ aProxy, aParams.mType, aParams.mOnWaitingForKeyEvent)),
+ mProxy(aProxy) {}
+
+RefPtr<MediaDataDecoder::DecodePromise> EMEMediaDataDecoderProxy::Decode(
+ MediaRawData* aSample) {
+ RefPtr<EMEMediaDataDecoderProxy> self = this;
+ RefPtr<MediaRawData> sample = aSample;
+ return InvokeAsync(mThread, __func__, [self, this, sample]() {
+ RefPtr<DecodePromise> p = mDecodePromise.Ensure(__func__);
+ mSamplesWaitingForKey->WaitIfKeyNotUsable(sample)
+ ->Then(
+ mThread, __func__,
+ [self, this](RefPtr<MediaRawData> aSample) {
+ mKeyRequest.Complete();
+
+ MediaDataDecoderProxy::Decode(aSample)
+ ->Then(mThread, __func__,
+ [self,
+ this](DecodePromise::ResolveOrRejectValue&& aValue) {
+ mDecodeRequest.Complete();
+ mDecodePromise.ResolveOrReject(std::move(aValue),
+ __func__);
+ })
+ ->Track(mDecodeRequest);
+ },
+ [self]() {
+ self->mKeyRequest.Complete();
+ MOZ_CRASH("Should never get here");
+ })
+ ->Track(mKeyRequest);
+
+ return p;
+ });
+}
+
+RefPtr<MediaDataDecoder::FlushPromise> EMEMediaDataDecoderProxy::Flush() {
+ RefPtr<EMEMediaDataDecoderProxy> self = this;
+ return InvokeAsync(mThread, __func__, [self, this]() {
+ mKeyRequest.DisconnectIfExists();
+ mDecodeRequest.DisconnectIfExists();
+ mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ return MediaDataDecoderProxy::Flush();
+ });
+}
+
+RefPtr<ShutdownPromise> EMEMediaDataDecoderProxy::Shutdown() {
+ RefPtr<EMEMediaDataDecoderProxy> self = this;
+ return InvokeAsync(mThread, __func__, [self, this]() {
+ mSamplesWaitingForKey->BreakCycles();
+ mSamplesWaitingForKey = nullptr;
+ mProxy = nullptr;
+ return MediaDataDecoderProxy::Shutdown();
+ });
+}
+
+EMEDecoderModule::EMEDecoderModule(CDMProxy* aProxy, PDMFactory* aPDM)
+ : mProxy(aProxy), mPDM(aPDM) {}
+
+EMEDecoderModule::~EMEDecoderModule() = default;
+
+static already_AddRefed<MediaDataDecoderProxy> CreateDecoderWrapper(
+ CDMProxy* aProxy, const CreateDecoderParams& aParams) {
+ RefPtr<gmp::GeckoMediaPluginService> s(
+ gmp::GeckoMediaPluginService::GetGeckoMediaPluginService());
+ if (!s) {
+ return nullptr;
+ }
+ nsCOMPtr<nsISerialEventTarget> thread(s->GetGMPThread());
+ if (!thread) {
+ return nullptr;
+ }
+ RefPtr<MediaDataDecoderProxy> decoder(
+ new EMEMediaDataDecoderProxy(aParams,
+ do_AddRef(new ChromiumCDMVideoDecoder(
+ GMPVideoDecoderParams(aParams), aProxy)),
+ thread.forget(), aProxy));
+ return decoder.forget();
+}
+
+RefPtr<EMEDecoderModule::CreateDecoderPromise>
+EMEDecoderModule::AsyncCreateDecoder(const CreateDecoderParams& aParams) {
+ MOZ_ASSERT(aParams.mConfig.mCrypto.IsEncrypted());
+ MOZ_ASSERT(mPDM);
+
+ if (aParams.mConfig.IsVideo()) {
+ if (StaticPrefs::media_eme_video_blank()) {
+ EME_LOG(
+ "EMEDecoderModule::CreateVideoDecoder() creating a blank decoder.");
+ RefPtr<PlatformDecoderModule> m(BlankDecoderModule::Create());
+ RefPtr<MediaDataDecoder> decoder = m->CreateVideoDecoder(aParams);
+ return EMEDecoderModule::CreateDecoderPromise::CreateAndResolve(decoder,
+ __func__);
+ }
+
+ if (!SupportsMimeType(aParams.mConfig.mMimeType, nullptr).isEmpty()) {
+ // GMP decodes. Assume that means it can decrypt too.
+ return EMEDecoderModule::CreateDecoderPromise::CreateAndResolve(
+ CreateDecoderWrapper(mProxy, aParams), __func__);
+ }
+
+ RefPtr<EMEDecoderModule::CreateDecoderPromise> p =
+ mPDM->CreateDecoder(aParams)->Then(
+ GetCurrentSerialEventTarget(), __func__,
+ [self = RefPtr{this},
+ params = CreateDecoderParamsForAsync(aParams)](
+ RefPtr<MediaDataDecoder>&& aDecoder) {
+ RefPtr<MediaDataDecoder> emeDecoder(
+ new EMEDecryptor(aDecoder, self->mProxy, params.mType,
+ params.mOnWaitingForKeyEvent));
+ return EMEDecoderModule::CreateDecoderPromise::CreateAndResolve(
+ emeDecoder, __func__);
+ },
+ [](const MediaResult& aError) {
+ return EMEDecoderModule::CreateDecoderPromise::CreateAndReject(
+ aError, __func__);
+ });
+ return p;
+ }
+
+ MOZ_ASSERT(aParams.mConfig.IsAudio());
+
+ // We don't support using the GMP to decode audio.
+ MOZ_ASSERT(SupportsMimeType(aParams.mConfig.mMimeType, nullptr).isEmpty());
+ MOZ_ASSERT(mPDM);
+
+ if (StaticPrefs::media_eme_audio_blank()) {
+ EME_LOG("EMEDecoderModule::CreateAudioDecoder() creating a blank decoder.");
+ RefPtr<PlatformDecoderModule> m(BlankDecoderModule::Create());
+ RefPtr<MediaDataDecoder> decoder = m->CreateAudioDecoder(aParams);
+ return EMEDecoderModule::CreateDecoderPromise::CreateAndResolve(decoder,
+ __func__);
+ }
+
+ UniquePtr<ADTSSampleConverter> converter = nullptr;
+ if (MP4Decoder::IsAAC(aParams.mConfig.mMimeType)) {
+ // The CDM expects encrypted AAC to be in ADTS format.
+ // See bug 1433344.
+ converter = MakeUnique<ADTSSampleConverter>(aParams.AudioConfig());
+ }
+
+ RefPtr<EMEDecoderModule::CreateDecoderPromise> p =
+ mPDM->CreateDecoder(aParams)->Then(
+ GetCurrentSerialEventTarget(), __func__,
+ [self = RefPtr{this}, params = CreateDecoderParamsForAsync(aParams),
+ converter = std::move(converter)](
+ RefPtr<MediaDataDecoder>&& aDecoder) mutable {
+ RefPtr<MediaDataDecoder> emeDecoder(new EMEDecryptor(
+ aDecoder, self->mProxy, params.mType,
+ params.mOnWaitingForKeyEvent, std::move(converter)));
+ return EMEDecoderModule::CreateDecoderPromise::CreateAndResolve(
+ emeDecoder, __func__);
+ },
+ [](const MediaResult& aError) {
+ return EMEDecoderModule::CreateDecoderPromise::CreateAndReject(
+ aError, __func__);
+ });
+ return p;
+}
+
+media::DecodeSupportSet EMEDecoderModule::SupportsMimeType(
+ const nsACString& aMimeType, DecoderDoctorDiagnostics* aDiagnostics) const {
+ Maybe<nsCString> keySystem;
+ keySystem.emplace(NS_ConvertUTF16toUTF8(mProxy->KeySystem()));
+ return GMPDecoderModule::SupportsMimeType(
+ aMimeType, nsLiteralCString(CHROMIUM_CDM_API), keySystem);
+}
+
+} // namespace mozilla
diff --git a/dom/media/platforms/agnostic/eme/EMEDecoderModule.h b/dom/media/platforms/agnostic/eme/EMEDecoderModule.h
new file mode 100644
index 0000000000..06fea6e650
--- /dev/null
+++ b/dom/media/platforms/agnostic/eme/EMEDecoderModule.h
@@ -0,0 +1,79 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#if !defined(EMEDecoderModule_h_)
+# define EMEDecoderModule_h_
+
+# include "MediaDataDecoderProxy.h"
+# include "PlatformDecoderModule.h"
+# include "SamplesWaitingForKey.h"
+
+namespace mozilla {
+
+class CDMProxy;
+class PDMFactory;
+
+class EMEDecoderModule : public PlatformDecoderModule {
+ public:
+ EMEDecoderModule(CDMProxy* aProxy, PDMFactory* aPDM);
+
+ protected:
+ RefPtr<CreateDecoderPromise> AsyncCreateDecoder(
+ const CreateDecoderParams& aParams) override;
+
+ // Decode thread.
+ already_AddRefed<MediaDataDecoder> CreateVideoDecoder(
+ const CreateDecoderParams& aParams) override {
+ MOZ_CRASH("Not used");
+ }
+
+ // Decode thread.
+ already_AddRefed<MediaDataDecoder> CreateAudioDecoder(
+ const CreateDecoderParams& aParams) override {
+ MOZ_CRASH("Not used");
+ }
+
+ media::DecodeSupportSet SupportsMimeType(
+ const nsACString& aMimeType,
+ DecoderDoctorDiagnostics* aDiagnostics) const override;
+
+ private:
+ virtual ~EMEDecoderModule();
+ RefPtr<CDMProxy> mProxy;
+ // Will be null if CDM has decoding capability.
+ RefPtr<PDMFactory> mPDM;
+};
+
+DDLoggedTypeDeclNameAndBase(EMEMediaDataDecoderProxy, MediaDataDecoderProxy);
+
+class EMEMediaDataDecoderProxy
+ : public MediaDataDecoderProxy,
+ public DecoderDoctorLifeLogger<EMEMediaDataDecoderProxy> {
+ public:
+ EMEMediaDataDecoderProxy(const CreateDecoderParams& aParams,
+ already_AddRefed<MediaDataDecoder> aProxyDecoder,
+ already_AddRefed<nsISerialEventTarget> aProxyThread,
+ CDMProxy* aProxy);
+ EMEMediaDataDecoderProxy(const CreateDecoderParams& aParams,
+ already_AddRefed<MediaDataDecoder> aProxyDecoder,
+ CDMProxy* aProxy);
+
+ RefPtr<DecodePromise> Decode(MediaRawData* aSample) override;
+ RefPtr<FlushPromise> Flush() override;
+ RefPtr<ShutdownPromise> Shutdown() override;
+
+ private:
+ nsCOMPtr<nsISerialEventTarget> mThread;
+ RefPtr<SamplesWaitingForKey> mSamplesWaitingForKey;
+ MozPromiseRequestHolder<SamplesWaitingForKey::WaitForKeyPromise> mKeyRequest;
+ MozPromiseHolder<DecodePromise> mDecodePromise;
+ MozPromiseRequestHolder<DecodePromise> mDecodeRequest;
+ RefPtr<CDMProxy> mProxy;
+};
+
+} // namespace mozilla
+
+#endif // EMEDecoderModule_h_
diff --git a/dom/media/platforms/agnostic/eme/SamplesWaitingForKey.cpp b/dom/media/platforms/agnostic/eme/SamplesWaitingForKey.cpp
new file mode 100644
index 0000000000..23d79ce56a
--- /dev/null
+++ b/dom/media/platforms/agnostic/eme/SamplesWaitingForKey.cpp
@@ -0,0 +1,79 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "SamplesWaitingForKey.h"
+
+#include "MediaData.h"
+#include "MediaEventSource.h"
+#include "mozilla/CDMCaps.h"
+#include "mozilla/CDMProxy.h"
+#include "mozilla/TaskQueue.h"
+
+namespace mozilla {
+
+SamplesWaitingForKey::SamplesWaitingForKey(
+ CDMProxy* aProxy, TrackInfo::TrackType aType,
+ const std::function<MediaEventProducer<TrackInfo::TrackType>*()>&
+ aOnWaitingForKeyEvent)
+ : mMutex("SamplesWaitingForKey"),
+ mProxy(aProxy),
+ mType(aType),
+ mOnWaitingForKeyEvent(aOnWaitingForKeyEvent) {}
+
+SamplesWaitingForKey::~SamplesWaitingForKey() { Flush(); }
+
+RefPtr<SamplesWaitingForKey::WaitForKeyPromise>
+SamplesWaitingForKey::WaitIfKeyNotUsable(MediaRawData* aSample) {
+ if (!aSample || !aSample->mCrypto.IsEncrypted() || !mProxy) {
+ return WaitForKeyPromise::CreateAndResolve(aSample, __func__);
+ }
+ auto caps = mProxy->Capabilites().Lock();
+ const auto& keyid = aSample->mCrypto.mKeyId;
+ if (caps->IsKeyUsable(keyid)) {
+ return WaitForKeyPromise::CreateAndResolve(aSample, __func__);
+ }
+ SampleEntry entry;
+ entry.mSample = aSample;
+ RefPtr<WaitForKeyPromise> p = entry.mPromise.Ensure(__func__);
+ {
+ MutexAutoLock lock(mMutex);
+ mSamples.AppendElement(std::move(entry));
+ }
+ if (mOnWaitingForKeyEvent && mOnWaitingForKeyEvent()) {
+ mOnWaitingForKeyEvent()->Notify(mType);
+ }
+ caps->NotifyWhenKeyIdUsable(aSample->mCrypto.mKeyId, this);
+ return p;
+}
+
+void SamplesWaitingForKey::NotifyUsable(const CencKeyId& aKeyId) {
+ MutexAutoLock lock(mMutex);
+ size_t i = 0;
+ while (i < mSamples.Length()) {
+ auto& entry = mSamples[i];
+ if (aKeyId == entry.mSample->mCrypto.mKeyId) {
+ entry.mPromise.Resolve(entry.mSample, __func__);
+ mSamples.RemoveElementAt(i);
+ } else {
+ i++;
+ }
+ }
+}
+
+void SamplesWaitingForKey::Flush() {
+ MutexAutoLock lock(mMutex);
+ for (auto& sample : mSamples) {
+ sample.mPromise.Reject(true, __func__);
+ }
+ mSamples.Clear();
+}
+
+void SamplesWaitingForKey::BreakCycles() {
+ MutexAutoLock lock(mMutex);
+ mProxy = nullptr;
+}
+
+} // namespace mozilla
diff --git a/dom/media/platforms/agnostic/eme/SamplesWaitingForKey.h b/dom/media/platforms/agnostic/eme/SamplesWaitingForKey.h
new file mode 100644
index 0000000000..06d72e3aae
--- /dev/null
+++ b/dom/media/platforms/agnostic/eme/SamplesWaitingForKey.h
@@ -0,0 +1,68 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef SamplesWaitingForKey_h_
+#define SamplesWaitingForKey_h_
+
+#include <functional>
+
+#include "MediaInfo.h"
+#include "mozilla/MozPromise.h"
+#include "mozilla/Mutex.h"
+#include "mozilla/RefPtr.h"
+
+namespace mozilla {
+
+typedef nsTArray<uint8_t> CencKeyId;
+
+class CDMProxy;
+template <typename... Es>
+class MediaEventProducer;
+class MediaRawData;
+
+// Encapsulates the task of waiting for the CDMProxy to have the necessary
+// keys to decrypt a given sample.
+class SamplesWaitingForKey {
+ public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(SamplesWaitingForKey)
+
+ typedef MozPromise<RefPtr<MediaRawData>, bool, /* IsExclusive = */ true>
+ WaitForKeyPromise;
+
+ SamplesWaitingForKey(
+ CDMProxy* aProxy, TrackInfo::TrackType aType,
+ const std::function<MediaEventProducer<TrackInfo::TrackType>*()>&
+ aOnWaitingForKeyEvent);
+
+ // Returns a promise that will be resolved if or when a key for decoding the
+ // sample becomes usable.
+ RefPtr<WaitForKeyPromise> WaitIfKeyNotUsable(MediaRawData* aSample);
+
+ void NotifyUsable(const CencKeyId& aKeyId);
+
+ void Flush();
+
+ void BreakCycles();
+
+ protected:
+ ~SamplesWaitingForKey();
+
+ private:
+ Mutex mMutex MOZ_UNANNOTATED;
+ RefPtr<CDMProxy> mProxy;
+ struct SampleEntry {
+ RefPtr<MediaRawData> mSample;
+ MozPromiseHolder<WaitForKeyPromise> mPromise;
+ };
+ nsTArray<SampleEntry> mSamples;
+ const TrackInfo::TrackType mType;
+ const std::function<MediaEventProducer<TrackInfo::TrackType>*()>
+ mOnWaitingForKeyEvent;
+};
+
+} // namespace mozilla
+
+#endif // SamplesWaitingForKey_h_
diff --git a/dom/media/platforms/agnostic/eme/moz.build b/dom/media/platforms/agnostic/eme/moz.build
new file mode 100644
index 0000000000..34f0007b3b
--- /dev/null
+++ b/dom/media/platforms/agnostic/eme/moz.build
@@ -0,0 +1,22 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+EXPORTS += [
+ "ChromiumCDMVideoDecoder.h",
+ "DecryptThroughputLimit.h",
+ "EMEDecoderModule.h",
+ "SamplesWaitingForKey.h",
+]
+
+UNIFIED_SOURCES += [
+ "ChromiumCDMVideoDecoder.cpp",
+ "EMEDecoderModule.cpp",
+ "SamplesWaitingForKey.cpp",
+]
+
+include("/ipc/chromium/chromium-config.mozbuild")
+
+FINAL_LIBRARY = "xul"
diff --git a/dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp b/dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp
new file mode 100644
index 0000000000..f01c7e94e4
--- /dev/null
+++ b/dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp
@@ -0,0 +1,94 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "GMPDecoderModule.h"
+
+#include "DecoderDoctorDiagnostics.h"
+#include "GMPService.h"
+#include "GMPUtils.h"
+#include "GMPVideoDecoder.h"
+#include "MP4Decoder.h"
+#include "MediaDataDecoderProxy.h"
+#include "VPXDecoder.h"
+#include "VideoUtils.h"
+#include "gmp-video-decode.h"
+#include "mozilla/StaticMutex.h"
+#include "nsServiceManagerUtils.h"
+#ifdef XP_WIN
+# include "WMFDecoderModule.h"
+#endif
+
+namespace mozilla {
+
+static already_AddRefed<MediaDataDecoderProxy> CreateDecoderWrapper(
+ GMPVideoDecoderParams&& aParams) {
+ RefPtr<gmp::GeckoMediaPluginService> s(
+ gmp::GeckoMediaPluginService::GetGeckoMediaPluginService());
+ if (!s) {
+ return nullptr;
+ }
+ nsCOMPtr<nsISerialEventTarget> thread(s->GetGMPThread());
+ if (!thread) {
+ return nullptr;
+ }
+
+ RefPtr<MediaDataDecoderProxy> decoder(new MediaDataDecoderProxy(
+ do_AddRef(new GMPVideoDecoder(std::move(aParams))), thread.forget()));
+ return decoder.forget();
+}
+
+already_AddRefed<MediaDataDecoder> GMPDecoderModule::CreateVideoDecoder(
+ const CreateDecoderParams& aParams) {
+ if (!MP4Decoder::IsH264(aParams.mConfig.mMimeType) &&
+ !VPXDecoder::IsVP8(aParams.mConfig.mMimeType) &&
+ !VPXDecoder::IsVP9(aParams.mConfig.mMimeType)) {
+ return nullptr;
+ }
+
+ return CreateDecoderWrapper(GMPVideoDecoderParams(aParams));
+}
+
+already_AddRefed<MediaDataDecoder> GMPDecoderModule::CreateAudioDecoder(
+ const CreateDecoderParams& aParams) {
+ return nullptr;
+}
+
+/* static */
+media::DecodeSupportSet GMPDecoderModule::SupportsMimeType(
+ const nsACString& aMimeType, const nsACString& aApi,
+ const Maybe<nsCString>& aKeySystem) {
+ AutoTArray<nsCString, 2> tags;
+ if (MP4Decoder::IsH264(aMimeType)) {
+ tags.AppendElement("h264"_ns);
+ } else if (VPXDecoder::IsVP9(aMimeType)) {
+ tags.AppendElement("vp9"_ns);
+ } else if (VPXDecoder::IsVP8(aMimeType)) {
+ tags.AppendElement("vp8"_ns);
+ } else {
+ return media::DecodeSupportSet{};
+ }
+
+ // Optional tag for EME GMP plugins.
+ if (aKeySystem) {
+ tags.AppendElement(*aKeySystem);
+ }
+
+ // GMP plugins are always software based.
+ return HaveGMPFor(aApi, tags) ? media::DecodeSupport::SoftwareDecode
+ : media::DecodeSupportSet{};
+}
+
+media::DecodeSupportSet GMPDecoderModule::SupportsMimeType(
+ const nsACString& aMimeType, DecoderDoctorDiagnostics* aDiagnostics) const {
+ return SupportsMimeType(aMimeType, "decode-video"_ns, Nothing());
+}
+
+/* static */
+already_AddRefed<PlatformDecoderModule> GMPDecoderModule::Create() {
+ return MakeAndAddRef<GMPDecoderModule>();
+}
+
+} // namespace mozilla
diff --git a/dom/media/platforms/agnostic/gmp/GMPDecoderModule.h b/dom/media/platforms/agnostic/gmp/GMPDecoderModule.h
new file mode 100644
index 0000000000..1a131dc154
--- /dev/null
+++ b/dom/media/platforms/agnostic/gmp/GMPDecoderModule.h
@@ -0,0 +1,58 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#if !defined(GMPDecoderModule_h_)
+# define GMPDecoderModule_h_
+
+# include "PlatformDecoderModule.h"
+# include "mozilla/Maybe.h"
+
+// The special NodeId we use when doing unencrypted decoding using the GMP's
+// decoder. This ensures that each GMP MediaDataDecoder we create doesn't
+// require spinning up a new process, but instead we run all instances of
+// GMP decoders in the one process, to reduce overhead.
+//
+// Note: GMP storage is isolated by NodeId, and non persistent for this
+// special NodeId, and the only way a GMP can communicate with the outside
+// world is through the EME GMP APIs, and we never run EME with this NodeID
+// (because NodeIds are random strings which can't contain the '-' character),
+// so there's no way a malicious GMP can harvest, store, and then report any
+// privacy sensitive data about what users are watching.
+# define SHARED_GMP_DECODING_NODE_ID "gmp-shared-decoding"_ns
+
+namespace mozilla {
+
+class GMPDecoderModule : public PlatformDecoderModule {
+ template <typename T, typename... Args>
+ friend already_AddRefed<T> MakeAndAddRef(Args&&...);
+
+ public:
+ static already_AddRefed<PlatformDecoderModule> Create();
+
+ // Decode thread.
+ already_AddRefed<MediaDataDecoder> CreateVideoDecoder(
+ const CreateDecoderParams& aParams) override;
+
+ // Decode thread.
+ already_AddRefed<MediaDataDecoder> CreateAudioDecoder(
+ const CreateDecoderParams& aParams) override;
+
+ media::DecodeSupportSet SupportsMimeType(
+ const nsACString& aMimeType,
+ DecoderDoctorDiagnostics* aDiagnostics) const override;
+
+ static media::DecodeSupportSet SupportsMimeType(
+ const nsACString& aMimeType, const nsACString& aApi,
+ const Maybe<nsCString>& aKeySystem);
+
+ private:
+ GMPDecoderModule() = default;
+ virtual ~GMPDecoderModule() = default;
+};
+
+} // namespace mozilla
+
+#endif // GMPDecoderModule_h_
diff --git a/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.cpp b/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.cpp
new file mode 100644
index 0000000000..b964036a4a
--- /dev/null
+++ b/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.cpp
@@ -0,0 +1,489 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "GMPVideoDecoder.h"
+#include "GMPDecoderModule.h"
+#include "GMPVideoHost.h"
+#include "GMPLog.h"
+#include "MediaData.h"
+#include "mozilla/EndianUtils.h"
+#include "mozilla/StaticPrefs_media.h"
+#include "nsServiceManagerUtils.h"
+#include "AnnexB.h"
+#include "H264.h"
+#include "MP4Decoder.h"
+#include "prsystem.h"
+#include "VPXDecoder.h"
+#include "VideoUtils.h"
+
+namespace mozilla {
+
+#if defined(DEBUG)
+static bool IsOnGMPThread() {
+ nsCOMPtr<mozIGeckoMediaPluginService> mps =
+ do_GetService("@mozilla.org/gecko-media-plugin-service;1");
+ MOZ_ASSERT(mps);
+
+ nsCOMPtr<nsIThread> gmpThread;
+ nsresult rv = mps->GetThread(getter_AddRefs(gmpThread));
+ MOZ_ASSERT(NS_SUCCEEDED(rv) && gmpThread);
+ return gmpThread->IsOnCurrentThread();
+}
+#endif
+
+GMPVideoDecoderParams::GMPVideoDecoderParams(const CreateDecoderParams& aParams)
+ : mConfig(aParams.VideoConfig()),
+ mImageContainer(aParams.mImageContainer),
+ mCrashHelper(aParams.mCrashHelper),
+ mKnowsCompositor(aParams.mKnowsCompositor),
+ mTrackingId(aParams.mTrackingId) {}
+
+nsCString GMPVideoDecoder::GetCodecName() const {
+ if (MP4Decoder::IsH264(mConfig.mMimeType)) {
+ return "h264"_ns;
+ } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) {
+ return "vp8"_ns;
+ } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) {
+ return "vp9"_ns;
+ }
+ return "unknown"_ns;
+}
+
+void GMPVideoDecoder::Decoded(GMPVideoi420Frame* aDecodedFrame) {
+ GMP_LOG_DEBUG("GMPVideoDecoder::Decoded");
+
+ GMPUniquePtr<GMPVideoi420Frame> decodedFrame(aDecodedFrame);
+ MOZ_ASSERT(IsOnGMPThread());
+
+ VideoData::YCbCrBuffer b;
+ for (int i = 0; i < kGMPNumOfPlanes; ++i) {
+ b.mPlanes[i].mData = decodedFrame->Buffer(GMPPlaneType(i));
+ b.mPlanes[i].mStride = decodedFrame->Stride(GMPPlaneType(i));
+ if (i == kGMPYPlane) {
+ b.mPlanes[i].mWidth = decodedFrame->Width();
+ b.mPlanes[i].mHeight = decodedFrame->Height();
+ } else {
+ b.mPlanes[i].mWidth = (decodedFrame->Width() + 1) / 2;
+ b.mPlanes[i].mHeight = (decodedFrame->Height() + 1) / 2;
+ }
+ b.mPlanes[i].mSkip = 0;
+ }
+
+ b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT;
+ b.mYUVColorSpace =
+ DefaultColorSpace({decodedFrame->Width(), decodedFrame->Height()});
+
+ UniquePtr<SampleMetadata> sampleData;
+ if (auto entryHandle = mSamples.Lookup(decodedFrame->Timestamp())) {
+ sampleData = std::move(entryHandle.Data());
+ entryHandle.Remove();
+ } else {
+ GMP_LOG_DEBUG(
+ "GMPVideoDecoder::Decoded(this=%p) missing sample metadata for "
+ "time %" PRIu64,
+ this, decodedFrame->Timestamp());
+ if (mSamples.IsEmpty()) {
+ // If we have no remaining samples in the table, then we have processed
+ // all outstanding decode requests.
+ ProcessReorderQueue(mDecodePromise, __func__);
+ }
+ return;
+ }
+
+ MOZ_ASSERT(sampleData);
+
+ gfx::IntRect pictureRegion(0, 0, decodedFrame->Width(),
+ decodedFrame->Height());
+ Result<already_AddRefed<VideoData>, MediaResult> r =
+ VideoData::CreateAndCopyData(
+ mConfig, mImageContainer, sampleData->mOffset,
+ media::TimeUnit::FromMicroseconds(decodedFrame->UpdatedTimestamp()),
+ media::TimeUnit::FromMicroseconds(decodedFrame->Duration()), b,
+ sampleData->mKeyframe, media::TimeUnit::FromMicroseconds(-1),
+ pictureRegion, mKnowsCompositor);
+ if (r.isErr()) {
+ mReorderQueue.Clear();
+ mUnorderedData.Clear();
+ mSamples.Clear();
+ mDecodePromise.RejectIfExists(r.unwrapErr(), __func__);
+ return;
+ }
+
+ RefPtr<VideoData> v = r.unwrap();
+ MOZ_ASSERT(v);
+ mPerformanceRecorder.Record(static_cast<int64_t>(decodedFrame->Timestamp()),
+ [&](DecodeStage& aStage) {
+ aStage.SetImageFormat(DecodeStage::YUV420P);
+ aStage.SetResolution(decodedFrame->Width(),
+ decodedFrame->Height());
+ aStage.SetYUVColorSpace(b.mYUVColorSpace);
+ aStage.SetColorDepth(b.mColorDepth);
+ aStage.SetColorRange(b.mColorRange);
+ });
+
+ if (mReorderFrames) {
+ mReorderQueue.Push(std::move(v));
+ } else {
+ mUnorderedData.AppendElement(std::move(v));
+ }
+
+ if (mSamples.IsEmpty()) {
+ // If we have no remaining samples in the table, then we have processed
+ // all outstanding decode requests.
+ ProcessReorderQueue(mDecodePromise, __func__);
+ }
+}
+
+void GMPVideoDecoder::ReceivedDecodedReferenceFrame(const uint64_t aPictureId) {
+ GMP_LOG_DEBUG("GMPVideoDecoder::ReceivedDecodedReferenceFrame");
+ MOZ_ASSERT(IsOnGMPThread());
+}
+
+void GMPVideoDecoder::ReceivedDecodedFrame(const uint64_t aPictureId) {
+ GMP_LOG_DEBUG("GMPVideoDecoder::ReceivedDecodedFrame");
+ MOZ_ASSERT(IsOnGMPThread());
+}
+
+void GMPVideoDecoder::InputDataExhausted() {
+ GMP_LOG_DEBUG("GMPVideoDecoder::InputDataExhausted");
+ MOZ_ASSERT(IsOnGMPThread());
+ mSamples.Clear();
+ ProcessReorderQueue(mDecodePromise, __func__);
+}
+
+void GMPVideoDecoder::DrainComplete() {
+ GMP_LOG_DEBUG("GMPVideoDecoder::DrainComplete");
+ MOZ_ASSERT(IsOnGMPThread());
+ mSamples.Clear();
+
+ if (mDrainPromise.IsEmpty()) {
+ return;
+ }
+
+ DecodedData results;
+ if (mReorderFrames) {
+ results.SetCapacity(mReorderQueue.Length());
+ while (!mReorderQueue.IsEmpty()) {
+ results.AppendElement(mReorderQueue.Pop());
+ }
+ } else {
+ results = std::move(mUnorderedData);
+ }
+
+ mDrainPromise.Resolve(std::move(results), __func__);
+}
+
+void GMPVideoDecoder::ResetComplete() {
+ GMP_LOG_DEBUG("GMPVideoDecoder::ResetComplete");
+ MOZ_ASSERT(IsOnGMPThread());
+ mPerformanceRecorder.Record(std::numeric_limits<int64_t>::max());
+ mFlushPromise.ResolveIfExists(true, __func__);
+}
+
+void GMPVideoDecoder::Error(GMPErr aErr) {
+ GMP_LOG_DEBUG("GMPVideoDecoder::Error");
+ MOZ_ASSERT(IsOnGMPThread());
+ auto error = MediaResult(aErr == GMPDecodeErr ? NS_ERROR_DOM_MEDIA_DECODE_ERR
+ : NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ RESULT_DETAIL("GMPErr:%x", aErr));
+ mDecodePromise.RejectIfExists(error, __func__);
+ mDrainPromise.RejectIfExists(error, __func__);
+ mFlushPromise.RejectIfExists(error, __func__);
+}
+
+void GMPVideoDecoder::Terminated() {
+ GMP_LOG_DEBUG("GMPVideoDecoder::Terminated");
+ MOZ_ASSERT(IsOnGMPThread());
+ Error(GMPErr::GMPAbortedErr);
+}
+
+void GMPVideoDecoder::ProcessReorderQueue(
+ MozPromiseHolder<DecodePromise>& aPromise, const char* aMethodName) {
+ if (aPromise.IsEmpty()) {
+ return;
+ }
+
+ if (!mReorderFrames) {
+ aPromise.Resolve(std::move(mUnorderedData), aMethodName);
+ return;
+ }
+
+ DecodedData results;
+ size_t availableFrames = mReorderQueue.Length();
+ if (availableFrames > mMaxRefFrames) {
+ size_t resolvedFrames = availableFrames - mMaxRefFrames;
+ results.SetCapacity(resolvedFrames);
+ do {
+ results.AppendElement(mReorderQueue.Pop());
+ } while (--resolvedFrames > 0);
+ }
+
+ aPromise.Resolve(std::move(results), aMethodName);
+}
+
+GMPVideoDecoder::GMPVideoDecoder(const GMPVideoDecoderParams& aParams)
+ : mConfig(aParams.mConfig),
+ mGMP(nullptr),
+ mHost(nullptr),
+ mConvertNALUnitLengths(false),
+ mCrashHelper(aParams.mCrashHelper),
+ mImageContainer(aParams.mImageContainer),
+ mKnowsCompositor(aParams.mKnowsCompositor),
+ mTrackingId(aParams.mTrackingId),
+ mCanDecodeBatch(StaticPrefs::media_gmp_decoder_decode_batch()),
+ mReorderFrames(StaticPrefs::media_gmp_decoder_reorder_frames()) {}
+
+void GMPVideoDecoder::InitTags(nsTArray<nsCString>& aTags) {
+ if (MP4Decoder::IsH264(mConfig.mMimeType)) {
+ aTags.AppendElement("h264"_ns);
+ } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) {
+ aTags.AppendElement("vp8"_ns);
+ } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) {
+ aTags.AppendElement("vp9"_ns);
+ }
+}
+
+nsCString GMPVideoDecoder::GetNodeId() { return SHARED_GMP_DECODING_NODE_ID; }
+
+GMPUniquePtr<GMPVideoEncodedFrame> GMPVideoDecoder::CreateFrame(
+ MediaRawData* aSample) {
+ GMPVideoFrame* ftmp = nullptr;
+ GMPErr err = mHost->CreateFrame(kGMPEncodedVideoFrame, &ftmp);
+ if (GMP_FAILED(err)) {
+ return nullptr;
+ }
+
+ GMPUniquePtr<GMPVideoEncodedFrame> frame(
+ static_cast<GMPVideoEncodedFrame*>(ftmp));
+ err = frame->CreateEmptyFrame(aSample->Size());
+ if (GMP_FAILED(err)) {
+ return nullptr;
+ }
+
+ memcpy(frame->Buffer(), aSample->Data(), frame->Size());
+
+ // Convert 4-byte NAL unit lengths to host-endian 4-byte buffer lengths to
+ // suit the GMP API.
+ if (mConvertNALUnitLengths) {
+ const int kNALLengthSize = 4;
+ uint8_t* buf = frame->Buffer();
+ while (buf < frame->Buffer() + frame->Size() - kNALLengthSize) {
+ uint32_t length = BigEndian::readUint32(buf) + kNALLengthSize;
+ *reinterpret_cast<uint32_t*>(buf) = length;
+ buf += length;
+ }
+ }
+
+ frame->SetBufferType(GMP_BufferLength32);
+
+ frame->SetEncodedWidth(mConfig.mDisplay.width);
+ frame->SetEncodedHeight(mConfig.mDisplay.height);
+ frame->SetTimeStamp(aSample->mTime.ToMicroseconds());
+ frame->SetCompleteFrame(true);
+ frame->SetDuration(aSample->mDuration.ToMicroseconds());
+ frame->SetFrameType(aSample->mKeyframe ? kGMPKeyFrame : kGMPDeltaFrame);
+
+ return frame;
+}
+
+const VideoInfo& GMPVideoDecoder::GetConfig() const { return mConfig; }
+
+void GMPVideoDecoder::GMPInitDone(GMPVideoDecoderProxy* aGMP,
+ GMPVideoHost* aHost) {
+ MOZ_ASSERT(IsOnGMPThread());
+
+ if (!aGMP) {
+ mInitPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
+ return;
+ }
+ MOZ_ASSERT(aHost);
+
+ if (mInitPromise.IsEmpty()) {
+ // GMP must have been shutdown while we were waiting for Init operation
+ // to complete.
+ aGMP->Close();
+ return;
+ }
+
+ bool isOpenH264 = aGMP->GetPluginType() == GMPPluginType::OpenH264;
+
+ GMPVideoCodec codec;
+ memset(&codec, 0, sizeof(codec));
+
+ codec.mGMPApiVersion = kGMPVersion34;
+ nsTArray<uint8_t> codecSpecific;
+ if (MP4Decoder::IsH264(mConfig.mMimeType)) {
+ codec.mCodecType = kGMPVideoCodecH264;
+ codecSpecific.AppendElement(0); // mPacketizationMode.
+ codecSpecific.AppendElements(mConfig.mExtraData->Elements(),
+ mConfig.mExtraData->Length());
+ // OpenH264 expects pseudo-AVCC, but others must be passed
+ // AnnexB for H264.
+ mConvertToAnnexB = !isOpenH264;
+ mMaxRefFrames = H264::ComputeMaxRefFrames(mConfig.mExtraData);
+ } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) {
+ codec.mCodecType = kGMPVideoCodecVP8;
+ } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) {
+ codec.mCodecType = kGMPVideoCodecVP9;
+ } else {
+ // Unrecognized mime type
+ aGMP->Close();
+ mInitPromise.Reject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
+ return;
+ }
+ codec.mWidth = mConfig.mImage.width;
+ codec.mHeight = mConfig.mImage.height;
+ codec.mUseThreadedDecode = StaticPrefs::media_gmp_decoder_multithreaded();
+ codec.mLogLevel = GetGMPLibraryLogLevel();
+
+ nsresult rv =
+ aGMP->InitDecode(codec, codecSpecific, this, PR_GetNumberOfProcessors());
+ if (NS_FAILED(rv)) {
+ aGMP->Close();
+ mInitPromise.Reject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
+ return;
+ }
+
+ mGMP = aGMP;
+ mHost = aHost;
+
+ // GMP implementations have interpreted the meaning of GMP_BufferLength32
+ // differently. The OpenH264 GMP expects GMP_BufferLength32 to behave as
+ // specified in the GMP API, where each buffer is prefixed by a 32-bit
+ // host-endian buffer length that includes the size of the buffer length
+ // field. Other existing GMPs currently expect GMP_BufferLength32 (when
+ // combined with kGMPVideoCodecH264) to mean "like AVCC but restricted to
+ // 4-byte NAL lengths" (i.e. buffer lengths are specified in big-endian
+ // and do not include the length of the buffer length field.
+ mConvertNALUnitLengths = isOpenH264;
+
+ mInitPromise.Resolve(TrackInfo::kVideoTrack, __func__);
+}
+
+RefPtr<MediaDataDecoder::InitPromise> GMPVideoDecoder::Init() {
+ MOZ_ASSERT(IsOnGMPThread());
+
+ mMPS = do_GetService("@mozilla.org/gecko-media-plugin-service;1");
+ MOZ_ASSERT(mMPS);
+
+ RefPtr<InitPromise> promise(mInitPromise.Ensure(__func__));
+
+ nsTArray<nsCString> tags;
+ InitTags(tags);
+ UniquePtr<GetGMPVideoDecoderCallback> callback(new GMPInitDoneCallback(this));
+ if (NS_FAILED(mMPS->GetGMPVideoDecoder(mCrashHelper, &tags, GetNodeId(),
+ std::move(callback)))) {
+ mInitPromise.Reject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
+ }
+
+ return promise;
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> GMPVideoDecoder::Decode(
+ MediaRawData* aSample) {
+ MOZ_ASSERT(IsOnGMPThread());
+
+ RefPtr<MediaRawData> sample(aSample);
+ if (!mGMP) {
+ return DecodePromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ RESULT_DETAIL("mGMP not initialized")),
+ __func__);
+ }
+
+ if (mTrackingId) {
+ MediaInfoFlag flag = MediaInfoFlag::None;
+ flag |= (aSample->mKeyframe ? MediaInfoFlag::KeyFrame
+ : MediaInfoFlag::NonKeyFrame);
+ if (mGMP->GetPluginType() == GMPPluginType::OpenH264) {
+ flag |= MediaInfoFlag::SoftwareDecoding;
+ }
+ if (MP4Decoder::IsH264(mConfig.mMimeType)) {
+ flag |= MediaInfoFlag::VIDEO_H264;
+ } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) {
+ flag |= MediaInfoFlag::VIDEO_VP8;
+ } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) {
+ flag |= MediaInfoFlag::VIDEO_VP9;
+ }
+ mPerformanceRecorder.Start(aSample->mTime.ToMicroseconds(),
+ "GMPVideoDecoder"_ns, *mTrackingId, flag);
+ }
+
+ GMPUniquePtr<GMPVideoEncodedFrame> frame = CreateFrame(sample);
+ if (!frame) {
+ return DecodePromise::CreateAndReject(
+ MediaResult(NS_ERROR_OUT_OF_MEMORY,
+ RESULT_DETAIL("CreateFrame returned null")),
+ __func__);
+ }
+
+ uint64_t frameTimestamp = frame->TimeStamp();
+ RefPtr<DecodePromise> p = mDecodePromise.Ensure(__func__);
+ nsTArray<uint8_t> info; // No codec specific per-frame info to pass.
+ nsresult rv = mGMP->Decode(std::move(frame), false, info, 0);
+ if (NS_FAILED(rv)) {
+ mDecodePromise.Reject(MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
+ RESULT_DETAIL("mGMP->Decode:%" PRIx32,
+ static_cast<uint32_t>(rv))),
+ __func__);
+ }
+
+ // If we have multiple outstanding frames, we need to track which offset
+ // belongs to which frame. During seek, it is possible to get the same frame
+ // requested twice, if the old frame is still outstanding. We will simply drop
+ // the extra decoded frame and request more input if the last outstanding.
+ mSamples.WithEntryHandle(frameTimestamp, [&](auto entryHandle) {
+ auto sampleData = MakeUnique<SampleMetadata>(sample);
+ entryHandle.InsertOrUpdate(std::move(sampleData));
+ });
+
+ return p;
+}
+
+RefPtr<MediaDataDecoder::FlushPromise> GMPVideoDecoder::Flush() {
+ MOZ_ASSERT(IsOnGMPThread());
+
+ mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ mDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+
+ RefPtr<FlushPromise> p = mFlushPromise.Ensure(__func__);
+ if (!mGMP || NS_FAILED(mGMP->Reset())) {
+ // Abort the flush.
+ mPerformanceRecorder.Record(std::numeric_limits<int64_t>::max());
+ mFlushPromise.Resolve(true, __func__);
+ }
+ return p;
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> GMPVideoDecoder::Drain() {
+ MOZ_ASSERT(IsOnGMPThread());
+
+ MOZ_ASSERT(mDecodePromise.IsEmpty(), "Must wait for decoding to complete");
+
+ RefPtr<DecodePromise> p = mDrainPromise.Ensure(__func__);
+ if (!mGMP || NS_FAILED(mGMP->Drain())) {
+ mDrainPromise.Resolve(DecodedData(), __func__);
+ }
+
+ return p;
+}
+
+RefPtr<ShutdownPromise> GMPVideoDecoder::Shutdown() {
+ mInitPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ mFlushPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+
+ // Note that this *may* be called from the proxy thread also.
+ // TODO: If that's the case, then this code is racy.
+ if (!mGMP) {
+ return ShutdownPromise::CreateAndResolve(true, __func__);
+ }
+ // Note this unblocks flush and drain operations waiting for callbacks.
+ mGMP->Close();
+ mGMP = nullptr;
+ return ShutdownPromise::CreateAndResolve(true, __func__);
+}
+
+} // namespace mozilla
diff --git a/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.h b/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.h
new file mode 100644
index 0000000000..1f0f59c685
--- /dev/null
+++ b/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.h
@@ -0,0 +1,129 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/layers/KnowsCompositor.h"
+#if !defined(GMPVideoDecoder_h_)
+# define GMPVideoDecoder_h_
+
+# include "GMPVideoDecoderProxy.h"
+# include "ImageContainer.h"
+# include "MediaDataDecoderProxy.h"
+# include "MediaInfo.h"
+# include "PerformanceRecorder.h"
+# include "PlatformDecoderModule.h"
+# include "ReorderQueue.h"
+# include "mozIGeckoMediaPluginService.h"
+# include "nsClassHashtable.h"
+
+namespace mozilla {
+
+struct MOZ_STACK_CLASS GMPVideoDecoderParams {
+ explicit GMPVideoDecoderParams(const CreateDecoderParams& aParams);
+
+ const VideoInfo& mConfig;
+ layers::ImageContainer* mImageContainer;
+ GMPCrashHelper* mCrashHelper;
+ layers::KnowsCompositor* mKnowsCompositor;
+ const Maybe<TrackingId> mTrackingId;
+};
+
+DDLoggedTypeDeclNameAndBase(GMPVideoDecoder, MediaDataDecoder);
+
+class GMPVideoDecoder final : public MediaDataDecoder,
+ public GMPVideoDecoderCallbackProxy,
+ public DecoderDoctorLifeLogger<GMPVideoDecoder> {
+ public:
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(GMPVideoDecoder, final);
+
+ explicit GMPVideoDecoder(const GMPVideoDecoderParams& aParams);
+
+ RefPtr<InitPromise> Init() override;
+ RefPtr<DecodePromise> Decode(MediaRawData* aSample) override;
+ RefPtr<DecodePromise> Drain() override;
+ RefPtr<FlushPromise> Flush() override;
+ RefPtr<ShutdownPromise> Shutdown() override;
+ nsCString GetDescriptionName() const override {
+ return "gmp video decoder"_ns;
+ }
+ nsCString GetCodecName() const override;
+ ConversionRequired NeedsConversion() const override {
+ return mConvertToAnnexB ? ConversionRequired::kNeedAnnexB
+ : ConversionRequired::kNeedAVCC;
+ }
+ bool CanDecodeBatch() const override { return mCanDecodeBatch; }
+
+ // GMPVideoDecoderCallbackProxy
+ // All those methods are called on the GMP thread.
+ void Decoded(GMPVideoi420Frame* aDecodedFrame) override;
+ void ReceivedDecodedReferenceFrame(const uint64_t aPictureId) override;
+ void ReceivedDecodedFrame(const uint64_t aPictureId) override;
+ void InputDataExhausted() override;
+ void DrainComplete() override;
+ void ResetComplete() override;
+ void Error(GMPErr aErr) override;
+ void Terminated() override;
+
+ protected:
+ virtual void InitTags(nsTArray<nsCString>& aTags);
+ virtual nsCString GetNodeId();
+ virtual GMPUniquePtr<GMPVideoEncodedFrame> CreateFrame(MediaRawData* aSample);
+ virtual const VideoInfo& GetConfig() const;
+ void ProcessReorderQueue(MozPromiseHolder<DecodePromise>& aPromise,
+ const char* aMethodName);
+
+ private:
+ ~GMPVideoDecoder() = default;
+
+ class GMPInitDoneCallback : public GetGMPVideoDecoderCallback {
+ public:
+ explicit GMPInitDoneCallback(GMPVideoDecoder* aDecoder)
+ : mDecoder(aDecoder) {}
+
+ void Done(GMPVideoDecoderProxy* aGMP, GMPVideoHost* aHost) override {
+ mDecoder->GMPInitDone(aGMP, aHost);
+ }
+
+ private:
+ RefPtr<GMPVideoDecoder> mDecoder;
+ };
+ void GMPInitDone(GMPVideoDecoderProxy* aGMP, GMPVideoHost* aHost);
+
+ const VideoInfo mConfig;
+ nsCOMPtr<mozIGeckoMediaPluginService> mMPS;
+ GMPVideoDecoderProxy* mGMP;
+ GMPVideoHost* mHost;
+ bool mConvertNALUnitLengths;
+ MozPromiseHolder<InitPromise> mInitPromise;
+ RefPtr<GMPCrashHelper> mCrashHelper;
+
+ struct SampleMetadata {
+ explicit SampleMetadata(MediaRawData* aSample)
+ : mOffset(aSample->mOffset), mKeyframe(aSample->mKeyframe) {}
+ int64_t mOffset;
+ bool mKeyframe;
+ };
+
+ nsClassHashtable<nsUint64HashKey, SampleMetadata> mSamples;
+ RefPtr<layers::ImageContainer> mImageContainer;
+ RefPtr<layers::KnowsCompositor> mKnowsCompositor;
+ PerformanceRecorderMulti<DecodeStage> mPerformanceRecorder;
+ const Maybe<TrackingId> mTrackingId;
+
+ uint32_t mMaxRefFrames = 0;
+ ReorderQueue mReorderQueue;
+ DecodedData mUnorderedData;
+
+ MozPromiseHolder<DecodePromise> mDecodePromise;
+ MozPromiseHolder<DecodePromise> mDrainPromise;
+ MozPromiseHolder<FlushPromise> mFlushPromise;
+ bool mConvertToAnnexB = false;
+ bool mCanDecodeBatch = false;
+ bool mReorderFrames = true;
+};
+
+} // namespace mozilla
+
+#endif // GMPVideoDecoder_h_
diff --git a/dom/media/platforms/agnostic/gmp/moz.build b/dom/media/platforms/agnostic/gmp/moz.build
new file mode 100644
index 0000000000..8e49f4a0e8
--- /dev/null
+++ b/dom/media/platforms/agnostic/gmp/moz.build
@@ -0,0 +1,24 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+LOCAL_INCLUDES += [
+ "/dom/media/gmp", # for GMPLog.h,
+]
+
+EXPORTS += [
+ "GMPDecoderModule.h",
+ "GMPVideoDecoder.h",
+]
+
+UNIFIED_SOURCES += [
+ "GMPDecoderModule.cpp",
+ "GMPVideoDecoder.cpp",
+]
+
+# GMPVideoEncodedFrameImpl.h needs IPC
+include("/ipc/chromium/chromium-config.mozbuild")
+
+FINAL_LIBRARY = "xul"