diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
commit | 26a029d407be480d791972afb5975cf62c9360a6 (patch) | |
tree | f435a8308119effd964b339f76abb83a57c29483 /dom/media/platforms/agnostic | |
parent | Initial commit. (diff) | |
download | firefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz firefox-26a029d407be480d791972afb5975cf62c9360a6.zip |
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/media/platforms/agnostic')
40 files changed, 10721 insertions, 0 deletions
diff --git a/dom/media/platforms/agnostic/AOMDecoder.cpp b/dom/media/platforms/agnostic/AOMDecoder.cpp new file mode 100644 index 0000000000..cb7f784848 --- /dev/null +++ b/dom/media/platforms/agnostic/AOMDecoder.cpp @@ -0,0 +1,1073 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "AOMDecoder.h" + +#include <algorithm> + +#include "BitWriter.h" +#include "BitReader.h" +#include "ImageContainer.h" +#include "MediaResult.h" +#include "TimeUnits.h" +#include <aom/aom_image.h> +#include <aom/aomdx.h> +#include "gfx2DGlue.h" +#include "gfxUtils.h" +#include "mozilla/PodOperations.h" +#include "mozilla/SyncRunnable.h" +#include "mozilla/TaskQueue.h" +#include "nsError.h" +#include "nsThreadUtils.h" +#include "prsystem.h" +#include "VideoUtils.h" + +#undef LOG +#define LOG(arg, ...) \ + DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, __func__, \ + ##__VA_ARGS__) +#define LOG_RESULT(code, message, ...) \ + DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, "::%s: %s (code %d) " message, \ + __func__, aom_codec_err_to_string(code), (int)code, ##__VA_ARGS__) +#define LOGEX_RESULT(_this, code, message, ...) \ + DDMOZ_LOGEX(_this, sPDMLog, mozilla::LogLevel::Debug, \ + "::%s: %s (code %d) " message, __func__, \ + aom_codec_err_to_string(code), (int)code, ##__VA_ARGS__) +#define LOG_STATIC_RESULT(code, message, ...) \ + MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, \ + ("AOMDecoder::%s: %s (code %d) " message, __func__, \ + aom_codec_err_to_string(code), (int)code, ##__VA_ARGS__)) + +#define ASSERT_BYTE_ALIGNED(bitIO) MOZ_ASSERT((bitIO).BitCount() % 8 == 0) + +namespace mozilla { + +using namespace gfx; +using namespace layers; +using gfx::CICP::ColourPrimaries; +using gfx::CICP::MatrixCoefficients; +using gfx::CICP::TransferCharacteristics; + +static MediaResult InitContext(AOMDecoder& aAOMDecoder, aom_codec_ctx_t* aCtx, + const VideoInfo& aInfo) { + aom_codec_iface_t* dx = aom_codec_av1_dx(); + if (!dx) { + return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("Couldn't get AV1 decoder interface.")); + } + + size_t decode_threads = 2; + if (aInfo.mDisplay.width >= 2048) { + decode_threads = 8; + } else if (aInfo.mDisplay.width >= 1024) { + decode_threads = 4; + } + decode_threads = std::min(decode_threads, GetNumberOfProcessors()); + + aom_codec_dec_cfg_t config; + PodZero(&config); + config.threads = static_cast<unsigned int>(decode_threads); + config.w = config.h = 0; // set after decode + config.allow_lowbitdepth = true; + + aom_codec_flags_t flags = 0; + + auto res = aom_codec_dec_init(aCtx, dx, &config, flags); + if (res != AOM_CODEC_OK) { + LOGEX_RESULT(&aAOMDecoder, res, "Codec initialization failed, res=%d", + int(res)); + return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("AOM error initializing AV1 decoder: %s", + aom_codec_err_to_string(res))); + } + return NS_OK; +} + +AOMDecoder::AOMDecoder(const CreateDecoderParams& aParams) + : mImageContainer(aParams.mImageContainer), + mTaskQueue(TaskQueue::Create( + GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER), "AOMDecoder")), + mInfo(aParams.VideoConfig()), + mTrackingId(aParams.mTrackingId) { + PodZero(&mCodec); +} + +AOMDecoder::~AOMDecoder() = default; + +RefPtr<ShutdownPromise> AOMDecoder::Shutdown() { + RefPtr<AOMDecoder> self = this; + return InvokeAsync(mTaskQueue, __func__, [self]() { + auto res = aom_codec_destroy(&self->mCodec); + if (res != AOM_CODEC_OK) { + LOGEX_RESULT(self.get(), res, "aom_codec_destroy"); + } + return self->mTaskQueue->BeginShutdown(); + }); +} + +RefPtr<MediaDataDecoder::InitPromise> AOMDecoder::Init() { + MediaResult rv = InitContext(*this, &mCodec, mInfo); + if (NS_FAILED(rv)) { + return AOMDecoder::InitPromise::CreateAndReject(rv, __func__); + } + return AOMDecoder::InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, + __func__); +} + +RefPtr<MediaDataDecoder::FlushPromise> AOMDecoder::Flush() { + return InvokeAsync(mTaskQueue, __func__, [this, self = RefPtr(this)]() { + mPerformanceRecorder.Record(std::numeric_limits<int64_t>::max()); + return FlushPromise::CreateAndResolve(true, __func__); + }); +} + +// UniquePtr dtor wrapper for aom_image_t. +struct AomImageFree { + void operator()(aom_image_t* img) { aom_img_free(img); } +}; + +RefPtr<MediaDataDecoder::DecodePromise> AOMDecoder::ProcessDecode( + MediaRawData* aSample) { + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + +#if defined(DEBUG) + NS_ASSERTION( + IsKeyframe(*aSample) == aSample->mKeyframe, + "AOM Decode Keyframe error sample->mKeyframe and si.si_kf out of sync"); +#endif + + MediaInfoFlag flag = MediaInfoFlag::None; + flag |= (aSample->mKeyframe ? MediaInfoFlag::KeyFrame + : MediaInfoFlag::NonKeyFrame); + flag |= MediaInfoFlag::SoftwareDecoding; + flag |= MediaInfoFlag::VIDEO_AV1; + + mTrackingId.apply([&](const auto& aId) { + mPerformanceRecorder.Start(aSample->mTimecode.ToMicroseconds(), + "AOMDecoder"_ns, aId, flag); + }); + + if (aom_codec_err_t r = aom_codec_decode(&mCodec, aSample->Data(), + aSample->Size(), nullptr)) { + LOG_RESULT(r, "Decode error!"); + return DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, + RESULT_DETAIL("AOM error decoding AV1 sample: %s", + aom_codec_err_to_string(r))), + __func__); + } + + aom_codec_iter_t iter = nullptr; + aom_image_t* img; + UniquePtr<aom_image_t, AomImageFree> img8; + DecodedData results; + + while ((img = aom_codec_get_frame(&mCodec, &iter))) { + NS_ASSERTION( + img->fmt == AOM_IMG_FMT_I420 || img->fmt == AOM_IMG_FMT_I42016 || + img->fmt == AOM_IMG_FMT_I444 || img->fmt == AOM_IMG_FMT_I44416, + "AV1 image format not I420 or I444"); + + // Chroma shifts are rounded down as per the decoding examples in the SDK + VideoData::YCbCrBuffer b; + b.mPlanes[0].mData = img->planes[0]; + b.mPlanes[0].mStride = img->stride[0]; + b.mPlanes[0].mHeight = img->d_h; + b.mPlanes[0].mWidth = img->d_w; + b.mPlanes[0].mSkip = 0; + + b.mPlanes[1].mData = img->planes[1]; + b.mPlanes[1].mStride = img->stride[1]; + b.mPlanes[1].mSkip = 0; + + b.mPlanes[2].mData = img->planes[2]; + b.mPlanes[2].mStride = img->stride[2]; + b.mPlanes[2].mSkip = 0; + + if (img->fmt == AOM_IMG_FMT_I420 || img->fmt == AOM_IMG_FMT_I42016) { + b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT; + + b.mPlanes[1].mHeight = (img->d_h + 1) >> img->y_chroma_shift; + b.mPlanes[1].mWidth = (img->d_w + 1) >> img->x_chroma_shift; + + b.mPlanes[2].mHeight = (img->d_h + 1) >> img->y_chroma_shift; + b.mPlanes[2].mWidth = (img->d_w + 1) >> img->x_chroma_shift; + } else if (img->fmt == AOM_IMG_FMT_I444 || img->fmt == AOM_IMG_FMT_I44416) { + b.mPlanes[1].mHeight = img->d_h; + b.mPlanes[1].mWidth = img->d_w; + + b.mPlanes[2].mHeight = img->d_h; + b.mPlanes[2].mWidth = img->d_w; + } else { + LOG("AOM Unknown image format"); + return DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, + RESULT_DETAIL("AOM Unknown image format")), + __func__); + } + + if (img->bit_depth == 10) { + b.mColorDepth = ColorDepth::COLOR_10; + } else if (img->bit_depth == 12) { + b.mColorDepth = ColorDepth::COLOR_12; + } + + switch (img->mc) { + case AOM_CICP_MC_BT_601: + b.mYUVColorSpace = YUVColorSpace::BT601; + break; + case AOM_CICP_MC_BT_2020_NCL: + case AOM_CICP_MC_BT_2020_CL: + b.mYUVColorSpace = YUVColorSpace::BT2020; + break; + case AOM_CICP_MC_BT_709: + b.mYUVColorSpace = YUVColorSpace::BT709; + break; + default: + b.mYUVColorSpace = DefaultColorSpace({img->d_w, img->d_h}); + break; + } + b.mColorRange = img->range == AOM_CR_FULL_RANGE ? ColorRange::FULL + : ColorRange::LIMITED; + + switch (img->cp) { + case AOM_CICP_CP_BT_709: + b.mColorPrimaries = ColorSpace2::BT709; + break; + case AOM_CICP_CP_BT_2020: + b.mColorPrimaries = ColorSpace2::BT2020; + break; + default: + b.mColorPrimaries = ColorSpace2::BT709; + break; + } + + Result<already_AddRefed<VideoData>, MediaResult> r = + VideoData::CreateAndCopyData( + mInfo, mImageContainer, aSample->mOffset, aSample->mTime, + aSample->mDuration, b, aSample->mKeyframe, aSample->mTimecode, + mInfo.ScaledImageRect(img->d_w, img->d_h), nullptr); + + if (r.isErr()) { + MediaResult rs = r.unwrapErr(); + LOG("VideoData::CreateAndCopyData error (source %ux%u display %ux%u " + "picture %ux%u) - %s: %s", + img->d_w, img->d_h, mInfo.mDisplay.width, mInfo.mDisplay.height, + mInfo.mImage.width, mInfo.mImage.height, rs.ErrorName().get(), + rs.Message().get()); + + return DecodePromise::CreateAndReject(std::move(rs), __func__); + } + + RefPtr<VideoData> v = r.unwrap(); + MOZ_ASSERT(v); + + mPerformanceRecorder.Record( + aSample->mTimecode.ToMicroseconds(), [&](DecodeStage& aStage) { + aStage.SetResolution(mInfo.mImage.width, mInfo.mImage.height); + auto format = [&]() -> Maybe<DecodeStage::ImageFormat> { + switch (img->fmt) { + case AOM_IMG_FMT_I420: + case AOM_IMG_FMT_I42016: + return Some(DecodeStage::YUV420P); + case AOM_IMG_FMT_I444: + case AOM_IMG_FMT_I44416: + return Some(DecodeStage::YUV444P); + default: + return Nothing(); + } + }(); + format.apply([&](auto& aFmt) { aStage.SetImageFormat(aFmt); }); + aStage.SetYUVColorSpace(b.mYUVColorSpace); + aStage.SetColorRange(b.mColorRange); + aStage.SetColorDepth(b.mColorDepth); + }); + results.AppendElement(std::move(v)); + } + return DecodePromise::CreateAndResolve(std::move(results), __func__); +} + +RefPtr<MediaDataDecoder::DecodePromise> AOMDecoder::Decode( + MediaRawData* aSample) { + return InvokeAsync<MediaRawData*>(mTaskQueue, this, __func__, + &AOMDecoder::ProcessDecode, aSample); +} + +RefPtr<MediaDataDecoder::DecodePromise> AOMDecoder::Drain() { + return InvokeAsync(mTaskQueue, __func__, [] { + return DecodePromise::CreateAndResolve(DecodedData(), __func__); + }); +} + +/* static */ +bool AOMDecoder::IsAV1(const nsACString& aMimeType) { + return aMimeType.EqualsLiteral("video/av1"); +} + +/* static */ +bool AOMDecoder::IsKeyframe(Span<const uint8_t> aBuffer) { + aom_codec_stream_info_t info; + PodZero(&info); + + auto res = aom_codec_peek_stream_info(aom_codec_av1_dx(), aBuffer.Elements(), + aBuffer.Length(), &info); + if (res != AOM_CODEC_OK) { + LOG_STATIC_RESULT( + res, "couldn't get keyframe flag with aom_codec_peek_stream_info"); + return false; + } + + return bool(info.is_kf); +} + +/* static */ +gfx::IntSize AOMDecoder::GetFrameSize(Span<const uint8_t> aBuffer) { + aom_codec_stream_info_t info; + PodZero(&info); + + auto res = aom_codec_peek_stream_info(aom_codec_av1_dx(), aBuffer.Elements(), + aBuffer.Length(), &info); + if (res != AOM_CODEC_OK) { + LOG_STATIC_RESULT( + res, "couldn't get frame size with aom_codec_peek_stream_info"); + } + + return gfx::IntSize(info.w, info.h); +} + +/* static */ +AOMDecoder::OBUIterator AOMDecoder::ReadOBUs(const Span<const uint8_t>& aData) { + return OBUIterator(aData); +} + +void AOMDecoder::OBUIterator::UpdateNext() { + // If mGoNext is not set, we don't need to load a new OBU. + if (!mGoNext) { + return; + } + // Check if we've reached the end of the data. Allow mGoNext to stay true so + // that HasNext() will return false. + if (mPosition >= mData.Length()) { + return; + } + mGoNext = false; + + // If retrieving the next OBU fails, reset the current OBU and set the + // position past the end of the data so that HasNext() returns false. + auto resetExit = MakeScopeExit([&]() { + mCurrent = OBUInfo(); + mPosition = mData.Length(); + }); + + auto subspan = mData.Subspan(mPosition, mData.Length() - mPosition); + BitReader br(subspan.Elements(), subspan.Length() * 8); + OBUInfo temp; + + // AV1 spec available at: + // https://aomediacodec.github.io/av1-spec/ + // or https://aomediacodec.github.io/av1-spec/av1-spec.pdf + + // begin open_bitstream_unit( ) + // https://aomediacodec.github.io/av1-spec/#general-obu-syntax + + // begin obu_header( ) + // https://aomediacodec.github.io/av1-spec/#obu-header-syntax + br.ReadBit(); // obu_forbidden_bit + temp.mType = static_cast<OBUType>(br.ReadBits(4)); + if (!temp.IsValid()) { + // Non-fatal error, unknown OBUs can be skipped as long as the size field + // is properly specified. + NS_WARNING(nsPrintfCString("Encountered unknown OBU type (%" PRIu8 + ", OBU may be invalid", + static_cast<uint8_t>(temp.mType)) + .get()); + } + temp.mExtensionFlag = br.ReadBit(); + bool hasSizeField = br.ReadBit(); + br.ReadBit(); // obu_reserved_1bit + + // begin obu_extension_header( ) (5.3.3) + if (temp.mExtensionFlag) { + if (br.BitsLeft() < 8) { + mResult = MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, + "Not enough bits left for an OBU extension header"); + return; + } + br.ReadBits(3); // temporal_id + br.ReadBits(2); // spatial_id + br.ReadBits(3); // extension_header_reserved_3bits + } + // end obu_extension_header( ) + // end obu_header( ) + + // Get the size of the remaining OBU data attached to the header in + // bytes. + size_t size; + if (hasSizeField) { + if (br.BitsLeft() < 8) { + mResult = MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, + "Not enough bits left for an OBU size field"); + return; + } + CheckedUint32 checkedSize = br.ReadULEB128().toChecked<uint32_t>(); + // Spec requires that the value ULEB128 reads is (1 << 32) - 1 or below. + // See leb128(): https://aomediacodec.github.io/av1-spec/#leb128 + if (!checkedSize.isValid()) { + mResult = + MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, "OBU size was too large"); + return; + } + size = checkedSize.value(); + } else { + // This case should rarely happen in practice. To support the Annex B + // format in the specification, we would have to parse every header type + // to skip over them, but this allows us to at least iterate once to + // retrieve the first OBU in the data. + size = mData.Length() - 1 - temp.mExtensionFlag; + } + + if (br.BitsLeft() / 8 < size) { + mResult = MediaResult( + NS_ERROR_DOM_MEDIA_DECODE_ERR, + nsPrintfCString("Size specified by the OBU header (%zu) is more " + "than the actual remaining OBU data (%zu)", + size, br.BitsLeft() / 8) + .get()); + return; + } + + ASSERT_BYTE_ALIGNED(br); + + size_t bytes = br.BitCount() / 8; + temp.mContents = mData.Subspan(mPosition + bytes, size); + mCurrent = temp; + // end open_bitstream_unit( ) + + mPosition += bytes + size; + resetExit.release(); + mResult = NS_OK; +} + +/* static */ +already_AddRefed<MediaByteBuffer> AOMDecoder::CreateOBU( + const OBUType aType, const Span<const uint8_t>& aContents) { + RefPtr<MediaByteBuffer> buffer = new MediaByteBuffer(); + + BitWriter bw(buffer); + bw.WriteBits(0, 1); // obu_forbidden_bit + bw.WriteBits(static_cast<uint8_t>(aType), 4); + bw.WriteBit(false); // obu_extension_flag + bw.WriteBit(true); // obu_has_size_field + bw.WriteBits(0, 1); // obu_reserved_1bit + ASSERT_BYTE_ALIGNED(bw); + bw.WriteULEB128(aContents.Length()); + ASSERT_BYTE_ALIGNED(bw); + + buffer->AppendElements(aContents.Elements(), aContents.Length()); + return buffer.forget(); +} + +/* static */ +MediaResult AOMDecoder::ReadSequenceHeaderInfo( + const Span<const uint8_t>& aSample, AV1SequenceInfo& aDestInfo) { + // We need to get the last sequence header OBU, the specification does not + // limit a temporal unit to one sequence header. + OBUIterator iter = ReadOBUs(aSample); + OBUInfo seqOBU; + + while (true) { + if (!iter.HasNext()) { + // Pass along the error from parsing the OBU. + MediaResult result = iter.GetResult(); + if (result.Code() != NS_OK) { + return result; + } + break; + } + OBUInfo obu = iter.Next(); + if (obu.mType == OBUType::SequenceHeader) { + seqOBU = obu; + } + } + + if (seqOBU.mType != OBUType::SequenceHeader) { + return NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA; + } + + // Sequence header syntax is specified here: + // https://aomediacodec.github.io/av1-spec/#sequence-header-obu-syntax + // Section 5.5: Sequence header OBU syntax + + // See also Section 6.4: Sequence header OBU semantics + // https://aomediacodec.github.io/av1-spec/#sequence-header-obu-semantics + // This section defines all the fields used in the sequence header. + BitReader br(seqOBU.mContents.Elements(), seqOBU.mContents.Length() * 8); + AV1SequenceInfo tempInfo; + + // begin sequence_header_obu( ) + // https://aomediacodec.github.io/av1-spec/#general-sequence-header-obu-syntax + tempInfo.mProfile = br.ReadBits(3); + const bool stillPicture = br.ReadBit(); + const bool reducedStillPicture = br.ReadBit(); + if (!stillPicture && reducedStillPicture) { + return MediaResult( + NS_ERROR_DOM_MEDIA_DECODE_ERR, + "reduced_still_picture is true while still_picture is false"); + } + + if (reducedStillPicture) { + OperatingPoint op; + op.mLayers = 0; + op.mLevel = br.ReadBits(5); // seq_level_idx[0] + op.mTier = 0; + tempInfo.mOperatingPoints.SetCapacity(1); + tempInfo.mOperatingPoints.AppendElement(op); + } else { + bool decoderModelInfoPresent; + uint8_t operatingPointCountMinusOne; + + if (br.ReadBit()) { // timing_info_present_flag + // begin timing_info( ) + // https://aomediacodec.github.io/av1-spec/#timing-info-syntax + br.ReadBits(32); // num_units_in_display_tick + br.ReadBits(32); // time_scale + if (br.ReadBit()) { // equal_picture_interval + br.ReadUE(); // num_ticks_per_picture_minus_1 + } + // end timing_info( ) + + decoderModelInfoPresent = br.ReadBit(); + if (decoderModelInfoPresent) { + // begin decoder_model_info( ) + // https://aomediacodec.github.io/av1-spec/#decoder-model-info-syntax + br.ReadBits(5); // buffer_delay_length_minus_1 + br.ReadBits(32); // num_units_in_decoding_tick + br.ReadBits(5); // buffer_removal_time_length_minus_1 + br.ReadBits(5); // frame_presentation_time_length_minus_1 + // end decoder_model_info( ) + } + } else { + decoderModelInfoPresent = false; + } + + bool initialDisplayDelayPresent = br.ReadBit(); + operatingPointCountMinusOne = br.ReadBits(5); + tempInfo.mOperatingPoints.SetCapacity(operatingPointCountMinusOne + 1); + for (uint8_t i = 0; i <= operatingPointCountMinusOne; i++) { + OperatingPoint op; + op.mLayers = br.ReadBits(12); // operating_point_idc[ i ] + op.mLevel = br.ReadBits(5); // seq_level_idx[ i ] + op.mTier = op.mLevel > 7 ? br.ReadBits(1) : 0; + if (decoderModelInfoPresent) { + br.ReadBit(); // decoder_model_present_for_this_op[ i ] + } + if (initialDisplayDelayPresent) { + if (br.ReadBit()) { // initial_display_delay_present_for_this_op[ i ] + br.ReadBits(4); + } + } + tempInfo.mOperatingPoints.AppendElement(op); + } + } + + uint8_t frameWidthBits = br.ReadBits(4) + 1; + uint8_t frameHeightBits = br.ReadBits(4) + 1; + uint32_t maxWidth = br.ReadBits(frameWidthBits) + 1; + uint32_t maxHeight = br.ReadBits(frameHeightBits) + 1; + tempInfo.mImage = gfx::IntSize(maxWidth, maxHeight); + + if (!reducedStillPicture) { + if (br.ReadBit()) { // frame_id_numbers_present_flag + br.ReadBits(4); // delta_frame_id_length_minus_2 + br.ReadBits(3); // additional_frame_id_legnth_minus_1 + } + } + + br.ReadBit(); // use_128x128_superblock + br.ReadBit(); // enable_filter_intra + br.ReadBit(); // enable_intra_edge_filter + + if (reducedStillPicture) { + // enable_interintra_compound = 0 + // enable_masked_compound = 0 + // enable_warped_motion = 0 + // enable_dual_filter = 0 + // enable_order_hint = 0 + // enable_jnt_comp = 0 + // enable_ref_frame_mvs = 0 + // seq_force_screen_content_tools = SELECT_SCREEN_CONTENT_TOOLS + // seq_force_integer_mv = SELECT_INTEGER_MV + // OrderHintBits = 0 + } else { + br.ReadBit(); // enable_interintra_compound + br.ReadBit(); // enable_masked_compound + br.ReadBit(); // enable_warped_motion + br.ReadBit(); // enable_dual_filter + + const bool enableOrderHint = br.ReadBit(); + + if (enableOrderHint) { + br.ReadBit(); // enable_jnt_comp + br.ReadBit(); // enable_ref_frame_mvs + } + + uint8_t forceScreenContentTools; + + if (br.ReadBit()) { // seq_choose_screen_content_tools + forceScreenContentTools = 2; // SELECT_SCREEN_CONTENT_TOOLS + } else { + forceScreenContentTools = br.ReadBits(1); + } + + if (forceScreenContentTools > 0) { + if (!br.ReadBit()) { // seq_choose_integer_mv + br.ReadBit(); // seq_force_integer_mv + } + } + + if (enableOrderHint) { + br.ReadBits(3); // order_hint_bits_minus_1 + } + } + + br.ReadBit(); // enable_superres + br.ReadBit(); // enable_cdef + br.ReadBit(); // enable_restoration + + // begin color_config( ) + // https://aomediacodec.github.io/av1-spec/#color-config-syntax + const bool highBitDepth = br.ReadBit(); + if (tempInfo.mProfile == 2 && highBitDepth) { + const bool twelveBit = br.ReadBit(); + tempInfo.mBitDepth = twelveBit ? 12 : 10; + } else { + tempInfo.mBitDepth = highBitDepth ? 10 : 8; + } + + tempInfo.mMonochrome = tempInfo.mProfile == 1 ? false : br.ReadBit(); + + VideoColorSpace* colors = &tempInfo.mColorSpace; + + if (br.ReadBit()) { // color_description_present_flag + colors->mPrimaries = static_cast<ColourPrimaries>(br.ReadBits(8)); + colors->mTransfer = static_cast<TransferCharacteristics>(br.ReadBits(8)); + colors->mMatrix = static_cast<MatrixCoefficients>(br.ReadBits(8)); + } else { + colors->mPrimaries = ColourPrimaries::CP_UNSPECIFIED; + colors->mTransfer = TransferCharacteristics::TC_UNSPECIFIED; + colors->mMatrix = MatrixCoefficients::MC_UNSPECIFIED; + } + + if (tempInfo.mMonochrome) { + colors->mRange = br.ReadBit() ? ColorRange::FULL : ColorRange::LIMITED; + tempInfo.mSubsamplingX = true; + tempInfo.mSubsamplingY = true; + tempInfo.mChromaSamplePosition = ChromaSamplePosition::Unknown; + } else if (colors->mPrimaries == ColourPrimaries::CP_BT709 && + colors->mTransfer == TransferCharacteristics::TC_SRGB && + colors->mMatrix == MatrixCoefficients::MC_IDENTITY) { + colors->mRange = ColorRange::FULL; + tempInfo.mSubsamplingX = false; + tempInfo.mSubsamplingY = false; + } else { + colors->mRange = br.ReadBit() ? ColorRange::FULL : ColorRange::LIMITED; + switch (tempInfo.mProfile) { + case 0: + tempInfo.mSubsamplingX = true; + tempInfo.mSubsamplingY = true; + break; + case 1: + tempInfo.mSubsamplingX = false; + tempInfo.mSubsamplingY = false; + break; + case 2: + if (tempInfo.mBitDepth == 12) { + tempInfo.mSubsamplingX = br.ReadBit(); + tempInfo.mSubsamplingY = + tempInfo.mSubsamplingX ? br.ReadBit() : false; + } else { + tempInfo.mSubsamplingX = true; + tempInfo.mSubsamplingY = false; + } + break; + } + tempInfo.mChromaSamplePosition = + tempInfo.mSubsamplingX && tempInfo.mSubsamplingY + ? static_cast<ChromaSamplePosition>(br.ReadBits(2)) + : ChromaSamplePosition::Unknown; + } + + br.ReadBit(); // separate_uv_delta_q + // end color_config( ) + + br.ReadBit(); // film_grain_params_present + // end sequence_header_obu( ) + + // begin trailing_bits( ) + // https://aomediacodec.github.io/av1-spec/#trailing-bits-syntax + if (br.BitsLeft() > 8) { + NS_WARNING( + "AV1 sequence header finished reading with more than " + "a byte of aligning bits, may indicate an error"); + } + // Ensure that data is read correctly by checking trailing bits. + bool correct = br.ReadBit(); + correct &= br.ReadBits(br.BitsLeft() % 8) == 0; + while (br.BitsLeft() > 0) { + correct &= br.ReadBits(8) == 0; + } + if (!correct) { + return MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, + "AV1 sequence header was corrupted"); + } + // end trailing_bits( ) + + aDestInfo = tempInfo; + return NS_OK; +} + +/* static */ +already_AddRefed<MediaByteBuffer> AOMDecoder::CreateSequenceHeader( + const AV1SequenceInfo& aInfo, nsresult& aResult) { + aResult = NS_ERROR_FAILURE; + + RefPtr<MediaByteBuffer> seqHdrBuffer = new MediaByteBuffer(); + BitWriter bw(seqHdrBuffer); + + // See 5.5.1: General sequence header OBU syntax + // https://aomediacodec.github.io/av1-spec/#general-sequence-header-obu-syntax + bw.WriteBits(aInfo.mProfile, 3); + bw.WriteBit(false); // still_picture + bw.WriteBit(false); // reduced_still_picture_header + + bw.WriteBit(false); // timing_info_present_flag + // if ( timing_info_present_flag ) {...} + bw.WriteBit(false); // initial_display_delay_present_flag + + size_t opCount = aInfo.mOperatingPoints.Length(); + bw.WriteBits(opCount - 1, 5); // operating_points_cnt_minus_1 + for (size_t i = 0; i < opCount; i++) { + OperatingPoint op = aInfo.mOperatingPoints[i]; + bw.WriteBits(op.mLayers, 12); // operating_point_idc[ i ] + bw.WriteBits(op.mLevel, 5); + if (op.mLevel > 7) { + bw.WriteBits(op.mTier, 1); + } else { + // seq_tier[ i ] = 0 + if (op.mTier != 0) { + NS_WARNING("Operating points cannot specify tier for levels under 8."); + return nullptr; + } + } + // if ( decoder_model_info_present_flag ) {...} + // else + // decoder_model_info_present_for_this_op[ i ] = 0 + // if ( initial_display_delay_present_flag ) {...} + } + + if (aInfo.mImage.IsEmpty()) { + NS_WARNING("Sequence header requires a valid image size"); + return nullptr; + } + auto getBits = [](int32_t value) { + uint8_t bit = 0; + do { + value >>= 1; + bit++; + } while (value > 0); + return bit; + }; + uint8_t bitsW = getBits(aInfo.mImage.Width()); + uint8_t bitsH = getBits(aInfo.mImage.Height()); + bw.WriteBits(bitsW - 1, 4); + bw.WriteBits(bitsH - 1, 4); + bw.WriteBits(aInfo.mImage.Width() - 1, bitsW); + bw.WriteBits(aInfo.mImage.Height() - 1, bitsH); + + // if ( !reduced_still_picture_header ) + bw.WriteBit(false); // frame_id_numbers_present_flag + // if ( frame_id_numbers_present_flag ) {...} + // end if ( !reduced_still_picture_header ) + + // Values below are derived from a 1080p YouTube AV1 stream. + // The values are unused currently for determining the usable + // decoder, and are only included to allow successful validation + // of the generated sequence header. + + bw.WriteBit(true); // use_128x128_superblock + bw.WriteBit(true); // enable_filter_intra + bw.WriteBit(true); // enable_intra_edge_filter + + // if ( !reduced_still_picture_header) + bw.WriteBit(false); // enable_interintra_compound + bw.WriteBit(true); // enable_masked_compound + bw.WriteBit(true); // enable_warped_motion + bw.WriteBit(false); // enable_dual_filter + + bw.WriteBit(true); // enable_order_hint + // if ( enable_order_hint ) + bw.WriteBit(false); // enable_jnt_comp + bw.WriteBit(true); // enable_ref_frame_mvs + // end if ( enable_order_hint ) + + bw.WriteBit(true); // seq_choose_screen_content_tools + // if ( seq_choose_screen_content_tools ) + // seq_force_screen_content_tools = SELECT_SCREEN_CONTENT_TOOLS (2) + // else + // seq_force_screen_content_tools = f(1) + + // if ( seq_force_screen_content_tools > 0 ) + bw.WriteBit(true); // seq_choose_integer_mv + // if ( !seq_choose_integer_mv ) {...} + // end if ( seq_force_screen_content_tools > 0 ) + + // if ( enable_order_hint ) + bw.WriteBits(6, 3); // order_hint_bits_minus_1 + // end if ( enable_order_hint ) + // end if ( !reduced_still_picture_header ) + + bw.WriteBit(false); // enable_superres + bw.WriteBit(false); // enable_cdef + bw.WriteBit(true); // enable_restoration + + // Begin color_config( ) + // https://aomediacodec.github.io/av1-spec/#color-config-syntax + bool highBitDepth = aInfo.mBitDepth >= 10; + bw.WriteBit(highBitDepth); + + if (aInfo.mBitDepth == 12 && aInfo.mProfile != 2) { + NS_WARNING("Profile must be 2 for 12-bit"); + return nullptr; + } + if (aInfo.mProfile == 2 && highBitDepth) { + bw.WriteBit(aInfo.mBitDepth == 12); // twelve_bit + } + + if (aInfo.mMonochrome && aInfo.mProfile == 1) { + NS_WARNING("Profile 1 does not support monochrome"); + return nullptr; + } + if (aInfo.mProfile != 1) { + bw.WriteBit(aInfo.mMonochrome); + } + + const VideoColorSpace colors = aInfo.mColorSpace; + bool colorsPresent = + colors.mPrimaries != ColourPrimaries::CP_UNSPECIFIED || + colors.mTransfer != TransferCharacteristics::TC_UNSPECIFIED || + colors.mMatrix != MatrixCoefficients::MC_UNSPECIFIED; + bw.WriteBit(colorsPresent); + + if (colorsPresent) { + bw.WriteBits(static_cast<uint8_t>(colors.mPrimaries), 8); + bw.WriteBits(static_cast<uint8_t>(colors.mTransfer), 8); + bw.WriteBits(static_cast<uint8_t>(colors.mMatrix), 8); + } + + if (aInfo.mMonochrome) { + if (!aInfo.mSubsamplingX || !aInfo.mSubsamplingY) { + NS_WARNING("Monochrome requires 4:0:0 subsampling"); + return nullptr; + } + if (aInfo.mChromaSamplePosition != ChromaSamplePosition::Unknown) { + NS_WARNING( + "Cannot specify chroma sample position on monochrome sequence"); + return nullptr; + } + bw.WriteBit(colors.mRange == ColorRange::FULL); + } else if (colors.mPrimaries == ColourPrimaries::CP_BT709 && + colors.mTransfer == TransferCharacteristics::TC_SRGB && + colors.mMatrix == MatrixCoefficients::MC_IDENTITY) { + if (aInfo.mSubsamplingX || aInfo.mSubsamplingY || + colors.mRange != ColorRange::FULL || + aInfo.mChromaSamplePosition != ChromaSamplePosition::Unknown) { + NS_WARNING("sRGB requires 4:4:4 subsampling with full color range"); + return nullptr; + } + } else { + bw.WriteBit(colors.mRange == ColorRange::FULL); + switch (aInfo.mProfile) { + case 0: + if (!aInfo.mSubsamplingX || !aInfo.mSubsamplingY) { + NS_WARNING("Main Profile requires 4:2:0 subsampling"); + return nullptr; + } + break; + case 1: + if (aInfo.mSubsamplingX || aInfo.mSubsamplingY) { + NS_WARNING("High Profile requires 4:4:4 subsampling"); + return nullptr; + } + break; + case 2: + if (aInfo.mBitDepth == 12) { + bw.WriteBit(aInfo.mSubsamplingX); + if (aInfo.mSubsamplingX) { + bw.WriteBit(aInfo.mSubsamplingY); + } + } else { + if (!aInfo.mSubsamplingX || aInfo.mSubsamplingY) { + NS_WARNING( + "Professional Profile < 12-bit requires 4:2:2 subsampling"); + return nullptr; + } + } + break; + } + + if (aInfo.mSubsamplingX && aInfo.mSubsamplingY) { + bw.WriteBits(static_cast<uint8_t>(aInfo.mChromaSamplePosition), 2); + } else { + if (aInfo.mChromaSamplePosition != ChromaSamplePosition::Unknown) { + NS_WARNING("Only 4:2:0 subsampling can specify chroma position"); + return nullptr; + } + } + } + + bw.WriteBit(false); // separate_uv_delta_q + // end color_config( ) + + bw.WriteBit(true); // film_grain_params_present + + // trailing_bits( ) + // https://aomediacodec.github.io/av1-spec/#trailing-bits-syntax + size_t numTrailingBits = 8 - (bw.BitCount() % 8); + bw.WriteBit(true); + bw.WriteBits(0, numTrailingBits - 1); + ASSERT_BYTE_ALIGNED(bw); + + Span<const uint8_t> seqHdr(seqHdrBuffer->Elements(), seqHdrBuffer->Length()); + aResult = NS_OK; + return CreateOBU(OBUType::SequenceHeader, seqHdr); +} + +/* static */ +void AOMDecoder::TryReadAV1CBox(const MediaByteBuffer* aBox, + AV1SequenceInfo& aDestInfo, + MediaResult& aSeqHdrResult) { + // See av1C specification: + // https://aomediacodec.github.io/av1-isobmff/#av1codecconfigurationbox-section + BitReader br(aBox); + + br.ReadBits(8); // marker, version + + aDestInfo.mProfile = br.ReadBits(3); + + OperatingPoint op; + op.mLevel = br.ReadBits(5); + op.mTier = br.ReadBits(1); + aDestInfo.mOperatingPoints.AppendElement(op); + + bool highBitDepth = br.ReadBit(); + bool twelveBit = br.ReadBit(); + aDestInfo.mBitDepth = highBitDepth ? twelveBit ? 12 : 10 : 8; + + aDestInfo.mMonochrome = br.ReadBit(); + aDestInfo.mSubsamplingX = br.ReadBit(); + aDestInfo.mSubsamplingY = br.ReadBit(); + aDestInfo.mChromaSamplePosition = + static_cast<ChromaSamplePosition>(br.ReadBits(2)); + + br.ReadBits(3); // reserved + br.ReadBit(); // initial_presentation_delay_present + br.ReadBits(4); // initial_presentation_delay_minus_one or reserved + + ASSERT_BYTE_ALIGNED(br); + + size_t skipBytes = br.BitCount() / 8; + Span<const uint8_t> obus(aBox->Elements() + skipBytes, + aBox->Length() - skipBytes); + + // Minimum possible OBU header size + if (obus.Length() < 1) { + aSeqHdrResult = NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA; + return; + } + + // If present, the sequence header will be redundant to some values, but any + // values stored in it should be treated as more accurate than av1C. + aSeqHdrResult = ReadSequenceHeaderInfo(obus, aDestInfo); +} + +/* static */ +void AOMDecoder::WriteAV1CBox(const AV1SequenceInfo& aInfo, + MediaByteBuffer* aDestBox, bool& aHasSeqHdr) { + aHasSeqHdr = false; + + BitWriter bw(aDestBox); + + bw.WriteBit(true); // marker + bw.WriteBits(1, 7); // version + + bw.WriteBits(aInfo.mProfile, 3); + + MOZ_DIAGNOSTIC_ASSERT(aInfo.mOperatingPoints.Length() > 0); + bw.WriteBits(aInfo.mOperatingPoints[0].mLevel, 5); + bw.WriteBits(aInfo.mOperatingPoints[0].mTier, 1); + + bw.WriteBit(aInfo.mBitDepth >= 10); // high_bitdepth + bw.WriteBit(aInfo.mBitDepth == 12); // twelve_bit + + bw.WriteBit(aInfo.mMonochrome); + bw.WriteBit(aInfo.mSubsamplingX); + bw.WriteBit(aInfo.mSubsamplingY); + bw.WriteBits(static_cast<uint8_t>(aInfo.mChromaSamplePosition), 2); + + bw.WriteBits(0, 3); // reserved + bw.WriteBit(false); // initial_presentation_delay_present + bw.WriteBits(0, 4); // initial_presentation_delay_minus_one or reserved + + ASSERT_BYTE_ALIGNED(bw); + + nsresult rv; + RefPtr<MediaByteBuffer> seqHdrBuffer = CreateSequenceHeader(aInfo, rv); + + if (NS_SUCCEEDED(rv)) { + aDestBox->AppendElements(seqHdrBuffer->Elements(), seqHdrBuffer->Length()); + aHasSeqHdr = true; + } +} + +/* static */ +Maybe<AOMDecoder::AV1SequenceInfo> AOMDecoder::CreateSequenceInfoFromCodecs( + const nsAString& aCodec) { + AV1SequenceInfo info; + OperatingPoint op; + uint8_t chromaSamplePosition; + if (!ExtractAV1CodecDetails(aCodec, info.mProfile, op.mLevel, op.mTier, + info.mBitDepth, info.mMonochrome, + info.mSubsamplingX, info.mSubsamplingY, + chromaSamplePosition, info.mColorSpace)) { + return Nothing(); + } + info.mOperatingPoints.AppendElement(op); + info.mChromaSamplePosition = + static_cast<ChromaSamplePosition>(chromaSamplePosition); + return Some(info); +} + +/* static */ +bool AOMDecoder::SetVideoInfo(VideoInfo* aDestInfo, const nsAString& aCodec) { + Maybe<AV1SequenceInfo> info = CreateSequenceInfoFromCodecs(aCodec); + if (info.isNothing()) { + return false; + } + + if (!aDestInfo->mImage.IsEmpty()) { + info->mImage = aDestInfo->mImage; + } + + RefPtr<MediaByteBuffer> extraData = new MediaByteBuffer(); + bool hasSeqHdr; + WriteAV1CBox(info.value(), extraData, hasSeqHdr); + aDestInfo->mExtraData = extraData; + return true; +} + +} // namespace mozilla +#undef LOG +#undef ASSERT_BYTE_ALIGNED diff --git a/dom/media/platforms/agnostic/AOMDecoder.h b/dom/media/platforms/agnostic/AOMDecoder.h new file mode 100644 index 0000000000..f9620339e0 --- /dev/null +++ b/dom/media/platforms/agnostic/AOMDecoder.h @@ -0,0 +1,287 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#if !defined(AOMDecoder_h_) +# define AOMDecoder_h_ + +# include <stdint.h> + +# include "PerformanceRecorder.h" +# include "PlatformDecoderModule.h" +# include <aom/aom_decoder.h> +# include "mozilla/Span.h" +# include "VideoUtils.h" + +namespace mozilla { + +DDLoggedTypeDeclNameAndBase(AOMDecoder, MediaDataDecoder); + +class AOMDecoder final : public MediaDataDecoder, + public DecoderDoctorLifeLogger<AOMDecoder> { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AOMDecoder, final); + + explicit AOMDecoder(const CreateDecoderParams& aParams); + + RefPtr<InitPromise> Init() override; + RefPtr<DecodePromise> Decode(MediaRawData* aSample) override; + RefPtr<DecodePromise> Drain() override; + RefPtr<FlushPromise> Flush() override; + RefPtr<ShutdownPromise> Shutdown() override; + nsCString GetDescriptionName() const override { + return "av1 libaom video decoder"_ns; + } + nsCString GetCodecName() const override { return "av1"_ns; } + + // Return true if aMimeType is a one of the strings used + // by our demuxers to identify AV1 streams. + static bool IsAV1(const nsACString& aMimeType); + + // Return true if a sample is a keyframe. + static bool IsKeyframe(Span<const uint8_t> aBuffer); + + // Return the frame dimensions for a sample. + static gfx::IntSize GetFrameSize(Span<const uint8_t> aBuffer); + + // obu_type defined at: + // https://aomediacodec.github.io/av1-spec/av1-spec.pdf#page=123 + enum class OBUType : uint8_t { + Reserved = 0, + SequenceHeader = 1, + TemporalDelimiter = 2, + FrameHeader = 3, + TileGroup = 4, + Metadata = 5, + Frame = 6, + RedundantFrameHeader = 7, + TileList = 8, + Padding = 15 + }; + + struct OBUInfo { + OBUType mType = OBUType::Reserved; + bool mExtensionFlag = false; + Span<const uint8_t> mContents; + + bool IsValid() const { + switch (mType) { + case OBUType::SequenceHeader: + case OBUType::TemporalDelimiter: + case OBUType::FrameHeader: + case OBUType::TileGroup: + case OBUType::Metadata: + case OBUType::Frame: + case OBUType::RedundantFrameHeader: + case OBUType::TileList: + case OBUType::Padding: + return true; + default: + return false; + } + } + }; + + struct OBUIterator { + public: + explicit OBUIterator(const Span<const uint8_t>& aData) + : mData(aData), mPosition(0), mGoNext(true), mResult(NS_OK) {} + bool HasNext() { + UpdateNext(); + return !mGoNext; + } + OBUInfo Next() { + UpdateNext(); + mGoNext = true; + return mCurrent; + } + MediaResult GetResult() const { return mResult; } + + private: + const Span<const uint8_t>& mData; + size_t mPosition; + OBUInfo mCurrent; + bool mGoNext; + MediaResult mResult; + + // Used to fill mCurrent with the next OBU in the iterator. + // mGoNext must be set to false if the next OBU is retrieved, + // otherwise it will be true so that HasNext() returns false. + // When an invalid OBU is read, the iterator will finish and + // mCurrent will be reset to default OBUInfo(). + void UpdateNext(); + }; + + // Create an iterator to parse Open Bitstream Units from a buffer. + static OBUIterator ReadOBUs(const Span<const uint8_t>& aData); + // Writes an Open Bitstream Unit header type and the contained subheader. + // Extension flag is set to 0 and size field is always present. + static already_AddRefed<MediaByteBuffer> CreateOBU( + const OBUType aType, const Span<const uint8_t>& aContents); + + // chroma_sample_position defined at: + // https://aomediacodec.github.io/av1-spec/av1-spec.pdf#page=131 + enum class ChromaSamplePosition : uint8_t { + Unknown = 0, + Vertical = 1, + Colocated = 2, + Reserved = 3 + }; + + struct OperatingPoint { + // https://aomediacodec.github.io/av1-spec/av1-spec.pdf#page=125 + // operating_point_idc[ i ]: A set of bitwise flags determining + // the temporal and spatial layers to decode. + // A value of 0 indicates that scalability is not being used. + uint16_t mLayers = 0; + // https://aomediacodec.github.io/av1-spec/av1-spec.pdf#page=650 + // See A.3: Levels for a definition of the available levels. + uint8_t mLevel = 0; + // https://aomediacodec.github.io/av1-spec/av1-spec.pdf#page=126 + // seq_tier[ i ]: The tier for the selected operating point. + uint8_t mTier = 0; + + bool operator==(const OperatingPoint& aOther) const { + return mLayers == aOther.mLayers && mLevel == aOther.mLevel && + mTier == aOther.mTier; + } + bool operator!=(const OperatingPoint& aOther) const { + return !(*this == aOther); + } + }; + + struct AV1SequenceInfo { + AV1SequenceInfo() = default; + + AV1SequenceInfo(const AV1SequenceInfo& aOther) { *this = aOther; } + + // Profiles, levels and tiers defined at: + // https://aomediacodec.github.io/av1-spec/av1-spec.pdf#page=650 + uint8_t mProfile = 0; + + // choose_operating_point( ) defines that the operating points are + // specified in order of preference by the encoder. Higher operating + // points indices in the header will allow a tradeoff of quality for + // performance, dropping some data from the decoding process. + // Normally we are only interested in the first operating point. + // See: https://aomediacodec.github.io/av1-spec/av1-spec.pdf#page=126 + nsTArray<OperatingPoint> mOperatingPoints = nsTArray<OperatingPoint>(1); + + gfx::IntSize mImage = {0, 0}; + + // Color configs explained at: + // https://aomediacodec.github.io/av1-spec/av1-spec.pdf#page=129 + uint8_t mBitDepth = 8; + bool mMonochrome = false; + bool mSubsamplingX = true; + bool mSubsamplingY = true; + ChromaSamplePosition mChromaSamplePosition = ChromaSamplePosition::Unknown; + + VideoColorSpace mColorSpace; + + gfx::ColorDepth ColorDepth() const { + return gfx::ColorDepthForBitDepth(mBitDepth); + } + + bool operator==(const AV1SequenceInfo& aOther) const { + if (mProfile != aOther.mProfile || mImage != aOther.mImage || + mBitDepth != aOther.mBitDepth || mMonochrome != aOther.mMonochrome || + mSubsamplingX != aOther.mSubsamplingX || + mSubsamplingY != aOther.mSubsamplingY || + mChromaSamplePosition != aOther.mChromaSamplePosition || + mColorSpace != aOther.mColorSpace) { + return false; + } + + size_t opCount = mOperatingPoints.Length(); + if (opCount != aOther.mOperatingPoints.Length()) { + return false; + } + for (size_t i = 0; i < opCount; i++) { + if (mOperatingPoints[i] != aOther.mOperatingPoints[i]) { + return false; + } + } + + return true; + } + bool operator!=(const AV1SequenceInfo& aOther) const { + return !(*this == aOther); + } + AV1SequenceInfo& operator=(const AV1SequenceInfo& aOther) { + mProfile = aOther.mProfile; + + size_t opCount = aOther.mOperatingPoints.Length(); + mOperatingPoints.ClearAndRetainStorage(); + mOperatingPoints.SetCapacity(opCount); + for (size_t i = 0; i < opCount; i++) { + mOperatingPoints.AppendElement(aOther.mOperatingPoints[i]); + } + + mImage = aOther.mImage; + mBitDepth = aOther.mBitDepth; + mMonochrome = aOther.mMonochrome; + mSubsamplingX = aOther.mSubsamplingX; + mSubsamplingY = aOther.mSubsamplingY; + mChromaSamplePosition = aOther.mChromaSamplePosition; + mColorSpace = aOther.mColorSpace; + return *this; + } + }; + + // Get a sequence header's info from a sample. + // Returns a MediaResult with codes: + // NS_OK: Sequence header was successfully found and read. + // NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA: Sequence header was not present. + // Other errors will indicate that the data was corrupt. + static MediaResult ReadSequenceHeaderInfo(const Span<const uint8_t>& aSample, + AV1SequenceInfo& aDestInfo); + // Writes a sequence header OBU to the buffer. + static already_AddRefed<MediaByteBuffer> CreateSequenceHeader( + const AV1SequenceInfo& aInfo, nsresult& aResult); + + // Reads the raw data of an ISOBMFF-compatible av1 configuration box (av1C), + // including any included sequence header. + static void TryReadAV1CBox(const MediaByteBuffer* aBox, + AV1SequenceInfo& aDestInfo, + MediaResult& aSeqHdrResult); + // Reads the raw data of an ISOBMFF-compatible av1 configuration box (av1C), + // including any included sequence header. + // This function should only be called for av1C boxes made by WriteAV1CBox, as + // it will assert that the box and its contained OBUs are not corrupted. + static void ReadAV1CBox(const MediaByteBuffer* aBox, + AV1SequenceInfo& aDestInfo, bool& aHadSeqHdr) { + MediaResult seqHdrResult; + TryReadAV1CBox(aBox, aDestInfo, seqHdrResult); + nsresult code = seqHdrResult.Code(); + MOZ_ASSERT(code == NS_OK || code == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA); + aHadSeqHdr = code == NS_OK; + } + // Writes an ISOBMFF-compatible av1 configuration box (av1C) to the buffer. + static void WriteAV1CBox(const AV1SequenceInfo& aInfo, + MediaByteBuffer* aDestBox, bool& aHasSeqHdr); + + // Create sequence info from a MIME codecs string. + static Maybe<AV1SequenceInfo> CreateSequenceInfoFromCodecs( + const nsAString& aCodec); + static bool SetVideoInfo(VideoInfo* aDestInfo, const nsAString& aCodec); + + private: + ~AOMDecoder(); + RefPtr<DecodePromise> ProcessDecode(MediaRawData* aSample); + + const RefPtr<layers::ImageContainer> mImageContainer; + const RefPtr<TaskQueue> mTaskQueue; + + // AOM decoder state + aom_codec_ctx_t mCodec; + + const VideoInfo mInfo; + const Maybe<TrackingId> mTrackingId; + PerformanceRecorderMulti<DecodeStage> mPerformanceRecorder; +}; + +} // namespace mozilla + +#endif // AOMDecoder_h_ diff --git a/dom/media/platforms/agnostic/AgnosticDecoderModule.cpp b/dom/media/platforms/agnostic/AgnosticDecoderModule.cpp new file mode 100644 index 0000000000..7bdc30b432 --- /dev/null +++ b/dom/media/platforms/agnostic/AgnosticDecoderModule.cpp @@ -0,0 +1,185 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "AgnosticDecoderModule.h" + +#include "TheoraDecoder.h" +#include "VPXDecoder.h" +#include "mozilla/Logging.h" +#include "mozilla/StaticPrefs_media.h" +#include "VideoUtils.h" + +#ifdef MOZ_AV1 +# include "AOMDecoder.h" +# include "DAV1DDecoder.h" +#endif + +namespace mozilla { + +enum class DecoderType { +#ifdef MOZ_AV1 + AV1, +#endif + Opus, + Theora, + Vorbis, + VPX, + Wave, +}; + +static bool IsAvailableInDefault(DecoderType type) { + switch (type) { +#ifdef MOZ_AV1 + case DecoderType::AV1: + return StaticPrefs::media_av1_enabled(); +#endif + case DecoderType::Opus: + case DecoderType::Theora: + case DecoderType::Vorbis: + case DecoderType::VPX: + case DecoderType::Wave: + return true; + default: + return false; + } +} + +static bool IsAvailableInRdd(DecoderType type) { + switch (type) { +#ifdef MOZ_AV1 + case DecoderType::AV1: + return StaticPrefs::media_av1_enabled(); +#endif + case DecoderType::Opus: + return StaticPrefs::media_rdd_opus_enabled(); + case DecoderType::Theora: + return StaticPrefs::media_rdd_theora_enabled(); + case DecoderType::Vorbis: +#if defined(__MINGW32__) + // If this is a MinGW build we need to force AgnosticDecoderModule to + // handle the decision to support Vorbis decoding (instead of + // RDD/RemoteDecoderModule) because of Bug 1597408 (Vorbis decoding on + // RDD causing sandboxing failure on MinGW-clang). Typically this + // would be dealt with using defines in StaticPrefList.yaml, but we + // must handle it here because of Bug 1598426 (the __MINGW32__ define + // isn't supported in StaticPrefList.yaml). + return false; +#else + return StaticPrefs::media_rdd_vorbis_enabled(); +#endif + case DecoderType::VPX: + return StaticPrefs::media_rdd_vpx_enabled(); + case DecoderType::Wave: + return StaticPrefs::media_rdd_wav_enabled(); + default: + return false; + } +} + +static bool IsAvailableInUtility(DecoderType type) { + switch (type) { + case DecoderType::Opus: + return StaticPrefs::media_utility_opus_enabled(); + case DecoderType::Vorbis: + return StaticPrefs::media_utility_vorbis_enabled(); + case DecoderType::Wave: + return StaticPrefs::media_utility_wav_enabled(); + case DecoderType::Theora: // Video codecs, dont take care of them + case DecoderType::VPX: + default: + return false; + } +} + +// Checks if decoder is available in the current process +static bool IsAvailable(DecoderType type) { + return XRE_IsRDDProcess() ? IsAvailableInRdd(type) + : XRE_IsUtilityProcess() ? IsAvailableInUtility(type) + : IsAvailableInDefault(type); +} + +media::DecodeSupportSet AgnosticDecoderModule::SupportsMimeType( + const nsACString& aMimeType, DecoderDoctorDiagnostics* aDiagnostics) const { + UniquePtr<TrackInfo> trackInfo = CreateTrackInfoWithMIMEType(aMimeType); + if (!trackInfo) { + return media::DecodeSupportSet{}; + } + return Supports(SupportDecoderParams(*trackInfo), aDiagnostics); +} + +media::DecodeSupportSet AgnosticDecoderModule::Supports( + const SupportDecoderParams& aParams, + DecoderDoctorDiagnostics* aDiagnostics) const { + // This should only be supported by MFMediaEngineDecoderModule. + if (aParams.mMediaEngineId) { + return media::DecodeSupportSet{}; + } + + const auto& trackInfo = aParams.mConfig; + const nsACString& mimeType = trackInfo.mMimeType; + + bool supports = +#ifdef MOZ_AV1 + // We remove support for decoding AV1 here if RDD is enabled so that + // decoding on the content process doesn't accidentally happen in case + // something goes wrong with launching the RDD process. + (AOMDecoder::IsAV1(mimeType) && IsAvailable(DecoderType::AV1)) || +#endif + (VPXDecoder::IsVPX(mimeType) && IsAvailable(DecoderType::VPX)) || + (TheoraDecoder::IsTheora(mimeType) && IsAvailable(DecoderType::Theora)); + MOZ_LOG(sPDMLog, LogLevel::Debug, + ("Agnostic decoder %s requested type '%s'", + supports ? "supports" : "rejects", mimeType.BeginReading())); + if (supports) { + return media::DecodeSupport::SoftwareDecode; + } + return media::DecodeSupportSet{}; +} + +already_AddRefed<MediaDataDecoder> AgnosticDecoderModule::CreateVideoDecoder( + const CreateDecoderParams& aParams) { + if (Supports(SupportDecoderParams(aParams), nullptr /* diagnostic */) + .isEmpty()) { + return nullptr; + } + RefPtr<MediaDataDecoder> m; + + if (VPXDecoder::IsVPX(aParams.mConfig.mMimeType)) { + m = new VPXDecoder(aParams); + } +#ifdef MOZ_AV1 + // We remove support for decoding AV1 here if RDD is enabled so that + // decoding on the content process doesn't accidentally happen in case + // something goes wrong with launching the RDD process. + if (StaticPrefs::media_av1_enabled() && + (!StaticPrefs::media_rdd_process_enabled() || XRE_IsRDDProcess()) && + AOMDecoder::IsAV1(aParams.mConfig.mMimeType)) { + if (StaticPrefs::media_av1_use_dav1d()) { + m = new DAV1DDecoder(aParams); + } else { + m = new AOMDecoder(aParams); + } + } +#endif + else if (TheoraDecoder::IsTheora(aParams.mConfig.mMimeType)) { + m = new TheoraDecoder(aParams); + } + + return m.forget(); +} + +already_AddRefed<MediaDataDecoder> AgnosticDecoderModule::CreateAudioDecoder( + const CreateDecoderParams& aParams) { + return nullptr; +} + +/* static */ +already_AddRefed<PlatformDecoderModule> AgnosticDecoderModule::Create() { + RefPtr<PlatformDecoderModule> pdm = new AgnosticDecoderModule(); + return pdm.forget(); +} + +} // namespace mozilla diff --git a/dom/media/platforms/agnostic/AgnosticDecoderModule.h b/dom/media/platforms/agnostic/AgnosticDecoderModule.h new file mode 100644 index 0000000000..bac5f4bf42 --- /dev/null +++ b/dom/media/platforms/agnostic/AgnosticDecoderModule.h @@ -0,0 +1,39 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(AgnosticDecoderModule_h_) +# define AgnosticDecoderModule_h_ + +# include "PlatformDecoderModule.h" + +namespace mozilla { + +class AgnosticDecoderModule : public PlatformDecoderModule { + public: + static already_AddRefed<PlatformDecoderModule> Create(); + + media::DecodeSupportSet SupportsMimeType( + const nsACString& aMimeType, + DecoderDoctorDiagnostics* aDiagnostics) const override; + media::DecodeSupportSet Supports( + const SupportDecoderParams& aParams, + DecoderDoctorDiagnostics* aDiagnostics) const override; + + protected: + AgnosticDecoderModule() = default; + virtual ~AgnosticDecoderModule() = default; + // Decode thread. + already_AddRefed<MediaDataDecoder> CreateVideoDecoder( + const CreateDecoderParams& aParams) override; + + // Decode thread. + already_AddRefed<MediaDataDecoder> CreateAudioDecoder( + const CreateDecoderParams& aParams) override; +}; + +} // namespace mozilla + +#endif /* AgnosticDecoderModule_h_ */ diff --git a/dom/media/platforms/agnostic/BlankDecoderModule.cpp b/dom/media/platforms/agnostic/BlankDecoderModule.cpp new file mode 100644 index 0000000000..05ce51d0c1 --- /dev/null +++ b/dom/media/platforms/agnostic/BlankDecoderModule.cpp @@ -0,0 +1,146 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "BlankDecoderModule.h" + +#include "mozilla/CheckedInt.h" +#include "mozilla/UniquePtrExtensions.h" +#include "mozilla/RefPtr.h" +#include "mozilla/gfx/Rect.h" +#include "mozilla/gfx/Point.h" +#include "ImageContainer.h" +#include "MediaData.h" +#include "MediaInfo.h" +#include "VideoUtils.h" + +namespace mozilla { + +BlankVideoDataCreator::BlankVideoDataCreator( + uint32_t aFrameWidth, uint32_t aFrameHeight, + layers::ImageContainer* aImageContainer) + : mFrameWidth(aFrameWidth), + mFrameHeight(aFrameHeight), + mImageContainer(aImageContainer) { + mInfo.mDisplay = gfx::IntSize(mFrameWidth, mFrameHeight); + mPicture = gfx::IntRect(0, 0, mFrameWidth, mFrameHeight); +} + +already_AddRefed<MediaData> BlankVideoDataCreator::Create( + MediaRawData* aSample) { + // Create a fake YUV buffer in a 420 format. That is, an 8bpp Y plane, + // with a U and V plane that are half the size of the Y plane, i.e 8 bit, + // 2x2 subsampled. Have the data pointer of each frame point to the + // first plane, they'll always be zero'd memory anyway. + const CheckedUint32 size = CheckedUint32(mFrameWidth) * mFrameHeight; + if (!size.isValid()) { + // Overflow happened. + return nullptr; + } + auto frame = MakeUniqueFallible<uint8_t[]>(size.value()); + if (!frame) { + return nullptr; + } + memset(frame.get(), 0, mFrameWidth * mFrameHeight); + VideoData::YCbCrBuffer buffer; + + // Y plane. + buffer.mPlanes[0].mData = frame.get(); + buffer.mPlanes[0].mStride = mFrameWidth; + buffer.mPlanes[0].mHeight = mFrameHeight; + buffer.mPlanes[0].mWidth = mFrameWidth; + buffer.mPlanes[0].mSkip = 0; + + // Cb plane. + buffer.mPlanes[1].mData = frame.get(); + buffer.mPlanes[1].mStride = (mFrameWidth + 1) / 2; + buffer.mPlanes[1].mHeight = (mFrameHeight + 1) / 2; + buffer.mPlanes[1].mWidth = (mFrameWidth + 1) / 2; + buffer.mPlanes[1].mSkip = 0; + + // Cr plane. + buffer.mPlanes[2].mData = frame.get(); + buffer.mPlanes[2].mStride = (mFrameWidth + 1) / 2; + buffer.mPlanes[2].mHeight = (mFrameHeight + 1) / 2; + buffer.mPlanes[2].mWidth = (mFrameWidth + 1) / 2; + buffer.mPlanes[2].mSkip = 0; + + buffer.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT; + buffer.mYUVColorSpace = gfx::YUVColorSpace::BT601; + buffer.mColorPrimaries = gfx::ColorSpace2::BT709; + + Result<already_AddRefed<VideoData>, MediaResult> result = + VideoData::CreateAndCopyData(mInfo, mImageContainer, aSample->mOffset, + aSample->mTime, aSample->mDuration, buffer, + aSample->mKeyframe, aSample->mTime, mPicture, + nullptr); + return result.unwrapOr(nullptr); +} + +BlankAudioDataCreator::BlankAudioDataCreator(uint32_t aChannelCount, + uint32_t aSampleRate) + : mFrameSum(0), mChannelCount(aChannelCount), mSampleRate(aSampleRate) {} + +already_AddRefed<MediaData> BlankAudioDataCreator::Create( + MediaRawData* aSample) { + // Convert duration to frames. We add 1 to duration to account for + // rounding errors, so we get a consistent tone. + CheckedInt64 frames = + UsecsToFrames(aSample->mDuration.ToMicroseconds() + 1, mSampleRate); + if (!frames.isValid() || !mChannelCount || !mSampleRate || + frames.value() > (UINT32_MAX / mChannelCount)) { + return nullptr; + } + AlignedAudioBuffer samples(frames.value() * mChannelCount); + if (!samples) { + return nullptr; + } + // Fill the sound buffer with an A4 tone. + static const float pi = 3.14159265f; + static const float noteHz = 440.0f; + for (int i = 0; i < frames.value(); i++) { + float f = sin(2 * pi * noteHz * mFrameSum / mSampleRate); + for (unsigned c = 0; c < mChannelCount; c++) { + samples[i * mChannelCount + c] = AudioDataValue(f); + } + mFrameSum++; + } + RefPtr<AudioData> data(new AudioData(aSample->mOffset, aSample->mTime, + std::move(samples), mChannelCount, + mSampleRate)); + return data.forget(); +} + +already_AddRefed<MediaDataDecoder> BlankDecoderModule::CreateVideoDecoder( + const CreateDecoderParams& aParams) { + const VideoInfo& config = aParams.VideoConfig(); + UniquePtr<DummyDataCreator> creator = MakeUnique<BlankVideoDataCreator>( + config.mDisplay.width, config.mDisplay.height, aParams.mImageContainer); + RefPtr<MediaDataDecoder> decoder = new DummyMediaDataDecoder( + std::move(creator), "blank media data decoder"_ns, aParams); + return decoder.forget(); +} + +already_AddRefed<MediaDataDecoder> BlankDecoderModule::CreateAudioDecoder( + const CreateDecoderParams& aParams) { + const AudioInfo& config = aParams.AudioConfig(); + UniquePtr<DummyDataCreator> creator = + MakeUnique<BlankAudioDataCreator>(config.mChannels, config.mRate); + RefPtr<MediaDataDecoder> decoder = new DummyMediaDataDecoder( + std::move(creator), "blank media data decoder"_ns, aParams); + return decoder.forget(); +} + +media::DecodeSupportSet BlankDecoderModule::SupportsMimeType( + const nsACString& aMimeType, DecoderDoctorDiagnostics* aDiagnostics) const { + return media::DecodeSupport::SoftwareDecode; +} + +/* static */ +already_AddRefed<PlatformDecoderModule> BlankDecoderModule::Create() { + return MakeAndAddRef<BlankDecoderModule>(); +} + +} // namespace mozilla diff --git a/dom/media/platforms/agnostic/BlankDecoderModule.h b/dom/media/platforms/agnostic/BlankDecoderModule.h new file mode 100644 index 0000000000..65e5d4479e --- /dev/null +++ b/dom/media/platforms/agnostic/BlankDecoderModule.h @@ -0,0 +1,68 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#if !defined(BlankDecoderModule_h_) +# define BlankDecoderModule_h_ + +# include "DummyMediaDataDecoder.h" +# include "PlatformDecoderModule.h" + +namespace mozilla { + +namespace layers { +class ImageContainer; +} + +class MediaData; +class MediaRawData; + +class BlankVideoDataCreator : public DummyDataCreator { + public: + BlankVideoDataCreator(uint32_t aFrameWidth, uint32_t aFrameHeight, + layers::ImageContainer* aImageContainer); + + already_AddRefed<MediaData> Create(MediaRawData* aSample) override; + + private: + VideoInfo mInfo; + gfx::IntRect mPicture; + uint32_t mFrameWidth; + uint32_t mFrameHeight; + RefPtr<layers::ImageContainer> mImageContainer; +}; + +class BlankAudioDataCreator : public DummyDataCreator { + public: + BlankAudioDataCreator(uint32_t aChannelCount, uint32_t aSampleRate); + + already_AddRefed<MediaData> Create(MediaRawData* aSample) override; + + private: + int64_t mFrameSum; + uint32_t mChannelCount; + uint32_t mSampleRate; +}; + +class BlankDecoderModule : public PlatformDecoderModule { + template <typename T, typename... Args> + friend already_AddRefed<T> MakeAndAddRef(Args&&...); + + public: + static already_AddRefed<PlatformDecoderModule> Create(); + + already_AddRefed<MediaDataDecoder> CreateVideoDecoder( + const CreateDecoderParams& aParams) override; + + already_AddRefed<MediaDataDecoder> CreateAudioDecoder( + const CreateDecoderParams& aParams) override; + + media::DecodeSupportSet SupportsMimeType( + const nsACString& aMimeType, + DecoderDoctorDiagnostics* aDiagnostics) const override; +}; + +} // namespace mozilla + +#endif /* BlankDecoderModule_h_ */ diff --git a/dom/media/platforms/agnostic/DAV1DDecoder.cpp b/dom/media/platforms/agnostic/DAV1DDecoder.cpp new file mode 100644 index 0000000000..e93ceb27a1 --- /dev/null +++ b/dom/media/platforms/agnostic/DAV1DDecoder.cpp @@ -0,0 +1,403 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "DAV1DDecoder.h" + +#include "gfxUtils.h" +#include "ImageContainer.h" +#include "mozilla/StaticPrefs_media.h" +#include "mozilla/TaskQueue.h" +#include "nsThreadUtils.h" +#include "PerformanceRecorder.h" +#include "VideoUtils.h" + +#undef LOG +#define LOG(arg, ...) \ + DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, __func__, \ + ##__VA_ARGS__) + +namespace mozilla { + +static int GetDecodingThreadCount(uint32_t aCodedHeight) { + /** + * Based on the result we print out from the dav1decoder [1], the + * following information shows the number of tiles for AV1 videos served on + * Youtube. Each Tile can be decoded in parallel, so we would like to make + * sure we at least use enough threads to match the number of tiles. + * + * ---------------------------- + * | resolution row col total | + * | 480p 2 1 2 | + * | 720p 2 2 4 | + * | 1080p 4 2 8 | + * | 1440p 4 2 8 | + * | 2160p 8 4 32 | + * ---------------------------- + * + * Besides the tile thread count, the frame thread count also needs to be + * considered. As we didn't find anything about what the best number is for + * the count of frame thread, just simply use 2 for parallel jobs, which + * is similar with Chromium's implementation. They uses 3 frame threads for + * 720p+ but less tile threads, so we will still use more total threads. In + * addition, their data is measured on 2019, our data should be closer to the + * current real world situation. + * [1] + * https://searchfox.org/mozilla-central/rev/2f5ed7b7244172d46f538051250b14fb4d8f1a5f/third_party/dav1d/src/decode.c#2940 + */ + int tileThreads = 2, frameThreads = 2; + if (aCodedHeight >= 2160) { + tileThreads = 32; + } else if (aCodedHeight >= 1080) { + tileThreads = 8; + } else if (aCodedHeight >= 720) { + tileThreads = 4; + } + return tileThreads * frameThreads; +} + +DAV1DDecoder::DAV1DDecoder(const CreateDecoderParams& aParams) + : mInfo(aParams.VideoConfig()), + mTaskQueue(TaskQueue::Create( + GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER), + "Dav1dDecoder")), + mImageContainer(aParams.mImageContainer), + mImageAllocator(aParams.mKnowsCompositor), + mTrackingId(aParams.mTrackingId), + mLowLatency( + aParams.mOptions.contains(CreateDecoderParams::Option::LowLatency)) {} + +DAV1DDecoder::~DAV1DDecoder() = default; + +RefPtr<MediaDataDecoder::InitPromise> DAV1DDecoder::Init() { + Dav1dSettings settings; + dav1d_default_settings(&settings); + if (mLowLatency) { + settings.max_frame_delay = 1; + } + size_t decoder_threads = 2; + if (mInfo.mDisplay.width >= 2048) { + decoder_threads = 8; + } else if (mInfo.mDisplay.width >= 1024) { + decoder_threads = 4; + } + if (StaticPrefs::media_av1_new_thread_count_strategy()) { + decoder_threads = GetDecodingThreadCount(mInfo.mImage.Height()); + } + // Still need to consider the amount of physical cores in order to achieve + // best performance. + settings.n_threads = + static_cast<int>(std::min(decoder_threads, GetNumberOfProcessors())); + if (int32_t count = StaticPrefs::media_av1_force_thread_count(); count > 0) { + settings.n_threads = count; + } + + int res = dav1d_open(&mContext, &settings); + if (res < 0) { + return DAV1DDecoder::InitPromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("Couldn't get dAV1d decoder interface.")), + __func__); + } + return DAV1DDecoder::InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, + __func__); +} + +RefPtr<MediaDataDecoder::DecodePromise> DAV1DDecoder::Decode( + MediaRawData* aSample) { + return InvokeAsync<MediaRawData*>(mTaskQueue, this, __func__, + &DAV1DDecoder::InvokeDecode, aSample); +} + +void ReleaseDataBuffer_s(const uint8_t* buf, void* user_data) { + MOZ_ASSERT(user_data); + MOZ_ASSERT(buf); + DAV1DDecoder* d = static_cast<DAV1DDecoder*>(user_data); + d->ReleaseDataBuffer(buf); +} + +void DAV1DDecoder::ReleaseDataBuffer(const uint8_t* buf) { + // The release callback may be called on a different thread defined by the + // third party dav1d execution. In that case post a task into TaskQueue to + // ensure that mDecodingBuffers is only ever accessed on the TaskQueue. + RefPtr<DAV1DDecoder> self = this; + auto releaseBuffer = [self, buf] { + MOZ_ASSERT(self->mTaskQueue->IsCurrentThreadIn()); + DebugOnly<bool> found = self->mDecodingBuffers.Remove(buf); + MOZ_ASSERT(found); + }; + + if (mTaskQueue->IsCurrentThreadIn()) { + releaseBuffer(); + } else { + nsresult rv = mTaskQueue->Dispatch(NS_NewRunnableFunction( + "DAV1DDecoder::ReleaseDataBuffer", std::move(releaseBuffer))); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; + } +} + +RefPtr<MediaDataDecoder::DecodePromise> DAV1DDecoder::InvokeDecode( + MediaRawData* aSample) { + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + MOZ_ASSERT(aSample); + + MediaInfoFlag flag = MediaInfoFlag::None; + flag |= (aSample->mKeyframe ? MediaInfoFlag::KeyFrame + : MediaInfoFlag::NonKeyFrame); + flag |= MediaInfoFlag::SoftwareDecoding; + flag |= MediaInfoFlag::VIDEO_AV1; + mTrackingId.apply([&](const auto& aId) { + mPerformanceRecorder.Start(aSample->mTimecode.ToMicroseconds(), + "DAV1DDecoder"_ns, aId, flag); + }); + + // Add the buffer to the hashtable in order to increase + // the ref counter and keep it alive. When dav1d does not + // need it any more will call it's release callback. Remove + // the buffer, in there, to reduce the ref counter and eventually + // free it. We need a hashtable and not an array because the + // release callback are not coming in the same order that the + // buffers have been added in the decoder (threading ordering + // inside decoder) + mDecodingBuffers.InsertOrUpdate(aSample->Data(), RefPtr{aSample}); + Dav1dData data; + int res = dav1d_data_wrap(&data, aSample->Data(), aSample->Size(), + ReleaseDataBuffer_s, this); + data.m.timestamp = aSample->mTimecode.ToMicroseconds(); + data.m.duration = aSample->mDuration.ToMicroseconds(); + data.m.offset = aSample->mOffset; + + if (res < 0) { + LOG("Create decoder data error."); + return DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__), __func__); + } + DecodedData results; + do { + res = dav1d_send_data(mContext, &data); + if (res < 0 && res != DAV1D_ERR(EAGAIN)) { + LOG("Decode error: %d", res); + return DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, __func__), __func__); + } + // Alway consume the whole buffer on success. + // At this point only DAV1D_ERR(EAGAIN) is expected. + MOZ_ASSERT((res == 0 && !data.sz) || + (res == DAV1D_ERR(EAGAIN) && data.sz == aSample->Size())); + + Result<already_AddRefed<VideoData>, MediaResult> r = GetPicture(); + if (r.isOk()) { + results.AppendElement(r.unwrap()); + } else { + MediaResult rs = r.unwrapErr(); + if (rs.Code() == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) { + // No frames ready to return. This is not an error, in some + // circumstances, we need to feed it with a certain amount of frames + // before we get a picture. + continue; + } + // Skip if rs is NS_OK, which can happen if picture layout is I400. + if (NS_FAILED(rs.Code())) { + return DecodePromise::CreateAndReject(rs, __func__); + } + } + } while (data.sz > 0); + + return DecodePromise::CreateAndResolve(std::move(results), __func__); +} + +Result<already_AddRefed<VideoData>, MediaResult> DAV1DDecoder::GetPicture() { + class Dav1dPictureWrapper { + public: + Dav1dPicture* operator&() { return &p; } + const Dav1dPicture& operator*() const { return p; } + ~Dav1dPictureWrapper() { dav1d_picture_unref(&p); } + + private: + Dav1dPicture p = Dav1dPicture(); + }; + Dav1dPictureWrapper picture; + + int res = dav1d_get_picture(mContext, &picture); + if (res < 0) { + auto r = MediaResult(res == DAV1D_ERR(EAGAIN) + ? NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA + : NS_ERROR_DOM_MEDIA_DECODE_ERR, + RESULT_DETAIL("dav1d_get_picture: %d", res)); + LOG("%s", r.Message().get()); + return Err(r); + } + + if ((*picture).p.layout == DAV1D_PIXEL_LAYOUT_I400) { + // Use NS_OK to indicate that this picture should be skipped. + auto r = MediaResult( + NS_OK, + RESULT_DETAIL("I400 picture: No chroma data to construct an image")); + LOG("%s", r.Message().get()); + return Err(r); + } + + Result<already_AddRefed<VideoData>, MediaResult> r = ConstructImage(*picture); + return r.mapErr([&](const MediaResult& aResult) { + LOG("ConstructImage (%ux%u display %ux%u picture %ux%u ) error - %s: %s", + (*picture).p.w, (*picture).p.h, mInfo.mDisplay.width, + mInfo.mDisplay.height, mInfo.mImage.width, mInfo.mImage.height, + aResult.ErrorName().get(), aResult.Message().get()); + return aResult; + }); +} + +/* static */ +Maybe<gfx::YUVColorSpace> DAV1DDecoder::GetColorSpace( + const Dav1dPicture& aPicture, LazyLogModule& aLogger) { + // When returning Nothing(), the caller chooses the appropriate default. + if (!aPicture.seq_hdr || !aPicture.seq_hdr->color_description_present) { + return Nothing(); + } + + return gfxUtils::CicpToColorSpace( + static_cast<gfx::CICP::MatrixCoefficients>(aPicture.seq_hdr->mtrx), + static_cast<gfx::CICP::ColourPrimaries>(aPicture.seq_hdr->pri), aLogger); +} + +/* static */ +Maybe<gfx::ColorSpace2> DAV1DDecoder::GetColorPrimaries( + const Dav1dPicture& aPicture, LazyLogModule& aLogger) { + // When returning Nothing(), the caller chooses the appropriate default. + if (!aPicture.seq_hdr || !aPicture.seq_hdr->color_description_present) { + return Nothing(); + } + + return gfxUtils::CicpToColorPrimaries( + static_cast<gfx::CICP::ColourPrimaries>(aPicture.seq_hdr->pri), aLogger); +} + +Result<already_AddRefed<VideoData>, MediaResult> DAV1DDecoder::ConstructImage( + const Dav1dPicture& aPicture) { + VideoData::YCbCrBuffer b; + if (aPicture.p.bpc == 10) { + b.mColorDepth = gfx::ColorDepth::COLOR_10; + } else if (aPicture.p.bpc == 12) { + b.mColorDepth = gfx::ColorDepth::COLOR_12; + } else { + b.mColorDepth = gfx::ColorDepth::COLOR_8; + } + + b.mYUVColorSpace = + DAV1DDecoder::GetColorSpace(aPicture, sPDMLog) + .valueOr(DefaultColorSpace({aPicture.p.w, aPicture.p.h})); + b.mColorPrimaries = DAV1DDecoder::GetColorPrimaries(aPicture, sPDMLog) + .valueOr(gfx::ColorSpace2::BT709); + b.mColorRange = aPicture.seq_hdr->color_range ? gfx::ColorRange::FULL + : gfx::ColorRange::LIMITED; + + b.mPlanes[0].mData = static_cast<uint8_t*>(aPicture.data[0]); + b.mPlanes[0].mStride = aPicture.stride[0]; + b.mPlanes[0].mHeight = aPicture.p.h; + b.mPlanes[0].mWidth = aPicture.p.w; + b.mPlanes[0].mSkip = 0; + + b.mPlanes[1].mData = static_cast<uint8_t*>(aPicture.data[1]); + b.mPlanes[1].mStride = aPicture.stride[1]; + b.mPlanes[1].mSkip = 0; + + b.mPlanes[2].mData = static_cast<uint8_t*>(aPicture.data[2]); + b.mPlanes[2].mStride = aPicture.stride[1]; + b.mPlanes[2].mSkip = 0; + + // https://code.videolan.org/videolan/dav1d/blob/master/tools/output/yuv.c#L67 + const int ss_ver = aPicture.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = aPicture.p.layout != DAV1D_PIXEL_LAYOUT_I444; + + b.mPlanes[1].mHeight = (aPicture.p.h + ss_ver) >> ss_ver; + b.mPlanes[1].mWidth = (aPicture.p.w + ss_hor) >> ss_hor; + + b.mPlanes[2].mHeight = (aPicture.p.h + ss_ver) >> ss_ver; + b.mPlanes[2].mWidth = (aPicture.p.w + ss_hor) >> ss_hor; + + if (ss_ver) { + b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT; + } else if (ss_hor) { + b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH; + } + + // Timestamp, duration and offset used here are wrong. + // We need to take those values from the decoder. Latest + // dav1d version allows for that. + media::TimeUnit timecode = + media::TimeUnit::FromMicroseconds(aPicture.m.timestamp); + media::TimeUnit duration = + media::TimeUnit::FromMicroseconds(aPicture.m.duration); + int64_t offset = aPicture.m.offset; + bool keyframe = aPicture.frame_hdr->frame_type == DAV1D_FRAME_TYPE_KEY; + + mPerformanceRecorder.Record(aPicture.m.timestamp, [&](DecodeStage& aStage) { + aStage.SetResolution(aPicture.p.w, aPicture.p.h); + auto format = [&]() -> Maybe<DecodeStage::ImageFormat> { + switch (aPicture.p.layout) { + case DAV1D_PIXEL_LAYOUT_I420: + return Some(DecodeStage::YUV420P); + case DAV1D_PIXEL_LAYOUT_I422: + return Some(DecodeStage::YUV422P); + case DAV1D_PIXEL_LAYOUT_I444: + return Some(DecodeStage::YUV444P); + default: + return Nothing(); + } + }(); + format.apply([&](auto& aFmt) { aStage.SetImageFormat(aFmt); }); + aStage.SetYUVColorSpace(b.mYUVColorSpace); + aStage.SetColorRange(b.mColorRange); + aStage.SetColorDepth(b.mColorDepth); + }); + + return VideoData::CreateAndCopyData( + mInfo, mImageContainer, offset, timecode, duration, b, keyframe, timecode, + mInfo.ScaledImageRect(aPicture.p.w, aPicture.p.h), mImageAllocator); +} + +RefPtr<MediaDataDecoder::DecodePromise> DAV1DDecoder::Drain() { + RefPtr<DAV1DDecoder> self = this; + return InvokeAsync(mTaskQueue, __func__, [self, this] { + DecodedData results; + while (true) { + Result<already_AddRefed<VideoData>, MediaResult> r = GetPicture(); + if (r.isOk()) { + results.AppendElement(r.unwrap()); + } else { + MediaResult rs = r.unwrapErr(); + if (rs.Code() == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) { + break; + } + // Skip if rs is NS_OK, which can happen if picture layout is I400. + if (NS_FAILED(rs.Code())) { + return DecodePromise::CreateAndReject(rs, __func__); + } + } + } + return DecodePromise::CreateAndResolve(std::move(results), __func__); + }); +} + +RefPtr<MediaDataDecoder::FlushPromise> DAV1DDecoder::Flush() { + RefPtr<DAV1DDecoder> self = this; + return InvokeAsync(mTaskQueue, __func__, [this, self]() { + dav1d_flush(self->mContext); + mPerformanceRecorder.Record(std::numeric_limits<int64_t>::max()); + return FlushPromise::CreateAndResolve(true, __func__); + }); +} + +RefPtr<ShutdownPromise> DAV1DDecoder::Shutdown() { + RefPtr<DAV1DDecoder> self = this; + return InvokeAsync(mTaskQueue, __func__, [self]() { + dav1d_close(&self->mContext); + return self->mTaskQueue->BeginShutdown(); + }); +} + +} // namespace mozilla +#undef LOG diff --git a/dom/media/platforms/agnostic/DAV1DDecoder.h b/dom/media/platforms/agnostic/DAV1DDecoder.h new file mode 100644 index 0000000000..d928846e36 --- /dev/null +++ b/dom/media/platforms/agnostic/DAV1DDecoder.h @@ -0,0 +1,71 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#if !defined(DAV1DDecoder_h_) +# define DAV1DDecoder_h_ + +# include "PerformanceRecorder.h" +# include "PlatformDecoderModule.h" +# include "dav1d/dav1d.h" +# include "mozilla/Result.h" +# include "nsRefPtrHashtable.h" + +namespace mozilla { + +DDLoggedTypeDeclNameAndBase(DAV1DDecoder, MediaDataDecoder); + +typedef nsRefPtrHashtable<nsPtrHashKey<const uint8_t>, MediaRawData> + MediaRawDataHashtable; + +class DAV1DDecoder final : public MediaDataDecoder, + public DecoderDoctorLifeLogger<DAV1DDecoder> { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DAV1DDecoder, final); + + explicit DAV1DDecoder(const CreateDecoderParams& aParams); + + RefPtr<InitPromise> Init() override; + RefPtr<DecodePromise> Decode(MediaRawData* aSample) override; + RefPtr<DecodePromise> Drain() override; + RefPtr<FlushPromise> Flush() override; + RefPtr<ShutdownPromise> Shutdown() override; + nsCString GetDescriptionName() const override { + return "av1 libdav1d video decoder"_ns; + } + nsCString GetCodecName() const override { return "av1"_ns; } + + void ReleaseDataBuffer(const uint8_t* buf); + + static Maybe<gfx::YUVColorSpace> GetColorSpace(const Dav1dPicture& aPicture, + LazyLogModule& aLogger); + + static Maybe<gfx::ColorSpace2> GetColorPrimaries(const Dav1dPicture& aPicture, + LazyLogModule& aLogger); + + private: + virtual ~DAV1DDecoder(); + RefPtr<DecodePromise> InvokeDecode(MediaRawData* aSample); + Result<already_AddRefed<VideoData>, MediaResult> GetPicture(); + Result<already_AddRefed<VideoData>, MediaResult> ConstructImage( + const Dav1dPicture& aPicture); + + Dav1dContext* mContext = nullptr; + + const VideoInfo mInfo; + const RefPtr<TaskQueue> mTaskQueue; + const RefPtr<layers::ImageContainer> mImageContainer; + const RefPtr<layers::KnowsCompositor> mImageAllocator; + const Maybe<TrackingId> mTrackingId; + const bool mLowLatency; + PerformanceRecorderMulti<DecodeStage> mPerformanceRecorder; + + // Keep the buffers alive until dav1d + // does not need them any more. + MediaRawDataHashtable mDecodingBuffers; +}; + +} // namespace mozilla + +#endif // DAV1DDecoder_h_ diff --git a/dom/media/platforms/agnostic/DummyMediaDataDecoder.cpp b/dom/media/platforms/agnostic/DummyMediaDataDecoder.cpp new file mode 100644 index 0000000000..172c7f3dd9 --- /dev/null +++ b/dom/media/platforms/agnostic/DummyMediaDataDecoder.cpp @@ -0,0 +1,80 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "DummyMediaDataDecoder.h" +#include "AnnexB.h" +#include "H264.h" +#include "MP4Decoder.h" + +namespace mozilla { + +DummyDataCreator::~DummyDataCreator() = default; + +DummyMediaDataDecoder::DummyMediaDataDecoder( + UniquePtr<DummyDataCreator>&& aCreator, const nsACString& aDescription, + const CreateDecoderParams& aParams) + : mCreator(std::move(aCreator)), + mIsH264(MP4Decoder::IsH264(aParams.mConfig.mMimeType)), + mMaxRefFrames(mIsH264 ? H264::HasSPS(aParams.VideoConfig().mExtraData) + ? H264::ComputeMaxRefFrames( + aParams.VideoConfig().mExtraData) + : 16 + : 0), + mType(aParams.mConfig.GetType()), + mDescription(aDescription) {} + +RefPtr<MediaDataDecoder::InitPromise> DummyMediaDataDecoder::Init() { + return InitPromise::CreateAndResolve(mType, __func__); +} + +RefPtr<ShutdownPromise> DummyMediaDataDecoder::Shutdown() { + return ShutdownPromise::CreateAndResolve(true, __func__); +} + +RefPtr<MediaDataDecoder::DecodePromise> DummyMediaDataDecoder::Decode( + MediaRawData* aSample) { + RefPtr<MediaData> data = mCreator->Create(aSample); + + if (!data) { + return DecodePromise::CreateAndReject(NS_ERROR_OUT_OF_MEMORY, __func__); + } + + // Frames come out in DTS order but we need to output them in PTS order. + mReorderQueue.Push(std::move(data)); + + if (mReorderQueue.Length() > mMaxRefFrames) { + return DecodePromise::CreateAndResolve(DecodedData{mReorderQueue.Pop()}, + __func__); + } + return DecodePromise::CreateAndResolve(DecodedData(), __func__); +} + +RefPtr<MediaDataDecoder::DecodePromise> DummyMediaDataDecoder::Drain() { + DecodedData samples; + while (!mReorderQueue.IsEmpty()) { + samples.AppendElement(mReorderQueue.Pop()); + } + return DecodePromise::CreateAndResolve(std::move(samples), __func__); +} + +RefPtr<MediaDataDecoder::FlushPromise> DummyMediaDataDecoder::Flush() { + mReorderQueue.Clear(); + return FlushPromise::CreateAndResolve(true, __func__); +} + +nsCString DummyMediaDataDecoder::GetDescriptionName() const { + return "blank media data decoder"_ns; +} + +nsCString DummyMediaDataDecoder::GetCodecName() const { return "unknown"_ns; } + +MediaDataDecoder::ConversionRequired DummyMediaDataDecoder::NeedsConversion() + const { + return mIsH264 ? ConversionRequired::kNeedAVCC + : ConversionRequired::kNeedNone; +} + +} // namespace mozilla diff --git a/dom/media/platforms/agnostic/DummyMediaDataDecoder.h b/dom/media/platforms/agnostic/DummyMediaDataDecoder.h new file mode 100644 index 0000000000..562d289bd9 --- /dev/null +++ b/dom/media/platforms/agnostic/DummyMediaDataDecoder.h @@ -0,0 +1,68 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(DummyMediaDataDecoder_h_) +# define DummyMediaDataDecoder_h_ + +# include "MediaInfo.h" +# include "mozilla/UniquePtr.h" +# include "PlatformDecoderModule.h" +# include "ReorderQueue.h" + +namespace mozilla { + +class MediaRawData; + +class DummyDataCreator { + public: + virtual ~DummyDataCreator(); + virtual already_AddRefed<MediaData> Create(MediaRawData* aSample) = 0; +}; + +DDLoggedTypeDeclNameAndBase(DummyMediaDataDecoder, MediaDataDecoder); + +// Decoder that uses a passed in object's Create function to create Null +// MediaData objects. +class DummyMediaDataDecoder final + : public MediaDataDecoder, + public DecoderDoctorLifeLogger<DummyMediaDataDecoder> { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DummyMediaDataDecoder, final); + + DummyMediaDataDecoder(UniquePtr<DummyDataCreator>&& aCreator, + const nsACString& aDescription, + const CreateDecoderParams& aParams); + + RefPtr<InitPromise> Init() override; + + RefPtr<ShutdownPromise> Shutdown() override; + + RefPtr<DecodePromise> Decode(MediaRawData* aSample) override; + + RefPtr<DecodePromise> Drain() override; + + RefPtr<FlushPromise> Flush() override; + + nsCString GetDescriptionName() const override; + + nsCString GetCodecName() const override; + + ConversionRequired NeedsConversion() const override; + + private: + ~DummyMediaDataDecoder() = default; + + UniquePtr<DummyDataCreator> mCreator; + const bool mIsH264; + const uint32_t mMaxRefFrames; + ReorderQueue mReorderQueue; + TrackInfo::TrackType mType; + nsCString mDescription; +}; + +} // namespace mozilla + +#endif // !defined(DummyMediaDataDecoder_h_) diff --git a/dom/media/platforms/agnostic/NullDecoderModule.cpp b/dom/media/platforms/agnostic/NullDecoderModule.cpp new file mode 100644 index 0000000000..2893a4af67 --- /dev/null +++ b/dom/media/platforms/agnostic/NullDecoderModule.cpp @@ -0,0 +1,57 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "DummyMediaDataDecoder.h" +#include "ImageContainer.h" + +namespace mozilla { + +class NullVideoDataCreator : public DummyDataCreator { + public: + NullVideoDataCreator() = default; + + already_AddRefed<MediaData> Create(MediaRawData* aSample) override { + // Create a dummy VideoData with an empty image. This gives us something to + // send to media streams if necessary. + RefPtr<layers::PlanarYCbCrImage> image = + new layers::RecyclingPlanarYCbCrImage(new layers::BufferRecycleBin()); + return VideoData::CreateFromImage(gfx::IntSize(), aSample->mOffset, + aSample->mTime, aSample->mDuration, image, + aSample->mKeyframe, aSample->mTimecode); + } +}; + +class NullDecoderModule : public PlatformDecoderModule { + public: + // Decode thread. + already_AddRefed<MediaDataDecoder> CreateVideoDecoder( + const CreateDecoderParams& aParams) override { + UniquePtr<DummyDataCreator> creator = MakeUnique<NullVideoDataCreator>(); + RefPtr<MediaDataDecoder> decoder = new DummyMediaDataDecoder( + std::move(creator), "null media data decoder"_ns, aParams); + return decoder.forget(); + } + + // Decode thread. + already_AddRefed<MediaDataDecoder> CreateAudioDecoder( + const CreateDecoderParams& aParams) override { + MOZ_ASSERT(false, "Audio decoders are unsupported."); + return nullptr; + } + + media::DecodeSupportSet SupportsMimeType( + const nsACString& aMimeType, + DecoderDoctorDiagnostics* aDiagnostics) const override { + return media::DecodeSupport::SoftwareDecode; + } +}; + +already_AddRefed<PlatformDecoderModule> CreateNullDecoderModule() { + RefPtr<PlatformDecoderModule> pdm = new NullDecoderModule(); + return pdm.forget(); +} + +} // namespace mozilla diff --git a/dom/media/platforms/agnostic/TheoraDecoder.cpp b/dom/media/platforms/agnostic/TheoraDecoder.cpp new file mode 100644 index 0000000000..d60093a204 --- /dev/null +++ b/dom/media/platforms/agnostic/TheoraDecoder.cpp @@ -0,0 +1,269 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "TheoraDecoder.h" + +#include <algorithm> +#include <ogg/ogg.h> + +#include "ImageContainer.h" +#include "TimeUnits.h" +#include "XiphExtradata.h" +#include "gfx2DGlue.h" +#include "mozilla/PodOperations.h" +#include "mozilla/TaskQueue.h" +#include "nsError.h" +#include "PerformanceRecorder.h" +#include "VideoUtils.h" + +#undef LOG +#define LOG(arg, ...) \ + DDMOZ_LOG(gMediaDecoderLog, mozilla::LogLevel::Debug, "::%s: " arg, \ + __func__, ##__VA_ARGS__) + +namespace mozilla { + +using namespace gfx; +using namespace layers; + +extern LazyLogModule gMediaDecoderLog; + +ogg_packet InitTheoraPacket(const unsigned char* aData, size_t aLength, + bool aBOS, bool aEOS, int64_t aGranulepos, + int64_t aPacketNo) { + ogg_packet packet; + packet.packet = const_cast<unsigned char*>(aData); + packet.bytes = aLength; + packet.b_o_s = aBOS; + packet.e_o_s = aEOS; + packet.granulepos = aGranulepos; + packet.packetno = aPacketNo; + return packet; +} + +TheoraDecoder::TheoraDecoder(const CreateDecoderParams& aParams) + : mImageAllocator(aParams.mKnowsCompositor), + mImageContainer(aParams.mImageContainer), + mTaskQueue(TaskQueue::Create( + GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER), + "TheoraDecoder")), + mTheoraInfo{}, + mTheoraComment{}, + mTheoraSetupInfo(nullptr), + mTheoraDecoderContext(nullptr), + mPacketCount(0), + mInfo(aParams.VideoConfig()), + mTrackingId(aParams.mTrackingId) { + MOZ_COUNT_CTOR(TheoraDecoder); +} + +TheoraDecoder::~TheoraDecoder() { + MOZ_COUNT_DTOR(TheoraDecoder); + th_setup_free(mTheoraSetupInfo); + th_comment_clear(&mTheoraComment); + th_info_clear(&mTheoraInfo); +} + +RefPtr<ShutdownPromise> TheoraDecoder::Shutdown() { + RefPtr<TheoraDecoder> self = this; + return InvokeAsync(mTaskQueue, __func__, [self, this]() { + if (mTheoraDecoderContext) { + th_decode_free(mTheoraDecoderContext); + mTheoraDecoderContext = nullptr; + } + return mTaskQueue->BeginShutdown(); + }); +} + +RefPtr<MediaDataDecoder::InitPromise> TheoraDecoder::Init() { + th_comment_init(&mTheoraComment); + th_info_init(&mTheoraInfo); + + nsTArray<unsigned char*> headers; + nsTArray<size_t> headerLens; + if (!XiphExtradataToHeaders(headers, headerLens, + mInfo.mCodecSpecificConfig->Elements(), + mInfo.mCodecSpecificConfig->Length())) { + return InitPromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("Could not get theora header.")), + __func__); + } + for (size_t i = 0; i < headers.Length(); i++) { + if (NS_FAILED(DoDecodeHeader(headers[i], headerLens[i]))) { + return InitPromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("Could not decode theora header.")), + __func__); + } + } + if (mPacketCount != 3) { + return InitPromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("Packet count is wrong.")), + __func__); + } + + mTheoraDecoderContext = th_decode_alloc(&mTheoraInfo, mTheoraSetupInfo); + if (mTheoraDecoderContext) { + return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__); + } else { + return InitPromise::CreateAndReject( + MediaResult(NS_ERROR_OUT_OF_MEMORY, + RESULT_DETAIL("Could not allocate theora decoder.")), + __func__); + } +} + +RefPtr<MediaDataDecoder::FlushPromise> TheoraDecoder::Flush() { + return InvokeAsync(mTaskQueue, __func__, []() { + return FlushPromise::CreateAndResolve(true, __func__); + }); +} + +nsresult TheoraDecoder::DoDecodeHeader(const unsigned char* aData, + size_t aLength) { + bool bos = mPacketCount == 0; + ogg_packet pkt = + InitTheoraPacket(aData, aLength, bos, false, 0, mPacketCount++); + + int r = th_decode_headerin(&mTheoraInfo, &mTheoraComment, &mTheoraSetupInfo, + &pkt); + return r > 0 ? NS_OK : NS_ERROR_FAILURE; +} + +RefPtr<MediaDataDecoder::DecodePromise> TheoraDecoder::ProcessDecode( + MediaRawData* aSample) { + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + + MediaInfoFlag flag = MediaInfoFlag::None; + flag |= (aSample->mKeyframe ? MediaInfoFlag::KeyFrame + : MediaInfoFlag::NonKeyFrame); + flag |= MediaInfoFlag::SoftwareDecoding; + flag |= MediaInfoFlag::VIDEO_THEORA; + Maybe<PerformanceRecorder<DecodeStage>> rec = + mTrackingId.map([&](const auto& aId) { + return PerformanceRecorder<DecodeStage>("TheoraDecoder"_ns, aId, flag); + }); + + const unsigned char* aData = aSample->Data(); + size_t aLength = aSample->Size(); + + bool bos = mPacketCount == 0; + ogg_packet pkt = + InitTheoraPacket(aData, aLength, bos, false, + aSample->mTimecode.ToMicroseconds(), mPacketCount++); + + int ret = th_decode_packetin(mTheoraDecoderContext, &pkt, nullptr); + if (ret == 0 || ret == TH_DUPFRAME) { + th_ycbcr_buffer ycbcr; + th_decode_ycbcr_out(mTheoraDecoderContext, ycbcr); + + int hdec = !(mTheoraInfo.pixel_fmt & 1); + int vdec = !(mTheoraInfo.pixel_fmt & 2); + + VideoData::YCbCrBuffer b; + b.mPlanes[0].mData = ycbcr[0].data; + b.mPlanes[0].mStride = ycbcr[0].stride; + b.mPlanes[0].mHeight = mTheoraInfo.frame_height; + b.mPlanes[0].mWidth = mTheoraInfo.frame_width; + b.mPlanes[0].mSkip = 0; + + b.mPlanes[1].mData = ycbcr[1].data; + b.mPlanes[1].mStride = ycbcr[1].stride; + b.mPlanes[1].mHeight = mTheoraInfo.frame_height >> vdec; + b.mPlanes[1].mWidth = mTheoraInfo.frame_width >> hdec; + b.mPlanes[1].mSkip = 0; + + b.mPlanes[2].mData = ycbcr[2].data; + b.mPlanes[2].mStride = ycbcr[2].stride; + b.mPlanes[2].mHeight = mTheoraInfo.frame_height >> vdec; + b.mPlanes[2].mWidth = mTheoraInfo.frame_width >> hdec; + b.mPlanes[2].mSkip = 0; + + if (vdec) { + b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT; + } else if (hdec) { + b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH; + } + + b.mYUVColorSpace = + DefaultColorSpace({mTheoraInfo.frame_width, mTheoraInfo.frame_height}); + + IntRect pictureArea(mTheoraInfo.pic_x, mTheoraInfo.pic_y, + mTheoraInfo.pic_width, mTheoraInfo.pic_height); + + VideoInfo info; + info.mDisplay = mInfo.mDisplay; + Result<already_AddRefed<VideoData>, MediaResult> r = + VideoData::CreateAndCopyData( + info, mImageContainer, aSample->mOffset, aSample->mTime, + aSample->mDuration, b, aSample->mKeyframe, aSample->mTimecode, + mInfo.ScaledImageRect(mTheoraInfo.frame_width, + mTheoraInfo.frame_height), + mImageAllocator); + if (r.isErr()) { + LOG("Image allocation error source %ux%u display %ux%u picture %ux%u", + mTheoraInfo.frame_width, mTheoraInfo.frame_height, + mInfo.mDisplay.width, mInfo.mDisplay.height, mInfo.mImage.width, + mInfo.mImage.height); + return DecodePromise::CreateAndReject(r.unwrapErr(), __func__); + } + + RefPtr<VideoData> v = r.unwrap(); + MOZ_ASSERT(v); + + rec.apply([&](auto& aRec) { + aRec.Record([&](DecodeStage& aStage) { + aStage.SetResolution(static_cast<int>(mTheoraInfo.frame_width), + static_cast<int>(mTheoraInfo.frame_height)); + auto format = [&]() -> Maybe<DecodeStage::ImageFormat> { + switch (mTheoraInfo.pixel_fmt) { + case TH_PF_420: + return Some(DecodeStage::YUV420P); + case TH_PF_422: + return Some(DecodeStage::YUV422P); + case TH_PF_444: + return Some(DecodeStage::YUV444P); + default: + return Nothing(); + } + }(); + format.apply([&](auto& aFmt) { aStage.SetImageFormat(aFmt); }); + aStage.SetYUVColorSpace(b.mYUVColorSpace); + aStage.SetColorRange(b.mColorRange); + aStage.SetColorDepth(b.mColorDepth); + }); + }); + + return DecodePromise::CreateAndResolve(DecodedData{v}, __func__); + } + LOG("Theora Decode error: %d", ret); + return DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, + RESULT_DETAIL("Theora decode error:%d", ret)), + __func__); +} + +RefPtr<MediaDataDecoder::DecodePromise> TheoraDecoder::Decode( + MediaRawData* aSample) { + return InvokeAsync<MediaRawData*>(mTaskQueue, this, __func__, + &TheoraDecoder::ProcessDecode, aSample); +} + +RefPtr<MediaDataDecoder::DecodePromise> TheoraDecoder::Drain() { + return InvokeAsync(mTaskQueue, __func__, [] { + return DecodePromise::CreateAndResolve(DecodedData(), __func__); + }); +} + +/* static */ +bool TheoraDecoder::IsTheora(const nsACString& aMimeType) { + return aMimeType.EqualsLiteral("video/theora"); +} + +} // namespace mozilla +#undef LOG diff --git a/dom/media/platforms/agnostic/TheoraDecoder.h b/dom/media/platforms/agnostic/TheoraDecoder.h new file mode 100644 index 0000000000..f209bb61ec --- /dev/null +++ b/dom/media/platforms/agnostic/TheoraDecoder.h @@ -0,0 +1,63 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#if !defined(TheoraDecoder_h_) +# define TheoraDecoder_h_ + +# include <stdint.h> + +# include "PlatformDecoderModule.h" +# include <theora/theoradec.h> + +namespace mozilla { + +DDLoggedTypeDeclNameAndBase(TheoraDecoder, MediaDataDecoder); + +class TheoraDecoder final : public MediaDataDecoder, + public DecoderDoctorLifeLogger<TheoraDecoder> { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TheoraDecoder, final); + + explicit TheoraDecoder(const CreateDecoderParams& aParams); + + RefPtr<InitPromise> Init() override; + RefPtr<DecodePromise> Decode(MediaRawData* aSample) override; + RefPtr<DecodePromise> Drain() override; + RefPtr<FlushPromise> Flush() override; + RefPtr<ShutdownPromise> Shutdown() override; + + // Return true if mimetype is a Theora codec + static bool IsTheora(const nsACString& aMimeType); + + nsCString GetDescriptionName() const override { + return "theora video decoder"_ns; + } + + nsCString GetCodecName() const override { return "theora"_ns; } + + private: + ~TheoraDecoder(); + nsresult DoDecodeHeader(const unsigned char* aData, size_t aLength); + + RefPtr<DecodePromise> ProcessDecode(MediaRawData* aSample); + + const RefPtr<layers::KnowsCompositor> mImageAllocator; + const RefPtr<layers::ImageContainer> mImageContainer; + const RefPtr<TaskQueue> mTaskQueue; + + // Theora header & decoder state + th_info mTheoraInfo; + th_comment mTheoraComment; + th_setup_info* mTheoraSetupInfo; + th_dec_ctx* mTheoraDecoderContext; + int mPacketCount; + + const VideoInfo mInfo; + const Maybe<TrackingId> mTrackingId; +}; + +} // namespace mozilla + +#endif diff --git a/dom/media/platforms/agnostic/VPXDecoder.cpp b/dom/media/platforms/agnostic/VPXDecoder.cpp new file mode 100644 index 0000000000..1b07606bd5 --- /dev/null +++ b/dom/media/platforms/agnostic/VPXDecoder.cpp @@ -0,0 +1,679 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "VPXDecoder.h" + +#include <algorithm> +#include <vpx/vpx_image.h> + +#include "BitReader.h" +#include "BitWriter.h" +#include "ImageContainer.h" +#include "TimeUnits.h" +#include "gfx2DGlue.h" +#include "gfxUtils.h" +#include "mozilla/PodOperations.h" +#include "mozilla/SyncRunnable.h" +#include "mozilla/TaskQueue.h" +#include "mozilla/Unused.h" +#include "nsError.h" +#include "PerformanceRecorder.h" +#include "prsystem.h" +#include "VideoUtils.h" + +#undef LOG +#define LOG(arg, ...) \ + DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, __func__, \ + ##__VA_ARGS__) + +namespace mozilla { + +using namespace gfx; +using namespace layers; + +static VPXDecoder::Codec MimeTypeToCodec(const nsACString& aMimeType) { + if (aMimeType.EqualsLiteral("video/vp8")) { + return VPXDecoder::Codec::VP8; + } else if (aMimeType.EqualsLiteral("video/vp9")) { + return VPXDecoder::Codec::VP9; + } + return VPXDecoder::Codec::Unknown; +} + +static nsresult InitContext(vpx_codec_ctx_t* aCtx, const VideoInfo& aInfo, + const VPXDecoder::Codec aCodec, bool aLowLatency) { + int decode_threads = 2; + + vpx_codec_iface_t* dx = nullptr; + if (aCodec == VPXDecoder::Codec::VP8) { + dx = vpx_codec_vp8_dx(); + } else if (aCodec == VPXDecoder::Codec::VP9) { + dx = vpx_codec_vp9_dx(); + if (aInfo.mDisplay.width >= 2048) { + decode_threads = 8; + } else if (aInfo.mDisplay.width >= 1024) { + decode_threads = 4; + } + } + decode_threads = std::min(decode_threads, PR_GetNumberOfProcessors()); + + vpx_codec_dec_cfg_t config; + config.threads = aLowLatency ? 1 : decode_threads; + config.w = config.h = 0; // set after decode + + if (!dx || vpx_codec_dec_init(aCtx, dx, &config, 0)) { + return NS_ERROR_FAILURE; + } + return NS_OK; +} + +VPXDecoder::VPXDecoder(const CreateDecoderParams& aParams) + : mImageContainer(aParams.mImageContainer), + mImageAllocator(aParams.mKnowsCompositor), + mTaskQueue(TaskQueue::Create( + GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER), "VPXDecoder")), + mInfo(aParams.VideoConfig()), + mCodec(MimeTypeToCodec(aParams.VideoConfig().mMimeType)), + mLowLatency( + aParams.mOptions.contains(CreateDecoderParams::Option::LowLatency)), + mTrackingId(aParams.mTrackingId) { + MOZ_COUNT_CTOR(VPXDecoder); + PodZero(&mVPX); + PodZero(&mVPXAlpha); +} + +VPXDecoder::~VPXDecoder() { MOZ_COUNT_DTOR(VPXDecoder); } + +RefPtr<ShutdownPromise> VPXDecoder::Shutdown() { + RefPtr<VPXDecoder> self = this; + return InvokeAsync(mTaskQueue, __func__, [self]() { + vpx_codec_destroy(&self->mVPX); + vpx_codec_destroy(&self->mVPXAlpha); + return self->mTaskQueue->BeginShutdown(); + }); +} + +RefPtr<MediaDataDecoder::InitPromise> VPXDecoder::Init() { + if (NS_FAILED(InitContext(&mVPX, mInfo, mCodec, mLowLatency))) { + return VPXDecoder::InitPromise::CreateAndReject( + NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + } + if (mInfo.HasAlpha()) { + if (NS_FAILED(InitContext(&mVPXAlpha, mInfo, mCodec, mLowLatency))) { + return VPXDecoder::InitPromise::CreateAndReject( + NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + } + } + return VPXDecoder::InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, + __func__); +} + +RefPtr<MediaDataDecoder::FlushPromise> VPXDecoder::Flush() { + return InvokeAsync(mTaskQueue, __func__, []() { + return FlushPromise::CreateAndResolve(true, __func__); + }); +} + +RefPtr<MediaDataDecoder::DecodePromise> VPXDecoder::ProcessDecode( + MediaRawData* aSample) { + MOZ_ASSERT(mTaskQueue->IsOnCurrentThread()); + + MediaInfoFlag flag = MediaInfoFlag::None; + flag |= (aSample->mKeyframe ? MediaInfoFlag::KeyFrame + : MediaInfoFlag::NonKeyFrame); + flag |= MediaInfoFlag::SoftwareDecoding; + switch (mCodec) { + case Codec::VP8: + flag |= MediaInfoFlag::VIDEO_VP8; + break; + case Codec::VP9: + flag |= MediaInfoFlag::VIDEO_VP9; + break; + default: + break; + } + flag |= MediaInfoFlag::VIDEO_THEORA; + auto rec = mTrackingId.map([&](const auto& aId) { + return PerformanceRecorder<DecodeStage>("VPXDecoder"_ns, aId, flag); + }); + + if (vpx_codec_err_t r = vpx_codec_decode(&mVPX, aSample->Data(), + aSample->Size(), nullptr, 0)) { + LOG("VPX Decode error: %s", vpx_codec_err_to_string(r)); + return DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, + RESULT_DETAIL("VPX error: %s", vpx_codec_err_to_string(r))), + __func__); + } + + vpx_codec_iter_t iter = nullptr; + vpx_image_t* img; + vpx_image_t* img_alpha = nullptr; + bool alpha_decoded = false; + DecodedData results; + + while ((img = vpx_codec_get_frame(&mVPX, &iter))) { + NS_ASSERTION(img->fmt == VPX_IMG_FMT_I420 || img->fmt == VPX_IMG_FMT_I444, + "WebM image format not I420 or I444"); + NS_ASSERTION(!alpha_decoded, + "Multiple frames per packet that contains alpha"); + + if (aSample->AlphaSize() > 0) { + if (!alpha_decoded) { + MediaResult rv = DecodeAlpha(&img_alpha, aSample); + if (NS_FAILED(rv)) { + return DecodePromise::CreateAndReject(rv, __func__); + } + alpha_decoded = true; + } + } + // Chroma shifts are rounded down as per the decoding examples in the SDK + VideoData::YCbCrBuffer b; + b.mPlanes[0].mData = img->planes[0]; + b.mPlanes[0].mStride = img->stride[0]; + b.mPlanes[0].mHeight = img->d_h; + b.mPlanes[0].mWidth = img->d_w; + b.mPlanes[0].mSkip = 0; + + b.mPlanes[1].mData = img->planes[1]; + b.mPlanes[1].mStride = img->stride[1]; + b.mPlanes[1].mSkip = 0; + + b.mPlanes[2].mData = img->planes[2]; + b.mPlanes[2].mStride = img->stride[2]; + b.mPlanes[2].mSkip = 0; + + if (img->fmt == VPX_IMG_FMT_I420) { + b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT; + + b.mPlanes[1].mHeight = (img->d_h + 1) >> img->y_chroma_shift; + b.mPlanes[1].mWidth = (img->d_w + 1) >> img->x_chroma_shift; + + b.mPlanes[2].mHeight = (img->d_h + 1) >> img->y_chroma_shift; + b.mPlanes[2].mWidth = (img->d_w + 1) >> img->x_chroma_shift; + } else if (img->fmt == VPX_IMG_FMT_I444) { + b.mPlanes[1].mHeight = img->d_h; + b.mPlanes[1].mWidth = img->d_w; + + b.mPlanes[2].mHeight = img->d_h; + b.mPlanes[2].mWidth = img->d_w; + } else { + LOG("VPX Unknown image format"); + return DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, + RESULT_DETAIL("VPX Unknown image format")), + __func__); + } + b.mYUVColorSpace = [&]() { + switch (img->cs) { + case VPX_CS_BT_601: + case VPX_CS_SMPTE_170: + case VPX_CS_SMPTE_240: + return gfx::YUVColorSpace::BT601; + case VPX_CS_BT_709: + return gfx::YUVColorSpace::BT709; + case VPX_CS_BT_2020: + return gfx::YUVColorSpace::BT2020; + default: + return DefaultColorSpace({img->d_w, img->d_h}); + } + }(); + b.mColorRange = img->range == VPX_CR_FULL_RANGE ? gfx::ColorRange::FULL + : gfx::ColorRange::LIMITED; + + RefPtr<VideoData> v; + if (!img_alpha) { + Result<already_AddRefed<VideoData>, MediaResult> r = + VideoData::CreateAndCopyData( + mInfo, mImageContainer, aSample->mOffset, aSample->mTime, + aSample->mDuration, b, aSample->mKeyframe, aSample->mTimecode, + mInfo.ScaledImageRect(img->d_w, img->d_h), mImageAllocator); + // TODO: Reject DecodePromise below with r's error return. + v = r.unwrapOr(nullptr); + } else { + VideoData::YCbCrBuffer::Plane alpha_plane; + alpha_plane.mData = img_alpha->planes[0]; + alpha_plane.mStride = img_alpha->stride[0]; + alpha_plane.mHeight = img_alpha->d_h; + alpha_plane.mWidth = img_alpha->d_w; + alpha_plane.mSkip = 0; + v = VideoData::CreateAndCopyData( + mInfo, mImageContainer, aSample->mOffset, aSample->mTime, + aSample->mDuration, b, alpha_plane, aSample->mKeyframe, + aSample->mTimecode, mInfo.ScaledImageRect(img->d_w, img->d_h)); + } + + if (!v) { + LOG("Image allocation error source %ux%u display %ux%u picture %ux%u", + img->d_w, img->d_h, mInfo.mDisplay.width, mInfo.mDisplay.height, + mInfo.mImage.width, mInfo.mImage.height); + return DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__), __func__); + } + + rec.apply([&](auto& aRec) { + return aRec.Record([&](DecodeStage& aStage) { + aStage.SetResolution(static_cast<int>(img->d_w), + static_cast<int>(img->d_h)); + auto format = [&]() -> Maybe<DecodeStage::ImageFormat> { + switch (img->fmt) { + case VPX_IMG_FMT_I420: + return Some(DecodeStage::YUV420P); + case VPX_IMG_FMT_I444: + return Some(DecodeStage::YUV444P); + default: + return Nothing(); + } + }(); + format.apply([&](auto& aFmt) { aStage.SetImageFormat(aFmt); }); + aStage.SetYUVColorSpace(b.mYUVColorSpace); + aStage.SetColorRange(b.mColorRange); + aStage.SetColorDepth(b.mColorDepth); + }); + }); + + results.AppendElement(std::move(v)); + } + return DecodePromise::CreateAndResolve(std::move(results), __func__); +} + +RefPtr<MediaDataDecoder::DecodePromise> VPXDecoder::Decode( + MediaRawData* aSample) { + return InvokeAsync<MediaRawData*>(mTaskQueue, this, __func__, + &VPXDecoder::ProcessDecode, aSample); +} + +RefPtr<MediaDataDecoder::DecodePromise> VPXDecoder::Drain() { + return InvokeAsync(mTaskQueue, __func__, [] { + return DecodePromise::CreateAndResolve(DecodedData(), __func__); + }); +} + +MediaResult VPXDecoder::DecodeAlpha(vpx_image_t** aImgAlpha, + const MediaRawData* aSample) { + vpx_codec_err_t r = vpx_codec_decode(&mVPXAlpha, aSample->AlphaData(), + aSample->AlphaSize(), nullptr, 0); + if (r) { + LOG("VPX decode alpha error: %s", vpx_codec_err_to_string(r)); + return MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, + RESULT_DETAIL("VPX decode alpha error: %s", + vpx_codec_err_to_string(r))); + } + + vpx_codec_iter_t iter = nullptr; + + *aImgAlpha = vpx_codec_get_frame(&mVPXAlpha, &iter); + NS_ASSERTION((*aImgAlpha)->fmt == VPX_IMG_FMT_I420 || + (*aImgAlpha)->fmt == VPX_IMG_FMT_I444, + "WebM image format not I420 or I444"); + + return NS_OK; +} + +nsCString VPXDecoder::GetCodecName() const { + switch (mCodec) { + case Codec::VP8: + return "vp8"_ns; + case Codec::VP9: + return "vp9"_ns; + default: + return "unknown"_ns; + } +} + +/* static */ +bool VPXDecoder::IsVPX(const nsACString& aMimeType, uint8_t aCodecMask) { + return ((aCodecMask & VPXDecoder::VP8) && + aMimeType.EqualsLiteral("video/vp8")) || + ((aCodecMask & VPXDecoder::VP9) && + aMimeType.EqualsLiteral("video/vp9")); +} + +/* static */ +bool VPXDecoder::IsVP8(const nsACString& aMimeType) { + return IsVPX(aMimeType, VPXDecoder::VP8); +} + +/* static */ +bool VPXDecoder::IsVP9(const nsACString& aMimeType) { + return IsVPX(aMimeType, VPXDecoder::VP9); +} + +/* static */ +bool VPXDecoder::IsKeyframe(Span<const uint8_t> aBuffer, Codec aCodec) { + VPXStreamInfo info; + return GetStreamInfo(aBuffer, info, aCodec) && info.mKeyFrame; +} + +/* static */ +gfx::IntSize VPXDecoder::GetFrameSize(Span<const uint8_t> aBuffer, + Codec aCodec) { + VPXStreamInfo info; + if (!GetStreamInfo(aBuffer, info, aCodec)) { + return gfx::IntSize(); + } + return info.mImage; +} + +/* static */ +gfx::IntSize VPXDecoder::GetDisplaySize(Span<const uint8_t> aBuffer, + Codec aCodec) { + VPXStreamInfo info; + if (!GetStreamInfo(aBuffer, info, aCodec)) { + return gfx::IntSize(); + } + return info.mDisplay; +} + +/* static */ +int VPXDecoder::GetVP9Profile(Span<const uint8_t> aBuffer) { + VPXStreamInfo info; + if (!GetStreamInfo(aBuffer, info, Codec::VP9)) { + return -1; + } + return info.mProfile; +} + +/* static */ +bool VPXDecoder::GetStreamInfo(Span<const uint8_t> aBuffer, + VPXDecoder::VPXStreamInfo& aInfo, Codec aCodec) { + if (aBuffer.IsEmpty()) { + // Can't be good. + return false; + } + + aInfo = VPXStreamInfo(); + + if (aCodec == Codec::VP8) { + aInfo.mKeyFrame = (aBuffer[0] & 1) == + 0; // frame type (0 for key frames, 1 for interframes) + if (!aInfo.mKeyFrame) { + // We can't retrieve the required information from interframes. + return true; + } + if (aBuffer.Length() < 10) { + return false; + } + uint8_t version = (aBuffer[0] >> 1) & 0x7; + if (version > 3) { + return false; + } + uint8_t start_code_byte_0 = aBuffer[3]; + uint8_t start_code_byte_1 = aBuffer[4]; + uint8_t start_code_byte_2 = aBuffer[5]; + if (start_code_byte_0 != 0x9d || start_code_byte_1 != 0x01 || + start_code_byte_2 != 0x2a) { + return false; + } + uint16_t width = (aBuffer[6] | aBuffer[7] << 8) & 0x3fff; + uint16_t height = (aBuffer[8] | aBuffer[9] << 8) & 0x3fff; + + // aspect ratio isn't found in the VP8 frame header. + aInfo.mImage = gfx::IntSize(width, height); + aInfo.mDisplayAndImageDifferent = false; + aInfo.mDisplay = aInfo.mImage; + return true; + } + + BitReader br(aBuffer.Elements(), aBuffer.Length() * 8); + uint32_t frameMarker = br.ReadBits(2); // frame_marker + if (frameMarker != 2) { + // That's not a valid vp9 header. + return false; + } + uint32_t profile = br.ReadBits(1); // profile_low_bit + profile |= br.ReadBits(1) << 1; // profile_high_bit + if (profile == 3) { + profile += br.ReadBits(1); // reserved_zero + if (profile > 3) { + // reserved_zero wasn't zero. + return false; + } + } + + aInfo.mProfile = profile; + + bool show_existing_frame = br.ReadBits(1); + if (show_existing_frame) { + if (profile == 3 && aBuffer.Length() < 2) { + return false; + } + Unused << br.ReadBits(3); // frame_to_show_map_idx + return true; + } + + if (aBuffer.Length() < 10) { + // Header too small; + return false; + } + + aInfo.mKeyFrame = !br.ReadBits(1); + bool show_frame = br.ReadBits(1); + bool error_resilient_mode = br.ReadBits(1); + + auto frame_sync_code = [&]() -> bool { + uint8_t frame_sync_byte_1 = br.ReadBits(8); + uint8_t frame_sync_byte_2 = br.ReadBits(8); + uint8_t frame_sync_byte_3 = br.ReadBits(8); + return frame_sync_byte_1 == 0x49 && frame_sync_byte_2 == 0x83 && + frame_sync_byte_3 == 0x42; + }; + + auto color_config = [&]() -> bool { + aInfo.mBitDepth = 8; + if (profile >= 2) { + bool ten_or_twelve_bit = br.ReadBits(1); + aInfo.mBitDepth = ten_or_twelve_bit ? 12 : 10; + } + aInfo.mColorSpace = br.ReadBits(3); + if (aInfo.mColorSpace != 7 /* CS_RGB */) { + aInfo.mFullRange = br.ReadBits(1); + if (profile == 1 || profile == 3) { + aInfo.mSubSampling_x = br.ReadBits(1); + aInfo.mSubSampling_y = br.ReadBits(1); + if (br.ReadBits(1)) { // reserved_zero + return false; + }; + } else { + aInfo.mSubSampling_x = true; + aInfo.mSubSampling_y = true; + } + } else { + aInfo.mFullRange = true; + if (profile == 1 || profile == 3) { + aInfo.mSubSampling_x = false; + aInfo.mSubSampling_y = false; + if (br.ReadBits(1)) { // reserved_zero + return false; + }; + } else { + // sRGB color space is only available with VP9 profile 1. + return false; + } + } + return true; + }; + + auto frame_size = [&]() { + int32_t width = static_cast<int32_t>(br.ReadBits(16)) + 1; + int32_t height = static_cast<int32_t>(br.ReadBits(16)) + 1; + aInfo.mImage = gfx::IntSize(width, height); + }; + + auto render_size = [&]() { + // render_and_frame_size_different + aInfo.mDisplayAndImageDifferent = br.ReadBits(1); + if (aInfo.mDisplayAndImageDifferent) { + int32_t width = static_cast<int32_t>(br.ReadBits(16)) + 1; + int32_t height = static_cast<int32_t>(br.ReadBits(16)) + 1; + aInfo.mDisplay = gfx::IntSize(width, height); + } else { + aInfo.mDisplay = aInfo.mImage; + } + }; + + if (aInfo.mKeyFrame) { + if (!frame_sync_code()) { + return false; + } + if (!color_config()) { + return false; + } + frame_size(); + render_size(); + } else { + bool intra_only = show_frame ? false : br.ReadBit(); + if (!error_resilient_mode) { + Unused << br.ReadBits(2); // reset_frame_context + } + if (intra_only) { + if (!frame_sync_code()) { + return false; + } + if (profile > 0) { + if (!color_config()) { + return false; + } + } else { + aInfo.mColorSpace = 1; // CS_BT_601 + aInfo.mSubSampling_x = true; + aInfo.mSubSampling_y = true; + aInfo.mBitDepth = 8; + } + Unused << br.ReadBits(8); // refresh_frame_flags + frame_size(); + render_size(); + } + } + return true; +} + +// Ref: "VP Codec ISO Media File Format Binding, v1.0, 2017-03-31" +// <https://www.webmproject.org/vp9/mp4/> +// +// class VPCodecConfigurationBox extends FullBox('vpcC', version = 1, 0) +// { +// VPCodecConfigurationRecord() vpcConfig; +// } +// +// aligned (8) class VPCodecConfigurationRecord { +// unsigned int (8) profile; +// unsigned int (8) level; +// unsigned int (4) bitDepth; +// unsigned int (3) chromaSubsampling; +// unsigned int (1) videoFullRangeFlag; +// unsigned int (8) colourPrimaries; +// unsigned int (8) transferCharacteristics; +// unsigned int (8) matrixCoefficients; +// unsigned int (16) codecIntializationDataSize; +// unsigned int (8)[] codecIntializationData; +// } + +/* static */ +void VPXDecoder::GetVPCCBox(MediaByteBuffer* aDestBox, + const VPXStreamInfo& aInfo) { + BitWriter writer(aDestBox); + + int chroma = [&]() { + if (aInfo.mSubSampling_x && aInfo.mSubSampling_y) { + return 1; // 420 Colocated; + } + if (aInfo.mSubSampling_x && !aInfo.mSubSampling_y) { + return 2; // 422 + } + if (!aInfo.mSubSampling_x && !aInfo.mSubSampling_y) { + return 3; // 444 + } + // This indicates 4:4:0 subsampling, which is not expressable in the + // 'vpcC' box. Default to 4:2:0. + return 1; + }(); + + writer.WriteU8(1); // version + writer.WriteBits(0, 24); // flags + + writer.WriteU8(aInfo.mProfile); // profile + writer.WriteU8(10); // level set it to 1.0 + + writer.WriteBits(aInfo.mBitDepth, 4); // bitdepth + writer.WriteBits(chroma, 3); // chroma + writer.WriteBit(aInfo.mFullRange); // full/restricted range + + // See VPXDecoder::VPXStreamInfo enums + writer.WriteU8(aInfo.mColorPrimaries); // color primaries + writer.WriteU8(aInfo.mTransferFunction); // transfer characteristics + writer.WriteU8(2); // matrix coefficients: unspecified + + writer.WriteBits(0, + 16); // codecIntializationDataSize (must be 0 for VP8/VP9) +} + +/* static */ +bool VPXDecoder::SetVideoInfo(VideoInfo* aDestInfo, const nsAString& aCodec) { + VPXDecoder::VPXStreamInfo info; + uint8_t level = 0; + uint8_t chroma = 1; + VideoColorSpace colorSpace; + if (!ExtractVPXCodecDetails(aCodec, info.mProfile, level, info.mBitDepth, + chroma, colorSpace)) { + return false; + } + + aDestInfo->mColorPrimaries = + gfxUtils::CicpToColorPrimaries(colorSpace.mPrimaries, sPDMLog); + aDestInfo->mTransferFunction = + gfxUtils::CicpToTransferFunction(colorSpace.mTransfer); + aDestInfo->mColorDepth = gfx::ColorDepthForBitDepth(info.mBitDepth); + VPXDecoder::SetChroma(info, chroma); + info.mFullRange = colorSpace.mRange == ColorRange::FULL; + RefPtr<MediaByteBuffer> extraData = new MediaByteBuffer(); + VPXDecoder::GetVPCCBox(extraData, info); + aDestInfo->mExtraData = extraData; + return true; +} + +/* static */ +void VPXDecoder::SetChroma(VPXStreamInfo& aDestInfo, uint8_t chroma) { + switch (chroma) { + case 0: + case 1: + aDestInfo.mSubSampling_x = true; + aDestInfo.mSubSampling_y = true; + break; + case 2: + aDestInfo.mSubSampling_x = true; + aDestInfo.mSubSampling_y = false; + break; + case 3: + aDestInfo.mSubSampling_x = false; + aDestInfo.mSubSampling_y = false; + break; + } +} + +/* static */ +void VPXDecoder::ReadVPCCBox(VPXStreamInfo& aDestInfo, MediaByteBuffer* aBox) { + BitReader reader(aBox); + + reader.ReadBits(8); // version + reader.ReadBits(24); // flags + aDestInfo.mProfile = reader.ReadBits(8); + reader.ReadBits(8); // level + + aDestInfo.mBitDepth = reader.ReadBits(4); + SetChroma(aDestInfo, reader.ReadBits(3)); + aDestInfo.mFullRange = reader.ReadBit(); + + aDestInfo.mColorPrimaries = reader.ReadBits(8); // color primaries + aDestInfo.mTransferFunction = reader.ReadBits(8); // transfer characteristics + reader.ReadBits(8); // matrix coefficients + + MOZ_ASSERT(reader.ReadBits(16) == + 0); // codecInitializationDataSize (must be 0 for VP8/VP9) +} + +} // namespace mozilla +#undef LOG diff --git a/dom/media/platforms/agnostic/VPXDecoder.h b/dom/media/platforms/agnostic/VPXDecoder.h new file mode 100644 index 0000000000..ea9073ad6b --- /dev/null +++ b/dom/media/platforms/agnostic/VPXDecoder.h @@ -0,0 +1,208 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#if !defined(VPXDecoder_h_) +# define VPXDecoder_h_ + +# include <stdint.h> +# include <vpx/vp8dx.h> +# include <vpx/vpx_codec.h> +# include <vpx/vpx_decoder.h> + +# include "PlatformDecoderModule.h" +# include "mozilla/Span.h" +# include "mozilla/gfx/Types.h" + +namespace mozilla { + +DDLoggedTypeDeclNameAndBase(VPXDecoder, MediaDataDecoder); + +class VPXDecoder final : public MediaDataDecoder, + public DecoderDoctorLifeLogger<VPXDecoder> { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VPXDecoder, final); + + explicit VPXDecoder(const CreateDecoderParams& aParams); + + RefPtr<InitPromise> Init() override; + RefPtr<DecodePromise> Decode(MediaRawData* aSample) override; + RefPtr<DecodePromise> Drain() override; + RefPtr<FlushPromise> Flush() override; + RefPtr<ShutdownPromise> Shutdown() override; + nsCString GetDescriptionName() const override { + return "libvpx video decoder"_ns; + } + nsCString GetCodecName() const override; + + enum Codec : uint8_t { + VP8 = 1 << 0, + VP9 = 1 << 1, + Unknown = 1 << 7, + }; + + // Return true if aMimeType is a one of the strings used by our demuxers to + // identify VPX of the specified type. Does not parse general content type + // strings, i.e. white space matters. + static bool IsVPX(const nsACString& aMimeType, + uint8_t aCodecMask = VP8 | VP9); + static bool IsVP8(const nsACString& aMimeType); + static bool IsVP9(const nsACString& aMimeType); + + // Return true if a sample is a keyframe for the specified codec. + static bool IsKeyframe(Span<const uint8_t> aBuffer, Codec aCodec); + + // Return the frame dimensions for a sample for the specified codec. + static gfx::IntSize GetFrameSize(Span<const uint8_t> aBuffer, Codec aCodec); + // Return the display dimensions for a sample for the specified codec. + static gfx::IntSize GetDisplaySize(Span<const uint8_t> aBuffer, Codec aCodec); + + // Return the VP9 profile as per https://www.webmproject.org/vp9/profiles/ + // Return negative value if error. + static int GetVP9Profile(Span<const uint8_t> aBuffer); + + struct VPXStreamInfo { + gfx::IntSize mImage; + bool mDisplayAndImageDifferent = false; + gfx::IntSize mDisplay; + bool mKeyFrame = false; + + uint8_t mProfile = 0; + uint8_t mBitDepth = 8; + /* + 0 CS_UNKNOWN Unknown (in this case the color space must be signaled outside + the VP9 bitstream). + 1 CS_BT_601 Rec. ITU-R BT.601-7 + 2 CS_BT_709 Rec. ITU-R BT.709-6 + 3 CS_SMPTE_170 SMPTE-170 + 4 CS_SMPTE_240 SMPTE-240 + 5 CS_BT_2020 Rec. ITU-R BT.2020-2 + 6 CS_RESERVED Reserved + 7 CS_RGB sRGB (IEC 61966-2-1) + */ + int mColorSpace = 1; // CS_BT_601 + + gfx::YUVColorSpace ColorSpace() const { + switch (mColorSpace) { + case 1: + case 3: + case 4: + return gfx::YUVColorSpace::BT601; + case 2: + return gfx::YUVColorSpace::BT709; + case 5: + return gfx::YUVColorSpace::BT2020; + default: + return gfx::YUVColorSpace::Default; + } + } + + uint8_t mColorPrimaries = gfx::CICP::ColourPrimaries::CP_UNSPECIFIED; + gfx::ColorSpace2 ColorPrimaries() const { + switch (mColorPrimaries) { + case gfx::CICP::ColourPrimaries::CP_BT709: + return gfx::ColorSpace2::BT709; + case gfx::CICP::ColourPrimaries::CP_UNSPECIFIED: + return gfx::ColorSpace2::BT709; + case gfx::CICP::ColourPrimaries::CP_BT2020: + return gfx::ColorSpace2::BT2020; + default: + return gfx::ColorSpace2::BT709; + } + } + + uint8_t mTransferFunction = + gfx::CICP::TransferCharacteristics::TC_UNSPECIFIED; + gfx::TransferFunction TransferFunction() const { + switch (mTransferFunction) { + case gfx::CICP::TransferCharacteristics::TC_BT709: + return gfx::TransferFunction::BT709; + case gfx::CICP::TransferCharacteristics::TC_SRGB: + return gfx::TransferFunction::SRGB; + case gfx::CICP::TransferCharacteristics::TC_SMPTE2084: + return gfx::TransferFunction::PQ; + case gfx::CICP::TransferCharacteristics::TC_HLG: + return gfx::TransferFunction::HLG; + default: + return gfx::TransferFunction::BT709; + } + } + + /* + mFullRange == false then: + For BitDepth equals 8: + Y is between 16 and 235 inclusive. + U and V are between 16 and 240 inclusive. + For BitDepth equals 10: + Y is between 64 and 940 inclusive. + U and V are between 64 and 960 inclusive. + For BitDepth equals 12: + Y is between 256 and 3760. + U and V are between 256 and 3840 inclusive. + mFullRange == true then: + No restriction on Y, U, V values. + */ + bool mFullRange = false; + + gfx::ColorRange ColorRange() const { + return mFullRange ? gfx::ColorRange::FULL : gfx::ColorRange::LIMITED; + } + + /* + Sub-sampling, used only for non sRGB colorspace. + subsampling_x subsampling_y Description + 0 0 YUV 4:4:4 + 0 1 YUV 4:4:0 + 1 0 YUV 4:2:2 + 1 1 YUV 4:2:0 + */ + bool mSubSampling_x = true; + bool mSubSampling_y = true; + + bool IsCompatible(const VPXStreamInfo& aOther) const { + return mImage == aOther.mImage && mProfile == aOther.mProfile && + mBitDepth == aOther.mBitDepth && + mSubSampling_x == aOther.mSubSampling_x && + mSubSampling_y == aOther.mSubSampling_y && + mColorSpace == aOther.mColorSpace && + mFullRange == aOther.mFullRange; + } + }; + + static bool GetStreamInfo(Span<const uint8_t> aBuffer, VPXStreamInfo& aInfo, + Codec aCodec); + + static void GetVPCCBox(MediaByteBuffer* aDestBox, const VPXStreamInfo& aInfo); + // Set extradata for a VP8/VP9 track, returning false if the codec was + // invalid. + static bool SetVideoInfo(VideoInfo* aDestInfo, const nsAString& aCodec); + + static void SetChroma(VPXStreamInfo& aDestInfo, uint8_t chroma); + static void ReadVPCCBox(VPXStreamInfo& aDestInfo, MediaByteBuffer* aBox); + + private: + ~VPXDecoder(); + RefPtr<DecodePromise> ProcessDecode(MediaRawData* aSample); + MediaResult DecodeAlpha(vpx_image_t** aImgAlpha, const MediaRawData* aSample); + + const RefPtr<layers::ImageContainer> mImageContainer; + RefPtr<layers::KnowsCompositor> mImageAllocator; + const RefPtr<TaskQueue> mTaskQueue; + + // VPx decoder state + vpx_codec_ctx_t mVPX; + + // VPx alpha decoder state + vpx_codec_ctx_t mVPXAlpha; + + const VideoInfo mInfo; + + const Codec mCodec; + const bool mLowLatency; + const Maybe<TrackingId> mTrackingId; +}; + +} // namespace mozilla + +#endif diff --git a/dom/media/platforms/agnostic/bytestreams/Adts.cpp b/dom/media/platforms/agnostic/bytestreams/Adts.cpp new file mode 100644 index 0000000000..5f31904d9c --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/Adts.cpp @@ -0,0 +1,94 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "Adts.h" +#include "MediaData.h" +#include "mozilla/Array.h" +#include "mozilla/ArrayUtils.h" + +namespace mozilla { + +static const int kADTSHeaderSize = 7; + +int8_t Adts::GetFrequencyIndex(uint32_t aSamplesPerSecond) { + static const uint32_t freq_lookup[] = {96000, 88200, 64000, 48000, 44100, + 32000, 24000, 22050, 16000, 12000, + 11025, 8000, 7350, 0}; + + int8_t i = 0; + while (freq_lookup[i] && aSamplesPerSecond < freq_lookup[i]) { + i++; + } + + if (!freq_lookup[i]) { + return -1; + } + + return i; +} + +bool Adts::ConvertSample(uint16_t aChannelCount, int8_t aFrequencyIndex, + int8_t aProfile, MediaRawData* aSample) { + size_t newSize = aSample->Size() + kADTSHeaderSize; + + // ADTS header uses 13 bits for packet size. + if (newSize >= (1 << 13) || aChannelCount > 15 || aFrequencyIndex < 0 || + aProfile < 1 || aProfile > 4) { + return false; + } + + Array<uint8_t, kADTSHeaderSize> header; + header[0] = 0xff; + header[1] = 0xf1; + header[2] = + ((aProfile - 1) << 6) + (aFrequencyIndex << 2) + (aChannelCount >> 2); + header[3] = ((aChannelCount & 0x3) << 6) + (newSize >> 11); + header[4] = (newSize & 0x7ff) >> 3; + header[5] = ((newSize & 7) << 5) + 0x1f; + header[6] = 0xfc; + + UniquePtr<MediaRawDataWriter> writer(aSample->CreateWriter()); + if (!writer->Prepend(&header[0], ArrayLength(header))) { + return false; + } + + if (aSample->mCrypto.IsEncrypted()) { + if (aSample->mCrypto.mPlainSizes.Length() == 0) { + writer->mCrypto.mPlainSizes.AppendElement(kADTSHeaderSize); + writer->mCrypto.mEncryptedSizes.AppendElement(aSample->Size() - + kADTSHeaderSize); + } else { + writer->mCrypto.mPlainSizes[0] += kADTSHeaderSize; + } + } + + return true; +} + +bool Adts::RevertSample(MediaRawData* aSample) { + if (aSample->Size() < kADTSHeaderSize) { + return false; + } + + { + const uint8_t* header = aSample->Data(); + if (header[0] != 0xff || header[1] != 0xf1 || header[6] != 0xfc) { + // Not ADTS. + return false; + } + } + + UniquePtr<MediaRawDataWriter> writer(aSample->CreateWriter()); + writer->PopFront(kADTSHeaderSize); + + if (aSample->mCrypto.IsEncrypted()) { + if (aSample->mCrypto.mPlainSizes.Length() > 0 && + writer->mCrypto.mPlainSizes[0] >= kADTSHeaderSize) { + writer->mCrypto.mPlainSizes[0] -= kADTSHeaderSize; + } + } + + return true; +} +} // namespace mozilla diff --git a/dom/media/platforms/agnostic/bytestreams/Adts.h b/dom/media/platforms/agnostic/bytestreams/Adts.h new file mode 100644 index 0000000000..c2b6b558b6 --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/Adts.h @@ -0,0 +1,22 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ADTS_H_ +#define ADTS_H_ + +#include <stdint.h> + +namespace mozilla { +class MediaRawData; + +class Adts { + public: + static int8_t GetFrequencyIndex(uint32_t aSamplesPerSecond); + static bool ConvertSample(uint16_t aChannelCount, int8_t aFrequencyIndex, + int8_t aProfile, mozilla::MediaRawData* aSample); + static bool RevertSample(MediaRawData* aSample); +}; +} // namespace mozilla + +#endif diff --git a/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp b/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp new file mode 100644 index 0000000000..086936dcc6 --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp @@ -0,0 +1,542 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/ArrayUtils.h" +#include "mozilla/EndianUtils.h" +#include "mozilla/ResultExtensions.h" +#include "mozilla/Try.h" +#include "mozilla/Unused.h" +#include "AnnexB.h" +#include "BufferReader.h" +#include "ByteWriter.h" +#include "H264.h" +#include "H265.h" +#include "MediaData.h" + +mozilla::LazyLogModule gAnnexB("AnnexB"); + +#define LOG(msg, ...) MOZ_LOG(gAnnexB, LogLevel::Debug, (msg, ##__VA_ARGS__)) +#define LOGV(msg, ...) MOZ_LOG(gAnnexB, LogLevel::Verbose, (msg, ##__VA_ARGS__)) + +namespace mozilla { + +static const uint8_t kAnnexBDelimiter[] = {0, 0, 0, 1}; + +/* static */ +Result<Ok, nsresult> AnnexB::ConvertAVCCSampleToAnnexB( + mozilla::MediaRawData* aSample, bool aAddSPS) { + MOZ_ASSERT(aSample); + + if (!IsAVCC(aSample)) { + return Ok(); + } + MOZ_ASSERT(aSample->Data()); + + MOZ_TRY(ConvertAVCCTo4BytesAVCC(aSample)); + + if (aSample->Size() < 4) { + // Nothing to do, it's corrupted anyway. + return Ok(); + } + + BufferReader reader(aSample->Data(), aSample->Size()); + + nsTArray<uint8_t> tmp; + ByteWriter<BigEndian> writer(tmp); + + while (reader.Remaining() >= 4) { + uint32_t nalLen; + MOZ_TRY_VAR(nalLen, reader.ReadU32()); + const uint8_t* p = reader.Read(nalLen); + + if (!writer.Write(kAnnexBDelimiter, ArrayLength(kAnnexBDelimiter))) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + if (!p) { + break; + } + if (!writer.Write(p, nalLen)) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + } + + UniquePtr<MediaRawDataWriter> samplewriter(aSample->CreateWriter()); + + if (!samplewriter->Replace(tmp.Elements(), tmp.Length())) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + + // Prepend the Annex B NAL with SPS and PPS tables to keyframes. + if (aAddSPS && aSample->mKeyframe) { + RefPtr<MediaByteBuffer> annexB = + ConvertAVCCExtraDataToAnnexB(aSample->mExtraData); + if (!samplewriter->Prepend(annexB->Elements(), annexB->Length())) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + + // Prepending the NAL with SPS/PPS will mess up the encryption subsample + // offsets. So we need to account for the extra bytes by increasing + // the length of the first clear data subsample. Otherwise decryption + // will fail. + if (aSample->mCrypto.IsEncrypted()) { + if (aSample->mCrypto.mPlainSizes.Length() == 0) { + CheckedUint32 plainSize{annexB->Length()}; + CheckedUint32 encryptedSize{samplewriter->Size()}; + encryptedSize -= annexB->Length(); + samplewriter->mCrypto.mPlainSizes.AppendElement(plainSize.value()); + samplewriter->mCrypto.mEncryptedSizes.AppendElement( + encryptedSize.value()); + } else { + CheckedUint32 newSize{samplewriter->mCrypto.mPlainSizes[0]}; + newSize += annexB->Length(); + samplewriter->mCrypto.mPlainSizes[0] = newSize.value(); + } + } + } + + return Ok(); +} + +/* static */ +Result<Ok, nsresult> AnnexB::ConvertHVCCSampleToAnnexB( + mozilla::MediaRawData* aSample, bool aAddSPS) { + MOZ_ASSERT(aSample); + if (!IsHVCC(aSample)) { + LOG("Not HVCC?"); + return Ok(); + } + MOZ_ASSERT(aSample->Data()); + + MOZ_TRY(ConvertHVCCTo4BytesHVCC(aSample)); + if (aSample->Size() < 4) { + // Nothing to do, it's corrupted anyway. + LOG("Corrupted HVCC sample?"); + return Ok(); + } + + BufferReader reader(aSample->Data(), aSample->Size()); + nsTArray<uint8_t> tmp; + ByteWriter<BigEndian> writer(tmp); + while (reader.Remaining() >= 4) { + uint32_t nalLen; + MOZ_TRY_VAR(nalLen, reader.ReadU32()); + const uint8_t* p = reader.Read(nalLen); + if (!writer.Write(kAnnexBDelimiter, ArrayLength(kAnnexBDelimiter))) { + LOG("Failed to write kAnnexBDelimiter, OOM?"); + return Err(NS_ERROR_OUT_OF_MEMORY); + } + if (!p) { + break; + } + if (!writer.Write(p, nalLen)) { + LOG("Failed to write nalu, OOM?"); + return Err(NS_ERROR_OUT_OF_MEMORY); + } + } + + UniquePtr<MediaRawDataWriter> samplewriter(aSample->CreateWriter()); + if (!samplewriter->Replace(tmp.Elements(), tmp.Length())) { + LOG("Failed to write sample, OOM?"); + return Err(NS_ERROR_OUT_OF_MEMORY); + } + + // Prepend the Annex B NAL with SPS and PPS tables to keyframes. + if (aAddSPS && aSample->mKeyframe) { + RefPtr<MediaByteBuffer> annexB = + ConvertHVCCExtraDataToAnnexB(aSample->mExtraData); + if (!annexB) { + LOG("Failed to convert HVCC extradata to AnnexB"); + return Err(NS_ERROR_FAILURE); + } + if (!samplewriter->Prepend(annexB->Elements(), annexB->Length())) { + LOG("Failed to append annexB extradata"); + return Err(NS_ERROR_OUT_OF_MEMORY); + } + + // Prepending the NAL with SPS/PPS will mess up the encryption subsample + // offsets. So we need to account for the extra bytes by increasing + // the length of the first clear data subsample. Otherwise decryption + // will fail. + if (aSample->mCrypto.IsEncrypted()) { + if (aSample->mCrypto.mPlainSizes.Length() == 0) { + CheckedUint32 plainSize{annexB->Length()}; + CheckedUint32 encryptedSize{samplewriter->Size()}; + encryptedSize -= annexB->Length(); + samplewriter->mCrypto.mPlainSizes.AppendElement(plainSize.value()); + samplewriter->mCrypto.mEncryptedSizes.AppendElement( + encryptedSize.value()); + } else { + CheckedUint32 newSize{samplewriter->mCrypto.mPlainSizes[0]}; + newSize += annexB->Length(); + samplewriter->mCrypto.mPlainSizes[0] = newSize.value(); + } + } + } + return Ok(); +} + +already_AddRefed<mozilla::MediaByteBuffer> AnnexB::ConvertAVCCExtraDataToAnnexB( + const mozilla::MediaByteBuffer* aExtraData) { + // AVCC 6 byte header looks like: + // +------+------+------+------+------+------+------+------+ + // [0] | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | + // +------+------+------+------+------+------+------+------+ + // [1] | profile | + // +------+------+------+------+------+------+------+------+ + // [2] | compatiblity | + // +------+------+------+------+------+------+------+------+ + // [3] | level | + // +------+------+------+------+------+------+------+------+ + // [4] | unused | nalLenSiz-1 | + // +------+------+------+------+------+------+------+------+ + // [5] | unused | numSps | + // +------+------+------+------+------+------+------+------+ + + RefPtr<mozilla::MediaByteBuffer> annexB = new mozilla::MediaByteBuffer; + + BufferReader reader(*aExtraData); + const uint8_t* ptr = reader.Read(5); + if (ptr && ptr[0] == 1) { + // Append SPS then PPS + Unused << reader.ReadU8().map( + [&](uint8_t x) { return ConvertSPSOrPPS(reader, x & 31, annexB); }); + Unused << reader.ReadU8().map( + [&](uint8_t x) { return ConvertSPSOrPPS(reader, x, annexB); }); + // MP4Box adds extra bytes that we ignore. I don't know what they do. + } + + return annexB.forget(); +} + +already_AddRefed<mozilla::MediaByteBuffer> AnnexB::ConvertHVCCExtraDataToAnnexB( + const mozilla::MediaByteBuffer* aExtraData) { + auto rv = HVCCConfig::Parse(aExtraData); + if (rv.isErr()) { + return nullptr; + } + const HVCCConfig hvcc = rv.unwrap(); + RefPtr<mozilla::MediaByteBuffer> annexB = new mozilla::MediaByteBuffer; + for (const auto& nalu : hvcc.mNALUs) { + annexB->AppendElements(kAnnexBDelimiter, ArrayLength(kAnnexBDelimiter)); + annexB->AppendElements(nalu.mNALU.Elements(), nalu.mNALU.Length()); + LOGV("Insert NALU (type=%hhu, size=%zu) to AnnexB (size=%zu)", + nalu.mNalUnitType, nalu.mNALU.Length(), annexB->Length()); + } + return annexB.forget(); +} + +Result<mozilla::Ok, nsresult> AnnexB::ConvertSPSOrPPS( + BufferReader& aReader, uint8_t aCount, mozilla::MediaByteBuffer* aAnnexB) { + for (int i = 0; i < aCount; i++) { + uint16_t length; + MOZ_TRY_VAR(length, aReader.ReadU16()); + + const uint8_t* ptr = aReader.Read(length); + if (!ptr) { + return Err(NS_ERROR_FAILURE); + } + aAnnexB->AppendElements(kAnnexBDelimiter, ArrayLength(kAnnexBDelimiter)); + aAnnexB->AppendElements(ptr, length); + } + return Ok(); +} + +static Result<Ok, nsresult> FindStartCodeInternal(BufferReader& aBr) { + size_t offset = aBr.Offset(); + + for (uint32_t i = 0; i < aBr.Align() && aBr.Remaining() >= 3; i++) { + auto res = aBr.PeekU24(); + if (res.isOk() && (res.unwrap() == 0x000001)) { + return Ok(); + } + mozilla::Unused << aBr.Read(1); + } + + while (aBr.Remaining() >= 6) { + uint32_t x32; + MOZ_TRY_VAR(x32, aBr.PeekU32()); + if ((x32 - 0x01010101) & (~x32) & 0x80808080) { + if ((x32 >> 8) == 0x000001) { + return Ok(); + } + if (x32 == 0x000001) { + mozilla::Unused << aBr.Read(1); + return Ok(); + } + if ((x32 & 0xff) == 0) { + const uint8_t* p = aBr.Peek(1); + if ((x32 & 0xff00) == 0 && p[4] == 1) { + mozilla::Unused << aBr.Read(2); + return Ok(); + } + if (p[4] == 0 && p[5] == 1) { + mozilla::Unused << aBr.Read(3); + return Ok(); + } + } + } + mozilla::Unused << aBr.Read(4); + } + + while (aBr.Remaining() >= 3) { + uint32_t data; + MOZ_TRY_VAR(data, aBr.PeekU24()); + if (data == 0x000001) { + return Ok(); + } + mozilla::Unused << aBr.Read(1); + } + + // No start code were found; Go back to the beginning. + mozilla::Unused << aBr.Seek(offset); + return Err(NS_ERROR_FAILURE); +} + +static Result<Ok, nsresult> FindStartCode(BufferReader& aBr, + size_t& aStartSize) { + if (FindStartCodeInternal(aBr).isErr()) { + aStartSize = 0; + return Err(NS_ERROR_FAILURE); + } + + aStartSize = 3; + if (aBr.Offset()) { + // Check if it's 4-bytes start code + aBr.Rewind(1); + uint8_t data; + MOZ_TRY_VAR(data, aBr.ReadU8()); + if (data == 0) { + aStartSize = 4; + } + } + mozilla::Unused << aBr.Read(3); + return Ok(); +} + +/* static */ +void AnnexB::ParseNALEntries(const Span<const uint8_t>& aSpan, + nsTArray<AnnexB::NALEntry>& aEntries) { + BufferReader reader(aSpan.data(), aSpan.Length()); + size_t startSize; + auto rv = FindStartCode(reader, startSize); + size_t startOffset = reader.Offset(); + if (rv.isOk()) { + while (FindStartCode(reader, startSize).isOk()) { + int64_t offset = reader.Offset(); + int64_t sizeNAL = offset - startOffset - startSize; + aEntries.AppendElement(AnnexB::NALEntry(startOffset, sizeNAL)); + reader.Seek(startOffset); + reader.Read(sizeNAL + startSize); + startOffset = offset; + } + } + int64_t sizeNAL = reader.Remaining(); + if (sizeNAL) { + aEntries.AppendElement(AnnexB::NALEntry(startOffset, sizeNAL)); + } +} + +static Result<mozilla::Ok, nsresult> ParseNALUnits(ByteWriter<BigEndian>& aBw, + BufferReader& aBr) { + size_t startSize; + + auto rv = FindStartCode(aBr, startSize); + if (rv.isOk()) { + size_t startOffset = aBr.Offset(); + while (FindStartCode(aBr, startSize).isOk()) { + size_t offset = aBr.Offset(); + size_t sizeNAL = offset - startOffset - startSize; + aBr.Seek(startOffset); + if (!aBw.WriteU32(sizeNAL) || !aBw.Write(aBr.Read(sizeNAL), sizeNAL)) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + aBr.Read(startSize); + startOffset = offset; + } + } + size_t sizeNAL = aBr.Remaining(); + if (sizeNAL) { + if (!aBw.WriteU32(sizeNAL) || !aBw.Write(aBr.Read(sizeNAL), sizeNAL)) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + } + return Ok(); +} + +bool AnnexB::ConvertSampleToAVCC(mozilla::MediaRawData* aSample, + const RefPtr<MediaByteBuffer>& aAVCCHeader) { + if (IsAVCC(aSample)) { + return ConvertAVCCTo4BytesAVCC(aSample).isOk(); + } + if (!IsAnnexB(aSample)) { + // Not AnnexB, nothing to convert. + return true; + } + + nsTArray<uint8_t> nalu; + ByteWriter<BigEndian> writer(nalu); + BufferReader reader(aSample->Data(), aSample->Size()); + + if (ParseNALUnits(writer, reader).isErr()) { + return false; + } + UniquePtr<MediaRawDataWriter> samplewriter(aSample->CreateWriter()); + if (!samplewriter->Replace(nalu.Elements(), nalu.Length())) { + return false; + } + + if (aAVCCHeader) { + aSample->mExtraData = aAVCCHeader; + return true; + } + + // Create the AVCC header. + auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>(); + static const uint8_t kFakeExtraData[] = { + 1 /* version */, + 0x64 /* profile (High) */, + 0 /* profile compat (0) */, + 40 /* level (40) */, + 0xfc | 3 /* nal size - 1 */, + 0xe0 /* num SPS (0) */, + 0 /* num PPS (0) */ + }; + // XXX(Bug 1631371) Check if this should use a fallible operation as it + // pretended earlier. + extradata->AppendElements(kFakeExtraData, ArrayLength(kFakeExtraData)); + aSample->mExtraData = std::move(extradata); + return true; +} + +/* static */ +Result<mozilla::Ok, nsresult> AnnexB::ConvertSampleToHVCC( + mozilla::MediaRawData* aSample) { + if (IsHVCC(aSample)) { + return ConvertHVCCTo4BytesHVCC(aSample); + } + if (!IsAnnexB(aSample)) { + // Not AnnexB, nothing to convert. + return Ok(); + } + + nsTArray<uint8_t> nalu; + ByteWriter<BigEndian> writer(nalu); + BufferReader reader(aSample->Data(), aSample->Size()); + if (auto rv = ParseNALUnits(writer, reader); rv.isErr()) { + LOG("Failed fo parse AnnexB NALU for HVCC"); + return rv; + } + UniquePtr<MediaRawDataWriter> samplewriter(aSample->CreateWriter()); + if (!samplewriter->Replace(nalu.Elements(), nalu.Length())) { + LOG("Failed fo replace NALU"); + return Err(NS_ERROR_OUT_OF_MEMORY); + } + if (aSample->mExtraData && HVCCConfig::Parse(aSample).isErr()) { + LOG("Failed to parse invalid hvcc extradata"); + return Err(NS_ERROR_DOM_MEDIA_METADATA_ERR); + } + // TODO : currently we don't set the fake header because we expect the sample + // already has a valid extradata. (set by the media change monitor) We can + // support setting a specific/fake header if we want to support HEVC encoding. + return Ok(); +} + +/* static */ +Result<mozilla::Ok, nsresult> AnnexB::ConvertAVCCTo4BytesAVCC( + mozilla::MediaRawData* aSample) { + auto avcc = AVCCConfig::Parse(aSample); + MOZ_ASSERT(avcc.isOk()); + return ConvertNALUTo4BytesNALU(aSample, avcc.unwrap().NALUSize()); +} + +/* static */ +Result<mozilla::Ok, nsresult> AnnexB::ConvertHVCCTo4BytesHVCC( + mozilla::MediaRawData* aSample) { + auto hvcc = HVCCConfig::Parse(aSample); + MOZ_ASSERT(hvcc.isOk()); + return ConvertNALUTo4BytesNALU(aSample, hvcc.unwrap().NALUSize()); +} + +/* static */ +bool AnnexB::IsAVCC(const mozilla::MediaRawData* aSample) { + return AVCCConfig::Parse(aSample).isOk(); +} + +/* static */ +bool AnnexB::IsHVCC(const mozilla::MediaRawData* aSample) { + return HVCCConfig::Parse(aSample).isOk(); +} + +/* static */ +bool AnnexB::IsAnnexB(const mozilla::MediaRawData* aSample) { + if (aSample->Size() < 4) { + return false; + } + uint32_t header = mozilla::BigEndian::readUint32(aSample->Data()); + return header == 0x00000001 || (header >> 8) == 0x000001; +} + +/* static */ mozilla::Result<mozilla::Ok, nsresult> +AnnexB::ConvertNALUTo4BytesNALU(mozilla::MediaRawData* aSample, + uint8_t aNALUSize) { + // NALSize should be between 1 to 4. + if (aNALUSize == 0 || aNALUSize > 4) { + return Err(NS_ERROR_FAILURE); + } + + // If the nalLenSize is already 4, we can only check if the data is corrupt + // without replacing data in aSample. + bool needConversion = aNALUSize != 4; + + MOZ_ASSERT(aSample); + nsTArray<uint8_t> dest; + ByteWriter<BigEndian> writer(dest); + BufferReader reader(aSample->Data(), aSample->Size()); + while (reader.Remaining() > aNALUSize) { + uint32_t nalLen; + switch (aNALUSize) { + case 1: + MOZ_TRY_VAR(nalLen, reader.ReadU8()); + break; + case 2: + MOZ_TRY_VAR(nalLen, reader.ReadU16()); + break; + case 3: + MOZ_TRY_VAR(nalLen, reader.ReadU24()); + break; + case 4: + MOZ_TRY_VAR(nalLen, reader.ReadU32()); + break; + default: + MOZ_ASSERT_UNREACHABLE("Bytes of the NAL body length must be in [1,4]"); + return Err(NS_ERROR_ILLEGAL_VALUE); + } + const uint8_t* p = reader.Read(nalLen); + if (!p) { + // The data may be corrupt. + return Err(NS_ERROR_UNEXPECTED); + } + if (!needConversion) { + // We only parse aSample to see if it's corrupt. + continue; + } + if (!writer.WriteU32(nalLen) || !writer.Write(p, nalLen)) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + } + if (!needConversion) { + // We've parsed all the data, and it's all good. + return Ok(); + } + UniquePtr<MediaRawDataWriter> samplewriter(aSample->CreateWriter()); + if (!samplewriter->Replace(dest.Elements(), dest.Length())) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + return Ok(); +} + +#undef LOG +#undef LOGV + +} // namespace mozilla diff --git a/dom/media/platforms/agnostic/bytestreams/AnnexB.h b/dom/media/platforms/agnostic/bytestreams/AnnexB.h new file mode 100644 index 0000000000..d3b2b8892d --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/AnnexB.h @@ -0,0 +1,89 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_ANNEX_B_H_ +#define DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_ANNEX_B_H_ + +#include "ErrorList.h" +#include "mozilla/RefPtr.h" +#include "mozilla/Result.h" + +template <class> +class nsTArray; + +namespace mozilla { +class BufferReader; +class MediaRawData; +class MediaByteBuffer; + +class AnnexB { + public: + struct NALEntry { + NALEntry(int64_t aOffset, int64_t aSize) : mOffset(aOffset), mSize(aSize) { + MOZ_ASSERT(mOffset >= 0); + MOZ_ASSERT(mSize >= 0); + } + // They should be non-negative, so we use int64_t to assert their value when + // assigning value to them. + int64_t mOffset; + int64_t mSize; + }; + // All conversions assume size of NAL length field is 4 bytes. + // Convert a sample from AVCC format to Annex B. + static mozilla::Result<mozilla::Ok, nsresult> ConvertAVCCSampleToAnnexB( + mozilla::MediaRawData* aSample, bool aAddSPS = true); + // All conversions assume size of NAL length field is 4 bytes. + // Convert a sample from HVCC format to Annex B. + static mozilla::Result<mozilla::Ok, nsresult> ConvertHVCCSampleToAnnexB( + mozilla::MediaRawData* aSample, bool aAddSPS = true); + + // Convert a sample from Annex B to AVCC. + // an AVCC extradata must not be set. + static bool ConvertSampleToAVCC( + mozilla::MediaRawData* aSample, + const RefPtr<mozilla::MediaByteBuffer>& aAVCCHeader = nullptr); + // Convert a sample from Annex B to HVCC. An HVCC extradata must not be set. + static Result<mozilla::Ok, nsresult> ConvertSampleToHVCC( + mozilla::MediaRawData* aSample); + + // Covert sample to 4 bytes NALU byte stream. + static mozilla::Result<mozilla::Ok, nsresult> ConvertAVCCTo4BytesAVCC( + mozilla::MediaRawData* aSample); + static mozilla::Result<mozilla::Ok, nsresult> ConvertHVCCTo4BytesHVCC( + mozilla::MediaRawData* aSample); + + // Parse an AVCC extradata and construct the Annex B sample header. + static already_AddRefed<mozilla::MediaByteBuffer> + ConvertAVCCExtraDataToAnnexB(const mozilla::MediaByteBuffer* aExtraData); + // Parse a HVCC extradata and construct the Annex B sample header. + static already_AddRefed<mozilla::MediaByteBuffer> + ConvertHVCCExtraDataToAnnexB(const mozilla::MediaByteBuffer* aExtraData); + + // Returns true if format is AVCC and sample has valid extradata. + static bool IsAVCC(const mozilla::MediaRawData* aSample); + // Returns true if format is HVCC and sample has valid extradata. + static bool IsHVCC(const mozilla::MediaRawData* aSample); + // Returns true if format is AnnexB. + static bool IsAnnexB(const mozilla::MediaRawData* aSample); + + // Parse NAL entries from the bytes stream to know the offset and the size of + // each NAL in the bytes stream. + static void ParseNALEntries(const Span<const uint8_t>& aSpan, + nsTArray<AnnexB::NALEntry>& aEntries); + + private: + // AVCC box parser helper. + static mozilla::Result<mozilla::Ok, nsresult> ConvertSPSOrPPS( + mozilla::BufferReader& aReader, uint8_t aCount, + mozilla::MediaByteBuffer* aAnnexB); + + // NALU size can vary, this function converts the sample to always 4 bytes + // NALU. + static mozilla::Result<mozilla::Ok, nsresult> ConvertNALUTo4BytesNALU( + mozilla::MediaRawData* aSample, uint8_t aNALUSize); +}; + +} // namespace mozilla + +#endif // DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_ANNEX_B_H_ diff --git a/dom/media/platforms/agnostic/bytestreams/ByteStreamsUtils.h b/dom/media/platforms/agnostic/bytestreams/ByteStreamsUtils.h new file mode 100644 index 0000000000..464c35911a --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/ByteStreamsUtils.h @@ -0,0 +1,70 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_BYTESTREAMUTILS_H_ +#define DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_BYTESTREAMUTILS_H_ + +namespace mozilla { + +// Described in ISO 23001-8:2016 +// H264 spec Table 2, H265 spec Table E.3 +enum class PrimaryID : uint8_t { + INVALID = 0, + BT709 = 1, + UNSPECIFIED = 2, + BT470M = 4, + BT470BG = 5, + SMPTE170M = 6, + SMPTE240M = 7, + FILM = 8, + BT2020 = 9, + SMPTEST428_1 = 10, + SMPTEST431_2 = 11, + SMPTEST432_1 = 12, + EBU_3213_E = 22 +}; + +// H264 spec Table 3, H265 spec Table E.4 +enum class TransferID : uint8_t { + INVALID = 0, + BT709 = 1, + UNSPECIFIED = 2, + GAMMA22 = 4, + GAMMA28 = 5, + SMPTE170M = 6, + SMPTE240M = 7, + LINEAR = 8, + LOG = 9, + LOG_SQRT = 10, + IEC61966_2_4 = 11, + BT1361_ECG = 12, + IEC61966_2_1 = 13, + BT2020_10 = 14, + BT2020_12 = 15, + SMPTEST2084 = 16, + SMPTEST428_1 = 17, + + // Not yet standardized + ARIB_STD_B67 = 18, // AKA hybrid-log gamma, HLG. +}; + +// H264 spec Table 4, H265 spec Table E.5 +enum class MatrixID : uint8_t { + RGB = 0, + BT709 = 1, + UNSPECIFIED = 2, + FCC = 4, + BT470BG = 5, + SMPTE170M = 6, + SMPTE240M = 7, + YCOCG = 8, + BT2020_NCL = 9, + BT2020_CL = 10, + YDZDX = 11, + INVALID = 255, +}; + +} // namespace mozilla + +#endif // DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_H265_H_ diff --git a/dom/media/platforms/agnostic/bytestreams/H264.cpp b/dom/media/platforms/agnostic/bytestreams/H264.cpp new file mode 100644 index 0000000000..113be67d0e --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/H264.cpp @@ -0,0 +1,1333 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "H264.h" +#include <limits> +#include "AnnexB.h" +#include "BitReader.h" +#include "BitWriter.h" +#include "BufferReader.h" +#include "ByteStreamsUtils.h" +#include "ByteWriter.h" +#include "mozilla/PodOperations.h" +#include "mozilla/ResultExtensions.h" +#include "mozilla/Try.h" + +#define READSE(var, min, max) \ + { \ + int32_t val = br.ReadSE(); \ + if (val < min || val > max) { \ + return false; \ + } \ + aDest.var = val; \ + } + +#define READUE(var, max) \ + { \ + uint32_t uval = br.ReadUE(); \ + if (uval > max) { \ + return false; \ + } \ + aDest.var = uval; \ + } + +mozilla::LazyLogModule gH264("H264"); + +#define LOG(msg, ...) MOZ_LOG(gH264, LogLevel::Debug, (msg, ##__VA_ARGS__)) + +namespace mozilla { + +// Default scaling lists (per spec). +// ITU H264: +// Table 7-2 – Assignment of mnemonic names to scaling list indices and +// specification of fall-back rule +static const uint8_t Default_4x4_Intra[16] = {6, 13, 13, 20, 20, 20, 28, 28, + 28, 28, 32, 32, 32, 37, 37, 42}; + +static const uint8_t Default_4x4_Inter[16] = {10, 14, 14, 20, 20, 20, 24, 24, + 24, 24, 27, 27, 27, 30, 30, 34}; + +static const uint8_t Default_8x8_Intra[64] = { + 6, 10, 10, 13, 11, 13, 16, 16, 16, 16, 18, 18, 18, 18, 18, 23, + 23, 23, 23, 23, 23, 25, 25, 25, 25, 25, 25, 25, 27, 27, 27, 27, + 27, 27, 27, 27, 29, 29, 29, 29, 29, 29, 29, 31, 31, 31, 31, 31, + 31, 33, 33, 33, 33, 33, 36, 36, 36, 36, 38, 38, 38, 40, 40, 42}; + +static const uint8_t Default_8x8_Inter[64] = { + 9, 13, 13, 15, 13, 15, 17, 17, 17, 17, 19, 19, 19, 19, 19, 21, + 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 24, 24, 24, 24, + 24, 24, 24, 24, 25, 25, 25, 25, 25, 25, 25, 27, 27, 27, 27, 27, + 27, 28, 28, 28, 28, 28, 30, 30, 30, 30, 32, 32, 32, 33, 33, 35}; + +namespace detail { +static void scaling_list(BitReader& aBr, uint8_t* aScalingList, + int aSizeOfScalingList, const uint8_t* aDefaultList, + const uint8_t* aFallbackList) { + int32_t lastScale = 8; + int32_t nextScale = 8; + int32_t deltaScale; + + // (pic|seq)_scaling_list_present_flag[i] + if (!aBr.ReadBit()) { + if (aFallbackList) { + memcpy(aScalingList, aFallbackList, aSizeOfScalingList); + } + return; + } + + for (int i = 0; i < aSizeOfScalingList; i++) { + if (nextScale != 0) { + deltaScale = aBr.ReadSE(); + nextScale = (lastScale + deltaScale + 256) % 256; + if (!i && !nextScale) { + memcpy(aScalingList, aDefaultList, aSizeOfScalingList); + return; + } + } + aScalingList[i] = (nextScale == 0) ? lastScale : nextScale; + lastScale = aScalingList[i]; + } +} +} // namespace detail. + +template <size_t N> +static void scaling_list(BitReader& aBr, uint8_t (&aScalingList)[N], + const uint8_t (&aDefaultList)[N], + const uint8_t (&aFallbackList)[N]) { + detail::scaling_list(aBr, aScalingList, N, aDefaultList, aFallbackList); +} + +template <size_t N> +static void scaling_list(BitReader& aBr, uint8_t (&aScalingList)[N], + const uint8_t (&aDefaultList)[N]) { + detail::scaling_list(aBr, aScalingList, N, aDefaultList, nullptr); +} + +SPSData::SPSData() { + PodZero(this); + // Default values when they aren't defined as per ITU-T H.264 (2014/02). + chroma_format_idc = 1; + video_format = 5; + colour_primaries = 2; + transfer_characteristics = 2; + sample_ratio = 1.0; + memset(scaling_matrix4x4, 16, sizeof(scaling_matrix4x4)); + memset(scaling_matrix8x8, 16, sizeof(scaling_matrix8x8)); +} + +bool SPSData::operator==(const SPSData& aOther) const { + return this->valid && aOther.valid && !memcmp(this, &aOther, sizeof(SPSData)); +} + +bool SPSData::operator!=(const SPSData& aOther) const { + return !(operator==(aOther)); +} + +static PrimaryID GetPrimaryID(int aPrimary) { + if (aPrimary < 1 || aPrimary > 22 || aPrimary == 3) { + return PrimaryID::INVALID; + } + if (aPrimary > 12 && aPrimary < 22) { + return PrimaryID::INVALID; + } + return static_cast<PrimaryID>(aPrimary); +} + +static TransferID GetTransferID(int aTransfer) { + if (aTransfer < 1 || aTransfer > 18 || aTransfer == 3) { + return TransferID::INVALID; + } + return static_cast<TransferID>(aTransfer); +} + +static MatrixID GetMatrixID(int aMatrix) { + if (aMatrix < 0 || aMatrix > 11 || aMatrix == 3) { + return MatrixID::INVALID; + } + return static_cast<MatrixID>(aMatrix); +} + +gfx::YUVColorSpace SPSData::ColorSpace() const { + // Bitfield, note that guesses with higher values take precedence over + // guesses with lower values. + enum Guess { + GUESS_BT601 = 1 << 0, + GUESS_BT709 = 1 << 1, + GUESS_BT2020 = 1 << 2, + }; + + uint32_t guess = 0; + + switch (GetPrimaryID(colour_primaries)) { + case PrimaryID::BT709: + guess |= GUESS_BT709; + break; + case PrimaryID::BT470M: + case PrimaryID::BT470BG: + case PrimaryID::SMPTE170M: + case PrimaryID::SMPTE240M: + guess |= GUESS_BT601; + break; + case PrimaryID::BT2020: + guess |= GUESS_BT2020; + break; + case PrimaryID::FILM: + case PrimaryID::SMPTEST428_1: + case PrimaryID::SMPTEST431_2: + case PrimaryID::SMPTEST432_1: + case PrimaryID::EBU_3213_E: + case PrimaryID::INVALID: + case PrimaryID::UNSPECIFIED: + break; + } + + switch (GetTransferID(transfer_characteristics)) { + case TransferID::BT709: + guess |= GUESS_BT709; + break; + case TransferID::GAMMA22: + case TransferID::GAMMA28: + case TransferID::SMPTE170M: + case TransferID::SMPTE240M: + guess |= GUESS_BT601; + break; + case TransferID::BT2020_10: + case TransferID::BT2020_12: + guess |= GUESS_BT2020; + break; + case TransferID::LINEAR: + case TransferID::LOG: + case TransferID::LOG_SQRT: + case TransferID::IEC61966_2_4: + case TransferID::BT1361_ECG: + case TransferID::IEC61966_2_1: + case TransferID::SMPTEST2084: + case TransferID::SMPTEST428_1: + case TransferID::ARIB_STD_B67: + case TransferID::INVALID: + case TransferID::UNSPECIFIED: + break; + } + + switch (GetMatrixID(matrix_coefficients)) { + case MatrixID::BT709: + guess |= GUESS_BT709; + break; + case MatrixID::BT470BG: + case MatrixID::SMPTE170M: + case MatrixID::SMPTE240M: + guess |= GUESS_BT601; + break; + case MatrixID::BT2020_NCL: + case MatrixID::BT2020_CL: + guess |= GUESS_BT2020; + break; + case MatrixID::RGB: + case MatrixID::FCC: + case MatrixID::YCOCG: + case MatrixID::YDZDX: + case MatrixID::INVALID: + case MatrixID::UNSPECIFIED: + break; + } + + // Removes lowest bit until only a single bit remains. + while (guess & (guess - 1)) { + guess &= guess - 1; + } + if (!guess) { + // A better default to BT601 which should die a slow death. + guess = GUESS_BT709; + } + + switch (guess) { + case GUESS_BT601: + return gfx::YUVColorSpace::BT601; + case GUESS_BT709: + return gfx::YUVColorSpace::BT709; + case GUESS_BT2020: + return gfx::YUVColorSpace::BT2020; + default: + MOZ_CRASH("not possible to get here but makes compiler happy"); + } +} + +gfx::ColorDepth SPSData::ColorDepth() const { + if (bit_depth_luma_minus8 != 0 && bit_depth_luma_minus8 != 2 && + bit_depth_luma_minus8 != 4) { + // We don't know what that is, just assume 8 bits to prevent decoding + // regressions if we ever encounter those. + return gfx::ColorDepth::COLOR_8; + } + return gfx::ColorDepthForBitDepth(bit_depth_luma_minus8 + 8); +} + +// SPSNAL and SPSNALIterator do not own their data. +class SPSNAL { + public: + SPSNAL(const uint8_t* aPtr, size_t aLength) { + MOZ_ASSERT(aPtr); + + if (aLength == 0 || (*aPtr & 0x1f) != H264_NAL_SPS) { + return; + } + mDecodedNAL = H264::DecodeNALUnit(aPtr, aLength); + if (mDecodedNAL) { + mLength = BitReader::GetBitLength(mDecodedNAL); + } + } + + SPSNAL() = default; + + bool IsValid() const { return mDecodedNAL; } + + bool operator==(const SPSNAL& aOther) const { + if (!mDecodedNAL || !aOther.mDecodedNAL) { + return false; + } + + SPSData decodedSPS1; + SPSData decodedSPS2; + if (!GetSPSData(decodedSPS1) || !aOther.GetSPSData(decodedSPS2)) { + // Couldn't decode one SPS, perform a binary comparison + if (mLength != aOther.mLength) { + return false; + } + MOZ_ASSERT(mLength / 8 <= mDecodedNAL->Length()); + + if (memcmp(mDecodedNAL->Elements(), aOther.mDecodedNAL->Elements(), + mLength / 8) != 0) { + return false; + } + + uint32_t remaining = mLength - (mLength & ~7); + + BitReader b1(mDecodedNAL->Elements() + mLength / 8, remaining); + BitReader b2(aOther.mDecodedNAL->Elements() + mLength / 8, remaining); + for (uint32_t i = 0; i < remaining; i++) { + if (b1.ReadBit() != b2.ReadBit()) { + return false; + } + } + return true; + } + + return decodedSPS1 == decodedSPS2; + } + + bool operator!=(const SPSNAL& aOther) const { return !(operator==(aOther)); } + + bool GetSPSData(SPSData& aDest) const { + return H264::DecodeSPS(mDecodedNAL, aDest); + } + + private: + RefPtr<mozilla::MediaByteBuffer> mDecodedNAL; + uint32_t mLength = 0; +}; + +class SPSNALIterator { + public: + explicit SPSNALIterator(const mozilla::MediaByteBuffer* aExtraData) + : mExtraDataPtr(aExtraData->Elements()), mReader(aExtraData) { + if (!mReader.Read(5)) { + return; + } + + auto res = mReader.ReadU8(); + mNumSPS = res.isOk() ? res.unwrap() & 0x1f : 0; + if (mNumSPS == 0) { + return; + } + mValid = true; + } + + SPSNALIterator& operator++() { + if (mEOS || !mValid) { + return *this; + } + if (--mNumSPS == 0) { + mEOS = true; + } + auto res = mReader.ReadU16(); + uint16_t length = res.isOk() ? res.unwrap() : 0; + if (length == 0 || !mReader.Read(length)) { + mEOS = true; + } + return *this; + } + + explicit operator bool() const { return mValid && !mEOS; } + + SPSNAL operator*() const { + MOZ_ASSERT(bool(*this)); + BufferReader reader(mExtraDataPtr + mReader.Offset(), mReader.Remaining()); + + auto res = reader.ReadU16(); + uint16_t length = res.isOk() ? res.unwrap() : 0; + const uint8_t* ptr = reader.Read(length); + if (!ptr || !length) { + return SPSNAL(); + } + return SPSNAL(ptr, length); + } + + private: + const uint8_t* mExtraDataPtr; + BufferReader mReader; + bool mValid = false; + bool mEOS = false; + uint8_t mNumSPS = 0; +}; + +/* static */ already_AddRefed<mozilla::MediaByteBuffer> H264::DecodeNALUnit( + const uint8_t* aNAL, size_t aLength) { + MOZ_ASSERT(aNAL); + + if (aLength < 4) { + return nullptr; + } + + RefPtr<mozilla::MediaByteBuffer> rbsp = new mozilla::MediaByteBuffer; + BufferReader reader(aNAL, aLength); + auto res = reader.ReadU8(); + if (res.isErr()) { + return nullptr; + } + uint8_t nal_unit_type = res.unwrap() & 0x1f; + uint32_t nalUnitHeaderBytes = 1; + if (nal_unit_type == H264_NAL_PREFIX || nal_unit_type == H264_NAL_SLICE_EXT || + nal_unit_type == H264_NAL_SLICE_EXT_DVC) { + bool svc_extension_flag = false; + bool avc_3d_extension_flag = false; + if (nal_unit_type != H264_NAL_SLICE_EXT_DVC) { + res = reader.PeekU8(); + if (res.isErr()) { + return nullptr; + } + svc_extension_flag = res.unwrap() & 0x80; + } else { + res = reader.PeekU8(); + if (res.isErr()) { + return nullptr; + } + avc_3d_extension_flag = res.unwrap() & 0x80; + } + if (svc_extension_flag) { + nalUnitHeaderBytes += 3; + } else if (avc_3d_extension_flag) { + nalUnitHeaderBytes += 2; + } else { + nalUnitHeaderBytes += 3; + } + } + if (!reader.Read(nalUnitHeaderBytes - 1)) { + return nullptr; + } + uint32_t lastbytes = 0xffff; + while (reader.Remaining()) { + auto res = reader.ReadU8(); + if (res.isErr()) { + return nullptr; + } + uint8_t byte = res.unwrap(); + if ((lastbytes & 0xffff) == 0 && byte == 0x03) { + // reset last two bytes, to detect the 0x000003 sequence again. + lastbytes = 0xffff; + } else { + rbsp->AppendElement(byte); + } + lastbytes = (lastbytes << 8) | byte; + } + return rbsp.forget(); +} + +// The reverse of DecodeNALUnit. To allow the distinction between Annex B (that +// uses 0x000001 as marker) and AVCC, the pattern 0x00 0x00 0x0n (where n is +// between 0 and 3) can't be found in the bytestream. A 0x03 byte is inserted +// after the second 0. Eg. 0x00 0x00 0x00 becomes 0x00 0x00 0x03 0x00 +/* static */ already_AddRefed<mozilla::MediaByteBuffer> H264::EncodeNALUnit( + const uint8_t* aNAL, size_t aLength) { + MOZ_ASSERT(aNAL); + RefPtr<MediaByteBuffer> rbsp = new MediaByteBuffer(); + BufferReader reader(aNAL, aLength); + + auto res = reader.ReadU8(); + if (res.isErr()) { + return rbsp.forget(); + } + rbsp->AppendElement(res.unwrap()); + + res = reader.ReadU8(); + if (res.isErr()) { + return rbsp.forget(); + } + rbsp->AppendElement(res.unwrap()); + + while ((res = reader.ReadU8()).isOk()) { + uint8_t val = res.unwrap(); + if (val <= 0x03 && rbsp->ElementAt(rbsp->Length() - 2) == 0 && + rbsp->ElementAt(rbsp->Length() - 1) == 0) { + rbsp->AppendElement(0x03); + } + rbsp->AppendElement(val); + } + return rbsp.forget(); +} + +static int32_t ConditionDimension(float aValue) { + // This will exclude NaNs and too-big values. + if (aValue > 1.0 && aValue <= float(INT32_MAX) / 2) { + return int32_t(aValue); + } + return 0; +} + +/* static */ +bool H264::DecodeSPS(const mozilla::MediaByteBuffer* aSPS, SPSData& aDest) { + if (!aSPS) { + return false; + } + BitReader br(aSPS, BitReader::GetBitLength(aSPS)); + + aDest.profile_idc = br.ReadBits(8); + aDest.constraint_set0_flag = br.ReadBit(); + aDest.constraint_set1_flag = br.ReadBit(); + aDest.constraint_set2_flag = br.ReadBit(); + aDest.constraint_set3_flag = br.ReadBit(); + aDest.constraint_set4_flag = br.ReadBit(); + aDest.constraint_set5_flag = br.ReadBit(); + br.ReadBits(2); // reserved_zero_2bits + aDest.level_idc = br.ReadBits(8); + READUE(seq_parameter_set_id, MAX_SPS_COUNT - 1); + + if (aDest.profile_idc == 100 || aDest.profile_idc == 110 || + aDest.profile_idc == 122 || aDest.profile_idc == 244 || + aDest.profile_idc == 44 || aDest.profile_idc == 83 || + aDest.profile_idc == 86 || aDest.profile_idc == 118 || + aDest.profile_idc == 128 || aDest.profile_idc == 138 || + aDest.profile_idc == 139 || aDest.profile_idc == 134) { + READUE(chroma_format_idc, 3); + if (aDest.chroma_format_idc == 3) { + aDest.separate_colour_plane_flag = br.ReadBit(); + } + READUE(bit_depth_luma_minus8, 6); + READUE(bit_depth_chroma_minus8, 6); + br.ReadBit(); // qpprime_y_zero_transform_bypass_flag + aDest.seq_scaling_matrix_present_flag = br.ReadBit(); + if (aDest.seq_scaling_matrix_present_flag) { + scaling_list(br, aDest.scaling_matrix4x4[0], Default_4x4_Intra, + Default_4x4_Intra); + scaling_list(br, aDest.scaling_matrix4x4[1], Default_4x4_Intra, + aDest.scaling_matrix4x4[0]); + scaling_list(br, aDest.scaling_matrix4x4[2], Default_4x4_Intra, + aDest.scaling_matrix4x4[1]); + scaling_list(br, aDest.scaling_matrix4x4[3], Default_4x4_Inter, + Default_4x4_Inter); + scaling_list(br, aDest.scaling_matrix4x4[4], Default_4x4_Inter, + aDest.scaling_matrix4x4[3]); + scaling_list(br, aDest.scaling_matrix4x4[5], Default_4x4_Inter, + aDest.scaling_matrix4x4[4]); + + scaling_list(br, aDest.scaling_matrix8x8[0], Default_8x8_Intra, + Default_8x8_Intra); + scaling_list(br, aDest.scaling_matrix8x8[1], Default_8x8_Inter, + Default_8x8_Inter); + if (aDest.chroma_format_idc == 3) { + scaling_list(br, aDest.scaling_matrix8x8[2], Default_8x8_Intra, + aDest.scaling_matrix8x8[0]); + scaling_list(br, aDest.scaling_matrix8x8[3], Default_8x8_Inter, + aDest.scaling_matrix8x8[1]); + scaling_list(br, aDest.scaling_matrix8x8[4], Default_8x8_Intra, + aDest.scaling_matrix8x8[2]); + scaling_list(br, aDest.scaling_matrix8x8[5], Default_8x8_Inter, + aDest.scaling_matrix8x8[3]); + } + } + } else if (aDest.profile_idc == 183) { + aDest.chroma_format_idc = 0; + } else { + // default value if chroma_format_idc isn't set. + aDest.chroma_format_idc = 1; + } + READUE(log2_max_frame_num, 12); + aDest.log2_max_frame_num += 4; + READUE(pic_order_cnt_type, 2); + if (aDest.pic_order_cnt_type == 0) { + READUE(log2_max_pic_order_cnt_lsb, 12); + aDest.log2_max_pic_order_cnt_lsb += 4; + } else if (aDest.pic_order_cnt_type == 1) { + aDest.delta_pic_order_always_zero_flag = br.ReadBit(); + READSE(offset_for_non_ref_pic, -231, 230); + READSE(offset_for_top_to_bottom_field, -231, 230); + uint32_t num_ref_frames_in_pic_order_cnt_cycle = br.ReadUE(); + for (uint32_t i = 0; i < num_ref_frames_in_pic_order_cnt_cycle; i++) { + br.ReadSE(); // offset_for_ref_frame[i] + } + } + aDest.max_num_ref_frames = br.ReadUE(); + aDest.gaps_in_frame_num_allowed_flag = br.ReadBit(); + aDest.pic_width_in_mbs = br.ReadUE() + 1; + aDest.pic_height_in_map_units = br.ReadUE() + 1; + aDest.frame_mbs_only_flag = br.ReadBit(); + if (!aDest.frame_mbs_only_flag) { + aDest.pic_height_in_map_units *= 2; + aDest.mb_adaptive_frame_field_flag = br.ReadBit(); + } + aDest.direct_8x8_inference_flag = br.ReadBit(); + aDest.frame_cropping_flag = br.ReadBit(); + if (aDest.frame_cropping_flag) { + aDest.frame_crop_left_offset = br.ReadUE(); + aDest.frame_crop_right_offset = br.ReadUE(); + aDest.frame_crop_top_offset = br.ReadUE(); + aDest.frame_crop_bottom_offset = br.ReadUE(); + } + + aDest.sample_ratio = 1.0f; + aDest.vui_parameters_present_flag = br.ReadBit(); + if (aDest.vui_parameters_present_flag) { + if (!vui_parameters(br, aDest)) { + return false; + } + } + + // Calculate common values. + + uint8_t ChromaArrayType = + aDest.separate_colour_plane_flag ? 0 : aDest.chroma_format_idc; + // Calculate width. + uint32_t CropUnitX = 1; + uint32_t SubWidthC = aDest.chroma_format_idc == 3 ? 1 : 2; + if (ChromaArrayType != 0) { + CropUnitX = SubWidthC; + } + + // Calculate Height + uint32_t CropUnitY = 2 - aDest.frame_mbs_only_flag; + uint32_t SubHeightC = aDest.chroma_format_idc <= 1 ? 2 : 1; + if (ChromaArrayType != 0) { + CropUnitY *= SubHeightC; + } + + uint32_t width = aDest.pic_width_in_mbs * 16; + uint32_t height = aDest.pic_height_in_map_units * 16; + if (aDest.frame_crop_left_offset <= + std::numeric_limits<int32_t>::max() / 4 / CropUnitX && + aDest.frame_crop_right_offset <= + std::numeric_limits<int32_t>::max() / 4 / CropUnitX && + aDest.frame_crop_top_offset <= + std::numeric_limits<int32_t>::max() / 4 / CropUnitY && + aDest.frame_crop_bottom_offset <= + std::numeric_limits<int32_t>::max() / 4 / CropUnitY && + (aDest.frame_crop_left_offset + aDest.frame_crop_right_offset) * + CropUnitX < + width && + (aDest.frame_crop_top_offset + aDest.frame_crop_bottom_offset) * + CropUnitY < + height) { + aDest.crop_left = aDest.frame_crop_left_offset * CropUnitX; + aDest.crop_right = aDest.frame_crop_right_offset * CropUnitX; + aDest.crop_top = aDest.frame_crop_top_offset * CropUnitY; + aDest.crop_bottom = aDest.frame_crop_bottom_offset * CropUnitY; + } else { + // Nonsensical value, ignore them. + aDest.crop_left = aDest.crop_right = aDest.crop_top = aDest.crop_bottom = 0; + } + + aDest.pic_width = width - aDest.crop_left - aDest.crop_right; + aDest.pic_height = height - aDest.crop_top - aDest.crop_bottom; + + aDest.interlaced = !aDest.frame_mbs_only_flag; + + // Determine display size. + if (aDest.sample_ratio > 1.0) { + // Increase the intrinsic width + aDest.display_width = ConditionDimension( + AssertedCast<float>(aDest.pic_width) * aDest.sample_ratio); + aDest.display_height = aDest.pic_height; + } else { + // Increase the intrinsic height + aDest.display_width = aDest.pic_width; + aDest.display_height = ConditionDimension( + AssertedCast<float>(aDest.pic_height) / aDest.sample_ratio); + } + + aDest.valid = true; + + return true; +} + +/* static */ +bool H264::vui_parameters(BitReader& aBr, SPSData& aDest) { + aDest.aspect_ratio_info_present_flag = aBr.ReadBit(); + if (aDest.aspect_ratio_info_present_flag) { + aDest.aspect_ratio_idc = aBr.ReadBits(8); + aDest.sar_width = aDest.sar_height = 0; + + // From E.2.1 VUI parameters semantics (ITU-T H.264 02/2014) + switch (aDest.aspect_ratio_idc) { + case 0: + // Unspecified + break; + case 1: + /* + 1:1 + 7680x4320 16:9 frame without horizontal overscan + 3840x2160 16:9 frame without horizontal overscan + 1280x720 16:9 frame without horizontal overscan + 1920x1080 16:9 frame without horizontal overscan (cropped from + 1920x1088) 640x480 4:3 frame without horizontal overscan + */ + aDest.sample_ratio = 1.0f; + break; + case 2: + /* + 12:11 + 720x576 4:3 frame with horizontal overscan + 352x288 4:3 frame without horizontal overscan + */ + aDest.sample_ratio = 12.0 / 11.0; + break; + case 3: + /* + 10:11 + 720x480 4:3 frame with horizontal overscan + 352x240 4:3 frame without horizontal overscan + */ + aDest.sample_ratio = 10.0 / 11.0; + break; + case 4: + /* + 16:11 + 720x576 16:9 frame with horizontal overscan + 528x576 4:3 frame without horizontal overscan + */ + aDest.sample_ratio = 16.0 / 11.0; + break; + case 5: + /* + 40:33 + 720x480 16:9 frame with horizontal overscan + 528x480 4:3 frame without horizontal overscan + */ + aDest.sample_ratio = 40.0 / 33.0; + break; + case 6: + /* + 24:11 + 352x576 4:3 frame without horizontal overscan + 480x576 16:9 frame with horizontal overscan + */ + aDest.sample_ratio = 24.0 / 11.0; + break; + case 7: + /* + 20:11 + 352x480 4:3 frame without horizontal overscan + 480x480 16:9 frame with horizontal overscan + */ + aDest.sample_ratio = 20.0 / 11.0; + break; + case 8: + /* + 32:11 + 352x576 16:9 frame without horizontal overscan + */ + aDest.sample_ratio = 32.0 / 11.0; + break; + case 9: + /* + 80:33 + 352x480 16:9 frame without horizontal overscan + */ + aDest.sample_ratio = 80.0 / 33.0; + break; + case 10: + /* + 18:11 + 480x576 4:3 frame with horizontal overscan + */ + aDest.sample_ratio = 18.0 / 11.0; + break; + case 11: + /* + 15:11 + 480x480 4:3 frame with horizontal overscan + */ + aDest.sample_ratio = 15.0 / 11.0; + break; + case 12: + /* + 64:33 + 528x576 16:9 frame with horizontal overscan + */ + aDest.sample_ratio = 64.0 / 33.0; + break; + case 13: + /* + 160:99 + 528x480 16:9 frame without horizontal overscan + */ + aDest.sample_ratio = 160.0 / 99.0; + break; + case 14: + /* + 4:3 + 1440x1080 16:9 frame without horizontal overscan + */ + aDest.sample_ratio = 4.0 / 3.0; + break; + case 15: + /* + 3:2 + 1280x1080 16:9 frame without horizontal overscan + */ + aDest.sample_ratio = 3.2 / 2.0; + break; + case 16: + /* + 2:1 + 960x1080 16:9 frame without horizontal overscan + */ + aDest.sample_ratio = 2.0 / 1.0; + break; + case 255: + /* Extended_SAR */ + aDest.sar_width = aBr.ReadBits(16); + aDest.sar_height = aBr.ReadBits(16); + if (aDest.sar_width && aDest.sar_height) { + aDest.sample_ratio = float(aDest.sar_width) / float(aDest.sar_height); + } + break; + default: + break; + } + } + + if (aBr.ReadBit()) { // overscan_info_present_flag + aDest.overscan_appropriate_flag = aBr.ReadBit(); + } + + if (aBr.ReadBit()) { // video_signal_type_present_flag + aDest.video_format = aBr.ReadBits(3); + aDest.video_full_range_flag = aBr.ReadBit(); + aDest.colour_description_present_flag = aBr.ReadBit(); + if (aDest.colour_description_present_flag) { + aDest.colour_primaries = aBr.ReadBits(8); + aDest.transfer_characteristics = aBr.ReadBits(8); + aDest.matrix_coefficients = aBr.ReadBits(8); + } + } + + aDest.chroma_loc_info_present_flag = aBr.ReadBit(); + if (aDest.chroma_loc_info_present_flag) { + BitReader& br = aBr; // so that macro READUE works + READUE(chroma_sample_loc_type_top_field, 5); + READUE(chroma_sample_loc_type_bottom_field, 5); + } + + bool timing_info_present_flag = aBr.ReadBit(); + if (timing_info_present_flag) { + aBr.ReadBits(32); // num_units_in_tick + aBr.ReadBits(32); // time_scale + aBr.ReadBit(); // fixed_frame_rate_flag + } + return true; +} + +/* static */ +bool H264::DecodeSPSFromExtraData(const mozilla::MediaByteBuffer* aExtraData, + SPSData& aDest) { + SPSNALIterator it(aExtraData); + if (!it) { + return false; + } + return (*it).GetSPSData(aDest); +} + +/* static */ +bool H264::EnsureSPSIsSane(SPSData& aSPS) { + bool valid = true; + static const float default_aspect = 4.0f / 3.0f; + if (aSPS.sample_ratio <= 0.0f || aSPS.sample_ratio > 6.0f) { + if (aSPS.pic_width && aSPS.pic_height) { + aSPS.sample_ratio = (float)aSPS.pic_width / (float)aSPS.pic_height; + } else { + aSPS.sample_ratio = default_aspect; + } + aSPS.display_width = aSPS.pic_width; + aSPS.display_height = aSPS.pic_height; + valid = false; + } + if (aSPS.max_num_ref_frames > 16) { + aSPS.max_num_ref_frames = 16; + valid = false; + } + return valid; +} + +/* static */ +uint32_t H264::ComputeMaxRefFrames(const mozilla::MediaByteBuffer* aExtraData) { + uint32_t maxRefFrames = 4; + // Retrieve video dimensions from H264 SPS NAL. + SPSData spsdata; + if (DecodeSPSFromExtraData(aExtraData, spsdata)) { + // max_num_ref_frames determines the size of the sliding window + // we need to queue that many frames in order to guarantee proper + // pts frames ordering. Use a minimum of 4 to ensure proper playback of + // non compliant videos. + maxRefFrames = + std::min(std::max(maxRefFrames, spsdata.max_num_ref_frames + 1), 16u); + } + return maxRefFrames; +} + +/* static */ H264::FrameType H264::GetFrameType( + const mozilla::MediaRawData* aSample) { + auto avcc = AVCCConfig::Parse(aSample); + if (avcc.isErr()) { + // We must have a valid AVCC frame with extradata. + return FrameType::INVALID; + } + MOZ_ASSERT(aSample->Data()); + + int nalLenSize = avcc.unwrap().NALUSize(); + + BufferReader reader(aSample->Data(), aSample->Size()); + + while (reader.Remaining() >= nalLenSize) { + uint32_t nalLen = 0; + switch (nalLenSize) { + case 1: + nalLen = reader.ReadU8().unwrapOr(0); + break; + case 2: + nalLen = reader.ReadU16().unwrapOr(0); + break; + case 3: + nalLen = reader.ReadU24().unwrapOr(0); + break; + case 4: + nalLen = reader.ReadU32().unwrapOr(0); + break; + default: + MOZ_ASSERT_UNREACHABLE("NAL length is up to 4 bytes"); + } + if (!nalLen) { + continue; + } + const uint8_t* p = reader.Read(nalLen); + if (!p) { + return FrameType::INVALID; + } + int8_t nalType = AssertedCast<int8_t>(*p & 0x1f); + if (nalType == H264_NAL_IDR_SLICE) { + // IDR NAL. + return FrameType::I_FRAME; + } + if (nalType == H264_NAL_SEI) { + RefPtr<mozilla::MediaByteBuffer> decodedNAL = DecodeNALUnit(p, nalLen); + SEIRecoveryData data; + if (DecodeRecoverySEI(decodedNAL, data)) { + return FrameType::I_FRAME; + } + } else if (nalType == H264_NAL_SLICE) { + RefPtr<mozilla::MediaByteBuffer> decodedNAL = DecodeNALUnit(p, nalLen); + if (DecodeISlice(decodedNAL)) { + return FrameType::I_FRAME; + } + } + } + + return FrameType::OTHER; +} + +/* static */ already_AddRefed<mozilla::MediaByteBuffer> H264::ExtractExtraData( + const mozilla::MediaRawData* aSample) { + auto avcc = AVCCConfig::Parse(aSample); + MOZ_ASSERT(avcc.isOk()); + + RefPtr<mozilla::MediaByteBuffer> extradata = new mozilla::MediaByteBuffer; + + // SPS content + nsTArray<uint8_t> sps; + ByteWriter<BigEndian> spsw(sps); + int numSps = 0; + // PPS content + nsTArray<uint8_t> pps; + ByteWriter<BigEndian> ppsw(pps); + int numPps = 0; + + int nalLenSize = avcc.unwrap().NALUSize(); + + size_t sampleSize = aSample->Size(); + if (aSample->mCrypto.IsEncrypted()) { + // The content is encrypted, we can only parse the non-encrypted data. + MOZ_ASSERT(aSample->mCrypto.mPlainSizes.Length() > 0); + if (aSample->mCrypto.mPlainSizes.Length() == 0 || + aSample->mCrypto.mPlainSizes[0] > sampleSize) { + // This is invalid content. + return nullptr; + } + sampleSize = aSample->mCrypto.mPlainSizes[0]; + } + + BufferReader reader(aSample->Data(), sampleSize); + + nsTArray<SPSData> SPSTable; + // If we encounter SPS with the same id but different content, we will stop + // attempting to detect duplicates. + bool checkDuplicate = true; + + // Find SPS and PPS NALUs in AVCC data + while (reader.Remaining() > nalLenSize) { + uint32_t nalLen = 0; + switch (nalLenSize) { + case 1: + Unused << reader.ReadU8().map( + [&](uint8_t x) mutable { return nalLen = x; }); + break; + case 2: + Unused << reader.ReadU16().map( + [&](uint16_t x) mutable { return nalLen = x; }); + break; + case 3: + Unused << reader.ReadU24().map( + [&](uint32_t x) mutable { return nalLen = x; }); + break; + case 4: + Unused << reader.ReadU32().map( + [&](uint32_t x) mutable { return nalLen = x; }); + break; + default: + MOZ_ASSERT_UNREACHABLE("NAL length size is at most 4 bytes"); + } + const uint8_t* p = reader.Read(nalLen); + if (!p) { + // The read failed, but we may already have some SPS + PPS data so + // break out of reading and process what we have, if any. + break; + } + uint8_t nalType = *p & 0x1f; + + if (nalType == H264_NAL_SPS) { + RefPtr<mozilla::MediaByteBuffer> sps = DecodeNALUnit(p, nalLen); + SPSData data; + if (!DecodeSPS(sps, data)) { + // Invalid SPS, ignore. + continue; + } + uint8_t spsId = data.seq_parameter_set_id; + if (spsId >= SPSTable.Length()) { + if (!SPSTable.SetLength(spsId + 1, fallible)) { + // OOM. + return nullptr; + } + } + if (checkDuplicate && SPSTable[spsId].valid && SPSTable[spsId] == data) { + // Duplicate ignore. + continue; + } + if (SPSTable[spsId].valid) { + // We already have detected a SPS with this Id. Just to be safe we + // disable SPS duplicate detection. + checkDuplicate = false; + } else { + SPSTable[spsId] = data; + } + numSps++; + if (!spsw.WriteU16(nalLen) || !spsw.Write(p, nalLen)) { + return extradata.forget(); + } + } else if (nalType == H264_NAL_PPS) { + numPps++; + if (!ppsw.WriteU16(nalLen) || !ppsw.Write(p, nalLen)) { + return extradata.forget(); + } + } + } + + // We ignore PPS data if we didn't find a SPS as we would be unable to + // decode it anyway. + numPps = numSps ? numPps : 0; + + if (numSps && sps.Length() > 5) { + extradata->AppendElement(1); // version + extradata->AppendElement(sps[3]); // profile + extradata->AppendElement(sps[4]); // profile compat + extradata->AppendElement(sps[5]); // level + extradata->AppendElement(0xfc | 3); // nal size - 1 + extradata->AppendElement(0xe0 | numSps); + extradata->AppendElements(sps.Elements(), sps.Length()); + extradata->AppendElement(numPps); + if (numPps) { + extradata->AppendElements(pps.Elements(), pps.Length()); + } + } + + return extradata.forget(); +} + +/* static */ +uint8_t H264::NumSPS(const mozilla::MediaByteBuffer* aExtraData) { + auto avcc = AVCCConfig::Parse(aExtraData); + return avcc.isErr() ? 0 : avcc.unwrap().mNumSPS; +} + +/* static */ +bool H264::HasSPS(const mozilla::MediaByteBuffer* aExtraData) { + return H264::NumSPS(aExtraData) > 0; +} + +/* static */ +bool H264::CompareExtraData(const mozilla::MediaByteBuffer* aExtraData1, + const mozilla::MediaByteBuffer* aExtraData2) { + if (aExtraData1 == aExtraData2) { + return true; + } + uint8_t numSPS = NumSPS(aExtraData1); + if (numSPS == 0 || numSPS != NumSPS(aExtraData2)) { + return false; + } + + // We only compare if the SPS are the same as the various H264 decoders can + // deal with in-band change of PPS. + + SPSNALIterator it1(aExtraData1); + SPSNALIterator it2(aExtraData2); + + while (it1 && it2) { + if (*it1 != *it2) { + return false; + } + ++it1; + ++it2; + } + return true; +} + +static inline Result<Ok, nsresult> ReadSEIInt(BufferReader& aBr, + uint32_t& aOutput) { + uint8_t tmpByte; + + aOutput = 0; + MOZ_TRY_VAR(tmpByte, aBr.ReadU8()); + while (tmpByte == 0xFF) { + aOutput += 255; + MOZ_TRY_VAR(tmpByte, aBr.ReadU8()); + } + aOutput += tmpByte; // this is the last byte + return Ok(); +} + +/* static */ +bool H264::DecodeISlice(const mozilla::MediaByteBuffer* aSlice) { + if (!aSlice) { + return false; + } + + // According to ITU-T Rec H.264 Table 7.3.3, read the slice type from + // slice_header, and the slice type 2 and 7 are representing I slice. + BitReader br(aSlice); + // Skip `first_mb_in_slice` + br.ReadUE(); + // The value of slice type can go from 0 to 9, but the value between 5 to + // 9 are actually equal to 0 to 4. + const uint32_t sliceType = br.ReadUE() % 5; + return sliceType == SLICE_TYPES::I_SLICE || sliceType == SI_SLICE; +} + +/* static */ +bool H264::DecodeRecoverySEI(const mozilla::MediaByteBuffer* aSEI, + SEIRecoveryData& aDest) { + if (!aSEI) { + return false; + } + // sei_rbsp() as per 7.3.2.3 Supplemental enhancement information RBSP syntax + BufferReader br(aSEI); + + do { + // sei_message() as per + // 7.3.2.3.1 Supplemental enhancement information message syntax + uint32_t payloadType = 0; + if (ReadSEIInt(br, payloadType).isErr()) { + return false; + } + + uint32_t payloadSize = 0; + if (ReadSEIInt(br, payloadSize).isErr()) { + return false; + } + + // sei_payload(payloadType, payloadSize) as per + // D.1 SEI payload syntax. + const uint8_t* p = br.Read(payloadSize); + if (!p) { + return false; + } + if (payloadType == 6) { // SEI_RECOVERY_POINT + if (payloadSize == 0) { + // Invalid content, ignore. + continue; + } + // D.1.7 Recovery point SEI message syntax + BitReader br(p, payloadSize * 8); + aDest.recovery_frame_cnt = br.ReadUE(); + aDest.exact_match_flag = br.ReadBit(); + aDest.broken_link_flag = br.ReadBit(); + aDest.changing_slice_group_idc = br.ReadBits(2); + return true; + } + } while (br.PeekU8().isOk() && + br.PeekU8().unwrap() != + 0x80); // more_rbsp_data() msg[offset] != 0x80 + // ignore the trailing bits rbsp_trailing_bits(); + return false; +} + +/*static */ already_AddRefed<mozilla::MediaByteBuffer> H264::CreateExtraData( + uint8_t aProfile, uint8_t aConstraints, uint8_t aLevel, + const gfx::IntSize& aSize) { + // SPS of a 144p video. + const uint8_t originSPS[] = {0x4d, 0x40, 0x0c, 0xe8, 0x80, 0x80, 0x9d, + 0x80, 0xb5, 0x01, 0x01, 0x01, 0x40, 0x00, + 0x00, 0x00, 0x40, 0x00, 0x00, 0x0f, 0x03, + 0xc5, 0x0a, 0x44, 0x80}; + + RefPtr<MediaByteBuffer> extraData = new MediaByteBuffer(); + extraData->AppendElements(originSPS, sizeof(originSPS)); + BitReader br(extraData, BitReader::GetBitLength(extraData)); + + RefPtr<MediaByteBuffer> sps = new MediaByteBuffer(); + BitWriter bw(sps); + + br.ReadBits(8); // Skip original profile_idc + bw.WriteU8(aProfile); + br.ReadBits(8); // Skip original constraint flags && reserved_zero_2bits + aConstraints = + aConstraints & ~0x3; // Ensure reserved_zero_2bits are set to 0 + bw.WriteBits(aConstraints, 8); + br.ReadBits(8); // Skip original level_idc + bw.WriteU8(aLevel); + bw.WriteUE(br.ReadUE()); // seq_parameter_set_id (0 stored on 1 bit) + + if (aProfile == 100 || aProfile == 110 || aProfile == 122 || + aProfile == 244 || aProfile == 44 || aProfile == 83 || aProfile == 86 || + aProfile == 118 || aProfile == 128 || aProfile == 138 || + aProfile == 139 || aProfile == 134) { + bw.WriteUE(1); // chroma_format_idc -> always set to 4:2:0 chroma format + bw.WriteUE(0); // bit_depth_luma_minus8 -> always 8 bits here + bw.WriteUE(0); // bit_depth_chroma_minus8 -> always 8 bits here + bw.WriteBit(false); // qpprime_y_zero_transform_bypass_flag + bw.WriteBit(false); // seq_scaling_matrix_present_flag + } + + bw.WriteBits(br.ReadBits(11), + 11); // log2_max_frame_num to gaps_in_frame_num_allowed_flag + + // skip over original exp-golomb encoded width/height + br.ReadUE(); // skip width + br.ReadUE(); // skip height + uint32_t width = aSize.width; + uint32_t widthNeeded = width % 16 != 0 ? (width / 16 + 1) * 16 : width; + uint32_t height = aSize.height; + uint32_t heightNeeded = height % 16 != 0 ? (height / 16 + 1) * 16 : height; + bw.WriteUE(widthNeeded / 16 - 1); + bw.WriteUE(heightNeeded / 16 - 1); + bw.WriteBit(br.ReadBit()); // write frame_mbs_only_flag + bw.WriteBit(br.ReadBit()); // write direct_8x8_inference_flag; + if (widthNeeded != width || heightNeeded != height) { + // Write cropping value + bw.WriteBit(true); // skip frame_cropping_flag + bw.WriteUE(0); // frame_crop_left_offset + bw.WriteUE((widthNeeded - width) / 2); // frame_crop_right_offset + bw.WriteUE(0); // frame_crop_top_offset + bw.WriteUE((heightNeeded - height) / 2); // frame_crop_bottom_offset + } else { + bw.WriteBit(false); // skip frame_cropping_flag + } + br.ReadBit(); // skip frame_cropping_flag; + // Write the remainings of the original sps (vui_parameters which sets an + // aspect ration of 1.0) + while (br.BitsLeft()) { + bw.WriteBit(br.ReadBit()); + } + bw.CloseWithRbspTrailing(); + + RefPtr<MediaByteBuffer> encodedSPS = + EncodeNALUnit(sps->Elements(), sps->Length()); + extraData->Clear(); + + const uint8_t PPS[] = {0xeb, 0xef, 0x20}; + + WriteExtraData( + extraData, aProfile, aConstraints, aLevel, + Span<const uint8_t>(encodedSPS->Elements(), encodedSPS->Length()), + Span<const uint8_t>(PPS, sizeof(PPS))); + + return extraData.forget(); +} + +void H264::WriteExtraData(MediaByteBuffer* aDestExtraData, + const uint8_t aProfile, const uint8_t aConstraints, + const uint8_t aLevel, const Span<const uint8_t> aSPS, + const Span<const uint8_t> aPPS) { + aDestExtraData->AppendElement(1); + aDestExtraData->AppendElement(aProfile); + aDestExtraData->AppendElement(aConstraints); + aDestExtraData->AppendElement(aLevel); + aDestExtraData->AppendElement(3); // nalLENSize-1 + aDestExtraData->AppendElement(1); // numPPS + uint8_t c[2]; + mozilla::BigEndian::writeUint16(&c[0], aSPS.Length() + 1); + aDestExtraData->AppendElements(c, 2); + aDestExtraData->AppendElement((0x00 << 7) | (0x3 << 5) | H264_NAL_SPS); + aDestExtraData->AppendElements(aSPS.Elements(), aSPS.Length()); + + aDestExtraData->AppendElement(1); // numPPS + mozilla::BigEndian::writeUint16(&c[0], aPPS.Length() + 1); + aDestExtraData->AppendElements(c, 2); + aDestExtraData->AppendElement((0x00 << 7) | (0x3 << 5) | H264_NAL_PPS); + aDestExtraData->AppendElements(aPPS.Elements(), aPPS.Length()); +} + +/* static */ Result<AVCCConfig, nsresult> AVCCConfig::Parse( + const mozilla::MediaRawData* aSample) { + if (!aSample || aSample->Size() < 3) { + return mozilla::Err(NS_ERROR_FAILURE); + } + if (aSample->mTrackInfo && + !aSample->mTrackInfo->mMimeType.EqualsLiteral("video/avc")) { + LOG("Only allow 'video/avc' (mimeType=%s)", + aSample->mTrackInfo->mMimeType.get()); + return mozilla::Err(NS_ERROR_FAILURE); + } + return AVCCConfig::Parse(aSample->mExtraData); +} + +/* static */ Result<AVCCConfig, nsresult> AVCCConfig::Parse( + const mozilla::MediaByteBuffer* aExtraData) { + if (!aExtraData || aExtraData->Length() < 7) { + return mozilla::Err(NS_ERROR_FAILURE); + } + const auto& byteBuffer = *aExtraData; + if (byteBuffer[0] != 1) { + return mozilla::Err(NS_ERROR_FAILURE); + } + AVCCConfig avcc{}; + avcc.mConfigurationVersion = byteBuffer[0]; + avcc.mAVCProfileIndication = byteBuffer[1]; + avcc.mProfileCompatibility = byteBuffer[2]; + avcc.mAVCLevelIndication = byteBuffer[3]; + avcc.mLengthSizeMinusOne = byteBuffer[4] & 0x3; + avcc.mNumSPS = byteBuffer[5] & 0x1F; + return avcc; +} + +#undef READUE +#undef READSE + +} // namespace mozilla + +#undef LOG diff --git a/dom/media/platforms/agnostic/bytestreams/H264.h b/dom/media/platforms/agnostic/bytestreams/H264.h new file mode 100644 index 0000000000..c3651d1a0f --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/H264.h @@ -0,0 +1,547 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MP4_DEMUXER_H264_H_ +#define MP4_DEMUXER_H264_H_ + +#include <stdint.h> +#include "DecoderData.h" +#include "mozilla/gfx/Types.h" + +namespace mozilla { +class BitReader; + +// Spec 7.4.2.1 +#define MAX_SPS_COUNT 32 +#define MAX_PPS_COUNT 256 + +// NAL unit types +enum NAL_TYPES { + H264_NAL_SLICE = 1, + H264_NAL_DPA = 2, + H264_NAL_DPB = 3, + H264_NAL_DPC = 4, + H264_NAL_IDR_SLICE = 5, + H264_NAL_SEI = 6, + H264_NAL_SPS = 7, + H264_NAL_PPS = 8, + H264_NAL_AUD = 9, + H264_NAL_END_SEQUENCE = 10, + H264_NAL_END_STREAM = 11, + H264_NAL_FILLER_DATA = 12, + H264_NAL_SPS_EXT = 13, + H264_NAL_PREFIX = 14, + H264_NAL_AUXILIARY_SLICE = 19, + H264_NAL_SLICE_EXT = 20, + H264_NAL_SLICE_EXT_DVC = 21, +}; + +// According to ITU-T Rec H.264 (2017/04) Table 7.6. +enum SLICE_TYPES { + P_SLICE = 0, + B_SLICE = 1, + I_SLICE = 2, + SP_SLICE = 3, + SI_SLICE = 4, +}; + +struct SPSData { + bool operator==(const SPSData& aOther) const; + bool operator!=(const SPSData& aOther) const; + + gfx::YUVColorSpace ColorSpace() const; + gfx::ColorDepth ColorDepth() const; + + bool valid = {}; + + /* Decoded Members */ + /* + pic_width is the decoded width according to: + pic_width = ((pic_width_in_mbs_minus1 + 1) * 16) + - (frame_crop_left_offset + frame_crop_right_offset) * 2 + */ + uint32_t pic_width = {}; + /* + pic_height is the decoded height according to: + pic_height = (2 - frame_mbs_only_flag) * ((pic_height_in_map_units_minus1 + + 1) * 16) + - (frame_crop_top_offset + frame_crop_bottom_offset) * 2 + */ + uint32_t pic_height = {}; + + bool interlaced = {}; + + /* + Displayed size. + display_width and display_height are adjusted according to the display + sample aspect ratio. + */ + uint32_t display_width = {}; + uint32_t display_height = {}; + + float sample_ratio = {}; + + uint32_t crop_left = {}; + uint32_t crop_right = {}; + uint32_t crop_top = {}; + uint32_t crop_bottom = {}; + + /* + H264 decoding parameters according to ITU-T H.264 (T-REC-H.264-201402-I/en) + http://www.itu.int/rec/T-REC-H.264-201402-I/en + */ + + bool constraint_set0_flag = {}; + bool constraint_set1_flag = {}; + bool constraint_set2_flag = {}; + bool constraint_set3_flag = {}; + bool constraint_set4_flag = {}; + bool constraint_set5_flag = {}; + + /* + profile_idc and level_idc indicate the profile and level to which the coded + video sequence conforms when the SVC sequence parameter set is the active + SVC sequence parameter set. + */ + uint8_t profile_idc = {}; + uint8_t level_idc = {}; + + /* + seq_parameter_set_id identifies the sequence parameter set that is referred + to by the picture parameter set. The value of seq_parameter_set_id shall be + in the range of 0 to 31, inclusive. + */ + uint8_t seq_parameter_set_id = {}; + + /* + chroma_format_idc specifies the chroma sampling relative to the luma + sampling as specified in clause 6.2. The value of chroma_format_idc shall be + in the range of 0 to 3, inclusive. When chroma_format_idc is not present, + it shall be inferred to be equal to 1 (4:2:0 chroma format). + When profile_idc is equal to 183, chroma_format_idc shall be equal to 0 + (4:0:0 chroma format). + */ + uint8_t chroma_format_idc = {}; + + /* + bit_depth_luma_minus8 specifies the bit depth of the samples of the luma + array and the value of the luma quantisation parameter range offset + QpBdOffset Y , as specified by + BitDepth Y = 8 + bit_depth_luma_minus8 (7-3) + QpBdOffset Y = 6 * bit_depth_luma_minus8 (7-4) + When bit_depth_luma_minus8 is not present, it shall be inferred to be equal + to 0. bit_depth_luma_minus8 shall be in the range of 0 to 6, inclusive. + */ + uint8_t bit_depth_luma_minus8 = {}; + + /* + bit_depth_chroma_minus8 specifies the bit depth of the samples of the chroma + arrays and the value of the chroma quantisation parameter range offset + QpBdOffset C , as specified by + BitDepth C = 8 + bit_depth_chroma_minus8 (7-5) + QpBdOffset C = 6 * bit_depth_chroma_minus8 (7-6) + When bit_depth_chroma_minus8 is not present, it shall be inferred to be + equal to 0. bit_depth_chroma_minus8 shall be in the range of 0 to 6, + inclusive. + */ + uint8_t bit_depth_chroma_minus8 = {}; + + /* + separate_colour_plane_flag equal to 1 specifies that the three colour + components of the 4:4:4 chroma format are coded separately. + separate_colour_plane_flag equal to 0 specifies that the colour components + are not coded separately. When separate_colour_plane_flag is not present, + it shall be inferred to be equal to 0. When separate_colour_plane_flag is + equal to 1, the primary coded picture consists of three separate components, + each of which consists of coded samples of one colour plane (Y, Cb or Cr) + that each use the monochrome coding syntax. In this case, each colour plane + is associated with a specific colour_plane_id value. + */ + bool separate_colour_plane_flag = {}; + + /* + seq_scaling_matrix_present_flag equal to 1 specifies that the flags + seq_scaling_list_present_flag[ i ] for i = 0..7 or + i = 0..11 are present. seq_scaling_matrix_present_flag equal to 0 specifies + that these flags are not present and the sequence-level scaling list + specified by Flat_4x4_16 shall be inferred for i = 0..5 and the + sequence-level scaling list specified by Flat_8x8_16 shall be inferred for + i = 6..11. When seq_scaling_matrix_present_flag is not present, it shall be + inferred to be equal to 0. + */ + bool seq_scaling_matrix_present_flag = {}; + + /* + log2_max_frame_num_minus4 specifies the value of the variable + MaxFrameNum that is used in frame_num related derivations as + follows: + + MaxFrameNum = 2( log2_max_frame_num_minus4 + 4 ). The value of + log2_max_frame_num_minus4 shall be in the range of 0 to 12, inclusive. + */ + uint8_t log2_max_frame_num = {}; + + /* + pic_order_cnt_type specifies the method to decode picture order + count (as specified in subclause 8.2.1). The value of + pic_order_cnt_type shall be in the range of 0 to 2, inclusive. + */ + uint8_t pic_order_cnt_type = {}; + + /* + log2_max_pic_order_cnt_lsb_minus4 specifies the value of the + variable MaxPicOrderCntLsb that is used in the decoding + process for picture order count as specified in subclause + 8.2.1 as follows: + + MaxPicOrderCntLsb = 2( log2_max_pic_order_cnt_lsb_minus4 + 4 ) + + The value of log2_max_pic_order_cnt_lsb_minus4 shall be in + the range of 0 to 12, inclusive. + */ + uint8_t log2_max_pic_order_cnt_lsb = {}; + + /* + delta_pic_order_always_zero_flag equal to 1 specifies that + delta_pic_order_cnt[ 0 ] and delta_pic_order_cnt[ 1 ] are + not present in the slice headers of the sequence and shall + be inferred to be equal to 0. + */ + bool delta_pic_order_always_zero_flag = {}; + + /* + offset_for_non_ref_pic is used to calculate the picture + order count of a non-reference picture as specified in + 8.2.1. The value of offset_for_non_ref_pic shall be in the + range of -231 to 231 - 1, inclusive. + */ + int8_t offset_for_non_ref_pic = {}; + + /* + offset_for_top_to_bottom_field is used to calculate the + picture order count of a bottom field as specified in + subclause 8.2.1. The value of offset_for_top_to_bottom_field + shall be in the range of -231 to 231 - 1, inclusive. + */ + int8_t offset_for_top_to_bottom_field = {}; + + /* + max_num_ref_frames specifies the maximum number of short-term and + long-term reference frames, complementary reference field pairs, + and non-paired reference fields that may be used by the decoding + process for inter prediction of any picture in the + sequence. max_num_ref_frames also determines the size of the sliding + window operation as specified in subclause 8.2.5.3. The value of + max_num_ref_frames shall be in the range of 0 to MaxDpbFrames (as + specified in subclause A.3.1 or A.3.2), inclusive. + */ + uint32_t max_num_ref_frames = {}; + + /* + gaps_in_frame_num_value_allowed_flag specifies the allowed + values of frame_num as specified in subclause 7.4.3 and the + decoding process in case of an inferred gap between values of + frame_num as specified in subclause 8.2.5.2. + */ + bool gaps_in_frame_num_allowed_flag = {}; + + /* + pic_width_in_mbs_minus1 plus 1 specifies the width of each + decoded picture in units of macroblocks. 16 macroblocks in a row + */ + uint32_t pic_width_in_mbs = {}; + + /* + pic_height_in_map_units_minus1 plus 1 specifies the height in + slice group map units of a decoded frame or field. 16 + macroblocks in each column. + */ + uint32_t pic_height_in_map_units = {}; + + /* + frame_mbs_only_flag equal to 0 specifies that coded pictures of + the coded video sequence may either be coded fields or coded + frames. frame_mbs_only_flag equal to 1 specifies that every + coded picture of the coded video sequence is a coded frame + containing only frame macroblocks. + */ + bool frame_mbs_only_flag = {}; + + /* + mb_adaptive_frame_field_flag equal to 0 specifies no + switching between frame and field macroblocks within a + picture. mb_adaptive_frame_field_flag equal to 1 specifies + the possible use of switching between frame and field + macroblocks within frames. When mb_adaptive_frame_field_flag + is not present, it shall be inferred to be equal to 0. + */ + bool mb_adaptive_frame_field_flag = {}; + + /* + direct_8x8_inference_flag specifies the method used in the derivation + process for luma motion vectors for B_Skip, B_Direct_16x16 and B_Direct_8x8 + as specified in clause 8.4.1.2. When frame_mbs_only_flag is equal to 0, + direct_8x8_inference_flag shall be equal to 1. + */ + bool direct_8x8_inference_flag = {}; + + /* + frame_cropping_flag equal to 1 specifies that the frame cropping + offset parameters follow next in the sequence parameter + set. frame_cropping_flag equal to 0 specifies that the frame + cropping offset parameters are not present. + */ + bool frame_cropping_flag = {}; + uint32_t frame_crop_left_offset = {}; + uint32_t frame_crop_right_offset = {}; + uint32_t frame_crop_top_offset = {}; + uint32_t frame_crop_bottom_offset = {}; + + // VUI Parameters + + /* + vui_parameters_present_flag equal to 1 specifies that the + vui_parameters( ) syntax structure as specified in Annex E is + present. vui_parameters_present_flag equal to 0 specifies that + the vui_parameters( ) syntax structure as specified in Annex E + is not present. + */ + bool vui_parameters_present_flag = {}; + + /* + aspect_ratio_info_present_flag equal to 1 specifies that + aspect_ratio_idc is present. aspect_ratio_info_present_flag + equal to 0 specifies that aspect_ratio_idc is not present. + */ + bool aspect_ratio_info_present_flag = {}; + + /* + aspect_ratio_idc specifies the value of the sample aspect + ratio of the luma samples. Table E-1 shows the meaning of + the code. When aspect_ratio_idc indicates Extended_SAR, the + sample aspect ratio is represented by sar_width and + sar_height. When the aspect_ratio_idc syntax element is not + present, aspect_ratio_idc value shall be inferred to be + equal to 0. + */ + uint8_t aspect_ratio_idc = {}; + uint32_t sar_width = {}; + uint32_t sar_height = {}; + + /* + video_signal_type_present_flag equal to 1 specifies that video_format, + video_full_range_flag and colour_description_present_flag are present. + video_signal_type_present_flag equal to 0, specify that video_format, + video_full_range_flag and colour_description_present_flag are not present. + */ + bool video_signal_type_present_flag = {}; + + /* + overscan_info_present_flag equal to1 specifies that the + overscan_appropriate_flag is present. When overscan_info_present_flag is + equal to 0 or is not present, the preferred display method for the video + signal is unspecified (Unspecified). + */ + bool overscan_info_present_flag = {}; + /* + overscan_appropriate_flag equal to 1 indicates that the cropped decoded + pictures output are suitable for display using overscan. + overscan_appropriate_flag equal to 0 indicates that the cropped decoded + pictures output contain visually important information in the entire region + out to the edges of the cropping rectangle of the picture + */ + bool overscan_appropriate_flag = {}; + + /* + video_format indicates the representation of the pictures as specified in + Table E-2, before being coded in accordance with this + Recommendation | International Standard. When the video_format syntax + element is not present, video_format value shall be inferred to be equal + to 5. (Unspecified video format) + */ + uint8_t video_format = {}; + + /* + video_full_range_flag indicates the black level and range of the luma and + chroma signals as derived from E′Y, E′PB, and E′PR or E′R, E′G, and E′B + real-valued component signals. + When the video_full_range_flag syntax element is not present, the value of + video_full_range_flag shall be inferred to be equal to 0. + */ + bool video_full_range_flag = {}; + + /* + colour_description_present_flag equal to1 specifies that colour_primaries, + transfer_characteristics and matrix_coefficients are present. + colour_description_present_flag equal to 0 specifies that colour_primaries, + transfer_characteristics and matrix_coefficients are not present. + */ + bool colour_description_present_flag = {}; + + /* + colour_primaries indicates the chromaticity coordinates of the source + primaries as specified in Table E-3 in terms of the CIE 1931 definition of + x and y as specified by ISO 11664-1. + When the colour_primaries syntax element is not present, the value of + colour_primaries shall be inferred to be equal to 2 (the chromaticity is + unspecified or is determined by the application). + */ + uint8_t colour_primaries = {}; + + /* + transfer_characteristics indicates the opto-electronic transfer + characteristic of the source picture as specified in Table E-4 as a function + of a linear optical intensity input Lc with a nominal real-valued range of 0 + to 1. + When the transfer_characteristics syntax element is not present, the value + of transfer_characteristics shall be inferred to be equal to 2 + (the transfer characteristics are unspecified or are determined by the + application). + */ + uint8_t transfer_characteristics = {}; + + uint8_t matrix_coefficients = {}; + bool chroma_loc_info_present_flag = {}; + /* + The value of chroma_sample_loc_type_top_field and + chroma_sample_loc_type_bottom_field shall be in the range of 0 to 5, + inclusive + */ + uint8_t chroma_sample_loc_type_top_field = {}; + uint8_t chroma_sample_loc_type_bottom_field = {}; + + bool scaling_matrix_present = {}; + uint8_t scaling_matrix4x4[6][16] = {}; + uint8_t scaling_matrix8x8[6][64] = {}; + + SPSData(); +}; + +struct SEIRecoveryData { + /* + recovery_frame_cnt specifies the recovery point of output pictures in output + order. All decoded pictures in output order are indicated to be correct or + approximately correct in content starting at the output order position of + the reference picture having the frame_num equal to the frame_num of the VCL + NAL units for the current access unit incremented by recovery_frame_cnt in + modulo MaxFrameNum arithmetic. recovery_frame_cnt shall be in the range of 0 + to MaxFrameNum − 1, inclusive. + */ + uint32_t recovery_frame_cnt = 0; + /* + exact_match_flag indicates whether decoded pictures at and subsequent to the + specified recovery point in output order derived by starting the decoding + process at the access unit associated with the recovery point SEI message + shall be an exact match to the pictures that would be produced by starting + the decoding process at the location of a previous IDR access unit in the + NAL unit stream. The value 0 indicates that the match need not be exact and + the value 1 indicates that the match shall be exact. + */ + bool exact_match_flag = false; + /* + broken_link_flag indicates the presence or absence of a broken link in the + NAL unit stream at the location of the recovery point SEI message */ + bool broken_link_flag = false; + /* + changing_slice_group_idc equal to 0 indicates that decoded pictures are + correct or approximately correct in content at and subsequent to the + recovery point in output order when all macroblocks of the primary coded + pictures are decoded within the changing slice group period + */ + uint8_t changing_slice_group_idc = 0; +}; + +class H264 { + public: + /* Check if out of band extradata contains a SPS NAL */ + static bool HasSPS(const mozilla::MediaByteBuffer* aExtraData); + // Extract SPS and PPS NALs from aSample by looking into each NALs. + // aSample must be in AVCC format. + static already_AddRefed<mozilla::MediaByteBuffer> ExtractExtraData( + const mozilla::MediaRawData* aSample); + // Return true if both extradata are equal. + static bool CompareExtraData(const mozilla::MediaByteBuffer* aExtraData1, + const mozilla::MediaByteBuffer* aExtraData2); + + // Ensure that SPS data makes sense, Return true if SPS data was, and false + // otherwise. If false, then content will be adjusted accordingly. + static bool EnsureSPSIsSane(SPSData& aSPS); + + static bool DecodeSPSFromExtraData(const mozilla::MediaByteBuffer* aExtraData, + SPSData& aDest); + /* Decode SPS NAL RBSP and fill SPSData structure */ + static bool DecodeSPS(const mozilla::MediaByteBuffer* aSPS, SPSData& aDest); + + // If the given aExtraData is valid, return the aExtraData.max_num_ref_frames + // clamped to be in the range of [4, 16]; otherwise return 4. + static uint32_t ComputeMaxRefFrames( + const mozilla::MediaByteBuffer* aExtraData); + + enum class FrameType { + I_FRAME, + OTHER, + INVALID, + }; + + // Returns the frame type. Returns I_FRAME if the sample is an IDR + // (Instantaneous Decoding Refresh) Picture. + static FrameType GetFrameType(const mozilla::MediaRawData* aSample); + // Create a dummy extradata, useful to create a decoder and test the + // capabilities of the decoder. + static already_AddRefed<mozilla::MediaByteBuffer> CreateExtraData( + uint8_t aProfile, uint8_t aConstraints, uint8_t aLevel, + const gfx::IntSize& aSize); + static void WriteExtraData(mozilla::MediaByteBuffer* aDestExtraData, + const uint8_t aProfile, const uint8_t aConstraints, + const uint8_t aLevel, + const Span<const uint8_t> aSPS, + const Span<const uint8_t> aPPS); + + private: + friend class SPSNAL; + /* Extract RAW BYTE SEQUENCE PAYLOAD from NAL content. + Returns nullptr if invalid content. + This is compliant to ITU H.264 7.3.1 Syntax in tabular form NAL unit syntax + */ + static already_AddRefed<mozilla::MediaByteBuffer> DecodeNALUnit( + const uint8_t* aNAL, size_t aLength); + static already_AddRefed<mozilla::MediaByteBuffer> EncodeNALUnit( + const uint8_t* aNAL, size_t aLength); + static bool vui_parameters(mozilla::BitReader& aBr, SPSData& aDest); + // Read HRD parameters, all data is ignored. + static void hrd_parameters(mozilla::BitReader& aBr); + static uint8_t NumSPS(const mozilla::MediaByteBuffer* aExtraData); + // Decode SEI payload and return true if the SEI NAL indicates a recovery + // point. + static bool DecodeRecoverySEI(const mozilla::MediaByteBuffer* aSEI, + SEIRecoveryData& aDest); + // Decode NAL Slice payload and return true if its slice type is I slice or SI + // slice. + static bool DecodeISlice(const mozilla::MediaByteBuffer* aSlice); +}; + +// ISO/IEC 14496-15 : avcC. We only parse partial attributes, not all of them. +struct AVCCConfig final { + public: + static Result<AVCCConfig, nsresult> Parse( + const mozilla::MediaRawData* aSample); + static Result<AVCCConfig, nsresult> Parse( + const mozilla::MediaByteBuffer* aExtraData); + + uint8_t NALUSize() const { return mLengthSizeMinusOne + 1; } + + uint8_t mConfigurationVersion; + uint8_t mAVCProfileIndication; + uint8_t mProfileCompatibility; + uint8_t mAVCLevelIndication; + uint8_t mLengthSizeMinusOne; + uint8_t mNumSPS; + + private: + AVCCConfig() = default; +}; + +} // namespace mozilla + +#endif // MP4_DEMUXER_H264_H_ diff --git a/dom/media/platforms/agnostic/bytestreams/H265.cpp b/dom/media/platforms/agnostic/bytestreams/H265.cpp new file mode 100644 index 0000000000..bf2fe4b6b2 --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/H265.cpp @@ -0,0 +1,1300 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "H265.h" +#include <stdint.h> + +#include <cmath> +#include <limits> + +#include "AnnexB.h" +#include "BitReader.h" +#include "BitWriter.h" +#include "BufferReader.h" +#include "ByteStreamsUtils.h" +#include "ByteWriter.h" +#include "MediaData.h" +#include "MediaInfo.h" +#include "mozilla/ArrayUtils.h" +#include "mozilla/PodOperations.h" +#include "mozilla/ResultExtensions.h" +#include "mozilla/Span.h" + +mozilla::LazyLogModule gH265("H265"); + +#define LOG(msg, ...) MOZ_LOG(gH265, LogLevel::Debug, (msg, ##__VA_ARGS__)) +#define LOGV(msg, ...) MOZ_LOG(gH265, LogLevel::Verbose, (msg, ##__VA_ARGS__)) + +#define TRUE_OR_RETURN(condition) \ + do { \ + if (!(condition)) { \ + LOG(#condition " should be true!"); \ + return mozilla::Err(NS_ERROR_FAILURE); \ + } \ + } while (0) + +#define IN_RANGE_OR_RETURN(val, min, max) \ + do { \ + int64_t temp = AssertedCast<int64_t>(val); \ + if ((temp) < (min) || (max) < (temp)) { \ + LOG(#val " is not in the range of [" #min "," #max "]"); \ + return mozilla::Err(NS_ERROR_FAILURE); \ + } \ + } while (0) + +#define NON_ZERO_OR_RETURN(dest, val) \ + do { \ + int64_t temp = AssertedCast<int64_t>(val); \ + if ((temp) != 0) { \ + (dest) = (temp); \ + } else { \ + LOG(#dest " should be non-zero"); \ + return mozilla::Err(NS_ERROR_FAILURE); \ + } \ + } while (0) + +namespace mozilla { + +H265NALU::H265NALU(const uint8_t* aData, uint32_t aByteSize) + : mNALU(aData, aByteSize) { + // Per 7.3.1 NAL unit syntax + BitReader reader(aData, aByteSize * 8); + Unused << reader.ReadBit(); // forbidden_zero_bit + mNalUnitType = reader.ReadBits(6); + mNuhLayerId = reader.ReadBits(6); + mNuhTemporalIdPlus1 = reader.ReadBits(3); + LOGV("Created H265NALU, type=%hhu, size=%u", mNalUnitType, aByteSize); +} + +/* static */ Result<HVCCConfig, nsresult> HVCCConfig::Parse( + const mozilla::MediaRawData* aSample) { + if (!aSample) { + LOG("No sample"); + return mozilla::Err(NS_ERROR_FAILURE); + } + if (aSample->Size() < 3) { + LOG("Incorrect sample size %zu", aSample->Size()); + return mozilla::Err(NS_ERROR_FAILURE); + } + if (aSample->mTrackInfo && + !aSample->mTrackInfo->mMimeType.EqualsLiteral("video/hevc")) { + LOG("Only allow 'video/hevc' (mimeType=%s)", + aSample->mTrackInfo->mMimeType.get()); + return mozilla::Err(NS_ERROR_FAILURE); + } + return HVCCConfig::Parse(aSample->mExtraData); +} + +/* static */ +Result<HVCCConfig, nsresult> HVCCConfig::Parse( + const mozilla::MediaByteBuffer* aExtraData) { + // From configurationVersion to numOfArrays, total 184 bits (23 bytes) + if (!aExtraData) { + LOG("No extra-data"); + return mozilla::Err(NS_ERROR_FAILURE); + } + if (aExtraData->Length() < 23) { + LOG("Incorrect extra-data size %zu", aExtraData->Length()); + return mozilla::Err(NS_ERROR_FAILURE); + } + const auto& byteBuffer = *aExtraData; + if (byteBuffer[0] != 1) { + LOG("Version should always be 1"); + return mozilla::Err(NS_ERROR_FAILURE); + } + HVCCConfig hvcc; + + BitReader reader(aExtraData); + hvcc.configurationVersion = reader.ReadBits(8); + hvcc.general_profile_space = reader.ReadBits(2); + hvcc.general_tier_flag = reader.ReadBit(); + hvcc.general_profile_idc = reader.ReadBits(5); + hvcc.general_profile_compatibility_flags = reader.ReadU32(); + + uint32_t flagHigh = reader.ReadU32(); + uint16_t flagLow = reader.ReadBits(16); + hvcc.general_constraint_indicator_flags = + ((uint64_t)(flagHigh) << 16) | (uint64_t)(flagLow); + + hvcc.general_level_idc = reader.ReadBits(8); + Unused << reader.ReadBits(4); // reserved + hvcc.min_spatial_segmentation_idc = reader.ReadBits(12); + Unused << reader.ReadBits(6); // reserved + hvcc.parallelismType = reader.ReadBits(2); + Unused << reader.ReadBits(6); // reserved + hvcc.chroma_format_idc = reader.ReadBits(2); + Unused << reader.ReadBits(5); // reserved + hvcc.bit_depth_luma_minus8 = reader.ReadBits(3); + Unused << reader.ReadBits(5); // reserved + hvcc.bit_depth_chroma_minus8 = reader.ReadBits(3); + hvcc.avgFrameRate = reader.ReadBits(16); + hvcc.constantFrameRate = reader.ReadBits(2); + hvcc.numTemporalLayers = reader.ReadBits(3); + hvcc.temporalIdNested = reader.ReadBit(); + hvcc.lengthSizeMinusOne = reader.ReadBits(2); + const uint8_t numOfArrays = reader.ReadBits(8); + for (uint8_t idx = 0; idx < numOfArrays; idx++) { + Unused << reader.ReadBits(2); // array_completeness + reserved + const uint8_t nalUnitType = reader.ReadBits(6); + const uint16_t numNalus = reader.ReadBits(16); + LOGV("nalu-type=%u, nalu-num=%u", nalUnitType, numNalus); + for (uint16_t nIdx = 0; nIdx < numNalus; nIdx++) { + const uint16_t nalUnitLength = reader.ReadBits(16); + if (reader.BitsLeft() < nalUnitLength * 8) { + LOG("Aborting parsing, NALU size (%u bits) is larger than remaining " + "(%zu bits)!", + nalUnitLength * 8u, reader.BitsLeft()); + // We return what we've parsed so far and ignore the rest. + hvcc.mByteBuffer = aExtraData; + return hvcc; + } + const uint8_t* currentPtr = + aExtraData->Elements() + reader.BitCount() / 8; + H265NALU nalu(currentPtr, nalUnitLength); + // ReadBits can only read at most 32 bits at a time. + uint32_t nalSize = nalUnitLength * 8; + while (nalSize > 0) { + uint32_t readBits = nalSize > 32 ? 32 : nalSize; + reader.ReadBits(readBits); + nalSize -= readBits; + } + // Per ISO_IEC-14496-15-2022, 8.3.2.1.3 Semantics, NALU should only be + // SPS/PPS/VPS or SEI, ignore all the other types of NALU. + if (nalu.IsSPS() || nalu.IsPPS() || nalu.IsVPS() || nalu.IsSEI()) { + hvcc.mNALUs.AppendElement(nalu); + } else { + LOG("Ignore NALU (%u) which is not SPS/PPS/VPS or SEI", + nalu.mNalUnitType); + } + } + } + hvcc.mByteBuffer = aExtraData; + return hvcc; +} + +uint32_t HVCCConfig::NumSPS() const { + uint32_t spsCounter = 0; + for (const auto& nalu : mNALUs) { + if (nalu.IsSPS()) { + spsCounter++; + } + } + return spsCounter; +} + +bool HVCCConfig::HasSPS() const { + bool hasSPS = false; + for (const auto& nalu : mNALUs) { + if (nalu.IsSPS()) { + hasSPS = true; + break; + } + } + return hasSPS; +} + +/* static */ +Result<H265SPS, nsresult> H265::DecodeSPSFromSPSNALU(const H265NALU& aSPSNALU) { + MOZ_ASSERT(aSPSNALU.IsSPS()); + RefPtr<MediaByteBuffer> rbsp = H265::DecodeNALUnit(aSPSNALU.mNALU); + if (!rbsp) { + LOG("Failed to decode NALU"); + return Err(NS_ERROR_FAILURE); + } + + // H265 spec, 7.3.2.2.1 seq_parameter_set_rbsp + H265SPS sps; + BitReader reader(rbsp); + sps.sps_video_parameter_set_id = reader.ReadBits(4); + IN_RANGE_OR_RETURN(sps.sps_video_parameter_set_id, 0, 15); + sps.sps_max_sub_layers_minus1 = reader.ReadBits(3); + IN_RANGE_OR_RETURN(sps.sps_max_sub_layers_minus1, 0, 6); + sps.sps_temporal_id_nesting_flag = reader.ReadBit(); + + if (auto rv = ParseProfileTierLevel( + reader, true /* aProfilePresentFlag, true per spec*/, + sps.sps_max_sub_layers_minus1, sps.profile_tier_level); + rv.isErr()) { + LOG("Failed to parse the profile tier level."); + return Err(NS_ERROR_FAILURE); + } + + sps.sps_seq_parameter_set_id = reader.ReadUE(); + IN_RANGE_OR_RETURN(sps.sps_seq_parameter_set_id, 0, 15); + sps.chroma_format_idc = reader.ReadUE(); + IN_RANGE_OR_RETURN(sps.chroma_format_idc, 0, 3); + + if (sps.chroma_format_idc == 3) { + sps.separate_colour_plane_flag = reader.ReadBit(); + } + + // From Table 6-1. + if (sps.chroma_format_idc == 1) { + sps.subWidthC = sps.subHeightC = 2; + } else if (sps.chroma_format_idc == 2) { + sps.subWidthC = 2; + sps.subHeightC = 1; + } else { + sps.subWidthC = sps.subHeightC = 1; + } + + NON_ZERO_OR_RETURN(sps.pic_width_in_luma_samples, reader.ReadUE()); + NON_ZERO_OR_RETURN(sps.pic_height_in_luma_samples, reader.ReadUE()); + { + // (A-2) Calculate maxDpbSize + const auto maxLumaPs = sps.profile_tier_level.GetMaxLumaPs(); + CheckedUint32 picSize = sps.pic_height_in_luma_samples; + picSize *= sps.pic_width_in_luma_samples; + if (!picSize.isValid()) { + LOG("Invalid picture size"); + return Err(NS_ERROR_FAILURE); + } + const auto picSizeInSamplesY = picSize.value(); + const auto maxDpbPicBuf = sps.profile_tier_level.GetDpbMaxPicBuf(); + if (picSizeInSamplesY <= (maxLumaPs >> 2)) { + sps.maxDpbSize = std::min(4 * maxDpbPicBuf, 16u); + } else if (picSizeInSamplesY <= (maxLumaPs >> 1)) { + sps.maxDpbSize = std::min(2 * maxDpbPicBuf, 16u); + } else if (picSizeInSamplesY <= ((3 * maxLumaPs) >> 2)) { + sps.maxDpbSize = std::min((4 * maxDpbPicBuf) / 3, 16u); + } else { + sps.maxDpbSize = maxDpbPicBuf; + } + } + + sps.conformance_window_flag = reader.ReadBit(); + if (sps.conformance_window_flag) { + sps.conf_win_left_offset = reader.ReadUE(); + sps.conf_win_right_offset = reader.ReadUE(); + sps.conf_win_top_offset = reader.ReadUE(); + sps.conf_win_bottom_offset = reader.ReadUE(); + } + sps.bit_depth_luma_minus8 = reader.ReadUE(); + IN_RANGE_OR_RETURN(sps.bit_depth_luma_minus8, 0, 8); + sps.bit_depth_chroma_minus8 = reader.ReadUE(); + IN_RANGE_OR_RETURN(sps.bit_depth_chroma_minus8, 0, 8); + sps.log2_max_pic_order_cnt_lsb_minus4 = reader.ReadUE(); + IN_RANGE_OR_RETURN(sps.log2_max_pic_order_cnt_lsb_minus4, 0, 12); + sps.sps_sub_layer_ordering_info_present_flag = reader.ReadBit(); + for (auto i = sps.sps_sub_layer_ordering_info_present_flag + ? 0 + : sps.sps_max_sub_layers_minus1; + i <= sps.sps_max_sub_layers_minus1; i++) { + sps.sps_max_dec_pic_buffering_minus1[i] = reader.ReadUE(); + IN_RANGE_OR_RETURN(sps.sps_max_dec_pic_buffering_minus1[i], 0, + sps.maxDpbSize - 1); + sps.sps_max_num_reorder_pics[i] = reader.ReadUE(); + IN_RANGE_OR_RETURN(sps.sps_max_num_reorder_pics[i], 0, + sps.sps_max_dec_pic_buffering_minus1[i]); + // 7.4.3.2.1, see sps_max_dec_pic_buffering_minus1 and + // sps_max_num_reorder_pics, "When i is greater than 0, ....". + if (i > 0) { + TRUE_OR_RETURN(sps.sps_max_dec_pic_buffering_minus1[i] >= + sps.sps_max_dec_pic_buffering_minus1[i - 1]); + TRUE_OR_RETURN(sps.sps_max_num_reorder_pics[i] >= + sps.sps_max_num_reorder_pics[i - 1]); + } + sps.sps_max_latency_increase_plus1[i] = reader.ReadUE(); + IN_RANGE_OR_RETURN(sps.sps_max_latency_increase_plus1[i], 0, 0xFFFFFFFE); + } + sps.log2_min_luma_coding_block_size_minus3 = reader.ReadUE(); + sps.log2_diff_max_min_luma_coding_block_size = reader.ReadUE(); + sps.log2_min_luma_transform_block_size_minus2 = reader.ReadUE(); + sps.log2_diff_max_min_luma_transform_block_size = reader.ReadUE(); + sps.max_transform_hierarchy_depth_inter = reader.ReadUE(); + sps.max_transform_hierarchy_depth_intra = reader.ReadUE(); + const auto scaling_list_enabled_flag = reader.ReadBit(); + if (scaling_list_enabled_flag) { + Unused << reader.ReadBit(); // sps_scaling_list_data_present_flag + if (auto rv = ParseAndIgnoreScalingListData(reader); rv.isErr()) { + LOG("Failed to parse scaling list data."); + return Err(NS_ERROR_FAILURE); + } + } + + // amp_enabled_flag and sample_adaptive_offset_enabled_flag + Unused << reader.ReadBits(2); + + sps.pcm_enabled_flag = reader.ReadBit(); + if (sps.pcm_enabled_flag) { + sps.pcm_sample_bit_depth_luma_minus1 = reader.ReadBits(3); + IN_RANGE_OR_RETURN(sps.pcm_sample_bit_depth_luma_minus1, 0, + sps.BitDepthLuma()); + sps.pcm_sample_bit_depth_chroma_minus1 = reader.ReadBits(3); + IN_RANGE_OR_RETURN(sps.pcm_sample_bit_depth_chroma_minus1, 0, + sps.BitDepthChroma()); + sps.log2_min_pcm_luma_coding_block_size_minus3 = reader.ReadUE(); + IN_RANGE_OR_RETURN(sps.log2_min_pcm_luma_coding_block_size_minus3, 0, 2); + uint32_t log2MinIpcmCbSizeY{sps.log2_min_pcm_luma_coding_block_size_minus3 + + 3}; + sps.log2_diff_max_min_pcm_luma_coding_block_size = reader.ReadUE(); + { + // Validate value + CheckedUint32 log2MaxIpcmCbSizeY{ + sps.log2_diff_max_min_pcm_luma_coding_block_size}; + log2MaxIpcmCbSizeY += log2MinIpcmCbSizeY; + CheckedUint32 minCbLog2SizeY{sps.log2_min_luma_coding_block_size_minus3}; + minCbLog2SizeY += 3; // (7-10) + CheckedUint32 ctbLog2SizeY{minCbLog2SizeY}; + ctbLog2SizeY += sps.log2_diff_max_min_luma_coding_block_size; // (7-11) + IN_RANGE_OR_RETURN(log2MaxIpcmCbSizeY.value(), 0, + std::min(ctbLog2SizeY.value(), uint32_t(5))); + } + sps.pcm_loop_filter_disabled_flag = reader.ReadBit(); + } + + sps.num_short_term_ref_pic_sets = reader.ReadUE(); + IN_RANGE_OR_RETURN(sps.num_short_term_ref_pic_sets, 0, + kMaxShortTermRefPicSets); + for (auto i = 0; i < sps.num_short_term_ref_pic_sets; i++) { + if (auto rv = ParseStRefPicSet(reader, i, sps); rv.isErr()) { + LOG("Failed to parse short-term reference picture set."); + return Err(NS_ERROR_FAILURE); + } + } + const auto long_term_ref_pics_present_flag = reader.ReadBit(); + if (long_term_ref_pics_present_flag) { + uint32_t num_long_term_ref_pics_sps; + num_long_term_ref_pics_sps = reader.ReadUE(); + IN_RANGE_OR_RETURN(num_long_term_ref_pics_sps, 0, kMaxLongTermRefPicSets); + for (auto i = 0; i < num_long_term_ref_pics_sps; i++) { + Unused << reader.ReadBits(sps.log2_max_pic_order_cnt_lsb_minus4 + + 4); // lt_ref_pic_poc_lsb_sps[i] + Unused << reader.ReadBit(); // used_by_curr_pic_lt_sps_flag + } + } + sps.sps_temporal_mvp_enabled_flag = reader.ReadBit(); + sps.strong_intra_smoothing_enabled_flag = reader.ReadBit(); + const auto vui_parameters_present_flag = reader.ReadBit(); + if (vui_parameters_present_flag) { + if (auto rv = ParseVuiParameters(reader, sps); rv.isErr()) { + LOG("Failed to parse VUI parameter."); + return Err(NS_ERROR_FAILURE); + } + } + + // The rest is extension data we don't care about, so no need to parse them. + return sps; +} + +/* static */ +Result<H265SPS, nsresult> H265::DecodeSPSFromHVCCExtraData( + const mozilla::MediaByteBuffer* aExtraData) { + auto rv = HVCCConfig::Parse(aExtraData); + if (rv.isErr()) { + LOG("Only support HVCC extra-data"); + return Err(NS_ERROR_FAILURE); + } + const auto& hvcc = rv.unwrap(); + const H265NALU* spsNALU = nullptr; + for (const auto& nalu : hvcc.mNALUs) { + if (nalu.IsSPS()) { + spsNALU = &nalu; + break; + } + } + if (!spsNALU) { + LOG("No sps found"); + return Err(NS_ERROR_FAILURE); + } + return DecodeSPSFromSPSNALU(*spsNALU); +} + +/* static */ +Result<Ok, nsresult> H265::ParseProfileTierLevel( + BitReader& aReader, bool aProfilePresentFlag, + uint8_t aMaxNumSubLayersMinus1, H265ProfileTierLevel& aProfile) { + // H265 spec, 7.3.3 Profile, tier and level syntax + if (aProfilePresentFlag) { + aProfile.general_profile_space = aReader.ReadBits(2); + aProfile.general_tier_flag = aReader.ReadBit(); + aProfile.general_profile_idc = aReader.ReadBits(5); + IN_RANGE_OR_RETURN(aProfile.general_profile_idc, 0, 11); + aProfile.general_profile_compatibility_flags = aReader.ReadU32(); + aProfile.general_progressive_source_flag = aReader.ReadBit(); + aProfile.general_interlaced_source_flag = aReader.ReadBit(); + aProfile.general_non_packed_constraint_flag = aReader.ReadBit(); + aProfile.general_frame_only_constraint_flag = aReader.ReadBit(); + // ignored attributes, in total general_reserved_zero_43bits + Unused << aReader.ReadBits(32); + Unused << aReader.ReadBits(11); + // general_inbld_flag or general_reserved_zero_bit + Unused << aReader.ReadBit(); + } + aProfile.general_level_idc = aReader.ReadBits(8); + + // Following are all ignored attributes. + bool sub_layer_profile_present_flag[8]; + bool sub_layer_level_present_flag[8]; + for (auto i = 0; i < aMaxNumSubLayersMinus1; i++) { + sub_layer_profile_present_flag[i] = aReader.ReadBit(); + sub_layer_level_present_flag[i] = aReader.ReadBit(); + } + if (aMaxNumSubLayersMinus1 > 0) { + for (auto i = aMaxNumSubLayersMinus1; i < 8; i++) { + // reserved_zero_2bits + Unused << aReader.ReadBits(2); + } + } + for (auto i = 0; i < aMaxNumSubLayersMinus1; i++) { + if (sub_layer_profile_present_flag[i]) { + // sub_layer_profile_space, sub_layer_tier_flag, sub_layer_profile_idc + Unused << aReader.ReadBits(8); + // sub_layer_profile_compatibility_flag + Unused << aReader.ReadBits(32); + // sub_layer_progressive_source_flag, sub_layer_interlaced_source_flag, + // sub_layer_non_packed_constraint_flag, + // sub_layer_frame_only_constraint_flag + Unused << aReader.ReadBits(4); + // ignored attributes, in total general_reserved_zero_43bits + Unused << aReader.ReadBits(32); + Unused << aReader.ReadBits(11); + // sub_layer_inbld_flag or reserved_zero_bit + Unused << aReader.ReadBit(); + } + if (sub_layer_level_present_flag[i]) { + Unused << aReader.ReadBits(8); // sub_layer_level_idc + } + } + return Ok(); +} + +uint32_t H265ProfileTierLevel::GetMaxLumaPs() const { + // From Table A.8 - General tier and level limits. + // "general_level_idc and sub_layer_level_idc[ i ] shall be set equal to a + // value of 30 times the level number specified in Table A.8". + if (general_level_idc <= 30) { // level 1 + return 36864; + } + if (general_level_idc <= 60) { // level 2 + return 122880; + } + if (general_level_idc <= 63) { // level 2.1 + return 245760; + } + if (general_level_idc <= 90) { // level 3 + return 552960; + } + if (general_level_idc <= 93) { // level 3.1 + return 983040; + } + if (general_level_idc <= 123) { // level 4, 4.1 + return 2228224; + } + if (general_level_idc <= 156) { // level 5, 5.1, 5.2 + return 8912896; + } + // level 6, 6.1, 6.2 - beyond that there's no actual limit. + return 35651584; +} + +uint32_t H265ProfileTierLevel::GetDpbMaxPicBuf() const { + // From A.4.2 - Profile-specific level limits for the video profiles. + // "maxDpbPicBuf is equal to 6 for all profiles where the value of + // sps_curr_pic_ref_enabled_flag is required to be equal to 0 and 7 for all + // profiles where the value of sps_curr_pic_ref_enabled_flag is not required + // to be equal to 0." From A.3 Profile, the flag in the main, main still, + // range extensions and high throughput is required to be zero. + return (general_profile_idc >= H265ProfileIdc::kProfileIdcMain && + general_profile_idc <= H265ProfileIdc::kProfileIdcHighThroughput) + ? 6 + : 7; +} + +/* static */ +Result<Ok, nsresult> H265::ParseAndIgnoreScalingListData(BitReader& aReader) { + // H265 spec, 7.3.4 Scaling list data syntax + for (auto sizeIdx = 0; sizeIdx < 4; sizeIdx++) { + for (auto matrixIdx = 0; matrixIdx < 6; + matrixIdx += (sizeIdx == 3) ? 3 : 1) { + const auto scaling_list_pred_mode_flag = aReader.ReadBit(); + if (!scaling_list_pred_mode_flag) { + Unused << aReader.ReadUE(); // scaling_list_pred_matrix_id_delta + } else { + int32_t coefNum = std::min(64, (1 << (4 + (sizeIdx << 1)))); + if (sizeIdx > 1) { + Unused << aReader.ReadSE(); // scaling_list_dc_coef_minus8 + } + for (auto i = 0; i < coefNum; i++) { + Unused << aReader.ReadSE(); // scaling_list_delta_coef + } + } + } + } + return Ok(); +} + +/* static */ +Result<Ok, nsresult> H265::ParseStRefPicSet(BitReader& aReader, + uint32_t aStRpsIdx, H265SPS& aSPS) { + // H265 Spec, 7.3.7 Short-term reference picture set syntax + MOZ_ASSERT(aStRpsIdx < kMaxShortTermRefPicSets); + bool inter_ref_pic_set_prediction_flag = false; + H265StRefPicSet& curStRefPicSet = aSPS.st_ref_pic_set[aStRpsIdx]; + if (aStRpsIdx != 0) { + inter_ref_pic_set_prediction_flag = aReader.ReadBit(); + } + if (inter_ref_pic_set_prediction_flag) { + int delta_idx_minus1 = 0; + if (aStRpsIdx == aSPS.num_short_term_ref_pic_sets) { + delta_idx_minus1 = aReader.ReadUE(); + IN_RANGE_OR_RETURN(delta_idx_minus1, 0, aStRpsIdx - 1); + } + const uint32_t RefRpsIdx = aStRpsIdx - (delta_idx_minus1 + 1); // (7-59) + const bool delta_rps_sign = aReader.ReadBit(); + const uint32_t abs_delta_rps_minus1 = aReader.ReadUE(); + IN_RANGE_OR_RETURN(abs_delta_rps_minus1, 0, 0x7FFF); + const int32_t deltaRps = + (1 - 2 * delta_rps_sign) * + AssertedCast<int32_t>(abs_delta_rps_minus1 + 1); // (7-60) + + bool used_by_curr_pic_flag[kMaxShortTermRefPicSets] = {}; + bool use_delta_flag[kMaxShortTermRefPicSets] = {}; + // 7.4.8 - use_delta_flag defaults to 1 if not present. + std::fill_n(use_delta_flag, kMaxShortTermRefPicSets, true); + const H265StRefPicSet& refSet = aSPS.st_ref_pic_set[RefRpsIdx]; + for (auto j = 0; j <= refSet.numDeltaPocs; j++) { + used_by_curr_pic_flag[j] = aReader.ReadBit(); + if (!used_by_curr_pic_flag[j]) { + use_delta_flag[j] = aReader.ReadBit(); + } + } + // Calculate fields (7-61) + uint32_t i = 0; + for (int64_t j = static_cast<int64_t>(refSet.num_positive_pics) - 1; j >= 0; + j--) { + MOZ_DIAGNOSTIC_ASSERT(j < kMaxShortTermRefPicSets); + int64_t d_poc = refSet.deltaPocS1[j] + deltaRps; + if (d_poc < 0 && use_delta_flag[refSet.num_negative_pics + j]) { + curStRefPicSet.deltaPocS0[i] = d_poc; + curStRefPicSet.usedByCurrPicS0[i++] = + used_by_curr_pic_flag[refSet.num_negative_pics + j]; + } + } + if (deltaRps < 0 && use_delta_flag[refSet.numDeltaPocs]) { + curStRefPicSet.deltaPocS0[i] = deltaRps; + curStRefPicSet.usedByCurrPicS0[i++] = + used_by_curr_pic_flag[refSet.numDeltaPocs]; + } + for (auto j = 0; j < refSet.num_negative_pics; j++) { + MOZ_DIAGNOSTIC_ASSERT(j < kMaxShortTermRefPicSets); + int64_t d_poc = refSet.deltaPocS0[j] + deltaRps; + if (d_poc < 0 && use_delta_flag[j]) { + curStRefPicSet.deltaPocS0[i] = d_poc; + curStRefPicSet.usedByCurrPicS0[i++] = used_by_curr_pic_flag[j]; + } + } + curStRefPicSet.num_negative_pics = i; + // Calculate fields (7-62) + i = 0; + for (int64_t j = static_cast<int64_t>(refSet.num_negative_pics) - 1; j >= 0; + j--) { + MOZ_DIAGNOSTIC_ASSERT(j < kMaxShortTermRefPicSets); + int64_t d_poc = refSet.deltaPocS0[j] + deltaRps; + if (d_poc > 0 && use_delta_flag[j]) { + curStRefPicSet.deltaPocS1[i] = d_poc; + curStRefPicSet.usedByCurrPicS1[i++] = used_by_curr_pic_flag[j]; + } + } + if (deltaRps > 0 && use_delta_flag[refSet.numDeltaPocs]) { + curStRefPicSet.deltaPocS1[i] = deltaRps; + curStRefPicSet.usedByCurrPicS1[i++] = + used_by_curr_pic_flag[refSet.numDeltaPocs]; + } + for (auto j = 0; j < refSet.num_positive_pics; j++) { + MOZ_DIAGNOSTIC_ASSERT(j < kMaxShortTermRefPicSets); + int64_t d_poc = refSet.deltaPocS1[j] + deltaRps; + if (d_poc > 0 && use_delta_flag[refSet.num_negative_pics + j]) { + curStRefPicSet.deltaPocS1[i] = d_poc; + curStRefPicSet.usedByCurrPicS1[i++] = + used_by_curr_pic_flag[refSet.num_negative_pics + j]; + } + } + curStRefPicSet.num_positive_pics = i; + } else { + curStRefPicSet.num_negative_pics = aReader.ReadUE(); + curStRefPicSet.num_positive_pics = aReader.ReadUE(); + const uint32_t spsMaxDecPicBufferingMinus1 = + aSPS.sps_max_dec_pic_buffering_minus1[aSPS.sps_max_sub_layers_minus1]; + IN_RANGE_OR_RETURN(curStRefPicSet.num_negative_pics, 0, + spsMaxDecPicBufferingMinus1); + CheckedUint32 maxPositivePics{spsMaxDecPicBufferingMinus1}; + maxPositivePics -= curStRefPicSet.num_negative_pics; + IN_RANGE_OR_RETURN(curStRefPicSet.num_positive_pics, 0, + maxPositivePics.value()); + for (auto i = 0; i < curStRefPicSet.num_negative_pics; i++) { + const uint32_t delta_poc_s0_minus1 = aReader.ReadUE(); + IN_RANGE_OR_RETURN(delta_poc_s0_minus1, 0, 0x7FFF); + if (i == 0) { + // (7-67) + curStRefPicSet.deltaPocS0[i] = -(delta_poc_s0_minus1 + 1); + } else { + // (7-69) + curStRefPicSet.deltaPocS0[i] = + curStRefPicSet.deltaPocS0[i - 1] - (delta_poc_s0_minus1 + 1); + } + curStRefPicSet.usedByCurrPicS0[i] = aReader.ReadBit(); + } + for (auto i = 0; i < curStRefPicSet.num_positive_pics; i++) { + const int delta_poc_s1_minus1 = aReader.ReadUE(); + IN_RANGE_OR_RETURN(delta_poc_s1_minus1, 0, 0x7FFF); + if (i == 0) { + // (7-68) + curStRefPicSet.deltaPocS1[i] = delta_poc_s1_minus1 + 1; + } else { + // (7-70) + curStRefPicSet.deltaPocS1[i] = + curStRefPicSet.deltaPocS1[i - 1] + delta_poc_s1_minus1 + 1; + } + curStRefPicSet.usedByCurrPicS1[i] = aReader.ReadBit(); + } + } + // (7-71) + curStRefPicSet.numDeltaPocs = + curStRefPicSet.num_negative_pics + curStRefPicSet.num_positive_pics; + return Ok(); +} + +/* static */ +Result<Ok, nsresult> H265::ParseVuiParameters(BitReader& aReader, + H265SPS& aSPS) { + // VUI parameters: Table E.1 "Interpretation of sample aspect ratio indicator" + static constexpr int kTableSarWidth[] = {0, 1, 12, 10, 16, 40, 24, 20, 32, + 80, 18, 15, 64, 160, 4, 3, 2}; + static constexpr int kTableSarHeight[] = {0, 1, 11, 11, 11, 33, 11, 11, 11, + 33, 11, 11, 33, 99, 3, 2, 1}; + static_assert(std::size(kTableSarWidth) == std::size(kTableSarHeight), + "sar tables must have the same size"); + aSPS.vui_parameters = Some(H265VUIParameters()); + H265VUIParameters* vui = aSPS.vui_parameters.ptr(); + + const auto aspect_ratio_info_present_flag = aReader.ReadBit(); + if (aspect_ratio_info_present_flag) { + const auto aspect_ratio_idc = aReader.ReadBits(8); + constexpr int kExtendedSar = 255; + if (aspect_ratio_idc == kExtendedSar) { + vui->sar_width = aReader.ReadBits(16); + vui->sar_height = aReader.ReadBits(16); + } else { + const auto max_aspect_ratio_idc = std::size(kTableSarWidth) - 1; + IN_RANGE_OR_RETURN(aspect_ratio_idc, 0, max_aspect_ratio_idc); + vui->sar_width = kTableSarWidth[aspect_ratio_idc]; + vui->sar_height = kTableSarHeight[aspect_ratio_idc]; + } + } + + const auto overscan_info_present_flag = aReader.ReadBit(); + if (overscan_info_present_flag) { + Unused << aReader.ReadBit(); // overscan_appropriate_flag + } + + const auto video_signal_type_present_flag = aReader.ReadBit(); + if (video_signal_type_present_flag) { + Unused << aReader.ReadBits(3); // video_format + vui->video_full_range_flag = aReader.ReadBit(); + const auto colour_description_present_flag = aReader.ReadBit(); + if (colour_description_present_flag) { + vui->colour_primaries.emplace(aReader.ReadBits(8)); + vui->transfer_characteristics.emplace(aReader.ReadBits(8)); + vui->matrix_coeffs.emplace(aReader.ReadBits(8)); + } + } + + const auto chroma_loc_info_present_flag = aReader.ReadBit(); + if (chroma_loc_info_present_flag) { + Unused << aReader.ReadUE(); // chroma_sample_loc_type_top_field + Unused << aReader.ReadUE(); // chroma_sample_loc_type_bottom_field + } + + // Ignore neutral_chroma_indication_flag, field_seq_flag and + // frame_field_info_present_flag. + Unused << aReader.ReadBits(3); + + const auto default_display_window_flag = aReader.ReadBit(); + if (default_display_window_flag) { + uint32_t def_disp_win_left_offset = aReader.ReadUE(); + uint32_t def_disp_win_right_offset = aReader.ReadUE(); + uint32_t def_disp_win_top_offset = aReader.ReadUE(); + uint32_t def_disp_win_bottom_offset = aReader.ReadUE(); + // (E-68) + (E-69) + aSPS.mDisplayWidth = aSPS.subWidthC; + aSPS.mDisplayWidth *= + (aSPS.conf_win_left_offset + def_disp_win_left_offset); + aSPS.mDisplayWidth *= + (aSPS.conf_win_right_offset + def_disp_win_right_offset); + if (!aSPS.mDisplayWidth.isValid()) { + LOG("mDisplayWidth overflow!"); + return Err(NS_ERROR_FAILURE); + } + IN_RANGE_OR_RETURN(aSPS.mDisplayWidth.value(), 0, + aSPS.pic_width_in_luma_samples); + + // (E-70) + (E-71) + aSPS.mDisplayHeight = aSPS.subHeightC; + aSPS.mDisplayHeight *= (aSPS.conf_win_top_offset + def_disp_win_top_offset); + aSPS.mDisplayHeight *= + (aSPS.conf_win_bottom_offset + def_disp_win_bottom_offset); + if (!aSPS.mDisplayHeight.isValid()) { + LOG("mDisplayHeight overflow!"); + return Err(NS_ERROR_FAILURE); + } + IN_RANGE_OR_RETURN(aSPS.mDisplayHeight.value(), 0, + aSPS.pic_height_in_luma_samples); + } + + const auto vui_timing_info_present_flag = aReader.ReadBit(); + if (vui_timing_info_present_flag) { + Unused << aReader.ReadU32(); // vui_num_units_in_tick + Unused << aReader.ReadU32(); // vui_time_scale + const auto vui_poc_proportional_to_timing_flag = aReader.ReadBit(); + if (vui_poc_proportional_to_timing_flag) { + Unused << aReader.ReadUE(); // vui_num_ticks_poc_diff_one_minus1 + } + const auto vui_hrd_parameters_present_flag = aReader.ReadBit(); + if (vui_hrd_parameters_present_flag) { + if (auto rv = ParseAndIgnoreHrdParameters(aReader, true, + aSPS.sps_max_sub_layers_minus1); + rv.isErr()) { + LOG("Failed to parse Hrd parameters"); + return rv; + } + } + } + + const auto bitstream_restriction_flag = aReader.ReadBit(); + if (bitstream_restriction_flag) { + // Skip tiles_fixed_structure_flag, motion_vectors_over_pic_boundaries_flag + // and restricted_ref_pic_lists_flag. + Unused << aReader.ReadBits(3); + Unused << aReader.ReadUE(); // min_spatial_segmentation_idc + Unused << aReader.ReadUE(); // max_bytes_per_pic_denom + Unused << aReader.ReadUE(); // max_bits_per_min_cu_denom + Unused << aReader.ReadUE(); // log2_max_mv_length_horizontal + Unused << aReader.ReadUE(); // log2_max_mv_length_vertical + } + return Ok(); +} + +/* static */ +Result<Ok, nsresult> H265::ParseAndIgnoreHrdParameters( + BitReader& aReader, bool aCommonInfPresentFlag, + int aMaxNumSubLayersMinus1) { + // H265 Spec, E.2.2 HRD parameters syntax + bool nal_hrd_parameters_present_flag = false; + bool vcl_hrd_parameters_present_flag = false; + bool sub_pic_hrd_params_present_flag = false; + if (aCommonInfPresentFlag) { + nal_hrd_parameters_present_flag = aReader.ReadBit(); + vcl_hrd_parameters_present_flag = aReader.ReadBit(); + if (nal_hrd_parameters_present_flag || vcl_hrd_parameters_present_flag) { + sub_pic_hrd_params_present_flag = aReader.ReadBit(); + if (sub_pic_hrd_params_present_flag) { + Unused << aReader.ReadBits(8); // tick_divisor_minus2 + // du_cpb_removal_delay_increment_length_minus1 + Unused << aReader.ReadBits(5); + // sub_pic_cpb_params_in_pic_timing_sei_flag + Unused << aReader.ReadBits(1); + Unused << aReader.ReadBits(5); // dpb_output_delay_du_length_minus1 + } + + Unused << aReader.ReadBits(4); // bit_rate_scale + Unused << aReader.ReadBits(4); // cpb_size_scale + if (sub_pic_hrd_params_present_flag) { + Unused << aReader.ReadBits(4); // cpb_size_du_scale + } + Unused << aReader.ReadBits(5); // initial_cpb_removal_delay_length_minus1 + Unused << aReader.ReadBits(5); // au_cpb_removal_delay_length_minus1 + Unused << aReader.ReadBits(5); // dpb_output_delay_length_minus1 + } + } + for (int i = 0; i <= aMaxNumSubLayersMinus1; i++) { + bool fixed_pic_rate_within_cvs_flag = false; + if (auto fixed_pic_rate_general_flag = aReader.ReadBit(); + !fixed_pic_rate_general_flag) { + fixed_pic_rate_within_cvs_flag = aReader.ReadBit(); + } + bool low_delay_hrd_flag = false; + if (fixed_pic_rate_within_cvs_flag) { + Unused << aReader.ReadUE(); // elemental_duration_in_tc_minus1 + } else { + low_delay_hrd_flag = aReader.ReadBit(); + } + int cpb_cnt_minus1 = 0; + if (!low_delay_hrd_flag) { + cpb_cnt_minus1 = aReader.ReadUE(); + IN_RANGE_OR_RETURN(cpb_cnt_minus1, 0, 31); + } + if (nal_hrd_parameters_present_flag) { + if (auto rv = ParseAndIgnoreSubLayerHrdParameters( + aReader, cpb_cnt_minus1 + 1, sub_pic_hrd_params_present_flag); + rv.isErr()) { + LOG("Failed to parse nal Hrd parameters"); + return rv; + }; + } + if (vcl_hrd_parameters_present_flag) { + if (auto rv = ParseAndIgnoreSubLayerHrdParameters( + aReader, cpb_cnt_minus1 + 1, sub_pic_hrd_params_present_flag); + rv.isErr()) { + LOG("Failed to parse vcl Hrd parameters"); + return rv; + } + } + } + return Ok(); +} + +/* static */ +Result<Ok, nsresult> H265::ParseAndIgnoreSubLayerHrdParameters( + BitReader& aReader, int aCpbCnt, bool aSubPicHrdParamsPresentFlag) { + // H265 Spec, E.2.3 Sub-layer HRD parameters syntax + for (auto i = 0; i < aCpbCnt; i++) { + Unused << aReader.ReadUE(); // bit_rate_value_minus1 + Unused << aReader.ReadUE(); // cpb_size_value_minus1 + if (aSubPicHrdParamsPresentFlag) { + Unused << aReader.ReadUE(); // cpb_size_du_value_minus1 + Unused << aReader.ReadUE(); // bit_rate_du_value_minus1 + } + Unused << aReader.ReadBit(); // cbr_flag + } + return Ok(); +} + +bool H265SPS::operator==(const H265SPS& aOther) const { + return memcmp(this, &aOther, sizeof(H265SPS)) == 0; +} + +bool H265SPS::operator!=(const H265SPS& aOther) const { + return !(operator==(aOther)); +} + +gfx::IntSize H265SPS::GetImageSize() const { + return gfx::IntSize(pic_width_in_luma_samples, pic_height_in_luma_samples); +} + +gfx::IntSize H265SPS::GetDisplaySize() const { + if (mDisplayWidth.value() == 0 || mDisplayHeight.value() == 0) { + return GetImageSize(); + } + return gfx::IntSize(mDisplayWidth.value(), mDisplayHeight.value()); +} + +gfx::ColorDepth H265SPS::ColorDepth() const { + if (bit_depth_luma_minus8 != 0 && bit_depth_luma_minus8 != 2 && + bit_depth_luma_minus8 != 4) { + // We don't know what that is, just assume 8 bits to prevent decoding + // regressions if we ever encounter those. + return gfx::ColorDepth::COLOR_8; + } + return gfx::ColorDepthForBitDepth(BitDepthLuma()); +} + +// PrimaryID, TransferID and MatrixID are defined in ByteStreamsUtils.h +static PrimaryID GetPrimaryID(const Maybe<uint8_t>& aPrimary) { + if (!aPrimary || *aPrimary < 1 || *aPrimary > 22 || *aPrimary == 3) { + return PrimaryID::INVALID; + } + if (*aPrimary > 12 && *aPrimary < 22) { + return PrimaryID::INVALID; + } + return static_cast<PrimaryID>(*aPrimary); +} + +static TransferID GetTransferID(const Maybe<uint8_t>& aTransfer) { + if (!aTransfer || *aTransfer < 1 || *aTransfer > 18 || *aTransfer == 3) { + return TransferID::INVALID; + } + return static_cast<TransferID>(*aTransfer); +} + +static MatrixID GetMatrixID(const Maybe<uint8_t>& aMatrix) { + if (!aMatrix || *aMatrix > 11 || *aMatrix == 3) { + return MatrixID::INVALID; + } + return static_cast<MatrixID>(*aMatrix); +} + +gfx::YUVColorSpace H265SPS::ColorSpace() const { + // Bitfield, note that guesses with higher values take precedence over + // guesses with lower values. + enum Guess { + GUESS_BT601 = 1 << 0, + GUESS_BT709 = 1 << 1, + GUESS_BT2020 = 1 << 2, + }; + + uint32_t guess = 0; + if (vui_parameters) { + switch (GetPrimaryID(vui_parameters->colour_primaries)) { + case PrimaryID::BT709: + guess |= GUESS_BT709; + break; + case PrimaryID::BT470M: + case PrimaryID::BT470BG: + case PrimaryID::SMPTE170M: + case PrimaryID::SMPTE240M: + guess |= GUESS_BT601; + break; + case PrimaryID::BT2020: + guess |= GUESS_BT2020; + break; + case PrimaryID::FILM: + case PrimaryID::SMPTEST428_1: + case PrimaryID::SMPTEST431_2: + case PrimaryID::SMPTEST432_1: + case PrimaryID::EBU_3213_E: + case PrimaryID::INVALID: + case PrimaryID::UNSPECIFIED: + break; + } + + switch (GetTransferID(vui_parameters->transfer_characteristics)) { + case TransferID::BT709: + guess |= GUESS_BT709; + break; + case TransferID::GAMMA22: + case TransferID::GAMMA28: + case TransferID::SMPTE170M: + case TransferID::SMPTE240M: + guess |= GUESS_BT601; + break; + case TransferID::BT2020_10: + case TransferID::BT2020_12: + guess |= GUESS_BT2020; + break; + case TransferID::LINEAR: + case TransferID::LOG: + case TransferID::LOG_SQRT: + case TransferID::IEC61966_2_4: + case TransferID::BT1361_ECG: + case TransferID::IEC61966_2_1: + case TransferID::SMPTEST2084: + case TransferID::SMPTEST428_1: + case TransferID::ARIB_STD_B67: + case TransferID::INVALID: + case TransferID::UNSPECIFIED: + break; + } + + switch (GetMatrixID(vui_parameters->matrix_coeffs)) { + case MatrixID::BT709: + guess |= GUESS_BT709; + break; + case MatrixID::BT470BG: + case MatrixID::SMPTE170M: + case MatrixID::SMPTE240M: + guess |= GUESS_BT601; + break; + case MatrixID::BT2020_NCL: + case MatrixID::BT2020_CL: + guess |= GUESS_BT2020; + break; + case MatrixID::RGB: + case MatrixID::FCC: + case MatrixID::YCOCG: + case MatrixID::YDZDX: + case MatrixID::INVALID: + case MatrixID::UNSPECIFIED: + break; + } + } + + // Removes lowest bit until only a single bit remains. + while (guess & (guess - 1)) { + guess &= guess - 1; + } + if (!guess) { + // A better default to BT601 which should die a slow death. + guess = GUESS_BT709; + } + + switch (guess) { + case GUESS_BT601: + return gfx::YUVColorSpace::BT601; + case GUESS_BT709: + return gfx::YUVColorSpace::BT709; + default: + MOZ_DIAGNOSTIC_ASSERT(guess == GUESS_BT2020); + return gfx::YUVColorSpace::BT2020; + } +} + +bool H265SPS::IsFullColorRange() const { + return vui_parameters ? vui_parameters->video_full_range_flag : false; +} + +uint8_t H265SPS::ColorPrimaries() const { + // Per H265 spec E.3.1, "When the colour_primaries syntax element is not + // present, the value of colour_primaries is inferred to be equal to 2 (the + // chromaticity is unspecified or is determined by the application).". + if (!vui_parameters || !vui_parameters->colour_primaries) { + return 2; + } + return vui_parameters->colour_primaries.value(); +} + +uint8_t H265SPS::TransferFunction() const { + // Per H265 spec E.3.1, "When the transfer_characteristics syntax element is + // not present, the value of transfer_characteristics is inferred to be equal + // to 2 (the transfer characteristics are unspecified or are determined by the + // application)." + if (!vui_parameters || !vui_parameters->transfer_characteristics) { + return 2; + } + return vui_parameters->transfer_characteristics.value(); +} + +/* static */ +already_AddRefed<mozilla::MediaByteBuffer> H265::DecodeNALUnit( + const Span<const uint8_t>& aNALU) { + RefPtr<mozilla::MediaByteBuffer> rbsp = new mozilla::MediaByteBuffer; + BufferReader reader(aNALU.Elements(), aNALU.Length()); + auto header = reader.ReadU16(); + if (header.isErr()) { + return nullptr; + } + uint32_t lastbytes = 0xffff; + while (reader.Remaining()) { + auto res = reader.ReadU8(); + if (res.isErr()) { + return nullptr; + } + uint8_t byte = res.unwrap(); + if ((lastbytes & 0xffff) == 0 && byte == 0x03) { + // reset last two bytes, to detect the 0x000003 sequence again. + lastbytes = 0xffff; + } else { + rbsp->AppendElement(byte); + } + lastbytes = (lastbytes << 8) | byte; + } + return rbsp.forget(); +} + +/* static */ +already_AddRefed<mozilla::MediaByteBuffer> H265::ExtractHVCCExtraData( + const mozilla::MediaRawData* aSample) { + size_t sampleSize = aSample->Size(); + if (aSample->mCrypto.IsEncrypted()) { + // The content is encrypted, we can only parse the non-encrypted data. + MOZ_ASSERT(aSample->mCrypto.mPlainSizes.Length() > 0); + if (aSample->mCrypto.mPlainSizes.Length() == 0 || + aSample->mCrypto.mPlainSizes[0] > sampleSize) { + LOG("Invalid crypto content"); + return nullptr; + } + sampleSize = aSample->mCrypto.mPlainSizes[0]; + } + + auto hvcc = HVCCConfig::Parse(aSample); + if (hvcc.isErr()) { + LOG("Only support extracting extradata from HVCC"); + return nullptr; + } + const auto nalLenSize = hvcc.unwrap().NALUSize(); + BufferReader reader(aSample->Data(), sampleSize); + + nsTArray<Maybe<H265SPS>> spsRefTable; + nsTArray<H265NALU> spsNALUs; + // If we encounter SPS with the same id but different content, we will stop + // attempting to detect duplicates. + bool checkDuplicate = true; + const H265SPS* firstSPS = nullptr; + + RefPtr<mozilla::MediaByteBuffer> extradata = new mozilla::MediaByteBuffer; + while (reader.Remaining() > nalLenSize) { + // ISO/IEC 14496-15, 4.2.3.2 Syntax. (NALUSample) Reading the size of NALU. + uint32_t nalLen = 0; + switch (nalLenSize) { + case 1: + Unused << reader.ReadU8().map( + [&](uint8_t x) mutable { return nalLen = x; }); + break; + case 2: + Unused << reader.ReadU16().map( + [&](uint16_t x) mutable { return nalLen = x; }); + break; + case 3: + Unused << reader.ReadU24().map( + [&](uint32_t x) mutable { return nalLen = x; }); + break; + default: + MOZ_DIAGNOSTIC_ASSERT(nalLenSize == 4); + Unused << reader.ReadU32().map( + [&](uint32_t x) mutable { return nalLen = x; }); + break; + } + const uint8_t* p = reader.Read(nalLen); + if (!p) { + // The read failed, but we may already have some SPS data so break out of + // reading and process what we have, if any. + break; + } + const H265NALU nalu(p, nalLen); + LOGV("Found NALU, type=%u", nalu.mNalUnitType); + if (nalu.IsSPS()) { + auto rv = H265::DecodeSPSFromSPSNALU(nalu); + if (rv.isErr()) { + // Invalid SPS, ignore. + LOG("Ignore invalid SPS"); + continue; + } + const H265SPS sps = rv.unwrap(); + const uint8_t spsId = sps.sps_seq_parameter_set_id; // 0~15 + if (spsId >= spsRefTable.Length()) { + if (!spsRefTable.SetLength(spsId + 1, fallible)) { + NS_WARNING("OOM while expanding spsRefTable!"); + return nullptr; + } + } + if (checkDuplicate && spsRefTable[spsId] && + *(spsRefTable[spsId]) == sps) { + // Duplicate ignore. + continue; + } + if (spsRefTable[spsId]) { + // We already have detected a SPS with this Id. Just to be safe we + // disable SPS duplicate detection. + checkDuplicate = false; + } else { + spsRefTable[spsId] = Some(sps); + spsNALUs.AppendElement(nalu); + if (!firstSPS) { + firstSPS = spsRefTable[spsId].ptr(); + } + } + } + } + + LOGV("Found %zu SPS NALU", spsNALUs.Length()); + if (!spsNALUs.IsEmpty()) { + MOZ_ASSERT(firstSPS); + BitWriter writer(extradata); + + // ISO/IEC 14496-15, HEVCDecoderConfigurationRecord. But we only append SPS. + writer.WriteBits(1, 8); // version + const auto& profile = firstSPS->profile_tier_level; + writer.WriteBits(profile.general_profile_space, 2); + writer.WriteBits(profile.general_tier_flag, 1); + writer.WriteBits(profile.general_profile_idc, 5); + writer.WriteU32(profile.general_profile_compatibility_flags); + + // general_constraint_indicator_flags + writer.WriteBit(profile.general_progressive_source_flag); + writer.WriteBit(profile.general_interlaced_source_flag); + writer.WriteBit(profile.general_non_packed_constraint_flag); + writer.WriteBit(profile.general_frame_only_constraint_flag); + writer.WriteBits(0, 44); /* ignored 44 bits */ + + writer.WriteU8(profile.general_level_idc); + writer.WriteBits(0, 4); // reserved + writer.WriteBits(0, 12); // min_spatial_segmentation_idc + writer.WriteBits(0, 6); // reserved + writer.WriteBits(0, 2); // parallelismType + writer.WriteBits(0, 6); // reserved + writer.WriteBits(firstSPS->chroma_format_idc, 2); + writer.WriteBits(0, 5); // reserved + writer.WriteBits(firstSPS->bit_depth_luma_minus8, 3); + writer.WriteBits(0, 5); // reserved + writer.WriteBits(firstSPS->bit_depth_chroma_minus8, 3); + // avgFrameRate + constantFrameRate + numTemporalLayers + temporalIdNested + writer.WriteBits(0, 22); + writer.WriteBits(nalLenSize - 1, 2); // lengthSizeMinusOne + writer.WriteU8(1); // numOfArrays, only SPS + for (auto j = 0; j < 1; j++) { + writer.WriteBits(0, 2); // array_completeness + reserved + writer.WriteBits(H265NALU::SPS_NUT, 6); // NAL_unit_type + writer.WriteBits(spsNALUs.Length(), 16); // numNalus + for (auto i = 0; i < spsNALUs.Length(); i++) { + writer.WriteBits(spsNALUs[i].mNALU.Length(), + 16); // nalUnitLength + MOZ_ASSERT(writer.BitCount() % 8 == 0); + extradata->AppendElements(spsNALUs[i].mNALU.Elements(), + spsNALUs[i].mNALU.Length()); + writer.AdvanceBytes(spsNALUs[i].mNALU.Length()); + } + } + } + + return extradata.forget(); +} + +class SPSIterator final { + public: + explicit SPSIterator(const HVCCConfig& aConfig) : mConfig(aConfig) {} + + SPSIterator& operator++() { + size_t idx = 0; + for (idx = mNextIdx; idx < mConfig.mNALUs.Length(); idx++) { + if (mConfig.mNALUs[idx].IsSPS()) { + mSPS = &mConfig.mNALUs[idx]; + break; + } + } + mNextIdx = idx + 1; + return *this; + } + + explicit operator bool() const { return mNextIdx < mConfig.mNALUs.Length(); } + + const H265NALU* operator*() const { return mSPS ? mSPS : nullptr; } + + private: + size_t mNextIdx = 0; + const H265NALU* mSPS = nullptr; + const HVCCConfig& mConfig; +}; + +/* static */ +bool AreTwoSPSIdentical(const H265NALU& aLhs, const H265NALU& aRhs) { + MOZ_ASSERT(aLhs.IsSPS() && aRhs.IsSPS()); + auto rv1 = H265::DecodeSPSFromSPSNALU(aLhs); + auto rv2 = H265::DecodeSPSFromSPSNALU(aRhs); + if (rv1.isErr() || rv2.isErr()) { + return false; + } + return rv1.unwrap() == rv2.unwrap(); +} + +/* static */ +bool H265::CompareExtraData(const mozilla::MediaByteBuffer* aExtraData1, + const mozilla::MediaByteBuffer* aExtraData2) { + if (aExtraData1 == aExtraData2) { + return true; + } + + auto config1 = HVCCConfig::Parse(aExtraData1); + auto config2 = HVCCConfig::Parse(aExtraData2); + if (config1.isErr() || config2.isErr()) { + return false; + } + + uint8_t numSPS = config1.unwrap().NumSPS(); + if (numSPS == 0 || numSPS != config2.unwrap().NumSPS()) { + return false; + } + + // We only compare if the SPS are the same as the various HEVC decoders can + // deal with in-band change of PPS. + SPSIterator it1(config1.unwrap()); + SPSIterator it2(config2.unwrap()); + while (it1 && it2) { + const H265NALU* nalu1 = *it1; + const H265NALU* nalu2 = *it2; + if (!nalu1 || !nalu2) { + return false; + } + if (!AreTwoSPSIdentical(*nalu1, *nalu2)) { + return false; + } + ++it1; + ++it2; + } + return true; +} + +#undef LOG +#undef LOGV + +} // namespace mozilla diff --git a/dom/media/platforms/agnostic/bytestreams/H265.h b/dom/media/platforms/agnostic/bytestreams/H265.h new file mode 100644 index 0000000000..5aca28e5cb --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/H265.h @@ -0,0 +1,356 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_H265_H_ +#define DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_H265_H_ + +#include <stdint.h> + +#include "mozilla/CheckedInt.h" +#include "mozilla/gfx/Point.h" +#include "mozilla/Maybe.h" +#include "mozilla/Result.h" +#include "mozilla/Span.h" +#include "nsTArray.h" + +namespace mozilla { + +class BitReader; +class MediaByteBuffer; +class MediaRawData; + +// Most classes in this file are implemented according to the H265 spec +// (https://www.itu.int/rec/T-REC-H.265-202108-I/en), except the HVCCConfig, +// which is in the ISO/IEC 14496-15. To make it easier to read the +// implementation with the spec, the naming style in this file follows the spec +// instead of our usual style. + +enum { + kMaxLongTermRefPicSets = 32, // See num_long_term_ref_pics_sps + kMaxShortTermRefPicSets = 64, // See num_short_term_ref_pic_sets + kMaxSubLayers = 7, // See [v/s]ps_max_sub_layers_minus1 +}; + +// Spec 7.3.1 NAL unit syntax +class H265NALU final { + public: + H265NALU(const uint8_t* aData, uint32_t aByteSize); + H265NALU() = default; + + // Table 7-1 + enum NAL_TYPES { + TRAIL_N = 0, + TRAIL_R = 1, + TSA_N = 2, + TSA_R = 3, + STSA_N = 4, + STSA_R = 5, + RADL_N = 6, + RADL_R = 7, + RASL_N = 8, + RASL_R = 9, + RSV_VCL_N10 = 10, + RSV_VCL_R11 = 11, + RSV_VCL_N12 = 12, + RSV_VCL_R13 = 13, + RSV_VCL_N14 = 14, + RSV_VCL_R15 = 15, + BLA_W_LP = 16, + BLA_W_RADL = 17, + BLA_N_LP = 18, + IDR_W_RADL = 19, + IDR_N_LP = 20, + CRA_NUT = 21, + RSV_IRAP_VCL22 = 22, + RSV_IRAP_VCL23 = 23, + RSV_VCL24 = 24, + RSV_VCL25 = 25, + RSV_VCL26 = 26, + RSV_VCL27 = 27, + RSV_VCL28 = 28, + RSV_VCL29 = 29, + RSV_VCL30 = 30, + RSV_VCL31 = 31, + VPS_NUT = 32, + SPS_NUT = 33, + PPS_NUT = 34, + AUD_NUT = 35, + EOS_NUT = 36, + EOB_NUT = 37, + FD_NUT = 38, + PREFIX_SEI_NUT = 39, + SUFFIX_SEI_NUT = 40, + RSV_NVCL41 = 41, + RSV_NVCL42 = 42, + RSV_NVCL43 = 43, + RSV_NVCL44 = 44, + RSV_NVCL45 = 45, + RSV_NVCL46 = 46, + RSV_NVCL47 = 47, + UNSPEC48 = 48, + UNSPEC49 = 49, + UNSPEC50 = 50, + UNSPEC51 = 51, + UNSPEC52 = 52, + UNSPEC53 = 53, + UNSPEC54 = 54, + UNSPEC55 = 55, + UNSPEC56 = 56, + UNSPEC57 = 57, + UNSPEC58 = 58, + UNSPEC59 = 59, + UNSPEC60 = 60, + UNSPEC61 = 61, + UNSPEC62 = 62, + UNSPEC63 = 63, + }; + + bool IsIframe() const { + return mNalUnitType == NAL_TYPES::IDR_W_RADL || + mNalUnitType == NAL_TYPES::IDR_N_LP; + } + + bool IsSPS() const { return mNalUnitType == NAL_TYPES::SPS_NUT; } + bool IsVPS() const { return mNalUnitType == NAL_TYPES::VPS_NUT; } + bool IsPPS() const { return mNalUnitType == NAL_TYPES::PPS_NUT; } + bool IsSEI() const { + return mNalUnitType == NAL_TYPES::PREFIX_SEI_NUT || + mNalUnitType == NAL_TYPES::SUFFIX_SEI_NUT; + } + + uint8_t mNalUnitType; + uint8_t mNuhLayerId; + uint8_t mNuhTemporalIdPlus1; + // This contain the full content of NALU, which can be used to decode rbsp. + const Span<const uint8_t> mNALU; +}; + +// H265 spec, 7.3.3 Profile, tier and level syntax +struct H265ProfileTierLevel final { + H265ProfileTierLevel() = default; + + enum H265ProfileIdc { + kProfileIdcMain = 1, + kProfileIdcMain10 = 2, + kProfileIdcMainStill = 3, + kProfileIdcRangeExtensions = 4, + kProfileIdcHighThroughput = 5, + kProfileIdcMultiviewMain = 6, + kProfileIdcScalableMain = 7, + kProfileIdc3dMain = 8, + kProfileIdcScreenContentCoding = 9, + kProfileIdcScalableRangeExtensions = 10, + kProfileIdcHighThroughputScreenContentCoding = 11, + }; + + // From Table A.8 - General tier and level limits. + uint32_t GetMaxLumaPs() const; + + // From A.4.2 - Profile-specific level limits for the video profiles. + uint32_t GetDpbMaxPicBuf() const; + + // Syntax elements. + uint8_t general_profile_space = {}; + bool general_tier_flag = {}; + uint8_t general_profile_idc = {}; + uint32_t general_profile_compatibility_flags = {}; + bool general_progressive_source_flag = {}; + bool general_interlaced_source_flag = {}; + bool general_non_packed_constraint_flag = {}; + bool general_frame_only_constraint_flag = {}; + uint8_t general_level_idc = {}; +}; + +// H265 spec, 7.3.7 Short-term reference picture set syntax +struct H265StRefPicSet final { + H265StRefPicSet() = default; + + // Syntax elements. + uint32_t num_negative_pics = {}; + uint32_t num_positive_pics = {}; + + // Calculated fields + // From the H265 spec 7.4.8 + bool usedByCurrPicS0[kMaxShortTermRefPicSets] = {}; // (7-65) + bool usedByCurrPicS1[kMaxShortTermRefPicSets] = {}; // (7-66) + uint32_t deltaPocS0[kMaxShortTermRefPicSets] = {}; // (7-67) + (7-69) + uint32_t deltaPocS1[kMaxShortTermRefPicSets] = {}; // (7-68) + (7-70) + uint32_t numDeltaPocs = {}; // (7-72) +}; + +// H265 spec, E.2.1 VUI parameters syntax +struct H265VUIParameters { + H265VUIParameters() = default; + + // Syntax elements. + uint32_t sar_width = {}; + uint32_t sar_height = {}; + bool video_full_range_flag = {}; + Maybe<uint8_t> colour_primaries; + Maybe<uint8_t> transfer_characteristics; + Maybe<uint8_t> matrix_coeffs; +}; + +// H265 spec, 7.3.2.2 Sequence parameter set RBSP syntax +struct H265SPS final { + H265SPS() = default; + + bool operator==(const H265SPS& aOther) const; + bool operator!=(const H265SPS& aOther) const; + + // Syntax elements. + uint8_t sps_video_parameter_set_id = {}; + uint8_t sps_max_sub_layers_minus1 = {}; + bool sps_temporal_id_nesting_flag = {}; + H265ProfileTierLevel profile_tier_level = {}; + uint32_t sps_seq_parameter_set_id = {}; + uint32_t chroma_format_idc = {}; + bool separate_colour_plane_flag = {}; + uint32_t pic_width_in_luma_samples = {}; + uint32_t pic_height_in_luma_samples = {}; + + bool conformance_window_flag = {}; + uint32_t conf_win_left_offset = {}; + uint32_t conf_win_right_offset = {}; + uint32_t conf_win_top_offset = {}; + uint32_t conf_win_bottom_offset = {}; + + uint32_t bit_depth_luma_minus8 = {}; + uint32_t bit_depth_chroma_minus8 = {}; + uint32_t log2_max_pic_order_cnt_lsb_minus4 = {}; + bool sps_sub_layer_ordering_info_present_flag = {}; + uint32_t sps_max_dec_pic_buffering_minus1[kMaxSubLayers] = {}; + uint32_t sps_max_num_reorder_pics[kMaxSubLayers] = {}; + uint32_t sps_max_latency_increase_plus1[kMaxSubLayers] = {}; + uint32_t log2_min_luma_coding_block_size_minus3 = {}; + uint32_t log2_diff_max_min_luma_coding_block_size = {}; + uint32_t log2_min_luma_transform_block_size_minus2 = {}; + uint32_t log2_diff_max_min_luma_transform_block_size = {}; + uint32_t max_transform_hierarchy_depth_inter = {}; + uint32_t max_transform_hierarchy_depth_intra = {}; + + bool pcm_enabled_flag = {}; + uint8_t pcm_sample_bit_depth_luma_minus1 = {}; + uint8_t pcm_sample_bit_depth_chroma_minus1 = {}; + uint32_t log2_min_pcm_luma_coding_block_size_minus3 = {}; + uint32_t log2_diff_max_min_pcm_luma_coding_block_size = {}; + bool pcm_loop_filter_disabled_flag = {}; + + uint32_t num_short_term_ref_pic_sets = {}; + H265StRefPicSet st_ref_pic_set[kMaxShortTermRefPicSets] = {}; + + bool sps_temporal_mvp_enabled_flag = {}; + bool strong_intra_smoothing_enabled_flag = {}; + Maybe<H265VUIParameters> vui_parameters; + + // Calculated fields + uint32_t subWidthC = {}; // From Table 6-1. + uint32_t subHeightC = {}; // From Table 6-1. + CheckedUint32 mDisplayWidth; // Per (E-68) + (E-69) + CheckedUint32 mDisplayHeight; // Per (E-70) + (E-71) + uint32_t maxDpbSize = {}; + + // Often used information + uint32_t BitDepthLuma() const { return bit_depth_luma_minus8 + 8; } + uint32_t BitDepthChroma() const { return bit_depth_chroma_minus8 + 8; } + gfx::IntSize GetImageSize() const; + gfx::IntSize GetDisplaySize() const; + gfx::ColorDepth ColorDepth() const; + gfx::YUVColorSpace ColorSpace() const; + bool IsFullColorRange() const; + uint8_t ColorPrimaries() const; + uint8_t TransferFunction() const; +}; + +// ISO/IEC 14496-15 : hvcC. +struct HVCCConfig final { + public: + static Result<HVCCConfig, nsresult> Parse( + const mozilla::MediaRawData* aSample); + static Result<HVCCConfig, nsresult> Parse( + const mozilla::MediaByteBuffer* aExtraData); + + uint8_t NALUSize() const { return lengthSizeMinusOne + 1; } + uint32_t NumSPS() const; + bool HasSPS() const; + + uint8_t configurationVersion; + uint8_t general_profile_space; + bool general_tier_flag; + uint8_t general_profile_idc; + uint32_t general_profile_compatibility_flags; + uint64_t general_constraint_indicator_flags; + uint8_t general_level_idc; + uint16_t min_spatial_segmentation_idc; + uint8_t parallelismType; + uint8_t chroma_format_idc; + uint8_t bit_depth_luma_minus8; + uint8_t bit_depth_chroma_minus8; + uint16_t avgFrameRate; + uint8_t constantFrameRate; + uint8_t numTemporalLayers; + bool temporalIdNested; + uint8_t lengthSizeMinusOne; + + nsTArray<H265NALU> mNALUs; + + // Keep the orginal buffer alive in order to let H265NALU always access to + // valid data if there is any NALU. + RefPtr<const MediaByteBuffer> mByteBuffer; + + private: + HVCCConfig() = default; +}; + +class H265 final { + public: + static Result<H265SPS, nsresult> DecodeSPSFromHVCCExtraData( + const mozilla::MediaByteBuffer* aExtraData); + static Result<H265SPS, nsresult> DecodeSPSFromSPSNALU( + const H265NALU& aSPSNALU); + + // Extract SPS and PPS NALs from aSample by looking into each NALs. + static already_AddRefed<mozilla::MediaByteBuffer> ExtractHVCCExtraData( + const mozilla::MediaRawData* aSample); + + // Return true if both extradata are equal. + static bool CompareExtraData(const mozilla::MediaByteBuffer* aExtraData1, + const mozilla::MediaByteBuffer* aExtraData2); + + private: + // Return RAW BYTE SEQUENCE PAYLOAD (rbsp) from NAL content. + static already_AddRefed<mozilla::MediaByteBuffer> DecodeNALUnit( + const Span<const uint8_t>& aNALU); + + // Parse the profile level based on the H265 spec, 7.3.3. MUST use a bit + // reader which starts from the position of the first bit of the data. + static Result<Ok, nsresult> ParseProfileTierLevel( + BitReader& aReader, bool aProfilePresentFlag, + uint8_t aMaxNumSubLayersMinus1, H265ProfileTierLevel& aProfile); + + // Parse the short-term reference picture set based on the H265 spec, 7.3.7. + // MUST use a bit reader which starts from the position of the first bit of + // the data. + static Result<Ok, nsresult> ParseStRefPicSet(BitReader& aReader, + uint32_t aStRpsIdx, + H265SPS& aSPS); + + // Parse the VUI parameters based on the H265 spec, E.2.1. MUST use a bit + // reader which starts from the position of the first bit of the data. + static Result<Ok, nsresult> ParseVuiParameters(BitReader& aReader, + H265SPS& aSPS); + + // Parse and ignore the structure. MUST use a bitreader which starts from the + // position of the first bit of the data. + static Result<Ok, nsresult> ParseAndIgnoreScalingListData(BitReader& aReader); + static Result<Ok, nsresult> ParseAndIgnoreHrdParameters( + BitReader& aReader, bool aCommonInfPresentFlag, + int aMaxNumSubLayersMinus1); + static Result<Ok, nsresult> ParseAndIgnoreSubLayerHrdParameters( + BitReader& aReader, int aCpbCnt, bool aSubPicHrdParamsPresentFlag); +}; + +} // namespace mozilla + +#endif // DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_H265_H_ diff --git a/dom/media/platforms/agnostic/bytestreams/gtest/TestByteStreams.cpp b/dom/media/platforms/agnostic/bytestreams/gtest/TestByteStreams.cpp new file mode 100644 index 0000000000..911f10f193 --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/gtest/TestByteStreams.cpp @@ -0,0 +1,787 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "gtest/gtest.h" + +#include "AnnexB.h" +#include "BufferReader.h" +#include "ByteWriter.h" +#include "H264.h" +#include "H265.h" +#include "mozilla/Types.h" + +namespace mozilla { + +// Create AVCC style extra data (the contents on an AVCC box). Note +// NALLengthSize will be 4 so AVCC samples need to set their data up +// accordingly. +static already_AddRefed<MediaByteBuffer> GetExtraData() { + // Extra data with + // - baseline profile(0x42 == 66). + // - constraint flags 0 and 1 set(0xc0) -- normal for baseline profile. + // - level 4.0 (0x28 == 40). + // - 1280 * 720 resolution. + return H264::CreateExtraData(0x42, 0xc0, 0x28, {1280, 720}); +} + +// Create an AVCC style sample with requested size in bytes. This sample is +// setup to contain a single NAL (in practice samples can contain many). The +// sample sets its NAL size to aSampleSize - 4 and stores that size in the first +// 4 bytes. Aside from the NAL size at the start, the data is uninitialized +// (beware)! aSampleSize is a uint32_t as samples larger than can be expressed +// by a uint32_t are not to spec. +static already_AddRefed<MediaRawData> GetAvccSample(uint32_t aSampleSize) { + if (aSampleSize < 4) { + // Stop tests asking for insane samples. + EXPECT_FALSE(true) << "Samples should be requested with sane sizes"; + } + nsTArray<uint8_t> sampleData; + + // Write the NAL size. + ByteWriter<BigEndian> writer(sampleData); + EXPECT_TRUE(writer.WriteU32(aSampleSize - 4)); + + // Write the 'NAL'. Beware, this data is uninitialized. + sampleData.AppendElements(static_cast<size_t>(aSampleSize) - 4); + RefPtr<MediaRawData> rawData = + new MediaRawData{sampleData.Elements(), sampleData.Length()}; + EXPECT_NE(rawData->Data(), nullptr); + + // Set extra data. + rawData->mExtraData = GetExtraData(); + return rawData.forget(); +} + +static const uint8_t sHvccBytesBuffer[] = { + 1 /* version */, + 1 /* general_profile_space/general_tier_flag/general_profile_idc */, + 0x60 /* general_profile_compatibility_flags 1/4 */, + 0 /* general_profile_compatibility_flags 2/4 */, + 0 /* general_profile_compatibility_flags 3/4 */, + 0 /* general_profile_compatibility_flags 4/4 */, + 0x90 /* general_constraint_indicator_flags 1/6 */, + 0 /* general_constraint_indicator_flags 2/6 */, + 0 /* general_constraint_indicator_flags 3/6 */, + 0 /* general_constraint_indicator_flags 4/6 */, + 0 /* general_constraint_indicator_flags 5/6 */, + 0 /* general_constraint_indicator_flags 6/6 */, + 0x5A /* general_level_idc */, + 0 /* min_spatial_segmentation_idc 1/2 */, + 0 /* min_spatial_segmentation_idc 2/2 */, + 0 /* parallelismType */, + 1 /* chroma_format_idc */, + 0 /* bit_depth_luma_minus8 */, + 0 /* bit_depth_chroma_minus8 */, + 0 /* avgFrameRate 1/2 */, + 0 /* avgFrameRate 2/2 */, + 0x0F /* constantFrameRate/numTemporalLayers/temporalIdNested/lengthSizeMinusOne + */ + , + 2 /* numOfArrays */, + /* SPS Array */ + 0x21 /* NAL_unit_type (SPS) */, + 0 /* numNalus 1/2 */, + 1 /* numNalus 2/2 */, + + /* SPS */ + 0 /* nalUnitLength 1/2 */, + 8 /* nalUnitLength 2/2 (header + rsbp) */, + 0x42 /* NALU header 1/2 */, + 0 /* NALU header 2/2 */, + 0 /* rbsp 1/6 */, + 0 /* rbsp 2/6 */, + 0 /* rbsp 3/6 */, + 0 /* rbsp 4/6 */, + 0 /* rbsp 5/6 */, + 0 /* rbsp 6/6 */, + + /* PPS Array */ + 0x22 /* NAL_unit_type (PPS) */, + 0 /* numNalus 1/2 */, + 1 /* numNalus 2/2 */, + + /* PPS */ + 0 /* nalUnitLength 1/2 */, + 3 /* nalUnitLength 2/2 (header + rsbp) */, + 0x44 /* NALU header 1/2 */, + 0 /* NALU header 2/2 */, + 0 /* rbsp */, +}; + +// Create a HVCC sample, which contain fake data, in given size. +static already_AddRefed<MediaRawData> GetHVCCSample(uint32_t aSampleSize) { + if (aSampleSize < 4) { + // Stop tests asking for insane samples. + EXPECT_FALSE(true) << "Samples should be requested with sane sizes"; + } + auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>(); + extradata->AppendElements(sHvccBytesBuffer, ArrayLength(sHvccBytesBuffer)); + + // Write the NAL size. + nsTArray<uint8_t> sampleData; + ByteWriter<BigEndian> writer(sampleData); + EXPECT_TRUE(writer.WriteU32(aSampleSize - 4)); // Assume it's a 4 bytes NALU + + // Fill fake empty data + for (uint32_t idx = 0; idx < aSampleSize - 4; idx++) { + sampleData.AppendElement(0); + } + RefPtr<MediaRawData> rawData = + new MediaRawData{sampleData.Elements(), sampleData.Length()}; + EXPECT_NE(rawData->Data(), nullptr); + EXPECT_EQ(rawData->Size(), aSampleSize); + rawData->mExtraData = extradata; + return rawData.forget(); +} + +// Create a HVCC sample by using given data in given size. +static already_AddRefed<MediaRawData> GetHVCCSample( + const uint8_t* aData, const uint32_t aDataLength) { + if (aDataLength < 4) { + // Stop tests asking for insane samples. + EXPECT_FALSE(true) << "Samples should be requested with sane sizes"; + } + auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>(); + extradata->AppendElements(sHvccBytesBuffer, ArrayLength(sHvccBytesBuffer)); + + // Write the NAL size. + nsTArray<uint8_t> sampleData; + ByteWriter<BigEndian> writer(sampleData); + EXPECT_TRUE(writer.WriteU32(aDataLength)); // Assume it's a 4 bytes NALU + sampleData.AppendElements(aData, aDataLength); + + RefPtr<MediaRawData> rawData = + new MediaRawData{sampleData.Elements(), sampleData.Length()}; + EXPECT_NE(rawData->Data(), nullptr); + EXPECT_EQ(rawData->Size(), aDataLength + 4); + rawData->mExtraData = extradata; + return rawData.forget(); +} + +// Test that conversion from AVCC to AnnexB works as expected. +TEST(AnnexB, AVCCToAnnexBConversion) +{ + RefPtr<MediaRawData> rawData{GetAvccSample(128)}; + + { + // Test conversion of data when not adding SPS works as expected. + RefPtr<MediaRawData> rawDataClone = rawData->Clone(); + Result<Ok, nsresult> result = + AnnexB::ConvertAVCCSampleToAnnexB(rawDataClone, /* aAddSps */ false); + EXPECT_TRUE(result.isOk()) << "Conversion should succeed"; + EXPECT_EQ(rawDataClone->Size(), rawData->Size()) + << "AnnexB sample should be the same size as the AVCC sample -- the 4 " + "byte NAL length data (AVCC) is replaced with 4 bytes of NAL " + "separator (AnnexB)"; + EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone)) + << "The sample should be AnnexB following conversion"; + } + + { + // Test that the SPS data is not added if the frame is not a keyframe. + RefPtr<MediaRawData> rawDataClone = rawData->Clone(); + rawDataClone->mKeyframe = + false; // false is the default, but let's be sure. + Result<Ok, nsresult> result = + AnnexB::ConvertAVCCSampleToAnnexB(rawDataClone, /* aAddSps */ true); + EXPECT_TRUE(result.isOk()) << "Conversion should succeed"; + EXPECT_EQ(rawDataClone->Size(), rawData->Size()) + << "AnnexB sample should be the same size as the AVCC sample -- the 4 " + "byte NAL length data (AVCC) is replaced with 4 bytes of NAL " + "separator (AnnexB) and SPS data is not added as the frame is not a " + "keyframe"; + EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone)) + << "The sample should be AnnexB following conversion"; + } + + { + // Test that the SPS data is added to keyframes. + RefPtr<MediaRawData> rawDataClone = rawData->Clone(); + rawDataClone->mKeyframe = true; + Result<Ok, nsresult> result = + AnnexB::ConvertAVCCSampleToAnnexB(rawDataClone, /* aAddSps */ true); + EXPECT_TRUE(result.isOk()) << "Conversion should succeed"; + EXPECT_GT(rawDataClone->Size(), rawData->Size()) + << "AnnexB sample should be larger than the AVCC sample because we've " + "added SPS data"; + EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone)) + << "The sample should be AnnexB following conversion"; + // We could verify the SPS and PPS data we add, but we don't have great + // tooling to do so. Consider doing so in future. + } + + { + // Test conversion involving subsample encryption doesn't overflow vlaues. + const uint32_t sampleSize = UINT16_MAX * 2; + RefPtr<MediaRawData> rawCryptoData{GetAvccSample(sampleSize)}; + // Need to be a keyframe to test prepending SPS + PPS to sample. + rawCryptoData->mKeyframe = true; + UniquePtr<MediaRawDataWriter> rawDataWriter = rawCryptoData->CreateWriter(); + + rawDataWriter->mCrypto.mCryptoScheme = CryptoScheme::Cenc; + + // We want to check that the clear size doesn't overflow during conversion. + // This size originates in a uint16_t, but since it can grow during AnnexB + // we cover it here. + const uint16_t clearSize = UINT16_MAX - 10; + // Set a clear size very close to uint16_t max value. + rawDataWriter->mCrypto.mPlainSizes.AppendElement(clearSize); + rawDataWriter->mCrypto.mEncryptedSizes.AppendElement(sampleSize - + clearSize); + + RefPtr<MediaRawData> rawCryptoDataClone = rawCryptoData->Clone(); + Result<Ok, nsresult> result = AnnexB::ConvertAVCCSampleToAnnexB( + rawCryptoDataClone, /* aAddSps */ true); + EXPECT_TRUE(result.isOk()) << "Conversion should succeed"; + EXPECT_GT(rawCryptoDataClone->Size(), rawCryptoData->Size()) + << "AnnexB sample should be larger than the AVCC sample because we've " + "added SPS data"; + EXPECT_GT(rawCryptoDataClone->mCrypto.mPlainSizes[0], + rawCryptoData->mCrypto.mPlainSizes[0]) + << "Conversion should have increased clear data sizes without overflow"; + EXPECT_EQ(rawCryptoDataClone->mCrypto.mEncryptedSizes[0], + rawCryptoData->mCrypto.mEncryptedSizes[0]) + << "Conversion should not affect encrypted sizes"; + EXPECT_TRUE(AnnexB::IsAnnexB(rawCryptoDataClone)) + << "The sample should be AnnexB following conversion"; + } +} + +TEST(AnnexB, HVCCToAnnexBConversion) +{ + RefPtr<MediaRawData> rawData{GetHVCCSample(128)}; + { + // Test conversion of data when not adding SPS works as expected. + RefPtr<MediaRawData> rawDataClone = rawData->Clone(); + Result<Ok, nsresult> result = + AnnexB::ConvertHVCCSampleToAnnexB(rawDataClone, /* aAddSps */ false); + EXPECT_TRUE(result.isOk()) << "Conversion should succeed"; + EXPECT_EQ(rawDataClone->Size(), rawData->Size()) + << "AnnexB sample should be the same size as the HVCC sample -- the 4 " + "byte NAL length data (HVCC) is replaced with 4 bytes of NAL " + "separator (AnnexB)"; + EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone)) + << "The sample should be AnnexB following conversion"; + } + { + // Test that the SPS data is not added if the frame is not a keyframe. + RefPtr<MediaRawData> rawDataClone = rawData->Clone(); + rawDataClone->mKeyframe = + false; // false is the default, but let's be sure. + Result<Ok, nsresult> result = + AnnexB::ConvertHVCCSampleToAnnexB(rawDataClone, /* aAddSps */ true); + EXPECT_TRUE(result.isOk()) << "Conversion should succeed"; + EXPECT_EQ(rawDataClone->Size(), rawData->Size()) + << "AnnexB sample should be the same size as the HVCC sample -- the 4 " + "byte NAL length data (HVCC) is replaced with 4 bytes of NAL " + "separator (AnnexB) and SPS data is not added as the frame is not a " + "keyframe"; + EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone)) + << "The sample should be AnnexB following conversion"; + } + { + // Test that the SPS data is added to keyframes. + RefPtr<MediaRawData> rawDataClone = rawData->Clone(); + rawDataClone->mKeyframe = true; + Result<Ok, nsresult> result = + AnnexB::ConvertHVCCSampleToAnnexB(rawDataClone, /* aAddSps */ true); + EXPECT_TRUE(result.isOk()) << "Conversion should succeed"; + EXPECT_GT(rawDataClone->Size(), rawData->Size()) + << "AnnexB sample should be larger than the HVCC sample because we've " + "added SPS data"; + EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone)) + << "The sample should be AnnexB following conversion"; + // We could verify the SPS and PPS data we add, but we don't have great + // tooling to do so. Consider doing so in future. + } + { + // Test conversion involving subsample encryption doesn't overflow values. + const uint32_t sampleSize = UINT16_MAX * 2; + RefPtr<MediaRawData> rawCryptoData{GetHVCCSample(sampleSize)}; + // Need to be a keyframe to test prepending SPS + PPS to sample. + rawCryptoData->mKeyframe = true; + UniquePtr<MediaRawDataWriter> rawDataWriter = rawCryptoData->CreateWriter(); + + rawDataWriter->mCrypto.mCryptoScheme = CryptoScheme::Cenc; + + // We want to check that the clear size doesn't overflow during conversion. + // This size originates in a uint16_t, but since it can grow during AnnexB + // we cover it here. + const uint16_t clearSize = UINT16_MAX - 10; + // Set a clear size very close to uint16_t max value. + rawDataWriter->mCrypto.mPlainSizes.AppendElement(clearSize); + rawDataWriter->mCrypto.mEncryptedSizes.AppendElement(sampleSize - + clearSize); + + RefPtr<MediaRawData> rawCryptoDataClone = rawCryptoData->Clone(); + Result<Ok, nsresult> result = AnnexB::ConvertHVCCSampleToAnnexB( + rawCryptoDataClone, /* aAddSps */ true); + EXPECT_TRUE(result.isOk()) << "Conversion should succeed"; + EXPECT_GT(rawCryptoDataClone->Size(), rawCryptoData->Size()) + << "AnnexB sample should be larger than the HVCC sample because we've " + "added SPS data"; + EXPECT_GT(rawCryptoDataClone->mCrypto.mPlainSizes[0], + rawCryptoData->mCrypto.mPlainSizes[0]) + << "Conversion should have increased clear data sizes without overflow"; + EXPECT_EQ(rawCryptoDataClone->mCrypto.mEncryptedSizes[0], + rawCryptoData->mCrypto.mEncryptedSizes[0]) + << "Conversion should not affect encrypted sizes"; + EXPECT_TRUE(AnnexB::IsAnnexB(rawCryptoDataClone)) + << "The sample should be AnnexB following conversion"; + } +} + +TEST(H264, AVCCParsingSuccess) +{ + auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>(); + uint8_t avccBytesBuffer[] = { + 1 /* version */, + 0x64 /* profile (High) */, + 0 /* profile compat (0) */, + 40 /* level (40) */, + 0xfc | 3 /* nal size - 1 */, + 0xe0 /* num SPS (0) */, + 0 /* num PPS (0) */ + }; + extradata->AppendElements(avccBytesBuffer, ArrayLength(avccBytesBuffer)); + auto rv = AVCCConfig::Parse(extradata); + EXPECT_TRUE(rv.isOk()); + const auto avcc = rv.unwrap(); + EXPECT_EQ(avcc.mConfigurationVersion, 1); + EXPECT_EQ(avcc.mAVCProfileIndication, 0x64); + EXPECT_EQ(avcc.mProfileCompatibility, 0); + EXPECT_EQ(avcc.mAVCLevelIndication, 40); + EXPECT_EQ(avcc.NALUSize(), 4); + EXPECT_EQ(avcc.mNumSPS, 0); +} + +TEST(H264, AVCCParsingFailure) +{ + { + // Incorrect version + auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>(); + uint8_t avccBytesBuffer[] = { + 2 /* version */, + 0x64 /* profile (High) */, + 0 /* profile compat (0) */, + 40 /* level (40) */, + 0xfc | 3 /* nal size - 1 */, + 0xe0 /* num SPS (0) */, + 0 /* num PPS (0) */ + }; + extradata->AppendElements(avccBytesBuffer, ArrayLength(avccBytesBuffer)); + auto avcc = AVCCConfig::Parse(extradata); + EXPECT_TRUE(avcc.isErr()); + } + { + // Insuffient data (lacking of PPS) + auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>(); + uint8_t avccBytesBuffer[] = { + 1 /* version */, + 0x64 /* profile (High) */, + 0 /* profile compat (0) */, + 40 /* level (40) */, + 0xfc | 3 /* nal size - 1 */, + 0xe0 /* num SPS (0) */, + }; + extradata->AppendElements(avccBytesBuffer, ArrayLength(avccBytesBuffer)); + auto avcc = AVCCConfig::Parse(extradata); + EXPECT_TRUE(avcc.isErr()); + } +} + +TEST(H265, HVCCParsingSuccess) +{ + { + auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>(); + uint8_t hvccBytesBuffer[] = { + 1 /* version */, + 1 /* general_profile_space/general_tier_flag/general_profile_idc */, + 0x60 /* general_profile_compatibility_flags 1/4 */, + 0 /* general_profile_compatibility_flags 2/4 */, + 0 /* general_profile_compatibility_flags 3/4 */, + 0 /* general_profile_compatibility_flags 4/4 */, + 0x90 /* general_constraint_indicator_flags 1/6 */, + 0 /* general_constraint_indicator_flags 2/6 */, + 0 /* general_constraint_indicator_flags 3/6 */, + 0 /* general_constraint_indicator_flags 4/6 */, + 0 /* general_constraint_indicator_flags 5/6 */, + 0 /* general_constraint_indicator_flags 6/6 */, + 0x5A /* general_level_idc */, + 0 /* min_spatial_segmentation_idc 1/2 */, + 0 /* min_spatial_segmentation_idc 2/2 */, + 0 /* parallelismType */, + 1 /* chroma_format_idc */, + 0 /* bit_depth_luma_minus8 */, + 0 /* bit_depth_chroma_minus8 */, + 0 /* avgFrameRate 1/2 */, + 0 /* avgFrameRate 2/2 */, + 0x0F /* constantFrameRate/numTemporalLayers/temporalIdNested/lengthSizeMinusOne + */ + , + 0 /* numOfArrays */, + }; + extradata->AppendElements(hvccBytesBuffer, ArrayLength(hvccBytesBuffer)); + auto rv = HVCCConfig::Parse(extradata); + EXPECT_TRUE(rv.isOk()); + auto hvcc = rv.unwrap(); + EXPECT_EQ(hvcc.configurationVersion, 1); + EXPECT_EQ(hvcc.general_profile_space, 0); + EXPECT_EQ(hvcc.general_tier_flag, false); + EXPECT_EQ(hvcc.general_profile_idc, 1); + EXPECT_EQ(hvcc.general_profile_compatibility_flags, (uint32_t)0x60000000); + EXPECT_EQ(hvcc.general_constraint_indicator_flags, + (uint64_t)0x900000000000); + EXPECT_EQ(hvcc.general_level_idc, 0x5A); + EXPECT_EQ(hvcc.min_spatial_segmentation_idc, 0); + EXPECT_EQ(hvcc.parallelismType, 0); + EXPECT_EQ(hvcc.chroma_format_idc, 1); + EXPECT_EQ(hvcc.bit_depth_luma_minus8, 0); + EXPECT_EQ(hvcc.bit_depth_chroma_minus8, 0); + EXPECT_EQ(hvcc.avgFrameRate, 0); + EXPECT_EQ(hvcc.constantFrameRate, 0); + EXPECT_EQ(hvcc.numTemporalLayers, 1); + EXPECT_EQ(hvcc.temporalIdNested, true); + EXPECT_EQ(hvcc.NALUSize(), 4); + EXPECT_EQ(hvcc.mNALUs.Length(), uint32_t(0)); + } + { + // Multple NALUs + auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>(); + uint8_t hvccBytesBuffer[] = { + 1 /* version */, + 1 /* general_profile_space/general_tier_flag/general_profile_idc */, + 0x60 /* general_profile_compatibility_flags 1/4 */, + 0 /* general_profile_compatibility_flags 2/4 */, + 0 /* general_profile_compatibility_flags 3/4 */, + 0 /* general_profile_compatibility_flags 4/4 */, + 0x90 /* general_constraint_indicator_flags 1/6 */, + 0 /* general_constraint_indicator_flags 2/6 */, + 0 /* general_constraint_indicator_flags 3/6 */, + 0 /* general_constraint_indicator_flags 4/6 */, + 0 /* general_constraint_indicator_flags 5/6 */, + 0 /* general_constraint_indicator_flags 6/6 */, + 0x5A /* general_level_idc */, + 0 /* min_spatial_segmentation_idc 1/2 */, + 0 /* min_spatial_segmentation_idc 2/2 */, + 0 /* parallelismType */, + 1 /* chroma_format_idc */, + 0 /* bit_depth_luma_minus8 */, + 0 /* bit_depth_chroma_minus8 */, + 0 /* avgFrameRate 1/2 */, + 0 /* avgFrameRate 2/2 */, + 0x0F /* constantFrameRate/numTemporalLayers/temporalIdNested/lengthSizeMinusOne + */ + , + 2 /* numOfArrays */, + /* SPS Array */ + 0x21 /* NAL_unit_type (SPS) */, + 0 /* numNalus 1/2 */, + 1 /* numNalus 2/2 */, + + /* SPS */ + 0 /* nalUnitLength 1/2 */, + 8 /* nalUnitLength 2/2 (header + rsbp) */, + 0x42 /* NALU header 1/2 */, + 0 /* NALU header 2/2 */, + 0 /* rbsp 1/6 */, + 0 /* rbsp 2/6 */, + 0 /* rbsp 3/6 */, + 0 /* rbsp 4/6 */, + 0 /* rbsp 5/6 */, + 0 /* rbsp 6/6 */, + + /* PPS Array */ + 0x22 /* NAL_unit_type (PPS) */, + 0 /* numNalus 1/2 */, + 2 /* numNalus 2/2 */, + + /* PPS 1 */ + 0 /* nalUnitLength 1/2 */, + 3 /* nalUnitLength 2/2 (header + rsbp) */, + 0x44 /* NALU header 1/2 */, + 0 /* NALU header 2/2 */, + 0 /* rbsp */, + + /* PPS 2 */ + 0 /* nalUnitLength 1/2 */, + 3 /* nalUnitLength 2/2 (header + rsbp) */, + 0x44 /* NALU header 1/2 */, + 0 /* NALU header 2/2 */, + 0 /* rbsp */, + }; + extradata->AppendElements(hvccBytesBuffer, ArrayLength(hvccBytesBuffer)); + auto rv = HVCCConfig::Parse(extradata); + EXPECT_TRUE(rv.isOk()); + auto hvcc = rv.unwrap(); + // Check NALU, it should contain 1 SPS and 2 PPS. + EXPECT_EQ(hvcc.mNALUs.Length(), uint32_t(3)); + EXPECT_EQ(hvcc.mNALUs[0].mNalUnitType, H265NALU::NAL_TYPES::SPS_NUT); + EXPECT_EQ(hvcc.mNALUs[0].mNuhLayerId, 0); + EXPECT_EQ(hvcc.mNALUs[0].mNuhTemporalIdPlus1, 0); + EXPECT_EQ(hvcc.mNALUs[0].IsSPS(), true); + EXPECT_EQ(hvcc.mNALUs[0].mNALU.Length(), 8u); + + EXPECT_EQ(hvcc.mNALUs[1].mNalUnitType, H265NALU::NAL_TYPES::PPS_NUT); + EXPECT_EQ(hvcc.mNALUs[1].mNuhLayerId, 0); + EXPECT_EQ(hvcc.mNALUs[1].mNuhTemporalIdPlus1, 0); + EXPECT_EQ(hvcc.mNALUs[1].IsSPS(), false); + EXPECT_EQ(hvcc.mNALUs[1].mNALU.Length(), 3u); + + EXPECT_EQ(hvcc.mNALUs[2].mNalUnitType, H265NALU::NAL_TYPES::PPS_NUT); + EXPECT_EQ(hvcc.mNALUs[2].mNuhLayerId, 0); + EXPECT_EQ(hvcc.mNALUs[2].mNuhTemporalIdPlus1, 0); + EXPECT_EQ(hvcc.mNALUs[2].IsSPS(), false); + EXPECT_EQ(hvcc.mNALUs[2].mNALU.Length(), 3u); + } +} + +TEST(H265, HVCCParsingFailure) +{ + { + // Incorrect version + auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>(); + uint8_t hvccBytesBuffer[] = { + 2 /* version */, + 1 /* general_profile_space/general_tier_flag/general_profile_idc */, + 0x60 /* general_profile_compatibility_flags 1/4 */, + 0 /* general_profile_compatibility_flags 2/4 */, + 0 /* general_profile_compatibility_flags 3/4 */, + 0 /* general_profile_compatibility_flags 4/4 */, + 0x90 /* general_constraint_indicator_flags 1/6 */, + 0 /* general_constraint_indicator_flags 2/6 */, + 0 /* general_constraint_indicator_flags 3/6 */, + 0 /* general_constraint_indicator_flags 4/6 */, + 0 /* general_constraint_indicator_flags 5/6 */, + 0 /* general_constraint_indicator_flags 6/6 */, + 0x5A /* general_level_idc */, + 0 /* min_spatial_segmentation_idc 1/2 */, + 0 /* min_spatial_segmentation_idc 2/2 */, + 0 /* parallelismType */, + 1 /* chroma_format_idc */, + 0 /* bit_depth_luma_minus8 */, + 0 /* bit_depth_chroma_minus8 */, + 0 /* avgFrameRate 1/2 */, + 0 /* avgFrameRate 2/2 */, + 0x0F /* constantFrameRate/numTemporalLayers/temporalIdNested/lengthSizeMinusOne + */ + , + 0 /* numOfArrays */, + }; + extradata->AppendElements(hvccBytesBuffer, ArrayLength(hvccBytesBuffer)); + auto avcc = HVCCConfig::Parse(extradata); + EXPECT_TRUE(avcc.isErr()); + } + { + // Insuffient data + auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>(); + uint8_t hvccBytesBuffer[] = { + 1 /* version */, + 1 /* general_profile_space/general_tier_flag/general_profile_idc */, + 0x60 /* general_profile_compatibility_flags 1/4 */, + 0 /* general_profile_compatibility_flags 2/4 */, + 0 /* general_profile_compatibility_flags 3/4 */, + 0 /* general_profile_compatibility_flags 4/4 */, + 0x90 /* general_constraint_indicator_flags 1/6 */, + 0 /* general_constraint_indicator_flags 2/6 */, + 0 /* general_constraint_indicator_flags 3/6 */, + 0 /* general_constraint_indicator_flags 4/6 */, + 0 /* general_constraint_indicator_flags 5/6 */, + 0 /* general_constraint_indicator_flags 6/6 */, + 0x5A /* general_level_idc */ + }; + extradata->AppendElements(hvccBytesBuffer, ArrayLength(hvccBytesBuffer)); + auto avcc = HVCCConfig::Parse(extradata); + EXPECT_TRUE(avcc.isErr()); + } +} + +TEST(H265, HVCCToAnnexB) +{ + auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>(); + uint8_t hvccBytesBuffer[] = { + 1 /* version */, + 1 /* general_profile_space/general_tier_flag/general_profile_idc */, + 0x60 /* general_profile_compatibility_flags 1/4 */, + 0 /* general_profile_compatibility_flags 2/4 */, + 0 /* general_profile_compatibility_flags 3/4 */, + 0 /* general_profile_compatibility_flags 4/4 */, + 0x90 /* general_constraint_indicator_flags 1/6 */, + 0 /* general_constraint_indicator_flags 2/6 */, + 0 /* general_constraint_indicator_flags 3/6 */, + 0 /* general_constraint_indicator_flags 4/6 */, + 0 /* general_constraint_indicator_flags 5/6 */, + 0 /* general_constraint_indicator_flags 6/6 */, + 0x5A /* general_level_idc */, + 0 /* min_spatial_segmentation_idc 1/2 */, + 0 /* min_spatial_segmentation_idc 2/2 */, + 0 /* parallelismType */, + 1 /* chroma_format_idc */, + 0 /* bit_depth_luma_minus8 */, + 0 /* bit_depth_chroma_minus8 */, + 0 /* avgFrameRate 1/2 */, + 0 /* avgFrameRate 2/2 */, + 0x0F /* constantFrameRate/numTemporalLayers/temporalIdNested/lengthSizeMinusOne + */ + , + 2 /* numOfArrays */, + /* SPS Array */ + 0x21 /* NAL_unit_type (SPS) */, + 0 /* numNalus 1/2 */, + 1 /* numNalus 2/2 */, + + /* SPS */ + 0 /* nalUnitLength 1/2 */, + 3 /* nalUnitLength 2/2 (header + rsbp) */, + 0x42 /* NALU header 1/2 */, + 0 /* NALU header 2/2 */, + 0 /* rbsp */, + + /* PPS Array */ + 0x22 /* NAL_unit_type (PPS) */, + 0 /* numNalus 1/2 */, + 1 /* numNalus 2/2 */, + + /* PPS */ + 0 /* nalUnitLength 1/2 */, + 3 /* nalUnitLength 2/2 (header + rsbp) */, + 0x44 /* NALU header 1/2 */, + 0 /* NALU header 2/2 */, + 0 /* rbsp */, + }; + extradata->AppendElements(hvccBytesBuffer, ArrayLength(hvccBytesBuffer)); + + // We convert hvcc extra-data to annexb format, then parse each nalu to see if + // they are still correct or not. + const size_t naluBytesSize = 3; // NAL size is 3, see nalUnitLength above + const size_t delimiterBytesSize = 4; // 0x00000001 + const size_t naluPlusDelimiterBytesSize = naluBytesSize + delimiterBytesSize; + RefPtr<mozilla::MediaByteBuffer> annexBExtraData = + AnnexB::ConvertHVCCExtraDataToAnnexB(extradata); + // 2 NALU, sps and pps + EXPECT_EQ(annexBExtraData->Length(), naluPlusDelimiterBytesSize * 2); + + H265NALU sps( + static_cast<uint8_t*>(annexBExtraData->Elements() + delimiterBytesSize), + naluBytesSize); + EXPECT_EQ(sps.mNalUnitType, H265NALU::NAL_TYPES::SPS_NUT); + EXPECT_EQ(sps.mNuhLayerId, 0); + EXPECT_EQ(sps.mNuhTemporalIdPlus1, 0); + EXPECT_EQ(sps.IsSPS(), true); + EXPECT_EQ(sps.mNALU.Length(), 3u); + + H265NALU pps( + static_cast<uint8_t*>(annexBExtraData->Elements() + + naluPlusDelimiterBytesSize + delimiterBytesSize), + naluBytesSize); + EXPECT_EQ(pps.mNalUnitType, H265NALU::NAL_TYPES::PPS_NUT); + EXPECT_EQ(pps.mNuhLayerId, 0); + EXPECT_EQ(pps.mNuhTemporalIdPlus1, 0); + EXPECT_EQ(pps.IsSPS(), false); + EXPECT_EQ(pps.mNALU.Length(), 3u); +} + +TEST(H265, AnnexBToHVCC) +{ + RefPtr<MediaRawData> rawData{GetHVCCSample(128)}; + RefPtr<MediaRawData> rawDataClone = rawData->Clone(); + Result<Ok, nsresult> result = + AnnexB::ConvertHVCCSampleToAnnexB(rawDataClone, /* aAddSps */ false); + EXPECT_TRUE(result.isOk()) << "HVCC to AnnexB Conversion should succeed"; + EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone)) + << "The sample should be AnnexB following conversion"; + + auto rv = AnnexB::ConvertSampleToHVCC(rawDataClone); + EXPECT_TRUE(rv.isOk()) << "AnnexB to HVCC Conversion should succeed"; + EXPECT_TRUE(AnnexB::IsHVCC(rawDataClone)) + << "The sample should be HVCC following conversion"; +} + +// This is SPS from 'hevc_white_frame.mp4' +static const uint8_t sSps[] = { + 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, + 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x5d, 0xa0, 0x02, 0x00, 0x80, + 0x30, 0x16, 0x59, 0x59, 0xa4, 0x93, 0x2b, 0xc0, 0x5a, 0x02, 0x00, + 0x00, 0x03, 0x00, 0x02, 0x00, 0x00, 0x03, 0x00, 0x3c, 0x10}; + +TEST(H265, ExtractHVCCExtraData) +{ + RefPtr<MediaRawData> rawData{GetHVCCSample(sSps, ArrayLength(sSps))}; + RefPtr<MediaByteBuffer> extradata = H265::ExtractHVCCExtraData(rawData); + EXPECT_TRUE(extradata); + auto rv = HVCCConfig::Parse(extradata); + EXPECT_TRUE(rv.isOk()); + auto hvcc = rv.unwrap(); + EXPECT_EQ(hvcc.mNALUs.Length(), 1u); + EXPECT_EQ(hvcc.mNALUs[0].mNalUnitType, H265NALU::NAL_TYPES::SPS_NUT); + EXPECT_EQ(hvcc.mNALUs[0].mNuhLayerId, 0u); + EXPECT_EQ(hvcc.mNALUs[0].mNuhTemporalIdPlus1, 1); + EXPECT_EQ(hvcc.mNALUs[0].IsSPS(), true); + EXPECT_EQ(hvcc.mNALUs[0].mNALU.Length(), 43u); +} + +TEST(H265, DecodeSPSFromSPSNALU) +{ + H265NALU nalu{sSps, ArrayLength(sSps)}; + auto rv = H265::DecodeSPSFromSPSNALU(nalu); + EXPECT_TRUE(rv.isOk()); + auto sps = rv.unwrap(); + // Examine the value by using HEVCESBrowser. + EXPECT_EQ(sps.sps_video_parameter_set_id, 0u); + EXPECT_EQ(sps.sps_max_sub_layers_minus1, 0u); + EXPECT_EQ(sps.sps_temporal_id_nesting_flag, 1); + EXPECT_EQ(sps.profile_tier_level.general_profile_space, 0u); + EXPECT_EQ(sps.profile_tier_level.general_tier_flag, false); + EXPECT_EQ(sps.profile_tier_level.general_profile_idc, 1u); + EXPECT_EQ(sps.profile_tier_level.general_profile_compatibility_flags, + 0x60000000u); + EXPECT_EQ(sps.profile_tier_level.general_progressive_source_flag, true); + EXPECT_EQ(sps.profile_tier_level.general_interlaced_source_flag, false); + EXPECT_EQ(sps.profile_tier_level.general_non_packed_constraint_flag, false); + EXPECT_EQ(sps.profile_tier_level.general_frame_only_constraint_flag, true); + EXPECT_EQ(sps.profile_tier_level.general_level_idc, 93u); + EXPECT_EQ(sps.sps_seq_parameter_set_id, 0u); + EXPECT_EQ(sps.chroma_format_idc, 1u); + EXPECT_EQ(sps.separate_colour_plane_flag, false); + EXPECT_EQ(sps.pic_width_in_luma_samples, 1024u); + EXPECT_EQ(sps.pic_height_in_luma_samples, 768u); + EXPECT_EQ(sps.conformance_window_flag, false); + EXPECT_EQ(sps.bit_depth_luma_minus8, 0u); + EXPECT_EQ(sps.bit_depth_chroma_minus8, 0u); + EXPECT_EQ(sps.log2_max_pic_order_cnt_lsb_minus4, 4u); + EXPECT_EQ(sps.sps_sub_layer_ordering_info_present_flag, true); + EXPECT_EQ(sps.sps_max_dec_pic_buffering_minus1[0], 4u); + EXPECT_EQ(sps.sps_max_num_reorder_pics[0], 2u); + EXPECT_EQ(sps.sps_max_latency_increase_plus1[0], 5u); + EXPECT_EQ(sps.log2_min_luma_coding_block_size_minus3, 0u); + EXPECT_EQ(sps.log2_diff_max_min_luma_coding_block_size, 3u); + EXPECT_EQ(sps.log2_min_luma_transform_block_size_minus2, 0u); + EXPECT_EQ(sps.log2_diff_max_min_luma_transform_block_size, 3u); + EXPECT_EQ(sps.max_transform_hierarchy_depth_inter, 0u); + EXPECT_EQ(sps.max_transform_hierarchy_depth_inter, 0u); + EXPECT_EQ(sps.pcm_enabled_flag, false); + EXPECT_EQ(sps.num_short_term_ref_pic_sets, 0u); + EXPECT_EQ(sps.sps_temporal_mvp_enabled_flag, true); + EXPECT_EQ(sps.strong_intra_smoothing_enabled_flag, true); + EXPECT_TRUE(sps.vui_parameters); + EXPECT_EQ(sps.vui_parameters->video_full_range_flag, false); + + // Test public methods + EXPECT_EQ(sps.BitDepthLuma(), 8u); + EXPECT_EQ(sps.BitDepthChroma(), 8u); + const auto imgSize = sps.GetImageSize(); + EXPECT_EQ(imgSize.Width(), 1024); + EXPECT_EQ(imgSize.Height(), 768); + const auto disSize = sps.GetDisplaySize(); + EXPECT_EQ(disSize, imgSize); + EXPECT_EQ(sps.ColorDepth(), gfx::ColorDepth::COLOR_8); + EXPECT_EQ(sps.ColorSpace(), gfx::YUVColorSpace::BT709); + EXPECT_EQ(sps.IsFullColorRange(), false); + EXPECT_EQ(sps.ColorPrimaries(), 2u); + EXPECT_EQ(sps.TransferFunction(), 2u); +} + +} // namespace mozilla diff --git a/dom/media/platforms/agnostic/bytestreams/gtest/moz.build b/dom/media/platforms/agnostic/bytestreams/gtest/moz.build new file mode 100644 index 0000000000..5351becdaf --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/gtest/moz.build @@ -0,0 +1,11 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+UNIFIED_SOURCES += [
+ "TestByteStreams.cpp",
+]
+
+FINAL_LIBRARY = "xul-gtest"
diff --git a/dom/media/platforms/agnostic/bytestreams/moz.build b/dom/media/platforms/agnostic/bytestreams/moz.build new file mode 100644 index 0000000000..0244a62d41 --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/moz.build @@ -0,0 +1,38 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("**"): + BUG_COMPONENT = ("Core", "Audio/Video: Playback") + +TEST_DIRS += [ + "gtest", +] + +EXPORTS += [ + "Adts.h", + "AnnexB.h", + "ByteStreamsUtils.h", + "H264.h", + "H265.h", +] + +UNIFIED_SOURCES += [ + "Adts.cpp", + "AnnexB.cpp", + "H264.cpp", + "H265.cpp", +] + +LOCAL_INCLUDES += [ + "../../../mp4/", +] + +FINAL_LIBRARY = "xul" + +# Suppress warnings for now. +CXXFLAGS += [ + "-Wno-sign-compare", +] diff --git a/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp new file mode 100644 index 0000000000..e71632e6d3 --- /dev/null +++ b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.cpp @@ -0,0 +1,156 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ChromiumCDMVideoDecoder.h" +#include "ChromiumCDMProxy.h" +#include "content_decryption_module.h" +#include "GMPService.h" +#include "GMPVideoDecoder.h" +#include "MP4Decoder.h" +#include "VPXDecoder.h" + +namespace mozilla { + +ChromiumCDMVideoDecoder::ChromiumCDMVideoDecoder( + const GMPVideoDecoderParams& aParams, CDMProxy* aCDMProxy) + : mCDMParent(aCDMProxy->AsChromiumCDMProxy()->GetCDMParent()), + mConfig(aParams.mConfig), + mCrashHelper(aParams.mCrashHelper), + mGMPThread(GetGMPThread()), + mImageContainer(aParams.mImageContainer), + mKnowsCompositor(aParams.mKnowsCompositor) {} + +ChromiumCDMVideoDecoder::~ChromiumCDMVideoDecoder() = default; + +static uint32_t ToCDMH264Profile(uint8_t aProfile) { + switch (aProfile) { + case 66: + return cdm::VideoCodecProfile::kH264ProfileBaseline; + case 77: + return cdm::VideoCodecProfile::kH264ProfileMain; + case 88: + return cdm::VideoCodecProfile::kH264ProfileExtended; + case 100: + return cdm::VideoCodecProfile::kH264ProfileHigh; + case 110: + return cdm::VideoCodecProfile::kH264ProfileHigh10; + case 122: + return cdm::VideoCodecProfile::kH264ProfileHigh422; + case 144: + return cdm::VideoCodecProfile::kH264ProfileHigh444Predictive; + } + return cdm::VideoCodecProfile::kUnknownVideoCodecProfile; +} + +RefPtr<MediaDataDecoder::InitPromise> ChromiumCDMVideoDecoder::Init() { + if (!mCDMParent) { + // Must have failed to get the CDMParent from the ChromiumCDMProxy + // in our constructor; the MediaKeys must have shut down the CDM + // before we had a chance to start up the decoder. + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + } + + gmp::CDMVideoDecoderConfig config; + if (MP4Decoder::IsH264(mConfig.mMimeType)) { + config.mCodec() = cdm::VideoCodec::kCodecH264; + config.mProfile() = + ToCDMH264Profile(mConfig.mExtraData->SafeElementAt(1, 0)); + config.mExtraData() = mConfig.mExtraData->Clone(); + mConvertToAnnexB = true; + } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) { + config.mCodec() = cdm::VideoCodec::kCodecVp8; + config.mProfile() = cdm::VideoCodecProfile::kProfileNotNeeded; + } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) { + config.mCodec() = cdm::VideoCodec::kCodecVp9; + config.mProfile() = cdm::VideoCodecProfile::kProfileNotNeeded; + } else { + return MediaDataDecoder::InitPromise::CreateAndReject( + NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + } + config.mImageWidth() = mConfig.mImage.width; + config.mImageHeight() = mConfig.mImage.height; + config.mEncryptionScheme() = cdm::EncryptionScheme::kUnencrypted; + switch (mConfig.mCrypto.mCryptoScheme) { + case CryptoScheme::None: + break; + case CryptoScheme::Cenc: + config.mEncryptionScheme() = cdm::EncryptionScheme::kCenc; + break; + case CryptoScheme::Cbcs: + config.mEncryptionScheme() = cdm::EncryptionScheme::kCbcs; + break; + default: + MOZ_ASSERT_UNREACHABLE("Should not have unrecognized encryption type"); + break; + } + + RefPtr<gmp::ChromiumCDMParent> cdm = mCDMParent; + VideoInfo info = mConfig; + RefPtr<layers::ImageContainer> imageContainer = mImageContainer; + RefPtr<layers::KnowsCompositor> knowsCompositor = mKnowsCompositor; + return InvokeAsync(mGMPThread, __func__, + [cdm, config, info, imageContainer, knowsCompositor]() { + return cdm->InitializeVideoDecoder( + config, info, imageContainer, knowsCompositor); + }); +} + +nsCString ChromiumCDMVideoDecoder::GetDescriptionName() const { + return "chromium cdm video decoder"_ns; +} + +nsCString ChromiumCDMVideoDecoder::GetCodecName() const { + if (MP4Decoder::IsH264(mConfig.mMimeType)) { + return "h264"_ns; + } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) { + return "vp8"_ns; + } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) { + return "vp9"_ns; + } + return "unknown"_ns; +} + +MediaDataDecoder::ConversionRequired ChromiumCDMVideoDecoder::NeedsConversion() + const { + return mConvertToAnnexB ? ConversionRequired::kNeedAnnexB + : ConversionRequired::kNeedNone; +} + +RefPtr<MediaDataDecoder::DecodePromise> ChromiumCDMVideoDecoder::Decode( + MediaRawData* aSample) { + RefPtr<gmp::ChromiumCDMParent> cdm = mCDMParent; + RefPtr<MediaRawData> sample = aSample; + return InvokeAsync(mGMPThread, __func__, [cdm, sample]() { + return cdm->DecryptAndDecodeFrame(sample); + }); +} + +RefPtr<MediaDataDecoder::FlushPromise> ChromiumCDMVideoDecoder::Flush() { + MOZ_ASSERT(mCDMParent); + RefPtr<gmp::ChromiumCDMParent> cdm = mCDMParent; + return InvokeAsync(mGMPThread, __func__, + [cdm]() { return cdm->FlushVideoDecoder(); }); +} + +RefPtr<MediaDataDecoder::DecodePromise> ChromiumCDMVideoDecoder::Drain() { + MOZ_ASSERT(mCDMParent); + RefPtr<gmp::ChromiumCDMParent> cdm = mCDMParent; + return InvokeAsync(mGMPThread, __func__, [cdm]() { return cdm->Drain(); }); +} + +RefPtr<ShutdownPromise> ChromiumCDMVideoDecoder::Shutdown() { + if (!mCDMParent) { + // Must have failed to get the CDMParent from the ChromiumCDMProxy + // in our constructor; the MediaKeys must have shut down the CDM + // before we had a chance to start up the decoder. + return ShutdownPromise::CreateAndResolve(true, __func__); + } + RefPtr<gmp::ChromiumCDMParent> cdm = mCDMParent; + return InvokeAsync(mGMPThread, __func__, + [cdm]() { return cdm->ShutdownVideoDecoder(); }); +} + +} // namespace mozilla diff --git a/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.h b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.h new file mode 100644 index 0000000000..c177bf2e48 --- /dev/null +++ b/dom/media/platforms/agnostic/eme/ChromiumCDMVideoDecoder.h @@ -0,0 +1,54 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ChromiumCDMVideoDecoder_h_ +#define ChromiumCDMVideoDecoder_h_ + +#include "ChromiumCDMParent.h" +#include "PlatformDecoderModule.h" +#include "mozilla/layers/KnowsCompositor.h" + +namespace mozilla { + +class CDMProxy; +struct GMPVideoDecoderParams; + +DDLoggedTypeDeclNameAndBase(ChromiumCDMVideoDecoder, MediaDataDecoder); + +class ChromiumCDMVideoDecoder final + : public MediaDataDecoder, + public DecoderDoctorLifeLogger<ChromiumCDMVideoDecoder> { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ChromiumCDMVideoDecoder, final); + + ChromiumCDMVideoDecoder(const GMPVideoDecoderParams& aParams, + CDMProxy* aCDMProxy); + + RefPtr<InitPromise> Init() override; + RefPtr<DecodePromise> Decode(MediaRawData* aSample) override; + RefPtr<FlushPromise> Flush() override; + RefPtr<DecodePromise> Drain() override; + RefPtr<ShutdownPromise> Shutdown() override; + nsCString GetDescriptionName() const override; + nsCString GetCodecName() const override; + ConversionRequired NeedsConversion() const override; + + private: + ~ChromiumCDMVideoDecoder(); + + RefPtr<gmp::ChromiumCDMParent> mCDMParent; + const VideoInfo mConfig; + RefPtr<GMPCrashHelper> mCrashHelper; + nsCOMPtr<nsISerialEventTarget> mGMPThread; + RefPtr<layers::ImageContainer> mImageContainer; + RefPtr<layers::KnowsCompositor> mKnowsCompositor; + MozPromiseHolder<InitPromise> mInitPromise; + bool mConvertToAnnexB = false; +}; + +} // namespace mozilla + +#endif // ChromiumCDMVideoDecoder_h_ diff --git a/dom/media/platforms/agnostic/eme/DecryptThroughputLimit.h b/dom/media/platforms/agnostic/eme/DecryptThroughputLimit.h new file mode 100644 index 0000000000..bafb387f83 --- /dev/null +++ b/dom/media/platforms/agnostic/eme/DecryptThroughputLimit.h @@ -0,0 +1,103 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DecryptThroughputLimit_h +#define DecryptThroughputLimit_h + +#include <deque> + +#include "MediaTimer.h" +#include "PlatformDecoderModule.h" + +namespace mozilla { + +// We throttle our decrypt so that we don't decrypt more than a certain +// duration of samples per second. This is to work around bugs in the +// Widevine CDM. See bugs 1338924, 1342822, 1718223. +class DecryptThroughputLimit { + public: + explicit DecryptThroughputLimit(nsISerialEventTarget* aTargetThread, + uint32_t aMaxThroughputMs) + : mThrottleScheduler(aTargetThread), + mMaxThroughput(aMaxThroughputMs / 1000.0) {} + + typedef MozPromise<RefPtr<MediaRawData>, MediaResult, true> ThrottlePromise; + + // Resolves promise after a delay if necessary in order to reduce the + // throughput of samples sent through the CDM for decryption. + RefPtr<ThrottlePromise> Throttle(MediaRawData* aSample) { + // We should only have one decrypt request being processed at once. + MOZ_RELEASE_ASSERT(!mThrottleScheduler.IsScheduled()); + + const TimeDuration WindowSize = TimeDuration::FromSeconds(0.1); + const TimeDuration MaxThroughput = + TimeDuration::FromSeconds(mMaxThroughput); + + // Forget decrypts that happened before the start of our window. + const TimeStamp now = TimeStamp::Now(); + while (!mDecrypts.empty() && + mDecrypts.front().mTimestamp < now - WindowSize) { + mDecrypts.pop_front(); + } + + // How much time duration of the media would we have decrypted inside the + // time window if we did decrypt this block? + TimeDuration sampleDuration = aSample->mDuration.ToTimeDuration(); + TimeDuration durationDecrypted = sampleDuration; + for (const DecryptedJob& job : mDecrypts) { + durationDecrypted += job.mSampleDuration; + } + + if (durationDecrypted < MaxThroughput) { + // If we decrypted a sample of this duration, we would *not* have + // decrypted more than our threshold for max throughput, over the + // preceding wall time window. So we're safe to proceed with this + // decrypt. + mDecrypts.push_back(DecryptedJob({now, sampleDuration})); + return ThrottlePromise::CreateAndResolve(aSample, __func__); + } + + // Otherwise, we need to delay until decrypting won't exceed our + // throughput threshold. + + RefPtr<ThrottlePromise> p = mPromiseHolder.Ensure(__func__); + + TimeDuration delay = durationDecrypted - MaxThroughput; + TimeStamp target = now + delay; + RefPtr<MediaRawData> sample(aSample); + mThrottleScheduler.Ensure( + target, + [this, sample, sampleDuration]() { + mThrottleScheduler.CompleteRequest(); + mDecrypts.push_back(DecryptedJob({TimeStamp::Now(), sampleDuration})); + mPromiseHolder.Resolve(sample, __func__); + }, + []() { MOZ_DIAGNOSTIC_ASSERT(false); }); + + return p; + } + + void Flush() { + mThrottleScheduler.Reset(); + mPromiseHolder.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + } + + private: + DelayedScheduler mThrottleScheduler; + MozPromiseHolder<ThrottlePromise> mPromiseHolder; + + double mMaxThroughput; + + struct DecryptedJob { + TimeStamp mTimestamp; + TimeDuration mSampleDuration; + }; + std::deque<DecryptedJob> mDecrypts; +}; + +} // namespace mozilla + +#endif // DecryptThroughputLimit_h diff --git a/dom/media/platforms/agnostic/eme/EMEDecoderModule.cpp b/dom/media/platforms/agnostic/eme/EMEDecoderModule.cpp new file mode 100644 index 0000000000..c143172073 --- /dev/null +++ b/dom/media/platforms/agnostic/eme/EMEDecoderModule.cpp @@ -0,0 +1,481 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "EMEDecoderModule.h" + +#include <inttypes.h> + +#include "Adts.h" +#include "BlankDecoderModule.h" +#include "ChromiumCDMVideoDecoder.h" +#include "DecryptThroughputLimit.h" +#include "GMPDecoderModule.h" +#include "GMPService.h" +#include "GMPVideoDecoder.h" +#include "MP4Decoder.h" +#include "MediaInfo.h" +#include "PDMFactory.h" +#include "mozilla/CDMProxy.h" +#include "mozilla/EMEUtils.h" +#include "mozilla/StaticPrefs_media.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/Unused.h" +#include "nsClassHashtable.h" +#include "nsServiceManagerUtils.h" + +namespace mozilla { + +typedef MozPromiseRequestHolder<DecryptPromise> DecryptPromiseRequestHolder; + +DDLoggedTypeDeclNameAndBase(EMEDecryptor, MediaDataDecoder); + +class ADTSSampleConverter { + public: + explicit ADTSSampleConverter(const AudioInfo& aInfo) + : mNumChannels(aInfo.mChannels) + // Note: we set profile to 2 if we encounter an extended profile (which + // set mProfile to 0 and then set mExtendedProfile) such as HE-AACv2 + // (profile 5). These can then pass through conversion to ADTS and back. + // This is done as ADTS only has 2 bits for profile, and the transform + // subtracts one from the value. We check if the profile supplied is > 4 + // for safety. 2 is used as a fallback value, though it seems the CDM + // doesn't care what is set. + , + mProfile(aInfo.mProfile < 1 || aInfo.mProfile > 4 ? 2 : aInfo.mProfile), + mFrequencyIndex(Adts::GetFrequencyIndex(aInfo.mRate)) { + EME_LOG("ADTSSampleConvertor(): aInfo.mProfile=%" PRIi8 + " aInfo.mExtendedProfile=%" PRIi8, + aInfo.mProfile, aInfo.mExtendedProfile); + if (aInfo.mProfile < 1 || aInfo.mProfile > 4) { + EME_LOG( + "ADTSSampleConvertor(): Profile not in [1, 4]! Samples will " + "their profile set to 2!"); + } + } + bool Convert(MediaRawData* aSample) const { + return Adts::ConvertSample(mNumChannels, mFrequencyIndex, mProfile, + aSample); + } + bool Revert(MediaRawData* aSample) const { + return Adts::RevertSample(aSample); + } + + private: + const uint32_t mNumChannels; + const uint8_t mProfile; + const uint8_t mFrequencyIndex; +}; + +class EMEDecryptor final : public MediaDataDecoder, + public DecoderDoctorLifeLogger<EMEDecryptor> { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(EMEDecryptor, final); + + EMEDecryptor(MediaDataDecoder* aDecoder, CDMProxy* aProxy, + TrackInfo::TrackType aType, + const std::function<MediaEventProducer<TrackInfo::TrackType>*()>& + aOnWaitingForKey, + UniquePtr<ADTSSampleConverter> aConverter = nullptr) + : mDecoder(aDecoder), + mProxy(aProxy), + mSamplesWaitingForKey( + new SamplesWaitingForKey(mProxy, aType, aOnWaitingForKey)), + mADTSSampleConverter(std::move(aConverter)), + mIsShutdown(false) { + DDLINKCHILD("decoder", mDecoder.get()); + } + + RefPtr<InitPromise> Init() override { + MOZ_ASSERT(!mIsShutdown); + mThread = GetCurrentSerialEventTarget(); + uint32_t maxThroughputMs = StaticPrefs::media_eme_max_throughput_ms(); + EME_LOG("EME max-throughput-ms=%" PRIu32, maxThroughputMs); + mThroughputLimiter.emplace(mThread, maxThroughputMs); + + return mDecoder->Init(); + } + + RefPtr<DecodePromise> Decode(MediaRawData* aSample) override { + MOZ_ASSERT(mThread->IsOnCurrentThread()); + MOZ_RELEASE_ASSERT(mDecrypts.Count() == 0, + "Can only process one sample at a time"); + RefPtr<DecodePromise> p = mDecodePromise.Ensure(__func__); + + RefPtr<EMEDecryptor> self = this; + mSamplesWaitingForKey->WaitIfKeyNotUsable(aSample) + ->Then( + mThread, __func__, + [self](const RefPtr<MediaRawData>& aSample) { + self->mKeyRequest.Complete(); + self->ThrottleDecode(aSample); + }, + [self]() { self->mKeyRequest.Complete(); }) + ->Track(mKeyRequest); + return p; + } + + void ThrottleDecode(MediaRawData* aSample) { + MOZ_ASSERT(mThread->IsOnCurrentThread()); + + RefPtr<EMEDecryptor> self = this; + mThroughputLimiter->Throttle(aSample) + ->Then( + mThread, __func__, + [self](RefPtr<MediaRawData> aSample) { + self->mThrottleRequest.Complete(); + self->AttemptDecode(aSample); + }, + [self]() { self->mThrottleRequest.Complete(); }) + ->Track(mThrottleRequest); + } + + void AttemptDecode(MediaRawData* aSample) { + MOZ_ASSERT(mThread->IsOnCurrentThread()); + if (mIsShutdown) { + NS_WARNING("EME encrypted sample arrived after shutdown"); + mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + return; + } + + if (mADTSSampleConverter && !mADTSSampleConverter->Convert(aSample)) { + mDecodePromise.RejectIfExists( + MediaResult( + NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("Failed to convert encrypted AAC sample to ADTS")), + __func__); + return; + } + + const auto& decrypt = mDecrypts.InsertOrUpdate( + aSample, MakeUnique<DecryptPromiseRequestHolder>()); + mProxy->Decrypt(aSample) + ->Then(mThread, __func__, this, &EMEDecryptor::Decrypted, + &EMEDecryptor::Decrypted) + ->Track(*decrypt); + } + + void Decrypted(const DecryptResult& aDecrypted) { + MOZ_ASSERT(mThread->IsOnCurrentThread()); + MOZ_ASSERT(aDecrypted.mSample); + + UniquePtr<DecryptPromiseRequestHolder> holder; + mDecrypts.Remove(aDecrypted.mSample, &holder); + if (holder) { + holder->Complete(); + } else { + // Decryption is not in the list of decrypt operations waiting + // for a result. It must have been flushed or drained. Ignore result. + return; + } + + if (mADTSSampleConverter && + !mADTSSampleConverter->Revert(aDecrypted.mSample)) { + mDecodePromise.RejectIfExists( + MediaResult( + NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("Failed to revert decrypted ADTS sample to AAC")), + __func__); + return; + } + + if (mIsShutdown) { + NS_WARNING("EME decrypted sample arrived after shutdown"); + return; + } + + if (aDecrypted.mStatus == eme::NoKeyErr) { + // Key became unusable after we sent the sample to CDM to decrypt. + // Call Decode() again, so that the sample is enqueued for decryption + // if the key becomes usable again. + AttemptDecode(aDecrypted.mSample); + } else if (aDecrypted.mStatus != eme::Ok) { + mDecodePromise.RejectIfExists( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("decrypted.mStatus=%u", + uint32_t(aDecrypted.mStatus))), + __func__); + } else { + MOZ_ASSERT(!mIsShutdown); + // The sample is no longer encrypted, so clear its crypto metadata. + UniquePtr<MediaRawDataWriter> writer(aDecrypted.mSample->CreateWriter()); + writer->mCrypto = CryptoSample(); + RefPtr<EMEDecryptor> self = this; + mDecoder->Decode(aDecrypted.mSample) + ->Then(mThread, __func__, + [self](DecodePromise::ResolveOrRejectValue&& aValue) { + self->mDecodeRequest.Complete(); + self->mDecodePromise.ResolveOrReject(std::move(aValue), + __func__); + }) + ->Track(mDecodeRequest); + } + } + + RefPtr<FlushPromise> Flush() override { + MOZ_ASSERT(mThread->IsOnCurrentThread()); + MOZ_ASSERT(!mIsShutdown); + mKeyRequest.DisconnectIfExists(); + mThrottleRequest.DisconnectIfExists(); + mDecodeRequest.DisconnectIfExists(); + mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + mThroughputLimiter->Flush(); + for (auto iter = mDecrypts.Iter(); !iter.Done(); iter.Next()) { + auto holder = iter.UserData(); + holder->DisconnectIfExists(); + iter.Remove(); + } + RefPtr<SamplesWaitingForKey> k = mSamplesWaitingForKey; + return mDecoder->Flush()->Then(mThread, __func__, [k]() { + k->Flush(); + return FlushPromise::CreateAndResolve(true, __func__); + }); + } + + RefPtr<DecodePromise> Drain() override { + MOZ_ASSERT(mThread->IsOnCurrentThread()); + MOZ_ASSERT(!mIsShutdown); + MOZ_ASSERT(mDecodePromise.IsEmpty() && !mDecodeRequest.Exists(), + "Must wait for decoding to complete"); + for (auto iter = mDecrypts.Iter(); !iter.Done(); iter.Next()) { + auto holder = iter.UserData(); + holder->DisconnectIfExists(); + iter.Remove(); + } + return mDecoder->Drain(); + } + + RefPtr<ShutdownPromise> Shutdown() override { + // mThread may not be set if Init hasn't been called first. + MOZ_ASSERT(!mThread || mThread->IsOnCurrentThread()); + MOZ_ASSERT(!mIsShutdown); + mIsShutdown = true; + mSamplesWaitingForKey->BreakCycles(); + mSamplesWaitingForKey = nullptr; + RefPtr<MediaDataDecoder> decoder = std::move(mDecoder); + mProxy = nullptr; + return decoder->Shutdown(); + } + + nsCString GetProcessName() const override { + return mDecoder->GetProcessName(); + } + + nsCString GetDescriptionName() const override { + return mDecoder->GetDescriptionName(); + } + + nsCString GetCodecName() const override { return mDecoder->GetCodecName(); } + + ConversionRequired NeedsConversion() const override { + return mDecoder->NeedsConversion(); + } + + private: + ~EMEDecryptor() = default; + + RefPtr<MediaDataDecoder> mDecoder; + nsCOMPtr<nsISerialEventTarget> mThread; + RefPtr<CDMProxy> mProxy; + nsClassHashtable<nsRefPtrHashKey<MediaRawData>, DecryptPromiseRequestHolder> + mDecrypts; + RefPtr<SamplesWaitingForKey> mSamplesWaitingForKey; + MozPromiseRequestHolder<SamplesWaitingForKey::WaitForKeyPromise> mKeyRequest; + Maybe<DecryptThroughputLimit> mThroughputLimiter; + MozPromiseRequestHolder<DecryptThroughputLimit::ThrottlePromise> + mThrottleRequest; + MozPromiseHolder<DecodePromise> mDecodePromise; + MozPromiseHolder<DecodePromise> mDrainPromise; + MozPromiseHolder<FlushPromise> mFlushPromise; + MozPromiseRequestHolder<DecodePromise> mDecodeRequest; + UniquePtr<ADTSSampleConverter> mADTSSampleConverter; + bool mIsShutdown; +}; + +EMEMediaDataDecoderProxy::EMEMediaDataDecoderProxy( + const CreateDecoderParams& aParams, + already_AddRefed<MediaDataDecoder> aProxyDecoder, + already_AddRefed<nsISerialEventTarget> aProxyThread, CDMProxy* aProxy) + : MediaDataDecoderProxy(std::move(aProxyDecoder), std::move(aProxyThread)), + mThread(GetCurrentSerialEventTarget()), + mSamplesWaitingForKey(new SamplesWaitingForKey( + aProxy, aParams.mType, aParams.mOnWaitingForKeyEvent)), + mProxy(aProxy) {} + +EMEMediaDataDecoderProxy::EMEMediaDataDecoderProxy( + const CreateDecoderParams& aParams, + already_AddRefed<MediaDataDecoder> aProxyDecoder, CDMProxy* aProxy) + : MediaDataDecoderProxy(std::move(aProxyDecoder), + do_AddRef(GetCurrentSerialEventTarget())), + mThread(GetCurrentSerialEventTarget()), + mSamplesWaitingForKey(new SamplesWaitingForKey( + aProxy, aParams.mType, aParams.mOnWaitingForKeyEvent)), + mProxy(aProxy) {} + +RefPtr<MediaDataDecoder::DecodePromise> EMEMediaDataDecoderProxy::Decode( + MediaRawData* aSample) { + RefPtr<EMEMediaDataDecoderProxy> self = this; + RefPtr<MediaRawData> sample = aSample; + return InvokeAsync(mThread, __func__, [self, this, sample]() { + RefPtr<DecodePromise> p = mDecodePromise.Ensure(__func__); + mSamplesWaitingForKey->WaitIfKeyNotUsable(sample) + ->Then( + mThread, __func__, + [self, this](RefPtr<MediaRawData> aSample) { + mKeyRequest.Complete(); + + MediaDataDecoderProxy::Decode(aSample) + ->Then(mThread, __func__, + [self, + this](DecodePromise::ResolveOrRejectValue&& aValue) { + mDecodeRequest.Complete(); + mDecodePromise.ResolveOrReject(std::move(aValue), + __func__); + }) + ->Track(mDecodeRequest); + }, + [self]() { + self->mKeyRequest.Complete(); + MOZ_CRASH("Should never get here"); + }) + ->Track(mKeyRequest); + + return p; + }); +} + +RefPtr<MediaDataDecoder::FlushPromise> EMEMediaDataDecoderProxy::Flush() { + RefPtr<EMEMediaDataDecoderProxy> self = this; + return InvokeAsync(mThread, __func__, [self, this]() { + mKeyRequest.DisconnectIfExists(); + mDecodeRequest.DisconnectIfExists(); + mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + return MediaDataDecoderProxy::Flush(); + }); +} + +RefPtr<ShutdownPromise> EMEMediaDataDecoderProxy::Shutdown() { + RefPtr<EMEMediaDataDecoderProxy> self = this; + return InvokeAsync(mThread, __func__, [self, this]() { + mSamplesWaitingForKey->BreakCycles(); + mSamplesWaitingForKey = nullptr; + mProxy = nullptr; + return MediaDataDecoderProxy::Shutdown(); + }); +} + +EMEDecoderModule::EMEDecoderModule(CDMProxy* aProxy, PDMFactory* aPDM) + : mProxy(aProxy), mPDM(aPDM) {} + +EMEDecoderModule::~EMEDecoderModule() = default; + +static already_AddRefed<MediaDataDecoderProxy> CreateDecoderWrapper( + CDMProxy* aProxy, const CreateDecoderParams& aParams) { + RefPtr<gmp::GeckoMediaPluginService> s( + gmp::GeckoMediaPluginService::GetGeckoMediaPluginService()); + if (!s) { + return nullptr; + } + nsCOMPtr<nsISerialEventTarget> thread(s->GetGMPThread()); + if (!thread) { + return nullptr; + } + RefPtr<MediaDataDecoderProxy> decoder( + new EMEMediaDataDecoderProxy(aParams, + do_AddRef(new ChromiumCDMVideoDecoder( + GMPVideoDecoderParams(aParams), aProxy)), + thread.forget(), aProxy)); + return decoder.forget(); +} + +RefPtr<EMEDecoderModule::CreateDecoderPromise> +EMEDecoderModule::AsyncCreateDecoder(const CreateDecoderParams& aParams) { + MOZ_ASSERT(aParams.mConfig.mCrypto.IsEncrypted()); + MOZ_ASSERT(mPDM); + + if (aParams.mConfig.IsVideo()) { + if (StaticPrefs::media_eme_video_blank()) { + EME_LOG( + "EMEDecoderModule::CreateVideoDecoder() creating a blank decoder."); + RefPtr<PlatformDecoderModule> m(BlankDecoderModule::Create()); + RefPtr<MediaDataDecoder> decoder = m->CreateVideoDecoder(aParams); + return EMEDecoderModule::CreateDecoderPromise::CreateAndResolve(decoder, + __func__); + } + + if (!SupportsMimeType(aParams.mConfig.mMimeType, nullptr).isEmpty()) { + // GMP decodes. Assume that means it can decrypt too. + return EMEDecoderModule::CreateDecoderPromise::CreateAndResolve( + CreateDecoderWrapper(mProxy, aParams), __func__); + } + + RefPtr<EMEDecoderModule::CreateDecoderPromise> p = + mPDM->CreateDecoder(aParams)->Then( + GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, + params = CreateDecoderParamsForAsync(aParams)]( + RefPtr<MediaDataDecoder>&& aDecoder) { + RefPtr<MediaDataDecoder> emeDecoder( + new EMEDecryptor(aDecoder, self->mProxy, params.mType, + params.mOnWaitingForKeyEvent)); + return EMEDecoderModule::CreateDecoderPromise::CreateAndResolve( + emeDecoder, __func__); + }, + [](const MediaResult& aError) { + return EMEDecoderModule::CreateDecoderPromise::CreateAndReject( + aError, __func__); + }); + return p; + } + + MOZ_ASSERT(aParams.mConfig.IsAudio()); + + // We don't support using the GMP to decode audio. + MOZ_ASSERT(SupportsMimeType(aParams.mConfig.mMimeType, nullptr).isEmpty()); + MOZ_ASSERT(mPDM); + + if (StaticPrefs::media_eme_audio_blank()) { + EME_LOG("EMEDecoderModule::CreateAudioDecoder() creating a blank decoder."); + RefPtr<PlatformDecoderModule> m(BlankDecoderModule::Create()); + RefPtr<MediaDataDecoder> decoder = m->CreateAudioDecoder(aParams); + return EMEDecoderModule::CreateDecoderPromise::CreateAndResolve(decoder, + __func__); + } + + UniquePtr<ADTSSampleConverter> converter = nullptr; + if (MP4Decoder::IsAAC(aParams.mConfig.mMimeType)) { + // The CDM expects encrypted AAC to be in ADTS format. + // See bug 1433344. + converter = MakeUnique<ADTSSampleConverter>(aParams.AudioConfig()); + } + + RefPtr<EMEDecoderModule::CreateDecoderPromise> p = + mPDM->CreateDecoder(aParams)->Then( + GetCurrentSerialEventTarget(), __func__, + [self = RefPtr{this}, params = CreateDecoderParamsForAsync(aParams), + converter = std::move(converter)]( + RefPtr<MediaDataDecoder>&& aDecoder) mutable { + RefPtr<MediaDataDecoder> emeDecoder(new EMEDecryptor( + aDecoder, self->mProxy, params.mType, + params.mOnWaitingForKeyEvent, std::move(converter))); + return EMEDecoderModule::CreateDecoderPromise::CreateAndResolve( + emeDecoder, __func__); + }, + [](const MediaResult& aError) { + return EMEDecoderModule::CreateDecoderPromise::CreateAndReject( + aError, __func__); + }); + return p; +} + +media::DecodeSupportSet EMEDecoderModule::SupportsMimeType( + const nsACString& aMimeType, DecoderDoctorDiagnostics* aDiagnostics) const { + Maybe<nsCString> keySystem; + keySystem.emplace(NS_ConvertUTF16toUTF8(mProxy->KeySystem())); + return GMPDecoderModule::SupportsMimeType( + aMimeType, nsLiteralCString(CHROMIUM_CDM_API), keySystem); +} + +} // namespace mozilla diff --git a/dom/media/platforms/agnostic/eme/EMEDecoderModule.h b/dom/media/platforms/agnostic/eme/EMEDecoderModule.h new file mode 100644 index 0000000000..06fea6e650 --- /dev/null +++ b/dom/media/platforms/agnostic/eme/EMEDecoderModule.h @@ -0,0 +1,79 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(EMEDecoderModule_h_) +# define EMEDecoderModule_h_ + +# include "MediaDataDecoderProxy.h" +# include "PlatformDecoderModule.h" +# include "SamplesWaitingForKey.h" + +namespace mozilla { + +class CDMProxy; +class PDMFactory; + +class EMEDecoderModule : public PlatformDecoderModule { + public: + EMEDecoderModule(CDMProxy* aProxy, PDMFactory* aPDM); + + protected: + RefPtr<CreateDecoderPromise> AsyncCreateDecoder( + const CreateDecoderParams& aParams) override; + + // Decode thread. + already_AddRefed<MediaDataDecoder> CreateVideoDecoder( + const CreateDecoderParams& aParams) override { + MOZ_CRASH("Not used"); + } + + // Decode thread. + already_AddRefed<MediaDataDecoder> CreateAudioDecoder( + const CreateDecoderParams& aParams) override { + MOZ_CRASH("Not used"); + } + + media::DecodeSupportSet SupportsMimeType( + const nsACString& aMimeType, + DecoderDoctorDiagnostics* aDiagnostics) const override; + + private: + virtual ~EMEDecoderModule(); + RefPtr<CDMProxy> mProxy; + // Will be null if CDM has decoding capability. + RefPtr<PDMFactory> mPDM; +}; + +DDLoggedTypeDeclNameAndBase(EMEMediaDataDecoderProxy, MediaDataDecoderProxy); + +class EMEMediaDataDecoderProxy + : public MediaDataDecoderProxy, + public DecoderDoctorLifeLogger<EMEMediaDataDecoderProxy> { + public: + EMEMediaDataDecoderProxy(const CreateDecoderParams& aParams, + already_AddRefed<MediaDataDecoder> aProxyDecoder, + already_AddRefed<nsISerialEventTarget> aProxyThread, + CDMProxy* aProxy); + EMEMediaDataDecoderProxy(const CreateDecoderParams& aParams, + already_AddRefed<MediaDataDecoder> aProxyDecoder, + CDMProxy* aProxy); + + RefPtr<DecodePromise> Decode(MediaRawData* aSample) override; + RefPtr<FlushPromise> Flush() override; + RefPtr<ShutdownPromise> Shutdown() override; + + private: + nsCOMPtr<nsISerialEventTarget> mThread; + RefPtr<SamplesWaitingForKey> mSamplesWaitingForKey; + MozPromiseRequestHolder<SamplesWaitingForKey::WaitForKeyPromise> mKeyRequest; + MozPromiseHolder<DecodePromise> mDecodePromise; + MozPromiseRequestHolder<DecodePromise> mDecodeRequest; + RefPtr<CDMProxy> mProxy; +}; + +} // namespace mozilla + +#endif // EMEDecoderModule_h_ diff --git a/dom/media/platforms/agnostic/eme/SamplesWaitingForKey.cpp b/dom/media/platforms/agnostic/eme/SamplesWaitingForKey.cpp new file mode 100644 index 0000000000..23d79ce56a --- /dev/null +++ b/dom/media/platforms/agnostic/eme/SamplesWaitingForKey.cpp @@ -0,0 +1,79 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "SamplesWaitingForKey.h" + +#include "MediaData.h" +#include "MediaEventSource.h" +#include "mozilla/CDMCaps.h" +#include "mozilla/CDMProxy.h" +#include "mozilla/TaskQueue.h" + +namespace mozilla { + +SamplesWaitingForKey::SamplesWaitingForKey( + CDMProxy* aProxy, TrackInfo::TrackType aType, + const std::function<MediaEventProducer<TrackInfo::TrackType>*()>& + aOnWaitingForKeyEvent) + : mMutex("SamplesWaitingForKey"), + mProxy(aProxy), + mType(aType), + mOnWaitingForKeyEvent(aOnWaitingForKeyEvent) {} + +SamplesWaitingForKey::~SamplesWaitingForKey() { Flush(); } + +RefPtr<SamplesWaitingForKey::WaitForKeyPromise> +SamplesWaitingForKey::WaitIfKeyNotUsable(MediaRawData* aSample) { + if (!aSample || !aSample->mCrypto.IsEncrypted() || !mProxy) { + return WaitForKeyPromise::CreateAndResolve(aSample, __func__); + } + auto caps = mProxy->Capabilites().Lock(); + const auto& keyid = aSample->mCrypto.mKeyId; + if (caps->IsKeyUsable(keyid)) { + return WaitForKeyPromise::CreateAndResolve(aSample, __func__); + } + SampleEntry entry; + entry.mSample = aSample; + RefPtr<WaitForKeyPromise> p = entry.mPromise.Ensure(__func__); + { + MutexAutoLock lock(mMutex); + mSamples.AppendElement(std::move(entry)); + } + if (mOnWaitingForKeyEvent && mOnWaitingForKeyEvent()) { + mOnWaitingForKeyEvent()->Notify(mType); + } + caps->NotifyWhenKeyIdUsable(aSample->mCrypto.mKeyId, this); + return p; +} + +void SamplesWaitingForKey::NotifyUsable(const CencKeyId& aKeyId) { + MutexAutoLock lock(mMutex); + size_t i = 0; + while (i < mSamples.Length()) { + auto& entry = mSamples[i]; + if (aKeyId == entry.mSample->mCrypto.mKeyId) { + entry.mPromise.Resolve(entry.mSample, __func__); + mSamples.RemoveElementAt(i); + } else { + i++; + } + } +} + +void SamplesWaitingForKey::Flush() { + MutexAutoLock lock(mMutex); + for (auto& sample : mSamples) { + sample.mPromise.Reject(true, __func__); + } + mSamples.Clear(); +} + +void SamplesWaitingForKey::BreakCycles() { + MutexAutoLock lock(mMutex); + mProxy = nullptr; +} + +} // namespace mozilla diff --git a/dom/media/platforms/agnostic/eme/SamplesWaitingForKey.h b/dom/media/platforms/agnostic/eme/SamplesWaitingForKey.h new file mode 100644 index 0000000000..06d72e3aae --- /dev/null +++ b/dom/media/platforms/agnostic/eme/SamplesWaitingForKey.h @@ -0,0 +1,68 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef SamplesWaitingForKey_h_ +#define SamplesWaitingForKey_h_ + +#include <functional> + +#include "MediaInfo.h" +#include "mozilla/MozPromise.h" +#include "mozilla/Mutex.h" +#include "mozilla/RefPtr.h" + +namespace mozilla { + +typedef nsTArray<uint8_t> CencKeyId; + +class CDMProxy; +template <typename... Es> +class MediaEventProducer; +class MediaRawData; + +// Encapsulates the task of waiting for the CDMProxy to have the necessary +// keys to decrypt a given sample. +class SamplesWaitingForKey { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(SamplesWaitingForKey) + + typedef MozPromise<RefPtr<MediaRawData>, bool, /* IsExclusive = */ true> + WaitForKeyPromise; + + SamplesWaitingForKey( + CDMProxy* aProxy, TrackInfo::TrackType aType, + const std::function<MediaEventProducer<TrackInfo::TrackType>*()>& + aOnWaitingForKeyEvent); + + // Returns a promise that will be resolved if or when a key for decoding the + // sample becomes usable. + RefPtr<WaitForKeyPromise> WaitIfKeyNotUsable(MediaRawData* aSample); + + void NotifyUsable(const CencKeyId& aKeyId); + + void Flush(); + + void BreakCycles(); + + protected: + ~SamplesWaitingForKey(); + + private: + Mutex mMutex MOZ_UNANNOTATED; + RefPtr<CDMProxy> mProxy; + struct SampleEntry { + RefPtr<MediaRawData> mSample; + MozPromiseHolder<WaitForKeyPromise> mPromise; + }; + nsTArray<SampleEntry> mSamples; + const TrackInfo::TrackType mType; + const std::function<MediaEventProducer<TrackInfo::TrackType>*()> + mOnWaitingForKeyEvent; +}; + +} // namespace mozilla + +#endif // SamplesWaitingForKey_h_ diff --git a/dom/media/platforms/agnostic/eme/moz.build b/dom/media/platforms/agnostic/eme/moz.build new file mode 100644 index 0000000000..34f0007b3b --- /dev/null +++ b/dom/media/platforms/agnostic/eme/moz.build @@ -0,0 +1,22 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXPORTS += [ + "ChromiumCDMVideoDecoder.h", + "DecryptThroughputLimit.h", + "EMEDecoderModule.h", + "SamplesWaitingForKey.h", +] + +UNIFIED_SOURCES += [ + "ChromiumCDMVideoDecoder.cpp", + "EMEDecoderModule.cpp", + "SamplesWaitingForKey.cpp", +] + +include("/ipc/chromium/chromium-config.mozbuild") + +FINAL_LIBRARY = "xul" diff --git a/dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp b/dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp new file mode 100644 index 0000000000..f01c7e94e4 --- /dev/null +++ b/dom/media/platforms/agnostic/gmp/GMPDecoderModule.cpp @@ -0,0 +1,94 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "GMPDecoderModule.h" + +#include "DecoderDoctorDiagnostics.h" +#include "GMPService.h" +#include "GMPUtils.h" +#include "GMPVideoDecoder.h" +#include "MP4Decoder.h" +#include "MediaDataDecoderProxy.h" +#include "VPXDecoder.h" +#include "VideoUtils.h" +#include "gmp-video-decode.h" +#include "mozilla/StaticMutex.h" +#include "nsServiceManagerUtils.h" +#ifdef XP_WIN +# include "WMFDecoderModule.h" +#endif + +namespace mozilla { + +static already_AddRefed<MediaDataDecoderProxy> CreateDecoderWrapper( + GMPVideoDecoderParams&& aParams) { + RefPtr<gmp::GeckoMediaPluginService> s( + gmp::GeckoMediaPluginService::GetGeckoMediaPluginService()); + if (!s) { + return nullptr; + } + nsCOMPtr<nsISerialEventTarget> thread(s->GetGMPThread()); + if (!thread) { + return nullptr; + } + + RefPtr<MediaDataDecoderProxy> decoder(new MediaDataDecoderProxy( + do_AddRef(new GMPVideoDecoder(std::move(aParams))), thread.forget())); + return decoder.forget(); +} + +already_AddRefed<MediaDataDecoder> GMPDecoderModule::CreateVideoDecoder( + const CreateDecoderParams& aParams) { + if (!MP4Decoder::IsH264(aParams.mConfig.mMimeType) && + !VPXDecoder::IsVP8(aParams.mConfig.mMimeType) && + !VPXDecoder::IsVP9(aParams.mConfig.mMimeType)) { + return nullptr; + } + + return CreateDecoderWrapper(GMPVideoDecoderParams(aParams)); +} + +already_AddRefed<MediaDataDecoder> GMPDecoderModule::CreateAudioDecoder( + const CreateDecoderParams& aParams) { + return nullptr; +} + +/* static */ +media::DecodeSupportSet GMPDecoderModule::SupportsMimeType( + const nsACString& aMimeType, const nsACString& aApi, + const Maybe<nsCString>& aKeySystem) { + AutoTArray<nsCString, 2> tags; + if (MP4Decoder::IsH264(aMimeType)) { + tags.AppendElement("h264"_ns); + } else if (VPXDecoder::IsVP9(aMimeType)) { + tags.AppendElement("vp9"_ns); + } else if (VPXDecoder::IsVP8(aMimeType)) { + tags.AppendElement("vp8"_ns); + } else { + return media::DecodeSupportSet{}; + } + + // Optional tag for EME GMP plugins. + if (aKeySystem) { + tags.AppendElement(*aKeySystem); + } + + // GMP plugins are always software based. + return HaveGMPFor(aApi, tags) ? media::DecodeSupport::SoftwareDecode + : media::DecodeSupportSet{}; +} + +media::DecodeSupportSet GMPDecoderModule::SupportsMimeType( + const nsACString& aMimeType, DecoderDoctorDiagnostics* aDiagnostics) const { + return SupportsMimeType(aMimeType, "decode-video"_ns, Nothing()); +} + +/* static */ +already_AddRefed<PlatformDecoderModule> GMPDecoderModule::Create() { + return MakeAndAddRef<GMPDecoderModule>(); +} + +} // namespace mozilla diff --git a/dom/media/platforms/agnostic/gmp/GMPDecoderModule.h b/dom/media/platforms/agnostic/gmp/GMPDecoderModule.h new file mode 100644 index 0000000000..1a131dc154 --- /dev/null +++ b/dom/media/platforms/agnostic/gmp/GMPDecoderModule.h @@ -0,0 +1,58 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(GMPDecoderModule_h_) +# define GMPDecoderModule_h_ + +# include "PlatformDecoderModule.h" +# include "mozilla/Maybe.h" + +// The special NodeId we use when doing unencrypted decoding using the GMP's +// decoder. This ensures that each GMP MediaDataDecoder we create doesn't +// require spinning up a new process, but instead we run all instances of +// GMP decoders in the one process, to reduce overhead. +// +// Note: GMP storage is isolated by NodeId, and non persistent for this +// special NodeId, and the only way a GMP can communicate with the outside +// world is through the EME GMP APIs, and we never run EME with this NodeID +// (because NodeIds are random strings which can't contain the '-' character), +// so there's no way a malicious GMP can harvest, store, and then report any +// privacy sensitive data about what users are watching. +# define SHARED_GMP_DECODING_NODE_ID "gmp-shared-decoding"_ns + +namespace mozilla { + +class GMPDecoderModule : public PlatformDecoderModule { + template <typename T, typename... Args> + friend already_AddRefed<T> MakeAndAddRef(Args&&...); + + public: + static already_AddRefed<PlatformDecoderModule> Create(); + + // Decode thread. + already_AddRefed<MediaDataDecoder> CreateVideoDecoder( + const CreateDecoderParams& aParams) override; + + // Decode thread. + already_AddRefed<MediaDataDecoder> CreateAudioDecoder( + const CreateDecoderParams& aParams) override; + + media::DecodeSupportSet SupportsMimeType( + const nsACString& aMimeType, + DecoderDoctorDiagnostics* aDiagnostics) const override; + + static media::DecodeSupportSet SupportsMimeType( + const nsACString& aMimeType, const nsACString& aApi, + const Maybe<nsCString>& aKeySystem); + + private: + GMPDecoderModule() = default; + virtual ~GMPDecoderModule() = default; +}; + +} // namespace mozilla + +#endif // GMPDecoderModule_h_ diff --git a/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.cpp b/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.cpp new file mode 100644 index 0000000000..b964036a4a --- /dev/null +++ b/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.cpp @@ -0,0 +1,489 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "GMPVideoDecoder.h" +#include "GMPDecoderModule.h" +#include "GMPVideoHost.h" +#include "GMPLog.h" +#include "MediaData.h" +#include "mozilla/EndianUtils.h" +#include "mozilla/StaticPrefs_media.h" +#include "nsServiceManagerUtils.h" +#include "AnnexB.h" +#include "H264.h" +#include "MP4Decoder.h" +#include "prsystem.h" +#include "VPXDecoder.h" +#include "VideoUtils.h" + +namespace mozilla { + +#if defined(DEBUG) +static bool IsOnGMPThread() { + nsCOMPtr<mozIGeckoMediaPluginService> mps = + do_GetService("@mozilla.org/gecko-media-plugin-service;1"); + MOZ_ASSERT(mps); + + nsCOMPtr<nsIThread> gmpThread; + nsresult rv = mps->GetThread(getter_AddRefs(gmpThread)); + MOZ_ASSERT(NS_SUCCEEDED(rv) && gmpThread); + return gmpThread->IsOnCurrentThread(); +} +#endif + +GMPVideoDecoderParams::GMPVideoDecoderParams(const CreateDecoderParams& aParams) + : mConfig(aParams.VideoConfig()), + mImageContainer(aParams.mImageContainer), + mCrashHelper(aParams.mCrashHelper), + mKnowsCompositor(aParams.mKnowsCompositor), + mTrackingId(aParams.mTrackingId) {} + +nsCString GMPVideoDecoder::GetCodecName() const { + if (MP4Decoder::IsH264(mConfig.mMimeType)) { + return "h264"_ns; + } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) { + return "vp8"_ns; + } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) { + return "vp9"_ns; + } + return "unknown"_ns; +} + +void GMPVideoDecoder::Decoded(GMPVideoi420Frame* aDecodedFrame) { + GMP_LOG_DEBUG("GMPVideoDecoder::Decoded"); + + GMPUniquePtr<GMPVideoi420Frame> decodedFrame(aDecodedFrame); + MOZ_ASSERT(IsOnGMPThread()); + + VideoData::YCbCrBuffer b; + for (int i = 0; i < kGMPNumOfPlanes; ++i) { + b.mPlanes[i].mData = decodedFrame->Buffer(GMPPlaneType(i)); + b.mPlanes[i].mStride = decodedFrame->Stride(GMPPlaneType(i)); + if (i == kGMPYPlane) { + b.mPlanes[i].mWidth = decodedFrame->Width(); + b.mPlanes[i].mHeight = decodedFrame->Height(); + } else { + b.mPlanes[i].mWidth = (decodedFrame->Width() + 1) / 2; + b.mPlanes[i].mHeight = (decodedFrame->Height() + 1) / 2; + } + b.mPlanes[i].mSkip = 0; + } + + b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT; + b.mYUVColorSpace = + DefaultColorSpace({decodedFrame->Width(), decodedFrame->Height()}); + + UniquePtr<SampleMetadata> sampleData; + if (auto entryHandle = mSamples.Lookup(decodedFrame->Timestamp())) { + sampleData = std::move(entryHandle.Data()); + entryHandle.Remove(); + } else { + GMP_LOG_DEBUG( + "GMPVideoDecoder::Decoded(this=%p) missing sample metadata for " + "time %" PRIu64, + this, decodedFrame->Timestamp()); + if (mSamples.IsEmpty()) { + // If we have no remaining samples in the table, then we have processed + // all outstanding decode requests. + ProcessReorderQueue(mDecodePromise, __func__); + } + return; + } + + MOZ_ASSERT(sampleData); + + gfx::IntRect pictureRegion(0, 0, decodedFrame->Width(), + decodedFrame->Height()); + Result<already_AddRefed<VideoData>, MediaResult> r = + VideoData::CreateAndCopyData( + mConfig, mImageContainer, sampleData->mOffset, + media::TimeUnit::FromMicroseconds(decodedFrame->UpdatedTimestamp()), + media::TimeUnit::FromMicroseconds(decodedFrame->Duration()), b, + sampleData->mKeyframe, media::TimeUnit::FromMicroseconds(-1), + pictureRegion, mKnowsCompositor); + if (r.isErr()) { + mReorderQueue.Clear(); + mUnorderedData.Clear(); + mSamples.Clear(); + mDecodePromise.RejectIfExists(r.unwrapErr(), __func__); + return; + } + + RefPtr<VideoData> v = r.unwrap(); + MOZ_ASSERT(v); + mPerformanceRecorder.Record(static_cast<int64_t>(decodedFrame->Timestamp()), + [&](DecodeStage& aStage) { + aStage.SetImageFormat(DecodeStage::YUV420P); + aStage.SetResolution(decodedFrame->Width(), + decodedFrame->Height()); + aStage.SetYUVColorSpace(b.mYUVColorSpace); + aStage.SetColorDepth(b.mColorDepth); + aStage.SetColorRange(b.mColorRange); + }); + + if (mReorderFrames) { + mReorderQueue.Push(std::move(v)); + } else { + mUnorderedData.AppendElement(std::move(v)); + } + + if (mSamples.IsEmpty()) { + // If we have no remaining samples in the table, then we have processed + // all outstanding decode requests. + ProcessReorderQueue(mDecodePromise, __func__); + } +} + +void GMPVideoDecoder::ReceivedDecodedReferenceFrame(const uint64_t aPictureId) { + GMP_LOG_DEBUG("GMPVideoDecoder::ReceivedDecodedReferenceFrame"); + MOZ_ASSERT(IsOnGMPThread()); +} + +void GMPVideoDecoder::ReceivedDecodedFrame(const uint64_t aPictureId) { + GMP_LOG_DEBUG("GMPVideoDecoder::ReceivedDecodedFrame"); + MOZ_ASSERT(IsOnGMPThread()); +} + +void GMPVideoDecoder::InputDataExhausted() { + GMP_LOG_DEBUG("GMPVideoDecoder::InputDataExhausted"); + MOZ_ASSERT(IsOnGMPThread()); + mSamples.Clear(); + ProcessReorderQueue(mDecodePromise, __func__); +} + +void GMPVideoDecoder::DrainComplete() { + GMP_LOG_DEBUG("GMPVideoDecoder::DrainComplete"); + MOZ_ASSERT(IsOnGMPThread()); + mSamples.Clear(); + + if (mDrainPromise.IsEmpty()) { + return; + } + + DecodedData results; + if (mReorderFrames) { + results.SetCapacity(mReorderQueue.Length()); + while (!mReorderQueue.IsEmpty()) { + results.AppendElement(mReorderQueue.Pop()); + } + } else { + results = std::move(mUnorderedData); + } + + mDrainPromise.Resolve(std::move(results), __func__); +} + +void GMPVideoDecoder::ResetComplete() { + GMP_LOG_DEBUG("GMPVideoDecoder::ResetComplete"); + MOZ_ASSERT(IsOnGMPThread()); + mPerformanceRecorder.Record(std::numeric_limits<int64_t>::max()); + mFlushPromise.ResolveIfExists(true, __func__); +} + +void GMPVideoDecoder::Error(GMPErr aErr) { + GMP_LOG_DEBUG("GMPVideoDecoder::Error"); + MOZ_ASSERT(IsOnGMPThread()); + auto error = MediaResult(aErr == GMPDecodeErr ? NS_ERROR_DOM_MEDIA_DECODE_ERR + : NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("GMPErr:%x", aErr)); + mDecodePromise.RejectIfExists(error, __func__); + mDrainPromise.RejectIfExists(error, __func__); + mFlushPromise.RejectIfExists(error, __func__); +} + +void GMPVideoDecoder::Terminated() { + GMP_LOG_DEBUG("GMPVideoDecoder::Terminated"); + MOZ_ASSERT(IsOnGMPThread()); + Error(GMPErr::GMPAbortedErr); +} + +void GMPVideoDecoder::ProcessReorderQueue( + MozPromiseHolder<DecodePromise>& aPromise, const char* aMethodName) { + if (aPromise.IsEmpty()) { + return; + } + + if (!mReorderFrames) { + aPromise.Resolve(std::move(mUnorderedData), aMethodName); + return; + } + + DecodedData results; + size_t availableFrames = mReorderQueue.Length(); + if (availableFrames > mMaxRefFrames) { + size_t resolvedFrames = availableFrames - mMaxRefFrames; + results.SetCapacity(resolvedFrames); + do { + results.AppendElement(mReorderQueue.Pop()); + } while (--resolvedFrames > 0); + } + + aPromise.Resolve(std::move(results), aMethodName); +} + +GMPVideoDecoder::GMPVideoDecoder(const GMPVideoDecoderParams& aParams) + : mConfig(aParams.mConfig), + mGMP(nullptr), + mHost(nullptr), + mConvertNALUnitLengths(false), + mCrashHelper(aParams.mCrashHelper), + mImageContainer(aParams.mImageContainer), + mKnowsCompositor(aParams.mKnowsCompositor), + mTrackingId(aParams.mTrackingId), + mCanDecodeBatch(StaticPrefs::media_gmp_decoder_decode_batch()), + mReorderFrames(StaticPrefs::media_gmp_decoder_reorder_frames()) {} + +void GMPVideoDecoder::InitTags(nsTArray<nsCString>& aTags) { + if (MP4Decoder::IsH264(mConfig.mMimeType)) { + aTags.AppendElement("h264"_ns); + } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) { + aTags.AppendElement("vp8"_ns); + } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) { + aTags.AppendElement("vp9"_ns); + } +} + +nsCString GMPVideoDecoder::GetNodeId() { return SHARED_GMP_DECODING_NODE_ID; } + +GMPUniquePtr<GMPVideoEncodedFrame> GMPVideoDecoder::CreateFrame( + MediaRawData* aSample) { + GMPVideoFrame* ftmp = nullptr; + GMPErr err = mHost->CreateFrame(kGMPEncodedVideoFrame, &ftmp); + if (GMP_FAILED(err)) { + return nullptr; + } + + GMPUniquePtr<GMPVideoEncodedFrame> frame( + static_cast<GMPVideoEncodedFrame*>(ftmp)); + err = frame->CreateEmptyFrame(aSample->Size()); + if (GMP_FAILED(err)) { + return nullptr; + } + + memcpy(frame->Buffer(), aSample->Data(), frame->Size()); + + // Convert 4-byte NAL unit lengths to host-endian 4-byte buffer lengths to + // suit the GMP API. + if (mConvertNALUnitLengths) { + const int kNALLengthSize = 4; + uint8_t* buf = frame->Buffer(); + while (buf < frame->Buffer() + frame->Size() - kNALLengthSize) { + uint32_t length = BigEndian::readUint32(buf) + kNALLengthSize; + *reinterpret_cast<uint32_t*>(buf) = length; + buf += length; + } + } + + frame->SetBufferType(GMP_BufferLength32); + + frame->SetEncodedWidth(mConfig.mDisplay.width); + frame->SetEncodedHeight(mConfig.mDisplay.height); + frame->SetTimeStamp(aSample->mTime.ToMicroseconds()); + frame->SetCompleteFrame(true); + frame->SetDuration(aSample->mDuration.ToMicroseconds()); + frame->SetFrameType(aSample->mKeyframe ? kGMPKeyFrame : kGMPDeltaFrame); + + return frame; +} + +const VideoInfo& GMPVideoDecoder::GetConfig() const { return mConfig; } + +void GMPVideoDecoder::GMPInitDone(GMPVideoDecoderProxy* aGMP, + GMPVideoHost* aHost) { + MOZ_ASSERT(IsOnGMPThread()); + + if (!aGMP) { + mInitPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + return; + } + MOZ_ASSERT(aHost); + + if (mInitPromise.IsEmpty()) { + // GMP must have been shutdown while we were waiting for Init operation + // to complete. + aGMP->Close(); + return; + } + + bool isOpenH264 = aGMP->GetPluginType() == GMPPluginType::OpenH264; + + GMPVideoCodec codec; + memset(&codec, 0, sizeof(codec)); + + codec.mGMPApiVersion = kGMPVersion34; + nsTArray<uint8_t> codecSpecific; + if (MP4Decoder::IsH264(mConfig.mMimeType)) { + codec.mCodecType = kGMPVideoCodecH264; + codecSpecific.AppendElement(0); // mPacketizationMode. + codecSpecific.AppendElements(mConfig.mExtraData->Elements(), + mConfig.mExtraData->Length()); + // OpenH264 expects pseudo-AVCC, but others must be passed + // AnnexB for H264. + mConvertToAnnexB = !isOpenH264; + mMaxRefFrames = H264::ComputeMaxRefFrames(mConfig.mExtraData); + } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) { + codec.mCodecType = kGMPVideoCodecVP8; + } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) { + codec.mCodecType = kGMPVideoCodecVP9; + } else { + // Unrecognized mime type + aGMP->Close(); + mInitPromise.Reject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + return; + } + codec.mWidth = mConfig.mImage.width; + codec.mHeight = mConfig.mImage.height; + codec.mUseThreadedDecode = StaticPrefs::media_gmp_decoder_multithreaded(); + codec.mLogLevel = GetGMPLibraryLogLevel(); + + nsresult rv = + aGMP->InitDecode(codec, codecSpecific, this, PR_GetNumberOfProcessors()); + if (NS_FAILED(rv)) { + aGMP->Close(); + mInitPromise.Reject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + return; + } + + mGMP = aGMP; + mHost = aHost; + + // GMP implementations have interpreted the meaning of GMP_BufferLength32 + // differently. The OpenH264 GMP expects GMP_BufferLength32 to behave as + // specified in the GMP API, where each buffer is prefixed by a 32-bit + // host-endian buffer length that includes the size of the buffer length + // field. Other existing GMPs currently expect GMP_BufferLength32 (when + // combined with kGMPVideoCodecH264) to mean "like AVCC but restricted to + // 4-byte NAL lengths" (i.e. buffer lengths are specified in big-endian + // and do not include the length of the buffer length field. + mConvertNALUnitLengths = isOpenH264; + + mInitPromise.Resolve(TrackInfo::kVideoTrack, __func__); +} + +RefPtr<MediaDataDecoder::InitPromise> GMPVideoDecoder::Init() { + MOZ_ASSERT(IsOnGMPThread()); + + mMPS = do_GetService("@mozilla.org/gecko-media-plugin-service;1"); + MOZ_ASSERT(mMPS); + + RefPtr<InitPromise> promise(mInitPromise.Ensure(__func__)); + + nsTArray<nsCString> tags; + InitTags(tags); + UniquePtr<GetGMPVideoDecoderCallback> callback(new GMPInitDoneCallback(this)); + if (NS_FAILED(mMPS->GetGMPVideoDecoder(mCrashHelper, &tags, GetNodeId(), + std::move(callback)))) { + mInitPromise.Reject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + } + + return promise; +} + +RefPtr<MediaDataDecoder::DecodePromise> GMPVideoDecoder::Decode( + MediaRawData* aSample) { + MOZ_ASSERT(IsOnGMPThread()); + + RefPtr<MediaRawData> sample(aSample); + if (!mGMP) { + return DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("mGMP not initialized")), + __func__); + } + + if (mTrackingId) { + MediaInfoFlag flag = MediaInfoFlag::None; + flag |= (aSample->mKeyframe ? MediaInfoFlag::KeyFrame + : MediaInfoFlag::NonKeyFrame); + if (mGMP->GetPluginType() == GMPPluginType::OpenH264) { + flag |= MediaInfoFlag::SoftwareDecoding; + } + if (MP4Decoder::IsH264(mConfig.mMimeType)) { + flag |= MediaInfoFlag::VIDEO_H264; + } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) { + flag |= MediaInfoFlag::VIDEO_VP8; + } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) { + flag |= MediaInfoFlag::VIDEO_VP9; + } + mPerformanceRecorder.Start(aSample->mTime.ToMicroseconds(), + "GMPVideoDecoder"_ns, *mTrackingId, flag); + } + + GMPUniquePtr<GMPVideoEncodedFrame> frame = CreateFrame(sample); + if (!frame) { + return DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_OUT_OF_MEMORY, + RESULT_DETAIL("CreateFrame returned null")), + __func__); + } + + uint64_t frameTimestamp = frame->TimeStamp(); + RefPtr<DecodePromise> p = mDecodePromise.Ensure(__func__); + nsTArray<uint8_t> info; // No codec specific per-frame info to pass. + nsresult rv = mGMP->Decode(std::move(frame), false, info, 0); + if (NS_FAILED(rv)) { + mDecodePromise.Reject(MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, + RESULT_DETAIL("mGMP->Decode:%" PRIx32, + static_cast<uint32_t>(rv))), + __func__); + } + + // If we have multiple outstanding frames, we need to track which offset + // belongs to which frame. During seek, it is possible to get the same frame + // requested twice, if the old frame is still outstanding. We will simply drop + // the extra decoded frame and request more input if the last outstanding. + mSamples.WithEntryHandle(frameTimestamp, [&](auto entryHandle) { + auto sampleData = MakeUnique<SampleMetadata>(sample); + entryHandle.InsertOrUpdate(std::move(sampleData)); + }); + + return p; +} + +RefPtr<MediaDataDecoder::FlushPromise> GMPVideoDecoder::Flush() { + MOZ_ASSERT(IsOnGMPThread()); + + mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + mDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + + RefPtr<FlushPromise> p = mFlushPromise.Ensure(__func__); + if (!mGMP || NS_FAILED(mGMP->Reset())) { + // Abort the flush. + mPerformanceRecorder.Record(std::numeric_limits<int64_t>::max()); + mFlushPromise.Resolve(true, __func__); + } + return p; +} + +RefPtr<MediaDataDecoder::DecodePromise> GMPVideoDecoder::Drain() { + MOZ_ASSERT(IsOnGMPThread()); + + MOZ_ASSERT(mDecodePromise.IsEmpty(), "Must wait for decoding to complete"); + + RefPtr<DecodePromise> p = mDrainPromise.Ensure(__func__); + if (!mGMP || NS_FAILED(mGMP->Drain())) { + mDrainPromise.Resolve(DecodedData(), __func__); + } + + return p; +} + +RefPtr<ShutdownPromise> GMPVideoDecoder::Shutdown() { + mInitPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + mFlushPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + + // Note that this *may* be called from the proxy thread also. + // TODO: If that's the case, then this code is racy. + if (!mGMP) { + return ShutdownPromise::CreateAndResolve(true, __func__); + } + // Note this unblocks flush and drain operations waiting for callbacks. + mGMP->Close(); + mGMP = nullptr; + return ShutdownPromise::CreateAndResolve(true, __func__); +} + +} // namespace mozilla diff --git a/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.h b/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.h new file mode 100644 index 0000000000..1f0f59c685 --- /dev/null +++ b/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.h @@ -0,0 +1,129 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/layers/KnowsCompositor.h" +#if !defined(GMPVideoDecoder_h_) +# define GMPVideoDecoder_h_ + +# include "GMPVideoDecoderProxy.h" +# include "ImageContainer.h" +# include "MediaDataDecoderProxy.h" +# include "MediaInfo.h" +# include "PerformanceRecorder.h" +# include "PlatformDecoderModule.h" +# include "ReorderQueue.h" +# include "mozIGeckoMediaPluginService.h" +# include "nsClassHashtable.h" + +namespace mozilla { + +struct MOZ_STACK_CLASS GMPVideoDecoderParams { + explicit GMPVideoDecoderParams(const CreateDecoderParams& aParams); + + const VideoInfo& mConfig; + layers::ImageContainer* mImageContainer; + GMPCrashHelper* mCrashHelper; + layers::KnowsCompositor* mKnowsCompositor; + const Maybe<TrackingId> mTrackingId; +}; + +DDLoggedTypeDeclNameAndBase(GMPVideoDecoder, MediaDataDecoder); + +class GMPVideoDecoder final : public MediaDataDecoder, + public GMPVideoDecoderCallbackProxy, + public DecoderDoctorLifeLogger<GMPVideoDecoder> { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(GMPVideoDecoder, final); + + explicit GMPVideoDecoder(const GMPVideoDecoderParams& aParams); + + RefPtr<InitPromise> Init() override; + RefPtr<DecodePromise> Decode(MediaRawData* aSample) override; + RefPtr<DecodePromise> Drain() override; + RefPtr<FlushPromise> Flush() override; + RefPtr<ShutdownPromise> Shutdown() override; + nsCString GetDescriptionName() const override { + return "gmp video decoder"_ns; + } + nsCString GetCodecName() const override; + ConversionRequired NeedsConversion() const override { + return mConvertToAnnexB ? ConversionRequired::kNeedAnnexB + : ConversionRequired::kNeedAVCC; + } + bool CanDecodeBatch() const override { return mCanDecodeBatch; } + + // GMPVideoDecoderCallbackProxy + // All those methods are called on the GMP thread. + void Decoded(GMPVideoi420Frame* aDecodedFrame) override; + void ReceivedDecodedReferenceFrame(const uint64_t aPictureId) override; + void ReceivedDecodedFrame(const uint64_t aPictureId) override; + void InputDataExhausted() override; + void DrainComplete() override; + void ResetComplete() override; + void Error(GMPErr aErr) override; + void Terminated() override; + + protected: + virtual void InitTags(nsTArray<nsCString>& aTags); + virtual nsCString GetNodeId(); + virtual GMPUniquePtr<GMPVideoEncodedFrame> CreateFrame(MediaRawData* aSample); + virtual const VideoInfo& GetConfig() const; + void ProcessReorderQueue(MozPromiseHolder<DecodePromise>& aPromise, + const char* aMethodName); + + private: + ~GMPVideoDecoder() = default; + + class GMPInitDoneCallback : public GetGMPVideoDecoderCallback { + public: + explicit GMPInitDoneCallback(GMPVideoDecoder* aDecoder) + : mDecoder(aDecoder) {} + + void Done(GMPVideoDecoderProxy* aGMP, GMPVideoHost* aHost) override { + mDecoder->GMPInitDone(aGMP, aHost); + } + + private: + RefPtr<GMPVideoDecoder> mDecoder; + }; + void GMPInitDone(GMPVideoDecoderProxy* aGMP, GMPVideoHost* aHost); + + const VideoInfo mConfig; + nsCOMPtr<mozIGeckoMediaPluginService> mMPS; + GMPVideoDecoderProxy* mGMP; + GMPVideoHost* mHost; + bool mConvertNALUnitLengths; + MozPromiseHolder<InitPromise> mInitPromise; + RefPtr<GMPCrashHelper> mCrashHelper; + + struct SampleMetadata { + explicit SampleMetadata(MediaRawData* aSample) + : mOffset(aSample->mOffset), mKeyframe(aSample->mKeyframe) {} + int64_t mOffset; + bool mKeyframe; + }; + + nsClassHashtable<nsUint64HashKey, SampleMetadata> mSamples; + RefPtr<layers::ImageContainer> mImageContainer; + RefPtr<layers::KnowsCompositor> mKnowsCompositor; + PerformanceRecorderMulti<DecodeStage> mPerformanceRecorder; + const Maybe<TrackingId> mTrackingId; + + uint32_t mMaxRefFrames = 0; + ReorderQueue mReorderQueue; + DecodedData mUnorderedData; + + MozPromiseHolder<DecodePromise> mDecodePromise; + MozPromiseHolder<DecodePromise> mDrainPromise; + MozPromiseHolder<FlushPromise> mFlushPromise; + bool mConvertToAnnexB = false; + bool mCanDecodeBatch = false; + bool mReorderFrames = true; +}; + +} // namespace mozilla + +#endif // GMPVideoDecoder_h_ diff --git a/dom/media/platforms/agnostic/gmp/moz.build b/dom/media/platforms/agnostic/gmp/moz.build new file mode 100644 index 0000000000..8e49f4a0e8 --- /dev/null +++ b/dom/media/platforms/agnostic/gmp/moz.build @@ -0,0 +1,24 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +LOCAL_INCLUDES += [ + "/dom/media/gmp", # for GMPLog.h, +] + +EXPORTS += [ + "GMPDecoderModule.h", + "GMPVideoDecoder.h", +] + +UNIFIED_SOURCES += [ + "GMPDecoderModule.cpp", + "GMPVideoDecoder.cpp", +] + +# GMPVideoEncodedFrameImpl.h needs IPC +include("/ipc/chromium/chromium-config.mozbuild") + +FINAL_LIBRARY = "xul" |