diff options
Diffstat (limited to 'dom/media/platforms/agnostic/bytestreams')
| -rw-r--r-- | dom/media/platforms/agnostic/bytestreams/Adts.cpp | 94 | ||||
| -rw-r--r-- | dom/media/platforms/agnostic/bytestreams/Adts.h | 22 | ||||
| -rw-r--r-- | dom/media/platforms/agnostic/bytestreams/AnnexB.cpp | 364 | ||||
| -rw-r--r-- | dom/media/platforms/agnostic/bytestreams/AnnexB.h | 66 | ||||
| -rw-r--r-- | dom/media/platforms/agnostic/bytestreams/H264.cpp | 1356 | ||||
| -rw-r--r-- | dom/media/platforms/agnostic/bytestreams/H264.h | 525 | ||||
| -rw-r--r-- | dom/media/platforms/agnostic/bytestreams/gtest/TestAnnexB.cpp | 144 | ||||
| -rw-r--r-- | dom/media/platforms/agnostic/bytestreams/gtest/moz.build | 11 | ||||
| -rw-r--r-- | dom/media/platforms/agnostic/bytestreams/moz.build | 35 |
9 files changed, 2617 insertions, 0 deletions
diff --git a/dom/media/platforms/agnostic/bytestreams/Adts.cpp b/dom/media/platforms/agnostic/bytestreams/Adts.cpp new file mode 100644 index 0000000000..5f31904d9c --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/Adts.cpp @@ -0,0 +1,94 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "Adts.h" +#include "MediaData.h" +#include "mozilla/Array.h" +#include "mozilla/ArrayUtils.h" + +namespace mozilla { + +static const int kADTSHeaderSize = 7; + +int8_t Adts::GetFrequencyIndex(uint32_t aSamplesPerSecond) { + static const uint32_t freq_lookup[] = {96000, 88200, 64000, 48000, 44100, + 32000, 24000, 22050, 16000, 12000, + 11025, 8000, 7350, 0}; + + int8_t i = 0; + while (freq_lookup[i] && aSamplesPerSecond < freq_lookup[i]) { + i++; + } + + if (!freq_lookup[i]) { + return -1; + } + + return i; +} + +bool Adts::ConvertSample(uint16_t aChannelCount, int8_t aFrequencyIndex, + int8_t aProfile, MediaRawData* aSample) { + size_t newSize = aSample->Size() + kADTSHeaderSize; + + // ADTS header uses 13 bits for packet size. + if (newSize >= (1 << 13) || aChannelCount > 15 || aFrequencyIndex < 0 || + aProfile < 1 || aProfile > 4) { + return false; + } + + Array<uint8_t, kADTSHeaderSize> header; + header[0] = 0xff; + header[1] = 0xf1; + header[2] = + ((aProfile - 1) << 6) + (aFrequencyIndex << 2) + (aChannelCount >> 2); + header[3] = ((aChannelCount & 0x3) << 6) + (newSize >> 11); + header[4] = (newSize & 0x7ff) >> 3; + header[5] = ((newSize & 7) << 5) + 0x1f; + header[6] = 0xfc; + + UniquePtr<MediaRawDataWriter> writer(aSample->CreateWriter()); + if (!writer->Prepend(&header[0], ArrayLength(header))) { + return false; + } + + if (aSample->mCrypto.IsEncrypted()) { + if (aSample->mCrypto.mPlainSizes.Length() == 0) { + writer->mCrypto.mPlainSizes.AppendElement(kADTSHeaderSize); + writer->mCrypto.mEncryptedSizes.AppendElement(aSample->Size() - + kADTSHeaderSize); + } else { + writer->mCrypto.mPlainSizes[0] += kADTSHeaderSize; + } + } + + return true; +} + +bool Adts::RevertSample(MediaRawData* aSample) { + if (aSample->Size() < kADTSHeaderSize) { + return false; + } + + { + const uint8_t* header = aSample->Data(); + if (header[0] != 0xff || header[1] != 0xf1 || header[6] != 0xfc) { + // Not ADTS. + return false; + } + } + + UniquePtr<MediaRawDataWriter> writer(aSample->CreateWriter()); + writer->PopFront(kADTSHeaderSize); + + if (aSample->mCrypto.IsEncrypted()) { + if (aSample->mCrypto.mPlainSizes.Length() > 0 && + writer->mCrypto.mPlainSizes[0] >= kADTSHeaderSize) { + writer->mCrypto.mPlainSizes[0] -= kADTSHeaderSize; + } + } + + return true; +} +} // namespace mozilla diff --git a/dom/media/platforms/agnostic/bytestreams/Adts.h b/dom/media/platforms/agnostic/bytestreams/Adts.h new file mode 100644 index 0000000000..c2b6b558b6 --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/Adts.h @@ -0,0 +1,22 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef ADTS_H_ +#define ADTS_H_ + +#include <stdint.h> + +namespace mozilla { +class MediaRawData; + +class Adts { + public: + static int8_t GetFrequencyIndex(uint32_t aSamplesPerSecond); + static bool ConvertSample(uint16_t aChannelCount, int8_t aFrequencyIndex, + int8_t aProfile, mozilla::MediaRawData* aSample); + static bool RevertSample(MediaRawData* aSample); +}; +} // namespace mozilla + +#endif diff --git a/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp b/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp new file mode 100644 index 0000000000..07e9c2dde8 --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/AnnexB.cpp @@ -0,0 +1,364 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/ArrayUtils.h" +#include "mozilla/EndianUtils.h" +#include "mozilla/ResultExtensions.h" +#include "mozilla/Unused.h" +#include "AnnexB.h" +#include "BufferReader.h" +#include "ByteWriter.h" +#include "MediaData.h" + +namespace mozilla { + +static const uint8_t kAnnexBDelimiter[] = {0, 0, 0, 1}; + +Result<Ok, nsresult> AnnexB::ConvertSampleToAnnexB( + mozilla::MediaRawData* aSample, bool aAddSPS) { + MOZ_ASSERT(aSample); + + if (!IsAVCC(aSample)) { + return Ok(); + } + MOZ_ASSERT(aSample->Data()); + + MOZ_TRY(ConvertSampleTo4BytesAVCC(aSample)); + + if (aSample->Size() < 4) { + // Nothing to do, it's corrupted anyway. + return Ok(); + } + + BufferReader reader(aSample->Data(), aSample->Size()); + + nsTArray<uint8_t> tmp; + ByteWriter<BigEndian> writer(tmp); + + while (reader.Remaining() >= 4) { + uint32_t nalLen; + MOZ_TRY_VAR(nalLen, reader.ReadU32()); + const uint8_t* p = reader.Read(nalLen); + + if (!writer.Write(kAnnexBDelimiter, ArrayLength(kAnnexBDelimiter))) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + if (!p) { + break; + } + if (!writer.Write(p, nalLen)) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + } + + UniquePtr<MediaRawDataWriter> samplewriter(aSample->CreateWriter()); + + if (!samplewriter->Replace(tmp.Elements(), tmp.Length())) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + + // Prepend the Annex B NAL with SPS and PPS tables to keyframes. + if (aAddSPS && aSample->mKeyframe) { + RefPtr<MediaByteBuffer> annexB = + ConvertExtraDataToAnnexB(aSample->mExtraData); + if (!samplewriter->Prepend(annexB->Elements(), annexB->Length())) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + + // Prepending the NAL with SPS/PPS will mess up the encryption subsample + // offsets. So we need to account for the extra bytes by increasing + // the length of the first clear data subsample. Otherwise decryption + // will fail. + if (aSample->mCrypto.IsEncrypted()) { + if (aSample->mCrypto.mPlainSizes.Length() == 0) { + CheckedUint32 plainSize{annexB->Length()}; + CheckedUint32 encryptedSize{samplewriter->Size()}; + encryptedSize -= annexB->Length(); + samplewriter->mCrypto.mPlainSizes.AppendElement(plainSize.value()); + samplewriter->mCrypto.mEncryptedSizes.AppendElement( + encryptedSize.value()); + } else { + CheckedUint32 newSize{samplewriter->mCrypto.mPlainSizes[0]}; + newSize += annexB->Length(); + samplewriter->mCrypto.mPlainSizes[0] = newSize.value(); + } + } + } + + return Ok(); +} + +already_AddRefed<mozilla::MediaByteBuffer> AnnexB::ConvertExtraDataToAnnexB( + const mozilla::MediaByteBuffer* aExtraData) { + // AVCC 6 byte header looks like: + // +------+------+------+------+------+------+------+------+ + // [0] | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | + // +------+------+------+------+------+------+------+------+ + // [1] | profile | + // +------+------+------+------+------+------+------+------+ + // [2] | compatiblity | + // +------+------+------+------+------+------+------+------+ + // [3] | level | + // +------+------+------+------+------+------+------+------+ + // [4] | unused | nalLenSiz-1 | + // +------+------+------+------+------+------+------+------+ + // [5] | unused | numSps | + // +------+------+------+------+------+------+------+------+ + + RefPtr<mozilla::MediaByteBuffer> annexB = new mozilla::MediaByteBuffer; + + BufferReader reader(*aExtraData); + const uint8_t* ptr = reader.Read(5); + if (ptr && ptr[0] == 1) { + // Append SPS then PPS + Unused << reader.ReadU8().map( + [&](uint8_t x) { return ConvertSPSOrPPS(reader, x & 31, annexB); }); + Unused << reader.ReadU8().map( + [&](uint8_t x) { return ConvertSPSOrPPS(reader, x, annexB); }); + // MP4Box adds extra bytes that we ignore. I don't know what they do. + } + + return annexB.forget(); +} + +Result<mozilla::Ok, nsresult> AnnexB::ConvertSPSOrPPS( + BufferReader& aReader, uint8_t aCount, mozilla::MediaByteBuffer* aAnnexB) { + for (int i = 0; i < aCount; i++) { + uint16_t length; + MOZ_TRY_VAR(length, aReader.ReadU16()); + + const uint8_t* ptr = aReader.Read(length); + if (!ptr) { + return Err(NS_ERROR_FAILURE); + } + aAnnexB->AppendElements(kAnnexBDelimiter, ArrayLength(kAnnexBDelimiter)); + aAnnexB->AppendElements(ptr, length); + } + return Ok(); +} + +static Result<Ok, nsresult> FindStartCodeInternal(BufferReader& aBr) { + size_t offset = aBr.Offset(); + + for (uint32_t i = 0; i < aBr.Align() && aBr.Remaining() >= 3; i++) { + auto res = aBr.PeekU24(); + if (res.isOk() && (res.unwrap() == 0x000001)) { + return Ok(); + } + mozilla::Unused << aBr.Read(1); + } + + while (aBr.Remaining() >= 6) { + uint32_t x32; + MOZ_TRY_VAR(x32, aBr.PeekU32()); + if ((x32 - 0x01010101) & (~x32) & 0x80808080) { + if ((x32 >> 8) == 0x000001) { + return Ok(); + } + if (x32 == 0x000001) { + mozilla::Unused << aBr.Read(1); + return Ok(); + } + if ((x32 & 0xff) == 0) { + const uint8_t* p = aBr.Peek(1); + if ((x32 & 0xff00) == 0 && p[4] == 1) { + mozilla::Unused << aBr.Read(2); + return Ok(); + } + if (p[4] == 0 && p[5] == 1) { + mozilla::Unused << aBr.Read(3); + return Ok(); + } + } + } + mozilla::Unused << aBr.Read(4); + } + + while (aBr.Remaining() >= 3) { + uint32_t data; + MOZ_TRY_VAR(data, aBr.PeekU24()); + if (data == 0x000001) { + return Ok(); + } + mozilla::Unused << aBr.Read(1); + } + + // No start code were found; Go back to the beginning. + mozilla::Unused << aBr.Seek(offset); + return Err(NS_ERROR_FAILURE); +} + +static Result<Ok, nsresult> FindStartCode(BufferReader& aBr, + size_t& aStartSize) { + if (FindStartCodeInternal(aBr).isErr()) { + aStartSize = 0; + return Err(NS_ERROR_FAILURE); + } + + aStartSize = 3; + if (aBr.Offset()) { + // Check if it's 4-bytes start code + aBr.Rewind(1); + uint8_t data; + MOZ_TRY_VAR(data, aBr.ReadU8()); + if (data == 0) { + aStartSize = 4; + } + } + mozilla::Unused << aBr.Read(3); + return Ok(); +} + +/* static */ +void AnnexB::ParseNALEntries(const Span<const uint8_t>& aSpan, + nsTArray<AnnexB::NALEntry>& aEntries) { + BufferReader reader(aSpan.data(), aSpan.Length()); + size_t startSize; + auto rv = FindStartCode(reader, startSize); + size_t startOffset = reader.Offset(); + if (rv.isOk()) { + while (FindStartCode(reader, startSize).isOk()) { + int64_t offset = reader.Offset(); + int64_t sizeNAL = offset - startOffset - startSize; + aEntries.AppendElement(AnnexB::NALEntry(startOffset, sizeNAL)); + reader.Seek(startOffset); + reader.Read(sizeNAL + startSize); + startOffset = offset; + } + } + int64_t sizeNAL = reader.Remaining(); + if (sizeNAL) { + aEntries.AppendElement(AnnexB::NALEntry(startOffset, sizeNAL)); + } +} + +static Result<mozilla::Ok, nsresult> ParseNALUnits(ByteWriter<BigEndian>& aBw, + BufferReader& aBr) { + size_t startSize; + + auto rv = FindStartCode(aBr, startSize); + if (rv.isOk()) { + size_t startOffset = aBr.Offset(); + while (FindStartCode(aBr, startSize).isOk()) { + size_t offset = aBr.Offset(); + size_t sizeNAL = offset - startOffset - startSize; + aBr.Seek(startOffset); + if (!aBw.WriteU32(sizeNAL) || !aBw.Write(aBr.Read(sizeNAL), sizeNAL)) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + aBr.Read(startSize); + startOffset = offset; + } + } + size_t sizeNAL = aBr.Remaining(); + if (sizeNAL) { + if (!aBw.WriteU32(sizeNAL) || !aBw.Write(aBr.Read(sizeNAL), sizeNAL)) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + } + return Ok(); +} + +bool AnnexB::ConvertSampleToAVCC(mozilla::MediaRawData* aSample, + const RefPtr<MediaByteBuffer>& aAVCCHeader) { + if (IsAVCC(aSample)) { + return ConvertSampleTo4BytesAVCC(aSample).isOk(); + } + if (!IsAnnexB(aSample)) { + // Not AnnexB, nothing to convert. + return true; + } + + nsTArray<uint8_t> nalu; + ByteWriter<BigEndian> writer(nalu); + BufferReader reader(aSample->Data(), aSample->Size()); + + if (ParseNALUnits(writer, reader).isErr()) { + return false; + } + UniquePtr<MediaRawDataWriter> samplewriter(aSample->CreateWriter()); + if (!samplewriter->Replace(nalu.Elements(), nalu.Length())) { + return false; + } + + if (aAVCCHeader) { + aSample->mExtraData = aAVCCHeader; + return true; + } + + // Create the AVCC header. + auto extradata = MakeRefPtr<mozilla::MediaByteBuffer>(); + static const uint8_t kFakeExtraData[] = { + 1 /* version */, + 0x64 /* profile (High) */, + 0 /* profile compat (0) */, + 40 /* level (40) */, + 0xfc | 3 /* nal size - 1 */, + 0xe0 /* num SPS (0) */, + 0 /* num PPS (0) */ + }; + // XXX(Bug 1631371) Check if this should use a fallible operation as it + // pretended earlier. + extradata->AppendElements(kFakeExtraData, ArrayLength(kFakeExtraData)); + aSample->mExtraData = std::move(extradata); + return true; +} + +Result<mozilla::Ok, nsresult> AnnexB::ConvertSampleTo4BytesAVCC( + mozilla::MediaRawData* aSample) { + MOZ_ASSERT(IsAVCC(aSample)); + + int nalLenSize = ((*aSample->mExtraData)[4] & 3) + 1; + + if (nalLenSize == 4) { + return Ok(); + } + nsTArray<uint8_t> dest; + ByteWriter<BigEndian> writer(dest); + BufferReader reader(aSample->Data(), aSample->Size()); + while (reader.Remaining() > nalLenSize) { + uint32_t nalLen; + switch (nalLenSize) { + case 1: + MOZ_TRY_VAR(nalLen, reader.ReadU8()); + break; + case 2: + MOZ_TRY_VAR(nalLen, reader.ReadU16()); + break; + case 3: + MOZ_TRY_VAR(nalLen, reader.ReadU24()); + break; + } + + MOZ_ASSERT(nalLenSize != 4); + + const uint8_t* p = reader.Read(nalLen); + if (!p) { + return Ok(); + } + if (!writer.WriteU32(nalLen) || !writer.Write(p, nalLen)) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + } + UniquePtr<MediaRawDataWriter> samplewriter(aSample->CreateWriter()); + if (!samplewriter->Replace(dest.Elements(), dest.Length())) { + return Err(NS_ERROR_OUT_OF_MEMORY); + } + return Ok(); +} + +bool AnnexB::IsAVCC(const mozilla::MediaRawData* aSample) { + return aSample->Size() >= 3 && aSample->mExtraData && + aSample->mExtraData->Length() >= 7 && (*aSample->mExtraData)[0] == 1; +} + +bool AnnexB::IsAnnexB(const mozilla::MediaRawData* aSample) { + if (aSample->Size() < 4) { + return false; + } + uint32_t header = mozilla::BigEndian::readUint32(aSample->Data()); + return header == 0x00000001 || (header >> 8) == 0x000001; +} + +} // namespace mozilla diff --git a/dom/media/platforms/agnostic/bytestreams/AnnexB.h b/dom/media/platforms/agnostic/bytestreams/AnnexB.h new file mode 100644 index 0000000000..dbb8a7c3e1 --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/AnnexB.h @@ -0,0 +1,66 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_ANNEX_B_H_ +#define DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_ANNEX_B_H_ + +#include "ErrorList.h" +#include "mozilla/RefPtr.h" +#include "mozilla/Result.h" + +template <class> +class nsTArray; + +namespace mozilla { +class BufferReader; +class MediaRawData; +class MediaByteBuffer; + +class AnnexB { + public: + struct NALEntry { + NALEntry(int64_t aOffset, int64_t aSize) : mOffset(aOffset), mSize(aSize) { + MOZ_ASSERT(mOffset >= 0); + MOZ_ASSERT(mSize >= 0); + } + // They should be non-negative, so we use int64_t to assert their value when + // assigning value to them. + int64_t mOffset; + int64_t mSize; + }; + // All conversions assume size of NAL length field is 4 bytes. + // Convert a sample from AVCC format to Annex B. + static mozilla::Result<mozilla::Ok, nsresult> ConvertSampleToAnnexB( + mozilla::MediaRawData* aSample, bool aAddSPS = true); + // Convert a sample from Annex B to AVCC. + // an AVCC extradata must not be set. + static bool ConvertSampleToAVCC( + mozilla::MediaRawData* aSample, + const RefPtr<mozilla::MediaByteBuffer>& aAVCCHeader = nullptr); + static mozilla::Result<mozilla::Ok, nsresult> ConvertSampleTo4BytesAVCC( + mozilla::MediaRawData* aSample); + + // Parse an AVCC extradata and construct the Annex B sample header. + static already_AddRefed<mozilla::MediaByteBuffer> ConvertExtraDataToAnnexB( + const mozilla::MediaByteBuffer* aExtraData); + // Returns true if format is AVCC and sample has valid extradata. + static bool IsAVCC(const mozilla::MediaRawData* aSample); + // Returns true if format is AnnexB. + static bool IsAnnexB(const mozilla::MediaRawData* aSample); + + // Parse NAL entries from the bytes stream to know the offset and the size of + // each NAL in the bytes stream. + static void ParseNALEntries(const Span<const uint8_t>& aSpan, + nsTArray<AnnexB::NALEntry>& aEntries); + + private: + // AVCC box parser helper. + static mozilla::Result<mozilla::Ok, nsresult> ConvertSPSOrPPS( + mozilla::BufferReader& aReader, uint8_t aCount, + mozilla::MediaByteBuffer* aAnnexB); +}; + +} // namespace mozilla + +#endif // DOM_MEDIA_PLATFORMS_AGNOSTIC_BYTESTREAMS_ANNEX_B_H_ diff --git a/dom/media/platforms/agnostic/bytestreams/H264.cpp b/dom/media/platforms/agnostic/bytestreams/H264.cpp new file mode 100644 index 0000000000..4dc33e1763 --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/H264.cpp @@ -0,0 +1,1356 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "H264.h" +#include <cmath> +#include <limits> +#include "AnnexB.h" +#include "BitReader.h" +#include "BitWriter.h" +#include "BufferReader.h" +#include "ByteWriter.h" +#include "mozilla/ArrayUtils.h" +#include "mozilla/PodOperations.h" +#include "mozilla/ResultExtensions.h" + +#define READSE(var, min, max) \ + { \ + int32_t val = br.ReadSE(); \ + if (val < min || val > max) { \ + return false; \ + } \ + aDest.var = val; \ + } + +#define READUE(var, max) \ + { \ + uint32_t uval = br.ReadUE(); \ + if (uval > max) { \ + return false; \ + } \ + aDest.var = uval; \ + } + +namespace mozilla { + +// Default scaling lists (per spec). +// ITU H264: +// Table 7-2 – Assignment of mnemonic names to scaling list indices and +// specification of fall-back rule +static const uint8_t Default_4x4_Intra[16] = {6, 13, 13, 20, 20, 20, 28, 28, + 28, 28, 32, 32, 32, 37, 37, 42}; + +static const uint8_t Default_4x4_Inter[16] = {10, 14, 14, 20, 20, 20, 24, 24, + 24, 24, 27, 27, 27, 30, 30, 34}; + +static const uint8_t Default_8x8_Intra[64] = { + 6, 10, 10, 13, 11, 13, 16, 16, 16, 16, 18, 18, 18, 18, 18, 23, + 23, 23, 23, 23, 23, 25, 25, 25, 25, 25, 25, 25, 27, 27, 27, 27, + 27, 27, 27, 27, 29, 29, 29, 29, 29, 29, 29, 31, 31, 31, 31, 31, + 31, 33, 33, 33, 33, 33, 36, 36, 36, 36, 38, 38, 38, 40, 40, 42}; + +static const uint8_t Default_8x8_Inter[64] = { + 9, 13, 13, 15, 13, 15, 17, 17, 17, 17, 19, 19, 19, 19, 19, 21, + 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 24, 24, 24, 24, + 24, 24, 24, 24, 25, 25, 25, 25, 25, 25, 25, 27, 27, 27, 27, 27, + 27, 28, 28, 28, 28, 28, 30, 30, 30, 30, 32, 32, 32, 33, 33, 35}; + +namespace detail { +static void scaling_list(BitReader& aBr, uint8_t* aScalingList, + int aSizeOfScalingList, const uint8_t* aDefaultList, + const uint8_t* aFallbackList) { + int32_t lastScale = 8; + int32_t nextScale = 8; + int32_t deltaScale; + + // (pic|seq)_scaling_list_present_flag[i] + if (!aBr.ReadBit()) { + if (aFallbackList) { + memcpy(aScalingList, aFallbackList, aSizeOfScalingList); + } + return; + } + + for (int i = 0; i < aSizeOfScalingList; i++) { + if (nextScale != 0) { + deltaScale = aBr.ReadSE(); + nextScale = (lastScale + deltaScale + 256) % 256; + if (!i && !nextScale) { + memcpy(aScalingList, aDefaultList, aSizeOfScalingList); + return; + } + } + aScalingList[i] = (nextScale == 0) ? lastScale : nextScale; + lastScale = aScalingList[i]; + } +} +} // namespace detail. + +template <size_t N> +static void scaling_list(BitReader& aBr, uint8_t (&aScalingList)[N], + const uint8_t (&aDefaultList)[N], + const uint8_t (&aFallbackList)[N]) { + detail::scaling_list(aBr, aScalingList, N, aDefaultList, aFallbackList); +} + +template <size_t N> +static void scaling_list(BitReader& aBr, uint8_t (&aScalingList)[N], + const uint8_t (&aDefaultList)[N]) { + detail::scaling_list(aBr, aScalingList, N, aDefaultList, nullptr); +} + +SPSData::SPSData() { + PodZero(this); + // Default values when they aren't defined as per ITU-T H.264 (2014/02). + chroma_format_idc = 1; + video_format = 5; + colour_primaries = 2; + transfer_characteristics = 2; + sample_ratio = 1.0; + memset(scaling_matrix4x4, 16, sizeof(scaling_matrix4x4)); + memset(scaling_matrix8x8, 16, sizeof(scaling_matrix8x8)); +} + +bool SPSData::operator==(const SPSData& aOther) const { + return this->valid && aOther.valid && !memcmp(this, &aOther, sizeof(SPSData)); +} + +bool SPSData::operator!=(const SPSData& aOther) const { + return !(operator==(aOther)); +} + +// Described in ISO 23001-8:2016 +// Table 2 +enum class PrimaryID : uint8_t { + INVALID = 0, + BT709 = 1, + UNSPECIFIED = 2, + BT470M = 4, + BT470BG = 5, + SMPTE170M = 6, + SMPTE240M = 7, + FILM = 8, + BT2020 = 9, + SMPTEST428_1 = 10, + SMPTEST431_2 = 11, + SMPTEST432_1 = 12, + EBU_3213_E = 22 +}; + +// Table 3 +enum class TransferID : uint8_t { + INVALID = 0, + BT709 = 1, + UNSPECIFIED = 2, + GAMMA22 = 4, + GAMMA28 = 5, + SMPTE170M = 6, + SMPTE240M = 7, + LINEAR = 8, + LOG = 9, + LOG_SQRT = 10, + IEC61966_2_4 = 11, + BT1361_ECG = 12, + IEC61966_2_1 = 13, + BT2020_10 = 14, + BT2020_12 = 15, + SMPTEST2084 = 16, + SMPTEST428_1 = 17, + + // Not yet standardized + ARIB_STD_B67 = 18, // AKA hybrid-log gamma, HLG. +}; + +// Table 4 +enum class MatrixID : uint8_t { + RGB = 0, + BT709 = 1, + UNSPECIFIED = 2, + FCC = 4, + BT470BG = 5, + SMPTE170M = 6, + SMPTE240M = 7, + YCOCG = 8, + BT2020_NCL = 9, + BT2020_CL = 10, + YDZDX = 11, + INVALID = 255, +}; + +static PrimaryID GetPrimaryID(int aPrimary) { + if (aPrimary < 1 || aPrimary > 22 || aPrimary == 3) { + return PrimaryID::INVALID; + } + if (aPrimary > 12 && aPrimary < 22) { + return PrimaryID::INVALID; + } + return static_cast<PrimaryID>(aPrimary); +} + +static TransferID GetTransferID(int aTransfer) { + if (aTransfer < 1 || aTransfer > 18 || aTransfer == 3) { + return TransferID::INVALID; + } + return static_cast<TransferID>(aTransfer); +} + +static MatrixID GetMatrixID(int aMatrix) { + if (aMatrix < 0 || aMatrix > 11 || aMatrix == 3) { + return MatrixID::INVALID; + } + return static_cast<MatrixID>(aMatrix); +} + +gfx::YUVColorSpace SPSData::ColorSpace() const { + // Bitfield, note that guesses with higher values take precedence over + // guesses with lower values. + enum Guess { + GUESS_BT601 = 1 << 0, + GUESS_BT709 = 1 << 1, + GUESS_BT2020 = 1 << 2, + }; + + uint32_t guess = 0; + + switch (GetPrimaryID(colour_primaries)) { + case PrimaryID::BT709: + guess |= GUESS_BT709; + break; + case PrimaryID::BT470M: + case PrimaryID::BT470BG: + case PrimaryID::SMPTE170M: + case PrimaryID::SMPTE240M: + guess |= GUESS_BT601; + break; + case PrimaryID::BT2020: + guess |= GUESS_BT2020; + break; + case PrimaryID::FILM: + case PrimaryID::SMPTEST428_1: + case PrimaryID::SMPTEST431_2: + case PrimaryID::SMPTEST432_1: + case PrimaryID::EBU_3213_E: + case PrimaryID::INVALID: + case PrimaryID::UNSPECIFIED: + break; + } + + switch (GetTransferID(transfer_characteristics)) { + case TransferID::BT709: + guess |= GUESS_BT709; + break; + case TransferID::GAMMA22: + case TransferID::GAMMA28: + case TransferID::SMPTE170M: + case TransferID::SMPTE240M: + guess |= GUESS_BT601; + break; + case TransferID::BT2020_10: + case TransferID::BT2020_12: + guess |= GUESS_BT2020; + break; + case TransferID::LINEAR: + case TransferID::LOG: + case TransferID::LOG_SQRT: + case TransferID::IEC61966_2_4: + case TransferID::BT1361_ECG: + case TransferID::IEC61966_2_1: + case TransferID::SMPTEST2084: + case TransferID::SMPTEST428_1: + case TransferID::ARIB_STD_B67: + case TransferID::INVALID: + case TransferID::UNSPECIFIED: + break; + } + + switch (GetMatrixID(matrix_coefficients)) { + case MatrixID::BT709: + guess |= GUESS_BT709; + break; + case MatrixID::BT470BG: + case MatrixID::SMPTE170M: + case MatrixID::SMPTE240M: + guess |= GUESS_BT601; + break; + case MatrixID::BT2020_NCL: + case MatrixID::BT2020_CL: + guess |= GUESS_BT2020; + break; + case MatrixID::RGB: + case MatrixID::FCC: + case MatrixID::YCOCG: + case MatrixID::YDZDX: + case MatrixID::INVALID: + case MatrixID::UNSPECIFIED: + break; + } + + // Removes lowest bit until only a single bit remains. + while (guess & (guess - 1)) { + guess &= guess - 1; + } + if (!guess) { + // A better default to BT601 which should die a slow death. + guess = GUESS_BT709; + } + + switch (guess) { + case GUESS_BT601: + return gfx::YUVColorSpace::BT601; + case GUESS_BT709: + return gfx::YUVColorSpace::BT709; + case GUESS_BT2020: + return gfx::YUVColorSpace::BT2020; + default: + MOZ_CRASH("not possible to get here but makes compiler happy"); + } +} + +gfx::ColorDepth SPSData::ColorDepth() const { + if (bit_depth_luma_minus8 != 0 && bit_depth_luma_minus8 != 2 && + bit_depth_luma_minus8 != 4) { + // We don't know what that is, just assume 8 bits to prevent decoding + // regressions if we ever encounter those. + return gfx::ColorDepth::COLOR_8; + } + return gfx::ColorDepthForBitDepth(bit_depth_luma_minus8 + 8); +} + +// SPSNAL and SPSNALIterator do not own their data. +class SPSNAL { + public: + SPSNAL(const uint8_t* aPtr, size_t aLength) { + MOZ_ASSERT(aPtr); + + if (aLength == 0 || (*aPtr & 0x1f) != H264_NAL_SPS) { + return; + } + mDecodedNAL = H264::DecodeNALUnit(aPtr, aLength); + if (mDecodedNAL) { + mLength = BitReader::GetBitLength(mDecodedNAL); + } + } + + SPSNAL() = default; + + bool IsValid() const { return mDecodedNAL; } + + bool operator==(const SPSNAL& aOther) const { + if (!mDecodedNAL || !aOther.mDecodedNAL) { + return false; + } + + SPSData decodedSPS1; + SPSData decodedSPS2; + if (!GetSPSData(decodedSPS1) || !aOther.GetSPSData(decodedSPS2)) { + // Couldn't decode one SPS, perform a binary comparison + if (mLength != aOther.mLength) { + return false; + } + MOZ_ASSERT(mLength / 8 <= mDecodedNAL->Length()); + + if (memcmp(mDecodedNAL->Elements(), aOther.mDecodedNAL->Elements(), + mLength / 8)) { + return false; + } + + uint32_t remaining = mLength - (mLength & ~7); + + BitReader b1(mDecodedNAL->Elements() + mLength / 8, remaining); + BitReader b2(aOther.mDecodedNAL->Elements() + mLength / 8, remaining); + for (uint32_t i = 0; i < remaining; i++) { + if (b1.ReadBit() != b2.ReadBit()) { + return false; + } + } + return true; + } + + return decodedSPS1 == decodedSPS2; + } + + bool operator!=(const SPSNAL& aOther) const { return !(operator==(aOther)); } + + bool GetSPSData(SPSData& aDest) const { + return H264::DecodeSPS(mDecodedNAL, aDest); + } + + private: + RefPtr<mozilla::MediaByteBuffer> mDecodedNAL; + uint32_t mLength = 0; +}; + +class SPSNALIterator { + public: + explicit SPSNALIterator(const mozilla::MediaByteBuffer* aExtraData) + : mExtraDataPtr(aExtraData->Elements()), mReader(aExtraData) { + if (!mReader.Read(5)) { + return; + } + + auto res = mReader.ReadU8(); + mNumSPS = res.isOk() ? res.unwrap() & 0x1f : 0; + if (mNumSPS == 0) { + return; + } + mValid = true; + } + + SPSNALIterator& operator++() { + if (mEOS || !mValid) { + return *this; + } + if (--mNumSPS == 0) { + mEOS = true; + } + auto res = mReader.ReadU16(); + uint16_t length = res.isOk() ? res.unwrap() : 0; + if (length == 0 || !mReader.Read(length)) { + mEOS = true; + } + return *this; + } + + explicit operator bool() const { return mValid && !mEOS; } + + SPSNAL operator*() const { + MOZ_ASSERT(bool(*this)); + BufferReader reader(mExtraDataPtr + mReader.Offset(), mReader.Remaining()); + + auto res = reader.ReadU16(); + uint16_t length = res.isOk() ? res.unwrap() : 0; + const uint8_t* ptr = reader.Read(length); + if (!ptr || !length) { + return SPSNAL(); + } + return SPSNAL(ptr, length); + } + + private: + const uint8_t* mExtraDataPtr; + BufferReader mReader; + bool mValid = false; + bool mEOS = false; + uint8_t mNumSPS = 0; +}; + +/* static */ already_AddRefed<mozilla::MediaByteBuffer> H264::DecodeNALUnit( + const uint8_t* aNAL, size_t aLength) { + MOZ_ASSERT(aNAL); + + if (aLength < 4) { + return nullptr; + } + + RefPtr<mozilla::MediaByteBuffer> rbsp = new mozilla::MediaByteBuffer; + BufferReader reader(aNAL, aLength); + auto res = reader.ReadU8(); + if (res.isErr()) { + return nullptr; + } + uint8_t nal_unit_type = res.unwrap() & 0x1f; + uint32_t nalUnitHeaderBytes = 1; + if (nal_unit_type == H264_NAL_PREFIX || nal_unit_type == H264_NAL_SLICE_EXT || + nal_unit_type == H264_NAL_SLICE_EXT_DVC) { + bool svc_extension_flag = false; + bool avc_3d_extension_flag = false; + if (nal_unit_type != H264_NAL_SLICE_EXT_DVC) { + res = reader.PeekU8(); + if (res.isErr()) { + return nullptr; + } + svc_extension_flag = res.unwrap() & 0x80; + } else { + res = reader.PeekU8(); + if (res.isErr()) { + return nullptr; + } + avc_3d_extension_flag = res.unwrap() & 0x80; + } + if (svc_extension_flag) { + nalUnitHeaderBytes += 3; + } else if (avc_3d_extension_flag) { + nalUnitHeaderBytes += 2; + } else { + nalUnitHeaderBytes += 3; + } + } + if (!reader.Read(nalUnitHeaderBytes - 1)) { + return nullptr; + } + uint32_t lastbytes = 0xffff; + while (reader.Remaining()) { + auto res = reader.ReadU8(); + if (res.isErr()) { + return nullptr; + } + uint8_t byte = res.unwrap(); + if ((lastbytes & 0xffff) == 0 && byte == 0x03) { + // reset last two bytes, to detect the 0x000003 sequence again. + lastbytes = 0xffff; + } else { + rbsp->AppendElement(byte); + } + lastbytes = (lastbytes << 8) | byte; + } + return rbsp.forget(); +} + +// The reverse of DecodeNALUnit. To allow the distinction between Annex B (that +// uses 0x000001 as marker) and AVCC, the pattern 0x00 0x00 0x0n (where n is +// between 0 and 3) can't be found in the bytestream. A 0x03 byte is inserted +// after the second 0. Eg. 0x00 0x00 0x00 becomes 0x00 0x00 0x03 0x00 +/* static */ already_AddRefed<mozilla::MediaByteBuffer> H264::EncodeNALUnit( + const uint8_t* aNAL, size_t aLength) { + MOZ_ASSERT(aNAL); + RefPtr<MediaByteBuffer> rbsp = new MediaByteBuffer(); + BufferReader reader(aNAL, aLength); + + auto res = reader.ReadU8(); + if (res.isErr()) { + return rbsp.forget(); + } + rbsp->AppendElement(res.unwrap()); + + res = reader.ReadU8(); + if (res.isErr()) { + return rbsp.forget(); + } + rbsp->AppendElement(res.unwrap()); + + while ((res = reader.ReadU8()).isOk()) { + uint8_t val = res.unwrap(); + if (val <= 0x03 && rbsp->ElementAt(rbsp->Length() - 2) == 0 && + rbsp->ElementAt(rbsp->Length() - 1) == 0) { + rbsp->AppendElement(0x03); + } + rbsp->AppendElement(val); + } + return rbsp.forget(); +} + +static int32_t ConditionDimension(float aValue) { + // This will exclude NaNs and too-big values. + if (aValue > 1.0 && aValue <= float(INT32_MAX) / 2) { + return int32_t(aValue); + } + return 0; +} + +/* static */ +bool H264::DecodeSPS(const mozilla::MediaByteBuffer* aSPS, SPSData& aDest) { + if (!aSPS) { + return false; + } + BitReader br(aSPS, BitReader::GetBitLength(aSPS)); + + aDest.profile_idc = br.ReadBits(8); + aDest.constraint_set0_flag = br.ReadBit(); + aDest.constraint_set1_flag = br.ReadBit(); + aDest.constraint_set2_flag = br.ReadBit(); + aDest.constraint_set3_flag = br.ReadBit(); + aDest.constraint_set4_flag = br.ReadBit(); + aDest.constraint_set5_flag = br.ReadBit(); + br.ReadBits(2); // reserved_zero_2bits + aDest.level_idc = br.ReadBits(8); + READUE(seq_parameter_set_id, MAX_SPS_COUNT - 1); + + if (aDest.profile_idc == 100 || aDest.profile_idc == 110 || + aDest.profile_idc == 122 || aDest.profile_idc == 244 || + aDest.profile_idc == 44 || aDest.profile_idc == 83 || + aDest.profile_idc == 86 || aDest.profile_idc == 118 || + aDest.profile_idc == 128 || aDest.profile_idc == 138 || + aDest.profile_idc == 139 || aDest.profile_idc == 134) { + READUE(chroma_format_idc, 3); + if (aDest.chroma_format_idc == 3) { + aDest.separate_colour_plane_flag = br.ReadBit(); + } + READUE(bit_depth_luma_minus8, 6); + READUE(bit_depth_chroma_minus8, 6); + br.ReadBit(); // qpprime_y_zero_transform_bypass_flag + aDest.seq_scaling_matrix_present_flag = br.ReadBit(); + if (aDest.seq_scaling_matrix_present_flag) { + scaling_list(br, aDest.scaling_matrix4x4[0], Default_4x4_Intra, + Default_4x4_Intra); + scaling_list(br, aDest.scaling_matrix4x4[1], Default_4x4_Intra, + aDest.scaling_matrix4x4[0]); + scaling_list(br, aDest.scaling_matrix4x4[2], Default_4x4_Intra, + aDest.scaling_matrix4x4[1]); + scaling_list(br, aDest.scaling_matrix4x4[3], Default_4x4_Inter, + Default_4x4_Inter); + scaling_list(br, aDest.scaling_matrix4x4[4], Default_4x4_Inter, + aDest.scaling_matrix4x4[3]); + scaling_list(br, aDest.scaling_matrix4x4[5], Default_4x4_Inter, + aDest.scaling_matrix4x4[4]); + + scaling_list(br, aDest.scaling_matrix8x8[0], Default_8x8_Intra, + Default_8x8_Intra); + scaling_list(br, aDest.scaling_matrix8x8[1], Default_8x8_Inter, + Default_8x8_Inter); + if (aDest.chroma_format_idc == 3) { + scaling_list(br, aDest.scaling_matrix8x8[2], Default_8x8_Intra, + aDest.scaling_matrix8x8[0]); + scaling_list(br, aDest.scaling_matrix8x8[3], Default_8x8_Inter, + aDest.scaling_matrix8x8[1]); + scaling_list(br, aDest.scaling_matrix8x8[4], Default_8x8_Intra, + aDest.scaling_matrix8x8[2]); + scaling_list(br, aDest.scaling_matrix8x8[5], Default_8x8_Inter, + aDest.scaling_matrix8x8[3]); + } + } + } else if (aDest.profile_idc == 183) { + aDest.chroma_format_idc = 0; + } else { + // default value if chroma_format_idc isn't set. + aDest.chroma_format_idc = 1; + } + READUE(log2_max_frame_num, 12); + aDest.log2_max_frame_num += 4; + READUE(pic_order_cnt_type, 2); + if (aDest.pic_order_cnt_type == 0) { + READUE(log2_max_pic_order_cnt_lsb, 12); + aDest.log2_max_pic_order_cnt_lsb += 4; + } else if (aDest.pic_order_cnt_type == 1) { + aDest.delta_pic_order_always_zero_flag = br.ReadBit(); + READSE(offset_for_non_ref_pic, -231, 230); + READSE(offset_for_top_to_bottom_field, -231, 230); + uint32_t num_ref_frames_in_pic_order_cnt_cycle = br.ReadUE(); + for (uint32_t i = 0; i < num_ref_frames_in_pic_order_cnt_cycle; i++) { + br.ReadSE(); // offset_for_ref_frame[i] + } + } + aDest.max_num_ref_frames = br.ReadUE(); + aDest.gaps_in_frame_num_allowed_flag = br.ReadBit(); + aDest.pic_width_in_mbs = br.ReadUE() + 1; + aDest.pic_height_in_map_units = br.ReadUE() + 1; + aDest.frame_mbs_only_flag = br.ReadBit(); + if (!aDest.frame_mbs_only_flag) { + aDest.pic_height_in_map_units *= 2; + aDest.mb_adaptive_frame_field_flag = br.ReadBit(); + } + aDest.direct_8x8_inference_flag = br.ReadBit(); + aDest.frame_cropping_flag = br.ReadBit(); + if (aDest.frame_cropping_flag) { + aDest.frame_crop_left_offset = br.ReadUE(); + aDest.frame_crop_right_offset = br.ReadUE(); + aDest.frame_crop_top_offset = br.ReadUE(); + aDest.frame_crop_bottom_offset = br.ReadUE(); + } + + aDest.sample_ratio = 1.0f; + aDest.vui_parameters_present_flag = br.ReadBit(); + if (aDest.vui_parameters_present_flag) { + if (!vui_parameters(br, aDest)) { + return false; + } + } + + // Calculate common values. + + uint8_t ChromaArrayType = + aDest.separate_colour_plane_flag ? 0 : aDest.chroma_format_idc; + // Calculate width. + uint32_t CropUnitX = 1; + uint32_t SubWidthC = aDest.chroma_format_idc == 3 ? 1 : 2; + if (ChromaArrayType != 0) { + CropUnitX = SubWidthC; + } + + // Calculate Height + uint32_t CropUnitY = 2 - aDest.frame_mbs_only_flag; + uint32_t SubHeightC = aDest.chroma_format_idc <= 1 ? 2 : 1; + if (ChromaArrayType != 0) { + CropUnitY *= SubHeightC; + } + + uint32_t width = aDest.pic_width_in_mbs * 16; + uint32_t height = aDest.pic_height_in_map_units * 16; + if (aDest.frame_crop_left_offset <= + std::numeric_limits<int32_t>::max() / 4 / CropUnitX && + aDest.frame_crop_right_offset <= + std::numeric_limits<int32_t>::max() / 4 / CropUnitX && + aDest.frame_crop_top_offset <= + std::numeric_limits<int32_t>::max() / 4 / CropUnitY && + aDest.frame_crop_bottom_offset <= + std::numeric_limits<int32_t>::max() / 4 / CropUnitY && + (aDest.frame_crop_left_offset + aDest.frame_crop_right_offset) * + CropUnitX < + width && + (aDest.frame_crop_top_offset + aDest.frame_crop_bottom_offset) * + CropUnitY < + height) { + aDest.crop_left = aDest.frame_crop_left_offset * CropUnitX; + aDest.crop_right = aDest.frame_crop_right_offset * CropUnitX; + aDest.crop_top = aDest.frame_crop_top_offset * CropUnitY; + aDest.crop_bottom = aDest.frame_crop_bottom_offset * CropUnitY; + } else { + // Nonsensical value, ignore them. + aDest.crop_left = aDest.crop_right = aDest.crop_top = aDest.crop_bottom = 0; + } + + aDest.pic_width = width - aDest.crop_left - aDest.crop_right; + aDest.pic_height = height - aDest.crop_top - aDest.crop_bottom; + + aDest.interlaced = !aDest.frame_mbs_only_flag; + + // Determine display size. + if (aDest.sample_ratio > 1.0) { + // Increase the intrinsic width + aDest.display_width = + ConditionDimension(aDest.pic_width * aDest.sample_ratio); + aDest.display_height = aDest.pic_height; + } else { + // Increase the intrinsic height + aDest.display_width = aDest.pic_width; + aDest.display_height = + ConditionDimension(aDest.pic_height / aDest.sample_ratio); + } + + aDest.valid = true; + + return true; +} + +/* static */ +bool H264::vui_parameters(BitReader& aBr, SPSData& aDest) { + aDest.aspect_ratio_info_present_flag = aBr.ReadBit(); + if (aDest.aspect_ratio_info_present_flag) { + aDest.aspect_ratio_idc = aBr.ReadBits(8); + aDest.sar_width = aDest.sar_height = 0; + + // From E.2.1 VUI parameters semantics (ITU-T H.264 02/2014) + switch (aDest.aspect_ratio_idc) { + case 0: + // Unspecified + break; + case 1: + /* + 1:1 + 7680x4320 16:9 frame without horizontal overscan + 3840x2160 16:9 frame without horizontal overscan + 1280x720 16:9 frame without horizontal overscan + 1920x1080 16:9 frame without horizontal overscan (cropped from + 1920x1088) 640x480 4:3 frame without horizontal overscan + */ + aDest.sample_ratio = 1.0f; + break; + case 2: + /* + 12:11 + 720x576 4:3 frame with horizontal overscan + 352x288 4:3 frame without horizontal overscan + */ + aDest.sample_ratio = 12.0 / 11.0; + break; + case 3: + /* + 10:11 + 720x480 4:3 frame with horizontal overscan + 352x240 4:3 frame without horizontal overscan + */ + aDest.sample_ratio = 10.0 / 11.0; + break; + case 4: + /* + 16:11 + 720x576 16:9 frame with horizontal overscan + 528x576 4:3 frame without horizontal overscan + */ + aDest.sample_ratio = 16.0 / 11.0; + break; + case 5: + /* + 40:33 + 720x480 16:9 frame with horizontal overscan + 528x480 4:3 frame without horizontal overscan + */ + aDest.sample_ratio = 40.0 / 33.0; + break; + case 6: + /* + 24:11 + 352x576 4:3 frame without horizontal overscan + 480x576 16:9 frame with horizontal overscan + */ + aDest.sample_ratio = 24.0 / 11.0; + break; + case 7: + /* + 20:11 + 352x480 4:3 frame without horizontal overscan + 480x480 16:9 frame with horizontal overscan + */ + aDest.sample_ratio = 20.0 / 11.0; + break; + case 8: + /* + 32:11 + 352x576 16:9 frame without horizontal overscan + */ + aDest.sample_ratio = 32.0 / 11.0; + break; + case 9: + /* + 80:33 + 352x480 16:9 frame without horizontal overscan + */ + aDest.sample_ratio = 80.0 / 33.0; + break; + case 10: + /* + 18:11 + 480x576 4:3 frame with horizontal overscan + */ + aDest.sample_ratio = 18.0 / 11.0; + break; + case 11: + /* + 15:11 + 480x480 4:3 frame with horizontal overscan + */ + aDest.sample_ratio = 15.0 / 11.0; + break; + case 12: + /* + 64:33 + 528x576 16:9 frame with horizontal overscan + */ + aDest.sample_ratio = 64.0 / 33.0; + break; + case 13: + /* + 160:99 + 528x480 16:9 frame without horizontal overscan + */ + aDest.sample_ratio = 160.0 / 99.0; + break; + case 14: + /* + 4:3 + 1440x1080 16:9 frame without horizontal overscan + */ + aDest.sample_ratio = 4.0 / 3.0; + break; + case 15: + /* + 3:2 + 1280x1080 16:9 frame without horizontal overscan + */ + aDest.sample_ratio = 3.2 / 2.0; + break; + case 16: + /* + 2:1 + 960x1080 16:9 frame without horizontal overscan + */ + aDest.sample_ratio = 2.0 / 1.0; + break; + case 255: + /* Extended_SAR */ + aDest.sar_width = aBr.ReadBits(16); + aDest.sar_height = aBr.ReadBits(16); + if (aDest.sar_width && aDest.sar_height) { + aDest.sample_ratio = float(aDest.sar_width) / float(aDest.sar_height); + } + break; + default: + break; + } + } + + if (aBr.ReadBit()) { // overscan_info_present_flag + aDest.overscan_appropriate_flag = aBr.ReadBit(); + } + + if (aBr.ReadBit()) { // video_signal_type_present_flag + aDest.video_format = aBr.ReadBits(3); + aDest.video_full_range_flag = aBr.ReadBit(); + aDest.colour_description_present_flag = aBr.ReadBit(); + if (aDest.colour_description_present_flag) { + aDest.colour_primaries = aBr.ReadBits(8); + aDest.transfer_characteristics = aBr.ReadBits(8); + aDest.matrix_coefficients = aBr.ReadBits(8); + } + } + + aDest.chroma_loc_info_present_flag = aBr.ReadBit(); + if (aDest.chroma_loc_info_present_flag) { + BitReader& br = aBr; // so that macro READUE works + READUE(chroma_sample_loc_type_top_field, 5); + READUE(chroma_sample_loc_type_bottom_field, 5); + } + + bool timing_info_present_flag = aBr.ReadBit(); + if (timing_info_present_flag) { + aBr.ReadBits(32); // num_units_in_tick + aBr.ReadBits(32); // time_scale + aBr.ReadBit(); // fixed_frame_rate_flag + } + return true; +} + +/* static */ +bool H264::DecodeSPSFromExtraData(const mozilla::MediaByteBuffer* aExtraData, + SPSData& aDest) { + SPSNALIterator it(aExtraData); + if (!it) { + return false; + } + return (*it).GetSPSData(aDest); +} + +/* static */ +bool H264::EnsureSPSIsSane(SPSData& aSPS) { + bool valid = true; + static const float default_aspect = 4.0f / 3.0f; + if (aSPS.sample_ratio <= 0.0f || aSPS.sample_ratio > 6.0f) { + if (aSPS.pic_width && aSPS.pic_height) { + aSPS.sample_ratio = (float)aSPS.pic_width / (float)aSPS.pic_height; + } else { + aSPS.sample_ratio = default_aspect; + } + aSPS.display_width = aSPS.pic_width; + aSPS.display_height = aSPS.pic_height; + valid = false; + } + if (aSPS.max_num_ref_frames > 16) { + aSPS.max_num_ref_frames = 16; + valid = false; + } + return valid; +} + +/* static */ +uint32_t H264::ComputeMaxRefFrames(const mozilla::MediaByteBuffer* aExtraData) { + uint32_t maxRefFrames = 4; + // Retrieve video dimensions from H264 SPS NAL. + SPSData spsdata; + if (DecodeSPSFromExtraData(aExtraData, spsdata)) { + // max_num_ref_frames determines the size of the sliding window + // we need to queue that many frames in order to guarantee proper + // pts frames ordering. Use a minimum of 4 to ensure proper playback of + // non compliant videos. + maxRefFrames = + std::min(std::max(maxRefFrames, spsdata.max_num_ref_frames + 1), 16u); + } + return maxRefFrames; +} + +/* static */ H264::FrameType H264::GetFrameType( + const mozilla::MediaRawData* aSample) { + if (!AnnexB::IsAVCC(aSample)) { + // We must have a valid AVCC frame with extradata. + return FrameType::INVALID; + } + MOZ_ASSERT(aSample->Data()); + + int nalLenSize = ((*aSample->mExtraData)[4] & 3) + 1; + + BufferReader reader(aSample->Data(), aSample->Size()); + + while (reader.Remaining() >= nalLenSize) { + uint32_t nalLen = 0; + switch (nalLenSize) { + case 1: + nalLen = reader.ReadU8().unwrapOr(0); + break; + case 2: + nalLen = reader.ReadU16().unwrapOr(0); + break; + case 3: + nalLen = reader.ReadU24().unwrapOr(0); + break; + case 4: + nalLen = reader.ReadU32().unwrapOr(0); + break; + } + if (!nalLen) { + continue; + } + const uint8_t* p = reader.Read(nalLen); + if (!p) { + return FrameType::INVALID; + } + int8_t nalType = *p & 0x1f; + if (nalType == H264_NAL_IDR_SLICE) { + // IDR NAL. + return FrameType::I_FRAME; + } else if (nalType == H264_NAL_SEI) { + RefPtr<mozilla::MediaByteBuffer> decodedNAL = DecodeNALUnit(p, nalLen); + SEIRecoveryData data; + if (DecodeRecoverySEI(decodedNAL, data)) { + return FrameType::I_FRAME; + } + } else if (nalType == H264_NAL_SLICE) { + RefPtr<mozilla::MediaByteBuffer> decodedNAL = DecodeNALUnit(p, nalLen); + if (DecodeISlice(decodedNAL)) { + return FrameType::I_FRAME; + } + } + } + + return FrameType::OTHER; +} + +/* static */ already_AddRefed<mozilla::MediaByteBuffer> H264::ExtractExtraData( + const mozilla::MediaRawData* aSample) { + MOZ_ASSERT(AnnexB::IsAVCC(aSample)); + + RefPtr<mozilla::MediaByteBuffer> extradata = new mozilla::MediaByteBuffer; + + // SPS content + nsTArray<uint8_t> sps; + ByteWriter<BigEndian> spsw(sps); + int numSps = 0; + // PPS content + nsTArray<uint8_t> pps; + ByteWriter<BigEndian> ppsw(pps); + int numPps = 0; + + int nalLenSize = ((*aSample->mExtraData)[4] & 3) + 1; + + size_t sampleSize = aSample->Size(); + if (aSample->mCrypto.IsEncrypted()) { + // The content is encrypted, we can only parse the non-encrypted data. + MOZ_ASSERT(aSample->mCrypto.mPlainSizes.Length() > 0); + if (aSample->mCrypto.mPlainSizes.Length() == 0 || + aSample->mCrypto.mPlainSizes[0] > sampleSize) { + // This is invalid content. + return nullptr; + } + sampleSize = aSample->mCrypto.mPlainSizes[0]; + } + + BufferReader reader(aSample->Data(), sampleSize); + + nsTArray<SPSData> SPSTable; + // If we encounter SPS with the same id but different content, we will stop + // attempting to detect duplicates. + bool checkDuplicate = true; + + // Find SPS and PPS NALUs in AVCC data + while (reader.Remaining() > nalLenSize) { + uint32_t nalLen = 0; + switch (nalLenSize) { + case 1: + Unused << reader.ReadU8().map( + [&](uint8_t x) mutable { return nalLen = x; }); + break; + case 2: + Unused << reader.ReadU16().map( + [&](uint16_t x) mutable { return nalLen = x; }); + break; + case 3: + Unused << reader.ReadU24().map( + [&](uint32_t x) mutable { return nalLen = x; }); + break; + case 4: + Unused << reader.ReadU32().map( + [&](uint32_t x) mutable { return nalLen = x; }); + break; + } + const uint8_t* p = reader.Read(nalLen); + if (!p) { + // The read failed, but we may already have some SPS + PPS data so + // break out of reading and process what we have, if any. + break; + } + uint8_t nalType = *p & 0x1f; + + if (nalType == H264_NAL_SPS) { + RefPtr<mozilla::MediaByteBuffer> sps = DecodeNALUnit(p, nalLen); + SPSData data; + if (!DecodeSPS(sps, data)) { + // Invalid SPS, ignore. + continue; + } + uint8_t spsId = data.seq_parameter_set_id; + if (spsId >= SPSTable.Length()) { + if (!SPSTable.SetLength(spsId + 1, fallible)) { + // OOM. + return nullptr; + } + } + if (checkDuplicate && SPSTable[spsId].valid && SPSTable[spsId] == data) { + // Duplicate ignore. + continue; + } + if (SPSTable[spsId].valid) { + // We already have detected a SPS with this Id. Just to be safe we + // disable SPS duplicate detection. + checkDuplicate = false; + } else { + SPSTable[spsId] = data; + } + numSps++; + if (!spsw.WriteU16(nalLen) || !spsw.Write(p, nalLen)) { + return extradata.forget(); + } + } else if (nalType == H264_NAL_PPS) { + numPps++; + if (!ppsw.WriteU16(nalLen) || !ppsw.Write(p, nalLen)) { + return extradata.forget(); + } + } + } + + // We ignore PPS data if we didn't find a SPS as we would be unable to + // decode it anyway. + numPps = numSps ? numPps : 0; + + if (numSps && sps.Length() > 5) { + extradata->AppendElement(1); // version + extradata->AppendElement(sps[3]); // profile + extradata->AppendElement(sps[4]); // profile compat + extradata->AppendElement(sps[5]); // level + extradata->AppendElement(0xfc | 3); // nal size - 1 + extradata->AppendElement(0xe0 | numSps); + extradata->AppendElements(sps.Elements(), sps.Length()); + extradata->AppendElement(numPps); + if (numPps) { + extradata->AppendElements(pps.Elements(), pps.Length()); + } + } + + return extradata.forget(); +} + +/* static */ +bool H264::HasSPS(const mozilla::MediaByteBuffer* aExtraData) { + return NumSPS(aExtraData) > 0; +} + +/* static */ +uint8_t H264::NumSPS(const mozilla::MediaByteBuffer* aExtraData) { + if (!aExtraData || aExtraData->IsEmpty()) { + return 0; + } + + BufferReader reader(aExtraData); + if (!reader.Read(5)) { + return 0; + } + auto res = reader.ReadU8(); + if (res.isErr()) { + return 0; + } + return res.unwrap() & 0x1f; +} + +/* static */ +bool H264::CompareExtraData(const mozilla::MediaByteBuffer* aExtraData1, + const mozilla::MediaByteBuffer* aExtraData2) { + if (aExtraData1 == aExtraData2) { + return true; + } + uint8_t numSPS = NumSPS(aExtraData1); + if (numSPS == 0 || numSPS != NumSPS(aExtraData2)) { + return false; + } + + // We only compare if the SPS are the same as the various H264 decoders can + // deal with in-band change of PPS. + + SPSNALIterator it1(aExtraData1); + SPSNALIterator it2(aExtraData2); + + while (it1 && it2) { + if (*it1 != *it2) { + return false; + } + ++it1; + ++it2; + } + return true; +} + +static inline Result<Ok, nsresult> ReadSEIInt(BufferReader& aBr, + uint32_t& aOutput) { + uint8_t tmpByte; + + aOutput = 0; + MOZ_TRY_VAR(tmpByte, aBr.ReadU8()); + while (tmpByte == 0xFF) { + aOutput += 255; + MOZ_TRY_VAR(tmpByte, aBr.ReadU8()); + } + aOutput += tmpByte; // this is the last byte + return Ok(); +} + +/* static */ +bool H264::DecodeISlice(const mozilla::MediaByteBuffer* aSlice) { + if (!aSlice) { + return false; + } + + // According to ITU-T Rec H.264 Table 7.3.3, read the slice type from + // slice_header, and the slice type 2 and 7 are representing I slice. + BitReader br(aSlice); + // Skip `first_mb_in_slice` + br.ReadUE(); + // The value of slice type can go from 0 to 9, but the value between 5 to + // 9 are actually equal to 0 to 4. + const uint32_t sliceType = br.ReadUE() % 5; + return sliceType == SLICE_TYPES::I_SLICE || sliceType == SI_SLICE; +} + +/* static */ +bool H264::DecodeRecoverySEI(const mozilla::MediaByteBuffer* aSEI, + SEIRecoveryData& aDest) { + if (!aSEI) { + return false; + } + // sei_rbsp() as per 7.3.2.3 Supplemental enhancement information RBSP syntax + BufferReader br(aSEI); + + do { + // sei_message() as per + // 7.3.2.3.1 Supplemental enhancement information message syntax + uint32_t payloadType = 0; + if (ReadSEIInt(br, payloadType).isErr()) { + return false; + } + + uint32_t payloadSize = 0; + if (ReadSEIInt(br, payloadSize).isErr()) { + return false; + } + + // sei_payload(payloadType, payloadSize) as per + // D.1 SEI payload syntax. + const uint8_t* p = br.Read(payloadSize); + if (!p) { + return false; + } + if (payloadType == 6) { // SEI_RECOVERY_POINT + if (payloadSize == 0) { + // Invalid content, ignore. + continue; + } + // D.1.7 Recovery point SEI message syntax + BitReader br(p, payloadSize * 8); + aDest.recovery_frame_cnt = br.ReadUE(); + aDest.exact_match_flag = br.ReadBit(); + aDest.broken_link_flag = br.ReadBit(); + aDest.changing_slice_group_idc = br.ReadBits(2); + return true; + } + } while (br.PeekU8().isOk() && + br.PeekU8().unwrap() != + 0x80); // more_rbsp_data() msg[offset] != 0x80 + // ignore the trailing bits rbsp_trailing_bits(); + return false; +} + +/*static */ already_AddRefed<mozilla::MediaByteBuffer> H264::CreateExtraData( + uint8_t aProfile, uint8_t aConstraints, uint8_t aLevel, + const gfx::IntSize& aSize) { + // SPS of a 144p video. + const uint8_t originSPS[] = {0x4d, 0x40, 0x0c, 0xe8, 0x80, 0x80, 0x9d, + 0x80, 0xb5, 0x01, 0x01, 0x01, 0x40, 0x00, + 0x00, 0x00, 0x40, 0x00, 0x00, 0x0f, 0x03, + 0xc5, 0x0a, 0x44, 0x80}; + + RefPtr<MediaByteBuffer> extraData = new MediaByteBuffer(); + extraData->AppendElements(originSPS, sizeof(originSPS)); + BitReader br(extraData, BitReader::GetBitLength(extraData)); + + RefPtr<MediaByteBuffer> sps = new MediaByteBuffer(); + BitWriter bw(sps); + + br.ReadBits(8); // Skip original profile_idc + bw.WriteU8(aProfile); + br.ReadBits(8); // Skip original constraint flags && reserved_zero_2bits + aConstraints = + aConstraints & ~0x3; // Ensure reserved_zero_2bits are set to 0 + bw.WriteBits(aConstraints, 8); + br.ReadBits(8); // Skip original level_idc + bw.WriteU8(aLevel); + bw.WriteUE(br.ReadUE()); // seq_parameter_set_id (0 stored on 1 bit) + + if (aProfile == 100 || aProfile == 110 || aProfile == 122 || + aProfile == 244 || aProfile == 44 || aProfile == 83 || aProfile == 86 || + aProfile == 118 || aProfile == 128 || aProfile == 138 || + aProfile == 139 || aProfile == 134) { + bw.WriteUE(1); // chroma_format_idc -> always set to 4:2:0 chroma format + bw.WriteUE(0); // bit_depth_luma_minus8 -> always 8 bits here + bw.WriteUE(0); // bit_depth_chroma_minus8 -> always 8 bits here + bw.WriteBit(false); // qpprime_y_zero_transform_bypass_flag + bw.WriteBit(false); // seq_scaling_matrix_present_flag + } + + bw.WriteBits(br.ReadBits(11), + 11); // log2_max_frame_num to gaps_in_frame_num_allowed_flag + + // skip over original exp-golomb encoded width/height + br.ReadUE(); // skip width + br.ReadUE(); // skip height + uint32_t width = aSize.width; + uint32_t widthNeeded = width % 16 != 0 ? (width / 16 + 1) * 16 : width; + uint32_t height = aSize.height; + uint32_t heightNeeded = height % 16 != 0 ? (height / 16 + 1) * 16 : height; + bw.WriteUE(widthNeeded / 16 - 1); + bw.WriteUE(heightNeeded / 16 - 1); + bw.WriteBit(br.ReadBit()); // write frame_mbs_only_flag + bw.WriteBit(br.ReadBit()); // write direct_8x8_inference_flag; + if (widthNeeded != width || heightNeeded != height) { + // Write cropping value + bw.WriteBit(true); // skip frame_cropping_flag + bw.WriteUE(0); // frame_crop_left_offset + bw.WriteUE((widthNeeded - width) / 2); // frame_crop_right_offset + bw.WriteUE(0); // frame_crop_top_offset + bw.WriteUE((heightNeeded - height) / 2); // frame_crop_bottom_offset + } else { + bw.WriteBit(false); // skip frame_cropping_flag + } + br.ReadBit(); // skip frame_cropping_flag; + // Write the remainings of the original sps (vui_parameters which sets an + // aspect ration of 1.0) + while (br.BitsLeft()) { + bw.WriteBit(br.ReadBit()); + } + bw.CloseWithRbspTrailing(); + + RefPtr<MediaByteBuffer> encodedSPS = + EncodeNALUnit(sps->Elements(), sps->Length()); + extraData->Clear(); + + const uint8_t PPS[] = {0xeb, 0xef, 0x20}; + + WriteExtraData( + extraData, aProfile, aConstraints, aLevel, + Span<const uint8_t>(encodedSPS->Elements(), encodedSPS->Length()), + Span<const uint8_t>(PPS, sizeof(PPS))); + + return extraData.forget(); +} + +void H264::WriteExtraData(MediaByteBuffer* aDestExtraData, + const uint8_t aProfile, const uint8_t aConstraints, + const uint8_t aLevel, const Span<const uint8_t> aSPS, + const Span<const uint8_t> aPPS) { + aDestExtraData->AppendElement(1); + aDestExtraData->AppendElement(aProfile); + aDestExtraData->AppendElement(aConstraints); + aDestExtraData->AppendElement(aLevel); + aDestExtraData->AppendElement(3); // nalLENSize-1 + aDestExtraData->AppendElement(1); // numPPS + uint8_t c[2]; + mozilla::BigEndian::writeUint16(&c[0], aSPS.Length() + 1); + aDestExtraData->AppendElements(c, 2); + aDestExtraData->AppendElement((0x00 << 7) | (0x3 << 5) | H264_NAL_SPS); + aDestExtraData->AppendElements(aSPS.Elements(), aSPS.Length()); + + aDestExtraData->AppendElement(1); // numPPS + mozilla::BigEndian::writeUint16(&c[0], aPPS.Length() + 1); + aDestExtraData->AppendElements(c, 2); + aDestExtraData->AppendElement((0x00 << 7) | (0x3 << 5) | H264_NAL_PPS); + aDestExtraData->AppendElements(aPPS.Elements(), aPPS.Length()); +} + +#undef READUE +#undef READSE + +} // namespace mozilla diff --git a/dom/media/platforms/agnostic/bytestreams/H264.h b/dom/media/platforms/agnostic/bytestreams/H264.h new file mode 100644 index 0000000000..37c0e5bc14 --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/H264.h @@ -0,0 +1,525 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MP4_DEMUXER_H264_H_ +#define MP4_DEMUXER_H264_H_ + +#include "DecoderData.h" +#include "mozilla/gfx/Types.h" + +namespace mozilla { +class BitReader; + +// Spec 7.4.2.1 +#define MAX_SPS_COUNT 32 +#define MAX_PPS_COUNT 256 + +// NAL unit types +enum NAL_TYPES { + H264_NAL_SLICE = 1, + H264_NAL_DPA = 2, + H264_NAL_DPB = 3, + H264_NAL_DPC = 4, + H264_NAL_IDR_SLICE = 5, + H264_NAL_SEI = 6, + H264_NAL_SPS = 7, + H264_NAL_PPS = 8, + H264_NAL_AUD = 9, + H264_NAL_END_SEQUENCE = 10, + H264_NAL_END_STREAM = 11, + H264_NAL_FILLER_DATA = 12, + H264_NAL_SPS_EXT = 13, + H264_NAL_PREFIX = 14, + H264_NAL_AUXILIARY_SLICE = 19, + H264_NAL_SLICE_EXT = 20, + H264_NAL_SLICE_EXT_DVC = 21, +}; + +// According to ITU-T Rec H.264 (2017/04) Table 7.6. +enum SLICE_TYPES { + P_SLICE = 0, + B_SLICE = 1, + I_SLICE = 2, + SP_SLICE = 3, + SI_SLICE = 4, +}; + +struct SPSData { + bool operator==(const SPSData& aOther) const; + bool operator!=(const SPSData& aOther) const; + + gfx::YUVColorSpace ColorSpace() const; + gfx::ColorDepth ColorDepth() const; + + bool valid; + + /* Decoded Members */ + /* + pic_width is the decoded width according to: + pic_width = ((pic_width_in_mbs_minus1 + 1) * 16) + - (frame_crop_left_offset + frame_crop_right_offset) * 2 + */ + uint32_t pic_width; + /* + pic_height is the decoded height according to: + pic_height = (2 - frame_mbs_only_flag) * ((pic_height_in_map_units_minus1 + + 1) * 16) + - (frame_crop_top_offset + frame_crop_bottom_offset) * 2 + */ + uint32_t pic_height; + + bool interlaced; + + /* + Displayed size. + display_width and display_height are adjusted according to the display + sample aspect ratio. + */ + uint32_t display_width; + uint32_t display_height; + + float sample_ratio; + + uint32_t crop_left; + uint32_t crop_right; + uint32_t crop_top; + uint32_t crop_bottom; + + /* + H264 decoding parameters according to ITU-T H.264 (T-REC-H.264-201402-I/en) + http://www.itu.int/rec/T-REC-H.264-201402-I/en + */ + + bool constraint_set0_flag; + bool constraint_set1_flag; + bool constraint_set2_flag; + bool constraint_set3_flag; + bool constraint_set4_flag; + bool constraint_set5_flag; + + /* + profile_idc and level_idc indicate the profile and level to which the coded + video sequence conforms when the SVC sequence parameter set is the active + SVC sequence parameter set. + */ + uint8_t profile_idc; + uint8_t level_idc; + + /* + seq_parameter_set_id identifies the sequence parameter set that is referred + to by the picture parameter set. The value of seq_parameter_set_id shall be + in the range of 0 to 31, inclusive. + */ + uint8_t seq_parameter_set_id; + + /* + chroma_format_idc specifies the chroma sampling relative to the luma + sampling as specified in clause 6.2. The value of chroma_format_idc shall be + in the range of 0 to 3, inclusive. When chroma_format_idc is not present, + it shall be inferred to be equal to 1 (4:2:0 chroma format). + When profile_idc is equal to 183, chroma_format_idc shall be equal to 0 + (4:0:0 chroma format). + */ + uint8_t chroma_format_idc; + + /* + bit_depth_luma_minus8 specifies the bit depth of the samples of the luma + array and the value of the luma quantisation parameter range offset + QpBdOffset Y , as specified by + BitDepth Y = 8 + bit_depth_luma_minus8 (7-3) + QpBdOffset Y = 6 * bit_depth_luma_minus8 (7-4) + When bit_depth_luma_minus8 is not present, it shall be inferred to be equal + to 0. bit_depth_luma_minus8 shall be in the range of 0 to 6, inclusive. + */ + uint8_t bit_depth_luma_minus8; + + /* + bit_depth_chroma_minus8 specifies the bit depth of the samples of the chroma + arrays and the value of the chroma quantisation parameter range offset + QpBdOffset C , as specified by + BitDepth C = 8 + bit_depth_chroma_minus8 (7-5) + QpBdOffset C = 6 * bit_depth_chroma_minus8 (7-6) + When bit_depth_chroma_minus8 is not present, it shall be inferred to be + equal to 0. bit_depth_chroma_minus8 shall be in the range of 0 to 6, + inclusive. + */ + uint8_t bit_depth_chroma_minus8; + + /* + separate_colour_plane_flag equal to 1 specifies that the three colour + components of the 4:4:4 chroma format are coded separately. + separate_colour_plane_flag equal to 0 specifies that the colour components + are not coded separately. When separate_colour_plane_flag is not present, + it shall be inferred to be equal to 0. When separate_colour_plane_flag is + equal to 1, the primary coded picture consists of three separate components, + each of which consists of coded samples of one colour plane (Y, Cb or Cr) + that each use the monochrome coding syntax. In this case, each colour plane + is associated with a specific colour_plane_id value. + */ + bool separate_colour_plane_flag; + + /* + seq_scaling_matrix_present_flag equal to 1 specifies that the flags + seq_scaling_list_present_flag[ i ] for i = 0..7 or + i = 0..11 are present. seq_scaling_matrix_present_flag equal to 0 specifies + that these flags are not present and the sequence-level scaling list + specified by Flat_4x4_16 shall be inferred for i = 0..5 and the + sequence-level scaling list specified by Flat_8x8_16 shall be inferred for + i = 6..11. When seq_scaling_matrix_present_flag is not present, it shall be + inferred to be equal to 0. + */ + bool seq_scaling_matrix_present_flag; + + /* + log2_max_frame_num_minus4 specifies the value of the variable + MaxFrameNum that is used in frame_num related derivations as + follows: + + MaxFrameNum = 2( log2_max_frame_num_minus4 + 4 ). The value of + log2_max_frame_num_minus4 shall be in the range of 0 to 12, inclusive. + */ + uint8_t log2_max_frame_num; + + /* + pic_order_cnt_type specifies the method to decode picture order + count (as specified in subclause 8.2.1). The value of + pic_order_cnt_type shall be in the range of 0 to 2, inclusive. + */ + uint8_t pic_order_cnt_type; + + /* + log2_max_pic_order_cnt_lsb_minus4 specifies the value of the + variable MaxPicOrderCntLsb that is used in the decoding + process for picture order count as specified in subclause + 8.2.1 as follows: + + MaxPicOrderCntLsb = 2( log2_max_pic_order_cnt_lsb_minus4 + 4 ) + + The value of log2_max_pic_order_cnt_lsb_minus4 shall be in + the range of 0 to 12, inclusive. + */ + uint8_t log2_max_pic_order_cnt_lsb; + + /* + delta_pic_order_always_zero_flag equal to 1 specifies that + delta_pic_order_cnt[ 0 ] and delta_pic_order_cnt[ 1 ] are + not present in the slice headers of the sequence and shall + be inferred to be equal to 0. + */ + bool delta_pic_order_always_zero_flag; + + /* + offset_for_non_ref_pic is used to calculate the picture + order count of a non-reference picture as specified in + 8.2.1. The value of offset_for_non_ref_pic shall be in the + range of -231 to 231 - 1, inclusive. + */ + int8_t offset_for_non_ref_pic; + + /* + offset_for_top_to_bottom_field is used to calculate the + picture order count of a bottom field as specified in + subclause 8.2.1. The value of offset_for_top_to_bottom_field + shall be in the range of -231 to 231 - 1, inclusive. + */ + int8_t offset_for_top_to_bottom_field; + + /* + max_num_ref_frames specifies the maximum number of short-term and + long-term reference frames, complementary reference field pairs, + and non-paired reference fields that may be used by the decoding + process for inter prediction of any picture in the + sequence. max_num_ref_frames also determines the size of the sliding + window operation as specified in subclause 8.2.5.3. The value of + max_num_ref_frames shall be in the range of 0 to MaxDpbFrames (as + specified in subclause A.3.1 or A.3.2), inclusive. + */ + uint32_t max_num_ref_frames; + + /* + gaps_in_frame_num_value_allowed_flag specifies the allowed + values of frame_num as specified in subclause 7.4.3 and the + decoding process in case of an inferred gap between values of + frame_num as specified in subclause 8.2.5.2. + */ + bool gaps_in_frame_num_allowed_flag; + + /* + pic_width_in_mbs_minus1 plus 1 specifies the width of each + decoded picture in units of macroblocks. 16 macroblocks in a row + */ + uint32_t pic_width_in_mbs; + + /* + pic_height_in_map_units_minus1 plus 1 specifies the height in + slice group map units of a decoded frame or field. 16 + macroblocks in each column. + */ + uint32_t pic_height_in_map_units; + + /* + frame_mbs_only_flag equal to 0 specifies that coded pictures of + the coded video sequence may either be coded fields or coded + frames. frame_mbs_only_flag equal to 1 specifies that every + coded picture of the coded video sequence is a coded frame + containing only frame macroblocks. + */ + bool frame_mbs_only_flag; + + /* + mb_adaptive_frame_field_flag equal to 0 specifies no + switching between frame and field macroblocks within a + picture. mb_adaptive_frame_field_flag equal to 1 specifies + the possible use of switching between frame and field + macroblocks within frames. When mb_adaptive_frame_field_flag + is not present, it shall be inferred to be equal to 0. + */ + bool mb_adaptive_frame_field_flag; + + /* + direct_8x8_inference_flag specifies the method used in the derivation + process for luma motion vectors for B_Skip, B_Direct_16x16 and B_Direct_8x8 + as specified in clause 8.4.1.2. When frame_mbs_only_flag is equal to 0, + direct_8x8_inference_flag shall be equal to 1. + */ + bool direct_8x8_inference_flag; + + /* + frame_cropping_flag equal to 1 specifies that the frame cropping + offset parameters follow next in the sequence parameter + set. frame_cropping_flag equal to 0 specifies that the frame + cropping offset parameters are not present. + */ + bool frame_cropping_flag; + uint32_t frame_crop_left_offset; + uint32_t frame_crop_right_offset; + uint32_t frame_crop_top_offset; + uint32_t frame_crop_bottom_offset; + + // VUI Parameters + + /* + vui_parameters_present_flag equal to 1 specifies that the + vui_parameters( ) syntax structure as specified in Annex E is + present. vui_parameters_present_flag equal to 0 specifies that + the vui_parameters( ) syntax structure as specified in Annex E + is not present. + */ + bool vui_parameters_present_flag; + + /* + aspect_ratio_info_present_flag equal to 1 specifies that + aspect_ratio_idc is present. aspect_ratio_info_present_flag + equal to 0 specifies that aspect_ratio_idc is not present. + */ + bool aspect_ratio_info_present_flag; + + /* + aspect_ratio_idc specifies the value of the sample aspect + ratio of the luma samples. Table E-1 shows the meaning of + the code. When aspect_ratio_idc indicates Extended_SAR, the + sample aspect ratio is represented by sar_width and + sar_height. When the aspect_ratio_idc syntax element is not + present, aspect_ratio_idc value shall be inferred to be + equal to 0. + */ + uint8_t aspect_ratio_idc; + uint32_t sar_width; + uint32_t sar_height; + + /* + video_signal_type_present_flag equal to 1 specifies that video_format, + video_full_range_flag and colour_description_present_flag are present. + video_signal_type_present_flag equal to 0, specify that video_format, + video_full_range_flag and colour_description_present_flag are not present. + */ + bool video_signal_type_present_flag; + + /* + overscan_info_present_flag equal to1 specifies that the + overscan_appropriate_flag is present. When overscan_info_present_flag is + equal to 0 or is not present, the preferred display method for the video + signal is unspecified (Unspecified). + */ + bool overscan_info_present_flag; + /* + overscan_appropriate_flag equal to 1 indicates that the cropped decoded + pictures output are suitable for display using overscan. + overscan_appropriate_flag equal to 0 indicates that the cropped decoded + pictures output contain visually important information in the entire region + out to the edges of the cropping rectangle of the picture + */ + bool overscan_appropriate_flag; + + /* + video_format indicates the representation of the pictures as specified in + Table E-2, before being coded in accordance with this + Recommendation | International Standard. When the video_format syntax + element is not present, video_format value shall be inferred to be equal + to 5. (Unspecified video format) + */ + uint8_t video_format; + + /* + video_full_range_flag indicates the black level and range of the luma and + chroma signals as derived from E′Y, E′PB, and E′PR or E′R, E′G, and E′B + real-valued component signals. + When the video_full_range_flag syntax element is not present, the value of + video_full_range_flag shall be inferred to be equal to 0. + */ + bool video_full_range_flag; + + /* + colour_description_present_flag equal to1 specifies that colour_primaries, + transfer_characteristics and matrix_coefficients are present. + colour_description_present_flag equal to 0 specifies that colour_primaries, + transfer_characteristics and matrix_coefficients are not present. + */ + bool colour_description_present_flag; + + /* + colour_primaries indicates the chromaticity coordinates of the source + primaries as specified in Table E-3 in terms of the CIE 1931 definition of + x and y as specified by ISO 11664-1. + When the colour_primaries syntax element is not present, the value of + colour_primaries shall be inferred to be equal to 2 (the chromaticity is + unspecified or is determined by the application). + */ + uint8_t colour_primaries; + + /* + transfer_characteristics indicates the opto-electronic transfer + characteristic of the source picture as specified in Table E-4 as a function + of a linear optical intensity input Lc with a nominal real-valued range of 0 + to 1. + When the transfer_characteristics syntax element is not present, the value + of transfer_characteristics shall be inferred to be equal to 2 + (the transfer characteristics are unspecified or are determined by the + application). + */ + uint8_t transfer_characteristics; + + uint8_t matrix_coefficients; + bool chroma_loc_info_present_flag; + /* + The value of chroma_sample_loc_type_top_field and + chroma_sample_loc_type_bottom_field shall be in the range of 0 to 5, + inclusive + */ + uint8_t chroma_sample_loc_type_top_field; + uint8_t chroma_sample_loc_type_bottom_field; + + bool scaling_matrix_present; + uint8_t scaling_matrix4x4[6][16]; + uint8_t scaling_matrix8x8[6][64]; + + SPSData(); +}; + +struct SEIRecoveryData { + /* + recovery_frame_cnt specifies the recovery point of output pictures in output + order. All decoded pictures in output order are indicated to be correct or + approximately correct in content starting at the output order position of + the reference picture having the frame_num equal to the frame_num of the VCL + NAL units for the current access unit incremented by recovery_frame_cnt in + modulo MaxFrameNum arithmetic. recovery_frame_cnt shall be in the range of 0 + to MaxFrameNum − 1, inclusive. + */ + uint32_t recovery_frame_cnt = 0; + /* + exact_match_flag indicates whether decoded pictures at and subsequent to the + specified recovery point in output order derived by starting the decoding + process at the access unit associated with the recovery point SEI message + shall be an exact match to the pictures that would be produced by starting + the decoding process at the location of a previous IDR access unit in the + NAL unit stream. The value 0 indicates that the match need not be exact and + the value 1 indicates that the match shall be exact. + */ + bool exact_match_flag = false; + /* + broken_link_flag indicates the presence or absence of a broken link in the + NAL unit stream at the location of the recovery point SEI message */ + bool broken_link_flag = false; + /* + changing_slice_group_idc equal to 0 indicates that decoded pictures are + correct or approximately correct in content at and subsequent to the + recovery point in output order when all macroblocks of the primary coded + pictures are decoded within the changing slice group period + */ + uint8_t changing_slice_group_idc = 0; +}; + +class H264 { + public: + /* Check if out of band extradata contains a SPS NAL */ + static bool HasSPS(const mozilla::MediaByteBuffer* aExtraData); + // Extract SPS and PPS NALs from aSample by looking into each NALs. + // aSample must be in AVCC format. + static already_AddRefed<mozilla::MediaByteBuffer> ExtractExtraData( + const mozilla::MediaRawData* aSample); + // Return true if both extradata are equal. + static bool CompareExtraData(const mozilla::MediaByteBuffer* aExtraData1, + const mozilla::MediaByteBuffer* aExtraData2); + + // Ensure that SPS data makes sense, Return true if SPS data was, and false + // otherwise. If false, then content will be adjusted accordingly. + static bool EnsureSPSIsSane(SPSData& aSPS); + + static bool DecodeSPSFromExtraData(const mozilla::MediaByteBuffer* aExtraData, + SPSData& aDest); + /* Decode SPS NAL RBSP and fill SPSData structure */ + static bool DecodeSPS(const mozilla::MediaByteBuffer* aSPS, SPSData& aDest); + + // If the given aExtraData is valid, return the aExtraData.max_num_ref_frames + // clamped to be in the range of [4, 16]; otherwise return 4. + static uint32_t ComputeMaxRefFrames( + const mozilla::MediaByteBuffer* aExtraData); + + enum class FrameType { + I_FRAME, + OTHER, + INVALID, + }; + + // Returns the frame type. Returns I_FRAME if the sample is an IDR + // (Instantaneous Decoding Refresh) Picture. + static FrameType GetFrameType(const mozilla::MediaRawData* aSample); + // Create a dummy extradata, useful to create a decoder and test the + // capabilities of the decoder. + static already_AddRefed<mozilla::MediaByteBuffer> CreateExtraData( + uint8_t aProfile, uint8_t aConstraints, uint8_t aLevel, + const gfx::IntSize& aSize); + static void WriteExtraData(mozilla::MediaByteBuffer* aDestExtraData, + const uint8_t aProfile, const uint8_t aConstraints, + const uint8_t aLevel, + const Span<const uint8_t> aSPS, + const Span<const uint8_t> aPPS); + + private: + friend class SPSNAL; + /* Extract RAW BYTE SEQUENCE PAYLOAD from NAL content. + Returns nullptr if invalid content. + This is compliant to ITU H.264 7.3.1 Syntax in tabular form NAL unit syntax + */ + static already_AddRefed<mozilla::MediaByteBuffer> DecodeNALUnit( + const uint8_t* aNAL, size_t aLength); + static already_AddRefed<mozilla::MediaByteBuffer> EncodeNALUnit( + const uint8_t* aNAL, size_t aLength); + static bool vui_parameters(mozilla::BitReader& aBr, SPSData& aDest); + // Read HRD parameters, all data is ignored. + static void hrd_parameters(mozilla::BitReader& aBr); + static uint8_t NumSPS(const mozilla::MediaByteBuffer* aExtraData); + // Decode SEI payload and return true if the SEI NAL indicates a recovery + // point. + static bool DecodeRecoverySEI(const mozilla::MediaByteBuffer* aSEI, + SEIRecoveryData& aDest); + // Decode NAL Slice payload and return true if its slice type is I slice or SI + // slice. + static bool DecodeISlice(const mozilla::MediaByteBuffer* aSlice); +}; + +} // namespace mozilla + +#endif // MP4_DEMUXER_H264_H_ diff --git a/dom/media/platforms/agnostic/bytestreams/gtest/TestAnnexB.cpp b/dom/media/platforms/agnostic/bytestreams/gtest/TestAnnexB.cpp new file mode 100644 index 0000000000..abc4b2ae8d --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/gtest/TestAnnexB.cpp @@ -0,0 +1,144 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#include "gtest/gtest.h" + +#include "AnnexB.h" +#include "ByteWriter.h" +#include "H264.h" + +namespace mozilla { + +// Create AVCC style extra data (the contents on an AVCC box). Note +// NALLengthSize will be 4 so AVCC samples need to set their data up +// accordingly. +static already_AddRefed<MediaByteBuffer> GetExtraData() { + // Extra data with + // - baseline profile(0x42 == 66). + // - constraint flags 0 and 1 set(0xc0) -- normal for baseline profile. + // - level 4.0 (0x28 == 40). + // - 1280 * 720 resolution. + return H264::CreateExtraData(0x42, 0xc0, 0x28, {1280, 720}); +} + +// Create an AVCC style sample with requested size in bytes. This sample is +// setup to contain a single NAL (in practice samples can contain many). The +// sample sets its NAL size to aSampleSize - 4 and stores that size in the first +// 4 bytes. Aside from the NAL size at the start, the data is uninitialized +// (beware)! aSampleSize is a uint32_t as samples larger than can be expressed +// by a uint32_t are not to spec. +static already_AddRefed<MediaRawData> GetAvccSample(uint32_t aSampleSize) { + if (aSampleSize < 4) { + // Stop tests asking for insane samples. + EXPECT_FALSE(true) << "Samples should be requested with sane sizes"; + } + nsTArray<uint8_t> sampleData; + + // Write the NAL size. + ByteWriter<BigEndian> writer(sampleData); + EXPECT_TRUE(writer.WriteU32(aSampleSize - 4)); + + // Write the 'NAL'. Beware, this data is uninitialized. + sampleData.AppendElements(static_cast<size_t>(aSampleSize) - 4); + RefPtr<MediaRawData> rawData = + new MediaRawData{sampleData.Elements(), sampleData.Length()}; + EXPECT_NE(rawData->Data(), nullptr); + + // Set extra data. + rawData->mExtraData = GetExtraData(); + return rawData.forget(); +} + +// Test that conversion from AVCC to AnnexB works as expected. +TEST(AnnexB, AnnexBConversion) +{ + RefPtr<MediaRawData> rawData{GetAvccSample(128)}; + + { + // Test conversion of data when not adding SPS works as expected. + RefPtr<MediaRawData> rawDataClone = rawData->Clone(); + Result<Ok, nsresult> result = + AnnexB::ConvertSampleToAnnexB(rawDataClone, /* aAddSps */ false); + EXPECT_TRUE(result.isOk()) << "Conversion should succeed"; + EXPECT_EQ(rawDataClone->Size(), rawData->Size()) + << "AnnexB sample should be the same size as the AVCC sample -- the 4 " + "byte NAL length data (AVCC) is replaced with 4 bytes of NAL " + "separator (AnnexB)"; + EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone)) + << "The sample should be AnnexB following conversion"; + } + + { + // Test that the SPS data is not added if the frame is not a keyframe. + RefPtr<MediaRawData> rawDataClone = rawData->Clone(); + rawDataClone->mKeyframe = + false; // false is the default, but let's be sure. + Result<Ok, nsresult> result = + AnnexB::ConvertSampleToAnnexB(rawDataClone, /* aAddSps */ true); + EXPECT_TRUE(result.isOk()) << "Conversion should succeed"; + EXPECT_EQ(rawDataClone->Size(), rawData->Size()) + << "AnnexB sample should be the same size as the AVCC sample -- the 4 " + "byte NAL length data (AVCC) is replaced with 4 bytes of NAL " + "separator (AnnexB) and SPS data is not added as the frame is not a " + "keyframe"; + EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone)) + << "The sample should be AnnexB following conversion"; + } + + { + // Test that the SPS data is added to keyframes. + RefPtr<MediaRawData> rawDataClone = rawData->Clone(); + rawDataClone->mKeyframe = true; + Result<Ok, nsresult> result = + AnnexB::ConvertSampleToAnnexB(rawDataClone, /* aAddSps */ true); + EXPECT_TRUE(result.isOk()) << "Conversion should succeed"; + EXPECT_GT(rawDataClone->Size(), rawData->Size()) + << "AnnexB sample should be larger than the AVCC sample because we've " + "added SPS data"; + EXPECT_TRUE(AnnexB::IsAnnexB(rawDataClone)) + << "The sample should be AnnexB following conversion"; + // We could verify the SPS and PPS data we add, but we don't have great + // tooling to do so. Consider doing so in future. + } + + { + // Test conversion involving subsample encryption doesn't overflow vlaues. + const uint32_t sampleSize = UINT16_MAX * 2; + RefPtr<MediaRawData> rawCryptoData{GetAvccSample(sampleSize)}; + // Need to be a keyframe to test prepending SPS + PPS to sample. + rawCryptoData->mKeyframe = true; + UniquePtr<MediaRawDataWriter> rawDataWriter = rawCryptoData->CreateWriter(); + + rawDataWriter->mCrypto.mCryptoScheme = CryptoScheme::Cenc; + + // We want to check that the clear size doesn't overflow during conversion. + // This size originates in a uint16_t, but since it can grow during AnnexB + // we cover it here. + const uint16_t clearSize = UINT16_MAX - 10; + // Set a clear size very close to uint16_t max value. + rawDataWriter->mCrypto.mPlainSizes.AppendElement(clearSize); + rawDataWriter->mCrypto.mEncryptedSizes.AppendElement(sampleSize - + clearSize); + + RefPtr<MediaRawData> rawCryptoDataClone = rawCryptoData->Clone(); + Result<Ok, nsresult> result = + AnnexB::ConvertSampleToAnnexB(rawCryptoDataClone, /* aAddSps */ true); + EXPECT_TRUE(result.isOk()) << "Conversion should succeed"; + EXPECT_GT(rawCryptoDataClone->Size(), rawCryptoData->Size()) + << "AnnexB sample should be larger than the AVCC sample because we've " + "added SPS data"; + EXPECT_GT(rawCryptoDataClone->mCrypto.mPlainSizes[0], + rawCryptoData->mCrypto.mPlainSizes[0]) + << "Conversion should have increased clear data sizes without overflow"; + EXPECT_EQ(rawCryptoDataClone->mCrypto.mEncryptedSizes[0], + rawCryptoData->mCrypto.mEncryptedSizes[0]) + << "Conversion should not affect encrypted sizes"; + EXPECT_TRUE(AnnexB::IsAnnexB(rawCryptoDataClone)) + << "The sample should be AnnexB following conversion"; + } +} + +} // namespace mozilla diff --git a/dom/media/platforms/agnostic/bytestreams/gtest/moz.build b/dom/media/platforms/agnostic/bytestreams/gtest/moz.build new file mode 100644 index 0000000000..be0bb9b3ae --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/gtest/moz.build @@ -0,0 +1,11 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+UNIFIED_SOURCES += [
+ "TestAnnexB.cpp",
+]
+
+FINAL_LIBRARY = "xul-gtest"
diff --git a/dom/media/platforms/agnostic/bytestreams/moz.build b/dom/media/platforms/agnostic/bytestreams/moz.build new file mode 100644 index 0000000000..225cb427f8 --- /dev/null +++ b/dom/media/platforms/agnostic/bytestreams/moz.build @@ -0,0 +1,35 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +with Files("**"): + BUG_COMPONENT = ("Core", "Audio/Video: Playback") + +TEST_DIRS += [ + "gtest", +] + +EXPORTS += [ + "Adts.h", + "AnnexB.h", + "H264.h", +] + +UNIFIED_SOURCES += [ + "Adts.cpp", + "AnnexB.cpp", + "H264.cpp", +] + +LOCAL_INCLUDES += [ + "../../../mp4/", +] + +FINAL_LIBRARY = "xul" + +# Suppress warnings for now. +CXXFLAGS += [ + "-Wno-sign-compare", +] |