1 files changed, 557 insertions, 0 deletions
diff --git a/dom/media/mp4/MP4Demuxer.cpp b/dom/media/mp4/MP4Demuxer.cpp
new file mode 100644
index 0000000000..59bba16577
--- /dev/null
+++ b/dom/media/mp4/MP4Demuxer.cpp
@@ -0,0 +1,557 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include <algorithm>
+#include <limits>
+#include <stdint.h>
+
+#include "MP4Demuxer.h"
+
+#include "AnnexB.h"
+#include "BufferStream.h"
+#include "H264.h"
+#include "Index.h"
+#include "MP4Decoder.h"
+#include "MP4Metadata.h"
+#include "MoofParser.h"
+#include "ResourceStream.h"
+#include "VPXDecoder.h"
+#include "mozilla/Span.h"
+#include "mozilla/StaticPrefs_media.h"
+#include "mozilla/Telemetry.h"
+#include "nsPrintfCString.h"
+
+extern mozilla::LazyLogModule gMediaDemuxerLog;
+mozilla::LogModule* GetDemuxerLog() { return gMediaDemuxerLog; }
+
+#define LOG(arg, ...)                                                 \
+  DDMOZ_LOG(gMediaDemuxerLog, mozilla::LogLevel::Debug, "::%s: " arg, \
+            __func__, ##__VA_ARGS__)
+
+namespace mozilla {
+
+DDLoggedTypeDeclNameAndBase(MP4TrackDemuxer, MediaTrackDemuxer);
+
+class MP4TrackDemuxer : public MediaTrackDemuxer,
+                        public DecoderDoctorLifeLogger<MP4TrackDemuxer> {
+ public:
+  MP4TrackDemuxer(MediaResource* aResource, UniquePtr<TrackInfo>&& aInfo,
+                  const IndiceWrapper& aIndices);
+
+  UniquePtr<TrackInfo> GetInfo() const override;
+
+  RefPtr<SeekPromise> Seek(const media::TimeUnit& aTime) override;
+
+  RefPtr<SamplesPromise> GetSamples(int32_t aNumSamples = 1) override;
+
+  void Reset() override;
+
+  nsresult GetNextRandomAccessPoint(media::TimeUnit* aTime) override;
+
+  RefPtr<SkipAccessPointPromise> SkipToNextRandomAccessPoint(
+      const media::TimeUnit& aTimeThreshold) override;
+
+  media::TimeIntervals GetBuffered() override;
+
+  void NotifyDataRemoved();
+  void NotifyDataArrived();
+
+ private:
+  already_AddRefed<MediaRawData> GetNextSample();
+  void EnsureUpToDateIndex();
+  void SetNextKeyFrameTime();
+  RefPtr<MediaResource> mResource;
+  RefPtr<ResourceStream> mStream;
+  UniquePtr<TrackInfo> mInfo;
+  RefPtr<Index> mIndex;
+  UniquePtr<SampleIterator> mIterator;
+  Maybe<media::TimeUnit> mNextKeyframeTime;
+  // Queued samples extracted by the demuxer, but not yet returned.
+  RefPtr<MediaRawData> mQueuedSample;
+  bool mNeedReIndex;
+  enum CodecType { kH264, kVP9, kOther } mType = kOther;
+};
+
+MP4Demuxer::MP4Demuxer(MediaResource* aResource)
+    : mResource(aResource),
+      mStream(new ResourceStream(aResource)),
+      mIsSeekable(false) {
+  DDLINKCHILD("resource", aResource);
+  DDLINKCHILD("stream", mStream.get());
+}
+
+RefPtr<MP4Demuxer::InitPromise> MP4Demuxer::Init() {
+  AutoPinned<ResourceStream> stream(mStream);
+
+  // 'result' will capture the first warning, if any.
+  MediaResult result{NS_OK};
+
+  MP4Metadata::ResultAndByteBuffer initData = MP4Metadata::Metadata(stream);
+  if (!initData.Ref()) {
+    return InitPromise::CreateAndReject(
+        NS_FAILED(initData.Result())
+            ? std::move(initData.Result())
+            : MediaResult(NS_ERROR_DOM_MEDIA_DEMUXER_ERR,
+                          RESULT_DETAIL("Invalid MP4 metadata or OOM")),
+        __func__);
+  } else if (NS_FAILED(initData.Result()) && result == NS_OK) {
+    result = std::move(initData.Result());
+  }
+
+  RefPtr<BufferStream> bufferstream = new BufferStream(initData.Ref());
+
+  MP4Metadata metadata{bufferstream};
+  DDLINKCHILD("metadata", &metadata);
+  nsresult rv = metadata.Parse();
+  if (NS_FAILED(rv)) {
+    return InitPromise::CreateAndReject(
+        MediaResult(rv, RESULT_DETAIL("Parse MP4 metadata failed")), __func__);
+  }
+
+  auto audioTrackCount = metadata.GetNumberTracks(TrackInfo::kAudioTrack);
+  if (audioTrackCount.Ref() == MP4Metadata::NumberTracksError()) {
+    if (StaticPrefs::media_playback_warnings_as_errors()) {
+      return InitPromise::CreateAndReject(
+          MediaResult(
+              NS_ERROR_DOM_MEDIA_DEMUXER_ERR,
+              RESULT_DETAIL("Invalid audio track (%s)",
+                            audioTrackCount.Result().Description().get())),
+          __func__);
+    }
+    audioTrackCount.Ref() = 0;
+  }
+
+  auto videoTrackCount = metadata.GetNumberTracks(TrackInfo::kVideoTrack);
+  if (videoTrackCount.Ref() == MP4Metadata::NumberTracksError()) {
+    if (StaticPrefs::media_playback_warnings_as_errors()) {
+      return InitPromise::CreateAndReject(
+          MediaResult(
+              NS_ERROR_DOM_MEDIA_DEMUXER_ERR,
+              RESULT_DETAIL("Invalid video track (%s)",
+                            videoTrackCount.Result().Description().get())),
+          __func__);
+    }
+    videoTrackCount.Ref() = 0;
+  }
+
+  if (audioTrackCount.Ref() == 0 && videoTrackCount.Ref() == 0) {
+    return InitPromise::CreateAndReject(
+        MediaResult(
+            NS_ERROR_DOM_MEDIA_DEMUXER_ERR,
+            RESULT_DETAIL("No MP4 audio (%s) or video (%s) tracks",
+                          audioTrackCount.Result().Description().get(),
+                          videoTrackCount.Result().Description().get())),
+        __func__);
+  }
+
+  if (NS_FAILED(audioTrackCount.Result()) && result == NS_OK) {
+    result = std::move(audioTrackCount.Result());
+  }
+  if (NS_FAILED(videoTrackCount.Result()) && result == NS_OK) {
+    result = std::move(videoTrackCount.Result());
+  }
+
+  if (audioTrackCount.Ref() != 0) {
+    for (size_t i = 0; i < audioTrackCount.Ref(); i++) {
+      MP4Metadata::ResultAndTrackInfo info =
+          metadata.GetTrackInfo(TrackInfo::kAudioTrack, i);
+      if (!info.Ref()) {
+        if (StaticPrefs::media_playback_warnings_as_errors()) {
+          return InitPromise::CreateAndReject(
+              MediaResult(NS_ERROR_DOM_MEDIA_DEMUXER_ERR,
+                          RESULT_DETAIL("Invalid MP4 audio track (%s)",
+                                        info.Result().Description().get())),
+              __func__);
+        }
+        if (result == NS_OK) {
+          result =
+              MediaResult(NS_ERROR_DOM_MEDIA_DEMUXER_ERR,
+                          RESULT_DETAIL("Invalid MP4 audio track (%s)",
+                                        info.Result().Description().get()));
+        }
+        continue;
+      } else if (NS_FAILED(info.Result()) && result == NS_OK) {
+        result = std::move(info.Result());
+      }
+      MP4Metadata::ResultAndIndice indices =
+          metadata.GetTrackIndice(info.Ref()->mTrackId);
+      if (!indices.Ref()) {
+        if (NS_FAILED(info.Result()) && result == NS_OK) {
+          result = std::move(indices.Result());
+        }
+        continue;
+      }
+      RefPtr<MP4TrackDemuxer> demuxer = new MP4TrackDemuxer(
+          mResource, std::move(info.Ref()), *indices.Ref().get());
+      DDLINKCHILD("audio demuxer", demuxer.get());
+      mAudioDemuxers.AppendElement(std::move(demuxer));
+    }
+  }
+
+  if (videoTrackCount.Ref() != 0) {
+    for (size_t i = 0; i < videoTrackCount.Ref(); i++) {
+      MP4Metadata::ResultAndTrackInfo info =
+          metadata.GetTrackInfo(TrackInfo::kVideoTrack, i);
+      if (!info.Ref()) {
+        if (StaticPrefs::media_playback_warnings_as_errors()) {
+          return InitPromise::CreateAndReject(
+              MediaResult(NS_ERROR_DOM_MEDIA_DEMUXER_ERR,
+                          RESULT_DETAIL("Invalid MP4 video track (%s)",
+                                        info.Result().Description().get())),
+              __func__);
+        }
+        if (result == NS_OK) {
+          result =
+              MediaResult(NS_ERROR_DOM_MEDIA_DEMUXER_ERR,
+                          RESULT_DETAIL("Invalid MP4 video track (%s)",
+                                        info.Result().Description().get()));
+        }
+        continue;
+      } else if (NS_FAILED(info.Result()) && result == NS_OK) {
+        result = std::move(info.Result());
+      }
+      MP4Metadata::ResultAndIndice indices =
+          metadata.GetTrackIndice(info.Ref()->mTrackId);
+      if (!indices.Ref()) {
+        if (NS_FAILED(info.Result()) && result == NS_OK) {
+          result = std::move(indices.Result());
+        }
+        continue;
+      }
+      RefPtr<MP4TrackDemuxer> demuxer = new MP4TrackDemuxer(
+          mResource, std::move(info.Ref()), *indices.Ref().get());
+      DDLINKCHILD("video demuxer", demuxer.get());
+      mVideoDemuxers.AppendElement(std::move(demuxer));
+    }
+  }
+
+  MP4Metadata::ResultAndCryptoFile cryptoFile = metadata.Crypto();
+  if (NS_FAILED(cryptoFile.Result()) && result == NS_OK) {
+    result = std::move(cryptoFile.Result());
+  }
+  MOZ_ASSERT(cryptoFile.Ref());
+  if (cryptoFile.Ref()->valid) {
+    const nsTArray<PsshInfo>& psshs = cryptoFile.Ref()->pssh;
+    for (uint32_t i = 0; i < psshs.Length(); i++) {
+      mCryptoInitData.AppendElements(psshs[i].data);
+    }
+  }
+
+  mIsSeekable = metadata.CanSeek();
+
+  return InitPromise::CreateAndResolve(result, __func__);
+}
+
+uint32_t MP4Demuxer::GetNumberTracks(TrackInfo::TrackType aType) const {
+  switch (aType) {
+    case TrackInfo::kAudioTrack:
+      return uint32_t(mAudioDemuxers.Length());
+    case TrackInfo::kVideoTrack:
+      return uint32_t(mVideoDemuxers.Length());
+    default:
+      return 0;
+  }
+}
+
+already_AddRefed<MediaTrackDemuxer> MP4Demuxer::GetTrackDemuxer(
+    TrackInfo::TrackType aType, uint32_t aTrackNumber) {
+  switch (aType) {
+    case TrackInfo::kAudioTrack:
+      if (aTrackNumber >= uint32_t(mAudioDemuxers.Length())) {
+        return nullptr;
+      }
+      return RefPtr<MediaTrackDemuxer>(mAudioDemuxers[aTrackNumber]).forget();
+    case TrackInfo::kVideoTrack:
+      if (aTrackNumber >= uint32_t(mVideoDemuxers.Length())) {
+        return nullptr;
+      }
+      return RefPtr<MediaTrackDemuxer>(mVideoDemuxers[aTrackNumber]).forget();
+    default:
+      return nullptr;
+  }
+}
+
+bool MP4Demuxer::IsSeekable() const { return mIsSeekable; }
+
+void MP4Demuxer::NotifyDataArrived() {
+  for (auto& dmx : mAudioDemuxers) {
+    dmx->NotifyDataArrived();
+  }
+  for (auto& dmx : mVideoDemuxers) {
+    dmx->NotifyDataArrived();
+  }
+}
+
+void MP4Demuxer::NotifyDataRemoved() {
+  for (auto& dmx : mAudioDemuxers) {
+    dmx->NotifyDataRemoved();
+  }
+  for (auto& dmx : mVideoDemuxers) {
+    dmx->NotifyDataRemoved();
+  }
+}
+
+UniquePtr<EncryptionInfo> MP4Demuxer::GetCrypto() {
+  UniquePtr<EncryptionInfo> crypto;
+  if (!mCryptoInitData.IsEmpty()) {
+    crypto.reset(new EncryptionInfo{});
+    crypto->AddInitData(u"cenc"_ns, mCryptoInitData);
+  }
+  return crypto;
+}
+
+MP4TrackDemuxer::MP4TrackDemuxer(MediaResource* aResource,
+                                 UniquePtr<TrackInfo>&& aInfo,
+                                 const IndiceWrapper& aIndices)
+    : mResource(aResource),
+      mStream(new ResourceStream(aResource)),
+      mInfo(std::move(aInfo)),
+      mIndex(new Index(aIndices, mStream, mInfo->mTrackId, mInfo->IsAudio())),
+      mIterator(MakeUnique<SampleIterator>(mIndex)),
+      mNeedReIndex(true) {
+  EnsureUpToDateIndex();  // Force update of index
+
+  VideoInfo* videoInfo = mInfo->GetAsVideoInfo();
+  if (videoInfo && MP4Decoder::IsH264(mInfo->mMimeType)) {
+    mType = kH264;
+    RefPtr<MediaByteBuffer> extraData = videoInfo->mExtraData;
+    SPSData spsdata;
+    if (H264::DecodeSPSFromExtraData(extraData, spsdata) &&
+        spsdata.pic_width > 0 && spsdata.pic_height > 0 &&
+        H264::EnsureSPSIsSane(spsdata)) {
+      videoInfo->mImage.width = spsdata.pic_width;
+      videoInfo->mImage.height = spsdata.pic_height;
+      videoInfo->mDisplay.width = spsdata.display_width;
+      videoInfo->mDisplay.height = spsdata.display_height;
+    }
+  } else {
+    if (videoInfo && VPXDecoder::IsVP9(mInfo->mMimeType)) {
+      mType = kVP9;
+    }
+  }
+}
+
+UniquePtr<TrackInfo> MP4TrackDemuxer::GetInfo() const { return mInfo->Clone(); }
+
+void MP4TrackDemuxer::EnsureUpToDateIndex() {
+  if (!mNeedReIndex) {
+    return;
+  }
+  AutoPinned<MediaResource> resource(mResource);
+  MediaByteRangeSet byteRanges;
+  nsresult rv = resource->GetCachedRanges(byteRanges);
+  if (NS_FAILED(rv)) {
+    return;
+  }
+  mIndex->UpdateMoofIndex(byteRanges);
+  mNeedReIndex = false;
+}
+
+RefPtr<MP4TrackDemuxer::SeekPromise> MP4TrackDemuxer::Seek(
+    const media::TimeUnit& aTime) {
+  auto seekTime = aTime;
+  mQueuedSample = nullptr;
+
+  mIterator->Seek(seekTime.ToMicroseconds());
+
+  // Check what time we actually seeked to.
+  do {
+    RefPtr<MediaRawData> sample = GetNextSample();
+    if (!sample) {
+      return SeekPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_END_OF_STREAM,
+                                          __func__);
+    }
+    if (!sample->Size()) {
+      // This sample can't be decoded, continue searching.
+      continue;
+    }
+    if (sample->mKeyframe) {
+      MOZ_DIAGNOSTIC_ASSERT(sample->HasValidTime());
+      mQueuedSample = sample;
+      seekTime = mQueuedSample->mTime;
+    }
+  } while (!mQueuedSample);
+
+  SetNextKeyFrameTime();
+
+  return SeekPromise::CreateAndResolve(seekTime, __func__);
+}
+
+already_AddRefed<MediaRawData> MP4TrackDemuxer::GetNextSample() {
+  RefPtr<MediaRawData> sample = mIterator->GetNext();
+  if (!sample) {
+    return nullptr;
+  }
+  if (mInfo->GetAsVideoInfo()) {
+    sample->mExtraData = mInfo->GetAsVideoInfo()->mExtraData;
+    if (mType == kH264 && !sample->mCrypto.IsEncrypted()) {
+      H264::FrameType type = H264::GetFrameType(sample);
+      switch (type) {
+        case H264::FrameType::I_FRAME:
+          [[fallthrough]];
+        case H264::FrameType::OTHER: {
+          bool keyframe = type == H264::FrameType::I_FRAME;
+          if (sample->mKeyframe != keyframe) {
+            NS_WARNING(nsPrintfCString("Frame incorrectly marked as %skeyframe "
+                                       "@ pts:%" PRId64 " dur:%" PRId64
+                                       " dts:%" PRId64,
+                                       keyframe ? "" : "non-",
+                                       sample->mTime.ToMicroseconds(),
+                                       sample->mDuration.ToMicroseconds(),
+                                       sample->mTimecode.ToMicroseconds())
+                           .get());
+            sample->mKeyframe = keyframe;
+          }
+          break;
+        }
+        case H264::FrameType::INVALID:
+          NS_WARNING(nsPrintfCString("Invalid H264 frame @ pts:%" PRId64
+                                     " dur:%" PRId64 " dts:%" PRId64,
+                                     sample->mTime.ToMicroseconds(),
+                                     sample->mDuration.ToMicroseconds(),
+                                     sample->mTimecode.ToMicroseconds())
+                         .get());
+          // We could reject the sample now, however demuxer errors are fatal.
+          // So we keep the invalid frame, relying on the H264 decoder to
+          // handle the error later.
+          // TODO: make demuxer errors non-fatal.
+          break;
+      }
+    } else if (mType == kVP9 && !sample->mCrypto.IsEncrypted()) {
+      bool keyframe = VPXDecoder::IsKeyframe(
+          Span<const uint8_t>(sample->Data(), sample->Size()),
+          VPXDecoder::Codec::VP9);
+      if (sample->mKeyframe != keyframe) {
+        NS_WARNING(nsPrintfCString(
+                       "Frame incorrectly marked as %skeyframe "
+                       "@ pts:%" PRId64 " dur:%" PRId64 " dts:%" PRId64,
+                       keyframe ? "" : "non-", sample->mTime.ToMicroseconds(),
+                       sample->mDuration.ToMicroseconds(),
+                       sample->mTimecode.ToMicroseconds())
+                       .get());
+        sample->mKeyframe = keyframe;
+      }
+    }
+  }
+
+  return sample.forget();
+}
+
+RefPtr<MP4TrackDemuxer::SamplesPromise> MP4TrackDemuxer::GetSamples(
+    int32_t aNumSamples) {
+  EnsureUpToDateIndex();
+  RefPtr<SamplesHolder> samples = new SamplesHolder;
+  if (!aNumSamples) {
+    return SamplesPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_DEMUXER_ERR,
+                                           __func__);
+  }
+
+  if (mQueuedSample) {
+    NS_ASSERTION(mQueuedSample->mKeyframe, "mQueuedSample must be a keyframe");
+    samples->AppendSample(mQueuedSample);
+    mQueuedSample = nullptr;
+    aNumSamples--;
+  }
+  RefPtr<MediaRawData> sample;
+  while (aNumSamples && (sample = GetNextSample())) {
+    if (!sample->Size()) {
+      continue;
+    }
+    MOZ_DIAGNOSTIC_ASSERT(sample->HasValidTime());
+    samples->AppendSample(sample);
+    aNumSamples--;
+  }
+
+  if (samples->GetSamples().IsEmpty()) {
+    return SamplesPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_END_OF_STREAM,
+                                           __func__);
+  }
+
+  if (mNextKeyframeTime.isNothing() ||
+      samples->GetSamples().LastElement()->mTime >= mNextKeyframeTime.value()) {
+    SetNextKeyFrameTime();
+  }
+  return SamplesPromise::CreateAndResolve(samples, __func__);
+}
+
+void MP4TrackDemuxer::SetNextKeyFrameTime() {
+  mNextKeyframeTime.reset();
+  Microseconds frameTime = mIterator->GetNextKeyframeTime();
+  if (frameTime != -1) {
+    mNextKeyframeTime.emplace(media::TimeUnit::FromMicroseconds(frameTime));
+  }
+}
+
+void MP4TrackDemuxer::Reset() {
+  mQueuedSample = nullptr;
+  // TODO, Seek to first frame available, which isn't always 0.
+  mIterator->Seek(0);
+  SetNextKeyFrameTime();
+}
+
+nsresult MP4TrackDemuxer::GetNextRandomAccessPoint(media::TimeUnit* aTime) {
+  if (mNextKeyframeTime.isNothing()) {
+    // There's no next key frame.
+    *aTime = media::TimeUnit::FromInfinity();
+  } else {
+    *aTime = mNextKeyframeTime.value();
+  }
+  return NS_OK;
+}
+
+RefPtr<MP4TrackDemuxer::SkipAccessPointPromise>
+MP4TrackDemuxer::SkipToNextRandomAccessPoint(
+    const media::TimeUnit& aTimeThreshold) {
+  mQueuedSample = nullptr;
+  // Loop until we reach the next keyframe after the threshold.
+  uint32_t parsed = 0;
+  bool found = false;
+  RefPtr<MediaRawData> sample;
+  while (!found && (sample = GetNextSample())) {
+    parsed++;
+    MOZ_DIAGNOSTIC_ASSERT(sample->HasValidTime());
+    if (sample->mKeyframe && sample->mTime >= aTimeThreshold) {
+      found = true;
+      mQueuedSample = sample;
+    }
+  }
+  SetNextKeyFrameTime();
+  if (found) {
+    return SkipAccessPointPromise::CreateAndResolve(parsed, __func__);
+  }
+  SkipFailureHolder failure(NS_ERROR_DOM_MEDIA_END_OF_STREAM, parsed);
+  return SkipAccessPointPromise::CreateAndReject(std::move(failure), __func__);
+}
+
+media::TimeIntervals MP4TrackDemuxer::GetBuffered() {
+  EnsureUpToDateIndex();
+  AutoPinned<MediaResource> resource(mResource);
+  MediaByteRangeSet byteRanges;
+  nsresult rv = resource->GetCachedRanges(byteRanges);
+
+  if (NS_FAILED(rv)) {
+    return media::TimeIntervals();
+  }
+
+  return mIndex->ConvertByteRangesToTimeRanges(byteRanges);
+}
+
+void MP4TrackDemuxer::NotifyDataArrived() { mNeedReIndex = true; }
+
+void MP4TrackDemuxer::NotifyDataRemoved() {
+  AutoPinned<MediaResource> resource(mResource);
+  MediaByteRangeSet byteRanges;
+  nsresult rv = resource->GetCachedRanges(byteRanges);
+  if (NS_FAILED(rv)) {
+    return;
+  }
+  mIndex->UpdateMoofIndex(byteRanges, true /* can evict */);
+  mNeedReIndex = false;
+}
+
+}  // namespace mozilla
+
+#undef LOG