Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 530 insertions, 0 deletions

diff --git a/dom/media/mediasource/MediaSourceDemuxer.cpp b/dom/media/mediasource/MediaSourceDemuxer.cpp
new file mode 100644
index 0000000000..9d08eda4fa
--- /dev/null
+++ b/dom/media/mediasource/MediaSourceDemuxer.cpp
@@ -0,0 +1,530 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "MediaSourceDemuxer.h"
+
+#include "MediaSourceUtils.h"
+#include "OpusDecoder.h"
+#include "SourceBufferList.h"
+#include "VorbisDecoder.h"
+#include "VideoUtils.h"
+#include "nsPrintfCString.h"
+
+#include <algorithm>
+#include <limits>
+#include <stdint.h>
+
+namespace mozilla {
+
+typedef TrackInfo::TrackType TrackType;
+using media::TimeIntervals;
+using media::TimeUnit;
+
+MediaSourceDemuxer::MediaSourceDemuxer(AbstractThread* aAbstractMainThread)
+    : mTaskQueue(
+          TaskQueue::Create(GetMediaThreadPool(MediaThreadType::SUPERVISOR),
+                            "MediaSourceDemuxer::mTaskQueue")),
+      mMonitor("MediaSourceDemuxer") {
+  MOZ_ASSERT(NS_IsMainThread());
+}
+
+constexpr TimeUnit MediaSourceDemuxer::EOS_FUZZ;
+constexpr TimeUnit MediaSourceDemuxer::EOS_FUZZ_START;
+
+RefPtr<MediaSourceDemuxer::InitPromise> MediaSourceDemuxer::Init() {
+  RefPtr<MediaSourceDemuxer> self = this;
+  return InvokeAsync(GetTaskQueue(), __func__, [self]() {
+    if (self->ScanSourceBuffersForContent()) {
+      return InitPromise::CreateAndResolve(NS_OK, __func__);
+    }
+
+    RefPtr<InitPromise> p = self->mInitPromise.Ensure(__func__);
+
+    return p;
+  });
+}
+
+void MediaSourceDemuxer::AddSizeOfResources(
+    MediaSourceDecoder::ResourceSizes* aSizes) {
+  MOZ_ASSERT(NS_IsMainThread());
+
+  // NB: The track buffers must only be accessed on the TaskQueue.
+  RefPtr<MediaSourceDemuxer> self = this;
+  RefPtr<MediaSourceDecoder::ResourceSizes> sizes = aSizes;
+  nsCOMPtr<nsIRunnable> task = NS_NewRunnableFunction(
+      "MediaSourceDemuxer::AddSizeOfResources", [self, sizes]() {
+        for (const RefPtr<TrackBuffersManager>& manager :
+             self->mSourceBuffers) {
+          manager->AddSizeOfResources(sizes);
+        }
+      });
+
+  nsresult rv = GetTaskQueue()->Dispatch(task.forget());
+  MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+  Unused << rv;
+}
+
+void MediaSourceDemuxer::NotifyInitDataArrived() {
+  RefPtr<MediaSourceDemuxer> self = this;
+  nsCOMPtr<nsIRunnable> task = NS_NewRunnableFunction(
+      "MediaSourceDemuxer::NotifyInitDataArrived", [self]() {
+        if (self->mInitPromise.IsEmpty()) {
+          return;
+        }
+        if (self->ScanSourceBuffersForContent()) {
+          self->mInitPromise.ResolveIfExists(NS_OK, __func__);
+        }
+      });
+  nsresult rv = GetTaskQueue()->Dispatch(task.forget());
+  MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+  Unused << rv;
+}
+
+bool MediaSourceDemuxer::ScanSourceBuffersForContent() {
+  MOZ_ASSERT(OnTaskQueue());
+
+  if (mSourceBuffers.IsEmpty()) {
+    return false;
+  }
+
+  MonitorAutoLock mon(mMonitor);
+
+  bool haveEmptySourceBuffer = false;
+  for (const auto& sourceBuffer : mSourceBuffers) {
+    MediaInfo info = sourceBuffer->GetMetadata();
+    if (!info.HasAudio() && !info.HasVideo()) {
+      haveEmptySourceBuffer = true;
+    }
+    if (info.HasAudio() && !mAudioTrack) {
+      mInfo.mAudio = info.mAudio;
+      mAudioTrack = sourceBuffer;
+    }
+    if (info.HasVideo() && !mVideoTrack) {
+      mInfo.mVideo = info.mVideo;
+      mVideoTrack = sourceBuffer;
+    }
+    if (info.IsEncrypted() && !mInfo.IsEncrypted()) {
+      mInfo.mCrypto = info.mCrypto;
+    }
+  }
+  if (mInfo.HasAudio() && mInfo.HasVideo()) {
+    // We have both audio and video. We can ignore non-ready source buffer.
+    return true;
+  }
+  return !haveEmptySourceBuffer;
+}
+
+uint32_t MediaSourceDemuxer::GetNumberTracks(TrackType aType) const {
+  MonitorAutoLock mon(mMonitor);
+
+  switch (aType) {
+    case TrackType::kAudioTrack:
+      return mInfo.HasAudio() ? 1u : 0;
+    case TrackType::kVideoTrack:
+      return mInfo.HasVideo() ? 1u : 0;
+    default:
+      return 0;
+  }
+}
+
+already_AddRefed<MediaTrackDemuxer> MediaSourceDemuxer::GetTrackDemuxer(
+    TrackType aType, uint32_t aTrackNumber) {
+  MonitorAutoLock mon(mMonitor);
+  RefPtr<TrackBuffersManager> manager = GetManager(aType);
+  if (!manager) {
+    return nullptr;
+  }
+  RefPtr<MediaSourceTrackDemuxer> e =
+      new MediaSourceTrackDemuxer(this, aType, manager);
+  DDLINKCHILD("track demuxer", e.get());
+  mDemuxers.AppendElement(e);
+  return e.forget();
+}
+
+bool MediaSourceDemuxer::IsSeekable() const { return true; }
+
+UniquePtr<EncryptionInfo> MediaSourceDemuxer::GetCrypto() {
+  MonitorAutoLock mon(mMonitor);
+  auto crypto = MakeUnique<EncryptionInfo>();
+  *crypto = mInfo.mCrypto;
+  return crypto;
+}
+
+void MediaSourceDemuxer::AttachSourceBuffer(
+    RefPtr<TrackBuffersManager>& aSourceBuffer) {
+  nsCOMPtr<nsIRunnable> task = NewRunnableMethod<RefPtr<TrackBuffersManager>&&>(
+      "MediaSourceDemuxer::DoAttachSourceBuffer", this,
+      &MediaSourceDemuxer::DoAttachSourceBuffer, aSourceBuffer);
+  nsresult rv = GetTaskQueue()->Dispatch(task.forget());
+  MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+  Unused << rv;
+}
+
+void MediaSourceDemuxer::DoAttachSourceBuffer(
+    RefPtr<mozilla::TrackBuffersManager>&& aSourceBuffer) {
+  MOZ_ASSERT(OnTaskQueue());
+  mSourceBuffers.AppendElement(std::move(aSourceBuffer));
+  ScanSourceBuffersForContent();
+}
+
+void MediaSourceDemuxer::DetachSourceBuffer(
+    RefPtr<TrackBuffersManager>& aSourceBuffer) {
+  nsCOMPtr<nsIRunnable> task =
+      NS_NewRunnableFunction("MediaSourceDemuxer::DoDetachSourceBuffer",
+                             [self = RefPtr{this}, aSourceBuffer]() {
+                               self->DoDetachSourceBuffer(aSourceBuffer);
+                             });
+  nsresult rv = GetTaskQueue()->Dispatch(task.forget());
+  MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+  Unused << rv;
+}
+
+void MediaSourceDemuxer::DoDetachSourceBuffer(
+    const RefPtr<TrackBuffersManager>& aSourceBuffer) {
+  MOZ_ASSERT(OnTaskQueue());
+  mSourceBuffers.RemoveElementsBy(
+      [&aSourceBuffer](const RefPtr<TrackBuffersManager> aLinkedSourceBuffer) {
+        return aLinkedSourceBuffer == aSourceBuffer;
+      });
+
+  AutoTArray<RefPtr<MediaSourceTrackDemuxer>, 2> matchingDemuxers;
+  {
+    MonitorAutoLock mon(mMonitor);
+    if (aSourceBuffer == mAudioTrack) {
+      mAudioTrack = nullptr;
+    }
+    if (aSourceBuffer == mVideoTrack) {
+      mVideoTrack = nullptr;
+    }
+
+    mDemuxers.RemoveElementsBy(
+        [&](RefPtr<MediaSourceTrackDemuxer>& elementRef) {
+          if (!elementRef->HasManager(aSourceBuffer)) {
+            return false;
+          }
+          matchingDemuxers.AppendElement(std::move(elementRef));
+          return true;
+        });
+  }
+
+  for (MediaSourceTrackDemuxer* demuxer : matchingDemuxers) {
+    demuxer->DetachManager();
+  }
+  ScanSourceBuffersForContent();
+}
+
+TrackInfo* MediaSourceDemuxer::GetTrackInfo(TrackType aTrack) {
+  switch (aTrack) {
+    case TrackType::kAudioTrack:
+      return &mInfo.mAudio;
+    case TrackType::kVideoTrack:
+      return &mInfo.mVideo;
+    default:
+      return nullptr;
+  }
+}
+
+RefPtr<TrackBuffersManager> MediaSourceDemuxer::GetManager(TrackType aTrack) {
+  switch (aTrack) {
+    case TrackType::kAudioTrack:
+      return mAudioTrack;
+    case TrackType::kVideoTrack:
+      return mVideoTrack;
+    default:
+      return nullptr;
+  }
+}
+
+MediaSourceDemuxer::~MediaSourceDemuxer() {
+  mInitPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+}
+
+RefPtr<GenericPromise> MediaSourceDemuxer::GetDebugInfo(
+    dom::MediaSourceDemuxerDebugInfo& aInfo) const {
+  MonitorAutoLock mon(mMonitor);
+  nsTArray<RefPtr<GenericPromise>> promises;
+  if (mAudioTrack) {
+    promises.AppendElement(mAudioTrack->RequestDebugInfo(aInfo.mAudioTrack));
+  }
+  if (mVideoTrack) {
+    promises.AppendElement(mVideoTrack->RequestDebugInfo(aInfo.mVideoTrack));
+  }
+  return GenericPromise::All(GetCurrentSerialEventTarget(), promises)
+      ->Then(
+          GetCurrentSerialEventTarget(), __func__,
+          []() { return GenericPromise::CreateAndResolve(true, __func__); },
+          [] {
+            return GenericPromise::CreateAndReject(NS_ERROR_FAILURE, __func__);
+          });
+}
+
+MediaSourceTrackDemuxer::MediaSourceTrackDemuxer(MediaSourceDemuxer* aParent,
+                                                 TrackInfo::TrackType aType,
+                                                 TrackBuffersManager* aManager)
+    : mParent(aParent),
+      mTaskQueue(mParent->GetTaskQueue()),
+      mType(aType),
+      mMonitor("MediaSourceTrackDemuxer"),
+      mManager(aManager),
+      mReset(true),
+      mPreRoll(TimeUnit::FromMicroseconds(
+          OpusDataDecoder::IsOpus(mParent->GetTrackInfo(mType)->mMimeType) ||
+                  VorbisDataDecoder::IsVorbis(
+                      mParent->GetTrackInfo(mType)->mMimeType)
+              ? 80000
+          : mParent->GetTrackInfo(mType)->mMimeType.EqualsLiteral(
+                "audio/mp4a-latm")
+              // AAC encoder delay is by default 2112 audio frames.
+              // See
+              // https://developer.apple.com/library/content/documentation/QuickTime/QTFF/QTFFAppenG/QTFFAppenG.html
+              // So we always seek 2112 frames
+              ? (2112 * 1000000ULL /
+                 mParent->GetTrackInfo(mType)->GetAsAudioInfo()->mRate)
+              : 0)) {
+  MOZ_ASSERT(mParent);
+  MOZ_ASSERT(mTaskQueue);
+}
+
+UniquePtr<TrackInfo> MediaSourceTrackDemuxer::GetInfo() const {
+  MonitorAutoLock mon(mParent->mMonitor);
+  return mParent->GetTrackInfo(mType)->Clone();
+}
+
+RefPtr<MediaSourceTrackDemuxer::SeekPromise> MediaSourceTrackDemuxer::Seek(
+    const TimeUnit& aTime) {
+  MOZ_ASSERT(mParent, "Called after BreackCycle()");
+  return InvokeAsync(mParent->GetTaskQueue(), this, __func__,
+                     &MediaSourceTrackDemuxer::DoSeek, aTime);
+}
+
+RefPtr<MediaSourceTrackDemuxer::SamplesPromise>
+MediaSourceTrackDemuxer::GetSamples(int32_t aNumSamples) {
+  MOZ_ASSERT(mParent, "Called after BreackCycle()");
+  return InvokeAsync(mParent->GetTaskQueue(), this, __func__,
+                     &MediaSourceTrackDemuxer::DoGetSamples, aNumSamples);
+}
+
+void MediaSourceTrackDemuxer::Reset() {
+  MOZ_ASSERT(mParent, "Called after BreackCycle()");
+  RefPtr<MediaSourceTrackDemuxer> self = this;
+  nsCOMPtr<nsIRunnable> task =
+      NS_NewRunnableFunction("MediaSourceTrackDemuxer::Reset", [self]() {
+        self->mNextSample.reset();
+        self->mReset = true;
+        if (!self->mManager) {
+          return;
+        }
+        MOZ_ASSERT(self->OnTaskQueue());
+        self->mManager->Seek(self->mType, TimeUnit::Zero(), TimeUnit::Zero());
+        {
+          MonitorAutoLock mon(self->mMonitor);
+          self->mNextRandomAccessPoint =
+              self->mManager->GetNextRandomAccessPoint(
+                  self->mType, MediaSourceDemuxer::EOS_FUZZ);
+        }
+      });
+  nsresult rv = mParent->GetTaskQueue()->Dispatch(task.forget());
+  MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+  Unused << rv;
+}
+
+nsresult MediaSourceTrackDemuxer::GetNextRandomAccessPoint(TimeUnit* aTime) {
+  MonitorAutoLock mon(mMonitor);
+  *aTime = mNextRandomAccessPoint;
+  return NS_OK;
+}
+
+RefPtr<MediaSourceTrackDemuxer::SkipAccessPointPromise>
+MediaSourceTrackDemuxer::SkipToNextRandomAccessPoint(
+    const TimeUnit& aTimeThreshold) {
+  return InvokeAsync(mParent->GetTaskQueue(), this, __func__,
+                     &MediaSourceTrackDemuxer::DoSkipToNextRandomAccessPoint,
+                     aTimeThreshold);
+}
+
+media::TimeIntervals MediaSourceTrackDemuxer::GetBuffered() {
+  MonitorAutoLock mon(mMonitor);
+  if (!mManager) {
+    return media::TimeIntervals();
+  }
+  return mManager->Buffered();
+}
+
+void MediaSourceTrackDemuxer::BreakCycles() {
+  RefPtr<MediaSourceTrackDemuxer> self = this;
+  nsCOMPtr<nsIRunnable> task =
+      NS_NewRunnableFunction("MediaSourceTrackDemuxer::BreakCycles", [self]() {
+        self->DetachManager();
+        self->mParent = nullptr;
+      });
+  nsresult rv = mParent->GetTaskQueue()->Dispatch(task.forget());
+  MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+  Unused << rv;
+}
+
+RefPtr<MediaSourceTrackDemuxer::SeekPromise> MediaSourceTrackDemuxer::DoSeek(
+    const TimeUnit& aTime) {
+  if (!mManager) {
+    return SeekPromise::CreateAndReject(
+        MediaResult(NS_ERROR_DOM_MEDIA_CANCELED,
+                    RESULT_DETAIL("manager is detached.")),
+        __func__);
+  }
+
+  MOZ_ASSERT(OnTaskQueue());
+  TimeIntervals buffered = mManager->Buffered(mType);
+  // Fuzz factor represents a +/- threshold. So when seeking it allows the gap
+  // to be twice as big as the fuzz value. We only want to allow EOS_FUZZ gap.
+  buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2);
+  TimeUnit seekTime = std::max(aTime - mPreRoll, TimeUnit::Zero());
+
+  if (mManager->IsEnded() && seekTime >= buffered.GetEnd()) {
+    // We're attempting to seek past the end time. Cap seekTime so that we seek
+    // to the last sample instead.
+    seekTime = std::max(mManager->HighestStartTime(mType) - mPreRoll,
+                        TimeUnit::Zero());
+  }
+  if (!buffered.ContainsWithStrictEnd(seekTime)) {
+    if (!buffered.ContainsWithStrictEnd(aTime)) {
+      // We don't have the data to seek to.
+      return SeekPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA,
+                                          __func__);
+    }
+    // Theoretically we should reject the promise with WAITING_FOR_DATA,
+    // however, to avoid unwanted regressions we assume that if at this time
+    // we don't have the wanted data it won't come later.
+    // Instead of using the pre-rolled time, use the earliest time available in
+    // the interval.
+    TimeIntervals::IndexType index = buffered.Find(aTime);
+    MOZ_ASSERT(index != TimeIntervals::NoIndex);
+    seekTime = buffered[index].mStart;
+  }
+  seekTime = mManager->Seek(mType, seekTime, MediaSourceDemuxer::EOS_FUZZ);
+  MediaResult result = NS_OK;
+  RefPtr<MediaRawData> sample =
+      mManager->GetSample(mType, TimeUnit::Zero(), result);
+  MOZ_ASSERT(NS_SUCCEEDED(result) && sample);
+  mNextSample = Some(sample);
+  mReset = false;
+  {
+    MonitorAutoLock mon(mMonitor);
+    mNextRandomAccessPoint =
+        mManager->GetNextRandomAccessPoint(mType, MediaSourceDemuxer::EOS_FUZZ);
+  }
+  return SeekPromise::CreateAndResolve(seekTime, __func__);
+}
+
+RefPtr<MediaSourceTrackDemuxer::SamplesPromise>
+MediaSourceTrackDemuxer::DoGetSamples(int32_t aNumSamples) {
+  if (!mManager) {
+    return SamplesPromise::CreateAndReject(
+        MediaResult(NS_ERROR_DOM_MEDIA_CANCELED,
+                    RESULT_DETAIL("manager is detached.")),
+        __func__);
+  }
+
+  MOZ_ASSERT(OnTaskQueue());
+  if (mReset) {
+    // If a reset was recently performed, we ensure that the data
+    // we are about to retrieve is still available.
+    TimeIntervals buffered = mManager->Buffered(mType);
+    if (buffered.IsEmpty() && mManager->IsEnded()) {
+      return SamplesPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_END_OF_STREAM,
+                                             __func__);
+    }
+
+    // We use a larger fuzz to determine the presentation start
+    // time than the fuzz we use to determine acceptable gaps between
+    // frames. This is needed to fix embedded video issues as seen in the wild
+    // from different muxed stream start times.
+    // See: https://www.w3.org/TR/media-source-2/#presentation-start-time
+    buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ_START);
+    if (!buffered.ContainsWithStrictEnd(TimeUnit::Zero())) {
+      return SamplesPromise::CreateAndReject(
+          NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
+    }
+    mReset = false;
+  }
+  RefPtr<MediaRawData> sample;
+  MediaResult result = NS_OK;
+  if (mNextSample) {
+    sample = mNextSample.ref();
+    mNextSample.reset();
+  } else {
+    sample = mManager->GetSample(mType, MediaSourceDemuxer::EOS_FUZZ, result);
+  }
+  if (!sample) {
+    if (result == NS_ERROR_DOM_MEDIA_END_OF_STREAM ||
+        result == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) {
+      return SamplesPromise::CreateAndReject(
+          (result == NS_ERROR_DOM_MEDIA_END_OF_STREAM && mManager->IsEnded())
+              ? NS_ERROR_DOM_MEDIA_END_OF_STREAM
+              : NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA,
+          __func__);
+    }
+    return SamplesPromise::CreateAndReject(result, __func__);
+  }
+  RefPtr<SamplesHolder> samples = new SamplesHolder;
+  samples->AppendSample(sample);
+  {
+    MonitorAutoLock mon(mMonitor);  // spurious warning will be given
+    // Diagnostic asserts for bug 1810396
+    MOZ_DIAGNOSTIC_ASSERT(sample, "Invalid sample pointer found!");
+    MOZ_DIAGNOSTIC_ASSERT(sample->HasValidTime(), "Invalid sample time found!");
+    if (!sample) {
+      return SamplesPromise::CreateAndReject(NS_ERROR_NULL_POINTER, __func__);
+    }
+    if (mNextRandomAccessPoint <= sample->mTime) {
+      mNextRandomAccessPoint = mManager->GetNextRandomAccessPoint(
+          mType, MediaSourceDemuxer::EOS_FUZZ);
+    }
+  }
+  return SamplesPromise::CreateAndResolve(samples, __func__);
+}
+
+RefPtr<MediaSourceTrackDemuxer::SkipAccessPointPromise>
+MediaSourceTrackDemuxer::DoSkipToNextRandomAccessPoint(
+    const TimeUnit& aTimeThreadshold) {
+  if (!mManager) {
+    return SkipAccessPointPromise::CreateAndReject(
+        SkipFailureHolder(MediaResult(NS_ERROR_DOM_MEDIA_CANCELED,
+                                      RESULT_DETAIL("manager is detached.")),
+                          0),
+        __func__);
+  }
+
+  MOZ_ASSERT(OnTaskQueue());
+  uint32_t parsed = 0;
+  // Ensure that the data we are about to skip to is still available.
+  TimeIntervals buffered = mManager->Buffered(mType);
+  buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2);
+  if (buffered.ContainsWithStrictEnd(aTimeThreadshold)) {
+    bool found;
+    parsed = mManager->SkipToNextRandomAccessPoint(
+        mType, aTimeThreadshold, MediaSourceDemuxer::EOS_FUZZ, found);
+    if (found) {
+      return SkipAccessPointPromise::CreateAndResolve(parsed, __func__);
+    }
+  }
+  SkipFailureHolder holder(mManager->IsEnded()
+                               ? NS_ERROR_DOM_MEDIA_END_OF_STREAM
+                               : NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA,
+                           parsed);
+  return SkipAccessPointPromise::CreateAndReject(holder, __func__);
+}
+
+bool MediaSourceTrackDemuxer::HasManager(TrackBuffersManager* aManager) const {
+  MOZ_ASSERT(OnTaskQueue());
+  return mManager == aManager;
+}
+
+void MediaSourceTrackDemuxer::DetachManager() {
+  MOZ_ASSERT(OnTaskQueue());
+  MonitorAutoLock mon(mMonitor);
+  mManager = nullptr;
+}
+
+}  // namespace mozilla