1 files changed, 372 insertions, 0 deletions

diff --git a/dom/media/mediasource/MediaSourceDecoder.cpp b/dom/media/mediasource/MediaSourceDecoder.cpp
new file mode 100644
index 0000000000..24a74e261b
--- /dev/null
+++ b/dom/media/mediasource/MediaSourceDecoder.cpp
@@ -0,0 +1,372 @@
+/* -*- mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+#include "MediaSourceDecoder.h"
+
+#include "base/process_util.h"
+#include "mozilla/Logging.h"
+#include "ExternalEngineStateMachine.h"
+#include "MediaDecoder.h"
+#include "MediaDecoderStateMachine.h"
+#include "MediaShutdownManager.h"
+#include "MediaSource.h"
+#include "MediaSourceDemuxer.h"
+#include "MediaSourceUtils.h"
+#include "SourceBuffer.h"
+#include "SourceBufferList.h"
+#include "VideoUtils.h"
+#include <algorithm>
+
+extern mozilla::LogModule* GetMediaSourceLog();
+
+#define MSE_DEBUG(arg, ...)                                              \
+  DDMOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Debug, "::%s: " arg, \
+            __func__, ##__VA_ARGS__)
+#define MSE_DEBUGV(arg, ...)                                               \
+  DDMOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Verbose, "::%s: " arg, \
+            __func__, ##__VA_ARGS__)
+
+using namespace mozilla::media;
+
+namespace mozilla {
+
+MediaSourceDecoder::MediaSourceDecoder(MediaDecoderInit& aInit)
+    : MediaDecoder(aInit), mMediaSource(nullptr), mEnded(false) {
+  mExplicitDuration.emplace(UnspecifiedNaN<double>());
+}
+
+MediaDecoderStateMachineBase* MediaSourceDecoder::CreateStateMachine(
+    bool aDisableExternalEngine) {
+  MOZ_ASSERT(NS_IsMainThread());
+  // if `mDemuxer` already exists, that means we're in the process of recreating
+  // the state machine. The track buffers are tied to the demuxer so we would
+  // need to reuse it.
+  if (!mDemuxer) {
+    mDemuxer = new MediaSourceDemuxer(AbstractMainThread());
+  }
+  MediaFormatReaderInit init;
+  init.mVideoFrameContainer = GetVideoFrameContainer();
+  init.mKnowsCompositor = GetCompositor();
+  init.mCrashHelper = GetOwner()->CreateGMPCrashHelper();
+  init.mFrameStats = mFrameStats;
+  init.mMediaDecoderOwnerID = mOwner;
+  static Atomic<uint32_t> sTrackingIdCounter(0);
+  init.mTrackingId.emplace(TrackingId::Source::MSEDecoder, sTrackingIdCounter++,
+                           TrackingId::TrackAcrossProcesses::Yes);
+  mReader = new MediaFormatReader(init, mDemuxer);
+#ifdef MOZ_WMF_MEDIA_ENGINE
+  // TODO : Only for testing development for now. In the future this should be
+  // used for encrypted content only.
+  if (StaticPrefs::media_wmf_media_engine_enabled() &&
+      !aDisableExternalEngine) {
+    return new ExternalEngineStateMachine(this, mReader);
+  }
+#endif
+  return new MediaDecoderStateMachine(this, mReader);
+}
+
+nsresult MediaSourceDecoder::Load(nsIPrincipal* aPrincipal) {
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(!GetStateMachine());
+
+  mPrincipal = aPrincipal;
+
+  nsresult rv = MediaShutdownManager::Instance().Register(this);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+  return CreateAndInitStateMachine(!mEnded);
+}
+
+template <typename IntervalType>
+IntervalType MediaSourceDecoder::GetSeekableImpl() {
+  MOZ_ASSERT(NS_IsMainThread());
+  if (!mMediaSource) {
+    NS_WARNING("MediaSource element isn't attached");
+    return IntervalType();
+  }
+
+  TimeIntervals seekable;
+  double duration = mMediaSource->Duration();
+  if (std::isnan(duration)) {
+    // Return empty range.
+  } else if (duration > 0 && std::isinf(duration)) {
+    media::TimeIntervals buffered = GetBuffered();
+
+    // 1. If live seekable range is not empty:
+    if (mMediaSource->HasLiveSeekableRange()) {
+      // 1. Let union ranges be the union of live seekable range and the
+      // HTMLMediaElement.buffered attribute.
+      TimeRanges unionRanges =
+          media::TimeRanges(buffered) + mMediaSource->LiveSeekableRange();
+      // 2. Return a single range with a start time equal to the earliest start
+      // time in union ranges and an end time equal to the highest end time in
+      // union ranges and abort these steps.
+      if constexpr (std::is_same<IntervalType, TimeRanges>::value) {
+        TimeRanges seekableRange = media::TimeRanges(
+            TimeRange(unionRanges.GetStart(), unionRanges.GetEnd()));
+        return seekableRange;
+      } else {
+        MOZ_RELEASE_ASSERT(false);
+      }
+    }
+
+    if (!buffered.IsEmpty()) {
+      seekable += media::TimeInterval(TimeUnit::Zero(), buffered.GetEnd());
+    }
+  } else {
+    if constexpr (std::is_same<IntervalType, TimeRanges>::value) {
+      // Common case: seekable in entire range of the media.
+      return TimeRanges(TimeRange(0, duration));
+    } else if constexpr (std::is_same<IntervalType, TimeIntervals>::value) {
+      seekable += media::TimeInterval(TimeUnit::Zero(),
+                                      mDuration.match(DurationToTimeUnit()));
+    } else {
+      MOZ_RELEASE_ASSERT(false);
+    }
+  }
+  MSE_DEBUG("ranges=%s", DumpTimeRanges(seekable).get());
+  return IntervalType(seekable);
+}
+
+media::TimeIntervals MediaSourceDecoder::GetSeekable() {
+  return GetSeekableImpl<media::TimeIntervals>();
+}
+
+media::TimeRanges MediaSourceDecoder::GetSeekableTimeRanges() {
+  return GetSeekableImpl<media::TimeRanges>();
+}
+
+media::TimeIntervals MediaSourceDecoder::GetBuffered() {
+  MOZ_ASSERT(NS_IsMainThread());
+
+  if (!mMediaSource) {
+    NS_WARNING("MediaSource element isn't attached");
+    return media::TimeIntervals::Invalid();
+  }
+  dom::SourceBufferList* sourceBuffers = mMediaSource->ActiveSourceBuffers();
+  if (!sourceBuffers) {
+    // Media source object is shutting down.
+    return TimeIntervals();
+  }
+  TimeUnit highestEndTime;
+  nsTArray<media::TimeIntervals> activeRanges;
+  media::TimeIntervals buffered;
+
+  for (uint32_t i = 0; i < sourceBuffers->Length(); i++) {
+    bool found;
+    dom::SourceBuffer* sb = sourceBuffers->IndexedGetter(i, found);
+    MOZ_ASSERT(found);
+
+    activeRanges.AppendElement(sb->GetTimeIntervals());
+    highestEndTime =
+        std::max(highestEndTime, activeRanges.LastElement().GetEnd());
+  }
+
+  buffered += media::TimeInterval(TimeUnit::Zero(), highestEndTime);
+
+  for (auto& range : activeRanges) {
+    if (mEnded && !range.IsEmpty()) {
+      // Set the end time on the last range to highestEndTime by adding a
+      // new range spanning the current end time to highestEndTime, which
+      // Normalize() will then merge with the old last range.
+      range += media::TimeInterval(range.GetEnd(), highestEndTime);
+    }
+    buffered.Intersection(range);
+  }
+
+  MSE_DEBUG("ranges=%s", DumpTimeRanges(buffered).get());
+  return buffered;
+}
+
+void MediaSourceDecoder::Shutdown() {
+  MOZ_ASSERT(NS_IsMainThread());
+  MSE_DEBUG("Shutdown");
+  // Detach first so that TrackBuffers are unused on the main thread when
+  // shut down on the decode task queue.
+  if (mMediaSource) {
+    mMediaSource->Detach();
+  }
+  mDemuxer = nullptr;
+
+  MediaDecoder::Shutdown();
+}
+
+void MediaSourceDecoder::AttachMediaSource(dom::MediaSource* aMediaSource) {
+  MOZ_ASSERT(!mMediaSource && !GetStateMachine() && NS_IsMainThread());
+  mMediaSource = aMediaSource;
+  DDLINKCHILD("mediasource", aMediaSource);
+}
+
+void MediaSourceDecoder::DetachMediaSource() {
+  MOZ_ASSERT(mMediaSource && NS_IsMainThread());
+  DDUNLINKCHILD(mMediaSource);
+  mMediaSource = nullptr;
+}
+
+void MediaSourceDecoder::Ended(bool aEnded) {
+  MOZ_ASSERT(NS_IsMainThread());
+  if (aEnded) {
+    // We want the MediaSourceReader to refresh its buffered range as it may
+    // have been modified (end lined up).
+    NotifyDataArrived();
+  }
+  mEnded = aEnded;
+  GetStateMachine()->DispatchIsLiveStream(!mEnded);
+}
+
+void MediaSourceDecoder::AddSizeOfResources(ResourceSizes* aSizes) {
+  MOZ_ASSERT(NS_IsMainThread());
+  if (GetDemuxer()) {
+    GetDemuxer()->AddSizeOfResources(aSizes);
+  }
+}
+
+void MediaSourceDecoder::SetInitialDuration(const TimeUnit& aDuration) {
+  MOZ_ASSERT(NS_IsMainThread());
+  // Only use the decoded duration if one wasn't already
+  // set.
+  if (!mMediaSource || !std::isnan(ExplicitDuration())) {
+    return;
+  }
+  SetMediaSourceDuration(aDuration);
+}
+
+void MediaSourceDecoder::SetMediaSourceDuration(const TimeUnit& aDuration) {
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(!IsShutdown());
+  if (aDuration.IsPositiveOrZero()) {
+    SetExplicitDuration(ToMicrosecondResolution(aDuration.ToSeconds()));
+  } else {
+    SetExplicitDuration(PositiveInfinity<double>());
+  }
+}
+
+void MediaSourceDecoder::SetMediaSourceDuration(double aDuration) {
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(!IsShutdown());
+  if (aDuration >= 0) {
+    SetExplicitDuration(aDuration);
+  } else {
+    SetExplicitDuration(PositiveInfinity<double>());
+  }
+}
+
+RefPtr<GenericPromise> MediaSourceDecoder::RequestDebugInfo(
+    dom::MediaSourceDecoderDebugInfo& aInfo) {
+  // This should be safe to call off main thead, but there's no such usage at
+  // time of writing. Can be carefully relaxed if needed.
+  MOZ_ASSERT(NS_IsMainThread(), "Expects to be called on main thread.");
+  nsTArray<RefPtr<GenericPromise>> promises;
+  if (mReader) {
+    promises.AppendElement(mReader->RequestDebugInfo(aInfo.mReader));
+  }
+  if (mDemuxer) {
+    promises.AppendElement(mDemuxer->GetDebugInfo(aInfo.mDemuxer));
+  }
+  return GenericPromise::All(GetCurrentSerialEventTarget(), promises)
+      ->Then(
+          GetCurrentSerialEventTarget(), __func__,
+          []() { return GenericPromise::CreateAndResolve(true, __func__); },
+          [] {
+            return GenericPromise::CreateAndReject(NS_ERROR_FAILURE, __func__);
+          });
+}
+
+double MediaSourceDecoder::GetDuration() {
+  MOZ_ASSERT(NS_IsMainThread());
+  return ExplicitDuration();
+}
+
+MediaDecoderOwner::NextFrameStatus
+MediaSourceDecoder::NextFrameBufferedStatus() {
+  MOZ_ASSERT(NS_IsMainThread());
+
+  if (!mMediaSource ||
+      mMediaSource->ReadyState() == dom::MediaSourceReadyState::Closed) {
+    return MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE;
+  }
+
+  // Next frame hasn't been decoded yet.
+  // Use the buffered range to consider if we have the next frame available.
+  auto currentPosition = CurrentPosition();
+  TimeIntervals buffered = GetBuffered();
+  buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2);
+  TimeInterval interval(
+      currentPosition, currentPosition + DEFAULT_NEXT_FRAME_AVAILABLE_BUFFERED);
+  return buffered.ContainsWithStrictEnd(ClampIntervalToEnd(interval))
+             ? MediaDecoderOwner::NEXT_FRAME_AVAILABLE
+             : MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE;
+}
+
+bool MediaSourceDecoder::CanPlayThroughImpl() {
+  MOZ_ASSERT(NS_IsMainThread());
+
+  if (NextFrameBufferedStatus() == MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE) {
+    return false;
+  }
+
+  if (std::isnan(mMediaSource->Duration())) {
+    // Don't have any data yet.
+    return false;
+  }
+  TimeUnit duration = TimeUnit::FromSeconds(mMediaSource->Duration());
+  auto currentPosition = CurrentPosition();
+  if (duration <= currentPosition) {
+    return true;
+  }
+  // If we have data up to the mediasource's duration or 3s ahead, we can
+  // assume that we can play without interruption.
+  dom::SourceBufferList* sourceBuffers = mMediaSource->ActiveSourceBuffers();
+  TimeUnit bufferedEnd = sourceBuffers->GetHighestBufferedEndTime();
+  TimeUnit timeAhead =
+      std::min(duration, currentPosition + TimeUnit::FromSeconds(3));
+  TimeInterval interval(currentPosition, timeAhead);
+  return bufferedEnd >= timeAhead;
+}
+
+TimeInterval MediaSourceDecoder::ClampIntervalToEnd(
+    const TimeInterval& aInterval) {
+  MOZ_ASSERT(NS_IsMainThread());
+
+  if (!mEnded) {
+    return aInterval;
+  }
+  TimeUnit duration = mDuration.match(DurationToTimeUnit());
+  if (duration < aInterval.mStart) {
+    return aInterval;
+  }
+  return TimeInterval(aInterval.mStart, std::min(aInterval.mEnd, duration),
+                      aInterval.mFuzz);
+}
+
+void MediaSourceDecoder::NotifyInitDataArrived() {
+  MOZ_ASSERT(NS_IsMainThread());
+  if (mDemuxer) {
+    mDemuxer->NotifyInitDataArrived();
+  }
+}
+
+void MediaSourceDecoder::NotifyDataArrived() {
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_DIAGNOSTIC_ASSERT(!IsShutdown());
+  NotifyReaderDataArrived();
+  GetOwner()->DownloadProgressed();
+}
+
+already_AddRefed<nsIPrincipal> MediaSourceDecoder::GetCurrentPrincipal() {
+  MOZ_ASSERT(NS_IsMainThread());
+  return do_AddRef(mPrincipal);
+}
+
+bool MediaSourceDecoder::HadCrossOriginRedirects() {
+  MOZ_ASSERT(NS_IsMainThread());
+  return false;
+}
+
+#undef MSE_DEBUG
+#undef MSE_DEBUGV
+
+}  // namespace mozilla