summaryrefslogtreecommitdiffstats
path: root/dom/html/TextTrackManager.cpp
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
commit36d22d82aa202bb199967e9512281e9a53db42c9 (patch)
tree105e8c98ddea1c1e4784a60a5a6410fa416be2de /dom/html/TextTrackManager.cpp
parentInitial commit. (diff)
downloadfirefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.tar.xz
firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.zip
Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/html/TextTrackManager.cpp')
-rw-r--r--dom/html/TextTrackManager.cpp878
1 files changed, 878 insertions, 0 deletions
diff --git a/dom/html/TextTrackManager.cpp b/dom/html/TextTrackManager.cpp
new file mode 100644
index 0000000000..02e44592f2
--- /dev/null
+++ b/dom/html/TextTrackManager.cpp
@@ -0,0 +1,878 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/dom/TextTrackManager.h"
+#include "mozilla/ClearOnShutdown.h"
+#include "mozilla/CycleCollectedJSContext.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/Telemetry.h"
+#include "mozilla/dom/Document.h"
+#include "mozilla/dom/Event.h"
+#include "mozilla/dom/HTMLMediaElement.h"
+#include "mozilla/dom/HTMLTrackElement.h"
+#include "mozilla/dom/HTMLVideoElement.h"
+#include "mozilla/dom/TextTrack.h"
+#include "mozilla/dom/TextTrackCue.h"
+#include "nsComponentManagerUtils.h"
+#include "nsGlobalWindow.h"
+#include "nsIFrame.h"
+#include "nsIWebVTTParserWrapper.h"
+#include "nsVariant.h"
+#include "nsVideoFrame.h"
+
+mozilla::LazyLogModule gTextTrackLog("WebVTT");
+
+#define WEBVTT_LOG(msg, ...) \
+ MOZ_LOG(gTextTrackLog, LogLevel::Debug, \
+ ("TextTrackManager=%p, " msg, this, ##__VA_ARGS__))
+#define WEBVTT_LOGV(msg, ...) \
+ MOZ_LOG(gTextTrackLog, LogLevel::Verbose, \
+ ("TextTrackManager=%p, " msg, this, ##__VA_ARGS__))
+
+namespace mozilla::dom {
+
+NS_IMPL_ISUPPORTS(TextTrackManager::ShutdownObserverProxy, nsIObserver);
+
+void TextTrackManager::ShutdownObserverProxy::Unregister() {
+ nsContentUtils::UnregisterShutdownObserver(this);
+ mManager = nullptr;
+}
+
+CompareTextTracks::CompareTextTracks(HTMLMediaElement* aMediaElement) {
+ mMediaElement = aMediaElement;
+}
+
+Maybe<uint32_t> CompareTextTracks::TrackChildPosition(
+ TextTrack* aTextTrack) const {
+ MOZ_DIAGNOSTIC_ASSERT(aTextTrack);
+ HTMLTrackElement* trackElement = aTextTrack->GetTrackElement();
+ if (!trackElement) {
+ return Nothing();
+ }
+ return mMediaElement->ComputeIndexOf(trackElement);
+}
+
+bool CompareTextTracks::Equals(TextTrack* aOne, TextTrack* aTwo) const {
+ // Two tracks can never be equal. If they have corresponding TrackElements
+ // they would need to occupy the same tree position (impossible) and in the
+ // case of tracks coming from AddTextTrack source we put the newest at the
+ // last position, so they won't be equal as well.
+ return false;
+}
+
+bool CompareTextTracks::LessThan(TextTrack* aOne, TextTrack* aTwo) const {
+ // Protect against nullptr TextTrack objects; treat them as
+ // sorting toward the end.
+ if (!aOne) {
+ return false;
+ }
+ if (!aTwo) {
+ return true;
+ }
+ TextTrackSource sourceOne = aOne->GetTextTrackSource();
+ TextTrackSource sourceTwo = aTwo->GetTextTrackSource();
+ if (sourceOne != sourceTwo) {
+ return sourceOne == TextTrackSource::Track ||
+ (sourceOne == TextTrackSource::AddTextTrack &&
+ sourceTwo == TextTrackSource::MediaResourceSpecific);
+ }
+ switch (sourceOne) {
+ case TextTrackSource::Track: {
+ Maybe<uint32_t> positionOne = TrackChildPosition(aOne);
+ Maybe<uint32_t> positionTwo = TrackChildPosition(aTwo);
+ // If either position one or positiontwo are Nothing then something has
+ // gone wrong. In this case we should just put them at the back of the
+ // list.
+ return positionOne.isSome() && positionTwo.isSome() &&
+ *positionOne < *positionTwo;
+ }
+ case TextTrackSource::AddTextTrack:
+ // For AddTextTrack sources the tracks will already be in the correct
+ // relative order in the source array. Assume we're called in iteration
+ // order and can therefore always report aOne < aTwo to maintain the
+ // original temporal ordering.
+ return true;
+ case TextTrackSource::MediaResourceSpecific:
+ // No rules for Media Resource Specific tracks yet.
+ break;
+ }
+ return true;
+}
+
+NS_IMPL_CYCLE_COLLECTION(TextTrackManager, mMediaElement, mTextTracks,
+ mPendingTextTracks, mNewCues)
+
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(TextTrackManager)
+ NS_INTERFACE_MAP_ENTRY(nsIDOMEventListener)
+NS_INTERFACE_MAP_END
+
+NS_IMPL_CYCLE_COLLECTING_ADDREF(TextTrackManager)
+NS_IMPL_CYCLE_COLLECTING_RELEASE(TextTrackManager)
+
+StaticRefPtr<nsIWebVTTParserWrapper> TextTrackManager::sParserWrapper;
+
+TextTrackManager::TextTrackManager(HTMLMediaElement* aMediaElement)
+ : mMediaElement(aMediaElement),
+ mHasSeeked(false),
+ mLastTimeMarchesOnCalled(media::TimeUnit::Zero()),
+ mTimeMarchesOnDispatched(false),
+ mUpdateCueDisplayDispatched(false),
+ performedTrackSelection(false),
+ mShutdown(false) {
+ nsISupports* parentObject = mMediaElement->OwnerDoc()->GetParentObject();
+
+ NS_ENSURE_TRUE_VOID(parentObject);
+ WEBVTT_LOG("Create TextTrackManager");
+ nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(parentObject);
+ mNewCues = new TextTrackCueList(window);
+ mTextTracks = new TextTrackList(window, this);
+ mPendingTextTracks = new TextTrackList(window, this);
+
+ if (!sParserWrapper) {
+ nsCOMPtr<nsIWebVTTParserWrapper> parserWrapper =
+ do_CreateInstance(NS_WEBVTTPARSERWRAPPER_CONTRACTID);
+ MOZ_ASSERT(parserWrapper, "Can't create nsIWebVTTParserWrapper");
+ sParserWrapper = parserWrapper;
+ ClearOnShutdown(&sParserWrapper);
+ }
+ mShutdownProxy = new ShutdownObserverProxy(this);
+}
+
+TextTrackManager::~TextTrackManager() {
+ WEBVTT_LOG("~TextTrackManager");
+ mShutdownProxy->Unregister();
+}
+
+TextTrackList* TextTrackManager::GetTextTracks() const { return mTextTracks; }
+
+already_AddRefed<TextTrack> TextTrackManager::AddTextTrack(
+ TextTrackKind aKind, const nsAString& aLabel, const nsAString& aLanguage,
+ TextTrackMode aMode, TextTrackReadyState aReadyState,
+ TextTrackSource aTextTrackSource) {
+ if (!mMediaElement || !mTextTracks) {
+ return nullptr;
+ }
+ RefPtr<TextTrack> track = mTextTracks->AddTextTrack(
+ aKind, aLabel, aLanguage, aMode, aReadyState, aTextTrackSource,
+ CompareTextTracks(mMediaElement));
+ WEBVTT_LOG("AddTextTrack %p kind %" PRIu32 " Label %s Language %s",
+ track.get(), static_cast<uint32_t>(aKind),
+ NS_ConvertUTF16toUTF8(aLabel).get(),
+ NS_ConvertUTF16toUTF8(aLanguage).get());
+ AddCues(track);
+
+ if (aTextTrackSource == TextTrackSource::Track) {
+ RefPtr<nsIRunnable> task = NewRunnableMethod(
+ "dom::TextTrackManager::HonorUserPreferencesForTrackSelection", this,
+ &TextTrackManager::HonorUserPreferencesForTrackSelection);
+ NS_DispatchToMainThread(task.forget());
+ }
+
+ return track.forget();
+}
+
+void TextTrackManager::AddTextTrack(TextTrack* aTextTrack) {
+ if (!mMediaElement || !mTextTracks) {
+ return;
+ }
+ WEBVTT_LOG("AddTextTrack TextTrack %p", aTextTrack);
+ mTextTracks->AddTextTrack(aTextTrack, CompareTextTracks(mMediaElement));
+ AddCues(aTextTrack);
+
+ if (aTextTrack->GetTextTrackSource() == TextTrackSource::Track) {
+ RefPtr<nsIRunnable> task = NewRunnableMethod(
+ "dom::TextTrackManager::HonorUserPreferencesForTrackSelection", this,
+ &TextTrackManager::HonorUserPreferencesForTrackSelection);
+ NS_DispatchToMainThread(task.forget());
+ }
+}
+
+void TextTrackManager::AddCues(TextTrack* aTextTrack) {
+ if (!mNewCues) {
+ WEBVTT_LOG("AddCues mNewCues is null");
+ return;
+ }
+
+ TextTrackCueList* cueList = aTextTrack->GetCues();
+ if (cueList) {
+ bool dummy;
+ WEBVTT_LOGV("AddCues, CuesNum=%d", cueList->Length());
+ for (uint32_t i = 0; i < cueList->Length(); ++i) {
+ mNewCues->AddCue(*cueList->IndexedGetter(i, dummy));
+ }
+ MaybeRunTimeMarchesOn();
+ }
+}
+
+void TextTrackManager::RemoveTextTrack(TextTrack* aTextTrack,
+ bool aPendingListOnly) {
+ if (!mPendingTextTracks || !mTextTracks) {
+ return;
+ }
+
+ WEBVTT_LOG("RemoveTextTrack TextTrack %p", aTextTrack);
+ mPendingTextTracks->RemoveTextTrack(aTextTrack);
+ if (aPendingListOnly) {
+ return;
+ }
+
+ mTextTracks->RemoveTextTrack(aTextTrack);
+ // Remove the cues in mNewCues belong to aTextTrack.
+ TextTrackCueList* removeCueList = aTextTrack->GetCues();
+ if (removeCueList) {
+ WEBVTT_LOGV("RemoveTextTrack removeCuesNum=%d", removeCueList->Length());
+ for (uint32_t i = 0; i < removeCueList->Length(); ++i) {
+ mNewCues->RemoveCue(*((*removeCueList)[i]));
+ }
+ MaybeRunTimeMarchesOn();
+ }
+}
+
+void TextTrackManager::DidSeek() {
+ WEBVTT_LOG("DidSeek");
+ mHasSeeked = true;
+}
+
+void TextTrackManager::UpdateCueDisplay() {
+ WEBVTT_LOG("UpdateCueDisplay");
+ mUpdateCueDisplayDispatched = false;
+
+ if (!mMediaElement || !mTextTracks || IsShutdown()) {
+ WEBVTT_LOG("Abort UpdateCueDisplay.");
+ return;
+ }
+
+ nsIFrame* frame = mMediaElement->GetPrimaryFrame();
+ nsVideoFrame* videoFrame = do_QueryFrame(frame);
+ if (!videoFrame) {
+ WEBVTT_LOG("Abort UpdateCueDisplay, because of no video frame.");
+ return;
+ }
+
+ nsCOMPtr<nsIContent> overlay = videoFrame->GetCaptionOverlay();
+ if (!overlay) {
+ WEBVTT_LOG("Abort UpdateCueDisplay, because of no overlay.");
+ return;
+ }
+
+ RefPtr<nsPIDOMWindowInner> window =
+ mMediaElement->OwnerDoc()->GetInnerWindow();
+ if (!window) {
+ WEBVTT_LOG("Abort UpdateCueDisplay, because of no window.");
+ }
+
+ nsTArray<RefPtr<TextTrackCue>> showingCues;
+ mTextTracks->GetShowingCues(showingCues);
+
+ WEBVTT_LOG("UpdateCueDisplay, processCues, showingCuesNum=%zu",
+ showingCues.Length());
+ RefPtr<nsVariantCC> jsCues = new nsVariantCC();
+ jsCues->SetAsArray(nsIDataType::VTYPE_INTERFACE, &NS_GET_IID(EventTarget),
+ showingCues.Length(),
+ static_cast<void*>(showingCues.Elements()));
+ nsCOMPtr<nsIContent> controls = videoFrame->GetVideoControls();
+
+ nsContentUtils::AddScriptRunner(NS_NewRunnableFunction(
+ "TextTrackManager::UpdateCueDisplay",
+ [window, jsCues, overlay, controls]() {
+ if (sParserWrapper) {
+ sParserWrapper->ProcessCues(window, jsCues, overlay, controls);
+ }
+ }));
+}
+
+void TextTrackManager::NotifyCueAdded(TextTrackCue& aCue) {
+ WEBVTT_LOG("NotifyCueAdded, cue=%p", &aCue);
+ if (mNewCues) {
+ mNewCues->AddCue(aCue);
+ }
+ MaybeRunTimeMarchesOn();
+}
+
+void TextTrackManager::NotifyCueRemoved(TextTrackCue& aCue) {
+ WEBVTT_LOG("NotifyCueRemoved, cue=%p", &aCue);
+ if (mNewCues) {
+ mNewCues->RemoveCue(aCue);
+ }
+ MaybeRunTimeMarchesOn();
+ DispatchUpdateCueDisplay();
+}
+
+void TextTrackManager::PopulatePendingList() {
+ if (!mTextTracks || !mPendingTextTracks || !mMediaElement) {
+ return;
+ }
+ uint32_t len = mTextTracks->Length();
+ bool dummy;
+ for (uint32_t index = 0; index < len; ++index) {
+ TextTrack* ttrack = mTextTracks->IndexedGetter(index, dummy);
+ if (ttrack && ttrack->Mode() != TextTrackMode::Disabled &&
+ ttrack->ReadyState() == TextTrackReadyState::Loading) {
+ mPendingTextTracks->AddTextTrack(ttrack,
+ CompareTextTracks(mMediaElement));
+ }
+ }
+}
+
+void TextTrackManager::AddListeners() {
+ if (mMediaElement) {
+ mMediaElement->AddEventListener(u"resizecaption"_ns, this, false, false);
+ mMediaElement->AddEventListener(u"resizevideocontrols"_ns, this, false,
+ false);
+ mMediaElement->AddEventListener(u"seeked"_ns, this, false, false);
+ mMediaElement->AddEventListener(u"controlbarchange"_ns, this, false, true);
+ }
+}
+
+void TextTrackManager::HonorUserPreferencesForTrackSelection() {
+ if (performedTrackSelection || !mTextTracks) {
+ return;
+ }
+ WEBVTT_LOG("HonorUserPreferencesForTrackSelection");
+ TextTrackKind ttKinds[] = {TextTrackKind::Captions, TextTrackKind::Subtitles};
+
+ // Steps 1 - 3: Perform automatic track selection for different TextTrack
+ // Kinds.
+ PerformTrackSelection(ttKinds, ArrayLength(ttKinds));
+ PerformTrackSelection(TextTrackKind::Descriptions);
+ PerformTrackSelection(TextTrackKind::Chapters);
+
+ // Step 4: Set all TextTracks with a kind of metadata that are disabled
+ // to hidden.
+ for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
+ TextTrack* track = (*mTextTracks)[i];
+ if (track->Kind() == TextTrackKind::Metadata && TrackIsDefault(track) &&
+ track->Mode() == TextTrackMode::Disabled) {
+ track->SetMode(TextTrackMode::Hidden);
+ }
+ }
+
+ performedTrackSelection = true;
+}
+
+bool TextTrackManager::TrackIsDefault(TextTrack* aTextTrack) {
+ HTMLTrackElement* trackElement = aTextTrack->GetTrackElement();
+ if (!trackElement) {
+ return false;
+ }
+ return trackElement->Default();
+}
+
+void TextTrackManager::PerformTrackSelection(TextTrackKind aTextTrackKind) {
+ TextTrackKind ttKinds[] = {aTextTrackKind};
+ PerformTrackSelection(ttKinds, ArrayLength(ttKinds));
+}
+
+void TextTrackManager::PerformTrackSelection(TextTrackKind aTextTrackKinds[],
+ uint32_t size) {
+ nsTArray<TextTrack*> candidates;
+ GetTextTracksOfKinds(aTextTrackKinds, size, candidates);
+
+ // Step 3: If any TextTracks in candidates are showing then abort these steps.
+ for (uint32_t i = 0; i < candidates.Length(); i++) {
+ if (candidates[i]->Mode() == TextTrackMode::Showing) {
+ WEBVTT_LOGV("PerformTrackSelection Showing return kind %d",
+ static_cast<int>(candidates[i]->Kind()));
+ return;
+ }
+ }
+
+ // Step 4: Honor user preferences for track selection, otherwise, set the
+ // first TextTrack in candidates with a default attribute to showing.
+ // TODO: Bug 981691 - Honor user preferences for text track selection.
+ for (uint32_t i = 0; i < candidates.Length(); i++) {
+ if (TrackIsDefault(candidates[i]) &&
+ candidates[i]->Mode() == TextTrackMode::Disabled) {
+ candidates[i]->SetMode(TextTrackMode::Showing);
+ WEBVTT_LOGV("PerformTrackSelection set Showing kind %d",
+ static_cast<int>(candidates[i]->Kind()));
+ return;
+ }
+ }
+}
+
+void TextTrackManager::GetTextTracksOfKinds(TextTrackKind aTextTrackKinds[],
+ uint32_t size,
+ nsTArray<TextTrack*>& aTextTracks) {
+ for (uint32_t i = 0; i < size; i++) {
+ GetTextTracksOfKind(aTextTrackKinds[i], aTextTracks);
+ }
+}
+
+void TextTrackManager::GetTextTracksOfKind(TextTrackKind aTextTrackKind,
+ nsTArray<TextTrack*>& aTextTracks) {
+ if (!mTextTracks) {
+ return;
+ }
+ for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
+ TextTrack* textTrack = (*mTextTracks)[i];
+ if (textTrack->Kind() == aTextTrackKind) {
+ aTextTracks.AppendElement(textTrack);
+ }
+ }
+}
+
+NS_IMETHODIMP
+TextTrackManager::HandleEvent(Event* aEvent) {
+ if (!mTextTracks) {
+ return NS_OK;
+ }
+
+ nsAutoString type;
+ aEvent->GetType(type);
+ WEBVTT_LOG("Handle event %s", NS_ConvertUTF16toUTF8(type).get());
+
+ const bool setDirty = type.EqualsLiteral("seeked") ||
+ type.EqualsLiteral("resizecaption") ||
+ type.EqualsLiteral("resizevideocontrols");
+ const bool updateDisplay = type.EqualsLiteral("controlbarchange") ||
+ type.EqualsLiteral("resizecaption");
+
+ if (setDirty) {
+ for (uint32_t i = 0; i < mTextTracks->Length(); i++) {
+ ((*mTextTracks)[i])->SetCuesDirty();
+ }
+ }
+ if (updateDisplay) {
+ UpdateCueDisplay();
+ }
+
+ return NS_OK;
+}
+
+class SimpleTextTrackEvent : public Runnable {
+ public:
+ friend class CompareSimpleTextTrackEvents;
+ SimpleTextTrackEvent(const nsAString& aEventName, double aTime,
+ TextTrack* aTrack, TextTrackCue* aCue)
+ : Runnable("dom::SimpleTextTrackEvent"),
+ mName(aEventName),
+ mTime(aTime),
+ mTrack(aTrack),
+ mCue(aCue) {}
+
+ NS_IMETHOD Run() override {
+ WEBVTT_LOGV("SimpleTextTrackEvent cue %p mName %s mTime %lf", mCue.get(),
+ NS_ConvertUTF16toUTF8(mName).get(), mTime);
+ mCue->DispatchTrustedEvent(mName);
+ return NS_OK;
+ }
+
+ void Dispatch() {
+ if (nsCOMPtr<nsIGlobalObject> global = mCue->GetOwnerGlobal()) {
+ global->Dispatch(TaskCategory::Other, do_AddRef(this));
+ } else {
+ NS_DispatchToMainThread(do_AddRef(this));
+ }
+ }
+
+ private:
+ nsString mName;
+ double mTime;
+ TextTrack* mTrack;
+ RefPtr<TextTrackCue> mCue;
+};
+
+class CompareSimpleTextTrackEvents {
+ private:
+ Maybe<uint32_t> TrackChildPosition(SimpleTextTrackEvent* aEvent) const {
+ if (aEvent->mTrack) {
+ HTMLTrackElement* trackElement = aEvent->mTrack->GetTrackElement();
+ if (trackElement) {
+ return mMediaElement->ComputeIndexOf(trackElement);
+ }
+ }
+ return Nothing();
+ }
+ HTMLMediaElement* mMediaElement;
+
+ public:
+ explicit CompareSimpleTextTrackEvents(HTMLMediaElement* aMediaElement) {
+ mMediaElement = aMediaElement;
+ }
+
+ bool Equals(SimpleTextTrackEvent* aOne, SimpleTextTrackEvent* aTwo) const {
+ return false;
+ }
+
+ bool LessThan(SimpleTextTrackEvent* aOne, SimpleTextTrackEvent* aTwo) const {
+ // TimeMarchesOn step 13.1.
+ if (aOne->mTime < aTwo->mTime) {
+ return true;
+ }
+ if (aOne->mTime > aTwo->mTime) {
+ return false;
+ }
+
+ // TimeMarchesOn step 13.2 text track cue order.
+ // TextTrack position in TextTrackList
+ TextTrack* t1 = aOne->mTrack;
+ TextTrack* t2 = aTwo->mTrack;
+ MOZ_ASSERT(t1, "CompareSimpleTextTrackEvents t1 is null");
+ MOZ_ASSERT(t2, "CompareSimpleTextTrackEvents t2 is null");
+ if (t1 != t2) {
+ TextTrackList* tList = t1->GetTextTrackList();
+ MOZ_ASSERT(tList, "CompareSimpleTextTrackEvents tList is null");
+ nsTArray<RefPtr<TextTrack>>& textTracks = tList->GetTextTrackArray();
+ auto index1 = textTracks.IndexOf(t1);
+ auto index2 = textTracks.IndexOf(t2);
+ if (index1 < index2) {
+ return true;
+ }
+ if (index1 > index2) {
+ return false;
+ }
+ }
+
+ MOZ_ASSERT(t1 == t2, "CompareSimpleTextTrackEvents t1 != t2");
+ // c1 and c2 are both belongs to t1.
+ TextTrackCue* c1 = aOne->mCue;
+ TextTrackCue* c2 = aTwo->mCue;
+ if (c1 != c2) {
+ if (c1->StartTime() < c2->StartTime()) {
+ return true;
+ }
+ if (c1->StartTime() > c2->StartTime()) {
+ return false;
+ }
+ if (c1->EndTime() < c2->EndTime()) {
+ return true;
+ }
+ if (c1->EndTime() > c2->EndTime()) {
+ return false;
+ }
+
+ TextTrackCueList* cueList = t1->GetCues();
+ MOZ_ASSERT(cueList);
+ nsTArray<RefPtr<TextTrackCue>>& cues = cueList->GetCuesArray();
+ auto index1 = cues.IndexOf(c1);
+ auto index2 = cues.IndexOf(c2);
+ if (index1 < index2) {
+ return true;
+ }
+ if (index1 > index2) {
+ return false;
+ }
+ }
+
+ // TimeMarchesOn step 13.3.
+ if (aOne->mName.EqualsLiteral("enter") ||
+ aTwo->mName.EqualsLiteral("exit")) {
+ return true;
+ }
+ return false;
+ }
+};
+
+class TextTrackListInternal {
+ public:
+ void AddTextTrack(TextTrack* aTextTrack,
+ const CompareTextTracks& aCompareTT) {
+ if (!mTextTracks.Contains(aTextTrack)) {
+ mTextTracks.InsertElementSorted(aTextTrack, aCompareTT);
+ }
+ }
+ uint32_t Length() const { return mTextTracks.Length(); }
+ TextTrack* operator[](uint32_t aIndex) {
+ return mTextTracks.SafeElementAt(aIndex, nullptr);
+ }
+
+ private:
+ nsTArray<RefPtr<TextTrack>> mTextTracks;
+};
+
+void TextTrackManager::DispatchUpdateCueDisplay() {
+ if (!mUpdateCueDisplayDispatched && !IsShutdown()) {
+ WEBVTT_LOG("DispatchUpdateCueDisplay");
+ nsPIDOMWindowInner* win = mMediaElement->OwnerDoc()->GetInnerWindow();
+ if (win) {
+ nsGlobalWindowInner::Cast(win)->Dispatch(
+ TaskCategory::Other,
+ NewRunnableMethod("dom::TextTrackManager::UpdateCueDisplay", this,
+ &TextTrackManager::UpdateCueDisplay));
+ mUpdateCueDisplayDispatched = true;
+ }
+ }
+}
+
+void TextTrackManager::DispatchTimeMarchesOn() {
+ // Run the algorithm if no previous instance is still running, otherwise
+ // enqueue the current playback position and whether only that changed
+ // through its usual monotonic increase during normal playback; current
+ // executing call upon completion will check queue for further 'work'.
+ if (!mTimeMarchesOnDispatched && !IsShutdown()) {
+ WEBVTT_LOG("DispatchTimeMarchesOn");
+ nsPIDOMWindowInner* win = mMediaElement->OwnerDoc()->GetInnerWindow();
+ if (win) {
+ nsGlobalWindowInner::Cast(win)->Dispatch(
+ TaskCategory::Other,
+ NewRunnableMethod("dom::TextTrackManager::TimeMarchesOn", this,
+ &TextTrackManager::TimeMarchesOn));
+ mTimeMarchesOnDispatched = true;
+ }
+ }
+}
+
+// https://html.spec.whatwg.org/multipage/embedded-content.html#time-marches-on
+void TextTrackManager::TimeMarchesOn() {
+ NS_ASSERTION(NS_IsMainThread(), "Wrong thread!");
+ mTimeMarchesOnDispatched = false;
+
+ CycleCollectedJSContext* context = CycleCollectedJSContext::Get();
+ if (context && context->IsInStableOrMetaStableState()) {
+ // FireTimeUpdate can be called while at stable state following a
+ // current position change which triggered a state watcher in MediaDecoder
+ // (see bug 1443429).
+ // TimeMarchesOn() will modify JS attributes which is forbidden while in
+ // stable state. So we dispatch a task to perform such operation later
+ // instead.
+ DispatchTimeMarchesOn();
+ return;
+ }
+ WEBVTT_LOG("TimeMarchesOn");
+
+ // Early return if we don't have any TextTracks or shutting down.
+ if (!mTextTracks || mTextTracks->Length() == 0 || IsShutdown() ||
+ !mMediaElement) {
+ return;
+ }
+
+ if (mMediaElement->ReadyState() == HTMLMediaElement_Binding::HAVE_NOTHING) {
+ WEBVTT_LOG(
+ "TimeMarchesOn return because media doesn't contain any data yet");
+ return;
+ }
+
+ if (mMediaElement->Seeking()) {
+ WEBVTT_LOG("TimeMarchesOn return during seeking");
+ return;
+ }
+
+ // Step 1, 2.
+ nsISupports* parentObject = mMediaElement->OwnerDoc()->GetParentObject();
+ if (NS_WARN_IF(!parentObject)) {
+ return;
+ }
+ nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(parentObject);
+ RefPtr<TextTrackCueList> currentCues = new TextTrackCueList(window);
+ RefPtr<TextTrackCueList> otherCues = new TextTrackCueList(window);
+
+ // Step 3.
+ auto currentPlaybackTime =
+ media::TimeUnit::FromSeconds(mMediaElement->CurrentTime());
+ bool hasNormalPlayback = !mHasSeeked;
+ mHasSeeked = false;
+ WEBVTT_LOG(
+ "TimeMarchesOn mLastTimeMarchesOnCalled %lf currentPlaybackTime %lf "
+ "hasNormalPlayback %d",
+ mLastTimeMarchesOnCalled.ToSeconds(), currentPlaybackTime.ToSeconds(),
+ hasNormalPlayback);
+
+ // The reason we collect other cues is (1) to change active cues to inactive,
+ // (2) find missing cues, so we actually no need to process all cues. We just
+ // need to handle cues which are in the time interval [lastTime:currentTime]
+ // or [currentTime:lastTime] (seeking forward). That can help us to reduce the
+ // size of other cues, which can improve execution time.
+ auto start = std::min(mLastTimeMarchesOnCalled, currentPlaybackTime);
+ auto end = std::max(mLastTimeMarchesOnCalled, currentPlaybackTime);
+ media::TimeInterval interval(start, end);
+ WEBVTT_LOGV("TimeMarchesOn Time interval [%f:%f]", start.ToSeconds(),
+ end.ToSeconds());
+ for (uint32_t idx = 0; idx < mTextTracks->Length(); ++idx) {
+ TextTrack* track = (*mTextTracks)[idx];
+ if (track) {
+ track->GetCurrentCuesAndOtherCues(currentCues, otherCues, interval);
+ }
+ }
+
+ // Step 4.
+ RefPtr<TextTrackCueList> missedCues = new TextTrackCueList(window);
+ if (hasNormalPlayback) {
+ for (uint32_t i = 0; i < otherCues->Length(); ++i) {
+ TextTrackCue* cue = (*otherCues)[i];
+ if (cue->StartTime() >= mLastTimeMarchesOnCalled.ToSeconds() &&
+ cue->EndTime() <= currentPlaybackTime.ToSeconds()) {
+ missedCues->AddCue(*cue);
+ }
+ }
+ }
+
+ WEBVTT_LOGV("TimeMarchesOn currentCues %d", currentCues->Length());
+ WEBVTT_LOGV("TimeMarchesOn otherCues %d", otherCues->Length());
+ WEBVTT_LOGV("TimeMarchesOn missedCues %d", missedCues->Length());
+ // Step 5. Empty now.
+ // TODO: Step 6: fire timeupdate?
+
+ // Step 7. Abort steps if condition 1, 2, 3 are satisfied.
+ // 1. All of the cues in current cues have their active flag set.
+ // 2. None of the cues in other cues have their active flag set.
+ // 3. Missed cues is empty.
+ bool c1 = true;
+ for (uint32_t i = 0; i < currentCues->Length(); ++i) {
+ if (!(*currentCues)[i]->GetActive()) {
+ c1 = false;
+ break;
+ }
+ }
+ bool c2 = true;
+ for (uint32_t i = 0; i < otherCues->Length(); ++i) {
+ if ((*otherCues)[i]->GetActive()) {
+ c2 = false;
+ break;
+ }
+ }
+ bool c3 = (missedCues->Length() == 0);
+ if (c1 && c2 && c3) {
+ mLastTimeMarchesOnCalled = currentPlaybackTime;
+ WEBVTT_LOG("TimeMarchesOn step 7 return, mLastTimeMarchesOnCalled %lf",
+ mLastTimeMarchesOnCalled.ToSeconds());
+ return;
+ }
+
+ // Step 8. Respect PauseOnExit flag if not seek.
+ if (hasNormalPlayback) {
+ for (uint32_t i = 0; i < otherCues->Length(); ++i) {
+ TextTrackCue* cue = (*otherCues)[i];
+ if (cue && cue->PauseOnExit() && cue->GetActive()) {
+ WEBVTT_LOG("TimeMarchesOn pause the MediaElement");
+ mMediaElement->Pause();
+ break;
+ }
+ }
+ for (uint32_t i = 0; i < missedCues->Length(); ++i) {
+ TextTrackCue* cue = (*missedCues)[i];
+ if (cue && cue->PauseOnExit()) {
+ WEBVTT_LOG("TimeMarchesOn pause the MediaElement");
+ mMediaElement->Pause();
+ break;
+ }
+ }
+ }
+
+ // Step 15.
+ // Sort text tracks in the same order as the text tracks appear
+ // in the media element's list of text tracks, and remove
+ // duplicates.
+ TextTrackListInternal affectedTracks;
+ // Step 13, 14.
+ nsTArray<RefPtr<SimpleTextTrackEvent>> eventList;
+ // Step 9, 10.
+ // For each text track cue in missed cues, prepare an event named
+ // enter for the TextTrackCue object with the cue start time.
+ for (uint32_t i = 0; i < missedCues->Length(); ++i) {
+ TextTrackCue* cue = (*missedCues)[i];
+ if (cue) {
+ WEBVTT_LOG("Prepare 'enter' event for cue %p [%f, %f] in missing cues",
+ cue, cue->StartTime(), cue->EndTime());
+ SimpleTextTrackEvent* event = new SimpleTextTrackEvent(
+ u"enter"_ns, cue->StartTime(), cue->GetTrack(), cue);
+ eventList.InsertElementSorted(
+ event, CompareSimpleTextTrackEvents(mMediaElement));
+ affectedTracks.AddTextTrack(cue->GetTrack(),
+ CompareTextTracks(mMediaElement));
+ }
+ }
+
+ // Step 11, 17.
+ for (uint32_t i = 0; i < otherCues->Length(); ++i) {
+ TextTrackCue* cue = (*otherCues)[i];
+ if (cue->GetActive() || missedCues->IsCueExist(cue)) {
+ double time =
+ cue->StartTime() > cue->EndTime() ? cue->StartTime() : cue->EndTime();
+ WEBVTT_LOG("Prepare 'exit' event for cue %p [%f, %f] in other cues", cue,
+ cue->StartTime(), cue->EndTime());
+ SimpleTextTrackEvent* event =
+ new SimpleTextTrackEvent(u"exit"_ns, time, cue->GetTrack(), cue);
+ eventList.InsertElementSorted(
+ event, CompareSimpleTextTrackEvents(mMediaElement));
+ affectedTracks.AddTextTrack(cue->GetTrack(),
+ CompareTextTracks(mMediaElement));
+ }
+ cue->SetActive(false);
+ }
+
+ // Step 12, 17.
+ for (uint32_t i = 0; i < currentCues->Length(); ++i) {
+ TextTrackCue* cue = (*currentCues)[i];
+ if (!cue->GetActive()) {
+ WEBVTT_LOG("Prepare 'enter' event for cue %p [%f, %f] in current cues",
+ cue, cue->StartTime(), cue->EndTime());
+ SimpleTextTrackEvent* event = new SimpleTextTrackEvent(
+ u"enter"_ns, cue->StartTime(), cue->GetTrack(), cue);
+ eventList.InsertElementSorted(
+ event, CompareSimpleTextTrackEvents(mMediaElement));
+ affectedTracks.AddTextTrack(cue->GetTrack(),
+ CompareTextTracks(mMediaElement));
+ }
+ cue->SetActive(true);
+ }
+
+ // Fire the eventList
+ for (uint32_t i = 0; i < eventList.Length(); ++i) {
+ eventList[i]->Dispatch();
+ }
+
+ // Step 16.
+ for (uint32_t i = 0; i < affectedTracks.Length(); ++i) {
+ TextTrack* ttrack = affectedTracks[i];
+ if (ttrack) {
+ ttrack->DispatchAsyncTrustedEvent(u"cuechange"_ns);
+ HTMLTrackElement* trackElement = ttrack->GetTrackElement();
+ if (trackElement) {
+ trackElement->DispatchTrackRunnable(u"cuechange"_ns);
+ }
+ }
+ }
+
+ mLastTimeMarchesOnCalled = currentPlaybackTime;
+
+ // Step 18.
+ UpdateCueDisplay();
+}
+
+void TextTrackManager::NotifyCueUpdated(TextTrackCue* aCue) {
+ // TODO: Add/Reorder the cue to mNewCues if we have some optimization?
+ WEBVTT_LOG("NotifyCueUpdated, cue=%p", aCue);
+ MaybeRunTimeMarchesOn();
+ // For the case "Texttrack.mode = hidden/showing", if the mode
+ // changing between showing and hidden, TimeMarchesOn
+ // doesn't render the cue. Call DispatchUpdateCueDisplay() explicitly.
+ DispatchUpdateCueDisplay();
+}
+
+void TextTrackManager::NotifyReset() {
+ // https://html.spec.whatwg.org/multipage/media.html#text-track-cue-active-flag
+ // This will unset all cues' active flag and update the cue display.
+ WEBVTT_LOG("NotifyReset");
+ mLastTimeMarchesOnCalled = media::TimeUnit::Zero();
+ for (uint32_t idx = 0; idx < mTextTracks->Length(); ++idx) {
+ (*mTextTracks)[idx]->SetCuesInactive();
+ }
+ UpdateCueDisplay();
+}
+
+bool TextTrackManager::IsLoaded() {
+ return mTextTracks ? mTextTracks->AreTextTracksLoaded() : true;
+}
+
+bool TextTrackManager::IsShutdown() const {
+ return (mShutdown || !sParserWrapper);
+}
+
+void TextTrackManager::MaybeRunTimeMarchesOn() {
+ MOZ_ASSERT(mMediaElement);
+ // According to spec, we should check media element's show poster flag before
+ // running `TimeMarchesOn` in following situations, (1) add cue (2) remove cue
+ // (3) cue's start time changes (4) cues's end time changes
+ // https://html.spec.whatwg.org/multipage/media.html#playing-the-media-resource:time-marches-on
+ // https://html.spec.whatwg.org/multipage/media.html#text-track-api:time-marches-on
+ if (mMediaElement->GetShowPosterFlag()) {
+ return;
+ }
+ TimeMarchesOn();
+}
+
+} // namespace mozilla::dom