diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
commit | 26a029d407be480d791972afb5975cf62c9360a6 (patch) | |
tree | f435a8308119effd964b339f76abb83a57c29483 /dom/html/TextTrackManager.cpp | |
parent | Initial commit. (diff) | |
download | firefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz firefox-26a029d407be480d791972afb5975cf62c9360a6.zip |
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/html/TextTrackManager.cpp')
-rw-r--r-- | dom/html/TextTrackManager.cpp | 874 |
1 files changed, 874 insertions, 0 deletions
diff --git a/dom/html/TextTrackManager.cpp b/dom/html/TextTrackManager.cpp new file mode 100644 index 0000000000..7aacb5bf73 --- /dev/null +++ b/dom/html/TextTrackManager.cpp @@ -0,0 +1,874 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/TextTrackManager.h" +#include "mozilla/ClearOnShutdown.h" +#include "mozilla/CycleCollectedJSContext.h" +#include "mozilla/Maybe.h" +#include "mozilla/Telemetry.h" +#include "mozilla/dom/Document.h" +#include "mozilla/dom/Event.h" +#include "mozilla/dom/HTMLMediaElement.h" +#include "mozilla/dom/HTMLTrackElement.h" +#include "mozilla/dom/HTMLVideoElement.h" +#include "mozilla/dom/TextTrack.h" +#include "mozilla/dom/TextTrackCue.h" +#include "nsComponentManagerUtils.h" +#include "nsGlobalWindowInner.h" +#include "nsIFrame.h" +#include "nsIWebVTTParserWrapper.h" +#include "nsVariant.h" +#include "nsVideoFrame.h" + +mozilla::LazyLogModule gTextTrackLog("WebVTT"); + +#define WEBVTT_LOG(msg, ...) \ + MOZ_LOG(gTextTrackLog, LogLevel::Debug, \ + ("TextTrackManager=%p, " msg, this, ##__VA_ARGS__)) +#define WEBVTT_LOGV(msg, ...) \ + MOZ_LOG(gTextTrackLog, LogLevel::Verbose, \ + ("TextTrackManager=%p, " msg, this, ##__VA_ARGS__)) + +namespace mozilla::dom { + +NS_IMPL_ISUPPORTS(TextTrackManager::ShutdownObserverProxy, nsIObserver); + +void TextTrackManager::ShutdownObserverProxy::Unregister() { + nsContentUtils::UnregisterShutdownObserver(this); + mManager = nullptr; +} + +CompareTextTracks::CompareTextTracks(HTMLMediaElement* aMediaElement) { + mMediaElement = aMediaElement; +} + +Maybe<uint32_t> CompareTextTracks::TrackChildPosition( + TextTrack* aTextTrack) const { + MOZ_DIAGNOSTIC_ASSERT(aTextTrack); + HTMLTrackElement* trackElement = aTextTrack->GetTrackElement(); + if (!trackElement) { + return Nothing(); + } + return mMediaElement->ComputeIndexOf(trackElement); +} + +bool CompareTextTracks::Equals(TextTrack* aOne, TextTrack* aTwo) const { + // Two tracks can never be equal. If they have corresponding TrackElements + // they would need to occupy the same tree position (impossible) and in the + // case of tracks coming from AddTextTrack source we put the newest at the + // last position, so they won't be equal as well. + return false; +} + +bool CompareTextTracks::LessThan(TextTrack* aOne, TextTrack* aTwo) const { + // Protect against nullptr TextTrack objects; treat them as + // sorting toward the end. + if (!aOne) { + return false; + } + if (!aTwo) { + return true; + } + TextTrackSource sourceOne = aOne->GetTextTrackSource(); + TextTrackSource sourceTwo = aTwo->GetTextTrackSource(); + if (sourceOne != sourceTwo) { + return sourceOne == TextTrackSource::Track || + (sourceOne == TextTrackSource::AddTextTrack && + sourceTwo == TextTrackSource::MediaResourceSpecific); + } + switch (sourceOne) { + case TextTrackSource::Track: { + Maybe<uint32_t> positionOne = TrackChildPosition(aOne); + Maybe<uint32_t> positionTwo = TrackChildPosition(aTwo); + // If either position one or positiontwo are Nothing then something has + // gone wrong. In this case we should just put them at the back of the + // list. + return positionOne.isSome() && positionTwo.isSome() && + *positionOne < *positionTwo; + } + case TextTrackSource::AddTextTrack: + // For AddTextTrack sources the tracks will already be in the correct + // relative order in the source array. Assume we're called in iteration + // order and can therefore always report aOne < aTwo to maintain the + // original temporal ordering. + return true; + case TextTrackSource::MediaResourceSpecific: + // No rules for Media Resource Specific tracks yet. + break; + } + return true; +} + +NS_IMPL_CYCLE_COLLECTION(TextTrackManager, mMediaElement, mTextTracks, + mPendingTextTracks, mNewCues) + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(TextTrackManager) + NS_INTERFACE_MAP_ENTRY(nsIDOMEventListener) +NS_INTERFACE_MAP_END + +NS_IMPL_CYCLE_COLLECTING_ADDREF(TextTrackManager) +NS_IMPL_CYCLE_COLLECTING_RELEASE(TextTrackManager) + +StaticRefPtr<nsIWebVTTParserWrapper> TextTrackManager::sParserWrapper; + +TextTrackManager::TextTrackManager(HTMLMediaElement* aMediaElement) + : mMediaElement(aMediaElement), + mHasSeeked(false), + mLastTimeMarchesOnCalled(media::TimeUnit::Zero()), + mTimeMarchesOnDispatched(false), + mUpdateCueDisplayDispatched(false), + performedTrackSelection(false), + mShutdown(false) { + nsISupports* parentObject = mMediaElement->OwnerDoc()->GetParentObject(); + + NS_ENSURE_TRUE_VOID(parentObject); + WEBVTT_LOG("Create TextTrackManager"); + nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(parentObject); + mNewCues = new TextTrackCueList(window); + mTextTracks = new TextTrackList(window, this); + mPendingTextTracks = new TextTrackList(window, this); + + if (!sParserWrapper) { + nsCOMPtr<nsIWebVTTParserWrapper> parserWrapper = + do_CreateInstance(NS_WEBVTTPARSERWRAPPER_CONTRACTID); + MOZ_ASSERT(parserWrapper, "Can't create nsIWebVTTParserWrapper"); + sParserWrapper = parserWrapper; + ClearOnShutdown(&sParserWrapper); + } + mShutdownProxy = new ShutdownObserverProxy(this); +} + +TextTrackManager::~TextTrackManager() { + WEBVTT_LOG("~TextTrackManager"); + mShutdownProxy->Unregister(); +} + +TextTrackList* TextTrackManager::GetTextTracks() const { return mTextTracks; } + +already_AddRefed<TextTrack> TextTrackManager::AddTextTrack( + TextTrackKind aKind, const nsAString& aLabel, const nsAString& aLanguage, + TextTrackMode aMode, TextTrackReadyState aReadyState, + TextTrackSource aTextTrackSource) { + if (!mMediaElement || !mTextTracks) { + return nullptr; + } + RefPtr<TextTrack> track = mTextTracks->AddTextTrack( + aKind, aLabel, aLanguage, aMode, aReadyState, aTextTrackSource, + CompareTextTracks(mMediaElement)); + WEBVTT_LOG("AddTextTrack %p kind %" PRIu32 " Label %s Language %s", + track.get(), static_cast<uint32_t>(aKind), + NS_ConvertUTF16toUTF8(aLabel).get(), + NS_ConvertUTF16toUTF8(aLanguage).get()); + AddCues(track); + + if (aTextTrackSource == TextTrackSource::Track) { + RefPtr<nsIRunnable> task = NewRunnableMethod( + "dom::TextTrackManager::HonorUserPreferencesForTrackSelection", this, + &TextTrackManager::HonorUserPreferencesForTrackSelection); + NS_DispatchToMainThread(task.forget()); + } + + return track.forget(); +} + +void TextTrackManager::AddTextTrack(TextTrack* aTextTrack) { + if (!mMediaElement || !mTextTracks) { + return; + } + WEBVTT_LOG("AddTextTrack TextTrack %p", aTextTrack); + mTextTracks->AddTextTrack(aTextTrack, CompareTextTracks(mMediaElement)); + AddCues(aTextTrack); + + if (aTextTrack->GetTextTrackSource() == TextTrackSource::Track) { + RefPtr<nsIRunnable> task = NewRunnableMethod( + "dom::TextTrackManager::HonorUserPreferencesForTrackSelection", this, + &TextTrackManager::HonorUserPreferencesForTrackSelection); + NS_DispatchToMainThread(task.forget()); + } +} + +void TextTrackManager::AddCues(TextTrack* aTextTrack) { + if (!mNewCues) { + WEBVTT_LOG("AddCues mNewCues is null"); + return; + } + + TextTrackCueList* cueList = aTextTrack->GetCues(); + if (cueList) { + bool dummy; + WEBVTT_LOGV("AddCues, CuesNum=%d", cueList->Length()); + for (uint32_t i = 0; i < cueList->Length(); ++i) { + mNewCues->AddCue(*cueList->IndexedGetter(i, dummy)); + } + MaybeRunTimeMarchesOn(); + } +} + +void TextTrackManager::RemoveTextTrack(TextTrack* aTextTrack, + bool aPendingListOnly) { + if (!mPendingTextTracks || !mTextTracks) { + return; + } + + WEBVTT_LOG("RemoveTextTrack TextTrack %p", aTextTrack); + mPendingTextTracks->RemoveTextTrack(aTextTrack); + if (aPendingListOnly) { + return; + } + + mTextTracks->RemoveTextTrack(aTextTrack); + // Remove the cues in mNewCues belong to aTextTrack. + TextTrackCueList* removeCueList = aTextTrack->GetCues(); + if (removeCueList) { + WEBVTT_LOGV("RemoveTextTrack removeCuesNum=%d", removeCueList->Length()); + for (uint32_t i = 0; i < removeCueList->Length(); ++i) { + mNewCues->RemoveCue(*((*removeCueList)[i])); + } + MaybeRunTimeMarchesOn(); + } +} + +void TextTrackManager::DidSeek() { + WEBVTT_LOG("DidSeek"); + mHasSeeked = true; +} + +void TextTrackManager::UpdateCueDisplay() { + WEBVTT_LOG("UpdateCueDisplay"); + mUpdateCueDisplayDispatched = false; + + if (!mMediaElement || !mTextTracks || IsShutdown()) { + WEBVTT_LOG("Abort UpdateCueDisplay."); + return; + } + + nsIFrame* frame = mMediaElement->GetPrimaryFrame(); + nsVideoFrame* videoFrame = do_QueryFrame(frame); + if (!videoFrame) { + WEBVTT_LOG("Abort UpdateCueDisplay, because of no video frame."); + return; + } + + nsCOMPtr<nsIContent> overlay = videoFrame->GetCaptionOverlay(); + if (!overlay) { + WEBVTT_LOG("Abort UpdateCueDisplay, because of no overlay."); + return; + } + + RefPtr<nsPIDOMWindowInner> window = + mMediaElement->OwnerDoc()->GetInnerWindow(); + if (!window) { + WEBVTT_LOG("Abort UpdateCueDisplay, because of no window."); + } + + nsTArray<RefPtr<TextTrackCue>> showingCues; + mTextTracks->GetShowingCues(showingCues); + + WEBVTT_LOG("UpdateCueDisplay, processCues, showingCuesNum=%zu", + showingCues.Length()); + RefPtr<nsVariantCC> jsCues = new nsVariantCC(); + jsCues->SetAsArray(nsIDataType::VTYPE_INTERFACE, &NS_GET_IID(EventTarget), + showingCues.Length(), + static_cast<void*>(showingCues.Elements())); + nsCOMPtr<nsIContent> controls = videoFrame->GetVideoControls(); + + nsContentUtils::AddScriptRunner(NS_NewRunnableFunction( + "TextTrackManager::UpdateCueDisplay", + [window, jsCues, overlay, controls]() { + if (sParserWrapper) { + sParserWrapper->ProcessCues(window, jsCues, overlay, controls); + } + })); +} + +void TextTrackManager::NotifyCueAdded(TextTrackCue& aCue) { + WEBVTT_LOG("NotifyCueAdded, cue=%p", &aCue); + if (mNewCues) { + mNewCues->AddCue(aCue); + } + MaybeRunTimeMarchesOn(); +} + +void TextTrackManager::NotifyCueRemoved(TextTrackCue& aCue) { + WEBVTT_LOG("NotifyCueRemoved, cue=%p", &aCue); + if (mNewCues) { + mNewCues->RemoveCue(aCue); + } + MaybeRunTimeMarchesOn(); + DispatchUpdateCueDisplay(); +} + +void TextTrackManager::PopulatePendingList() { + if (!mTextTracks || !mPendingTextTracks || !mMediaElement) { + return; + } + uint32_t len = mTextTracks->Length(); + bool dummy; + for (uint32_t index = 0; index < len; ++index) { + TextTrack* ttrack = mTextTracks->IndexedGetter(index, dummy); + if (ttrack && ttrack->Mode() != TextTrackMode::Disabled && + ttrack->ReadyState() == TextTrackReadyState::Loading) { + mPendingTextTracks->AddTextTrack(ttrack, + CompareTextTracks(mMediaElement)); + } + } +} + +void TextTrackManager::AddListeners() { + if (mMediaElement) { + mMediaElement->AddEventListener(u"resizecaption"_ns, this, false, false); + mMediaElement->AddEventListener(u"resizevideocontrols"_ns, this, false, + false); + mMediaElement->AddEventListener(u"seeked"_ns, this, false, false); + mMediaElement->AddEventListener(u"controlbarchange"_ns, this, false, true); + } +} + +void TextTrackManager::HonorUserPreferencesForTrackSelection() { + if (performedTrackSelection || !mTextTracks) { + return; + } + WEBVTT_LOG("HonorUserPreferencesForTrackSelection"); + TextTrackKind ttKinds[] = {TextTrackKind::Captions, TextTrackKind::Subtitles}; + + // Steps 1 - 3: Perform automatic track selection for different TextTrack + // Kinds. + PerformTrackSelection(ttKinds, ArrayLength(ttKinds)); + PerformTrackSelection(TextTrackKind::Descriptions); + PerformTrackSelection(TextTrackKind::Chapters); + + // Step 4: Set all TextTracks with a kind of metadata that are disabled + // to hidden. + for (uint32_t i = 0; i < mTextTracks->Length(); i++) { + TextTrack* track = (*mTextTracks)[i]; + if (track->Kind() == TextTrackKind::Metadata && TrackIsDefault(track) && + track->Mode() == TextTrackMode::Disabled) { + track->SetMode(TextTrackMode::Hidden); + } + } + + performedTrackSelection = true; +} + +bool TextTrackManager::TrackIsDefault(TextTrack* aTextTrack) { + HTMLTrackElement* trackElement = aTextTrack->GetTrackElement(); + if (!trackElement) { + return false; + } + return trackElement->Default(); +} + +void TextTrackManager::PerformTrackSelection(TextTrackKind aTextTrackKind) { + TextTrackKind ttKinds[] = {aTextTrackKind}; + PerformTrackSelection(ttKinds, ArrayLength(ttKinds)); +} + +void TextTrackManager::PerformTrackSelection(TextTrackKind aTextTrackKinds[], + uint32_t size) { + nsTArray<TextTrack*> candidates; + GetTextTracksOfKinds(aTextTrackKinds, size, candidates); + + // Step 3: If any TextTracks in candidates are showing then abort these steps. + for (uint32_t i = 0; i < candidates.Length(); i++) { + if (candidates[i]->Mode() == TextTrackMode::Showing) { + WEBVTT_LOGV("PerformTrackSelection Showing return kind %d", + static_cast<int>(candidates[i]->Kind())); + return; + } + } + + // Step 4: Honor user preferences for track selection, otherwise, set the + // first TextTrack in candidates with a default attribute to showing. + // TODO: Bug 981691 - Honor user preferences for text track selection. + for (uint32_t i = 0; i < candidates.Length(); i++) { + if (TrackIsDefault(candidates[i]) && + candidates[i]->Mode() == TextTrackMode::Disabled) { + candidates[i]->SetMode(TextTrackMode::Showing); + WEBVTT_LOGV("PerformTrackSelection set Showing kind %d", + static_cast<int>(candidates[i]->Kind())); + return; + } + } +} + +void TextTrackManager::GetTextTracksOfKinds(TextTrackKind aTextTrackKinds[], + uint32_t size, + nsTArray<TextTrack*>& aTextTracks) { + for (uint32_t i = 0; i < size; i++) { + GetTextTracksOfKind(aTextTrackKinds[i], aTextTracks); + } +} + +void TextTrackManager::GetTextTracksOfKind(TextTrackKind aTextTrackKind, + nsTArray<TextTrack*>& aTextTracks) { + if (!mTextTracks) { + return; + } + for (uint32_t i = 0; i < mTextTracks->Length(); i++) { + TextTrack* textTrack = (*mTextTracks)[i]; + if (textTrack->Kind() == aTextTrackKind) { + aTextTracks.AppendElement(textTrack); + } + } +} + +NS_IMETHODIMP +TextTrackManager::HandleEvent(Event* aEvent) { + if (!mTextTracks) { + return NS_OK; + } + + nsAutoString type; + aEvent->GetType(type); + WEBVTT_LOG("Handle event %s", NS_ConvertUTF16toUTF8(type).get()); + + const bool setDirty = type.EqualsLiteral("seeked") || + type.EqualsLiteral("resizecaption") || + type.EqualsLiteral("resizevideocontrols"); + const bool updateDisplay = type.EqualsLiteral("controlbarchange") || + type.EqualsLiteral("resizecaption"); + + if (setDirty) { + for (uint32_t i = 0; i < mTextTracks->Length(); i++) { + ((*mTextTracks)[i])->SetCuesDirty(); + } + } + if (updateDisplay) { + UpdateCueDisplay(); + } + + return NS_OK; +} + +class SimpleTextTrackEvent : public Runnable { + public: + friend class CompareSimpleTextTrackEvents; + SimpleTextTrackEvent(const nsAString& aEventName, double aTime, + TextTrack* aTrack, TextTrackCue* aCue) + : Runnable("dom::SimpleTextTrackEvent"), + mName(aEventName), + mTime(aTime), + mTrack(aTrack), + mCue(aCue) {} + + NS_IMETHOD Run() override { + WEBVTT_LOGV("SimpleTextTrackEvent cue %p mName %s mTime %lf", mCue.get(), + NS_ConvertUTF16toUTF8(mName).get(), mTime); + mCue->DispatchTrustedEvent(mName); + return NS_OK; + } + + void Dispatch() { + if (nsCOMPtr<nsIGlobalObject> global = mCue->GetOwnerGlobal()) { + global->Dispatch(do_AddRef(this)); + } else { + NS_DispatchToMainThread(do_AddRef(this)); + } + } + + private: + nsString mName; + double mTime; + TextTrack* mTrack; + RefPtr<TextTrackCue> mCue; +}; + +class CompareSimpleTextTrackEvents { + private: + Maybe<uint32_t> TrackChildPosition(SimpleTextTrackEvent* aEvent) const { + if (aEvent->mTrack) { + HTMLTrackElement* trackElement = aEvent->mTrack->GetTrackElement(); + if (trackElement) { + return mMediaElement->ComputeIndexOf(trackElement); + } + } + return Nothing(); + } + HTMLMediaElement* mMediaElement; + + public: + explicit CompareSimpleTextTrackEvents(HTMLMediaElement* aMediaElement) { + mMediaElement = aMediaElement; + } + + bool Equals(SimpleTextTrackEvent* aOne, SimpleTextTrackEvent* aTwo) const { + return false; + } + + bool LessThan(SimpleTextTrackEvent* aOne, SimpleTextTrackEvent* aTwo) const { + // TimeMarchesOn step 13.1. + if (aOne->mTime < aTwo->mTime) { + return true; + } + if (aOne->mTime > aTwo->mTime) { + return false; + } + + // TimeMarchesOn step 13.2 text track cue order. + // TextTrack position in TextTrackList + TextTrack* t1 = aOne->mTrack; + TextTrack* t2 = aTwo->mTrack; + MOZ_ASSERT(t1, "CompareSimpleTextTrackEvents t1 is null"); + MOZ_ASSERT(t2, "CompareSimpleTextTrackEvents t2 is null"); + if (t1 != t2) { + TextTrackList* tList = t1->GetTextTrackList(); + MOZ_ASSERT(tList, "CompareSimpleTextTrackEvents tList is null"); + nsTArray<RefPtr<TextTrack>>& textTracks = tList->GetTextTrackArray(); + auto index1 = textTracks.IndexOf(t1); + auto index2 = textTracks.IndexOf(t2); + if (index1 < index2) { + return true; + } + if (index1 > index2) { + return false; + } + } + + MOZ_ASSERT(t1 == t2, "CompareSimpleTextTrackEvents t1 != t2"); + // c1 and c2 are both belongs to t1. + TextTrackCue* c1 = aOne->mCue; + TextTrackCue* c2 = aTwo->mCue; + if (c1 != c2) { + if (c1->StartTime() < c2->StartTime()) { + return true; + } + if (c1->StartTime() > c2->StartTime()) { + return false; + } + if (c1->EndTime() < c2->EndTime()) { + return true; + } + if (c1->EndTime() > c2->EndTime()) { + return false; + } + + TextTrackCueList* cueList = t1->GetCues(); + MOZ_ASSERT(cueList); + nsTArray<RefPtr<TextTrackCue>>& cues = cueList->GetCuesArray(); + auto index1 = cues.IndexOf(c1); + auto index2 = cues.IndexOf(c2); + if (index1 < index2) { + return true; + } + if (index1 > index2) { + return false; + } + } + + // TimeMarchesOn step 13.3. + if (aOne->mName.EqualsLiteral("enter") || + aTwo->mName.EqualsLiteral("exit")) { + return true; + } + return false; + } +}; + +class TextTrackListInternal { + public: + void AddTextTrack(TextTrack* aTextTrack, + const CompareTextTracks& aCompareTT) { + if (!mTextTracks.Contains(aTextTrack)) { + mTextTracks.InsertElementSorted(aTextTrack, aCompareTT); + } + } + uint32_t Length() const { return mTextTracks.Length(); } + TextTrack* operator[](uint32_t aIndex) { + return mTextTracks.SafeElementAt(aIndex, nullptr); + } + + private: + nsTArray<RefPtr<TextTrack>> mTextTracks; +}; + +void TextTrackManager::DispatchUpdateCueDisplay() { + if (!mUpdateCueDisplayDispatched && !IsShutdown()) { + WEBVTT_LOG("DispatchUpdateCueDisplay"); + if (nsPIDOMWindowInner* win = mMediaElement->OwnerDoc()->GetInnerWindow()) { + nsGlobalWindowInner::Cast(win)->Dispatch( + NewRunnableMethod("dom::TextTrackManager::UpdateCueDisplay", this, + &TextTrackManager::UpdateCueDisplay)); + mUpdateCueDisplayDispatched = true; + } + } +} + +void TextTrackManager::DispatchTimeMarchesOn() { + // Run the algorithm if no previous instance is still running, otherwise + // enqueue the current playback position and whether only that changed + // through its usual monotonic increase during normal playback; current + // executing call upon completion will check queue for further 'work'. + if (!mTimeMarchesOnDispatched && !IsShutdown()) { + WEBVTT_LOG("DispatchTimeMarchesOn"); + if (nsPIDOMWindowInner* win = mMediaElement->OwnerDoc()->GetInnerWindow()) { + nsGlobalWindowInner::Cast(win)->Dispatch( + NewRunnableMethod("dom::TextTrackManager::TimeMarchesOn", this, + &TextTrackManager::TimeMarchesOn)); + mTimeMarchesOnDispatched = true; + } + } +} + +// https://html.spec.whatwg.org/multipage/embedded-content.html#time-marches-on +void TextTrackManager::TimeMarchesOn() { + NS_ASSERTION(NS_IsMainThread(), "Wrong thread!"); + mTimeMarchesOnDispatched = false; + + CycleCollectedJSContext* context = CycleCollectedJSContext::Get(); + if (context && context->IsInStableOrMetaStableState()) { + // FireTimeUpdate can be called while at stable state following a + // current position change which triggered a state watcher in MediaDecoder + // (see bug 1443429). + // TimeMarchesOn() will modify JS attributes which is forbidden while in + // stable state. So we dispatch a task to perform such operation later + // instead. + DispatchTimeMarchesOn(); + return; + } + WEBVTT_LOG("TimeMarchesOn"); + + // Early return if we don't have any TextTracks or shutting down. + if (!mTextTracks || mTextTracks->Length() == 0 || IsShutdown() || + !mMediaElement) { + return; + } + + if (mMediaElement->ReadyState() == HTMLMediaElement_Binding::HAVE_NOTHING) { + WEBVTT_LOG( + "TimeMarchesOn return because media doesn't contain any data yet"); + return; + } + + if (mMediaElement->Seeking()) { + WEBVTT_LOG("TimeMarchesOn return during seeking"); + return; + } + + // Step 1, 2. + nsISupports* parentObject = mMediaElement->OwnerDoc()->GetParentObject(); + if (NS_WARN_IF(!parentObject)) { + return; + } + nsCOMPtr<nsPIDOMWindowInner> window = do_QueryInterface(parentObject); + RefPtr<TextTrackCueList> currentCues = new TextTrackCueList(window); + RefPtr<TextTrackCueList> otherCues = new TextTrackCueList(window); + + // Step 3. + auto currentPlaybackTime = + media::TimeUnit::FromSeconds(mMediaElement->CurrentTime()); + bool hasNormalPlayback = !mHasSeeked; + mHasSeeked = false; + WEBVTT_LOG( + "TimeMarchesOn mLastTimeMarchesOnCalled %lf currentPlaybackTime %lf " + "hasNormalPlayback %d", + mLastTimeMarchesOnCalled.ToSeconds(), currentPlaybackTime.ToSeconds(), + hasNormalPlayback); + + // The reason we collect other cues is (1) to change active cues to inactive, + // (2) find missing cues, so we actually no need to process all cues. We just + // need to handle cues which are in the time interval [lastTime:currentTime] + // or [currentTime:lastTime] (seeking forward). That can help us to reduce the + // size of other cues, which can improve execution time. + auto start = std::min(mLastTimeMarchesOnCalled, currentPlaybackTime); + auto end = std::max(mLastTimeMarchesOnCalled, currentPlaybackTime); + media::TimeInterval interval(start, end); + WEBVTT_LOGV("TimeMarchesOn Time interval [%f:%f]", start.ToSeconds(), + end.ToSeconds()); + for (uint32_t idx = 0; idx < mTextTracks->Length(); ++idx) { + TextTrack* track = (*mTextTracks)[idx]; + if (track) { + track->GetCurrentCuesAndOtherCues(currentCues, otherCues, interval); + } + } + + // Step 4. + RefPtr<TextTrackCueList> missedCues = new TextTrackCueList(window); + if (hasNormalPlayback) { + for (uint32_t i = 0; i < otherCues->Length(); ++i) { + TextTrackCue* cue = (*otherCues)[i]; + if (cue->StartTime() >= mLastTimeMarchesOnCalled.ToSeconds() && + cue->EndTime() <= currentPlaybackTime.ToSeconds()) { + missedCues->AddCue(*cue); + } + } + } + + WEBVTT_LOGV("TimeMarchesOn currentCues %d", currentCues->Length()); + WEBVTT_LOGV("TimeMarchesOn otherCues %d", otherCues->Length()); + WEBVTT_LOGV("TimeMarchesOn missedCues %d", missedCues->Length()); + // Step 5. Empty now. + // TODO: Step 6: fire timeupdate? + + // Step 7. Abort steps if condition 1, 2, 3 are satisfied. + // 1. All of the cues in current cues have their active flag set. + // 2. None of the cues in other cues have their active flag set. + // 3. Missed cues is empty. + bool c1 = true; + for (uint32_t i = 0; i < currentCues->Length(); ++i) { + if (!(*currentCues)[i]->GetActive()) { + c1 = false; + break; + } + } + bool c2 = true; + for (uint32_t i = 0; i < otherCues->Length(); ++i) { + if ((*otherCues)[i]->GetActive()) { + c2 = false; + break; + } + } + bool c3 = (missedCues->Length() == 0); + if (c1 && c2 && c3) { + mLastTimeMarchesOnCalled = currentPlaybackTime; + WEBVTT_LOG("TimeMarchesOn step 7 return, mLastTimeMarchesOnCalled %lf", + mLastTimeMarchesOnCalled.ToSeconds()); + return; + } + + // Step 8. Respect PauseOnExit flag if not seek. + if (hasNormalPlayback) { + for (uint32_t i = 0; i < otherCues->Length(); ++i) { + TextTrackCue* cue = (*otherCues)[i]; + if (cue && cue->PauseOnExit() && cue->GetActive()) { + WEBVTT_LOG("TimeMarchesOn pause the MediaElement"); + mMediaElement->Pause(); + break; + } + } + for (uint32_t i = 0; i < missedCues->Length(); ++i) { + TextTrackCue* cue = (*missedCues)[i]; + if (cue && cue->PauseOnExit()) { + WEBVTT_LOG("TimeMarchesOn pause the MediaElement"); + mMediaElement->Pause(); + break; + } + } + } + + // Step 15. + // Sort text tracks in the same order as the text tracks appear + // in the media element's list of text tracks, and remove + // duplicates. + TextTrackListInternal affectedTracks; + // Step 13, 14. + nsTArray<RefPtr<SimpleTextTrackEvent>> eventList; + // Step 9, 10. + // For each text track cue in missed cues, prepare an event named + // enter for the TextTrackCue object with the cue start time. + for (uint32_t i = 0; i < missedCues->Length(); ++i) { + TextTrackCue* cue = (*missedCues)[i]; + if (cue) { + WEBVTT_LOG("Prepare 'enter' event for cue %p [%f, %f] in missing cues", + cue, cue->StartTime(), cue->EndTime()); + SimpleTextTrackEvent* event = new SimpleTextTrackEvent( + u"enter"_ns, cue->StartTime(), cue->GetTrack(), cue); + eventList.InsertElementSorted( + event, CompareSimpleTextTrackEvents(mMediaElement)); + affectedTracks.AddTextTrack(cue->GetTrack(), + CompareTextTracks(mMediaElement)); + } + } + + // Step 11, 17. + for (uint32_t i = 0; i < otherCues->Length(); ++i) { + TextTrackCue* cue = (*otherCues)[i]; + if (cue->GetActive() || missedCues->IsCueExist(cue)) { + double time = + cue->StartTime() > cue->EndTime() ? cue->StartTime() : cue->EndTime(); + WEBVTT_LOG("Prepare 'exit' event for cue %p [%f, %f] in other cues", cue, + cue->StartTime(), cue->EndTime()); + SimpleTextTrackEvent* event = + new SimpleTextTrackEvent(u"exit"_ns, time, cue->GetTrack(), cue); + eventList.InsertElementSorted( + event, CompareSimpleTextTrackEvents(mMediaElement)); + affectedTracks.AddTextTrack(cue->GetTrack(), + CompareTextTracks(mMediaElement)); + } + cue->SetActive(false); + } + + // Step 12, 17. + for (uint32_t i = 0; i < currentCues->Length(); ++i) { + TextTrackCue* cue = (*currentCues)[i]; + if (!cue->GetActive()) { + WEBVTT_LOG("Prepare 'enter' event for cue %p [%f, %f] in current cues", + cue, cue->StartTime(), cue->EndTime()); + SimpleTextTrackEvent* event = new SimpleTextTrackEvent( + u"enter"_ns, cue->StartTime(), cue->GetTrack(), cue); + eventList.InsertElementSorted( + event, CompareSimpleTextTrackEvents(mMediaElement)); + affectedTracks.AddTextTrack(cue->GetTrack(), + CompareTextTracks(mMediaElement)); + } + cue->SetActive(true); + } + + // Fire the eventList + for (uint32_t i = 0; i < eventList.Length(); ++i) { + eventList[i]->Dispatch(); + } + + // Step 16. + for (uint32_t i = 0; i < affectedTracks.Length(); ++i) { + TextTrack* ttrack = affectedTracks[i]; + if (ttrack) { + ttrack->DispatchAsyncTrustedEvent(u"cuechange"_ns); + HTMLTrackElement* trackElement = ttrack->GetTrackElement(); + if (trackElement) { + trackElement->DispatchTrackRunnable(u"cuechange"_ns); + } + } + } + + mLastTimeMarchesOnCalled = currentPlaybackTime; + + // Step 18. + UpdateCueDisplay(); +} + +void TextTrackManager::NotifyCueUpdated(TextTrackCue* aCue) { + // TODO: Add/Reorder the cue to mNewCues if we have some optimization? + WEBVTT_LOG("NotifyCueUpdated, cue=%p", aCue); + MaybeRunTimeMarchesOn(); + // For the case "Texttrack.mode = hidden/showing", if the mode + // changing between showing and hidden, TimeMarchesOn + // doesn't render the cue. Call DispatchUpdateCueDisplay() explicitly. + DispatchUpdateCueDisplay(); +} + +void TextTrackManager::NotifyReset() { + // https://html.spec.whatwg.org/multipage/media.html#text-track-cue-active-flag + // This will unset all cues' active flag and update the cue display. + WEBVTT_LOG("NotifyReset"); + mLastTimeMarchesOnCalled = media::TimeUnit::Zero(); + for (uint32_t idx = 0; idx < mTextTracks->Length(); ++idx) { + (*mTextTracks)[idx]->SetCuesInactive(); + } + UpdateCueDisplay(); +} + +bool TextTrackManager::IsLoaded() { + return mTextTracks ? mTextTracks->AreTextTracksLoaded() : true; +} + +bool TextTrackManager::IsShutdown() const { + return (mShutdown || !sParserWrapper); +} + +void TextTrackManager::MaybeRunTimeMarchesOn() { + MOZ_ASSERT(mMediaElement); + // According to spec, we should check media element's show poster flag before + // running `TimeMarchesOn` in following situations, (1) add cue (2) remove cue + // (3) cue's start time changes (4) cues's end time changes + // https://html.spec.whatwg.org/multipage/media.html#playing-the-media-resource:time-marches-on + // https://html.spec.whatwg.org/multipage/media.html#text-track-api:time-marches-on + if (mMediaElement->GetShowPosterFlag()) { + return; + } + TimeMarchesOn(); +} + +} // namespace mozilla::dom |