From 26a029d407be480d791972afb5975cf62c9360a6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 02:47:55 +0200 Subject: Adding upstream version 124.0.1. Signed-off-by: Daniel Baumann --- dom/media/webvtt/TextTrack.cpp | 383 + dom/media/webvtt/TextTrack.h | 148 + dom/media/webvtt/TextTrackCue.cpp | 258 + dom/media/webvtt/TextTrackCue.h | 342 + dom/media/webvtt/TextTrackCueList.cpp | 125 + dom/media/webvtt/TextTrackCueList.h | 73 + dom/media/webvtt/TextTrackList.cpp | 190 + dom/media/webvtt/TextTrackList.h | 79 + dom/media/webvtt/TextTrackRegion.cpp | 58 + dom/media/webvtt/TextTrackRegion.h | 138 + dom/media/webvtt/WebVTTListener.cpp | 212 + dom/media/webvtt/WebVTTListener.h | 69 + dom/media/webvtt/WebVTTParserWrapper.sys.mjs | 56 + dom/media/webvtt/components.conf | 14 + dom/media/webvtt/moz.build | 52 + dom/media/webvtt/nsIWebVTTListener.idl | 37 + dom/media/webvtt/nsIWebVTTParserWrapper.idl | 94 + dom/media/webvtt/package.json | 6 + dom/media/webvtt/test/crashtests/1304948.html | 33 + dom/media/webvtt/test/crashtests/1319486.html | 27 + dom/media/webvtt/test/crashtests/1533909.html | 17 + dom/media/webvtt/test/crashtests/882549.html | 13 + dom/media/webvtt/test/crashtests/894104.html | 20 + dom/media/webvtt/test/crashtests/crashtests.list | 5 + dom/media/webvtt/test/mochitest/bad-signature.vtt | 1 + dom/media/webvtt/test/mochitest/basic.vtt | 29 + dom/media/webvtt/test/mochitest/bug883173.vtt | 16 + dom/media/webvtt/test/mochitest/long.vtt | 8001 ++++++++++++++++++++ dom/media/webvtt/test/mochitest/manifest.js | 27 + dom/media/webvtt/test/mochitest/mochitest.toml | 81 + dom/media/webvtt/test/mochitest/parser.vtt | 6 + dom/media/webvtt/test/mochitest/region.vtt | 6 + dom/media/webvtt/test/mochitest/sequential.vtt | 10 + .../webvtt/test/mochitest/test_bug1018933.html | 50 + .../webvtt/test/mochitest/test_bug1242594.html | 46 + .../webvtt/test/mochitest/test_bug883173.html | 39 + .../webvtt/test/mochitest/test_bug895091.html | 60 + .../webvtt/test/mochitest/test_bug957847.html | 30 + .../mochitest/test_testtrack_cors_no_response.html | 41 + .../webvtt/test/mochitest/test_texttrack.html | 158 + .../test_texttrack_cors_preload_none.html | 40 + .../test_texttrack_mode_change_during_loading.html | 75 + .../webvtt/test/mochitest/test_texttrack_moz.html | 60 + .../webvtt/test/mochitest/test_texttrackcue.html | 299 + .../test/mochitest/test_texttrackcue_moz.html | 34 + .../test/mochitest/test_texttrackevents_video.html | 91 + .../webvtt/test/mochitest/test_texttracklist.html | 51 + .../test/mochitest/test_texttracklist_moz.html | 34 + .../test/mochitest/test_texttrackregion.html | 58 + .../test/mochitest/test_trackelementevent.html | 77 + .../test/mochitest/test_trackelementsrc.html | 50 + .../webvtt/test/mochitest/test_trackevent.html | 69 + .../webvtt/test/mochitest/test_vttparser.html | 44 + .../mochitest/test_webvtt_empty_displaystate.html | 98 + .../mochitest/test_webvtt_event_same_time.html | 63 + .../test_webvtt_infinite_processing_loop.html | 49 + .../mochitest/test_webvtt_overlapping_time.html | 100 + .../test/mochitest/test_webvtt_positionalign.html | 113 + .../webvtt/test/mochitest/test_webvtt_seeking.html | 110 + ...pdate_display_after_adding_or_removing_cue.html | 93 + .../webvtt/test/mochitest/vttPositionAlign.vtt | 86 + dom/media/webvtt/test/reftest/black.mp4 | Bin 0 -> 15036 bytes .../test/reftest/cues_time_overlapping.webvtt | 7 + dom/media/webvtt/test/reftest/reftest.list | 3 + .../test/reftest/vtt_overlapping_time-ref.html | 29 + .../webvtt/test/reftest/vtt_overlapping_time.html | 30 + .../test/reftest/vtt_reflow_display-ref.html | 28 + .../webvtt/test/reftest/vtt_reflow_display.css | 33 + .../webvtt/test/reftest/vtt_reflow_display.html | 37 + .../vtt_update_display_after_removed_cue.html | 36 + .../vtt_update_display_after_removed_cue_ref.html | 6 + dom/media/webvtt/test/reftest/white.webm | Bin 0 -> 10880 bytes dom/media/webvtt/test/xpcshell/test_parser.js | 158 + dom/media/webvtt/test/xpcshell/xpcshell.toml | 3 + dom/media/webvtt/update-webvtt.js | 61 + dom/media/webvtt/vtt.sys.mjs | 1644 ++++ 76 files changed, 14819 insertions(+) create mode 100644 dom/media/webvtt/TextTrack.cpp create mode 100644 dom/media/webvtt/TextTrack.h create mode 100644 dom/media/webvtt/TextTrackCue.cpp create mode 100644 dom/media/webvtt/TextTrackCue.h create mode 100644 dom/media/webvtt/TextTrackCueList.cpp create mode 100644 dom/media/webvtt/TextTrackCueList.h create mode 100644 dom/media/webvtt/TextTrackList.cpp create mode 100644 dom/media/webvtt/TextTrackList.h create mode 100644 dom/media/webvtt/TextTrackRegion.cpp create mode 100644 dom/media/webvtt/TextTrackRegion.h create mode 100644 dom/media/webvtt/WebVTTListener.cpp create mode 100644 dom/media/webvtt/WebVTTListener.h create mode 100644 dom/media/webvtt/WebVTTParserWrapper.sys.mjs create mode 100644 dom/media/webvtt/components.conf create mode 100644 dom/media/webvtt/moz.build create mode 100644 dom/media/webvtt/nsIWebVTTListener.idl create mode 100644 dom/media/webvtt/nsIWebVTTParserWrapper.idl create mode 100644 dom/media/webvtt/package.json create mode 100644 dom/media/webvtt/test/crashtests/1304948.html create mode 100644 dom/media/webvtt/test/crashtests/1319486.html create mode 100644 dom/media/webvtt/test/crashtests/1533909.html create mode 100644 dom/media/webvtt/test/crashtests/882549.html create mode 100644 dom/media/webvtt/test/crashtests/894104.html create mode 100644 dom/media/webvtt/test/crashtests/crashtests.list create mode 100644 dom/media/webvtt/test/mochitest/bad-signature.vtt create mode 100644 dom/media/webvtt/test/mochitest/basic.vtt create mode 100644 dom/media/webvtt/test/mochitest/bug883173.vtt create mode 100644 dom/media/webvtt/test/mochitest/long.vtt create mode 100644 dom/media/webvtt/test/mochitest/manifest.js create mode 100644 dom/media/webvtt/test/mochitest/mochitest.toml create mode 100644 dom/media/webvtt/test/mochitest/parser.vtt create mode 100644 dom/media/webvtt/test/mochitest/region.vtt create mode 100644 dom/media/webvtt/test/mochitest/sequential.vtt create mode 100644 dom/media/webvtt/test/mochitest/test_bug1018933.html create mode 100644 dom/media/webvtt/test/mochitest/test_bug1242594.html create mode 100644 dom/media/webvtt/test/mochitest/test_bug883173.html create mode 100644 dom/media/webvtt/test/mochitest/test_bug895091.html create mode 100644 dom/media/webvtt/test/mochitest/test_bug957847.html create mode 100644 dom/media/webvtt/test/mochitest/test_testtrack_cors_no_response.html create mode 100644 dom/media/webvtt/test/mochitest/test_texttrack.html create mode 100644 dom/media/webvtt/test/mochitest/test_texttrack_cors_preload_none.html create mode 100644 dom/media/webvtt/test/mochitest/test_texttrack_mode_change_during_loading.html create mode 100644 dom/media/webvtt/test/mochitest/test_texttrack_moz.html create mode 100644 dom/media/webvtt/test/mochitest/test_texttrackcue.html create mode 100644 dom/media/webvtt/test/mochitest/test_texttrackcue_moz.html create mode 100644 dom/media/webvtt/test/mochitest/test_texttrackevents_video.html create mode 100644 dom/media/webvtt/test/mochitest/test_texttracklist.html create mode 100644 dom/media/webvtt/test/mochitest/test_texttracklist_moz.html create mode 100644 dom/media/webvtt/test/mochitest/test_texttrackregion.html create mode 100644 dom/media/webvtt/test/mochitest/test_trackelementevent.html create mode 100644 dom/media/webvtt/test/mochitest/test_trackelementsrc.html create mode 100644 dom/media/webvtt/test/mochitest/test_trackevent.html create mode 100644 dom/media/webvtt/test/mochitest/test_vttparser.html create mode 100644 dom/media/webvtt/test/mochitest/test_webvtt_empty_displaystate.html create mode 100644 dom/media/webvtt/test/mochitest/test_webvtt_event_same_time.html create mode 100644 dom/media/webvtt/test/mochitest/test_webvtt_infinite_processing_loop.html create mode 100644 dom/media/webvtt/test/mochitest/test_webvtt_overlapping_time.html create mode 100644 dom/media/webvtt/test/mochitest/test_webvtt_positionalign.html create mode 100644 dom/media/webvtt/test/mochitest/test_webvtt_seeking.html create mode 100644 dom/media/webvtt/test/mochitest/test_webvtt_update_display_after_adding_or_removing_cue.html create mode 100644 dom/media/webvtt/test/mochitest/vttPositionAlign.vtt create mode 100644 dom/media/webvtt/test/reftest/black.mp4 create mode 100644 dom/media/webvtt/test/reftest/cues_time_overlapping.webvtt create mode 100644 dom/media/webvtt/test/reftest/reftest.list create mode 100644 dom/media/webvtt/test/reftest/vtt_overlapping_time-ref.html create mode 100644 dom/media/webvtt/test/reftest/vtt_overlapping_time.html create mode 100644 dom/media/webvtt/test/reftest/vtt_reflow_display-ref.html create mode 100644 dom/media/webvtt/test/reftest/vtt_reflow_display.css create mode 100644 dom/media/webvtt/test/reftest/vtt_reflow_display.html create mode 100644 dom/media/webvtt/test/reftest/vtt_update_display_after_removed_cue.html create mode 100644 dom/media/webvtt/test/reftest/vtt_update_display_after_removed_cue_ref.html create mode 100644 dom/media/webvtt/test/reftest/white.webm create mode 100644 dom/media/webvtt/test/xpcshell/test_parser.js create mode 100644 dom/media/webvtt/test/xpcshell/xpcshell.toml create mode 100644 dom/media/webvtt/update-webvtt.js create mode 100644 dom/media/webvtt/vtt.sys.mjs (limited to 'dom/media/webvtt') diff --git a/dom/media/webvtt/TextTrack.cpp b/dom/media/webvtt/TextTrack.cpp new file mode 100644 index 0000000000..7ee9ee63d2 --- /dev/null +++ b/dom/media/webvtt/TextTrack.cpp @@ -0,0 +1,383 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 et tw=78: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/AsyncEventDispatcher.h" +#include "mozilla/dom/TextTrack.h" +#include "mozilla/dom/TextTrackBinding.h" +#include "mozilla/dom/TextTrackList.h" +#include "mozilla/dom/TextTrackCue.h" +#include "mozilla/dom/TextTrackCueList.h" +#include "mozilla/dom/TextTrackRegion.h" +#include "mozilla/dom/HTMLMediaElement.h" +#include "mozilla/dom/HTMLTrackElement.h" +#include "nsGlobalWindowInner.h" + +extern mozilla::LazyLogModule gTextTrackLog; + +#define WEBVTT_LOG(msg, ...) \ + MOZ_LOG(gTextTrackLog, LogLevel::Debug, \ + ("TextTrack=%p, " msg, this, ##__VA_ARGS__)) + +namespace mozilla::dom { + +static const char* ToStateStr(const TextTrackMode aMode) { + switch (aMode) { + case TextTrackMode::Disabled: + return "DISABLED"; + case TextTrackMode::Hidden: + return "HIDDEN"; + case TextTrackMode::Showing: + return "SHOWING"; + default: + MOZ_ASSERT_UNREACHABLE("Invalid state."); + } + return "Unknown"; +} + +static const char* ToReadyStateStr(const TextTrackReadyState aState) { + switch (aState) { + case TextTrackReadyState::NotLoaded: + return "NotLoaded"; + case TextTrackReadyState::Loading: + return "Loading"; + case TextTrackReadyState::Loaded: + return "Loaded"; + case TextTrackReadyState::FailedToLoad: + return "FailedToLoad"; + default: + MOZ_ASSERT_UNREACHABLE("Invalid state."); + } + return "Unknown"; +} + +static const char* ToTextTrackKindStr(const TextTrackKind aKind) { + switch (aKind) { + case TextTrackKind::Subtitles: + return "Subtitles"; + case TextTrackKind::Captions: + return "Captions"; + case TextTrackKind::Descriptions: + return "Descriptions"; + case TextTrackKind::Chapters: + return "Chapters"; + case TextTrackKind::Metadata: + return "Metadata"; + default: + MOZ_ASSERT_UNREACHABLE("Invalid kind."); + } + return "Unknown"; +} + +NS_IMPL_CYCLE_COLLECTION_INHERITED(TextTrack, DOMEventTargetHelper, mCueList, + mActiveCueList, mTextTrackList, + mTrackElement) + +NS_IMPL_ADDREF_INHERITED(TextTrack, DOMEventTargetHelper) +NS_IMPL_RELEASE_INHERITED(TextTrack, DOMEventTargetHelper) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(TextTrack) +NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper) + +TextTrack::TextTrack(nsPIDOMWindowInner* aOwnerWindow, TextTrackKind aKind, + const nsAString& aLabel, const nsAString& aLanguage, + TextTrackMode aMode, TextTrackReadyState aReadyState, + TextTrackSource aTextTrackSource) + : DOMEventTargetHelper(aOwnerWindow), + mKind(aKind), + mLabel(aLabel), + mLanguage(aLanguage), + mMode(aMode), + mReadyState(aReadyState), + mTextTrackSource(aTextTrackSource) { + SetDefaultSettings(); +} + +TextTrack::TextTrack(nsPIDOMWindowInner* aOwnerWindow, + TextTrackList* aTextTrackList, TextTrackKind aKind, + const nsAString& aLabel, const nsAString& aLanguage, + TextTrackMode aMode, TextTrackReadyState aReadyState, + TextTrackSource aTextTrackSource) + : DOMEventTargetHelper(aOwnerWindow), + mTextTrackList(aTextTrackList), + mKind(aKind), + mLabel(aLabel), + mLanguage(aLanguage), + mMode(aMode), + mReadyState(aReadyState), + mTextTrackSource(aTextTrackSource) { + SetDefaultSettings(); +} + +TextTrack::~TextTrack() = default; + +void TextTrack::SetDefaultSettings() { + nsPIDOMWindowInner* ownerWindow = GetOwner(); + mCueList = new TextTrackCueList(ownerWindow); + mActiveCueList = new TextTrackCueList(ownerWindow); + mCuePos = 0; + mDirty = false; +} + +JSObject* TextTrack::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + return TextTrack_Binding::Wrap(aCx, this, aGivenProto); +} + +void TextTrack::SetMode(TextTrackMode aValue) { + if (mMode == aValue) { + return; + } + WEBVTT_LOG("Set mode=%s for track kind %s", ToStateStr(aValue), + ToTextTrackKindStr(mKind)); + mMode = aValue; + + HTMLMediaElement* mediaElement = GetMediaElement(); + if (aValue == TextTrackMode::Disabled) { + for (size_t i = 0; i < mCueList->Length() && mediaElement; ++i) { + mediaElement->NotifyCueRemoved(*(*mCueList)[i]); + } + SetCuesInactive(); + } else { + for (size_t i = 0; i < mCueList->Length() && mediaElement; ++i) { + mediaElement->NotifyCueAdded(*(*mCueList)[i]); + } + } + if (mediaElement) { + mediaElement->NotifyTextTrackModeChanged(); + } + // https://html.spec.whatwg.org/multipage/media.html#sourcing-out-of-band-text-tracks:start-the-track-processing-model + // Run the `start-the-track-processing-model` to track's corresponding track + // element whenever track's mode changes. + if (mTrackElement) { + mTrackElement->MaybeDispatchLoadResource(); + } + // Ensure the TimeMarchesOn is called in case that the mCueList + // is empty. + NotifyCueUpdated(nullptr); +} + +void TextTrack::GetId(nsAString& aId) const { + // If the track has a track element then its id should be the same as the + // track element's id. + if (mTrackElement) { + mTrackElement->GetAttr(nsGkAtoms::id, aId); + } +} + +void TextTrack::AddCue(TextTrackCue& aCue) { + WEBVTT_LOG("AddCue %p [%f:%f]", &aCue, aCue.StartTime(), aCue.EndTime()); + TextTrack* oldTextTrack = aCue.GetTrack(); + if (oldTextTrack) { + ErrorResult dummy; + oldTextTrack->RemoveCue(aCue, dummy); + } + mCueList->AddCue(aCue); + aCue.SetTrack(this); + HTMLMediaElement* mediaElement = GetMediaElement(); + if (mediaElement && (mMode != TextTrackMode::Disabled)) { + mediaElement->NotifyCueAdded(aCue); + } +} + +void TextTrack::RemoveCue(TextTrackCue& aCue, ErrorResult& aRv) { + WEBVTT_LOG("RemoveCue %p", &aCue); + // Bug1304948, check the aCue belongs to the TextTrack. + mCueList->RemoveCue(aCue, aRv); + if (aRv.Failed()) { + return; + } + aCue.SetActive(false); + aCue.SetTrack(nullptr); + HTMLMediaElement* mediaElement = GetMediaElement(); + if (mediaElement) { + mediaElement->NotifyCueRemoved(aCue); + } +} + +void TextTrack::ClearAllCues() { + WEBVTT_LOG("ClearAllCues"); + ErrorResult dummy; + while (!mCueList->IsEmpty()) { + RemoveCue(*(*mCueList)[0], dummy); + } +} + +void TextTrack::SetCuesDirty() { + for (uint32_t i = 0; i < mCueList->Length(); i++) { + ((*mCueList)[i])->Reset(); + } +} + +TextTrackCueList* TextTrack::GetActiveCues() { + if (mMode != TextTrackMode::Disabled) { + return mActiveCueList; + } + return nullptr; +} + +void TextTrack::GetActiveCueArray(nsTArray >& aCues) { + if (mMode != TextTrackMode::Disabled) { + mActiveCueList->GetArray(aCues); + } +} + +TextTrackReadyState TextTrack::ReadyState() const { return mReadyState; } + +void TextTrack::SetReadyState(TextTrackReadyState aState) { + WEBVTT_LOG("SetReadyState=%s", ToReadyStateStr(aState)); + mReadyState = aState; + HTMLMediaElement* mediaElement = GetMediaElement(); + if (mediaElement && (mReadyState == TextTrackReadyState::Loaded || + mReadyState == TextTrackReadyState::FailedToLoad)) { + mediaElement->RemoveTextTrack(this, true); + mediaElement->UpdateReadyState(); + } +} + +TextTrackList* TextTrack::GetTextTrackList() { return mTextTrackList; } + +void TextTrack::SetTextTrackList(TextTrackList* aTextTrackList) { + mTextTrackList = aTextTrackList; +} + +HTMLTrackElement* TextTrack::GetTrackElement() { return mTrackElement; } + +void TextTrack::SetTrackElement(HTMLTrackElement* aTrackElement) { + mTrackElement = aTrackElement; +} + +void TextTrack::SetCuesInactive() { + WEBVTT_LOG("SetCuesInactive"); + mCueList->SetCuesInactive(); +} + +void TextTrack::NotifyCueUpdated(TextTrackCue* aCue) { + WEBVTT_LOG("NotifyCueUpdated, cue=%p", aCue); + mCueList->NotifyCueUpdated(aCue); + HTMLMediaElement* mediaElement = GetMediaElement(); + if (mediaElement) { + mediaElement->NotifyCueUpdated(aCue); + } +} + +void TextTrack::GetLabel(nsAString& aLabel) const { + if (mTrackElement) { + mTrackElement->GetLabel(aLabel); + } else { + aLabel = mLabel; + } +} +void TextTrack::GetLanguage(nsAString& aLanguage) const { + if (mTrackElement) { + mTrackElement->GetSrclang(aLanguage); + } else { + aLanguage = mLanguage; + } +} + +void TextTrack::DispatchAsyncTrustedEvent(const nsString& aEventName) { + nsPIDOMWindowInner* win = GetOwner(); + if (!win) { + return; + } + RefPtr self = this; + nsGlobalWindowInner::Cast(win)->Dispatch(NS_NewRunnableFunction( + "dom::TextTrack::DispatchAsyncTrustedEvent", + [self, aEventName]() { self->DispatchTrustedEvent(aEventName); })); +} + +bool TextTrack::IsLoaded() { + if (mMode == TextTrackMode::Disabled) { + return true; + } + // If the TrackElement's src is null, we can not block the + // MediaElement. + if (mTrackElement) { + nsAutoString src; + if (!(mTrackElement->GetAttr(nsGkAtoms::src, src))) { + return true; + } + } + return mReadyState >= TextTrackReadyState::Loaded; +} + +void TextTrack::NotifyCueActiveStateChanged(TextTrackCue* aCue) { + MOZ_ASSERT(aCue); + if (aCue->GetActive()) { + MOZ_ASSERT(!mActiveCueList->IsCueExist(aCue)); + WEBVTT_LOG("NotifyCueActiveStateChanged, add cue %p to the active list", + aCue); + mActiveCueList->AddCue(*aCue); + } else { + MOZ_ASSERT(mActiveCueList->IsCueExist(aCue)); + WEBVTT_LOG( + "NotifyCueActiveStateChanged, remove cue %p from the active list", + aCue); + mActiveCueList->RemoveCue(*aCue); + } +} + +void TextTrack::GetCurrentCuesAndOtherCues( + RefPtr& aCurrentCues, + RefPtr& aOtherCues, + const media::TimeInterval& aInterval) const { + const HTMLMediaElement* mediaElement = GetMediaElement(); + if (!mediaElement) { + return; + } + + if (Mode() == TextTrackMode::Disabled) { + return; + } + + // According to `time marches on` step1, current cue list contains the cues + // whose start times are less than or equal to the current playback position + // and whose end times are greater than the current playback position. + // https://html.spec.whatwg.org/multipage/media.html#time-marches-on + MOZ_ASSERT(aCurrentCues && aOtherCues); + const double playbackTime = mediaElement->CurrentTime(); + for (uint32_t idx = 0; idx < mCueList->Length(); idx++) { + TextTrackCue* cue = (*mCueList)[idx]; + WEBVTT_LOG("cue %p [%f:%f], playbackTime=%f", cue, cue->StartTime(), + cue->EndTime(), playbackTime); + if (cue->StartTime() <= playbackTime && cue->EndTime() > playbackTime) { + WEBVTT_LOG("Add cue %p [%f:%f] to current cue list", cue, + cue->StartTime(), cue->EndTime()); + aCurrentCues->AddCue(*cue); + } else { + // As the spec didn't have a restriction for the negative duration, it + // does happen sometime if user sets it explictly. It would be treated as + // a `missing cue` later in the `TimeMarchesOn` but it won't be displayed. + if (cue->EndTime() < cue->StartTime()) { + // Add cue into `otherCue` only when its start time is contained by the + // current time interval. + if (aInterval.Contains( + media::TimeUnit::FromSeconds(cue->StartTime()))) { + WEBVTT_LOG("[Negative duration] Add cue %p [%f:%f] to other cue list", + cue, cue->StartTime(), cue->EndTime()); + aOtherCues->AddCue(*cue); + } + continue; + } + media::TimeInterval cueInterval( + media::TimeUnit::FromSeconds(cue->StartTime()), + media::TimeUnit::FromSeconds(cue->EndTime())); + // cues are completely outside the time interval. + if (!aInterval.Touches(cueInterval)) { + continue; + } + // contains any cues which are overlapping within the time interval. + WEBVTT_LOG("Add cue %p [%f:%f] to other cue list", cue, cue->StartTime(), + cue->EndTime()); + aOtherCues->AddCue(*cue); + } + } +} + +HTMLMediaElement* TextTrack::GetMediaElement() const { + return mTextTrackList ? mTextTrackList->GetMediaElement() : nullptr; +} + +} // namespace mozilla::dom diff --git a/dom/media/webvtt/TextTrack.h b/dom/media/webvtt/TextTrack.h new file mode 100644 index 0000000000..1adf8f1838 --- /dev/null +++ b/dom/media/webvtt/TextTrack.h @@ -0,0 +1,148 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 et tw=78: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_TextTrack_h +#define mozilla_dom_TextTrack_h + +#include "mozilla/DOMEventTargetHelper.h" +#include "mozilla/dom/TextTrackBinding.h" +#include "nsCOMPtr.h" +#include "nsCycleCollectionParticipant.h" +#include "nsString.h" +#include "TimeUnits.h" + +namespace mozilla::dom { + +class TextTrackList; +class TextTrackCue; +class TextTrackCueList; +class HTMLTrackElement; +class HTMLMediaElement; + +enum class TextTrackSource : uint8_t { + Track, + AddTextTrack, + MediaResourceSpecific, +}; + +// Constants for numeric readyState property values. +enum class TextTrackReadyState : uint8_t { + NotLoaded = 0U, + Loading = 1U, + Loaded = 2U, + FailedToLoad = 3U +}; + +class TextTrack final : public DOMEventTargetHelper { + public: + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(TextTrack, DOMEventTargetHelper) + + TextTrack(nsPIDOMWindowInner* aOwnerWindow, TextTrackKind aKind, + const nsAString& aLabel, const nsAString& aLanguage, + TextTrackMode aMode, TextTrackReadyState aReadyState, + TextTrackSource aTextTrackSource); + TextTrack(nsPIDOMWindowInner* aOwnerWindow, TextTrackList* aTextTrackList, + TextTrackKind aKind, const nsAString& aLabel, + const nsAString& aLanguage, TextTrackMode aMode, + TextTrackReadyState aReadyState, TextTrackSource aTextTrackSource); + + void SetDefaultSettings(); + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + TextTrackKind Kind() const { return mKind; } + void GetLabel(nsAString& aLabel) const; + void GetLanguage(nsAString& aLanguage) const; + void GetInBandMetadataTrackDispatchType(nsAString& aType) const { + aType = mType; + } + void GetId(nsAString& aId) const; + + TextTrackMode Mode() const { return mMode; } + void SetMode(TextTrackMode aValue); + + TextTrackCueList* GetCues() const { + if (mMode == TextTrackMode::Disabled) { + return nullptr; + } + return mCueList; + } + + TextTrackCueList* GetActiveCues(); + void GetActiveCueArray(nsTArray >& aCues); + + TextTrackReadyState ReadyState() const; + void SetReadyState(TextTrackReadyState aState); + + void AddCue(TextTrackCue& aCue); + void RemoveCue(TextTrackCue& aCue, ErrorResult& aRv); + void SetDirty() { mDirty = true; } + void SetCuesDirty(); + + TextTrackList* GetTextTrackList(); + void SetTextTrackList(TextTrackList* aTextTrackList); + + IMPL_EVENT_HANDLER(cuechange) + + HTMLTrackElement* GetTrackElement(); + void SetTrackElement(HTMLTrackElement* aTrackElement); + + TextTrackSource GetTextTrackSource() { return mTextTrackSource; } + + void SetCuesInactive(); + + void NotifyCueUpdated(TextTrackCue* aCue); + + void DispatchAsyncTrustedEvent(const nsString& aEventName); + + bool IsLoaded(); + + // Called when associated cue's active flag has been changed, and then we + // would add or remove the cue to the active cue list. + void NotifyCueActiveStateChanged(TextTrackCue* aCue); + + // Use this function to request current cues, which start time are less than + // or equal to the current playback position and whose end times are greater + // than the current playback position, and other cues, which are not in the + // current cues. Because there would be LOTS of cues in the other cues, and we + // don't actually need all of them. Therefore, we use a time interval to get + // the cues which are overlapping within the time interval. + void GetCurrentCuesAndOtherCues(RefPtr& aCurrentCues, + RefPtr& aOtherCues, + const media::TimeInterval& aInterval) const; + + void ClearAllCues(); + + private: + ~TextTrack(); + + HTMLMediaElement* GetMediaElement() const; + + RefPtr mTextTrackList; + + TextTrackKind mKind; + nsString mLabel; + nsString mLanguage; + nsString mType; + TextTrackMode mMode; + + RefPtr mCueList; + RefPtr mActiveCueList; + RefPtr mTrackElement; + + uint32_t mCuePos; + TextTrackReadyState mReadyState; + bool mDirty; + + // An enum that represents where the track was sourced from. + TextTrackSource mTextTrackSource; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_TextTrack_h diff --git a/dom/media/webvtt/TextTrackCue.cpp b/dom/media/webvtt/TextTrackCue.cpp new file mode 100644 index 0000000000..7364d6083b --- /dev/null +++ b/dom/media/webvtt/TextTrackCue.cpp @@ -0,0 +1,258 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/TextTrackCue.h" + +#include "mozilla/dom/Document.h" +#include "mozilla/dom/HTMLTrackElement.h" +#include "mozilla/dom/TextTrackList.h" +#include "mozilla/dom/TextTrackRegion.h" +#include "nsComponentManagerUtils.h" +#include "mozilla/ClearOnShutdown.h" +#include "mozilla/intl/Bidi.h" + +extern mozilla::LazyLogModule gTextTrackLog; + +#define LOG(msg, ...) \ + MOZ_LOG(gTextTrackLog, LogLevel::Debug, \ + ("TextTrackCue=%p, " msg, this, ##__VA_ARGS__)) + +namespace mozilla::dom { + +NS_IMPL_CYCLE_COLLECTION_INHERITED(TextTrackCue, DOMEventTargetHelper, + mDocument, mTrack, mTrackElement, + mDisplayState, mRegion) + +NS_IMPL_ADDREF_INHERITED(TextTrackCue, DOMEventTargetHelper) +NS_IMPL_RELEASE_INHERITED(TextTrackCue, DOMEventTargetHelper) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(TextTrackCue) +NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper) + +StaticRefPtr TextTrackCue::sParserWrapper; + +// Set default value for cue, spec https://w3c.github.io/webvtt/#model-cues +void TextTrackCue::SetDefaultCueSettings() { + mPositionIsAutoKeyword = true; + // Spec https://www.w3.org/TR/webvtt1/#webvtt-cue-position-automatic-alignment + mPositionAlign = PositionAlignSetting::Auto; + mSize = 100.0; + mPauseOnExit = false; + mSnapToLines = true; + mLineIsAutoKeyword = true; + mAlign = AlignSetting::Center; + mLineAlign = LineAlignSetting::Start; + mVertical = DirectionSetting::_empty; + mActive = false; +} + +TextTrackCue::TextTrackCue(nsPIDOMWindowInner* aOwnerWindow, double aStartTime, + double aEndTime, const nsAString& aText, + ErrorResult& aRv) + : DOMEventTargetHelper(aOwnerWindow), + mText(aText), + mStartTime(aStartTime), + mEndTime(aEndTime), + mPosition(0.0), + mLine(0.0), + mReset(false, "TextTrackCue::mReset"), + mHaveStartedWatcher(false), + mWatchManager(this, AbstractThread::MainThread()) { + LOG("create TextTrackCue"); + SetDefaultCueSettings(); + MOZ_ASSERT(aOwnerWindow); + if (NS_FAILED(StashDocument())) { + aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); + } +} + +TextTrackCue::TextTrackCue(nsPIDOMWindowInner* aOwnerWindow, double aStartTime, + double aEndTime, const nsAString& aText, + HTMLTrackElement* aTrackElement, ErrorResult& aRv) + : DOMEventTargetHelper(aOwnerWindow), + mText(aText), + mStartTime(aStartTime), + mEndTime(aEndTime), + mTrackElement(aTrackElement), + mPosition(0.0), + mLine(0.0), + mReset(false, "TextTrackCue::mReset"), + mHaveStartedWatcher(false), + mWatchManager(this, AbstractThread::MainThread()) { + LOG("create TextTrackCue"); + SetDefaultCueSettings(); + MOZ_ASSERT(aOwnerWindow); + if (NS_FAILED(StashDocument())) { + aRv.Throw(NS_ERROR_DOM_INVALID_STATE_ERR); + } +} + +TextTrackCue::~TextTrackCue() = default; + +/** Save a reference to our creating document so we don't have to + * keep getting it from our window. + */ +nsresult TextTrackCue::StashDocument() { + nsPIDOMWindowInner* window = GetOwner(); + if (!window) { + return NS_ERROR_NO_INTERFACE; + } + mDocument = window->GetDoc(); + if (!mDocument) { + return NS_ERROR_NOT_AVAILABLE; + } + return NS_OK; +} + +already_AddRefed TextTrackCue::GetCueAsHTML() { + // mDocument may be null during cycle collector shutdown. + // See bug 941701. + if (!mDocument) { + return nullptr; + } + + if (!sParserWrapper) { + nsresult rv; + nsCOMPtr parserWrapper = + do_CreateInstance(NS_WEBVTTPARSERWRAPPER_CONTRACTID, &rv); + if (NS_FAILED(rv)) { + return mDocument->CreateDocumentFragment(); + } + sParserWrapper = parserWrapper; + ClearOnShutdown(&sParserWrapper); + } + + nsPIDOMWindowInner* window = mDocument->GetInnerWindow(); + if (!window) { + return mDocument->CreateDocumentFragment(); + } + + RefPtr frag; + sParserWrapper->ConvertCueToDOMTree(window, static_cast(this), + getter_AddRefs(frag)); + if (!frag) { + return mDocument->CreateDocumentFragment(); + } + return frag.forget(); +} + +void TextTrackCue::SetTrackElement(HTMLTrackElement* aTrackElement) { + mTrackElement = aTrackElement; +} + +JSObject* TextTrackCue::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + return VTTCue_Binding::Wrap(aCx, this, aGivenProto); +} + +TextTrackRegion* TextTrackCue::GetRegion() { return mRegion; } + +void TextTrackCue::SetRegion(TextTrackRegion* aRegion) { + if (mRegion == aRegion) { + return; + } + mRegion = aRegion; + mReset = true; +} + +double TextTrackCue::ComputedLine() { + // See spec https://w3c.github.io/webvtt/#cue-computed-line + if (!mLineIsAutoKeyword && !mSnapToLines && (mLine < 0.0 || mLine > 100.0)) { + return 100.0; + } else if (!mLineIsAutoKeyword) { + return mLine; + } else if (mLineIsAutoKeyword && !mSnapToLines) { + return 100.0; + } else if (!mTrack || !mTrack->GetTextTrackList() || + !mTrack->GetTextTrackList()->GetMediaElement()) { + return -1.0; + } + + RefPtr trackList = mTrack->GetTextTrackList(); + bool dummy; + uint32_t showingTracksNum = 0; + for (uint32_t idx = 0; idx < trackList->Length(); idx++) { + RefPtr track = trackList->IndexedGetter(idx, dummy); + if (track->Mode() == TextTrackMode::Showing) { + showingTracksNum++; + } + + if (mTrack == track) { + break; + } + } + + return (-1.0) * showingTracksNum; +} + +double TextTrackCue::ComputedPosition() { + // See spec https://w3c.github.io/webvtt/#cue-computed-position + if (!mPositionIsAutoKeyword) { + return mPosition; + } + if (ComputedPositionAlign() == PositionAlignSetting::Line_left) { + return 0.0; + } + if (ComputedPositionAlign() == PositionAlignSetting::Line_right) { + return 100.0; + } + return 50.0; +} + +PositionAlignSetting TextTrackCue::ComputedPositionAlign() { + // See spec https://w3c.github.io/webvtt/#cue-computed-position-alignment + if (mPositionAlign != PositionAlignSetting::Auto) { + return mPositionAlign; + } else if (mAlign == AlignSetting::Left) { + return PositionAlignSetting::Line_left; + } else if (mAlign == AlignSetting::Right) { + return PositionAlignSetting::Line_right; + } else if (mAlign == AlignSetting::Start) { + return IsTextBaseDirectionLTR() ? PositionAlignSetting::Line_left + : PositionAlignSetting::Line_right; + } else if (mAlign == AlignSetting::End) { + return IsTextBaseDirectionLTR() ? PositionAlignSetting::Line_right + : PositionAlignSetting::Line_left; + } + return PositionAlignSetting::Center; +} + +bool TextTrackCue::IsTextBaseDirectionLTR() const { + // The result returned by `GetBaseDirection` might be `neutral` if the text + // only contains neutral charaters. In this case, we would treat its base + // direction as LTR. + return intl::Bidi::GetBaseDirection(mText) != intl::Bidi::BaseDirection::RTL; +} + +void TextTrackCue::NotifyDisplayStatesChanged() { + if (!mReset) { + return; + } + + if (!mTrack || !mTrack->GetTextTrackList() || + !mTrack->GetTextTrackList()->GetMediaElement()) { + return; + } + + mTrack->GetTextTrackList() + ->GetMediaElement() + ->NotifyCueDisplayStatesChanged(); +} + +void TextTrackCue::SetActive(bool aActive) { + if (mActive == aActive) { + return; + } + + LOG("TextTrackCue, SetActive=%d", aActive); + mActive = aActive; + mDisplayState = mActive ? mDisplayState : nullptr; + if (mTrack) { + mTrack->NotifyCueActiveStateChanged(this); + } +} + +#undef LOG + +} // namespace mozilla::dom diff --git a/dom/media/webvtt/TextTrackCue.h b/dom/media/webvtt/TextTrackCue.h new file mode 100644 index 0000000000..90ce0a571d --- /dev/null +++ b/dom/media/webvtt/TextTrackCue.h @@ -0,0 +1,342 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 et tw=78: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_TextTrackCue_h +#define mozilla_dom_TextTrackCue_h + +#include "mozilla/DOMEventTargetHelper.h" +#include "mozilla/dom/DocumentFragment.h" +#include "mozilla/dom/VTTCueBinding.h" +#include "nsCycleCollectionParticipant.h" +#include "nsIWebVTTParserWrapper.h" +#include "mozilla/StaticPtr.h" +#include "mozilla/dom/HTMLDivElement.h" +#include "mozilla/dom/TextTrack.h" +#include "mozilla/StateWatching.h" + +namespace mozilla::dom { + +class Document; +class HTMLTrackElement; +class TextTrackRegion; + +class TextTrackCue final : public DOMEventTargetHelper { + public: + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(TextTrackCue, DOMEventTargetHelper) + + // TextTrackCue WebIDL + // See bug 868509 about splitting out the WebVTT-specific interfaces. + static already_AddRefed Constructor(GlobalObject& aGlobal, + double aStartTime, + double aEndTime, + const nsAString& aText, + ErrorResult& aRv) { + nsCOMPtr window = + do_QueryInterface(aGlobal.GetAsSupports()); + RefPtr ttcue = + new TextTrackCue(window, aStartTime, aEndTime, aText, aRv); + return ttcue.forget(); + } + TextTrackCue(nsPIDOMWindowInner* aGlobal, double aStartTime, double aEndTime, + const nsAString& aText, ErrorResult& aRv); + + TextTrackCue(nsPIDOMWindowInner* aGlobal, double aStartTime, double aEndTime, + const nsAString& aText, HTMLTrackElement* aTrackElement, + ErrorResult& aRv); + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + TextTrack* GetTrack() const { return mTrack; } + + void GetId(nsAString& aId) const { aId = mId; } + + void SetId(const nsAString& aId) { + if (mId == aId) { + return; + } + + mId = aId; + } + + double StartTime() const { return mStartTime; } + + void SetStartTime(double aStartTime) { + if (mStartTime == aStartTime) { + return; + } + + mStartTime = aStartTime; + mReset = true; + NotifyCueUpdated(this); + } + + double EndTime() const { return mEndTime; } + + void SetEndTime(double aEndTime) { + if (mEndTime == aEndTime) { + return; + } + + mEndTime = aEndTime; + mReset = true; + NotifyCueUpdated(this); + } + + bool PauseOnExit() { return mPauseOnExit; } + + void SetPauseOnExit(bool aPauseOnExit) { + if (mPauseOnExit == aPauseOnExit) { + return; + } + + mPauseOnExit = aPauseOnExit; + NotifyCueUpdated(nullptr); + } + + TextTrackRegion* GetRegion(); + void SetRegion(TextTrackRegion* aRegion); + + DirectionSetting Vertical() const { return mVertical; } + + void SetVertical(const DirectionSetting& aVertical) { + if (mVertical == aVertical) { + return; + } + + mReset = true; + mVertical = aVertical; + } + + bool SnapToLines() { return mSnapToLines; } + + void SetSnapToLines(bool aSnapToLines) { + if (mSnapToLines == aSnapToLines) { + return; + } + + mReset = true; + mSnapToLines = aSnapToLines; + } + + void GetLine(OwningDoubleOrAutoKeyword& aLine) const { + if (mLineIsAutoKeyword) { + aLine.SetAsAutoKeyword() = AutoKeyword::Auto; + return; + } + aLine.SetAsDouble() = mLine; + } + + void SetLine(const DoubleOrAutoKeyword& aLine) { + if (aLine.IsDouble() && + (mLineIsAutoKeyword || (aLine.GetAsDouble() != mLine))) { + mLineIsAutoKeyword = false; + mLine = aLine.GetAsDouble(); + mReset = true; + return; + } + if (aLine.IsAutoKeyword() && !mLineIsAutoKeyword) { + mLineIsAutoKeyword = true; + mReset = true; + } + } + + LineAlignSetting LineAlign() const { return mLineAlign; } + + void SetLineAlign(LineAlignSetting& aLineAlign, ErrorResult& aRv) { + if (mLineAlign == aLineAlign) { + return; + } + + mReset = true; + mLineAlign = aLineAlign; + } + + void GetPosition(OwningDoubleOrAutoKeyword& aPosition) const { + if (mPositionIsAutoKeyword) { + aPosition.SetAsAutoKeyword() = AutoKeyword::Auto; + return; + } + aPosition.SetAsDouble() = mPosition; + } + + void SetPosition(const DoubleOrAutoKeyword& aPosition, ErrorResult& aRv) { + if (!aPosition.IsAutoKeyword() && + (aPosition.GetAsDouble() > 100.0 || aPosition.GetAsDouble() < 0.0)) { + aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR); + return; + } + + if (aPosition.IsDouble() && + (mPositionIsAutoKeyword || (aPosition.GetAsDouble() != mPosition))) { + mPositionIsAutoKeyword = false; + mPosition = aPosition.GetAsDouble(); + mReset = true; + return; + } + + if (aPosition.IsAutoKeyword() && !mPositionIsAutoKeyword) { + mPositionIsAutoKeyword = true; + mReset = true; + } + } + + PositionAlignSetting PositionAlign() const { return mPositionAlign; } + + void SetPositionAlign(PositionAlignSetting aPositionAlign, ErrorResult& aRv) { + if (mPositionAlign == aPositionAlign) { + return; + } + + mReset = true; + mPositionAlign = aPositionAlign; + } + + double Size() const { return mSize; } + + void SetSize(double aSize, ErrorResult& aRv) { + if (mSize == aSize) { + return; + } + + if (aSize < 0.0 || aSize > 100.0) { + aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR); + return; + } + + mReset = true; + mSize = aSize; + } + + AlignSetting Align() const { return mAlign; } + + void SetAlign(AlignSetting& aAlign) { + if (mAlign == aAlign) { + return; + } + + mReset = true; + mAlign = aAlign; + } + + void GetText(nsAString& aText) const { aText = mText; } + + void SetText(const nsAString& aText) { + if (mText == aText) { + return; + } + + mReset = true; + mText = aText; + } + + IMPL_EVENT_HANDLER(enter) + IMPL_EVENT_HANDLER(exit) + + HTMLDivElement* GetDisplayState() { + return static_cast(mDisplayState.get()); + } + + void SetDisplayState(HTMLDivElement* aDisplayState) { + mDisplayState = aDisplayState; + mReset = false; + } + + void Reset() { mReset = true; } + + bool HasBeenReset() { return mReset; } + + double ComputedLine(); + double ComputedPosition(); + PositionAlignSetting ComputedPositionAlign(); + + // Helper functions for implementation. + const nsAString& Id() const { return mId; } + + void SetTrack(TextTrack* aTextTrack) { + mTrack = aTextTrack; + if (!mHaveStartedWatcher && aTextTrack) { + mHaveStartedWatcher = true; + mWatchManager.Watch(mReset, &TextTrackCue::NotifyDisplayStatesChanged); + } else if (mHaveStartedWatcher && !aTextTrack) { + mHaveStartedWatcher = false; + mWatchManager.Unwatch(mReset, &TextTrackCue::NotifyDisplayStatesChanged); + } + } + + /** + * Produces a tree of anonymous content based on the tree of the processed + * cue text. + * + * Returns a DocumentFragment that is the head of the tree of anonymous + * content. + */ + already_AddRefed GetCueAsHTML(); + + void SetTrackElement(HTMLTrackElement* aTrackElement); + + void SetActive(bool aActive); + + bool GetActive() { return mActive; } + + private: + ~TextTrackCue(); + + void NotifyCueUpdated(TextTrackCue* aCue) { + if (mTrack) { + mTrack->NotifyCueUpdated(aCue); + } + } + + void NotifyDisplayStatesChanged(); + + void SetDefaultCueSettings(); + nsresult StashDocument(); + + bool IsTextBaseDirectionLTR() const; + + RefPtr mDocument; + nsString mText; + double mStartTime; + double mEndTime; + + RefPtr mTrack; + RefPtr mTrackElement; + nsString mId; + double mPosition; + bool mPositionIsAutoKeyword; + PositionAlignSetting mPositionAlign; + double mSize; + bool mPauseOnExit; + bool mSnapToLines; + RefPtr mRegion; + DirectionSetting mVertical; + bool mLineIsAutoKeyword; + double mLine; + AlignSetting mAlign; + LineAlignSetting mLineAlign; + + // Holds the computed DOM elements that represent the parsed cue text. + // http://www.whatwg.org/specs/web-apps/current-work/#text-track-cue-display-state + RefPtr mDisplayState; + // Tells whether or not we need to recompute mDisplayState. This is set + // anytime a property that relates to the display of the TextTrackCue is + // changed. + Watchable mReset; + + bool mActive; + + static StaticRefPtr sParserWrapper; + + // Only start watcher after the cue has text track. + bool mHaveStartedWatcher; + WatchManager mWatchManager; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_TextTrackCue_h diff --git a/dom/media/webvtt/TextTrackCueList.cpp b/dom/media/webvtt/TextTrackCueList.cpp new file mode 100644 index 0000000000..d6fb8baedc --- /dev/null +++ b/dom/media/webvtt/TextTrackCueList.cpp @@ -0,0 +1,125 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/TextTrackCueList.h" +#include "mozilla/dom/TextTrackCueListBinding.h" +#include "mozilla/dom/TextTrackCue.h" + +namespace mozilla::dom { + +class CompareCuesByTime { + public: + bool Equals(TextTrackCue* aOne, TextTrackCue* aTwo) const { return false; } + bool LessThan(TextTrackCue* aOne, TextTrackCue* aTwo) const { + return aOne->StartTime() < aTwo->StartTime() || + (aOne->StartTime() == aTwo->StartTime() && + aOne->EndTime() >= aTwo->EndTime()); + } +}; + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(TextTrackCueList, mParent, mList) + +NS_IMPL_CYCLE_COLLECTING_ADDREF(TextTrackCueList) +NS_IMPL_CYCLE_COLLECTING_RELEASE(TextTrackCueList) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(TextTrackCueList) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +TextTrackCueList::TextTrackCueList(nsISupports* aParent) : mParent(aParent) {} + +TextTrackCueList::~TextTrackCueList() = default; + +JSObject* TextTrackCueList::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + return TextTrackCueList_Binding::Wrap(aCx, this, aGivenProto); +} + +TextTrackCue* TextTrackCueList::IndexedGetter(uint32_t aIndex, bool& aFound) { + aFound = aIndex < mList.Length(); + if (!aFound) { + return nullptr; + } + return mList[aIndex]; +} + +TextTrackCue* TextTrackCueList::operator[](uint32_t aIndex) { + return mList.SafeElementAt(aIndex, nullptr); +} + +TextTrackCueList& TextTrackCueList::operator=(const TextTrackCueList& aOther) { + mList = aOther.mList.Clone(); + return *this; +} + +TextTrackCue* TextTrackCueList::GetCueById(const nsAString& aId) { + if (aId.IsEmpty()) { + return nullptr; + } + + for (uint32_t i = 0; i < mList.Length(); i++) { + if (aId.Equals(mList[i]->Id())) { + return mList[i]; + } + } + return nullptr; +} + +void TextTrackCueList::AddCue(TextTrackCue& aCue) { + if (mList.Contains(&aCue)) { + return; + } + mList.InsertElementSorted(&aCue, CompareCuesByTime()); +} + +void TextTrackCueList::RemoveCue(TextTrackCue& aCue, ErrorResult& aRv) { + if (!mList.Contains(&aCue)) { + aRv.Throw(NS_ERROR_DOM_NOT_FOUND_ERR); + return; + } + mList.RemoveElement(&aCue); +} + +void TextTrackCueList::RemoveCue(TextTrackCue& aCue) { + mList.RemoveElement(&aCue); +} + +void TextTrackCueList::RemoveCueAt(uint32_t aIndex) { + if (aIndex < mList.Length()) { + mList.RemoveElementAt(aIndex); + } +} + +void TextTrackCueList::RemoveAll() { mList.Clear(); } + +void TextTrackCueList::GetArray(nsTArray>& aCues) { + aCues = mList.Clone(); +} + +void TextTrackCueList::SetCuesInactive() { + for (uint32_t i = 0; i < mList.Length(); ++i) { + mList[i]->SetActive(false); + } +} + +void TextTrackCueList::NotifyCueUpdated(TextTrackCue* aCue) { + if (aCue) { + mList.RemoveElement(aCue); + mList.InsertElementSorted(aCue, CompareCuesByTime()); + } +} + +bool TextTrackCueList::IsCueExist(TextTrackCue* aCue) { + if (aCue && mList.Contains(aCue)) { + return true; + } + return false; +} + +nsTArray>& TextTrackCueList::GetCuesArray() { + return mList; +} + +} // namespace mozilla::dom diff --git a/dom/media/webvtt/TextTrackCueList.h b/dom/media/webvtt/TextTrackCueList.h new file mode 100644 index 0000000000..f590f94d8c --- /dev/null +++ b/dom/media/webvtt/TextTrackCueList.h @@ -0,0 +1,73 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 et tw=78: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_TextTrackCueList_h +#define mozilla_dom_TextTrackCueList_h + +#include "nsTArray.h" +#include "nsCOMPtr.h" +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" + +namespace mozilla { +class ErrorResult; + +namespace dom { + +class TextTrackCue; + +class TextTrackCueList final : public nsISupports, public nsWrapperCache { + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(TextTrackCueList) + + // TextTrackCueList WebIDL + explicit TextTrackCueList(nsISupports* aParent); + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + nsISupports* GetParentObject() const { return mParent; } + + uint32_t Length() const { return mList.Length(); } + + bool IsEmpty() const { return mList.Length() == 0; } + + TextTrackCue* IndexedGetter(uint32_t aIndex, bool& aFound); + TextTrackCue* operator[](uint32_t aIndex); + TextTrackCue* GetCueById(const nsAString& aId); + TextTrackCueList& operator=(const TextTrackCueList& aOther); + // Adds a cue to mList by performing an insertion sort on mList. + // We expect most files to already be sorted, so an insertion sort starting + // from the end of the current array should be more efficient than a general + // sort step after all cues are loaded. + void AddCue(TextTrackCue& aCue); + void RemoveCue(TextTrackCue& aCue); + void RemoveCue(TextTrackCue& aCue, ErrorResult& aRv); + void RemoveCueAt(uint32_t aIndex); + void RemoveAll(); + void GetArray(nsTArray>& aCues); + + void SetCuesInactive(); + + void NotifyCueUpdated(TextTrackCue* aCue); + bool IsCueExist(TextTrackCue* aCue); + nsTArray>& GetCuesArray(); + + private: + ~TextTrackCueList(); + + nsCOMPtr mParent; + + // A sorted list of TextTrackCues sorted by earliest start time. If the start + // times are equal then it will be sorted by end time, earliest first. + nsTArray> mList; +}; + +} // namespace dom +} // namespace mozilla + +#endif // mozilla_dom_TextTrackCueList_h diff --git a/dom/media/webvtt/TextTrackList.cpp b/dom/media/webvtt/TextTrackList.cpp new file mode 100644 index 0000000000..5d51ff48fe --- /dev/null +++ b/dom/media/webvtt/TextTrackList.cpp @@ -0,0 +1,190 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/TextTrackList.h" + +#include "mozilla/DebugOnly.h" +#include "mozilla/dom/Event.h" +#include "mozilla/dom/TextTrackListBinding.h" +#include "mozilla/dom/TrackEvent.h" +#include "nsThreadUtils.h" +#include "nsGlobalWindowInner.h" +#include "mozilla/dom/TextTrackCue.h" +#include "mozilla/dom/TextTrackManager.h" + +namespace mozilla::dom { + +NS_IMPL_CYCLE_COLLECTION_INHERITED(TextTrackList, DOMEventTargetHelper, + mTextTracks, mTextTrackManager) + +NS_IMPL_ADDREF_INHERITED(TextTrackList, DOMEventTargetHelper) +NS_IMPL_RELEASE_INHERITED(TextTrackList, DOMEventTargetHelper) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(TextTrackList) +NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper) + +TextTrackList::TextTrackList(nsPIDOMWindowInner* aOwnerWindow) + : DOMEventTargetHelper(aOwnerWindow) {} + +TextTrackList::TextTrackList(nsPIDOMWindowInner* aOwnerWindow, + TextTrackManager* aTextTrackManager) + : DOMEventTargetHelper(aOwnerWindow), + mTextTrackManager(aTextTrackManager) {} + +TextTrackList::~TextTrackList() = default; + +void TextTrackList::GetShowingCues(nsTArray>& aCues) { + // Only Subtitles and Captions can show on the screen. + nsTArray> cues; + for (uint32_t i = 0; i < Length(); i++) { + if (mTextTracks[i]->Mode() == TextTrackMode::Showing && + (mTextTracks[i]->Kind() == TextTrackKind::Subtitles || + mTextTracks[i]->Kind() == TextTrackKind::Captions)) { + mTextTracks[i]->GetActiveCueArray(cues); + aCues.AppendElements(cues); + } + } +} + +JSObject* TextTrackList::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + return TextTrackList_Binding::Wrap(aCx, this, aGivenProto); +} + +TextTrack* TextTrackList::IndexedGetter(uint32_t aIndex, bool& aFound) { + aFound = aIndex < mTextTracks.Length(); + if (!aFound) { + return nullptr; + } + return mTextTracks[aIndex]; +} + +TextTrack* TextTrackList::operator[](uint32_t aIndex) { + return mTextTracks.SafeElementAt(aIndex, nullptr); +} + +already_AddRefed TextTrackList::AddTextTrack( + TextTrackKind aKind, const nsAString& aLabel, const nsAString& aLanguage, + TextTrackMode aMode, TextTrackReadyState aReadyState, + TextTrackSource aTextTrackSource, const CompareTextTracks& aCompareTT) { + RefPtr track = + new TextTrack(GetOwner(), this, aKind, aLabel, aLanguage, aMode, + aReadyState, aTextTrackSource); + AddTextTrack(track, aCompareTT); + return track.forget(); +} + +void TextTrackList::AddTextTrack(TextTrack* aTextTrack, + const CompareTextTracks& aCompareTT) { + if (mTextTracks.Contains(aTextTrack)) { + return; + } + mTextTracks.InsertElementSorted(aTextTrack, aCompareTT); + aTextTrack->SetTextTrackList(this); + CreateAndDispatchTrackEventRunner(aTextTrack, u"addtrack"_ns); +} + +TextTrack* TextTrackList::GetTrackById(const nsAString& aId) { + nsAutoString id; + for (uint32_t i = 0; i < Length(); i++) { + mTextTracks[i]->GetId(id); + if (aId.Equals(id)) { + return mTextTracks[i]; + } + } + return nullptr; +} + +void TextTrackList::RemoveTextTrack(TextTrack* aTrack) { + if (mTextTracks.RemoveElement(aTrack)) { + CreateAndDispatchTrackEventRunner(aTrack, u"removetrack"_ns); + } +} + +class TrackEventRunner : public Runnable { + public: + TrackEventRunner(TextTrackList* aList, Event* aEvent) + : Runnable("dom::TrackEventRunner"), mList(aList), mEvent(aEvent) {} + + NS_IMETHOD Run() override { return mList->DispatchTrackEvent(mEvent); } + + RefPtr mList; + + private: + RefPtr mEvent; +}; + +nsresult TextTrackList::DispatchTrackEvent(Event* aEvent) { + return DispatchTrustedEvent(aEvent); +} + +void TextTrackList::CreateAndDispatchChangeEvent() { + MOZ_ASSERT(NS_IsMainThread()); + nsPIDOMWindowInner* win = GetOwner(); + if (!win) { + return; + } + + RefPtr event = NS_NewDOMEvent(this, nullptr, nullptr); + + event->InitEvent(u"change"_ns, false, false); + event->SetTrusted(true); + + nsCOMPtr eventRunner = new TrackEventRunner(this, event); + nsGlobalWindowInner::Cast(win)->Dispatch(eventRunner.forget()); +} + +void TextTrackList::CreateAndDispatchTrackEventRunner( + TextTrack* aTrack, const nsAString& aEventName) { + nsCOMPtr target = GetMainThreadSerialEventTarget(); + if (!target) { + // If we are not able to get the main-thread object we are shutting down. + return; + } + + TrackEventInit eventInit; + eventInit.mTrack.SetValue().SetAsTextTrack() = aTrack; + RefPtr event = + TrackEvent::Constructor(this, aEventName, eventInit); + + // Dispatch the TrackEvent asynchronously. + DebugOnly rv = target->Dispatch( + do_AddRef(new TrackEventRunner(this, event)), NS_DISPATCH_NORMAL); + + // If we are shutting down this can file but it's still ok. + NS_WARNING_ASSERTION(NS_SUCCEEDED(rv), "Dispatch failed"); +} + +HTMLMediaElement* TextTrackList::GetMediaElement() { + if (mTextTrackManager) { + return mTextTrackManager->mMediaElement; + } + return nullptr; +} + +void TextTrackList::SetTextTrackManager(TextTrackManager* aTextTrackManager) { + mTextTrackManager = aTextTrackManager; +} + +void TextTrackList::SetCuesInactive() { + for (uint32_t i = 0; i < Length(); i++) { + mTextTracks[i]->SetCuesInactive(); + } +} + +bool TextTrackList::AreTextTracksLoaded() { + // Return false if any texttrack is not loaded. + for (uint32_t i = 0; i < Length(); i++) { + if (!mTextTracks[i]->IsLoaded()) { + return false; + } + } + return true; +} + +nsTArray>& TextTrackList::GetTextTrackArray() { + return mTextTracks; +} + +} // namespace mozilla::dom diff --git a/dom/media/webvtt/TextTrackList.h b/dom/media/webvtt/TextTrackList.h new file mode 100644 index 0000000000..712e3ae9c4 --- /dev/null +++ b/dom/media/webvtt/TextTrackList.h @@ -0,0 +1,79 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 et tw=78: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_TextTrackList_h +#define mozilla_dom_TextTrackList_h + +#include "mozilla/DOMEventTargetHelper.h" +#include "mozilla/dom/TextTrack.h" +#include "nsCycleCollectionParticipant.h" + +namespace mozilla::dom { + +class Event; +class HTMLMediaElement; +class TextTrackManager; +class CompareTextTracks; +class TrackEvent; +class TrackEventRunner; + +class TextTrackList final : public DOMEventTargetHelper { + public: + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(TextTrackList, DOMEventTargetHelper) + + explicit TextTrackList(nsPIDOMWindowInner* aOwnerWindow); + TextTrackList(nsPIDOMWindowInner* aOwnerWindow, + TextTrackManager* aTextTrackManager); + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + uint32_t Length() const { return mTextTracks.Length(); } + + // Get all the current active cues. + void GetShowingCues(nsTArray>& aCues); + + TextTrack* IndexedGetter(uint32_t aIndex, bool& aFound); + TextTrack* operator[](uint32_t aIndex); + + already_AddRefed AddTextTrack( + TextTrackKind aKind, const nsAString& aLabel, const nsAString& aLanguage, + TextTrackMode aMode, TextTrackReadyState aReadyState, + TextTrackSource aTextTrackSource, const CompareTextTracks& aCompareTT); + TextTrack* GetTrackById(const nsAString& aId); + + void AddTextTrack(TextTrack* aTextTrack, const CompareTextTracks& aCompareTT); + + void RemoveTextTrack(TextTrack* aTrack); + + HTMLMediaElement* GetMediaElement(); + void SetTextTrackManager(TextTrackManager* aTextTrackManager); + + nsresult DispatchTrackEvent(Event* aEvent); + void CreateAndDispatchChangeEvent(); + void SetCuesInactive(); + + bool AreTextTracksLoaded(); + nsTArray>& GetTextTrackArray(); + + IMPL_EVENT_HANDLER(change) + IMPL_EVENT_HANDLER(addtrack) + IMPL_EVENT_HANDLER(removetrack) + + private: + ~TextTrackList(); + + nsTArray> mTextTracks; + RefPtr mTextTrackManager; + + void CreateAndDispatchTrackEventRunner(TextTrack* aTrack, + const nsAString& aEventName); +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_TextTrackList_h diff --git a/dom/media/webvtt/TextTrackRegion.cpp b/dom/media/webvtt/TextTrackRegion.cpp new file mode 100644 index 0000000000..d883659579 --- /dev/null +++ b/dom/media/webvtt/TextTrackRegion.cpp @@ -0,0 +1,58 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 et tw=78: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/dom/TextTrackRegion.h" + +namespace mozilla::dom { + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(TextTrackRegion, mParent) +NS_IMPL_CYCLE_COLLECTING_ADDREF(TextTrackRegion) +NS_IMPL_CYCLE_COLLECTING_RELEASE(TextTrackRegion) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(TextTrackRegion) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +JSObject* TextTrackRegion::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + return VTTRegion_Binding::Wrap(aCx, this, aGivenProto); +} + +already_AddRefed TextTrackRegion::Constructor( + const GlobalObject& aGlobal, ErrorResult& aRv) { + nsCOMPtr window = + do_QueryInterface(aGlobal.GetAsSupports()); + if (!window) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + RefPtr region = new TextTrackRegion(aGlobal.GetAsSupports()); + return region.forget(); +} + +TextTrackRegion::TextTrackRegion(nsISupports* aGlobal) + : mParent(aGlobal), + mWidth(100), + mLines(3), + mRegionAnchorX(0), + mRegionAnchorY(100), + mViewportAnchorX(0), + mViewportAnchorY(100), + mScroll(ScrollSetting::_empty) {} + +void TextTrackRegion::CopyValues(TextTrackRegion& aRegion) { + mId = aRegion.Id(); + mWidth = aRegion.Width(); + mLines = aRegion.Lines(); + mRegionAnchorX = aRegion.RegionAnchorX(); + mRegionAnchorY = aRegion.RegionAnchorY(); + mViewportAnchorX = aRegion.ViewportAnchorX(); + mViewportAnchorY = aRegion.ViewportAnchorY(); + mScroll = aRegion.Scroll(); +} + +} // namespace mozilla::dom diff --git a/dom/media/webvtt/TextTrackRegion.h b/dom/media/webvtt/TextTrackRegion.h new file mode 100644 index 0000000000..d316d7a30c --- /dev/null +++ b/dom/media/webvtt/TextTrackRegion.h @@ -0,0 +1,138 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 et tw=78: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_TextTrackRegion_h +#define mozilla_dom_TextTrackRegion_h + +#include "nsCycleCollectionParticipant.h" +#include "nsString.h" +#include "nsWrapperCache.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/dom/TextTrack.h" +#include "mozilla/dom/VTTRegionBinding.h" +#include "mozilla/Preferences.h" + +namespace mozilla::dom { + +class GlobalObject; +class TextTrack; + +class TextTrackRegion final : public nsISupports, public nsWrapperCache { + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS(TextTrackRegion) + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + nsISupports* GetParentObject() const { return mParent; } + + explicit TextTrackRegion(nsISupports* aGlobal); + + /** WebIDL Methods. */ + + static already_AddRefed Constructor( + const GlobalObject& aGlobal, ErrorResult& aRv); + + double Lines() const { return mLines; } + + void SetLines(double aLines, ErrorResult& aRv) { + if (aLines < 0) { + aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR); + } else { + mLines = aLines; + } + } + + double Width() const { return mWidth; } + + void SetWidth(double aWidth, ErrorResult& aRv) { + if (!InvalidValue(aWidth, aRv)) { + mWidth = aWidth; + } + } + + double RegionAnchorX() const { return mRegionAnchorX; } + + void SetRegionAnchorX(double aVal, ErrorResult& aRv) { + if (!InvalidValue(aVal, aRv)) { + mRegionAnchorX = aVal; + } + } + + double RegionAnchorY() const { return mRegionAnchorY; } + + void SetRegionAnchorY(double aVal, ErrorResult& aRv) { + if (!InvalidValue(aVal, aRv)) { + mRegionAnchorY = aVal; + } + } + + double ViewportAnchorX() const { return mViewportAnchorX; } + + void SetViewportAnchorX(double aVal, ErrorResult& aRv) { + if (!InvalidValue(aVal, aRv)) { + mViewportAnchorX = aVal; + } + } + + double ViewportAnchorY() const { return mViewportAnchorY; } + + void SetViewportAnchorY(double aVal, ErrorResult& aRv) { + if (!InvalidValue(aVal, aRv)) { + mViewportAnchorY = aVal; + } + } + + ScrollSetting Scroll() const { return mScroll; } + + void SetScroll(const ScrollSetting& aScroll) { + if (aScroll == ScrollSetting::_empty || aScroll == ScrollSetting::Up) { + mScroll = aScroll; + } + } + + void GetId(nsAString& aId) const { aId = mId; } + + void SetId(const nsAString& aId) { mId = aId; } + + /** end WebIDL Methods. */ + + // Helper to aid copying of a given TextTrackRegion's width, lines, + // anchor, viewport and scroll values. + void CopyValues(TextTrackRegion& aRegion); + + // -----helpers------- + const nsAString& Id() const { return mId; } + + private: + ~TextTrackRegion() = default; + + nsCOMPtr mParent; + nsString mId; + double mWidth; + long mLines; + double mRegionAnchorX; + double mRegionAnchorY; + double mViewportAnchorX; + double mViewportAnchorY; + ScrollSetting mScroll; + + // Helper to ensure new value is in the range: 0.0% - 100.0%; throws + // an IndexSizeError otherwise. + inline bool InvalidValue(double aValue, ErrorResult& aRv) { + if (aValue < 0.0 || aValue > 100.0) { + aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR); + return true; + } + + return false; + } +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_TextTrackRegion_h diff --git a/dom/media/webvtt/WebVTTListener.cpp b/dom/media/webvtt/WebVTTListener.cpp new file mode 100644 index 0000000000..3f8d99a8f3 --- /dev/null +++ b/dom/media/webvtt/WebVTTListener.cpp @@ -0,0 +1,212 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "WebVTTListener.h" +#include "mozilla/CycleCollectedJSContext.h" +#include "mozilla/dom/Document.h" +#include "mozilla/dom/HTMLTrackElement.h" +#include "mozilla/dom/TextTrackCue.h" +#include "mozilla/dom/TextTrackRegion.h" +#include "mozilla/dom/VTTRegionBinding.h" +#include "nsComponentManagerUtils.h" +#include "nsIAsyncVerifyRedirectCallback.h" +#include "nsIInputStream.h" + +extern mozilla::LazyLogModule gTextTrackLog; +#define LOG(msg, ...) \ + MOZ_LOG(gTextTrackLog, LogLevel::Debug, \ + ("WebVTTListener=%p, " msg, this, ##__VA_ARGS__)) +#define LOG_WIHTOUT_ADDRESS(msg, ...) \ + MOZ_LOG(gTextTrackLog, LogLevel::Debug, (msg, ##__VA_ARGS__)) + +namespace mozilla::dom { + +NS_IMPL_CYCLE_COLLECTION(WebVTTListener, mElement, mParserWrapper) + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(WebVTTListener) + NS_INTERFACE_MAP_ENTRY(nsIWebVTTListener) + NS_INTERFACE_MAP_ENTRY(nsIStreamListener) + NS_INTERFACE_MAP_ENTRY(nsIChannelEventSink) + NS_INTERFACE_MAP_ENTRY(nsIInterfaceRequestor) + NS_INTERFACE_MAP_ENTRY_AMBIGUOUS(nsISupports, nsIWebVTTListener) +NS_INTERFACE_MAP_END + +NS_IMPL_CYCLE_COLLECTING_ADDREF(WebVTTListener) +NS_IMPL_CYCLE_COLLECTING_RELEASE(WebVTTListener) + +WebVTTListener::WebVTTListener(HTMLTrackElement* aElement) + : mElement(aElement), mParserWrapperError(NS_OK) { + MOZ_ASSERT(mElement, "Must pass an element to the callback"); + LOG("Created listener for track element %p", aElement); + MOZ_DIAGNOSTIC_ASSERT( + CycleCollectedJSContext::Get() && + !CycleCollectedJSContext::Get()->IsInStableOrMetaStableState()); + mParserWrapper = do_CreateInstance(NS_WEBVTTPARSERWRAPPER_CONTRACTID, + &mParserWrapperError); + if (NS_SUCCEEDED(mParserWrapperError)) { + nsPIDOMWindowInner* window = mElement->OwnerDoc()->GetInnerWindow(); + mParserWrapperError = mParserWrapper->LoadParser(window); + } + if (NS_SUCCEEDED(mParserWrapperError)) { + mParserWrapperError = mParserWrapper->Watch(this); + } +} + +WebVTTListener::~WebVTTListener() { LOG("destroyed."); } + +NS_IMETHODIMP +WebVTTListener::GetInterface(const nsIID& aIID, void** aResult) { + return QueryInterface(aIID, aResult); +} + +nsresult WebVTTListener::LoadResource() { + if (IsCanceled()) { + return NS_OK; + } + // Exit if we failed to create the WebVTTParserWrapper (vtt.jsm) + NS_ENSURE_SUCCESS(mParserWrapperError, mParserWrapperError); + + mElement->SetReadyState(TextTrackReadyState::Loading); + return NS_OK; +} + +NS_IMETHODIMP +WebVTTListener::AsyncOnChannelRedirect(nsIChannel* aOldChannel, + nsIChannel* aNewChannel, uint32_t aFlags, + nsIAsyncVerifyRedirectCallback* cb) { + if (IsCanceled()) { + return NS_OK; + } + if (mElement) { + mElement->OnChannelRedirect(aOldChannel, aNewChannel, aFlags); + } + cb->OnRedirectVerifyCallback(NS_OK); + return NS_OK; +} + +NS_IMETHODIMP +WebVTTListener::OnStartRequest(nsIRequest* aRequest) { + if (IsCanceled()) { + return NS_OK; + } + + LOG("OnStartRequest"); + mElement->DispatchTestEvent(u"mozStartedLoadingTextTrack"_ns); + return NS_OK; +} + +NS_IMETHODIMP +WebVTTListener::OnStopRequest(nsIRequest* aRequest, nsresult aStatus) { + if (IsCanceled()) { + return NS_OK; + } + + LOG("OnStopRequest"); + if (NS_FAILED(aStatus)) { + LOG("Got error status"); + mElement->SetReadyState(TextTrackReadyState::FailedToLoad); + } + // Attempt to parse any final data the parser might still have. + mParserWrapper->Flush(); + if (mElement->ReadyState() != TextTrackReadyState::FailedToLoad) { + mElement->SetReadyState(TextTrackReadyState::Loaded); + } + + mElement->CancelChannelAndListener(); + + return aStatus; +} + +nsresult WebVTTListener::ParseChunk(nsIInputStream* aInStream, void* aClosure, + const char* aFromSegment, + uint32_t aToOffset, uint32_t aCount, + uint32_t* aWriteCount) { + nsCString buffer(aFromSegment, aCount); + WebVTTListener* listener = static_cast(aClosure); + MOZ_ASSERT(!listener->IsCanceled()); + + if (NS_FAILED(listener->mParserWrapper->Parse(buffer))) { + LOG_WIHTOUT_ADDRESS( + "WebVTTListener=%p, Unable to parse chunk of WEBVTT text. Aborting.", + listener); + *aWriteCount = 0; + return NS_ERROR_FAILURE; + } + + *aWriteCount = aCount; + return NS_OK; +} + +NS_IMETHODIMP +WebVTTListener::OnDataAvailable(nsIRequest* aRequest, nsIInputStream* aStream, + uint64_t aOffset, uint32_t aCount) { + if (IsCanceled()) { + return NS_OK; + } + + LOG("OnDataAvailable"); + uint32_t count = aCount; + while (count > 0) { + uint32_t read; + nsresult rv = aStream->ReadSegments(ParseChunk, this, count, &read); + NS_ENSURE_SUCCESS(rv, rv); + if (!read) { + return NS_ERROR_FAILURE; + } + count -= read; + } + + return NS_OK; +} + +NS_IMETHODIMP +WebVTTListener::OnCue(JS::Handle aCue, JSContext* aCx) { + MOZ_ASSERT(!IsCanceled()); + if (!aCue.isObject()) { + return NS_ERROR_FAILURE; + } + + JS::Rooted obj(aCx, &aCue.toObject()); + TextTrackCue* cue = nullptr; + nsresult rv = UNWRAP_OBJECT(VTTCue, &obj, cue); + NS_ENSURE_SUCCESS(rv, rv); + + cue->SetTrackElement(mElement); + mElement->mTrack->AddCue(*cue); + + return NS_OK; +} + +NS_IMETHODIMP +WebVTTListener::OnRegion(JS::Handle aRegion, JSContext* aCx) { + MOZ_ASSERT(!IsCanceled()); + // Nothing for this callback to do. + return NS_OK; +} + +NS_IMETHODIMP +WebVTTListener::OnParsingError(int32_t errorCode, JSContext* cx) { + MOZ_ASSERT(!IsCanceled()); + // We only care about files that have a bad WebVTT file signature right now + // as that means the file failed to load. + if (errorCode == ErrorCodes::BadSignature) { + LOG("parsing error"); + mElement->SetReadyState(TextTrackReadyState::FailedToLoad); + } + return NS_OK; +} + +bool WebVTTListener::IsCanceled() const { return mCancel; } + +void WebVTTListener::Cancel() { + MOZ_ASSERT(!IsCanceled(), "Do not cancel canceled listener again!"); + LOG("Cancel listen to channel's response."); + mCancel = true; + mParserWrapper->Cancel(); + mParserWrapper = nullptr; + mElement = nullptr; +} + +} // namespace mozilla::dom diff --git a/dom/media/webvtt/WebVTTListener.h b/dom/media/webvtt/WebVTTListener.h new file mode 100644 index 0000000000..45f6bc90d6 --- /dev/null +++ b/dom/media/webvtt/WebVTTListener.h @@ -0,0 +1,69 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_WebVTTLoadListener_h +#define mozilla_dom_WebVTTLoadListener_h + +#include "nsIWebVTTListener.h" +#include "nsIStreamListener.h" +#include "nsIChannelEventSink.h" +#include "nsIInterfaceRequestor.h" +#include "nsCOMPtr.h" +#include "nsCycleCollectionParticipant.h" + +class nsIWebVTTParserWrapper; + +namespace mozilla::dom { + +class HTMLTrackElement; + +class WebVTTListener final : public nsIWebVTTListener, + public nsIStreamListener, + public nsIChannelEventSink, + public nsIInterfaceRequestor { + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_NSIWEBVTTLISTENER + NS_DECL_NSIREQUESTOBSERVER + NS_DECL_NSISTREAMLISTENER + NS_DECL_NSICHANNELEVENTSINK + NS_DECL_NSIINTERFACEREQUESTOR + + NS_DECL_CYCLE_COLLECTION_CLASS_AMBIGUOUS(WebVTTListener, nsIStreamListener) + + public: + explicit WebVTTListener(HTMLTrackElement* aElement); + + /** + * Loads the WebVTTListener. Must call this in order for the listener to be + * ready to parse data that is passed to it. + */ + nsresult LoadResource(); + + /** + * When this listener is not going to be used anymore, its owner should take + * a responsibility to call `Cancel()` to prevent this listener making any + * changes for the track element. + */ + bool IsCanceled() const; + void Cancel(); + + private: + ~WebVTTListener(); + + // List of error codes returned from the WebVTT parser that we care about. + enum ErrorCodes { BadSignature = 0 }; + static nsresult ParseChunk(nsIInputStream* aInStream, void* aClosure, + const char* aFromSegment, uint32_t aToOffset, + uint32_t aCount, uint32_t* aWriteCount); + + RefPtr mElement; + nsCOMPtr mParserWrapper; + nsresult mParserWrapperError; + bool mCancel = false; +}; + +} // namespace mozilla::dom + +#endif // mozilla_dom_WebVTTListener_h diff --git a/dom/media/webvtt/WebVTTParserWrapper.sys.mjs b/dom/media/webvtt/WebVTTParserWrapper.sys.mjs new file mode 100644 index 0000000000..19cf08ca55 --- /dev/null +++ b/dom/media/webvtt/WebVTTParserWrapper.sys.mjs @@ -0,0 +1,56 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import { WebVTT } from "resource://gre/modules/vtt.sys.mjs"; + +export function WebVTTParserWrapper() { + // Nothing +} + +WebVTTParserWrapper.prototype = { + loadParser(window) { + this.parser = new WebVTT.Parser(window, new TextDecoder("utf8")); + }, + + parse(data) { + // We can safely translate the string data to a Uint8Array as we are + // guaranteed character codes only from \u0000 => \u00ff + var buffer = new Uint8Array(data.length); + for (var i = 0; i < data.length; i++) { + buffer[i] = data.charCodeAt(i); + } + + this.parser.parse(buffer); + }, + + flush() { + this.parser.flush(); + }, + + watch(callback) { + this.parser.oncue = callback.onCue; + this.parser.onregion = callback.onRegion; + this.parser.onparsingerror = function (e) { + // Passing the just the error code back is enough for our needs. + callback.onParsingError("code" in e ? e.code : -1); + }; + }, + + cancel() { + this.parser.oncue = null; + this.parser.onregion = null; + this.parser.onparsingerror = null; + }, + + convertCueToDOMTree(window, cue) { + return WebVTT.convertCueToDOMTree(window, cue.text); + }, + + processCues(window, cues, overlay, controls) { + WebVTT.processCues(window, cues, overlay, controls); + }, + + classDescription: "Wrapper for the JS WebVTT implementation (vtt.js)", + QueryInterface: ChromeUtils.generateQI(["nsIWebVTTParserWrapper"]), +}; diff --git a/dom/media/webvtt/components.conf b/dom/media/webvtt/components.conf new file mode 100644 index 0000000000..21fc95c13d --- /dev/null +++ b/dom/media/webvtt/components.conf @@ -0,0 +1,14 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Classes = [ + { + 'cid': '{acf6e493-0092-4b26-b172-241e375c57ab}', + 'contract_ids': ['@mozilla.org/webvttParserWrapper;1'], + 'esModule': 'resource://gre/modules/WebVTTParserWrapper.sys.mjs', + 'constructor': 'WebVTTParserWrapper', + }, +] diff --git a/dom/media/webvtt/moz.build b/dom/media/webvtt/moz.build new file mode 100644 index 0000000000..c44f394c5f --- /dev/null +++ b/dom/media/webvtt/moz.build @@ -0,0 +1,52 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXPORTS.mozilla.dom += [ + "TextTrack.h", + "TextTrackCue.h", + "TextTrackCueList.h", + "TextTrackList.h", + "TextTrackRegion.h", + "WebVTTListener.h", +] + +UNIFIED_SOURCES += [ + "TextTrack.cpp", + "TextTrackCue.cpp", + "TextTrackCueList.cpp", + "TextTrackList.cpp", + "TextTrackRegion.cpp", + "WebVTTListener.cpp", +] + +XPIDL_SOURCES += [ + "nsIWebVTTListener.idl", + "nsIWebVTTParserWrapper.idl", +] + +XPIDL_MODULE = "webvtt" + +EXTRA_JS_MODULES += [ + "WebVTTParserWrapper.sys.mjs", +] + +XPCOM_MANIFESTS += [ + "components.conf", +] + +EXTRA_JS_MODULES += [ + "vtt.sys.mjs", +] + +XPCSHELL_TESTS_MANIFESTS += ["test/xpcshell/xpcshell.toml"] + +MOCHITEST_MANIFESTS += ["test/mochitest/mochitest.toml"] + +REFTEST_MANIFESTS += ["test/reftest/reftest.list"] + +CRASHTEST_MANIFESTS += ["test/crashtests/crashtests.list"] + +FINAL_LIBRARY = "xul" diff --git a/dom/media/webvtt/nsIWebVTTListener.idl b/dom/media/webvtt/nsIWebVTTListener.idl new file mode 100644 index 0000000000..b99d4cf517 --- /dev/null +++ b/dom/media/webvtt/nsIWebVTTListener.idl @@ -0,0 +1,37 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +/** + * Listener for a JS WebVTT parser (vtt.js). + */ +[scriptable, uuid(8a2d7780-2045-4a29-99f4-df15cae5fc49)] +interface nsIWebVTTListener : nsISupports +{ + /** + * Is called when the WebVTTParser successfully parses a WebVTT cue. + * + * @param cue An object representing the data of a parsed WebVTT cue. + */ + [implicit_jscontext] + void onCue(in jsval cue); + + /** + * Is called when the WebVTT parser successfully parses a WebVTT region. + * + * @param region An object representing the data of a parsed + * WebVTT region. + */ + [implicit_jscontext] + void onRegion(in jsval region); + + /** + * Is called when the WebVTT parser encounters a parsing error. + * + * @param error The error code of the ParserError the occured. + */ + [implicit_jscontext] + void onParsingError(in long errorCode); +}; diff --git a/dom/media/webvtt/nsIWebVTTParserWrapper.idl b/dom/media/webvtt/nsIWebVTTParserWrapper.idl new file mode 100644 index 0000000000..76725d568b --- /dev/null +++ b/dom/media/webvtt/nsIWebVTTParserWrapper.idl @@ -0,0 +1,94 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +interface nsIWebVTTListener; +interface mozIDOMWindow; +interface nsIVariant; + +webidl DocumentFragment; + +/** + * Interface for a wrapper of a JS WebVTT parser (vtt.js). + */ +[scriptable, uuid(8dfe016e-1701-4618-9f5e-9a6154e853f0)] +interface nsIWebVTTParserWrapper : nsISupports +{ + /** + * Loads the JS WebVTTParser and sets it to use the passed window to create + * VTTRegions and VTTCues. This function must be called before calling + * parse, flush, or watch. + * + * @param window The window that the parser will use to create VTTCues and + * VTTRegions. + * + */ + void loadParser(in mozIDOMWindow window); + + /** + * Attempts to parse the stream's data as WebVTT format. When it successfully + * parses a WebVTT region or WebVTT cue it will create a VTTRegion or VTTCue + * object and pass it back to the callee through its callbacks. + * + * @param data The buffer that contains the WebVTT data received by the + * Necko consumer so far. + */ + void parse(in ACString data); + + /** + * Flush indicates that no more data is expected from the stream. As such the + * parser should try to parse any kind of partial data it has. + */ + void flush(); + + /** + * Set this parser object to use an nsIWebVTTListener object for its onCue + * and onRegion callbacks. + * + * @param callback The nsIWebVTTListener object that exposes onCue and + * onRegion callbacks for the parser. + */ + void watch(in nsIWebVTTListener callback); + + /** + * Cancel watching notifications which parser would send. + */ + void cancel(); + + /** + * Convert the text content of a WebVTT cue to a document fragment so that + * we can display it on the page. + * + * @param window A window object with which the document fragment will be + * created. + * @param cue The cue whose content will be converted to a document + * fragment. + */ + DocumentFragment convertCueToDOMTree(in mozIDOMWindow window, + in nsISupports cue); + + + /** + * Compute the display state of the VTTCues in cues along with any VTTRegions + * that they might be in. First, it computes the positioning and styling of + * the cues and regions passed and converts them into a DOM tree rooted at + * a containing HTMLDivElement. It then adjusts those computed divs for + * overlap avoidance using the dimensions of 'overlay'. Finally, it adds the + * computed divs to the VTTCues display state property for use later. + * + * @param window A window object with which it will create the DOM tree + * and containing div element. + * @param cues An array of VTTCues who need there display state to be + * computed. + * @param overlay The HTMLElement that the cues will be displayed within. + * @param controls The video control element that will affect cues position. + */ + void processCues(in mozIDOMWindow window, in nsIVariant cues, + in nsISupports overlay, in nsISupports controls); +}; + +%{C++ +#define NS_WEBVTTPARSERWRAPPER_CONTRACTID "@mozilla.org/webvttParserWrapper;1" +%} diff --git a/dom/media/webvtt/package.json b/dom/media/webvtt/package.json new file mode 100644 index 0000000000..2952b78c01 --- /dev/null +++ b/dom/media/webvtt/package.json @@ -0,0 +1,6 @@ +{ + "devDependencies": { + "gift": "~0.0.6", + "optimist": "~0.6.0" + } +} diff --git a/dom/media/webvtt/test/crashtests/1304948.html b/dom/media/webvtt/test/crashtests/1304948.html new file mode 100644 index 0000000000..667a13d06a --- /dev/null +++ b/dom/media/webvtt/test/crashtests/1304948.html @@ -0,0 +1,33 @@ + + + Bug 1304948 : Crash if a texttrack remove a cue not belongs to it. + + + + + diff --git a/dom/media/webvtt/test/crashtests/1319486.html b/dom/media/webvtt/test/crashtests/1319486.html new file mode 100644 index 0000000000..74bdb2147c --- /dev/null +++ b/dom/media/webvtt/test/crashtests/1319486.html @@ -0,0 +1,27 @@ + + + + Bug 1319486 : Crash if a texttrackcue added to different texttracks. + + + + + + + + diff --git a/dom/media/webvtt/test/crashtests/1533909.html b/dom/media/webvtt/test/crashtests/1533909.html new file mode 100644 index 0000000000..ee73ecb7b8 --- /dev/null +++ b/dom/media/webvtt/test/crashtests/1533909.html @@ -0,0 +1,17 @@ + + + + +