/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim: set ts=8 sts=2 et sw=2 tw=80: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #ifndef MOZILLA_TRACKBUFFERSMANAGER_H_ #define MOZILLA_TRACKBUFFERSMANAGER_H_ #include "mozilla/Atomics.h" #include "mozilla/EventTargetCapability.h" #include "mozilla/Maybe.h" #include "mozilla/Mutex.h" #include "mozilla/NotNull.h" #include "mozilla/TaskQueue.h" #include "mozilla/dom/MediaDebugInfoBinding.h" #include "MediaContainerType.h" #include "MediaData.h" #include "MediaDataDemuxer.h" #include "MediaResult.h" #include "MediaSourceDecoder.h" #include "MediaSpan.h" #include "SourceBufferTask.h" #include "TimeUnits.h" #include "nsTArray.h" namespace mozilla { class AbstractThread; class ContainerParser; class MediaByteBuffer; class MediaRawData; class MediaSourceDemuxer; class SourceBufferResource; class SourceBufferTaskQueue { public: SourceBufferTaskQueue() = default; ~SourceBufferTaskQueue() { MOZ_ASSERT(mQueue.IsEmpty(), "All tasks must have been processed"); } void Push(SourceBufferTask* aTask) { mQueue.AppendElement(aTask); } already_AddRefed Pop() { if (!mQueue.Length()) { return nullptr; } RefPtr task = std::move(mQueue[0]); mQueue.RemoveElementAt(0); return task.forget(); } nsTArray>::size_type Length() const { return mQueue.Length(); } private: nsTArray> mQueue; }; DDLoggedTypeDeclName(TrackBuffersManager); class TrackBuffersManager final : public DecoderDoctorLifeLogger { public: NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackBuffersManager); enum class EvictDataResult : int8_t { NO_DATA_EVICTED, CANT_EVICT, BUFFER_FULL, }; typedef TrackInfo::TrackType TrackType; typedef MediaData::Type MediaType; typedef nsTArray> TrackBuffer; typedef SourceBufferTask::AppendPromise AppendPromise; typedef SourceBufferTask::RangeRemovalPromise RangeRemovalPromise; // Interface for SourceBuffer TrackBuffersManager(MediaSourceDecoder* aParentDecoder, const MediaContainerType& aType); // Queue a task to add data to the end of the input buffer and run the MSE // Buffer Append Algorithm // 3.5.5 Buffer Append Algorithm. // http://w3c.github.io/media-source/index.html#sourcebuffer-buffer-append RefPtr AppendData(already_AddRefed aData, const SourceBufferAttributes& aAttributes); // Queue a task to abort any pending AppendData. // Does nothing at this stage. void AbortAppendData(); // Queue a task to run MSE Reset Parser State Algorithm. // 3.5.2 Reset Parser State void ResetParserState(SourceBufferAttributes& aAttributes); // Queue a task to run the MSE range removal algorithm. // http://w3c.github.io/media-source/#sourcebuffer-coded-frame-removal RefPtr RangeRemoval(media::TimeUnit aStart, media::TimeUnit aEnd); // Schedule data eviction if necessary as the next call to AppendData will // add aSize bytes. // Eviction is done in two steps, first remove data up to aPlaybackTime // and if still more space is needed remove from the end. EvictDataResult EvictData(const media::TimeUnit& aPlaybackTime, int64_t aSize); // Queue a task to run ChangeType void ChangeType(const MediaContainerType& aType); // Returns the buffered range currently managed. // This may be called on any thread. // Buffered must conform to // http://w3c.github.io/media-source/index.html#widl-SourceBuffer-buffered media::TimeIntervals Buffered() const; media::TimeUnit HighestStartTime() const; media::TimeUnit HighestEndTime() const; // Return the size of the data managed by this SourceBufferContentManager. int64_t GetSize() const; // Indicate that the MediaSource parent object got into "ended" state. void Ended(); // The parent SourceBuffer is about to be destroyed. void Detach(); int64_t EvictionThreshold() const; // Interface for MediaSourceDemuxer MediaInfo GetMetadata() const; const TrackBuffer& GetTrackBuffer(TrackInfo::TrackType aTrack) const; const media::TimeIntervals& Buffered(TrackInfo::TrackType) const; const media::TimeUnit& HighestStartTime(TrackInfo::TrackType) const; media::TimeIntervals SafeBuffered(TrackInfo::TrackType) const; bool IsEnded() const { return mEnded; } uint32_t Evictable(TrackInfo::TrackType aTrack) const; media::TimeUnit Seek(TrackInfo::TrackType aTrack, const media::TimeUnit& aTime, const media::TimeUnit& aFuzz); uint32_t SkipToNextRandomAccessPoint(TrackInfo::TrackType aTrack, const media::TimeUnit& aTimeThreadshold, const media::TimeUnit& aFuzz, bool& aFound); already_AddRefed GetSample(TrackInfo::TrackType aTrack, const media::TimeUnit& aFuzz, MediaResult& aResult); int32_t FindCurrentPosition(TrackInfo::TrackType aTrack, const media::TimeUnit& aFuzz) const MOZ_REQUIRES(mTaskQueueCapability); // Will set the next GetSample index if needed. This information is determined // through the value of mNextSampleTimecode. Return false if the index // couldn't be determined or if there's nothing more that could be demuxed. // This occurs if either the track buffer doesn't contain the required // timecode or is empty. nsresult SetNextGetSampleIndexIfNeeded(TrackInfo::TrackType aTrack, const media::TimeUnit& aFuzz) MOZ_REQUIRES(mTaskQueueCapability); media::TimeUnit GetNextRandomAccessPoint(TrackInfo::TrackType aTrack, const media::TimeUnit& aFuzz); // Requests that the TrackBuffersManager populates aInfo with debug // information. This may be done asynchronously, and aInfo should *not* be // accessed by the caller until the returned promise is resolved or rejected. RefPtr RequestDebugInfo( dom::TrackBuffersManagerDebugInfo& aInfo) const; void AddSizeOfResources(MediaSourceDecoder::ResourceSizes* aSizes) const; private: typedef MozPromise CodedFrameProcessingPromise; ~TrackBuffersManager(); // All following functions run on the taskqueue. RefPtr DoAppendData(already_AddRefed aData, const SourceBufferAttributes& aAttributes); void ScheduleSegmentParserLoop() MOZ_REQUIRES(mTaskQueueCapability); void SegmentParserLoop() MOZ_REQUIRES(mTaskQueueCapability); void InitializationSegmentReceived() MOZ_REQUIRES(mTaskQueueCapability); void ShutdownDemuxers() MOZ_REQUIRES(mTaskQueueCapability); void CreateDemuxerforMIMEType() MOZ_REQUIRES(mTaskQueueCapability); void ResetDemuxingState() MOZ_REQUIRES(mTaskQueueCapability); void NeedMoreData() MOZ_REQUIRES(mTaskQueueCapability); void RejectAppend(const MediaResult& aRejectValue, const char* aName) MOZ_REQUIRES(mTaskQueueCapability); // Will return a promise that will be resolved once all frames of the current // media segment have been processed. RefPtr CodedFrameProcessing() MOZ_REQUIRES(mTaskQueueCapability); void CompleteCodedFrameProcessing() MOZ_REQUIRES(mTaskQueueCapability); // Called by ResetParserState. void CompleteResetParserState() MOZ_REQUIRES(mTaskQueueCapability); RefPtr CodedFrameRemovalWithPromise( media::TimeInterval aInterval) MOZ_REQUIRES(mTaskQueueCapability); bool CodedFrameRemoval(media::TimeInterval aInterval) MOZ_REQUIRES(mTaskQueueCapability); // Removes all coded frames -- this is not to spec and should be used as a // last resort to clear buffers only if other methods cannot. void RemoveAllCodedFrames() MOZ_REQUIRES(mTaskQueueCapability); void SetAppendState(SourceBufferAttributes::AppendState aAppendState) MOZ_REQUIRES(mTaskQueueCapability); bool HasVideo() const { return mVideoTracks.mNumTracks > 0; } bool HasAudio() const { return mAudioTracks.mNumTracks > 0; } // The input buffer as per // http://w3c.github.io/media-source/index.html#sourcebuffer-input-buffer Maybe mInputBuffer MOZ_GUARDED_BY(mTaskQueueCapability); // Buffer full flag as per // https://w3c.github.io/media-source/#sourcebuffer-buffer-full-flag. Accessed // on both the main thread and the task queue. Atomic mBufferFull; bool mFirstInitializationSegmentReceived MOZ_GUARDED_BY(mTaskQueueCapability); bool mChangeTypeReceived MOZ_GUARDED_BY(mTaskQueueCapability); // Set to true once a new segment is started. bool mNewMediaSegmentStarted MOZ_GUARDED_BY(mTaskQueueCapability); bool mActiveTrack MOZ_GUARDED_BY(mTaskQueueCapability); MediaContainerType mType MOZ_GUARDED_BY(mTaskQueueCapability); // ContainerParser objects and methods. // Those are used to parse the incoming input buffer. // Recreate the ContainerParser and if aReuseInitData is true then // feed it with the previous init segment found. void RecreateParser(bool aReuseInitData) MOZ_REQUIRES(mTaskQueueCapability); UniquePtr mParser; // Demuxer objects and methods. void AppendDataToCurrentInputBuffer(const MediaSpan& aData) MOZ_REQUIRES(mTaskQueueCapability); RefPtr mInitData MOZ_GUARDED_BY(mTaskQueueCapability); // Checks if a new set of init data is a repeat of the last set of init data // received. Because streams may retransmit the same init data (or // functionally equivalent init data) we do not want to perform costly // operations each time we receive init data, only when it's actually // different data. bool IsRepeatInitData(const MediaInfo& aNewMediaInfo) const MOZ_REQUIRES(mTaskQueueCapability); // Temporary input buffer to handle partial media segment header. // We store the current input buffer content into it should we need to // reinitialize the demuxer once we have some samples and a discontinuity is // detected. Maybe mPendingInputBuffer MOZ_GUARDED_BY(mTaskQueueCapability); RefPtr mCurrentInputBuffer MOZ_GUARDED_BY(mTaskQueueCapability); RefPtr mInputDemuxer MOZ_GUARDED_BY(mTaskQueueCapability); // Length already processed in current media segment. uint64_t mProcessedInput MOZ_GUARDED_BY(mTaskQueueCapability); Maybe mLastParsedEndTime MOZ_GUARDED_BY(mTaskQueueCapability); void OnDemuxerInitDone(const MediaResult& aResult); void OnDemuxerInitFailed(const MediaResult& aFailure); void OnDemuxerResetDone(const MediaResult& aResult) MOZ_REQUIRES(mTaskQueueCapability); MozPromiseRequestHolder mDemuxerInitRequest; void OnDemuxFailed(TrackType aTrack, const MediaResult& aError) MOZ_REQUIRES(mTaskQueueCapability); void DoDemuxVideo() MOZ_REQUIRES(mTaskQueueCapability); void OnVideoDemuxCompleted(RefPtr aSamples); void OnVideoDemuxFailed(const MediaResult& aError) { mVideoTracks.mDemuxRequest.Complete(); mTaskQueueCapability->AssertOnCurrentThread(); OnDemuxFailed(TrackType::kVideoTrack, aError); } void DoDemuxAudio() MOZ_REQUIRES(mTaskQueueCapability); void OnAudioDemuxCompleted(RefPtr aSamples); void OnAudioDemuxFailed(const MediaResult& aError) { mAudioTracks.mDemuxRequest.Complete(); mTaskQueueCapability->AssertOnCurrentThread(); OnDemuxFailed(TrackType::kAudioTrack, aError); } // Dispatches an "encrypted" event is any sample in array has initData // present. void MaybeDispatchEncryptedEvent( const nsTArray>& aSamples); void DoEvictData(const media::TimeUnit& aPlaybackTime, int64_t aSizeToEvict) MOZ_REQUIRES(mTaskQueueCapability); void GetDebugInfo(dom::TrackBuffersManagerDebugInfo& aInfo) const MOZ_REQUIRES(mTaskQueueCapability); struct TrackData { TrackData() : mNumTracks(0), mNeedRandomAccessPoint(true), mSizeBuffer(0) {} Atomic mNumTracks; // Definition of variables: // https://w3c.github.io/media-source/#track-buffers // Last decode timestamp variable that stores the decode timestamp of the // last coded frame appended in the current coded frame group. // The variable is initially unset to indicate that no coded frames have // been appended yet. Maybe mLastDecodeTimestamp; // Last frame duration variable that stores the coded frame duration of the // last coded frame appended in the current coded frame group. // The variable is initially unset to indicate that no coded frames have // been appended yet. Maybe mLastFrameDuration; // Highest end timestamp variable that stores the highest coded frame end // timestamp across all coded frames in the current coded frame group that // were appended to this track buffer. // The variable is initially unset to indicate that no coded frames have // been appended yet. Maybe mHighestEndTimestamp; // Highest presentation timestamp in track buffer. // Protected by global monitor, except when reading on the task queue as it // is only written there. media::TimeUnit mHighestStartTimestamp; // Longest frame duration seen since last random access point. // Only ever accessed when mLastDecodeTimestamp and mLastFrameDuration are // set. media::TimeUnit mLongestFrameDuration; // Need random access point flag variable that keeps track of whether the // track buffer is waiting for a random access point coded frame. // The variable is initially set to true to indicate that random access // point coded frame is needed before anything can be added to the track // buffer. bool mNeedRandomAccessPoint; RefPtr mDemuxer; MozPromiseRequestHolder mDemuxRequest; // Highest end timestamp of the last media segment demuxed. media::TimeUnit mLastParsedEndTime; // If set, position where the next contiguous frame will be inserted. // If a discontinuity is detected, it will be unset and recalculated upon // the next insertion. Maybe mNextInsertionIndex; // Samples just demuxed, but not yet parsed. TrackBuffer mQueuedSamples; const TrackBuffer& GetTrackBuffer() const { MOZ_RELEASE_ASSERT(mBuffers.Length(), "TrackBuffer must have been created"); return mBuffers.LastElement(); } TrackBuffer& GetTrackBuffer() { MOZ_RELEASE_ASSERT(mBuffers.Length(), "TrackBuffer must have been created"); return mBuffers.LastElement(); } // We only manage a single track of each type at this time. nsTArray mBuffers; // Track buffer ranges variable that represents the presentation time ranges // occupied by the coded frames currently stored in the track buffer. media::TimeIntervals mBufferedRanges; // Sanitized mBufferedRanges with a fuzz of half a sample's duration applied // This buffered ranges is the basis of what is exposed to the JS. media::TimeIntervals mSanitizedBufferedRanges; // Byte size of all samples contained in this track buffer. uint32_t mSizeBuffer; // TrackInfo of the first metadata received. RefPtr mInfo; // TrackInfo of the last metadata parsed (updated with each init segment. RefPtr mLastInfo; // If set, position of the next sample to be retrieved by GetSample(). // If the position is equal to the TrackBuffer's length, it indicates that // we've reached EOS. Maybe mNextGetSampleIndex; // Approximation of the next sample's decode timestamp. media::TimeUnit mNextSampleTimecode; // Approximation of the next sample's presentation timestamp. media::TimeUnit mNextSampleTime; struct EvictionIndex { EvictionIndex() { Reset(); } void Reset() { mEvictable = 0; mLastIndex = 0; } uint32_t mEvictable; uint32_t mLastIndex; }; // Size of data that can be safely evicted during the next eviction // cycle. // We consider as evictable all frames up to the last keyframe prior to // mNextGetSampleIndex. If mNextGetSampleIndex isn't set, then we assume // that we can't yet evict data. // Protected by global monitor, except when reading on the task queue as it // is only written there. EvictionIndex mEvictionIndex; void ResetAppendState() { mLastDecodeTimestamp.reset(); mLastFrameDuration.reset(); mHighestEndTimestamp.reset(); mNeedRandomAccessPoint = true; mNextInsertionIndex.reset(); } void Reset() { ResetAppendState(); mEvictionIndex.Reset(); for (auto& buffer : mBuffers) { buffer.Clear(); } mSizeBuffer = 0; mNextGetSampleIndex.reset(); mBufferedRanges.Clear(); mSanitizedBufferedRanges.Clear(); } void AddSizeOfResources(MediaSourceDecoder::ResourceSizes* aSizes) const; }; void CheckSequenceDiscontinuity(const media::TimeUnit& aPresentationTime) MOZ_REQUIRES(mTaskQueueCapability); void ProcessFrames(TrackBuffer& aSamples, TrackData& aTrackData) MOZ_REQUIRES(mTaskQueueCapability); media::TimeInterval PresentationInterval(const TrackBuffer& aSamples) const MOZ_REQUIRES(mTaskQueueCapability); bool CheckNextInsertionIndex(TrackData& aTrackData, const media::TimeUnit& aSampleTime) MOZ_REQUIRES(mTaskQueueCapability); void InsertFrames(TrackBuffer& aSamples, const media::TimeIntervals& aIntervals, TrackData& aTrackData) MOZ_REQUIRES(mTaskQueueCapability); void UpdateHighestTimestamp(TrackData& aTrackData, const media::TimeUnit& aHighestTime) MOZ_REQUIRES(mTaskQueueCapability); // Remove all frames and their dependencies contained in aIntervals. // Return the index at which frames were first removed or 0 if no frames // removed. enum class RemovalMode { kRemoveFrame, kTruncateFrame, }; uint32_t RemoveFrames(const media::TimeIntervals& aIntervals, TrackData& aTrackData, uint32_t aStartIndex, RemovalMode aMode); // Recalculate track's evictable amount. void ResetEvictionIndex(TrackData& aTrackData); void UpdateEvictionIndex(TrackData& aTrackData, uint32_t aCurrentIndex); // Find index of sample. Return a negative value if not found. uint32_t FindSampleIndex(const TrackBuffer& aTrackBuffer, const media::TimeInterval& aInterval); const MediaRawData* GetSample(TrackInfo::TrackType aTrack, uint32_t aIndex, const media::TimeUnit& aExpectedDts, const media::TimeUnit& aExpectedPts, const media::TimeUnit& aFuzz); void UpdateBufferedRanges(); void RejectProcessing(const MediaResult& aRejectValue, const char* aName); void ResolveProcessing(bool aResolveValue, const char* aName); MozPromiseRequestHolder mProcessingRequest; MozPromiseHolder mProcessingPromise; // Trackbuffers definition. nsTArray GetTracksList() const; nsTArray GetTracksList(); TrackData& GetTracksData(TrackType aTrack) { switch (aTrack) { case TrackType::kVideoTrack: return mVideoTracks; case TrackType::kAudioTrack: default: return mAudioTracks; } } const TrackData& GetTracksData(TrackType aTrack) const { switch (aTrack) { case TrackType::kVideoTrack: return mVideoTracks; case TrackType::kAudioTrack: default: return mAudioTracks; } } TrackData mVideoTracks; TrackData mAudioTracks; // TaskQueue methods and objects. RefPtr GetTaskQueueSafe() const { MutexAutoLock mut(mMutex); return mTaskQueue; } NotNull TaskQueueFromTaskQueue() const { #ifdef DEBUG RefPtr taskQueue = GetTaskQueueSafe(); MOZ_ASSERT(taskQueue && taskQueue->IsCurrentThreadIn()); #endif return WrapNotNull(mTaskQueue.get()); } bool OnTaskQueue() const { auto taskQueue = TaskQueueFromTaskQueue(); return taskQueue->IsCurrentThreadIn(); } void ResetTaskQueue() { MutexAutoLock mut(mMutex); mTaskQueue = nullptr; } // SourceBuffer Queues and running context. SourceBufferTaskQueue mQueue; void QueueTask(SourceBufferTask* aTask); void ProcessTasks(); // Set if the TrackBuffersManager is currently processing a task. // At this stage, this task is always a AppendBufferTask. RefPtr mCurrentTask MOZ_GUARDED_BY(mTaskQueueCapability); // Current SourceBuffer state for ongoing task. // Its content is returned to the SourceBuffer once the AppendBufferTask has // completed. UniquePtr mSourceBufferAttributes MOZ_GUARDED_BY(mTaskQueueCapability); // The current sourcebuffer append window. It's content is equivalent to // mSourceBufferAttributes.mAppendWindowStart/End media::TimeInterval mAppendWindow MOZ_GUARDED_BY(mTaskQueueCapability); // Strong references to external objects. nsMainThreadPtrHandle mParentDecoder; const RefPtr mAbstractMainThread; // Return public highest end time across all aTracks. // Monitor must be held. media::TimeUnit HighestEndTime( nsTArray& aTracks) const; // Set to true if mediasource state changed to ended. Atomic mEnded; // Global size of this source buffer content. Atomic mSizeSourceBuffer; const int64_t mVideoEvictionThreshold; const int64_t mAudioEvictionThreshold; enum class EvictionState { NO_EVICTION_NEEDED, EVICTION_NEEDED, EVICTION_COMPLETED, }; Atomic mEvictionState; // Monitor to protect following objects accessed across multiple threads. mutable Mutex mMutex MOZ_UNANNOTATED; // mTaskQueue is only ever written after construction on the task queue. // As such, it can be accessed while on task queue without the need for the // mutex. RefPtr mTaskQueue; // Stable audio and video track time ranges. media::TimeIntervals mVideoBufferedRanges; media::TimeIntervals mAudioBufferedRanges; // MediaInfo of the first init segment read. MediaInfo mInfo; // End mutex protected members. // EventTargetCapability used to ensure we're running on the task queue // as expected for various accesses. // TODO: we could store only this and dispatch to it, rather than also having // mTaskQueue. However, there's special locking around mTaskQueue, so we keep // both for now. Maybe> mTaskQueueCapability; }; } // namespace mozilla #endif /* MOZILLA_TRACKBUFFERSMANAGER_H_ */