/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim:set ts=2 sw=2 sts=2 et cindent: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "MediaSourceDemuxer.h" #include "MediaSourceUtils.h" #include "OpusDecoder.h" #include "SourceBufferList.h" #include "VorbisDecoder.h" #include "VideoUtils.h" #include "nsPrintfCString.h" #include #include #include namespace mozilla { typedef TrackInfo::TrackType TrackType; using media::TimeIntervals; using media::TimeUnit; MediaSourceDemuxer::MediaSourceDemuxer(AbstractThread* aAbstractMainThread) : mTaskQueue( TaskQueue::Create(GetMediaThreadPool(MediaThreadType::SUPERVISOR), "MediaSourceDemuxer::mTaskQueue")), mMonitor("MediaSourceDemuxer") { MOZ_ASSERT(NS_IsMainThread()); } constexpr TimeUnit MediaSourceDemuxer::EOS_FUZZ; constexpr TimeUnit MediaSourceDemuxer::EOS_FUZZ_START; RefPtr MediaSourceDemuxer::Init() { RefPtr self = this; return InvokeAsync(GetTaskQueue(), __func__, [self]() { if (self->ScanSourceBuffersForContent()) { return InitPromise::CreateAndResolve(NS_OK, __func__); } RefPtr p = self->mInitPromise.Ensure(__func__); return p; }); } void MediaSourceDemuxer::AddSizeOfResources( MediaSourceDecoder::ResourceSizes* aSizes) { MOZ_ASSERT(NS_IsMainThread()); // NB: The track buffers must only be accessed on the TaskQueue. RefPtr self = this; RefPtr sizes = aSizes; nsCOMPtr task = NS_NewRunnableFunction( "MediaSourceDemuxer::AddSizeOfResources", [self, sizes]() { for (const RefPtr& manager : self->mSourceBuffers) { manager->AddSizeOfResources(sizes); } }); nsresult rv = GetTaskQueue()->Dispatch(task.forget()); MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); Unused << rv; } void MediaSourceDemuxer::NotifyInitDataArrived() { RefPtr self = this; nsCOMPtr task = NS_NewRunnableFunction( "MediaSourceDemuxer::NotifyInitDataArrived", [self]() { if (self->mInitPromise.IsEmpty()) { return; } if (self->ScanSourceBuffersForContent()) { self->mInitPromise.ResolveIfExists(NS_OK, __func__); } }); nsresult rv = GetTaskQueue()->Dispatch(task.forget()); MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); Unused << rv; } bool MediaSourceDemuxer::ScanSourceBuffersForContent() { MOZ_ASSERT(OnTaskQueue()); if (mSourceBuffers.IsEmpty()) { return false; } MonitorAutoLock mon(mMonitor); bool haveEmptySourceBuffer = false; for (const auto& sourceBuffer : mSourceBuffers) { MediaInfo info = sourceBuffer->GetMetadata(); if (!info.HasAudio() && !info.HasVideo()) { haveEmptySourceBuffer = true; } if (info.HasAudio() && !mAudioTrack) { mInfo.mAudio = info.mAudio; mAudioTrack = sourceBuffer; } if (info.HasVideo() && !mVideoTrack) { mInfo.mVideo = info.mVideo; mVideoTrack = sourceBuffer; } if (info.IsEncrypted() && !mInfo.IsEncrypted()) { mInfo.mCrypto = info.mCrypto; } } if (mInfo.HasAudio() && mInfo.HasVideo()) { // We have both audio and video. We can ignore non-ready source buffer. return true; } return !haveEmptySourceBuffer; } uint32_t MediaSourceDemuxer::GetNumberTracks(TrackType aType) const { MonitorAutoLock mon(mMonitor); switch (aType) { case TrackType::kAudioTrack: return mInfo.HasAudio() ? 1u : 0; case TrackType::kVideoTrack: return mInfo.HasVideo() ? 1u : 0; default: return 0; } } already_AddRefed MediaSourceDemuxer::GetTrackDemuxer( TrackType aType, uint32_t aTrackNumber) { MonitorAutoLock mon(mMonitor); RefPtr manager = GetManager(aType); if (!manager) { return nullptr; } RefPtr e = new MediaSourceTrackDemuxer(this, aType, manager); DDLINKCHILD("track demuxer", e.get()); mDemuxers.AppendElement(e); return e.forget(); } bool MediaSourceDemuxer::IsSeekable() const { return true; } UniquePtr MediaSourceDemuxer::GetCrypto() { MonitorAutoLock mon(mMonitor); auto crypto = MakeUnique(); *crypto = mInfo.mCrypto; return crypto; } void MediaSourceDemuxer::AttachSourceBuffer( RefPtr& aSourceBuffer) { nsCOMPtr task = NewRunnableMethod&&>( "MediaSourceDemuxer::DoAttachSourceBuffer", this, &MediaSourceDemuxer::DoAttachSourceBuffer, aSourceBuffer); nsresult rv = GetTaskQueue()->Dispatch(task.forget()); MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); Unused << rv; } void MediaSourceDemuxer::DoAttachSourceBuffer( RefPtr&& aSourceBuffer) { MOZ_ASSERT(OnTaskQueue()); mSourceBuffers.AppendElement(std::move(aSourceBuffer)); ScanSourceBuffersForContent(); } void MediaSourceDemuxer::DetachSourceBuffer( RefPtr& aSourceBuffer) { nsCOMPtr task = NS_NewRunnableFunction("MediaSourceDemuxer::DoDetachSourceBuffer", [self = RefPtr{this}, aSourceBuffer]() { self->DoDetachSourceBuffer(aSourceBuffer); }); nsresult rv = GetTaskQueue()->Dispatch(task.forget()); MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); Unused << rv; } void MediaSourceDemuxer::DoDetachSourceBuffer( const RefPtr& aSourceBuffer) { MOZ_ASSERT(OnTaskQueue()); mSourceBuffers.RemoveElementsBy( [&aSourceBuffer](const RefPtr aLinkedSourceBuffer) { return aLinkedSourceBuffer == aSourceBuffer; }); AutoTArray, 2> matchingDemuxers; { MonitorAutoLock mon(mMonitor); if (aSourceBuffer == mAudioTrack) { mAudioTrack = nullptr; } if (aSourceBuffer == mVideoTrack) { mVideoTrack = nullptr; } mDemuxers.RemoveElementsBy( [&](RefPtr& elementRef) { if (!elementRef->HasManager(aSourceBuffer)) { return false; } matchingDemuxers.AppendElement(std::move(elementRef)); return true; }); } for (MediaSourceTrackDemuxer* demuxer : matchingDemuxers) { demuxer->DetachManager(); } ScanSourceBuffersForContent(); } TrackInfo* MediaSourceDemuxer::GetTrackInfo(TrackType aTrack) { switch (aTrack) { case TrackType::kAudioTrack: return &mInfo.mAudio; case TrackType::kVideoTrack: return &mInfo.mVideo; default: return nullptr; } } RefPtr MediaSourceDemuxer::GetManager(TrackType aTrack) { switch (aTrack) { case TrackType::kAudioTrack: return mAudioTrack; case TrackType::kVideoTrack: return mVideoTrack; default: return nullptr; } } MediaSourceDemuxer::~MediaSourceDemuxer() { mInitPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); } RefPtr MediaSourceDemuxer::GetDebugInfo( dom::MediaSourceDemuxerDebugInfo& aInfo) const { MonitorAutoLock mon(mMonitor); nsTArray> promises; if (mAudioTrack) { promises.AppendElement(mAudioTrack->RequestDebugInfo(aInfo.mAudioTrack)); } if (mVideoTrack) { promises.AppendElement(mVideoTrack->RequestDebugInfo(aInfo.mVideoTrack)); } return GenericPromise::All(GetCurrentSerialEventTarget(), promises) ->Then( GetCurrentSerialEventTarget(), __func__, []() { return GenericPromise::CreateAndResolve(true, __func__); }, [] { return GenericPromise::CreateAndReject(NS_ERROR_FAILURE, __func__); }); } MediaSourceTrackDemuxer::MediaSourceTrackDemuxer(MediaSourceDemuxer* aParent, TrackInfo::TrackType aType, TrackBuffersManager* aManager) : mParent(aParent), mTaskQueue(mParent->GetTaskQueue()), mType(aType), mMonitor("MediaSourceTrackDemuxer"), mManager(aManager), mReset(true), mPreRoll(TimeUnit::FromMicroseconds( OpusDataDecoder::IsOpus(mParent->GetTrackInfo(mType)->mMimeType) || VorbisDataDecoder::IsVorbis( mParent->GetTrackInfo(mType)->mMimeType) ? 80000 : mParent->GetTrackInfo(mType)->mMimeType.EqualsLiteral( "audio/mp4a-latm") // AAC encoder delay is by default 2112 audio frames. // See // https://developer.apple.com/library/content/documentation/QuickTime/QTFF/QTFFAppenG/QTFFAppenG.html // So we always seek 2112 frames ? (2112 * 1000000ULL / mParent->GetTrackInfo(mType)->GetAsAudioInfo()->mRate) : 0)) { MOZ_ASSERT(mParent); MOZ_ASSERT(mTaskQueue); } UniquePtr MediaSourceTrackDemuxer::GetInfo() const { MonitorAutoLock mon(mParent->mMonitor); return mParent->GetTrackInfo(mType)->Clone(); } RefPtr MediaSourceTrackDemuxer::Seek( const TimeUnit& aTime) { MOZ_ASSERT(mParent, "Called after BreackCycle()"); return InvokeAsync(mParent->GetTaskQueue(), this, __func__, &MediaSourceTrackDemuxer::DoSeek, aTime); } RefPtr MediaSourceTrackDemuxer::GetSamples(int32_t aNumSamples) { MOZ_ASSERT(mParent, "Called after BreackCycle()"); return InvokeAsync(mParent->GetTaskQueue(), this, __func__, &MediaSourceTrackDemuxer::DoGetSamples, aNumSamples); } void MediaSourceTrackDemuxer::Reset() { MOZ_ASSERT(mParent, "Called after BreackCycle()"); RefPtr self = this; nsCOMPtr task = NS_NewRunnableFunction("MediaSourceTrackDemuxer::Reset", [self]() { self->mNextSample.reset(); self->mReset = true; if (!self->mManager) { return; } MOZ_ASSERT(self->OnTaskQueue()); self->mManager->Seek(self->mType, TimeUnit::Zero(), TimeUnit::Zero()); { MonitorAutoLock mon(self->mMonitor); self->mNextRandomAccessPoint = self->mManager->GetNextRandomAccessPoint( self->mType, MediaSourceDemuxer::EOS_FUZZ); } }); nsresult rv = mParent->GetTaskQueue()->Dispatch(task.forget()); MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); Unused << rv; } nsresult MediaSourceTrackDemuxer::GetNextRandomAccessPoint(TimeUnit* aTime) { MonitorAutoLock mon(mMonitor); *aTime = mNextRandomAccessPoint; return NS_OK; } RefPtr MediaSourceTrackDemuxer::SkipToNextRandomAccessPoint( const TimeUnit& aTimeThreshold) { return InvokeAsync(mParent->GetTaskQueue(), this, __func__, &MediaSourceTrackDemuxer::DoSkipToNextRandomAccessPoint, aTimeThreshold); } media::TimeIntervals MediaSourceTrackDemuxer::GetBuffered() { MonitorAutoLock mon(mMonitor); if (!mManager) { return media::TimeIntervals(); } return mManager->Buffered(); } void MediaSourceTrackDemuxer::BreakCycles() { RefPtr self = this; nsCOMPtr task = NS_NewRunnableFunction("MediaSourceTrackDemuxer::BreakCycles", [self]() { self->DetachManager(); self->mParent = nullptr; }); nsresult rv = mParent->GetTaskQueue()->Dispatch(task.forget()); MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); Unused << rv; } RefPtr MediaSourceTrackDemuxer::DoSeek( const TimeUnit& aTime) { if (!mManager) { return SeekPromise::CreateAndReject( MediaResult(NS_ERROR_DOM_MEDIA_CANCELED, RESULT_DETAIL("manager is detached.")), __func__); } MOZ_ASSERT(OnTaskQueue()); TimeIntervals buffered = mManager->Buffered(mType); // Fuzz factor represents a +/- threshold. So when seeking it allows the gap // to be twice as big as the fuzz value. We only want to allow EOS_FUZZ gap. buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2); TimeUnit seekTime = std::max(aTime - mPreRoll, TimeUnit::Zero()); if (mManager->IsEnded() && seekTime >= buffered.GetEnd()) { // We're attempting to seek past the end time. Cap seekTime so that we seek // to the last sample instead. seekTime = std::max(mManager->HighestStartTime(mType) - mPreRoll, TimeUnit::Zero()); } if (!buffered.ContainsWithStrictEnd(seekTime)) { if (!buffered.ContainsWithStrictEnd(aTime)) { // We don't have the data to seek to. return SeekPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__); } // Theoretically we should reject the promise with WAITING_FOR_DATA, // however, to avoid unwanted regressions we assume that if at this time // we don't have the wanted data it won't come later. // Instead of using the pre-rolled time, use the earliest time available in // the interval. TimeIntervals::IndexType index = buffered.Find(aTime); MOZ_ASSERT(index != TimeIntervals::NoIndex); seekTime = buffered[index].mStart; } seekTime = mManager->Seek(mType, seekTime, MediaSourceDemuxer::EOS_FUZZ); MediaResult result = NS_OK; RefPtr sample = mManager->GetSample(mType, TimeUnit::Zero(), result); MOZ_ASSERT(NS_SUCCEEDED(result) && sample); mNextSample = Some(sample); mReset = false; { MonitorAutoLock mon(mMonitor); mNextRandomAccessPoint = mManager->GetNextRandomAccessPoint(mType, MediaSourceDemuxer::EOS_FUZZ); } return SeekPromise::CreateAndResolve(seekTime, __func__); } RefPtr MediaSourceTrackDemuxer::DoGetSamples(int32_t aNumSamples) { if (!mManager) { return SamplesPromise::CreateAndReject( MediaResult(NS_ERROR_DOM_MEDIA_CANCELED, RESULT_DETAIL("manager is detached.")), __func__); } MOZ_ASSERT(OnTaskQueue()); if (mReset) { // If a reset was recently performed, we ensure that the data // we are about to retrieve is still available. TimeIntervals buffered = mManager->Buffered(mType); if (buffered.IsEmpty() && mManager->IsEnded()) { return SamplesPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_END_OF_STREAM, __func__); } // We use a larger fuzz to determine the presentation start // time than the fuzz we use to determine acceptable gaps between // frames. This is needed to fix embedded video issues as seen in the wild // from different muxed stream start times. // See: https://www.w3.org/TR/media-source-2/#presentation-start-time buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ_START); if (!buffered.ContainsWithStrictEnd(TimeUnit::Zero())) { return SamplesPromise::CreateAndReject( NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__); } mReset = false; } RefPtr sample; MediaResult result = NS_OK; if (mNextSample) { sample = mNextSample.ref(); mNextSample.reset(); } else { sample = mManager->GetSample(mType, MediaSourceDemuxer::EOS_FUZZ, result); } if (!sample) { if (result == NS_ERROR_DOM_MEDIA_END_OF_STREAM || result == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) { return SamplesPromise::CreateAndReject( (result == NS_ERROR_DOM_MEDIA_END_OF_STREAM && mManager->IsEnded()) ? NS_ERROR_DOM_MEDIA_END_OF_STREAM : NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__); } return SamplesPromise::CreateAndReject(result, __func__); } RefPtr samples = new SamplesHolder; samples->AppendSample(sample); { MonitorAutoLock mon(mMonitor); // spurious warning will be given // Diagnostic asserts for bug 1810396 MOZ_DIAGNOSTIC_ASSERT(sample, "Invalid sample pointer found!"); MOZ_DIAGNOSTIC_ASSERT(sample->HasValidTime(), "Invalid sample time found!"); if (!sample) { return SamplesPromise::CreateAndReject(NS_ERROR_NULL_POINTER, __func__); } if (mNextRandomAccessPoint <= sample->mTime) { mNextRandomAccessPoint = mManager->GetNextRandomAccessPoint( mType, MediaSourceDemuxer::EOS_FUZZ); } } return SamplesPromise::CreateAndResolve(samples, __func__); } RefPtr MediaSourceTrackDemuxer::DoSkipToNextRandomAccessPoint( const TimeUnit& aTimeThreadshold) { if (!mManager) { return SkipAccessPointPromise::CreateAndReject( SkipFailureHolder(MediaResult(NS_ERROR_DOM_MEDIA_CANCELED, RESULT_DETAIL("manager is detached.")), 0), __func__); } MOZ_ASSERT(OnTaskQueue()); uint32_t parsed = 0; // Ensure that the data we are about to skip to is still available. TimeIntervals buffered = mManager->Buffered(mType); buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2); if (buffered.ContainsWithStrictEnd(aTimeThreadshold)) { bool found; parsed = mManager->SkipToNextRandomAccessPoint( mType, aTimeThreadshold, MediaSourceDemuxer::EOS_FUZZ, found); if (found) { return SkipAccessPointPromise::CreateAndResolve(parsed, __func__); } } SkipFailureHolder holder(mManager->IsEnded() ? NS_ERROR_DOM_MEDIA_END_OF_STREAM : NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, parsed); return SkipAccessPointPromise::CreateAndReject(holder, __func__); } bool MediaSourceTrackDemuxer::HasManager(TrackBuffersManager* aManager) const { MOZ_ASSERT(OnTaskQueue()); return mManager == aManager; } void MediaSourceTrackDemuxer::DetachManager() { MOZ_ASSERT(OnTaskQueue()); MonitorAutoLock mon(mMonitor); mManager = nullptr; } } // namespace mozilla