/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim: set ts=8 sts=2 et sw=2 tw=80: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "MediaDecoder.h" #include "AudioDeviceInfo.h" #include "DOMMediaStream.h" #include "DecoderBenchmark.h" #include "ImageContainer.h" #include "MediaDecoderStateMachineBase.h" #include "MediaFormatReader.h" #include "MediaResource.h" #include "MediaShutdownManager.h" #include "MediaTrackGraph.h" #include "TelemetryProbesReporter.h" #include "VideoFrameContainer.h" #include "VideoUtils.h" #include "mozilla/AbstractThread.h" #include "mozilla/dom/DOMTypes.h" #include "mozilla/FloatingPoint.h" #include "mozilla/MathAlgorithms.h" #include "mozilla/Preferences.h" #include "mozilla/StaticPrefs_media.h" #include "mozilla/StaticPtr.h" #include "mozilla/Telemetry.h" #include "mozilla/Unused.h" #include "nsComponentManagerUtils.h" #include "nsContentUtils.h" #include "nsError.h" #include "nsIMemoryReporter.h" #include "nsPrintfCString.h" #include "nsServiceManagerUtils.h" #include "nsTArray.h" #include "WindowRenderer.h" #include #include #include using namespace mozilla::dom; using namespace mozilla::layers; using namespace mozilla::media; namespace mozilla { // avoid redefined macro in unified build #undef LOG #undef DUMP LazyLogModule gMediaDecoderLog("MediaDecoder"); #define LOG(x, ...) \ DDMOZ_LOG(gMediaDecoderLog, LogLevel::Debug, x, ##__VA_ARGS__) #define DUMP(x, ...) printf_stderr(x "\n", ##__VA_ARGS__) #define NS_DispatchToMainThread(...) CompileError_UseAbstractMainThreadInstead static const char* ToPlayStateStr(MediaDecoder::PlayState aState) { switch (aState) { case MediaDecoder::PLAY_STATE_LOADING: return "LOADING"; case MediaDecoder::PLAY_STATE_PAUSED: return "PAUSED"; case MediaDecoder::PLAY_STATE_PLAYING: return "PLAYING"; case MediaDecoder::PLAY_STATE_ENDED: return "ENDED"; case MediaDecoder::PLAY_STATE_SHUTDOWN: return "SHUTDOWN"; default: MOZ_ASSERT_UNREACHABLE("Invalid playState."); } return "UNKNOWN"; } class MediaMemoryTracker : public nsIMemoryReporter { virtual ~MediaMemoryTracker(); NS_DECL_THREADSAFE_ISUPPORTS NS_DECL_NSIMEMORYREPORTER MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf); MediaMemoryTracker(); void InitMemoryReporter(); static StaticRefPtr sUniqueInstance; static MediaMemoryTracker* UniqueInstance() { if (!sUniqueInstance) { sUniqueInstance = new MediaMemoryTracker(); sUniqueInstance->InitMemoryReporter(); } return sUniqueInstance; } using DecodersArray = nsTArray; static DecodersArray& Decoders() { return UniqueInstance()->mDecoders; } DecodersArray mDecoders; public: static void AddMediaDecoder(MediaDecoder* aDecoder) { Decoders().AppendElement(aDecoder); } static void RemoveMediaDecoder(MediaDecoder* aDecoder) { DecodersArray& decoders = Decoders(); decoders.RemoveElement(aDecoder); if (decoders.IsEmpty()) { sUniqueInstance = nullptr; } } static RefPtr GetSizes(dom::Document* aDoc) { MOZ_ASSERT(NS_IsMainThread()); DecodersArray& decoders = Decoders(); // if we don't have any decoder, we can bail if (decoders.IsEmpty()) { // and release the instance that was created by calling Decoders() sUniqueInstance = nullptr; return MediaMemoryPromise::CreateAndResolve(MediaMemoryInfo(), __func__); } RefPtr resourceSizes = new MediaDecoder::ResourceSizes(MediaMemoryTracker::MallocSizeOf); size_t videoSize = 0; size_t audioSize = 0; for (auto&& decoder : decoders) { if (decoder->GetOwner() && decoder->GetOwner()->GetDocument() == aDoc) { videoSize += decoder->SizeOfVideoQueue(); audioSize += decoder->SizeOfAudioQueue(); decoder->AddSizeOfResources(resourceSizes); } } return resourceSizes->Promise()->Then( AbstractThread::MainThread(), __func__, [videoSize, audioSize](size_t resourceSize) { return MediaMemoryPromise::CreateAndResolve( MediaMemoryInfo(videoSize, audioSize, resourceSize), __func__); }, [](size_t) { return MediaMemoryPromise::CreateAndReject(NS_ERROR_FAILURE, __func__); }); } }; StaticRefPtr MediaMemoryTracker::sUniqueInstance; RefPtr GetMediaMemorySizes(dom::Document* aDoc) { return MediaMemoryTracker::GetSizes(aDoc); } LazyLogModule gMediaTimerLog("MediaTimer"); constexpr TimeUnit MediaDecoder::DEFAULT_NEXT_FRAME_AVAILABLE_BUFFERED; void MediaDecoder::InitStatics() { MOZ_ASSERT(NS_IsMainThread()); // Eagerly init gMediaDecoderLog to work around bug 1415441. MOZ_LOG(gMediaDecoderLog, LogLevel::Info, ("MediaDecoder::InitStatics")); } NS_IMPL_ISUPPORTS(MediaMemoryTracker, nsIMemoryReporter) void MediaDecoder::NotifyOwnerActivityChanged(bool aIsOwnerInvisible, bool aIsOwnerConnected) { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); SetElementVisibility(aIsOwnerInvisible, aIsOwnerConnected); NotifyCompositor(); } void MediaDecoder::Pause() { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); LOG("Pause"); if (mPlayState == PLAY_STATE_LOADING || IsEnded()) { mNextState = PLAY_STATE_PAUSED; return; } ChangeState(PLAY_STATE_PAUSED); } void MediaDecoder::SetVolume(double aVolume) { MOZ_ASSERT(NS_IsMainThread()); mVolume = aVolume; } RefPtr MediaDecoder::SetSink(AudioDeviceInfo* aSinkDevice) { MOZ_ASSERT(NS_IsMainThread()); mSinkDevice = aSinkDevice; return GetStateMachine()->InvokeSetSink(aSinkDevice); } void MediaDecoder::SetOutputCaptureState(OutputCaptureState aState, SharedDummyTrack* aDummyTrack) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load()."); MOZ_ASSERT_IF(aState == OutputCaptureState::Capture, aDummyTrack); mOutputCaptureState = aState; if (mOutputDummyTrack.Ref().get() != aDummyTrack) { mOutputDummyTrack = nsMainThreadPtrHandle( MakeAndAddRef>( "MediaDecoder::mOutputDummyTrack", aDummyTrack)); } } void MediaDecoder::AddOutputTrack(RefPtr aTrack) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load()."); CopyableTArray> tracks = mOutputTracks; tracks.AppendElement(std::move(aTrack)); mOutputTracks = tracks; } void MediaDecoder::RemoveOutputTrack( const RefPtr& aTrack) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load()."); CopyableTArray> tracks = mOutputTracks; if (tracks.RemoveElement(aTrack)) { mOutputTracks = tracks; } } void MediaDecoder::SetOutputTracksPrincipal( const RefPtr& aPrincipal) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(mDecoderStateMachine, "Must be called after Load()."); mOutputPrincipal = MakePrincipalHandle(aPrincipal); } double MediaDecoder::GetDuration() { MOZ_ASSERT(NS_IsMainThread()); return ToMicrosecondResolution(mDuration.match(DurationToDouble())); } bool MediaDecoder::IsInfinite() const { MOZ_ASSERT(NS_IsMainThread()); return std::isinf(mDuration.match(DurationToDouble())); } #define INIT_MIRROR(name, val) \ name(mOwner->AbstractMainThread(), val, "MediaDecoder::" #name " (Mirror)") #define INIT_CANONICAL(name, val) \ name(mOwner->AbstractMainThread(), val, "MediaDecoder::" #name " (Canonical)") MediaDecoder::MediaDecoder(MediaDecoderInit& aInit) : mWatchManager(this, aInit.mOwner->AbstractMainThread()), mLogicalPosition(0.0), mDuration(TimeUnit::Invalid()), mOwner(aInit.mOwner), mAbstractMainThread(aInit.mOwner->AbstractMainThread()), mFrameStats(new FrameStatistics()), mDecoderBenchmark(new DecoderBenchmark()), mVideoFrameContainer(aInit.mOwner->GetVideoFrameContainer()), mMinimizePreroll(aInit.mMinimizePreroll), mFiredMetadataLoaded(false), mIsOwnerInvisible(false), mIsOwnerConnected(false), mForcedHidden(false), mHasSuspendTaint(aInit.mHasSuspendTaint), mShouldResistFingerprinting( aInit.mOwner->ShouldResistFingerprinting(RFPTarget::Unknown)), mPlaybackRate(aInit.mPlaybackRate), mLogicallySeeking(false, "MediaDecoder::mLogicallySeeking"), INIT_MIRROR(mBuffered, TimeIntervals()), INIT_MIRROR(mCurrentPosition, TimeUnit::Zero()), INIT_MIRROR(mStateMachineDuration, NullableTimeUnit()), INIT_MIRROR(mIsAudioDataAudible, false), INIT_CANONICAL(mVolume, aInit.mVolume), INIT_CANONICAL(mPreservesPitch, aInit.mPreservesPitch), INIT_CANONICAL(mLooping, aInit.mLooping), INIT_CANONICAL(mStreamName, aInit.mStreamName), INIT_CANONICAL(mSinkDevice, nullptr), INIT_CANONICAL(mSecondaryVideoContainer, nullptr), INIT_CANONICAL(mOutputCaptureState, OutputCaptureState::None), INIT_CANONICAL(mOutputDummyTrack, nullptr), INIT_CANONICAL(mOutputTracks, nsTArray>()), INIT_CANONICAL(mOutputPrincipal, PRINCIPAL_HANDLE_NONE), INIT_CANONICAL(mPlayState, PLAY_STATE_LOADING), mSameOriginMedia(false), mVideoDecodingOberver( new BackgroundVideoDecodingPermissionObserver(this)), mIsBackgroundVideoDecodingAllowed(false), mTelemetryReported(false), mContainerType(aInit.mContainerType), mTelemetryProbesReporter( new TelemetryProbesReporter(aInit.mReporterOwner)) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(mAbstractMainThread); MediaMemoryTracker::AddMediaDecoder(this); // // Initialize watchers. // // mDuration mWatchManager.Watch(mStateMachineDuration, &MediaDecoder::DurationChanged); // readyState mWatchManager.Watch(mPlayState, &MediaDecoder::UpdateReadyState); // ReadyState computation depends on MediaDecoder::CanPlayThrough, which // depends on the download rate. mWatchManager.Watch(mBuffered, &MediaDecoder::UpdateReadyState); // mLogicalPosition mWatchManager.Watch(mCurrentPosition, &MediaDecoder::UpdateLogicalPosition); mWatchManager.Watch(mPlayState, &MediaDecoder::UpdateLogicalPosition); mWatchManager.Watch(mLogicallySeeking, &MediaDecoder::UpdateLogicalPosition); mWatchManager.Watch(mIsAudioDataAudible, &MediaDecoder::NotifyAudibleStateChanged); mWatchManager.Watch(mVolume, &MediaDecoder::NotifyVolumeChanged); mVideoDecodingOberver->RegisterEvent(); } #undef INIT_MIRROR #undef INIT_CANONICAL void MediaDecoder::Shutdown() { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); // Unwatch all watch targets to prevent further notifications. mWatchManager.Shutdown(); DiscardOngoingSeekIfExists(); // This changes the decoder state to SHUTDOWN and does other things // necessary to unblock the state machine thread if it's blocked, so // the asynchronous shutdown in nsDestroyStateMachine won't deadlock. if (mDecoderStateMachine) { ShutdownStateMachine()->Then(mAbstractMainThread, __func__, this, &MediaDecoder::FinishShutdown, &MediaDecoder::FinishShutdown); } else { // Ensure we always unregister asynchronously in order not to disrupt // the hashtable iterating in MediaShutdownManager::Shutdown(). RefPtr self = this; nsCOMPtr r = NS_NewRunnableFunction( "MediaDecoder::Shutdown", [self]() { self->ShutdownInternal(); }); mAbstractMainThread->Dispatch(r.forget()); } ChangeState(PLAY_STATE_SHUTDOWN); mVideoDecodingOberver->UnregisterEvent(); mVideoDecodingOberver = nullptr; mOwner = nullptr; } void MediaDecoder::NotifyXPCOMShutdown() { MOZ_ASSERT(NS_IsMainThread()); // NotifyXPCOMShutdown will clear its reference to mDecoder. So we must ensure // that this MediaDecoder stays alive until completion. RefPtr kungFuDeathGrip = this; if (auto* owner = GetOwner()) { owner->NotifyXPCOMShutdown(); } else if (!IsShutdown()) { Shutdown(); } MOZ_DIAGNOSTIC_ASSERT(IsShutdown()); } MediaDecoder::~MediaDecoder() { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(IsShutdown()); MediaMemoryTracker::RemoveMediaDecoder(this); } void MediaDecoder::OnPlaybackEvent(MediaPlaybackEvent&& aEvent) { switch (aEvent.mType) { case MediaPlaybackEvent::PlaybackEnded: PlaybackEnded(); break; case MediaPlaybackEvent::SeekStarted: SeekingStarted(); break; case MediaPlaybackEvent::Invalidate: Invalidate(); break; case MediaPlaybackEvent::EnterVideoSuspend: GetOwner()->DispatchAsyncEvent(u"mozentervideosuspend"_ns); mTelemetryProbesReporter->OnDecodeSuspended(); mIsVideoDecodingSuspended = true; break; case MediaPlaybackEvent::ExitVideoSuspend: GetOwner()->DispatchAsyncEvent(u"mozexitvideosuspend"_ns); mTelemetryProbesReporter->OnDecodeResumed(); mIsVideoDecodingSuspended = false; break; case MediaPlaybackEvent::StartVideoSuspendTimer: GetOwner()->DispatchAsyncEvent(u"mozstartvideosuspendtimer"_ns); break; case MediaPlaybackEvent::CancelVideoSuspendTimer: GetOwner()->DispatchAsyncEvent(u"mozcancelvideosuspendtimer"_ns); break; case MediaPlaybackEvent::VideoOnlySeekBegin: GetOwner()->DispatchAsyncEvent(u"mozvideoonlyseekbegin"_ns); break; case MediaPlaybackEvent::VideoOnlySeekCompleted: GetOwner()->DispatchAsyncEvent(u"mozvideoonlyseekcompleted"_ns); break; default: break; } } bool MediaDecoder::IsVideoDecodingSuspended() const { return mIsVideoDecodingSuspended; } void MediaDecoder::OnPlaybackErrorEvent(const MediaResult& aError) { MOZ_ASSERT(NS_IsMainThread()); #ifndef MOZ_WMF_MEDIA_ENGINE DecodeError(aError); #else if (aError != NS_ERROR_DOM_MEDIA_EXTERNAL_ENGINE_NOT_SUPPORTED_ERR && aError != NS_ERROR_DOM_MEDIA_CDM_PROXY_NOT_SUPPORTED_ERR) { DecodeError(aError); return; } // Already in shutting down decoder, no need to create another state machine. if (mPlayState == PLAY_STATE_SHUTDOWN) { return; } // External engine can't play the resource or we intentionally disable it, try // to use our own state machine again. Here we will create a new state machine // immediately and asynchrously shutdown the old one because we don't want to // dispatch any task to the old state machine. Therefore, we will disconnect // anything related with the old state machine, create a new state machine and // setup events/mirror/etc, then shutdown the old one and release its // reference once it finishes shutdown. RefPtr discardStateMachine = mDecoderStateMachine; // Disconnect mirror and events first. SetStateMachine(nullptr); DisconnectEvents(); // Recreate a state machine and shutdown the old one. bool needExternalEngine = false; if (aError == NS_ERROR_DOM_MEDIA_CDM_PROXY_NOT_SUPPORTED_ERR) { # ifdef MOZ_WMF_CDM if (aError.GetCDMProxy()->AsWMFCDMProxy()) { needExternalEngine = true; } # endif } LOG("Need to create a new %s state machine", needExternalEngine ? "external engine" : "normal"); nsresult rv = CreateAndInitStateMachine( false /* live stream */, !needExternalEngine /* disable external engine */); if (NS_WARN_IF(NS_FAILED(rv))) { LOG("Failed to create a new state machine!"); } // Some attributes might have been set on the destroyed state machine, and // won't be reflected on the new MDSM by the state mirroring. We need to // update them manually later, after MDSM finished reading the // metadata because the MDSM might not be ready to perform the operations yet. mPendingStatusUpdateForNewlyCreatedStateMachine = true; // If there is ongoing seek performed on the old MDSM, cancel it because we // will perform seeking later again and don't want the old seeking affecting // us. DiscardOngoingSeekIfExists(); discardStateMachine->BeginShutdown()->Then( AbstractThread::MainThread(), __func__, [discardStateMachine] {}); #endif } void MediaDecoder::OnDecoderDoctorEvent(DecoderDoctorEvent aEvent) { MOZ_ASSERT(NS_IsMainThread()); // OnDecoderDoctorEvent is disconnected at shutdown time. MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); Document* doc = GetOwner()->GetDocument(); if (!doc) { return; } DecoderDoctorDiagnostics diags; diags.StoreEvent(doc, aEvent, __func__); } static const char* NextFrameStatusToStr( MediaDecoderOwner::NextFrameStatus aStatus) { switch (aStatus) { case MediaDecoderOwner::NEXT_FRAME_AVAILABLE: return "NEXT_FRAME_AVAILABLE"; case MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE: return "NEXT_FRAME_UNAVAILABLE"; case MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING: return "NEXT_FRAME_UNAVAILABLE_BUFFERING"; case MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING: return "NEXT_FRAME_UNAVAILABLE_SEEKING"; case MediaDecoderOwner::NEXT_FRAME_UNINITIALIZED: return "NEXT_FRAME_UNINITIALIZED"; } return "UNKNOWN"; } void MediaDecoder::OnNextFrameStatus( MediaDecoderOwner::NextFrameStatus aStatus) { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); if (mNextFrameStatus != aStatus) { LOG("Changed mNextFrameStatus to %s", NextFrameStatusToStr(aStatus)); mNextFrameStatus = aStatus; UpdateReadyState(); } } void MediaDecoder::OnTrackInfoUpdated(const VideoInfo& aVideoInfo, const AudioInfo& aAudioInfo) { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); // Note that we don't check HasVideo() or HasAudio() here, because // those are checks for existing validity. If we always set the values // to what we receive, then we can go from not-video to video, for // example. mInfo->mVideo = aVideoInfo; mInfo->mAudio = aAudioInfo; Invalidate(); EnsureTelemetryReported(); } void MediaDecoder::OnSecondaryVideoContainerInstalled( const RefPtr& aSecondaryVideoContainer) { MOZ_ASSERT(NS_IsMainThread()); GetOwner()->OnSecondaryVideoContainerInstalled(aSecondaryVideoContainer); } void MediaDecoder::OnStoreDecoderBenchmark(const VideoInfo& aInfo) { MOZ_ASSERT(NS_IsMainThread()); int32_t videoFrameRate = aInfo.GetFrameRate().ref(); if (mFrameStats && videoFrameRate) { DecoderBenchmarkInfo benchmarkInfo{ aInfo.mMimeType, aInfo.mDisplay.width, aInfo.mDisplay.height, videoFrameRate, BitDepthForColorDepth(aInfo.mColorDepth), }; LOG("Store benchmark: Video width=%d, height=%d, frameRate=%d, content " "type = %s\n", benchmarkInfo.mWidth, benchmarkInfo.mHeight, benchmarkInfo.mFrameRate, benchmarkInfo.mContentType.BeginReading()); mDecoderBenchmark->Store(benchmarkInfo, mFrameStats); } } void MediaDecoder::ShutdownInternal() { MOZ_ASSERT(NS_IsMainThread()); mVideoFrameContainer = nullptr; mSecondaryVideoContainer = nullptr; MediaShutdownManager::Instance().Unregister(this); } void MediaDecoder::FinishShutdown() { MOZ_ASSERT(NS_IsMainThread()); SetStateMachine(nullptr); ShutdownInternal(); } nsresult MediaDecoder::CreateAndInitStateMachine(bool aIsLiveStream, bool aDisableExternalEngine) { MOZ_ASSERT(NS_IsMainThread()); SetStateMachine(CreateStateMachine(aDisableExternalEngine)); NS_ENSURE_TRUE(GetStateMachine(), NS_ERROR_FAILURE); GetStateMachine()->DispatchIsLiveStream(aIsLiveStream); nsresult rv = mDecoderStateMachine->Init(this); NS_ENSURE_SUCCESS(rv, rv); // If some parameters got set before the state machine got created, // set them now SetStateMachineParameters(); return NS_OK; } void MediaDecoder::SetStateMachineParameters() { MOZ_ASSERT(NS_IsMainThread()); if (mPlaybackRate != 1 && mPlaybackRate != 0) { mDecoderStateMachine->DispatchSetPlaybackRate(mPlaybackRate); } mTimedMetadataListener = mDecoderStateMachine->TimedMetadataEvent().Connect( mAbstractMainThread, this, &MediaDecoder::OnMetadataUpdate); mMetadataLoadedListener = mDecoderStateMachine->MetadataLoadedEvent().Connect( mAbstractMainThread, this, &MediaDecoder::MetadataLoaded); mFirstFrameLoadedListener = mDecoderStateMachine->FirstFrameLoadedEvent().Connect( mAbstractMainThread, this, &MediaDecoder::FirstFrameLoaded); mOnPlaybackEvent = mDecoderStateMachine->OnPlaybackEvent().Connect( mAbstractMainThread, this, &MediaDecoder::OnPlaybackEvent); mOnPlaybackErrorEvent = mDecoderStateMachine->OnPlaybackErrorEvent().Connect( mAbstractMainThread, this, &MediaDecoder::OnPlaybackErrorEvent); mOnDecoderDoctorEvent = mDecoderStateMachine->OnDecoderDoctorEvent().Connect( mAbstractMainThread, this, &MediaDecoder::OnDecoderDoctorEvent); mOnMediaNotSeekable = mDecoderStateMachine->OnMediaNotSeekable().Connect( mAbstractMainThread, this, &MediaDecoder::OnMediaNotSeekable); mOnNextFrameStatus = mDecoderStateMachine->OnNextFrameStatus().Connect( mAbstractMainThread, this, &MediaDecoder::OnNextFrameStatus); mOnTrackInfoUpdated = mDecoderStateMachine->OnTrackInfoUpdatedEvent().Connect( mAbstractMainThread, this, &MediaDecoder::OnTrackInfoUpdated); mOnSecondaryVideoContainerInstalled = mDecoderStateMachine->OnSecondaryVideoContainerInstalled().Connect( mAbstractMainThread, this, &MediaDecoder::OnSecondaryVideoContainerInstalled); mOnStoreDecoderBenchmark = mReader->OnStoreDecoderBenchmark().Connect( mAbstractMainThread, this, &MediaDecoder::OnStoreDecoderBenchmark); mOnEncrypted = mReader->OnEncrypted().Connect( mAbstractMainThread, GetOwner(), &MediaDecoderOwner::DispatchEncrypted); mOnWaitingForKey = mReader->OnWaitingForKey().Connect( mAbstractMainThread, GetOwner(), &MediaDecoderOwner::NotifyWaitingForKey); mOnDecodeWarning = mReader->OnDecodeWarning().Connect( mAbstractMainThread, GetOwner(), &MediaDecoderOwner::DecodeWarning); } void MediaDecoder::DisconnectEvents() { MOZ_ASSERT(NS_IsMainThread()); mTimedMetadataListener.Disconnect(); mMetadataLoadedListener.Disconnect(); mFirstFrameLoadedListener.Disconnect(); mOnPlaybackEvent.Disconnect(); mOnPlaybackErrorEvent.Disconnect(); mOnDecoderDoctorEvent.Disconnect(); mOnMediaNotSeekable.Disconnect(); mOnEncrypted.Disconnect(); mOnWaitingForKey.Disconnect(); mOnDecodeWarning.Disconnect(); mOnNextFrameStatus.Disconnect(); mOnTrackInfoUpdated.Disconnect(); mOnSecondaryVideoContainerInstalled.Disconnect(); mOnStoreDecoderBenchmark.Disconnect(); } RefPtr MediaDecoder::ShutdownStateMachine() { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(GetStateMachine()); DisconnectEvents(); return mDecoderStateMachine->BeginShutdown(); } void MediaDecoder::Play() { MOZ_ASSERT(NS_IsMainThread()); NS_ASSERTION(mDecoderStateMachine != nullptr, "Should have state machine."); LOG("Play"); if (mPlaybackRate == 0) { return; } if (IsEnded()) { Seek(0, SeekTarget::PrevSyncPoint); return; } if (mPlayState == PLAY_STATE_LOADING) { mNextState = PLAY_STATE_PLAYING; return; } ChangeState(PLAY_STATE_PLAYING); } void MediaDecoder::Seek(double aTime, SeekTarget::Type aSeekType) { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); MOZ_ASSERT(aTime >= 0.0, "Cannot seek to a negative value."); LOG("Seek"); auto time = TimeUnit::FromSeconds(aTime); mLogicalPosition = aTime; mLogicallySeeking = true; SeekTarget target = SeekTarget(time, aSeekType); CallSeek(target); if (mPlayState == PLAY_STATE_ENDED) { ChangeState(GetOwner()->GetPaused() ? PLAY_STATE_PAUSED : PLAY_STATE_PLAYING); } } void MediaDecoder::SetDelaySeekMode(bool aShouldDelaySeek) { MOZ_ASSERT(NS_IsMainThread()); LOG("SetDelaySeekMode, shouldDelaySeek=%d", aShouldDelaySeek); if (mShouldDelaySeek == aShouldDelaySeek) { return; } mShouldDelaySeek = aShouldDelaySeek; if (!mShouldDelaySeek && mDelayedSeekTarget) { Seek(mDelayedSeekTarget->GetTime().ToSeconds(), mDelayedSeekTarget->GetType()); mDelayedSeekTarget.reset(); } } void MediaDecoder::DiscardOngoingSeekIfExists() { MOZ_ASSERT(NS_IsMainThread()); mSeekRequest.DisconnectIfExists(); } void MediaDecoder::CallSeek(const SeekTarget& aTarget) { MOZ_ASSERT(NS_IsMainThread()); if (mShouldDelaySeek) { LOG("Delay seek to %f and store it to delayed seek target", mDelayedSeekTarget->GetTime().ToSeconds()); mDelayedSeekTarget = Some(aTarget); return; } DiscardOngoingSeekIfExists(); mDecoderStateMachine->InvokeSeek(aTarget) ->Then(mAbstractMainThread, __func__, this, &MediaDecoder::OnSeekResolved, &MediaDecoder::OnSeekRejected) ->Track(mSeekRequest); } double MediaDecoder::GetCurrentTime() { MOZ_ASSERT(NS_IsMainThread()); return mLogicalPosition; } void MediaDecoder::OnMetadataUpdate(TimedMetadata&& aMetadata) { MOZ_ASSERT(NS_IsMainThread()); MetadataLoaded(MakeUnique(*aMetadata.mInfo), UniquePtr(std::move(aMetadata.mTags)), MediaDecoderEventVisibility::Observable); FirstFrameLoaded(std::move(aMetadata.mInfo), MediaDecoderEventVisibility::Observable); } void MediaDecoder::MetadataLoaded( UniquePtr aInfo, UniquePtr aTags, MediaDecoderEventVisibility aEventVisibility) { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); LOG("MetadataLoaded, channels=%u rate=%u hasAudio=%d hasVideo=%d", aInfo->mAudio.mChannels, aInfo->mAudio.mRate, aInfo->HasAudio(), aInfo->HasVideo()); mMediaSeekable = aInfo->mMediaSeekable; mMediaSeekableOnlyInBufferedRanges = aInfo->mMediaSeekableOnlyInBufferedRanges; mInfo = std::move(aInfo); mTelemetryProbesReporter->OnMediaContentChanged( TelemetryProbesReporter::MediaInfoToMediaContent(*mInfo)); // Make sure the element and the frame (if any) are told about // our new size. if (aEventVisibility != MediaDecoderEventVisibility::Suppressed) { mFiredMetadataLoaded = true; GetOwner()->MetadataLoaded(mInfo.get(), std::move(aTags)); } // Invalidate() will end up calling GetOwner()->UpdateMediaSize with the last // dimensions retrieved from the video frame container. The video frame // container contains more up to date dimensions than aInfo. // So we call Invalidate() after calling GetOwner()->MetadataLoaded to ensure // the media element has the latest dimensions. Invalidate(); #ifdef MOZ_WMF_MEDIA_ENGINE if (mPendingStatusUpdateForNewlyCreatedStateMachine) { mPendingStatusUpdateForNewlyCreatedStateMachine = false; LOG("Set pending statuses if necessary (mLogicallySeeking=%d, " "mLogicalPosition=%f, mPlaybackRate=%f)", mLogicallySeeking.Ref(), mLogicalPosition, mPlaybackRate); if (mLogicalPosition != 0) { Seek(mLogicalPosition, SeekTarget::Accurate); } if (mPlaybackRate != 0 && mPlaybackRate != 1.0) { mDecoderStateMachine->DispatchSetPlaybackRate(mPlaybackRate); } } #endif EnsureTelemetryReported(); } void MediaDecoder::EnsureTelemetryReported() { MOZ_ASSERT(NS_IsMainThread()); if (mTelemetryReported || !mInfo) { // Note: sometimes we get multiple MetadataLoaded calls (for example // for chained ogg). So we ensure we don't report duplicate results for // these resources. return; } nsTArray codecs; if (mInfo->HasAudio() && !mInfo->mAudio.GetAsAudioInfo()->mMimeType.IsEmpty()) { codecs.AppendElement(mInfo->mAudio.GetAsAudioInfo()->mMimeType); } if (mInfo->HasVideo() && !mInfo->mVideo.GetAsVideoInfo()->mMimeType.IsEmpty()) { codecs.AppendElement(mInfo->mVideo.GetAsVideoInfo()->mMimeType); } if (codecs.IsEmpty()) { codecs.AppendElement(nsPrintfCString( "resource; %s", ContainerType().OriginalString().Data())); } for (const nsCString& codec : codecs) { LOG("Telemetry MEDIA_CODEC_USED= '%s'", codec.get()); Telemetry::Accumulate(Telemetry::HistogramID::MEDIA_CODEC_USED, codec); } mTelemetryReported = true; } const char* MediaDecoder::PlayStateStr() { MOZ_ASSERT(NS_IsMainThread()); return ToPlayStateStr(mPlayState); } void MediaDecoder::FirstFrameLoaded( UniquePtr aInfo, MediaDecoderEventVisibility aEventVisibility) { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); LOG("FirstFrameLoaded, channels=%u rate=%u hasAudio=%d hasVideo=%d " "mPlayState=%s transportSeekable=%d", aInfo->mAudio.mChannels, aInfo->mAudio.mRate, aInfo->HasAudio(), aInfo->HasVideo(), PlayStateStr(), IsTransportSeekable()); mInfo = std::move(aInfo); mTelemetryProbesReporter->OnMediaContentChanged( TelemetryProbesReporter::MediaInfoToMediaContent(*mInfo)); Invalidate(); // The element can run javascript via events // before reaching here, so only change the // state if we're still set to the original // loading state. if (mPlayState == PLAY_STATE_LOADING) { ChangeState(mNextState); } // GetOwner()->FirstFrameLoaded() might call us back. Put it at the bottom of // this function to avoid unexpected shutdown from reentrant calls. if (aEventVisibility != MediaDecoderEventVisibility::Suppressed) { GetOwner()->FirstFrameLoaded(); } } void MediaDecoder::NetworkError(const MediaResult& aError) { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); GetOwner()->NetworkError(aError); } void MediaDecoder::DecodeError(const MediaResult& aError) { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); GetOwner()->DecodeError(aError); } void MediaDecoder::UpdateSameOriginStatus(bool aSameOrigin) { MOZ_ASSERT(NS_IsMainThread()); mSameOriginMedia = aSameOrigin; } bool MediaDecoder::IsSeeking() const { MOZ_ASSERT(NS_IsMainThread()); return mLogicallySeeking; } bool MediaDecoder::OwnerHasError() const { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); return GetOwner()->HasError(); } bool MediaDecoder::IsEnded() const { MOZ_ASSERT(NS_IsMainThread()); return mPlayState == PLAY_STATE_ENDED; } bool MediaDecoder::IsShutdown() const { MOZ_ASSERT(NS_IsMainThread()); return mPlayState == PLAY_STATE_SHUTDOWN; } void MediaDecoder::PlaybackEnded() { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); if (mLogicallySeeking || mPlayState == PLAY_STATE_LOADING || mPlayState == PLAY_STATE_ENDED) { LOG("MediaDecoder::PlaybackEnded bailed out, " "mLogicallySeeking=%d mPlayState=%s", mLogicallySeeking.Ref(), ToPlayStateStr(mPlayState)); return; } LOG("MediaDecoder::PlaybackEnded"); ChangeState(PLAY_STATE_ENDED); InvalidateWithFlags(VideoFrameContainer::INVALIDATE_FORCE); GetOwner()->PlaybackEnded(); } void MediaDecoder::NotifyPrincipalChanged() { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); GetOwner()->NotifyDecoderPrincipalChanged(); } void MediaDecoder::OnSeekResolved() { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); LOG("MediaDecoder::OnSeekResolved"); mLogicallySeeking = false; // Ensure logical position is updated after seek. UpdateLogicalPositionInternal(); mSeekRequest.Complete(); GetOwner()->SeekCompleted(); } void MediaDecoder::OnSeekRejected() { MOZ_ASSERT(NS_IsMainThread()); LOG("MediaDecoder::OnSeekRejected"); mSeekRequest.Complete(); mLogicallySeeking = false; GetOwner()->SeekAborted(); } void MediaDecoder::SeekingStarted() { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); GetOwner()->SeekStarted(); } void MediaDecoder::ChangeState(PlayState aState) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(!IsShutdown(), "SHUTDOWN is the final state."); if (mNextState == aState) { mNextState = PLAY_STATE_PAUSED; } if (mPlayState != aState) { DDLOG(DDLogCategory::Property, "play_state", ToPlayStateStr(aState)); LOG("Play state changes from %s to %s", ToPlayStateStr(mPlayState), ToPlayStateStr(aState)); mPlayState = aState; UpdateTelemetryHelperBasedOnPlayState(aState); } } TelemetryProbesReporter::Visibility MediaDecoder::OwnerVisibility() const { return GetOwner()->IsActuallyInvisible() || mForcedHidden ? TelemetryProbesReporter::Visibility::eInvisible : TelemetryProbesReporter::Visibility::eVisible; } void MediaDecoder::UpdateTelemetryHelperBasedOnPlayState( PlayState aState) const { if (aState == PlayState::PLAY_STATE_PLAYING) { mTelemetryProbesReporter->OnPlay( OwnerVisibility(), TelemetryProbesReporter::MediaInfoToMediaContent(*mInfo), mVolume == 0.f); } else if (aState == PlayState::PLAY_STATE_PAUSED || aState == PlayState::PLAY_STATE_ENDED) { mTelemetryProbesReporter->OnPause(OwnerVisibility()); } else if (aState == PLAY_STATE_SHUTDOWN) { mTelemetryProbesReporter->OnShutdown(); } } MediaDecoder::PositionUpdate MediaDecoder::GetPositionUpdateReason( double aPrevPos, const TimeUnit& aCurPos) const { MOZ_ASSERT(NS_IsMainThread()); // If current position is earlier than previous position and we didn't do // seek, that means we looped back to the start position. const bool notSeeking = !mSeekRequest.Exists(); if (mLooping && notSeeking && aCurPos.ToSeconds() < aPrevPos) { return PositionUpdate::eSeamlessLoopingSeeking; } return aPrevPos != aCurPos.ToSeconds() && notSeeking ? PositionUpdate::ePeriodicUpdate : PositionUpdate::eOther; } void MediaDecoder::UpdateLogicalPositionInternal() { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); TimeUnit currentPosition = CurrentPosition(); if (mPlayState == PLAY_STATE_ENDED) { currentPosition = std::max(currentPosition, mDuration.match(DurationToTimeUnit())); } const PositionUpdate reason = GetPositionUpdateReason(mLogicalPosition, currentPosition); switch (reason) { case PositionUpdate::ePeriodicUpdate: SetLogicalPosition(currentPosition); // This is actually defined in `TimeMarchesOn`, but we do that in decoder. // https://html.spec.whatwg.org/multipage/media.html#playing-the-media-resource:event-media-timeupdate-7 // TODO (bug 1688137): should we move it back to `TimeMarchesOn`? GetOwner()->MaybeQueueTimeupdateEvent(); break; case PositionUpdate::eSeamlessLoopingSeeking: // When seamless seeking occurs, seeking was performed on the demuxer so // the decoder doesn't know. That means decoder still thinks it's in // playing. Therefore, we have to manually call those methods to notify // the owner about seeking. GetOwner()->SeekStarted(); SetLogicalPosition(currentPosition); GetOwner()->SeekCompleted(); break; default: MOZ_ASSERT(reason == PositionUpdate::eOther); SetLogicalPosition(currentPosition); break; } // Invalidate the frame so any video data is displayed. // Do this before the timeupdate event so that if that // event runs JavaScript that queries the media size, the // frame has reflowed and the size updated beforehand. Invalidate(); } void MediaDecoder::SetLogicalPosition(const TimeUnit& aNewPosition) { MOZ_ASSERT(NS_IsMainThread()); if (TimeUnit::FromSeconds(mLogicalPosition) == aNewPosition || mLogicalPosition == aNewPosition.ToSeconds()) { return; } mLogicalPosition = aNewPosition.ToSeconds(); DDLOG(DDLogCategory::Property, "currentTime", mLogicalPosition); } void MediaDecoder::DurationChanged() { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); Variant oldDuration = mDuration; // Use the explicit duration if we have one. // Otherwise use the duration mirrored from MDSM. if (mExplicitDuration.isSome()) { mDuration.emplace(mExplicitDuration.ref()); } else if (mStateMachineDuration.Ref().isSome()) { MOZ_ASSERT(mStateMachineDuration.Ref().ref().IsValid()); mDuration.emplace(mStateMachineDuration.Ref().ref()); } LOG("New duration: %s", mDuration.match(DurationToTimeUnit()).ToString().get()); if (oldDuration.is() && oldDuration.as().IsValid()) { LOG("Old Duration %s", oldDuration.match(DurationToTimeUnit()).ToString().get()); } if ((oldDuration.is() || oldDuration.as().IsValid())) { if (mDuration.match(DurationToDouble()) == oldDuration.match(DurationToDouble())) { return; } } LOG("Duration changed to %s", mDuration.match(DurationToTimeUnit()).ToString().get()); // See https://www.w3.org/Bugs/Public/show_bug.cgi?id=28822 for a discussion // of whether we should fire durationchange on explicit infinity. if (mFiredMetadataLoaded && (!std::isinf(mDuration.match(DurationToDouble())) || mExplicitDuration.isSome())) { GetOwner()->DispatchAsyncEvent(u"durationchange"_ns); } if (CurrentPosition().ToSeconds() > mDuration.match(DurationToDouble())) { Seek(mDuration.match(DurationToDouble()), SeekTarget::Accurate); } } already_AddRefed MediaDecoder::GetCompositor() { MediaDecoderOwner* owner = GetOwner(); Document* ownerDoc = owner ? owner->GetDocument() : nullptr; WindowRenderer* renderer = ownerDoc ? nsContentUtils::WindowRendererForDocument(ownerDoc) : nullptr; RefPtr knows = renderer ? renderer->AsKnowsCompositor() : nullptr; return knows ? knows->GetForMedia().forget() : nullptr; } void MediaDecoder::NotifyCompositor() { RefPtr knowsCompositor = GetCompositor(); if (knowsCompositor) { nsCOMPtr r = NewRunnableMethod&&>( "MediaFormatReader::UpdateCompositor", mReader, &MediaFormatReader::UpdateCompositor, knowsCompositor.forget()); Unused << mReader->OwnerThread()->Dispatch(r.forget()); } } void MediaDecoder::SetElementVisibility(bool aIsOwnerInvisible, bool aIsOwnerConnected) { MOZ_ASSERT(NS_IsMainThread()); mIsOwnerInvisible = aIsOwnerInvisible; mIsOwnerConnected = aIsOwnerConnected; mTelemetryProbesReporter->OnVisibilityChanged(OwnerVisibility()); UpdateVideoDecodeMode(); } void MediaDecoder::SetForcedHidden(bool aForcedHidden) { MOZ_ASSERT(NS_IsMainThread()); mForcedHidden = aForcedHidden; mTelemetryProbesReporter->OnVisibilityChanged(OwnerVisibility()); UpdateVideoDecodeMode(); } void MediaDecoder::SetSuspendTaint(bool aTainted) { MOZ_ASSERT(NS_IsMainThread()); mHasSuspendTaint = aTainted; UpdateVideoDecodeMode(); } void MediaDecoder::UpdateVideoDecodeMode() { MOZ_ASSERT(NS_IsMainThread()); // The MDSM may yet be set. if (!mDecoderStateMachine) { LOG("UpdateVideoDecodeMode(), early return because we don't have MDSM."); return; } // Seeking is required when leaving suspend mode. if (!mMediaSeekable) { LOG("UpdateVideoDecodeMode(), set Normal because the media is not " "seekable"); mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal); return; } // If mHasSuspendTaint is set, never suspend the video decoder. if (mHasSuspendTaint) { LOG("UpdateVideoDecodeMode(), set Normal because the element has been " "tainted."); mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal); return; } // If mSecondaryVideoContainer is set, never suspend the video decoder. if (mSecondaryVideoContainer.Ref()) { LOG("UpdateVideoDecodeMode(), set Normal because the element is cloning " "itself visually to another video container."); mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal); return; } // Don't suspend elements that is not in a connected tree. if (!mIsOwnerConnected) { LOG("UpdateVideoDecodeMode(), set Normal because the element is not in " "tree."); mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal); return; } // If mForcedHidden is set, suspend the video decoder anyway. if (mForcedHidden) { LOG("UpdateVideoDecodeMode(), set Suspend because the element is forced to " "be suspended."); mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Suspend); return; } // Resume decoding in the advance, even the element is in the background. if (mIsBackgroundVideoDecodingAllowed) { LOG("UpdateVideoDecodeMode(), set Normal because the tab is in background " "and hovered."); mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal); return; } if (mIsOwnerInvisible) { LOG("UpdateVideoDecodeMode(), set Suspend because of invisible element."); mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Suspend); } else { LOG("UpdateVideoDecodeMode(), set Normal because of visible element."); mDecoderStateMachine->SetVideoDecodeMode(VideoDecodeMode::Normal); } } void MediaDecoder::SetIsBackgroundVideoDecodingAllowed(bool aAllowed) { mIsBackgroundVideoDecodingAllowed = aAllowed; UpdateVideoDecodeMode(); } bool MediaDecoder::HasSuspendTaint() const { MOZ_ASSERT(NS_IsMainThread()); return mHasSuspendTaint; } void MediaDecoder::SetSecondaryVideoContainer( const RefPtr& aSecondaryVideoContainer) { MOZ_ASSERT(NS_IsMainThread()); if (mSecondaryVideoContainer.Ref() == aSecondaryVideoContainer) { return; } mSecondaryVideoContainer = aSecondaryVideoContainer; UpdateVideoDecodeMode(); } bool MediaDecoder::IsMediaSeekable() { MOZ_ASSERT(NS_IsMainThread()); NS_ENSURE_TRUE(GetStateMachine(), false); return mMediaSeekable; } namespace { // Returns zero, either as a TimeUnit or as a double. template constexpr T Zero() { if constexpr (std::is_same::value) { return 0.0; } else if constexpr (std::is_same::value) { return TimeUnit::Zero(); } MOZ_RELEASE_ASSERT(false); }; // Returns Infinity either as a TimeUnit or as a double. template constexpr T Infinity() { if constexpr (std::is_same::value) { return std::numeric_limits::infinity(); } else if constexpr (std::is_same::value) { return TimeUnit::FromInfinity(); } MOZ_RELEASE_ASSERT(false); }; }; // namespace // This method can be made to return either TimeIntervals, that is a set of // interval that are delimited with TimeUnit, or TimeRanges, that is a set of // intervals that are delimited by seconds, as doubles. // seekable often depends on the duration of a media, in the very common case // where the seekable range is [0, duration]. When playing a MediaSource, the // duration of a media element can be set as an arbitrary number, that are // 64-bits floating point values. // This allows returning an interval that is [0, duration], with duration being // a double that cannot be represented as a TimeUnit, either because it has too // many significant digits, or because it's outside of the int64_t range that // TimeUnit internally uses. template IntervalType MediaDecoder::GetSeekableImpl() { MOZ_ASSERT(NS_IsMainThread()); if (std::isnan(GetDuration())) { // We do not have a duration yet, we can't determine the seekable range. return IntervalType(); } // Compute [0, duration] -- When dealing with doubles, use ::GetDuration to // avoid rounding the value differently. When dealing with TimeUnit, it's // returned directly. typename IntervalType::InnerType duration; if constexpr (std::is_same::value) { duration = GetDuration(); } else { duration = mDuration.as(); } typename IntervalType::ElemType zeroToDuration = typename IntervalType::ElemType( Zero(), IsInfinite() ? Infinity() : duration); auto buffered = IntervalType(GetBuffered()); // Remove any negative range in the interval -- seeking to a non-positive // position isn't possible. auto positiveBuffered = buffered.Intersection(zeroToDuration); // We can seek in buffered range if the media is seekable. Also, we can seek // in unbuffered ranges if the transport level is seekable (local file or the // server supports range requests, etc.) or in cue-less WebMs if (mMediaSeekableOnlyInBufferedRanges) { return IntervalType(positiveBuffered); } if (!IsMediaSeekable()) { return IntervalType(); } if (!IsTransportSeekable()) { return IntervalType(positiveBuffered); } // Common case: seeking is possible at any point of the stream. return IntervalType(zeroToDuration); } media::TimeIntervals MediaDecoder::GetSeekable() { return GetSeekableImpl(); } media::TimeRanges MediaDecoder::GetSeekableTimeRanges() { return GetSeekableImpl(); } void MediaDecoder::SetFragmentEndTime(double aTime) { MOZ_ASSERT(NS_IsMainThread()); if (mDecoderStateMachine) { mDecoderStateMachine->DispatchSetFragmentEndTime( TimeUnit::FromSeconds(aTime)); } } void MediaDecoder::SetPlaybackRate(double aPlaybackRate) { MOZ_ASSERT(NS_IsMainThread()); double oldRate = mPlaybackRate; mPlaybackRate = aPlaybackRate; if (aPlaybackRate == 0) { Pause(); return; } if (oldRate == 0 && !GetOwner()->GetPaused()) { // PlaybackRate is no longer null. // Restart the playback if the media was playing. Play(); } if (mDecoderStateMachine) { mDecoderStateMachine->DispatchSetPlaybackRate(aPlaybackRate); } } void MediaDecoder::SetPreservesPitch(bool aPreservesPitch) { MOZ_ASSERT(NS_IsMainThread()); mPreservesPitch = aPreservesPitch; } void MediaDecoder::SetLooping(bool aLooping) { MOZ_ASSERT(NS_IsMainThread()); mLooping = aLooping; } void MediaDecoder::SetStreamName(const nsAutoString& aStreamName) { MOZ_ASSERT(NS_IsMainThread()); mStreamName = aStreamName; } void MediaDecoder::ConnectMirrors(MediaDecoderStateMachineBase* aObject) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(aObject); mStateMachineDuration.Connect(aObject->CanonicalDuration()); mBuffered.Connect(aObject->CanonicalBuffered()); mCurrentPosition.Connect(aObject->CanonicalCurrentPosition()); mIsAudioDataAudible.Connect(aObject->CanonicalIsAudioDataAudible()); } void MediaDecoder::DisconnectMirrors() { MOZ_ASSERT(NS_IsMainThread()); mStateMachineDuration.DisconnectIfConnected(); mBuffered.DisconnectIfConnected(); mCurrentPosition.DisconnectIfConnected(); mIsAudioDataAudible.DisconnectIfConnected(); } void MediaDecoder::SetStateMachine( MediaDecoderStateMachineBase* aStateMachine) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT_IF(aStateMachine, !mDecoderStateMachine); if (aStateMachine) { mDecoderStateMachine = aStateMachine; LOG("set state machine %p", mDecoderStateMachine.get()); ConnectMirrors(aStateMachine); UpdateVideoDecodeMode(); } else if (mDecoderStateMachine) { LOG("null out state machine %p", mDecoderStateMachine.get()); mDecoderStateMachine = nullptr; DisconnectMirrors(); } } ImageContainer* MediaDecoder::GetImageContainer() { return mVideoFrameContainer ? mVideoFrameContainer->GetImageContainer() : nullptr; } void MediaDecoder::InvalidateWithFlags(uint32_t aFlags) { if (mVideoFrameContainer) { mVideoFrameContainer->InvalidateWithFlags(aFlags); } } void MediaDecoder::Invalidate() { if (mVideoFrameContainer) { mVideoFrameContainer->Invalidate(); } } void MediaDecoder::Suspend() { MOZ_ASSERT(NS_IsMainThread()); GetStateMachine()->InvokeSuspendMediaSink(); } void MediaDecoder::Resume() { MOZ_ASSERT(NS_IsMainThread()); GetStateMachine()->InvokeResumeMediaSink(); } // Constructs the time ranges representing what segments of the media // are buffered and playable. media::TimeIntervals MediaDecoder::GetBuffered() { MOZ_ASSERT(NS_IsMainThread()); return mBuffered.Ref(); } size_t MediaDecoder::SizeOfVideoQueue() { MOZ_ASSERT(NS_IsMainThread()); if (mDecoderStateMachine) { return mDecoderStateMachine->SizeOfVideoQueue(); } return 0; } size_t MediaDecoder::SizeOfAudioQueue() { MOZ_ASSERT(NS_IsMainThread()); if (mDecoderStateMachine) { return mDecoderStateMachine->SizeOfAudioQueue(); } return 0; } void MediaDecoder::NotifyReaderDataArrived() { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); nsresult rv = mReader->OwnerThread()->Dispatch( NewRunnableMethod("MediaFormatReader::NotifyDataArrived", mReader.get(), &MediaFormatReader::NotifyDataArrived)); MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); Unused << rv; } // Provide access to the state machine object MediaDecoderStateMachineBase* MediaDecoder::GetStateMachine() const { MOZ_ASSERT(NS_IsMainThread()); return mDecoderStateMachine; } bool MediaDecoder::CanPlayThrough() { MOZ_ASSERT(NS_IsMainThread()); MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); return CanPlayThroughImpl(); } RefPtr MediaDecoder::SetCDMProxy(CDMProxy* aProxy) { MOZ_ASSERT(NS_IsMainThread()); #ifdef MOZ_WMF_MEDIA_ENGINE // Switch to another state machine if the current one doesn't support the // given CDM proxy. if (aProxy && !GetStateMachine()->IsCDMProxySupported(aProxy)) { LOG("CDM proxy not supported! Switch to another state machine."); OnPlaybackErrorEvent( MediaResult{NS_ERROR_DOM_MEDIA_CDM_PROXY_NOT_SUPPORTED_ERR, aProxy}); } #endif MOZ_DIAGNOSTIC_ASSERT_IF(aProxy, GetStateMachine()->IsCDMProxySupported(aProxy)); return GetStateMachine()->SetCDMProxy(aProxy); } bool MediaDecoder::IsOpusEnabled() { return StaticPrefs::media_opus_enabled(); } bool MediaDecoder::IsOggEnabled() { return StaticPrefs::media_ogg_enabled(); } bool MediaDecoder::IsWaveEnabled() { return StaticPrefs::media_wave_enabled(); } bool MediaDecoder::IsWebMEnabled() { return StaticPrefs::media_webm_enabled(); } NS_IMETHODIMP MediaMemoryTracker::CollectReports(nsIHandleReportCallback* aHandleReport, nsISupports* aData, bool aAnonymize) { // NB: When resourceSizes' ref count goes to 0 the promise will report the // resources memory and finish the asynchronous memory report. RefPtr resourceSizes = new MediaDecoder::ResourceSizes(MediaMemoryTracker::MallocSizeOf); nsCOMPtr handleReport = aHandleReport; nsCOMPtr data = aData; resourceSizes->Promise()->Then( AbstractThread::MainThread(), __func__, [handleReport, data](size_t size) { handleReport->Callback( ""_ns, "explicit/media/resources"_ns, KIND_HEAP, UNITS_BYTES, static_cast(size), nsLiteralCString("Memory used by media resources including " "streaming buffers, caches, etc."), data); nsCOMPtr imgr = do_GetService("@mozilla.org/memory-reporter-manager;1"); if (imgr) { imgr->EndReport(); } }, [](size_t) { /* unused reject function */ }); int64_t video = 0; int64_t audio = 0; DecodersArray& decoders = Decoders(); for (size_t i = 0; i < decoders.Length(); ++i) { MediaDecoder* decoder = decoders[i]; video += static_cast(decoder->SizeOfVideoQueue()); audio += static_cast(decoder->SizeOfAudioQueue()); decoder->AddSizeOfResources(resourceSizes); } MOZ_COLLECT_REPORT("explicit/media/decoded/video", KIND_HEAP, UNITS_BYTES, video, "Memory used by decoded video frames."); MOZ_COLLECT_REPORT("explicit/media/decoded/audio", KIND_HEAP, UNITS_BYTES, audio, "Memory used by decoded audio chunks."); return NS_OK; } MediaDecoderOwner* MediaDecoder::GetOwner() const { MOZ_ASSERT(NS_IsMainThread()); // mOwner is valid until shutdown. return mOwner; } MediaDecoderOwner::NextFrameStatus MediaDecoder::NextFrameBufferedStatus() { MOZ_ASSERT(NS_IsMainThread()); // Next frame hasn't been decoded yet. // Use the buffered range to consider if we have the next frame available. auto currentPosition = CurrentPosition(); media::TimeInterval interval( currentPosition, currentPosition + DEFAULT_NEXT_FRAME_AVAILABLE_BUFFERED); return GetBuffered().Contains(interval) ? MediaDecoderOwner::NEXT_FRAME_AVAILABLE : MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE; } void MediaDecoder::GetDebugInfo(dom::MediaDecoderDebugInfo& aInfo) { MOZ_ASSERT(NS_IsMainThread()); CopyUTF8toUTF16(nsPrintfCString("%p", this), aInfo.mInstance); aInfo.mChannels = mInfo ? mInfo->mAudio.mChannels : 0; aInfo.mRate = mInfo ? mInfo->mAudio.mRate : 0; aInfo.mHasAudio = mInfo ? mInfo->HasAudio() : false; aInfo.mHasVideo = mInfo ? mInfo->HasVideo() : false; CopyUTF8toUTF16(MakeStringSpan(PlayStateStr()), aInfo.mPlayState); aInfo.mContainerType = NS_ConvertUTF8toUTF16(ContainerType().Type().AsString()); } RefPtr MediaDecoder::RequestDebugInfo( MediaDecoderDebugInfo& aInfo) { MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); if (!NS_IsMainThread()) { // Run the request on the main thread if it's not already. return InvokeAsync(AbstractThread::MainThread(), __func__, [this, self = RefPtr{this}, &aInfo]() { return RequestDebugInfo(aInfo); }); } GetDebugInfo(aInfo); return mReader->RequestDebugInfo(aInfo.mReader) ->Then(AbstractThread::MainThread(), __func__, [this, self = RefPtr{this}, &aInfo] { if (!GetStateMachine()) { return GenericPromise::CreateAndResolve(true, __func__); } return GetStateMachine()->RequestDebugInfo(aInfo.mStateMachine); }); } void MediaDecoder::NotifyAudibleStateChanged() { MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); GetOwner()->SetAudibleState(mIsAudioDataAudible); mTelemetryProbesReporter->OnAudibleChanged( mIsAudioDataAudible ? TelemetryProbesReporter::AudibleState::eAudible : TelemetryProbesReporter::AudibleState::eNotAudible); } void MediaDecoder::NotifyVolumeChanged() { MOZ_DIAGNOSTIC_ASSERT(!IsShutdown()); mTelemetryProbesReporter->OnMutedChanged(mVolume == 0.f); } double MediaDecoder::GetTotalVideoPlayTimeInSeconds() const { return mTelemetryProbesReporter->GetTotalVideoPlayTimeInSeconds(); } double MediaDecoder::GetTotalVideoHDRPlayTimeInSeconds() const { return mTelemetryProbesReporter->GetTotalVideoHDRPlayTimeInSeconds(); } double MediaDecoder::GetVisibleVideoPlayTimeInSeconds() const { return mTelemetryProbesReporter->GetVisibleVideoPlayTimeInSeconds(); } double MediaDecoder::GetInvisibleVideoPlayTimeInSeconds() const { return mTelemetryProbesReporter->GetInvisibleVideoPlayTimeInSeconds(); } double MediaDecoder::GetVideoDecodeSuspendedTimeInSeconds() const { return mTelemetryProbesReporter->GetVideoDecodeSuspendedTimeInSeconds(); } double MediaDecoder::GetTotalAudioPlayTimeInSeconds() const { return mTelemetryProbesReporter->GetTotalAudioPlayTimeInSeconds(); } double MediaDecoder::GetAudiblePlayTimeInSeconds() const { return mTelemetryProbesReporter->GetAudiblePlayTimeInSeconds(); } double MediaDecoder::GetInaudiblePlayTimeInSeconds() const { return mTelemetryProbesReporter->GetInaudiblePlayTimeInSeconds(); } double MediaDecoder::GetMutedPlayTimeInSeconds() const { return mTelemetryProbesReporter->GetMutedPlayTimeInSeconds(); } MediaMemoryTracker::MediaMemoryTracker() = default; void MediaMemoryTracker::InitMemoryReporter() { RegisterWeakAsyncMemoryReporter(this); } MediaMemoryTracker::~MediaMemoryTracker() { UnregisterWeakMemoryReporter(this); } } // namespace mozilla // avoid redefined macro in unified build #undef DUMP #undef LOG #undef NS_DispatchToMainThread