/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim:set ts=2 sw=2 sts=2 et cindent: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include #include #include #include "mediasink/AudioSink.h" #include "mediasink/AudioSinkWrapper.h" #include "mediasink/DecodedStream.h" #include "mediasink/VideoSink.h" #include "mozilla/Logging.h" #include "mozilla/MathAlgorithms.h" #include "mozilla/NotNull.h" #include "mozilla/Preferences.h" #include "mozilla/ProfilerLabels.h" #include "mozilla/ProfilerMarkers.h" #include "mozilla/ProfilerMarkerTypes.h" #include "mozilla/SharedThreadPool.h" #include "mozilla/Sprintf.h" #include "mozilla/StaticPrefs_media.h" #include "mozilla/Telemetry.h" #include "mozilla/TaskQueue.h" #include "nsIMemoryReporter.h" #include "nsPrintfCString.h" #include "nsTArray.h" #include "AudioSegment.h" #include "DOMMediaStream.h" #include "ImageContainer.h" #include "MediaDecoder.h" #include "MediaDecoderStateMachine.h" #include "MediaShutdownManager.h" #include "MediaTrackGraph.h" #include "MediaTimer.h" #include "PerformanceRecorder.h" #include "ReaderProxy.h" #include "TimeUnits.h" #include "VideoSegment.h" #include "VideoUtils.h" namespace mozilla { using namespace mozilla::media; #define NS_DispatchToMainThread(...) \ CompileError_UseAbstractThreadDispatchInstead // avoid redefined macro in unified build #undef FMT #undef LOG #undef LOGV #undef LOGW #undef LOGE #undef SFMT #undef SLOG #undef SLOGW #undef SLOGE #define FMT(x, ...) "Decoder=%p " x, mDecoderID, ##__VA_ARGS__ #define LOG(x, ...) \ DDMOZ_LOG(gMediaDecoderLog, LogLevel::Debug, "Decoder=%p " x, mDecoderID, \ ##__VA_ARGS__) #define LOGV(x, ...) \ DDMOZ_LOG(gMediaDecoderLog, LogLevel::Verbose, "Decoder=%p " x, mDecoderID, \ ##__VA_ARGS__) #define LOGW(x, ...) NS_WARNING(nsPrintfCString(FMT(x, ##__VA_ARGS__)).get()) #define LOGE(x, ...) \ NS_DebugBreak(NS_DEBUG_WARNING, \ nsPrintfCString(FMT(x, ##__VA_ARGS__)).get(), nullptr, \ __FILE__, __LINE__) // Used by StateObject and its sub-classes #define SFMT(x, ...) \ "Decoder=%p state=%s " x, mMaster->mDecoderID, ToStateStr(GetState()), \ ##__VA_ARGS__ #define SLOG(x, ...) \ DDMOZ_LOGEX(mMaster, gMediaDecoderLog, LogLevel::Debug, "state=%s " x, \ ToStateStr(GetState()), ##__VA_ARGS__) #define SLOGW(x, ...) NS_WARNING(nsPrintfCString(SFMT(x, ##__VA_ARGS__)).get()) #define SLOGE(x, ...) \ NS_DebugBreak(NS_DEBUG_WARNING, \ nsPrintfCString(SFMT(x, ##__VA_ARGS__)).get(), nullptr, \ __FILE__, __LINE__) // Certain constants get stored as member variables and then adjusted by various // scale factors on a per-decoder basis. We want to make sure to avoid using // these constants directly, so we put them in a namespace. namespace detail { // Resume a suspended video decoder to the current playback position plus this // time premium for compensating the seeking delay. static constexpr auto RESUME_VIDEO_PREMIUM = TimeUnit::FromMicroseconds(125000); static const int64_t AMPLE_AUDIO_USECS = 2000000; // If more than this much decoded audio is queued, we'll hold off // decoding more audio. static constexpr auto AMPLE_AUDIO_THRESHOLD = TimeUnit::FromMicroseconds(AMPLE_AUDIO_USECS); } // namespace detail // If we have fewer than LOW_VIDEO_FRAMES decoded frames, and // we're not "prerolling video", we'll skip the video up to the next keyframe // which is at or after the current playback position. static const uint32_t LOW_VIDEO_FRAMES = 2; // Arbitrary "frame duration" when playing only audio. static const uint32_t AUDIO_DURATION_USECS = 40000; namespace detail { // If we have less than this much buffered data available, we'll consider // ourselves to be running low on buffered data. We determine how much // buffered data we have remaining using the reader's GetBuffered() // implementation. static const int64_t LOW_BUFFER_THRESHOLD_USECS = 5000000; static constexpr auto LOW_BUFFER_THRESHOLD = TimeUnit::FromMicroseconds(LOW_BUFFER_THRESHOLD_USECS); // LOW_BUFFER_THRESHOLD_USECS needs to be greater than AMPLE_AUDIO_USECS, // otherwise the skip-to-keyframe logic can activate when we're running low on // data. static_assert(LOW_BUFFER_THRESHOLD_USECS > AMPLE_AUDIO_USECS, "LOW_BUFFER_THRESHOLD_USECS is too small"); } // namespace detail // Amount of excess data to add in to the "should we buffer" calculation. static constexpr auto EXHAUSTED_DATA_MARGIN = TimeUnit::FromMicroseconds(100000); static const uint32_t MIN_VIDEO_QUEUE_SIZE = 3; static const uint32_t MAX_VIDEO_QUEUE_SIZE = 10; #ifdef MOZ_APPLEMEDIA static const uint32_t HW_VIDEO_QUEUE_SIZE = 10; #else static const uint32_t HW_VIDEO_QUEUE_SIZE = 3; #endif static const uint32_t VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE = 9999; static uint32_t sVideoQueueDefaultSize = MAX_VIDEO_QUEUE_SIZE; static uint32_t sVideoQueueHWAccelSize = HW_VIDEO_QUEUE_SIZE; static uint32_t sVideoQueueSendToCompositorSize = VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE; static void InitVideoQueuePrefs() { MOZ_ASSERT(NS_IsMainThread()); static bool sPrefInit = false; if (!sPrefInit) { sPrefInit = true; sVideoQueueDefaultSize = Preferences::GetUint( "media.video-queue.default-size", MAX_VIDEO_QUEUE_SIZE); sVideoQueueHWAccelSize = Preferences::GetUint( "media.video-queue.hw-accel-size", HW_VIDEO_QUEUE_SIZE); sVideoQueueSendToCompositorSize = Preferences::GetUint("media.video-queue.send-to-compositor-size", VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE); } } template static void DiscardFramesFromTail(MediaQueue& aQueue, const Function&& aTest) { while (aQueue.GetSize()) { if (aTest(aQueue.PeekBack()->mTime.ToMicroseconds())) { RefPtr releaseMe = aQueue.PopBack(); continue; } break; } } // Delay, in milliseconds, that tabs needs to be in background before video // decoding is suspended. static TimeDuration SuspendBackgroundVideoDelay() { return TimeDuration::FromMilliseconds( StaticPrefs::media_suspend_bkgnd_video_delay_ms()); } class MediaDecoderStateMachine::StateObject { public: virtual ~StateObject() = default; virtual void Exit() {} // Exit action. virtual void Step() {} // Perform a 'cycle' of this state object. virtual State GetState() const = 0; // Event handlers for various events. virtual void HandleAudioCaptured() {} virtual void HandleAudioDecoded(AudioData* aAudio) { Crash("Unexpected event!", __func__); } virtual void HandleVideoDecoded(VideoData* aVideo) { Crash("Unexpected event!", __func__); } virtual void HandleAudioWaited(MediaData::Type aType) { Crash("Unexpected event!", __func__); } virtual void HandleVideoWaited(MediaData::Type aType) { Crash("Unexpected event!", __func__); } virtual void HandleWaitingForAudio() { Crash("Unexpected event!", __func__); } virtual void HandleAudioCanceled() { Crash("Unexpected event!", __func__); } virtual void HandleEndOfAudio() { Crash("Unexpected event!", __func__); } virtual void HandleWaitingForVideo() { Crash("Unexpected event!", __func__); } virtual void HandleVideoCanceled() { Crash("Unexpected event!", __func__); } virtual void HandleEndOfVideo() { Crash("Unexpected event!", __func__); } virtual RefPtr HandleSeek( const SeekTarget& aTarget); virtual RefPtr HandleShutdown(); virtual void HandleVideoSuspendTimeout() = 0; virtual void HandleResumeVideoDecoding(const TimeUnit& aTarget); virtual void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) {} virtual void GetDebugInfo( dom::MediaDecoderStateMachineDecodingStateDebugInfo& aInfo) {} virtual void HandleLoopingChanged() {} private: template auto ReturnTypeHelper(R (S::*)(As...)) -> R; void Crash(const char* aReason, const char* aSite) { char buf[1024]; SprintfLiteral(buf, "%s state=%s callsite=%s", aReason, ToStateStr(GetState()), aSite); MOZ_ReportAssertionFailure(buf, __FILE__, __LINE__); MOZ_CRASH(); } protected: enum class EventVisibility : int8_t { Observable, Suppressed }; using Master = MediaDecoderStateMachine; explicit StateObject(Master* aPtr) : mMaster(aPtr) {} TaskQueue* OwnerThread() const { return mMaster->mTaskQueue; } ReaderProxy* Reader() const { return mMaster->mReader; } const MediaInfo& Info() const { return mMaster->Info(); } MediaQueue& AudioQueue() const { return mMaster->mAudioQueue; } MediaQueue& VideoQueue() const { return mMaster->mVideoQueue; } template auto CallEnterMemberFunction(S* aS, std::tuple& aTuple, std::index_sequence) -> decltype(ReturnTypeHelper(&S::Enter)) { AUTO_PROFILER_LABEL("StateObject::CallEnterMemberFunction", MEDIA_PLAYBACK); return aS->Enter(std::move(std::get(aTuple))...); } // Note this function will delete the current state object. // Don't access members to avoid UAF after this call. template auto SetState(Ts&&... aArgs) -> decltype(ReturnTypeHelper(&S::Enter)) { // |aArgs| must be passed by reference to avoid passing MOZ_NON_PARAM class // SeekJob by value. See bug 1287006 and bug 1338374. But we still *must* // copy the parameters, because |Exit()| can modify them. See bug 1312321. // So we 1) pass the parameters by reference, but then 2) immediately copy // them into a Tuple to be safe against modification, and finally 3) move // the elements of the Tuple into the final function call. auto copiedArgs = std::make_tuple(std::forward(aArgs)...); // Copy mMaster which will reset to null. auto* master = mMaster; auto* s = new S(master); // It's possible to seek again during seeking, otherwise the new state // should always be different from the original one. MOZ_ASSERT(GetState() != s->GetState() || GetState() == DECODER_STATE_SEEKING_ACCURATE || GetState() == DECODER_STATE_SEEKING_FROMDORMANT || GetState() == DECODER_STATE_SEEKING_NEXTFRAMESEEKING || GetState() == DECODER_STATE_SEEKING_VIDEOONLY); SLOG("change state to: %s", ToStateStr(s->GetState())); PROFILER_MARKER_TEXT("MDSM::StateChange", MEDIA_PLAYBACK, {}, nsPrintfCString("%s", ToStateStr(s->GetState()))); Exit(); // Delete the old state asynchronously to avoid UAF if the caller tries to // access its members after SetState() returns. master->OwnerThread()->DispatchDirectTask( NS_NewRunnableFunction("MDSM::StateObject::DeleteOldState", [toDelete = std::move(master->mStateObj)]() {})); // Also reset mMaster to catch potentail UAF. mMaster = nullptr; master->mStateObj.reset(s); return CallEnterMemberFunction(s, copiedArgs, std::index_sequence_for{}); } RefPtr SetSeekingState( SeekJob&& aSeekJob, EventVisibility aVisibility); void SetDecodingState(); // Take a raw pointer in order not to change the life cycle of MDSM. // It is guaranteed to be valid by MDSM. Master* mMaster; }; /** * Purpose: decode metadata like duration and dimensions of the media resource. * * Transition to other states when decoding metadata is done: * SHUTDOWN if failing to decode metadata. * DECODING_FIRSTFRAME otherwise. */ class MediaDecoderStateMachine::DecodeMetadataState : public MediaDecoderStateMachine::StateObject { public: explicit DecodeMetadataState(Master* aPtr) : StateObject(aPtr) {} void Enter() { MOZ_ASSERT(!mMaster->mVideoDecodeSuspended); MOZ_ASSERT(!mMetadataRequest.Exists()); SLOG("Dispatching AsyncReadMetadata"); // We disconnect mMetadataRequest in Exit() so it is fine to capture // a raw pointer here. Reader() ->ReadMetadata() ->Then( OwnerThread(), __func__, [this](MetadataHolder&& aMetadata) { OnMetadataRead(std::move(aMetadata)); }, [this](const MediaResult& aError) { OnMetadataNotRead(aError); }) ->Track(mMetadataRequest); } void Exit() override { mMetadataRequest.DisconnectIfExists(); } State GetState() const override { return DECODER_STATE_DECODING_METADATA; } RefPtr HandleSeek( const SeekTarget& aTarget) override { MOZ_DIAGNOSTIC_ASSERT(false, "Can't seek while decoding metadata."); return MediaDecoder::SeekPromise::CreateAndReject(true, __func__); } void HandleVideoSuspendTimeout() override { // Do nothing since no decoders are created yet. } void HandleResumeVideoDecoding(const TimeUnit&) override { // We never suspend video decoding in this state. MOZ_ASSERT(false, "Shouldn't have suspended video decoding."); } private: void OnMetadataRead(MetadataHolder&& aMetadata); void OnMetadataNotRead(const MediaResult& aError) { AUTO_PROFILER_LABEL("DecodeMetadataState::OnMetadataNotRead", MEDIA_PLAYBACK); mMetadataRequest.Complete(); SLOGE("Decode metadata failed, shutting down decoder"); mMaster->DecodeError(aError); } MozPromiseRequestHolder mMetadataRequest; }; /** * Purpose: release decoder resources to save memory and hardware resources. * * Transition to: * SEEKING if any seek request or play state changes to PLAYING. */ class MediaDecoderStateMachine::DormantState : public MediaDecoderStateMachine::StateObject { public: explicit DormantState(Master* aPtr) : StateObject(aPtr) {} void Enter() { if (mMaster->IsPlaying()) { mMaster->StopPlayback(); } // Calculate the position to seek to when exiting dormant. auto t = mMaster->mMediaSink->IsStarted() ? mMaster->GetClock() : mMaster->GetMediaTime(); mMaster->AdjustByLooping(t); mPendingSeek.mTarget.emplace(t, SeekTarget::Accurate); // SeekJob asserts |mTarget.IsValid() == !mPromise.IsEmpty()| so we // need to create the promise even it is not used at all. // The promise may be used when coming out of DormantState into // SeekingState. RefPtr x = mPendingSeek.mPromise.Ensure(__func__); // Reset the decoding state to ensure that any queued video frames are // released and don't consume video memory. mMaster->ResetDecode(); // No need to call StopMediaSink() here. // We will do it during seeking when exiting dormant. // Ignore WAIT_FOR_DATA since we won't decode in dormant. mMaster->mAudioWaitRequest.DisconnectIfExists(); mMaster->mVideoWaitRequest.DisconnectIfExists(); MaybeReleaseResources(); } void Exit() override { // mPendingSeek is either moved when exiting dormant or // should be rejected here before transition to SHUTDOWN. mPendingSeek.RejectIfExists(__func__); } State GetState() const override { return DECODER_STATE_DORMANT; } RefPtr HandleSeek( const SeekTarget& aTarget) override; void HandleVideoSuspendTimeout() override { // Do nothing since we've released decoders in Enter(). } void HandleResumeVideoDecoding(const TimeUnit&) override { // Do nothing since we won't resume decoding until exiting dormant. } void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) override; void HandleAudioDecoded(AudioData*) override { MaybeReleaseResources(); } void HandleVideoDecoded(VideoData*) override { MaybeReleaseResources(); } void HandleWaitingForAudio() override { MaybeReleaseResources(); } void HandleWaitingForVideo() override { MaybeReleaseResources(); } void HandleAudioCanceled() override { MaybeReleaseResources(); } void HandleVideoCanceled() override { MaybeReleaseResources(); } void HandleEndOfAudio() override { MaybeReleaseResources(); } void HandleEndOfVideo() override { MaybeReleaseResources(); } private: void MaybeReleaseResources() { if (!mMaster->mAudioDataRequest.Exists() && !mMaster->mVideoDataRequest.Exists()) { // Release decoders only when they are idle. Otherwise it might cause // decode error later when resetting decoders during seeking. mMaster->mReader->ReleaseResources(); } } SeekJob mPendingSeek; }; /** * Purpose: decode the 1st audio and video frames to fire the 'loadeddata' * event. * * Transition to: * SHUTDOWN if any decode error. * SEEKING if any seek request. * DECODING/LOOPING_DECODING when the 'loadeddata' event is fired. */ class MediaDecoderStateMachine::DecodingFirstFrameState : public MediaDecoderStateMachine::StateObject { public: explicit DecodingFirstFrameState(Master* aPtr) : StateObject(aPtr) {} void Enter(); void Exit() override { // mPendingSeek is either moved in MaybeFinishDecodeFirstFrame() // or should be rejected here before transition to SHUTDOWN. mPendingSeek.RejectIfExists(__func__); } State GetState() const override { return DECODER_STATE_DECODING_FIRSTFRAME; } void HandleAudioDecoded(AudioData* aAudio) override { mMaster->PushAudio(aAudio); MaybeFinishDecodeFirstFrame(); } void HandleVideoDecoded(VideoData* aVideo) override { mMaster->PushVideo(aVideo); MaybeFinishDecodeFirstFrame(); } void HandleWaitingForAudio() override { mMaster->WaitForData(MediaData::Type::AUDIO_DATA); } void HandleAudioCanceled() override { mMaster->RequestAudioData(); } void HandleEndOfAudio() override { AudioQueue().Finish(); MaybeFinishDecodeFirstFrame(); } void HandleWaitingForVideo() override { mMaster->WaitForData(MediaData::Type::VIDEO_DATA); } void HandleVideoCanceled() override { mMaster->RequestVideoData(media::TimeUnit()); } void HandleEndOfVideo() override { VideoQueue().Finish(); MaybeFinishDecodeFirstFrame(); } void HandleAudioWaited(MediaData::Type aType) override { mMaster->RequestAudioData(); } void HandleVideoWaited(MediaData::Type aType) override { mMaster->RequestVideoData(media::TimeUnit()); } void HandleVideoSuspendTimeout() override { // Do nothing for we need to decode the 1st video frame to get the // dimensions. } void HandleResumeVideoDecoding(const TimeUnit&) override { // We never suspend video decoding in this state. MOZ_ASSERT(false, "Shouldn't have suspended video decoding."); } RefPtr HandleSeek( const SeekTarget& aTarget) override { if (mMaster->mIsMSE) { return StateObject::HandleSeek(aTarget); } // Delay seek request until decoding first frames for non-MSE media. SLOG("Not Enough Data to seek at this stage, queuing seek"); mPendingSeek.RejectIfExists(__func__); mPendingSeek.mTarget.emplace(aTarget); return mPendingSeek.mPromise.Ensure(__func__); } private: // Notify FirstFrameLoaded if having decoded first frames and // transition to SEEKING if there is any pending seek, or DECODING otherwise. void MaybeFinishDecodeFirstFrame(); SeekJob mPendingSeek; }; /** * Purpose: decode audio/video data for playback. * * Transition to: * DORMANT if playback is paused for a while. * SEEKING if any seek request. * SHUTDOWN if any decode error. * BUFFERING if playback can't continue due to lack of decoded data. * COMPLETED when having decoded all audio/video data. * LOOPING_DECODING when media start seamless looping */ class MediaDecoderStateMachine::DecodingState : public MediaDecoderStateMachine::StateObject { public: explicit DecodingState(Master* aPtr) : StateObject(aPtr), mDormantTimer(OwnerThread()) {} void Enter(); void Exit() override { if (!mDecodeStartTime.IsNull()) { TimeDuration decodeDuration = TimeStamp::Now() - mDecodeStartTime; SLOG("Exiting DECODING, decoded for %.3lfs", decodeDuration.ToSeconds()); } mDormantTimer.Reset(); mOnAudioPopped.DisconnectIfExists(); mOnVideoPopped.DisconnectIfExists(); } void Step() override; State GetState() const override { return DECODER_STATE_DECODING; } void HandleAudioDecoded(AudioData* aAudio) override { mMaster->PushAudio(aAudio); DispatchDecodeTasksIfNeeded(); MaybeStopPrerolling(); } void HandleVideoDecoded(VideoData* aVideo) override { // We only do this check when we're not looping, which can be known by // checking the queue's offset. const auto currentTime = mMaster->GetMediaTime(); if (aVideo->GetEndTime() < currentTime && VideoQueue().GetOffset() == media::TimeUnit::Zero()) { if (!mVideoFirstLateTime) { mVideoFirstLateTime = Some(TimeStamp::Now()); } PROFILER_MARKER("Video falling behind", MEDIA_PLAYBACK, {}, VideoFallingBehindMarker, aVideo->mTime.ToMicroseconds(), currentTime.ToMicroseconds()); SLOG("video %" PRId64 " starts being late (current=%" PRId64 ")", aVideo->mTime.ToMicroseconds(), currentTime.ToMicroseconds()); } else { mVideoFirstLateTime.reset(); } mMaster->PushVideo(aVideo); DispatchDecodeTasksIfNeeded(); MaybeStopPrerolling(); } void HandleAudioCanceled() override { mMaster->RequestAudioData(); } void HandleVideoCanceled() override { mMaster->RequestVideoData(mMaster->GetMediaTime(), ShouldRequestNextKeyFrame()); } void HandleEndOfAudio() override; void HandleEndOfVideo() override; void HandleWaitingForAudio() override { mMaster->WaitForData(MediaData::Type::AUDIO_DATA); MaybeStopPrerolling(); } void HandleWaitingForVideo() override { mMaster->WaitForData(MediaData::Type::VIDEO_DATA); MaybeStopPrerolling(); } void HandleAudioWaited(MediaData::Type aType) override { mMaster->RequestAudioData(); } void HandleVideoWaited(MediaData::Type aType) override { mMaster->RequestVideoData(mMaster->GetMediaTime(), ShouldRequestNextKeyFrame()); } void HandleAudioCaptured() override { MaybeStopPrerolling(); // MediaSink is changed. Schedule Step() to check if we can start playback. mMaster->ScheduleStateMachine(); } void HandleVideoSuspendTimeout() override { // No video, so nothing to suspend. if (!mMaster->HasVideo()) { return; } PROFILER_MARKER_UNTYPED("MDSM::EnterVideoSuspend", MEDIA_PLAYBACK); mMaster->mVideoDecodeSuspended = true; mMaster->mOnPlaybackEvent.Notify(MediaPlaybackEvent::EnterVideoSuspend); Reader()->SetVideoBlankDecode(true); } void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) override { if (aPlayState == MediaDecoder::PLAY_STATE_PLAYING) { // Schedule Step() to check if we can start playback. mMaster->ScheduleStateMachine(); // Try to dispatch decoding tasks for mMinimizePreroll might be reset. DispatchDecodeTasksIfNeeded(); } if (aPlayState == MediaDecoder::PLAY_STATE_PAUSED) { StartDormantTimer(); mVideoFirstLateTime.reset(); } else { mDormantTimer.Reset(); } } void GetDebugInfo( dom::MediaDecoderStateMachineDecodingStateDebugInfo& aInfo) override { aInfo.mIsPrerolling = mIsPrerolling; } void HandleLoopingChanged() override { SetDecodingState(); } protected: virtual void EnsureAudioDecodeTaskQueued(); virtual void EnsureVideoDecodeTaskQueued(); virtual bool ShouldStopPrerolling() const { return mIsPrerolling && (DonePrerollingAudio() || IsWaitingData(MediaData::Type::AUDIO_DATA)) && (DonePrerollingVideo() || IsWaitingData(MediaData::Type::VIDEO_DATA)); } virtual bool IsWaitingData(MediaData::Type aType) const { if (aType == MediaData::Type::AUDIO_DATA) { return mMaster->IsWaitingAudioData(); } MOZ_ASSERT(aType == MediaData::Type::VIDEO_DATA); return mMaster->IsWaitingVideoData(); } void MaybeStopPrerolling() { if (ShouldStopPrerolling()) { mIsPrerolling = false; // Check if we can start playback. mMaster->ScheduleStateMachine(); } } bool ShouldRequestNextKeyFrame() const { if (!mVideoFirstLateTime) { return false; } const double elapsedTimeMs = (TimeStamp::Now() - *mVideoFirstLateTime).ToMilliseconds(); const bool rv = elapsedTimeMs >= StaticPrefs::media_decoder_skip_when_video_too_slow_ms(); if (rv) { PROFILER_MARKER_UNTYPED("Skipping to next keyframe", MEDIA_PLAYBACK); SLOG( "video has been late behind media time for %f ms, should skip to " "next key frame", elapsedTimeMs); } return rv; } virtual bool IsBufferingAllowed() const { return true; } private: void DispatchDecodeTasksIfNeeded(); void MaybeStartBuffering(); // At the start of decoding we want to "preroll" the decode until we've // got a few frames decoded before we consider whether decode is falling // behind. Otherwise our "we're falling behind" logic will trigger // unnecessarily if we start playing as soon as the first sample is // decoded. These two fields store how many video frames and audio // samples we must consume before are considered to be finished prerolling. TimeUnit AudioPrerollThreshold() const { return (mMaster->mAmpleAudioThreshold / 2) .MultDouble(mMaster->mPlaybackRate); } uint32_t VideoPrerollFrames() const { return std::min( static_cast( mMaster->GetAmpleVideoFrames() / 2. * mMaster->mPlaybackRate + 1), sVideoQueueDefaultSize); } bool DonePrerollingAudio() const { return !mMaster->IsAudioDecoding() || mMaster->GetDecodedAudioDuration() >= AudioPrerollThreshold(); } bool DonePrerollingVideo() const { return !mMaster->IsVideoDecoding() || static_cast(mMaster->VideoQueue().GetSize()) >= VideoPrerollFrames(); } void StartDormantTimer() { if (!mMaster->mMediaSeekable) { // Don't enter dormant if the media is not seekable because we need to // seek when exiting dormant. return; } auto timeout = StaticPrefs::media_dormant_on_pause_timeout_ms(); if (timeout < 0) { // Disabled when timeout is negative. return; } if (timeout == 0) { // Enter dormant immediately without scheduling a timer. SetState(); return; } if (mMaster->mMinimizePreroll) { SetState(); return; } TimeStamp target = TimeStamp::Now() + TimeDuration::FromMilliseconds(timeout); mDormantTimer.Ensure( target, [this]() { AUTO_PROFILER_LABEL("DecodingState::StartDormantTimer:SetDormant", MEDIA_PLAYBACK); mDormantTimer.CompleteRequest(); SetState(); }, [this]() { mDormantTimer.CompleteRequest(); }); } // Time at which we started decoding. TimeStamp mDecodeStartTime; // When we start decoding (either for the first time, or after a pause) // we may be low on decoded data. We don't want our "low data" logic to // kick in and decide that we're low on decoded data because the download // can't keep up with the decode, and cause us to pause playback. So we // have a "preroll" stage, where we ignore the results of our "low data" // logic during the first few frames of our decode. This occurs during // playback. bool mIsPrerolling = true; // Fired when playback is paused for a while to enter dormant. DelayedScheduler mDormantTimer; MediaEventListener mOnAudioPopped; MediaEventListener mOnVideoPopped; // If video has been later than the media time, this will records when the // video started being late. It will be reset once video catches up with the // media time. Maybe mVideoFirstLateTime; }; /** * Purpose: decode audio data for playback when media is in seamless * looping, we will adjust media time to make samples time monotonically * increasing. All its methods runs on its owner thread (MDSM thread). * * Transition to: * DORMANT if playback is paused for a while. * SEEKING if any seek request. * SHUTDOWN if any decode error. * BUFFERING if playback can't continue due to lack of decoded data. * COMPLETED when the media resource is closed and no data is available * anymore. * DECODING when media stops seamless looping. */ class MediaDecoderStateMachine::LoopingDecodingState : public MediaDecoderStateMachine::DecodingState { public: explicit LoopingDecodingState(Master* aPtr) : DecodingState(aPtr), mIsReachingAudioEOS(!mMaster->IsAudioDecoding()), mIsReachingVideoEOS(!mMaster->IsVideoDecoding()), mAudioEndedBeforeEnteringStateWithoutDuration(false), mVideoEndedBeforeEnteringStateWithoutDuration(false) { MOZ_ASSERT(mMaster->mLooping); // If the track has reached EOS and we already have its last data, then we // can know its duration. But if playback starts from EOS (due to seeking), // the decoded end time would be zero because none of data gets decoded yet. if (mIsReachingAudioEOS) { if (mMaster->HasLastDecodedData(MediaData::Type::AUDIO_DATA) && !mMaster->mAudioTrackDecodedDuration) { mMaster->mAudioTrackDecodedDuration.emplace( mMaster->mDecodedAudioEndTime); } else { mAudioEndedBeforeEnteringStateWithoutDuration = true; } } if (mIsReachingVideoEOS) { if (mMaster->HasLastDecodedData(MediaData::Type::VIDEO_DATA) && !mMaster->mVideoTrackDecodedDuration) { mMaster->mVideoTrackDecodedDuration.emplace( mMaster->mDecodedVideoEndTime); } else { mVideoEndedBeforeEnteringStateWithoutDuration = true; } } // If we've looped at least once before, the master's media queues have // already stored some adjusted data. If a track has reached EOS, we need to // update queue offset correctly. Otherwise, it would cause a/v unsync. if (mMaster->mOriginalDecodedDuration != media::TimeUnit::Zero()) { if (mIsReachingAudioEOS && mMaster->HasAudio()) { AudioQueue().SetOffset(AudioQueue().GetOffset() + mMaster->mOriginalDecodedDuration); } if (mIsReachingVideoEOS && mMaster->HasVideo()) { VideoQueue().SetOffset(VideoQueue().GetOffset() + mMaster->mOriginalDecodedDuration); } } } void Enter() { UpdatePlaybackPositionToZeroIfNeeded(); if (mMaster->HasAudio() && mIsReachingAudioEOS) { SLOG("audio has ended, request the data again."); RequestDataFromStartPosition(TrackInfo::TrackType::kAudioTrack); } if (mMaster->HasVideo() && mIsReachingVideoEOS) { SLOG("video has ended, request the data again."); RequestDataFromStartPosition(TrackInfo::TrackType::kVideoTrack); } DecodingState::Enter(); } void Exit() override { MOZ_DIAGNOSTIC_ASSERT(mMaster->OnTaskQueue()); SLOG("Leaving looping state, offset [a=%" PRId64 ",v=%" PRId64 "], endtime [a=%" PRId64 ",v=%" PRId64 "], track duration [a=%" PRId64 ",v=%" PRId64 "], waiting=%s", AudioQueue().GetOffset().ToMicroseconds(), VideoQueue().GetOffset().ToMicroseconds(), mMaster->mDecodedAudioEndTime.ToMicroseconds(), mMaster->mDecodedVideoEndTime.ToMicroseconds(), mMaster->mAudioTrackDecodedDuration ? mMaster->mAudioTrackDecodedDuration->ToMicroseconds() : 0, mMaster->mVideoTrackDecodedDuration ? mMaster->mVideoTrackDecodedDuration->ToMicroseconds() : 0, mDataWaitingTimestampAdjustment ? MediaData::TypeToStr(mDataWaitingTimestampAdjustment->mType) : "none"); if (ShouldDiscardLoopedData(MediaData::Type::AUDIO_DATA)) { DiscardLoopedData(MediaData::Type::AUDIO_DATA); } if (ShouldDiscardLoopedData(MediaData::Type::VIDEO_DATA)) { DiscardLoopedData(MediaData::Type::VIDEO_DATA); } if (mMaster->HasAudio() && HasDecodedLastAudioFrame()) { SLOG("Mark audio queue as finished"); mMaster->mAudioDataRequest.DisconnectIfExists(); mMaster->mAudioWaitRequest.DisconnectIfExists(); AudioQueue().Finish(); } if (mMaster->HasVideo() && HasDecodedLastVideoFrame()) { SLOG("Mark video queue as finished"); mMaster->mVideoDataRequest.DisconnectIfExists(); mMaster->mVideoWaitRequest.DisconnectIfExists(); VideoQueue().Finish(); } if (mWaitingAudioDataFromStart) { mMaster->mMediaSink->EnableTreatAudioUnderrunAsSilence(false); } // Clear waiting data should be done after marking queue as finished. mDataWaitingTimestampAdjustment = nullptr; mAudioDataRequest.DisconnectIfExists(); mVideoDataRequest.DisconnectIfExists(); mAudioSeekRequest.DisconnectIfExists(); mVideoSeekRequest.DisconnectIfExists(); DecodingState::Exit(); } ~LoopingDecodingState() { MOZ_DIAGNOSTIC_ASSERT(!mAudioDataRequest.Exists()); MOZ_DIAGNOSTIC_ASSERT(!mVideoDataRequest.Exists()); MOZ_DIAGNOSTIC_ASSERT(!mAudioSeekRequest.Exists()); MOZ_DIAGNOSTIC_ASSERT(!mVideoSeekRequest.Exists()); } State GetState() const override { return DECODER_STATE_LOOPING_DECODING; } void HandleAudioDecoded(AudioData* aAudio) override { // TODO : check if we need to update mOriginalDecodedDuration if (mWaitingAudioDataFromStart) { mMaster->mMediaSink->EnableTreatAudioUnderrunAsSilence(false); mWaitingAudioDataFromStart = false; } // After pushing data to the queue, timestamp might be adjusted. DecodingState::HandleAudioDecoded(aAudio); mMaster->mDecodedAudioEndTime = std::max(aAudio->GetEndTime(), mMaster->mDecodedAudioEndTime); SLOG("audio sample after time-adjustment [%" PRId64 ",%" PRId64 "]", aAudio->mTime.ToMicroseconds(), aAudio->GetEndTime().ToMicroseconds()); } void HandleVideoDecoded(VideoData* aVideo) override { // TODO : check if we need to update mOriginalDecodedDuration // After pushing data to the queue, timestamp might be adjusted. DecodingState::HandleVideoDecoded(aVideo); mMaster->mDecodedVideoEndTime = std::max(aVideo->GetEndTime(), mMaster->mDecodedVideoEndTime); SLOG("video sample after time-adjustment [%" PRId64 ",%" PRId64 "]", aVideo->mTime.ToMicroseconds(), aVideo->GetEndTime().ToMicroseconds()); } void HandleEndOfAudio() override { mIsReachingAudioEOS = true; if (!mMaster->mAudioTrackDecodedDuration && mMaster->HasLastDecodedData(MediaData::Type::AUDIO_DATA)) { mMaster->mAudioTrackDecodedDuration.emplace( mMaster->mDecodedAudioEndTime); } if (DetermineOriginalDecodedDurationIfNeeded()) { AudioQueue().SetOffset(AudioQueue().GetOffset() + mMaster->mOriginalDecodedDuration); } SLOG( "received audio EOS when seamless looping, starts seeking, " "audioLoopingOffset=[%" PRId64 "], mAudioTrackDecodedDuration=[%" PRId64 "]", AudioQueue().GetOffset().ToMicroseconds(), mMaster->mAudioTrackDecodedDuration->ToMicroseconds()); if (!IsRequestingDataFromStartPosition(MediaData::Type::AUDIO_DATA)) { RequestDataFromStartPosition(TrackInfo::TrackType::kAudioTrack); } ProcessSamplesWaitingAdjustmentIfAny(); } void HandleEndOfVideo() override { mIsReachingVideoEOS = true; if (!mMaster->mVideoTrackDecodedDuration && mMaster->HasLastDecodedData(MediaData::Type::VIDEO_DATA)) { mMaster->mVideoTrackDecodedDuration.emplace( mMaster->mDecodedVideoEndTime); } if (DetermineOriginalDecodedDurationIfNeeded()) { VideoQueue().SetOffset(VideoQueue().GetOffset() + mMaster->mOriginalDecodedDuration); } SLOG( "received video EOS when seamless looping, starts seeking, " "videoLoopingOffset=[%" PRId64 "], mVideoTrackDecodedDuration=[%" PRId64 "]", VideoQueue().GetOffset().ToMicroseconds(), mMaster->mVideoTrackDecodedDuration->ToMicroseconds()); if (!IsRequestingDataFromStartPosition(MediaData::Type::VIDEO_DATA)) { RequestDataFromStartPosition(TrackInfo::TrackType::kVideoTrack); } ProcessSamplesWaitingAdjustmentIfAny(); } private: void RequestDataFromStartPosition(TrackInfo::TrackType aType) { MOZ_DIAGNOSTIC_ASSERT(aType == TrackInfo::TrackType::kAudioTrack || aType == TrackInfo::TrackType::kVideoTrack); const bool isAudio = aType == TrackInfo::TrackType::kAudioTrack; MOZ_ASSERT_IF(isAudio, mMaster->HasAudio()); MOZ_ASSERT_IF(!isAudio, mMaster->HasVideo()); if (IsReaderSeeking()) { MOZ_ASSERT(!mPendingSeekingType); mPendingSeekingType = Some(aType); SLOG("Delay %s seeking until the reader finishes current seeking", isAudio ? "audio" : "video"); return; } auto& seekRequest = isAudio ? mAudioSeekRequest : mVideoSeekRequest; Reader()->ResetDecode(aType); Reader() ->Seek(SeekTarget(media::TimeUnit::Zero(), SeekTarget::Type::Accurate, isAudio ? SeekTarget::Track::AudioOnly : SeekTarget::Track::VideoOnly)) ->Then( OwnerThread(), __func__, [this, isAudio, master = RefPtr{mMaster}]() mutable -> void { AUTO_PROFILER_LABEL( nsPrintfCString( "LoopingDecodingState::RequestDataFromStartPosition(%s)::" "SeekResolved", isAudio ? "audio" : "video") .get(), MEDIA_PLAYBACK); if (auto& state = master->mStateObj; state && state->GetState() != DECODER_STATE_LOOPING_DECODING) { MOZ_RELEASE_ASSERT(false, "This shouldn't happen!"); return; } if (isAudio) { mAudioSeekRequest.Complete(); } else { mVideoSeekRequest.Complete(); } SLOG( "seeking completed, start to request first %s sample " "(queued=%zu, decoder-queued=%zu)", isAudio ? "audio" : "video", isAudio ? AudioQueue().GetSize() : VideoQueue().GetSize(), isAudio ? Reader()->SizeOfAudioQueueInFrames() : Reader()->SizeOfVideoQueueInFrames()); if (isAudio) { RequestAudioDataFromReaderAfterEOS(); } else { RequestVideoDataFromReaderAfterEOS(); } if (mPendingSeekingType) { auto seekingType = *mPendingSeekingType; mPendingSeekingType.reset(); SLOG("Perform pending %s seeking", TrackTypeToStr(seekingType)); RequestDataFromStartPosition(seekingType); } }, [this, isAudio, master = RefPtr{mMaster}]( const SeekRejectValue& aReject) mutable -> void { AUTO_PROFILER_LABEL( nsPrintfCString("LoopingDecodingState::" "RequestDataFromStartPosition(%s)::" "SeekRejected", isAudio ? "audio" : "video") .get(), MEDIA_PLAYBACK); if (auto& state = master->mStateObj; state && state->GetState() != DECODER_STATE_LOOPING_DECODING) { MOZ_RELEASE_ASSERT(false, "This shouldn't happen!"); return; } if (isAudio) { mAudioSeekRequest.Complete(); } else { mVideoSeekRequest.Complete(); } HandleError(aReject.mError, isAudio); }) ->Track(seekRequest); } void RequestAudioDataFromReaderAfterEOS() { MOZ_ASSERT(mMaster->HasAudio()); Reader() ->RequestAudioData() ->Then( OwnerThread(), __func__, [this, master = RefPtr{mMaster}](const RefPtr& aAudio) { AUTO_PROFILER_LABEL( "LoopingDecodingState::" "RequestAudioDataFromReader::" "RequestDataResolved", MEDIA_PLAYBACK); if (auto& state = master->mStateObj; state && state->GetState() != DECODER_STATE_LOOPING_DECODING) { MOZ_RELEASE_ASSERT(false, "This shouldn't happen!"); return; } mIsReachingAudioEOS = false; mAudioDataRequest.Complete(); SLOG( "got audio decoded sample " "[%" PRId64 ",%" PRId64 "]", aAudio->mTime.ToMicroseconds(), aAudio->GetEndTime().ToMicroseconds()); if (ShouldPutDataOnWaiting(MediaData::Type::AUDIO_DATA)) { SLOG( "decoded audio sample needs to wait for timestamp " "adjustment after EOS"); PutDataOnWaiting(aAudio); return; } HandleAudioDecoded(aAudio); ProcessSamplesWaitingAdjustmentIfAny(); }, [this, master = RefPtr{mMaster}](const MediaResult& aError) { AUTO_PROFILER_LABEL( "LoopingDecodingState::" "RequestAudioDataFromReader::" "RequestDataRejected", MEDIA_PLAYBACK); if (auto& state = master->mStateObj; state && state->GetState() != DECODER_STATE_LOOPING_DECODING) { MOZ_RELEASE_ASSERT(false, "This shouldn't happen!"); return; } mAudioDataRequest.Complete(); HandleError(aError, true /* isAudio */); }) ->Track(mAudioDataRequest); } void RequestVideoDataFromReaderAfterEOS() { MOZ_ASSERT(mMaster->HasVideo()); Reader() ->RequestVideoData(media::TimeUnit(), false /* aRequestNextVideoKeyFrame */) ->Then( OwnerThread(), __func__, [this, master = RefPtr{mMaster}](const RefPtr& aVideo) { AUTO_PROFILER_LABEL( "LoopingDecodingState::" "RequestVideoDataFromReaderAfterEOS()::" "RequestDataResolved", MEDIA_PLAYBACK); if (auto& state = master->mStateObj; state && state->GetState() != DECODER_STATE_LOOPING_DECODING) { MOZ_RELEASE_ASSERT(false, "This shouldn't happen!"); return; } mIsReachingVideoEOS = false; mVideoDataRequest.Complete(); SLOG( "got video decoded sample " "[%" PRId64 ",%" PRId64 "]", aVideo->mTime.ToMicroseconds(), aVideo->GetEndTime().ToMicroseconds()); if (ShouldPutDataOnWaiting(MediaData::Type::VIDEO_DATA)) { SLOG( "decoded video sample needs to wait for timestamp " "adjustment after EOS"); PutDataOnWaiting(aVideo); return; } mMaster->mBypassingSkipToNextKeyFrameCheck = true; HandleVideoDecoded(aVideo); ProcessSamplesWaitingAdjustmentIfAny(); }, [this, master = RefPtr{mMaster}](const MediaResult& aError) { AUTO_PROFILER_LABEL( "LoopingDecodingState::" "RequestVideoDataFromReaderAfterEOS()::" "RequestDataRejected", MEDIA_PLAYBACK); if (auto& state = master->mStateObj; state && state->GetState() != DECODER_STATE_LOOPING_DECODING) { MOZ_RELEASE_ASSERT(false, "This shouldn't happen!"); return; } mVideoDataRequest.Complete(); HandleError(aError, false /* isAudio */); }) ->Track(mVideoDataRequest); } void UpdatePlaybackPositionToZeroIfNeeded() { // Hasn't reached EOS, no need to adjust playback position. if (!mIsReachingAudioEOS || !mIsReachingVideoEOS) { return; } // If we have already reached EOS before starting media sink, the sink // has not started yet and the current position is larger than last decoded // end time, that means we directly seeked to EOS and playback would start // from the start position soon. Therefore, we should reset the position to // 0s so that when media sink starts we can make it start from 0s, not from // EOS position which would result in wrong estimation of decoded audio // duration because decoded data's time which can't be adjusted as offset is // zero would be always less than media sink time. if (!mMaster->mMediaSink->IsStarted() && (mMaster->mCurrentPosition.Ref() > mMaster->mDecodedAudioEndTime || mMaster->mCurrentPosition.Ref() > mMaster->mDecodedVideoEndTime)) { mMaster->UpdatePlaybackPositionInternal(TimeUnit::Zero()); } } void HandleError(const MediaResult& aError, bool aIsAudio); bool ShouldRequestData(MediaData::Type aType) const { MOZ_DIAGNOSTIC_ASSERT(aType == MediaData::Type::AUDIO_DATA || aType == MediaData::Type::VIDEO_DATA); if (aType == MediaData::Type::AUDIO_DATA && (mAudioSeekRequest.Exists() || mAudioDataRequest.Exists() || IsDataWaitingForTimestampAdjustment(MediaData::Type::AUDIO_DATA))) { return false; } if (aType == MediaData::Type::VIDEO_DATA && (mVideoSeekRequest.Exists() || mVideoDataRequest.Exists() || IsDataWaitingForTimestampAdjustment(MediaData::Type::VIDEO_DATA))) { return false; } return true; } void HandleAudioCanceled() override { if (ShouldRequestData(MediaData::Type::AUDIO_DATA)) { mMaster->RequestAudioData(); } } void HandleAudioWaited(MediaData::Type aType) override { if (ShouldRequestData(MediaData::Type::AUDIO_DATA)) { mMaster->RequestAudioData(); } } void HandleVideoCanceled() override { if (ShouldRequestData(MediaData::Type::VIDEO_DATA)) { mMaster->RequestVideoData(mMaster->GetMediaTime(), ShouldRequestNextKeyFrame()); }; } void HandleVideoWaited(MediaData::Type aType) override { if (ShouldRequestData(MediaData::Type::VIDEO_DATA)) { mMaster->RequestVideoData(mMaster->GetMediaTime(), ShouldRequestNextKeyFrame()); }; } void EnsureAudioDecodeTaskQueued() override { if (!ShouldRequestData(MediaData::Type::AUDIO_DATA)) { return; } DecodingState::EnsureAudioDecodeTaskQueued(); } void EnsureVideoDecodeTaskQueued() override { if (!ShouldRequestData(MediaData::Type::VIDEO_DATA)) { return; } DecodingState::EnsureVideoDecodeTaskQueued(); } bool DetermineOriginalDecodedDurationIfNeeded() { // Duration would only need to be set once, unless we get more data which is // larger than the duration. That can happen on MSE (reopen stream). if (mMaster->mOriginalDecodedDuration != media::TimeUnit::Zero()) { return true; } // Single track situations if (mMaster->HasAudio() && !mMaster->HasVideo()) { MOZ_ASSERT(mMaster->mAudioTrackDecodedDuration); mMaster->mOriginalDecodedDuration = *mMaster->mAudioTrackDecodedDuration; SLOG("audio only, duration=%" PRId64, mMaster->mOriginalDecodedDuration.ToMicroseconds()); return true; } if (mMaster->HasVideo() && !mMaster->HasAudio()) { MOZ_ASSERT(mMaster->mVideoTrackDecodedDuration); mMaster->mOriginalDecodedDuration = *mMaster->mVideoTrackDecodedDuration; SLOG("video only, duration=%" PRId64, mMaster->mOriginalDecodedDuration.ToMicroseconds()); return true; } MOZ_ASSERT(mMaster->HasAudio() && mMaster->HasVideo()); // Both tracks have ended so that we can check which track is longer. if (mMaster->mAudioTrackDecodedDuration && mMaster->mVideoTrackDecodedDuration) { mMaster->mOriginalDecodedDuration = std::max(*mMaster->mVideoTrackDecodedDuration, *mMaster->mAudioTrackDecodedDuration); SLOG("Both tracks ended, original duration=%" PRId64 " (a=%" PRId64 ", v=%" PRId64 ")", mMaster->mOriginalDecodedDuration.ToMicroseconds(), mMaster->mAudioTrackDecodedDuration->ToMicroseconds(), mMaster->mVideoTrackDecodedDuration->ToMicroseconds()); return true; } // When entering the state, video has ended but audio hasn't, which means // audio is longer. if (mMaster->mAudioTrackDecodedDuration && mVideoEndedBeforeEnteringStateWithoutDuration) { mMaster->mOriginalDecodedDuration = *mMaster->mAudioTrackDecodedDuration; mVideoEndedBeforeEnteringStateWithoutDuration = false; SLOG("audio is longer, duration=%" PRId64, mMaster->mOriginalDecodedDuration.ToMicroseconds()); return true; } // When entering the state, audio has ended but video hasn't, which means // video is longer. if (mMaster->mVideoTrackDecodedDuration && mAudioEndedBeforeEnteringStateWithoutDuration) { mMaster->mOriginalDecodedDuration = *mMaster->mVideoTrackDecodedDuration; mAudioEndedBeforeEnteringStateWithoutDuration = false; SLOG("video is longer, duration=%" PRId64, mMaster->mOriginalDecodedDuration.ToMicroseconds()); return true; } SLOG("Still waiting for another track ends..."); MOZ_ASSERT(!mMaster->mAudioTrackDecodedDuration || !mMaster->mVideoTrackDecodedDuration); MOZ_ASSERT(mMaster->mOriginalDecodedDuration == media::TimeUnit::Zero()); return false; } void ProcessSamplesWaitingAdjustmentIfAny() { if (!mDataWaitingTimestampAdjustment) { return; } RefPtr data = mDataWaitingTimestampAdjustment; mDataWaitingTimestampAdjustment = nullptr; const bool isAudio = data->mType == MediaData::Type::AUDIO_DATA; SLOG("process %s sample waiting for timestamp adjustment", isAudio ? "audio" : "video"); if (isAudio) { // Waiting sample is for next round of looping, so the queue offset // shouldn't be zero. This happens when the track has reached EOS before // entering the state (and looping never happens before). Same for below // video case. if (AudioQueue().GetOffset() == media::TimeUnit::Zero()) { AudioQueue().SetOffset(mMaster->mOriginalDecodedDuration); } HandleAudioDecoded(data->As()); } else { MOZ_DIAGNOSTIC_ASSERT(data->mType == MediaData::Type::VIDEO_DATA); if (VideoQueue().GetOffset() == media::TimeUnit::Zero()) { VideoQueue().SetOffset(mMaster->mOriginalDecodedDuration); } HandleVideoDecoded(data->As()); } } bool IsDataWaitingForTimestampAdjustment(MediaData::Type aType) const { return mDataWaitingTimestampAdjustment && mDataWaitingTimestampAdjustment->mType == aType; } bool ShouldPutDataOnWaiting(MediaData::Type aType) const { // If another track is already waiting, this track shouldn't be waiting. // This case only happens when both tracks reached EOS before entering the // looping decoding state, so we don't know the decoded duration yet (used // to adjust timestamp) But this is fine, because both tracks will start // from 0 so we don't need to adjust them now. if (mDataWaitingTimestampAdjustment && !IsDataWaitingForTimestampAdjustment(aType)) { return false; } // Only have one track, no need to wait. if ((aType == MediaData::Type::AUDIO_DATA && !mMaster->HasVideo()) || (aType == MediaData::Type::VIDEO_DATA && !mMaster->HasAudio())) { return false; } // We don't know the duration yet, so we can't calculate the looping offset. return mMaster->mOriginalDecodedDuration == media::TimeUnit::Zero(); } void PutDataOnWaiting(MediaData* aData) { MOZ_ASSERT(!mDataWaitingTimestampAdjustment); mDataWaitingTimestampAdjustment = aData; SLOG("put %s [%" PRId64 ",%" PRId64 "] on waiting", MediaData::TypeToStr(aData->mType), aData->mTime.ToMicroseconds(), aData->GetEndTime().ToMicroseconds()); MaybeStopPrerolling(); } bool ShouldDiscardLoopedData(MediaData::Type aType) const { if (!mMaster->mMediaSink->IsStarted()) { return false; } MOZ_DIAGNOSTIC_ASSERT(aType == MediaData::Type::AUDIO_DATA || aType == MediaData::Type::VIDEO_DATA); const bool isAudio = aType == MediaData::Type::AUDIO_DATA; if (isAudio && !mMaster->HasAudio()) { return false; } if (!isAudio && !mMaster->HasVideo()) { return false; } /** * If media cancels looping, we should check whether there is media data * whose time is later than EOS. If so, we should discard them because we * won't have a chance to play them. * * playback last decoded * position EOS data time * ----|---------------|------------|---------> (Increasing timeline) * mCurrent looping mMaster's * ClockTime offset mDecodedXXXEndTime * */ const auto offset = isAudio ? AudioQueue().GetOffset() : VideoQueue().GetOffset(); const auto endTime = isAudio ? mMaster->mDecodedAudioEndTime : mMaster->mDecodedVideoEndTime; const auto clockTime = mMaster->GetClock(); return (offset != media::TimeUnit::Zero() && clockTime < offset && offset < endTime); } void DiscardLoopedData(MediaData::Type aType) { MOZ_DIAGNOSTIC_ASSERT(aType == MediaData::Type::AUDIO_DATA || aType == MediaData::Type::VIDEO_DATA); const bool isAudio = aType == MediaData::Type::AUDIO_DATA; const auto offset = isAudio ? AudioQueue().GetOffset() : VideoQueue().GetOffset(); if (offset == media::TimeUnit::Zero()) { return; } SLOG("Discard %s frames after the time=%" PRId64, isAudio ? "audio" : "video", offset.ToMicroseconds()); if (isAudio) { DiscardFramesFromTail(AudioQueue(), [&](int64_t aSampleTime) { return aSampleTime > offset.ToMicroseconds(); }); } else { DiscardFramesFromTail(VideoQueue(), [&](int64_t aSampleTime) { return aSampleTime > offset.ToMicroseconds(); }); } } bool HasDecodedLastAudioFrame() const { // when we're going to leave looping state and have got EOS before, we // should mark audio queue as ended because we have got all data we need. return mAudioDataRequest.Exists() || mAudioSeekRequest.Exists() || ShouldDiscardLoopedData(MediaData::Type::AUDIO_DATA) || IsDataWaitingForTimestampAdjustment(MediaData::Type::AUDIO_DATA) || mIsReachingAudioEOS; } bool HasDecodedLastVideoFrame() const { // when we're going to leave looping state and have got EOS before, we // should mark video queue as ended because we have got all data we need. return mVideoDataRequest.Exists() || mVideoSeekRequest.Exists() || ShouldDiscardLoopedData(MediaData::Type::VIDEO_DATA) || IsDataWaitingForTimestampAdjustment(MediaData::Type::VIDEO_DATA) || mIsReachingVideoEOS; } bool ShouldStopPrerolling() const override { // These checks is used to handle the media queue aren't opened correctly // because they've been close before entering the looping state. Therefore, // we need to preroll data in order to let new data to reopen the queue // automatically. Otherwise, playback can't start successfully. bool isWaitingForNewData = false; if (mMaster->HasAudio()) { isWaitingForNewData |= (mIsReachingAudioEOS && AudioQueue().IsFinished()); } if (mMaster->HasVideo()) { isWaitingForNewData |= (mIsReachingVideoEOS && VideoQueue().IsFinished()); } return !isWaitingForNewData && DecodingState::ShouldStopPrerolling(); } bool IsReaderSeeking() const { return mAudioSeekRequest.Exists() || mVideoSeekRequest.Exists(); } bool IsWaitingData(MediaData::Type aType) const override { if (aType == MediaData::Type::AUDIO_DATA) { return mMaster->IsWaitingAudioData() || IsDataWaitingForTimestampAdjustment(MediaData::Type::AUDIO_DATA); } MOZ_DIAGNOSTIC_ASSERT(aType == MediaData::Type::VIDEO_DATA); return mMaster->IsWaitingVideoData() || IsDataWaitingForTimestampAdjustment(MediaData::Type::VIDEO_DATA); } bool IsRequestingDataFromStartPosition(MediaData::Type aType) const { MOZ_DIAGNOSTIC_ASSERT(aType == MediaData::Type::AUDIO_DATA || aType == MediaData::Type::VIDEO_DATA); if (aType == MediaData::Type::AUDIO_DATA) { return mAudioSeekRequest.Exists() || mAudioDataRequest.Exists(); } return mVideoSeekRequest.Exists() || mVideoDataRequest.Exists(); } bool IsBufferingAllowed() const override { return !mIsReachingAudioEOS && !mIsReachingVideoEOS; } bool mIsReachingAudioEOS; bool mIsReachingVideoEOS; /** * If we have both tracks which have different length, when one track ends * first, we can't adjust new data from that track if another longer track * hasn't ended yet. The adjusted timestamp needs to be based off the longer * track's last data's timestamp, because otherwise it would cause a deviation * and eventually a/v unsync. Those sample needs to be stored and we will * adjust their timestamp later. * * Following graph explains the situation in details. * o : decoded data with timestamp adjusted or no adjustment (not looping yet) * x : decoded data without timestamp adjustment. * - : stop decoding and nothing happens * EOS : the track reaches to the end. We now know the offset of the track. * * Timeline -----------------------------------> * Track1 : o EOS x - - o * Track2 : o o o EOS o o * * Before reaching track2's EOS, we can't adjust samples from track1 because * track2 might have longer duration than track1. The sample X would be * stored in `mDataWaitingTimestampAdjustment` and we would also stop decoding * for track1. * * After reaching track2's EOS, now we know another track's offset, and the * larger one would be used for `mOriginalDecodedDuration`. Once that duration * has been determined, we will no longer need to put samples on waiting * because we already know how to adjust timestamp. */ RefPtr mDataWaitingTimestampAdjustment; MozPromiseRequestHolder mAudioSeekRequest; MozPromiseRequestHolder mVideoSeekRequest; MozPromiseRequestHolder mAudioDataRequest; MozPromiseRequestHolder mVideoDataRequest; // The media format reader only allows seeking a track at a time, if we're // already in seeking, then delay the new seek until the current one finishes. Maybe mPendingSeekingType; // These are used to track a special case where the playback starts from EOS // position via seeking. So even if EOS has reached, none of data has been // decoded yet. They will be reset when `mOriginalDecodedDuration` is // determined. bool mAudioEndedBeforeEnteringStateWithoutDuration; bool mVideoEndedBeforeEnteringStateWithoutDuration; // True if the audio has reached EOS, but the data from the start position is // not avalible yet. We use this to determine whether we should enable // appending silence audio frames into audio backend while audio underrun in // order to keep audio clock running. bool mWaitingAudioDataFromStart = false; }; /** * Purpose: seek to a particular new playback position. * * Transition to: * SEEKING if any new seek request. * SHUTDOWN if seek failed. * COMPLETED if the new playback position is the end of the media resource. * NextFrameSeekingState if completing a NextFrameSeekingFromDormantState. * DECODING/LOOPING_DECODING otherwise. */ class MediaDecoderStateMachine::SeekingState : public MediaDecoderStateMachine::StateObject { public: explicit SeekingState(Master* aPtr) : StateObject(aPtr), mVisibility(static_cast(0)) {} RefPtr Enter(SeekJob&& aSeekJob, EventVisibility aVisibility) { mSeekJob = std::move(aSeekJob); mVisibility = aVisibility; // Suppressed visibility comes from two cases: (1) leaving dormant state, // and (2) resuming suspended video decoder. We want both cases to be // transparent to the user. So we only notify the change when the seek // request is from the user. if (mVisibility == EventVisibility::Observable) { // Don't stop playback for a video-only seek since we want to keep playing // audio and we don't need to stop playback while leaving dormant for the // playback should has been stopped. mMaster->StopPlayback(); mMaster->UpdatePlaybackPositionInternal(mSeekJob.mTarget->GetTime()); mMaster->mOnPlaybackEvent.Notify(MediaPlaybackEvent::SeekStarted); mMaster->mOnNextFrameStatus.Notify( MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING); } RefPtr p = mSeekJob.mPromise.Ensure(__func__); DoSeek(); return p; } virtual void Exit() override = 0; State GetState() const override = 0; void HandleAudioDecoded(AudioData* aAudio) override = 0; void HandleVideoDecoded(VideoData* aVideo) override = 0; void HandleAudioWaited(MediaData::Type aType) override = 0; void HandleVideoWaited(MediaData::Type aType) override = 0; void HandleVideoSuspendTimeout() override { // Do nothing since we want a valid video frame to show when seek is done. } void HandleResumeVideoDecoding(const TimeUnit&) override { // Do nothing. We will resume video decoding in the decoding state. } // We specially handle next frame seeks by ignoring them if we're already // seeking. RefPtr HandleSeek( const SeekTarget& aTarget) override { if (aTarget.IsNextFrame()) { // We ignore next frame seeks if we already have a seek pending SLOG("Already SEEKING, ignoring seekToNextFrame"); MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished"); return MediaDecoder::SeekPromise::CreateAndReject( /* aRejectValue = */ true, __func__); } return StateObject::HandleSeek(aTarget); } protected: SeekJob mSeekJob; EventVisibility mVisibility; virtual void DoSeek() = 0; // Transition to the next state (defined by the subclass) when seek is // completed. virtual void GoToNextState() { SetDecodingState(); } void SeekCompleted(); virtual TimeUnit CalculateNewCurrentTime() const = 0; }; class MediaDecoderStateMachine::AccurateSeekingState : public MediaDecoderStateMachine::SeekingState { public: explicit AccurateSeekingState(Master* aPtr) : SeekingState(aPtr) {} State GetState() const override { return DECODER_STATE_SEEKING_ACCURATE; } RefPtr Enter(SeekJob&& aSeekJob, EventVisibility aVisibility) { MOZ_ASSERT(aSeekJob.mTarget->IsAccurate() || aSeekJob.mTarget->IsFast()); mCurrentTimeBeforeSeek = mMaster->GetMediaTime(); return SeekingState::Enter(std::move(aSeekJob), aVisibility); } void Exit() override { // Disconnect MediaDecoder. mSeekJob.RejectIfExists(__func__); // Disconnect ReaderProxy. mSeekRequest.DisconnectIfExists(); mWaitRequest.DisconnectIfExists(); } void HandleAudioDecoded(AudioData* aAudio) override { MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking, "Seek shouldn't be finished"); MOZ_ASSERT(aAudio); AdjustFastSeekIfNeeded(aAudio); if (mSeekJob.mTarget->IsFast()) { // Non-precise seek; we can stop the seek at the first sample. mMaster->PushAudio(aAudio); mDoneAudioSeeking = true; } else { nsresult rv = DropAudioUpToSeekTarget(aAudio); if (NS_FAILED(rv)) { mMaster->DecodeError(rv); return; } } if (!mDoneAudioSeeking) { RequestAudioData(); return; } MaybeFinishSeek(); } void HandleVideoDecoded(VideoData* aVideo) override { MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking, "Seek shouldn't be finished"); MOZ_ASSERT(aVideo); AdjustFastSeekIfNeeded(aVideo); if (mSeekJob.mTarget->IsFast()) { // Non-precise seek. We can stop the seek at the first sample. mMaster->PushVideo(aVideo); mDoneVideoSeeking = true; } else { nsresult rv = DropVideoUpToSeekTarget(aVideo); if (NS_FAILED(rv)) { mMaster->DecodeError(rv); return; } } if (!mDoneVideoSeeking) { RequestVideoData(); return; } MaybeFinishSeek(); } void HandleWaitingForAudio() override { MOZ_ASSERT(!mDoneAudioSeeking); mMaster->WaitForData(MediaData::Type::AUDIO_DATA); } void HandleAudioCanceled() override { MOZ_ASSERT(!mDoneAudioSeeking); RequestAudioData(); } void HandleEndOfAudio() override { HandleEndOfAudioInternal(); MaybeFinishSeek(); } void HandleWaitingForVideo() override { MOZ_ASSERT(!mDoneVideoSeeking); mMaster->WaitForData(MediaData::Type::VIDEO_DATA); } void HandleVideoCanceled() override { MOZ_ASSERT(!mDoneVideoSeeking); RequestVideoData(); } void HandleEndOfVideo() override { HandleEndOfVideoInternal(); MaybeFinishSeek(); } void HandleAudioWaited(MediaData::Type aType) override { MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking, "Seek shouldn't be finished"); RequestAudioData(); } void HandleVideoWaited(MediaData::Type aType) override { MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking, "Seek shouldn't be finished"); RequestVideoData(); } void DoSeek() override { mDoneAudioSeeking = !Info().HasAudio(); mDoneVideoSeeking = !Info().HasVideo(); // Resetting decode should be called after stopping media sink, which can // ensure that we have an empty media queue before seeking the demuxer. mMaster->StopMediaSink(); mMaster->ResetDecode(); DemuxerSeek(); } TimeUnit CalculateNewCurrentTime() const override { const auto seekTime = mSeekJob.mTarget->GetTime(); // For the accurate seek, we always set the newCurrentTime = seekTime so // that the updated HTMLMediaElement.currentTime will always be the seek // target; we rely on the MediaSink to handles the gap between the // newCurrentTime and the real decoded samples' start time. if (mSeekJob.mTarget->IsAccurate()) { return seekTime; } // For the fast seek, we update the newCurrentTime with the decoded audio // and video samples, set it to be the one which is closet to the seekTime. if (mSeekJob.mTarget->IsFast()) { RefPtr audio = AudioQueue().PeekFront(); RefPtr video = VideoQueue().PeekFront(); // A situation that both audio and video approaches the end. if (!audio && !video) { return seekTime; } const int64_t audioStart = audio ? audio->mTime.ToMicroseconds() : INT64_MAX; const int64_t videoStart = video ? video->mTime.ToMicroseconds() : INT64_MAX; const int64_t audioGap = std::abs(audioStart - seekTime.ToMicroseconds()); const int64_t videoGap = std::abs(videoStart - seekTime.ToMicroseconds()); return TimeUnit::FromMicroseconds(audioGap <= videoGap ? audioStart : videoStart); } MOZ_ASSERT(false, "AccurateSeekTask doesn't handle other seek types."); return TimeUnit::Zero(); } protected: void DemuxerSeek() { // Request the demuxer to perform seek. Reader() ->Seek(mSeekJob.mTarget.ref()) ->Then( OwnerThread(), __func__, [this](const media::TimeUnit& aUnit) { OnSeekResolved(aUnit); }, [this](const SeekRejectValue& aReject) { OnSeekRejected(aReject); }) ->Track(mSeekRequest); } void OnSeekResolved(media::TimeUnit) { AUTO_PROFILER_LABEL("AccurateSeekingState::OnSeekResolved", MEDIA_PLAYBACK); mSeekRequest.Complete(); // We must decode the first samples of active streams, so we can determine // the new stream time. So dispatch tasks to do that. if (!mDoneVideoSeeking) { RequestVideoData(); } if (!mDoneAudioSeeking) { RequestAudioData(); } } void OnSeekRejected(const SeekRejectValue& aReject) { AUTO_PROFILER_LABEL("AccurateSeekingState::OnSeekRejected", MEDIA_PLAYBACK); mSeekRequest.Complete(); if (aReject.mError == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) { SLOG("OnSeekRejected reason=WAITING_FOR_DATA type=%s", MediaData::TypeToStr(aReject.mType)); MOZ_ASSERT_IF(aReject.mType == MediaData::Type::AUDIO_DATA, !mMaster->IsRequestingAudioData()); MOZ_ASSERT_IF(aReject.mType == MediaData::Type::VIDEO_DATA, !mMaster->IsRequestingVideoData()); MOZ_ASSERT_IF(aReject.mType == MediaData::Type::AUDIO_DATA, !mMaster->IsWaitingAudioData()); MOZ_ASSERT_IF(aReject.mType == MediaData::Type::VIDEO_DATA, !mMaster->IsWaitingVideoData()); // Fire 'waiting' to notify the player that we are waiting for data. mMaster->mOnNextFrameStatus.Notify( MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING); Reader() ->WaitForData(aReject.mType) ->Then( OwnerThread(), __func__, [this](MediaData::Type aType) { AUTO_PROFILER_LABEL( "AccurateSeekingState::OnSeekRejected:WaitDataResolved", MEDIA_PLAYBACK); SLOG("OnSeekRejected wait promise resolved"); mWaitRequest.Complete(); DemuxerSeek(); }, [this](const WaitForDataRejectValue& aRejection) { AUTO_PROFILER_LABEL( "AccurateSeekingState::OnSeekRejected:WaitDataRejected", MEDIA_PLAYBACK); SLOG("OnSeekRejected wait promise rejected"); mWaitRequest.Complete(); mMaster->DecodeError(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA); }) ->Track(mWaitRequest); return; } if (aReject.mError == NS_ERROR_DOM_MEDIA_END_OF_STREAM) { if (!mDoneAudioSeeking) { HandleEndOfAudioInternal(); } if (!mDoneVideoSeeking) { HandleEndOfVideoInternal(); } MaybeFinishSeek(); return; } MOZ_ASSERT(NS_FAILED(aReject.mError), "Cancels should also disconnect mSeekRequest"); mMaster->DecodeError(aReject.mError); } void RequestAudioData() { MOZ_ASSERT(!mDoneAudioSeeking); mMaster->RequestAudioData(); } virtual void RequestVideoData() { MOZ_ASSERT(!mDoneVideoSeeking); mMaster->RequestVideoData(media::TimeUnit()); } void AdjustFastSeekIfNeeded(MediaData* aSample) { if (mSeekJob.mTarget->IsFast() && mSeekJob.mTarget->GetTime() > mCurrentTimeBeforeSeek && aSample->mTime < mCurrentTimeBeforeSeek) { // We are doing a fastSeek, but we ended up *before* the previous // playback position. This is surprising UX, so switch to an accurate // seek and decode to the seek target. This is not conformant to the // spec, fastSeek should always be fast, but until we get the time to // change all Readers to seek to the keyframe after the currentTime // in this case, we'll just decode forward. Bug 1026330. mSeekJob.mTarget->SetType(SeekTarget::Accurate); } } nsresult DropAudioUpToSeekTarget(AudioData* aAudio) { MOZ_ASSERT(aAudio && mSeekJob.mTarget->IsAccurate()); if (mSeekJob.mTarget->GetTime() >= aAudio->GetEndTime()) { // Our seek target lies after the frames in this AudioData. Don't // push it onto the audio queue, and keep decoding forwards. return NS_OK; } if (aAudio->mTime > mSeekJob.mTarget->GetTime()) { // The seek target doesn't lie in the audio block just after the last // audio frames we've seen which were before the seek target. This // could have been the first audio data we've seen after seek, i.e. the // seek terminated after the seek target in the audio stream. Just // abort the audio decode-to-target, the state machine will play // silence to cover the gap. Typically this happens in poorly muxed // files. SLOGW("Audio not synced after seek, maybe a poorly muxed file?"); mMaster->PushAudio(aAudio); mDoneAudioSeeking = true; return NS_OK; } bool ok = aAudio->SetTrimWindow( {mSeekJob.mTarget->GetTime(), aAudio->GetEndTime()}); if (!ok) { return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR; } MOZ_ASSERT(AudioQueue().GetSize() == 0, "Should be the 1st sample after seeking"); mMaster->PushAudio(aAudio); mDoneAudioSeeking = true; return NS_OK; } nsresult DropVideoUpToSeekTarget(VideoData* aVideo) { MOZ_ASSERT(aVideo); SLOG("DropVideoUpToSeekTarget() frame [%" PRId64 ", %" PRId64 "]", aVideo->mTime.ToMicroseconds(), aVideo->GetEndTime().ToMicroseconds()); const auto target = GetSeekTarget(); // If the frame end time is less than the seek target, we won't want // to display this frame after the seek, so discard it. if (target >= aVideo->GetEndTime()) { SLOG("DropVideoUpToSeekTarget() pop video frame [%" PRId64 ", %" PRId64 "] target=%" PRId64, aVideo->mTime.ToMicroseconds(), aVideo->GetEndTime().ToMicroseconds(), target.ToMicroseconds()); PROFILER_MARKER_UNTYPED("MDSM::DropVideoUpToSeekTarget", MEDIA_PLAYBACK); mFirstVideoFrameAfterSeek = aVideo; } else { if (target >= aVideo->mTime && aVideo->GetEndTime() >= target) { // The seek target lies inside this frame's time slice. Adjust the // frame's start time to match the seek target. aVideo->UpdateTimestamp(target); } mFirstVideoFrameAfterSeek = nullptr; SLOG("DropVideoUpToSeekTarget() found video frame [%" PRId64 ", %" PRId64 "] containing target=%" PRId64, aVideo->mTime.ToMicroseconds(), aVideo->GetEndTime().ToMicroseconds(), target.ToMicroseconds()); MOZ_ASSERT(VideoQueue().GetSize() == 0, "Should be the 1st sample after seeking"); mMaster->PushVideo(aVideo); mDoneVideoSeeking = true; } return NS_OK; } void HandleEndOfAudioInternal() { MOZ_ASSERT(!mDoneAudioSeeking); AudioQueue().Finish(); mDoneAudioSeeking = true; } void HandleEndOfVideoInternal() { MOZ_ASSERT(!mDoneVideoSeeking); if (mFirstVideoFrameAfterSeek) { // Hit the end of stream. Move mFirstVideoFrameAfterSeek into // mSeekedVideoData so we have something to display after seeking. mMaster->PushVideo(mFirstVideoFrameAfterSeek); } VideoQueue().Finish(); mDoneVideoSeeking = true; } void MaybeFinishSeek() { if (mDoneAudioSeeking && mDoneVideoSeeking) { SeekCompleted(); } } /* * Track the current seek promise made by the reader. */ MozPromiseRequestHolder mSeekRequest; /* * Internal state. */ media::TimeUnit mCurrentTimeBeforeSeek; bool mDoneAudioSeeking = false; bool mDoneVideoSeeking = false; MozPromiseRequestHolder mWaitRequest; // This temporarily stores the first frame we decode after we seek. // This is so that if we hit end of stream while we're decoding to reach // the seek target, we will still have a frame that we can display as the // last frame in the media. RefPtr mFirstVideoFrameAfterSeek; private: virtual media::TimeUnit GetSeekTarget() const { return mSeekJob.mTarget->GetTime(); } }; /* * Remove samples from the queue until aCompare() returns false. * aCompare A function object with the signature bool(int64_t) which returns * true for samples that should be removed. */ template static void DiscardFrames(MediaQueue& aQueue, const Function& aCompare) { while (aQueue.GetSize() > 0) { if (aCompare(aQueue.PeekFront()->mTime.ToMicroseconds())) { RefPtr releaseMe = aQueue.PopFront(); continue; } break; } } class MediaDecoderStateMachine::NextFrameSeekingState : public MediaDecoderStateMachine::SeekingState { public: explicit NextFrameSeekingState(Master* aPtr) : SeekingState(aPtr) {} State GetState() const override { return DECODER_STATE_SEEKING_NEXTFRAMESEEKING; } RefPtr Enter(SeekJob&& aSeekJob, EventVisibility aVisibility) { MOZ_ASSERT(aSeekJob.mTarget->IsNextFrame()); mCurrentTime = mMaster->GetMediaTime(); mDuration = mMaster->Duration(); return SeekingState::Enter(std::move(aSeekJob), aVisibility); } void Exit() override { // Disconnect my async seek operation. if (mAsyncSeekTask) { mAsyncSeekTask->Cancel(); } // Disconnect MediaDecoder. mSeekJob.RejectIfExists(__func__); } void HandleAudioDecoded(AudioData* aAudio) override { mMaster->PushAudio(aAudio); } void HandleVideoDecoded(VideoData* aVideo) override { MOZ_ASSERT(aVideo); MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished"); MOZ_ASSERT(NeedMoreVideo()); if (aVideo->mTime > mCurrentTime) { mMaster->PushVideo(aVideo); FinishSeek(); } else { RequestVideoData(); } } void HandleWaitingForAudio() override { MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished"); // We don't care about audio decode errors in this state which will be // handled by other states after seeking. } void HandleAudioCanceled() override { MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished"); // We don't care about audio decode errors in this state which will be // handled by other states after seeking. } void HandleEndOfAudio() override { MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished"); // We don't care about audio decode errors in this state which will be // handled by other states after seeking. } void HandleWaitingForVideo() override { MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished"); MOZ_ASSERT(NeedMoreVideo()); mMaster->WaitForData(MediaData::Type::VIDEO_DATA); } void HandleVideoCanceled() override { MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished"); MOZ_ASSERT(NeedMoreVideo()); RequestVideoData(); } void HandleEndOfVideo() override { MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished"); MOZ_ASSERT(NeedMoreVideo()); VideoQueue().Finish(); FinishSeek(); } void HandleAudioWaited(MediaData::Type aType) override { // We don't care about audio in this state. } void HandleVideoWaited(MediaData::Type aType) override { MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished"); MOZ_ASSERT(NeedMoreVideo()); RequestVideoData(); } TimeUnit CalculateNewCurrentTime() const override { // The HTMLMediaElement.currentTime should be updated to the seek target // which has been updated to the next frame's time. return mSeekJob.mTarget->GetTime(); } void DoSeek() override { mMaster->StopMediaSink(); auto currentTime = mCurrentTime; DiscardFrames(VideoQueue(), [currentTime](int64_t aSampleTime) { return aSampleTime <= currentTime.ToMicroseconds(); }); // If there is a pending video request, finish the seeking if we don't need // more data, or wait for HandleVideoDecoded() to finish seeking. if (mMaster->IsRequestingVideoData()) { if (!NeedMoreVideo()) { FinishSeek(); } return; } // Otherwise, we need to do the seek operation asynchronously for a special // case (bug504613.ogv) which has no data at all, the 1st seekToNextFrame() // operation reaches the end of the media. If we did the seek operation // synchronously, we immediately resolve the SeekPromise in mSeekJob and // then switch to the CompletedState which dispatches an "ended" event. // However, the ThenValue of the SeekPromise has not yet been set, so the // promise resolving is postponed and then the JS developer receives the // "ended" event before the seek promise is resolved. // An asynchronous seek operation helps to solve this issue since while the // seek is actually performed, the ThenValue of SeekPromise has already // been set so that it won't be postponed. RefPtr r = mAsyncSeekTask = new AysncNextFrameSeekTask(this); nsresult rv = OwnerThread()->Dispatch(r.forget()); MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); Unused << rv; } private: void DoSeekInternal() { // We don't need to discard frames to the mCurrentTime here because we have // done it at DoSeek() and any video data received in between either // finishes the seek operation or be discarded, see HandleVideoDecoded(). if (!NeedMoreVideo()) { FinishSeek(); } else if (!mMaster->IsRequestingVideoData() && !mMaster->IsWaitingVideoData()) { RequestVideoData(); } } class AysncNextFrameSeekTask : public Runnable { public: explicit AysncNextFrameSeekTask(NextFrameSeekingState* aStateObject) : Runnable( "MediaDecoderStateMachine::NextFrameSeekingState::" "AysncNextFrameSeekTask"), mStateObj(aStateObject) {} void Cancel() { mStateObj = nullptr; } NS_IMETHOD Run() override { if (mStateObj) { AUTO_PROFILER_LABEL("AysncNextFrameSeekTask::Run", MEDIA_PLAYBACK); mStateObj->DoSeekInternal(); } return NS_OK; } private: NextFrameSeekingState* mStateObj; }; void RequestVideoData() { mMaster->RequestVideoData(media::TimeUnit()); } bool NeedMoreVideo() const { // Need to request video when we have none and video queue is not finished. return VideoQueue().GetSize() == 0 && !VideoQueue().IsFinished(); } // Update the seek target's time before resolving this seek task, the updated // time will be used in the MDSM::SeekCompleted() to update the MDSM's // position. void UpdateSeekTargetTime() { RefPtr data = VideoQueue().PeekFront(); if (data) { mSeekJob.mTarget->SetTime(data->mTime); } else { MOZ_ASSERT(VideoQueue().AtEndOfStream()); mSeekJob.mTarget->SetTime(mDuration); } } void FinishSeek() { MOZ_ASSERT(!NeedMoreVideo()); UpdateSeekTargetTime(); auto time = mSeekJob.mTarget->GetTime().ToMicroseconds(); DiscardFrames(AudioQueue(), [time](int64_t aSampleTime) { return aSampleTime < time; }); SeekCompleted(); } /* * Internal state. */ TimeUnit mCurrentTime; TimeUnit mDuration; RefPtr mAsyncSeekTask; }; class MediaDecoderStateMachine::NextFrameSeekingFromDormantState : public MediaDecoderStateMachine::AccurateSeekingState { public: explicit NextFrameSeekingFromDormantState(Master* aPtr) : AccurateSeekingState(aPtr) {} State GetState() const override { return DECODER_STATE_SEEKING_FROMDORMANT; } RefPtr Enter(SeekJob&& aCurrentSeekJob, SeekJob&& aFutureSeekJob) { mFutureSeekJob = std::move(aFutureSeekJob); AccurateSeekingState::Enter(std::move(aCurrentSeekJob), EventVisibility::Suppressed); // Once seekToNextFrame() is called, we assume the user is likely to keep // calling seekToNextFrame() repeatedly, and so, we should prevent the MDSM // from getting into Dormant state. mMaster->mMinimizePreroll = false; return mFutureSeekJob.mPromise.Ensure(__func__); } void Exit() override { mFutureSeekJob.RejectIfExists(__func__); AccurateSeekingState::Exit(); } private: SeekJob mFutureSeekJob; // We don't want to transition to DecodingState once this seek completes, // instead, we transition to NextFrameSeekingState. void GoToNextState() override { SetState(std::move(mFutureSeekJob), EventVisibility::Observable); } }; class MediaDecoderStateMachine::VideoOnlySeekingState : public MediaDecoderStateMachine::AccurateSeekingState { public: explicit VideoOnlySeekingState(Master* aPtr) : AccurateSeekingState(aPtr) {} State GetState() const override { return DECODER_STATE_SEEKING_VIDEOONLY; } RefPtr Enter(SeekJob&& aSeekJob, EventVisibility aVisibility) { MOZ_ASSERT(aSeekJob.mTarget->IsVideoOnly()); MOZ_ASSERT(aVisibility == EventVisibility::Suppressed); RefPtr p = AccurateSeekingState::Enter(std::move(aSeekJob), aVisibility); // Dispatch a mozvideoonlyseekbegin event to indicate UI for corresponding // changes. mMaster->mOnPlaybackEvent.Notify(MediaPlaybackEvent::VideoOnlySeekBegin); return p; } void Exit() override { // We are completing or discarding this video-only seek operation now, // dispatch an event so that the UI can change in response to the end // of video-only seek. mMaster->mOnPlaybackEvent.Notify( MediaPlaybackEvent::VideoOnlySeekCompleted); AccurateSeekingState::Exit(); } void HandleAudioDecoded(AudioData* aAudio) override { MOZ_ASSERT(mDoneAudioSeeking && !mDoneVideoSeeking, "Seek shouldn't be finished"); MOZ_ASSERT(aAudio); // Video-only seek doesn't reset audio decoder. There might be pending audio // requests when AccurateSeekTask::Seek() begins. We will just store the // data without checking |mDiscontinuity| or calling // DropAudioUpToSeekTarget(). mMaster->PushAudio(aAudio); } void HandleWaitingForAudio() override {} void HandleAudioCanceled() override {} void HandleEndOfAudio() override {} void HandleAudioWaited(MediaData::Type aType) override { MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking, "Seek shouldn't be finished"); // Ignore pending requests from video-only seek. } void DoSeek() override { // TODO: keep decoding audio. mDoneAudioSeeking = true; mDoneVideoSeeking = !Info().HasVideo(); const auto offset = VideoQueue().GetOffset(); mMaster->ResetDecode(TrackInfo::kVideoTrack); // Entering video-only state and we've looped at least once before, so we // need to set offset in order to let new video frames catch up with the // clock time. if (offset != media::TimeUnit::Zero()) { VideoQueue().SetOffset(offset); } DemuxerSeek(); } protected: // Allow skip-to-next-key-frame to kick in if we fall behind the current // playback position so decoding has a better chance to catch up. void RequestVideoData() override { MOZ_ASSERT(!mDoneVideoSeeking); auto clock = mMaster->mMediaSink->IsStarted() ? mMaster->GetClock() : mMaster->GetMediaTime(); mMaster->AdjustByLooping(clock); const auto& nextKeyFrameTime = GetNextKeyFrameTime(); auto threshold = clock; if (nextKeyFrameTime.IsValid() && clock >= (nextKeyFrameTime - sSkipToNextKeyFrameThreshold)) { threshold = nextKeyFrameTime; } mMaster->RequestVideoData(threshold); } private: // Trigger skip to next key frame if the current playback position is very // close the next key frame's time. static constexpr TimeUnit sSkipToNextKeyFrameThreshold = TimeUnit::FromMicroseconds(5000); // If the media is playing, drop video until catch up playback position. media::TimeUnit GetSeekTarget() const override { auto target = mMaster->mMediaSink->IsStarted() ? mMaster->GetClock() : mSeekJob.mTarget->GetTime(); mMaster->AdjustByLooping(target); return target; } media::TimeUnit GetNextKeyFrameTime() const { // We only call this method in RequestVideoData() and we only request video // data if we haven't done video seeking. MOZ_DIAGNOSTIC_ASSERT(!mDoneVideoSeeking); MOZ_DIAGNOSTIC_ASSERT(mMaster->VideoQueue().GetSize() == 0); if (mFirstVideoFrameAfterSeek) { return mFirstVideoFrameAfterSeek->NextKeyFrameTime(); } return TimeUnit::Invalid(); } }; constexpr TimeUnit MediaDecoderStateMachine::VideoOnlySeekingState:: sSkipToNextKeyFrameThreshold; RefPtr MediaDecoderStateMachine::DormantState::HandleSeek(const SeekTarget& aTarget) { if (aTarget.IsNextFrame()) { // NextFrameSeekingState doesn't reset the decoder unlike // AccurateSeekingState. So we first must come out of dormant by seeking to // mPendingSeek and continue later with the NextFrameSeek SLOG("Changed state to SEEKING (to %" PRId64 ")", aTarget.GetTime().ToMicroseconds()); SeekJob seekJob; seekJob.mTarget = Some(aTarget); return StateObject::SetState( std::move(mPendingSeek), std::move(seekJob)); } return StateObject::HandleSeek(aTarget); } /** * Purpose: stop playback until enough data is decoded to continue playback. * * Transition to: * SEEKING if any seek request. * SHUTDOWN if any decode error. * COMPLETED when having decoded all audio/video data. * DECODING/LOOPING_DECODING when having decoded enough data to continue * playback. */ class MediaDecoderStateMachine::BufferingState : public MediaDecoderStateMachine::StateObject { public: explicit BufferingState(Master* aPtr) : StateObject(aPtr) {} void Enter() { if (mMaster->IsPlaying()) { mMaster->StopPlayback(); } mBufferingStart = TimeStamp::Now(); mMaster->ScheduleStateMachineIn(TimeUnit::FromMicroseconds(USECS_PER_S)); mMaster->mOnNextFrameStatus.Notify( MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING); } void Step() override; State GetState() const override { return DECODER_STATE_BUFFERING; } void HandleAudioDecoded(AudioData* aAudio) override { mMaster->PushAudio(aAudio); if (!mMaster->HaveEnoughDecodedAudio()) { mMaster->RequestAudioData(); } // This might be the sample we need to exit buffering. // Schedule Step() to check it. mMaster->ScheduleStateMachine(); } void HandleVideoDecoded(VideoData* aVideo) override { mMaster->PushVideo(aVideo); if (!mMaster->HaveEnoughDecodedVideo()) { mMaster->RequestVideoData(media::TimeUnit()); } // This might be the sample we need to exit buffering. // Schedule Step() to check it. mMaster->ScheduleStateMachine(); } void HandleAudioCanceled() override { mMaster->RequestAudioData(); } void HandleVideoCanceled() override { mMaster->RequestVideoData(media::TimeUnit()); } void HandleWaitingForAudio() override { mMaster->WaitForData(MediaData::Type::AUDIO_DATA); } void HandleWaitingForVideo() override { mMaster->WaitForData(MediaData::Type::VIDEO_DATA); } void HandleAudioWaited(MediaData::Type aType) override { mMaster->RequestAudioData(); } void HandleVideoWaited(MediaData::Type aType) override { mMaster->RequestVideoData(media::TimeUnit()); } void HandleEndOfAudio() override; void HandleEndOfVideo() override; void HandleVideoSuspendTimeout() override { // No video, so nothing to suspend. if (!mMaster->HasVideo()) { return; } mMaster->mVideoDecodeSuspended = true; mMaster->mOnPlaybackEvent.Notify(MediaPlaybackEvent::EnterVideoSuspend); Reader()->SetVideoBlankDecode(true); } private: TimeStamp mBufferingStart; // The maximum number of second we spend buffering when we are short on // unbuffered data. const uint32_t mBufferingWait = 15; }; /** * Purpose: play all the decoded data and fire the 'ended' event. * * Transition to: * SEEKING if any seek request. * LOOPING_DECODING if MDSM enable looping. */ class MediaDecoderStateMachine::CompletedState : public MediaDecoderStateMachine::StateObject { public: explicit CompletedState(Master* aPtr) : StateObject(aPtr) {} void Enter() { // On Android, the life cycle of graphic buffer is equal to Android's codec, // we couldn't release it if we still need to render the frame. #ifndef MOZ_WIDGET_ANDROID if (!mMaster->mLooping) { // We've decoded all samples. // We don't need decoders anymore if not looping. Reader()->ReleaseResources(); } #endif bool hasNextFrame = (!mMaster->HasAudio() || !mMaster->mAudioCompleted) && (!mMaster->HasVideo() || !mMaster->mVideoCompleted); mMaster->mOnNextFrameStatus.Notify( hasNextFrame ? MediaDecoderOwner::NEXT_FRAME_AVAILABLE : MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE); Step(); } void Exit() override { mSentPlaybackEndedEvent = false; } void Step() override { if (mMaster->mPlayState != MediaDecoder::PLAY_STATE_PLAYING && mMaster->IsPlaying()) { mMaster->StopPlayback(); } // Play the remaining media. We want to run AdvanceFrame() at least // once to ensure the current playback position is advanced to the // end of the media, and so that we update the readyState. if ((mMaster->HasVideo() && !mMaster->mVideoCompleted) || (mMaster->HasAudio() && !mMaster->mAudioCompleted)) { // Start playback if necessary to play the remaining media. mMaster->MaybeStartPlayback(); mMaster->UpdatePlaybackPositionPeriodically(); MOZ_ASSERT(!mMaster->IsPlaying() || mMaster->IsStateMachineScheduled(), "Must have timer scheduled"); return; } // StopPlayback in order to reset the IsPlaying() state so audio // is restarted correctly. mMaster->StopPlayback(); if (!mSentPlaybackEndedEvent) { auto clockTime = std::max(mMaster->AudioEndTime(), mMaster->VideoEndTime()); // Correct the time over the end once looping was turned on. mMaster->AdjustByLooping(clockTime); if (mMaster->mDuration.Ref()->IsInfinite()) { // We have a finite duration when playback reaches the end. mMaster->mDuration = Some(clockTime); DDLOGEX(mMaster, DDLogCategory::Property, "duration_us", mMaster->mDuration.Ref()->ToMicroseconds()); } mMaster->UpdatePlaybackPosition(clockTime); // Ensure readyState is updated before firing the 'ended' event. mMaster->mOnNextFrameStatus.Notify( MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE); mMaster->mOnPlaybackEvent.Notify(MediaPlaybackEvent::PlaybackEnded); mSentPlaybackEndedEvent = true; // MediaSink::GetEndTime() must be called before stopping playback. mMaster->StopMediaSink(); } } State GetState() const override { return DECODER_STATE_COMPLETED; } void HandleLoopingChanged() override { if (mMaster->mLooping) { SetDecodingState(); } } void HandleAudioCaptured() override { // MediaSink is changed. Schedule Step() to check if we can start playback. mMaster->ScheduleStateMachine(); } void HandleVideoSuspendTimeout() override { // Do nothing since no decoding is going on. } void HandleResumeVideoDecoding(const TimeUnit&) override { // Resume the video decoder and seek to the last video frame. // This triggers a video-only seek which won't update the playback position. auto target = mMaster->mDecodedVideoEndTime; mMaster->AdjustByLooping(target); StateObject::HandleResumeVideoDecoding(target); } void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) override { if (aPlayState == MediaDecoder::PLAY_STATE_PLAYING) { // Schedule Step() to check if we can start playback. mMaster->ScheduleStateMachine(); } } private: bool mSentPlaybackEndedEvent = false; }; /** * Purpose: release all resources allocated by MDSM. * * Transition to: * None since this is the final state. * * Transition from: * Any states other than SHUTDOWN. */ class MediaDecoderStateMachine::ShutdownState : public MediaDecoderStateMachine::StateObject { public: explicit ShutdownState(Master* aPtr) : StateObject(aPtr) {} RefPtr Enter(); void Exit() override { MOZ_DIAGNOSTIC_ASSERT(false, "Shouldn't escape the SHUTDOWN state."); } State GetState() const override { return DECODER_STATE_SHUTDOWN; } RefPtr HandleSeek( const SeekTarget& aTarget) override { MOZ_DIAGNOSTIC_ASSERT(false, "Can't seek in shutdown state."); return MediaDecoder::SeekPromise::CreateAndReject(true, __func__); } RefPtr HandleShutdown() override { MOZ_DIAGNOSTIC_ASSERT(false, "Already shutting down."); return nullptr; } void HandleVideoSuspendTimeout() override { MOZ_DIAGNOSTIC_ASSERT(false, "Already shutting down."); } void HandleResumeVideoDecoding(const TimeUnit&) override { MOZ_DIAGNOSTIC_ASSERT(false, "Already shutting down."); } }; RefPtr MediaDecoderStateMachine::StateObject::HandleSeek(const SeekTarget& aTarget) { SLOG("Changed state to SEEKING (to %" PRId64 ")", aTarget.GetTime().ToMicroseconds()); SeekJob seekJob; seekJob.mTarget = Some(aTarget); return SetSeekingState(std::move(seekJob), EventVisibility::Observable); } RefPtr MediaDecoderStateMachine::StateObject::HandleShutdown() { return SetState(); } static void ReportRecoveryTelemetry(const TimeStamp& aRecoveryStart, const MediaInfo& aMediaInfo, bool aIsHardwareAccelerated) { MOZ_ASSERT(NS_IsMainThread()); if (!aMediaInfo.HasVideo()) { return; } // Keyed by audio+video or video alone, hardware acceleration, // and by a resolution range. nsCString key(aMediaInfo.HasAudio() ? "AV" : "V"); key.AppendASCII(aIsHardwareAccelerated ? "(hw)," : ","); static const struct { int32_t mH; const char* mRes; } sResolutions[] = {{240, "0-240"}, {480, "241-480"}, {720, "481-720"}, {1080, "721-1080"}, {2160, "1081-2160"}}; const char* resolution = "2161+"; int32_t height = aMediaInfo.mVideo.mImage.height; for (const auto& res : sResolutions) { if (height <= res.mH) { resolution = res.mRes; break; } } key.AppendASCII(resolution); TimeDuration duration = TimeStamp::Now() - aRecoveryStart; double duration_ms = duration.ToMilliseconds(); Telemetry::Accumulate(Telemetry::VIDEO_SUSPEND_RECOVERY_TIME_MS, key, static_cast(lround(duration_ms))); Telemetry::Accumulate(Telemetry::VIDEO_SUSPEND_RECOVERY_TIME_MS, "All"_ns, static_cast(lround(duration_ms))); } void MediaDecoderStateMachine::StateObject::HandleResumeVideoDecoding( const TimeUnit& aTarget) { MOZ_ASSERT(mMaster->mVideoDecodeSuspended); mMaster->mVideoDecodeSuspended = false; mMaster->mOnPlaybackEvent.Notify(MediaPlaybackEvent::ExitVideoSuspend); Reader()->SetVideoBlankDecode(false); // Start counting recovery time from right now. TimeStamp start = TimeStamp::Now(); // Local reference to mInfo, so that it will be copied in the lambda below. const auto& info = Info(); bool hw = Reader()->VideoIsHardwareAccelerated(); // Start video-only seek to the current time. SeekJob seekJob; // We use fastseek to optimize the resuming time. // FastSeek is only used for video-only media since we don't need to worry // about A/V sync. // Don't use fastSeek if we want to seek to the end because it might seek to a // keyframe before the last frame (if the last frame itself is not a keyframe) // and we always want to present the final frame to the user when seeking to // the end. const auto type = mMaster->HasAudio() || aTarget == mMaster->Duration() ? SeekTarget::Type::Accurate : SeekTarget::Type::PrevSyncPoint; seekJob.mTarget.emplace(aTarget, type, SeekTarget::Track::VideoOnly); SLOG("video-only seek target=%" PRId64 ", current time=%" PRId64, aTarget.ToMicroseconds(), mMaster->GetMediaTime().ToMicroseconds()); // Hold mMaster->mAbstractMainThread here because this->mMaster will be // invalid after the current state object is deleted in SetState(); RefPtr mainThread = mMaster->mAbstractMainThread; SetSeekingState(std::move(seekJob), EventVisibility::Suppressed) ->Then( mainThread, __func__, [start, info, hw]() { ReportRecoveryTelemetry(start, info, hw); }, []() {}); } RefPtr MediaDecoderStateMachine::StateObject::SetSeekingState( SeekJob&& aSeekJob, EventVisibility aVisibility) { if (aSeekJob.mTarget->IsAccurate() || aSeekJob.mTarget->IsFast()) { if (aSeekJob.mTarget->IsVideoOnly()) { return SetState(std::move(aSeekJob), aVisibility); } return SetState(std::move(aSeekJob), aVisibility); } if (aSeekJob.mTarget->IsNextFrame()) { return SetState(std::move(aSeekJob), aVisibility); } MOZ_ASSERT_UNREACHABLE("Unknown SeekTarget::Type."); return nullptr; } void MediaDecoderStateMachine::StateObject::SetDecodingState() { if (mMaster->IsInSeamlessLooping()) { SetState(); return; } SetState(); } void MediaDecoderStateMachine::DecodeMetadataState::OnMetadataRead( MetadataHolder&& aMetadata) { mMetadataRequest.Complete(); AUTO_PROFILER_LABEL("DecodeMetadataState::OnMetadataRead", MEDIA_PLAYBACK); mMaster->mInfo.emplace(*aMetadata.mInfo); mMaster->mMediaSeekable = Info().mMediaSeekable; mMaster->mMediaSeekableOnlyInBufferedRanges = Info().mMediaSeekableOnlyInBufferedRanges; if (Info().mMetadataDuration.isSome()) { mMaster->mDuration = Info().mMetadataDuration; } else if (Info().mUnadjustedMetadataEndTime.isSome()) { const TimeUnit unadjusted = Info().mUnadjustedMetadataEndTime.ref(); const TimeUnit adjustment = Info().mStartTime; mMaster->mInfo->mMetadataDuration.emplace(unadjusted - adjustment); mMaster->mDuration = Info().mMetadataDuration; } // If we don't know the duration by this point, we assume infinity, per spec. if (mMaster->mDuration.Ref().isNothing()) { mMaster->mDuration = Some(TimeUnit::FromInfinity()); } DDLOGEX(mMaster, DDLogCategory::Property, "duration_us", mMaster->mDuration.Ref()->ToMicroseconds()); if (mMaster->HasVideo()) { SLOG("Video decode HWAccel=%d videoQueueSize=%d", Reader()->VideoIsHardwareAccelerated(), mMaster->GetAmpleVideoFrames()); } MOZ_ASSERT(mMaster->mDuration.Ref().isSome()); mMaster->mMetadataLoadedEvent.Notify(std::move(aMetadata.mInfo), std::move(aMetadata.mTags), MediaDecoderEventVisibility::Observable); // Check whether the media satisfies the requirement of seamless looping. // TODO : after we ensure video seamless looping is stable enough, then we can // remove this to make the condition always true. mMaster->mSeamlessLoopingAllowed = StaticPrefs::media_seamless_looping(); if (mMaster->HasVideo()) { mMaster->mSeamlessLoopingAllowed = StaticPrefs::media_seamless_looping_video(); } SetState(); } void MediaDecoderStateMachine::DormantState::HandlePlayStateChanged( MediaDecoder::PlayState aPlayState) { if (aPlayState == MediaDecoder::PLAY_STATE_PLAYING) { // Exit dormant when the user wants to play. MOZ_ASSERT(mMaster->mSentFirstFrameLoadedEvent); SetSeekingState(std::move(mPendingSeek), EventVisibility::Suppressed); } } void MediaDecoderStateMachine::DecodingFirstFrameState::Enter() { // Transition to DECODING if we've decoded first frames. if (mMaster->mSentFirstFrameLoadedEvent) { SetDecodingState(); return; } MOZ_ASSERT(!mMaster->mVideoDecodeSuspended); // Dispatch tasks to decode first frames. if (mMaster->HasAudio()) { mMaster->RequestAudioData(); } if (mMaster->HasVideo()) { mMaster->RequestVideoData(media::TimeUnit()); } } void MediaDecoderStateMachine::DecodingFirstFrameState:: MaybeFinishDecodeFirstFrame() { MOZ_ASSERT(!mMaster->mSentFirstFrameLoadedEvent); if ((mMaster->IsAudioDecoding() && AudioQueue().GetSize() == 0) || (mMaster->IsVideoDecoding() && VideoQueue().GetSize() == 0)) { return; } mMaster->FinishDecodeFirstFrame(); if (mPendingSeek.Exists()) { SetSeekingState(std::move(mPendingSeek), EventVisibility::Observable); } else { SetDecodingState(); } } void MediaDecoderStateMachine::DecodingState::Enter() { MOZ_ASSERT(mMaster->mSentFirstFrameLoadedEvent); if (mMaster->mVideoDecodeSuspended && mMaster->mVideoDecodeMode == VideoDecodeMode::Normal) { StateObject::HandleResumeVideoDecoding(mMaster->GetMediaTime()); return; } if (mMaster->mVideoDecodeMode == VideoDecodeMode::Suspend && !mMaster->mVideoDecodeSuspendTimer.IsScheduled() && !mMaster->mVideoDecodeSuspended) { // If the VideoDecodeMode is Suspend and the timer is not schedule, it means // the timer has timed out and we should suspend video decoding now if // necessary. HandleVideoSuspendTimeout(); } // If we're in the normal decoding mode and the decoding has finished, then we // should go to `completed` state because we don't need to decode anything // later. However, if we're in the saemless decoding mode, we will restart // decoding ASAP so we can still stay in `decoding` state. if (!mMaster->IsVideoDecoding() && !mMaster->IsAudioDecoding() && !mMaster->IsInSeamlessLooping()) { SetState(); return; } mOnAudioPopped = AudioQueue().PopFrontEvent().Connect(OwnerThread(), [this]() { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::OnAudioPopped", MEDIA_PLAYBACK); if (mMaster->IsAudioDecoding() && !mMaster->HaveEnoughDecodedAudio()) { EnsureAudioDecodeTaskQueued(); } }); mOnVideoPopped = VideoQueue().PopFrontEvent().Connect(OwnerThread(), [this]() { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::OnVideoPopped", MEDIA_PLAYBACK); if (mMaster->IsVideoDecoding() && !mMaster->HaveEnoughDecodedVideo()) { EnsureVideoDecodeTaskQueued(); } }); mMaster->mOnNextFrameStatus.Notify(MediaDecoderOwner::NEXT_FRAME_AVAILABLE); mDecodeStartTime = TimeStamp::Now(); MaybeStopPrerolling(); // Ensure that we've got tasks enqueued to decode data if we need to. DispatchDecodeTasksIfNeeded(); mMaster->ScheduleStateMachine(); // Will enter dormant when playback is paused for a while. if (mMaster->mPlayState == MediaDecoder::PLAY_STATE_PAUSED) { StartDormantTimer(); } } void MediaDecoderStateMachine::DecodingState::Step() { if (mMaster->mPlayState != MediaDecoder::PLAY_STATE_PLAYING && mMaster->IsPlaying()) { // We're playing, but the element/decoder is in paused state. Stop // playing! mMaster->StopPlayback(); } // Start playback if necessary so that the clock can be properly queried. if (!mIsPrerolling) { mMaster->MaybeStartPlayback(); } mMaster->UpdatePlaybackPositionPeriodically(); MOZ_ASSERT(!mMaster->IsPlaying() || mMaster->IsStateMachineScheduled(), "Must have timer scheduled"); if (IsBufferingAllowed()) { MaybeStartBuffering(); } } void MediaDecoderStateMachine::DecodingState::HandleEndOfAudio() { AudioQueue().Finish(); if (!mMaster->IsVideoDecoding()) { SetState(); } else { MaybeStopPrerolling(); } } void MediaDecoderStateMachine::DecodingState::HandleEndOfVideo() { VideoQueue().Finish(); if (!mMaster->IsAudioDecoding()) { SetState(); } else { MaybeStopPrerolling(); } } void MediaDecoderStateMachine::DecodingState::DispatchDecodeTasksIfNeeded() { if (mMaster->IsAudioDecoding() && !mMaster->mMinimizePreroll && !mMaster->HaveEnoughDecodedAudio()) { EnsureAudioDecodeTaskQueued(); } if (mMaster->IsVideoDecoding() && !mMaster->mMinimizePreroll && !mMaster->HaveEnoughDecodedVideo()) { EnsureVideoDecodeTaskQueued(); } } void MediaDecoderStateMachine::DecodingState::EnsureAudioDecodeTaskQueued() { if (!mMaster->IsAudioDecoding() || mMaster->IsRequestingAudioData() || mMaster->IsWaitingAudioData()) { return; } mMaster->RequestAudioData(); } void MediaDecoderStateMachine::DecodingState::EnsureVideoDecodeTaskQueued() { if (!mMaster->IsVideoDecoding() || mMaster->IsRequestingVideoData() || mMaster->IsWaitingVideoData()) { return; } mMaster->RequestVideoData(mMaster->GetMediaTime(), ShouldRequestNextKeyFrame()); } void MediaDecoderStateMachine::DecodingState::MaybeStartBuffering() { // Buffering makes senses only after decoding first frames. MOZ_ASSERT(mMaster->mSentFirstFrameLoadedEvent); // Don't enter buffering when MediaDecoder is not playing. if (mMaster->mPlayState != MediaDecoder::PLAY_STATE_PLAYING) { return; } // Don't enter buffering while prerolling so that the decoder has a chance to // enqueue some decoded data before we give up and start buffering. if (!mMaster->IsPlaying()) { return; } // Note we could have a wait promise pending when playing non-MSE EME. if (mMaster->OutOfDecodedAudio() && mMaster->IsWaitingAudioData()) { PROFILER_MARKER_TEXT("MDSM::StartBuffering", MEDIA_PLAYBACK, {}, "OutOfDecodedAudio"); SLOG("Enter buffering due to out of decoded audio"); SetState(); return; } if (mMaster->OutOfDecodedVideo() && mMaster->IsWaitingVideoData()) { PROFILER_MARKER_TEXT("MDSM::StartBuffering", MEDIA_PLAYBACK, {}, "OutOfDecodedVideo"); SLOG("Enter buffering due to out of decoded video"); SetState(); return; } if (Reader()->UseBufferingHeuristics() && mMaster->HasLowDecodedData() && mMaster->HasLowBufferedData() && !mMaster->mCanPlayThrough) { PROFILER_MARKER_TEXT("MDSM::StartBuffering", MEDIA_PLAYBACK, {}, "BufferingHeuristics"); SLOG("Enter buffering due to buffering heruistics"); SetState(); } } void MediaDecoderStateMachine::LoopingDecodingState::HandleError( const MediaResult& aError, bool aIsAudio) { SLOG("%s looping failed, aError=%s", aIsAudio ? "audio" : "video", aError.ErrorName().get()); switch (aError.Code()) { case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA: if (aIsAudio) { HandleWaitingForAudio(); // Now we won't be able to get new audio from the start position. // This could happen for MSE, because data for the start hasn't been // appended yet. If we can get the data before all queued audio has been // consumed, then nothing special would happen. If not, then the audio // underrun would happen and the audio clock stalls, which means video // playback would also stall. Therefore, in this special situation, we // can treat those audio underrun as silent frames in order to keep // driving the clock. But we would cancel this behavior once the new // audio data comes, or we fallback to the non-seamless looping. mWaitingAudioDataFromStart = true; mMaster->mMediaSink->EnableTreatAudioUnderrunAsSilence(true); } else { HandleWaitingForVideo(); } [[fallthrough]]; case NS_ERROR_DOM_MEDIA_END_OF_STREAM: // This could happen after either the resource has been close, or the data // hasn't been appended in MSE, so that we won't be able to get any // sample and need to fallback to normal looping. if (mIsReachingAudioEOS && mIsReachingVideoEOS) { SetState(); } break; default: mMaster->DecodeError(aError); break; } } void MediaDecoderStateMachine::SeekingState::SeekCompleted() { const auto newCurrentTime = CalculateNewCurrentTime(); if ((newCurrentTime == mMaster->Duration() || newCurrentTime.EqualsAtLowestResolution( mMaster->Duration().ToBase(USECS_PER_S))) && !mMaster->mIsLiveStream) { SLOG("Seek completed, seeked to end: %s", newCurrentTime.ToString().get()); // will transition to COMPLETED immediately. Note we don't do // this when playing a live stream, since the end of media will advance // once we download more data! AudioQueue().Finish(); VideoQueue().Finish(); // We won't start MediaSink when paused. m{Audio,Video}Completed will // remain false and 'playbackEnded' won't be notified. Therefore we // need to set these flags explicitly when seeking to the end. mMaster->mAudioCompleted = true; mMaster->mVideoCompleted = true; // There might still be a pending audio request when doing video-only or // next-frame seek. Discard it so we won't break the invariants of the // COMPLETED state by adding audio samples to a finished queue. mMaster->mAudioDataRequest.DisconnectIfExists(); } // We want to resolve the seek request prior finishing the first frame // to ensure that the seeked event is fired prior loadeded. // Note: SeekJob.Resolve() resets SeekJob.mTarget. Don't use mSeekJob anymore // hereafter. mSeekJob.Resolve(__func__); // Notify FirstFrameLoaded now if we haven't since we've decoded some data // for readyState to transition to HAVE_CURRENT_DATA and fire 'loadeddata'. if (!mMaster->mSentFirstFrameLoadedEvent) { mMaster->FinishDecodeFirstFrame(); } // Ensure timestamps are up to date. // Suppressed visibility comes from two cases: (1) leaving dormant state, // and (2) resuming suspended video decoder. We want both cases to be // transparent to the user. So we only notify the change when the seek // request is from the user. if (mVisibility == EventVisibility::Observable) { // Don't update playback position for video-only seek. // Otherwise we might have |newCurrentTime > mMediaSink->GetPosition()| // and fail the assertion in GetClock() since we didn't stop MediaSink. mMaster->UpdatePlaybackPositionInternal(newCurrentTime); } // Try to decode another frame to detect if we're at the end... SLOG("Seek completed, mCurrentPosition=%" PRId64, mMaster->mCurrentPosition.Ref().ToMicroseconds()); if (mMaster->VideoQueue().PeekFront()) { mMaster->mMediaSink->Redraw(Info().mVideo); mMaster->mOnPlaybackEvent.Notify(MediaPlaybackEvent::Invalidate); } GoToNextState(); } void MediaDecoderStateMachine::BufferingState::Step() { TimeStamp now = TimeStamp::Now(); MOZ_ASSERT(!mBufferingStart.IsNull(), "Must know buffering start time."); if (Reader()->UseBufferingHeuristics()) { if (mMaster->IsWaitingAudioData() || mMaster->IsWaitingVideoData()) { // Can't exit buffering when we are still waiting for data. // Note we don't schedule next loop for we will do that when the wait // promise is resolved. return; } // With buffering heuristics, we exit buffering state when we: // 1. can play through or // 2. time out (specified by mBufferingWait) or // 3. have enough buffered data. TimeDuration elapsed = now - mBufferingStart; TimeDuration timeout = TimeDuration::FromSeconds(mBufferingWait * mMaster->mPlaybackRate); bool stopBuffering = mMaster->mCanPlayThrough || elapsed >= timeout || !mMaster->HasLowBufferedData(TimeUnit::FromSeconds(mBufferingWait)); if (!stopBuffering) { SLOG("Buffering: wait %ds, timeout in %.3lfs", mBufferingWait, mBufferingWait - elapsed.ToSeconds()); mMaster->ScheduleStateMachineIn(TimeUnit::FromMicroseconds(USECS_PER_S)); return; } } else if (mMaster->OutOfDecodedAudio() || mMaster->OutOfDecodedVideo()) { MOZ_ASSERT(!mMaster->OutOfDecodedAudio() || mMaster->IsRequestingAudioData() || mMaster->IsWaitingAudioData()); MOZ_ASSERT(!mMaster->OutOfDecodedVideo() || mMaster->IsRequestingVideoData() || mMaster->IsWaitingVideoData()); SLOG( "In buffering mode, waiting to be notified: outOfAudio: %d, " "mAudioStatus: %s, outOfVideo: %d, mVideoStatus: %s", mMaster->OutOfDecodedAudio(), mMaster->AudioRequestStatus(), mMaster->OutOfDecodedVideo(), mMaster->VideoRequestStatus()); return; } SLOG("Buffered for %.3lfs", (now - mBufferingStart).ToSeconds()); SetDecodingState(); } void MediaDecoderStateMachine::BufferingState::HandleEndOfAudio() { AudioQueue().Finish(); if (!mMaster->IsVideoDecoding()) { SetState(); } else { // Check if we can exit buffering. mMaster->ScheduleStateMachine(); } } void MediaDecoderStateMachine::BufferingState::HandleEndOfVideo() { VideoQueue().Finish(); if (!mMaster->IsAudioDecoding()) { SetState(); } else { // Check if we can exit buffering. mMaster->ScheduleStateMachine(); } } RefPtr MediaDecoderStateMachine::ShutdownState::Enter() { auto* master = mMaster; master->mDelayedScheduler.Reset(); // Shutdown happens while decode timer is active, we need to disconnect and // dispose of the timer. master->CancelSuspendTimer(); if (master->IsPlaying()) { master->StopPlayback(); } master->mAudioDataRequest.DisconnectIfExists(); master->mVideoDataRequest.DisconnectIfExists(); master->mAudioWaitRequest.DisconnectIfExists(); master->mVideoWaitRequest.DisconnectIfExists(); // Resetting decode should be called after stopping media sink, which can // ensure that we have an empty media queue before seeking the demuxer. master->StopMediaSink(); master->ResetDecode(); master->mMediaSink->Shutdown(); // Prevent dangling pointers by disconnecting the listeners. master->mAudioQueueListener.Disconnect(); master->mVideoQueueListener.Disconnect(); master->mMetadataManager.Disconnect(); master->mOnMediaNotSeekable.Disconnect(); master->mAudibleListener.DisconnectIfExists(); // Disconnect canonicals and mirrors before shutting down our task queue. master->mStreamName.DisconnectIfConnected(); master->mSinkDevice.DisconnectIfConnected(); master->mOutputCaptureState.DisconnectIfConnected(); master->mOutputDummyTrack.DisconnectIfConnected(); master->mOutputTracks.DisconnectIfConnected(); master->mOutputPrincipal.DisconnectIfConnected(); master->mDuration.DisconnectAll(); master->mCurrentPosition.DisconnectAll(); master->mIsAudioDataAudible.DisconnectAll(); // Shut down the watch manager to stop further notifications. master->mWatchManager.Shutdown(); return Reader()->Shutdown()->Then(OwnerThread(), __func__, master, &MediaDecoderStateMachine::FinishShutdown, &MediaDecoderStateMachine::FinishShutdown); } #define INIT_WATCHABLE(name, val) name(val, "MediaDecoderStateMachine::" #name) #define INIT_MIRROR(name, val) \ name(mTaskQueue, val, "MediaDecoderStateMachine::" #name " (Mirror)") #define INIT_CANONICAL(name, val) \ name(mTaskQueue, val, "MediaDecoderStateMachine::" #name " (Canonical)") MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, MediaFormatReader* aReader) : MediaDecoderStateMachineBase(aDecoder, aReader), mWatchManager(this, mTaskQueue), mDispatchedStateMachine(false), mDelayedScheduler(mTaskQueue, true /*aFuzzy*/), mCurrentFrameID(0), mAmpleAudioThreshold(detail::AMPLE_AUDIO_THRESHOLD), mVideoDecodeSuspended(false), mVideoDecodeSuspendTimer(mTaskQueue), mVideoDecodeMode(VideoDecodeMode::Normal), mIsMSE(aDecoder->IsMSE()), mShouldResistFingerprinting(aDecoder->ShouldResistFingerprinting()), mSeamlessLoopingAllowed(false), INIT_MIRROR(mStreamName, nsAutoString()), INIT_MIRROR(mSinkDevice, nullptr), INIT_MIRROR(mOutputCaptureState, MediaDecoder::OutputCaptureState::None), INIT_MIRROR(mOutputDummyTrack, nullptr), INIT_MIRROR(mOutputTracks, nsTArray>()), INIT_MIRROR(mOutputPrincipal, PRINCIPAL_HANDLE_NONE), INIT_CANONICAL(mCanonicalOutputTracks, nsTArray>()), INIT_CANONICAL(mCanonicalOutputPrincipal, PRINCIPAL_HANDLE_NONE) { MOZ_COUNT_CTOR(MediaDecoderStateMachine); NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); InitVideoQueuePrefs(); DDLINKCHILD("reader", aReader); } #undef INIT_WATCHABLE #undef INIT_MIRROR #undef INIT_CANONICAL MediaDecoderStateMachine::~MediaDecoderStateMachine() { MOZ_ASSERT(NS_IsMainThread(), "Should be on main thread."); MOZ_COUNT_DTOR(MediaDecoderStateMachine); } void MediaDecoderStateMachine::InitializationTask(MediaDecoder* aDecoder) { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::InitializationTask", MEDIA_PLAYBACK); MOZ_ASSERT(OnTaskQueue()); MediaDecoderStateMachineBase::InitializationTask(aDecoder); // Connect mirrors. mStreamName.Connect(aDecoder->CanonicalStreamName()); mSinkDevice.Connect(aDecoder->CanonicalSinkDevice()); mOutputCaptureState.Connect(aDecoder->CanonicalOutputCaptureState()); mOutputDummyTrack.Connect(aDecoder->CanonicalOutputDummyTrack()); mOutputTracks.Connect(aDecoder->CanonicalOutputTracks()); mOutputPrincipal.Connect(aDecoder->CanonicalOutputPrincipal()); // Initialize watchers. mWatchManager.Watch(mStreamName, &MediaDecoderStateMachine::StreamNameChanged); mWatchManager.Watch(mOutputCaptureState, &MediaDecoderStateMachine::UpdateOutputCaptured); mWatchManager.Watch(mOutputDummyTrack, &MediaDecoderStateMachine::UpdateOutputCaptured); mWatchManager.Watch(mOutputTracks, &MediaDecoderStateMachine::UpdateOutputCaptured); mWatchManager.Watch(mOutputTracks, &MediaDecoderStateMachine::OutputTracksChanged); mWatchManager.Watch(mOutputPrincipal, &MediaDecoderStateMachine::OutputPrincipalChanged); mMediaSink = CreateMediaSink(); MOZ_ASSERT(!mStateObj); auto* s = new DecodeMetadataState(this); mStateObj.reset(s); s->Enter(); } void MediaDecoderStateMachine::AudioAudibleChanged(bool aAudible) { mIsAudioDataAudible = aAudible; } MediaSink* MediaDecoderStateMachine::CreateAudioSink() { if (mOutputCaptureState != MediaDecoder::OutputCaptureState::None) { DecodedStream* stream = new DecodedStream( this, mOutputCaptureState == MediaDecoder::OutputCaptureState::Capture ? mOutputDummyTrack.Ref() : nullptr, mOutputTracks, mVolume, mPlaybackRate, mPreservesPitch, mAudioQueue, mVideoQueue, mSinkDevice.Ref()); mAudibleListener.DisconnectIfExists(); mAudibleListener = stream->AudibleEvent().Connect( OwnerThread(), this, &MediaDecoderStateMachine::AudioAudibleChanged); return stream; } auto audioSinkCreator = [s = RefPtr(this), this]() { MOZ_ASSERT(OnTaskQueue()); AudioSink* audioSink = new AudioSink(mTaskQueue, mAudioQueue, Info().mAudio, mShouldResistFingerprinting); mAudibleListener.DisconnectIfExists(); mAudibleListener = audioSink->AudibleEvent().Connect( mTaskQueue, this, &MediaDecoderStateMachine::AudioAudibleChanged); return audioSink; }; return new AudioSinkWrapper(mTaskQueue, mAudioQueue, audioSinkCreator, mVolume, mPlaybackRate, mPreservesPitch, mSinkDevice.Ref()); } already_AddRefed MediaDecoderStateMachine::CreateMediaSink() { MOZ_ASSERT(OnTaskQueue()); RefPtr audioSink = CreateAudioSink(); RefPtr mediaSink = new VideoSink(mTaskQueue, audioSink, mVideoQueue, mVideoFrameContainer, *mFrameStats, sVideoQueueSendToCompositorSize); if (mSecondaryVideoContainer.Ref()) { mediaSink->SetSecondaryVideoContainer(mSecondaryVideoContainer.Ref()); } return mediaSink.forget(); } TimeUnit MediaDecoderStateMachine::GetDecodedAudioDuration() const { MOZ_ASSERT(OnTaskQueue()); if (mMediaSink->IsStarted()) { return mMediaSink->UnplayedDuration(TrackInfo::kAudioTrack) + TimeUnit::FromMicroseconds(AudioQueue().Duration()); } // MediaSink not started. All audio samples are in the queue. return TimeUnit::FromMicroseconds(AudioQueue().Duration()); } bool MediaDecoderStateMachine::HaveEnoughDecodedAudio() const { MOZ_ASSERT(OnTaskQueue()); auto ampleAudio = mAmpleAudioThreshold.MultDouble(mPlaybackRate); return AudioQueue().GetSize() > 0 && GetDecodedAudioDuration() >= ampleAudio; } bool MediaDecoderStateMachine::HaveEnoughDecodedVideo() const { MOZ_ASSERT(OnTaskQueue()); return static_cast(VideoQueue().GetSize()) >= GetAmpleVideoFrames() * mPlaybackRate + 1 && IsVideoDataEnoughComparedWithAudio(); } bool MediaDecoderStateMachine::IsVideoDataEnoughComparedWithAudio() const { // HW decoding is usually fast enough and we don't need to worry about its // speed. // TODO : we can consider whether we need to enable this on other HW decoding // except VAAPI. When enabling VAAPI on Linux, ffmpeg is not able to store too // many frames because it has a limitation of amount of stored video frames. // See bug1716638 and 1718309. if (mReader->VideoIsHardwareAccelerated()) { return true; } // In extreme situations (e.g. 4k+ video without hardware acceleration), the // video decoding will be much slower than audio. So for 4K+ video, we want to // consider audio decoding speed as well in order to reduce frame drops. This // check tries to keep the decoded video buffered as much as audio. if (HasAudio() && Info().mVideo.mImage.width >= 3840 && Info().mVideo.mImage.height >= 2160) { return VideoQueue().Duration() >= AudioQueue().Duration(); } // For non-4k video, the video decoding is usually really fast so we won't // need to consider audio decoding speed to store extra frames. return true; } void MediaDecoderStateMachine::PushAudio(AudioData* aSample) { MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT(aSample); AudioQueue().Push(aSample); PROFILER_MARKER("MDSM::PushAudio", MEDIA_PLAYBACK, {}, MediaSampleMarker, aSample->mTime.ToMicroseconds(), aSample->GetEndTime().ToMicroseconds(), AudioQueue().GetSize()); } void MediaDecoderStateMachine::PushVideo(VideoData* aSample) { MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT(aSample); aSample->mFrameID = ++mCurrentFrameID; VideoQueue().Push(aSample); PROFILER_MARKER("MDSM::PushVideo", MEDIA_PLAYBACK, {}, MediaSampleMarker, aSample->mTime.ToMicroseconds(), aSample->GetEndTime().ToMicroseconds(), VideoQueue().GetSize()); } void MediaDecoderStateMachine::OnAudioPopped(const RefPtr& aSample) { MOZ_ASSERT(OnTaskQueue()); mPlaybackOffset = std::max(mPlaybackOffset, aSample->mOffset); } void MediaDecoderStateMachine::OnVideoPopped(const RefPtr& aSample) { MOZ_ASSERT(OnTaskQueue()); mPlaybackOffset = std::max(mPlaybackOffset, aSample->mOffset); } bool MediaDecoderStateMachine::IsAudioDecoding() { MOZ_ASSERT(OnTaskQueue()); return HasAudio() && !AudioQueue().IsFinished(); } bool MediaDecoderStateMachine::IsVideoDecoding() { MOZ_ASSERT(OnTaskQueue()); return HasVideo() && !VideoQueue().IsFinished(); } bool MediaDecoderStateMachine::IsPlaying() const { MOZ_ASSERT(OnTaskQueue()); return mMediaSink->IsPlaying(); } void MediaDecoderStateMachine::SetMediaNotSeekable() { mMediaSeekable = false; } nsresult MediaDecoderStateMachine::Init(MediaDecoder* aDecoder) { MOZ_ASSERT(NS_IsMainThread()); nsresult rv = MediaDecoderStateMachineBase::Init(aDecoder); if (NS_WARN_IF(NS_FAILED(rv))) { return rv; } mAudioQueueListener = AudioQueue().PopFrontEvent().Connect( mTaskQueue, this, &MediaDecoderStateMachine::OnAudioPopped); mVideoQueueListener = VideoQueue().PopFrontEvent().Connect( mTaskQueue, this, &MediaDecoderStateMachine::OnVideoPopped); mOnMediaNotSeekable = mReader->OnMediaNotSeekable().Connect( OwnerThread(), this, &MediaDecoderStateMachine::SetMediaNotSeekable); return NS_OK; } void MediaDecoderStateMachine::StopPlayback() { MOZ_ASSERT(OnTaskQueue()); LOG("StopPlayback()"); if (IsPlaying()) { mOnPlaybackEvent.Notify(MediaPlaybackEvent{ MediaPlaybackEvent::PlaybackStopped, mPlaybackOffset}); mMediaSink->SetPlaying(false); MOZ_ASSERT(!IsPlaying()); } } void MediaDecoderStateMachine::MaybeStartPlayback() { MOZ_ASSERT(OnTaskQueue()); // Should try to start playback only after decoding first frames. if (!mSentFirstFrameLoadedEvent) { LOG("MaybeStartPlayback: Not starting playback before loading first frame"); return; } if (IsPlaying()) { // Logging this case is really spammy - don't do it. return; } if (mIsMediaSinkSuspended) { LOG("MaybeStartPlayback: Not starting playback when sink is suspended"); return; } if (mPlayState != MediaDecoder::PLAY_STATE_PLAYING) { LOG("MaybeStartPlayback: Not starting playback [mPlayState=%d]", mPlayState.Ref()); return; } LOG("MaybeStartPlayback() starting playback"); StartMediaSink(); if (!IsPlaying()) { mMediaSink->SetPlaying(true); MOZ_ASSERT(IsPlaying()); } mOnPlaybackEvent.Notify( MediaPlaybackEvent{MediaPlaybackEvent::PlaybackStarted, mPlaybackOffset}); } void MediaDecoderStateMachine::UpdatePlaybackPositionInternal( const TimeUnit& aTime) { MOZ_ASSERT(OnTaskQueue()); LOGV("UpdatePlaybackPositionInternal(%" PRId64 ")", aTime.ToMicroseconds()); mCurrentPosition = aTime; NS_ASSERTION(mCurrentPosition.Ref() >= TimeUnit::Zero(), "CurrentTime should be positive!"); if (mDuration.Ref().ref() < mCurrentPosition.Ref()) { mDuration = Some(mCurrentPosition.Ref()); DDLOG(DDLogCategory::Property, "duration_us", mDuration.Ref()->ToMicroseconds()); } } void MediaDecoderStateMachine::UpdatePlaybackPosition(const TimeUnit& aTime) { MOZ_ASSERT(OnTaskQueue()); UpdatePlaybackPositionInternal(aTime); bool fragmentEnded = mFragmentEndTime.IsValid() && GetMediaTime() >= mFragmentEndTime; mMetadataManager.DispatchMetadataIfNeeded(aTime); if (fragmentEnded) { StopPlayback(); } } /* static */ const char* MediaDecoderStateMachine::ToStateStr(State aState) { switch (aState) { case DECODER_STATE_DECODING_METADATA: return "DECODING_METADATA"; case DECODER_STATE_DORMANT: return "DORMANT"; case DECODER_STATE_DECODING_FIRSTFRAME: return "DECODING_FIRSTFRAME"; case DECODER_STATE_DECODING: return "DECODING"; case DECODER_STATE_SEEKING_ACCURATE: return "SEEKING_ACCURATE"; case DECODER_STATE_SEEKING_FROMDORMANT: return "SEEKING_FROMDORMANT"; case DECODER_STATE_SEEKING_NEXTFRAMESEEKING: return "DECODER_STATE_SEEKING_NEXTFRAMESEEKING"; case DECODER_STATE_SEEKING_VIDEOONLY: return "SEEKING_VIDEOONLY"; case DECODER_STATE_BUFFERING: return "BUFFERING"; case DECODER_STATE_COMPLETED: return "COMPLETED"; case DECODER_STATE_SHUTDOWN: return "SHUTDOWN"; case DECODER_STATE_LOOPING_DECODING: return "LOOPING_DECODING"; default: MOZ_ASSERT_UNREACHABLE("Invalid state."); } return "UNKNOWN"; } const char* MediaDecoderStateMachine::ToStateStr() { MOZ_ASSERT(OnTaskQueue()); return ToStateStr(mStateObj->GetState()); } void MediaDecoderStateMachine::VolumeChanged() { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::VolumeChanged", MEDIA_PLAYBACK); MOZ_ASSERT(OnTaskQueue()); mMediaSink->SetVolume(mVolume); } RefPtr MediaDecoderStateMachine::Shutdown() { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::Shutdown", MEDIA_PLAYBACK); MOZ_ASSERT(OnTaskQueue()); return mStateObj->HandleShutdown(); } void MediaDecoderStateMachine::PlayStateChanged() { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::PlayStateChanged", MEDIA_PLAYBACK); MOZ_ASSERT(OnTaskQueue()); if (mPlayState != MediaDecoder::PLAY_STATE_PLAYING) { CancelSuspendTimer(); } else if (mMinimizePreroll) { // Once we start playing, we don't want to minimize our prerolling, as we // assume the user is likely to want to keep playing in future. This needs // to happen before we invoke StartDecoding(). mMinimizePreroll = false; } mStateObj->HandlePlayStateChanged(mPlayState); } void MediaDecoderStateMachine::SetVideoDecodeMode(VideoDecodeMode aMode) { MOZ_ASSERT(NS_IsMainThread()); nsCOMPtr r = NewRunnableMethod( "MediaDecoderStateMachine::SetVideoDecodeModeInternal", this, &MediaDecoderStateMachine::SetVideoDecodeModeInternal, aMode); OwnerThread()->DispatchStateChange(r.forget()); } void MediaDecoderStateMachine::SetVideoDecodeModeInternal( VideoDecodeMode aMode) { MOZ_ASSERT(OnTaskQueue()); LOG("SetVideoDecodeModeInternal(), VideoDecodeMode=(%s->%s), " "mVideoDecodeSuspended=%c", mVideoDecodeMode == VideoDecodeMode::Normal ? "Normal" : "Suspend", aMode == VideoDecodeMode::Normal ? "Normal" : "Suspend", mVideoDecodeSuspended ? 'T' : 'F'); // Should not suspend decoding if we don't turn on the pref. if (!StaticPrefs::media_suspend_bkgnd_video_enabled() && aMode == VideoDecodeMode::Suspend) { LOG("SetVideoDecodeModeInternal(), early return because preference off and " "set to Suspend"); return; } if (aMode == mVideoDecodeMode) { LOG("SetVideoDecodeModeInternal(), early return because the mode does not " "change"); return; } // Set new video decode mode. mVideoDecodeMode = aMode; // Start timer to trigger suspended video decoding. if (mVideoDecodeMode == VideoDecodeMode::Suspend) { TimeStamp target = TimeStamp::Now() + SuspendBackgroundVideoDelay(); RefPtr self = this; mVideoDecodeSuspendTimer.Ensure( target, [=]() { self->OnSuspendTimerResolved(); }, []() { MOZ_DIAGNOSTIC_ASSERT(false); }); mOnPlaybackEvent.Notify(MediaPlaybackEvent::StartVideoSuspendTimer); return; } // Resuming from suspended decoding // If suspend timer exists, destroy it. CancelSuspendTimer(); if (mVideoDecodeSuspended) { auto target = mMediaSink->IsStarted() ? GetClock() : GetMediaTime(); AdjustByLooping(target); mStateObj->HandleResumeVideoDecoding(target + detail::RESUME_VIDEO_PREMIUM); } } void MediaDecoderStateMachine::BufferedRangeUpdated() { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::BufferedRangeUpdated", MEDIA_PLAYBACK); MOZ_ASSERT(OnTaskQueue()); // While playing an unseekable stream of unknown duration, mDuration // is updated as we play. But if data is being downloaded // faster than played, mDuration won't reflect the end of playable data // since we haven't played the frame at the end of buffered data. So update // mDuration here as new data is downloaded to prevent such a lag. if (mBuffered.Ref().IsInvalid()) { return; } bool exists; media::TimeUnit end{mBuffered.Ref().GetEnd(&exists)}; if (!exists) { return; } // Use estimated duration from buffer ranges when mDuration is unknown or // the estimated duration is larger. if (mDuration.Ref().isNothing() || mDuration.Ref()->IsInfinite() || end > mDuration.Ref().ref()) { mDuration = Some(end); DDLOG(DDLogCategory::Property, "duration_us", mDuration.Ref()->ToMicroseconds()); } } RefPtr MediaDecoderStateMachine::Seek( const SeekTarget& aTarget) { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::Seek", MEDIA_PLAYBACK); PROFILER_MARKER_UNTYPED("MDSM::Seek", MEDIA_PLAYBACK); MOZ_ASSERT(OnTaskQueue()); // We need to be able to seek in some way if (!mMediaSeekable && !mMediaSeekableOnlyInBufferedRanges) { LOGW("Seek() should not be called on a non-seekable media"); return MediaDecoder::SeekPromise::CreateAndReject(/* aRejectValue = */ true, __func__); } if (aTarget.IsNextFrame() && !HasVideo()) { LOGW("Ignore a NextFrameSeekTask on a media file without video track."); return MediaDecoder::SeekPromise::CreateAndReject(/* aRejectValue = */ true, __func__); } MOZ_ASSERT(mDuration.Ref().isSome(), "We should have got duration already"); return mStateObj->HandleSeek(aTarget); } void MediaDecoderStateMachine::StopMediaSink() { MOZ_ASSERT(OnTaskQueue()); if (mMediaSink->IsStarted()) { LOG("Stop MediaSink"); mMediaSink->Stop(); mMediaSinkAudioEndedPromise.DisconnectIfExists(); mMediaSinkVideoEndedPromise.DisconnectIfExists(); } } void MediaDecoderStateMachine::RequestAudioData() { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::RequestAudioData", MEDIA_PLAYBACK); MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT(IsAudioDecoding()); MOZ_ASSERT(!IsRequestingAudioData()); MOZ_ASSERT(!IsWaitingAudioData()); LOGV("Queueing audio task - queued=%zu, decoder-queued=%zu", AudioQueue().GetSize(), mReader->SizeOfAudioQueueInFrames()); PerformanceRecorder perfRecorder(MediaStage::RequestData); RefPtr self = this; mReader->RequestAudioData() ->Then( OwnerThread(), __func__, [this, self, perfRecorder(std::move(perfRecorder))]( const RefPtr& aAudio) mutable { perfRecorder.Record(); AUTO_PROFILER_LABEL( "MediaDecoderStateMachine::RequestAudioData:Resolved", MEDIA_PLAYBACK); MOZ_ASSERT(aAudio); mAudioDataRequest.Complete(); // audio->GetEndTime() is not always mono-increasing in chained // ogg. mDecodedAudioEndTime = std::max(aAudio->GetEndTime(), mDecodedAudioEndTime); LOGV("OnAudioDecoded [%" PRId64 ",%" PRId64 "]", aAudio->mTime.ToMicroseconds(), aAudio->GetEndTime().ToMicroseconds()); mStateObj->HandleAudioDecoded(aAudio); }, [this, self](const MediaResult& aError) { AUTO_PROFILER_LABEL( "MediaDecoderStateMachine::RequestAudioData:Rejected", MEDIA_PLAYBACK); LOGV("OnAudioNotDecoded ErrorName=%s Message=%s", aError.ErrorName().get(), aError.Message().get()); mAudioDataRequest.Complete(); switch (aError.Code()) { case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA: mStateObj->HandleWaitingForAudio(); break; case NS_ERROR_DOM_MEDIA_CANCELED: mStateObj->HandleAudioCanceled(); break; case NS_ERROR_DOM_MEDIA_END_OF_STREAM: mStateObj->HandleEndOfAudio(); break; default: DecodeError(aError); } }) ->Track(mAudioDataRequest); } void MediaDecoderStateMachine::RequestVideoData( const media::TimeUnit& aCurrentTime, bool aRequestNextKeyFrame) { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::RequestVideoData", MEDIA_PLAYBACK); MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT(IsVideoDecoding()); MOZ_ASSERT(!IsRequestingVideoData()); MOZ_ASSERT(!IsWaitingVideoData()); LOGV( "Queueing video task - queued=%zu, decoder-queued=%zo" ", stime=%" PRId64 ", by-pass-skip=%d", VideoQueue().GetSize(), mReader->SizeOfVideoQueueInFrames(), aCurrentTime.ToMicroseconds(), mBypassingSkipToNextKeyFrameCheck); PerformanceRecorder perfRecorder(MediaStage::RequestData, Info().mVideo.mImage.height); RefPtr self = this; mReader ->RequestVideoData( mBypassingSkipToNextKeyFrameCheck ? media::TimeUnit() : aCurrentTime, mBypassingSkipToNextKeyFrameCheck ? false : aRequestNextKeyFrame) ->Then( OwnerThread(), __func__, [this, self, perfRecorder(std::move(perfRecorder))]( const RefPtr& aVideo) mutable { perfRecorder.Record(); AUTO_PROFILER_LABEL( "MediaDecoderStateMachine::RequestVideoData:Resolved", MEDIA_PLAYBACK); MOZ_ASSERT(aVideo); mVideoDataRequest.Complete(); // Handle abnormal or negative timestamps. mDecodedVideoEndTime = std::max(mDecodedVideoEndTime, aVideo->GetEndTime()); LOGV("OnVideoDecoded [%" PRId64 ",%" PRId64 "]", aVideo->mTime.ToMicroseconds(), aVideo->GetEndTime().ToMicroseconds()); mStateObj->HandleVideoDecoded(aVideo); }, [this, self](const MediaResult& aError) { AUTO_PROFILER_LABEL( "MediaDecoderStateMachine::RequestVideoData:Rejected", MEDIA_PLAYBACK); LOGV("OnVideoNotDecoded ErrorName=%s Message=%s", aError.ErrorName().get(), aError.Message().get()); mVideoDataRequest.Complete(); switch (aError.Code()) { case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA: mStateObj->HandleWaitingForVideo(); break; case NS_ERROR_DOM_MEDIA_CANCELED: mStateObj->HandleVideoCanceled(); break; case NS_ERROR_DOM_MEDIA_END_OF_STREAM: mStateObj->HandleEndOfVideo(); break; default: DecodeError(aError); } }) ->Track(mVideoDataRequest); } void MediaDecoderStateMachine::WaitForData(MediaData::Type aType) { MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT(aType == MediaData::Type::AUDIO_DATA || aType == MediaData::Type::VIDEO_DATA); RefPtr self = this; if (aType == MediaData::Type::AUDIO_DATA) { mReader->WaitForData(MediaData::Type::AUDIO_DATA) ->Then( OwnerThread(), __func__, [self](MediaData::Type aType) { AUTO_PROFILER_LABEL( "MediaDecoderStateMachine::WaitForData:AudioResolved", MEDIA_PLAYBACK); self->mAudioWaitRequest.Complete(); MOZ_ASSERT(aType == MediaData::Type::AUDIO_DATA); self->mStateObj->HandleAudioWaited(aType); }, [self](const WaitForDataRejectValue& aRejection) { AUTO_PROFILER_LABEL( "MediaDecoderStateMachine::WaitForData:AudioRejected", MEDIA_PLAYBACK); self->mAudioWaitRequest.Complete(); self->DecodeError(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA); }) ->Track(mAudioWaitRequest); } else { mReader->WaitForData(MediaData::Type::VIDEO_DATA) ->Then( OwnerThread(), __func__, [self](MediaData::Type aType) { AUTO_PROFILER_LABEL( "MediaDecoderStateMachine::WaitForData:VideoResolved", MEDIA_PLAYBACK); self->mVideoWaitRequest.Complete(); MOZ_ASSERT(aType == MediaData::Type::VIDEO_DATA); self->mStateObj->HandleVideoWaited(aType); }, [self](const WaitForDataRejectValue& aRejection) { AUTO_PROFILER_LABEL( "MediaDecoderStateMachine::WaitForData:VideoRejected", MEDIA_PLAYBACK); self->mVideoWaitRequest.Complete(); self->DecodeError(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA); }) ->Track(mVideoWaitRequest); } } nsresult MediaDecoderStateMachine::StartMediaSink() { MOZ_ASSERT(OnTaskQueue()); if (mMediaSink->IsStarted()) { return NS_OK; } mAudioCompleted = false; nsresult rv = mMediaSink->Start(GetMediaTime(), Info()); StreamNameChanged(); auto videoPromise = mMediaSink->OnEnded(TrackInfo::kVideoTrack); auto audioPromise = mMediaSink->OnEnded(TrackInfo::kAudioTrack); if (audioPromise) { audioPromise ->Then(OwnerThread(), __func__, this, &MediaDecoderStateMachine::OnMediaSinkAudioComplete, &MediaDecoderStateMachine::OnMediaSinkAudioError) ->Track(mMediaSinkAudioEndedPromise); } if (videoPromise) { videoPromise ->Then(OwnerThread(), __func__, this, &MediaDecoderStateMachine::OnMediaSinkVideoComplete, &MediaDecoderStateMachine::OnMediaSinkVideoError) ->Track(mMediaSinkVideoEndedPromise); } // Remember the initial offset when playback starts. This will be used // to calculate the rate at which bytes are consumed as playback moves on. RefPtr sample = mAudioQueue.PeekFront(); mPlaybackOffset = sample ? sample->mOffset : 0; sample = mVideoQueue.PeekFront(); if (sample && sample->mOffset > mPlaybackOffset) { mPlaybackOffset = sample->mOffset; } return rv; } bool MediaDecoderStateMachine::HasLowDecodedAudio() { MOZ_ASSERT(OnTaskQueue()); return IsAudioDecoding() && GetDecodedAudioDuration() < EXHAUSTED_DATA_MARGIN.MultDouble(mPlaybackRate); } bool MediaDecoderStateMachine::HasLowDecodedVideo() { MOZ_ASSERT(OnTaskQueue()); return IsVideoDecoding() && VideoQueue().GetSize() < static_cast(floorl(LOW_VIDEO_FRAMES * mPlaybackRate)); } bool MediaDecoderStateMachine::HasLowDecodedData() { MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT(mReader->UseBufferingHeuristics()); return HasLowDecodedAudio() || HasLowDecodedVideo(); } bool MediaDecoderStateMachine::OutOfDecodedAudio() { MOZ_ASSERT(OnTaskQueue()); return IsAudioDecoding() && !AudioQueue().IsFinished() && AudioQueue().GetSize() == 0 && !mMediaSink->HasUnplayedFrames(TrackInfo::kAudioTrack); } bool MediaDecoderStateMachine::HasLowBufferedData() { MOZ_ASSERT(OnTaskQueue()); return HasLowBufferedData(detail::LOW_BUFFER_THRESHOLD); } bool MediaDecoderStateMachine::HasLowBufferedData(const TimeUnit& aThreshold) { MOZ_ASSERT(OnTaskQueue()); // If we don't have a duration, mBuffered is probably not going to have // a useful buffered range. Return false here so that we don't get stuck in // buffering mode for live streams. if (Duration().IsInfinite()) { return false; } if (mBuffered.Ref().IsInvalid()) { return false; } // We are never low in decoded data when we don't have audio/video or have // decoded all audio/video samples. TimeUnit endOfDecodedVideo = (HasVideo() && !VideoQueue().IsFinished()) ? mDecodedVideoEndTime : TimeUnit::FromNegativeInfinity(); TimeUnit endOfDecodedAudio = (HasAudio() && !AudioQueue().IsFinished()) ? mDecodedAudioEndTime : TimeUnit::FromNegativeInfinity(); auto endOfDecodedData = std::max(endOfDecodedVideo, endOfDecodedAudio); if (Duration() < endOfDecodedData) { // Our duration is not up to date. No point buffering. return false; } if (endOfDecodedData.IsInfinite()) { // Have decoded all samples. No point buffering. return false; } auto start = endOfDecodedData; auto end = std::min(GetMediaTime() + aThreshold, Duration()); if (start >= end) { // Duration of decoded samples is greater than our threshold. return false; } media::TimeInterval interval(start, end); return !mBuffered.Ref().Contains(interval); } void MediaDecoderStateMachine::EnqueueFirstFrameLoadedEvent() { MOZ_ASSERT(OnTaskQueue()); // Track value of mSentFirstFrameLoadedEvent from before updating it bool firstFrameBeenLoaded = mSentFirstFrameLoadedEvent; mSentFirstFrameLoadedEvent = true; MediaDecoderEventVisibility visibility = firstFrameBeenLoaded ? MediaDecoderEventVisibility::Suppressed : MediaDecoderEventVisibility::Observable; mFirstFrameLoadedEvent.Notify(UniquePtr(new MediaInfo(Info())), visibility); } void MediaDecoderStateMachine::FinishDecodeFirstFrame() { MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT(!mSentFirstFrameLoadedEvent); LOG("FinishDecodeFirstFrame"); mMediaSink->Redraw(Info().mVideo); LOG("Media duration %" PRId64 ", mediaSeekable=%d", Duration().ToMicroseconds(), mMediaSeekable); // Get potentially updated metadata mReader->ReadUpdatedMetadata(mInfo.ptr()); EnqueueFirstFrameLoadedEvent(); } RefPtr MediaDecoderStateMachine::FinishShutdown() { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::FinishShutdown", MEDIA_PLAYBACK); MOZ_ASSERT(OnTaskQueue()); LOG("Shutting down state machine task queue"); return OwnerThread()->BeginShutdown(); } void MediaDecoderStateMachine::RunStateMachine() { MOZ_ASSERT(OnTaskQueue()); AUTO_PROFILER_LABEL("MediaDecoderStateMachine::RunStateMachine", MEDIA_PLAYBACK); mDelayedScheduler.Reset(); // Must happen on state machine task queue. mDispatchedStateMachine = false; mStateObj->Step(); } void MediaDecoderStateMachine::ResetDecode(const TrackSet& aTracks) { MOZ_ASSERT(OnTaskQueue()); LOG("MediaDecoderStateMachine::Reset"); // Assert that aTracks specifies to reset the video track because we // don't currently support resetting just the audio track. MOZ_ASSERT(aTracks.contains(TrackInfo::kVideoTrack)); if (aTracks.contains(TrackInfo::kVideoTrack)) { mDecodedVideoEndTime = TimeUnit::Zero(); mVideoCompleted = false; VideoQueue().Reset(); mVideoDataRequest.DisconnectIfExists(); mVideoWaitRequest.DisconnectIfExists(); } if (aTracks.contains(TrackInfo::kAudioTrack)) { mDecodedAudioEndTime = TimeUnit::Zero(); mAudioCompleted = false; AudioQueue().Reset(); mAudioDataRequest.DisconnectIfExists(); mAudioWaitRequest.DisconnectIfExists(); } mReader->ResetDecode(aTracks); } media::TimeUnit MediaDecoderStateMachine::GetClock( TimeStamp* aTimeStamp) const { MOZ_ASSERT(OnTaskQueue()); auto clockTime = mMediaSink->GetPosition(aTimeStamp); // This fails on Windows some times, see 1765563 #if defined(XP_WIN) NS_ASSERTION(GetMediaTime() <= clockTime, "Clock should go forwards."); #else MOZ_ASSERT(GetMediaTime() <= clockTime, "Clock should go forwards."); #endif return clockTime; } void MediaDecoderStateMachine::UpdatePlaybackPositionPeriodically() { MOZ_ASSERT(OnTaskQueue()); if (!IsPlaying()) { return; } // Cap the current time to the larger of the audio and video end time. // This ensures that if we're running off the system clock, we don't // advance the clock to after the media end time. if (VideoEndTime() > TimeUnit::Zero() || AudioEndTime() > TimeUnit::Zero()) { auto clockTime = GetClock(); // Once looping was turned on, the time is probably larger than the duration // of the media track, so the time over the end should be corrected. AdjustByLooping(clockTime); bool loopback = clockTime < GetMediaTime() && mLooping; if (loopback && mBypassingSkipToNextKeyFrameCheck) { LOG("media has looped back, no longer bypassing skip-to-next-key-frame"); mBypassingSkipToNextKeyFrameCheck = false; } // Skip frames up to the frame at the playback position, and figure out // the time remaining until it's time to display the next frame and drop // the current frame. NS_ASSERTION(clockTime >= TimeUnit::Zero(), "Should have positive clock time."); // These will be non -1 if we've displayed a video frame, or played an audio // frame. auto maxEndTime = std::max(VideoEndTime(), AudioEndTime()); auto t = std::min(clockTime, maxEndTime); // FIXME: Bug 1091422 - chained ogg files hit this assertion. // MOZ_ASSERT(t >= GetMediaTime()); if (loopback || t > GetMediaTime()) { UpdatePlaybackPosition(t); } } // Note we have to update playback position before releasing the monitor. // Otherwise, MediaDecoder::AddOutputTrack could kick in when we are outside // the monitor and get a staled value from GetCurrentTimeUs() which hits the // assertion in GetClock(). int64_t delay = std::max( 1, static_cast(AUDIO_DURATION_USECS / mPlaybackRate)); ScheduleStateMachineIn(TimeUnit::FromMicroseconds(delay)); // Notify the listener as we progress in the playback offset. Note it would // be too intensive to send notifications for each popped audio/video sample. // It is good enough to send 'PlaybackProgressed' events every 40us (defined // by AUDIO_DURATION_USECS), and we ensure 'PlaybackProgressed' events are // always sent after 'PlaybackStarted' and before 'PlaybackStopped'. mOnPlaybackEvent.Notify(MediaPlaybackEvent{ MediaPlaybackEvent::PlaybackProgressed, mPlaybackOffset}); } void MediaDecoderStateMachine::ScheduleStateMachine() { MOZ_ASSERT(OnTaskQueue()); if (mDispatchedStateMachine) { return; } mDispatchedStateMachine = true; nsresult rv = OwnerThread()->Dispatch( NewRunnableMethod("MediaDecoderStateMachine::RunStateMachine", this, &MediaDecoderStateMachine::RunStateMachine)); MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); Unused << rv; } void MediaDecoderStateMachine::ScheduleStateMachineIn(const TimeUnit& aTime) { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::ScheduleStateMachineIn", MEDIA_PLAYBACK); MOZ_ASSERT(OnTaskQueue()); // mDelayedScheduler.Ensure() may Disconnect() // the promise, which must happen on the state // machine task queue. MOZ_ASSERT(aTime > TimeUnit::Zero()); if (mDispatchedStateMachine) { return; } TimeStamp target = TimeStamp::Now() + aTime.ToTimeDuration(); // It is OK to capture 'this' without causing UAF because the callback // always happens before shutdown. RefPtr self = this; mDelayedScheduler.Ensure( target, [self]() { self->mDelayedScheduler.CompleteRequest(); self->RunStateMachine(); }, []() { MOZ_DIAGNOSTIC_ASSERT(false); }); } bool MediaDecoderStateMachine::IsStateMachineScheduled() const { MOZ_ASSERT(OnTaskQueue()); return mDispatchedStateMachine || mDelayedScheduler.IsScheduled(); } void MediaDecoderStateMachine::SetPlaybackRate(double aPlaybackRate) { MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT(aPlaybackRate != 0, "Should be handled by MediaDecoder::Pause()"); mPlaybackRate = aPlaybackRate; mMediaSink->SetPlaybackRate(mPlaybackRate); // Schedule next cycle to check if we can stop prerolling. ScheduleStateMachine(); } void MediaDecoderStateMachine::PreservesPitchChanged() { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::PreservesPitchChanged", MEDIA_PLAYBACK); MOZ_ASSERT(OnTaskQueue()); mMediaSink->SetPreservesPitch(mPreservesPitch); } void MediaDecoderStateMachine::LoopingChanged() { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::LoopingChanged", MEDIA_PLAYBACK); MOZ_ASSERT(OnTaskQueue()); LOGV("LoopingChanged, looping=%d", mLooping.Ref()); PROFILER_MARKER_TEXT("MDSM::LoopingChanged", MEDIA_PLAYBACK, {}, mLooping ? "true"_ns : "false"_ns); if (mSeamlessLoopingAllowed) { mStateObj->HandleLoopingChanged(); } } void MediaDecoderStateMachine::StreamNameChanged() { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::StreamNameChanged", MEDIA_PLAYBACK); MOZ_ASSERT(OnTaskQueue()); mMediaSink->SetStreamName(mStreamName); } void MediaDecoderStateMachine::UpdateOutputCaptured() { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::UpdateOutputCaptured", MEDIA_PLAYBACK); MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT_IF( mOutputCaptureState == MediaDecoder::OutputCaptureState::Capture, mOutputDummyTrack.Ref()); // Reset these flags so they are consistent with the status of the sink. // TODO: Move these flags into MediaSink to improve cohesion so we don't need // to reset these flags when switching MediaSinks. mAudioCompleted = false; mVideoCompleted = false; // Don't create a new media sink if we're still suspending media sink. if (!mIsMediaSinkSuspended) { const bool wasPlaying = IsPlaying(); // Stop and shut down the existing sink. StopMediaSink(); mMediaSink->Shutdown(); // Create a new sink according to whether output is captured. mMediaSink = CreateMediaSink(); if (wasPlaying) { DebugOnly rv = StartMediaSink(); MOZ_ASSERT(NS_SUCCEEDED(rv)); } } // Don't buffer as much when audio is captured because we don't need to worry // about high latency audio devices. mAmpleAudioThreshold = mOutputCaptureState != MediaDecoder::OutputCaptureState::None ? detail::AMPLE_AUDIO_THRESHOLD / 2 : detail::AMPLE_AUDIO_THRESHOLD; mStateObj->HandleAudioCaptured(); } void MediaDecoderStateMachine::OutputTracksChanged() { MOZ_ASSERT(OnTaskQueue()); LOG("OutputTracksChanged, tracks=%zu", mOutputTracks.Ref().Length()); mCanonicalOutputTracks = mOutputTracks; } void MediaDecoderStateMachine::OutputPrincipalChanged() { MOZ_ASSERT(OnTaskQueue()); mCanonicalOutputPrincipal = mOutputPrincipal; } RefPtr MediaDecoderStateMachine::InvokeSetSink( const RefPtr& aSink) { MOZ_ASSERT(NS_IsMainThread()); MOZ_ASSERT(aSink); return InvokeAsync(OwnerThread(), this, __func__, &MediaDecoderStateMachine::SetSink, aSink); } RefPtr MediaDecoderStateMachine::SetSink( const RefPtr& aDevice) { MOZ_ASSERT(OnTaskQueue()); if (mIsMediaSinkSuspended) { // Don't create a new media sink when suspended. return GenericPromise::CreateAndResolve(false, __func__); } if (mOutputCaptureState != MediaDecoder::OutputCaptureState::None) { // Not supported yet. return GenericPromise::CreateAndReject(NS_ERROR_ABORT, __func__); } if (mSinkDevice.Ref() != aDevice) { // A new sink was set before this ran. return GenericPromise::CreateAndResolve(IsPlaying(), __func__); } if (mMediaSink->AudioDevice() == aDevice) { // The sink has not changed. return GenericPromise::CreateAndResolve(IsPlaying(), __func__); } const bool wasPlaying = IsPlaying(); // Stop and shutdown the existing sink. StopMediaSink(); mMediaSink->Shutdown(); // Create a new sink according to whether audio is captured. mMediaSink = CreateMediaSink(); // Start the new sink if (wasPlaying) { nsresult rv = StartMediaSink(); if (NS_FAILED(rv)) { return GenericPromise::CreateAndReject(NS_ERROR_ABORT, __func__); } } return GenericPromise::CreateAndResolve(wasPlaying, __func__); } void MediaDecoderStateMachine::InvokeSuspendMediaSink() { MOZ_ASSERT(NS_IsMainThread()); nsresult rv = OwnerThread()->Dispatch( NewRunnableMethod("MediaDecoderStateMachine::SuspendMediaSink", this, &MediaDecoderStateMachine::SuspendMediaSink)); MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); Unused << rv; } void MediaDecoderStateMachine::SuspendMediaSink() { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::SuspendMediaSink", MEDIA_PLAYBACK); MOZ_ASSERT(OnTaskQueue()); if (mIsMediaSinkSuspended) { return; } LOG("SuspendMediaSink"); mIsMediaSinkSuspended = true; StopMediaSink(); mMediaSink->Shutdown(); } void MediaDecoderStateMachine::InvokeResumeMediaSink() { MOZ_ASSERT(NS_IsMainThread()); nsresult rv = OwnerThread()->Dispatch( NewRunnableMethod("MediaDecoderStateMachine::ResumeMediaSink", this, &MediaDecoderStateMachine::ResumeMediaSink)); MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); Unused << rv; } void MediaDecoderStateMachine::ResumeMediaSink() { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::ResumeMediaSink", MEDIA_PLAYBACK); MOZ_ASSERT(OnTaskQueue()); if (!mIsMediaSinkSuspended) { return; } LOG("ResumeMediaSink"); mIsMediaSinkSuspended = false; if (!mMediaSink->IsStarted()) { mMediaSink = CreateMediaSink(); MaybeStartPlayback(); } } void MediaDecoderStateMachine::UpdateSecondaryVideoContainer() { AUTO_PROFILER_LABEL("MediaDecoderStateMachine::UpdateSecondaryVideoContainer", MEDIA_PLAYBACK); MOZ_ASSERT(OnTaskQueue()); MOZ_DIAGNOSTIC_ASSERT(mMediaSink); mMediaSink->SetSecondaryVideoContainer(mSecondaryVideoContainer.Ref()); mOnSecondaryVideoContainerInstalled.Notify(mSecondaryVideoContainer.Ref()); } TimeUnit MediaDecoderStateMachine::AudioEndTime() const { MOZ_ASSERT(OnTaskQueue()); if (mMediaSink->IsStarted()) { return mMediaSink->GetEndTime(TrackInfo::kAudioTrack); } return GetMediaTime(); } TimeUnit MediaDecoderStateMachine::VideoEndTime() const { MOZ_ASSERT(OnTaskQueue()); if (mMediaSink->IsStarted()) { return mMediaSink->GetEndTime(TrackInfo::kVideoTrack); } return GetMediaTime(); } void MediaDecoderStateMachine::OnMediaSinkVideoComplete() { MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT(HasVideo()); AUTO_PROFILER_LABEL("MediaDecoderStateMachine::OnMediaSinkVideoComplete", MEDIA_PLAYBACK); LOG("[%s]", __func__); mMediaSinkVideoEndedPromise.Complete(); mVideoCompleted = true; ScheduleStateMachine(); } void MediaDecoderStateMachine::OnMediaSinkVideoError() { MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT(HasVideo()); AUTO_PROFILER_LABEL("MediaDecoderStateMachine::OnMediaSinkVideoError", MEDIA_PLAYBACK); LOGE("[%s]", __func__); mMediaSinkVideoEndedPromise.Complete(); mVideoCompleted = true; if (HasAudio()) { return; } DecodeError(MediaResult(NS_ERROR_DOM_MEDIA_MEDIASINK_ERR, __func__)); } void MediaDecoderStateMachine::OnMediaSinkAudioComplete() { MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT(HasAudio()); AUTO_PROFILER_LABEL("MediaDecoderStateMachine::OnMediaSinkAudioComplete", MEDIA_PLAYBACK); LOG("[%s]", __func__); mMediaSinkAudioEndedPromise.Complete(); mAudioCompleted = true; // To notify PlaybackEnded as soon as possible. ScheduleStateMachine(); // Report OK to Decoder Doctor (to know if issue may have been resolved). mOnDecoderDoctorEvent.Notify( DecoderDoctorEvent{DecoderDoctorEvent::eAudioSinkStartup, NS_OK}); } void MediaDecoderStateMachine::OnMediaSinkAudioError(nsresult aResult) { MOZ_ASSERT(OnTaskQueue()); MOZ_ASSERT(HasAudio()); AUTO_PROFILER_LABEL("MediaDecoderStateMachine::OnMediaSinkAudioError", MEDIA_PLAYBACK); LOGE("[%s]", __func__); mMediaSinkAudioEndedPromise.Complete(); mAudioCompleted = true; // Result should never be NS_OK in this *error* handler. Report to Dec-Doc. MOZ_ASSERT(NS_FAILED(aResult)); mOnDecoderDoctorEvent.Notify( DecoderDoctorEvent{DecoderDoctorEvent::eAudioSinkStartup, aResult}); // Make the best effort to continue playback when there is video. if (HasVideo()) { return; } // Otherwise notify media decoder/element about this error for it makes // no sense to play an audio-only file without sound output. DecodeError(MediaResult(NS_ERROR_DOM_MEDIA_MEDIASINK_ERR, __func__)); } uint32_t MediaDecoderStateMachine::GetAmpleVideoFrames() const { MOZ_ASSERT(OnTaskQueue()); return mReader->VideoIsHardwareAccelerated() ? std::max(sVideoQueueHWAccelSize, MIN_VIDEO_QUEUE_SIZE) : std::max(sVideoQueueDefaultSize, MIN_VIDEO_QUEUE_SIZE); } void MediaDecoderStateMachine::GetDebugInfo( dom::MediaDecoderStateMachineDebugInfo& aInfo) { MOZ_ASSERT(OnTaskQueue()); aInfo.mDuration = mDuration.Ref() ? mDuration.Ref().ref().ToMicroseconds() : -1; aInfo.mMediaTime = GetMediaTime().ToMicroseconds(); aInfo.mClock = mMediaSink->IsStarted() ? GetClock().ToMicroseconds() : -1; aInfo.mPlayState = int32_t(mPlayState.Ref()); aInfo.mSentFirstFrameLoadedEvent = mSentFirstFrameLoadedEvent; aInfo.mIsPlaying = IsPlaying(); CopyUTF8toUTF16(MakeStringSpan(AudioRequestStatus()), aInfo.mAudioRequestStatus); CopyUTF8toUTF16(MakeStringSpan(VideoRequestStatus()), aInfo.mVideoRequestStatus); aInfo.mDecodedAudioEndTime = mDecodedAudioEndTime.ToMicroseconds(); aInfo.mDecodedVideoEndTime = mDecodedVideoEndTime.ToMicroseconds(); aInfo.mAudioCompleted = mAudioCompleted; aInfo.mVideoCompleted = mVideoCompleted; mStateObj->GetDebugInfo(aInfo.mStateObj); mMediaSink->GetDebugInfo(aInfo.mMediaSink); } RefPtr MediaDecoderStateMachine::RequestDebugInfo( dom::MediaDecoderStateMachineDebugInfo& aInfo) { RefPtr p = new GenericPromise::Private(__func__); RefPtr self = this; nsresult rv = OwnerThread()->Dispatch( NS_NewRunnableFunction("MediaDecoderStateMachine::RequestDebugInfo", [self, p, &aInfo]() { self->GetDebugInfo(aInfo); p->Resolve(true, __func__); }), AbstractThread::TailDispatch); MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); Unused << rv; return p; } class VideoQueueMemoryFunctor : public nsDequeFunctor { public: VideoQueueMemoryFunctor() : mSize(0) {} MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf); virtual void operator()(VideoData* aObject) override { mSize += aObject->SizeOfIncludingThis(MallocSizeOf); } size_t mSize; }; class AudioQueueMemoryFunctor : public nsDequeFunctor { public: AudioQueueMemoryFunctor() : mSize(0) {} MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf); virtual void operator()(AudioData* aObject) override { mSize += aObject->SizeOfIncludingThis(MallocSizeOf); } size_t mSize; }; size_t MediaDecoderStateMachine::SizeOfVideoQueue() const { VideoQueueMemoryFunctor functor; mVideoQueue.LockedForEach(functor); return functor.mSize; } size_t MediaDecoderStateMachine::SizeOfAudioQueue() const { AudioQueueMemoryFunctor functor; mAudioQueue.LockedForEach(functor); return functor.mSize; } const char* MediaDecoderStateMachine::AudioRequestStatus() const { MOZ_ASSERT(OnTaskQueue()); if (IsRequestingAudioData()) { MOZ_DIAGNOSTIC_ASSERT(!IsWaitingAudioData()); return "pending"; } if (IsWaitingAudioData()) { return "waiting"; } return "idle"; } const char* MediaDecoderStateMachine::VideoRequestStatus() const { MOZ_ASSERT(OnTaskQueue()); if (IsRequestingVideoData()) { MOZ_DIAGNOSTIC_ASSERT(!IsWaitingVideoData()); return "pending"; } if (IsWaitingVideoData()) { return "waiting"; } return "idle"; } void MediaDecoderStateMachine::OnSuspendTimerResolved() { LOG("OnSuspendTimerResolved"); mVideoDecodeSuspendTimer.CompleteRequest(); mStateObj->HandleVideoSuspendTimeout(); } void MediaDecoderStateMachine::CancelSuspendTimer() { LOG("CancelSuspendTimer: State: %s, Timer.IsScheduled: %c", ToStateStr(mStateObj->GetState()), mVideoDecodeSuspendTimer.IsScheduled() ? 'T' : 'F'); MOZ_ASSERT(OnTaskQueue()); if (mVideoDecodeSuspendTimer.IsScheduled()) { mOnPlaybackEvent.Notify(MediaPlaybackEvent::CancelVideoSuspendTimer); } mVideoDecodeSuspendTimer.Reset(); } void MediaDecoderStateMachine::AdjustByLooping(media::TimeUnit& aTime) const { MOZ_ASSERT(OnTaskQueue()); // No need to adjust time. if (mOriginalDecodedDuration == media::TimeUnit::Zero()) { return; } // There are situations where we need to perform subtraction instead of modulo // to accurately adjust the clock. When we are not in a state of seamless // looping, it is usually necessary to normalize the clock time within the // range of [0, duration]. However, if the current clock time is greater than // the duration (i.e., duration+1) and not in looping, we should not adjust it // to 1 as we are not looping back to the starting position. Instead, we // should leave the clock time unchanged and trim it later to match the // maximum duration time. if (mStateObj->GetState() != DECODER_STATE_LOOPING_DECODING) { // Use the smaller offset rather than the larger one, as the larger offset // indicates the next round of looping. For example, if the duration is X // and the playback is currently in the third round of looping, both // queues will have an offset of 3X. However, if the audio decoding is // faster and the fourth round of data has already been added to the audio // queue, the audio offset will become 4X. Since playback is still in the // third round, we should use the smaller offset of 3X to adjust the time. TimeUnit offset = TimeUnit::FromInfinity(); if (HasAudio()) { offset = std::min(AudioQueue().GetOffset(), offset); } if (HasVideo()) { offset = std::min(VideoQueue().GetOffset(), offset); } if (aTime > offset) { aTime -= offset; return; } } // When seamless looping happens at least once, it doesn't matter if we're // looping or not. aTime = aTime % mOriginalDecodedDuration; } bool MediaDecoderStateMachine::IsInSeamlessLooping() const { return mLooping && mSeamlessLoopingAllowed; } bool MediaDecoderStateMachine::HasLastDecodedData(MediaData::Type aType) { MOZ_DIAGNOSTIC_ASSERT(aType == MediaData::Type::AUDIO_DATA || aType == MediaData::Type::VIDEO_DATA); if (aType == MediaData::Type::AUDIO_DATA) { return mDecodedAudioEndTime != TimeUnit::Zero(); } return mDecodedVideoEndTime != TimeUnit::Zero(); } bool MediaDecoderStateMachine::IsCDMProxySupported(CDMProxy* aProxy) { #ifdef MOZ_WMF_CDM MOZ_ASSERT(aProxy); // This proxy only works with the external state machine. return !aProxy->AsWMFCDMProxy(); #else return true; #endif } } // namespace mozilla // avoid redefined macro in unified build #undef LOG #undef LOGV #undef LOGW #undef LOGE #undef SLOGW #undef SLOGE #undef NS_DispatchToMainThread