diff options
Diffstat (limited to 'dom/media/MediaDecoderStateMachine.cpp')
| -rw-r--r-- | dom/media/MediaDecoderStateMachine.cpp | 4870 |
1 files changed, 4870 insertions, 0 deletions
diff --git a/dom/media/MediaDecoderStateMachine.cpp b/dom/media/MediaDecoderStateMachine.cpp new file mode 100644 index 0000000000..f3cd79047b --- /dev/null +++ b/dom/media/MediaDecoderStateMachine.cpp @@ -0,0 +1,4870 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <algorithm> +#include <stdint.h> +#include <utility> + +#include "mediasink/AudioSink.h" +#include "mediasink/AudioSinkWrapper.h" +#include "mediasink/DecodedStream.h" +#include "mediasink/VideoSink.h" +#include "mozilla/Logging.h" +#include "mozilla/MathAlgorithms.h" +#include "mozilla/NotNull.h" +#include "mozilla/Preferences.h" +#include "mozilla/ProfilerLabels.h" +#include "mozilla/ProfilerMarkers.h" +#include "mozilla/ProfilerMarkerTypes.h" +#include "mozilla/SharedThreadPool.h" +#include "mozilla/Sprintf.h" +#include "mozilla/StaticPrefs_media.h" +#include "mozilla/Telemetry.h" +#include "mozilla/TaskQueue.h" + +#include "nsIMemoryReporter.h" +#include "nsPrintfCString.h" +#include "nsTArray.h" +#include "AudioSegment.h" +#include "DOMMediaStream.h" +#include "ImageContainer.h" +#include "MediaDecoder.h" +#include "MediaDecoderStateMachine.h" +#include "MediaShutdownManager.h" +#include "MediaTrackGraph.h" +#include "MediaTimer.h" +#include "PerformanceRecorder.h" +#include "ReaderProxy.h" +#include "TimeUnits.h" +#include "VideoSegment.h" +#include "VideoUtils.h" + +namespace mozilla { + +using namespace mozilla::media; + +#define NS_DispatchToMainThread(...) \ + CompileError_UseAbstractThreadDispatchInstead + +// avoid redefined macro in unified build +#undef FMT +#undef LOG +#undef LOGV +#undef LOGW +#undef LOGE +#undef SFMT +#undef SLOG +#undef SLOGW +#undef SLOGE + +#define FMT(x, ...) "Decoder=%p " x, mDecoderID, ##__VA_ARGS__ +#define LOG(x, ...) \ + DDMOZ_LOG(gMediaDecoderLog, LogLevel::Debug, "Decoder=%p " x, mDecoderID, \ + ##__VA_ARGS__) +#define LOGV(x, ...) \ + DDMOZ_LOG(gMediaDecoderLog, LogLevel::Verbose, "Decoder=%p " x, mDecoderID, \ + ##__VA_ARGS__) +#define LOGW(x, ...) NS_WARNING(nsPrintfCString(FMT(x, ##__VA_ARGS__)).get()) +#define LOGE(x, ...) \ + NS_DebugBreak(NS_DEBUG_WARNING, \ + nsPrintfCString(FMT(x, ##__VA_ARGS__)).get(), nullptr, \ + __FILE__, __LINE__) + +// Used by StateObject and its sub-classes +#define SFMT(x, ...) \ + "Decoder=%p state=%s " x, mMaster->mDecoderID, ToStateStr(GetState()), \ + ##__VA_ARGS__ +#define SLOG(x, ...) \ + DDMOZ_LOGEX(mMaster, gMediaDecoderLog, LogLevel::Debug, "state=%s " x, \ + ToStateStr(GetState()), ##__VA_ARGS__) +#define SLOGW(x, ...) NS_WARNING(nsPrintfCString(SFMT(x, ##__VA_ARGS__)).get()) +#define SLOGE(x, ...) \ + NS_DebugBreak(NS_DEBUG_WARNING, \ + nsPrintfCString(SFMT(x, ##__VA_ARGS__)).get(), nullptr, \ + __FILE__, __LINE__) + +// Certain constants get stored as member variables and then adjusted by various +// scale factors on a per-decoder basis. We want to make sure to avoid using +// these constants directly, so we put them in a namespace. +namespace detail { + +// Resume a suspended video decoder to the current playback position plus this +// time premium for compensating the seeking delay. +static constexpr auto RESUME_VIDEO_PREMIUM = TimeUnit::FromMicroseconds(125000); + +static const int64_t AMPLE_AUDIO_USECS = 2000000; + +// If more than this much decoded audio is queued, we'll hold off +// decoding more audio. +static constexpr auto AMPLE_AUDIO_THRESHOLD = + TimeUnit::FromMicroseconds(AMPLE_AUDIO_USECS); + +} // namespace detail + +// If we have fewer than LOW_VIDEO_FRAMES decoded frames, and +// we're not "prerolling video", we'll skip the video up to the next keyframe +// which is at or after the current playback position. +static const uint32_t LOW_VIDEO_FRAMES = 2; + +// Arbitrary "frame duration" when playing only audio. +static const uint32_t AUDIO_DURATION_USECS = 40000; + +namespace detail { + +// If we have less than this much buffered data available, we'll consider +// ourselves to be running low on buffered data. We determine how much +// buffered data we have remaining using the reader's GetBuffered() +// implementation. +static const int64_t LOW_BUFFER_THRESHOLD_USECS = 5000000; + +static constexpr auto LOW_BUFFER_THRESHOLD = + TimeUnit::FromMicroseconds(LOW_BUFFER_THRESHOLD_USECS); + +// LOW_BUFFER_THRESHOLD_USECS needs to be greater than AMPLE_AUDIO_USECS, +// otherwise the skip-to-keyframe logic can activate when we're running low on +// data. +static_assert(LOW_BUFFER_THRESHOLD_USECS > AMPLE_AUDIO_USECS, + "LOW_BUFFER_THRESHOLD_USECS is too small"); + +} // namespace detail + +// Amount of excess data to add in to the "should we buffer" calculation. +static constexpr auto EXHAUSTED_DATA_MARGIN = + TimeUnit::FromMicroseconds(100000); + +static const uint32_t MIN_VIDEO_QUEUE_SIZE = 3; +static const uint32_t MAX_VIDEO_QUEUE_SIZE = 10; +#ifdef MOZ_APPLEMEDIA +static const uint32_t HW_VIDEO_QUEUE_SIZE = 10; +#else +static const uint32_t HW_VIDEO_QUEUE_SIZE = 3; +#endif +static const uint32_t VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE = 9999; + +static uint32_t sVideoQueueDefaultSize = MAX_VIDEO_QUEUE_SIZE; +static uint32_t sVideoQueueHWAccelSize = HW_VIDEO_QUEUE_SIZE; +static uint32_t sVideoQueueSendToCompositorSize = + VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE; + +static void InitVideoQueuePrefs() { + MOZ_ASSERT(NS_IsMainThread()); + static bool sPrefInit = false; + if (!sPrefInit) { + sPrefInit = true; + sVideoQueueDefaultSize = Preferences::GetUint( + "media.video-queue.default-size", MAX_VIDEO_QUEUE_SIZE); + sVideoQueueHWAccelSize = Preferences::GetUint( + "media.video-queue.hw-accel-size", HW_VIDEO_QUEUE_SIZE); + sVideoQueueSendToCompositorSize = + Preferences::GetUint("media.video-queue.send-to-compositor-size", + VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE); + } +} + +template <typename Type, typename Function> +static void DiscardFramesFromTail(MediaQueue<Type>& aQueue, + const Function&& aTest) { + while (aQueue.GetSize()) { + if (aTest(aQueue.PeekBack()->mTime.ToMicroseconds())) { + RefPtr<Type> releaseMe = aQueue.PopBack(); + continue; + } + break; + } +} + +// Delay, in milliseconds, that tabs needs to be in background before video +// decoding is suspended. +static TimeDuration SuspendBackgroundVideoDelay() { + return TimeDuration::FromMilliseconds( + StaticPrefs::media_suspend_background_video_delay_ms()); +} + +class MediaDecoderStateMachine::StateObject { + public: + virtual ~StateObject() = default; + virtual void Exit() {} // Exit action. + virtual void Step() {} // Perform a 'cycle' of this state object. + virtual State GetState() const = 0; + + // Event handlers for various events. + virtual void HandleAudioCaptured() {} + virtual void HandleAudioDecoded(AudioData* aAudio) { + Crash("Unexpected event!", __func__); + } + virtual void HandleVideoDecoded(VideoData* aVideo) { + Crash("Unexpected event!", __func__); + } + virtual void HandleAudioWaited(MediaData::Type aType) { + Crash("Unexpected event!", __func__); + } + virtual void HandleVideoWaited(MediaData::Type aType) { + Crash("Unexpected event!", __func__); + } + virtual void HandleWaitingForAudio() { Crash("Unexpected event!", __func__); } + virtual void HandleAudioCanceled() { Crash("Unexpected event!", __func__); } + virtual void HandleEndOfAudio() { Crash("Unexpected event!", __func__); } + virtual void HandleWaitingForVideo() { Crash("Unexpected event!", __func__); } + virtual void HandleVideoCanceled() { Crash("Unexpected event!", __func__); } + virtual void HandleEndOfVideo() { Crash("Unexpected event!", __func__); } + + virtual RefPtr<MediaDecoder::SeekPromise> HandleSeek( + const SeekTarget& aTarget); + + virtual RefPtr<ShutdownPromise> HandleShutdown(); + + virtual void HandleVideoSuspendTimeout() = 0; + + virtual void HandleResumeVideoDecoding(const TimeUnit& aTarget); + + virtual void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) {} + + virtual void GetDebugInfo( + dom::MediaDecoderStateMachineDecodingStateDebugInfo& aInfo) {} + + virtual void HandleLoopingChanged() {} + + private: + template <class S, typename R, typename... As> + auto ReturnTypeHelper(R (S::*)(As...)) -> R; + + void Crash(const char* aReason, const char* aSite) { + char buf[1024]; + SprintfLiteral(buf, "%s state=%s callsite=%s", aReason, + ToStateStr(GetState()), aSite); + MOZ_ReportAssertionFailure(buf, __FILE__, __LINE__); + MOZ_CRASH(); + } + + protected: + enum class EventVisibility : int8_t { Observable, Suppressed }; + + using Master = MediaDecoderStateMachine; + explicit StateObject(Master* aPtr) : mMaster(aPtr) {} + TaskQueue* OwnerThread() const { return mMaster->mTaskQueue; } + ReaderProxy* Reader() const { return mMaster->mReader; } + const MediaInfo& Info() const { return mMaster->Info(); } + MediaQueue<AudioData>& AudioQueue() const { return mMaster->mAudioQueue; } + MediaQueue<VideoData>& VideoQueue() const { return mMaster->mVideoQueue; } + + template <class S, typename... Args, size_t... Indexes> + auto CallEnterMemberFunction(S* aS, std::tuple<Args...>& aTuple, + std::index_sequence<Indexes...>) + -> decltype(ReturnTypeHelper(&S::Enter)) { + AUTO_PROFILER_LABEL("StateObject::CallEnterMemberFunction", MEDIA_PLAYBACK); + return aS->Enter(std::move(std::get<Indexes>(aTuple))...); + } + + // Note this function will delete the current state object. + // Don't access members to avoid UAF after this call. + template <class S, typename... Ts> + auto SetState(Ts&&... aArgs) -> decltype(ReturnTypeHelper(&S::Enter)) { + // |aArgs| must be passed by reference to avoid passing MOZ_NON_PARAM class + // SeekJob by value. See bug 1287006 and bug 1338374. But we still *must* + // copy the parameters, because |Exit()| can modify them. See bug 1312321. + // So we 1) pass the parameters by reference, but then 2) immediately copy + // them into a Tuple to be safe against modification, and finally 3) move + // the elements of the Tuple into the final function call. + auto copiedArgs = std::make_tuple(std::forward<Ts>(aArgs)...); + + // Copy mMaster which will reset to null. + auto* master = mMaster; + + auto* s = new S(master); + + // It's possible to seek again during seeking, otherwise the new state + // should always be different from the original one. + MOZ_ASSERT(GetState() != s->GetState() || + GetState() == DECODER_STATE_SEEKING_ACCURATE || + GetState() == DECODER_STATE_SEEKING_FROMDORMANT || + GetState() == DECODER_STATE_SEEKING_NEXTFRAMESEEKING || + GetState() == DECODER_STATE_SEEKING_VIDEOONLY); + + SLOG("change state to: %s", ToStateStr(s->GetState())); + PROFILER_MARKER_TEXT("MDSM::StateChange", MEDIA_PLAYBACK, {}, + nsPrintfCString("%s", ToStateStr(s->GetState()))); + + Exit(); + + // Delete the old state asynchronously to avoid UAF if the caller tries to + // access its members after SetState() returns. + master->OwnerThread()->DispatchDirectTask( + NS_NewRunnableFunction("MDSM::StateObject::DeleteOldState", + [toDelete = std::move(master->mStateObj)]() {})); + // Also reset mMaster to catch potentail UAF. + mMaster = nullptr; + + master->mStateObj.reset(s); + return CallEnterMemberFunction(s, copiedArgs, + std::index_sequence_for<Ts...>{}); + } + + RefPtr<MediaDecoder::SeekPromise> SetSeekingState( + SeekJob&& aSeekJob, EventVisibility aVisibility); + + void SetDecodingState(); + + // Take a raw pointer in order not to change the life cycle of MDSM. + // It is guaranteed to be valid by MDSM. + Master* mMaster; +}; + +/** + * Purpose: decode metadata like duration and dimensions of the media resource. + * + * Transition to other states when decoding metadata is done: + * SHUTDOWN if failing to decode metadata. + * DECODING_FIRSTFRAME otherwise. + */ +class MediaDecoderStateMachine::DecodeMetadataState + : public MediaDecoderStateMachine::StateObject { + public: + explicit DecodeMetadataState(Master* aPtr) : StateObject(aPtr) {} + + void Enter() { + MOZ_ASSERT(!mMaster->mVideoDecodeSuspended); + MOZ_ASSERT(!mMetadataRequest.Exists()); + SLOG("Dispatching AsyncReadMetadata"); + + // We disconnect mMetadataRequest in Exit() so it is fine to capture + // a raw pointer here. + Reader() + ->ReadMetadata() + ->Then( + OwnerThread(), __func__, + [this](MetadataHolder&& aMetadata) { + OnMetadataRead(std::move(aMetadata)); + }, + [this](const MediaResult& aError) { OnMetadataNotRead(aError); }) + ->Track(mMetadataRequest); + } + + void Exit() override { mMetadataRequest.DisconnectIfExists(); } + + State GetState() const override { return DECODER_STATE_DECODING_METADATA; } + + RefPtr<MediaDecoder::SeekPromise> HandleSeek( + const SeekTarget& aTarget) override { + MOZ_DIAGNOSTIC_ASSERT(false, "Can't seek while decoding metadata."); + return MediaDecoder::SeekPromise::CreateAndReject(true, __func__); + } + + void HandleVideoSuspendTimeout() override { + // Do nothing since no decoders are created yet. + } + + void HandleResumeVideoDecoding(const TimeUnit&) override { + // We never suspend video decoding in this state. + MOZ_ASSERT(false, "Shouldn't have suspended video decoding."); + } + + private: + void OnMetadataRead(MetadataHolder&& aMetadata); + + void OnMetadataNotRead(const MediaResult& aError) { + AUTO_PROFILER_LABEL("DecodeMetadataState::OnMetadataNotRead", + MEDIA_PLAYBACK); + + mMetadataRequest.Complete(); + SLOGE("Decode metadata failed, shutting down decoder"); + mMaster->DecodeError(aError); + } + + MozPromiseRequestHolder<MediaFormatReader::MetadataPromise> mMetadataRequest; +}; + +/** + * Purpose: release decoder resources to save memory and hardware resources. + * + * Transition to: + * SEEKING if any seek request or play state changes to PLAYING. + */ +class MediaDecoderStateMachine::DormantState + : public MediaDecoderStateMachine::StateObject { + public: + explicit DormantState(Master* aPtr) : StateObject(aPtr) {} + + void Enter() { + if (mMaster->IsPlaying()) { + mMaster->StopPlayback(); + } + + // Calculate the position to seek to when exiting dormant. + auto t = mMaster->mMediaSink->IsStarted() ? mMaster->GetClock() + : mMaster->GetMediaTime(); + mMaster->AdjustByLooping(t); + mPendingSeek.mTarget.emplace(t, SeekTarget::Accurate); + // SeekJob asserts |mTarget.IsValid() == !mPromise.IsEmpty()| so we + // need to create the promise even it is not used at all. + // The promise may be used when coming out of DormantState into + // SeekingState. + RefPtr<MediaDecoder::SeekPromise> x = + mPendingSeek.mPromise.Ensure(__func__); + + // Reset the decoding state to ensure that any queued video frames are + // released and don't consume video memory. + mMaster->ResetDecode(); + + // No need to call StopMediaSink() here. + // We will do it during seeking when exiting dormant. + + // Ignore WAIT_FOR_DATA since we won't decode in dormant. + mMaster->mAudioWaitRequest.DisconnectIfExists(); + mMaster->mVideoWaitRequest.DisconnectIfExists(); + + MaybeReleaseResources(); + } + + void Exit() override { + // mPendingSeek is either moved when exiting dormant or + // should be rejected here before transition to SHUTDOWN. + mPendingSeek.RejectIfExists(__func__); + } + + State GetState() const override { return DECODER_STATE_DORMANT; } + + RefPtr<MediaDecoder::SeekPromise> HandleSeek( + const SeekTarget& aTarget) override; + + void HandleVideoSuspendTimeout() override { + // Do nothing since we've released decoders in Enter(). + } + + void HandleResumeVideoDecoding(const TimeUnit&) override { + // Do nothing since we won't resume decoding until exiting dormant. + } + + void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) override; + + void HandleAudioDecoded(AudioData*) override { MaybeReleaseResources(); } + void HandleVideoDecoded(VideoData*) override { MaybeReleaseResources(); } + void HandleWaitingForAudio() override { MaybeReleaseResources(); } + void HandleWaitingForVideo() override { MaybeReleaseResources(); } + void HandleAudioCanceled() override { MaybeReleaseResources(); } + void HandleVideoCanceled() override { MaybeReleaseResources(); } + void HandleEndOfAudio() override { MaybeReleaseResources(); } + void HandleEndOfVideo() override { MaybeReleaseResources(); } + + private: + void MaybeReleaseResources() { + if (!mMaster->mAudioDataRequest.Exists() && + !mMaster->mVideoDataRequest.Exists()) { + // Release decoders only when they are idle. Otherwise it might cause + // decode error later when resetting decoders during seeking. + mMaster->mReader->ReleaseResources(); + } + } + + SeekJob mPendingSeek; +}; + +/** + * Purpose: decode the 1st audio and video frames to fire the 'loadeddata' + * event. + * + * Transition to: + * SHUTDOWN if any decode error. + * SEEKING if any seek request. + * DECODING/LOOPING_DECODING when the 'loadeddata' event is fired. + */ +class MediaDecoderStateMachine::DecodingFirstFrameState + : public MediaDecoderStateMachine::StateObject { + public: + explicit DecodingFirstFrameState(Master* aPtr) : StateObject(aPtr) {} + + void Enter(); + + void Exit() override { + // mPendingSeek is either moved in MaybeFinishDecodeFirstFrame() + // or should be rejected here before transition to SHUTDOWN. + mPendingSeek.RejectIfExists(__func__); + } + + State GetState() const override { return DECODER_STATE_DECODING_FIRSTFRAME; } + + void HandleAudioDecoded(AudioData* aAudio) override { + mMaster->PushAudio(aAudio); + MaybeFinishDecodeFirstFrame(); + } + + void HandleVideoDecoded(VideoData* aVideo) override { + mMaster->PushVideo(aVideo); + MaybeFinishDecodeFirstFrame(); + } + + void HandleWaitingForAudio() override { + mMaster->WaitForData(MediaData::Type::AUDIO_DATA); + } + + void HandleAudioCanceled() override { mMaster->RequestAudioData(); } + + void HandleEndOfAudio() override { + AudioQueue().Finish(); + MaybeFinishDecodeFirstFrame(); + } + + void HandleWaitingForVideo() override { + mMaster->WaitForData(MediaData::Type::VIDEO_DATA); + } + + void HandleVideoCanceled() override { + mMaster->RequestVideoData(media::TimeUnit()); + } + + void HandleEndOfVideo() override { + VideoQueue().Finish(); + MaybeFinishDecodeFirstFrame(); + } + + void HandleAudioWaited(MediaData::Type aType) override { + mMaster->RequestAudioData(); + } + + void HandleVideoWaited(MediaData::Type aType) override { + mMaster->RequestVideoData(media::TimeUnit()); + } + + void HandleVideoSuspendTimeout() override { + // Do nothing for we need to decode the 1st video frame to get the + // dimensions. + } + + void HandleResumeVideoDecoding(const TimeUnit&) override { + // We never suspend video decoding in this state. + MOZ_ASSERT(false, "Shouldn't have suspended video decoding."); + } + + RefPtr<MediaDecoder::SeekPromise> HandleSeek( + const SeekTarget& aTarget) override { + if (mMaster->mIsMSE) { + return StateObject::HandleSeek(aTarget); + } + // Delay seek request until decoding first frames for non-MSE media. + SLOG("Not Enough Data to seek at this stage, queuing seek"); + mPendingSeek.RejectIfExists(__func__); + mPendingSeek.mTarget.emplace(aTarget); + return mPendingSeek.mPromise.Ensure(__func__); + } + + private: + // Notify FirstFrameLoaded if having decoded first frames and + // transition to SEEKING if there is any pending seek, or DECODING otherwise. + void MaybeFinishDecodeFirstFrame(); + + SeekJob mPendingSeek; +}; + +/** + * Purpose: decode audio/video data for playback. + * + * Transition to: + * DORMANT if playback is paused for a while. + * SEEKING if any seek request. + * SHUTDOWN if any decode error. + * BUFFERING if playback can't continue due to lack of decoded data. + * COMPLETED when having decoded all audio/video data. + * LOOPING_DECODING when media start seamless looping + */ +class MediaDecoderStateMachine::DecodingState + : public MediaDecoderStateMachine::StateObject { + public: + explicit DecodingState(Master* aPtr) + : StateObject(aPtr), mDormantTimer(OwnerThread()) {} + + void Enter(); + + void Exit() override { + if (!mDecodeStartTime.IsNull()) { + TimeDuration decodeDuration = TimeStamp::Now() - mDecodeStartTime; + SLOG("Exiting DECODING, decoded for %.3lfs", decodeDuration.ToSeconds()); + } + mDormantTimer.Reset(); + mOnAudioPopped.DisconnectIfExists(); + mOnVideoPopped.DisconnectIfExists(); + } + + void Step() override; + + State GetState() const override { return DECODER_STATE_DECODING; } + + void HandleAudioDecoded(AudioData* aAudio) override { + mMaster->PushAudio(aAudio); + DispatchDecodeTasksIfNeeded(); + MaybeStopPrerolling(); + } + + void HandleVideoDecoded(VideoData* aVideo) override { + // We only do this check when we're not looping, which can be known by + // checking the queue's offset. + const auto currentTime = mMaster->GetMediaTime(); + if (aVideo->GetEndTime() < currentTime && + VideoQueue().GetOffset() == media::TimeUnit::Zero()) { + if (!mVideoFirstLateTime) { + mVideoFirstLateTime = Some(TimeStamp::Now()); + } + PROFILER_MARKER("Video falling behind", MEDIA_PLAYBACK, {}, + VideoFallingBehindMarker, aVideo->mTime.ToMicroseconds(), + currentTime.ToMicroseconds()); + SLOG("video %" PRId64 " starts being late (current=%" PRId64 ")", + aVideo->mTime.ToMicroseconds(), currentTime.ToMicroseconds()); + } else { + mVideoFirstLateTime.reset(); + } + mMaster->PushVideo(aVideo); + DispatchDecodeTasksIfNeeded(); + MaybeStopPrerolling(); + } + + void HandleAudioCanceled() override { mMaster->RequestAudioData(); } + + void HandleVideoCanceled() override { + mMaster->RequestVideoData(mMaster->GetMediaTime(), + ShouldRequestNextKeyFrame()); + } + + void HandleEndOfAudio() override; + void HandleEndOfVideo() override; + + void HandleWaitingForAudio() override { + mMaster->WaitForData(MediaData::Type::AUDIO_DATA); + MaybeStopPrerolling(); + } + + void HandleWaitingForVideo() override { + mMaster->WaitForData(MediaData::Type::VIDEO_DATA); + MaybeStopPrerolling(); + } + + void HandleAudioWaited(MediaData::Type aType) override { + mMaster->RequestAudioData(); + } + + void HandleVideoWaited(MediaData::Type aType) override { + mMaster->RequestVideoData(mMaster->GetMediaTime(), + ShouldRequestNextKeyFrame()); + } + + void HandleAudioCaptured() override { + MaybeStopPrerolling(); + // MediaSink is changed. Schedule Step() to check if we can start playback. + mMaster->ScheduleStateMachine(); + } + + void HandleVideoSuspendTimeout() override { + // No video, so nothing to suspend. + if (!mMaster->HasVideo()) { + return; + } + + PROFILER_MARKER_UNTYPED("MDSM::EnterVideoSuspend", MEDIA_PLAYBACK); + mMaster->mVideoDecodeSuspended = true; + mMaster->mOnPlaybackEvent.Notify(MediaPlaybackEvent::EnterVideoSuspend); + Reader()->SetVideoBlankDecode(true); + } + + void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) override { + if (aPlayState == MediaDecoder::PLAY_STATE_PLAYING) { + // Schedule Step() to check if we can start playback. + mMaster->ScheduleStateMachine(); + // Try to dispatch decoding tasks for mMinimizePreroll might be reset. + DispatchDecodeTasksIfNeeded(); + } + + if (aPlayState == MediaDecoder::PLAY_STATE_PAUSED) { + StartDormantTimer(); + mVideoFirstLateTime.reset(); + } else { + mDormantTimer.Reset(); + } + } + + void GetDebugInfo( + dom::MediaDecoderStateMachineDecodingStateDebugInfo& aInfo) override { + aInfo.mIsPrerolling = mIsPrerolling; + } + + void HandleLoopingChanged() override { SetDecodingState(); } + + protected: + virtual void EnsureAudioDecodeTaskQueued(); + virtual void EnsureVideoDecodeTaskQueued(); + + virtual bool ShouldStopPrerolling() const { + return mIsPrerolling && + (DonePrerollingAudio() || + IsWaitingData(MediaData::Type::AUDIO_DATA)) && + (DonePrerollingVideo() || + IsWaitingData(MediaData::Type::VIDEO_DATA)); + } + + virtual bool IsWaitingData(MediaData::Type aType) const { + if (aType == MediaData::Type::AUDIO_DATA) { + return mMaster->IsWaitingAudioData(); + } + MOZ_ASSERT(aType == MediaData::Type::VIDEO_DATA); + return mMaster->IsWaitingVideoData(); + } + + void MaybeStopPrerolling() { + if (ShouldStopPrerolling()) { + mIsPrerolling = false; + // Check if we can start playback. + mMaster->ScheduleStateMachine(); + } + } + + bool ShouldRequestNextKeyFrame() const { + if (!mVideoFirstLateTime) { + return false; + } + const double elapsedTimeMs = + (TimeStamp::Now() - *mVideoFirstLateTime).ToMilliseconds(); + const bool rv = elapsedTimeMs >= + StaticPrefs::media_decoder_skip_when_video_too_slow_ms(); + if (rv) { + PROFILER_MARKER_UNTYPED("Skipping to next keyframe", MEDIA_PLAYBACK); + SLOG( + "video has been late behind media time for %f ms, should skip to " + "next key frame", + elapsedTimeMs); + } + return rv; + } + + virtual bool IsBufferingAllowed() const { return true; } + + private: + void DispatchDecodeTasksIfNeeded(); + void MaybeStartBuffering(); + + // At the start of decoding we want to "preroll" the decode until we've + // got a few frames decoded before we consider whether decode is falling + // behind. Otherwise our "we're falling behind" logic will trigger + // unnecessarily if we start playing as soon as the first sample is + // decoded. These two fields store how many video frames and audio + // samples we must consume before are considered to be finished prerolling. + TimeUnit AudioPrerollThreshold() const { + return (mMaster->mAmpleAudioThreshold / 2) + .MultDouble(mMaster->mPlaybackRate); + } + + uint32_t VideoPrerollFrames() const { + return std::min( + static_cast<uint32_t>( + mMaster->GetAmpleVideoFrames() / 2. * mMaster->mPlaybackRate + 1), + sVideoQueueDefaultSize); + } + + bool DonePrerollingAudio() const { + return !mMaster->IsAudioDecoding() || + mMaster->GetDecodedAudioDuration() >= AudioPrerollThreshold(); + } + + bool DonePrerollingVideo() const { + return !mMaster->IsVideoDecoding() || + static_cast<uint32_t>(mMaster->VideoQueue().GetSize()) >= + VideoPrerollFrames(); + } + + void StartDormantTimer() { + if (!mMaster->mMediaSeekable) { + // Don't enter dormant if the media is not seekable because we need to + // seek when exiting dormant. + return; + } + + auto timeout = StaticPrefs::media_dormant_on_pause_timeout_ms(); + if (timeout < 0) { + // Disabled when timeout is negative. + return; + } + + if (timeout == 0) { + // Enter dormant immediately without scheduling a timer. + SetState<DormantState>(); + return; + } + + if (mMaster->mMinimizePreroll) { + SetState<DormantState>(); + return; + } + + TimeStamp target = + TimeStamp::Now() + TimeDuration::FromMilliseconds(timeout); + + mDormantTimer.Ensure( + target, + [this]() { + AUTO_PROFILER_LABEL("DecodingState::StartDormantTimer:SetDormant", + MEDIA_PLAYBACK); + mDormantTimer.CompleteRequest(); + SetState<DormantState>(); + }, + [this]() { mDormantTimer.CompleteRequest(); }); + } + + // Time at which we started decoding. + TimeStamp mDecodeStartTime; + + // When we start decoding (either for the first time, or after a pause) + // we may be low on decoded data. We don't want our "low data" logic to + // kick in and decide that we're low on decoded data because the download + // can't keep up with the decode, and cause us to pause playback. So we + // have a "preroll" stage, where we ignore the results of our "low data" + // logic during the first few frames of our decode. This occurs during + // playback. + bool mIsPrerolling = true; + + // Fired when playback is paused for a while to enter dormant. + DelayedScheduler mDormantTimer; + + MediaEventListener mOnAudioPopped; + MediaEventListener mOnVideoPopped; + + // If video has been later than the media time, this will records when the + // video started being late. It will be reset once video catches up with the + // media time. + Maybe<TimeStamp> mVideoFirstLateTime; +}; + +/** + * Purpose: decode audio data for playback when media is in seamless + * looping, we will adjust media time to make samples time monotonically + * increasing. All its methods runs on its owner thread (MDSM thread). + * + * Transition to: + * DORMANT if playback is paused for a while. + * SEEKING if any seek request. + * SHUTDOWN if any decode error. + * BUFFERING if playback can't continue due to lack of decoded data. + * COMPLETED when the media resource is closed and no data is available + * anymore. + * DECODING when media stops seamless looping. + */ +class MediaDecoderStateMachine::LoopingDecodingState + : public MediaDecoderStateMachine::DecodingState { + public: + explicit LoopingDecodingState(Master* aPtr) + : DecodingState(aPtr), + mIsReachingAudioEOS(!mMaster->IsAudioDecoding()), + mIsReachingVideoEOS(!mMaster->IsVideoDecoding()), + mAudioEndedBeforeEnteringStateWithoutDuration(false), + mVideoEndedBeforeEnteringStateWithoutDuration(false) { + MOZ_ASSERT(mMaster->mLooping); + SLOG( + "LoopingDecodingState ctor, mIsReachingAudioEOS=%d, " + "mIsReachingVideoEOS=%d", + mIsReachingAudioEOS, mIsReachingVideoEOS); + // If the track has reached EOS and we already have its last data, then we + // can know its duration. But if playback starts from EOS (due to seeking), + // the decoded end time would be zero because none of data gets decoded yet. + if (mIsReachingAudioEOS) { + if (mMaster->HasLastDecodedData(MediaData::Type::AUDIO_DATA) && + !mMaster->mAudioTrackDecodedDuration) { + mMaster->mAudioTrackDecodedDuration.emplace( + mMaster->mDecodedAudioEndTime); + SLOG("determine mAudioTrackDecodedDuration"); + } else { + mAudioEndedBeforeEnteringStateWithoutDuration = true; + SLOG("still don't know mAudioTrackDecodedDuration"); + } + } + + if (mIsReachingVideoEOS) { + if (mMaster->HasLastDecodedData(MediaData::Type::VIDEO_DATA) && + !mMaster->mVideoTrackDecodedDuration) { + mMaster->mVideoTrackDecodedDuration.emplace( + mMaster->mDecodedVideoEndTime); + SLOG("determine mVideoTrackDecodedDuration"); + } else { + mVideoEndedBeforeEnteringStateWithoutDuration = true; + SLOG("still don't know mVideoTrackDecodedDuration"); + } + } + + // We might be able to determine the duration already, let's check. + if (mIsReachingAudioEOS || mIsReachingVideoEOS) { + Unused << DetermineOriginalDecodedDurationIfNeeded(); + } + + // If we've looped at least once before, then we need to update queue offset + // correctly to make the media data time and the clock time consistent. + // Otherwise, it would cause a/v desync. + if (mMaster->mOriginalDecodedDuration != media::TimeUnit::Zero()) { + if (mIsReachingAudioEOS && mMaster->HasAudio()) { + AudioQueue().SetOffset(AudioQueue().GetOffset() + + mMaster->mOriginalDecodedDuration); + } + if (mIsReachingVideoEOS && mMaster->HasVideo()) { + VideoQueue().SetOffset(VideoQueue().GetOffset() + + mMaster->mOriginalDecodedDuration); + } + } + } + + void Enter() { + if (mMaster->HasAudio() && mIsReachingAudioEOS) { + SLOG("audio has ended, request the data again."); + RequestDataFromStartPosition(TrackInfo::TrackType::kAudioTrack); + } + if (mMaster->HasVideo() && mIsReachingVideoEOS) { + SLOG("video has ended, request the data again."); + RequestDataFromStartPosition(TrackInfo::TrackType::kVideoTrack); + } + DecodingState::Enter(); + } + + void Exit() override { + MOZ_DIAGNOSTIC_ASSERT(mMaster->OnTaskQueue()); + SLOG("Leaving looping state, offset [a=%" PRId64 ",v=%" PRId64 + "], endtime [a=%" PRId64 ",v=%" PRId64 "], track duration [a=%" PRId64 + ",v=%" PRId64 "], waiting=%s", + AudioQueue().GetOffset().ToMicroseconds(), + VideoQueue().GetOffset().ToMicroseconds(), + mMaster->mDecodedAudioEndTime.ToMicroseconds(), + mMaster->mDecodedVideoEndTime.ToMicroseconds(), + mMaster->mAudioTrackDecodedDuration + ? mMaster->mAudioTrackDecodedDuration->ToMicroseconds() + : 0, + mMaster->mVideoTrackDecodedDuration + ? mMaster->mVideoTrackDecodedDuration->ToMicroseconds() + : 0, + mDataWaitingTimestampAdjustment + ? MediaData::TypeToStr(mDataWaitingTimestampAdjustment->mType) + : "none"); + if (ShouldDiscardLoopedData(MediaData::Type::AUDIO_DATA)) { + DiscardLoopedData(MediaData::Type::AUDIO_DATA); + } + if (ShouldDiscardLoopedData(MediaData::Type::VIDEO_DATA)) { + DiscardLoopedData(MediaData::Type::VIDEO_DATA); + } + + if (mMaster->HasAudio() && HasDecodedLastAudioFrame()) { + SLOG("Mark audio queue as finished"); + mMaster->mAudioDataRequest.DisconnectIfExists(); + mMaster->mAudioWaitRequest.DisconnectIfExists(); + AudioQueue().Finish(); + } + if (mMaster->HasVideo() && HasDecodedLastVideoFrame()) { + SLOG("Mark video queue as finished"); + mMaster->mVideoDataRequest.DisconnectIfExists(); + mMaster->mVideoWaitRequest.DisconnectIfExists(); + VideoQueue().Finish(); + } + + // Clear waiting data should be done after marking queue as finished. + mDataWaitingTimestampAdjustment = nullptr; + + mAudioDataRequest.DisconnectIfExists(); + mVideoDataRequest.DisconnectIfExists(); + mAudioSeekRequest.DisconnectIfExists(); + mVideoSeekRequest.DisconnectIfExists(); + DecodingState::Exit(); + } + + ~LoopingDecodingState() { + MOZ_DIAGNOSTIC_ASSERT(!mAudioDataRequest.Exists()); + MOZ_DIAGNOSTIC_ASSERT(!mVideoDataRequest.Exists()); + MOZ_DIAGNOSTIC_ASSERT(!mAudioSeekRequest.Exists()); + MOZ_DIAGNOSTIC_ASSERT(!mVideoSeekRequest.Exists()); + } + + State GetState() const override { return DECODER_STATE_LOOPING_DECODING; } + + void HandleAudioDecoded(AudioData* aAudio) override { + // TODO : check if we need to update mOriginalDecodedDuration + + // After pushing data to the queue, timestamp might be adjusted. + DecodingState::HandleAudioDecoded(aAudio); + mMaster->mDecodedAudioEndTime = + std::max(aAudio->GetEndTime(), mMaster->mDecodedAudioEndTime); + SLOG("audio sample after time-adjustment [%" PRId64 ",%" PRId64 "]", + aAudio->mTime.ToMicroseconds(), aAudio->GetEndTime().ToMicroseconds()); + } + + void HandleVideoDecoded(VideoData* aVideo) override { + // TODO : check if we need to update mOriginalDecodedDuration + + // Here sample still keeps its original timestamp. + + // This indicates there is a shorter audio track, and it's the first time in + // the looping (audio ends but video is playing) so that we haven't been + // able to determine the decoded duration. Therefore, we fill the gap + // between two tracks before video ends. Afterward, this adjustment will be + // done in `HandleEndOfAudio()`. + if (mMaster->mOriginalDecodedDuration == media::TimeUnit::Zero() && + mMaster->mAudioTrackDecodedDuration && + aVideo->GetEndTime() > *mMaster->mAudioTrackDecodedDuration) { + media::TimeUnit gap; + // First time we fill gap between the video frame to the last audio. + if (auto prevVideo = VideoQueue().PeekBack(); + prevVideo && + prevVideo->GetEndTime() < *mMaster->mAudioTrackDecodedDuration) { + gap = + aVideo->GetEndTime().ToBase(*mMaster->mAudioTrackDecodedDuration) - + *mMaster->mAudioTrackDecodedDuration; + } + // Then fill the gap for all following videos. + else { + gap = aVideo->mDuration.ToBase(*mMaster->mAudioTrackDecodedDuration); + } + SLOG("Longer video %" PRId64 "%s (audio-durtaion=%" PRId64 + "%s), insert silence to fill the gap %" PRId64 "%s", + aVideo->GetEndTime().ToMicroseconds(), + aVideo->GetEndTime().ToString().get(), + mMaster->mAudioTrackDecodedDuration->ToMicroseconds(), + mMaster->mAudioTrackDecodedDuration->ToString().get(), + gap.ToMicroseconds(), gap.ToString().get()); + PushFakeAudioDataIfNeeded(gap); + } + + // After pushing data to the queue, timestamp might be adjusted. + DecodingState::HandleVideoDecoded(aVideo); + mMaster->mDecodedVideoEndTime = + std::max(aVideo->GetEndTime(), mMaster->mDecodedVideoEndTime); + SLOG("video sample after time-adjustment [%" PRId64 ",%" PRId64 "]", + aVideo->mTime.ToMicroseconds(), aVideo->GetEndTime().ToMicroseconds()); + } + + void HandleEndOfAudio() override { + mIsReachingAudioEOS = true; + if (!mMaster->mAudioTrackDecodedDuration && + mMaster->HasLastDecodedData(MediaData::Type::AUDIO_DATA)) { + mMaster->mAudioTrackDecodedDuration.emplace( + mMaster->mDecodedAudioEndTime); + } + if (DetermineOriginalDecodedDurationIfNeeded()) { + AudioQueue().SetOffset(AudioQueue().GetOffset() + + mMaster->mOriginalDecodedDuration); + } + + // This indicates that the audio track is shorter than the video track, so + // we need to add some silence to fill the gap. + if (mMaster->mAudioTrackDecodedDuration && + mMaster->mOriginalDecodedDuration > + *mMaster->mAudioTrackDecodedDuration) { + MOZ_ASSERT(mMaster->HasVideo()); + MOZ_ASSERT(mMaster->mVideoTrackDecodedDuration); + MOZ_ASSERT(mMaster->mOriginalDecodedDuration == + *mMaster->mVideoTrackDecodedDuration); + auto gap = mMaster->mOriginalDecodedDuration.ToBase( + *mMaster->mAudioTrackDecodedDuration) - + *mMaster->mAudioTrackDecodedDuration; + SLOG( + "Audio track is shorter than the original decoded duration " + "(a=%" PRId64 "%s, t=%" PRId64 + "%s), insert silence to fill the gap %" PRId64 "%s", + mMaster->mAudioTrackDecodedDuration->ToMicroseconds(), + mMaster->mAudioTrackDecodedDuration->ToString().get(), + mMaster->mOriginalDecodedDuration.ToMicroseconds(), + mMaster->mOriginalDecodedDuration.ToString().get(), + gap.ToMicroseconds(), gap.ToString().get()); + PushFakeAudioDataIfNeeded(gap); + } + + SLOG( + "received audio EOS when seamless looping, starts seeking, " + "audioLoopingOffset=[%" PRId64 "], mAudioTrackDecodedDuration=[%" PRId64 + "]", + AudioQueue().GetOffset().ToMicroseconds(), + mMaster->mAudioTrackDecodedDuration->ToMicroseconds()); + if (!IsRequestingDataFromStartPosition(MediaData::Type::AUDIO_DATA)) { + RequestDataFromStartPosition(TrackInfo::TrackType::kAudioTrack); + } + ProcessSamplesWaitingAdjustmentIfAny(); + } + + void HandleEndOfVideo() override { + mIsReachingVideoEOS = true; + if (!mMaster->mVideoTrackDecodedDuration && + mMaster->HasLastDecodedData(MediaData::Type::VIDEO_DATA)) { + mMaster->mVideoTrackDecodedDuration.emplace( + mMaster->mDecodedVideoEndTime); + } + if (DetermineOriginalDecodedDurationIfNeeded()) { + VideoQueue().SetOffset(VideoQueue().GetOffset() + + mMaster->mOriginalDecodedDuration); + } + + SLOG( + "received video EOS when seamless looping, starts seeking, " + "videoLoopingOffset=[%" PRId64 "], mVideoTrackDecodedDuration=[%" PRId64 + "]", + VideoQueue().GetOffset().ToMicroseconds(), + mMaster->mVideoTrackDecodedDuration->ToMicroseconds()); + if (!IsRequestingDataFromStartPosition(MediaData::Type::VIDEO_DATA)) { + RequestDataFromStartPosition(TrackInfo::TrackType::kVideoTrack); + } + ProcessSamplesWaitingAdjustmentIfAny(); + } + + private: + void RequestDataFromStartPosition(TrackInfo::TrackType aType) { + MOZ_DIAGNOSTIC_ASSERT(aType == TrackInfo::TrackType::kAudioTrack || + aType == TrackInfo::TrackType::kVideoTrack); + + const bool isAudio = aType == TrackInfo::TrackType::kAudioTrack; + MOZ_ASSERT_IF(isAudio, mMaster->HasAudio()); + MOZ_ASSERT_IF(!isAudio, mMaster->HasVideo()); + + if (IsReaderSeeking()) { + MOZ_ASSERT(!mPendingSeekingType); + mPendingSeekingType = Some(aType); + SLOG("Delay %s seeking until the reader finishes current seeking", + isAudio ? "audio" : "video"); + return; + } + + auto& seekRequest = isAudio ? mAudioSeekRequest : mVideoSeekRequest; + Reader()->ResetDecode(aType); + Reader() + ->Seek(SeekTarget(media::TimeUnit::Zero(), SeekTarget::Type::Accurate, + isAudio ? SeekTarget::Track::AudioOnly + : SeekTarget::Track::VideoOnly)) + ->Then( + OwnerThread(), __func__, + [this, isAudio, master = RefPtr{mMaster}]() mutable -> void { + AUTO_PROFILER_LABEL( + nsPrintfCString( + "LoopingDecodingState::RequestDataFromStartPosition(%s)::" + "SeekResolved", + isAudio ? "audio" : "video") + .get(), + MEDIA_PLAYBACK); + if (auto& state = master->mStateObj; + state && + state->GetState() != DECODER_STATE_LOOPING_DECODING) { + MOZ_RELEASE_ASSERT(false, "This shouldn't happen!"); + return; + } + if (isAudio) { + mAudioSeekRequest.Complete(); + } else { + mVideoSeekRequest.Complete(); + } + SLOG( + "seeking completed, start to request first %s sample " + "(queued=%zu, decoder-queued=%zu)", + isAudio ? "audio" : "video", + isAudio ? AudioQueue().GetSize() : VideoQueue().GetSize(), + isAudio ? Reader()->SizeOfAudioQueueInFrames() + : Reader()->SizeOfVideoQueueInFrames()); + if (isAudio) { + RequestAudioDataFromReaderAfterEOS(); + } else { + RequestVideoDataFromReaderAfterEOS(); + } + if (mPendingSeekingType) { + auto seekingType = *mPendingSeekingType; + mPendingSeekingType.reset(); + SLOG("Perform pending %s seeking", TrackTypeToStr(seekingType)); + RequestDataFromStartPosition(seekingType); + } + }, + [this, isAudio, master = RefPtr{mMaster}]( + const SeekRejectValue& aReject) mutable -> void { + AUTO_PROFILER_LABEL( + nsPrintfCString("LoopingDecodingState::" + "RequestDataFromStartPosition(%s)::" + "SeekRejected", + isAudio ? "audio" : "video") + .get(), + MEDIA_PLAYBACK); + if (auto& state = master->mStateObj; + state && + state->GetState() != DECODER_STATE_LOOPING_DECODING) { + MOZ_RELEASE_ASSERT(false, "This shouldn't happen!"); + return; + } + if (isAudio) { + mAudioSeekRequest.Complete(); + } else { + mVideoSeekRequest.Complete(); + } + HandleError(aReject.mError, isAudio); + }) + ->Track(seekRequest); + } + + void RequestAudioDataFromReaderAfterEOS() { + MOZ_ASSERT(mMaster->HasAudio()); + Reader() + ->RequestAudioData() + ->Then( + OwnerThread(), __func__, + [this, master = RefPtr{mMaster}](const RefPtr<AudioData>& aAudio) { + AUTO_PROFILER_LABEL( + "LoopingDecodingState::" + "RequestAudioDataFromReader::" + "RequestDataResolved", + MEDIA_PLAYBACK); + if (auto& state = master->mStateObj; + state && + state->GetState() != DECODER_STATE_LOOPING_DECODING) { + MOZ_RELEASE_ASSERT(false, "This shouldn't happen!"); + return; + } + mIsReachingAudioEOS = false; + mAudioDataRequest.Complete(); + SLOG( + "got audio decoded sample " + "[%" PRId64 ",%" PRId64 "]", + aAudio->mTime.ToMicroseconds(), + aAudio->GetEndTime().ToMicroseconds()); + if (ShouldPutDataOnWaiting(MediaData::Type::AUDIO_DATA)) { + SLOG( + "decoded audio sample needs to wait for timestamp " + "adjustment after EOS"); + PutDataOnWaiting(aAudio); + return; + } + HandleAudioDecoded(aAudio); + ProcessSamplesWaitingAdjustmentIfAny(); + }, + [this, master = RefPtr{mMaster}](const MediaResult& aError) { + AUTO_PROFILER_LABEL( + "LoopingDecodingState::" + "RequestAudioDataFromReader::" + "RequestDataRejected", + MEDIA_PLAYBACK); + if (auto& state = master->mStateObj; + state && + state->GetState() != DECODER_STATE_LOOPING_DECODING) { + MOZ_RELEASE_ASSERT(false, "This shouldn't happen!"); + return; + } + mAudioDataRequest.Complete(); + HandleError(aError, true /* isAudio */); + }) + ->Track(mAudioDataRequest); + } + + void RequestVideoDataFromReaderAfterEOS() { + MOZ_ASSERT(mMaster->HasVideo()); + Reader() + ->RequestVideoData(media::TimeUnit(), + false /* aRequestNextVideoKeyFrame */) + ->Then( + OwnerThread(), __func__, + [this, master = RefPtr{mMaster}](const RefPtr<VideoData>& aVideo) { + AUTO_PROFILER_LABEL( + "LoopingDecodingState::" + "RequestVideoDataFromReaderAfterEOS()::" + "RequestDataResolved", + MEDIA_PLAYBACK); + if (auto& state = master->mStateObj; + state && + state->GetState() != DECODER_STATE_LOOPING_DECODING) { + MOZ_RELEASE_ASSERT(false, "This shouldn't happen!"); + return; + } + mIsReachingVideoEOS = false; + mVideoDataRequest.Complete(); + SLOG( + "got video decoded sample " + "[%" PRId64 ",%" PRId64 "]", + aVideo->mTime.ToMicroseconds(), + aVideo->GetEndTime().ToMicroseconds()); + if (ShouldPutDataOnWaiting(MediaData::Type::VIDEO_DATA)) { + SLOG( + "decoded video sample needs to wait for timestamp " + "adjustment after EOS"); + PutDataOnWaiting(aVideo); + return; + } + mMaster->mBypassingSkipToNextKeyFrameCheck = true; + HandleVideoDecoded(aVideo); + ProcessSamplesWaitingAdjustmentIfAny(); + }, + [this, master = RefPtr{mMaster}](const MediaResult& aError) { + AUTO_PROFILER_LABEL( + "LoopingDecodingState::" + "RequestVideoDataFromReaderAfterEOS()::" + "RequestDataRejected", + MEDIA_PLAYBACK); + if (auto& state = master->mStateObj; + state && + state->GetState() != DECODER_STATE_LOOPING_DECODING) { + MOZ_RELEASE_ASSERT(false, "This shouldn't happen!"); + return; + } + mVideoDataRequest.Complete(); + HandleError(aError, false /* isAudio */); + }) + ->Track(mVideoDataRequest); + } + + void HandleError(const MediaResult& aError, bool aIsAudio); + + bool ShouldRequestData(MediaData::Type aType) const { + MOZ_DIAGNOSTIC_ASSERT(aType == MediaData::Type::AUDIO_DATA || + aType == MediaData::Type::VIDEO_DATA); + + if (aType == MediaData::Type::AUDIO_DATA && + (mAudioSeekRequest.Exists() || mAudioDataRequest.Exists() || + IsDataWaitingForTimestampAdjustment(MediaData::Type::AUDIO_DATA) || + mMaster->IsWaitingAudioData())) { + return false; + } + if (aType == MediaData::Type::VIDEO_DATA && + (mVideoSeekRequest.Exists() || mVideoDataRequest.Exists() || + IsDataWaitingForTimestampAdjustment(MediaData::Type::VIDEO_DATA) || + mMaster->IsWaitingVideoData())) { + return false; + } + return true; + } + + void HandleAudioCanceled() override { + if (ShouldRequestData(MediaData::Type::AUDIO_DATA)) { + mMaster->RequestAudioData(); + } + } + + void HandleAudioWaited(MediaData::Type aType) override { + if (ShouldRequestData(MediaData::Type::AUDIO_DATA)) { + mMaster->RequestAudioData(); + } + } + + void HandleVideoCanceled() override { + if (ShouldRequestData(MediaData::Type::VIDEO_DATA)) { + mMaster->RequestVideoData(mMaster->GetMediaTime(), + ShouldRequestNextKeyFrame()); + }; + } + + void HandleVideoWaited(MediaData::Type aType) override { + if (ShouldRequestData(MediaData::Type::VIDEO_DATA)) { + mMaster->RequestVideoData(mMaster->GetMediaTime(), + ShouldRequestNextKeyFrame()); + }; + } + + void EnsureAudioDecodeTaskQueued() override { + if (!ShouldRequestData(MediaData::Type::AUDIO_DATA)) { + return; + } + DecodingState::EnsureAudioDecodeTaskQueued(); + } + + void EnsureVideoDecodeTaskQueued() override { + if (!ShouldRequestData(MediaData::Type::VIDEO_DATA)) { + return; + } + DecodingState::EnsureVideoDecodeTaskQueued(); + } + + bool DetermineOriginalDecodedDurationIfNeeded() { + // Duration would only need to be set once, unless we get more data which is + // larger than the duration. That can happen on MSE (reopen stream). + if (mMaster->mOriginalDecodedDuration != media::TimeUnit::Zero()) { + return true; + } + + // Single track situations + if (mMaster->HasAudio() && !mMaster->HasVideo() && + mMaster->mAudioTrackDecodedDuration) { + mMaster->mOriginalDecodedDuration = *mMaster->mAudioTrackDecodedDuration; + SLOG("audio only, duration=%" PRId64, + mMaster->mOriginalDecodedDuration.ToMicroseconds()); + return true; + } + if (mMaster->HasVideo() && !mMaster->HasAudio() && + mMaster->mVideoTrackDecodedDuration) { + mMaster->mOriginalDecodedDuration = *mMaster->mVideoTrackDecodedDuration; + SLOG("video only, duration=%" PRId64, + mMaster->mOriginalDecodedDuration.ToMicroseconds()); + return true; + } + // Two tracks situation + if (mMaster->HasAudio() && mMaster->HasVideo()) { + // Both tracks have ended so that we can check which track is longer. + if (mMaster->mAudioTrackDecodedDuration && + mMaster->mVideoTrackDecodedDuration) { + mMaster->mOriginalDecodedDuration = + std::max(*mMaster->mVideoTrackDecodedDuration, + *mMaster->mAudioTrackDecodedDuration); + SLOG("Both tracks ended, original duration=%" PRId64 " (a=%" PRId64 + ", v=%" PRId64 ")", + mMaster->mOriginalDecodedDuration.ToMicroseconds(), + mMaster->mAudioTrackDecodedDuration->ToMicroseconds(), + mMaster->mVideoTrackDecodedDuration->ToMicroseconds()); + return true; + } + // When entering the state, video has ended but audio hasn't, which means + // audio is longer. + if (mMaster->mAudioTrackDecodedDuration && + mVideoEndedBeforeEnteringStateWithoutDuration) { + mMaster->mOriginalDecodedDuration = + *mMaster->mAudioTrackDecodedDuration; + mVideoEndedBeforeEnteringStateWithoutDuration = false; + SLOG("audio is longer, duration=%" PRId64, + mMaster->mOriginalDecodedDuration.ToMicroseconds()); + return true; + } + // When entering the state, audio has ended but video hasn't, which means + // video is longer. + if (mMaster->mVideoTrackDecodedDuration && + mAudioEndedBeforeEnteringStateWithoutDuration) { + mMaster->mOriginalDecodedDuration = + *mMaster->mVideoTrackDecodedDuration; + mAudioEndedBeforeEnteringStateWithoutDuration = false; + SLOG("video is longer, duration=%" PRId64, + mMaster->mOriginalDecodedDuration.ToMicroseconds()); + return true; + } + SLOG("Still waiting for another track ends..."); + MOZ_ASSERT(!mMaster->mAudioTrackDecodedDuration || + !mMaster->mVideoTrackDecodedDuration); + } + SLOG("can't determine the original decoded duration yet"); + MOZ_ASSERT(mMaster->mOriginalDecodedDuration == media::TimeUnit::Zero()); + return false; + } + + void ProcessSamplesWaitingAdjustmentIfAny() { + if (!mDataWaitingTimestampAdjustment) { + return; + } + + RefPtr<MediaData> data = mDataWaitingTimestampAdjustment; + mDataWaitingTimestampAdjustment = nullptr; + const bool isAudio = data->mType == MediaData::Type::AUDIO_DATA; + SLOG("process %s sample waiting for timestamp adjustment", + isAudio ? "audio" : "video"); + if (isAudio) { + // Waiting sample is for next round of looping, so the queue offset + // shouldn't be zero. This happens when the track has reached EOS before + // entering the state (and looping never happens before). Same for below + // video case. + if (AudioQueue().GetOffset() == media::TimeUnit::Zero()) { + AudioQueue().SetOffset(mMaster->mOriginalDecodedDuration); + } + HandleAudioDecoded(data->As<AudioData>()); + } else { + MOZ_DIAGNOSTIC_ASSERT(data->mType == MediaData::Type::VIDEO_DATA); + if (VideoQueue().GetOffset() == media::TimeUnit::Zero()) { + VideoQueue().SetOffset(mMaster->mOriginalDecodedDuration); + } + HandleVideoDecoded(data->As<VideoData>()); + } + } + + bool IsDataWaitingForTimestampAdjustment(MediaData::Type aType) const { + return mDataWaitingTimestampAdjustment && + mDataWaitingTimestampAdjustment->mType == aType; + } + + bool ShouldPutDataOnWaiting(MediaData::Type aType) const { + // If another track is already waiting, this track shouldn't be waiting. + // This case only happens when both tracks reached EOS before entering the + // looping decoding state, so we don't know the decoded duration yet (used + // to adjust timestamp) But this is fine, because both tracks will start + // from 0 so we don't need to adjust them now. + if (mDataWaitingTimestampAdjustment && + !IsDataWaitingForTimestampAdjustment(aType)) { + return false; + } + + // Only have one track, no need to wait. + if ((aType == MediaData::Type::AUDIO_DATA && !mMaster->HasVideo()) || + (aType == MediaData::Type::VIDEO_DATA && !mMaster->HasAudio())) { + return false; + } + + // We don't know the duration yet, so we can't calculate the looping offset. + return mMaster->mOriginalDecodedDuration == media::TimeUnit::Zero(); + } + + void PutDataOnWaiting(MediaData* aData) { + MOZ_ASSERT(!mDataWaitingTimestampAdjustment); + mDataWaitingTimestampAdjustment = aData; + SLOG("put %s [%" PRId64 ",%" PRId64 "] on waiting", + MediaData::TypeToStr(aData->mType), aData->mTime.ToMicroseconds(), + aData->GetEndTime().ToMicroseconds()); + MaybeStopPrerolling(); + } + + bool ShouldDiscardLoopedData(MediaData::Type aType) const { + if (!mMaster->mMediaSink->IsStarted()) { + return false; + } + + MOZ_DIAGNOSTIC_ASSERT(aType == MediaData::Type::AUDIO_DATA || + aType == MediaData::Type::VIDEO_DATA); + const bool isAudio = aType == MediaData::Type::AUDIO_DATA; + if (isAudio && !mMaster->HasAudio()) { + return false; + } + if (!isAudio && !mMaster->HasVideo()) { + return false; + } + + /** + * If media cancels looping, we should check whether there is media data + * whose time is later than EOS. If so, we should discard them because we + * won't have a chance to play them. + * + * playback last decoded + * position EOS data time + * ----|---------------|------------|---------> (Increasing timeline) + * mCurrent looping mMaster's + * ClockTime offset mDecodedXXXEndTime + * + */ + const auto offset = + isAudio ? AudioQueue().GetOffset() : VideoQueue().GetOffset(); + const auto endTime = + isAudio ? mMaster->mDecodedAudioEndTime : mMaster->mDecodedVideoEndTime; + const auto clockTime = mMaster->GetClock(); + return (offset != media::TimeUnit::Zero() && clockTime < offset && + offset < endTime); + } + + void DiscardLoopedData(MediaData::Type aType) { + MOZ_DIAGNOSTIC_ASSERT(aType == MediaData::Type::AUDIO_DATA || + aType == MediaData::Type::VIDEO_DATA); + const bool isAudio = aType == MediaData::Type::AUDIO_DATA; + const auto offset = + isAudio ? AudioQueue().GetOffset() : VideoQueue().GetOffset(); + if (offset == media::TimeUnit::Zero()) { + return; + } + + SLOG("Discard %s frames after the time=%" PRId64, + isAudio ? "audio" : "video", offset.ToMicroseconds()); + if (isAudio) { + DiscardFramesFromTail(AudioQueue(), [&](int64_t aSampleTime) { + return aSampleTime > offset.ToMicroseconds(); + }); + } else { + DiscardFramesFromTail(VideoQueue(), [&](int64_t aSampleTime) { + return aSampleTime > offset.ToMicroseconds(); + }); + } + } + + void PushFakeAudioDataIfNeeded(const media::TimeUnit& aDuration) { + MOZ_ASSERT(Info().HasAudio()); + + const auto& audioInfo = Info().mAudio; + CheckedInt64 frames = aDuration.ToTicksAtRate(audioInfo.mRate); + if (!frames.isValid() || !audioInfo.mChannels || !audioInfo.mRate) { + NS_WARNING("Can't create fake audio, invalid frames/channel/rate?"); + return; + } + + if (!frames.value()) { + NS_WARNING(nsPrintfCString("Duration (%s) too short, no frame needed", + aDuration.ToString().get()) + .get()); + return; + } + + // If we can get the last sample, use its frame. Otherwise, use common 1024. + int64_t typicalPacketFrameCount = 1024; + if (RefPtr<AudioData> audio = AudioQueue().PeekBack()) { + typicalPacketFrameCount = audio->Frames(); + } + + media::TimeUnit totalDuration = TimeUnit::Zero(audioInfo.mRate); + // Generate fake audio in a smaller size of audio chunk. + while (frames.value()) { + int64_t packetFrameCount = + std::min(frames.value(), typicalPacketFrameCount); + frames -= packetFrameCount; + AlignedAudioBuffer samples(packetFrameCount * audioInfo.mChannels); + if (!samples) { + NS_WARNING("Can't create audio buffer, OOM?"); + return; + } + // `mDecodedAudioEndTime` is adjusted time, and we want unadjusted time + // otherwise the time would be adjusted twice when pushing sample into the + // media queue. + media::TimeUnit startTime = mMaster->mDecodedAudioEndTime; + if (AudioQueue().GetOffset() != media::TimeUnit::Zero()) { + startTime -= AudioQueue().GetOffset(); + } + RefPtr<AudioData> data(new AudioData(0, startTime, std::move(samples), + audioInfo.mChannels, + audioInfo.mRate)); + SLOG("Created fake audio data (duration=%s, frame-left=%" PRId64 ")", + data->mDuration.ToString().get(), frames.value()); + totalDuration += data->mDuration; + HandleAudioDecoded(data); + } + SLOG("Pushed fake silence audio data in total duration=%" PRId64 "%s", + totalDuration.ToMicroseconds(), totalDuration.ToString().get()); + } + + bool HasDecodedLastAudioFrame() const { + // when we're going to leave looping state and have got EOS before, we + // should mark audio queue as ended because we have got all data we need. + return mAudioDataRequest.Exists() || mAudioSeekRequest.Exists() || + ShouldDiscardLoopedData(MediaData::Type::AUDIO_DATA) || + IsDataWaitingForTimestampAdjustment(MediaData::Type::AUDIO_DATA) || + mIsReachingAudioEOS; + } + + bool HasDecodedLastVideoFrame() const { + // when we're going to leave looping state and have got EOS before, we + // should mark video queue as ended because we have got all data we need. + return mVideoDataRequest.Exists() || mVideoSeekRequest.Exists() || + ShouldDiscardLoopedData(MediaData::Type::VIDEO_DATA) || + IsDataWaitingForTimestampAdjustment(MediaData::Type::VIDEO_DATA) || + mIsReachingVideoEOS; + } + + bool ShouldStopPrerolling() const override { + // These checks is used to handle the media queue aren't opened correctly + // because they've been close before entering the looping state. Therefore, + // we need to preroll data in order to let new data to reopen the queue + // automatically. Otherwise, playback can't start successfully. + bool isWaitingForNewData = false; + if (mMaster->HasAudio()) { + isWaitingForNewData |= (mIsReachingAudioEOS && AudioQueue().IsFinished()); + } + if (mMaster->HasVideo()) { + isWaitingForNewData |= (mIsReachingVideoEOS && VideoQueue().IsFinished()); + } + return !isWaitingForNewData && DecodingState::ShouldStopPrerolling(); + } + + bool IsReaderSeeking() const { + return mAudioSeekRequest.Exists() || mVideoSeekRequest.Exists(); + } + + bool IsWaitingData(MediaData::Type aType) const override { + if (aType == MediaData::Type::AUDIO_DATA) { + return mMaster->IsWaitingAudioData() || + IsDataWaitingForTimestampAdjustment(MediaData::Type::AUDIO_DATA); + } + MOZ_DIAGNOSTIC_ASSERT(aType == MediaData::Type::VIDEO_DATA); + return mMaster->IsWaitingVideoData() || + IsDataWaitingForTimestampAdjustment(MediaData::Type::VIDEO_DATA); + } + + bool IsRequestingDataFromStartPosition(MediaData::Type aType) const { + MOZ_DIAGNOSTIC_ASSERT(aType == MediaData::Type::AUDIO_DATA || + aType == MediaData::Type::VIDEO_DATA); + if (aType == MediaData::Type::AUDIO_DATA) { + return mAudioSeekRequest.Exists() || mAudioDataRequest.Exists(); + } + return mVideoSeekRequest.Exists() || mVideoDataRequest.Exists(); + } + + bool IsBufferingAllowed() const override { + return !mIsReachingAudioEOS && !mIsReachingVideoEOS; + } + + bool mIsReachingAudioEOS; + bool mIsReachingVideoEOS; + + /** + * If we have both tracks which have different length, when one track ends + * first, we can't adjust new data from that track if another longer track + * hasn't ended yet. The adjusted timestamp needs to be based off the longer + * track's last data's timestamp, because otherwise it would cause a deviation + * and eventually a/v unsync. Those sample needs to be stored and we will + * adjust their timestamp later. + * + * Following graph explains the situation in details. + * o : decoded data with timestamp adjusted or no adjustment (not looping yet) + * x : decoded data without timestamp adjustment. + * - : stop decoding and nothing happens + * EOS : the track reaches to the end. We now know the offset of the track. + * + * Timeline -----------------------------------> + * Track1 : o EOS x - - o + * Track2 : o o o EOS o o + * + * Before reaching track2's EOS, we can't adjust samples from track1 because + * track2 might have longer duration than track1. The sample X would be + * stored in `mDataWaitingTimestampAdjustment` and we would also stop decoding + * for track1. + * + * After reaching track2's EOS, now we know another track's offset, and the + * larger one would be used for `mOriginalDecodedDuration`. Once that duration + * has been determined, we will no longer need to put samples on waiting + * because we already know how to adjust timestamp. + */ + RefPtr<MediaData> mDataWaitingTimestampAdjustment; + + MozPromiseRequestHolder<MediaFormatReader::SeekPromise> mAudioSeekRequest; + MozPromiseRequestHolder<MediaFormatReader::SeekPromise> mVideoSeekRequest; + MozPromiseRequestHolder<AudioDataPromise> mAudioDataRequest; + MozPromiseRequestHolder<VideoDataPromise> mVideoDataRequest; + + // The media format reader only allows seeking a track at a time, if we're + // already in seeking, then delay the new seek until the current one finishes. + Maybe<TrackInfo::TrackType> mPendingSeekingType; + + // These are used to track a special case where the playback starts from EOS + // position via seeking. So even if EOS has reached, none of data has been + // decoded yet. They will be reset when `mOriginalDecodedDuration` is + // determined. + bool mAudioEndedBeforeEnteringStateWithoutDuration; + bool mVideoEndedBeforeEnteringStateWithoutDuration; +}; + +/** + * Purpose: seek to a particular new playback position. + * + * Transition to: + * SEEKING if any new seek request. + * SHUTDOWN if seek failed. + * COMPLETED if the new playback position is the end of the media resource. + * NextFrameSeekingState if completing a NextFrameSeekingFromDormantState. + * DECODING/LOOPING_DECODING otherwise. + */ +class MediaDecoderStateMachine::SeekingState + : public MediaDecoderStateMachine::StateObject { + public: + explicit SeekingState(Master* aPtr) + : StateObject(aPtr), mVisibility(static_cast<EventVisibility>(0)) {} + + RefPtr<MediaDecoder::SeekPromise> Enter(SeekJob&& aSeekJob, + EventVisibility aVisibility) { + mSeekJob = std::move(aSeekJob); + mVisibility = aVisibility; + + // Suppressed visibility comes from two cases: (1) leaving dormant state, + // and (2) resuming suspended video decoder. We want both cases to be + // transparent to the user. So we only notify the change when the seek + // request is from the user. + if (mVisibility == EventVisibility::Observable) { + // Don't stop playback for a video-only seek since we want to keep playing + // audio and we don't need to stop playback while leaving dormant for the + // playback should has been stopped. + mMaster->StopPlayback(); + mMaster->UpdatePlaybackPositionInternal(mSeekJob.mTarget->GetTime()); + mMaster->mOnPlaybackEvent.Notify(MediaPlaybackEvent::SeekStarted); + mMaster->mOnNextFrameStatus.Notify( + MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING); + } + + RefPtr<MediaDecoder::SeekPromise> p = mSeekJob.mPromise.Ensure(__func__); + + DoSeek(); + + return p; + } + + virtual void Exit() override = 0; + + State GetState() const override = 0; + + void HandleAudioDecoded(AudioData* aAudio) override = 0; + void HandleVideoDecoded(VideoData* aVideo) override = 0; + void HandleAudioWaited(MediaData::Type aType) override = 0; + void HandleVideoWaited(MediaData::Type aType) override = 0; + + void HandleVideoSuspendTimeout() override { + // Do nothing since we want a valid video frame to show when seek is done. + } + + void HandleResumeVideoDecoding(const TimeUnit&) override { + // Do nothing. We will resume video decoding in the decoding state. + } + + // We specially handle next frame seeks by ignoring them if we're already + // seeking. + RefPtr<MediaDecoder::SeekPromise> HandleSeek( + const SeekTarget& aTarget) override { + if (aTarget.IsNextFrame()) { + // We ignore next frame seeks if we already have a seek pending + SLOG("Already SEEKING, ignoring seekToNextFrame"); + MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished"); + return MediaDecoder::SeekPromise::CreateAndReject( + /* aRejectValue = */ true, __func__); + } + + return StateObject::HandleSeek(aTarget); + } + + protected: + SeekJob mSeekJob; + EventVisibility mVisibility; + + virtual void DoSeek() = 0; + // Transition to the next state (defined by the subclass) when seek is + // completed. + virtual void GoToNextState() { SetDecodingState(); } + void SeekCompleted(); + virtual TimeUnit CalculateNewCurrentTime() const = 0; +}; + +class MediaDecoderStateMachine::AccurateSeekingState + : public MediaDecoderStateMachine::SeekingState { + public: + explicit AccurateSeekingState(Master* aPtr) : SeekingState(aPtr) {} + + State GetState() const override { return DECODER_STATE_SEEKING_ACCURATE; } + + RefPtr<MediaDecoder::SeekPromise> Enter(SeekJob&& aSeekJob, + EventVisibility aVisibility) { + MOZ_ASSERT(aSeekJob.mTarget->IsAccurate() || aSeekJob.mTarget->IsFast()); + mCurrentTimeBeforeSeek = mMaster->GetMediaTime(); + return SeekingState::Enter(std::move(aSeekJob), aVisibility); + } + + void Exit() override { + // Disconnect MediaDecoder. + mSeekJob.RejectIfExists(__func__); + + // Disconnect ReaderProxy. + mSeekRequest.DisconnectIfExists(); + + mWaitRequest.DisconnectIfExists(); + } + + void HandleAudioDecoded(AudioData* aAudio) override { + MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking, + "Seek shouldn't be finished"); + MOZ_ASSERT(aAudio); + + AdjustFastSeekIfNeeded(aAudio); + + if (mSeekJob.mTarget->IsFast()) { + // Non-precise seek; we can stop the seek at the first sample. + mMaster->PushAudio(aAudio); + mDoneAudioSeeking = true; + } else { + nsresult rv = DropAudioUpToSeekTarget(aAudio); + if (NS_FAILED(rv)) { + mMaster->DecodeError(rv); + return; + } + } + + if (!mDoneAudioSeeking) { + RequestAudioData(); + return; + } + MaybeFinishSeek(); + } + + void HandleVideoDecoded(VideoData* aVideo) override { + MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking, + "Seek shouldn't be finished"); + MOZ_ASSERT(aVideo); + + AdjustFastSeekIfNeeded(aVideo); + + if (mSeekJob.mTarget->IsFast()) { + // Non-precise seek. We can stop the seek at the first sample. + mMaster->PushVideo(aVideo); + mDoneVideoSeeking = true; + } else { + nsresult rv = DropVideoUpToSeekTarget(aVideo); + if (NS_FAILED(rv)) { + mMaster->DecodeError(rv); + return; + } + } + + if (!mDoneVideoSeeking) { + RequestVideoData(); + return; + } + MaybeFinishSeek(); + } + + void HandleWaitingForAudio() override { + MOZ_ASSERT(!mDoneAudioSeeking); + mMaster->WaitForData(MediaData::Type::AUDIO_DATA); + } + + void HandleAudioCanceled() override { + MOZ_ASSERT(!mDoneAudioSeeking); + RequestAudioData(); + } + + void HandleEndOfAudio() override { + HandleEndOfAudioInternal(); + MaybeFinishSeek(); + } + + void HandleWaitingForVideo() override { + MOZ_ASSERT(!mDoneVideoSeeking); + mMaster->WaitForData(MediaData::Type::VIDEO_DATA); + } + + void HandleVideoCanceled() override { + MOZ_ASSERT(!mDoneVideoSeeking); + RequestVideoData(); + } + + void HandleEndOfVideo() override { + HandleEndOfVideoInternal(); + MaybeFinishSeek(); + } + + void HandleAudioWaited(MediaData::Type aType) override { + MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking, + "Seek shouldn't be finished"); + + RequestAudioData(); + } + + void HandleVideoWaited(MediaData::Type aType) override { + MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking, + "Seek shouldn't be finished"); + + RequestVideoData(); + } + + void DoSeek() override { + mDoneAudioSeeking = !Info().HasAudio(); + mDoneVideoSeeking = !Info().HasVideo(); + + // Resetting decode should be called after stopping media sink, which can + // ensure that we have an empty media queue before seeking the demuxer. + mMaster->StopMediaSink(); + mMaster->ResetDecode(); + + DemuxerSeek(); + } + + TimeUnit CalculateNewCurrentTime() const override { + const auto seekTime = mSeekJob.mTarget->GetTime(); + + // For the accurate seek, we always set the newCurrentTime = seekTime so + // that the updated HTMLMediaElement.currentTime will always be the seek + // target; we rely on the MediaSink to handles the gap between the + // newCurrentTime and the real decoded samples' start time. + if (mSeekJob.mTarget->IsAccurate()) { + return seekTime; + } + + // For the fast seek, we update the newCurrentTime with the decoded audio + // and video samples, set it to be the one which is closet to the seekTime. + if (mSeekJob.mTarget->IsFast()) { + RefPtr<AudioData> audio = AudioQueue().PeekFront(); + RefPtr<VideoData> video = VideoQueue().PeekFront(); + + // A situation that both audio and video approaches the end. + if (!audio && !video) { + return seekTime; + } + + const int64_t audioStart = + audio ? audio->mTime.ToMicroseconds() : INT64_MAX; + const int64_t videoStart = + video ? video->mTime.ToMicroseconds() : INT64_MAX; + const int64_t audioGap = std::abs(audioStart - seekTime.ToMicroseconds()); + const int64_t videoGap = std::abs(videoStart - seekTime.ToMicroseconds()); + return TimeUnit::FromMicroseconds(audioGap <= videoGap ? audioStart + : videoStart); + } + + MOZ_ASSERT(false, "AccurateSeekTask doesn't handle other seek types."); + return TimeUnit::Zero(); + } + + protected: + void DemuxerSeek() { + // Request the demuxer to perform seek. + Reader() + ->Seek(mSeekJob.mTarget.ref()) + ->Then( + OwnerThread(), __func__, + [this](const media::TimeUnit& aUnit) { OnSeekResolved(aUnit); }, + [this](const SeekRejectValue& aReject) { OnSeekRejected(aReject); }) + ->Track(mSeekRequest); + } + + void OnSeekResolved(media::TimeUnit) { + AUTO_PROFILER_LABEL("AccurateSeekingState::OnSeekResolved", MEDIA_PLAYBACK); + mSeekRequest.Complete(); + + // We must decode the first samples of active streams, so we can determine + // the new stream time. So dispatch tasks to do that. + if (!mDoneVideoSeeking) { + RequestVideoData(); + } + if (!mDoneAudioSeeking) { + RequestAudioData(); + } + } + + void OnSeekRejected(const SeekRejectValue& aReject) { + AUTO_PROFILER_LABEL("AccurateSeekingState::OnSeekRejected", MEDIA_PLAYBACK); + mSeekRequest.Complete(); + + if (aReject.mError == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) { + SLOG("OnSeekRejected reason=WAITING_FOR_DATA type=%s", + MediaData::TypeToStr(aReject.mType)); + MOZ_ASSERT_IF(aReject.mType == MediaData::Type::AUDIO_DATA, + !mMaster->IsRequestingAudioData()); + MOZ_ASSERT_IF(aReject.mType == MediaData::Type::VIDEO_DATA, + !mMaster->IsRequestingVideoData()); + MOZ_ASSERT_IF(aReject.mType == MediaData::Type::AUDIO_DATA, + !mMaster->IsWaitingAudioData()); + MOZ_ASSERT_IF(aReject.mType == MediaData::Type::VIDEO_DATA, + !mMaster->IsWaitingVideoData()); + + // Fire 'waiting' to notify the player that we are waiting for data. + mMaster->mOnNextFrameStatus.Notify( + MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING); + + Reader() + ->WaitForData(aReject.mType) + ->Then( + OwnerThread(), __func__, + [this](MediaData::Type aType) { + AUTO_PROFILER_LABEL( + "AccurateSeekingState::OnSeekRejected:WaitDataResolved", + MEDIA_PLAYBACK); + SLOG("OnSeekRejected wait promise resolved"); + mWaitRequest.Complete(); + DemuxerSeek(); + }, + [this](const WaitForDataRejectValue& aRejection) { + AUTO_PROFILER_LABEL( + "AccurateSeekingState::OnSeekRejected:WaitDataRejected", + MEDIA_PLAYBACK); + SLOG("OnSeekRejected wait promise rejected"); + mWaitRequest.Complete(); + mMaster->DecodeError(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA); + }) + ->Track(mWaitRequest); + return; + } + + if (aReject.mError == NS_ERROR_DOM_MEDIA_END_OF_STREAM) { + if (!mDoneAudioSeeking) { + HandleEndOfAudioInternal(); + } + if (!mDoneVideoSeeking) { + HandleEndOfVideoInternal(); + } + MaybeFinishSeek(); + return; + } + + MOZ_ASSERT(NS_FAILED(aReject.mError), + "Cancels should also disconnect mSeekRequest"); + mMaster->DecodeError(aReject.mError); + } + + void RequestAudioData() { + MOZ_ASSERT(!mDoneAudioSeeking); + mMaster->RequestAudioData(); + } + + virtual void RequestVideoData() { + MOZ_ASSERT(!mDoneVideoSeeking); + mMaster->RequestVideoData(media::TimeUnit()); + } + + void AdjustFastSeekIfNeeded(MediaData* aSample) { + if (mSeekJob.mTarget->IsFast() && + mSeekJob.mTarget->GetTime() > mCurrentTimeBeforeSeek && + aSample->mTime < mCurrentTimeBeforeSeek) { + // We are doing a fastSeek, but we ended up *before* the previous + // playback position. This is surprising UX, so switch to an accurate + // seek and decode to the seek target. This is not conformant to the + // spec, fastSeek should always be fast, but until we get the time to + // change all Readers to seek to the keyframe after the currentTime + // in this case, we'll just decode forward. Bug 1026330. + mSeekJob.mTarget->SetType(SeekTarget::Accurate); + } + } + + nsresult DropAudioUpToSeekTarget(AudioData* aAudio) { + MOZ_ASSERT(aAudio && mSeekJob.mTarget->IsAccurate()); + + if (mSeekJob.mTarget->GetTime() >= aAudio->GetEndTime()) { + // Our seek target lies after the frames in this AudioData. Don't + // push it onto the audio queue, and keep decoding forwards. + return NS_OK; + } + + if (aAudio->mTime > mSeekJob.mTarget->GetTime()) { + // The seek target doesn't lie in the audio block just after the last + // audio frames we've seen which were before the seek target. This + // could have been the first audio data we've seen after seek, i.e. the + // seek terminated after the seek target in the audio stream. Just + // abort the audio decode-to-target, the state machine will play + // silence to cover the gap. Typically this happens in poorly muxed + // files. + SLOGW("Audio not synced after seek, maybe a poorly muxed file?"); + mMaster->PushAudio(aAudio); + mDoneAudioSeeking = true; + return NS_OK; + } + + bool ok = aAudio->SetTrimWindow( + {mSeekJob.mTarget->GetTime().ToBase(aAudio->mTime), + aAudio->GetEndTime()}); + if (!ok) { + return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR; + } + + MOZ_ASSERT(AudioQueue().GetSize() == 0, + "Should be the 1st sample after seeking"); + mMaster->PushAudio(aAudio); + mDoneAudioSeeking = true; + + return NS_OK; + } + + nsresult DropVideoUpToSeekTarget(VideoData* aVideo) { + MOZ_ASSERT(aVideo); + SLOG("DropVideoUpToSeekTarget() frame [%" PRId64 ", %" PRId64 "]", + aVideo->mTime.ToMicroseconds(), aVideo->GetEndTime().ToMicroseconds()); + const auto target = GetSeekTarget(); + + // If the frame end time is less than the seek target, we won't want + // to display this frame after the seek, so discard it. + if (target >= aVideo->GetEndTime()) { + SLOG("DropVideoUpToSeekTarget() pop video frame [%" PRId64 ", %" PRId64 + "] target=%" PRId64, + aVideo->mTime.ToMicroseconds(), + aVideo->GetEndTime().ToMicroseconds(), target.ToMicroseconds()); + PROFILER_MARKER_UNTYPED("MDSM::DropVideoUpToSeekTarget", MEDIA_PLAYBACK); + mFirstVideoFrameAfterSeek = aVideo; + } else { + if (target >= aVideo->mTime && aVideo->GetEndTime() >= target) { + // The seek target lies inside this frame's time slice. Adjust the + // frame's start time to match the seek target. + aVideo->UpdateTimestamp(target); + } + mFirstVideoFrameAfterSeek = nullptr; + + SLOG("DropVideoUpToSeekTarget() found video frame [%" PRId64 ", %" PRId64 + "] containing target=%" PRId64, + aVideo->mTime.ToMicroseconds(), + aVideo->GetEndTime().ToMicroseconds(), target.ToMicroseconds()); + + MOZ_ASSERT(VideoQueue().GetSize() == 0, + "Should be the 1st sample after seeking"); + mMaster->PushVideo(aVideo); + mDoneVideoSeeking = true; + } + + return NS_OK; + } + + void HandleEndOfAudioInternal() { + MOZ_ASSERT(!mDoneAudioSeeking); + AudioQueue().Finish(); + mDoneAudioSeeking = true; + } + + void HandleEndOfVideoInternal() { + MOZ_ASSERT(!mDoneVideoSeeking); + if (mFirstVideoFrameAfterSeek) { + // Hit the end of stream. Move mFirstVideoFrameAfterSeek into + // mSeekedVideoData so we have something to display after seeking. + mMaster->PushVideo(mFirstVideoFrameAfterSeek); + } + VideoQueue().Finish(); + mDoneVideoSeeking = true; + } + + void MaybeFinishSeek() { + if (mDoneAudioSeeking && mDoneVideoSeeking) { + SeekCompleted(); + } + } + + /* + * Track the current seek promise made by the reader. + */ + MozPromiseRequestHolder<MediaFormatReader::SeekPromise> mSeekRequest; + + /* + * Internal state. + */ + media::TimeUnit mCurrentTimeBeforeSeek; + bool mDoneAudioSeeking = false; + bool mDoneVideoSeeking = false; + MozPromiseRequestHolder<WaitForDataPromise> mWaitRequest; + + // This temporarily stores the first frame we decode after we seek. + // This is so that if we hit end of stream while we're decoding to reach + // the seek target, we will still have a frame that we can display as the + // last frame in the media. + RefPtr<VideoData> mFirstVideoFrameAfterSeek; + + private: + virtual media::TimeUnit GetSeekTarget() const { + return mSeekJob.mTarget->GetTime(); + } +}; + +/* + * Remove samples from the queue until aCompare() returns false. + * aCompare A function object with the signature bool(int64_t) which returns + * true for samples that should be removed. + */ +template <typename Type, typename Function> +static void DiscardFrames(MediaQueue<Type>& aQueue, const Function& aCompare) { + while (aQueue.GetSize() > 0) { + if (aCompare(aQueue.PeekFront()->mTime.ToMicroseconds())) { + RefPtr<Type> releaseMe = aQueue.PopFront(); + continue; + } + break; + } +} + +class MediaDecoderStateMachine::NextFrameSeekingState + : public MediaDecoderStateMachine::SeekingState { + public: + explicit NextFrameSeekingState(Master* aPtr) : SeekingState(aPtr) {} + + State GetState() const override { + return DECODER_STATE_SEEKING_NEXTFRAMESEEKING; + } + + RefPtr<MediaDecoder::SeekPromise> Enter(SeekJob&& aSeekJob, + EventVisibility aVisibility) { + MOZ_ASSERT(aSeekJob.mTarget->IsNextFrame()); + mCurrentTime = mMaster->GetMediaTime(); + mDuration = mMaster->Duration(); + return SeekingState::Enter(std::move(aSeekJob), aVisibility); + } + + void Exit() override { + // Disconnect my async seek operation. + if (mAsyncSeekTask) { + mAsyncSeekTask->Cancel(); + } + + // Disconnect MediaDecoder. + mSeekJob.RejectIfExists(__func__); + } + + void HandleAudioDecoded(AudioData* aAudio) override { + mMaster->PushAudio(aAudio); + } + + void HandleVideoDecoded(VideoData* aVideo) override { + MOZ_ASSERT(aVideo); + MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished"); + MOZ_ASSERT(NeedMoreVideo()); + + if (aVideo->mTime > mCurrentTime) { + mMaster->PushVideo(aVideo); + FinishSeek(); + } else { + RequestVideoData(); + } + } + + void HandleWaitingForAudio() override { + MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished"); + // We don't care about audio decode errors in this state which will be + // handled by other states after seeking. + } + + void HandleAudioCanceled() override { + MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished"); + // We don't care about audio decode errors in this state which will be + // handled by other states after seeking. + } + + void HandleEndOfAudio() override { + MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished"); + // We don't care about audio decode errors in this state which will be + // handled by other states after seeking. + } + + void HandleWaitingForVideo() override { + MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished"); + MOZ_ASSERT(NeedMoreVideo()); + mMaster->WaitForData(MediaData::Type::VIDEO_DATA); + } + + void HandleVideoCanceled() override { + MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished"); + MOZ_ASSERT(NeedMoreVideo()); + RequestVideoData(); + } + + void HandleEndOfVideo() override { + MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished"); + MOZ_ASSERT(NeedMoreVideo()); + VideoQueue().Finish(); + FinishSeek(); + } + + void HandleAudioWaited(MediaData::Type aType) override { + // We don't care about audio in this state. + } + + void HandleVideoWaited(MediaData::Type aType) override { + MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished"); + MOZ_ASSERT(NeedMoreVideo()); + RequestVideoData(); + } + + TimeUnit CalculateNewCurrentTime() const override { + // The HTMLMediaElement.currentTime should be updated to the seek target + // which has been updated to the next frame's time. + return mSeekJob.mTarget->GetTime(); + } + + void DoSeek() override { + mMaster->StopMediaSink(); + + auto currentTime = mCurrentTime; + DiscardFrames(VideoQueue(), [currentTime](int64_t aSampleTime) { + return aSampleTime <= currentTime.ToMicroseconds(); + }); + + // If there is a pending video request, finish the seeking if we don't need + // more data, or wait for HandleVideoDecoded() to finish seeking. + if (mMaster->IsRequestingVideoData()) { + if (!NeedMoreVideo()) { + FinishSeek(); + } + return; + } + + // Otherwise, we need to do the seek operation asynchronously for a special + // case (bug504613.ogv) which has no data at all, the 1st seekToNextFrame() + // operation reaches the end of the media. If we did the seek operation + // synchronously, we immediately resolve the SeekPromise in mSeekJob and + // then switch to the CompletedState which dispatches an "ended" event. + // However, the ThenValue of the SeekPromise has not yet been set, so the + // promise resolving is postponed and then the JS developer receives the + // "ended" event before the seek promise is resolved. + // An asynchronous seek operation helps to solve this issue since while the + // seek is actually performed, the ThenValue of SeekPromise has already + // been set so that it won't be postponed. + RefPtr<Runnable> r = mAsyncSeekTask = new AysncNextFrameSeekTask(this); + nsresult rv = OwnerThread()->Dispatch(r.forget()); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; + } + + private: + void DoSeekInternal() { + // We don't need to discard frames to the mCurrentTime here because we have + // done it at DoSeek() and any video data received in between either + // finishes the seek operation or be discarded, see HandleVideoDecoded(). + + if (!NeedMoreVideo()) { + FinishSeek(); + } else if (!mMaster->IsRequestingVideoData() && + !mMaster->IsWaitingVideoData()) { + RequestVideoData(); + } + } + + class AysncNextFrameSeekTask : public Runnable { + public: + explicit AysncNextFrameSeekTask(NextFrameSeekingState* aStateObject) + : Runnable( + "MediaDecoderStateMachine::NextFrameSeekingState::" + "AysncNextFrameSeekTask"), + mStateObj(aStateObject) {} + + void Cancel() { mStateObj = nullptr; } + + NS_IMETHOD Run() override { + if (mStateObj) { + AUTO_PROFILER_LABEL("AysncNextFrameSeekTask::Run", MEDIA_PLAYBACK); + mStateObj->DoSeekInternal(); + } + return NS_OK; + } + + private: + NextFrameSeekingState* mStateObj; + }; + + void RequestVideoData() { mMaster->RequestVideoData(media::TimeUnit()); } + + bool NeedMoreVideo() const { + // Need to request video when we have none and video queue is not finished. + return VideoQueue().GetSize() == 0 && !VideoQueue().IsFinished(); + } + + // Update the seek target's time before resolving this seek task, the updated + // time will be used in the MDSM::SeekCompleted() to update the MDSM's + // position. + void UpdateSeekTargetTime() { + RefPtr<VideoData> data = VideoQueue().PeekFront(); + if (data) { + mSeekJob.mTarget->SetTime(data->mTime); + } else { + MOZ_ASSERT(VideoQueue().AtEndOfStream()); + mSeekJob.mTarget->SetTime(mDuration); + } + } + + void FinishSeek() { + MOZ_ASSERT(!NeedMoreVideo()); + UpdateSeekTargetTime(); + auto time = mSeekJob.mTarget->GetTime().ToMicroseconds(); + DiscardFrames(AudioQueue(), + [time](int64_t aSampleTime) { return aSampleTime < time; }); + SeekCompleted(); + } + + /* + * Internal state. + */ + TimeUnit mCurrentTime; + TimeUnit mDuration; + RefPtr<AysncNextFrameSeekTask> mAsyncSeekTask; +}; + +class MediaDecoderStateMachine::NextFrameSeekingFromDormantState + : public MediaDecoderStateMachine::AccurateSeekingState { + public: + explicit NextFrameSeekingFromDormantState(Master* aPtr) + : AccurateSeekingState(aPtr) {} + + State GetState() const override { return DECODER_STATE_SEEKING_FROMDORMANT; } + + RefPtr<MediaDecoder::SeekPromise> Enter(SeekJob&& aCurrentSeekJob, + SeekJob&& aFutureSeekJob) { + mFutureSeekJob = std::move(aFutureSeekJob); + + AccurateSeekingState::Enter(std::move(aCurrentSeekJob), + EventVisibility::Suppressed); + + // Once seekToNextFrame() is called, we assume the user is likely to keep + // calling seekToNextFrame() repeatedly, and so, we should prevent the MDSM + // from getting into Dormant state. + mMaster->mMinimizePreroll = false; + + return mFutureSeekJob.mPromise.Ensure(__func__); + } + + void Exit() override { + mFutureSeekJob.RejectIfExists(__func__); + AccurateSeekingState::Exit(); + } + + private: + SeekJob mFutureSeekJob; + + // We don't want to transition to DecodingState once this seek completes, + // instead, we transition to NextFrameSeekingState. + void GoToNextState() override { + SetState<NextFrameSeekingState>(std::move(mFutureSeekJob), + EventVisibility::Observable); + } +}; + +class MediaDecoderStateMachine::VideoOnlySeekingState + : public MediaDecoderStateMachine::AccurateSeekingState { + public: + explicit VideoOnlySeekingState(Master* aPtr) : AccurateSeekingState(aPtr) {} + + State GetState() const override { return DECODER_STATE_SEEKING_VIDEOONLY; } + + RefPtr<MediaDecoder::SeekPromise> Enter(SeekJob&& aSeekJob, + EventVisibility aVisibility) { + MOZ_ASSERT(aSeekJob.mTarget->IsVideoOnly()); + MOZ_ASSERT(aVisibility == EventVisibility::Suppressed); + + RefPtr<MediaDecoder::SeekPromise> p = + AccurateSeekingState::Enter(std::move(aSeekJob), aVisibility); + + // Dispatch a mozvideoonlyseekbegin event to indicate UI for corresponding + // changes. + mMaster->mOnPlaybackEvent.Notify(MediaPlaybackEvent::VideoOnlySeekBegin); + + return p; + } + + void Exit() override { + // We are completing or discarding this video-only seek operation now, + // dispatch an event so that the UI can change in response to the end + // of video-only seek. + mMaster->mOnPlaybackEvent.Notify( + MediaPlaybackEvent::VideoOnlySeekCompleted); + + AccurateSeekingState::Exit(); + } + + void HandleAudioDecoded(AudioData* aAudio) override { + MOZ_ASSERT(mDoneAudioSeeking && !mDoneVideoSeeking, + "Seek shouldn't be finished"); + MOZ_ASSERT(aAudio); + + // Video-only seek doesn't reset audio decoder. There might be pending audio + // requests when AccurateSeekTask::Seek() begins. We will just store the + // data without checking |mDiscontinuity| or calling + // DropAudioUpToSeekTarget(). + mMaster->PushAudio(aAudio); + } + + void HandleWaitingForAudio() override {} + + void HandleAudioCanceled() override {} + + void HandleEndOfAudio() override {} + + void HandleAudioWaited(MediaData::Type aType) override { + MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking, + "Seek shouldn't be finished"); + + // Ignore pending requests from video-only seek. + } + + void DoSeek() override { + // TODO: keep decoding audio. + mDoneAudioSeeking = true; + mDoneVideoSeeking = !Info().HasVideo(); + + const auto offset = VideoQueue().GetOffset(); + mMaster->ResetDecode(TrackInfo::kVideoTrack); + + // Entering video-only state and we've looped at least once before, so we + // need to set offset in order to let new video frames catch up with the + // clock time. + if (offset != media::TimeUnit::Zero()) { + VideoQueue().SetOffset(offset); + } + + DemuxerSeek(); + } + + protected: + // Allow skip-to-next-key-frame to kick in if we fall behind the current + // playback position so decoding has a better chance to catch up. + void RequestVideoData() override { + MOZ_ASSERT(!mDoneVideoSeeking); + + auto clock = mMaster->mMediaSink->IsStarted() ? mMaster->GetClock() + : mMaster->GetMediaTime(); + mMaster->AdjustByLooping(clock); + const auto& nextKeyFrameTime = GetNextKeyFrameTime(); + + auto threshold = clock; + + if (nextKeyFrameTime.IsValid() && + clock >= (nextKeyFrameTime - sSkipToNextKeyFrameThreshold)) { + threshold = nextKeyFrameTime; + } + + mMaster->RequestVideoData(threshold); + } + + private: + // Trigger skip to next key frame if the current playback position is very + // close the next key frame's time. + static constexpr TimeUnit sSkipToNextKeyFrameThreshold = + TimeUnit::FromMicroseconds(5000); + + // If the media is playing, drop video until catch up playback position. + media::TimeUnit GetSeekTarget() const override { + auto target = mMaster->mMediaSink->IsStarted() + ? mMaster->GetClock() + : mSeekJob.mTarget->GetTime(); + mMaster->AdjustByLooping(target); + return target; + } + + media::TimeUnit GetNextKeyFrameTime() const { + // We only call this method in RequestVideoData() and we only request video + // data if we haven't done video seeking. + MOZ_DIAGNOSTIC_ASSERT(!mDoneVideoSeeking); + MOZ_DIAGNOSTIC_ASSERT(mMaster->VideoQueue().GetSize() == 0); + + if (mFirstVideoFrameAfterSeek) { + return mFirstVideoFrameAfterSeek->NextKeyFrameTime(); + } + + return TimeUnit::Invalid(); + } +}; + +constexpr TimeUnit MediaDecoderStateMachine::VideoOnlySeekingState:: + sSkipToNextKeyFrameThreshold; + +RefPtr<MediaDecoder::SeekPromise> +MediaDecoderStateMachine::DormantState::HandleSeek(const SeekTarget& aTarget) { + if (aTarget.IsNextFrame()) { + // NextFrameSeekingState doesn't reset the decoder unlike + // AccurateSeekingState. So we first must come out of dormant by seeking to + // mPendingSeek and continue later with the NextFrameSeek + SLOG("Changed state to SEEKING (to %" PRId64 ")", + aTarget.GetTime().ToMicroseconds()); + SeekJob seekJob; + seekJob.mTarget = Some(aTarget); + return StateObject::SetState<NextFrameSeekingFromDormantState>( + std::move(mPendingSeek), std::move(seekJob)); + } + + return StateObject::HandleSeek(aTarget); +} + +/** + * Purpose: stop playback until enough data is decoded to continue playback. + * + * Transition to: + * SEEKING if any seek request. + * SHUTDOWN if any decode error. + * COMPLETED when having decoded all audio/video data. + * DECODING/LOOPING_DECODING when having decoded enough data to continue + * playback. + */ +class MediaDecoderStateMachine::BufferingState + : public MediaDecoderStateMachine::StateObject { + public: + explicit BufferingState(Master* aPtr) : StateObject(aPtr) {} + + void Enter() { + if (mMaster->IsPlaying()) { + mMaster->StopPlayback(); + } + + mBufferingStart = TimeStamp::Now(); + mMaster->ScheduleStateMachineIn(TimeUnit::FromMicroseconds(USECS_PER_S)); + mMaster->mOnNextFrameStatus.Notify( + MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_BUFFERING); + } + + void Step() override; + + State GetState() const override { return DECODER_STATE_BUFFERING; } + + void HandleAudioDecoded(AudioData* aAudio) override { + mMaster->PushAudio(aAudio); + if (!mMaster->HaveEnoughDecodedAudio()) { + mMaster->RequestAudioData(); + } + // This might be the sample we need to exit buffering. + // Schedule Step() to check it. + mMaster->ScheduleStateMachine(); + } + + void HandleVideoDecoded(VideoData* aVideo) override { + mMaster->PushVideo(aVideo); + if (!mMaster->HaveEnoughDecodedVideo()) { + mMaster->RequestVideoData(media::TimeUnit()); + } + // This might be the sample we need to exit buffering. + // Schedule Step() to check it. + mMaster->ScheduleStateMachine(); + } + + void HandleAudioCanceled() override { mMaster->RequestAudioData(); } + + void HandleVideoCanceled() override { + mMaster->RequestVideoData(media::TimeUnit()); + } + + void HandleWaitingForAudio() override { + mMaster->WaitForData(MediaData::Type::AUDIO_DATA); + } + + void HandleWaitingForVideo() override { + mMaster->WaitForData(MediaData::Type::VIDEO_DATA); + } + + void HandleAudioWaited(MediaData::Type aType) override { + mMaster->RequestAudioData(); + } + + void HandleVideoWaited(MediaData::Type aType) override { + mMaster->RequestVideoData(media::TimeUnit()); + } + + void HandleEndOfAudio() override; + void HandleEndOfVideo() override; + + void HandleVideoSuspendTimeout() override { + // No video, so nothing to suspend. + if (!mMaster->HasVideo()) { + return; + } + + mMaster->mVideoDecodeSuspended = true; + mMaster->mOnPlaybackEvent.Notify(MediaPlaybackEvent::EnterVideoSuspend); + Reader()->SetVideoBlankDecode(true); + } + + private: + TimeStamp mBufferingStart; + + // The maximum number of second we spend buffering when we are short on + // unbuffered data. + const uint32_t mBufferingWait = 15; +}; + +/** + * Purpose: play all the decoded data and fire the 'ended' event. + * + * Transition to: + * SEEKING if any seek request. + * LOOPING_DECODING if MDSM enable looping. + */ +class MediaDecoderStateMachine::CompletedState + : public MediaDecoderStateMachine::StateObject { + public: + explicit CompletedState(Master* aPtr) : StateObject(aPtr) {} + + void Enter() { + // On Android, the life cycle of graphic buffer is equal to Android's codec, + // we couldn't release it if we still need to render the frame. +#ifndef MOZ_WIDGET_ANDROID + if (!mMaster->mLooping) { + // We've decoded all samples. + // We don't need decoders anymore if not looping. + Reader()->ReleaseResources(); + } +#endif + bool hasNextFrame = (!mMaster->HasAudio() || !mMaster->mAudioCompleted) && + (!mMaster->HasVideo() || !mMaster->mVideoCompleted); + + mMaster->mOnNextFrameStatus.Notify( + hasNextFrame ? MediaDecoderOwner::NEXT_FRAME_AVAILABLE + : MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE); + + Step(); + } + + void Exit() override { mSentPlaybackEndedEvent = false; } + + void Step() override { + if (mMaster->mPlayState != MediaDecoder::PLAY_STATE_PLAYING && + mMaster->IsPlaying()) { + mMaster->StopPlayback(); + } + + // Play the remaining media. We want to run AdvanceFrame() at least + // once to ensure the current playback position is advanced to the + // end of the media, and so that we update the readyState. + if ((mMaster->HasVideo() && !mMaster->mVideoCompleted) || + (mMaster->HasAudio() && !mMaster->mAudioCompleted)) { + // Start playback if necessary to play the remaining media. + mMaster->MaybeStartPlayback(); + mMaster->UpdatePlaybackPositionPeriodically(); + MOZ_ASSERT(!mMaster->IsPlaying() || mMaster->IsStateMachineScheduled(), + "Must have timer scheduled"); + return; + } + + // StopPlayback in order to reset the IsPlaying() state so audio + // is restarted correctly. + mMaster->StopPlayback(); + + if (!mSentPlaybackEndedEvent) { + auto clockTime = + std::max(mMaster->AudioEndTime(), mMaster->VideoEndTime()); + // Correct the time over the end once looping was turned on. + mMaster->AdjustByLooping(clockTime); + if (mMaster->mDuration.Ref()->IsInfinite()) { + // We have a finite duration when playback reaches the end. + mMaster->mDuration = Some(clockTime); + DDLOGEX(mMaster, DDLogCategory::Property, "duration_us", + mMaster->mDuration.Ref()->ToMicroseconds()); + } + mMaster->UpdatePlaybackPosition(clockTime); + + // Ensure readyState is updated before firing the 'ended' event. + mMaster->mOnNextFrameStatus.Notify( + MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE); + + mMaster->mOnPlaybackEvent.Notify(MediaPlaybackEvent::PlaybackEnded); + + mSentPlaybackEndedEvent = true; + + // MediaSink::GetEndTime() must be called before stopping playback. + mMaster->StopMediaSink(); + } + } + + State GetState() const override { return DECODER_STATE_COMPLETED; } + + void HandleLoopingChanged() override { + if (mMaster->mLooping) { + SetDecodingState(); + } + } + + void HandleAudioCaptured() override { + // MediaSink is changed. Schedule Step() to check if we can start playback. + mMaster->ScheduleStateMachine(); + } + + void HandleVideoSuspendTimeout() override { + // Do nothing since no decoding is going on. + } + + void HandleResumeVideoDecoding(const TimeUnit&) override { + // Resume the video decoder and seek to the last video frame. + // This triggers a video-only seek which won't update the playback position. + auto target = mMaster->mDecodedVideoEndTime; + mMaster->AdjustByLooping(target); + StateObject::HandleResumeVideoDecoding(target); + } + + void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) override { + if (aPlayState == MediaDecoder::PLAY_STATE_PLAYING) { + // Schedule Step() to check if we can start playback. + mMaster->ScheduleStateMachine(); + } + } + + private: + bool mSentPlaybackEndedEvent = false; +}; + +/** + * Purpose: release all resources allocated by MDSM. + * + * Transition to: + * None since this is the final state. + * + * Transition from: + * Any states other than SHUTDOWN. + */ +class MediaDecoderStateMachine::ShutdownState + : public MediaDecoderStateMachine::StateObject { + public: + explicit ShutdownState(Master* aPtr) : StateObject(aPtr) {} + + RefPtr<ShutdownPromise> Enter(); + + void Exit() override { + MOZ_DIAGNOSTIC_ASSERT(false, "Shouldn't escape the SHUTDOWN state."); + } + + State GetState() const override { return DECODER_STATE_SHUTDOWN; } + + RefPtr<MediaDecoder::SeekPromise> HandleSeek( + const SeekTarget& aTarget) override { + MOZ_DIAGNOSTIC_ASSERT(false, "Can't seek in shutdown state."); + return MediaDecoder::SeekPromise::CreateAndReject(true, __func__); + } + + RefPtr<ShutdownPromise> HandleShutdown() override { + MOZ_DIAGNOSTIC_ASSERT(false, "Already shutting down."); + return nullptr; + } + + void HandleVideoSuspendTimeout() override { + MOZ_DIAGNOSTIC_ASSERT(false, "Already shutting down."); + } + + void HandleResumeVideoDecoding(const TimeUnit&) override { + MOZ_DIAGNOSTIC_ASSERT(false, "Already shutting down."); + } +}; + +RefPtr<MediaDecoder::SeekPromise> +MediaDecoderStateMachine::StateObject::HandleSeek(const SeekTarget& aTarget) { + SLOG("Changed state to SEEKING (to %" PRId64 ")", + aTarget.GetTime().ToMicroseconds()); + SeekJob seekJob; + seekJob.mTarget = Some(aTarget); + return SetSeekingState(std::move(seekJob), EventVisibility::Observable); +} + +RefPtr<ShutdownPromise> +MediaDecoderStateMachine::StateObject::HandleShutdown() { + return SetState<ShutdownState>(); +} + +static void ReportRecoveryTelemetry(const TimeStamp& aRecoveryStart, + const MediaInfo& aMediaInfo, + bool aIsHardwareAccelerated) { + MOZ_ASSERT(NS_IsMainThread()); + if (!aMediaInfo.HasVideo()) { + return; + } + + // Keyed by audio+video or video alone, hardware acceleration, + // and by a resolution range. + nsCString key(aMediaInfo.HasAudio() ? "AV" : "V"); + key.AppendASCII(aIsHardwareAccelerated ? "(hw)," : ","); + static const struct { + int32_t mH; + const char* mRes; + } sResolutions[] = {{240, "0-240"}, + {480, "241-480"}, + {720, "481-720"}, + {1080, "721-1080"}, + {2160, "1081-2160"}}; + const char* resolution = "2161+"; + int32_t height = aMediaInfo.mVideo.mImage.height; + for (const auto& res : sResolutions) { + if (height <= res.mH) { + resolution = res.mRes; + break; + } + } + key.AppendASCII(resolution); + + TimeDuration duration = TimeStamp::Now() - aRecoveryStart; + double duration_ms = duration.ToMilliseconds(); + Telemetry::Accumulate(Telemetry::VIDEO_SUSPEND_RECOVERY_TIME_MS, key, + static_cast<uint32_t>(lround(duration_ms))); + Telemetry::Accumulate(Telemetry::VIDEO_SUSPEND_RECOVERY_TIME_MS, "All"_ns, + static_cast<uint32_t>(lround(duration_ms))); +} + +void MediaDecoderStateMachine::StateObject::HandleResumeVideoDecoding( + const TimeUnit& aTarget) { + MOZ_ASSERT(mMaster->mVideoDecodeSuspended); + + mMaster->mVideoDecodeSuspended = false; + mMaster->mOnPlaybackEvent.Notify(MediaPlaybackEvent::ExitVideoSuspend); + Reader()->SetVideoBlankDecode(false); + + // Start counting recovery time from right now. + TimeStamp start = TimeStamp::Now(); + + // Local reference to mInfo, so that it will be copied in the lambda below. + const auto& info = Info(); + bool hw = Reader()->VideoIsHardwareAccelerated(); + + // Start video-only seek to the current time. + SeekJob seekJob; + + // We use fastseek to optimize the resuming time. + // FastSeek is only used for video-only media since we don't need to worry + // about A/V sync. + // Don't use fastSeek if we want to seek to the end because it might seek to a + // keyframe before the last frame (if the last frame itself is not a keyframe) + // and we always want to present the final frame to the user when seeking to + // the end. + const auto type = mMaster->HasAudio() || aTarget == mMaster->Duration() + ? SeekTarget::Type::Accurate + : SeekTarget::Type::PrevSyncPoint; + + seekJob.mTarget.emplace(aTarget, type, SeekTarget::Track::VideoOnly); + SLOG("video-only seek target=%" PRId64 ", current time=%" PRId64, + aTarget.ToMicroseconds(), mMaster->GetMediaTime().ToMicroseconds()); + + // Hold mMaster->mAbstractMainThread here because this->mMaster will be + // invalid after the current state object is deleted in SetState(); + RefPtr<AbstractThread> mainThread = mMaster->mAbstractMainThread; + + SetSeekingState(std::move(seekJob), EventVisibility::Suppressed) + ->Then( + mainThread, __func__, + [start, info, hw]() { ReportRecoveryTelemetry(start, info, hw); }, + []() {}); +} + +RefPtr<MediaDecoder::SeekPromise> +MediaDecoderStateMachine::StateObject::SetSeekingState( + SeekJob&& aSeekJob, EventVisibility aVisibility) { + if (aSeekJob.mTarget->IsAccurate() || aSeekJob.mTarget->IsFast()) { + if (aSeekJob.mTarget->IsVideoOnly()) { + return SetState<VideoOnlySeekingState>(std::move(aSeekJob), aVisibility); + } + return SetState<AccurateSeekingState>(std::move(aSeekJob), aVisibility); + } + + if (aSeekJob.mTarget->IsNextFrame()) { + return SetState<NextFrameSeekingState>(std::move(aSeekJob), aVisibility); + } + + MOZ_ASSERT_UNREACHABLE("Unknown SeekTarget::Type."); + return nullptr; +} + +void MediaDecoderStateMachine::StateObject::SetDecodingState() { + if (mMaster->IsInSeamlessLooping()) { + SetState<LoopingDecodingState>(); + return; + } + SetState<DecodingState>(); +} + +void MediaDecoderStateMachine::DecodeMetadataState::OnMetadataRead( + MetadataHolder&& aMetadata) { + mMetadataRequest.Complete(); + + AUTO_PROFILER_LABEL("DecodeMetadataState::OnMetadataRead", MEDIA_PLAYBACK); + mMaster->mInfo.emplace(*aMetadata.mInfo); + mMaster->mMediaSeekable = Info().mMediaSeekable; + mMaster->mMediaSeekableOnlyInBufferedRanges = + Info().mMediaSeekableOnlyInBufferedRanges; + + if (Info().mMetadataDuration.isSome()) { + mMaster->mDuration = Info().mMetadataDuration; + } else if (Info().mUnadjustedMetadataEndTime.isSome()) { + const TimeUnit unadjusted = Info().mUnadjustedMetadataEndTime.ref(); + const TimeUnit adjustment = Info().mStartTime; + mMaster->mInfo->mMetadataDuration.emplace(unadjusted - adjustment); + mMaster->mDuration = Info().mMetadataDuration; + } + + // If we don't know the duration by this point, we assume infinity, per spec. + if (mMaster->mDuration.Ref().isNothing()) { + mMaster->mDuration = Some(TimeUnit::FromInfinity()); + } + + DDLOGEX(mMaster, DDLogCategory::Property, "duration_us", + mMaster->mDuration.Ref()->ToMicroseconds()); + + if (mMaster->HasVideo()) { + SLOG("Video decode HWAccel=%d videoQueueSize=%d", + Reader()->VideoIsHardwareAccelerated(), + mMaster->GetAmpleVideoFrames()); + } + + MOZ_ASSERT(mMaster->mDuration.Ref().isSome()); + + mMaster->mMetadataLoadedEvent.Notify(std::move(aMetadata.mInfo), + std::move(aMetadata.mTags), + MediaDecoderEventVisibility::Observable); + + // Check whether the media satisfies the requirement of seamless looping. + // TODO : after we ensure video seamless looping is stable enough, then we can + // remove this to make the condition always true. + mMaster->mSeamlessLoopingAllowed = StaticPrefs::media_seamless_looping(); + if (mMaster->HasVideo()) { + mMaster->mSeamlessLoopingAllowed = + StaticPrefs::media_seamless_looping_video(); + } + + SetState<DecodingFirstFrameState>(); +} + +void MediaDecoderStateMachine::DormantState::HandlePlayStateChanged( + MediaDecoder::PlayState aPlayState) { + if (aPlayState == MediaDecoder::PLAY_STATE_PLAYING) { + // Exit dormant when the user wants to play. + MOZ_ASSERT(mMaster->mSentFirstFrameLoadedEvent); + SetSeekingState(std::move(mPendingSeek), EventVisibility::Suppressed); + } +} + +void MediaDecoderStateMachine::DecodingFirstFrameState::Enter() { + // Transition to DECODING if we've decoded first frames. + if (mMaster->mSentFirstFrameLoadedEvent) { + SetDecodingState(); + return; + } + + MOZ_ASSERT(!mMaster->mVideoDecodeSuspended); + + // Dispatch tasks to decode first frames. + if (mMaster->HasAudio()) { + mMaster->RequestAudioData(); + } + if (mMaster->HasVideo()) { + mMaster->RequestVideoData(media::TimeUnit()); + } +} + +void MediaDecoderStateMachine::DecodingFirstFrameState:: + MaybeFinishDecodeFirstFrame() { + MOZ_ASSERT(!mMaster->mSentFirstFrameLoadedEvent); + + if ((mMaster->IsAudioDecoding() && AudioQueue().GetSize() == 0) || + (mMaster->IsVideoDecoding() && VideoQueue().GetSize() == 0)) { + return; + } + + mMaster->FinishDecodeFirstFrame(); + if (mPendingSeek.Exists()) { + SetSeekingState(std::move(mPendingSeek), EventVisibility::Observable); + } else { + SetDecodingState(); + } +} + +void MediaDecoderStateMachine::DecodingState::Enter() { + MOZ_ASSERT(mMaster->mSentFirstFrameLoadedEvent); + + if (mMaster->mVideoDecodeSuspended && + mMaster->mVideoDecodeMode == VideoDecodeMode::Normal) { + StateObject::HandleResumeVideoDecoding(mMaster->GetMediaTime()); + return; + } + + if (mMaster->mVideoDecodeMode == VideoDecodeMode::Suspend && + !mMaster->mVideoDecodeSuspendTimer.IsScheduled() && + !mMaster->mVideoDecodeSuspended) { + // If the VideoDecodeMode is Suspend and the timer is not schedule, it means + // the timer has timed out and we should suspend video decoding now if + // necessary. + HandleVideoSuspendTimeout(); + } + + // If we're in the normal decoding mode and the decoding has finished, then we + // should go to `completed` state because we don't need to decode anything + // later. However, if we're in the saemless decoding mode, we will restart + // decoding ASAP so we can still stay in `decoding` state. + if (!mMaster->IsVideoDecoding() && !mMaster->IsAudioDecoding() && + !mMaster->IsInSeamlessLooping()) { + SetState<CompletedState>(); + return; + } + + mOnAudioPopped = + AudioQueue().PopFrontEvent().Connect(OwnerThread(), [this]() { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::OnAudioPopped", + MEDIA_PLAYBACK); + if (mMaster->IsAudioDecoding() && !mMaster->HaveEnoughDecodedAudio()) { + EnsureAudioDecodeTaskQueued(); + } + }); + mOnVideoPopped = + VideoQueue().PopFrontEvent().Connect(OwnerThread(), [this]() { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::OnVideoPopped", + MEDIA_PLAYBACK); + if (mMaster->IsVideoDecoding() && !mMaster->HaveEnoughDecodedVideo()) { + EnsureVideoDecodeTaskQueued(); + } + }); + + mMaster->mOnNextFrameStatus.Notify(MediaDecoderOwner::NEXT_FRAME_AVAILABLE); + + mDecodeStartTime = TimeStamp::Now(); + + MaybeStopPrerolling(); + + // Ensure that we've got tasks enqueued to decode data if we need to. + DispatchDecodeTasksIfNeeded(); + + mMaster->ScheduleStateMachine(); + + // Will enter dormant when playback is paused for a while. + if (mMaster->mPlayState == MediaDecoder::PLAY_STATE_PAUSED) { + StartDormantTimer(); + } +} + +void MediaDecoderStateMachine::DecodingState::Step() { + if (mMaster->mPlayState != MediaDecoder::PLAY_STATE_PLAYING && + mMaster->IsPlaying()) { + // We're playing, but the element/decoder is in paused state. Stop + // playing! + mMaster->StopPlayback(); + } + + // Start playback if necessary so that the clock can be properly queried. + if (!mIsPrerolling) { + mMaster->MaybeStartPlayback(); + } + + mMaster->UpdatePlaybackPositionPeriodically(); + MOZ_ASSERT(!mMaster->IsPlaying() || mMaster->IsStateMachineScheduled(), + "Must have timer scheduled"); + if (IsBufferingAllowed()) { + MaybeStartBuffering(); + } +} + +void MediaDecoderStateMachine::DecodingState::HandleEndOfAudio() { + AudioQueue().Finish(); + if (!mMaster->IsVideoDecoding()) { + SetState<CompletedState>(); + } else { + MaybeStopPrerolling(); + } +} + +void MediaDecoderStateMachine::DecodingState::HandleEndOfVideo() { + VideoQueue().Finish(); + if (!mMaster->IsAudioDecoding()) { + SetState<CompletedState>(); + } else { + MaybeStopPrerolling(); + } +} + +void MediaDecoderStateMachine::DecodingState::DispatchDecodeTasksIfNeeded() { + if (mMaster->IsAudioDecoding() && !mMaster->mMinimizePreroll && + !mMaster->HaveEnoughDecodedAudio()) { + EnsureAudioDecodeTaskQueued(); + } + + if (mMaster->IsVideoDecoding() && !mMaster->mMinimizePreroll && + !mMaster->HaveEnoughDecodedVideo()) { + EnsureVideoDecodeTaskQueued(); + } +} + +void MediaDecoderStateMachine::DecodingState::EnsureAudioDecodeTaskQueued() { + if (!mMaster->IsAudioDecoding() || mMaster->IsRequestingAudioData() || + mMaster->IsWaitingAudioData()) { + return; + } + mMaster->RequestAudioData(); +} + +void MediaDecoderStateMachine::DecodingState::EnsureVideoDecodeTaskQueued() { + if (!mMaster->IsVideoDecoding() || mMaster->IsRequestingVideoData() || + mMaster->IsWaitingVideoData()) { + return; + } + mMaster->RequestVideoData(mMaster->GetMediaTime(), + ShouldRequestNextKeyFrame()); +} + +void MediaDecoderStateMachine::DecodingState::MaybeStartBuffering() { + // Buffering makes senses only after decoding first frames. + MOZ_ASSERT(mMaster->mSentFirstFrameLoadedEvent); + + // Don't enter buffering when MediaDecoder is not playing. + if (mMaster->mPlayState != MediaDecoder::PLAY_STATE_PLAYING) { + return; + } + + // Don't enter buffering while prerolling so that the decoder has a chance to + // enqueue some decoded data before we give up and start buffering. + if (!mMaster->IsPlaying()) { + return; + } + + // Note we could have a wait promise pending when playing non-MSE EME. + if (mMaster->OutOfDecodedAudio() && mMaster->IsWaitingAudioData()) { + PROFILER_MARKER_TEXT("MDSM::StartBuffering", MEDIA_PLAYBACK, {}, + "OutOfDecodedAudio"); + SLOG("Enter buffering due to out of decoded audio"); + SetState<BufferingState>(); + return; + } + if (mMaster->OutOfDecodedVideo() && mMaster->IsWaitingVideoData()) { + PROFILER_MARKER_TEXT("MDSM::StartBuffering", MEDIA_PLAYBACK, {}, + "OutOfDecodedVideo"); + SLOG("Enter buffering due to out of decoded video"); + SetState<BufferingState>(); + return; + } + + if (Reader()->UseBufferingHeuristics() && mMaster->HasLowDecodedData() && + mMaster->HasLowBufferedData() && !mMaster->mCanPlayThrough) { + PROFILER_MARKER_TEXT("MDSM::StartBuffering", MEDIA_PLAYBACK, {}, + "BufferingHeuristics"); + SLOG("Enter buffering due to buffering heruistics"); + SetState<BufferingState>(); + } +} + +void MediaDecoderStateMachine::LoopingDecodingState::HandleError( + const MediaResult& aError, bool aIsAudio) { + SLOG("%s looping failed, aError=%s", aIsAudio ? "audio" : "video", + aError.ErrorName().get()); + switch (aError.Code()) { + case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA: + if (aIsAudio) { + HandleWaitingForAudio(); + } else { + HandleWaitingForVideo(); + } + [[fallthrough]]; + case NS_ERROR_DOM_MEDIA_END_OF_STREAM: + // This could happen after either the resource has been close, or the data + // hasn't been appended in MSE, so that we won't be able to get any + // sample and need to fallback to normal looping. + if (mIsReachingAudioEOS && mIsReachingVideoEOS) { + SetState<CompletedState>(); + } + break; + default: + mMaster->DecodeError(aError); + break; + } +} + +void MediaDecoderStateMachine::SeekingState::SeekCompleted() { + const auto newCurrentTime = CalculateNewCurrentTime(); + + if ((newCurrentTime == mMaster->Duration() || + newCurrentTime.EqualsAtLowestResolution( + mMaster->Duration().ToBase(USECS_PER_S))) && + !mMaster->mIsLiveStream) { + SLOG("Seek completed, seeked to end: %s", newCurrentTime.ToString().get()); + // will transition to COMPLETED immediately. Note we don't do + // this when playing a live stream, since the end of media will advance + // once we download more data! + AudioQueue().Finish(); + VideoQueue().Finish(); + + // We won't start MediaSink when paused. m{Audio,Video}Completed will + // remain false and 'playbackEnded' won't be notified. Therefore we + // need to set these flags explicitly when seeking to the end. + mMaster->mAudioCompleted = true; + mMaster->mVideoCompleted = true; + + // There might still be a pending audio request when doing video-only or + // next-frame seek. Discard it so we won't break the invariants of the + // COMPLETED state by adding audio samples to a finished queue. + mMaster->mAudioDataRequest.DisconnectIfExists(); + } + + // We want to resolve the seek request prior finishing the first frame + // to ensure that the seeked event is fired prior loadeded. + // Note: SeekJob.Resolve() resets SeekJob.mTarget. Don't use mSeekJob anymore + // hereafter. + mSeekJob.Resolve(__func__); + + // Notify FirstFrameLoaded now if we haven't since we've decoded some data + // for readyState to transition to HAVE_CURRENT_DATA and fire 'loadeddata'. + if (!mMaster->mSentFirstFrameLoadedEvent) { + mMaster->FinishDecodeFirstFrame(); + } + + // Ensure timestamps are up to date. + // Suppressed visibility comes from two cases: (1) leaving dormant state, + // and (2) resuming suspended video decoder. We want both cases to be + // transparent to the user. So we only notify the change when the seek + // request is from the user. + if (mVisibility == EventVisibility::Observable) { + // Don't update playback position for video-only seek. + // Otherwise we might have |newCurrentTime > mMediaSink->GetPosition()| + // and fail the assertion in GetClock() since we didn't stop MediaSink. + mMaster->UpdatePlaybackPositionInternal(newCurrentTime); + } + + // Try to decode another frame to detect if we're at the end... + SLOG("Seek completed, mCurrentPosition=%" PRId64, + mMaster->mCurrentPosition.Ref().ToMicroseconds()); + + if (mMaster->VideoQueue().PeekFront()) { + mMaster->mMediaSink->Redraw(Info().mVideo); + mMaster->mOnPlaybackEvent.Notify(MediaPlaybackEvent::Invalidate); + } + + GoToNextState(); +} + +void MediaDecoderStateMachine::BufferingState::Step() { + TimeStamp now = TimeStamp::Now(); + MOZ_ASSERT(!mBufferingStart.IsNull(), "Must know buffering start time."); + + if (Reader()->UseBufferingHeuristics()) { + if (mMaster->IsWaitingAudioData() || mMaster->IsWaitingVideoData()) { + // Can't exit buffering when we are still waiting for data. + // Note we don't schedule next loop for we will do that when the wait + // promise is resolved. + return; + } + // With buffering heuristics, we exit buffering state when we: + // 1. can play through or + // 2. time out (specified by mBufferingWait) or + // 3. have enough buffered data. + TimeDuration elapsed = now - mBufferingStart; + TimeDuration timeout = + TimeDuration::FromSeconds(mBufferingWait * mMaster->mPlaybackRate); + bool stopBuffering = + mMaster->mCanPlayThrough || elapsed >= timeout || + !mMaster->HasLowBufferedData(TimeUnit::FromSeconds(mBufferingWait)); + if (!stopBuffering) { + SLOG("Buffering: wait %ds, timeout in %.3lfs", mBufferingWait, + mBufferingWait - elapsed.ToSeconds()); + mMaster->ScheduleStateMachineIn(TimeUnit::FromMicroseconds(USECS_PER_S)); + return; + } + } else if (mMaster->OutOfDecodedAudio() || mMaster->OutOfDecodedVideo()) { + MOZ_ASSERT(!mMaster->OutOfDecodedAudio() || + mMaster->IsRequestingAudioData() || + mMaster->IsWaitingAudioData()); + MOZ_ASSERT(!mMaster->OutOfDecodedVideo() || + mMaster->IsRequestingVideoData() || + mMaster->IsWaitingVideoData()); + SLOG( + "In buffering mode, waiting to be notified: outOfAudio: %d, " + "mAudioStatus: %s, outOfVideo: %d, mVideoStatus: %s", + mMaster->OutOfDecodedAudio(), mMaster->AudioRequestStatus(), + mMaster->OutOfDecodedVideo(), mMaster->VideoRequestStatus()); + return; + } + + SLOG("Buffered for %.3lfs", (now - mBufferingStart).ToSeconds()); + SetDecodingState(); +} + +void MediaDecoderStateMachine::BufferingState::HandleEndOfAudio() { + AudioQueue().Finish(); + if (!mMaster->IsVideoDecoding()) { + SetState<CompletedState>(); + } else { + // Check if we can exit buffering. + mMaster->ScheduleStateMachine(); + } +} + +void MediaDecoderStateMachine::BufferingState::HandleEndOfVideo() { + VideoQueue().Finish(); + if (!mMaster->IsAudioDecoding()) { + SetState<CompletedState>(); + } else { + // Check if we can exit buffering. + mMaster->ScheduleStateMachine(); + } +} + +RefPtr<ShutdownPromise> MediaDecoderStateMachine::ShutdownState::Enter() { + auto* master = mMaster; + + master->mDelayedScheduler.Reset(); + + // Shutdown happens while decode timer is active, we need to disconnect and + // dispose of the timer. + master->CancelSuspendTimer(); + + if (master->IsPlaying()) { + master->StopPlayback(); + } + + master->mAudioDataRequest.DisconnectIfExists(); + master->mVideoDataRequest.DisconnectIfExists(); + master->mAudioWaitRequest.DisconnectIfExists(); + master->mVideoWaitRequest.DisconnectIfExists(); + + // Resetting decode should be called after stopping media sink, which can + // ensure that we have an empty media queue before seeking the demuxer. + master->StopMediaSink(); + master->ResetDecode(); + master->mMediaSink->Shutdown(); + + // Prevent dangling pointers by disconnecting the listeners. + master->mAudioQueueListener.Disconnect(); + master->mVideoQueueListener.Disconnect(); + master->mMetadataManager.Disconnect(); + master->mOnMediaNotSeekable.Disconnect(); + master->mAudibleListener.DisconnectIfExists(); + + // Disconnect canonicals and mirrors before shutting down our task queue. + master->mStreamName.DisconnectIfConnected(); + master->mSinkDevice.DisconnectIfConnected(); + master->mOutputCaptureState.DisconnectIfConnected(); + master->mOutputDummyTrack.DisconnectIfConnected(); + master->mOutputTracks.DisconnectIfConnected(); + master->mOutputPrincipal.DisconnectIfConnected(); + + master->mDuration.DisconnectAll(); + master->mCurrentPosition.DisconnectAll(); + master->mIsAudioDataAudible.DisconnectAll(); + + // Shut down the watch manager to stop further notifications. + master->mWatchManager.Shutdown(); + + return Reader()->Shutdown()->Then(OwnerThread(), __func__, master, + &MediaDecoderStateMachine::FinishShutdown, + &MediaDecoderStateMachine::FinishShutdown); +} + +#define INIT_WATCHABLE(name, val) name(val, "MediaDecoderStateMachine::" #name) +#define INIT_MIRROR(name, val) \ + name(mTaskQueue, val, "MediaDecoderStateMachine::" #name " (Mirror)") +#define INIT_CANONICAL(name, val) \ + name(mTaskQueue, val, "MediaDecoderStateMachine::" #name " (Canonical)") + +MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder, + MediaFormatReader* aReader) + : MediaDecoderStateMachineBase(aDecoder, aReader), + mWatchManager(this, mTaskQueue), + mDispatchedStateMachine(false), + mDelayedScheduler(mTaskQueue, true /*aFuzzy*/), + mCurrentFrameID(0), + mAmpleAudioThreshold(detail::AMPLE_AUDIO_THRESHOLD), + mVideoDecodeSuspended(false), + mVideoDecodeSuspendTimer(mTaskQueue), + mVideoDecodeMode(VideoDecodeMode::Normal), + mIsMSE(aDecoder->IsMSE()), + mShouldResistFingerprinting(aDecoder->ShouldResistFingerprinting()), + mSeamlessLoopingAllowed(false), + INIT_MIRROR(mStreamName, nsAutoString()), + INIT_MIRROR(mSinkDevice, nullptr), + INIT_MIRROR(mOutputCaptureState, MediaDecoder::OutputCaptureState::None), + INIT_MIRROR(mOutputDummyTrack, nullptr), + INIT_MIRROR(mOutputTracks, nsTArray<RefPtr<ProcessedMediaTrack>>()), + INIT_MIRROR(mOutputPrincipal, PRINCIPAL_HANDLE_NONE), + INIT_CANONICAL(mCanonicalOutputPrincipal, PRINCIPAL_HANDLE_NONE) { + MOZ_COUNT_CTOR(MediaDecoderStateMachine); + NS_ASSERTION(NS_IsMainThread(), "Should be on main thread."); + + InitVideoQueuePrefs(); + + DDLINKCHILD("reader", aReader); +} + +#undef INIT_WATCHABLE +#undef INIT_MIRROR +#undef INIT_CANONICAL + +MediaDecoderStateMachine::~MediaDecoderStateMachine() { + MOZ_ASSERT(NS_IsMainThread(), "Should be on main thread."); + MOZ_COUNT_DTOR(MediaDecoderStateMachine); +} + +void MediaDecoderStateMachine::InitializationTask(MediaDecoder* aDecoder) { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::InitializationTask", + MEDIA_PLAYBACK); + MOZ_ASSERT(OnTaskQueue()); + + MediaDecoderStateMachineBase::InitializationTask(aDecoder); + + // Initialize watchers. + mWatchManager.Watch(mStreamName, + &MediaDecoderStateMachine::StreamNameChanged); + mWatchManager.Watch(mOutputCaptureState, + &MediaDecoderStateMachine::UpdateOutputCaptured); + mWatchManager.Watch(mOutputDummyTrack, + &MediaDecoderStateMachine::UpdateOutputCaptured); + mWatchManager.Watch(mOutputTracks, + &MediaDecoderStateMachine::UpdateOutputCaptured); + mWatchManager.Watch(mOutputPrincipal, + &MediaDecoderStateMachine::OutputPrincipalChanged); + + mMediaSink = CreateMediaSink(); + + MOZ_ASSERT(!mStateObj); + auto* s = new DecodeMetadataState(this); + mStateObj.reset(s); + s->Enter(); +} + +void MediaDecoderStateMachine::AudioAudibleChanged(bool aAudible) { + mIsAudioDataAudible = aAudible; +} + +MediaSink* MediaDecoderStateMachine::CreateAudioSink() { + if (mOutputCaptureState != MediaDecoder::OutputCaptureState::None) { + DecodedStream* stream = new DecodedStream( + this, + mOutputCaptureState == MediaDecoder::OutputCaptureState::Capture + ? mOutputDummyTrack.Ref() + : nullptr, + mOutputTracks, mVolume, mPlaybackRate, mPreservesPitch, mAudioQueue, + mVideoQueue, mSinkDevice.Ref()); + mAudibleListener.DisconnectIfExists(); + mAudibleListener = stream->AudibleEvent().Connect( + OwnerThread(), this, &MediaDecoderStateMachine::AudioAudibleChanged); + return stream; + } + + auto audioSinkCreator = [s = RefPtr<MediaDecoderStateMachine>(this), this]() { + MOZ_ASSERT(OnTaskQueue()); + UniquePtr<AudioSink> audioSink{new AudioSink( + mTaskQueue, mAudioQueue, Info().mAudio, mShouldResistFingerprinting)}; + mAudibleListener.DisconnectIfExists(); + mAudibleListener = audioSink->AudibleEvent().Connect( + mTaskQueue, this, &MediaDecoderStateMachine::AudioAudibleChanged); + return audioSink; + }; + return new AudioSinkWrapper( + mTaskQueue, mAudioQueue, std::move(audioSinkCreator), mVolume, + mPlaybackRate, mPreservesPitch, mSinkDevice.Ref()); +} + +already_AddRefed<MediaSink> MediaDecoderStateMachine::CreateMediaSink() { + MOZ_ASSERT(OnTaskQueue()); + RefPtr<MediaSink> audioSink = CreateAudioSink(); + RefPtr<MediaSink> mediaSink = + new VideoSink(mTaskQueue, audioSink, mVideoQueue, mVideoFrameContainer, + *mFrameStats, sVideoQueueSendToCompositorSize); + if (mSecondaryVideoContainer.Ref()) { + mediaSink->SetSecondaryVideoContainer(mSecondaryVideoContainer.Ref()); + } + return mediaSink.forget(); +} + +TimeUnit MediaDecoderStateMachine::GetDecodedAudioDuration() const { + MOZ_ASSERT(OnTaskQueue()); + if (mMediaSink->IsStarted()) { + return mMediaSink->UnplayedDuration(TrackInfo::kAudioTrack) + + TimeUnit::FromMicroseconds(AudioQueue().Duration()); + } + // MediaSink not started. All audio samples are in the queue. + return TimeUnit::FromMicroseconds(AudioQueue().Duration()); +} + +bool MediaDecoderStateMachine::HaveEnoughDecodedAudio() const { + MOZ_ASSERT(OnTaskQueue()); + auto ampleAudio = mAmpleAudioThreshold.MultDouble(mPlaybackRate); + return AudioQueue().GetSize() > 0 && GetDecodedAudioDuration() >= ampleAudio; +} + +bool MediaDecoderStateMachine::HaveEnoughDecodedVideo() const { + MOZ_ASSERT(OnTaskQueue()); + return static_cast<double>(VideoQueue().GetSize()) >= + GetAmpleVideoFrames() * mPlaybackRate + 1 && + IsVideoDataEnoughComparedWithAudio(); +} + +bool MediaDecoderStateMachine::IsVideoDataEnoughComparedWithAudio() const { + // HW decoding is usually fast enough and we don't need to worry about its + // speed. + // TODO : we can consider whether we need to enable this on other HW decoding + // except VAAPI. When enabling VAAPI on Linux, ffmpeg is not able to store too + // many frames because it has a limitation of amount of stored video frames. + // See bug1716638 and 1718309. + if (mReader->VideoIsHardwareAccelerated()) { + return true; + } + // In extreme situations (e.g. 4k+ video without hardware acceleration), the + // video decoding will be much slower than audio. So for 4K+ video, we want to + // consider audio decoding speed as well in order to reduce frame drops. This + // check tries to keep the decoded video buffered as much as audio. + if (HasAudio() && Info().mVideo.mImage.width >= 3840 && + Info().mVideo.mImage.height >= 2160) { + return VideoQueue().Duration() >= AudioQueue().Duration(); + } + // For non-4k video, the video decoding is usually really fast so we won't + // need to consider audio decoding speed to store extra frames. + return true; +} + +void MediaDecoderStateMachine::PushAudio(AudioData* aSample) { + MOZ_ASSERT(OnTaskQueue()); + MOZ_ASSERT(aSample); + AudioQueue().Push(aSample); + PROFILER_MARKER("MDSM::PushAudio", MEDIA_PLAYBACK, {}, MediaSampleMarker, + aSample->mTime.ToMicroseconds(), + aSample->GetEndTime().ToMicroseconds(), + AudioQueue().GetSize()); +} + +void MediaDecoderStateMachine::PushVideo(VideoData* aSample) { + MOZ_ASSERT(OnTaskQueue()); + MOZ_ASSERT(aSample); + aSample->mFrameID = ++mCurrentFrameID; + VideoQueue().Push(aSample); + PROFILER_MARKER("MDSM::PushVideo", MEDIA_PLAYBACK, {}, MediaSampleMarker, + aSample->mTime.ToMicroseconds(), + aSample->GetEndTime().ToMicroseconds(), + VideoQueue().GetSize()); +} + +void MediaDecoderStateMachine::OnAudioPopped(const RefPtr<AudioData>& aSample) { + MOZ_ASSERT(OnTaskQueue()); + mPlaybackOffset = std::max(mPlaybackOffset, aSample->mOffset); +} + +void MediaDecoderStateMachine::OnVideoPopped(const RefPtr<VideoData>& aSample) { + MOZ_ASSERT(OnTaskQueue()); + mPlaybackOffset = std::max(mPlaybackOffset, aSample->mOffset); +} + +bool MediaDecoderStateMachine::IsAudioDecoding() { + MOZ_ASSERT(OnTaskQueue()); + return HasAudio() && !AudioQueue().IsFinished(); +} + +bool MediaDecoderStateMachine::IsVideoDecoding() { + MOZ_ASSERT(OnTaskQueue()); + return HasVideo() && !VideoQueue().IsFinished(); +} + +bool MediaDecoderStateMachine::IsPlaying() const { + MOZ_ASSERT(OnTaskQueue()); + return mMediaSink->IsPlaying(); +} + +void MediaDecoderStateMachine::SetMediaNotSeekable() { mMediaSeekable = false; } + +nsresult MediaDecoderStateMachine::Init(MediaDecoder* aDecoder) { + MOZ_ASSERT(NS_IsMainThread()); + + nsresult rv = MediaDecoderStateMachineBase::Init(aDecoder); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + + // Connect mirrors. + aDecoder->CanonicalStreamName().ConnectMirror(&mStreamName); + aDecoder->CanonicalSinkDevice().ConnectMirror(&mSinkDevice); + aDecoder->CanonicalOutputCaptureState().ConnectMirror(&mOutputCaptureState); + aDecoder->CanonicalOutputDummyTrack().ConnectMirror(&mOutputDummyTrack); + aDecoder->CanonicalOutputTracks().ConnectMirror(&mOutputTracks); + aDecoder->CanonicalOutputPrincipal().ConnectMirror(&mOutputPrincipal); + + mAudioQueueListener = AudioQueue().PopFrontEvent().Connect( + mTaskQueue, this, &MediaDecoderStateMachine::OnAudioPopped); + mVideoQueueListener = VideoQueue().PopFrontEvent().Connect( + mTaskQueue, this, &MediaDecoderStateMachine::OnVideoPopped); + mOnMediaNotSeekable = mReader->OnMediaNotSeekable().Connect( + OwnerThread(), this, &MediaDecoderStateMachine::SetMediaNotSeekable); + + return NS_OK; +} + +void MediaDecoderStateMachine::StopPlayback() { + MOZ_ASSERT(OnTaskQueue()); + LOG("StopPlayback()"); + + if (IsPlaying()) { + mOnPlaybackEvent.Notify(MediaPlaybackEvent{ + MediaPlaybackEvent::PlaybackStopped, mPlaybackOffset}); + mMediaSink->SetPlaying(false); + MOZ_ASSERT(!IsPlaying()); + } +} + +void MediaDecoderStateMachine::MaybeStartPlayback() { + MOZ_ASSERT(OnTaskQueue()); + // Should try to start playback only after decoding first frames. + if (!mSentFirstFrameLoadedEvent) { + LOG("MaybeStartPlayback: Not starting playback before loading first frame"); + return; + } + + if (IsPlaying()) { + // Logging this case is really spammy - don't do it. + return; + } + + if (mIsMediaSinkSuspended) { + LOG("MaybeStartPlayback: Not starting playback when sink is suspended"); + return; + } + + if (mPlayState != MediaDecoder::PLAY_STATE_PLAYING) { + LOG("MaybeStartPlayback: Not starting playback [mPlayState=%d]", + mPlayState.Ref()); + return; + } + + LOG("MaybeStartPlayback() starting playback"); + StartMediaSink(); + + if (!IsPlaying()) { + mMediaSink->SetPlaying(true); + MOZ_ASSERT(IsPlaying()); + } + + mOnPlaybackEvent.Notify( + MediaPlaybackEvent{MediaPlaybackEvent::PlaybackStarted, mPlaybackOffset}); +} + +void MediaDecoderStateMachine::UpdatePlaybackPositionInternal( + const TimeUnit& aTime) { + MOZ_ASSERT(OnTaskQueue()); + LOGV("UpdatePlaybackPositionInternal(%" PRId64 ")", aTime.ToMicroseconds()); + + // Ensure the position has a precision that matches other TimeUnit such as + // buffering ranges and duration. + mCurrentPosition = aTime.ToBase(1000000); + NS_ASSERTION(mCurrentPosition.Ref() >= TimeUnit::Zero(), + "CurrentTime should be positive!"); + if (mDuration.Ref().ref() < mCurrentPosition.Ref()) { + mDuration = Some(mCurrentPosition.Ref()); + DDLOG(DDLogCategory::Property, "duration_us", + mDuration.Ref()->ToMicroseconds()); + } +} + +void MediaDecoderStateMachine::UpdatePlaybackPosition(const TimeUnit& aTime) { + MOZ_ASSERT(OnTaskQueue()); + UpdatePlaybackPositionInternal(aTime); + + bool fragmentEnded = + mFragmentEndTime.IsValid() && GetMediaTime() >= mFragmentEndTime; + mMetadataManager.DispatchMetadataIfNeeded(aTime); + + if (fragmentEnded) { + StopPlayback(); + } +} + +/* static */ const char* MediaDecoderStateMachine::ToStateStr(State aState) { + switch (aState) { + case DECODER_STATE_DECODING_METADATA: + return "DECODING_METADATA"; + case DECODER_STATE_DORMANT: + return "DORMANT"; + case DECODER_STATE_DECODING_FIRSTFRAME: + return "DECODING_FIRSTFRAME"; + case DECODER_STATE_DECODING: + return "DECODING"; + case DECODER_STATE_SEEKING_ACCURATE: + return "SEEKING_ACCURATE"; + case DECODER_STATE_SEEKING_FROMDORMANT: + return "SEEKING_FROMDORMANT"; + case DECODER_STATE_SEEKING_NEXTFRAMESEEKING: + return "DECODER_STATE_SEEKING_NEXTFRAMESEEKING"; + case DECODER_STATE_SEEKING_VIDEOONLY: + return "SEEKING_VIDEOONLY"; + case DECODER_STATE_BUFFERING: + return "BUFFERING"; + case DECODER_STATE_COMPLETED: + return "COMPLETED"; + case DECODER_STATE_SHUTDOWN: + return "SHUTDOWN"; + case DECODER_STATE_LOOPING_DECODING: + return "LOOPING_DECODING"; + default: + MOZ_ASSERT_UNREACHABLE("Invalid state."); + } + return "UNKNOWN"; +} + +const char* MediaDecoderStateMachine::ToStateStr() { + MOZ_ASSERT(OnTaskQueue()); + return ToStateStr(mStateObj->GetState()); +} + +void MediaDecoderStateMachine::VolumeChanged() { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::VolumeChanged", + MEDIA_PLAYBACK); + MOZ_ASSERT(OnTaskQueue()); + mMediaSink->SetVolume(mVolume); +} + +RefPtr<ShutdownPromise> MediaDecoderStateMachine::Shutdown() { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::Shutdown", MEDIA_PLAYBACK); + MOZ_ASSERT(OnTaskQueue()); + return mStateObj->HandleShutdown(); +} + +void MediaDecoderStateMachine::PlayStateChanged() { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::PlayStateChanged", + MEDIA_PLAYBACK); + MOZ_ASSERT(OnTaskQueue()); + + if (mPlayState != MediaDecoder::PLAY_STATE_PLAYING) { + CancelSuspendTimer(); + } else if (mMinimizePreroll) { + // Once we start playing, we don't want to minimize our prerolling, as we + // assume the user is likely to want to keep playing in future. This needs + // to happen before we invoke StartDecoding(). + mMinimizePreroll = false; + } + + mStateObj->HandlePlayStateChanged(mPlayState); +} + +void MediaDecoderStateMachine::SetVideoDecodeMode(VideoDecodeMode aMode) { + MOZ_ASSERT(NS_IsMainThread()); + nsCOMPtr<nsIRunnable> r = NewRunnableMethod<VideoDecodeMode>( + "MediaDecoderStateMachine::SetVideoDecodeModeInternal", this, + &MediaDecoderStateMachine::SetVideoDecodeModeInternal, aMode); + OwnerThread()->DispatchStateChange(r.forget()); +} + +void MediaDecoderStateMachine::SetVideoDecodeModeInternal( + VideoDecodeMode aMode) { + MOZ_ASSERT(OnTaskQueue()); + + LOG("SetVideoDecodeModeInternal(), VideoDecodeMode=(%s->%s), " + "mVideoDecodeSuspended=%c", + mVideoDecodeMode == VideoDecodeMode::Normal ? "Normal" : "Suspend", + aMode == VideoDecodeMode::Normal ? "Normal" : "Suspend", + mVideoDecodeSuspended ? 'T' : 'F'); + + // Should not suspend decoding if we don't turn on the pref. + if (!StaticPrefs::media_suspend_background_video_enabled() && + aMode == VideoDecodeMode::Suspend) { + LOG("SetVideoDecodeModeInternal(), early return because preference off and " + "set to Suspend"); + return; + } + + if (aMode == mVideoDecodeMode) { + LOG("SetVideoDecodeModeInternal(), early return because the mode does not " + "change"); + return; + } + + // Set new video decode mode. + mVideoDecodeMode = aMode; + + // Start timer to trigger suspended video decoding. + if (mVideoDecodeMode == VideoDecodeMode::Suspend) { + TimeStamp target = TimeStamp::Now() + SuspendBackgroundVideoDelay(); + + RefPtr<MediaDecoderStateMachine> self = this; + mVideoDecodeSuspendTimer.Ensure( + target, [=]() { self->OnSuspendTimerResolved(); }, + []() { MOZ_DIAGNOSTIC_ASSERT(false); }); + mOnPlaybackEvent.Notify(MediaPlaybackEvent::StartVideoSuspendTimer); + return; + } + + // Resuming from suspended decoding + + // If suspend timer exists, destroy it. + CancelSuspendTimer(); + + if (mVideoDecodeSuspended) { + auto target = mMediaSink->IsStarted() ? GetClock() : GetMediaTime(); + AdjustByLooping(target); + mStateObj->HandleResumeVideoDecoding(target + detail::RESUME_VIDEO_PREMIUM); + } +} + +void MediaDecoderStateMachine::BufferedRangeUpdated() { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::BufferedRangeUpdated", + MEDIA_PLAYBACK); + MOZ_ASSERT(OnTaskQueue()); + + // While playing an unseekable stream of unknown duration, mDuration + // is updated as we play. But if data is being downloaded + // faster than played, mDuration won't reflect the end of playable data + // since we haven't played the frame at the end of buffered data. So update + // mDuration here as new data is downloaded to prevent such a lag. + if (mBuffered.Ref().IsInvalid()) { + return; + } + + bool exists; + media::TimeUnit end{mBuffered.Ref().GetEnd(&exists)}; + if (!exists) { + return; + } + + // Use estimated duration from buffer ranges when mDuration is unknown or + // the estimated duration is larger. + if (mDuration.Ref().isNothing() || mDuration.Ref()->IsInfinite() || + end > mDuration.Ref().ref()) { + mDuration = Some(end); + DDLOG(DDLogCategory::Property, "duration_us", + mDuration.Ref()->ToMicroseconds()); + } +} + +RefPtr<MediaDecoder::SeekPromise> MediaDecoderStateMachine::Seek( + const SeekTarget& aTarget) { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::Seek", MEDIA_PLAYBACK); + PROFILER_MARKER_UNTYPED("MDSM::Seek", MEDIA_PLAYBACK); + MOZ_ASSERT(OnTaskQueue()); + + // We need to be able to seek in some way + if (!mMediaSeekable && !mMediaSeekableOnlyInBufferedRanges) { + LOGW("Seek() should not be called on a non-seekable media"); + return MediaDecoder::SeekPromise::CreateAndReject(/* aRejectValue = */ true, + __func__); + } + + if (aTarget.IsNextFrame() && !HasVideo()) { + LOGW("Ignore a NextFrameSeekTask on a media file without video track."); + return MediaDecoder::SeekPromise::CreateAndReject(/* aRejectValue = */ true, + __func__); + } + + MOZ_ASSERT(mDuration.Ref().isSome(), "We should have got duration already"); + + return mStateObj->HandleSeek(aTarget); +} + +void MediaDecoderStateMachine::StopMediaSink() { + MOZ_ASSERT(OnTaskQueue()); + if (mMediaSink->IsStarted()) { + LOG("Stop MediaSink"); + mMediaSink->Stop(); + mMediaSinkAudioEndedPromise.DisconnectIfExists(); + mMediaSinkVideoEndedPromise.DisconnectIfExists(); + } +} + +void MediaDecoderStateMachine::RequestAudioData() { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::RequestAudioData", + MEDIA_PLAYBACK); + MOZ_ASSERT(OnTaskQueue()); + MOZ_ASSERT(IsAudioDecoding()); + MOZ_ASSERT(!IsRequestingAudioData()); + MOZ_ASSERT(!IsWaitingAudioData()); + LOGV("Queueing audio task - queued=%zu, decoder-queued=%zu", + AudioQueue().GetSize(), mReader->SizeOfAudioQueueInFrames()); + + PerformanceRecorder<PlaybackStage> perfRecorder(MediaStage::RequestData); + RefPtr<MediaDecoderStateMachine> self = this; + mReader->RequestAudioData() + ->Then( + OwnerThread(), __func__, + [this, self, perfRecorder(std::move(perfRecorder))]( + const RefPtr<AudioData>& aAudio) mutable { + perfRecorder.Record(); + AUTO_PROFILER_LABEL( + "MediaDecoderStateMachine::RequestAudioData:Resolved", + MEDIA_PLAYBACK); + MOZ_ASSERT(aAudio); + mAudioDataRequest.Complete(); + // audio->GetEndTime() is not always mono-increasing in chained + // ogg. + mDecodedAudioEndTime = + std::max(aAudio->GetEndTime(), mDecodedAudioEndTime); + LOGV("OnAudioDecoded [%" PRId64 ",%" PRId64 "]", + aAudio->mTime.ToMicroseconds(), + aAudio->GetEndTime().ToMicroseconds()); + mStateObj->HandleAudioDecoded(aAudio); + }, + [this, self](const MediaResult& aError) { + AUTO_PROFILER_LABEL( + "MediaDecoderStateMachine::RequestAudioData:Rejected", + MEDIA_PLAYBACK); + LOGV("OnAudioNotDecoded ErrorName=%s Message=%s", + aError.ErrorName().get(), aError.Message().get()); + mAudioDataRequest.Complete(); + switch (aError.Code()) { + case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA: + mStateObj->HandleWaitingForAudio(); + break; + case NS_ERROR_DOM_MEDIA_CANCELED: + mStateObj->HandleAudioCanceled(); + break; + case NS_ERROR_DOM_MEDIA_END_OF_STREAM: + mStateObj->HandleEndOfAudio(); + break; + default: + DecodeError(aError); + } + }) + ->Track(mAudioDataRequest); +} + +void MediaDecoderStateMachine::RequestVideoData( + const media::TimeUnit& aCurrentTime, bool aRequestNextKeyFrame) { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::RequestVideoData", + MEDIA_PLAYBACK); + MOZ_ASSERT(OnTaskQueue()); + MOZ_ASSERT(IsVideoDecoding()); + MOZ_ASSERT(!IsRequestingVideoData()); + MOZ_ASSERT(!IsWaitingVideoData()); + LOGV( + "Queueing video task - queued=%zu, decoder-queued=%zo" + ", stime=%" PRId64 ", by-pass-skip=%d", + VideoQueue().GetSize(), mReader->SizeOfVideoQueueInFrames(), + aCurrentTime.ToMicroseconds(), mBypassingSkipToNextKeyFrameCheck); + + PerformanceRecorder<PlaybackStage> perfRecorder(MediaStage::RequestData, + Info().mVideo.mImage.height); + RefPtr<MediaDecoderStateMachine> self = this; + mReader + ->RequestVideoData( + mBypassingSkipToNextKeyFrameCheck ? media::TimeUnit() : aCurrentTime, + mBypassingSkipToNextKeyFrameCheck ? false : aRequestNextKeyFrame) + ->Then( + OwnerThread(), __func__, + [this, self, perfRecorder(std::move(perfRecorder))]( + const RefPtr<VideoData>& aVideo) mutable { + perfRecorder.Record(); + AUTO_PROFILER_LABEL( + "MediaDecoderStateMachine::RequestVideoData:Resolved", + MEDIA_PLAYBACK); + MOZ_ASSERT(aVideo); + mVideoDataRequest.Complete(); + // Handle abnormal or negative timestamps. + mDecodedVideoEndTime = + std::max(mDecodedVideoEndTime, aVideo->GetEndTime()); + LOGV("OnVideoDecoded [%" PRId64 ",%" PRId64 "]", + aVideo->mTime.ToMicroseconds(), + aVideo->GetEndTime().ToMicroseconds()); + mStateObj->HandleVideoDecoded(aVideo); + }, + [this, self](const MediaResult& aError) { + AUTO_PROFILER_LABEL( + "MediaDecoderStateMachine::RequestVideoData:Rejected", + MEDIA_PLAYBACK); + LOGV("OnVideoNotDecoded ErrorName=%s Message=%s", + aError.ErrorName().get(), aError.Message().get()); + mVideoDataRequest.Complete(); + switch (aError.Code()) { + case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA: + mStateObj->HandleWaitingForVideo(); + break; + case NS_ERROR_DOM_MEDIA_CANCELED: + mStateObj->HandleVideoCanceled(); + break; + case NS_ERROR_DOM_MEDIA_END_OF_STREAM: + mStateObj->HandleEndOfVideo(); + break; + default: + DecodeError(aError); + } + }) + ->Track(mVideoDataRequest); +} + +void MediaDecoderStateMachine::WaitForData(MediaData::Type aType) { + MOZ_ASSERT(OnTaskQueue()); + MOZ_ASSERT(aType == MediaData::Type::AUDIO_DATA || + aType == MediaData::Type::VIDEO_DATA); + RefPtr<MediaDecoderStateMachine> self = this; + if (aType == MediaData::Type::AUDIO_DATA) { + mReader->WaitForData(MediaData::Type::AUDIO_DATA) + ->Then( + OwnerThread(), __func__, + [self](MediaData::Type aType) { + AUTO_PROFILER_LABEL( + "MediaDecoderStateMachine::WaitForData:AudioResolved", + MEDIA_PLAYBACK); + self->mAudioWaitRequest.Complete(); + MOZ_ASSERT(aType == MediaData::Type::AUDIO_DATA); + self->mStateObj->HandleAudioWaited(aType); + }, + [self](const WaitForDataRejectValue& aRejection) { + AUTO_PROFILER_LABEL( + "MediaDecoderStateMachine::WaitForData:AudioRejected", + MEDIA_PLAYBACK); + self->mAudioWaitRequest.Complete(); + self->DecodeError(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA); + }) + ->Track(mAudioWaitRequest); + } else { + mReader->WaitForData(MediaData::Type::VIDEO_DATA) + ->Then( + OwnerThread(), __func__, + [self](MediaData::Type aType) { + AUTO_PROFILER_LABEL( + "MediaDecoderStateMachine::WaitForData:VideoResolved", + MEDIA_PLAYBACK); + self->mVideoWaitRequest.Complete(); + MOZ_ASSERT(aType == MediaData::Type::VIDEO_DATA); + self->mStateObj->HandleVideoWaited(aType); + }, + [self](const WaitForDataRejectValue& aRejection) { + AUTO_PROFILER_LABEL( + "MediaDecoderStateMachine::WaitForData:VideoRejected", + MEDIA_PLAYBACK); + self->mVideoWaitRequest.Complete(); + self->DecodeError(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA); + }) + ->Track(mVideoWaitRequest); + } +} + +nsresult MediaDecoderStateMachine::StartMediaSink() { + MOZ_ASSERT(OnTaskQueue()); + + if (mMediaSink->IsStarted()) { + return NS_OK; + } + + mAudioCompleted = false; + const auto startTime = GetMediaTime(); + LOG("StartMediaSink, mediaTime=%" PRId64, startTime.ToMicroseconds()); + nsresult rv = mMediaSink->Start(startTime, Info()); + StreamNameChanged(); + + auto videoPromise = mMediaSink->OnEnded(TrackInfo::kVideoTrack); + auto audioPromise = mMediaSink->OnEnded(TrackInfo::kAudioTrack); + + if (audioPromise) { + audioPromise + ->Then(OwnerThread(), __func__, this, + &MediaDecoderStateMachine::OnMediaSinkAudioComplete, + &MediaDecoderStateMachine::OnMediaSinkAudioError) + ->Track(mMediaSinkAudioEndedPromise); + } + if (videoPromise) { + videoPromise + ->Then(OwnerThread(), __func__, this, + &MediaDecoderStateMachine::OnMediaSinkVideoComplete, + &MediaDecoderStateMachine::OnMediaSinkVideoError) + ->Track(mMediaSinkVideoEndedPromise); + } + // Remember the initial offset when playback starts. This will be used + // to calculate the rate at which bytes are consumed as playback moves on. + RefPtr<MediaData> sample = mAudioQueue.PeekFront(); + mPlaybackOffset = sample ? sample->mOffset : 0; + sample = mVideoQueue.PeekFront(); + if (sample && sample->mOffset > mPlaybackOffset) { + mPlaybackOffset = sample->mOffset; + } + return rv; +} + +bool MediaDecoderStateMachine::HasLowDecodedAudio() { + MOZ_ASSERT(OnTaskQueue()); + return IsAudioDecoding() && + GetDecodedAudioDuration() < + EXHAUSTED_DATA_MARGIN.MultDouble(mPlaybackRate); +} + +bool MediaDecoderStateMachine::HasLowDecodedVideo() { + MOZ_ASSERT(OnTaskQueue()); + return IsVideoDecoding() && + VideoQueue().GetSize() < + static_cast<size_t>(floorl(LOW_VIDEO_FRAMES * mPlaybackRate)); +} + +bool MediaDecoderStateMachine::HasLowDecodedData() { + MOZ_ASSERT(OnTaskQueue()); + MOZ_ASSERT(mReader->UseBufferingHeuristics()); + return HasLowDecodedAudio() || HasLowDecodedVideo(); +} + +bool MediaDecoderStateMachine::OutOfDecodedAudio() { + MOZ_ASSERT(OnTaskQueue()); + return IsAudioDecoding() && !AudioQueue().IsFinished() && + AudioQueue().GetSize() == 0 && + !mMediaSink->HasUnplayedFrames(TrackInfo::kAudioTrack); +} + +bool MediaDecoderStateMachine::HasLowBufferedData() { + MOZ_ASSERT(OnTaskQueue()); + return HasLowBufferedData(detail::LOW_BUFFER_THRESHOLD); +} + +bool MediaDecoderStateMachine::HasLowBufferedData(const TimeUnit& aThreshold) { + MOZ_ASSERT(OnTaskQueue()); + + // If we don't have a duration, mBuffered is probably not going to have + // a useful buffered range. Return false here so that we don't get stuck in + // buffering mode for live streams. + if (Duration().IsInfinite()) { + return false; + } + + if (mBuffered.Ref().IsInvalid()) { + return false; + } + + // We are never low in decoded data when we don't have audio/video or have + // decoded all audio/video samples. + TimeUnit endOfDecodedVideo = (HasVideo() && !VideoQueue().IsFinished()) + ? mDecodedVideoEndTime + : TimeUnit::FromNegativeInfinity(); + TimeUnit endOfDecodedAudio = (HasAudio() && !AudioQueue().IsFinished()) + ? mDecodedAudioEndTime + : TimeUnit::FromNegativeInfinity(); + + auto endOfDecodedData = std::max(endOfDecodedVideo, endOfDecodedAudio); + if (Duration() < endOfDecodedData) { + // Our duration is not up to date. No point buffering. + return false; + } + + if (endOfDecodedData.IsInfinite()) { + // Have decoded all samples. No point buffering. + return false; + } + + auto start = endOfDecodedData; + auto end = std::min(GetMediaTime() + aThreshold, Duration()); + if (start >= end) { + // Duration of decoded samples is greater than our threshold. + return false; + } + media::TimeInterval interval(start, end); + return !mBuffered.Ref().Contains(interval); +} + +void MediaDecoderStateMachine::EnqueueFirstFrameLoadedEvent() { + MOZ_ASSERT(OnTaskQueue()); + // Track value of mSentFirstFrameLoadedEvent from before updating it + bool firstFrameBeenLoaded = mSentFirstFrameLoadedEvent; + mSentFirstFrameLoadedEvent = true; + MediaDecoderEventVisibility visibility = + firstFrameBeenLoaded ? MediaDecoderEventVisibility::Suppressed + : MediaDecoderEventVisibility::Observable; + mFirstFrameLoadedEvent.Notify(UniquePtr<MediaInfo>(new MediaInfo(Info())), + visibility); +} + +void MediaDecoderStateMachine::FinishDecodeFirstFrame() { + MOZ_ASSERT(OnTaskQueue()); + MOZ_ASSERT(!mSentFirstFrameLoadedEvent); + LOG("FinishDecodeFirstFrame"); + + mMediaSink->Redraw(Info().mVideo); + + LOG("Media duration %" PRId64 ", mediaSeekable=%d", + Duration().ToMicroseconds(), mMediaSeekable); + + // Get potentially updated metadata + mReader->ReadUpdatedMetadata(mInfo.ptr()); + + EnqueueFirstFrameLoadedEvent(); +} + +RefPtr<ShutdownPromise> MediaDecoderStateMachine::FinishShutdown() { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::FinishShutdown", + MEDIA_PLAYBACK); + MOZ_ASSERT(OnTaskQueue()); + LOG("Shutting down state machine task queue"); + return OwnerThread()->BeginShutdown(); +} + +void MediaDecoderStateMachine::RunStateMachine() { + MOZ_ASSERT(OnTaskQueue()); + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::RunStateMachine", + MEDIA_PLAYBACK); + mDelayedScheduler.Reset(); // Must happen on state machine task queue. + mDispatchedStateMachine = false; + mStateObj->Step(); +} + +void MediaDecoderStateMachine::ResetDecode(const TrackSet& aTracks) { + MOZ_ASSERT(OnTaskQueue()); + LOG("MediaDecoderStateMachine::Reset"); + + // Assert that aTracks specifies to reset the video track because we + // don't currently support resetting just the audio track. + MOZ_ASSERT(aTracks.contains(TrackInfo::kVideoTrack)); + + if (aTracks.contains(TrackInfo::kVideoTrack)) { + mDecodedVideoEndTime = TimeUnit::Zero(); + mVideoCompleted = false; + VideoQueue().Reset(); + mVideoDataRequest.DisconnectIfExists(); + mVideoWaitRequest.DisconnectIfExists(); + } + + if (aTracks.contains(TrackInfo::kAudioTrack)) { + mDecodedAudioEndTime = TimeUnit::Zero(); + mAudioCompleted = false; + AudioQueue().Reset(); + mAudioDataRequest.DisconnectIfExists(); + mAudioWaitRequest.DisconnectIfExists(); + } + + mReader->ResetDecode(aTracks); +} + +media::TimeUnit MediaDecoderStateMachine::GetClock( + TimeStamp* aTimeStamp) const { + MOZ_ASSERT(OnTaskQueue()); + auto clockTime = mMediaSink->GetPosition(aTimeStamp); + // This fails on Windows some times, see 1765563 +#if defined(XP_WIN) + NS_ASSERTION(GetMediaTime() <= clockTime, "Clock should go forwards."); +#else + MOZ_ASSERT(GetMediaTime() <= clockTime, "Clock should go forwards."); +#endif + return clockTime; +} + +void MediaDecoderStateMachine::UpdatePlaybackPositionPeriodically() { + MOZ_ASSERT(OnTaskQueue()); + + if (!IsPlaying()) { + return; + } + + // Cap the current time to the larger of the audio and video end time. + // This ensures that if we're running off the system clock, we don't + // advance the clock to after the media end time. + if (VideoEndTime() > TimeUnit::Zero() || AudioEndTime() > TimeUnit::Zero()) { + auto clockTime = GetClock(); + // Once looping was turned on, the time is probably larger than the duration + // of the media track, so the time over the end should be corrected. + AdjustByLooping(clockTime); + bool loopback = clockTime < GetMediaTime() && mLooping; + if (loopback && mBypassingSkipToNextKeyFrameCheck) { + LOG("media has looped back, no longer bypassing skip-to-next-key-frame"); + mBypassingSkipToNextKeyFrameCheck = false; + } + + // Skip frames up to the frame at the playback position, and figure out + // the time remaining until it's time to display the next frame and drop + // the current frame. + NS_ASSERTION(clockTime >= TimeUnit::Zero(), + "Should have positive clock time."); + + // These will be non -1 if we've displayed a video frame, or played an audio + // frame. + auto maxEndTime = std::max(VideoEndTime(), AudioEndTime()); + auto t = std::min(clockTime, maxEndTime); + // FIXME: Bug 1091422 - chained ogg files hit this assertion. + // MOZ_ASSERT(t >= GetMediaTime()); + if (loopback || t > GetMediaTime()) { + UpdatePlaybackPosition(t); + } + } + // Note we have to update playback position before releasing the monitor. + // Otherwise, MediaDecoder::AddOutputTrack could kick in when we are outside + // the monitor and get a staled value from GetCurrentTimeUs() which hits the + // assertion in GetClock(). + + int64_t delay = std::max<int64_t>( + 1, static_cast<int64_t>(AUDIO_DURATION_USECS / mPlaybackRate)); + ScheduleStateMachineIn(TimeUnit::FromMicroseconds(delay)); + + // Notify the listener as we progress in the playback offset. Note it would + // be too intensive to send notifications for each popped audio/video sample. + // It is good enough to send 'PlaybackProgressed' events every 40us (defined + // by AUDIO_DURATION_USECS), and we ensure 'PlaybackProgressed' events are + // always sent after 'PlaybackStarted' and before 'PlaybackStopped'. + mOnPlaybackEvent.Notify(MediaPlaybackEvent{ + MediaPlaybackEvent::PlaybackProgressed, mPlaybackOffset}); +} + +void MediaDecoderStateMachine::ScheduleStateMachine() { + MOZ_ASSERT(OnTaskQueue()); + if (mDispatchedStateMachine) { + return; + } + mDispatchedStateMachine = true; + + nsresult rv = OwnerThread()->Dispatch( + NewRunnableMethod("MediaDecoderStateMachine::RunStateMachine", this, + &MediaDecoderStateMachine::RunStateMachine)); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; +} + +void MediaDecoderStateMachine::ScheduleStateMachineIn(const TimeUnit& aTime) { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::ScheduleStateMachineIn", + MEDIA_PLAYBACK); + MOZ_ASSERT(OnTaskQueue()); // mDelayedScheduler.Ensure() may Disconnect() + // the promise, which must happen on the state + // machine task queue. + MOZ_ASSERT(aTime > TimeUnit::Zero()); + if (mDispatchedStateMachine) { + return; + } + + TimeStamp target = TimeStamp::Now() + aTime.ToTimeDuration(); + + // It is OK to capture 'this' without causing UAF because the callback + // always happens before shutdown. + RefPtr<MediaDecoderStateMachine> self = this; + mDelayedScheduler.Ensure( + target, + [self]() { + self->mDelayedScheduler.CompleteRequest(); + self->RunStateMachine(); + }, + []() { MOZ_DIAGNOSTIC_ASSERT(false); }); +} + +bool MediaDecoderStateMachine::IsStateMachineScheduled() const { + MOZ_ASSERT(OnTaskQueue()); + return mDispatchedStateMachine || mDelayedScheduler.IsScheduled(); +} + +void MediaDecoderStateMachine::SetPlaybackRate(double aPlaybackRate) { + MOZ_ASSERT(OnTaskQueue()); + MOZ_ASSERT(aPlaybackRate != 0, "Should be handled by MediaDecoder::Pause()"); + + mPlaybackRate = aPlaybackRate; + mMediaSink->SetPlaybackRate(mPlaybackRate); + + // Schedule next cycle to check if we can stop prerolling. + ScheduleStateMachine(); +} + +void MediaDecoderStateMachine::PreservesPitchChanged() { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::PreservesPitchChanged", + MEDIA_PLAYBACK); + MOZ_ASSERT(OnTaskQueue()); + mMediaSink->SetPreservesPitch(mPreservesPitch); +} + +void MediaDecoderStateMachine::LoopingChanged() { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::LoopingChanged", + MEDIA_PLAYBACK); + MOZ_ASSERT(OnTaskQueue()); + LOGV("LoopingChanged, looping=%d", mLooping.Ref()); + PROFILER_MARKER_TEXT("MDSM::LoopingChanged", MEDIA_PLAYBACK, {}, + mLooping ? "true"_ns : "false"_ns); + if (mSeamlessLoopingAllowed) { + mStateObj->HandleLoopingChanged(); + } +} + +void MediaDecoderStateMachine::StreamNameChanged() { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::StreamNameChanged", + MEDIA_PLAYBACK); + MOZ_ASSERT(OnTaskQueue()); + + mMediaSink->SetStreamName(mStreamName); +} + +void MediaDecoderStateMachine::UpdateOutputCaptured() { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::UpdateOutputCaptured", + MEDIA_PLAYBACK); + MOZ_ASSERT(OnTaskQueue()); + MOZ_ASSERT_IF( + mOutputCaptureState == MediaDecoder::OutputCaptureState::Capture, + mOutputDummyTrack.Ref()); + + // Reset these flags so they are consistent with the status of the sink. + // TODO: Move these flags into MediaSink to improve cohesion so we don't need + // to reset these flags when switching MediaSinks. + mAudioCompleted = false; + mVideoCompleted = false; + + // Don't create a new media sink if we're still suspending media sink. + if (!mIsMediaSinkSuspended) { + const bool wasPlaying = IsPlaying(); + // Stop and shut down the existing sink. + StopMediaSink(); + mMediaSink->Shutdown(); + + // Create a new sink according to whether output is captured. + mMediaSink = CreateMediaSink(); + if (wasPlaying) { + DebugOnly<nsresult> rv = StartMediaSink(); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + } + } + + // Don't buffer as much when audio is captured because we don't need to worry + // about high latency audio devices. + mAmpleAudioThreshold = + mOutputCaptureState != MediaDecoder::OutputCaptureState::None + ? detail::AMPLE_AUDIO_THRESHOLD / 2 + : detail::AMPLE_AUDIO_THRESHOLD; + + mStateObj->HandleAudioCaptured(); +} + +void MediaDecoderStateMachine::OutputPrincipalChanged() { + MOZ_ASSERT(OnTaskQueue()); + mCanonicalOutputPrincipal = mOutputPrincipal; +} + +RefPtr<GenericPromise> MediaDecoderStateMachine::InvokeSetSink( + const RefPtr<AudioDeviceInfo>& aSink) { + MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(aSink); + + return InvokeAsync(OwnerThread(), this, __func__, + &MediaDecoderStateMachine::SetSink, aSink); +} + +RefPtr<GenericPromise> MediaDecoderStateMachine::SetSink( + RefPtr<AudioDeviceInfo> aDevice) { + MOZ_ASSERT(OnTaskQueue()); + if (mIsMediaSinkSuspended) { + // Don't create a new media sink when suspended. + return GenericPromise::CreateAndResolve(true, __func__); + } + + return mMediaSink->SetAudioDevice(std::move(aDevice)); +} + +void MediaDecoderStateMachine::InvokeSuspendMediaSink() { + MOZ_ASSERT(NS_IsMainThread()); + + nsresult rv = OwnerThread()->Dispatch( + NewRunnableMethod("MediaDecoderStateMachine::SuspendMediaSink", this, + &MediaDecoderStateMachine::SuspendMediaSink)); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; +} + +void MediaDecoderStateMachine::SuspendMediaSink() { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::SuspendMediaSink", + MEDIA_PLAYBACK); + MOZ_ASSERT(OnTaskQueue()); + if (mIsMediaSinkSuspended) { + return; + } + LOG("SuspendMediaSink"); + mIsMediaSinkSuspended = true; + StopMediaSink(); + mMediaSink->Shutdown(); +} + +void MediaDecoderStateMachine::InvokeResumeMediaSink() { + MOZ_ASSERT(NS_IsMainThread()); + + nsresult rv = OwnerThread()->Dispatch( + NewRunnableMethod("MediaDecoderStateMachine::ResumeMediaSink", this, + &MediaDecoderStateMachine::ResumeMediaSink)); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; +} + +void MediaDecoderStateMachine::ResumeMediaSink() { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::ResumeMediaSink", + MEDIA_PLAYBACK); + MOZ_ASSERT(OnTaskQueue()); + if (!mIsMediaSinkSuspended) { + return; + } + LOG("ResumeMediaSink"); + mIsMediaSinkSuspended = false; + if (!mMediaSink->IsStarted()) { + mMediaSink = CreateMediaSink(); + MaybeStartPlayback(); + } +} + +void MediaDecoderStateMachine::UpdateSecondaryVideoContainer() { + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::UpdateSecondaryVideoContainer", + MEDIA_PLAYBACK); + MOZ_ASSERT(OnTaskQueue()); + MOZ_DIAGNOSTIC_ASSERT(mMediaSink); + mMediaSink->SetSecondaryVideoContainer(mSecondaryVideoContainer.Ref()); + mOnSecondaryVideoContainerInstalled.Notify(mSecondaryVideoContainer.Ref()); +} + +TimeUnit MediaDecoderStateMachine::AudioEndTime() const { + MOZ_ASSERT(OnTaskQueue()); + if (mMediaSink->IsStarted()) { + return mMediaSink->GetEndTime(TrackInfo::kAudioTrack); + } + return GetMediaTime(); +} + +TimeUnit MediaDecoderStateMachine::VideoEndTime() const { + MOZ_ASSERT(OnTaskQueue()); + if (mMediaSink->IsStarted()) { + return mMediaSink->GetEndTime(TrackInfo::kVideoTrack); + } + return GetMediaTime(); +} + +void MediaDecoderStateMachine::OnMediaSinkVideoComplete() { + MOZ_ASSERT(OnTaskQueue()); + MOZ_ASSERT(HasVideo()); + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::OnMediaSinkVideoComplete", + MEDIA_PLAYBACK); + LOG("[%s]", __func__); + + mMediaSinkVideoEndedPromise.Complete(); + mVideoCompleted = true; + ScheduleStateMachine(); +} + +void MediaDecoderStateMachine::OnMediaSinkVideoError() { + MOZ_ASSERT(OnTaskQueue()); + MOZ_ASSERT(HasVideo()); + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::OnMediaSinkVideoError", + MEDIA_PLAYBACK); + LOGE("[%s]", __func__); + + mMediaSinkVideoEndedPromise.Complete(); + mVideoCompleted = true; + if (HasAudio()) { + return; + } + DecodeError(MediaResult(NS_ERROR_DOM_MEDIA_MEDIASINK_ERR, __func__)); +} + +void MediaDecoderStateMachine::OnMediaSinkAudioComplete() { + MOZ_ASSERT(OnTaskQueue()); + MOZ_ASSERT(HasAudio()); + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::OnMediaSinkAudioComplete", + MEDIA_PLAYBACK); + LOG("[%s]", __func__); + + mMediaSinkAudioEndedPromise.Complete(); + mAudioCompleted = true; + // To notify PlaybackEnded as soon as possible. + ScheduleStateMachine(); + + // Report OK to Decoder Doctor (to know if issue may have been resolved). + mOnDecoderDoctorEvent.Notify( + DecoderDoctorEvent{DecoderDoctorEvent::eAudioSinkStartup, NS_OK}); +} + +void MediaDecoderStateMachine::OnMediaSinkAudioError(nsresult aResult) { + MOZ_ASSERT(OnTaskQueue()); + MOZ_ASSERT(HasAudio()); + AUTO_PROFILER_LABEL("MediaDecoderStateMachine::OnMediaSinkAudioError", + MEDIA_PLAYBACK); + LOGE("[%s]", __func__); + + mMediaSinkAudioEndedPromise.Complete(); + mAudioCompleted = true; + + // Result should never be NS_OK in this *error* handler. Report to Dec-Doc. + MOZ_ASSERT(NS_FAILED(aResult)); + mOnDecoderDoctorEvent.Notify( + DecoderDoctorEvent{DecoderDoctorEvent::eAudioSinkStartup, aResult}); + + // Make the best effort to continue playback when there is video. + if (HasVideo()) { + return; + } + + // Otherwise notify media decoder/element about this error for it makes + // no sense to play an audio-only file without sound output. + DecodeError(MediaResult(NS_ERROR_DOM_MEDIA_MEDIASINK_ERR, __func__)); +} + +uint32_t MediaDecoderStateMachine::GetAmpleVideoFrames() const { + MOZ_ASSERT(OnTaskQueue()); + return mReader->VideoIsHardwareAccelerated() + ? std::max<uint32_t>(sVideoQueueHWAccelSize, MIN_VIDEO_QUEUE_SIZE) + : std::max<uint32_t>(sVideoQueueDefaultSize, MIN_VIDEO_QUEUE_SIZE); +} + +void MediaDecoderStateMachine::GetDebugInfo( + dom::MediaDecoderStateMachineDebugInfo& aInfo) { + MOZ_ASSERT(OnTaskQueue()); + aInfo.mDuration = + mDuration.Ref() ? mDuration.Ref().ref().ToMicroseconds() : -1; + aInfo.mMediaTime = GetMediaTime().ToMicroseconds(); + aInfo.mClock = mMediaSink->IsStarted() ? GetClock().ToMicroseconds() : -1; + aInfo.mPlayState = int32_t(mPlayState.Ref()); + aInfo.mSentFirstFrameLoadedEvent = mSentFirstFrameLoadedEvent; + aInfo.mIsPlaying = IsPlaying(); + CopyUTF8toUTF16(MakeStringSpan(AudioRequestStatus()), + aInfo.mAudioRequestStatus); + CopyUTF8toUTF16(MakeStringSpan(VideoRequestStatus()), + aInfo.mVideoRequestStatus); + aInfo.mDecodedAudioEndTime = mDecodedAudioEndTime.ToMicroseconds(); + aInfo.mDecodedVideoEndTime = mDecodedVideoEndTime.ToMicroseconds(); + aInfo.mAudioCompleted = mAudioCompleted; + aInfo.mVideoCompleted = mVideoCompleted; + mStateObj->GetDebugInfo(aInfo.mStateObj); + mMediaSink->GetDebugInfo(aInfo.mMediaSink); +} + +RefPtr<GenericPromise> MediaDecoderStateMachine::RequestDebugInfo( + dom::MediaDecoderStateMachineDebugInfo& aInfo) { + RefPtr<GenericPromise::Private> p = new GenericPromise::Private(__func__); + RefPtr<MediaDecoderStateMachine> self = this; + nsresult rv = OwnerThread()->Dispatch( + NS_NewRunnableFunction("MediaDecoderStateMachine::RequestDebugInfo", + [self, p, &aInfo]() { + self->GetDebugInfo(aInfo); + p->Resolve(true, __func__); + }), + AbstractThread::TailDispatch); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; + return p; +} + +class VideoQueueMemoryFunctor : public nsDequeFunctor<VideoData> { + public: + VideoQueueMemoryFunctor() : mSize(0) {} + + MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf); + + virtual void operator()(VideoData* aObject) override { + mSize += aObject->SizeOfIncludingThis(MallocSizeOf); + } + + size_t mSize; +}; + +class AudioQueueMemoryFunctor : public nsDequeFunctor<AudioData> { + public: + AudioQueueMemoryFunctor() : mSize(0) {} + + MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf); + + virtual void operator()(AudioData* aObject) override { + mSize += aObject->SizeOfIncludingThis(MallocSizeOf); + } + + size_t mSize; +}; + +size_t MediaDecoderStateMachine::SizeOfVideoQueue() const { + VideoQueueMemoryFunctor functor; + mVideoQueue.LockedForEach(functor); + return functor.mSize; +} + +size_t MediaDecoderStateMachine::SizeOfAudioQueue() const { + AudioQueueMemoryFunctor functor; + mAudioQueue.LockedForEach(functor); + return functor.mSize; +} + +const char* MediaDecoderStateMachine::AudioRequestStatus() const { + MOZ_ASSERT(OnTaskQueue()); + if (IsRequestingAudioData()) { + MOZ_DIAGNOSTIC_ASSERT(!IsWaitingAudioData()); + return "pending"; + } + + if (IsWaitingAudioData()) { + return "waiting"; + } + return "idle"; +} + +const char* MediaDecoderStateMachine::VideoRequestStatus() const { + MOZ_ASSERT(OnTaskQueue()); + if (IsRequestingVideoData()) { + MOZ_DIAGNOSTIC_ASSERT(!IsWaitingVideoData()); + return "pending"; + } + + if (IsWaitingVideoData()) { + return "waiting"; + } + return "idle"; +} + +void MediaDecoderStateMachine::OnSuspendTimerResolved() { + LOG("OnSuspendTimerResolved"); + mVideoDecodeSuspendTimer.CompleteRequest(); + mStateObj->HandleVideoSuspendTimeout(); +} + +void MediaDecoderStateMachine::CancelSuspendTimer() { + LOG("CancelSuspendTimer: State: %s, Timer.IsScheduled: %c", + ToStateStr(mStateObj->GetState()), + mVideoDecodeSuspendTimer.IsScheduled() ? 'T' : 'F'); + MOZ_ASSERT(OnTaskQueue()); + if (mVideoDecodeSuspendTimer.IsScheduled()) { + mOnPlaybackEvent.Notify(MediaPlaybackEvent::CancelVideoSuspendTimer); + } + mVideoDecodeSuspendTimer.Reset(); +} + +void MediaDecoderStateMachine::AdjustByLooping(media::TimeUnit& aTime) const { + MOZ_ASSERT(OnTaskQueue()); + + // No need to adjust time. + if (mOriginalDecodedDuration == media::TimeUnit::Zero()) { + return; + } + + // There are situations where we need to perform subtraction instead of modulo + // to accurately adjust the clock. When we are not in a state of seamless + // looping, it is usually necessary to normalize the clock time within the + // range of [0, duration]. However, if the current clock time is greater than + // the duration (i.e., duration+1) and not in looping, we should not adjust it + // to 1 as we are not looping back to the starting position. Instead, we + // should leave the clock time unchanged and trim it later to match the + // maximum duration time. + if (mStateObj->GetState() != DECODER_STATE_LOOPING_DECODING) { + // Use the smaller offset rather than the larger one, as the larger offset + // indicates the next round of looping. For example, if the duration is X + // and the playback is currently in the third round of looping, both + // queues will have an offset of 3X. However, if the audio decoding is + // faster and the fourth round of data has already been added to the audio + // queue, the audio offset will become 4X. Since playback is still in the + // third round, we should use the smaller offset of 3X to adjust the time. + TimeUnit offset = TimeUnit::FromInfinity(); + if (HasAudio()) { + offset = std::min(AudioQueue().GetOffset(), offset); + } + if (HasVideo()) { + offset = std::min(VideoQueue().GetOffset(), offset); + } + if (aTime > offset) { + aTime -= offset; + return; + } + } + + // When seamless looping happens at least once, it doesn't matter if we're + // looping or not. + aTime = aTime % mOriginalDecodedDuration; +} + +bool MediaDecoderStateMachine::IsInSeamlessLooping() const { + return mLooping && mSeamlessLoopingAllowed; +} + +bool MediaDecoderStateMachine::HasLastDecodedData(MediaData::Type aType) { + MOZ_DIAGNOSTIC_ASSERT(aType == MediaData::Type::AUDIO_DATA || + aType == MediaData::Type::VIDEO_DATA); + if (aType == MediaData::Type::AUDIO_DATA) { + return mDecodedAudioEndTime != TimeUnit::Zero(); + } + return mDecodedVideoEndTime != TimeUnit::Zero(); +} + +bool MediaDecoderStateMachine::IsCDMProxySupported(CDMProxy* aProxy) { +#ifdef MOZ_WMF_CDM + MOZ_ASSERT(aProxy); + // This proxy only works with the external state machine. + return !aProxy->AsWMFCDMProxy(); +#else + return true; +#endif +} + +} // namespace mozilla + +// avoid redefined macro in unified build +#undef LOG +#undef LOGV +#undef LOGW +#undef LOGE +#undef SLOGW +#undef SLOGE +#undef NS_DispatchToMainThread |