From deb3a053d8303b69333220d13f4757ea35e3c646 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 26 Jun 2024 08:16:28 +0200 Subject: Merging upstream version 127.0.2. Signed-off-by: Daniel Baumann --- dom/media/mediasource/TrackBuffersManager.cpp | 42 ++++++++++++++++++++++++--- dom/media/mediasource/TrackBuffersManager.h | 2 ++ dom/media/webm/NesteggPacketHolder.h | 20 +++++++++++-- dom/media/webm/WebMDemuxer.cpp | 27 +++++++++++++++-- dom/media/webm/WebMDemuxer.h | 6 +++- 5 files changed, 87 insertions(+), 10 deletions(-) (limited to 'dom') diff --git a/dom/media/mediasource/TrackBuffersManager.cpp b/dom/media/mediasource/TrackBuffersManager.cpp index c87fa239f1..58ba7c250b 100644 --- a/dom/media/mediasource/TrackBuffersManager.cpp +++ b/dom/media/mediasource/TrackBuffersManager.cpp @@ -922,7 +922,11 @@ void TrackBuffersManager::SegmentParserLoop() { if (mNewMediaSegmentStarted) { if (NS_SUCCEEDED(newData) && mLastParsedEndTime.isSome() && start < mLastParsedEndTime.ref()) { - MSE_DEBUG("Re-creating demuxer"); + MSE_DEBUG("Re-creating demuxer, new start (%" PRId64 + ") is smaller than last parsed end time (%" PRId64 ")", + start.ToMicroseconds(), + mLastParsedEndTime->ToMicroseconds()); + mFrameEndTimeBeforeRecreateDemuxer = Some(end); ResetDemuxingState(); return; } @@ -1038,8 +1042,15 @@ void TrackBuffersManager::CreateDemuxerforMIMEType() { if (mType.Type() == MEDIAMIMETYPE(VIDEO_WEBM) || mType.Type() == MEDIAMIMETYPE(AUDIO_WEBM)) { - mInputDemuxer = - new WebMDemuxer(mCurrentInputBuffer, true /* IsMediaSource*/); + if (mFrameEndTimeBeforeRecreateDemuxer) { + MSE_DEBUG( + "CreateDemuxerFromMimeType: " + "mFrameEndTimeBeforeRecreateDemuxer=%" PRId64, + mFrameEndTimeBeforeRecreateDemuxer->ToMicroseconds()); + } + mInputDemuxer = new WebMDemuxer(mCurrentInputBuffer, true, + mFrameEndTimeBeforeRecreateDemuxer); + mFrameEndTimeBeforeRecreateDemuxer.reset(); DDLINKCHILD("demuxer", mInputDemuxer.get()); return; } @@ -1047,6 +1058,7 @@ void TrackBuffersManager::CreateDemuxerforMIMEType() { if (mType.Type() == MEDIAMIMETYPE(VIDEO_MP4) || mType.Type() == MEDIAMIMETYPE(AUDIO_MP4)) { mInputDemuxer = new MP4Demuxer(mCurrentInputBuffer); + mFrameEndTimeBeforeRecreateDemuxer.reset(); DDLINKCHILD("demuxer", mInputDemuxer.get()); return; } @@ -1625,9 +1637,11 @@ void TrackBuffersManager::MaybeDispatchEncryptedEvent( void TrackBuffersManager::OnVideoDemuxCompleted( const RefPtr& aSamples) { mTaskQueueCapability->AssertOnCurrentThread(); - MSE_DEBUG("%zu video samples demuxed", aSamples->GetSamples().Length()); mVideoTracks.mDemuxRequest.Complete(); mVideoTracks.mQueuedSamples.AppendElements(aSamples->GetSamples()); + MSE_DEBUG("%zu video samples demuxed, queued-sz=%zu", + aSamples->GetSamples().Length(), + mVideoTracks.mQueuedSamples.Length()); MaybeDispatchEncryptedEvent(aSamples->GetSamples()); DoDemuxAudio(); @@ -1892,6 +1906,9 @@ void TrackBuffersManager::ProcessFrames(TrackBuffer& aSamples, // coded frame. if (!sample->mKeyframe) { previouslyDroppedSample = nullptr; + SAMPLE_DEBUGV("skipping sample [%" PRId64 ",%" PRId64 "]", + sample->mTime.ToMicroseconds(), + sample->GetEndTime().ToMicroseconds()); continue; } // 2. Set the need random access point flag on track buffer to false. @@ -1971,6 +1988,7 @@ void TrackBuffersManager::ProcessFrames(TrackBuffer& aSamples, // 4. Unset the highest end timestamp on all track buffers. // 5. Set the need random access point flag on all track buffers to // true. + MSE_DEBUG("Resetting append state"); track->ResetAppendState(); } // 6. Jump to the Loop Top step above to restart processing of the current @@ -2553,10 +2571,12 @@ void TrackBuffersManager::RecreateParser(bool aReuseInitData) { mParser = ContainerParser::CreateForMIMEType(mType); DDLINKCHILD("parser", mParser.get()); if (aReuseInitData && mInitData) { + MSE_DEBUG("Using existing init data to reset parser"); TimeUnit start, end; mParser->ParseStartAndEndTimestamps(MediaSpan(mInitData), start, end); mProcessedInput = mInitData->Length(); } else { + MSE_DEBUG("Resetting parser, not reusing init data"); mProcessedInput = 0; } } @@ -2843,12 +2863,20 @@ const MediaRawData* TrackBuffersManager::GetSample(TrackInfo::TrackType aTrack, const TrackBuffer& track = GetTrackBuffer(aTrack); if (aIndex >= track.Length()) { + MSE_DEBUGV( + "Can't get sample due to reaching to the end, index=%u, " + "length=%zu", + aIndex, track.Length()); // reached the end. return nullptr; } if (!(aExpectedDts + aFuzz).IsValid() || !(aExpectedPts + aFuzz).IsValid()) { // Time overflow, it seems like we also reached the end. + MSE_DEBUGV("Can't get sample due to time overflow, expectedPts=%" PRId64 + ", aExpectedDts=%" PRId64 ", fuzz=%" PRId64, + aExpectedPts.ToMicroseconds(), aExpectedPts.ToMicroseconds(), + aFuzz.ToMicroseconds()); return nullptr; } @@ -2859,6 +2887,12 @@ const MediaRawData* TrackBuffersManager::GetSample(TrackInfo::TrackType aTrack, return sample; } + MSE_DEBUGV("Can't get sample due to big gap, sample=%" PRId64 + ", expectedPts=%" PRId64 ", aExpectedDts=%" PRId64 + ", fuzz=%" PRId64, + sample->mTime.ToMicroseconds(), aExpectedPts.ToMicroseconds(), + aExpectedPts.ToMicroseconds(), aFuzz.ToMicroseconds()); + // Gap is too big. End of Stream or Waiting for Data. // TODO, check that we have continuous data based on the sanitized buffered // range instead. diff --git a/dom/media/mediasource/TrackBuffersManager.h b/dom/media/mediasource/TrackBuffersManager.h index c72f2f5453..0e9c857586 100644 --- a/dom/media/mediasource/TrackBuffersManager.h +++ b/dom/media/mediasource/TrackBuffersManager.h @@ -568,6 +568,8 @@ class TrackBuffersManager final // mTaskQueue. However, there's special locking around mTaskQueue, so we keep // both for now. Maybe> mTaskQueueCapability; + + Maybe mFrameEndTimeBeforeRecreateDemuxer; }; } // namespace mozilla diff --git a/dom/media/webm/NesteggPacketHolder.h b/dom/media/webm/NesteggPacketHolder.h index 7c74f752d3..e1e7822106 100644 --- a/dom/media/webm/NesteggPacketHolder.h +++ b/dom/media/webm/NesteggPacketHolder.h @@ -25,11 +25,12 @@ class NesteggPacketHolder { mOffset(-1), mTimestamp(-1), mDuration(-1), + mDefaultDuration(-1), mTrack(0), mIsKeyframe(false) {} - bool Init(nestegg_packet* aPacket, int64_t aOffset, unsigned aTrack, - bool aIsKeyframe) { + bool Init(nestegg_packet* aPacket, nestegg* aContext, int64_t aOffset, + unsigned aTrack, bool aIsKeyframe) { uint64_t timestamp_ns; if (nestegg_packet_tstamp(aPacket, ×tamp_ns) == -1) { return false; @@ -47,6 +48,10 @@ class NesteggPacketHolder { if (!nestegg_packet_duration(aPacket, &duration_ns)) { mDuration = duration_ns / 1000; } + if (!nestegg_track_default_duration(aContext, mTrack, &duration_ns)) { + mDefaultDuration = duration_ns / 1000; + } + return true; } @@ -66,6 +71,10 @@ class NesteggPacketHolder { MOZ_ASSERT(IsInitialized()); return mDuration; } + int64_t DefaultDuration() const { + MOZ_ASSERT(IsInitialized()); + return mDefaultDuration; + } unsigned Track() { MOZ_ASSERT(IsInitialized()); return mTrack; @@ -78,7 +87,7 @@ class NesteggPacketHolder { private: ~NesteggPacketHolder() { nestegg_free_packet(mPacket); } - bool IsInitialized() { return mOffset >= 0; } + bool IsInitialized() const { return mOffset >= 0; } nestegg_packet* mPacket; @@ -90,8 +99,13 @@ class NesteggPacketHolder { int64_t mTimestamp; // Packet duration in microseconds; -1 if unknown or retrieval failed. + // https://www.webmproject.org/docs/container/#BlockDuration int64_t mDuration; + // Default durtaion in microseconds; -1 if unknown or retrieval failed. + // https://www.webmproject.org/docs/container/#Duration + int64_t mDefaultDuration; + // Track ID. unsigned mTrack; diff --git a/dom/media/webm/WebMDemuxer.cpp b/dom/media/webm/WebMDemuxer.cpp index da14118205..48d3d57ed9 100644 --- a/dom/media/webm/WebMDemuxer.cpp +++ b/dom/media/webm/WebMDemuxer.cpp @@ -15,6 +15,7 @@ #include "gfx2DGlue.h" #include "gfxUtils.h" #include "mozilla/EndianUtils.h" +#include "mozilla/Maybe.h" #include "mozilla/SharedThreadPool.h" #include "MediaDataDemuxer.h" #include "nsAutoRef.h" @@ -150,7 +151,9 @@ int WebMDemuxer::NestEggContext::Init() { WebMDemuxer::WebMDemuxer(MediaResource* aResource) : WebMDemuxer(aResource, false) {} -WebMDemuxer::WebMDemuxer(MediaResource* aResource, bool aIsMediaSource) +WebMDemuxer::WebMDemuxer( + MediaResource* aResource, bool aIsMediaSource, + Maybe aFrameEndTimeBeforeRecreateDemuxer) : mVideoContext(this, aResource), mAudioContext(this, aResource), mBufferedState(nullptr), @@ -169,6 +172,14 @@ WebMDemuxer::WebMDemuxer(MediaResource* aResource, bool aIsMediaSource) // Audio/video contexts hold a MediaResourceIndex. DDLINKCHILD("video context", mVideoContext.GetResource()); DDLINKCHILD("audio context", mAudioContext.GetResource()); + + MOZ_ASSERT_IF(!aIsMediaSource, + aFrameEndTimeBeforeRecreateDemuxer.isNothing()); + if (aIsMediaSource && aFrameEndTimeBeforeRecreateDemuxer) { + mVideoFrameEndTimeBeforeReset = aFrameEndTimeBeforeRecreateDemuxer; + WEBM_DEBUG("Set mVideoFrameEndTimeBeforeReset=%" PRId64, + mVideoFrameEndTimeBeforeReset->ToMicroseconds()); + } } WebMDemuxer::~WebMDemuxer() { @@ -588,6 +599,12 @@ nsresult WebMDemuxer::GetNextPacket(TrackInfo::TrackType aType, } int64_t tstamp = holder->Timestamp(); int64_t duration = holder->Duration(); + int64_t defaultDuration = holder->DefaultDuration(); + if (aType == TrackInfo::TrackType::kVideoTrack) { + WEBM_DEBUG("GetNextPacket(video): tstamp=%" PRId64 ", duration=%" PRId64 + ", defaultDuration=%" PRId64, + tstamp, duration, defaultDuration); + } // The end time of this frame is the start time of the next frame. Fetch // the timestamp of the next packet for this track. If we've reached the @@ -610,6 +627,12 @@ nsresult WebMDemuxer::GetNextPacket(TrackInfo::TrackType aType, next_tstamp = tstamp + duration; } else if (lastFrameTime.isSome()) { next_tstamp = tstamp + (tstamp - lastFrameTime.ref()); + } else if (defaultDuration >= 0) { + next_tstamp = tstamp + defaultDuration; + } else if (mVideoFrameEndTimeBeforeReset) { + WEBM_DEBUG("Setting next timestamp to be %" PRId64 " us", + mVideoFrameEndTimeBeforeReset->ToMicroseconds()); + next_tstamp = mVideoFrameEndTimeBeforeReset->ToMicroseconds(); } else if (mIsMediaSource) { (this->*pushPacket)(holder); } else { @@ -930,7 +953,7 @@ nsresult WebMDemuxer::DemuxPacket(TrackInfo::TrackType aType, int64_t offset = Resource(aType).Tell(); RefPtr holder = new NesteggPacketHolder(); - if (!holder->Init(packet, offset, track, false)) { + if (!holder->Init(packet, Context(aType), offset, track, false)) { WEBM_DEBUG("NesteggPacketHolder::Init: error"); return NS_ERROR_DOM_MEDIA_DEMUXER_ERR; } diff --git a/dom/media/webm/WebMDemuxer.h b/dom/media/webm/WebMDemuxer.h index 3b3bdc21e2..28407908a9 100644 --- a/dom/media/webm/WebMDemuxer.h +++ b/dom/media/webm/WebMDemuxer.h @@ -94,7 +94,9 @@ class WebMDemuxer : public MediaDataDemuxer, explicit WebMDemuxer(MediaResource* aResource); // Indicate if the WebMDemuxer is to be used with MediaSource. In which // case the demuxer will stop reads to the last known complete block. - WebMDemuxer(MediaResource* aResource, bool aIsMediaSource); + WebMDemuxer( + MediaResource* aResource, bool aIsMediaSource, + Maybe aFrameEndTimeBeforeRecreateDemuxer = Nothing()); RefPtr Init() override; @@ -223,6 +225,8 @@ class WebMDemuxer : public MediaDataDemuxer, Maybe mLastAudioFrameTime; Maybe mLastVideoFrameTime; + Maybe mVideoFrameEndTimeBeforeReset; + // Codec ID of audio track int mAudioCodec; // Codec ID of video track -- cgit v1.2.3