diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
commit | 26a029d407be480d791972afb5975cf62c9360a6 (patch) | |
tree | f435a8308119effd964b339f76abb83a57c29483 /gfx/layers/composite/ImageComposite.cpp | |
parent | Initial commit. (diff) | |
download | firefox-upstream/124.0.1.tar.xz firefox-upstream/124.0.1.zip |
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'gfx/layers/composite/ImageComposite.cpp')
-rw-r--r-- | gfx/layers/composite/ImageComposite.cpp | 378 |
1 files changed, 378 insertions, 0 deletions
diff --git a/gfx/layers/composite/ImageComposite.cpp b/gfx/layers/composite/ImageComposite.cpp new file mode 100644 index 0000000000..7c36519843 --- /dev/null +++ b/gfx/layers/composite/ImageComposite.cpp @@ -0,0 +1,378 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ImageComposite.h" + +#include <inttypes.h> + +#include "mozilla/ProfilerMarkers.h" +#include "gfxPlatform.h" +#include "nsPrintfCString.h" + +namespace mozilla { + +using namespace gfx; + +namespace layers { + +/* static */ const float ImageComposite::BIAS_TIME_MS = 1.0f; + +ImageComposite::ImageComposite() = default; + +ImageComposite::~ImageComposite() = default; + +TimeStamp ImageComposite::GetBiasedTime(const TimeStamp& aInput) const { + switch (mBias) { + case ImageComposite::BIAS_NEGATIVE: + return aInput - TimeDuration::FromMilliseconds(BIAS_TIME_MS); + case ImageComposite::BIAS_POSITIVE: + return aInput + TimeDuration::FromMilliseconds(BIAS_TIME_MS); + default: + return aInput; + } +} + +void ImageComposite::UpdateBias(size_t aImageIndex, bool aFrameChanged) { + MOZ_ASSERT(aImageIndex < ImagesCount()); + + TimeStamp compositionTime = GetCompositionTime(); + TimeStamp compositedImageTime = mImages[aImageIndex].mTimeStamp; + TimeStamp nextImageTime = aImageIndex + 1 < ImagesCount() + ? mImages[aImageIndex + 1].mTimeStamp + : TimeStamp(); + + if (profiler_thread_is_being_profiled_for_markers() && compositedImageTime && + nextImageTime) { + TimeDuration offsetCurrent = compositedImageTime - compositionTime; + TimeDuration offsetNext = nextImageTime - compositionTime; + nsPrintfCString str("current %.2lfms, next %.2lfms", + offsetCurrent.ToMilliseconds(), + offsetNext.ToMilliseconds()); + PROFILER_MARKER_TEXT("Video frame offsets", GRAPHICS, {}, str); + } + + if (compositedImageTime.IsNull()) { + mBias = ImageComposite::BIAS_NONE; + return; + } + TimeDuration threshold = TimeDuration::FromMilliseconds(1.5); + if (compositionTime - compositedImageTime < threshold && + compositionTime - compositedImageTime > -threshold) { + // The chosen frame's time is very close to the composition time (probably + // just before the current composition time, but due to previously set + // negative bias, it could be just after the current composition time too). + // If the inter-frame time is almost exactly equal to (a multiple of) + // the inter-composition time, then we're in a dangerous situation because + // jitter might cause frames to fall one side or the other of the + // composition times, causing many frames to be skipped or duplicated. + // Try to prevent that by adding a negative bias to the frame times during + // the next composite; that should ensure the next frame's time is treated + // as falling just before a composite time. + mBias = ImageComposite::BIAS_NEGATIVE; + return; + } + if (!nextImageTime.IsNull() && nextImageTime - compositionTime < threshold && + nextImageTime - compositionTime > -threshold) { + // The next frame's time is very close to our composition time (probably + // just after the current composition time, but due to previously set + // positive bias, it could be just before the current composition time too). + // We're in a dangerous situation because jitter might cause frames to + // fall one side or the other of the composition times, causing many frames + // to be skipped or duplicated. + // Specifically, the next composite is at risk of picking the "next + 1" + // frame rather than the "next" frame, which would cause the "next" frame to + // be skipped. Try to prevent that by adding a positive bias to the frame + // times during the next composite; if the inter-frame time is almost + // exactly equal to the inter-composition time, that should ensure that the + // next + 1 frame falls just *after* the next composition time, and the next + // composite should then pick the next frame rather than the next + 1 frame. + mBias = ImageComposite::BIAS_POSITIVE; + return; + } + if (aFrameChanged) { + // The current and next video frames are a sufficient distance from the + // composition time and we can reliably pick the right frame without bias. + // Reset the bias. + // We only do this when the frame changed. Otherwise, when playing a 30fps + // video on a 60fps display, we'd keep resetting the bias during the "middle + // frames". + mBias = ImageComposite::BIAS_NONE; + } +} + +int ImageComposite::ChooseImageIndex() { + // ChooseImageIndex is called for all images in the layer when it is visible. + // Change to this behaviour would break dropped frames counting calculation: + // We rely on this assumption to determine if during successive runs an + // image is returned that isn't the one following immediately the previous one + if (mImages.IsEmpty()) { + return -1; + } + + TimeStamp compositionTime = GetCompositionTime(); + auto compositionOpportunityId = GetCompositionOpportunityId(); + if (compositionTime && + compositionOpportunityId != mLastChooseImageIndexComposition) { + // We are inside a composition, in the first call to ChooseImageIndex during + // this composition. + // Find the newest frame whose biased timestamp is at or before + // `compositionTime`. + uint32_t imageIndex = 0; + while (imageIndex + 1 < mImages.Length() && + mImages[imageIndex + 1].mTextureHost->IsValid() && + GetBiasedTime(mImages[imageIndex + 1].mTimeStamp) <= + compositionTime) { + ++imageIndex; + } + + if (!mImages[imageIndex].mTextureHost->IsValid()) { + // Still not ready to be shown. + return -1; + } + + bool wasVisibleAtPreviousComposition = + compositionOpportunityId == mLastChooseImageIndexComposition.Next(); + + bool frameChanged = + UpdateCompositedFrame(imageIndex, wasVisibleAtPreviousComposition); + UpdateBias(imageIndex, frameChanged); + + mLastChooseImageIndexComposition = compositionOpportunityId; + + return imageIndex; + } + + // We've been called before during this composition, or we're not in a + // composition. Just return the last image we picked (if it's one of the + // current images). + for (uint32_t i = 0; i < mImages.Length(); ++i) { + if (mImages[i].mFrameID == mLastFrameID && + mImages[i].mProducerID == mLastProducerID) { + return i; + } + } + + return 0; +} + +const ImageComposite::TimedImage* ImageComposite::ChooseImage() { + int index = ChooseImageIndex(); + return index >= 0 ? &mImages[index] : nullptr; +} + +void ImageComposite::RemoveImagesWithTextureHost(TextureHost* aTexture) { + for (int32_t i = mImages.Length() - 1; i >= 0; --i) { + if (mImages[i].mTextureHost == aTexture) { + aTexture->UnbindTextureSource(); + mImages.RemoveElementAt(i); + } + } +} + +void ImageComposite::ClearImages() { mImages.Clear(); } + +void ImageComposite::SetImages(nsTArray<TimedImage>&& aNewImages) { + if (!aNewImages.IsEmpty()) { + DetectTimeStampJitter(&aNewImages[0]); + + // Frames older than the first frame in aNewImages that we haven't shown yet + // will never be shown. + CountSkippedFrames(&aNewImages[0]); + + if (profiler_thread_is_being_profiled_for_markers()) { + int len = aNewImages.Length(); + const auto& first = aNewImages[0]; + const auto& last = aNewImages.LastElement(); + nsPrintfCString str("%d %s, frameID %" PRId32 " (prod %" PRId32 + ") to frameID %" PRId32 " (prod %" PRId32 ")", + len, len == 1 ? "image" : "images", first.mFrameID, + first.mProducerID, last.mFrameID, last.mProducerID); + PROFILER_MARKER_TEXT("ImageComposite::SetImages", GRAPHICS, {}, str); + } + } + mImages = std::move(aNewImages); +} + +// Returns whether the frame changed. +bool ImageComposite::UpdateCompositedFrame( + int aImageIndex, bool aWasVisibleAtPreviousComposition) { + MOZ_RELEASE_ASSERT(aImageIndex >= 0); + MOZ_RELEASE_ASSERT(aImageIndex < static_cast<int>(mImages.Length())); + const TimedImage& image = mImages[aImageIndex]; + + auto compositionOpportunityId = GetCompositionOpportunityId(); + TimeStamp compositionTime = GetCompositionTime(); + MOZ_RELEASE_ASSERT(compositionTime, + "Should only be called during a composition"); + + nsCString descr; + if (profiler_thread_is_being_profiled_for_markers()) { + nsCString relativeTimeString; + if (image.mTimeStamp) { + relativeTimeString.AppendPrintf( + " [relative timestamp %.1lfms]", + (image.mTimeStamp - compositionTime).ToMilliseconds()); + } + int remainingImages = mImages.Length() - 1 - aImageIndex; + static const char* kBiasStrings[] = {"NONE", "NEGATIVE", "POSITIVE"}; + descr.AppendPrintf( + "frameID %" PRId32 " (producerID %" PRId32 ") [composite %" PRIu64 + "] [bias %s] [%d remaining %s]%s", + image.mFrameID, image.mProducerID, compositionOpportunityId.mId, + kBiasStrings[mBias], remainingImages, + remainingImages == 1 ? "image" : "images", relativeTimeString.get()); + if (mLastProducerID != image.mProducerID) { + descr.AppendPrintf(", previous producerID: %" PRId32, mLastProducerID); + } else if (mLastFrameID != image.mFrameID) { + descr.AppendPrintf(", previous frameID: %" PRId32, mLastFrameID); + } else { + descr.AppendLiteral(", no change"); + } + } + PROFILER_MARKER_TEXT("UpdateCompositedFrame", GRAPHICS, {}, descr); + + if (mLastFrameID == image.mFrameID && mLastProducerID == image.mProducerID) { + // The frame didn't change. + return false; + } + + CountSkippedFrames(&image); + + int32_t dropped = mSkippedFramesSinceLastComposite; + mSkippedFramesSinceLastComposite = 0; + + if (!aWasVisibleAtPreviousComposition) { + // This video was not part of the on-screen scene during the previous + // composition opportunity, for example it may have been scrolled off-screen + // or in a background tab, or compositing might have been paused. + // Ignore any skipped frames and don't count them as dropped. + dropped = 0; + } + + if (dropped > 0) { + mDroppedFrames += dropped; + if (profiler_thread_is_being_profiled_for_markers()) { + const char* frameOrFrames = dropped == 1 ? "frame" : "frames"; + nsPrintfCString text("%" PRId32 " %s dropped: %" PRId32 " -> %" PRId32 + " (producer %" PRId32 ")", + dropped, frameOrFrames, mLastFrameID, image.mFrameID, + mLastProducerID); + PROFILER_MARKER_TEXT("Video frames dropped", GRAPHICS, {}, text); + } + } + + mLastFrameID = image.mFrameID; + mLastProducerID = image.mProducerID; + mLastFrameUpdateComposition = compositionOpportunityId; + + return true; +} + +void ImageComposite::OnFinishRendering(int aImageIndex, + const TimedImage* aImage, + base::ProcessId aProcessId, + const CompositableHandle& aHandle) { + if (mLastFrameUpdateComposition != GetCompositionOpportunityId()) { + // The frame did not change in this composition. + return; + } + + if (aHandle) { + ImageCompositeNotificationInfo info; + info.mImageBridgeProcessId = aProcessId; + info.mNotification = ImageCompositeNotification( + aHandle, aImage->mTimeStamp, GetCompositionTime(), mLastFrameID, + mLastProducerID); + AppendImageCompositeNotification(info); + } +} + +const ImageComposite::TimedImage* ImageComposite::GetImage( + size_t aIndex) const { + if (aIndex >= mImages.Length()) { + return nullptr; + } + return &mImages[aIndex]; +} + +void ImageComposite::CountSkippedFrames(const TimedImage* aImage) { + if (aImage->mProducerID != mLastProducerID) { + // Switched producers. + return; + } + + if (mImages.IsEmpty() || aImage->mFrameID <= mLastFrameID + 1) { + // No frames were skipped. + return; + } + + uint32_t targetFrameRate = gfxPlatform::TargetFrameRate(); + if (targetFrameRate == 0) { + // Can't know whether we could have reasonably displayed all video frames. + return; + } + + double targetFrameDurationMS = 1000.0 / targetFrameRate; + + // Count how many images in mImages were skipped between mLastFrameID and + // aImage.mFrameID. Only count frames for which we can estimate a duration by + // looking at the next frame's timestamp, and only if the video frame rate is + // no faster than the target frame rate. + int32_t skipped = 0; + for (size_t i = 0; i + 1 < mImages.Length(); i++) { + const auto& img = mImages[i]; + if (img.mProducerID != aImage->mProducerID || + img.mFrameID <= mLastFrameID || img.mFrameID >= aImage->mFrameID) { + continue; + } + + // We skipped img! Estimate img's time duration. + const auto& next = mImages[i + 1]; + if (next.mProducerID != aImage->mProducerID) { + continue; + } + + MOZ_ASSERT(next.mFrameID > img.mFrameID); + TimeDuration duration = next.mTimeStamp - img.mTimeStamp; + if (floor(duration.ToMilliseconds()) >= floor(targetFrameDurationMS)) { + // Count the frame. + skipped++; + } + } + + mSkippedFramesSinceLastComposite += skipped; +} + +void ImageComposite::DetectTimeStampJitter(const TimedImage* aNewImage) { + if (!profiler_thread_is_being_profiled_for_markers() || + aNewImage->mTimeStamp.IsNull()) { + return; + } + + // Find aNewImage in mImages and compute its timestamp delta, if found. + // Ideally, a given video frame should never change its timestamp (jitter + // should be zero). However, we re-adjust video frame timestamps based on the + // audio clock. If the audio clock drifts compared to the system clock, or if + // there are bugs or inaccuracies in the computation of these timestamps, + // jitter will be non-zero. + Maybe<TimeDuration> jitter; + for (const auto& img : mImages) { + if (img.mProducerID == aNewImage->mProducerID && + img.mFrameID == aNewImage->mFrameID) { + if (!img.mTimeStamp.IsNull()) { + jitter = Some(aNewImage->mTimeStamp - img.mTimeStamp); + } + break; + } + } + if (jitter) { + nsPrintfCString text("%.2lfms", jitter->ToMilliseconds()); + PROFILER_MARKER_TEXT("VideoFrameTimeStampJitter", GRAPHICS, {}, text); + } +} + +} // namespace layers +} // namespace mozilla |