summaryrefslogtreecommitdiffstats
path: root/gfx/layers/ImageContainer.h
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-19 00:47:55 +0000
commit26a029d407be480d791972afb5975cf62c9360a6 (patch)
treef435a8308119effd964b339f76abb83a57c29483 /gfx/layers/ImageContainer.h
parentInitial commit. (diff)
downloadfirefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz
firefox-26a029d407be480d791972afb5975cf62c9360a6.zip
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'gfx/layers/ImageContainer.h')
-rw-r--r--gfx/layers/ImageContainer.h968
1 files changed, 968 insertions, 0 deletions
diff --git a/gfx/layers/ImageContainer.h b/gfx/layers/ImageContainer.h
new file mode 100644
index 0000000000..253ba417eb
--- /dev/null
+++ b/gfx/layers/ImageContainer.h
@@ -0,0 +1,968 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef GFX_IMAGECONTAINER_H
+#define GFX_IMAGECONTAINER_H
+
+#include <stdint.h> // for int32_t, uint32_t, uint8_t, uint64_t
+#include "ImageTypes.h" // for ImageFormat, etc
+#include "mozilla/AlreadyAddRefed.h"
+#include "mozilla/Assertions.h" // for MOZ_ASSERT_HELPER2
+#include "mozilla/Mutex.h" // for Mutex
+#include "mozilla/RecursiveMutex.h" // for RecursiveMutex, etc
+#include "mozilla/ThreadSafeWeakPtr.h"
+#include "mozilla/TimeStamp.h" // for TimeStamp
+#include "mozilla/gfx/Point.h" // For IntSize
+#include "mozilla/gfx/Rect.h"
+#include "mozilla/gfx/Types.h" // For ColorDepth
+#include "mozilla/layers/LayersTypes.h" // for LayersBackend, etc
+#include "mozilla/layers/CompositorTypes.h"
+#include "mozilla/mozalloc.h" // for operator delete, etc
+#include "mozilla/TypedEnumBits.h"
+#include "nsDebug.h" // for NS_ASSERTION
+#include "nsISupportsImpl.h" // for Image::Release, etc
+#include "nsTArray.h" // for nsTArray
+#include "nsThreadUtils.h" // for NS_IsMainThread
+#include "mozilla/Atomics.h"
+#include "mozilla/gfx/2D.h"
+#include "mozilla/EnumeratedArray.h"
+#include "mozilla/UniquePtr.h"
+#include "nsTHashMap.h"
+
+#ifdef XP_WIN
+struct ID3D10Texture2D;
+struct ID3D10Device;
+struct ID3D10ShaderResourceView;
+#endif
+
+typedef void* HANDLE;
+
+namespace mozilla {
+
+enum class VideoRotation;
+
+namespace layers {
+
+class GPUVideoImage;
+class ImageClient;
+class ImageCompositeNotification;
+class ImageContainer;
+class ImageContainerChild;
+class SharedPlanarYCbCrImage;
+class SurfaceDescriptor;
+class PlanarYCbCrImage;
+class TextureClient;
+class TextureClientRecycleAllocator;
+class KnowsCompositor;
+class NVImage;
+class MemoryOrShmem;
+#ifdef XP_WIN
+class D3D11RecycleAllocator;
+class D3D11YCbCrRecycleAllocator;
+#endif
+#ifdef XP_MACOSX
+class MacIOSurfaceRecycleAllocator;
+#endif
+class SurfaceDescriptorBuffer;
+
+struct ImageBackendData {
+ virtual ~ImageBackendData() = default;
+
+ protected:
+ ImageBackendData() = default;
+};
+
+/* Forward declarations for Image derivatives. */
+class GLImage;
+class SharedRGBImage;
+#ifdef MOZ_WIDGET_ANDROID
+class SurfaceTextureImage;
+#elif defined(XP_MACOSX)
+class MacIOSurfaceImage;
+#elif MOZ_WIDGET_GTK
+class DMABUFSurfaceImage;
+#endif
+
+/**
+ * A class representing a buffer of pixel data. The data can be in one
+ * of various formats including YCbCr.
+ *
+ * Create an image using an ImageContainer. Fill the image with data, and
+ * then call ImageContainer::SetImage to display it. An image must not be
+ * modified after calling SetImage. Image implementations do not need to
+ * perform locking; when filling an Image, the Image client is responsible
+ * for ensuring only one thread accesses the Image at a time, and after
+ * SetImage the image is immutable.
+ *
+ * When resampling an Image, only pixels within the buffer should be
+ * sampled. For example, cairo images should be sampled in EXTEND_PAD mode.
+ */
+class Image {
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Image)
+
+ public:
+ ImageFormat GetFormat() const { return mFormat; }
+ void* GetImplData() const { return mImplData; }
+
+ virtual gfx::IntSize GetSize() const = 0;
+ virtual gfx::IntPoint GetOrigin() const { return gfx::IntPoint(0, 0); }
+ virtual gfx::IntRect GetPictureRect() const {
+ return gfx::IntRect(GetOrigin().x, GetOrigin().y, GetSize().width,
+ GetSize().height);
+ }
+ virtual gfx::ColorDepth GetColorDepth() const {
+ return gfx::ColorDepth::COLOR_8;
+ }
+
+ ImageBackendData* GetBackendData(LayersBackend aBackend) {
+ return mBackendData[aBackend].get();
+ }
+ void SetBackendData(LayersBackend aBackend, ImageBackendData* aData) {
+ mBackendData[aBackend] = mozilla::WrapUnique(aData);
+ }
+
+ int32_t GetSerial() const { return mSerial; }
+
+ bool IsDRM() const { return mIsDRM; }
+ virtual void SetIsDRM(bool aIsDRM) { mIsDRM = aIsDRM; }
+
+ virtual void OnPrepareForwardToHost() {}
+ virtual void OnAbandonForwardToHost() {}
+
+ virtual already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() = 0;
+
+ enum class BuildSdbFlags : uint8_t {
+ Default = 0,
+ RgbOnly = 1 << 0,
+ };
+
+ virtual nsresult BuildSurfaceDescriptorBuffer(
+ SurfaceDescriptorBuffer& aSdBuffer, BuildSdbFlags aFlags,
+ const std::function<MemoryOrShmem(uint32_t)>& aAllocate);
+
+ virtual bool IsValid() const { return true; }
+
+ /**
+ * For use with the TextureForwarder only (so that the later can
+ * synchronize the TextureClient with the TextureHost).
+ */
+ virtual TextureClient* GetTextureClient(KnowsCompositor* aKnowsCompositor) {
+ return nullptr;
+ }
+
+ /* Access to derived classes. */
+ virtual GLImage* AsGLImage() { return nullptr; }
+ virtual GPUVideoImage* AsGPUVideoImage() { return nullptr; }
+#ifdef MOZ_WIDGET_ANDROID
+ virtual SurfaceTextureImage* AsSurfaceTextureImage() { return nullptr; }
+#endif
+#ifdef XP_MACOSX
+ virtual MacIOSurfaceImage* AsMacIOSurfaceImage() { return nullptr; }
+#endif
+ virtual PlanarYCbCrImage* AsPlanarYCbCrImage() { return nullptr; }
+#ifdef MOZ_WIDGET_GTK
+ virtual DMABUFSurfaceImage* AsDMABUFSurfaceImage() { return nullptr; }
+#endif
+
+ virtual NVImage* AsNVImage() { return nullptr; }
+
+ virtual Maybe<SurfaceDescriptor> GetDesc();
+
+ static nsresult AllocateSurfaceDescriptorBufferRgb(
+ const gfx::IntSize& aSize, gfx::SurfaceFormat aFormat,
+ uint8_t*& aOutBuffer, SurfaceDescriptorBuffer& aSdBuffer,
+ int32_t& aStride,
+ const std::function<layers::MemoryOrShmem(uint32_t)>& aAllocate);
+
+ protected:
+ Maybe<SurfaceDescriptor> GetDescFromTexClient(
+ TextureClient* tcOverride = nullptr);
+
+ Image(void* aImplData, ImageFormat aFormat)
+ : mImplData(aImplData),
+ mSerial(++sSerialCounter),
+ mFormat(aFormat),
+ mIsDRM(false) {}
+
+ // Protected destructor, to discourage deletion outside of Release():
+ virtual ~Image() = default;
+
+ mozilla::EnumeratedArray<mozilla::layers::LayersBackend,
+ mozilla::layers::LayersBackend::LAYERS_LAST,
+ UniquePtr<ImageBackendData>>
+ mBackendData;
+
+ void* mImplData;
+ int32_t mSerial;
+ ImageFormat mFormat;
+ bool mIsDRM;
+
+ static mozilla::Atomic<int32_t> sSerialCounter;
+};
+
+MOZ_MAKE_ENUM_CLASS_BITWISE_OPERATORS(Image::BuildSdbFlags)
+
+/**
+ * A RecycleBin is owned by an ImageContainer. We store buffers in it that we
+ * want to recycle from one image to the next.It's a separate object from
+ * ImageContainer because images need to store a strong ref to their RecycleBin
+ * and we must avoid creating a reference loop between an ImageContainer and
+ * its active image.
+ */
+class BufferRecycleBin final {
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(BufferRecycleBin)
+
+ // typedef mozilla::gl::GLContext GLContext;
+
+ public:
+ BufferRecycleBin();
+
+ void RecycleBuffer(mozilla::UniquePtr<uint8_t[]> aBuffer, uint32_t aSize);
+ // Returns a recycled buffer of the right size, or allocates a new buffer.
+ mozilla::UniquePtr<uint8_t[]> GetBuffer(uint32_t aSize);
+ virtual void ClearRecycledBuffers();
+
+ private:
+ typedef mozilla::Mutex Mutex;
+
+ // Private destructor, to discourage deletion outside of Release():
+ ~BufferRecycleBin() = default;
+
+ // This protects mRecycledBuffers, mRecycledBufferSize, mRecycledTextures
+ // and mRecycledTextureSizes
+ Mutex mLock;
+
+ // We should probably do something to prune this list on a timer so we don't
+ // eat excess memory while video is paused...
+ nsTArray<mozilla::UniquePtr<uint8_t[]>> mRecycledBuffers
+ MOZ_GUARDED_BY(mLock);
+ // This is only valid if mRecycledBuffers is non-empty
+ uint32_t mRecycledBufferSize MOZ_GUARDED_BY(mLock);
+};
+
+/**
+ * A class that manages Image creation for a LayerManager. The only reason
+ * we need a separate class here is that LayerManagers aren't threadsafe
+ * (because layers can only be used on the main thread) and we want to
+ * be able to create images from any thread, to facilitate video playback
+ * without involving the main thread, for example.
+ * Different layer managers can implement child classes of this making it
+ * possible to create layer manager specific images.
+ * This class is not meant to be used directly but rather can be set on an
+ * image container. This is usually done by the layer system internally and
+ * not explicitly by users. For PlanarYCbCr or Cairo images the default
+ * implementation will creates images whose data lives in system memory, for
+ * MacIOSurfaces the default implementation will be a simple MacIOSurface
+ * wrapper.
+ */
+
+class ImageFactory {
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ImageFactory)
+ protected:
+ friend class ImageContainer;
+
+ ImageFactory() = default;
+ virtual ~ImageFactory() = default;
+
+ virtual RefPtr<PlanarYCbCrImage> CreatePlanarYCbCrImage(
+ const gfx::IntSize& aScaleHint, BufferRecycleBin* aRecycleBin);
+};
+
+// Used to notify ImageContainer::NotifyComposite()
+class ImageContainerListener final {
+ NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ImageContainerListener)
+
+ public:
+ explicit ImageContainerListener(ImageContainer* aImageContainer);
+
+ void NotifyComposite(const ImageCompositeNotification& aNotification);
+ void NotifyDropped(uint32_t aDropped);
+ void ClearImageContainer();
+ void DropImageClient();
+
+ private:
+ typedef mozilla::Mutex Mutex;
+
+ ~ImageContainerListener();
+
+ Mutex mLock;
+ ImageContainer* mImageContainer MOZ_GUARDED_BY(mLock);
+};
+
+/**
+ * A class that manages Images for an ImageLayer. The only reason
+ * we need a separate class here is that ImageLayers aren't threadsafe
+ * (because layers can only be used on the main thread) and we want to
+ * be able to set the current Image from any thread, to facilitate
+ * video playback without involving the main thread, for example.
+ *
+ * An ImageContainer can operate in one of these modes:
+ * 1) Normal. Triggered by constructing the ImageContainer with
+ * DISABLE_ASYNC or when compositing is happening on the main thread.
+ * SetCurrentImages changes ImageContainer state but nothing is sent to the
+ * compositor until the next layer transaction.
+ * 2) Asynchronous. Initiated by constructing the ImageContainer with
+ * ENABLE_ASYNC when compositing is happening on the main thread.
+ * SetCurrentImages sends a message through the ImageBridge to the compositor
+ * thread to update the image, without going through the main thread or
+ * a layer transaction.
+ * The ImageContainer uses a shared memory block containing a cross-process
+ * mutex to communicate with the compositor thread. SetCurrentImage
+ * synchronously updates the shared state to point to the new image and the old
+ * image is immediately released (not true in Normal or Asynchronous modes).
+ */
+class ImageContainer final : public SupportsThreadSafeWeakPtr<ImageContainer> {
+ friend class ImageContainerChild;
+
+ public:
+ MOZ_DECLARE_REFCOUNTED_TYPENAME(ImageContainer)
+
+ enum Mode { SYNCHRONOUS = 0x0, ASYNCHRONOUS = 0x01 };
+
+ static const uint64_t sInvalidAsyncContainerId = 0;
+
+ explicit ImageContainer(ImageContainer::Mode flag = SYNCHRONOUS);
+
+ /**
+ * Create ImageContainer just to hold another ASYNCHRONOUS ImageContainer's
+ * async container ID.
+ * @param aAsyncContainerID async container ID for which we are a proxy
+ */
+ explicit ImageContainer(const CompositableHandle& aHandle);
+
+ ~ImageContainer();
+
+ typedef ContainerFrameID FrameID;
+ typedef ContainerProducerID ProducerID;
+
+ RefPtr<PlanarYCbCrImage> CreatePlanarYCbCrImage();
+
+ // Factory methods for shared image types.
+ RefPtr<SharedRGBImage> CreateSharedRGBImage();
+
+ struct NonOwningImage {
+ explicit NonOwningImage(Image* aImage = nullptr,
+ TimeStamp aTimeStamp = TimeStamp(),
+ FrameID aFrameID = 0, ProducerID aProducerID = 0)
+ : mImage(aImage),
+ mTimeStamp(aTimeStamp),
+ mFrameID(aFrameID),
+ mProducerID(aProducerID) {}
+ Image* mImage;
+ TimeStamp mTimeStamp;
+ FrameID mFrameID;
+ ProducerID mProducerID;
+ };
+ /**
+ * Set aImages as the list of timestamped to display. The Images must have
+ * been created by this ImageContainer.
+ * Can be called on any thread. This method takes mRecursiveMutex
+ * when accessing thread-shared state.
+ * aImages must be non-empty. The first timestamp in the list may be
+ * null but the others must not be, and the timestamps must increase.
+ * Every element of aImages must have non-null mImage.
+ * mFrameID can be zero, in which case you won't get meaningful
+ * painted/dropped frame counts. Otherwise you should use a unique and
+ * increasing ID for each decoded and submitted frame (but it's OK to
+ * pass the same frame to SetCurrentImages).
+ * mProducerID is a unique ID for the stream of images. A change in the
+ * mProducerID means changing to a new mFrameID namespace. All frames in
+ * aImages must have the same mProducerID.
+ *
+ * The Image data must not be modified after this method is called!
+ * Note that this must not be called if ENABLE_ASYNC has not been set.
+ *
+ * The implementation calls CurrentImageChanged() while holding
+ * mRecursiveMutex.
+ *
+ * If this ImageContainer has an ImageClient for async video:
+ * Schedule a task to send the image to the compositor using the
+ * PImageBridge protcol without using the main thread.
+ */
+ void SetCurrentImages(const nsTArray<NonOwningImage>& aImages);
+
+ /**
+ * Clear all images. Let ImageClient release all TextureClients. Because we
+ * may release the lock after acquiring it in this method, it cannot be called
+ * with the lock held.
+ */
+ void ClearAllImages() MOZ_EXCLUDES(mRecursiveMutex);
+
+ /**
+ * Clear any resources that are not immediately necessary. This may be called
+ * in low-memory conditions.
+ */
+ void ClearCachedResources();
+
+ /**
+ * Clear the current images.
+ * This function is expect to be called only from a CompositableClient
+ * that belongs to ImageBridgeChild. Created to prevent dead lock.
+ * See Bug 901224.
+ */
+ void ClearImagesFromImageBridge();
+
+ /**
+ * Set an Image as the current image to display. The Image must have
+ * been created by this ImageContainer.
+ * Must be called on the main thread, within a layers transaction.
+ *
+ * This method takes mRecursiveMutex
+ * when accessing thread-shared state.
+ * aImage can be null. While it's null, nothing will be painted.
+ *
+ * The Image data must not be modified after this method is called!
+ * Note that this must not be called if ENABLE_ASYNC been set.
+ *
+ * You won't get meaningful painted/dropped counts when using this method.
+ */
+ void SetCurrentImageInTransaction(Image* aImage);
+ void SetCurrentImagesInTransaction(const nsTArray<NonOwningImage>& aImages);
+
+ /**
+ * Returns true if this ImageContainer uses the ImageBridge IPDL protocol.
+ *
+ * Can be called from any thread.
+ */
+ bool IsAsync() const;
+
+ /**
+ * If this ImageContainer uses ImageBridge, returns the ID associated to
+ * this container, for use in the ImageBridge protocol.
+ * Returns 0 if this ImageContainer does not use ImageBridge. Note that
+ * 0 is always an invalid ID for asynchronous image containers.
+ *
+ * Can be called from any thread.
+ */
+ CompositableHandle GetAsyncContainerHandle();
+
+ /**
+ * Returns if the container currently has an image.
+ * Can be called on any thread. This method takes mRecursiveMutex
+ * when accessing thread-shared state.
+ */
+ bool HasCurrentImage();
+
+ struct OwningImage {
+ OwningImage() : mFrameID(0), mProducerID(0), mComposited(false) {}
+ RefPtr<Image> mImage;
+ TimeStamp mTimeStamp;
+ FrameID mFrameID;
+ ProducerID mProducerID;
+ bool mComposited;
+ };
+ /**
+ * Copy the current Image list to aImages.
+ * This has to add references since otherwise there are race conditions
+ * where the current image is destroyed before the caller can add
+ * a reference.
+ * Can be called on any thread.
+ * May return an empty list to indicate there is no current image.
+ * If aGenerationCounter is non-null, sets *aGenerationCounter to a value
+ * that's unique for this ImageContainer state.
+ */
+ void GetCurrentImages(nsTArray<OwningImage>* aImages,
+ uint32_t* aGenerationCounter = nullptr);
+
+ /**
+ * Returns the size of the image in pixels.
+ * Can be called on any thread. This method takes mRecursiveMutex when
+ * accessing thread-shared state.
+ */
+ gfx::IntSize GetCurrentSize();
+
+ /**
+ * Sets a size that the image is expected to be rendered at.
+ * This is a hint for image backends to optimize scaling.
+ * Default implementation in this class is to ignore the hint.
+ * Can be called on any thread. This method takes mRecursiveMutex
+ * when accessing thread-shared state.
+ */
+ void SetScaleHint(const gfx::IntSize& aScaleHint) {
+ RecursiveMutexAutoLock lock(mRecursiveMutex);
+ mScaleHint = aScaleHint;
+ }
+
+ const gfx::IntSize GetScaleHint() const {
+ RecursiveMutexAutoLock lock(mRecursiveMutex);
+ return mScaleHint;
+ }
+
+ void SetTransformHint(const gfx::Matrix& aTransformHint) {
+ RecursiveMutexAutoLock lock(mRecursiveMutex);
+ mTransformHint = aTransformHint;
+ }
+
+ const gfx::Matrix GetTransformHint() const {
+ RecursiveMutexAutoLock lock(mRecursiveMutex);
+ return mTransformHint;
+ }
+
+ void SetRotation(VideoRotation aRotation) {
+ MOZ_ASSERT(NS_IsMainThread());
+ mRotation = aRotation;
+ }
+
+ VideoRotation GetRotation() const {
+ MOZ_ASSERT(NS_IsMainThread());
+ return mRotation;
+ }
+
+ void SetImageFactory(ImageFactory* aFactory) {
+ RecursiveMutexAutoLock lock(mRecursiveMutex);
+ mImageFactory = aFactory ? aFactory : new ImageFactory();
+ }
+
+ already_AddRefed<ImageFactory> GetImageFactory() const {
+ RecursiveMutexAutoLock lock(mRecursiveMutex);
+ return do_AddRef(mImageFactory);
+ }
+
+ void EnsureRecycleAllocatorForRDD(KnowsCompositor* aKnowsCompositor);
+
+#ifdef XP_WIN
+ already_AddRefed<D3D11RecycleAllocator> GetD3D11RecycleAllocator(
+ KnowsCompositor* aKnowsCompositor, gfx::SurfaceFormat aPreferredFormat);
+ already_AddRefed<D3D11YCbCrRecycleAllocator> GetD3D11YCbCrRecycleAllocator(
+ KnowsCompositor* aKnowsCompositor);
+#endif
+
+#ifdef XP_MACOSX
+ already_AddRefed<MacIOSurfaceRecycleAllocator>
+ GetMacIOSurfaceRecycleAllocator();
+#endif
+
+ /**
+ * Returns the delay between the last composited image's presentation
+ * timestamp and when it was first composited. It's possible for the delay
+ * to be negative if the first image in the list passed to SetCurrentImages
+ * has a presentation timestamp greater than "now".
+ * Returns 0 if the composited image had a null timestamp, or if no
+ * image has been composited yet.
+ */
+ TimeDuration GetPaintDelay() {
+ RecursiveMutexAutoLock lock(mRecursiveMutex);
+ return mPaintDelay;
+ }
+
+ /**
+ * Returns the number of images which have been contained in this container
+ * and painted at least once. Can be called from any thread.
+ */
+ uint32_t GetPaintCount() {
+ RecursiveMutexAutoLock lock(mRecursiveMutex);
+ return mPaintCount;
+ }
+
+ /**
+ * An entry in the current image list "expires" when the entry has an
+ * non-null timestamp, and in a SetCurrentImages call the new image list is
+ * non-empty, the timestamp of the first new image is non-null and greater
+ * than the timestamp associated with the image, and the first new image's
+ * frameID is not the same as the entry's.
+ * Every expired image that is never composited is counted as dropped.
+ */
+ uint32_t GetDroppedImageCount() { return mDroppedImageCount; }
+
+ void NotifyComposite(const ImageCompositeNotification& aNotification);
+ void NotifyDropped(uint32_t aDropped);
+
+ already_AddRefed<ImageContainerListener> GetImageContainerListener() const;
+
+ /**
+ * Get the ImageClient associated with this container. Returns only after
+ * validating, and it will recreate the image client if that fails.
+ * Returns nullptr if not applicable.
+ */
+ already_AddRefed<ImageClient> GetImageClient();
+
+ /**
+ * Main thread only.
+ */
+ static ProducerID AllocateProducerID();
+
+ void DropImageClient();
+
+ private:
+ typedef mozilla::RecursiveMutex RecursiveMutex;
+
+ void SetCurrentImageInternal(const nsTArray<NonOwningImage>& aImages);
+
+ // This is called to ensure we have an active image, this may not be true
+ // when we're storing image information in a RemoteImageData structure.
+ // NOTE: If we have remote data mRemoteDataMutex should be locked when
+ // calling this function!
+ void EnsureActiveImage();
+
+ void EnsureImageClient() MOZ_REQUIRES(mRecursiveMutex);
+
+ bool HasImageClient() const {
+ RecursiveMutexAutoLock lock(mRecursiveMutex);
+ return !!mImageClient;
+ }
+
+ // RecursiveMutex to protect thread safe access to the "current
+ // image", and any other state which is shared between threads.
+ mutable RecursiveMutex mRecursiveMutex;
+
+ RefPtr<TextureClientRecycleAllocator> mRecycleAllocator
+ MOZ_GUARDED_BY(mRecursiveMutex);
+
+#ifdef XP_WIN
+ RefPtr<D3D11RecycleAllocator> mD3D11RecycleAllocator
+ MOZ_GUARDED_BY(mRecursiveMutex);
+
+ RefPtr<D3D11YCbCrRecycleAllocator> mD3D11YCbCrRecycleAllocator
+ MOZ_GUARDED_BY(mRecursiveMutex);
+#endif
+#ifdef XP_MACOSX
+ RefPtr<MacIOSurfaceRecycleAllocator> mMacIOSurfaceRecycleAllocator
+ MOZ_GUARDED_BY(mRecursiveMutex);
+#endif
+
+ nsTArray<OwningImage> mCurrentImages MOZ_GUARDED_BY(mRecursiveMutex);
+
+ // Updates every time mActiveImage changes
+ uint32_t mGenerationCounter MOZ_GUARDED_BY(mRecursiveMutex);
+
+ // Number of contained images that have been painted at least once. It's up
+ // to the ImageContainer implementation to ensure accesses to this are
+ // threadsafe.
+ uint32_t mPaintCount MOZ_GUARDED_BY(mRecursiveMutex);
+
+ // See GetPaintDelay. Accessed only with mRecursiveMutex held.
+ TimeDuration mPaintDelay MOZ_GUARDED_BY(mRecursiveMutex);
+
+ // See GetDroppedImageCount.
+ mozilla::Atomic<uint32_t> mDroppedImageCount;
+
+ // This is the image factory used by this container, layer managers using
+ // this container can set an alternative image factory that will be used to
+ // create images for this container.
+ RefPtr<ImageFactory> mImageFactory MOZ_GUARDED_BY(mRecursiveMutex);
+
+ gfx::IntSize mScaleHint MOZ_GUARDED_BY(mRecursiveMutex);
+
+ gfx::Matrix mTransformHint MOZ_GUARDED_BY(mRecursiveMutex);
+
+ // Main thread only.
+ VideoRotation mRotation;
+
+ RefPtr<BufferRecycleBin> mRecycleBin MOZ_GUARDED_BY(mRecursiveMutex);
+
+ // This member points to an ImageClient if this ImageContainer was
+ // sucessfully created with ENABLE_ASYNC, or points to null otherwise.
+ // 'unsuccessful' in this case only means that the ImageClient could not
+ // be created, most likely because off-main-thread compositing is not enabled.
+ // In this case the ImageContainer is perfectly usable, but it will forward
+ // frames to the compositor through transactions in the main thread rather
+ // than asynchronusly using the ImageBridge IPDL protocol.
+ RefPtr<ImageClient> mImageClient MOZ_GUARDED_BY(mRecursiveMutex);
+
+ const bool mIsAsync;
+ CompositableHandle mAsyncContainerHandle MOZ_GUARDED_BY(mRecursiveMutex);
+
+ // ProducerID for last current image(s)
+ ProducerID mCurrentProducerID MOZ_GUARDED_BY(mRecursiveMutex);
+
+ RefPtr<ImageContainerListener> mNotifyCompositeListener;
+
+ static mozilla::Atomic<uint32_t> sGenerationCounter;
+};
+
+class AutoLockImage {
+ public:
+ explicit AutoLockImage(ImageContainer* aContainer) {
+ aContainer->GetCurrentImages(&mImages);
+ }
+
+ bool HasImage() const { return !mImages.IsEmpty(); }
+ Image* GetImage() const {
+ return mImages.IsEmpty() ? nullptr : mImages[0].mImage.get();
+ }
+
+ Image* GetImage(TimeStamp aTimeStamp) const {
+ if (mImages.IsEmpty()) {
+ return nullptr;
+ }
+
+ MOZ_ASSERT(!aTimeStamp.IsNull());
+ uint32_t chosenIndex = 0;
+
+ while (chosenIndex + 1 < mImages.Length() &&
+ mImages[chosenIndex + 1].mTimeStamp <= aTimeStamp) {
+ ++chosenIndex;
+ }
+
+ return mImages[chosenIndex].mImage.get();
+ }
+
+ private:
+ AutoTArray<ImageContainer::OwningImage, 4> mImages;
+};
+
+// This type is currently only used for AVIF and WebCodecs therefore makes some
+// specific assumptions (e.g., Alpha's bpc and stride is equal to Y's one)
+struct PlanarAlphaData {
+ uint8_t* mChannel = nullptr;
+ gfx::IntSize mSize = gfx::IntSize(0, 0);
+ gfx::ColorDepth mDepth = gfx::ColorDepth::COLOR_8;
+ bool mPremultiplied = false;
+};
+struct PlanarYCbCrData {
+ // Luminance buffer
+ uint8_t* mYChannel = nullptr;
+ int32_t mYStride = 0;
+ int32_t mYSkip = 0;
+ // Chroma buffers
+ uint8_t* mCbChannel = nullptr;
+ uint8_t* mCrChannel = nullptr;
+ int32_t mCbCrStride = 0;
+ int32_t mCbSkip = 0;
+ int32_t mCrSkip = 0;
+ // Alpha buffer and its metadata
+ Maybe<PlanarAlphaData> mAlpha = Nothing();
+ // Picture region
+ gfx::IntRect mPictureRect = gfx::IntRect(0, 0, 0, 0);
+ StereoMode mStereoMode = StereoMode::MONO;
+ gfx::ColorDepth mColorDepth = gfx::ColorDepth::COLOR_8;
+ gfx::YUVColorSpace mYUVColorSpace = gfx::YUVColorSpace::Default;
+ gfx::ColorSpace2 mColorPrimaries = gfx::ColorSpace2::UNKNOWN;
+ gfx::TransferFunction mTransferFunction = gfx::TransferFunction::BT709;
+ gfx::ColorRange mColorRange = gfx::ColorRange::LIMITED;
+ gfx::ChromaSubsampling mChromaSubsampling = gfx::ChromaSubsampling::FULL;
+
+ // The cropped picture size of the Y channel.
+ gfx::IntSize YPictureSize() const { return mPictureRect.Size(); }
+
+ // The cropped picture size of the Cb/Cr channels.
+ gfx::IntSize CbCrPictureSize() const {
+ return mCbCrStride > 0 ? gfx::ChromaSize(YPictureSize(), mChromaSubsampling)
+ : gfx::IntSize(0, 0);
+ }
+
+ // The total uncropped size of data in the Y channel.
+ gfx::IntSize YDataSize() const {
+ return gfx::IntSize(mPictureRect.XMost(), mPictureRect.YMost());
+ }
+
+ // The total uncropped size of data in the Cb/Cr channels.
+ gfx::IntSize CbCrDataSize() const {
+ return mCbCrStride > 0 ? gfx::ChromaSize(YDataSize(), mChromaSubsampling)
+ : gfx::IntSize(0, 0);
+ }
+
+ static Maybe<PlanarYCbCrData> From(const SurfaceDescriptorBuffer&);
+};
+
+/****** Image subtypes for the different formats ******/
+
+/**
+ * We assume that the image data is in the REC 470M color space (see
+ * Theora specification, section 4.3.1).
+ *
+ * The YCbCr format can be:
+ *
+ * 4:4:4 - CbCr width/height are the same as Y.
+ * 4:2:2 - CbCr width is half that of Y. Height is the same.
+ * 4:2:0 - CbCr width and height is half that of Y.
+ *
+ * mChromaSubsampling specifies which YCbCr subsampling scheme to use.
+ *
+ * The Image that is rendered is the picture region defined by mPictureRect.
+ *
+ * mYSkip, mCbSkip, mCrSkip are added to support various output
+ * formats from hardware decoder. They are per-pixel skips in the
+ * source image.
+ *
+ * For example when image width is 640, mYStride is 670, mYSkip is 2,
+ * the mYChannel buffer looks like:
+ *
+ * |<----------------------- mYStride ----------------------------->|
+ * |<----------------- YDataSize().width ---------->|
+ * 0 3 6 9 12 15 18 21 639 669
+ * |----------------------------------------------------------------|
+ * |Y___Y___Y___Y___Y___Y___Y___Y... |%%%%%%%%%%%%%%%|
+ * |Y___Y___Y___Y___Y___Y___Y___Y... |%%%%%%%%%%%%%%%|
+ * |Y___Y___Y___Y___Y___Y___Y___Y... |%%%%%%%%%%%%%%%|
+ * | |<->|
+ * mYSkip
+ */
+class PlanarYCbCrImage : public Image {
+ public:
+ typedef PlanarYCbCrData Data;
+
+ enum { MAX_DIMENSION = 16384 };
+
+ virtual ~PlanarYCbCrImage();
+
+ /**
+ * This makes a copy of the data buffers, in order to support functioning
+ * in all different layer managers.
+ */
+ virtual nsresult CopyData(const Data& aData) = 0;
+
+ /**
+ * This doesn't make a copy of the data buffers.
+ */
+ virtual nsresult AdoptData(const Data& aData);
+
+ /**
+ * This will create an empty data buffers according to the input data's size.
+ */
+ virtual nsresult CreateEmptyBuffer(const Data& aData,
+ const gfx::IntSize& aYSize,
+ const gfx::IntSize& aCbCrSize) {
+ return NS_ERROR_NOT_IMPLEMENTED;
+ }
+
+ /**
+ * Grab the original YUV data. This is optional.
+ */
+ virtual const Data* GetData() const { return &mData; }
+
+ /**
+ * Return the number of bytes of heap memory used to store this image.
+ */
+ uint32_t GetDataSize() const { return mBufferSize; }
+
+ bool IsValid() const override { return !!mBufferSize; }
+
+ gfx::IntSize GetSize() const override { return mSize; }
+
+ gfx::IntPoint GetOrigin() const override { return mOrigin; }
+
+ PlanarYCbCrImage();
+
+ virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const {
+ return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
+ }
+
+ virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const = 0;
+
+ PlanarYCbCrImage* AsPlanarYCbCrImage() override { return this; }
+
+ /**
+ * Build a SurfaceDescriptorBuffer with this image. A function to allocate
+ * a MemoryOrShmem with the given capacity must be provided.
+ */
+ nsresult BuildSurfaceDescriptorBuffer(
+ SurfaceDescriptorBuffer& aSdBuffer, BuildSdbFlags aFlags,
+ const std::function<MemoryOrShmem(uint32_t)>& aAllocate) override;
+
+ protected:
+ already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() override;
+
+ void SetOffscreenFormat(gfxImageFormat aFormat) {
+ mOffscreenFormat = aFormat;
+ }
+ gfxImageFormat GetOffscreenFormat() const;
+
+ Data mData;
+ gfx::IntPoint mOrigin;
+ gfx::IntSize mSize;
+ gfxImageFormat mOffscreenFormat;
+ RefPtr<gfx::DataSourceSurface> mSourceSurface;
+ uint32_t mBufferSize;
+};
+
+class RecyclingPlanarYCbCrImage : public PlanarYCbCrImage {
+ public:
+ explicit RecyclingPlanarYCbCrImage(BufferRecycleBin* aRecycleBin)
+ : mRecycleBin(aRecycleBin) {}
+ virtual ~RecyclingPlanarYCbCrImage();
+ nsresult CopyData(const Data& aData) override;
+ size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override;
+
+ protected:
+ /**
+ * Return a buffer to store image data in.
+ */
+ mozilla::UniquePtr<uint8_t[]> AllocateBuffer(uint32_t aSize);
+
+ RefPtr<BufferRecycleBin> mRecycleBin;
+ mozilla::UniquePtr<uint8_t[]> mBuffer;
+};
+
+/**
+ * NVImage is used to store YUV420SP_NV12 and YUV420SP_NV21 data natively, which
+ * are not supported by PlanarYCbCrImage. (PlanarYCbCrImage only stores YUV444P,
+ * YUV422P and YUV420P, it converts YUV420SP_NV12 and YUV420SP_NV21 data into
+ * YUV420P in its PlanarYCbCrImage::SetData() method.)
+ *
+ * PlanarYCbCrData is able to express all the YUV family and so we keep use it
+ * in NVImage.
+ */
+class NVImage final : public Image {
+ typedef PlanarYCbCrData Data;
+
+ public:
+ NVImage();
+ virtual ~NVImage();
+
+ // Methods inherited from layers::Image.
+ gfx::IntSize GetSize() const override;
+ gfx::IntRect GetPictureRect() const override;
+ already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() override;
+ nsresult BuildSurfaceDescriptorBuffer(
+ SurfaceDescriptorBuffer& aSdBuffer, BuildSdbFlags aFlags,
+ const std::function<MemoryOrShmem(uint32_t)>& aAllocate) override;
+ bool IsValid() const override;
+ NVImage* AsNVImage() override;
+
+ // Methods mimic layers::PlanarYCbCrImage.
+ bool SetData(const Data& aData);
+ const Data* GetData() const;
+ uint32_t GetBufferSize() const;
+
+ protected:
+ /**
+ * Return a buffer to store image data in.
+ */
+ mozilla::UniquePtr<uint8_t[]> AllocateBuffer(uint32_t aSize);
+
+ mozilla::UniquePtr<uint8_t[]> mBuffer;
+ uint32_t mBufferSize;
+ gfx::IntSize mSize;
+ Data mData;
+ RefPtr<gfx::DataSourceSurface> mSourceSurface;
+};
+
+/**
+ * Currently, the data in a SourceSurfaceImage surface is treated as being in
+ * the device output color space. This class is very simple as all backends have
+ * to know about how to deal with drawing a cairo image.
+ */
+class SourceSurfaceImage final : public Image {
+ public:
+ already_AddRefed<gfx::SourceSurface> GetAsSourceSurface() override {
+ RefPtr<gfx::SourceSurface> surface(mSourceSurface);
+ return surface.forget();
+ }
+
+ void SetTextureFlags(TextureFlags aTextureFlags) {
+ mTextureFlags = aTextureFlags;
+ }
+ TextureClient* GetTextureClient(KnowsCompositor* aKnowsCompositor) override;
+
+ gfx::IntSize GetSize() const override { return mSize; }
+
+ SourceSurfaceImage(const gfx::IntSize& aSize,
+ gfx::SourceSurface* aSourceSurface);
+ explicit SourceSurfaceImage(gfx::SourceSurface* aSourceSurface);
+ virtual ~SourceSurfaceImage();
+
+ private:
+ gfx::IntSize mSize;
+ RefPtr<gfx::SourceSurface> mSourceSurface;
+ nsTHashMap<uint32_t, RefPtr<TextureClient>> mTextureClients;
+ TextureFlags mTextureFlags;
+};
+
+} // namespace layers
+} // namespace mozilla
+
+#endif