/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim:set ts=2 sw=2 sts=2 et cindent: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "OffscreenCanvasDisplayHelper.h" #include "mozilla/dom/Document.h" #include "mozilla/dom/WorkerRef.h" #include "mozilla/dom/WorkerRunnable.h" #include "mozilla/gfx/2D.h" #include "mozilla/gfx/CanvasManagerChild.h" #include "mozilla/gfx/DataSurfaceHelpers.h" #include "mozilla/gfx/Swizzle.h" #include "mozilla/layers/ImageBridgeChild.h" #include "mozilla/layers/PersistentBufferProvider.h" #include "mozilla/layers/TextureClientSharedSurface.h" #include "mozilla/layers/TextureWrapperImage.h" #include "mozilla/StaticPrefs_gfx.h" #include "mozilla/SVGObserverUtils.h" #include "nsICanvasRenderingContextInternal.h" #include "nsRFPService.h" namespace mozilla::dom { OffscreenCanvasDisplayHelper::OffscreenCanvasDisplayHelper( HTMLCanvasElement* aCanvasElement, uint32_t aWidth, uint32_t aHeight) : mMutex("mozilla::dom::OffscreenCanvasDisplayHelper"), mCanvasElement(aCanvasElement), mImageProducerID(layers::ImageContainer::AllocateProducerID()) { mData.mSize.width = aWidth; mData.mSize.height = aHeight; } OffscreenCanvasDisplayHelper::~OffscreenCanvasDisplayHelper() = default; void OffscreenCanvasDisplayHelper::DestroyElement() { MOZ_ASSERT(NS_IsMainThread()); MutexAutoLock lock(mMutex); if (mImageContainer) { mImageContainer->ClearAllImages(); mImageContainer = nullptr; } mFrontBufferSurface = nullptr; mCanvasElement = nullptr; } void OffscreenCanvasDisplayHelper::DestroyCanvas() { if (auto* cm = gfx::CanvasManagerChild::Get()) { cm->EndCanvasTransaction(); } MutexAutoLock lock(mMutex); if (mImageContainer) { mImageContainer->ClearAllImages(); mImageContainer = nullptr; } mFrontBufferSurface = nullptr; mOffscreenCanvas = nullptr; mWorkerRef = nullptr; } bool OffscreenCanvasDisplayHelper::CanElementCaptureStream() const { MutexAutoLock lock(mMutex); return !!mWorkerRef; } bool OffscreenCanvasDisplayHelper::UsingElementCaptureStream() const { MutexAutoLock lock(mMutex); if (NS_WARN_IF(!NS_IsMainThread())) { MOZ_ASSERT_UNREACHABLE("Should not call off main-thread!"); return !!mCanvasElement; } return mCanvasElement && mCanvasElement->UsingCaptureStream(); } CanvasContextType OffscreenCanvasDisplayHelper::GetContextType() const { MutexAutoLock lock(mMutex); return mType; } RefPtr OffscreenCanvasDisplayHelper::GetImageContainer() const { MutexAutoLock lock(mMutex); return mImageContainer; } void OffscreenCanvasDisplayHelper::UpdateContext( OffscreenCanvas* aOffscreenCanvas, RefPtr&& aWorkerRef, CanvasContextType aType, const Maybe& aChildId) { RefPtr imageContainer = MakeRefPtr(layers::ImageContainer::ASYNCHRONOUS); MutexAutoLock lock(mMutex); mOffscreenCanvas = aOffscreenCanvas; mWorkerRef = std::move(aWorkerRef); mType = aType; mContextChildId = aChildId; mImageContainer = std::move(imageContainer); if (aChildId) { mContextManagerId = Some(gfx::CanvasManagerChild::Get()->Id()); } else { mContextManagerId.reset(); } MaybeQueueInvalidateElement(); } void OffscreenCanvasDisplayHelper::FlushForDisplay() { MOZ_ASSERT(NS_IsMainThread()); MutexAutoLock lock(mMutex); // Without an OffscreenCanvas object bound to us, we either have not drawn // using the canvas at all, or we have already destroyed it. if (!mOffscreenCanvas) { return; } // We assign/destroy the worker ref at the same time as the OffscreenCanvas // ref, so we can only have the canvas without a worker ref if it exists on // the main thread. if (!mWorkerRef) { // We queue to ensure that we have the same asynchronous update behaviour // for a main thread and a worker based OffscreenCanvas. mOffscreenCanvas->QueueCommitToCompositor(); return; } class FlushWorkerRunnable final : public WorkerThreadRunnable { public: FlushWorkerRunnable(WorkerPrivate* aWorkerPrivate, OffscreenCanvasDisplayHelper* aDisplayHelper) : WorkerThreadRunnable("FlushWorkerRunnable"), mDisplayHelper(aDisplayHelper) {} bool WorkerRun(JSContext*, WorkerPrivate*) override { // The OffscreenCanvas can only be freed on the worker thread, so we // cannot be racing with an OffscreenCanvas::DestroyCanvas call and its // destructor. We just need to make sure we don't call into // OffscreenCanvas while holding the lock since it calls back into the // OffscreenCanvasDisplayHelper. RefPtr canvas; { MutexAutoLock lock(mDisplayHelper->mMutex); canvas = mDisplayHelper->mOffscreenCanvas; } if (canvas) { canvas->CommitFrameToCompositor(); } return true; } private: RefPtr mDisplayHelper; }; // Otherwise we are calling from the main thread during painting to a canvas // on a worker thread. auto task = MakeRefPtr(mWorkerRef->Private(), this); task->Dispatch(mWorkerRef->Private()); } bool OffscreenCanvasDisplayHelper::CommitFrameToCompositor( nsICanvasRenderingContextInternal* aContext, layers::TextureType aTextureType, const Maybe& aData) { auto endTransaction = MakeScopeExit([&]() { if (auto* cm = gfx::CanvasManagerChild::Get()) { cm->EndCanvasTransaction(); } }); MutexAutoLock lock(mMutex); gfx::SurfaceFormat format = gfx::SurfaceFormat::B8G8R8A8; layers::TextureFlags flags = layers::TextureFlags::IMMUTABLE; if (!mCanvasElement) { // Our weak reference to the canvas element has been cleared, so we cannot // present directly anymore. return false; } if (aData) { mData = aData.ref(); MaybeQueueInvalidateElement(); } if (mData.mOwnerId.isSome()) { // No need to update the ImageContainer as the presentation itself is // handled in the compositor process. return true; } if (!mImageContainer) { return false; } if (mData.mIsOpaque) { flags |= layers::TextureFlags::IS_OPAQUE; format = gfx::SurfaceFormat::B8G8R8X8; } else if (!mData.mIsAlphaPremult) { flags |= layers::TextureFlags::NON_PREMULTIPLIED; } switch (mData.mOriginPos) { case gl::OriginPos::BottomLeft: flags |= layers::TextureFlags::ORIGIN_BOTTOM_LEFT; break; case gl::OriginPos::TopLeft: break; default: MOZ_ASSERT_UNREACHABLE("Unhandled origin position!"); break; } auto imageBridge = layers::ImageBridgeChild::GetSingleton(); if (!imageBridge) { return false; } bool paintCallbacks = mData.mDoPaintCallbacks; bool hasRemoteTextureDesc = false; RefPtr image; RefPtr texture; RefPtr surface; Maybe desc; RefPtr tracker; { MutexAutoUnlock unlock(mMutex); if (paintCallbacks) { aContext->OnBeforePaintTransaction(); } desc = aContext->PresentFrontBuffer(nullptr, aTextureType); if (desc) { hasRemoteTextureDesc = desc->type() == layers::SurfaceDescriptor::TSurfaceDescriptorRemoteTexture; if (hasRemoteTextureDesc) { tracker = aContext->UseCompositableForwarder(imageBridge); if (tracker) { flags |= layers::TextureFlags::WAIT_FOR_REMOTE_TEXTURE_OWNER; } } } else { if (layers::PersistentBufferProvider* provider = aContext->GetBufferProvider()) { texture = provider->GetTextureClient(); } if (!texture) { surface = aContext->GetFrontBufferSnapshot(/* requireAlphaPremult */ false); if (surface && surface->GetType() == gfx::SurfaceType::WEBGL) { // Ensure we can map in the surface. If we get a SourceSurfaceWebgl // surface, then it may not be backed by raw pixels yet. We need to // map it on the owning thread rather than the ImageBridge thread. gfx::DataSourceSurface::ScopedMap map( static_cast(surface.get()), gfx::DataSourceSurface::READ); if (!map.IsMapped()) { surface = nullptr; } } } } if (paintCallbacks) { aContext->OnDidPaintTransaction(); } } // We save any current surface because we might need it in GetSnapshot. If we // are on a worker thread and not WebGL, then this will be the only way we can // access the pixel data on the main thread. mFrontBufferSurface = surface; // We do not use the ImageContainer plumbing with remote textures, so if we // have that, we can just early return here. if (hasRemoteTextureDesc) { const auto& textureDesc = desc->get_SurfaceDescriptorRemoteTexture(); imageBridge->UpdateCompositable(mImageContainer, textureDesc.textureId(), textureDesc.ownerId(), mData.mSize, flags, tracker); return true; } if (surface) { auto surfaceImage = MakeRefPtr(surface); surfaceImage->SetTextureFlags(flags); image = surfaceImage; } else { if (desc && !texture) { texture = layers::SharedSurfaceTextureData::CreateTextureClient( *desc, format, mData.mSize, flags, imageBridge); } if (texture) { image = new layers::TextureWrapperImage( texture, gfx::IntRect(gfx::IntPoint(0, 0), texture->GetSize())); } } if (image) { AutoTArray imageList; imageList.AppendElement(layers::ImageContainer::NonOwningImage( image, TimeStamp(), mLastFrameID++, mImageProducerID)); mImageContainer->SetCurrentImages(imageList); } else { mImageContainer->ClearAllImages(); } return true; } void OffscreenCanvasDisplayHelper::MaybeQueueInvalidateElement() { if (!mPendingInvalidate) { mPendingInvalidate = true; NS_DispatchToMainThread(NS_NewRunnableFunction( "OffscreenCanvasDisplayHelper::InvalidateElement", [self = RefPtr{this}] { self->InvalidateElement(); })); } } void OffscreenCanvasDisplayHelper::InvalidateElement() { MOZ_ASSERT(NS_IsMainThread()); HTMLCanvasElement* canvasElement; gfx::IntSize size; { MutexAutoLock lock(mMutex); MOZ_ASSERT(mPendingInvalidate); mPendingInvalidate = false; canvasElement = mCanvasElement; size = mData.mSize; } if (canvasElement) { SVGObserverUtils::InvalidateDirectRenderingObservers(canvasElement); canvasElement->InvalidateCanvasPlaceholder(size.width, size.height); canvasElement->InvalidateCanvasContent(nullptr); } } already_AddRefed OffscreenCanvasDisplayHelper::TransformSurface(gfx::SourceSurface* aSurface, bool aHasAlpha, bool aIsAlphaPremult, gl::OriginPos aOriginPos) const { if (!aSurface) { return nullptr; } if (aOriginPos == gl::OriginPos::TopLeft && (!aHasAlpha || aIsAlphaPremult)) { // If we don't need to y-flip, and it is either opaque or premultiplied, // we can just return the same surface. return do_AddRef(aSurface); } // Otherwise we need to copy and apply the necessary transformations. RefPtr srcSurface = aSurface->GetDataSurface(); if (!srcSurface) { return nullptr; } const auto size = srcSurface->GetSize(); const auto format = srcSurface->GetFormat(); RefPtr dstSurface = gfx::Factory::CreateDataSourceSurface(size, format, /* aZero */ false); if (!dstSurface) { return nullptr; } gfx::DataSourceSurface::ScopedMap srcMap(srcSurface, gfx::DataSourceSurface::READ); gfx::DataSourceSurface::ScopedMap dstMap(dstSurface, gfx::DataSourceSurface::WRITE); if (!srcMap.IsMapped() || !dstMap.IsMapped()) { return nullptr; } bool success; switch (aOriginPos) { case gl::OriginPos::BottomLeft: if (aHasAlpha && !aIsAlphaPremult) { success = gfx::PremultiplyYFlipData( srcMap.GetData(), srcMap.GetStride(), format, dstMap.GetData(), dstMap.GetStride(), format, size); } else { success = gfx::SwizzleYFlipData(srcMap.GetData(), srcMap.GetStride(), format, dstMap.GetData(), dstMap.GetStride(), format, size); } break; case gl::OriginPos::TopLeft: if (aHasAlpha && !aIsAlphaPremult) { success = gfx::PremultiplyData(srcMap.GetData(), srcMap.GetStride(), format, dstMap.GetData(), dstMap.GetStride(), format, size); } else { success = gfx::SwizzleData(srcMap.GetData(), srcMap.GetStride(), format, dstMap.GetData(), dstMap.GetStride(), format, size); } break; default: MOZ_ASSERT_UNREACHABLE("Unhandled origin position!"); success = false; break; } if (!success) { return nullptr; } return dstSurface.forget(); } already_AddRefed OffscreenCanvasDisplayHelper::GetSurfaceSnapshot() { MOZ_ASSERT(NS_IsMainThread()); class SnapshotWorkerRunnable final : public MainThreadWorkerRunnable { public: SnapshotWorkerRunnable(WorkerPrivate* aWorkerPrivate, OffscreenCanvasDisplayHelper* aDisplayHelper) : MainThreadWorkerRunnable("SnapshotWorkerRunnable"), mMonitor("SnapshotWorkerRunnable::mMonitor"), mDisplayHelper(aDisplayHelper) {} bool WorkerRun(JSContext*, WorkerPrivate*) override { // The OffscreenCanvas can only be freed on the worker thread, so we // cannot be racing with an OffscreenCanvas::DestroyCanvas call and its // destructor. We just need to make sure we don't call into // OffscreenCanvas while holding the lock since it calls back into the // OffscreenCanvasDisplayHelper. RefPtr canvas; { MutexAutoLock lock(mDisplayHelper->mMutex); canvas = mDisplayHelper->mOffscreenCanvas; } // Now that we are on the correct thread, we can extract the snapshot. If // it is a Skia surface, perform a copy to threading issues. RefPtr surface; if (canvas) { if (auto* context = canvas->GetContext()) { surface = context->GetFrontBufferSnapshot(/* requireAlphaPremult */ false); if (surface && surface->GetType() == gfx::SurfaceType::SKIA) { surface = gfx::Factory::CopyDataSourceSurface( static_cast(surface.get())); } } } MonitorAutoLock lock(mMonitor); mSurface = std::move(surface); mComplete = true; lock.NotifyAll(); return true; } already_AddRefed Wait(int32_t aTimeoutMs) { MonitorAutoLock lock(mMonitor); TimeDuration timeout = TimeDuration::FromMilliseconds(aTimeoutMs); while (!mComplete) { if (lock.Wait(timeout) == CVStatus::Timeout) { return nullptr; } } return mSurface.forget(); } private: Monitor mMonitor; RefPtr mDisplayHelper; RefPtr mSurface MOZ_GUARDED_BY(mMonitor); bool mComplete MOZ_GUARDED_BY(mMonitor) = false; }; bool hasAlpha; bool isAlphaPremult; gl::OriginPos originPos; HTMLCanvasElement* canvasElement; RefPtr surface; RefPtr workerRunnable; { MutexAutoLock lock(mMutex); #ifdef MOZ_WIDGET_ANDROID // On Android, we cannot both display a GL context and read back the pixels. if (mCanvasElement) { return nullptr; } #endif hasAlpha = !mData.mIsOpaque; isAlphaPremult = mData.mIsAlphaPremult; originPos = mData.mOriginPos; canvasElement = mCanvasElement; if (mWorkerRef) { workerRunnable = MakeRefPtr(mWorkerRef->Private(), this); workerRunnable->Dispatch(mWorkerRef->Private()); } } if (workerRunnable) { // We transferred to a DOM worker, so we need to do the readback on the // owning thread and wait for the result. surface = workerRunnable->Wait( StaticPrefs::gfx_offscreencanvas_snapshot_timeout_ms()); } else if (canvasElement) { // If we have a context, it is owned by the main thread. const auto* offscreenCanvas = canvasElement->GetOffscreenCanvas(); if (nsICanvasRenderingContextInternal* context = offscreenCanvas->GetContext()) { surface = context->GetFrontBufferSnapshot(/* requireAlphaPremult */ false); } } return TransformSurface(surface, hasAlpha, isAlphaPremult, originPos); } already_AddRefed OffscreenCanvasDisplayHelper::GetAsImage() { MOZ_ASSERT(NS_IsMainThread()); RefPtr surface = GetSurfaceSnapshot(); if (!surface) { return nullptr; } return MakeAndAddRef(surface); } UniquePtr OffscreenCanvasDisplayHelper::GetImageBuffer( int32_t* aOutFormat, gfx::IntSize* aOutImageSize) { RefPtr surface = GetSurfaceSnapshot(); if (!surface) { return nullptr; } RefPtr dataSurface = surface->GetDataSurface(); if (!dataSurface) { return nullptr; } *aOutFormat = imgIEncoder::INPUT_FORMAT_HOSTARGB; *aOutImageSize = dataSurface->GetSize(); UniquePtr imageBuffer = gfx::SurfaceToPackedBGRA(dataSurface); if (!imageBuffer) { return nullptr; } bool resistFingerprinting; nsICookieJarSettings* cookieJarSettings = nullptr; { MutexAutoLock lock(mMutex); if (mCanvasElement) { Document* doc = mCanvasElement->OwnerDoc(); resistFingerprinting = doc->ShouldResistFingerprinting(RFPTarget::CanvasRandomization); if (resistFingerprinting) { cookieJarSettings = doc->CookieJarSettings(); } } else { resistFingerprinting = nsContentUtils::ShouldResistFingerprinting( "Fallback", RFPTarget::CanvasRandomization); } } if (resistFingerprinting) { nsRFPService::RandomizePixels( cookieJarSettings, imageBuffer.get(), dataSurface->GetSize().width, dataSurface->GetSize().height, dataSurface->GetSize().width * dataSurface->GetSize().height * 4, gfx::SurfaceFormat::A8R8G8B8_UINT32); } return imageBuffer; } } // namespace mozilla::dom