diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 09:22:09 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 09:22:09 +0000 |
commit | 43a97878ce14b72f0981164f87f2e35e14151312 (patch) | |
tree | 620249daf56c0258faa40cbdcf9cfba06de2a846 /dom/media/imagecapture | |
parent | Initial commit. (diff) | |
download | firefox-43a97878ce14b72f0981164f87f2e35e14151312.tar.xz firefox-43a97878ce14b72f0981164f87f2e35e14151312.zip |
Adding upstream version 110.0.1.upstream/110.0.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/media/imagecapture')
-rw-r--r-- | dom/media/imagecapture/CaptureTask.cpp | 197 | ||||
-rw-r--r-- | dom/media/imagecapture/CaptureTask.h | 90 | ||||
-rw-r--r-- | dom/media/imagecapture/ImageCapture.cpp | 212 | ||||
-rw-r--r-- | dom/media/imagecapture/ImageCapture.h | 95 | ||||
-rw-r--r-- | dom/media/imagecapture/moz.build | 16 |
5 files changed, 610 insertions, 0 deletions
diff --git a/dom/media/imagecapture/CaptureTask.cpp b/dom/media/imagecapture/CaptureTask.cpp new file mode 100644 index 0000000000..d83de76f5e --- /dev/null +++ b/dom/media/imagecapture/CaptureTask.cpp @@ -0,0 +1,197 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "CaptureTask.h" +#include "gfxUtils.h" +#include "mozilla/dom/BlobImpl.h" +#include "mozilla/dom/ImageCapture.h" +#include "mozilla/dom/ImageCaptureError.h" +#include "mozilla/dom/ImageEncoder.h" +#include "mozilla/dom/MediaStreamTrack.h" +#include "mozilla/dom/VideoStreamTrack.h" +#include "mozilla/SchedulerGroup.h" +#include "nsThreadUtils.h" +#include "VideoSegment.h" + +namespace mozilla { + +class CaptureTask::MediaTrackEventListener : public MediaTrackListener { + public: + explicit MediaTrackEventListener(CaptureTask* aCaptureTask) + : mCaptureTask(aCaptureTask){}; + + // MediaTrackListener methods. + void NotifyEnded(MediaTrackGraph* aGraph) override { + mCaptureTask->PostTrackEndEvent(); + } + + private: + CaptureTask* mCaptureTask; +}; + +CaptureTask::CaptureTask(dom::ImageCapture* aImageCapture) + : mImageCapture(aImageCapture), + mEventListener(new MediaTrackEventListener(this)), + mImageGrabbedOrTrackEnd(false), + mPrincipalChanged(false) {} + +nsresult CaptureTask::TaskComplete(already_AddRefed<dom::BlobImpl> aBlobImpl, + nsresult aRv) { + MOZ_ASSERT(NS_IsMainThread()); + + DetachTrack(); + + nsresult rv; + RefPtr<dom::BlobImpl> blobImpl(aBlobImpl); + + // We have to set the parent because the blob has been generated with a valid + // one. + RefPtr<dom::Blob> blob; + if (blobImpl) { + blob = dom::Blob::Create(mImageCapture->GetOwnerGlobal(), blobImpl); + if (NS_WARN_IF(!blob)) { + return NS_ERROR_FAILURE; + } + } + + if (mPrincipalChanged) { + aRv = NS_ERROR_DOM_SECURITY_ERR; + IC_LOG("MediaStream principal should not change during TakePhoto()."); + } + + if (NS_SUCCEEDED(aRv)) { + rv = mImageCapture->PostBlobEvent(blob); + } else { + rv = + mImageCapture->PostErrorEvent(dom::ImageCaptureError::PHOTO_ERROR, aRv); + } + + // Ensure ImageCapture dereference on main thread here because the TakePhoto() + // sequences stopped here. + mImageCapture = nullptr; + + return rv; +} + +void CaptureTask::AttachTrack() { + MOZ_ASSERT(NS_IsMainThread()); + + dom::MediaStreamTrack* track = mImageCapture->GetVideoStreamTrack(); + track->AddPrincipalChangeObserver(this); + track->AddListener(mEventListener.get()); + track->AddDirectListener(this); +} + +void CaptureTask::DetachTrack() { + MOZ_ASSERT(NS_IsMainThread()); + + dom::MediaStreamTrack* track = mImageCapture->GetVideoStreamTrack(); + track->RemovePrincipalChangeObserver(this); + track->RemoveListener(mEventListener.get()); + track->RemoveDirectListener(this); +} + +void CaptureTask::PrincipalChanged(dom::MediaStreamTrack* aMediaStreamTrack) { + MOZ_ASSERT(NS_IsMainThread()); + mPrincipalChanged = true; +} + +void CaptureTask::NotifyRealtimeTrackData(MediaTrackGraph* aGraph, + TrackTime aTrackOffset, + const MediaSegment& aMedia) { + MOZ_ASSERT(aMedia.GetType() == MediaSegment::VIDEO); + const VideoSegment& video = static_cast<const VideoSegment&>(aMedia); + + // Callback for encoding complete, it calls on main thread. + class EncodeComplete : public dom::EncodeCompleteCallback { + public: + explicit EncodeComplete(CaptureTask* aTask) : mTask(aTask) {} + + bool CanBeDeletedOnAnyThread() override { + // EncodeComplete is used from the main thread only. + return false; + } + + nsresult ReceiveBlobImpl( + already_AddRefed<dom::BlobImpl> aBlobImpl) override { + MOZ_ASSERT(NS_IsMainThread()); + RefPtr<dom::BlobImpl> blobImpl(aBlobImpl); + mTask->TaskComplete(blobImpl.forget(), NS_OK); + mTask = nullptr; + return NS_OK; + } + + protected: + RefPtr<CaptureTask> mTask; + }; + + for (VideoSegment::ConstChunkIterator iter(video); !iter.IsEnded(); + iter.Next()) { + VideoChunk chunk = *iter; + + // Extract the first valid video frame. + VideoFrame frame; + if (chunk.IsNull()) { + continue; + } + + RefPtr<layers::Image> image; + if (chunk.mFrame.GetForceBlack()) { + // Create a black image. + image = VideoFrame::CreateBlackImage(chunk.mFrame.GetIntrinsicSize()); + } else { + image = chunk.mFrame.GetImage(); + } + if (!image) { + MOZ_ASSERT(image); + continue; + } + + bool wasGrabbed = mImageGrabbedOrTrackEnd.exchange(true); + if (wasGrabbed) { + return; + } + + // Encode image. + nsresult rv; + nsAutoString type(u"image/jpeg"_ns); + nsAutoString options; + rv = dom::ImageEncoder::ExtractDataFromLayersImageAsync( + type, options, false, image, false, new EncodeComplete(this)); + if (NS_FAILED(rv)) { + PostTrackEndEvent(); + } + } +} + +void CaptureTask::PostTrackEndEvent() { + bool wasGrabbed = mImageGrabbedOrTrackEnd.exchange(true); + if (wasGrabbed) { + return; + } + + // Got track end or finish event, stop the task. + class TrackEndRunnable : public Runnable { + public: + explicit TrackEndRunnable(CaptureTask* aTask) + : mozilla::Runnable("TrackEndRunnable"), mTask(aTask) {} + + NS_IMETHOD Run() override { + mTask->TaskComplete(nullptr, NS_ERROR_FAILURE); + mTask = nullptr; + return NS_OK; + } + + protected: + RefPtr<CaptureTask> mTask; + }; + + IC_LOG("Got MediaTrack track removed or finished event."); + nsCOMPtr<nsIRunnable> event = new TrackEndRunnable(this); + SchedulerGroup::Dispatch(TaskCategory::Other, event.forget()); +} + +} // namespace mozilla diff --git a/dom/media/imagecapture/CaptureTask.h b/dom/media/imagecapture/CaptureTask.h new file mode 100644 index 0000000000..efcd0e84b2 --- /dev/null +++ b/dom/media/imagecapture/CaptureTask.h @@ -0,0 +1,90 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef CAPTURETASK_H +#define CAPTURETASK_H + +#include "MediaTrackGraph.h" +#include "MediaTrackListener.h" +#include "PrincipalChangeObserver.h" + +namespace mozilla { + +namespace dom { +class BlobImpl; +class ImageCapture; +class MediaStreamTrack; +} // namespace dom + +/** + * CaptureTask retrieves image from MediaTrack and encodes the image to jpeg in + * ImageEncoder. The whole procedures start at AttachTrack(), it will add this + * class into MediaTrack and retrieves an image in MediaTrackGraph thread. + * Once the image is retrieved, it will be sent to ImageEncoder and the encoded + * blob will be sent out via encoder callback in main thread. + * + * CaptureTask holds a reference of ImageCapture to ensure ImageCapture won't be + * released during the period of the capturing process described above. + */ +class CaptureTask : public DirectMediaTrackListener, + public dom::PrincipalChangeObserver<dom::MediaStreamTrack> { + public: + class MediaTrackEventListener; + + // DirectMediaTrackListener methods + void NotifyRealtimeTrackData(MediaTrackGraph* aGraph, TrackTime aTrackOffset, + const MediaSegment& aMedia) override; + + // PrincipalChangeObserver<MediaStreamTrack> methods + void PrincipalChanged(dom::MediaStreamTrack* aMediaStreamTrack) override; + + // CaptureTask methods. + + // It is called when aBlob is ready to post back to script in company with + // aRv == NS_OK. If aRv is not NS_OK, it will post an error event to script. + // + // Note: + // this function should be called on main thread. + nsresult TaskComplete(already_AddRefed<dom::BlobImpl> aBlobImpl, + nsresult aRv); + + // Add listeners into MediaStreamTrack and PrincipalChangeObserver. + // It should be on main thread only. + void AttachTrack(); + + // Remove listeners from MediaStreamTrack and PrincipalChangeObserver. + // It should be on main thread only. + void DetachTrack(); + + // CaptureTask should be created on main thread. + explicit CaptureTask(dom::ImageCapture* aImageCapture); + + protected: + virtual ~CaptureTask() = default; + + // Post a runnable on main thread to end this task and call TaskComplete to + // post error event to script. It is called off-main-thread. + void PostTrackEndEvent(); + + // The ImageCapture associates with this task. This reference count should not + // change in this class unless it clears this reference after a blob or error + // event to script. + RefPtr<dom::ImageCapture> mImageCapture; + + RefPtr<MediaTrackEventListener> mEventListener; + + // True when an image is retrieved from the video track, or MediaTrackGraph + // sends a track finish, end, or removed event. Any thread. + Atomic<bool> mImageGrabbedOrTrackEnd; + + // True after MediaStreamTrack principal changes while waiting for a photo + // to finish and we should raise a security error. + bool mPrincipalChanged; +}; + +} // namespace mozilla + +#endif // CAPTURETASK_H diff --git a/dom/media/imagecapture/ImageCapture.cpp b/dom/media/imagecapture/ImageCapture.cpp new file mode 100644 index 0000000000..0e71aa5603 --- /dev/null +++ b/dom/media/imagecapture/ImageCapture.cpp @@ -0,0 +1,212 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "ImageCapture.h" +#include "mozilla/dom/BlobEvent.h" +#include "mozilla/dom/DOMException.h" +#include "mozilla/dom/Event.h" +#include "mozilla/dom/File.h" +#include "mozilla/dom/ImageCaptureError.h" +#include "mozilla/dom/ImageCaptureErrorEvent.h" +#include "mozilla/dom/ImageCaptureErrorEventBinding.h" +#include "mozilla/dom/VideoStreamTrack.h" +#include "mozilla/dom/Document.h" +#include "CaptureTask.h" +#include "MediaEngineSource.h" + +namespace mozilla { + +LogModule* GetICLog() { + static LazyLogModule log("ImageCapture"); + return log; +} + +namespace dom { + +NS_IMPL_CYCLE_COLLECTION_INHERITED(ImageCapture, DOMEventTargetHelper, mTrack) + +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(ImageCapture) +NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper) + +NS_IMPL_ADDREF_INHERITED(ImageCapture, DOMEventTargetHelper) +NS_IMPL_RELEASE_INHERITED(ImageCapture, DOMEventTargetHelper) + +ImageCapture::ImageCapture(VideoStreamTrack* aTrack, + nsPIDOMWindowInner* aOwnerWindow) + : DOMEventTargetHelper(aOwnerWindow), mTrack(aTrack) { + MOZ_ASSERT(aOwnerWindow); + MOZ_ASSERT(aTrack); +} + +ImageCapture::~ImageCapture() { MOZ_ASSERT(NS_IsMainThread()); } + +already_AddRefed<ImageCapture> ImageCapture::Constructor( + const GlobalObject& aGlobal, MediaStreamTrack& aTrack, ErrorResult& aRv) { + nsCOMPtr<nsPIDOMWindowInner> win = do_QueryInterface(aGlobal.GetAsSupports()); + if (!win) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + if (!aTrack.AsVideoStreamTrack()) { + aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); + return nullptr; + } + + RefPtr<ImageCapture> object = + new ImageCapture(aTrack.AsVideoStreamTrack(), win); + + return object.forget(); +} + +MediaStreamTrack* ImageCapture::GetVideoStreamTrack() const { return mTrack; } + +nsresult ImageCapture::TakePhotoByMediaEngine() { + // Callback for TakPhoto(), it also monitor the principal. If principal + // changes, it returns PHOTO_ERROR with security error. + class TakePhotoCallback : public MediaEnginePhotoCallback, + public PrincipalChangeObserver<MediaStreamTrack> { + public: + TakePhotoCallback(VideoStreamTrack* aVideoTrack, + ImageCapture* aImageCapture) + : mVideoTrack(aVideoTrack), + mImageCapture(aImageCapture), + mPrincipalChanged(false) { + MOZ_ASSERT(NS_IsMainThread()); + mVideoTrack->AddPrincipalChangeObserver(this); + } + + void PrincipalChanged(MediaStreamTrack* aMediaStream) override { + mPrincipalChanged = true; + } + + nsresult PhotoComplete(already_AddRefed<Blob> aBlob) override { + RefPtr<Blob> blob = aBlob; + + if (mPrincipalChanged) { + return PhotoError(NS_ERROR_DOM_SECURITY_ERR); + } + return mImageCapture->PostBlobEvent(blob); + } + + nsresult PhotoError(nsresult aRv) override { + return mImageCapture->PostErrorEvent(ImageCaptureError::PHOTO_ERROR, aRv); + } + + protected: + ~TakePhotoCallback() { + MOZ_ASSERT(NS_IsMainThread()); + mVideoTrack->RemovePrincipalChangeObserver(this); + } + + const RefPtr<VideoStreamTrack> mVideoTrack; + const RefPtr<ImageCapture> mImageCapture; + bool mPrincipalChanged; + }; + + RefPtr<MediaEnginePhotoCallback> callback = + new TakePhotoCallback(mTrack, this); + return mTrack->GetSource().TakePhoto(callback); +} + +void ImageCapture::TakePhoto(ErrorResult& aResult) { + // According to spec, MediaStreamTrack.readyState must be "live"; however + // gecko doesn't implement it yet (bug 910249). Instead of readyState, we + // check MediaStreamTrack.enable before bug 910249 is fixed. + // The error code should be INVALID_TRACK, but spec doesn't define it in + // ImageCaptureError. So it returns PHOTO_ERROR here before spec updates. + if (!mTrack->Enabled()) { + PostErrorEvent(ImageCaptureError::PHOTO_ERROR, NS_ERROR_FAILURE); + return; + } + + // Try if MediaEngine supports taking photo. + nsresult rv = TakePhotoByMediaEngine(); + + // It falls back to MediaTrackGraph image capture if MediaEngine doesn't + // support TakePhoto(). + if (rv == NS_ERROR_NOT_IMPLEMENTED) { + IC_LOG( + "MediaEngine doesn't support TakePhoto(), it falls back to " + "MediaTrackGraph."); + RefPtr<CaptureTask> task = new CaptureTask(this); + + // It adds itself into MediaTrackGraph, so ImageCapture doesn't need to + // hold the reference. + task->AttachTrack(); + } +} + +nsresult ImageCapture::PostBlobEvent(Blob* aBlob) { + MOZ_ASSERT(NS_IsMainThread()); + if (!CheckPrincipal()) { + // Media is not same-origin, don't allow the data out. + return PostErrorEvent(ImageCaptureError::PHOTO_ERROR, + NS_ERROR_DOM_SECURITY_ERR); + } + + BlobEventInit init; + init.mBubbles = false; + init.mCancelable = false; + init.mData = aBlob; + + RefPtr<BlobEvent> blob_event = + BlobEvent::Constructor(this, u"photo"_ns, init); + + return DispatchTrustedEvent(blob_event); +} + +nsresult ImageCapture::PostErrorEvent(uint16_t aErrorCode, nsresult aReason) { + MOZ_ASSERT(NS_IsMainThread()); + nsresult rv = CheckCurrentGlobalCorrectness(); + NS_ENSURE_SUCCESS(rv, rv); + + nsString errorMsg; + if (NS_FAILED(aReason)) { + nsCString name, message; + rv = NS_GetNameAndMessageForDOMNSResult(aReason, name, message); + if (NS_SUCCEEDED(rv)) { + CopyASCIItoUTF16(message, errorMsg); + } + } + + RefPtr<ImageCaptureError> error = + new ImageCaptureError(this, aErrorCode, errorMsg); + + ImageCaptureErrorEventInit init; + init.mBubbles = false; + init.mCancelable = false; + init.mImageCaptureError = error; + + RefPtr<Event> event = + ImageCaptureErrorEvent::Constructor(this, u"error"_ns, init); + + return DispatchTrustedEvent(event); +} + +bool ImageCapture::CheckPrincipal() { + MOZ_ASSERT(NS_IsMainThread()); + + nsCOMPtr<nsIPrincipal> principal = mTrack->GetPrincipal(); + + if (!GetOwner()) { + return false; + } + nsCOMPtr<Document> doc = GetOwner()->GetExtantDoc(); + if (!doc || !principal) { + return false; + } + + bool subsumes; + if (NS_FAILED(doc->NodePrincipal()->Subsumes(principal, &subsumes))) { + return false; + } + + return subsumes; +} + +} // namespace dom +} // namespace mozilla diff --git a/dom/media/imagecapture/ImageCapture.h b/dom/media/imagecapture/ImageCapture.h new file mode 100644 index 0000000000..957a3ffccc --- /dev/null +++ b/dom/media/imagecapture/ImageCapture.h @@ -0,0 +1,95 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef IMAGECAPTURE_H +#define IMAGECAPTURE_H + +#include "mozilla/DOMEventTargetHelper.h" +#include "mozilla/dom/ImageCaptureBinding.h" +#include "mozilla/Logging.h" + +namespace mozilla { + +#ifndef IC_LOG +LogModule* GetICLog(); +# define IC_LOG(...) \ + MOZ_LOG(GetICLog(), mozilla::LogLevel::Debug, (__VA_ARGS__)) +#endif + +namespace dom { + +class Blob; +class MediaStreamTrack; +class VideoStreamTrack; + +/** + * Implementation of https://dvcs.w3.org/hg/dap/raw-file/default/media-stream- + * capture/ImageCapture.html. + * The ImageCapture accepts a video MediaStreamTrack as input source. The image + * will be sent back as a JPG format via Blob event. + * + * All the functions in ImageCapture are run in main thread. + * + * There are two ways to capture image, MediaEngineSource and MediaTrackGraph. + * When the implementation of MediaEngineSource supports TakePhoto(), + * it uses the platform camera to grab image. Otherwise, it falls back + * to the MediaTrackGraph way. + */ + +class ImageCapture final : public DOMEventTargetHelper { + public: + NS_DECL_ISUPPORTS_INHERITED + NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(ImageCapture, DOMEventTargetHelper) + + IMPL_EVENT_HANDLER(photo) + IMPL_EVENT_HANDLER(error) + + // WebIDL members. + void TakePhoto(ErrorResult& aResult); + + // The MediaStreamTrack passed into the constructor. + MediaStreamTrack* GetVideoStreamTrack() const; + + // nsWrapperCache member + JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override { + return ImageCapture_Binding::Wrap(aCx, this, aGivenProto); + } + + // ImageCapture class members + nsPIDOMWindowInner* GetParentObject() { return GetOwner(); } + + static already_AddRefed<ImageCapture> Constructor(const GlobalObject& aGlobal, + MediaStreamTrack& aTrack, + ErrorResult& aRv); + + ImageCapture(VideoStreamTrack* aTrack, nsPIDOMWindowInner* aOwnerWindow); + + // Post a Blob event to script. + nsresult PostBlobEvent(Blob* aBlob); + + // Post an error event to script. + // aErrorCode should be one of error codes defined in ImageCaptureError.h. + // aReason is the nsresult which maps to a error string in + // dom/base/domerr.msg. + nsresult PostErrorEvent(uint16_t aErrorCode, nsresult aReason = NS_OK); + + bool CheckPrincipal(); + + protected: + virtual ~ImageCapture(); + + // Capture image by MediaEngine. If it's not support taking photo, this + // function should return NS_ERROR_NOT_IMPLEMENTED. + nsresult TakePhotoByMediaEngine(); + + RefPtr<VideoStreamTrack> mTrack; +}; + +} // namespace dom +} // namespace mozilla + +#endif // IMAGECAPTURE_H diff --git a/dom/media/imagecapture/moz.build b/dom/media/imagecapture/moz.build new file mode 100644 index 0000000000..fdbc73430d --- /dev/null +++ b/dom/media/imagecapture/moz.build @@ -0,0 +1,16 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXPORTS.mozilla.dom += ["ImageCapture.h"] + +UNIFIED_SOURCES += [ + "CaptureTask.cpp", + "ImageCapture.cpp", +] + +include("/ipc/chromium/chromium-config.mozbuild") + +FINAL_LIBRARY = "xul" |