diff options
Diffstat (limited to 'dom/media/systemservices')
65 files changed, 10952 insertions, 0 deletions
diff --git a/dom/media/systemservices/CamerasChild.cpp b/dom/media/systemservices/CamerasChild.cpp new file mode 100644 index 0000000000..3b9ce64e46 --- /dev/null +++ b/dom/media/systemservices/CamerasChild.cpp @@ -0,0 +1,535 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "CamerasChild.h" + +#undef FF + +#include "mozilla/Assertions.h" +#include "mozilla/ipc/BackgroundChild.h" +#include "mozilla/ipc/PBackgroundChild.h" +#include "mozilla/Logging.h" +#include "mozilla/SyncRunnable.h" +#include "mozilla/WeakPtr.h" +#include "mozilla/Unused.h" +#include "MediaUtils.h" +#include "nsThreadUtils.h" + +#undef LOG +#undef LOG_ENABLED +mozilla::LazyLogModule gCamerasChildLog("CamerasChild"); +#define LOG(args) MOZ_LOG(gCamerasChildLog, mozilla::LogLevel::Debug, args) +#define LOG_ENABLED() MOZ_LOG_TEST(gCamerasChildLog, mozilla::LogLevel::Debug) + +namespace mozilla::camera { + +CamerasSingleton::CamerasSingleton() + : mCamerasMutex("CamerasSingleton::mCamerasMutex"), + mCameras(nullptr), + mCamerasChildThread(nullptr) { + LOG(("CamerasSingleton: %p", this)); +} + +CamerasSingleton::~CamerasSingleton() { LOG(("~CamerasSingleton: %p", this)); } + +class InitializeIPCThread : public Runnable { + public: + InitializeIPCThread() + : Runnable("camera::InitializeIPCThread"), mCamerasChild(nullptr) {} + + NS_IMETHOD Run() override { + // Try to get the PBackground handle + ipc::PBackgroundChild* existingBackgroundChild = + ipc::BackgroundChild::GetForCurrentThread(); + // If it's not spun up yet, block until it is, and retry + if (!existingBackgroundChild) { + LOG(("No existingBackgroundChild")); + existingBackgroundChild = + ipc::BackgroundChild::GetOrCreateForCurrentThread(); + LOG(("BackgroundChild: %p", existingBackgroundChild)); + if (!existingBackgroundChild) { + return NS_ERROR_FAILURE; + } + } + + // Create CamerasChild + // We will be returning the resulting pointer (synchronously) to our caller. + mCamerasChild = static_cast<mozilla::camera::CamerasChild*>( + existingBackgroundChild->SendPCamerasConstructor()); + + return NS_OK; + } + + CamerasChild* GetCamerasChild() { return mCamerasChild; } + + private: + CamerasChild* mCamerasChild; +}; + +CamerasChild* GetCamerasChild() { + CamerasSingleton::Mutex().AssertCurrentThreadOwns(); + if (!CamerasSingleton::Child()) { + MOZ_ASSERT(!NS_IsMainThread(), "Should not be on the main Thread"); + MOZ_ASSERT(!CamerasSingleton::Thread()); + LOG(("No sCameras, setting up IPC Thread")); + nsresult rv = NS_NewNamedThread("Cameras IPC", + getter_AddRefs(CamerasSingleton::Thread())); + if (NS_FAILED(rv)) { + LOG(("Error launching IPC Thread")); + return nullptr; + } + + // At this point we are in the MediaManager thread, and the thread we are + // dispatching to is the specific Cameras IPC thread that was just made + // above, so now we will fire off a runnable to run + // BackgroundChild::GetOrCreateForCurrentThread there, while we + // block in this thread. + // We block until the following happens in the Cameras IPC thread: + // 1) Creation of PBackground finishes + // 2) Creation of PCameras finishes by sending a message to the parent + RefPtr<InitializeIPCThread> runnable = new InitializeIPCThread(); + RefPtr<SyncRunnable> sr = new SyncRunnable(runnable); + sr->DispatchToThread(CamerasSingleton::Thread()); + CamerasSingleton::Child() = runnable->GetCamerasChild(); + } + if (!CamerasSingleton::Child()) { + LOG(("Failed to set up CamerasChild, are we in shutdown?")); + } + return CamerasSingleton::Child(); +} + +CamerasChild* GetCamerasChildIfExists() { + OffTheBooksMutexAutoLock lock(CamerasSingleton::Mutex()); + return CamerasSingleton::Child(); +} + +mozilla::ipc::IPCResult CamerasChild::RecvReplyFailure(void) { + LOG(("%s", __PRETTY_FUNCTION__)); + MonitorAutoLock monitor(mReplyMonitor); + mReceivedReply = true; + mReplySuccess = false; + monitor.Notify(); + return IPC_OK(); +} + +mozilla::ipc::IPCResult CamerasChild::RecvReplySuccess(void) { + LOG(("%s", __PRETTY_FUNCTION__)); + MonitorAutoLock monitor(mReplyMonitor); + mReceivedReply = true; + mReplySuccess = true; + monitor.Notify(); + return IPC_OK(); +} + +mozilla::ipc::IPCResult CamerasChild::RecvReplyNumberOfCapabilities( + const int& capabilityCount) { + LOG(("%s", __PRETTY_FUNCTION__)); + MonitorAutoLock monitor(mReplyMonitor); + mReceivedReply = true; + mReplySuccess = true; + mReplyInteger = capabilityCount; + monitor.Notify(); + return IPC_OK(); +} + +// Helper function to dispatch calls to the IPC Thread and +// CamerasChild object. Takes the needed locks and dispatches. +// Takes a "failed" value and a reference to the output variable +// as parameters, will return the right one depending on whether +// dispatching succeeded. +// +// The LockAndDispatch object in the caller must stay alive until after any +// reply data has been retreived (mReplyInteger, etc) so that the data is +// protected by the ReplyMonitor/RequestMutex +template <class T = int> +class LockAndDispatch { + public: + LockAndDispatch(CamerasChild* aCamerasChild, const char* aRequestingFunc, + nsIRunnable* aRunnable, T aFailureValue, + const T& aSuccessValue) + : mCamerasChild(aCamerasChild), + mRequestingFunc(aRequestingFunc), + mRunnable(aRunnable), + mReplyLock(aCamerasChild->mReplyMonitor), + mRequestLock(aCamerasChild->mRequestMutex), + mSuccess(true), + mFailureValue(aFailureValue), + mSuccessValue(aSuccessValue) { + Dispatch(); + } + + T ReturnValue() const { + if (mSuccess) { + return mSuccessValue; + } else { + return mFailureValue; + } + } + + const bool& Success() const { return mSuccess; } + + private: + void Dispatch() { + if (!mCamerasChild->DispatchToParent(mRunnable, mReplyLock)) { + LOG(("Cameras dispatch for IPC failed in %s", mRequestingFunc)); + mSuccess = false; + } + } + + CamerasChild* mCamerasChild; + const char* mRequestingFunc; + nsIRunnable* mRunnable; + // Prevent concurrent use of the reply variables by holding + // the mReplyMonitor. Note that this is unlocked while waiting for + // the reply to be filled in, necessitating the additional mRequestLock/Mutex; + MonitorAutoLock mReplyLock; + MutexAutoLock mRequestLock; + bool mSuccess; + const T mFailureValue; + const T& mSuccessValue; +}; + +bool CamerasChild::DispatchToParent(nsIRunnable* aRunnable, + MonitorAutoLock& aMonitor) { + CamerasSingleton::Mutex().AssertCurrentThreadOwns(); + mReplyMonitor.AssertCurrentThreadOwns(); + CamerasSingleton::Thread()->Dispatch(aRunnable, NS_DISPATCH_NORMAL); + // Guard against spurious wakeups. + mReceivedReply = false; + // Wait for a reply + do { + // If the parent has been shut down, then we won't receive a reply. + if (!mIPCIsAlive) { + return false; + } + aMonitor.Wait(); + } while (!mReceivedReply); + return mReplySuccess; +} + +int CamerasChild::NumberOfCapabilities(CaptureEngine aCapEngine, + const char* deviceUniqueIdUTF8) { + LOG(("%s", __PRETTY_FUNCTION__)); + LOG(("NumberOfCapabilities for %s", deviceUniqueIdUTF8)); + nsCString unique_id(deviceUniqueIdUTF8); + nsCOMPtr<nsIRunnable> runnable = + mozilla::NewRunnableMethod<CaptureEngine, nsCString>( + "camera::PCamerasChild::SendNumberOfCapabilities", this, + &CamerasChild::SendNumberOfCapabilities, aCapEngine, unique_id); + LockAndDispatch<> dispatcher(this, __func__, runnable, 0, mReplyInteger); + LOG(("Capture capability count: %d", dispatcher.ReturnValue())); + return dispatcher.ReturnValue(); +} + +int CamerasChild::NumberOfCaptureDevices(CaptureEngine aCapEngine) { + LOG(("%s", __PRETTY_FUNCTION__)); + nsCOMPtr<nsIRunnable> runnable = mozilla::NewRunnableMethod<CaptureEngine>( + "camera::PCamerasChild::SendNumberOfCaptureDevices", this, + &CamerasChild::SendNumberOfCaptureDevices, aCapEngine); + LockAndDispatch<> dispatcher(this, __func__, runnable, 0, mReplyInteger); + LOG(("Capture Devices: %d", dispatcher.ReturnValue())); + return dispatcher.ReturnValue(); +} + +mozilla::ipc::IPCResult CamerasChild::RecvReplyNumberOfCaptureDevices( + const int& aDeviceCount) { + LOG(("%s", __PRETTY_FUNCTION__)); + MonitorAutoLock monitor(mReplyMonitor); + mReceivedReply = true; + mReplySuccess = true; + mReplyInteger = aDeviceCount; + monitor.Notify(); + return IPC_OK(); +} + +int CamerasChild::EnsureInitialized(CaptureEngine aCapEngine) { + LOG(("%s", __PRETTY_FUNCTION__)); + nsCOMPtr<nsIRunnable> runnable = mozilla::NewRunnableMethod<CaptureEngine>( + "camera::PCamerasChild::SendEnsureInitialized", this, + &CamerasChild::SendEnsureInitialized, aCapEngine); + LockAndDispatch<> dispatcher(this, __func__, runnable, 0, mReplyInteger); + LOG(("Capture Devices: %d", dispatcher.ReturnValue())); + return dispatcher.ReturnValue(); +} + +int CamerasChild::GetCaptureCapability( + CaptureEngine aCapEngine, const char* unique_idUTF8, + const unsigned int capability_number, + webrtc::VideoCaptureCapability* capability) { + LOG(("GetCaptureCapability: %s %d", unique_idUTF8, capability_number)); + MOZ_ASSERT(capability); + nsCString unique_id(unique_idUTF8); + nsCOMPtr<nsIRunnable> runnable = + mozilla::NewRunnableMethod<CaptureEngine, nsCString, unsigned int>( + "camera::PCamerasChild::SendGetCaptureCapability", this, + &CamerasChild::SendGetCaptureCapability, aCapEngine, unique_id, + capability_number); + mReplyCapability = capability; + LockAndDispatch<> dispatcher(this, __func__, runnable, -1, mZero); + mReplyCapability = nullptr; + return dispatcher.ReturnValue(); +} + +mozilla::ipc::IPCResult CamerasChild::RecvReplyGetCaptureCapability( + const VideoCaptureCapability& ipcCapability) { + LOG(("%s", __PRETTY_FUNCTION__)); + MonitorAutoLock monitor(mReplyMonitor); + mReceivedReply = true; + mReplySuccess = true; + mReplyCapability->width = ipcCapability.width(); + mReplyCapability->height = ipcCapability.height(); + mReplyCapability->maxFPS = ipcCapability.maxFPS(); + mReplyCapability->videoType = + static_cast<webrtc::VideoType>(ipcCapability.videoType()); + mReplyCapability->interlaced = ipcCapability.interlaced(); + monitor.Notify(); + return IPC_OK(); +} + +int CamerasChild::GetCaptureDevice( + CaptureEngine aCapEngine, unsigned int list_number, char* device_nameUTF8, + const unsigned int device_nameUTF8Length, char* unique_idUTF8, + const unsigned int unique_idUTF8Length, bool* scary) { + LOG(("%s", __PRETTY_FUNCTION__)); + nsCOMPtr<nsIRunnable> runnable = + mozilla::NewRunnableMethod<CaptureEngine, unsigned int>( + "camera::PCamerasChild::SendGetCaptureDevice", this, + &CamerasChild::SendGetCaptureDevice, aCapEngine, list_number); + LockAndDispatch<> dispatcher(this, __func__, runnable, -1, mZero); + if (dispatcher.Success()) { + base::strlcpy(device_nameUTF8, mReplyDeviceName.get(), + device_nameUTF8Length); + base::strlcpy(unique_idUTF8, mReplyDeviceID.get(), unique_idUTF8Length); + if (scary) { + *scary = mReplyScary; + } + LOG(("Got %s name %s id", device_nameUTF8, unique_idUTF8)); + } + return dispatcher.ReturnValue(); +} + +mozilla::ipc::IPCResult CamerasChild::RecvReplyGetCaptureDevice( + const nsACString& device_name, const nsACString& device_id, + const bool& scary) { + LOG(("%s", __PRETTY_FUNCTION__)); + MonitorAutoLock monitor(mReplyMonitor); + mReceivedReply = true; + mReplySuccess = true; + mReplyDeviceName = device_name; + mReplyDeviceID = device_id; + mReplyScary = scary; + monitor.Notify(); + return IPC_OK(); +} + +int CamerasChild::AllocateCapture(CaptureEngine aCapEngine, + const char* unique_idUTF8, + uint64_t aWindowID) { + LOG(("%s", __PRETTY_FUNCTION__)); + nsCString unique_id(unique_idUTF8); + nsCOMPtr<nsIRunnable> runnable = + mozilla::NewRunnableMethod<CaptureEngine, nsCString, const uint64_t&>( + "camera::PCamerasChild::SendAllocateCapture", this, + &CamerasChild::SendAllocateCapture, aCapEngine, unique_id, aWindowID); + LockAndDispatch<> dispatcher(this, __func__, runnable, -1, mReplyInteger); + if (dispatcher.Success()) { + LOG(("Capture Device allocated: %d", mReplyInteger)); + } + return dispatcher.ReturnValue(); +} + +mozilla::ipc::IPCResult CamerasChild::RecvReplyAllocateCapture( + const int& aCaptureId) { + LOG(("%s", __PRETTY_FUNCTION__)); + MonitorAutoLock monitor(mReplyMonitor); + mReceivedReply = true; + mReplySuccess = true; + mReplyInteger = aCaptureId; + monitor.Notify(); + return IPC_OK(); +} + +int CamerasChild::ReleaseCapture(CaptureEngine aCapEngine, + const int capture_id) { + LOG(("%s", __PRETTY_FUNCTION__)); + nsCOMPtr<nsIRunnable> runnable = + mozilla::NewRunnableMethod<CaptureEngine, int>( + "camera::PCamerasChild::SendReleaseCapture", this, + &CamerasChild::SendReleaseCapture, aCapEngine, capture_id); + LockAndDispatch<> dispatcher(this, __func__, runnable, -1, mZero); + return dispatcher.ReturnValue(); +} + +void CamerasChild::AddCallback(const CaptureEngine aCapEngine, + const int capture_id, FrameRelay* render) { + MutexAutoLock lock(mCallbackMutex); + CapturerElement ce; + ce.engine = aCapEngine; + ce.id = capture_id; + ce.callback = render; + mCallbacks.AppendElement(ce); +} + +void CamerasChild::RemoveCallback(const CaptureEngine aCapEngine, + const int capture_id) { + MutexAutoLock lock(mCallbackMutex); + for (unsigned int i = 0; i < mCallbacks.Length(); i++) { + CapturerElement ce = mCallbacks[i]; + if (ce.engine == aCapEngine && ce.id == capture_id) { + mCallbacks.RemoveElementAt(i); + break; + } + } +} + +int CamerasChild::StartCapture(CaptureEngine aCapEngine, const int capture_id, + const webrtc::VideoCaptureCapability& webrtcCaps, + FrameRelay* cb) { + LOG(("%s", __PRETTY_FUNCTION__)); + AddCallback(aCapEngine, capture_id, cb); + VideoCaptureCapability capCap( + webrtcCaps.width, webrtcCaps.height, webrtcCaps.maxFPS, + static_cast<int>(webrtcCaps.videoType), webrtcCaps.interlaced); + nsCOMPtr<nsIRunnable> runnable = + mozilla::NewRunnableMethod<CaptureEngine, int, VideoCaptureCapability>( + "camera::PCamerasChild::SendStartCapture", this, + &CamerasChild::SendStartCapture, aCapEngine, capture_id, capCap); + LockAndDispatch<> dispatcher(this, __func__, runnable, -1, mZero); + return dispatcher.ReturnValue(); +} + +int CamerasChild::FocusOnSelectedSource(CaptureEngine aCapEngine, + const int aCaptureId) { + LOG(("%s", __PRETTY_FUNCTION__)); + nsCOMPtr<nsIRunnable> runnable = + mozilla::NewRunnableMethod<CaptureEngine, int>( + "camera::PCamerasChild::SendFocusOnSelectedSource", this, + &CamerasChild::SendFocusOnSelectedSource, aCapEngine, aCaptureId); + LockAndDispatch<> dispatcher(this, __func__, runnable, -1, mZero); + return dispatcher.ReturnValue(); +} + +int CamerasChild::StopCapture(CaptureEngine aCapEngine, const int capture_id) { + LOG(("%s", __PRETTY_FUNCTION__)); + nsCOMPtr<nsIRunnable> runnable = + mozilla::NewRunnableMethod<CaptureEngine, int>( + "camera::PCamerasChild::SendStopCapture", this, + &CamerasChild::SendStopCapture, aCapEngine, capture_id); + LockAndDispatch<> dispatcher(this, __func__, runnable, -1, mZero); + if (dispatcher.Success()) { + RemoveCallback(aCapEngine, capture_id); + } + return dispatcher.ReturnValue(); +} + +class ShutdownRunnable : public Runnable { + public: + explicit ShutdownRunnable(already_AddRefed<Runnable>&& aReplyEvent) + : Runnable("camera::ShutdownRunnable"), mReplyEvent(aReplyEvent){}; + + NS_IMETHOD Run() override { + LOG(("Closing BackgroundChild")); + // This will also destroy the CamerasChild. + ipc::BackgroundChild::CloseForCurrentThread(); + + NS_DispatchToMainThread(mReplyEvent.forget()); + + return NS_OK; + } + + private: + RefPtr<Runnable> mReplyEvent; +}; + +void Shutdown(void) { + // Called from both MediaEngineWebRTC::Shutdown() on the MediaManager thread + // and DeallocPCamerasChild() on the dedicated IPC thread. + OffTheBooksMutexAutoLock lock(CamerasSingleton::Mutex()); + + CamerasChild* child = CamerasSingleton::Child(); + if (!child) { + // We don't want to cause everything to get fired up if we're + // really already shut down. + LOG(("Shutdown when already shut down")); + return; + } + if (CamerasSingleton::Thread()) { + LOG(("PBackground thread exists, dispatching close")); + // The IPC thread is shut down on the main thread after the + // BackgroundChild is closed. + RefPtr<ShutdownRunnable> runnable = new ShutdownRunnable( + NewRunnableMethod("nsIThread::Shutdown", CamerasSingleton::Thread(), + &nsIThread::Shutdown)); + CamerasSingleton::Thread()->Dispatch(runnable.forget(), NS_DISPATCH_NORMAL); + } else { + LOG(("Shutdown called without PBackground thread")); + } + LOG(("Erasing sCameras & thread refs (original thread)")); + CamerasSingleton::Child() = nullptr; + CamerasSingleton::Thread() = nullptr; +} + +mozilla::ipc::IPCResult CamerasChild::RecvDeliverFrame( + const CaptureEngine& capEngine, const int& capId, + mozilla::ipc::Shmem&& shmem, const VideoFrameProperties& prop) { + MutexAutoLock lock(mCallbackMutex); + if (Callback(capEngine, capId)) { + unsigned char* image = shmem.get<unsigned char>(); + Callback(capEngine, capId)->DeliverFrame(image, prop); + } else { + LOG(("DeliverFrame called with dead callback")); + } + SendReleaseFrame(std::move(shmem)); + return IPC_OK(); +} + +mozilla::ipc::IPCResult CamerasChild::RecvDeviceChange() { + mDeviceListChangeEvent.Notify(); + return IPC_OK(); +} + +void CamerasChild::ActorDestroy(ActorDestroyReason aWhy) { + LOG(("ActorDestroy")); + MonitorAutoLock monitor(mReplyMonitor); + mIPCIsAlive = false; + // Hopefully prevent us from getting stuck + // on replies that'll never come. + monitor.NotifyAll(); +} + +CamerasChild::CamerasChild() + : mCallbackMutex("mozilla::cameras::CamerasChild::mCallbackMutex"), + mIPCIsAlive(true), + mRequestMutex("mozilla::cameras::CamerasChild::mRequestMutex"), + mReplyMonitor("mozilla::cameras::CamerasChild::mReplyMonitor"), + mReceivedReply(false), + mReplySuccess(false), + mZero(0), + mReplyInteger(0), + mReplyScary(false) { + LOG(("CamerasChild: %p", this)); + + MOZ_COUNT_CTOR(CamerasChild); +} + +CamerasChild::~CamerasChild() { + LOG(("~CamerasChild: %p", this)); + CamerasSingleton::AssertNoChild(); + MOZ_COUNT_DTOR(CamerasChild); +} + +FrameRelay* CamerasChild::Callback(CaptureEngine aCapEngine, int capture_id) { + for (unsigned int i = 0; i < mCallbacks.Length(); i++) { + CapturerElement ce = mCallbacks[i]; + if (ce.engine == aCapEngine && ce.id == capture_id) { + return ce.callback; + } + } + + return nullptr; +} + +} // namespace mozilla::camera diff --git a/dom/media/systemservices/CamerasChild.h b/dom/media/systemservices/CamerasChild.h new file mode 100644 index 0000000000..18bdeec251 --- /dev/null +++ b/dom/media/systemservices/CamerasChild.h @@ -0,0 +1,262 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_CamerasChild_h +#define mozilla_CamerasChild_h + +#include <utility> + +#include "MediaEventSource.h" +#include "mozilla/Mutex.h" +#include "mozilla/camera/PCamerasChild.h" +#include "mozilla/camera/PCamerasParent.h" +#include "nsCOMPtr.h" + +// conflicts with #include of scoped_ptr.h +#undef FF +#include "modules/video_capture/video_capture_defines.h" + +namespace mozilla { + +namespace ipc { +class BackgroundChildImpl; +} // namespace ipc + +namespace camera { + +class FrameRelay { + public: + virtual int DeliverFrame( + uint8_t* buffer, const mozilla::camera::VideoFrameProperties& props) = 0; +}; + +struct CapturerElement { + CaptureEngine engine; + int id; + FrameRelay* callback; +}; + +// Forward declaration so we can work with pointers to it. +class CamerasChild; +// Helper class in impl that we friend. +template <class T> +class LockAndDispatch; + +// We emulate the sync webrtc.org API with the help of singleton +// CamerasSingleton, which manages a pointer to an IPC object, a thread +// where IPC operations should run on, and a mutex. +// The static function Cameras() will use that Singleton to set up, +// if needed, both the thread and the associated IPC objects and return +// a pointer to the IPC object. Users can then do IPC calls on that object +// after dispatching them to aforementioned thread. + +// 2 Threads are involved in this code: +// - the MediaManager thread, which will call the (static, sync API) functions +// through MediaEngineRemoteVideoSource +// - the Cameras IPC thread, which will be doing our IPC to the parent process +// via PBackground + +// Our main complication is that we emulate a sync API while (having to do) +// async messaging. We dispatch the messages to another thread to send them +// async and hold a Monitor to wait for the result to be asynchronously received +// again. The requirement for async messaging originates on the parent side: +// it's not reasonable to block all PBackground IPC there while waiting for +// something like device enumeration to complete. + +class CamerasSingleton { + public: + static OffTheBooksMutex& Mutex() { return singleton().mCamerasMutex; } + + static CamerasChild*& Child() { + Mutex().AssertCurrentThreadOwns(); + return singleton().mCameras; + } + + static nsCOMPtr<nsIThread>& Thread() { + Mutex().AssertCurrentThreadOwns(); + return singleton().mCamerasChildThread; + } + // The mutex is not held because mCameras is known not to be modified + // concurrently when this is asserted. + static void AssertNoChild() { MOZ_ASSERT(!singleton().mCameras); } + + private: + CamerasSingleton(); + ~CamerasSingleton(); + + static CamerasSingleton& singleton() { + static CamerasSingleton camera; + return camera; + } + + // Reinitializing CamerasChild will change the pointers below. + // We don't want this to happen in the middle of preparing IPC. + // We will be alive on destruction, so this needs to be off the books. + mozilla::OffTheBooksMutex mCamerasMutex; + + // This is owned by the IPC code, and the same code controls the lifetime. + // It will set and clear this pointer as appropriate in setup/teardown. + // We'd normally make this a WeakPtr but unfortunately the IPC code already + // uses the WeakPtr mixin in a protected base class of CamerasChild, and in + // any case the object becomes unusable as soon as IPC is tearing down, which + // will be before actual destruction. + CamerasChild* mCameras; + nsCOMPtr<nsIThread> mCamerasChildThread; +}; + +// Get a pointer to a CamerasChild object we can use to do IPC with. +// This does everything needed to set up, including starting the IPC +// channel with PBackground, blocking until thats done, and starting the +// thread to do IPC on. This will fail if we're in shutdown. On success +// it will set up the CamerasSingleton. +CamerasChild* GetCamerasChild(); + +CamerasChild* GetCamerasChildIfExists(); + +// Shut down the IPC channel and everything associated, like WebRTC. +// This is a static call because the CamerasChild object may not even +// be alive when we're called. +void Shutdown(void); + +// Obtain the CamerasChild object (if possible, i.e. not shutting down), +// and maintain a grip on the object for the duration of the call. +template <class MEM_FUN, class... ARGS> +int GetChildAndCall(MEM_FUN&& f, ARGS&&... args) { + OffTheBooksMutexAutoLock lock(CamerasSingleton::Mutex()); + CamerasChild* child = GetCamerasChild(); + if (child) { + return (child->*f)(std::forward<ARGS>(args)...); + } else { + return -1; + } +} + +class CamerasChild final : public PCamerasChild { + friend class mozilla::ipc::BackgroundChildImpl; + template <class T> + friend class mozilla::camera::LockAndDispatch; + + public: + // We are owned by the PBackground thread only. CamerasSingleton + // takes a non-owning reference. + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(CamerasChild) + + // IPC messages recevied, received on the PBackground thread + // these are the actual callbacks with data + mozilla::ipc::IPCResult RecvDeliverFrame( + const CaptureEngine&, const int&, mozilla::ipc::Shmem&&, + const VideoFrameProperties& prop) override; + + mozilla::ipc::IPCResult RecvDeviceChange() override; + + // these are response messages to our outgoing requests + mozilla::ipc::IPCResult RecvReplyNumberOfCaptureDevices(const int&) override; + mozilla::ipc::IPCResult RecvReplyNumberOfCapabilities(const int&) override; + mozilla::ipc::IPCResult RecvReplyAllocateCapture(const int&) override; + mozilla::ipc::IPCResult RecvReplyGetCaptureCapability( + const VideoCaptureCapability& capability) override; + mozilla::ipc::IPCResult RecvReplyGetCaptureDevice( + const nsACString& device_name, const nsACString& device_id, + const bool& scary) override; + mozilla::ipc::IPCResult RecvReplyFailure(void) override; + mozilla::ipc::IPCResult RecvReplySuccess(void) override; + void ActorDestroy(ActorDestroyReason aWhy) override; + + // the webrtc.org ViECapture calls are mirrored here, but with access + // to a specific PCameras instance to communicate over. These also + // run on the MediaManager thread + int NumberOfCaptureDevices(CaptureEngine aCapEngine); + int NumberOfCapabilities(CaptureEngine aCapEngine, + const char* deviceUniqueIdUTF8); + int ReleaseCapture(CaptureEngine aCapEngine, const int capture_id); + int StartCapture(CaptureEngine aCapEngine, const int capture_id, + const webrtc::VideoCaptureCapability& capability, + FrameRelay* func); + int FocusOnSelectedSource(CaptureEngine aCapEngine, const int capture_id); + int StopCapture(CaptureEngine aCapEngine, const int capture_id); + // Returns a non-negative capture identifier or -1 on failure. + int AllocateCapture(CaptureEngine aCapEngine, const char* unique_idUTF8, + uint64_t aWindowID); + int GetCaptureCapability(CaptureEngine aCapEngine, const char* unique_idUTF8, + const unsigned int capability_number, + webrtc::VideoCaptureCapability* capability); + int GetCaptureDevice(CaptureEngine aCapEngine, unsigned int list_number, + char* device_nameUTF8, + const unsigned int device_nameUTF8Length, + char* unique_idUTF8, + const unsigned int unique_idUTF8Length, + bool* scary = nullptr); + int EnsureInitialized(CaptureEngine aCapEngine); + + template <typename This> + int ConnectDeviceListChangeListener(MediaEventListener* aListener, + AbstractThread* aTarget, This* aThis, + void (This::*aMethod)()) { + // According to the spec, if the script sets + // navigator.mediaDevices.ondevicechange and the permission state is + // "always granted", the User Agent MUST fires a devicechange event when + // a new media input device is made available, even the script never + // call getusermedia or enumerateDevices. + + // In order to detect the event, we need to init the camera engine. + // Currently EnsureInitialized(aCapEngine) is only called when one of + // CamerasParent api, e.g., RecvNumberOfCaptureDevices(), is called. + + // So here we setup camera engine via EnsureInitialized(aCapEngine) + + EnsureInitialized(CameraEngine); + *aListener = mDeviceListChangeEvent.Connect(aTarget, aThis, aMethod); + return IPC_OK(); + } + + FrameRelay* Callback(CaptureEngine aCapEngine, int capture_id); + + private: + CamerasChild(); + ~CamerasChild(); + // Dispatch a Runnable to the PCamerasParent, by executing it on the + // decidecated Cameras IPC/PBackground thread. + bool DispatchToParent(nsIRunnable* aRunnable, MonitorAutoLock& aMonitor); + void AddCallback(const CaptureEngine aCapEngine, const int capture_id, + FrameRelay* render); + void RemoveCallback(const CaptureEngine aCapEngine, const int capture_id); + + nsTArray<CapturerElement> mCallbacks; + // Protects the callback arrays + Mutex mCallbackMutex MOZ_UNANNOTATED; + + bool mIPCIsAlive; + + // Hold to prevent multiple outstanding requests. We don't use + // request IDs so we only support one at a time. Don't want try + // to use the webrtc.org API from multiple threads simultanously. + // The monitor below isn't sufficient for this, as it will drop + // the lock when Wait-ing for a response, allowing us to send a new + // request. The Notify on receiving the response will then unblock + // both waiters and one will be guaranteed to get the wrong result. + // Take this one before taking mReplyMonitor. + Mutex mRequestMutex MOZ_UNANNOTATED; + // Hold to wait for an async response to our calls *and* until the + // user of LockAndDispatch<> has read the data out. This is done by + // keeping the LockAndDispatch object alive. + Monitor mReplyMonitor MOZ_UNANNOTATED; + // Async response valid? + bool mReceivedReply; + // Async responses data contents; + bool mReplySuccess; + const int mZero; + int mReplyInteger; + webrtc::VideoCaptureCapability* mReplyCapability = nullptr; + nsCString mReplyDeviceName; + nsCString mReplyDeviceID; + bool mReplyScary; + MediaEventProducer<void> mDeviceListChangeEvent; +}; + +} // namespace camera +} // namespace mozilla + +#endif // mozilla_CamerasChild_h diff --git a/dom/media/systemservices/CamerasParent.cpp b/dom/media/systemservices/CamerasParent.cpp new file mode 100644 index 0000000000..1c63f0cee1 --- /dev/null +++ b/dom/media/systemservices/CamerasParent.cpp @@ -0,0 +1,1220 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "CamerasParent.h" + +#include <atomic> +#include "MediaEngineSource.h" +#include "PerformanceRecorder.h" +#include "VideoFrameUtils.h" + +#include "mozilla/AppShutdown.h" +#include "mozilla/Assertions.h" +#include "mozilla/BasePrincipal.h" +#include "mozilla/ProfilerMarkers.h" +#include "mozilla/Unused.h" +#include "mozilla/Services.h" +#include "mozilla/Logging.h" +#include "mozilla/ipc/BackgroundParent.h" +#include "mozilla/ipc/PBackgroundParent.h" +#include "mozilla/dom/CanonicalBrowsingContext.h" +#include "mozilla/dom/WindowGlobalParent.h" +#include "mozilla/media/MediaUtils.h" +#include "mozilla/Preferences.h" +#include "mozilla/StaticPrefs_permissions.h" +#include "nsIPermissionManager.h" +#include "nsIThread.h" +#include "nsThreadUtils.h" +#include "nsNetUtil.h" + +#include "api/video/video_frame_buffer.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" + +#if defined(_WIN32) +# include <process.h> +# define getpid() _getpid() +#endif + +#undef LOG +#undef LOG_VERBOSE +#undef LOG_ENABLED +mozilla::LazyLogModule gCamerasParentLog("CamerasParent"); +#define LOG(...) \ + MOZ_LOG(gCamerasParentLog, mozilla::LogLevel::Debug, (__VA_ARGS__)) +#define LOG_FUNCTION() \ + MOZ_LOG(gCamerasParentLog, mozilla::LogLevel::Debug, \ + ("CamerasParent(%p)::%s", this, __func__)) +#define LOG_VERBOSE(...) \ + MOZ_LOG(gCamerasParentLog, mozilla::LogLevel::Verbose, (__VA_ARGS__)) +#define LOG_ENABLED() MOZ_LOG_TEST(gCamerasParentLog, mozilla::LogLevel::Debug) + +namespace mozilla { +using media::ShutdownBlockingTicket; +namespace camera { + +std::map<uint32_t, const char*> sDeviceUniqueIDs; +std::map<uint32_t, webrtc::VideoCaptureCapability> sAllRequestedCapabilities; + +uint32_t ResolutionFeasibilityDistance(int32_t candidate, int32_t requested) { + // The purpose of this function is to find a smallest resolution + // which is larger than all requested capabilities. + // Then we can use down-scaling to fulfill each request. + + MOZ_DIAGNOSTIC_ASSERT(candidate >= 0, "Candidate unexpectedly negative"); + MOZ_DIAGNOSTIC_ASSERT(requested >= 0, "Requested unexpectedly negative"); + + if (candidate == 0) { + // Treat width|height capability of 0 as "can do any". + // This allows for orthogonal capabilities that are not in discrete steps. + return 0; + } + + uint32_t distance = + std::abs(candidate - requested) * 1000 / std::max(candidate, requested); + if (candidate >= requested) { + // This is a good case, the candidate covers the requested resolution. + return distance; + } + + // This is a bad case, the candidate is lower than the requested resolution. + // This is penalized with an added weight of 10000. + return 10000 + distance; +} + +uint32_t FeasibilityDistance(int32_t candidate, int32_t requested) { + MOZ_DIAGNOSTIC_ASSERT(candidate >= 0, "Candidate unexpectedly negative"); + MOZ_DIAGNOSTIC_ASSERT(requested >= 0, "Requested unexpectedly negative"); + + if (candidate == 0) { + // Treat maxFPS capability of 0 as "can do any". + // This allows for orthogonal capabilities that are not in discrete steps. + return 0; + } + + return std::abs(candidate - requested) * 1000 / + std::max(candidate, requested); +} + +class CamerasParent::VideoEngineArray + : public media::Refcountable<nsTArray<RefPtr<VideoEngine>>> {}; + +// Singleton video engines. The sEngines RefPtr is IPC background thread only +// and outlives the CamerasParent instances. The array elements are video +// capture thread only. +using VideoEngineArray = CamerasParent::VideoEngineArray; +static StaticRefPtr<VideoEngineArray> sEngines; +// Number of CamerasParents instances in the current process for which +// mVideoCaptureThread has been set. IPC background thread only. +static int32_t sNumCamerasParents = 0; +// Video processing thread - where webrtc.org capturer code runs. Outlives the +// CamerasParent instances. IPC background thread only. +static StaticRefPtr<nsIThread> sVideoCaptureThread; + +static already_AddRefed<nsISerialEventTarget> +MakeAndAddRefVideoCaptureThreadAndSingletons() { + ipc::AssertIsOnBackgroundThread(); + + MOZ_ASSERT_IF(sVideoCaptureThread, sNumCamerasParents > 0); + MOZ_ASSERT_IF(!sVideoCaptureThread, sNumCamerasParents == 0); + + if (!sVideoCaptureThread) { + LOG("Spinning up WebRTC Cameras Thread"); + nsIThreadManager::ThreadCreationOptions options; +#ifdef XP_WIN + // Windows desktop capture needs a UI thread + options.isUiThread = true; +#endif + nsCOMPtr<nsIThread> videoCaptureThread; + if (NS_FAILED(NS_NewNamedThread("VideoCapture", + getter_AddRefs(videoCaptureThread), nullptr, + options))) { + return nullptr; + } + sVideoCaptureThread = videoCaptureThread.forget(); + + sEngines = MakeRefPtr<VideoEngineArray>(); + sEngines->AppendElements(CaptureEngine::MaxEngine); + } + + ++sNumCamerasParents; + return do_AddRef(sVideoCaptureThread); +} + +static void ReleaseVideoCaptureThreadAndSingletons() { + ipc::AssertIsOnBackgroundThread(); + + if (--sNumCamerasParents > 0) { + // Other CamerasParent instances are using the singleton classes. + return; + } + + MOZ_ASSERT(sNumCamerasParents == 0, "Double release!"); + + // No other CamerasParent instances alive. Clean up. + LOG("Shutting down VideoEngines and the VideoCapture thread"); + MOZ_ALWAYS_SUCCEEDS(sVideoCaptureThread->Dispatch( + NS_NewRunnableFunction(__func__, [engines = RefPtr(sEngines.forget())] { + for (RefPtr<VideoEngine>& engine : *engines) { + if (engine) { + VideoEngine::Delete(engine); + engine = nullptr; + } + } + }))); + + MOZ_ALWAYS_SUCCEEDS(RefPtr(sVideoCaptureThread.forget())->AsyncShutdown()); +} + +// 3 threads are involved in this code: +// - the main thread for some setups, and occassionally for video capture setup +// calls that don't work correctly elsewhere. +// - the IPC thread on which PBackground is running and which receives and +// sends messages +// - a thread which will execute the actual (possibly slow) camera access +// called "VideoCapture". On Windows this is a thread with an event loop +// suitable for UI access. + +void CamerasParent::OnDeviceChange() { + LOG_FUNCTION(); + + mPBackgroundEventTarget->Dispatch( + NS_NewRunnableFunction(__func__, [this, self = RefPtr(this)]() { + if (IsShuttingDown()) { + LOG("OnDeviceChanged failure: parent shutting down."); + return; + } + Unused << SendDeviceChange(); + })); +}; + +class DeliverFrameRunnable : public mozilla::Runnable { + public: + DeliverFrameRunnable(CamerasParent* aParent, CaptureEngine aEngine, + uint32_t aStreamId, const TrackingId& aTrackingId, + const webrtc::VideoFrame& aFrame, + const VideoFrameProperties& aProperties) + : Runnable("camera::DeliverFrameRunnable"), + mParent(aParent), + mCapEngine(aEngine), + mStreamId(aStreamId), + mTrackingId(aTrackingId), + mProperties(aProperties), + mResult(0) { + // No ShmemBuffer (of the right size) was available, so make an + // extra buffer here. We have no idea when we are going to run and + // it will be potentially long after the webrtc frame callback has + // returned, so the copy needs to be no later than here. + // We will need to copy this back into a Shmem later on so we prefer + // using ShmemBuffers to avoid the extra copy. + PerformanceRecorder<CopyVideoStage> rec( + "CamerasParent::VideoFrameToAltBuffer"_ns, aTrackingId, aFrame.width(), + aFrame.height()); + mAlternateBuffer.reset(new unsigned char[aProperties.bufferSize()]); + VideoFrameUtils::CopyVideoFrameBuffers(mAlternateBuffer.get(), + aProperties.bufferSize(), aFrame); + rec.Record(); + } + + DeliverFrameRunnable(CamerasParent* aParent, CaptureEngine aEngine, + uint32_t aStreamId, const TrackingId& aTrackingId, + ShmemBuffer aBuffer, VideoFrameProperties& aProperties) + : Runnable("camera::DeliverFrameRunnable"), + mParent(aParent), + mCapEngine(aEngine), + mStreamId(aStreamId), + mTrackingId(aTrackingId), + mBuffer(std::move(aBuffer)), + mProperties(aProperties), + mResult(0){}; + + NS_IMETHOD Run() override { + // runs on BackgroundEventTarget + MOZ_ASSERT(GetCurrentSerialEventTarget() == + mParent->mPBackgroundEventTarget); + if (mParent->IsShuttingDown()) { + // Communication channel is being torn down + mResult = 0; + return NS_OK; + } + if (!mParent->DeliverFrameOverIPC(mCapEngine, mStreamId, mTrackingId, + std::move(mBuffer), + mAlternateBuffer.get(), mProperties)) { + mResult = -1; + } else { + mResult = 0; + } + return NS_OK; + } + + int GetResult() { return mResult; } + + private: + const RefPtr<CamerasParent> mParent; + const CaptureEngine mCapEngine; + const uint32_t mStreamId; + const TrackingId mTrackingId; + ShmemBuffer mBuffer; + UniquePtr<unsigned char[]> mAlternateBuffer; + const VideoFrameProperties mProperties; + int mResult; +}; + +int CamerasParent::DeliverFrameOverIPC(CaptureEngine aCapEngine, + uint32_t aStreamId, + const TrackingId& aTrackingId, + ShmemBuffer aBuffer, + unsigned char* aAltBuffer, + const VideoFrameProperties& aProps) { + // No ShmemBuffers were available, so construct one now of the right size + // and copy into it. That is an extra copy, but we expect this to be + // the exceptional case, because we just assured the next call *will* have a + // buffer of the right size. + if (aAltBuffer != nullptr) { + // Get a shared memory buffer from the pool, at least size big + ShmemBuffer shMemBuff = mShmemPool.Get(this, aProps.bufferSize()); + + if (!shMemBuff.Valid()) { + LOG("No usable Video shmem in DeliverFrame (out of buffers?)"); + // We can skip this frame if we run out of buffers, it's not a real error. + return 0; + } + + PerformanceRecorder<CopyVideoStage> rec( + "CamerasParent::AltBufferToShmem"_ns, aTrackingId, aProps.width(), + aProps.height()); + // get() and Size() check for proper alignment of the segment + memcpy(shMemBuff.GetBytes(), aAltBuffer, aProps.bufferSize()); + rec.Record(); + + if (!SendDeliverFrame(aCapEngine, aStreamId, std::move(shMemBuff.Get()), + aProps)) { + return -1; + } + } else { + MOZ_ASSERT(aBuffer.Valid()); + // ShmemBuffer was available, we're all good. A single copy happened + // in the original webrtc callback. + if (!SendDeliverFrame(aCapEngine, aStreamId, std::move(aBuffer.Get()), + aProps)) { + return -1; + } + } + + return 0; +} + +ShmemBuffer CamerasParent::GetBuffer(size_t aSize) { + return mShmemPool.GetIfAvailable(aSize); +} + +void CallbackHelper::OnFrame(const webrtc::VideoFrame& aVideoFrame) { + LOG_VERBOSE("CamerasParent(%p)::%s", mParent, __func__); + if (profiler_thread_is_being_profiled_for_markers()) { + PROFILER_MARKER_UNTYPED( + nsPrintfCString("CaptureVideoFrame %dx%d %s %s", aVideoFrame.width(), + aVideoFrame.height(), + webrtc::VideoFrameBufferTypeToString( + aVideoFrame.video_frame_buffer()->type()), + mTrackingId.ToString().get()), + MEDIA_RT); + } + RefPtr<DeliverFrameRunnable> runnable = nullptr; + // Get frame properties + camera::VideoFrameProperties properties; + VideoFrameUtils::InitFrameBufferProperties(aVideoFrame, properties); + // Get a shared memory buffer to copy the frame data into + ShmemBuffer shMemBuffer = mParent->GetBuffer(properties.bufferSize()); + if (!shMemBuffer.Valid()) { + // Either we ran out of buffers or they're not the right size yet + LOG("Correctly sized Video shmem not available in DeliverFrame"); + // We will do the copy into a(n extra) temporary buffer inside + // the DeliverFrameRunnable constructor. + } else { + // Shared memory buffers of the right size are available, do the copy here. + PerformanceRecorder<CopyVideoStage> rec( + "CamerasParent::VideoFrameToShmem"_ns, mTrackingId, aVideoFrame.width(), + aVideoFrame.height()); + VideoFrameUtils::CopyVideoFrameBuffers( + shMemBuffer.GetBytes(), properties.bufferSize(), aVideoFrame); + rec.Record(); + runnable = + new DeliverFrameRunnable(mParent, mCapEngine, mStreamId, mTrackingId, + std::move(shMemBuffer), properties); + } + if (!runnable) { + runnable = new DeliverFrameRunnable(mParent, mCapEngine, mStreamId, + mTrackingId, aVideoFrame, properties); + } + MOZ_ASSERT(mParent); + nsIEventTarget* target = mParent->GetBackgroundEventTarget(); + MOZ_ASSERT(target != nullptr); + target->Dispatch(runnable, NS_DISPATCH_NORMAL); +} + +ipc::IPCResult CamerasParent::RecvReleaseFrame(ipc::Shmem&& aShmem) { + MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); + + mShmemPool.Put(ShmemBuffer(aShmem)); + return IPC_OK(); +} + +void CamerasParent::CloseEngines() { + MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); + LOG_FUNCTION(); + + // Stop the callers + while (!mCallbacks.IsEmpty()) { + auto capEngine = mCallbacks[0]->mCapEngine; + auto streamNum = mCallbacks[0]->mStreamId; + LOG("Forcing shutdown of engine %d, capturer %d", capEngine, streamNum); + StopCapture(capEngine, streamNum); + Unused << ReleaseCapture(capEngine, streamNum); + } + + if (VideoEngine* engine = mEngines->ElementAt(CameraEngine); engine) { + auto device_info = engine->GetOrCreateVideoCaptureDeviceInfo(); + MOZ_ASSERT(device_info); + if (device_info) { + device_info->DeRegisterVideoInputFeedBack(this); + } + } +} + +VideoEngine* CamerasParent::EnsureInitialized(int aEngine) { + MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); + LOG_VERBOSE("CamerasParent(%p)::%s", this, __func__); + CaptureEngine capEngine = static_cast<CaptureEngine>(aEngine); + + if (VideoEngine* engine = mEngines->ElementAt(capEngine); engine) { + return engine; + } + + CaptureDeviceType captureDeviceType = CaptureDeviceType::Camera; + switch (capEngine) { + case ScreenEngine: + captureDeviceType = CaptureDeviceType::Screen; + break; + case BrowserEngine: + captureDeviceType = CaptureDeviceType::Browser; + break; + case WinEngine: + captureDeviceType = CaptureDeviceType::Window; + break; + case CameraEngine: + captureDeviceType = CaptureDeviceType::Camera; + break; + default: + LOG("Invalid webrtc Video engine"); + return nullptr; + } + + RefPtr<VideoEngine> engine = VideoEngine::Create(captureDeviceType); + if (!engine) { + LOG("VideoEngine::Create failed"); + return nullptr; + } + + if (capEngine == CameraEngine) { + auto device_info = engine->GetOrCreateVideoCaptureDeviceInfo(); + MOZ_ASSERT(device_info); + if (device_info) { + device_info->RegisterVideoInputFeedBack(this); + } + } + + return mEngines->ElementAt(capEngine) = std::move(engine); +} + +// Dispatch the runnable to do the camera operation on the +// specific Cameras thread, preventing us from blocking, and +// chain a runnable to send back the result on the IPC thread. +// It would be nice to get rid of the code duplication here, +// perhaps via Promises. +ipc::IPCResult CamerasParent::RecvNumberOfCaptureDevices( + const CaptureEngine& aCapEngine) { + MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); + MOZ_ASSERT(!mDestroyed); + + LOG_FUNCTION(); + LOG("CaptureEngine=%d", aCapEngine); + + using Promise = MozPromise<int, bool, true>; + InvokeAsync( + mVideoCaptureThread, __func__, + [this, self = RefPtr(this), aCapEngine] { + int num = -1; + if (auto* engine = EnsureInitialized(aCapEngine)) { + if (auto devInfo = engine->GetOrCreateVideoCaptureDeviceInfo()) { + num = static_cast<int>(devInfo->NumberOfDevices()); + } + } + return Promise::CreateAndResolve( + num, "CamerasParent::RecvNumberOfCaptureDevices"); + }) + ->Then( + mPBackgroundEventTarget, __func__, + [this, self = RefPtr(this)](Promise::ResolveOrRejectValue&& aValue) { + int nrDevices = aValue.ResolveValue(); + + if (mDestroyed) { + LOG("RecvNumberOfCaptureDevices failure: child not alive"); + return; + } + + if (nrDevices < 0) { + LOG("RecvNumberOfCaptureDevices couldn't find devices"); + Unused << SendReplyFailure(); + return; + } + + LOG("RecvNumberOfCaptureDevices: %d", nrDevices); + Unused << SendReplyNumberOfCaptureDevices(nrDevices); + }); + return IPC_OK(); +} + +ipc::IPCResult CamerasParent::RecvEnsureInitialized( + const CaptureEngine& aCapEngine) { + MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); + MOZ_ASSERT(!mDestroyed); + + LOG_FUNCTION(); + + using Promise = MozPromise<bool, bool, true>; + InvokeAsync(mVideoCaptureThread, __func__, + [this, self = RefPtr(this), aCapEngine] { + return Promise::CreateAndResolve( + EnsureInitialized(aCapEngine), + "CamerasParent::RecvEnsureInitialized"); + }) + ->Then( + mPBackgroundEventTarget, __func__, + [this, self = RefPtr(this)](Promise::ResolveOrRejectValue&& aValue) { + bool result = aValue.ResolveValue(); + + if (mDestroyed) { + LOG("RecvEnsureInitialized: child not alive"); + return; + } + + if (!result) { + LOG("RecvEnsureInitialized failed"); + Unused << SendReplyFailure(); + return; + } + + LOG("RecvEnsureInitialized succeeded"); + Unused << SendReplySuccess(); + }); + return IPC_OK(); +} + +ipc::IPCResult CamerasParent::RecvNumberOfCapabilities( + const CaptureEngine& aCapEngine, const nsACString& aUniqueId) { + MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); + MOZ_ASSERT(!mDestroyed); + + LOG_FUNCTION(); + LOG("Getting caps for %s", PromiseFlatCString(aUniqueId).get()); + + using Promise = MozPromise<int, bool, true>; + InvokeAsync( + mVideoCaptureThread, __func__, + [this, self = RefPtr(this), id = nsCString(aUniqueId), aCapEngine]() { + int num = -1; + if (auto* engine = EnsureInitialized(aCapEngine)) { + if (auto devInfo = engine->GetOrCreateVideoCaptureDeviceInfo()) { + num = devInfo->NumberOfCapabilities(id.get()); + } + } + return Promise::CreateAndResolve( + num, "CamerasParent::RecvNumberOfCapabilities"); + }) + ->Then( + mPBackgroundEventTarget, __func__, + [this, self = RefPtr(this)](Promise::ResolveOrRejectValue&& aValue) { + int aNrCapabilities = aValue.ResolveValue(); + + if (mDestroyed) { + LOG("RecvNumberOfCapabilities: child not alive"); + return; + } + + if (aNrCapabilities < 0) { + LOG("RecvNumberOfCapabilities couldn't find capabilities"); + Unused << SendReplyFailure(); + return; + } + + LOG("RecvNumberOfCapabilities: %d", aNrCapabilities); + Unused << SendReplyNumberOfCapabilities(aNrCapabilities); + }); + return IPC_OK(); +} + +ipc::IPCResult CamerasParent::RecvGetCaptureCapability( + const CaptureEngine& aCapEngine, const nsACString& aUniqueId, + const int& aIndex) { + MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); + MOZ_ASSERT(!mDestroyed); + + LOG_FUNCTION(); + LOG("RecvGetCaptureCapability: %s %d", PromiseFlatCString(aUniqueId).get(), + aIndex); + + using Promise = MozPromise<webrtc::VideoCaptureCapability, int, true>; + InvokeAsync( + mVideoCaptureThread, __func__, + [this, self = RefPtr(this), id = nsCString(aUniqueId), aCapEngine, + aIndex] { + webrtc::VideoCaptureCapability webrtcCaps; + int error = -1; + if (auto* engine = EnsureInitialized(aCapEngine)) { + if (auto devInfo = engine->GetOrCreateVideoCaptureDeviceInfo()) { + error = devInfo->GetCapability(id.get(), aIndex, webrtcCaps); + } + } + + if (!error && aCapEngine == CameraEngine) { + auto iter = mAllCandidateCapabilities.find(id); + if (iter == mAllCandidateCapabilities.end()) { + std::map<uint32_t, webrtc::VideoCaptureCapability> + candidateCapabilities; + candidateCapabilities.emplace(aIndex, webrtcCaps); + mAllCandidateCapabilities.emplace(id, candidateCapabilities); + } else { + (iter->second).emplace(aIndex, webrtcCaps); + } + } + if (error) { + return Promise::CreateAndReject( + error, "CamerasParent::RecvGetCaptureCapability"); + } + return Promise::CreateAndResolve( + webrtcCaps, "CamerasParent::RecvGetCaptureCapability"); + }) + ->Then( + mPBackgroundEventTarget, __func__, + [this, self = RefPtr(this)](Promise::ResolveOrRejectValue&& aValue) { + if (mDestroyed) { + LOG("RecvGetCaptureCapability: child not alive"); + return; + } + + if (aValue.IsReject()) { + LOG("RecvGetCaptureCapability: reply failure"); + Unused << SendReplyFailure(); + return; + } + + auto webrtcCaps = aValue.ResolveValue(); + VideoCaptureCapability capCap( + webrtcCaps.width, webrtcCaps.height, webrtcCaps.maxFPS, + static_cast<int>(webrtcCaps.videoType), webrtcCaps.interlaced); + LOG("Capability: %u %u %u %d %d", webrtcCaps.width, + webrtcCaps.height, webrtcCaps.maxFPS, + static_cast<int>(webrtcCaps.videoType), webrtcCaps.interlaced); + Unused << SendReplyGetCaptureCapability(capCap); + }); + return IPC_OK(); +} + +ipc::IPCResult CamerasParent::RecvGetCaptureDevice( + const CaptureEngine& aCapEngine, const int& aDeviceIndex) { + MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); + MOZ_ASSERT(!mDestroyed); + + LOG_FUNCTION(); + + using Data = std::tuple<nsCString, nsCString, pid_t, int>; + using Promise = MozPromise<Data, bool, true>; + InvokeAsync( + mVideoCaptureThread, __func__, + [this, self = RefPtr(this), aCapEngine, aDeviceIndex] { + char deviceName[MediaEngineSource::kMaxDeviceNameLength]; + char deviceUniqueId[MediaEngineSource::kMaxUniqueIdLength]; + nsCString name; + nsCString uniqueId; + pid_t devicePid = 0; + int error = -1; + if (auto* engine = EnsureInitialized(aCapEngine)) { + if (auto devInfo = engine->GetOrCreateVideoCaptureDeviceInfo()) { + error = devInfo->GetDeviceName( + aDeviceIndex, deviceName, sizeof(deviceName), deviceUniqueId, + sizeof(deviceUniqueId), nullptr, 0, &devicePid); + } + } + if (error == 0) { + name.Assign(deviceName); + uniqueId.Assign(deviceUniqueId); + } + return Promise::CreateAndResolve( + std::make_tuple(std::move(name), std::move(uniqueId), devicePid, + error), + "CamerasParent::RecvGetCaptureDevice"); + }) + ->Then( + mPBackgroundEventTarget, __func__, + [this, self = RefPtr(this)](Promise::ResolveOrRejectValue&& aValue) { + const auto& [name, uniqueId, devicePid, error] = + aValue.ResolveValue(); + if (mDestroyed) { + return; + } + if (error != 0) { + LOG("GetCaptureDevice failed: %d", error); + Unused << SendReplyFailure(); + return; + } + bool scary = (devicePid == getpid()); + + LOG("Returning %s name %s id (pid = %d)%s", name.get(), + uniqueId.get(), devicePid, (scary ? " (scary)" : "")); + Unused << SendReplyGetCaptureDevice(name, uniqueId, scary); + }); + return IPC_OK(); +} + +// Find out whether the given window with id has permission to use the +// camera. If the permission is not persistent, we'll make it a one-shot by +// removing the (session) permission. +static bool HasCameraPermission(const uint64_t& aWindowId) { + MOZ_ASSERT(NS_IsMainThread()); + + RefPtr<dom::WindowGlobalParent> window = + dom::WindowGlobalParent::GetByInnerWindowId(aWindowId); + if (!window) { + // Could not find window by id + return false; + } + + // If we delegate permission from first party, we should use the top level + // window + if (StaticPrefs::permissions_delegation_enabled()) { + RefPtr<dom::BrowsingContext> topBC = window->BrowsingContext()->Top(); + window = topBC->Canonical()->GetCurrentWindowGlobal(); + } + + // Return false if the window is not the currently-active window for its + // BrowsingContext. + if (!window || !window->IsCurrentGlobal()) { + return false; + } + + nsIPrincipal* principal = window->DocumentPrincipal(); + if (principal->GetIsNullPrincipal()) { + return false; + } + + if (principal->IsSystemPrincipal()) { + return true; + } + + MOZ_ASSERT(principal->GetIsContentPrincipal()); + + nsresult rv; + // Name used with nsIPermissionManager + static const nsLiteralCString cameraPermission = "MediaManagerVideo"_ns; + nsCOMPtr<nsIPermissionManager> mgr = + do_GetService(NS_PERMISSIONMANAGER_CONTRACTID, &rv); + if (NS_WARN_IF(NS_FAILED(rv))) { + return false; + } + + uint32_t video = nsIPermissionManager::UNKNOWN_ACTION; + rv = mgr->TestExactPermissionFromPrincipal(principal, cameraPermission, + &video); + if (NS_WARN_IF(NS_FAILED(rv))) { + return false; + } + + bool allowed = (video == nsIPermissionManager::ALLOW_ACTION); + + // Session permissions are removed after one use. + if (allowed) { + mgr->RemoveFromPrincipal(principal, cameraPermission); + } + + return allowed; +} + +ipc::IPCResult CamerasParent::RecvAllocateCapture( + const CaptureEngine& aCapEngine, const nsACString& aUniqueIdUTF8, + const uint64_t& aWindowID) { + MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); + MOZ_ASSERT(!mDestroyed); + + LOG("CamerasParent(%p)::%s: Verifying permissions", this, __func__); + + using Promise1 = MozPromise<bool, bool, true>; + using Data = std::tuple<int, int>; + using Promise2 = MozPromise<Data, bool, true>; + InvokeAsync(GetMainThreadSerialEventTarget(), __func__, + [aWindowID] { + // Verify whether the claimed origin has received permission + // to use the camera, either persistently or this session (one + // shot). + bool allowed = HasCameraPermission(aWindowID); + if (!allowed) { + // Developer preference for turning off permission check. + if (Preferences::GetBool( + "media.navigator.permission.disabled", false)) { + allowed = true; + LOG("No permission but checks are disabled"); + } else { + LOG("No camera permission for this origin"); + } + } + return Promise1::CreateAndResolve( + allowed, "CamerasParent::RecvAllocateCapture"); + }) + ->Then(mVideoCaptureThread, __func__, + [this, self = RefPtr(this), aCapEngine, + unique_id = nsCString(aUniqueIdUTF8)]( + Promise1::ResolveOrRejectValue&& aValue) { + bool allowed = aValue.ResolveValue(); + int captureId = -1; + int error = -1; + if (allowed && EnsureInitialized(aCapEngine)) { + VideoEngine* engine = mEngines->ElementAt(aCapEngine); + captureId = engine->CreateVideoCapture(unique_id.get()); + engine->WithEntry(captureId, + [&error](VideoEngine::CaptureEntry& cap) { + if (cap.VideoCapture()) { + error = 0; + } + }); + } + return Promise2::CreateAndResolve( + std::make_tuple(captureId, error), + "CamerasParent::RecvAllocateCapture"); + }) + ->Then( + mPBackgroundEventTarget, __func__, + [this, self = RefPtr(this)](Promise2::ResolveOrRejectValue&& aValue) { + const auto [captureId, error] = aValue.ResolveValue(); + if (mDestroyed) { + LOG("RecvAllocateCapture: child not alive"); + return; + } + + if (error != 0) { + Unused << SendReplyFailure(); + LOG("RecvAllocateCapture: WithEntry error"); + return; + } + + LOG("Allocated device nr %d", captureId); + Unused << SendReplyAllocateCapture(captureId); + }); + return IPC_OK(); +} + +int CamerasParent::ReleaseCapture(const CaptureEngine& aCapEngine, + int aCaptureId) { + MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); + int error = -1; + if (auto* engine = EnsureInitialized(aCapEngine)) { + error = engine->ReleaseVideoCapture(aCaptureId); + } + return error; +} + +ipc::IPCResult CamerasParent::RecvReleaseCapture( + const CaptureEngine& aCapEngine, const int& aCaptureId) { + MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); + MOZ_ASSERT(!mDestroyed); + + LOG_FUNCTION(); + LOG("RecvReleaseCamera device nr %d", aCaptureId); + + using Promise = MozPromise<int, bool, true>; + InvokeAsync(mVideoCaptureThread, __func__, + [this, self = RefPtr(this), aCapEngine, aCaptureId] { + return Promise::CreateAndResolve( + ReleaseCapture(aCapEngine, aCaptureId), + "CamerasParent::RecvReleaseCapture"); + }) + ->Then(mPBackgroundEventTarget, __func__, + [this, self = RefPtr(this), + aCaptureId](Promise::ResolveOrRejectValue&& aValue) { + int error = aValue.ResolveValue(); + + if (mDestroyed) { + LOG("RecvReleaseCapture: child not alive"); + return; + } + + if (error != 0) { + Unused << SendReplyFailure(); + LOG("RecvReleaseCapture: Failed to free device nr %d", + aCaptureId); + return; + } + + Unused << SendReplySuccess(); + LOG("Freed device nr %d", aCaptureId); + }); + return IPC_OK(); +} + +ipc::IPCResult CamerasParent::RecvStartCapture( + const CaptureEngine& aCapEngine, const int& aCaptureId, + const VideoCaptureCapability& aIpcCaps) { + MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); + MOZ_ASSERT(!mDestroyed); + + LOG_FUNCTION(); + + using Promise = MozPromise<int, bool, true>; + InvokeAsync( + mVideoCaptureThread, __func__, + [this, self = RefPtr(this), aCapEngine, aCaptureId, aIpcCaps] { + LOG_FUNCTION(); + CallbackHelper** cbh; + int error = -1; + + if (!EnsureInitialized(aCapEngine)) { + return Promise::CreateAndResolve(error, + "CamerasParent::RecvStartCapture"); + } + + cbh = mCallbacks.AppendElement(new CallbackHelper( + static_cast<CaptureEngine>(aCapEngine), aCaptureId, this)); + + mEngines->ElementAt(aCapEngine) + ->WithEntry(aCaptureId, [&](VideoEngine::CaptureEntry& cap) { + webrtc::VideoCaptureCapability capability; + capability.width = aIpcCaps.width(); + capability.height = aIpcCaps.height(); + capability.maxFPS = aIpcCaps.maxFPS(); + capability.videoType = + static_cast<webrtc::VideoType>(aIpcCaps.videoType()); + capability.interlaced = aIpcCaps.interlaced(); + +#ifndef FUZZING_SNAPSHOT + MOZ_DIAGNOSTIC_ASSERT(sDeviceUniqueIDs.find(aCaptureId) == + sDeviceUniqueIDs.end()); +#endif + sDeviceUniqueIDs.emplace(aCaptureId, + cap.VideoCapture()->CurrentDeviceName()); + +#ifndef FUZZING_SNAPSHOT + MOZ_DIAGNOSTIC_ASSERT( + sAllRequestedCapabilities.find(aCaptureId) == + sAllRequestedCapabilities.end()); +#endif + sAllRequestedCapabilities.emplace(aCaptureId, capability); + + if (aCapEngine == CameraEngine) { + for (const auto& it : sDeviceUniqueIDs) { + if (strcmp(it.second, + cap.VideoCapture()->CurrentDeviceName()) == 0) { + capability.width = + std::max(capability.width, + sAllRequestedCapabilities[it.first].width); + capability.height = + std::max(capability.height, + sAllRequestedCapabilities[it.first].height); + capability.maxFPS = + std::max(capability.maxFPS, + sAllRequestedCapabilities[it.first].maxFPS); + } + } + + auto candidateCapabilities = mAllCandidateCapabilities.find( + nsCString(cap.VideoCapture()->CurrentDeviceName())); + if ((candidateCapabilities != + mAllCandidateCapabilities.end()) && + (!candidateCapabilities->second.empty())) { + int32_t minIdx = -1; + uint64_t minDistance = UINT64_MAX; + + for (auto& candidateCapability : + candidateCapabilities->second) { + if (candidateCapability.second.videoType != + capability.videoType) { + continue; + } + // The first priority is finding a suitable resolution. + // So here we raise the weight of width and height + uint64_t distance = uint64_t(ResolutionFeasibilityDistance( + candidateCapability.second.width, + capability.width)) + + uint64_t(ResolutionFeasibilityDistance( + candidateCapability.second.height, + capability.height)) + + uint64_t(FeasibilityDistance( + candidateCapability.second.maxFPS, + capability.maxFPS)); + if (distance < minDistance) { + minIdx = static_cast<int32_t>(candidateCapability.first); + minDistance = distance; + } + } + MOZ_ASSERT(minIdx != -1); + capability = candidateCapabilities->second[minIdx]; + } + } else if (aCapEngine == ScreenEngine || + aCapEngine == BrowserEngine || + aCapEngine == WinEngine) { + for (const auto& it : sDeviceUniqueIDs) { + if (strcmp(it.second, + cap.VideoCapture()->CurrentDeviceName()) == 0) { + capability.maxFPS = + std::max(capability.maxFPS, + sAllRequestedCapabilities[it.first].maxFPS); + } + } + } + + cap.VideoCapture()->SetTrackingId( + (*cbh)->mTrackingId.mUniqueInProcId); + error = cap.VideoCapture()->StartCapture(capability); + + if (!error) { + cap.VideoCapture()->RegisterCaptureDataCallback( + static_cast<rtc::VideoSinkInterface<webrtc::VideoFrame>*>( + *cbh)); + } else { + sDeviceUniqueIDs.erase(aCaptureId); + sAllRequestedCapabilities.erase(aCaptureId); + } + }); + + return Promise::CreateAndResolve(error, + "CamerasParent::RecvStartCapture"); + }) + ->Then( + mPBackgroundEventTarget, __func__, + [this, self = RefPtr(this)](Promise::ResolveOrRejectValue&& aValue) { + int error = aValue.ResolveValue(); + + if (mDestroyed) { + LOG("RecvStartCapture failure: child is not alive"); + return; + } + + if (error != 0) { + LOG("RecvStartCapture failure: StartCapture failed"); + Unused << SendReplyFailure(); + return; + } + + Unused << SendReplySuccess(); + }); + return IPC_OK(); +} + +ipc::IPCResult CamerasParent::RecvFocusOnSelectedSource( + const CaptureEngine& aCapEngine, const int& aCaptureId) { + MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); + MOZ_ASSERT(!mDestroyed); + + LOG_FUNCTION(); + + using Promise = MozPromise<bool, bool, true>; + InvokeAsync(mVideoCaptureThread, __func__, + [this, self = RefPtr(this), aCapEngine, aCaptureId] { + bool result = false; + if (auto* engine = EnsureInitialized(aCapEngine)) { + engine->WithEntry( + aCaptureId, [&](VideoEngine::CaptureEntry& cap) { + if (cap.VideoCapture()) { + result = cap.VideoCapture()->FocusOnSelectedSource(); + } + }); + } + return Promise::CreateAndResolve( + result, "CamerasParent::RecvFocusOnSelectedSource"); + }) + ->Then( + mPBackgroundEventTarget, __func__, + [this, self = RefPtr(this)](Promise::ResolveOrRejectValue&& aValue) { + bool result = aValue.ResolveValue(); + if (mDestroyed) { + LOG("RecvFocusOnSelectedSource failure: child is not alive"); + return; + } + + if (!result) { + Unused << SendReplyFailure(); + LOG("RecvFocusOnSelectedSource failure."); + return; + } + + Unused << SendReplySuccess(); + }); + return IPC_OK(); +} + +void CamerasParent::StopCapture(const CaptureEngine& aCapEngine, + int aCaptureId) { + MOZ_ASSERT(mVideoCaptureThread->IsOnCurrentThread()); + if (auto* engine = EnsureInitialized(aCapEngine)) { + // we're removing elements, iterate backwards + for (size_t i = mCallbacks.Length(); i > 0; i--) { + if (mCallbacks[i - 1]->mCapEngine == aCapEngine && + mCallbacks[i - 1]->mStreamId == (uint32_t)aCaptureId) { + CallbackHelper* cbh = mCallbacks[i - 1]; + engine->WithEntry(aCaptureId, [cbh, &aCaptureId]( + VideoEngine::CaptureEntry& cap) { + if (cap.VideoCapture()) { + cap.VideoCapture()->DeRegisterCaptureDataCallback( + static_cast<rtc::VideoSinkInterface<webrtc::VideoFrame>*>(cbh)); + cap.VideoCapture()->StopCaptureIfAllClientsClose(); + + sDeviceUniqueIDs.erase(aCaptureId); + sAllRequestedCapabilities.erase(aCaptureId); + } + }); + + delete mCallbacks[i - 1]; + mCallbacks.RemoveElementAt(i - 1); + break; + } + } + } +} + +ipc::IPCResult CamerasParent::RecvStopCapture(const CaptureEngine& aCapEngine, + const int& aCaptureId) { + MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); + MOZ_ASSERT(!mDestroyed); + + LOG_FUNCTION(); + + nsresult rv = mVideoCaptureThread->Dispatch(NS_NewRunnableFunction( + __func__, [this, self = RefPtr(this), aCapEngine, aCaptureId] { + StopCapture(aCapEngine, aCaptureId); + })); + + if (mDestroyed) { + if (NS_FAILED(rv)) { + return IPC_FAIL_NO_REASON(this); + } + } else { + if (NS_SUCCEEDED(rv)) { + if (!SendReplySuccess()) { + return IPC_FAIL_NO_REASON(this); + } + } else { + if (!SendReplyFailure()) { + return IPC_FAIL_NO_REASON(this); + } + } + } + return IPC_OK(); +} + +void CamerasParent::ActorDestroy(ActorDestroyReason aWhy) { + MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); + LOG_FUNCTION(); + + // Release shared memory now, it's our last chance + mShmemPool.Cleanup(this); + // We don't want to receive callbacks or anything if we can't + // forward them anymore anyway. + mDestroyed = true; + // We don't need to listen for shutdown any longer. Disconnect the request. + // This breaks the reference cycle between CamerasParent and the shutdown + // promise's Then handler. + mShutdownRequest.DisconnectIfExists(); + + if (mVideoCaptureThread) { + // Shut down the WebRTC stack, on the video capture thread. + MOZ_ALWAYS_SUCCEEDS(mVideoCaptureThread->Dispatch( + NewRunnableMethod(__func__, this, &CamerasParent::CloseEngines))); + } +} + +void CamerasParent::OnShutdown() { + ipc::AssertIsOnBackgroundThread(); + LOG("CamerasParent(%p) ShutdownEvent", this); + mShutdownRequest.Complete(); + (void)Send__delete__(this); +} + +CamerasParent::CamerasParent() + : mShutdownBlocker(ShutdownBlockingTicket::Create( + u"CamerasParent"_ns, NS_LITERAL_STRING_FROM_CSTRING(__FILE__), + __LINE__)), + mVideoCaptureThread(mShutdownBlocker + ? MakeAndAddRefVideoCaptureThreadAndSingletons() + : nullptr), + mEngines(sEngines), + mShmemPool(CaptureEngine::MaxEngine), + mPBackgroundEventTarget(GetCurrentSerialEventTarget()), + mDestroyed(false) { + MOZ_ASSERT(mPBackgroundEventTarget != nullptr, + "GetCurrentThreadEventTarget failed"); + LOG("CamerasParent: %p", this); + + // Don't dispatch from the constructor a runnable that may toggle the + // reference count, because the IPC thread does not get a reference until + // after the constructor returns. +} + +// RecvPCamerasConstructor() is used because IPC messages, for +// Send__delete__(), cannot be sent from AllocPCamerasParent(). +ipc::IPCResult CamerasParent::RecvPCamerasConstructor() { + MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); + + // AsyncShutdown barriers are available only for ShutdownPhases as late as + // XPCOMWillShutdown. The IPC background thread shuts down during + // XPCOMShutdownThreads, so actors may be created when AsyncShutdown barriers + // are no longer available. Should shutdown be past XPCOMWillShutdown we end + // up with a null mShutdownBlocker. + + if (!mShutdownBlocker) { + LOG("CamerasParent(%p) Got no ShutdownBlockingTicket. We are already in " + "shutdown. Deleting.", + this); + return Send__delete__(this) ? IPC_OK() : IPC_FAIL(this, "Failed to send"); + } + + if (!mVideoCaptureThread) { + return Send__delete__(this) ? IPC_OK() : IPC_FAIL(this, "Failed to send"); + } + + MOZ_ASSERT(mEngines); + + mShutdownBlocker->ShutdownPromise() + ->Then(mPBackgroundEventTarget, "CamerasParent OnShutdown", + [this, self = RefPtr(this)]( + const ShutdownPromise::ResolveOrRejectValue& aValue) { + MOZ_ASSERT(aValue.IsResolve(), + "ShutdownBlockingTicket must have been destroyed " + "without us disconnecting the shutdown request"); + OnShutdown(); + }) + ->Track(mShutdownRequest); + + return IPC_OK(); +} + +CamerasParent::~CamerasParent() { + ipc::AssertIsOnBackgroundThread(); + LOG_FUNCTION(); + + if (!mVideoCaptureThread) { + // No video engines or video capture thread to shutdown here. + return; + } + + MOZ_ASSERT(mShutdownBlocker, + "A ShutdownBlocker is a prerequisite for mVideoCaptureThread"); + + ReleaseVideoCaptureThreadAndSingletons(); +} + +already_AddRefed<CamerasParent> CamerasParent::Create() { + ipc::AssertIsOnBackgroundThread(); + return MakeAndAddRef<CamerasParent>(); +} + +} // namespace camera +} // namespace mozilla diff --git a/dom/media/systemservices/CamerasParent.h b/dom/media/systemservices/CamerasParent.h new file mode 100644 index 0000000000..97e7ea0bd4 --- /dev/null +++ b/dom/media/systemservices/CamerasParent.h @@ -0,0 +1,157 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_CamerasParent_h +#define mozilla_CamerasParent_h + +#include "CamerasChild.h" +#include "VideoEngine.h" +#include "mozilla/Atomics.h" +#include "mozilla/camera/PCamerasParent.h" +#include "mozilla/ipc/Shmem.h" +#include "mozilla/ShmemPool.h" +#include "api/video/video_sink_interface.h" +#include "modules/video_capture/video_capture.h" +#include "modules/video_capture/video_capture_defines.h" +#include "video/render/incoming_video_stream.h" + +class nsIThread; + +namespace mozilla::camera { + +class CamerasParent; + +class CallbackHelper : public rtc::VideoSinkInterface<webrtc::VideoFrame> { + public: + CallbackHelper(CaptureEngine aCapEng, uint32_t aStreamId, + CamerasParent* aParent) + : mCapEngine(aCapEng), + mStreamId(aStreamId), + mTrackingId(CaptureEngineToTrackingSourceStr(aCapEng), aStreamId), + mParent(aParent){}; + + // These callbacks end up running on the VideoCapture thread. + // From VideoCaptureCallback + void OnFrame(const webrtc::VideoFrame& aVideoFrame) override; + + friend CamerasParent; + + private: + const CaptureEngine mCapEngine; + const uint32_t mStreamId; + const TrackingId mTrackingId; + CamerasParent* const mParent; +}; + +class DeliverFrameRunnable; + +class CamerasParent final : public PCamerasParent, + private webrtc::VideoInputFeedBack { + using ShutdownMozPromise = media::ShutdownBlockingTicket::ShutdownMozPromise; + + NS_INLINE_DECL_THREADSAFE_REFCOUNTING_WITH_DELETE_ON_EVENT_TARGET( + CamerasParent, mPBackgroundEventTarget) + + public: + class VideoEngineArray; + friend DeliverFrameRunnable; + + static already_AddRefed<CamerasParent> Create(); + + // Messages received from the child. These run on the IPC/PBackground thread. + mozilla::ipc::IPCResult RecvPCamerasConstructor(); + mozilla::ipc::IPCResult RecvAllocateCapture( + const CaptureEngine& aCapEngine, const nsACString& aUniqueIdUTF8, + const uint64_t& aWindowID) override; + mozilla::ipc::IPCResult RecvReleaseCapture(const CaptureEngine& aCapEngine, + const int& aCaptureId) override; + mozilla::ipc::IPCResult RecvNumberOfCaptureDevices( + const CaptureEngine& aCapEngine) override; + mozilla::ipc::IPCResult RecvNumberOfCapabilities( + const CaptureEngine& aCapEngine, const nsACString& aUniqueId) override; + mozilla::ipc::IPCResult RecvGetCaptureCapability( + const CaptureEngine& aCapEngine, const nsACString& aUniqueId, + const int& aIndex) override; + mozilla::ipc::IPCResult RecvGetCaptureDevice( + const CaptureEngine& aCapEngine, const int& aDeviceIndex) override; + mozilla::ipc::IPCResult RecvStartCapture( + const CaptureEngine& aCapEngine, const int& aCaptureId, + const VideoCaptureCapability& aIpcCaps) override; + mozilla::ipc::IPCResult RecvFocusOnSelectedSource( + const CaptureEngine& aCapEngine, const int& aCaptureId) override; + mozilla::ipc::IPCResult RecvStopCapture(const CaptureEngine& aCapEngine, + const int& aCaptureId) override; + mozilla::ipc::IPCResult RecvReleaseFrame( + mozilla::ipc::Shmem&& aShmem) override; + void ActorDestroy(ActorDestroyReason aWhy) override; + mozilla::ipc::IPCResult RecvEnsureInitialized( + const CaptureEngine& aCapEngine) override; + + nsIEventTarget* GetBackgroundEventTarget() { + return mPBackgroundEventTarget; + }; + bool IsShuttingDown() { + // the first 2 are pBackground only, the last is atomic + MOZ_ASSERT(mPBackgroundEventTarget->IsOnCurrentThread()); + return mDestroyed; + }; + ShmemBuffer GetBuffer(size_t aSize); + + // helper to forward to the PBackground thread + int DeliverFrameOverIPC(CaptureEngine aCapEngine, uint32_t aStreamId, + const TrackingId& aTrackingId, ShmemBuffer aBuffer, + unsigned char* aAltBuffer, + const VideoFrameProperties& aProps); + + CamerasParent(); + + private: + virtual ~CamerasParent(); + + // We use these helpers for shutdown and for the respective IPC commands. + void StopCapture(const CaptureEngine& aCapEngine, int aCaptureId); + int ReleaseCapture(const CaptureEngine& aCapEngine, int aCaptureId); + + // VideoInputFeedBack + void OnDeviceChange() override; + + VideoEngine* EnsureInitialized(int aEngine); + + // Stops any ongoing capturing and releases resources. Called on + // mVideoCaptureThread. Idempotent. + void CloseEngines(); + + void OnShutdown(); + + nsTArray<CallbackHelper*> mCallbacks; + // If existent, blocks xpcom shutdown while alive. + // Note that this makes a reference cycle that gets broken in ActorDestroy(). + const UniquePtr<media::ShutdownBlockingTicket> mShutdownBlocker; + // Tracks the mShutdownBlocker shutdown handler. mPBackgroundEventTarget only. + MozPromiseRequestHolder<ShutdownMozPromise> mShutdownRequest; + + // Local copy of sVideoCaptureThread. Guaranteed alive if non-null. + const nsCOMPtr<nsISerialEventTarget> mVideoCaptureThread; + + // Reference to same VideoEngineArray as sEngines. Video capture thread only. + const RefPtr<VideoEngineArray> mEngines; + + // image buffers + ShmemPool mShmemPool; + + // PBackgroundParent thread + const nsCOMPtr<nsISerialEventTarget> mPBackgroundEventTarget; + + // Set to true in ActorDestroy. PBackground only. + bool mDestroyed; + + std::map<nsCString, std::map<uint32_t, webrtc::VideoCaptureCapability>> + mAllCandidateCapabilities; +}; + +} // namespace mozilla::camera + +#endif // mozilla_CameraParent_h diff --git a/dom/media/systemservices/CamerasTypes.cpp b/dom/media/systemservices/CamerasTypes.cpp new file mode 100644 index 0000000000..7eda2f650b --- /dev/null +++ b/dom/media/systemservices/CamerasTypes.cpp @@ -0,0 +1,26 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "CamerasTypes.h" + +namespace mozilla::camera { + +TrackingId::Source CaptureEngineToTrackingSourceStr( + const CaptureEngine& aEngine) { + switch (aEngine) { + case ScreenEngine: + return TrackingId::Source::Screen; + case BrowserEngine: + return TrackingId::Source::Tab; + case WinEngine: + return TrackingId::Source::Window; + case CameraEngine: + return TrackingId::Source::Camera; + default: + return TrackingId::Source::Unimplemented; + } +} +} // namespace mozilla::camera diff --git a/dom/media/systemservices/CamerasTypes.h b/dom/media/systemservices/CamerasTypes.h new file mode 100644 index 0000000000..fed3941d9c --- /dev/null +++ b/dom/media/systemservices/CamerasTypes.h @@ -0,0 +1,38 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_CamerasTypes_h +#define mozilla_CamerasTypes_h + +#include "ipc/EnumSerializer.h" +#include "PerformanceRecorder.h" + +namespace mozilla::camera { + +enum CaptureEngine : int { + InvalidEngine = 0, + ScreenEngine, + BrowserEngine, + WinEngine, + CameraEngine, + MaxEngine +}; + +TrackingId::Source CaptureEngineToTrackingSourceStr( + const CaptureEngine& aEngine); + +} // namespace mozilla::camera + +namespace IPC { +template <> +struct ParamTraits<mozilla::camera::CaptureEngine> + : public ContiguousEnumSerializer< + mozilla::camera::CaptureEngine, + mozilla::camera::CaptureEngine::InvalidEngine, + mozilla::camera::CaptureEngine::MaxEngine> {}; +} // namespace IPC + +#endif // mozilla_CamerasTypes_h diff --git a/dom/media/systemservices/MediaChild.cpp b/dom/media/systemservices/MediaChild.cpp new file mode 100644 index 0000000000..34780ed74f --- /dev/null +++ b/dom/media/systemservices/MediaChild.cpp @@ -0,0 +1,95 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MediaChild.h" +#include "MediaParent.h" + +#include "nsGlobalWindow.h" +#include "mozilla/dom/ContentChild.h" +#include "mozilla/MediaManager.h" +#include "mozilla/Logging.h" +#include "nsQueryObject.h" + +#undef LOG +mozilla::LazyLogModule gMediaChildLog("MediaChild"); +#define LOG(args) MOZ_LOG(gMediaChildLog, mozilla::LogLevel::Debug, args) + +namespace mozilla::media { + +RefPtr<PrincipalKeyPromise> GetPrincipalKey( + const ipc::PrincipalInfo& aPrincipalInfo, bool aPersist) { + RefPtr<MediaManager> mgr = MediaManager::GetInstance(); + MOZ_ASSERT(mgr); + + if (XRE_GetProcessType() == GeckoProcessType_Default) { + auto p = MakeRefPtr<PrincipalKeyPromise::Private>(__func__); + + mgr->GetNonE10sParent()->RecvGetPrincipalKey( + aPrincipalInfo, aPersist, + [p](const nsACString& aKey) { p->Resolve(aKey, __func__); }); + return p; + } + return Child::Get() + ->SendGetPrincipalKey(aPrincipalInfo, aPersist) + ->Then(GetMainThreadSerialEventTarget(), __func__, + [](const Child::GetPrincipalKeyPromise::ResolveOrRejectValue& + aValue) { + if (aValue.IsReject() || aValue.ResolveValue().IsEmpty()) { + return PrincipalKeyPromise::CreateAndReject(NS_ERROR_FAILURE, + __func__); + } + return PrincipalKeyPromise::CreateAndResolve( + aValue.ResolveValue(), __func__); + }); +} + +void SanitizeOriginKeys(const uint64_t& aSinceWhen, bool aOnlyPrivateBrowsing) { + LOG(("SanitizeOriginKeys since %" PRIu64 " %s", aSinceWhen, + (aOnlyPrivateBrowsing ? "in Private Browsing." : "."))); + + if (XRE_GetProcessType() == GeckoProcessType_Default) { + // Avoid opening MediaManager in this case, since this is called by + // sanitize.js when cookies are cleared, which can happen on startup. + RefPtr<Parent<NonE10s>> tmpParent = new Parent<NonE10s>(); + tmpParent->RecvSanitizeOriginKeys(aSinceWhen, aOnlyPrivateBrowsing); + } else { + Child::Get()->SendSanitizeOriginKeys(aSinceWhen, aOnlyPrivateBrowsing); + } +} + +static Child* sChild; + +Child* Child::Get() { + MOZ_ASSERT(XRE_GetProcessType() == GeckoProcessType_Content); + MOZ_ASSERT(NS_IsMainThread()); + if (!sChild) { + sChild = static_cast<Child*>( + dom::ContentChild::GetSingleton()->SendPMediaConstructor()); + } + return sChild; +} + +Child::Child() : mActorDestroyed(false) { + LOG(("media::Child: %p", this)); + MOZ_COUNT_CTOR(Child); +} + +Child::~Child() { + LOG(("~media::Child: %p", this)); + sChild = nullptr; + MOZ_COUNT_DTOR(Child); +} + +void Child::ActorDestroy(ActorDestroyReason aWhy) { mActorDestroyed = true; } + +PMediaChild* AllocPMediaChild() { return new Child(); } + +bool DeallocPMediaChild(media::PMediaChild* aActor) { + delete static_cast<Child*>(aActor); + return true; +} + +} // namespace mozilla::media diff --git a/dom/media/systemservices/MediaChild.h b/dom/media/systemservices/MediaChild.h new file mode 100644 index 0000000000..b0ae776cce --- /dev/null +++ b/dom/media/systemservices/MediaChild.h @@ -0,0 +1,60 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_MediaChild_h +#define mozilla_MediaChild_h + +#include "mozilla/media/PMediaChild.h" +#include "mozilla/media/PMediaParent.h" +#include "MediaUtils.h" + +namespace mozilla { + +namespace ipc { +class PrincipalInfo; +} + +namespace media { + +typedef MozPromise<nsCString, nsresult, false> PrincipalKeyPromise; + +// media::Child implements proxying to the chrome process for some media-related +// functions, for the moment just: +// +// GetPrincipalKey() - get a cookie-like persisted unique key for a given +// principalInfo. +// +// SanitizeOriginKeys() - reset persisted unique keys. + +// GetPrincipalKey and SanitizeOriginKeys are asynchronous APIs that return +// pledges (promise-like objects) with the future value. Use pledge.Then(func) +// to access. + +RefPtr<PrincipalKeyPromise> GetPrincipalKey( + const mozilla::ipc::PrincipalInfo& aPrincipalInfo, bool aPersist); + +void SanitizeOriginKeys(const uint64_t& aSinceWhen, bool aOnlyPrivateBrowsing); + +class Child : public PMediaChild { + public: + static Child* Get(); + + Child(); + + void ActorDestroy(ActorDestroyReason aWhy) override; + virtual ~Child(); + + private: + bool mActorDestroyed; +}; + +PMediaChild* AllocPMediaChild(); +bool DeallocPMediaChild(PMediaChild* aActor); + +} // namespace media +} // namespace mozilla + +#endif // mozilla_MediaChild_h diff --git a/dom/media/systemservices/MediaParent.cpp b/dom/media/systemservices/MediaParent.cpp new file mode 100644 index 0000000000..d2fb06b8ae --- /dev/null +++ b/dom/media/systemservices/MediaParent.cpp @@ -0,0 +1,536 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MediaParent.h" + +#include "mozilla/Base64.h" +#include <mozilla/StaticMutex.h> + +#include "MediaUtils.h" +#include "MediaEngine.h" +#include "VideoUtils.h" +#include "nsClassHashtable.h" +#include "nsThreadUtils.h" +#include "nsNetCID.h" +#include "nsNetUtil.h" +#include "nsIInputStream.h" +#include "nsILineInputStream.h" +#include "nsIOutputStream.h" +#include "nsISafeOutputStream.h" +#include "nsAppDirectoryServiceDefs.h" +#include "nsIFile.h" +#include "nsISupportsImpl.h" +#include "mozilla/Logging.h" + +#undef LOG +mozilla::LazyLogModule gMediaParentLog("MediaParent"); +#define LOG(args) MOZ_LOG(gMediaParentLog, mozilla::LogLevel::Debug, args) + +// A file in the profile dir is used to persist mOriginKeys used to anonymize +// deviceIds to be unique per origin, to avoid them being supercookies. + +#define ORIGINKEYS_FILE u"enumerate_devices.txt" +#define ORIGINKEYS_VERSION "1" + +namespace mozilla::media { + +StaticMutex sOriginKeyStoreStsMutex; + +class OriginKeyStore { + NS_INLINE_DECL_REFCOUNTING(OriginKeyStore); + class OriginKey { + public: + static const size_t DecodedLength = 18; + static const size_t EncodedLength = DecodedLength * 4 / 3; + + explicit OriginKey(const nsACString& aKey, + int64_t aSecondsStamp = 0) // 0 = temporal + : mKey(aKey), mSecondsStamp(aSecondsStamp) {} + + nsCString mKey; // Base64 encoded. + int64_t mSecondsStamp; + }; + + class OriginKeysTable { + public: + OriginKeysTable() : mPersistCount(0) {} + + nsresult GetPrincipalKey(const ipc::PrincipalInfo& aPrincipalInfo, + nsCString& aResult, bool aPersist = false) { + nsAutoCString principalString; + PrincipalInfoToString(aPrincipalInfo, principalString); + + OriginKey* key; + if (!mKeys.Get(principalString, &key)) { + nsCString salt; // Make a new one + nsresult rv = GenerateRandomName(salt, OriginKey::EncodedLength); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + key = mKeys.InsertOrUpdate(principalString, MakeUnique<OriginKey>(salt)) + .get(); + } + if (aPersist && !key->mSecondsStamp) { + key->mSecondsStamp = PR_Now() / PR_USEC_PER_SEC; + mPersistCount++; + } + aResult = key->mKey; + return NS_OK; + } + + void Clear(int64_t aSinceWhen) { + // Avoid int64_t* <-> void* casting offset + OriginKey since(nsCString(), aSinceWhen / PR_USEC_PER_SEC); + for (auto iter = mKeys.Iter(); !iter.Done(); iter.Next()) { + auto originKey = iter.UserData(); + LOG((((originKey->mSecondsStamp >= since.mSecondsStamp) + ? "%s: REMOVE %" PRId64 " >= %" PRId64 + : "%s: KEEP %" PRId64 " < %" PRId64), + __FUNCTION__, originKey->mSecondsStamp, since.mSecondsStamp)); + + if (originKey->mSecondsStamp >= since.mSecondsStamp) { + iter.Remove(); + } + } + mPersistCount = 0; + } + + private: + void PrincipalInfoToString(const ipc::PrincipalInfo& aPrincipalInfo, + nsACString& aString) { + switch (aPrincipalInfo.type()) { + case ipc::PrincipalInfo::TSystemPrincipalInfo: + aString.AssignLiteral("[System Principal]"); + return; + + case ipc::PrincipalInfo::TNullPrincipalInfo: { + const ipc::NullPrincipalInfo& info = + aPrincipalInfo.get_NullPrincipalInfo(); + aString.Assign(info.spec()); + return; + } + + case ipc::PrincipalInfo::TContentPrincipalInfo: { + const ipc::ContentPrincipalInfo& info = + aPrincipalInfo.get_ContentPrincipalInfo(); + aString.Assign(info.originNoSuffix()); + + nsAutoCString suffix; + info.attrs().CreateSuffix(suffix); + aString.Append(suffix); + return; + } + + case ipc::PrincipalInfo::TExpandedPrincipalInfo: { + const ipc::ExpandedPrincipalInfo& info = + aPrincipalInfo.get_ExpandedPrincipalInfo(); + + aString.AssignLiteral("[Expanded Principal ["); + + for (uint32_t i = 0; i < info.allowlist().Length(); i++) { + nsAutoCString str; + PrincipalInfoToString(info.allowlist()[i], str); + + if (i != 0) { + aString.AppendLiteral(", "); + } + + aString.Append(str); + } + + aString.AppendLiteral("]]"); + return; + } + + default: + MOZ_CRASH("Unknown PrincipalInfo type!"); + } + } + + protected: + nsClassHashtable<nsCStringHashKey, OriginKey> mKeys; + size_t mPersistCount; + }; + + class OriginKeysLoader : public OriginKeysTable { + public: + OriginKeysLoader() = default; + + nsresult GetPrincipalKey(const ipc::PrincipalInfo& aPrincipalInfo, + nsCString& aResult, bool aPersist = false) { + auto before = mPersistCount; + nsresult rv = + OriginKeysTable::GetPrincipalKey(aPrincipalInfo, aResult, aPersist); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + + if (mPersistCount != before) { + Save(); + } + return NS_OK; + } + + already_AddRefed<nsIFile> GetFile() { + MOZ_ASSERT(mProfileDir); + nsCOMPtr<nsIFile> file; + nsresult rv = mProfileDir->Clone(getter_AddRefs(file)); + if (NS_WARN_IF(NS_FAILED(rv))) { + return nullptr; + } + file->Append(nsLiteralString(ORIGINKEYS_FILE)); + return file.forget(); + } + + // Format of file is key secondsstamp origin (first line is version #): + // + // 1 + // rOMAAbFujNwKyIpj4RJ3Wt5Q 1424733961 http://fiddle.jshell.net + // rOMAAbFujNwKyIpj4RJ3Wt5Q 1424734841 http://mozilla.github.io + // etc. + + nsresult Read() { + nsCOMPtr<nsIFile> file = GetFile(); + if (NS_WARN_IF(!file)) { + return NS_ERROR_UNEXPECTED; + } + bool exists; + nsresult rv = file->Exists(&exists); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + if (!exists) { + return NS_OK; + } + + nsCOMPtr<nsIInputStream> stream; + rv = NS_NewLocalFileInputStream(getter_AddRefs(stream), file); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + nsCOMPtr<nsILineInputStream> i = do_QueryInterface(stream); + MOZ_ASSERT(i); + MOZ_ASSERT(!mPersistCount); + + nsCString line; + bool hasMoreLines; + rv = i->ReadLine(line, &hasMoreLines); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + if (!line.EqualsLiteral(ORIGINKEYS_VERSION)) { + // If version on disk is newer than we can understand then ignore it. + return NS_OK; + } + + while (hasMoreLines) { + rv = i->ReadLine(line, &hasMoreLines); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + // Read key secondsstamp origin. + // Ignore any lines that don't fit format in the comment above exactly. + int32_t f = line.FindChar(' '); + if (f < 0) { + continue; + } + const nsACString& key = Substring(line, 0, f); + const nsACString& s = Substring(line, f + 1); + f = s.FindChar(' '); + if (f < 0) { + continue; + } + int64_t secondsstamp = Substring(s, 0, f).ToInteger64(&rv); + if (NS_FAILED(rv)) { + continue; + } + const nsACString& origin = Substring(s, f + 1); + + // Validate key + if (key.Length() != OriginKey::EncodedLength) { + continue; + } + nsCString dummy; + rv = Base64Decode(key, dummy); + if (NS_FAILED(rv)) { + continue; + } + mKeys.InsertOrUpdate(origin, MakeUnique<OriginKey>(key, secondsstamp)); + } + mPersistCount = mKeys.Count(); + return NS_OK; + } + + nsresult Write() { + nsCOMPtr<nsIFile> file = GetFile(); + if (NS_WARN_IF(!file)) { + return NS_ERROR_UNEXPECTED; + } + + nsCOMPtr<nsIOutputStream> stream; + nsresult rv = + NS_NewSafeLocalFileOutputStream(getter_AddRefs(stream), file); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + + nsAutoCString versionBuffer; + versionBuffer.AppendLiteral(ORIGINKEYS_VERSION); + versionBuffer.Append('\n'); + + uint32_t count; + rv = stream->Write(versionBuffer.Data(), versionBuffer.Length(), &count); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + if (count != versionBuffer.Length()) { + return NS_ERROR_UNEXPECTED; + } + for (const auto& entry : mKeys) { + const nsACString& origin = entry.GetKey(); + OriginKey* originKey = entry.GetWeak(); + + if (!originKey->mSecondsStamp) { + continue; // don't write temporal ones + } + + nsCString originBuffer; + originBuffer.Append(originKey->mKey); + originBuffer.Append(' '); + originBuffer.AppendInt(originKey->mSecondsStamp); + originBuffer.Append(' '); + originBuffer.Append(origin); + originBuffer.Append('\n'); + + rv = stream->Write(originBuffer.Data(), originBuffer.Length(), &count); + if (NS_WARN_IF(NS_FAILED(rv)) || count != originBuffer.Length()) { + break; + } + } + + nsCOMPtr<nsISafeOutputStream> safeStream = do_QueryInterface(stream); + MOZ_ASSERT(safeStream); + + rv = safeStream->Finish(); + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + return NS_OK; + } + + nsresult Load() { + nsresult rv = Read(); + if (NS_WARN_IF(NS_FAILED(rv))) { + Delete(); + } + return rv; + } + + nsresult Save() { + nsresult rv = Write(); + if (NS_WARN_IF(NS_FAILED(rv))) { + NS_WARNING("Failed to write data for EnumerateDevices id-persistence."); + Delete(); + } + return rv; + } + + void Clear(int64_t aSinceWhen) { + OriginKeysTable::Clear(aSinceWhen); + Delete(); + Save(); + } + + nsresult Delete() { + nsCOMPtr<nsIFile> file = GetFile(); + if (NS_WARN_IF(!file)) { + return NS_ERROR_UNEXPECTED; + } + nsresult rv = file->Remove(false); + if (rv == NS_ERROR_FILE_NOT_FOUND) { + return NS_OK; + } + if (NS_WARN_IF(NS_FAILED(rv))) { + return rv; + } + return NS_OK; + } + + void SetProfileDir(nsIFile* aProfileDir) { + MOZ_ASSERT(!NS_IsMainThread()); + bool first = !mProfileDir; + mProfileDir = aProfileDir; + // Load from disk when we first get a profileDir, but not subsequently. + if (first) { + Load(); + } + } + + private: + nsCOMPtr<nsIFile> mProfileDir; + }; + + private: + static OriginKeyStore* sOriginKeyStore; + + virtual ~OriginKeyStore() { + MOZ_ASSERT(NS_IsMainThread()); + sOriginKeyStore = nullptr; + LOG(("%s", __FUNCTION__)); + } + + public: + static RefPtr<OriginKeyStore> Get() { + MOZ_ASSERT(NS_IsMainThread()); + if (!sOriginKeyStore) { + sOriginKeyStore = new OriginKeyStore(); + } + return RefPtr(sOriginKeyStore); + } + + // Only accessed on StreamTS threads + OriginKeysLoader mOriginKeys MOZ_GUARDED_BY(sOriginKeyStoreStsMutex); + OriginKeysTable mPrivateBrowsingOriginKeys + MOZ_GUARDED_BY(sOriginKeyStoreStsMutex); +}; +OriginKeyStore* OriginKeyStore::sOriginKeyStore = nullptr; + +template <class Super> +mozilla::ipc::IPCResult Parent<Super>::RecvGetPrincipalKey( + const ipc::PrincipalInfo& aPrincipalInfo, const bool& aPersist, + PMediaParent::GetPrincipalKeyResolver&& aResolve) { + MOZ_ASSERT(NS_IsMainThread()); + + // First, get profile dir. + + nsCOMPtr<nsIFile> profileDir; + nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(profileDir)); + if (NS_WARN_IF(NS_FAILED(rv))) { + return IPCResult(this, false); + } + + // Resolver has to be called in MainThread but the key is discovered + // in a different thread. We wrap the resolver around a MozPromise to make + // it more flexible and pass it to the new task. When this is done the + // resolver is resolved in MainThread. + + // Then over to stream-transport thread (a thread pool) to do the actual + // file io. Stash a promise to hold the answer and get an id for this request. + + nsCOMPtr<nsIEventTarget> sts = + do_GetService(NS_STREAMTRANSPORTSERVICE_CONTRACTID); + MOZ_ASSERT(sts); + auto taskQueue = TaskQueue::Create(sts.forget(), "RecvGetPrincipalKey"); + RefPtr<Parent<Super>> that(this); + + InvokeAsync( + taskQueue, __func__, + [this, that, profileDir, aPrincipalInfo, aPersist]() { + MOZ_ASSERT(!NS_IsMainThread()); + + StaticMutexAutoLock lock(sOriginKeyStoreStsMutex); + mOriginKeyStore->mOriginKeys.SetProfileDir(profileDir); + + nsresult rv; + nsAutoCString result; + if (IsPrincipalInfoPrivate(aPrincipalInfo)) { + rv = mOriginKeyStore->mPrivateBrowsingOriginKeys.GetPrincipalKey( + aPrincipalInfo, result); + } else { + rv = mOriginKeyStore->mOriginKeys.GetPrincipalKey(aPrincipalInfo, + result, aPersist); + } + + if (NS_WARN_IF(NS_FAILED(rv))) { + return PrincipalKeyPromise::CreateAndReject(rv, __func__); + } + return PrincipalKeyPromise::CreateAndResolve(result, __func__); + }) + ->Then( + GetCurrentSerialEventTarget(), __func__, + [aResolve](const PrincipalKeyPromise::ResolveOrRejectValue& aValue) { + if (aValue.IsReject()) { + aResolve(""_ns); + } else { + aResolve(aValue.ResolveValue()); + } + }); + + return IPC_OK(); +} + +template <class Super> +mozilla::ipc::IPCResult Parent<Super>::RecvSanitizeOriginKeys( + const uint64_t& aSinceWhen, const bool& aOnlyPrivateBrowsing) { + MOZ_ASSERT(NS_IsMainThread()); + nsCOMPtr<nsIFile> profileDir; + nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(profileDir)); + if (NS_WARN_IF(NS_FAILED(rv))) { + return IPCResult(this, false); + } + // Over to stream-transport thread (a thread pool) to do the file io. + + nsCOMPtr<nsIEventTarget> sts = + do_GetService(NS_STREAMTRANSPORTSERVICE_CONTRACTID); + MOZ_ASSERT(sts); + RefPtr<Parent<Super>> that(this); + + rv = sts->Dispatch( + NewRunnableFrom( + [this, that, profileDir, aSinceWhen, aOnlyPrivateBrowsing]() { + MOZ_ASSERT(!NS_IsMainThread()); + StaticMutexAutoLock lock(sOriginKeyStoreStsMutex); + mOriginKeyStore->mPrivateBrowsingOriginKeys.Clear(aSinceWhen); + if (!aOnlyPrivateBrowsing) { + mOriginKeyStore->mOriginKeys.SetProfileDir(profileDir); + mOriginKeyStore->mOriginKeys.Clear(aSinceWhen); + } + return NS_OK; + }), + NS_DISPATCH_NORMAL); + if (NS_WARN_IF(NS_FAILED(rv))) { + return IPCResult(this, false); + } + return IPC_OK(); +} + +template <class Super> +void Parent<Super>::ActorDestroy(ActorDestroyReason aWhy) { + // No more IPC from here + mDestroyed = true; + LOG(("%s", __FUNCTION__)); +} + +template <class Super> +Parent<Super>::Parent() + : mOriginKeyStore(OriginKeyStore::Get()), mDestroyed(false) { + LOG(("media::Parent: %p", this)); +} + +template <class Super> +Parent<Super>::~Parent() { + NS_ReleaseOnMainThread("Parent<Super>::mOriginKeyStore", + mOriginKeyStore.forget()); + LOG(("~media::Parent: %p", this)); +} + +PMediaParent* AllocPMediaParent() { + Parent<PMediaParent>* obj = new Parent<PMediaParent>(); + obj->AddRef(); + return obj; +} + +bool DeallocPMediaParent(media::PMediaParent* aActor) { + static_cast<Parent<PMediaParent>*>(aActor)->Release(); + return true; +} + +} // namespace mozilla::media + +// Instantiate templates to satisfy linker +template class mozilla::media::Parent<mozilla::media::NonE10s>; diff --git a/dom/media/systemservices/MediaParent.h b/dom/media/systemservices/MediaParent.h new file mode 100644 index 0000000000..77cba312f3 --- /dev/null +++ b/dom/media/systemservices/MediaParent.h @@ -0,0 +1,91 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_MediaParent_h +#define mozilla_MediaParent_h + +#include "MediaChild.h" + +#include "mozilla/media/PMediaParent.h" + +namespace mozilla::media { + +// media::Parent implements the chrome-process side of ipc for media::Child APIs +// A same-process version may also be created to service non-e10s calls. + +class OriginKeyStore; + +class NonE10s { + typedef mozilla::ipc::IProtocol::ActorDestroyReason ActorDestroyReason; + + public: + virtual ~NonE10s() = default; + + protected: + virtual mozilla::ipc::IPCResult RecvGetPrincipalKey( + const mozilla::ipc::PrincipalInfo& aPrincipalInfo, const bool& aPersist, + PMediaParent::GetPrincipalKeyResolver&& aResolve) = 0; + virtual mozilla::ipc::IPCResult RecvSanitizeOriginKeys( + const uint64_t& aSinceWhen, const bool& aOnlyPrivateBrowsing) = 0; + virtual void ActorDestroy(ActorDestroyReason aWhy) = 0; +}; + +/** + * Dummy class to avoid a templated class being passed to the refcounting macro + * (see Bug 1334421 for what happens then) + */ +class RefCountedParent { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(RefCountedParent) + + protected: + virtual ~RefCountedParent() = default; +}; + +// Super = PMediaParent or NonE10s + +template <class Super> +class Parent : public RefCountedParent, public Super { + typedef mozilla::ipc::IProtocol::ActorDestroyReason ActorDestroyReason; + + public: + virtual mozilla::ipc::IPCResult RecvGetPrincipalKey( + const mozilla::ipc::PrincipalInfo& aPrincipalInfo, const bool& aPersist, + PMediaParent::GetPrincipalKeyResolver&& aResolve) override; + virtual mozilla::ipc::IPCResult RecvSanitizeOriginKeys( + const uint64_t& aSinceWhen, const bool& aOnlyPrivateBrowsing) override; + virtual void ActorDestroy(ActorDestroyReason aWhy) override; + + Parent(); + + private: + virtual ~Parent(); + + RefPtr<OriginKeyStore> mOriginKeyStore; + bool mDestroyed; +}; + +template <class Parent> +mozilla::ipc::IPCResult IPCResult(Parent* aSelf, bool aSuccess); + +template <> +inline mozilla::ipc::IPCResult IPCResult(Parent<PMediaParent>* aSelf, + bool aSuccess) { + return aSuccess ? IPC_OK() : IPC_FAIL_NO_REASON(aSelf); +} + +template <> +inline mozilla::ipc::IPCResult IPCResult(Parent<NonE10s>* aSelf, + bool aSuccess) { + return IPC_OK(); +} + +PMediaParent* AllocPMediaParent(); +bool DeallocPMediaParent(PMediaParent* aActor); + +} // namespace mozilla::media + +#endif // mozilla_MediaParent_h diff --git a/dom/media/systemservices/MediaSystemResourceClient.cpp b/dom/media/systemservices/MediaSystemResourceClient.cpp new file mode 100644 index 0000000000..50695fc76c --- /dev/null +++ b/dom/media/systemservices/MediaSystemResourceClient.cpp @@ -0,0 +1,67 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/Monitor.h" +#include "mozilla/ReentrantMonitor.h" + +#include "MediaSystemResourceClient.h" + +namespace mozilla { + +Atomic<uint32_t> MediaSystemResourceClient::sSerialCounter(0); + +MediaSystemResourceClient::MediaSystemResourceClient( + MediaSystemResourceType aReourceType) + : mResourceType(aReourceType), + mId(++sSerialCounter), + mListener(nullptr), + mResourceState(RESOURCE_STATE_START), + mIsSync(false), + mAcquireSyncWaitMonitor(nullptr), + mAcquireSyncWaitDone(nullptr) { + mManager = MediaSystemResourceManager::Get(); + if (mManager) { + mManager->Register(this); + } +} + +MediaSystemResourceClient::~MediaSystemResourceClient() { + ReleaseResource(); + if (mManager) { + mManager->Unregister(this); + } +} + +bool MediaSystemResourceClient::SetListener( + MediaSystemResourceReservationListener* aListener) { + if (!mManager) { + return false; + } + return mManager->SetListener(this, aListener); +} + +void MediaSystemResourceClient::Acquire() { + if (!mManager) { + return; + } + mManager->Acquire(this); +} + +bool MediaSystemResourceClient::AcquireSyncNoWait() { + if (!mManager) { + return false; + } + return mManager->AcquireSyncNoWait(this); +} + +void MediaSystemResourceClient::ReleaseResource() { + if (!mManager) { + return; + } + mManager->ReleaseResource(this); +} + +} // namespace mozilla diff --git a/dom/media/systemservices/MediaSystemResourceClient.h b/dom/media/systemservices/MediaSystemResourceClient.h new file mode 100644 index 0000000000..52cf5107e9 --- /dev/null +++ b/dom/media/systemservices/MediaSystemResourceClient.h @@ -0,0 +1,91 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(MediaSystemResourceClient_h_) +# define MediaSystemResourceClient_h_ + +# include "MediaSystemResourceManager.h" +# include "MediaSystemResourceTypes.h" +# include "mozilla/Atomics.h" +# include "mozilla/media/MediaSystemResourceTypes.h" +# include "mozilla/Monitor.h" +# include "mozilla/RefPtr.h" + +namespace mozilla { + +class MediaSystemResourceManager; + +/** + * This is a base class for listener callbacks. + * This callback is invoked when the media system resource reservation state + * is changed. + */ +class MediaSystemResourceReservationListener { + public: + virtual void ResourceReserved() = 0; + virtual void ResourceReserveFailed() = 0; +}; + +/** + * MediaSystemResourceClient is used to reserve a media system resource + * like hw decoder. When system has a limitation of a media resource, + * use this class to mediate use rights of the resource. + */ +class MediaSystemResourceClient { + public: + // Enumeration for the valid decoding states + enum ResourceState { + RESOURCE_STATE_START, + RESOURCE_STATE_WAITING, + RESOURCE_STATE_ACQUIRED, + RESOURCE_STATE_NOT_ACQUIRED, + RESOURCE_STATE_END + }; + + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaSystemResourceClient) + + explicit MediaSystemResourceClient(MediaSystemResourceType aReourceType); + + bool SetListener(MediaSystemResourceReservationListener* aListener); + + // Try to acquire media resource asynchronously. + // If the resource is used by others, wait until acquired. + void Acquire(); + + // Try to acquire media resource synchronously. If the resource is not + // immediately available, fail to acquire it. return false if resource is not + // acquired. return true if resource is acquired. + // + // This function should not be called on ImageBridge thread. + // It should be used only for compatibility with legacy code. + bool AcquireSyncNoWait(); + + void ReleaseResource(); + + private: + ~MediaSystemResourceClient(); + + RefPtr<MediaSystemResourceManager> mManager; + const MediaSystemResourceType mResourceType; + const uint32_t mId; + + // Modified only by MediaSystemResourceManager. + // Accessed and modified with MediaSystemResourceManager::mReentrantMonitor + // held. + MediaSystemResourceReservationListener* mListener; + ResourceState mResourceState; + bool mIsSync; + ReentrantMonitor* mAcquireSyncWaitMonitor; + bool* mAcquireSyncWaitDone; + + static mozilla::Atomic<uint32_t> sSerialCounter; + + friend class MediaSystemResourceManager; +}; + +} // namespace mozilla + +#endif diff --git a/dom/media/systemservices/MediaSystemResourceManager.cpp b/dom/media/systemservices/MediaSystemResourceManager.cpp new file mode 100644 index 0000000000..414ef8e81d --- /dev/null +++ b/dom/media/systemservices/MediaSystemResourceManager.cpp @@ -0,0 +1,358 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/TaskQueue.h" + +#include "MediaSystemResourceManagerChild.h" +#include "MediaSystemResourceClient.h" + +#include "mozilla/layers/ImageBridgeChild.h" + +#include "MediaSystemResourceManager.h" + +namespace mozilla { + +using namespace mozilla::ipc; +using namespace mozilla::layers; + +/* static */ +StaticRefPtr<MediaSystemResourceManager> MediaSystemResourceManager::sSingleton; + +/* static */ +MediaSystemResourceManager* MediaSystemResourceManager::Get() { + if (sSingleton) { + return sSingleton; + } + MediaSystemResourceManager::Init(); + return sSingleton; +} + +/* static */ +void MediaSystemResourceManager::Shutdown() { + MOZ_ASSERT(InImageBridgeChildThread()); + if (sSingleton) { + sSingleton->CloseIPC(); + sSingleton = nullptr; + } +} + +/* static */ +void MediaSystemResourceManager::Init() { + RefPtr<ImageBridgeChild> imageBridge = ImageBridgeChild::GetSingleton(); + if (!imageBridge) { + NS_WARNING("ImageBridge does not exist"); + return; + } + + if (InImageBridgeChildThread()) { + if (!sSingleton) { +#ifdef DEBUG + static int timesCreated = 0; + timesCreated++; + MOZ_ASSERT(timesCreated == 1); +#endif + sSingleton = new MediaSystemResourceManager(); + } + return; + } + + ReentrantMonitor barrier MOZ_UNANNOTATED("MediaSystemResourceManager::Init"); + ReentrantMonitorAutoEnter mainThreadAutoMon(barrier); + bool done = false; + + RefPtr<Runnable> runnable = + NS_NewRunnableFunction("MediaSystemResourceManager::Init", [&]() { + if (!sSingleton) { + sSingleton = new MediaSystemResourceManager(); + } + ReentrantMonitorAutoEnter childThreadAutoMon(barrier); + done = true; + barrier.NotifyAll(); + }); + + imageBridge->GetThread()->Dispatch(runnable.forget()); + + // should stop the thread until done. + while (!done) { + barrier.Wait(); + } +} + +MediaSystemResourceManager::MediaSystemResourceManager() + : mReentrantMonitor("MediaSystemResourceManager.mReentrantMonitor"), + mShutDown(false), + mChild(nullptr) { + MOZ_ASSERT(InImageBridgeChildThread()); + OpenIPC(); +} + +MediaSystemResourceManager::~MediaSystemResourceManager() { + MOZ_ASSERT(IsIpcClosed()); +} + +void MediaSystemResourceManager::OpenIPC() { + MOZ_ASSERT(InImageBridgeChildThread()); + MOZ_ASSERT(!mChild); + + media::PMediaSystemResourceManagerChild* child = + ImageBridgeChild::GetSingleton() + ->SendPMediaSystemResourceManagerConstructor(); + mChild = static_cast<media::MediaSystemResourceManagerChild*>(child); + mChild->SetManager(this); +} + +void MediaSystemResourceManager::CloseIPC() { + MOZ_ASSERT(InImageBridgeChildThread()); + + if (!mChild) { + return; + } + mChild->Destroy(); + mChild = nullptr; + mShutDown = true; +} + +void MediaSystemResourceManager::OnIpcClosed() { mChild = nullptr; } + +bool MediaSystemResourceManager::IsIpcClosed() { return mChild ? true : false; } + +void MediaSystemResourceManager::Register(MediaSystemResourceClient* aClient) { + ReentrantMonitorAutoEnter mon(mReentrantMonitor); + MOZ_ASSERT(aClient); + MOZ_ASSERT(!mResourceClients.Contains(aClient->mId)); + + mResourceClients.InsertOrUpdate(aClient->mId, aClient); +} + +void MediaSystemResourceManager::Unregister( + MediaSystemResourceClient* aClient) { + ReentrantMonitorAutoEnter mon(mReentrantMonitor); + MOZ_ASSERT(aClient); + MOZ_ASSERT(mResourceClients.Contains(aClient->mId)); + MOZ_ASSERT(mResourceClients.Get(aClient->mId) == aClient); + + mResourceClients.Remove(aClient->mId); +} + +bool MediaSystemResourceManager::SetListener( + MediaSystemResourceClient* aClient, + MediaSystemResourceReservationListener* aListener) { + ReentrantMonitorAutoEnter mon(mReentrantMonitor); + MOZ_ASSERT(aClient); + + MediaSystemResourceClient* client = mResourceClients.Get(aClient->mId); + MOZ_ASSERT(client); + + if (!client) { + return false; + } + // State Check + if (aClient->mResourceState != + MediaSystemResourceClient::RESOURCE_STATE_START) { + return false; + } + aClient->mListener = aListener; + return true; +} + +void MediaSystemResourceManager::Acquire(MediaSystemResourceClient* aClient) { + MOZ_ASSERT(aClient); + MOZ_ASSERT(!InImageBridgeChildThread()); + + ReentrantMonitorAutoEnter mon(mReentrantMonitor); + MediaSystemResourceClient* client = mResourceClients.Get(aClient->mId); + MOZ_ASSERT(client); + MOZ_ASSERT(client == aClient); + + aClient->mIsSync = false; // async request + + if (!client) { + HandleAcquireResult(aClient->mId, false); + return; + } + // State Check + if (aClient->mResourceState != + MediaSystemResourceClient::RESOURCE_STATE_START) { + HandleAcquireResult(aClient->mId, false); + return; + } + aClient->mResourceState = MediaSystemResourceClient::RESOURCE_STATE_WAITING; + ImageBridgeChild::GetSingleton()->GetThread()->Dispatch( + NewRunnableMethod<uint32_t>("MediaSystemResourceManager::DoAcquire", this, + &MediaSystemResourceManager::DoAcquire, + aClient->mId)); +} + +bool MediaSystemResourceManager::AcquireSyncNoWait( + MediaSystemResourceClient* aClient) { + MOZ_ASSERT(aClient); + MOZ_ASSERT(!InImageBridgeChildThread()); + + ReentrantMonitor barrier MOZ_UNANNOTATED( + "MediaSystemResourceManager::AcquireSyncNoWait"); + ReentrantMonitorAutoEnter autoMon(barrier); + bool done = false; + { + ReentrantMonitorAutoEnter mon(mReentrantMonitor); + MediaSystemResourceClient* client = mResourceClients.Get(aClient->mId); + MOZ_ASSERT(client); + MOZ_ASSERT(client == aClient); + + aClient->mIsSync = true; // sync request + + if (InImageBridgeChildThread()) { + HandleAcquireResult(aClient->mId, false); + return false; + } + if (!client || client != aClient) { + HandleAcquireResult(aClient->mId, false); + return false; + } + // State Check + if (aClient->mResourceState != + MediaSystemResourceClient::RESOURCE_STATE_START) { + HandleAcquireResult(aClient->mId, false); + return false; + } + // Hold barrier Monitor until acquire task end. + aClient->mAcquireSyncWaitMonitor = &barrier; + aClient->mAcquireSyncWaitDone = &done; + aClient->mResourceState = MediaSystemResourceClient::RESOURCE_STATE_WAITING; + } + + ImageBridgeChild::GetSingleton()->GetThread()->Dispatch( + NewRunnableMethod<uint32_t>("MediaSystemResourceManager::DoAcquire", this, + &MediaSystemResourceManager::DoAcquire, + aClient->mId)); + + // should stop the thread until done. + while (!done) { + barrier.Wait(); + } + + { + ReentrantMonitorAutoEnter mon(mReentrantMonitor); + if (aClient->mResourceState != + MediaSystemResourceClient::RESOURCE_STATE_ACQUIRED) { + return false; + } + return true; + } +} + +void MediaSystemResourceManager::DoAcquire(uint32_t aId) { + MOZ_ASSERT(InImageBridgeChildThread()); + if (mShutDown || !mChild) { + HandleAcquireResult(aId, false); + return; + } + { + ReentrantMonitorAutoEnter mon(mReentrantMonitor); + MediaSystemResourceClient* client = mResourceClients.Get(aId); + MOZ_ASSERT(client); + + if (!client || client->mResourceState != + MediaSystemResourceClient::RESOURCE_STATE_WAITING) { + HandleAcquireResult(aId, false); + return; + } + MOZ_ASSERT(aId == client->mId); + bool willWait = !client->mAcquireSyncWaitMonitor ? true : false; + mChild->SendAcquire(client->mId, client->mResourceType, willWait); + } +} + +void MediaSystemResourceManager::ReleaseResource( + MediaSystemResourceClient* aClient) { + MOZ_ASSERT(aClient); + { + ReentrantMonitorAutoEnter mon(mReentrantMonitor); + MediaSystemResourceClient* client = mResourceClients.Get(aClient->mId); + MOZ_ASSERT(client); + MOZ_ASSERT(client == aClient); + + if (!client || client != aClient || + aClient->mResourceState == + MediaSystemResourceClient::RESOURCE_STATE_START || + aClient->mResourceState == + MediaSystemResourceClient::RESOURCE_STATE_END) { + aClient->mResourceState = MediaSystemResourceClient::RESOURCE_STATE_END; + return; + } + + aClient->mResourceState = MediaSystemResourceClient::RESOURCE_STATE_END; + + ImageBridgeChild::GetSingleton()->GetThread()->Dispatch( + NewRunnableMethod<uint32_t>( + "MediaSystemResourceManager::DoRelease", this, + &MediaSystemResourceManager::DoRelease, aClient->mId)); + } +} + +void MediaSystemResourceManager::DoRelease(uint32_t aId) { + MOZ_ASSERT(InImageBridgeChildThread()); + if (mShutDown || !mChild) { + return; + } + mChild->SendRelease(aId); +} + +void MediaSystemResourceManager::RecvResponse(uint32_t aId, bool aSuccess) { + HandleAcquireResult(aId, aSuccess); +} + +void MediaSystemResourceManager::HandleAcquireResult(uint32_t aId, + bool aSuccess) { + if (!InImageBridgeChildThread()) { + ImageBridgeChild::GetSingleton()->GetThread()->Dispatch( + NewRunnableMethod<uint32_t, bool>( + "MediaSystemResourceManager::HandleAcquireResult", this, + &MediaSystemResourceManager::HandleAcquireResult, aId, aSuccess)); + return; + } + + ReentrantMonitorAutoEnter mon(mReentrantMonitor); + MediaSystemResourceClient* client = mResourceClients.Get(aId); + if (!client) { + // Client was already unregistered. + return; + } + if (client->mResourceState != + MediaSystemResourceClient::RESOURCE_STATE_WAITING) { + return; + } + + // Update state + if (aSuccess) { + client->mResourceState = MediaSystemResourceClient::RESOURCE_STATE_ACQUIRED; + } else { + client->mResourceState = + MediaSystemResourceClient::RESOURCE_STATE_NOT_ACQUIRED; + } + + if (client->mIsSync) { + if (client->mAcquireSyncWaitMonitor) { + // Notify AcquireSync() complete + MOZ_ASSERT(client->mAcquireSyncWaitDone); + ReentrantMonitorAutoEnter autoMon(*client->mAcquireSyncWaitMonitor); + *client->mAcquireSyncWaitDone = true; + client->mAcquireSyncWaitMonitor->NotifyAll(); + client->mAcquireSyncWaitMonitor = nullptr; + client->mAcquireSyncWaitDone = nullptr; + } + } else { + // Notify Acquire() result + if (client->mListener) { + if (aSuccess) { + client->mListener->ResourceReserved(); + } else { + client->mListener->ResourceReserveFailed(); + } + } + } +} + +} // namespace mozilla diff --git a/dom/media/systemservices/MediaSystemResourceManager.h b/dom/media/systemservices/MediaSystemResourceManager.h new file mode 100644 index 0000000000..293595ece0 --- /dev/null +++ b/dom/media/systemservices/MediaSystemResourceManager.h @@ -0,0 +1,81 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(MediaSystemResourceManager_h_) +# define MediaSystemResourceManager_h_ + +# include <queue> + +# include "MediaSystemResourceTypes.h" +# include "mozilla/ReentrantMonitor.h" +# include "mozilla/StaticPtr.h" +# include "nsTHashMap.h" +# include "nsISupportsImpl.h" + +namespace mozilla { + +namespace media { +class MediaSystemResourceManagerChild; +} // namespace media + +class MediaSystemResourceClient; +class MediaSystemResourceReservationListener; +class ReentrantMonitor; +class TaskQueue; + +/** + * Manage media system resource allocation requests within a process. + */ +class MediaSystemResourceManager { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaSystemResourceManager) + + static MediaSystemResourceManager* Get(); + static void Init(); + static void Shutdown(); + + void OnIpcClosed(); + + void Register(MediaSystemResourceClient* aClient); + void Unregister(MediaSystemResourceClient* aClient); + + bool SetListener(MediaSystemResourceClient* aClient, + MediaSystemResourceReservationListener* aListener); + + void Acquire(MediaSystemResourceClient* aClient); + bool AcquireSyncNoWait(MediaSystemResourceClient* aClient); + void ReleaseResource(MediaSystemResourceClient* aClient); + + void RecvResponse(uint32_t aId, bool aSuccess); + + private: + MediaSystemResourceManager(); + virtual ~MediaSystemResourceManager(); + + void OpenIPC(); + void CloseIPC(); + bool IsIpcClosed(); + + void DoAcquire(uint32_t aId); + + void DoRelease(uint32_t aId); + + void HandleAcquireResult(uint32_t aId, bool aSuccess); + + ReentrantMonitor mReentrantMonitor MOZ_UNANNOTATED; + + bool mShutDown; + + media::MediaSystemResourceManagerChild* mChild; + + nsTHashMap<nsUint32HashKey, MediaSystemResourceClient*> mResourceClients; + + static StaticRefPtr<MediaSystemResourceManager> sSingleton; +}; + +} // namespace mozilla + +#endif diff --git a/dom/media/systemservices/MediaSystemResourceManagerChild.cpp b/dom/media/systemservices/MediaSystemResourceManagerChild.cpp new file mode 100644 index 0000000000..ff671fdf8d --- /dev/null +++ b/dom/media/systemservices/MediaSystemResourceManagerChild.cpp @@ -0,0 +1,42 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MediaSystemResourceManager.h" + +#include "MediaSystemResourceManagerChild.h" + +namespace mozilla::media { + +MediaSystemResourceManagerChild::MediaSystemResourceManagerChild() + : mDestroyed(false), mManager(nullptr) {} + +MediaSystemResourceManagerChild::~MediaSystemResourceManagerChild() = default; + +mozilla::ipc::IPCResult MediaSystemResourceManagerChild::RecvResponse( + const uint32_t& aId, const bool& aSuccess) { + if (mManager) { + mManager->RecvResponse(aId, aSuccess); + } + return IPC_OK(); +} + +void MediaSystemResourceManagerChild::ActorDestroy( + ActorDestroyReason aActorDestroyReason) { + MOZ_ASSERT(!mDestroyed); + if (mManager) { + mManager->OnIpcClosed(); + } + mDestroyed = true; +} + +void MediaSystemResourceManagerChild::Destroy() { + if (mDestroyed) { + return; + } + SendRemoveResourceManager(); + // WARNING: |this| is dead, hands off +} + +} // namespace mozilla::media diff --git a/dom/media/systemservices/MediaSystemResourceManagerChild.h b/dom/media/systemservices/MediaSystemResourceManagerChild.h new file mode 100644 index 0000000000..66bf76cdd3 --- /dev/null +++ b/dom/media/systemservices/MediaSystemResourceManagerChild.h @@ -0,0 +1,65 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(MediaSystemResourceManagerChild_h_) +# define MediaSystemResourceManagerChild_h_ + +# include "mozilla/media/PMediaSystemResourceManagerChild.h" +# include "nsISupportsImpl.h" + +namespace mozilla { + +class MediaSystemResourceManager; + +namespace ipc { +class BackgroundChildImpl; +} // namespace ipc + +namespace media { + +/** + * Handle MediaSystemResourceManager's IPC + */ +class MediaSystemResourceManagerChild final + : public PMediaSystemResourceManagerChild { + friend class PMediaSystemResourceManagerChild; + + public: + struct ResourceListener { + /* The resource is reserved and can be granted. + * The client can allocate the requested resource. + */ + virtual void resourceReserved() = 0; + /* The resource is not reserved any more. + * The client should release the resource as soon as possible if the + * resource is still being held. + */ + virtual void resourceCanceled() = 0; + }; + + MediaSystemResourceManagerChild(); + virtual ~MediaSystemResourceManagerChild(); + + void Destroy(); + + void SetManager(MediaSystemResourceManager* aManager) { mManager = aManager; } + + protected: + mozilla::ipc::IPCResult RecvResponse(const uint32_t& aId, + const bool& aSuccess); + + private: + void ActorDestroy(ActorDestroyReason aActorDestroyReason) override; + + bool mDestroyed; + MediaSystemResourceManager* mManager; + + friend class mozilla::ipc::BackgroundChildImpl; +}; + +} // namespace media +} // namespace mozilla + +#endif diff --git a/dom/media/systemservices/MediaSystemResourceManagerParent.cpp b/dom/media/systemservices/MediaSystemResourceManagerParent.cpp new file mode 100644 index 0000000000..ec20079abc --- /dev/null +++ b/dom/media/systemservices/MediaSystemResourceManagerParent.cpp @@ -0,0 +1,75 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/Unused.h" +#include "mozilla/layers/PImageBridgeParent.h" + +#include "MediaSystemResourceManagerParent.h" + +namespace mozilla::media { + +using namespace ipc; + +MediaSystemResourceManagerParent::MediaSystemResourceManagerParent() + : mDestroyed(false) { + mMediaSystemResourceService = MediaSystemResourceService::Get(); +} + +MediaSystemResourceManagerParent::~MediaSystemResourceManagerParent() { + MOZ_ASSERT(mDestroyed); +} + +mozilla::ipc::IPCResult MediaSystemResourceManagerParent::RecvAcquire( + const uint32_t& aId, const MediaSystemResourceType& aResourceType, + const bool& aWillWait) { + mResourceRequests.WithEntryHandle(aId, [&](auto&& request) { + MOZ_ASSERT(!request); + if (request) { + // Send fail response + mozilla::Unused << SendResponse(aId, false /* fail */); + return; + } + + request.Insert(MakeUnique<MediaSystemResourceRequest>(aId, aResourceType)); + mMediaSystemResourceService->Acquire(this, aId, aResourceType, aWillWait); + }); + + return IPC_OK(); +} + +mozilla::ipc::IPCResult MediaSystemResourceManagerParent::RecvRelease( + const uint32_t& aId) { + MediaSystemResourceRequest* request = mResourceRequests.Get(aId); + if (!request) { + return IPC_OK(); + } + + mMediaSystemResourceService->ReleaseResource(this, aId, + request->mResourceType); + mResourceRequests.Remove(aId); + return IPC_OK(); +} + +mozilla::ipc::IPCResult +MediaSystemResourceManagerParent::RecvRemoveResourceManager() { + IProtocol* mgr = Manager(); + if (!PMediaSystemResourceManagerParent::Send__delete__(this)) { + return IPC_FAIL_NO_REASON(mgr); + } + return IPC_OK(); +} + +void MediaSystemResourceManagerParent::ActorDestroy( + ActorDestroyReason aReason) { + MOZ_ASSERT(!mDestroyed); + + // Release all resource requests of the MediaSystemResourceManagerParent. + // Clears all remaining pointers to this object. + mMediaSystemResourceService->ReleaseResource(this); + + mDestroyed = true; +} + +} // namespace mozilla::media diff --git a/dom/media/systemservices/MediaSystemResourceManagerParent.h b/dom/media/systemservices/MediaSystemResourceManagerParent.h new file mode 100644 index 0000000000..29ed219f2e --- /dev/null +++ b/dom/media/systemservices/MediaSystemResourceManagerParent.h @@ -0,0 +1,59 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(MediaSystemResourceManagerParent_h_) +# define MediaSystemResourceManagerParent_h_ + +# include "MediaSystemResourceManager.h" +# include "MediaSystemResourceService.h" +# include "MediaSystemResourceTypes.h" +# include "mozilla/media/PMediaSystemResourceManagerParent.h" + +namespace mozilla::media { + +/** + * Handle MediaSystemResourceManager's IPC + */ +class MediaSystemResourceManagerParent final + : public PMediaSystemResourceManagerParent { + friend class PMediaSystemResourceManagerParent; + + public: + MediaSystemResourceManagerParent(); + virtual ~MediaSystemResourceManagerParent(); + + protected: + mozilla::ipc::IPCResult RecvAcquire( + const uint32_t& aId, const MediaSystemResourceType& aResourceType, + const bool& aWillWait); + + mozilla::ipc::IPCResult RecvRelease(const uint32_t& aId); + + mozilla::ipc::IPCResult RecvRemoveResourceManager(); + + private: + void ActorDestroy(ActorDestroyReason aActorDestroyReason) override; + + struct MediaSystemResourceRequest { + MediaSystemResourceRequest() + : mId(-1), mResourceType(MediaSystemResourceType::INVALID_RESOURCE) {} + MediaSystemResourceRequest(uint32_t aId, + MediaSystemResourceType aResourceType) + : mId(aId), mResourceType(aResourceType) {} + int32_t mId; + MediaSystemResourceType mResourceType; + }; + + bool mDestroyed; + + RefPtr<MediaSystemResourceService> mMediaSystemResourceService; + + nsClassHashtable<nsUint32HashKey, MediaSystemResourceRequest> + mResourceRequests; +}; + +} // namespace mozilla::media + +#endif diff --git a/dom/media/systemservices/MediaSystemResourceMessageUtils.h b/dom/media/systemservices/MediaSystemResourceMessageUtils.h new file mode 100644 index 0000000000..f06da1467d --- /dev/null +++ b/dom/media/systemservices/MediaSystemResourceMessageUtils.h @@ -0,0 +1,24 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(MediaSystemResourceMessageUtils_h_) +# define MediaSystemResourceMessageUtils_h_ + +# include "ipc/EnumSerializer.h" +# include "MediaSystemResourceTypes.h" + +namespace IPC { + +template <> +struct ParamTraits<mozilla::MediaSystemResourceType> + : public ContiguousEnumSerializer< + mozilla::MediaSystemResourceType, + mozilla::MediaSystemResourceType::VIDEO_DECODER, + mozilla::MediaSystemResourceType::INVALID_RESOURCE> {}; + +} // namespace IPC + +#endif diff --git a/dom/media/systemservices/MediaSystemResourceService.cpp b/dom/media/systemservices/MediaSystemResourceService.cpp new file mode 100644 index 0000000000..88c4566e76 --- /dev/null +++ b/dom/media/systemservices/MediaSystemResourceService.cpp @@ -0,0 +1,222 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MediaSystemResourceManagerParent.h" +#include "mozilla/layers/CompositorThread.h" +#include "mozilla/Unused.h" + +#include "MediaSystemResourceService.h" + +using namespace mozilla::layers; + +namespace mozilla { + +/* static */ +StaticRefPtr<MediaSystemResourceService> MediaSystemResourceService::sSingleton; + +/* static */ +MediaSystemResourceService* MediaSystemResourceService::Get() { + if (sSingleton) { + return sSingleton; + } + Init(); + return sSingleton; +} + +/* static */ +void MediaSystemResourceService::Init() { + if (!sSingleton) { + sSingleton = new MediaSystemResourceService(); + } +} + +/* static */ +void MediaSystemResourceService::Shutdown() { + if (sSingleton) { + sSingleton->Destroy(); + sSingleton = nullptr; + } +} + +MediaSystemResourceService::MediaSystemResourceService() : mDestroyed(false) { + MOZ_ASSERT(CompositorThreadHolder::IsInCompositorThread()); +} + +MediaSystemResourceService::~MediaSystemResourceService() = default; + +void MediaSystemResourceService::Destroy() { mDestroyed = true; } + +void MediaSystemResourceService::Acquire( + media::MediaSystemResourceManagerParent* aParent, uint32_t aId, + MediaSystemResourceType aResourceType, bool aWillWait) { + MOZ_ASSERT(CompositorThreadHolder::IsInCompositorThread()); + MOZ_ASSERT(aParent); + + if (mDestroyed) { + return; + } + + MediaSystemResource* resource = + mResources.Get(static_cast<uint32_t>(aResourceType)); + + if (!resource || resource->mResourceCount == 0) { + // Resource does not exit + // Send fail response + mozilla::Unused << aParent->SendResponse(aId, false /* fail */); + return; + } + + // Try to acquire a resource + if (resource->mAcquiredRequests.size() < resource->mResourceCount) { + // Resource is available + resource->mAcquiredRequests.push_back( + MediaSystemResourceRequest(aParent, aId)); + // Send success response + mozilla::Unused << aParent->SendResponse(aId, true /* success */); + return; + } + + if (!aWillWait) { + // Resource is not available and do not wait. + // Send fail response + mozilla::Unused << aParent->SendResponse(aId, false /* fail */); + return; + } + // Wait until acquire. + resource->mWaitingRequests.push_back( + MediaSystemResourceRequest(aParent, aId)); +} + +void MediaSystemResourceService::ReleaseResource( + media::MediaSystemResourceManagerParent* aParent, uint32_t aId, + MediaSystemResourceType aResourceType) { + MOZ_ASSERT(CompositorThreadHolder::IsInCompositorThread()); + MOZ_ASSERT(aParent); + + if (mDestroyed) { + return; + } + + MediaSystemResource* resource = + mResources.Get(static_cast<uint32_t>(aResourceType)); + + if (!resource || resource->mResourceCount == 0) { + // Resource does not exit + return; + } + RemoveRequest(aParent, aId, aResourceType); + UpdateRequests(aResourceType); +} + +void MediaSystemResourceService::ReleaseResource( + media::MediaSystemResourceManagerParent* aParent) { + MOZ_ASSERT(aParent); + + if (mDestroyed) { + return; + } + + for (const uint32_t& key : mResources.Keys()) { + RemoveRequests(aParent, static_cast<MediaSystemResourceType>(key)); + UpdateRequests(static_cast<MediaSystemResourceType>(key)); + } +} + +void MediaSystemResourceService::RemoveRequest( + media::MediaSystemResourceManagerParent* aParent, uint32_t aId, + MediaSystemResourceType aResourceType) { + MOZ_ASSERT(aParent); + + MediaSystemResource* resource = + mResources.Get(static_cast<uint32_t>(aResourceType)); + if (!resource) { + return; + } + + std::deque<MediaSystemResourceRequest>::iterator it; + std::deque<MediaSystemResourceRequest>& acquiredRequests = + resource->mAcquiredRequests; + for (it = acquiredRequests.begin(); it != acquiredRequests.end(); it++) { + if (((*it).mParent == aParent) && ((*it).mId == aId)) { + acquiredRequests.erase(it); + return; + } + } + + std::deque<MediaSystemResourceRequest>& waitingRequests = + resource->mWaitingRequests; + for (it = waitingRequests.begin(); it != waitingRequests.end(); it++) { + if (((*it).mParent == aParent) && ((*it).mId == aId)) { + waitingRequests.erase(it); + return; + } + } +} + +void MediaSystemResourceService::RemoveRequests( + media::MediaSystemResourceManagerParent* aParent, + MediaSystemResourceType aResourceType) { + MOZ_ASSERT(aParent); + + MediaSystemResource* resource = + mResources.Get(static_cast<uint32_t>(aResourceType)); + + if (!resource || resource->mResourceCount == 0) { + // Resource does not exit + return; + } + + std::deque<MediaSystemResourceRequest>::iterator it; + std::deque<MediaSystemResourceRequest>& acquiredRequests = + resource->mAcquiredRequests; + for (it = acquiredRequests.begin(); it != acquiredRequests.end();) { + if ((*it).mParent == aParent) { + it = acquiredRequests.erase(it); + } else { + it++; + } + } + + std::deque<MediaSystemResourceRequest>& waitingRequests = + resource->mWaitingRequests; + for (it = waitingRequests.begin(); it != waitingRequests.end();) { + if ((*it).mParent == aParent) { + it = waitingRequests.erase(it); + } else { + it++; + } + } +} + +void MediaSystemResourceService::UpdateRequests( + MediaSystemResourceType aResourceType) { + MediaSystemResource* resource = + mResources.Get(static_cast<uint32_t>(aResourceType)); + + if (!resource || resource->mResourceCount == 0) { + // Resource does not exit + return; + } + + std::deque<MediaSystemResourceRequest>& acquiredRequests = + resource->mAcquiredRequests; + std::deque<MediaSystemResourceRequest>& waitingRequests = + resource->mWaitingRequests; + + while ((acquiredRequests.size() < resource->mResourceCount) && + (!waitingRequests.empty())) { + MediaSystemResourceRequest& request = waitingRequests.front(); + MOZ_ASSERT(request.mParent); + // Send response + mozilla::Unused << request.mParent->SendResponse(request.mId, + true /* success */); + // Move request to mAcquiredRequests + acquiredRequests.push_back(waitingRequests.front()); + waitingRequests.pop_front(); + } +} + +} // namespace mozilla diff --git a/dom/media/systemservices/MediaSystemResourceService.h b/dom/media/systemservices/MediaSystemResourceService.h new file mode 100644 index 0000000000..8a75a6cafd --- /dev/null +++ b/dom/media/systemservices/MediaSystemResourceService.h @@ -0,0 +1,83 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(MediaSystemResourceService_h_) +# define MediaSystemResourceService_h_ + +# include <deque> + +# include "MediaSystemResourceTypes.h" +# include "mozilla/StaticPtr.h" +# include "nsClassHashtable.h" + +namespace mozilla { + +namespace media { +class MediaSystemResourceManagerParent; +} // namespace media + +/** + * Manage media system resource allocation requests within system. + */ +class MediaSystemResourceService { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaSystemResourceService) + + static MediaSystemResourceService* Get(); + static void Init(); + static void Shutdown(); + + void Acquire(media::MediaSystemResourceManagerParent* aParent, uint32_t aId, + MediaSystemResourceType aResourceType, bool aWillWait); + + void ReleaseResource(media::MediaSystemResourceManagerParent* aParent, + uint32_t aId, MediaSystemResourceType aResourceType); + + void ReleaseResource(media::MediaSystemResourceManagerParent* aParent); + + private: + MediaSystemResourceService(); + ~MediaSystemResourceService(); + + struct MediaSystemResourceRequest { + MediaSystemResourceRequest() : mParent(nullptr), mId(-1) {} + MediaSystemResourceRequest(media::MediaSystemResourceManagerParent* aParent, + uint32_t aId) + : mParent(aParent), mId(aId) {} + media::MediaSystemResourceManagerParent* mParent; + uint32_t mId; + }; + + struct MediaSystemResource { + MediaSystemResource() : mResourceCount(0) {} + explicit MediaSystemResource(uint32_t aResourceCount) + : mResourceCount(aResourceCount) {} + + std::deque<MediaSystemResourceRequest> mWaitingRequests; + std::deque<MediaSystemResourceRequest> mAcquiredRequests; + uint32_t mResourceCount; + }; + + void Destroy(); + + void RemoveRequest(media::MediaSystemResourceManagerParent* aParent, + uint32_t aId, MediaSystemResourceType aResourceType); + + void RemoveRequests(media::MediaSystemResourceManagerParent* aParent, + MediaSystemResourceType aResourceType); + + void UpdateRequests(MediaSystemResourceType aResourceType); + + bool mDestroyed; + + nsClassHashtable<nsUint32HashKey, MediaSystemResource> mResources; + + static StaticRefPtr<MediaSystemResourceService> sSingleton; +}; + +} // namespace mozilla + +#endif diff --git a/dom/media/systemservices/MediaSystemResourceTypes.h b/dom/media/systemservices/MediaSystemResourceTypes.h new file mode 100644 index 0000000000..d294c2b364 --- /dev/null +++ b/dom/media/systemservices/MediaSystemResourceTypes.h @@ -0,0 +1,23 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(MediaSystemResourceTypes_h_) +# define MediaSystemResourceTypes_h_ + +namespace mozilla { + +enum class MediaSystemResourceType : uint32_t { + VIDEO_DECODER = 0, + AUDIO_DECODER, // Not supported currently. + VIDEO_ENCODER, + AUDIO_ENCODER, // Not supported currently. + CAMERA, // Not supported currently. + INVALID_RESOURCE, +}; + +} // namespace mozilla + +#endif diff --git a/dom/media/systemservices/MediaTaskUtils.h b/dom/media/systemservices/MediaTaskUtils.h new file mode 100644 index 0000000000..cbe464e015 --- /dev/null +++ b/dom/media/systemservices/MediaTaskUtils.h @@ -0,0 +1,52 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_MediaTaskUtils_h +#define mozilla_MediaTaskUtils_h + +#include "nsThreadUtils.h" + +// The main reason this file is separate from MediaUtils.h +#include "base/task.h" + +namespace mozilla { +namespace media { + +/* media::NewTaskFrom() - Create a Task from a lambda. + * + * Similar to media::NewRunnableFrom() - Create an nsRunnable from a lambda, + * but ignore the return value from the lambda. + * + * Prefer NS_NewRunnableFunction(), which provides a specific name, unless the + * lambda really must have a non-void return value that is to be ignored. + */ + +template <typename OnRunType> +class LambdaTask : public Runnable { + public: + explicit LambdaTask(OnRunType&& aOnRun) + : Runnable("media::LambdaTask"), mOnRun(std::move(aOnRun)) {} + + private: + NS_IMETHOD + Run() override { + mOnRun(); + return NS_OK; + } + OnRunType mOnRun; +}; + +template <typename OnRunType> +already_AddRefed<LambdaTask<OnRunType>> NewTaskFrom(OnRunType&& aOnRun) { + typedef LambdaTask<OnRunType> LambdaType; + RefPtr<LambdaType> lambda = new LambdaType(std::forward<OnRunType>(aOnRun)); + return lambda.forget(); +} + +} // namespace media +} // namespace mozilla + +#endif // mozilla_MediaTaskUtils_h diff --git a/dom/media/systemservices/MediaUtils.cpp b/dom/media/systemservices/MediaUtils.cpp new file mode 100644 index 0000000000..fad2fd4e2c --- /dev/null +++ b/dom/media/systemservices/MediaUtils.cpp @@ -0,0 +1,126 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MediaUtils.h" + +#include "mozilla/AppShutdown.h" +#include "mozilla/Services.h" + +namespace mozilla::media { + +nsCOMPtr<nsIAsyncShutdownClient> GetShutdownBarrier() { + nsCOMPtr<nsIAsyncShutdownService> svc = services::GetAsyncShutdownService(); + if (!svc) { + // We can fail to get the shutdown service if we're already shutting down. + return nullptr; + } + + nsCOMPtr<nsIAsyncShutdownClient> barrier; + nsresult rv = svc->GetProfileBeforeChange(getter_AddRefs(barrier)); + if (!barrier) { + // We are probably in a content process. We need to do cleanup at + // XPCOM shutdown in leakchecking builds. + rv = svc->GetXpcomWillShutdown(getter_AddRefs(barrier)); + } + MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); + MOZ_RELEASE_ASSERT(barrier); + return barrier; +} + +nsCOMPtr<nsIAsyncShutdownClient> MustGetShutdownBarrier() { + nsCOMPtr<nsIAsyncShutdownClient> barrier = GetShutdownBarrier(); + MOZ_RELEASE_ASSERT(barrier); + return barrier; +} + +NS_IMPL_ISUPPORTS(ShutdownBlocker, nsIAsyncShutdownBlocker) + +namespace { +class TicketBlocker : public ShutdownBlocker { + using ShutdownMozPromise = ShutdownBlockingTicket::ShutdownMozPromise; + + public: + explicit TicketBlocker(const nsAString& aName) + : ShutdownBlocker(aName), mPromise(mHolder.Ensure(__func__)) {} + + NS_IMETHOD + BlockShutdown(nsIAsyncShutdownClient* aProfileBeforeChange) override { + mHolder.Resolve(true, __func__); + return NS_OK; + } + + void RejectIfExists() { mHolder.RejectIfExists(false, __func__); } + + ShutdownMozPromise* ShutdownPromise() { return mPromise; } + + private: + ~TicketBlocker() = default; + + MozPromiseHolder<ShutdownMozPromise> mHolder; + const RefPtr<ShutdownMozPromise> mPromise; +}; + +class ShutdownBlockingTicketImpl : public ShutdownBlockingTicket { + private: + RefPtr<TicketBlocker> mBlocker; + + public: + explicit ShutdownBlockingTicketImpl(RefPtr<TicketBlocker> aBlocker) + : mBlocker(std::move(aBlocker)) {} + + static UniquePtr<ShutdownBlockingTicket> Create(const nsAString& aName, + const nsAString& aFileName, + int32_t aLineNr) { + auto blocker = MakeRefPtr<TicketBlocker>(aName); + NS_DispatchToMainThread(NS_NewRunnableFunction( + "ShutdownBlockingTicketImpl::AddBlocker", + [blocker, file = nsString(aFileName), aLineNr] { + MustGetShutdownBarrier()->AddBlocker(blocker, file, aLineNr, u""_ns); + })); + if (AppShutdown::IsInOrBeyond(ShutdownPhase::AppShutdown)) { + // Adding a blocker is not guaranteed to succeed. Remove the blocker in + // case it succeeded anyway, and bail. + NS_DispatchToMainThread(NS_NewRunnableFunction( + "ShutdownBlockingTicketImpl::RemoveBlocker", [blocker] { + MustGetShutdownBarrier()->RemoveBlocker(blocker); + blocker->RejectIfExists(); + })); + return nullptr; + } + + // Adding a blocker is now guaranteed to succeed: + // - If AppShutdown::IsInOrBeyond(AppShutdown) returned false, + // - then the AddBlocker main thread task was queued before AppShutdown's + // sCurrentShutdownPhase is set to ShutdownPhase::AppShutdown, + // - which is before AppShutdown will drain the (main thread) event queue to + // run the AddBlocker task, if not already run, + // - which is before profile-before-change (the earliest barrier we'd add a + // blocker to, see GetShutdownBarrier()) is notified, + // - which is when AsyncShutdown prevents further conditions (blockers) + // being added to the profile-before-change barrier. + return MakeUnique<ShutdownBlockingTicketImpl>(std::move(blocker)); + } + + ~ShutdownBlockingTicketImpl() { + MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread( + NS_NewRunnableFunction(__func__, [blocker = std::move(mBlocker)] { + GetShutdownBarrier()->RemoveBlocker(blocker); + blocker->RejectIfExists(); + }))); + } + + ShutdownMozPromise* ShutdownPromise() override { + return mBlocker->ShutdownPromise(); + } +}; +} // namespace + +UniquePtr<ShutdownBlockingTicket> ShutdownBlockingTicket::Create( + const nsAString& aName, const nsAString& aFileName, int32_t aLineNr) { + return ShutdownBlockingTicketImpl::Create(aName, aFileName, aLineNr); +} + +} // namespace mozilla::media diff --git a/dom/media/systemservices/MediaUtils.h b/dom/media/systemservices/MediaUtils.h new file mode 100644 index 0000000000..90e21ce9aa --- /dev/null +++ b/dom/media/systemservices/MediaUtils.h @@ -0,0 +1,332 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_MediaUtils_h +#define mozilla_MediaUtils_h + +#include <map> + +#include "mozilla/Assertions.h" +#include "mozilla/Monitor.h" +#include "mozilla/MozPromise.h" +#include "mozilla/Mutex.h" +#include "mozilla/RefPtr.h" +#include "mozilla/SharedThreadPool.h" +#include "mozilla/TaskQueue.h" +#include "mozilla/UniquePtr.h" +#include "MediaEventSource.h" +#include "nsCOMPtr.h" +#include "nsIAsyncShutdown.h" +#include "nsISupportsImpl.h" +#include "nsProxyRelease.h" +#include "nsThreadUtils.h" + +class nsIEventTarget; + +namespace mozilla::media { + +/* media::NewRunnableFrom() - Create a Runnable from a lambda. + * + * Passing variables (closures) to an async function is clunky with Runnable: + * + * void Foo() + * { + * class FooRunnable : public Runnable + * { + * public: + * FooRunnable(const Bar &aBar) : mBar(aBar) {} + * NS_IMETHOD Run() override + * { + * // Use mBar + * } + * private: + * RefPtr<Bar> mBar; + * }; + * + * RefPtr<Bar> bar = new Bar(); + * NS_DispatchToMainThread(new FooRunnable(bar); + * } + * + * It's worse with more variables. Lambdas have a leg up with variable capture: + * + * void Foo() + * { + * RefPtr<Bar> bar = new Bar(); + * NS_DispatchToMainThread(media::NewRunnableFrom([bar]() mutable { + * // use bar + * })); + * } + * + * Capture is by-copy by default, so the nsRefPtr 'bar' is safely copied for + * access on the other thread (threadsafe refcounting in bar is assumed). + * + * The 'mutable' keyword is only needed for non-const access to bar. + */ + +template <typename OnRunType> +class LambdaRunnable : public Runnable { + public: + explicit LambdaRunnable(OnRunType&& aOnRun) + : Runnable("media::LambdaRunnable"), mOnRun(std::move(aOnRun)) {} + + private: + NS_IMETHODIMP + Run() override { return mOnRun(); } + OnRunType mOnRun; +}; + +template <typename OnRunType> +already_AddRefed<LambdaRunnable<OnRunType>> NewRunnableFrom( + OnRunType&& aOnRun) { + typedef LambdaRunnable<OnRunType> LambdaType; + RefPtr<LambdaType> lambda = new LambdaType(std::forward<OnRunType>(aOnRun)); + return lambda.forget(); +} + +/* media::Refcountable - Add threadsafe ref-counting to something that isn't. + * + * Often, reference counting is the most practical way to share an object with + * another thread without imposing lifetime restrictions, even if there's + * otherwise no concurrent access happening on the object. For instance, an + * algorithm on another thread may find it more expedient to modify a passed-in + * object, rather than pass expensive copies back and forth. + * + * Lists in particular often aren't ref-countable, yet are expensive to copy, + * e.g. nsTArray<RefPtr<Foo>>. Refcountable can be used to make such objects + * (or owning smart-pointers to such objects) refcountable. + * + * Technical limitation: A template specialization is needed for types that take + * a constructor. Please add below (UniquePtr covers a lot of ground though). + */ + +class RefcountableBase { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(RefcountableBase) + protected: + virtual ~RefcountableBase() = default; +}; + +template <typename T> +class Refcountable : public T, public RefcountableBase { + public: + Refcountable& operator=(T&& aOther) { + T::operator=(std::move(aOther)); + return *this; + } + + Refcountable& operator=(T& aOther) { + T::operator=(aOther); + return *this; + } +}; + +template <typename T> +class Refcountable<UniquePtr<T>> : public UniquePtr<T>, + public RefcountableBase { + public: + explicit Refcountable(T* aPtr) : UniquePtr<T>(aPtr) {} +}; + +template <> +class Refcountable<bool> : public RefcountableBase { + public: + explicit Refcountable(bool aValue) : mValue(aValue) {} + + Refcountable& operator=(bool aOther) { + mValue = aOther; + return *this; + } + + Refcountable& operator=(const Refcountable& aOther) { + mValue = aOther.mValue; + return *this; + } + + explicit operator bool() const { return mValue; } + + private: + bool mValue; +}; + +/* + * Async shutdown helpers + */ + +nsCOMPtr<nsIAsyncShutdownClient> GetShutdownBarrier(); + +// Like GetShutdownBarrier but will release assert that the result is not null. +nsCOMPtr<nsIAsyncShutdownClient> MustGetShutdownBarrier(); + +class ShutdownBlocker : public nsIAsyncShutdownBlocker { + public: + ShutdownBlocker(const nsAString& aName) : mName(aName) {} + + NS_IMETHOD + BlockShutdown(nsIAsyncShutdownClient* aProfileBeforeChange) override = 0; + + NS_IMETHOD GetName(nsAString& aName) override { + aName = mName; + return NS_OK; + } + + NS_IMETHOD GetState(nsIPropertyBag**) override { return NS_OK; } + + NS_DECL_THREADSAFE_ISUPPORTS + protected: + virtual ~ShutdownBlocker() = default; + + private: + const nsString mName; +}; + +/** + * A convenience class representing a "ticket" that keeps the process from + * shutting down until it is destructed. It does this by blocking + * xpcom-will-shutdown. Constructed and destroyed on any thread. + */ +class ShutdownBlockingTicket { + public: + using ShutdownMozPromise = MozPromise<bool, bool, false>; + + /** + * Construct with an arbitrary name, __FILE__ and __LINE__. + * Note that __FILE__ needs to be made wide, typically through + * NS_LITERAL_STRING_FROM_CSTRING(__FILE__). + * Returns nullptr if we are too far in the shutdown sequence to add a + * blocker. Any thread. + */ + static UniquePtr<ShutdownBlockingTicket> Create(const nsAString& aName, + const nsAString& aFileName, + int32_t aLineNr); + + virtual ~ShutdownBlockingTicket() = default; + + /** + * MozPromise that gets resolved upon xpcom-will-shutdown. + * Should the ticket get destroyed before the MozPromise has been resolved, + * the MozPromise will get rejected. + */ + virtual ShutdownMozPromise* ShutdownPromise() = 0; +}; + +/** + * Await convenience methods to block until the promise has been resolved or + * rejected. The Resolve/Reject functions, while called on a different thread, + * would be running just as on the current thread thanks to the memory barrier + * provided by the monitor. + * For now Await can only be used with an exclusive MozPromise if passed a + * Resolve/Reject function. + * Await() can *NOT* be called from a task queue/nsISerialEventTarget used for + * resolving/rejecting aPromise, otherwise things will deadlock. + */ +template <typename ResolveValueType, typename RejectValueType, + typename ResolveFunction, typename RejectFunction> +void Await(already_AddRefed<nsIEventTarget> aPool, + RefPtr<MozPromise<ResolveValueType, RejectValueType, true>> aPromise, + ResolveFunction&& aResolveFunction, + RejectFunction&& aRejectFunction) { + RefPtr<TaskQueue> taskQueue = + TaskQueue::Create(std::move(aPool), "MozPromiseAwait"); + Monitor mon MOZ_UNANNOTATED(__func__); + bool done = false; + + aPromise->Then( + taskQueue, __func__, + [&](ResolveValueType&& aResolveValue) { + MonitorAutoLock lock(mon); + aResolveFunction(std::forward<ResolveValueType>(aResolveValue)); + done = true; + mon.Notify(); + }, + [&](RejectValueType&& aRejectValue) { + MonitorAutoLock lock(mon); + aRejectFunction(std::forward<RejectValueType>(aRejectValue)); + done = true; + mon.Notify(); + }); + + MonitorAutoLock lock(mon); + while (!done) { + mon.Wait(); + } +} + +template <typename ResolveValueType, typename RejectValueType, bool Excl> +typename MozPromise<ResolveValueType, RejectValueType, + Excl>::ResolveOrRejectValue +Await(already_AddRefed<nsIEventTarget> aPool, + RefPtr<MozPromise<ResolveValueType, RejectValueType, Excl>> aPromise) { + RefPtr<TaskQueue> taskQueue = + TaskQueue::Create(std::move(aPool), "MozPromiseAwait"); + Monitor mon MOZ_UNANNOTATED(__func__); + bool done = false; + + typename MozPromise<ResolveValueType, RejectValueType, + Excl>::ResolveOrRejectValue val; + aPromise->Then( + taskQueue, __func__, + [&](ResolveValueType aResolveValue) { + val.SetResolve(std::move(aResolveValue)); + MonitorAutoLock lock(mon); + done = true; + mon.Notify(); + }, + [&](RejectValueType aRejectValue) { + val.SetReject(std::move(aRejectValue)); + MonitorAutoLock lock(mon); + done = true; + mon.Notify(); + }); + + MonitorAutoLock lock(mon); + while (!done) { + mon.Wait(); + } + + return val; +} + +/** + * Similar to Await, takes an array of promises of the same type. + * MozPromise::All is used to handle the resolution/rejection of the promises. + */ +template <typename ResolveValueType, typename RejectValueType, + typename ResolveFunction, typename RejectFunction> +void AwaitAll( + already_AddRefed<nsIEventTarget> aPool, + nsTArray<RefPtr<MozPromise<ResolveValueType, RejectValueType, true>>>& + aPromises, + ResolveFunction&& aResolveFunction, RejectFunction&& aRejectFunction) { + typedef MozPromise<ResolveValueType, RejectValueType, true> Promise; + RefPtr<nsIEventTarget> pool = aPool; + RefPtr<TaskQueue> taskQueue = + TaskQueue::Create(do_AddRef(pool), "MozPromiseAwaitAll"); + RefPtr<typename Promise::AllPromiseType> p = + Promise::All(taskQueue, aPromises); + Await(pool.forget(), p, std::move(aResolveFunction), + std::move(aRejectFunction)); +} + +// Note: only works with exclusive MozPromise, as Promise::All would attempt +// to perform copy of nsTArrays which are disallowed. +template <typename ResolveValueType, typename RejectValueType> +typename MozPromise<ResolveValueType, RejectValueType, + true>::AllPromiseType::ResolveOrRejectValue +AwaitAll(already_AddRefed<nsIEventTarget> aPool, + nsTArray<RefPtr<MozPromise<ResolveValueType, RejectValueType, true>>>& + aPromises) { + typedef MozPromise<ResolveValueType, RejectValueType, true> Promise; + RefPtr<nsIEventTarget> pool = aPool; + RefPtr<TaskQueue> taskQueue = + TaskQueue::Create(do_AddRef(pool), "MozPromiseAwaitAll"); + RefPtr<typename Promise::AllPromiseType> p = + Promise::All(taskQueue, aPromises); + return Await(pool.forget(), p); +} + +} // namespace mozilla::media + +#endif // mozilla_MediaUtils_h diff --git a/dom/media/systemservices/OSXRunLoopSingleton.cpp b/dom/media/systemservices/OSXRunLoopSingleton.cpp new file mode 100644 index 0000000000..6dea084e27 --- /dev/null +++ b/dom/media/systemservices/OSXRunLoopSingleton.cpp @@ -0,0 +1,41 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "OSXRunLoopSingleton.h" +#include <mozilla/StaticMutex.h> + +#include <AudioUnit/AudioUnit.h> +#include <CoreAudio/AudioHardware.h> +#include <CoreAudio/HostTime.h> +#include <CoreFoundation/CoreFoundation.h> + +static bool gRunLoopSet = false; +static mozilla::StaticMutex gMutex MOZ_UNANNOTATED; + +void mozilla_set_coreaudio_notification_runloop_if_needed() { + mozilla::StaticMutexAutoLock lock(gMutex); + if (gRunLoopSet) { + return; + } + + /* This is needed so that AudioUnit listeners get called on this thread, and + * not the main thread. If we don't do that, they are not called, or a crash + * occur, depending on the OSX version. */ + AudioObjectPropertyAddress runloop_address = { + kAudioHardwarePropertyRunLoop, kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMaster}; + + CFRunLoopRef run_loop = nullptr; + + OSStatus r; + r = AudioObjectSetPropertyData(kAudioObjectSystemObject, &runloop_address, 0, + NULL, sizeof(CFRunLoopRef), &run_loop); + if (r != noErr) { + NS_WARNING( + "Could not make global CoreAudio notifications use their own thread."); + } + + gRunLoopSet = true; +} diff --git a/dom/media/systemservices/OSXRunLoopSingleton.h b/dom/media/systemservices/OSXRunLoopSingleton.h new file mode 100644 index 0000000000..10e7b0153f --- /dev/null +++ b/dom/media/systemservices/OSXRunLoopSingleton.h @@ -0,0 +1,24 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef OSXRUNLOOPSINGLETON_H_ +#define OSXRUNLOOPSINGLETON_H_ + +#include <mozilla/Types.h> + +#if defined(__cplusplus) +extern "C" { +#endif + +/* This function tells CoreAudio to use its own thread for device change + * notifications, and can be called from any thread without external + * synchronization. */ +void MOZ_EXPORT mozilla_set_coreaudio_notification_runloop_if_needed(); + +#if defined(__cplusplus) +} +#endif + +#endif // OSXRUNLOOPSINGLETON_H_ diff --git a/dom/media/systemservices/PCameras.ipdl b/dom/media/systemservices/PCameras.ipdl new file mode 100644 index 0000000000..535702cd68 --- /dev/null +++ b/dom/media/systemservices/PCameras.ipdl @@ -0,0 +1,93 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include protocol PContent; +include protocol PBackground; + +include PBackgroundSharedTypes; + +using mozilla::camera::CaptureEngine from "mozilla/media/CamerasTypes.h"; + +namespace mozilla { +namespace camera { + +// IPC analog for webrtc::VideoCaptureCapability +struct VideoCaptureCapability +{ + int width; + int height; + int maxFPS; + int videoType; + bool interlaced; +}; + + +// IPC analog for webrtc::VideoFrame +// the described buffer is transported seperately in a Shmem +// See VideoFrameUtils.h +struct VideoFrameProperties +{ + // Size of image data within the ShMem, + // the ShMem is at least this large + uint32_t bufferSize; + // From webrtc::VideoFrame + uint32_t timeStamp; + int64_t ntpTimeMs; + int64_t renderTimeMs; + // See webrtc/**/rotation.h + int rotation; + int yAllocatedSize; + int uAllocatedSize; + int vAllocatedSize; + // From webrtc::VideoFrameBuffer + int width; + int height; + int yStride; + int uStride; + int vStride; +}; + +[ManualDealloc, ChildImpl=virtual, ParentImpl=virtual] +async protocol PCameras +{ + manager PBackground; + +child: + // transfers ownership of |buffer| from parent to child + async DeliverFrame(CaptureEngine capEngine, int streamId, + Shmem buffer, VideoFrameProperties props); + async DeviceChange(); + async ReplyNumberOfCaptureDevices(int deviceCount); + async ReplyNumberOfCapabilities(int capabilityCount); + async ReplyAllocateCapture(int captureId); + async ReplyGetCaptureCapability(VideoCaptureCapability cap); + async ReplyGetCaptureDevice(nsCString device_name, nsCString device_id, bool scary); + async ReplyFailure(); + async ReplySuccess(); + async __delete__(); + +parent: + async NumberOfCaptureDevices(CaptureEngine engine); + async NumberOfCapabilities(CaptureEngine engine, nsCString deviceUniqueIdUTF8); + + async GetCaptureCapability(CaptureEngine engine, nsCString unique_idUTF8, + int capability_number); + async GetCaptureDevice(CaptureEngine engine, int deviceIndex); + + async AllocateCapture(CaptureEngine engine, nsCString unique_idUTF8, + uint64_t windowID); + async ReleaseCapture(CaptureEngine engine, int captureId); + async StartCapture(CaptureEngine engine, int captureId, + VideoCaptureCapability capability); + async FocusOnSelectedSource(CaptureEngine engine, int captureId); + async StopCapture(CaptureEngine engine, int captureId); + // transfers frame back + async ReleaseFrame(Shmem s); + + // setup camera engine + async EnsureInitialized(CaptureEngine engine); +}; + +} // namespace camera +} // namespace mozilla diff --git a/dom/media/systemservices/PMedia.ipdl b/dom/media/systemservices/PMedia.ipdl new file mode 100644 index 0000000000..c3d8476b5f --- /dev/null +++ b/dom/media/systemservices/PMedia.ipdl @@ -0,0 +1,55 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include protocol PContent; + +include PBackgroundSharedTypes; + +include "mozilla/media/MediaChild.h"; + +namespace mozilla { +namespace media { + +[ManualDealloc, ChildImpl="Child", ParentImpl=virtual] +protocol PMedia +{ + manager PContent; + +parent: + /** + * Requests a potentially persistent unique secret key for each principal. + * Has no expiry, but is cleared by age along with cookies. + * This is needed by mediaDevices.enumerateDevices() to produce persistent + * deviceIds that wont work cross-origin. + * + * If this OriginAttributes dictionary has the privateBrowsing flag set to + * false, a key for this origin is returned from a primary pool of temporal + * in-memory keys and persistent keys read from disk. If no key exists, a + * temporal one is created. + * If aPersist is true and key is temporal, the key is promoted to persistent. + * Once persistent, a key cannot become temporal again. + * + * If the OriginAttributes dictionary has the privateBrowsing flag set to + * true, a different key for this origin is returned from a secondary pool + * that is never persisted to disk, and aPersist is ignored. + */ + async GetPrincipalKey(PrincipalInfo aPrincipal, bool aPersist) returns(nsCString aKey); + + /** + * Clear per-orgin list of persistent deviceIds stored for enumerateDevices + * Fire and forget. + * + * aSinceTime - milliseconds since 1 January 1970 00:00:00 UTC. 0 = clear all + * + * aOnlyPrivateBrowsing - if true then only purge the separate in-memory + * per-origin list used in Private Browsing. + */ + async SanitizeOriginKeys(uint64_t aSinceWhen, bool aOnlyPrivateBrowsing); + +child: + async __delete__(); +}; + +} // namespace media +} // namespace mozilla diff --git a/dom/media/systemservices/PMediaSystemResourceManager.ipdl b/dom/media/systemservices/PMediaSystemResourceManager.ipdl new file mode 100644 index 0000000000..a682de60bf --- /dev/null +++ b/dom/media/systemservices/PMediaSystemResourceManager.ipdl @@ -0,0 +1,38 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include protocol PImageBridge; +include "mozilla/media/MediaSystemResourceMessageUtils.h"; + +using mozilla::MediaSystemResourceType from "mozilla/media/MediaSystemResourceTypes.h"; + +namespace mozilla { +namespace media { + +/* + * The PMediaSystemResourceManager is a sub-protocol in PImageBridge + */ +[ManualDealloc] +sync protocol PMediaSystemResourceManager +{ + manager PImageBridge; + +child: + async Response(uint32_t aId, bool aSuccess); + async __delete__(); + +parent: + async Acquire(uint32_t aId, MediaSystemResourceType aResourceType, bool aWillWait); + async Release(uint32_t aId); + + /** + * Asynchronously tell the parent side to remove the PMediaSystemResourceManager. + */ + async RemoveResourceManager(); +}; + +} // namespace media +} // namespace mozilla + diff --git a/dom/media/systemservices/ShmemPool.cpp b/dom/media/systemservices/ShmemPool.cpp new file mode 100644 index 0000000000..c39ed5e790 --- /dev/null +++ b/dom/media/systemservices/ShmemPool.cpp @@ -0,0 +1,99 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/ShmemPool.h" + +#include <utility> + +#include "mozilla/Assertions.h" +#include "mozilla/Logging.h" + +mozilla::LazyLogModule sShmemPoolLog("ShmemPool"); + +#define SHMEMPOOL_LOG_VERBOSE(args) \ + MOZ_LOG(sShmemPoolLog, mozilla::LogLevel::Verbose, args) + +namespace mozilla { + +ShmemPool::ShmemPool(size_t aPoolSize, PoolType aPoolType) + : mPoolType(aPoolType), + mMutex("mozilla::ShmemPool"), + mPoolFree(aPoolSize), + mErrorLogged(false) +#ifdef DEBUG + , + mMaxPoolUse(0) +#endif +{ + mShmemPool.SetLength(aPoolSize); +} + +mozilla::ShmemBuffer ShmemPool::GetIfAvailable(size_t aSize) { + MutexAutoLock lock(mMutex); + + // Pool is empty, don't block caller. + if (mPoolFree == 0) { + if (!mErrorLogged) { + // log "out of pool" once as error to avoid log spam + mErrorLogged = true; + SHMEMPOOL_LOG_ERROR( + ("ShmemPool is empty, future occurrences " + "will be logged as warnings")); + } else { + SHMEMPOOL_LOG_WARN(("ShmemPool is empty")); + } + // This isn't initialized, so will be understood as an error. + return ShmemBuffer(); + } + + ShmemBuffer& res = mShmemPool[mPoolFree - 1]; + + if (!res.mInitialized) { + SHMEMPOOL_LOG(("No free preallocated Shmem")); + return ShmemBuffer(); + } + + MOZ_ASSERT(res.mShmem.IsWritable(), "Pool in Shmem is not writable?"); + + if (res.mShmem.Size<uint8_t>() < aSize) { + SHMEMPOOL_LOG(("Free Shmem but not of the right size")); + return ShmemBuffer(); + } + + mPoolFree--; +#ifdef DEBUG + size_t poolUse = mShmemPool.Length() - mPoolFree; + if (poolUse > mMaxPoolUse) { + mMaxPoolUse = poolUse; + SHMEMPOOL_LOG( + ("Maximum ShmemPool use increased: %zu buffers", mMaxPoolUse)); + } +#endif + return std::move(res); +} + +void ShmemPool::Put(ShmemBuffer&& aShmem) { + MutexAutoLock lock(mMutex); + MOZ_ASSERT(mPoolFree < mShmemPool.Length()); + mShmemPool[mPoolFree] = std::move(aShmem); + mPoolFree++; +#ifdef DEBUG + size_t poolUse = mShmemPool.Length() - mPoolFree; + if (poolUse > 0) { + SHMEMPOOL_LOG_VERBOSE(("ShmemPool usage reduced to %zu buffers", poolUse)); + } +#endif +} + +ShmemPool::~ShmemPool() { +#ifdef DEBUG + for (size_t i = 0; i < mShmemPool.Length(); i++) { + MOZ_ASSERT(!mShmemPool[i].Valid()); + } +#endif +} + +} // namespace mozilla diff --git a/dom/media/systemservices/ShmemPool.h b/dom/media/systemservices/ShmemPool.h new file mode 100644 index 0000000000..e62ccff24a --- /dev/null +++ b/dom/media/systemservices/ShmemPool.h @@ -0,0 +1,181 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_ShmemPool_h +#define mozilla_ShmemPool_h + +#include "mozilla/Mutex.h" +#include "mozilla/ipc/Shmem.h" +#include "nsTArray.h" + +extern mozilla::LazyLogModule sShmemPoolLog; +#define SHMEMPOOL_LOG(args) \ + MOZ_LOG(sShmemPoolLog, mozilla::LogLevel::Debug, args) +#define SHMEMPOOL_LOG_WARN(args) \ + MOZ_LOG(sShmemPoolLog, mozilla::LogLevel::Warning, args) +#define SHMEMPOOL_LOG_ERROR(args) \ + MOZ_LOG(sShmemPoolLog, mozilla::LogLevel::Error, args) + +namespace mozilla { + +class ShmemPool; + +class ShmemBuffer { + public: + ShmemBuffer() : mInitialized(false) {} + explicit ShmemBuffer(mozilla::ipc::Shmem aShmem) { + mInitialized = true; + mShmem = aShmem; + } + + ShmemBuffer(ShmemBuffer&& rhs) { + mInitialized = rhs.mInitialized; + mShmem = std::move(rhs.mShmem); + } + + ShmemBuffer& operator=(ShmemBuffer&& rhs) { + MOZ_ASSERT(&rhs != this, "self-moves are prohibited"); + mInitialized = rhs.mInitialized; + mShmem = std::move(rhs.mShmem); + return *this; + } + + // No copies allowed + ShmemBuffer(const ShmemBuffer&) = delete; + ShmemBuffer& operator=(const ShmemBuffer&) = delete; + + bool Valid() { return mInitialized; } + + uint8_t* GetBytes() { return mShmem.get<uint8_t>(); } + + mozilla::ipc::Shmem& Get() { return mShmem; } + + private: + friend class ShmemPool; + + bool mInitialized; + mozilla::ipc::Shmem mShmem; +}; + +class ShmemPool final { + public: + enum class PoolType { StaticPool, DynamicPool }; + explicit ShmemPool(size_t aPoolSize, + PoolType aPoolType = PoolType::StaticPool); + ~ShmemPool(); + // Get/GetIfAvailable differ in what thread they can run on. GetIfAvailable + // can run anywhere but won't allocate if the right size isn't available. + ShmemBuffer GetIfAvailable(size_t aSize); + void Put(ShmemBuffer&& aShmem); + + // We need to use the allocation/deallocation functions + // of a specific IPC child/parent instance. + template <class T> + void Cleanup(T* aInstance) { + MutexAutoLock lock(mMutex); + for (size_t i = 0; i < mShmemPool.Length(); i++) { + if (mShmemPool[i].mInitialized) { + aInstance->DeallocShmem(mShmemPool[i].Get()); + mShmemPool[i].mInitialized = false; + } + } + } + + enum class AllocationPolicy { Default, Unsafe }; + + template <class T> + ShmemBuffer Get(T* aInstance, size_t aSize, + AllocationPolicy aPolicy = AllocationPolicy::Default) { + MutexAutoLock lock(mMutex); + + // Pool is empty, don't block caller. + if (mPoolFree == 0 && mPoolType == PoolType::StaticPool) { + if (!mErrorLogged) { + // log "out of pool" once as error to avoid log spam + mErrorLogged = true; + SHMEMPOOL_LOG_ERROR( + ("ShmemPool is empty, future occurrences " + "will be logged as warnings")); + } else { + SHMEMPOOL_LOG_WARN(("ShmemPool is empty")); + } + // This isn't initialized, so will be understood as an error. + return ShmemBuffer(); + } + if (mPoolFree == 0) { + MOZ_ASSERT(mPoolType == PoolType::DynamicPool); + SHMEMPOOL_LOG(("Dynamic ShmemPool empty, allocating extra Shmem buffer")); + ShmemBuffer newBuffer; + mShmemPool.InsertElementAt(0, std::move(newBuffer)); + mPoolFree++; + } + + ShmemBuffer& res = mShmemPool[mPoolFree - 1]; + + if (!res.mInitialized) { + SHMEMPOOL_LOG(("Initializing new Shmem in pool")); + if (!AllocateShmem(aInstance, aSize, res, aPolicy)) { + SHMEMPOOL_LOG(("Failure allocating new Shmem buffer")); + return ShmemBuffer(); + } + res.mInitialized = true; + } + + MOZ_DIAGNOSTIC_ASSERT(res.mShmem.IsWritable(), + "Shmem in Pool is not writable?"); + + // Prepare buffer, increase size if needed (we never shrink as we don't + // maintain seperate sized pools and we don't want to keep reallocating) + if (res.mShmem.Size<char>() < aSize) { + SHMEMPOOL_LOG(("Size change/increase in Shmem Pool")); + aInstance->DeallocShmem(res.mShmem); + res.mInitialized = false; + // this may fail; always check return value + if (!AllocateShmem(aInstance, aSize, res, aPolicy)) { + SHMEMPOOL_LOG(("Failure allocating resized Shmem buffer")); + return ShmemBuffer(); + } else { + res.mInitialized = true; + } + } + + MOZ_ASSERT(res.mShmem.IsWritable(), + "Shmem in Pool is not writable post resize?"); + + mPoolFree--; +#ifdef DEBUG + size_t poolUse = mShmemPool.Length() - mPoolFree; + if (poolUse > mMaxPoolUse) { + mMaxPoolUse = poolUse; + SHMEMPOOL_LOG( + ("Maximum ShmemPool use increased: %zu buffers", mMaxPoolUse)); + } +#endif + return std::move(res); + } + + private: + template <class T> + bool AllocateShmem(T* aInstance, size_t aSize, ShmemBuffer& aRes, + AllocationPolicy aPolicy) { + return (aPolicy == AllocationPolicy::Default && + aInstance->AllocShmem(aSize, &aRes.mShmem)) || + (aPolicy == AllocationPolicy::Unsafe && + aInstance->AllocUnsafeShmem(aSize, &aRes.mShmem)); + } + const PoolType mPoolType; + Mutex mMutex MOZ_UNANNOTATED; + size_t mPoolFree; + bool mErrorLogged; +#ifdef DEBUG + size_t mMaxPoolUse; +#endif + nsTArray<ShmemBuffer> mShmemPool; +}; + +} // namespace mozilla + +#endif // mozilla_ShmemPool_h diff --git a/dom/media/systemservices/VideoEngine.cpp b/dom/media/systemservices/VideoEngine.cpp new file mode 100644 index 0000000000..1d4bde6344 --- /dev/null +++ b/dom/media/systemservices/VideoEngine.cpp @@ -0,0 +1,245 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "VideoEngine.h" +#include "libwebrtcglue/SystemTime.h" +#include "video_engine/desktop_capture_impl.h" +#include "system_wrappers/include/clock.h" +#ifdef WEBRTC_ANDROID +# include "modules/video_capture/video_capture.h" +#endif + +#ifdef MOZ_WIDGET_ANDROID +# include "mozilla/jni/Utils.h" +#endif + +namespace mozilla::camera { + +#undef LOG +#undef LOG_ENABLED +mozilla::LazyLogModule gVideoEngineLog("VideoEngine"); +#define LOG(args) MOZ_LOG(gVideoEngineLog, mozilla::LogLevel::Debug, args) +#define LOG_ENABLED() MOZ_LOG_TEST(gVideoEngineLog, mozilla::LogLevel::Debug) + +#if defined(ANDROID) +int VideoEngine::SetAndroidObjects() { + LOG(("%s", __PRETTY_FUNCTION__)); + + JavaVM* const javaVM = mozilla::jni::GetVM(); + if (!javaVM || webrtc::SetCaptureAndroidVM(javaVM) != 0) { + LOG(("Could not set capture Android VM")); + return -1; + } +# ifdef WEBRTC_INCLUDE_INTERNAL_VIDEO_RENDER + if (webrtc::SetRenderAndroidVM(javaVM) != 0) { + LOG(("Could not set render Android VM")); + return -1; + } +# endif + return 0; +} +#endif + +int32_t VideoEngine::CreateVideoCapture(const char* aDeviceUniqueIdUTF8) { + LOG(("%s", __PRETTY_FUNCTION__)); + MOZ_ASSERT(aDeviceUniqueIdUTF8); + + int32_t id = GenerateId(); + LOG(("CaptureDeviceInfo.type=%s id=%d", mCaptureDevInfo.TypeName(), id)); + + for (auto& it : mCaps) { + if (it.second.VideoCapture() && + it.second.VideoCapture()->CurrentDeviceName() && + strcmp(it.second.VideoCapture()->CurrentDeviceName(), + aDeviceUniqueIdUTF8) == 0) { + mIdMap.emplace(id, it.first); + return id; + } + } + + CaptureEntry entry = {-1, nullptr}; + + if (mCaptureDevInfo.type == CaptureDeviceType::Camera) { + entry = CaptureEntry( + id, webrtc::VideoCaptureFactory::Create(aDeviceUniqueIdUTF8)); + if (entry.VideoCapture()) { + entry.VideoCapture()->SetApplyRotation(true); + } + } else { +#ifndef WEBRTC_ANDROID + entry = CaptureEntry( + id, rtc::scoped_refptr<webrtc::VideoCaptureModule>( + webrtc::DesktopCaptureImpl::Create(id, aDeviceUniqueIdUTF8, + mCaptureDevInfo.type))); +#else + MOZ_ASSERT("CreateVideoCapture NO DESKTOP CAPTURE IMPL ON ANDROID" == + nullptr); +#endif + } + mCaps.emplace(id, std::move(entry)); + mIdMap.emplace(id, id); + return id; +} + +int VideoEngine::ReleaseVideoCapture(const int32_t aId) { + bool found = false; + +#ifdef DEBUG + { + auto it = mIdMap.find(aId); + MOZ_ASSERT(it != mIdMap.end()); + Unused << it; + } +#endif + + for (auto& it : mIdMap) { + if (it.first != aId && it.second == mIdMap[aId]) { + // There are other tracks still using this hardware. + found = true; + } + } + + if (!found) { + WithEntry(aId, [&found](CaptureEntry& cap) { + cap.mVideoCaptureModule = nullptr; + found = true; + }); + MOZ_ASSERT(found); + if (found) { + auto it = mCaps.find(mIdMap[aId]); + MOZ_ASSERT(it != mCaps.end()); + mCaps.erase(it); + } + } + + mIdMap.erase(aId); + return found ? 0 : (-1); +} + +std::shared_ptr<webrtc::VideoCaptureModule::DeviceInfo> +VideoEngine::GetOrCreateVideoCaptureDeviceInfo() { + LOG(("%s", __PRETTY_FUNCTION__)); + webrtc::Timestamp currentTime = webrtc::Timestamp::Micros(0); + + const char* capDevTypeName = + CaptureDeviceInfo(mCaptureDevInfo.type).TypeName(); + + if (mDeviceInfo) { + LOG(("Device cache available.")); + // Camera cache is invalidated by HW change detection elsewhere + if (mCaptureDevInfo.type == CaptureDeviceType::Camera) { + LOG(("returning cached CaptureDeviceInfo of type %s", capDevTypeName)); + return mDeviceInfo; + } + // Screen sharing cache is invalidated after the expiration time + currentTime = WebrtcSystemTime(); + LOG(("Checking expiry, fetched current time of: %" PRId64, + currentTime.ms())); + LOG(("device cache expiration is %" PRId64, mExpiryTime.ms())); + if (currentTime <= mExpiryTime) { + LOG(("returning cached CaptureDeviceInfo of type %s", capDevTypeName)); + return mDeviceInfo; + } + } + + if (currentTime.IsZero()) { + currentTime = WebrtcSystemTime(); + LOG(("Fetched current time of: %" PRId64, currentTime.ms())); + } + mExpiryTime = currentTime + webrtc::TimeDelta::Millis(kCacheExpiryPeriodMs); + LOG(("new device cache expiration is %" PRId64, mExpiryTime.ms())); + LOG(("creating a new VideoCaptureDeviceInfo of type %s", capDevTypeName)); + + switch (mCaptureDevInfo.type) { + case CaptureDeviceType::Camera: { +#ifdef MOZ_WIDGET_ANDROID + if (SetAndroidObjects()) { + LOG(("VideoEngine::SetAndroidObjects Failed")); + break; + } +#endif + mDeviceInfo.reset(webrtc::VideoCaptureFactory::CreateDeviceInfo()); + LOG(("CaptureDeviceType::Camera: Finished creating new device.")); + break; + } + // Window and Screen and Browser (tab) types are handled by DesktopCapture + case CaptureDeviceType::Browser: + case CaptureDeviceType::Window: + case CaptureDeviceType::Screen: { +#if !defined(WEBRTC_ANDROID) && !defined(WEBRTC_IOS) + mDeviceInfo = webrtc::DesktopCaptureImpl::CreateDeviceInfo( + mId, mCaptureDevInfo.type); + LOG(("screen capture: Finished creating new device.")); +#else + MOZ_ASSERT( + "GetVideoCaptureDeviceInfo NO DESKTOP CAPTURE IMPL ON ANDROID" == + nullptr); + mDeviceInfo.reset(); +#endif + break; + } + } + LOG(("EXIT %s", __PRETTY_FUNCTION__)); + return mDeviceInfo; +} + +already_AddRefed<VideoEngine> VideoEngine::Create( + const CaptureDeviceType& aCaptureDeviceType) { + LOG(("%s", __PRETTY_FUNCTION__)); + return do_AddRef(new VideoEngine(aCaptureDeviceType)); +} + +VideoEngine::CaptureEntry::CaptureEntry( + int32_t aCapnum, rtc::scoped_refptr<webrtc::VideoCaptureModule> aCapture) + : mCapnum(aCapnum), mVideoCaptureModule(aCapture) {} + +rtc::scoped_refptr<webrtc::VideoCaptureModule> +VideoEngine::CaptureEntry::VideoCapture() { + return mVideoCaptureModule; +} + +int32_t VideoEngine::CaptureEntry::Capnum() const { return mCapnum; } + +bool VideoEngine::WithEntry( + const int32_t entryCapnum, + const std::function<void(CaptureEntry& entry)>&& fn) { +#ifdef DEBUG + { + auto it = mIdMap.find(entryCapnum); + MOZ_ASSERT(it != mIdMap.end()); + Unused << it; + } +#endif + + auto it = mCaps.find(mIdMap[entryCapnum]); + MOZ_ASSERT(it != mCaps.end()); + if (it == mCaps.end()) { + return false; + } + fn(it->second); + return true; +} + +int32_t VideoEngine::GenerateId() { + // XXX Something better than this (a map perhaps, or a simple boolean TArray, + // given the number in-use is O(1) normally!) + static int sId = 0; + return mId = sId++; +} + +VideoEngine::VideoEngine(const CaptureDeviceType& aCaptureDeviceType) + : mId(0), mCaptureDevInfo(aCaptureDeviceType), mDeviceInfo(nullptr) { + LOG(("%s", __PRETTY_FUNCTION__)); + LOG(("Creating new VideoEngine with CaptureDeviceType %s", + mCaptureDevInfo.TypeName())); +} + +VideoEngine::~VideoEngine() { + MOZ_ASSERT(mCaps.empty()); + MOZ_ASSERT(mIdMap.empty()); +} + +} // namespace mozilla::camera diff --git a/dom/media/systemservices/VideoEngine.h b/dom/media/systemservices/VideoEngine.h new file mode 100644 index 0000000000..e7a34b34e6 --- /dev/null +++ b/dom/media/systemservices/VideoEngine.h @@ -0,0 +1,117 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_VideoEngine_h +#define mozilla_VideoEngine_h + +#include "MediaEngine.h" +#include "VideoFrameUtils.h" +#include "mozilla/media/MediaUtils.h" +#include "modules/video_capture/video_capture_impl.h" +#include "modules/video_capture/video_capture_defines.h" +#include "modules/video_capture/video_capture_factory.h" +#include <memory> +#include <functional> + +namespace mozilla::camera { + +enum class CaptureDeviceType { Camera, Screen, Window, Browser }; + +struct CaptureDeviceInfo { + CaptureDeviceType type; + + CaptureDeviceInfo() : type(CaptureDeviceType::Camera) {} + explicit CaptureDeviceInfo(CaptureDeviceType t) : type(t) {} + + const char* TypeName() const { + switch (type) { + case CaptureDeviceType::Camera: { + return "Camera"; + } + case CaptureDeviceType::Screen: { + return "Screen"; + } + case CaptureDeviceType::Window: { + return "Window"; + } + case CaptureDeviceType::Browser: { + return "Browser"; + } + } + assert(false); + return "UNKOWN-CaptureDeviceType!"; + } +}; + +// Historically the video engine was part of webrtc +// it was removed (and reimplemented in Talk) +class VideoEngine { + private: + virtual ~VideoEngine(); + + // Base cache expiration period + // Note because cameras use HW plug event detection, this + // only applies to screen based modes. + static const int64_t kCacheExpiryPeriodMs = 2000; + + public: + NS_INLINE_DECL_REFCOUNTING(VideoEngine) + + static already_AddRefed<VideoEngine> Create( + const CaptureDeviceType& aCaptureDeviceType); +#if defined(ANDROID) + static int SetAndroidObjects(); +#endif + // Returns a non-negative capture identifier or -1 on failure. + int32_t CreateVideoCapture(const char* aDeviceUniqueIdUTF8); + + int ReleaseVideoCapture(const int32_t aId); + + // VideoEngine is responsible for any cleanup in its modules + static void Delete(VideoEngine* aEngine) {} + + /** Returns an existing or creates a new new DeviceInfo. + * Camera info is cached to prevent repeated lengthy polling for "realness" + * of the hardware devices. Other types of capture, e.g. screen share info, + * are cached for 1 second. This could be handled in a more elegant way in + * the future. + * @return on failure the shared_ptr will be null, otherwise it will contain + * a DeviceInfo. + * @see bug 1305212 https://bugzilla.mozilla.org/show_bug.cgi?id=1305212 + */ + std::shared_ptr<webrtc::VideoCaptureModule::DeviceInfo> + GetOrCreateVideoCaptureDeviceInfo(); + + class CaptureEntry { + public: + CaptureEntry(int32_t aCapnum, + rtc::scoped_refptr<webrtc::VideoCaptureModule> aCapture); + int32_t Capnum() const; + rtc::scoped_refptr<webrtc::VideoCaptureModule> VideoCapture(); + + private: + int32_t mCapnum; + rtc::scoped_refptr<webrtc::VideoCaptureModule> mVideoCaptureModule; + friend class VideoEngine; + }; + + // Returns true iff an entry for capnum exists + bool WithEntry(const int32_t entryCapnum, + const std::function<void(CaptureEntry& entry)>&& fn); + + private: + explicit VideoEngine(const CaptureDeviceType& aCaptureDeviceType); + int32_t mId; + const CaptureDeviceInfo mCaptureDevInfo; + std::shared_ptr<webrtc::VideoCaptureModule::DeviceInfo> mDeviceInfo; + std::map<int32_t, CaptureEntry> mCaps; + std::map<int32_t, int32_t> mIdMap; + // The validity period for non-camera capture device infos` + webrtc::Timestamp mExpiryTime = webrtc::Timestamp::Micros(0); + int32_t GenerateId(); +}; +} // namespace mozilla::camera +#endif diff --git a/dom/media/systemservices/VideoFrameUtils.cpp b/dom/media/systemservices/VideoFrameUtils.cpp new file mode 100644 index 0000000000..00ead56a7b --- /dev/null +++ b/dom/media/systemservices/VideoFrameUtils.cpp @@ -0,0 +1,90 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "VideoFrameUtils.h" +#include "api/video/video_frame.h" +#include "mozilla/ShmemPool.h" + +namespace mozilla { + +uint32_t VideoFrameUtils::TotalRequiredBufferSize( + const webrtc::VideoFrame& aVideoFrame) { + auto i420 = aVideoFrame.video_frame_buffer()->ToI420(); + auto height = i420->height(); + size_t size = height * i420->StrideY() + + ((height + 1) / 2) * i420->StrideU() + + ((height + 1) / 2) * i420->StrideV(); + MOZ_RELEASE_ASSERT(size < std::numeric_limits<uint32_t>::max()); + return static_cast<uint32_t>(size); +} + +void VideoFrameUtils::InitFrameBufferProperties( + const webrtc::VideoFrame& aVideoFrame, + camera::VideoFrameProperties& aDestProps) { + // The VideoFrameBuffer image data stored in the accompanying buffer + // the buffer is at least this size of larger. + aDestProps.bufferSize() = TotalRequiredBufferSize(aVideoFrame); + + aDestProps.timeStamp() = aVideoFrame.timestamp(); + aDestProps.ntpTimeMs() = aVideoFrame.ntp_time_ms(); + aDestProps.renderTimeMs() = aVideoFrame.render_time_ms(); + + aDestProps.rotation() = aVideoFrame.rotation(); + + auto i420 = aVideoFrame.video_frame_buffer()->ToI420(); + auto height = i420->height(); + aDestProps.yAllocatedSize() = height * i420->StrideY(); + aDestProps.uAllocatedSize() = ((height + 1) / 2) * i420->StrideU(); + aDestProps.vAllocatedSize() = ((height + 1) / 2) * i420->StrideV(); + + aDestProps.width() = i420->width(); + aDestProps.height() = height; + + aDestProps.yStride() = i420->StrideY(); + aDestProps.uStride() = i420->StrideU(); + aDestProps.vStride() = i420->StrideV(); +} + +void VideoFrameUtils::CopyVideoFrameBuffers(uint8_t* aDestBuffer, + const size_t aDestBufferSize, + const webrtc::VideoFrame& aFrame) { + size_t aggregateSize = TotalRequiredBufferSize(aFrame); + + MOZ_ASSERT(aDestBufferSize >= aggregateSize); + auto i420 = aFrame.video_frame_buffer()->ToI420(); + + // If planes are ordered YUV and contiguous then do a single copy + if ((i420->DataY() != nullptr) && + // Check that the three planes are ordered + (i420->DataY() < i420->DataU()) && (i420->DataU() < i420->DataV()) && + // Check that the last plane ends at firstPlane[totalsize] + (&i420->DataY()[aggregateSize] == + &i420->DataV()[((i420->height() + 1) / 2) * i420->StrideV()])) { + memcpy(aDestBuffer, i420->DataY(), aggregateSize); + return; + } + + // Copy each plane + size_t offset = 0; + size_t size; + auto height = i420->height(); + size = height * i420->StrideY(); + memcpy(&aDestBuffer[offset], i420->DataY(), size); + offset += size; + size = ((height + 1) / 2) * i420->StrideU(); + memcpy(&aDestBuffer[offset], i420->DataU(), size); + offset += size; + size = ((height + 1) / 2) * i420->StrideV(); + memcpy(&aDestBuffer[offset], i420->DataV(), size); +} + +void VideoFrameUtils::CopyVideoFrameBuffers( + ShmemBuffer& aDestShmem, const webrtc::VideoFrame& aVideoFrame) { + CopyVideoFrameBuffers(aDestShmem.Get().get<uint8_t>(), + aDestShmem.Get().Size<uint8_t>(), aVideoFrame); +} + +} // namespace mozilla diff --git a/dom/media/systemservices/VideoFrameUtils.h b/dom/media/systemservices/VideoFrameUtils.h new file mode 100644 index 0000000000..23ebf4f316 --- /dev/null +++ b/dom/media/systemservices/VideoFrameUtils.h @@ -0,0 +1,48 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set sw=2 ts=8 et ft=cpp : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_VideoFrameUtil_h +#define mozilla_VideoFrameUtil_h + +#include "mozilla/camera/PCameras.h" + +namespace webrtc { +class VideoFrame; +} + +namespace mozilla { +class ShmemBuffer; + +// Util methods for working with webrtc::VideoFrame(s) and +// the IPC classes that are used to deliver their contents to the +// MediaEnginge + +class VideoFrameUtils { + public: + // Returns the total number of bytes necessary to copy a VideoFrame's buffer + // across all planes. + static uint32_t TotalRequiredBufferSize(const webrtc::VideoFrame& frame); + + // Initializes a camera::VideoFrameProperties from a VideoFrameBuffer + static void InitFrameBufferProperties( + const webrtc::VideoFrame& aVideoFrame, + camera::VideoFrameProperties& aDestProperties); + + // Copies the buffers out of a VideoFrameBuffer into a buffer. + // Attempts to make as few memcopies as possible. + static void CopyVideoFrameBuffers(uint8_t* aDestBuffer, + const size_t aDestBufferSize, + const webrtc::VideoFrame& aVideoFrame); + + // Copies the buffers in a VideoFrameBuffer into a Shmem + // returns the eno from the underlying memcpy. + static void CopyVideoFrameBuffers(ShmemBuffer& aDestShmem, + const webrtc::VideoFrame& aVideoFrame); +}; + +} /* namespace mozilla */ + +#endif diff --git a/dom/media/systemservices/android_video_capture/device_info_android.cc b/dom/media/systemservices/android_video_capture/device_info_android.cc new file mode 100644 index 0000000000..581040eb94 --- /dev/null +++ b/dom/media/systemservices/android_video_capture/device_info_android.cc @@ -0,0 +1,316 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "device_info_android.h" + +#include <algorithm> +#include <string> +#include <sstream> +#include <vector> + +#include "rtc_base/logging.h" +#include "modules/utility/include/helpers_android.h" + +#include "mozilla/jni/Utils.h" + +namespace webrtc { + +namespace videocapturemodule { + +// Helper for storing lists of pairs of ints. Used e.g. for resolutions & FPS +// ranges. +typedef std::pair<int, int> IntPair; +typedef std::vector<IntPair> IntPairs; + +static std::string IntPairsToString(const IntPairs& pairs, char separator) { + std::stringstream stream; + for (size_t i = 0; i < pairs.size(); ++i) { + if (i > 0) { + stream << ", "; + } + stream << "(" << pairs[i].first << separator << pairs[i].second << ")"; + } + return stream.str(); +} + +struct AndroidCameraInfo { + std::string name; + bool front_facing; + int orientation; + IntPairs resolutions; // Pairs are: (width,height). + // Pairs are (min,max) in units of FPS*1000 ("milli-frame-per-second"). + IntPairs mfpsRanges; + + std::string ToString() { + std::stringstream stream; + stream << "Name: [" << name << "], MFPS ranges: [" + << IntPairsToString(mfpsRanges, ':') + << "], front_facing: " << front_facing + << ", orientation: " << orientation << ", resolutions: [" + << IntPairsToString(resolutions, 'x') << "]"; + return stream.str(); + } +}; + +// Camera info; populated during DeviceInfoAndroid::Refresh() +static std::vector<AndroidCameraInfo>* g_camera_info = NULL; + +static JavaVM* g_jvm_dev_info = NULL; + +// Set |*index| to the index of |name| in g_camera_info or return false if no +// match found. +static bool FindCameraIndexByName(const std::string& name, size_t* index) { + for (size_t i = 0; i < g_camera_info->size(); ++i) { + if (g_camera_info->at(i).name == name) { + *index = i; + return true; + } + } + return false; +} + +// Returns a pointer to the named member of g_camera_info, or NULL if no match +// is found. +static AndroidCameraInfo* FindCameraInfoByName(const std::string& name) { + size_t index = 0; + if (FindCameraIndexByName(name, &index)) { + return &g_camera_info->at(index); + } + return NULL; +} + +// static +void DeviceInfoAndroid::Initialize(JavaVM* javaVM) { + // TODO(henrike): this "if" would make a lot more sense as an assert, but + // Java_org_webrtc_videoengineapp_ViEAndroidJavaAPI_GetVideoEngine() and + // Java_org_webrtc_videoengineapp_ViEAndroidJavaAPI_Terminate() conspire to + // prevent this. Once that code is made to only + // VideoEngine::SetAndroidObjects() once per process, this can turn into an + // assert. + if (g_camera_info) { + return; + } + + g_jvm_dev_info = javaVM; + BuildDeviceList(); +} + +void DeviceInfoAndroid::BuildDeviceList() { + if (!g_jvm_dev_info) { + return; + } + + AttachThreadScoped ats(g_jvm_dev_info); + JNIEnv* jni = ats.env(); + + g_camera_info = new std::vector<AndroidCameraInfo>(); + jclass j_info_class = mozilla::jni::GetClassRef( + jni, "org/webrtc/videoengine/VideoCaptureDeviceInfoAndroid"); + jclass j_cap_class = mozilla::jni::GetClassRef( + jni, "org/webrtc/videoengine/CaptureCapabilityAndroid"); + assert(j_info_class); + jmethodID j_get_device_info = jni->GetStaticMethodID( + j_info_class, "getDeviceInfo", + "()[Lorg/webrtc/videoengine/CaptureCapabilityAndroid;"); + jarray j_camera_caps = static_cast<jarray>( + jni->CallStaticObjectMethod(j_info_class, j_get_device_info)); + if (jni->ExceptionCheck()) { + jni->ExceptionClear(); + RTC_LOG(LS_INFO) << __FUNCTION__ << ": Failed to get camera capabilities."; + return; + } + if (j_camera_caps == nullptr) { + RTC_LOG(LS_INFO) << __FUNCTION__ << ": Failed to get camera capabilities."; + return; + } + + const jsize capLength = jni->GetArrayLength(j_camera_caps); + + jfieldID widthField = jni->GetFieldID(j_cap_class, "width", "[I"); + jfieldID heightField = jni->GetFieldID(j_cap_class, "height", "[I"); + jfieldID maxFpsField = jni->GetFieldID(j_cap_class, "maxMilliFPS", "I"); + jfieldID minFpsField = jni->GetFieldID(j_cap_class, "minMilliFPS", "I"); + jfieldID orientationField = jni->GetFieldID(j_cap_class, "orientation", "I"); + jfieldID frontFacingField = jni->GetFieldID(j_cap_class, "frontFacing", "Z"); + jfieldID nameField = + jni->GetFieldID(j_cap_class, "name", "Ljava/lang/String;"); + if (widthField == NULL || heightField == NULL || maxFpsField == NULL || + minFpsField == NULL || orientationField == NULL || + frontFacingField == NULL || nameField == NULL) { + RTC_LOG(LS_INFO) << __FUNCTION__ << ": Failed to get field Id."; + return; + } + + for (jsize i = 0; i < capLength; i++) { + jobject capabilityElement = + jni->GetObjectArrayElement((jobjectArray)j_camera_caps, i); + + AndroidCameraInfo info; + jstring camName = + static_cast<jstring>(jni->GetObjectField(capabilityElement, nameField)); + const char* camChars = jni->GetStringUTFChars(camName, nullptr); + info.name = std::string(camChars); + jni->ReleaseStringUTFChars(camName, camChars); + + info.orientation = jni->GetIntField(capabilityElement, orientationField); + info.front_facing = + jni->GetBooleanField(capabilityElement, frontFacingField); + jint min_mfps = jni->GetIntField(capabilityElement, minFpsField); + jint max_mfps = jni->GetIntField(capabilityElement, maxFpsField); + + jintArray widthResArray = static_cast<jintArray>( + jni->GetObjectField(capabilityElement, widthField)); + jintArray heightResArray = static_cast<jintArray>( + jni->GetObjectField(capabilityElement, heightField)); + + const jsize numRes = jni->GetArrayLength(widthResArray); + + jint* widths = jni->GetIntArrayElements(widthResArray, nullptr); + jint* heights = jni->GetIntArrayElements(heightResArray, nullptr); + + for (jsize j = 0; j < numRes; ++j) { + info.resolutions.push_back(std::make_pair(widths[j], heights[j])); + } + + info.mfpsRanges.push_back(std::make_pair(min_mfps, max_mfps)); + g_camera_info->push_back(info); + + jni->ReleaseIntArrayElements(widthResArray, widths, JNI_ABORT); + jni->ReleaseIntArrayElements(heightResArray, heights, JNI_ABORT); + } + + jni->DeleteLocalRef(j_info_class); + jni->DeleteLocalRef(j_cap_class); +} + +void DeviceInfoAndroid::DeInitialize() { + if (g_camera_info) { + delete g_camera_info; + g_camera_info = NULL; + } +} + +int32_t DeviceInfoAndroid::Refresh() { + if (!g_camera_info || g_camera_info->size() == 0) { + DeviceInfoAndroid::BuildDeviceList(); +#ifdef DEBUG + int frontFacingIndex = -1; + for (uint32_t i = 0; i < g_camera_info->size(); i++) { + if (g_camera_info->at(i).front_facing) { + frontFacingIndex = i; + } + } + // Either there is a front-facing camera, and it's first in the list, or + // there is no front-facing camera. + MOZ_ASSERT(frontFacingIndex == 0 || frontFacingIndex == -1); +#endif + } + return 0; +} + +VideoCaptureModule::DeviceInfo* VideoCaptureImpl::CreateDeviceInfo() { + return new videocapturemodule::DeviceInfoAndroid(); +} + +DeviceInfoAndroid::DeviceInfoAndroid() : DeviceInfoImpl() {} + +DeviceInfoAndroid::~DeviceInfoAndroid() {} + +bool DeviceInfoAndroid::FindCameraIndex(const char* deviceUniqueIdUTF8, + size_t* index) { + return FindCameraIndexByName(deviceUniqueIdUTF8, index); +} + +int32_t DeviceInfoAndroid::Init() { return 0; } + +uint32_t DeviceInfoAndroid::NumberOfDevices() { + Refresh(); + return g_camera_info->size(); +} + +int32_t DeviceInfoAndroid::GetDeviceName( + uint32_t deviceNumber, char* deviceNameUTF8, uint32_t deviceNameLength, + char* deviceUniqueIdUTF8, uint32_t deviceUniqueIdUTF8Length, + char* /*productUniqueIdUTF8*/, uint32_t /*productUniqueIdUTF8Length*/, + pid_t* /*pid*/) { + if (deviceNumber >= g_camera_info->size()) { + return -1; + } + const AndroidCameraInfo& info = g_camera_info->at(deviceNumber); + if (info.name.length() + 1 > deviceNameLength || + info.name.length() + 1 > deviceUniqueIdUTF8Length) { + return -1; + } + memcpy(deviceNameUTF8, info.name.c_str(), info.name.length() + 1); + memcpy(deviceUniqueIdUTF8, info.name.c_str(), info.name.length() + 1); + return 0; +} + +int32_t DeviceInfoAndroid::CreateCapabilityMap(const char* deviceUniqueIdUTF8) { + _captureCapabilities.clear(); + const AndroidCameraInfo* info = FindCameraInfoByName(deviceUniqueIdUTF8); + if (info == NULL) { + return -1; + } + + for (size_t i = 0; i < info->resolutions.size(); ++i) { + for (size_t j = 0; j < info->mfpsRanges.size(); ++j) { + const IntPair& size = info->resolutions[i]; + const IntPair& mfpsRange = info->mfpsRanges[j]; + VideoCaptureCapability cap; + cap.width = size.first; + cap.height = size.second; + cap.maxFPS = mfpsRange.second / 1000; + cap.videoType = VideoType::kNV21; + _captureCapabilities.push_back(cap); + } + } + return _captureCapabilities.size(); +} + +int32_t DeviceInfoAndroid::GetOrientation(const char* deviceUniqueIdUTF8, + VideoRotation& orientation) { + const AndroidCameraInfo* info = FindCameraInfoByName(deviceUniqueIdUTF8); + if (info == NULL || VideoCaptureImpl::RotationFromDegrees( + info->orientation, &orientation) != 0) { + return -1; + } + return 0; +} + +void DeviceInfoAndroid::GetMFpsRange(const char* deviceUniqueIdUTF8, + int max_fps_to_match, int* min_mfps, + int* max_mfps) { + const AndroidCameraInfo* info = FindCameraInfoByName(deviceUniqueIdUTF8); + if (info == NULL) { + return; + } + int desired_mfps = max_fps_to_match * 1000; + int best_diff_mfps = 0; + RTC_LOG(LS_INFO) << "Search for best target mfps " << desired_mfps; + // Search for best fps range with preference shifted to constant fps modes. + for (size_t i = 0; i < info->mfpsRanges.size(); ++i) { + int diff_mfps = + abs(info->mfpsRanges[i].first - desired_mfps) + + abs(info->mfpsRanges[i].second - desired_mfps) + + (info->mfpsRanges[i].second - info->mfpsRanges[i].first) / 2; + RTC_LOG(LS_INFO) << "Fps range " << info->mfpsRanges[i].first << ":" + << info->mfpsRanges[i].second + << ". Distance: " << diff_mfps; + if (i == 0 || diff_mfps < best_diff_mfps) { + best_diff_mfps = diff_mfps; + *min_mfps = info->mfpsRanges[i].first; + *max_mfps = info->mfpsRanges[i].second; + } + } +} + +} // namespace videocapturemodule +} // namespace webrtc diff --git a/dom/media/systemservices/android_video_capture/device_info_android.h b/dom/media/systemservices/android_video_capture/device_info_android.h new file mode 100644 index 0000000000..ac88b2b8ba --- /dev/null +++ b/dom/media/systemservices/android_video_capture/device_info_android.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_VIDEO_CAPTURE_MAIN_SOURCE_ANDROID_DEVICE_INFO_ANDROID_H_ +#define WEBRTC_MODULES_VIDEO_CAPTURE_MAIN_SOURCE_ANDROID_DEVICE_INFO_ANDROID_H_ + +#include <jni.h> + +#include "modules/video_capture/device_info_impl.h" +#include "modules/video_capture/video_capture_impl.h" + +#define AndroidJavaCaptureDeviceInfoClass \ + "org/webrtc/videoengine/VideoCaptureDeviceInfoAndroid" +#define AndroidJavaCaptureCapabilityClass \ + "org/webrtc/videoengine/CaptureCapabilityAndroid" + +namespace webrtc { +namespace videocapturemodule { + +class DeviceInfoAndroid : public DeviceInfoImpl { + public: + static void Initialize(JavaVM* javaVM); + static void DeInitialize(); + + DeviceInfoAndroid(); + virtual ~DeviceInfoAndroid(); + + // Set |*index| to the index of the camera matching |deviceUniqueIdUTF8|, or + // return false if no match. + bool FindCameraIndex(const char* deviceUniqueIdUTF8, size_t* index); + + virtual int32_t Init(); + virtual uint32_t NumberOfDevices(); + virtual int32_t Refresh(); + virtual int32_t GetDeviceName(uint32_t deviceNumber, char* deviceNameUTF8, + uint32_t deviceNameLength, + char* deviceUniqueIdUTF8, + uint32_t deviceUniqueIdUTF8Length, + char* productUniqueIdUTF8 = 0, + uint32_t productUniqueIdUTF8Length = 0, + pid_t* pid = 0); + virtual int32_t CreateCapabilityMap(const char* deviceUniqueIdUTF8) + RTC_EXCLUSIVE_LOCKS_REQUIRED(_apiLock); + + virtual int32_t DisplayCaptureSettingsDialogBox( + const char* /*deviceUniqueIdUTF8*/, const char* /*dialogTitleUTF8*/, + void* /*parentWindow*/, uint32_t /*positionX*/, uint32_t /*positionY*/) { + return -1; + } + virtual int32_t GetOrientation(const char* deviceUniqueIdUTF8, + VideoRotation& orientation); + + // Populate |min_mfps| and |max_mfps| with the closest supported range of the + // device to |max_fps_to_match|. + void GetMFpsRange(const char* deviceUniqueIdUTF8, int max_fps_to_match, + int* min_mfps, int* max_mfps); + + private: + enum { kExpectedCaptureDelay = 190 }; + static void BuildDeviceList(); +}; + +} // namespace videocapturemodule +} // namespace webrtc + +#endif // WEBRTC_MODULES_VIDEO_CAPTURE_MAIN_SOURCE_ANDROID_DEVICE_INFO_ANDROID_H_ diff --git a/dom/media/systemservices/android_video_capture/java/src/org/webrtc/videoengine/CaptureCapabilityAndroid.java b/dom/media/systemservices/android_video_capture/java/src/org/webrtc/videoengine/CaptureCapabilityAndroid.java new file mode 100644 index 0000000000..305fc74804 --- /dev/null +++ b/dom/media/systemservices/android_video_capture/java/src/org/webrtc/videoengine/CaptureCapabilityAndroid.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc.videoengine; + +import org.mozilla.gecko.annotation.WebRTCJNITarget; + +@WebRTCJNITarget +public class CaptureCapabilityAndroid { + public String name; + public int width[]; + public int height[]; + public int minMilliFPS; + public int maxMilliFPS; + public boolean frontFacing; + public boolean infrared; + public int orientation; +} diff --git a/dom/media/systemservices/android_video_capture/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java b/dom/media/systemservices/android_video_capture/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java new file mode 100644 index 0000000000..cc54009a7b --- /dev/null +++ b/dom/media/systemservices/android_video_capture/java/src/org/webrtc/videoengine/VideoCaptureAndroid.java @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc.videoengine; + +import java.io.IOException; +import java.util.List; + +import android.content.Context; +import android.util.Log; +import android.view.Surface; +import android.view.WindowManager; + +import java.util.concurrent.CountDownLatch; + +import org.mozilla.gecko.annotation.WebRTCJNITarget; + +import org.webrtc.CameraEnumerator; +import org.webrtc.Camera1Enumerator; +import org.webrtc.Camera2Enumerator; +import org.webrtc.CameraVideoCapturer; +import org.webrtc.CapturerObserver; +import org.webrtc.EglBase; +import org.webrtc.SurfaceTextureHelper; +import org.webrtc.VideoFrame; +import org.webrtc.VideoFrame.I420Buffer; + +public class VideoCaptureAndroid implements CameraVideoCapturer.CameraEventsHandler, CapturerObserver { + private final static String TAG = "WEBRTC-JC"; + + private final String deviceName; + private volatile long native_capturer; // |VideoCaptureAndroid*| in C++. + private Context context; + private CameraVideoCapturer cameraVideoCapturer; + private EglBase eglBase; + private SurfaceTextureHelper surfaceTextureHelper; + + // This class is recreated everytime we start/stop capture, so we + // can safely create the CountDownLatches here. + private final CountDownLatch capturerStarted = new CountDownLatch(1); + private boolean capturerStartedSucceeded = false; + private final CountDownLatch capturerStopped = new CountDownLatch(1); + + @WebRTCJNITarget + public VideoCaptureAndroid(String deviceName) { + // Remove the camera facing information from the name. + String[] parts = deviceName.split("Facing (front|back):"); + if (parts.length == 2) { + this.deviceName = parts[1].replace(" (infrared)", ""); + } else { + Log.e(TAG, "VideoCaptureAndroid: Expected facing mode as part of name: " + deviceName); + this.deviceName = deviceName; + } + this.context = GetContext(); + + CameraEnumerator enumerator; + if (Camera2Enumerator.isSupported(context)) { + enumerator = new Camera2Enumerator(context); + } else { + enumerator = new Camera1Enumerator(); + } + try { + cameraVideoCapturer = enumerator.createCapturer(this.deviceName, this); + eglBase = EglBase.create(); + surfaceTextureHelper = SurfaceTextureHelper.create("VideoCaptureAndroidSurfaceTextureHelper", eglBase.getEglBaseContext()); + cameraVideoCapturer.initialize(surfaceTextureHelper, context, this); + } catch (java.lang.IllegalArgumentException e) { + Log.e(TAG, "VideoCaptureAndroid: Exception while creating capturer: " + e); + } + } + + // Return the global application context. + @WebRTCJNITarget + private static native Context GetContext(); + + // Called by native code. Returns true if capturer is started. + // + // Note that this actually opens the camera, and Camera callbacks run on the + // thread that calls open(), so this is done on the CameraThread. Since ViE + // API needs a synchronous success return value we wait for the result. + @WebRTCJNITarget + private synchronized boolean startCapture( + final int width, final int height, + final int min_mfps, final int max_mfps, + long native_capturer) { + Log.d(TAG, "startCapture: " + width + "x" + height + "@" + + min_mfps + ":" + max_mfps); + + if (cameraVideoCapturer == null) { + return false; + } + + cameraVideoCapturer.startCapture(width, height, max_mfps); + try { + capturerStarted.await(); + } catch (InterruptedException e) { + return false; + } + if (capturerStartedSucceeded) { + this.native_capturer = native_capturer; + } + return capturerStartedSucceeded; + } + + // Called by native code. Returns true when camera is known to be stopped. + @WebRTCJNITarget + private synchronized boolean stopCapture() { + Log.d(TAG, "stopCapture"); + if (cameraVideoCapturer == null) { + return false; + } + + native_capturer = 0; + try { + cameraVideoCapturer.stopCapture(); + capturerStopped.await(); + } catch (InterruptedException e) { + return false; + } + Log.d(TAG, "stopCapture done"); + return true; + } + + @WebRTCJNITarget + private int getDeviceOrientation() { + int orientation = 0; + if (context != null) { + WindowManager wm = (WindowManager) context.getSystemService( + Context.WINDOW_SERVICE); + switch(wm.getDefaultDisplay().getRotation()) { + case Surface.ROTATION_90: + orientation = 90; + break; + case Surface.ROTATION_180: + orientation = 180; + break; + case Surface.ROTATION_270: + orientation = 270; + break; + case Surface.ROTATION_0: + default: + orientation = 0; + break; + } + } + return orientation; + } + + @WebRTCJNITarget + private native void ProvideCameraFrame( + int width, int height, + java.nio.ByteBuffer dataY, int strideY, + java.nio.ByteBuffer dataU, int strideU, + java.nio.ByteBuffer dataV, int strideV, + int rotation, long timeStamp, long captureObject); + + // + // CameraVideoCapturer.CameraEventsHandler interface + // + + // Camera error handler - invoked when camera can not be opened + // or any camera exception happens on camera thread. + public void onCameraError(String errorDescription) {} + + // Called when camera is disconnected. + public void onCameraDisconnected() {} + + // Invoked when camera stops receiving frames. + public void onCameraFreezed(String errorDescription) {} + + // Callback invoked when camera is opening. + public void onCameraOpening(String cameraName) {} + + // Callback invoked when first camera frame is available after camera is started. + public void onFirstFrameAvailable() {} + + // Callback invoked when camera is closed. + public void onCameraClosed() {} + + // + // CapturerObserver interface + // + + // Notify if the capturer have been started successfully or not. + public void onCapturerStarted(boolean success) { + capturerStartedSucceeded = success; + capturerStarted.countDown(); + } + + // Notify that the capturer has been stopped. + public void onCapturerStopped() { + capturerStopped.countDown(); + } + + // Delivers a captured frame. + public void onFrameCaptured(VideoFrame frame) { + if (native_capturer != 0) { + I420Buffer i420Buffer = frame.getBuffer().toI420(); + ProvideCameraFrame(i420Buffer.getWidth(), i420Buffer.getHeight(), + i420Buffer.getDataY(), i420Buffer.getStrideY(), + i420Buffer.getDataU(), i420Buffer.getStrideU(), + i420Buffer.getDataV(), i420Buffer.getStrideV(), + frame.getRotation(), + frame.getTimestampNs() / 1000000, native_capturer); + + i420Buffer.release(); + } + } +} diff --git a/dom/media/systemservices/android_video_capture/java/src/org/webrtc/videoengine/VideoCaptureDeviceInfoAndroid.java b/dom/media/systemservices/android_video_capture/java/src/org/webrtc/videoengine/VideoCaptureDeviceInfoAndroid.java new file mode 100644 index 0000000000..8ad8453955 --- /dev/null +++ b/dom/media/systemservices/android_video_capture/java/src/org/webrtc/videoengine/VideoCaptureDeviceInfoAndroid.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc.videoengine; + +import java.util.ArrayList; +import java.util.List; + +import android.Manifest; +import android.app.Activity; +import android.content.Context; +import android.util.Log; + +import org.mozilla.gecko.GeckoAppShell; +import org.mozilla.gecko.annotation.WebRTCJNITarget; + +import org.webrtc.CameraEnumerator; +import org.webrtc.CameraEnumerationAndroid.CaptureFormat; +import org.webrtc.Camera1Enumerator; +import org.webrtc.Camera2Enumerator; + +public class VideoCaptureDeviceInfoAndroid { + private final static String TAG = "WEBRTC-JC"; + + // Returns information about all cameras on the device. + // Since this reflects static information about the hardware present, there is + // no need to call this function more than once in a single process. It is + // marked "private" as it is only called by native code. + @WebRTCJNITarget + private static CaptureCapabilityAndroid[] getDeviceInfo() { + final Context context = GeckoAppShell.getApplicationContext(); + + if (Camera2Enumerator.isSupported(context)) { + return createDeviceList(new Camera2Enumerator(context)); + } else { + return createDeviceList(new Camera1Enumerator()); + } + } + + private static CaptureCapabilityAndroid[] createDeviceList(CameraEnumerator enumerator) { + + ArrayList<CaptureCapabilityAndroid> allDevices = new ArrayList<CaptureCapabilityAndroid>(); + ArrayList<CaptureCapabilityAndroid> IRDevices = new ArrayList<CaptureCapabilityAndroid>(); + + for (String camera: enumerator.getDeviceNames()) { + List<CaptureFormat> formats = enumerator.getSupportedFormats(camera); + int numFormats = formats.size(); + if (numFormats <= 0) { + continue; + } + + CaptureCapabilityAndroid device = new CaptureCapabilityAndroid(); + + // The only way to plumb through whether the device is front facing + // or not is by the name, but the name we receive depends upon the + // camera API in use. For the Camera1 API, this information is + // already present, but that is not the case when using Camera2. + // Later on, we look up the camera by name, so we have to use a + // format this is easy to undo. Ideally, libwebrtc would expose + // camera facing in VideoCaptureCapability and none of this would be + // necessary. + device.name = "Facing " + (enumerator.isFrontFacing(camera) ? "front" : "back") + ":" + camera; + + + boolean ir = enumerator.isInfrared(camera); + device.infrared = ir; + if (ir) { + device.name += " (infrared)"; + } + + // This isn't part of the new API, but we don't call + // GetDeviceOrientation() anywhere, so this value is unused. + device.orientation = 0; + + device.width = new int[numFormats]; + device.height = new int[numFormats]; + device.minMilliFPS = formats.get(0).framerate.min; + device.maxMilliFPS = formats.get(0).framerate.max; + int i = 0; + for (CaptureFormat format: formats) { + device.width[i] = format.width; + device.height[i] = format.height; + if (format.framerate.min < device.minMilliFPS) { + device.minMilliFPS = format.framerate.min; + } + if (format.framerate.max > device.maxMilliFPS) { + device.maxMilliFPS = format.framerate.max; + } + i++; + } + device.frontFacing = enumerator.isFrontFacing(camera); + // Infrared devices always last (but front facing ones before + // non-front-facing ones), front-facing non IR first, other in + // the middle. + if (!device.infrared) { + if (device.frontFacing) { + allDevices.add(0, device); + } else { + allDevices.add(device); + } + } else { + if (device.frontFacing) { + IRDevices.add(0, device); + } else { + IRDevices.add(device); + } + } + } + + allDevices.addAll(IRDevices); + + return allDevices.toArray(new CaptureCapabilityAndroid[0]); + } +} diff --git a/dom/media/systemservices/android_video_capture/video_capture_android.cc b/dom/media/systemservices/android_video_capture/video_capture_android.cc new file mode 100644 index 0000000000..7c9cd72ccb --- /dev/null +++ b/dom/media/systemservices/android_video_capture/video_capture_android.cc @@ -0,0 +1,270 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video_capture_android.h" + +#include "device_info_android.h" +#include "modules/utility/include/helpers_android.h" +#include "rtc_base/logging.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/time_utils.h" + +#include "AndroidBridge.h" + +static JavaVM* g_jvm_capture = NULL; +static jclass g_java_capturer_class = NULL; // VideoCaptureAndroid.class. +static jobject g_context = NULL; // Owned android.content.Context. + +namespace webrtc { + +jobject JniCommon_allocateNativeByteBuffer(JNIEnv* env, jclass, jint size) { + void* new_data = ::operator new(size); + jobject byte_buffer = env->NewDirectByteBuffer(new_data, size); + return byte_buffer; +} + +void JniCommon_freeNativeByteBuffer(JNIEnv* env, jclass, jobject byte_buffer) { + void* data = env->GetDirectBufferAddress(byte_buffer); + ::operator delete(data); +} + +// Called by Java to get the global application context. +jobject JNICALL GetContext(JNIEnv* env, jclass) { + assert(g_context); + return g_context; +} + +// Called by Java when the camera has a new frame to deliver. +void JNICALL ProvideCameraFrame(JNIEnv* env, jobject, jint width, jint height, + jobject javaDataY, jint strideY, + jobject javaDataU, jint strideU, + jobject javaDataV, jint strideV, jint rotation, + jlong timeStamp, jlong context) { + if (!context) { + return; + } + + webrtc::videocapturemodule::VideoCaptureAndroid* captureModule = + reinterpret_cast<webrtc::videocapturemodule::VideoCaptureAndroid*>( + context); + uint8_t* dataY = + reinterpret_cast<uint8_t*>(env->GetDirectBufferAddress(javaDataY)); + uint8_t* dataU = + reinterpret_cast<uint8_t*>(env->GetDirectBufferAddress(javaDataU)); + uint8_t* dataV = + reinterpret_cast<uint8_t*>(env->GetDirectBufferAddress(javaDataV)); + + rtc::scoped_refptr<I420Buffer> i420Buffer = I420Buffer::Copy( + width, height, dataY, strideY, dataU, strideU, dataV, strideV); + + captureModule->OnIncomingFrame(i420Buffer, rotation, timeStamp); +} + +int32_t SetCaptureAndroidVM(JavaVM* javaVM) { + if (g_java_capturer_class) { + return 0; + } + + if (javaVM) { + assert(!g_jvm_capture); + g_jvm_capture = javaVM; + AttachThreadScoped ats(g_jvm_capture); + + g_context = mozilla::AndroidBridge::Bridge()->GetGlobalContextRef(); + + videocapturemodule::DeviceInfoAndroid::Initialize(g_jvm_capture); + + { + jclass clsRef = mozilla::jni::GetClassRef( + ats.env(), "org/webrtc/videoengine/VideoCaptureAndroid"); + g_java_capturer_class = + static_cast<jclass>(ats.env()->NewGlobalRef(clsRef)); + ats.env()->DeleteLocalRef(clsRef); + assert(g_java_capturer_class); + + JNINativeMethod native_methods[] = { + {"GetContext", "()Landroid/content/Context;", + reinterpret_cast<void*>(&GetContext)}, + {"ProvideCameraFrame", + "(IILjava/nio/ByteBuffer;ILjava/nio/ByteBuffer;ILjava/nio/" + "ByteBuffer;IIJJ)V", + reinterpret_cast<void*>(&ProvideCameraFrame)}}; + if (ats.env()->RegisterNatives(g_java_capturer_class, native_methods, + 2) != 0) + assert(false); + } + + { + jclass clsRef = + mozilla::jni::GetClassRef(ats.env(), "org/webrtc/JniCommon"); + + JNINativeMethod native_methods[] = { + {"nativeAllocateByteBuffer", "(I)Ljava/nio/ByteBuffer;", + reinterpret_cast<void*>(&JniCommon_allocateNativeByteBuffer)}, + {"nativeFreeByteBuffer", "(Ljava/nio/ByteBuffer;)V", + reinterpret_cast<void*>(&JniCommon_freeNativeByteBuffer)}}; + if (ats.env()->RegisterNatives(clsRef, native_methods, 2) != 0) + assert(false); + } + } else { + if (g_jvm_capture) { + AttachThreadScoped ats(g_jvm_capture); + ats.env()->UnregisterNatives(g_java_capturer_class); + ats.env()->DeleteGlobalRef(g_java_capturer_class); + g_java_capturer_class = NULL; + g_context = NULL; + videocapturemodule::DeviceInfoAndroid::DeInitialize(); + g_jvm_capture = NULL; + } + } + + return 0; +} + +namespace videocapturemodule { + +rtc::scoped_refptr<VideoCaptureModule> VideoCaptureImpl::Create( + const char* deviceUniqueIdUTF8) { + rtc::scoped_refptr<VideoCaptureAndroid> implementation( + new rtc::RefCountedObject<VideoCaptureAndroid>()); + if (implementation->Init(deviceUniqueIdUTF8) != 0) { + implementation = nullptr; + } + return implementation; +} + +void VideoCaptureAndroid::OnIncomingFrame(rtc::scoped_refptr<I420Buffer> buffer, + int32_t degrees, + int64_t captureTime) { + MutexLock lock(&api_lock_); + + VideoRotation rotation = + (degrees <= 45 || degrees > 315) ? kVideoRotation_0 + : (degrees > 45 && degrees <= 135) ? kVideoRotation_90 + : (degrees > 135 && degrees <= 225) ? kVideoRotation_180 + : (degrees > 225 && degrees <= 315) ? kVideoRotation_270 + : kVideoRotation_0; // Impossible. + + // Historically, we have ignored captureTime. Why? + VideoFrame captureFrame(I420Buffer::Rotate(*buffer, rotation), 0, + rtc::TimeMillis(), rotation); + + DeliverCapturedFrame(captureFrame); +} + +VideoCaptureAndroid::VideoCaptureAndroid() + : VideoCaptureImpl(), + _deviceInfo(), + _jCapturer(NULL), + _captureStarted(false) {} + +int32_t VideoCaptureAndroid::Init(const char* deviceUniqueIdUTF8) { + const int nameLength = strlen(deviceUniqueIdUTF8); + if (nameLength >= kVideoCaptureUniqueNameLength) return -1; + + // Store the device name + RTC_LOG(LS_INFO) << "VideoCaptureAndroid::Init: " << deviceUniqueIdUTF8; + _deviceUniqueId = new char[nameLength + 1]; + memcpy(_deviceUniqueId, deviceUniqueIdUTF8, nameLength + 1); + + AttachThreadScoped ats(g_jvm_capture); + JNIEnv* env = ats.env(); + jmethodID ctor = env->GetMethodID(g_java_capturer_class, "<init>", + "(Ljava/lang/String;)V"); + assert(ctor); + jstring j_deviceName = env->NewStringUTF(_deviceUniqueId); + _jCapturer = env->NewGlobalRef( + env->NewObject(g_java_capturer_class, ctor, j_deviceName)); + assert(_jCapturer); + return 0; +} + +VideoCaptureAndroid::~VideoCaptureAndroid() { + // Ensure Java camera is released even if our caller didn't explicitly Stop. + if (_captureStarted) StopCapture(); + AttachThreadScoped ats(g_jvm_capture); + JNIEnv* env = ats.env(); + env->DeleteGlobalRef(_jCapturer); +} + +int32_t VideoCaptureAndroid::StartCapture( + const VideoCaptureCapability& capability) { + AttachThreadScoped ats(g_jvm_capture); + JNIEnv* env = ats.env(); + int width = 0; + int height = 0; + int min_mfps = 0; + int max_mfps = 0; + { + MutexLock lock(&api_lock_); + + if (_deviceInfo.GetBestMatchedCapability(_deviceUniqueId, capability, + _captureCapability) < 0) { + RTC_LOG(LS_ERROR) << __FUNCTION__ + << "s: GetBestMatchedCapability failed: " + << capability.width << "x" << capability.height; + return -1; + } + + width = _captureCapability.width; + height = _captureCapability.height; + _deviceInfo.GetMFpsRange(_deviceUniqueId, _captureCapability.maxFPS, + &min_mfps, &max_mfps); + + // Exit critical section to avoid blocking camera thread inside + // onIncomingFrame() call. + } + + jmethodID j_start = + env->GetMethodID(g_java_capturer_class, "startCapture", "(IIIIJ)Z"); + assert(j_start); + jlong j_this = reinterpret_cast<intptr_t>(this); + bool started = env->CallBooleanMethod(_jCapturer, j_start, width, height, + min_mfps, max_mfps, j_this); + if (started) { + MutexLock lock(&api_lock_); + _requestedCapability = capability; + _captureStarted = true; + } + return started ? 0 : -1; +} + +int32_t VideoCaptureAndroid::StopCapture() { + AttachThreadScoped ats(g_jvm_capture); + JNIEnv* env = ats.env(); + { + MutexLock lock(&api_lock_); + + memset(&_requestedCapability, 0, sizeof(_requestedCapability)); + memset(&_captureCapability, 0, sizeof(_captureCapability)); + _captureStarted = false; + // Exit critical section to avoid blocking camera thread inside + // onIncomingFrame() call. + } + + // try to stop the capturer. + jmethodID j_stop = + env->GetMethodID(g_java_capturer_class, "stopCapture", "()Z"); + return env->CallBooleanMethod(_jCapturer, j_stop) ? 0 : -1; +} + +bool VideoCaptureAndroid::CaptureStarted() { + MutexLock lock(&api_lock_); + return _captureStarted; +} + +int32_t VideoCaptureAndroid::CaptureSettings(VideoCaptureCapability& settings) { + MutexLock lock(&api_lock_); + settings = _requestedCapability; + return 0; +} + +} // namespace videocapturemodule +} // namespace webrtc diff --git a/dom/media/systemservices/android_video_capture/video_capture_android.h b/dom/media/systemservices/android_video_capture/video_capture_android.h new file mode 100644 index 0000000000..720c28e70b --- /dev/null +++ b/dom/media/systemservices/android_video_capture/video_capture_android.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_VIDEO_CAPTURE_MAIN_SOURCE_ANDROID_VIDEO_CAPTURE_ANDROID_H_ +#define WEBRTC_MODULES_VIDEO_CAPTURE_MAIN_SOURCE_ANDROID_VIDEO_CAPTURE_ANDROID_H_ + +#include <jni.h> + +#include "device_info_android.h" +#include "api/video/i420_buffer.h" +#include "modules/video_capture/video_capture_impl.h" + +namespace webrtc { +namespace videocapturemodule { + +class VideoCaptureAndroid : public VideoCaptureImpl { + public: + VideoCaptureAndroid(); + virtual int32_t Init(const char* deviceUniqueIdUTF8); + + virtual int32_t StartCapture(const VideoCaptureCapability& capability); + virtual int32_t StopCapture(); + virtual bool CaptureStarted(); + virtual int32_t CaptureSettings(VideoCaptureCapability& settings); + + void OnIncomingFrame(rtc::scoped_refptr<I420Buffer> buffer, int32_t degrees, + int64_t captureTime = 0); + + protected: + virtual ~VideoCaptureAndroid(); + + DeviceInfoAndroid _deviceInfo; + jobject _jCapturer; // Global ref to Java VideoCaptureAndroid object. + VideoCaptureCapability _captureCapability; + bool _captureStarted; +}; + +} // namespace videocapturemodule +} // namespace webrtc +#endif // WEBRTC_MODULES_VIDEO_CAPTURE_MAIN_SOURCE_ANDROID_VIDEO_CAPTURE_ANDROID_H_ diff --git a/dom/media/systemservices/moz.build b/dom/media/systemservices/moz.build new file mode 100644 index 0000000000..09a20b9f15 --- /dev/null +++ b/dom/media/systemservices/moz.build @@ -0,0 +1,112 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +include("/dom/media/webrtc/third_party_build/webrtc.mozbuild") + +if CONFIG["MOZ_WEBRTC"]: + EXPORTS += [ + "CamerasChild.h", + "CamerasParent.h", + "VideoEngine.h", + "VideoFrameUtils.h", + ] + UNIFIED_SOURCES += [ + "CamerasChild.cpp", + "CamerasParent.cpp", + "VideoEngine.cpp", + "VideoFrameUtils.cpp", + ] + LOCAL_INCLUDES += [ + "/dom/media/webrtc", + "/media/libyuv/libyuv/include", + "/mfbt", + "/third_party/libwebrtc", + "/third_party/libwebrtc/third_party/abseil-cpp", + "/tools/profiler/public", + ] + + if CONFIG["OS_TARGET"] == "Android": + UNIFIED_SOURCES += [ + "android_video_capture/device_info_android.cc", + "android_video_capture/video_capture_android.cc", + ] + elif CONFIG["OS_TARGET"] == "Darwin": + UNIFIED_SOURCES += [ + "objc_video_capture/device_info.mm", + "objc_video_capture/device_info_avfoundation.mm", + "objc_video_capture/device_info_objc.mm", + "objc_video_capture/rtc_video_capture_objc.mm", + "objc_video_capture/video_capture.mm", + "objc_video_capture/video_capture_avfoundation.mm", + ] + LOCAL_INCLUDES += [ + "/third_party/libwebrtc/sdk/objc", + "/third_party/libwebrtc/sdk/objc/base", + ] + CMMFLAGS += [ + "-fobjc-arc", + ] + + if CONFIG["OS_TARGET"] != "Android": + UNIFIED_SOURCES += [ + "video_engine/desktop_capture_impl.cc", + "video_engine/desktop_device_info.cc", + "video_engine/tab_capturer.cc", + ] + + +if CONFIG["OS_TARGET"] == "Android": + DEFINES["WEBRTC_ANDROID"] = True + +if CONFIG["MOZ_WIDGET_TOOLKIT"] == "cocoa": + UNIFIED_SOURCES += ["OSXRunLoopSingleton.cpp"] + EXPORTS += ["OSXRunLoopSingleton.h"] + +EXPORTS.mozilla += [ + "ShmemPool.h", +] + +EXPORTS.mozilla.media += [ + "CamerasTypes.h", + "MediaChild.h", + "MediaParent.h", + "MediaSystemResourceClient.h", + "MediaSystemResourceManager.h", + "MediaSystemResourceManagerChild.h", + "MediaSystemResourceManagerParent.h", + "MediaSystemResourceMessageUtils.h", + "MediaSystemResourceService.h", + "MediaSystemResourceTypes.h", + "MediaTaskUtils.h", + "MediaUtils.h", +] +UNIFIED_SOURCES += [ + "CamerasTypes.cpp", + "MediaChild.cpp", + "MediaParent.cpp", + "MediaSystemResourceClient.cpp", + "MediaSystemResourceManager.cpp", + "MediaSystemResourceManagerChild.cpp", + "MediaSystemResourceManagerParent.cpp", + "MediaSystemResourceService.cpp", + "MediaUtils.cpp", + "ShmemPool.cpp", +] +IPDL_SOURCES += [ + "PCameras.ipdl", + "PMedia.ipdl", + "PMediaSystemResourceManager.ipdl", +] +# /dom/base needed for nsGlobalWindow.h in MediaChild.cpp +LOCAL_INCLUDES += [ + "/dom/base", +] + +include("/ipc/chromium/chromium-config.mozbuild") + +FINAL_LIBRARY = "xul" + +with Files("android_video_capture/**"): + SCHEDULES.exclusive = ["android"] diff --git a/dom/media/systemservices/objc_video_capture/device_info.h b/dom/media/systemservices/objc_video_capture/device_info.h new file mode 100644 index 0000000000..d146cfcfda --- /dev/null +++ b/dom/media/systemservices/objc_video_capture/device_info.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CAPTURE_OBJC_DEVICE_INFO_H_ +#define MODULES_VIDEO_CAPTURE_OBJC_DEVICE_INFO_H_ + +#include "modules/video_capture/device_info_impl.h" + +#include <map> +#include <string> + +@class DeviceInfoIosObjC; + +namespace webrtc::videocapturemodule { +class DeviceInfoIos : public DeviceInfoImpl { + public: + DeviceInfoIos(); + virtual ~DeviceInfoIos(); + + // Implementation of DeviceInfoImpl. + int32_t Init() override; + uint32_t NumberOfDevices() override; + int32_t GetDeviceName(uint32_t deviceNumber, char* deviceNameUTF8, uint32_t deviceNameLength, + char* deviceUniqueIdUTF8, uint32_t deviceUniqueIdUTF8Length, + char* productUniqueIdUTF8 = 0, uint32_t productUniqueIdUTF8Length = 0, + pid_t* pid = 0) override; + + int32_t NumberOfCapabilities(const char* deviceUniqueIdUTF8) override; + + int32_t GetCapability(const char* deviceUniqueIdUTF8, const uint32_t deviceCapabilityNumber, + VideoCaptureCapability& capability) override; + + int32_t DisplayCaptureSettingsDialogBox(const char* deviceUniqueIdUTF8, + const char* dialogTitleUTF8, void* parentWindow, + uint32_t positionX, uint32_t positionY) override; + + int32_t GetOrientation(const char* deviceUniqueIdUTF8, VideoRotation& orientation) override; + + int32_t CreateCapabilityMap(const char* device_unique_id_utf8) override + RTC_EXCLUSIVE_LOCKS_REQUIRED(_apiLock); + + private: + std::map<std::string, VideoCaptureCapabilities> _capabilitiesMap; + DeviceInfoIosObjC* _captureInfo; +}; + +} // namespace webrtc::videocapturemodule + +#endif // MODULES_VIDEO_CAPTURE_OBJC_DEVICE_INFO_H_ diff --git a/dom/media/systemservices/objc_video_capture/device_info.mm b/dom/media/systemservices/objc_video_capture/device_info.mm new file mode 100644 index 0000000000..d0299a9ec9 --- /dev/null +++ b/dom/media/systemservices/objc_video_capture/device_info.mm @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#if !defined(__has_feature) || !__has_feature(objc_arc) +# error "This file requires ARC support." +#endif + +#include <AVFoundation/AVFoundation.h> + +#include <string> + +#include "device_info.h" +#include "device_info_objc.h" +#include "modules/video_capture/video_capture_impl.h" +#include "mozilla/StaticPrefs_media.h" +#include "objc_video_capture/device_info_avfoundation.h" +#include "rtc_base/logging.h" + +using namespace mozilla; +using namespace webrtc; +using namespace videocapturemodule; + +static NSArray* camera_presets = @[ + AVCaptureSessionPreset352x288, AVCaptureSessionPreset640x480, AVCaptureSessionPreset1280x720 +]; + +#define IOS_UNSUPPORTED() \ + RTC_LOG(LS_ERROR) << __FUNCTION__ << " is not supported on the iOS platform."; \ + return -1; + +VideoCaptureModule::DeviceInfo* VideoCaptureImpl::CreateDeviceInfo() { + if (StaticPrefs::media_getusermedia_camera_macavf_enabled_AtStartup()) { + return new DeviceInfoAvFoundation(); + } + return new DeviceInfoIos(); +} + +DeviceInfoIos::DeviceInfoIos() { this->Init(); } + +DeviceInfoIos::~DeviceInfoIos() { [_captureInfo registerOwner:nil]; } + +int32_t DeviceInfoIos::Init() { + _captureInfo = [[DeviceInfoIosObjC alloc] init]; + [_captureInfo registerOwner:this]; + + // Fill in all device capabilities. + int deviceCount = [DeviceInfoIosObjC captureDeviceCount]; + + for (int i = 0; i < deviceCount; i++) { + AVCaptureDevice* avDevice = [DeviceInfoIosObjC captureDeviceForIndex:i]; + VideoCaptureCapabilities capabilityVector; + + for (NSString* preset in camera_presets) { + BOOL support = [avDevice supportsAVCaptureSessionPreset:preset]; + if (support) { + VideoCaptureCapability capability = [DeviceInfoIosObjC capabilityForPreset:preset]; + capabilityVector.push_back(capability); + } + } + + char deviceNameUTF8[256]; + char deviceId[256]; + int error = this->GetDeviceName(i, deviceNameUTF8, 256, deviceId, 256); + if (error) { + return error; + } + std::string deviceIdCopy(deviceId); + std::pair<std::string, VideoCaptureCapabilities> mapPair = + std::pair<std::string, VideoCaptureCapabilities>(deviceIdCopy, capabilityVector); + _capabilitiesMap.insert(mapPair); + } + + return 0; +} + +uint32_t DeviceInfoIos::NumberOfDevices() { return [DeviceInfoIosObjC captureDeviceCount]; } + +int32_t DeviceInfoIos::GetDeviceName(uint32_t deviceNumber, char* deviceNameUTF8, + uint32_t deviceNameUTF8Length, char* deviceUniqueIdUTF8, + uint32_t deviceUniqueIdUTF8Length, char* productUniqueIdUTF8, + uint32_t productUniqueIdUTF8Length, pid_t* pid) { + if (deviceNumber >= NumberOfDevices()) { + return -1; + } + + NSString* deviceName = [DeviceInfoIosObjC deviceNameForIndex:deviceNumber]; + + NSString* deviceUniqueId = [DeviceInfoIosObjC deviceUniqueIdForIndex:deviceNumber]; + + strncpy(deviceNameUTF8, [deviceName UTF8String], deviceNameUTF8Length); + deviceNameUTF8[deviceNameUTF8Length - 1] = '\0'; + + strncpy(deviceUniqueIdUTF8, deviceUniqueId.UTF8String, deviceUniqueIdUTF8Length); + deviceUniqueIdUTF8[deviceUniqueIdUTF8Length - 1] = '\0'; + + if (productUniqueIdUTF8) { + productUniqueIdUTF8[0] = '\0'; + } + + return 0; +} + +int32_t DeviceInfoIos::NumberOfCapabilities(const char* deviceUniqueIdUTF8) { + int32_t numberOfCapabilities = 0; + std::string deviceUniqueId(deviceUniqueIdUTF8); + std::map<std::string, VideoCaptureCapabilities>::iterator it = + _capabilitiesMap.find(deviceUniqueId); + + if (it != _capabilitiesMap.end()) { + numberOfCapabilities = it->second.size(); + } + return numberOfCapabilities; +} + +int32_t DeviceInfoIos::GetCapability(const char* deviceUniqueIdUTF8, + const uint32_t deviceCapabilityNumber, + VideoCaptureCapability& capability) { + std::string deviceUniqueId(deviceUniqueIdUTF8); + std::map<std::string, VideoCaptureCapabilities>::iterator it = + _capabilitiesMap.find(deviceUniqueId); + + if (it != _capabilitiesMap.end()) { + VideoCaptureCapabilities deviceCapabilities = it->second; + + if (deviceCapabilityNumber < deviceCapabilities.size()) { + VideoCaptureCapability cap; + cap = deviceCapabilities[deviceCapabilityNumber]; + capability = cap; + return 0; + } + } + + return -1; +} + +int32_t DeviceInfoIos::DisplayCaptureSettingsDialogBox(const char* deviceUniqueIdUTF8, + const char* dialogTitleUTF8, + void* parentWindow, uint32_t positionX, + uint32_t positionY) { + IOS_UNSUPPORTED(); +} + +int32_t DeviceInfoIos::GetOrientation(const char* deviceUniqueIdUTF8, VideoRotation& orientation) { + if (strcmp(deviceUniqueIdUTF8, "Front Camera") == 0) { + orientation = kVideoRotation_0; + } else { + orientation = kVideoRotation_90; + } + return orientation; +} + +int32_t DeviceInfoIos::CreateCapabilityMap(const char* deviceUniqueIdUTF8) { + std::string deviceName(deviceUniqueIdUTF8); + std::map<std::string, std::vector<VideoCaptureCapability>>::iterator it = + _capabilitiesMap.find(deviceName); + VideoCaptureCapabilities deviceCapabilities; + if (it != _capabilitiesMap.end()) { + _captureCapabilities = it->second; + return 0; + } + + return -1; +} diff --git a/dom/media/systemservices/objc_video_capture/device_info_avfoundation.h b/dom/media/systemservices/objc_video_capture/device_info_avfoundation.h new file mode 100644 index 0000000000..9a698480fa --- /dev/null +++ b/dom/media/systemservices/objc_video_capture/device_info_avfoundation.h @@ -0,0 +1,71 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_SYSTEMSERVICES_OBJC_VIDEO_CAPTURE_DEVICE_INFO_AVFOUNDATION_H_ +#define DOM_MEDIA_SYSTEMSERVICES_OBJC_VIDEO_CAPTURE_DEVICE_INFO_AVFOUNDATION_H_ + +#include <map> +#include <string> + +#include "api/sequence_checker.h" +#include "device_info_objc.h" +#include "modules/video_capture/device_info_impl.h" + +namespace webrtc::videocapturemodule { + +/** + * DeviceInfo implementation for the libwebrtc ios/mac sdk camera backend. + * Single threaded except for DeviceChange() that happens on a platform callback + * thread. + */ +class DeviceInfoAvFoundation : public DeviceInfoImpl { + public: + static int32_t ConvertAVFrameRateToCapabilityFPS(Float64 aRate); + static webrtc::VideoType ConvertFourCCToVideoType(FourCharCode aCode); + + DeviceInfoAvFoundation(); + virtual ~DeviceInfoAvFoundation(); + + // Implementation of DeviceInfoImpl. + int32_t Init() override { return 0; } + void DeviceChange() override; + uint32_t NumberOfDevices() override; + int32_t GetDeviceName(uint32_t aDeviceNumber, char* aDeviceNameUTF8, + uint32_t aDeviceNameLength, char* aDeviceUniqueIdUTF8, + uint32_t aDeviceUniqueIdUTF8Length, + char* aProductUniqueIdUTF8 = nullptr, + uint32_t aProductUniqueIdUTF8Length = 0, + pid_t* aPid = nullptr) override; + int32_t NumberOfCapabilities(const char* aDeviceUniqueIdUTF8) override; + int32_t GetCapability(const char* aDeviceUniqueIdUTF8, + const uint32_t aDeviceCapabilityNumber, + VideoCaptureCapability& aCapability) override; + int32_t DisplayCaptureSettingsDialogBox(const char* aDeviceUniqueIdUTF8, + const char* aDialogTitleUTF8, + void* aParentWindow, + uint32_t aPositionX, + uint32_t aPositionY) override { + return -1; + } + int32_t CreateCapabilityMap(const char* aDeviceUniqueIdUTF8) override + RTC_EXCLUSIVE_LOCKS_REQUIRED(_apiLock); + + private: + const std::tuple<std::string, std::string, VideoCaptureCapabilities>* + FindDeviceAndCapabilities(const std::string& aDeviceUniqueId) const; + void EnsureCapabilitiesMap(); + + SequenceChecker mChecker; + std::atomic<bool> mInvalidateCapabilities; + // [{uniqueId, name, capabilities}] + std::vector<std::tuple<std::string, std::string, VideoCaptureCapabilities>> + mDevicesAndCapabilities RTC_GUARDED_BY(mChecker); + const DeviceInfoIosObjC* mDeviceChangeCaptureInfo RTC_GUARDED_BY(mChecker); +}; + +} // namespace webrtc::videocapturemodule + +#endif diff --git a/dom/media/systemservices/objc_video_capture/device_info_avfoundation.mm b/dom/media/systemservices/objc_video_capture/device_info_avfoundation.mm new file mode 100644 index 0000000000..fae65ff343 --- /dev/null +++ b/dom/media/systemservices/objc_video_capture/device_info_avfoundation.mm @@ -0,0 +1,213 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "device_info_avfoundation.h" +#include <CoreVideo/CVPixelBuffer.h> + +#include <string> + +#include "components/capturer/RTCCameraVideoCapturer.h" +#import "helpers/NSString+StdString.h" +#include "media/base/video_common.h" +#include "modules/video_capture/video_capture_defines.h" +#include "rtc_base/logging.h" + +namespace webrtc::videocapturemodule { +/* static */ +int32_t DeviceInfoAvFoundation::ConvertAVFrameRateToCapabilityFPS(Float64 aRate) { + return static_cast<int32_t>(aRate); +} + +/* static */ +webrtc::VideoType DeviceInfoAvFoundation::ConvertFourCCToVideoType(FourCharCode aCode) { + switch (aCode) { + case kCVPixelFormatType_420YpCbCr8Planar: + case kCVPixelFormatType_420YpCbCr8PlanarFullRange: + return webrtc::VideoType::kI420; + case kCVPixelFormatType_24BGR: + return webrtc::VideoType::kRGB24; + case kCVPixelFormatType_32ABGR: + return webrtc::VideoType::kABGR; + case kCMPixelFormat_32ARGB: + return webrtc::VideoType::kBGRA; + case kCMPixelFormat_32BGRA: + return webrtc::VideoType::kARGB; + case kCMPixelFormat_16LE565: + return webrtc::VideoType::kRGB565; + case kCMPixelFormat_16LE555: + case kCMPixelFormat_16LE5551: + return webrtc::VideoType::kARGB1555; + case kCMPixelFormat_422YpCbCr8_yuvs: + return webrtc::VideoType::kYUY2; + case kCMPixelFormat_422YpCbCr8: + return webrtc::VideoType::kUYVY; + case kCMVideoCodecType_JPEG: + case kCMVideoCodecType_JPEG_OpenDML: + return webrtc::VideoType::kMJPEG; + case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: + case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: + return webrtc::VideoType::kNV12; + default: + RTC_LOG(LS_WARNING) << "Unhandled FourCharCode" << aCode; + return webrtc::VideoType::kUnknown; + } +} + +DeviceInfoAvFoundation::DeviceInfoAvFoundation() + : mInvalidateCapabilities(false), mDeviceChangeCaptureInfo([[DeviceInfoIosObjC alloc] init]) { + [mDeviceChangeCaptureInfo registerOwner:this]; +} + +DeviceInfoAvFoundation::~DeviceInfoAvFoundation() { [mDeviceChangeCaptureInfo registerOwner:nil]; } + +void DeviceInfoAvFoundation::DeviceChange() { + mInvalidateCapabilities = true; + DeviceInfo::DeviceChange(); +} + +uint32_t DeviceInfoAvFoundation::NumberOfDevices() { + RTC_DCHECK_RUN_ON(&mChecker); + EnsureCapabilitiesMap(); + return mDevicesAndCapabilities.size(); +} + +int32_t DeviceInfoAvFoundation::GetDeviceName(uint32_t aDeviceNumber, char* aDeviceNameUTF8, + uint32_t aDeviceNameLength, char* aDeviceUniqueIdUTF8, + uint32_t aDeviceUniqueIdUTF8Length, + char* /* aProductUniqueIdUTF8 */, + uint32_t /* aProductUniqueIdUTF8Length */, + pid_t* /* aPid */) { + RTC_DCHECK_RUN_ON(&mChecker); + // Don't EnsureCapabilitiesMap() here, since: + // 1) That might invalidate the capabilities map + // 2) This function depends on the device index + + if (aDeviceNumber >= mDevicesAndCapabilities.size()) { + return -1; + } + + const auto& [uniqueId, name, _] = mDevicesAndCapabilities[aDeviceNumber]; + + strncpy(aDeviceUniqueIdUTF8, uniqueId.c_str(), aDeviceUniqueIdUTF8Length); + aDeviceUniqueIdUTF8[aDeviceUniqueIdUTF8Length - 1] = '\0'; + + strncpy(aDeviceNameUTF8, name.c_str(), aDeviceNameLength); + aDeviceNameUTF8[aDeviceNameLength - 1] = '\0'; + + return 0; +} + +int32_t DeviceInfoAvFoundation::NumberOfCapabilities(const char* aDeviceUniqueIdUTF8) { + RTC_DCHECK_RUN_ON(&mChecker); + + std::string deviceUniqueId(aDeviceUniqueIdUTF8); + const auto* tup = FindDeviceAndCapabilities(deviceUniqueId); + if (!tup) { + return 0; + } + + const auto& [_, __, capabilities] = *tup; + return static_cast<int32_t>(capabilities.size()); +} + +int32_t DeviceInfoAvFoundation::GetCapability(const char* aDeviceUniqueIdUTF8, + const uint32_t aDeviceCapabilityNumber, + VideoCaptureCapability& aCapability) { + RTC_DCHECK_RUN_ON(&mChecker); + + std::string deviceUniqueId(aDeviceUniqueIdUTF8); + const auto* tup = FindDeviceAndCapabilities(deviceUniqueId); + if (!tup) { + return -1; + } + + const auto& [_, __, capabilities] = *tup; + if (aDeviceCapabilityNumber >= capabilities.size()) { + return -1; + } + + aCapability = capabilities[aDeviceCapabilityNumber]; + return 0; +} + +int32_t DeviceInfoAvFoundation::CreateCapabilityMap(const char* aDeviceUniqueIdUTF8) { + RTC_DCHECK_RUN_ON(&mChecker); + + const size_t deviceUniqueIdUTF8Length = strlen(aDeviceUniqueIdUTF8); + if (deviceUniqueIdUTF8Length > kVideoCaptureUniqueNameLength) { + RTC_LOG(LS_INFO) << "Device name too long"; + return -1; + } + RTC_LOG(LS_INFO) << "CreateCapabilityMap called for device " << aDeviceUniqueIdUTF8; + std::string deviceUniqueId(aDeviceUniqueIdUTF8); + const auto* tup = FindDeviceAndCapabilities(deviceUniqueId); + if (!tup) { + RTC_LOG(LS_INFO) << "no matching device found"; + return -1; + } + + // Store the new used device name + _lastUsedDeviceNameLength = deviceUniqueIdUTF8Length; + _lastUsedDeviceName = + static_cast<char*>(realloc(_lastUsedDeviceName, _lastUsedDeviceNameLength + 1)); + memcpy(_lastUsedDeviceName, aDeviceUniqueIdUTF8, _lastUsedDeviceNameLength + 1); + + const auto& [_, __, capabilities] = *tup; + _captureCapabilities = capabilities; + return static_cast<int32_t>(_captureCapabilities.size()); +} + +auto DeviceInfoAvFoundation::FindDeviceAndCapabilities(const std::string& aDeviceUniqueId) const + -> const std::tuple<std::string, std::string, VideoCaptureCapabilities>* { + RTC_DCHECK_RUN_ON(&mChecker); + for (const auto& tup : mDevicesAndCapabilities) { + if (std::get<0>(tup) == aDeviceUniqueId) { + return &tup; + } + } + return nullptr; +} + +void DeviceInfoAvFoundation::EnsureCapabilitiesMap() { + RTC_DCHECK_RUN_ON(&mChecker); + + if (mInvalidateCapabilities.exchange(false)) { + mDevicesAndCapabilities.clear(); + } + + if (!mDevicesAndCapabilities.empty()) { + return; + } + + for (AVCaptureDevice* device in [RTCCameraVideoCapturer captureDevices]) { + std::string uniqueId = [NSString stdStringForString:device.uniqueID]; + std::string name = [NSString stdStringForString:device.localizedName]; + auto& [_, __, capabilities] = + mDevicesAndCapabilities.emplace_back(uniqueId, name, VideoCaptureCapabilities()); + + for (AVCaptureDeviceFormat* format in + [RTCCameraVideoCapturer supportedFormatsForDevice:device]) { + VideoCaptureCapability cap; + FourCharCode fourcc = CMFormatDescriptionGetMediaSubType(format.formatDescription); + cap.videoType = ConvertFourCCToVideoType(fourcc); + CMVideoDimensions dimensions = + CMVideoFormatDescriptionGetDimensions(format.formatDescription); + cap.width = dimensions.width; + cap.height = dimensions.height; + + for (AVFrameRateRange* range in format.videoSupportedFrameRateRanges) { + cap.maxFPS = ConvertAVFrameRateToCapabilityFPS(range.maxFrameRate); + capabilities.push_back(cap); + } + + if (capabilities.empty()) { + cap.maxFPS = 30; + capabilities.push_back(cap); + } + } + } +} +} // namespace webrtc::videocapturemodule diff --git a/dom/media/systemservices/objc_video_capture/device_info_objc.h b/dom/media/systemservices/objc_video_capture/device_info_objc.h new file mode 100644 index 0000000000..1ddedb471e --- /dev/null +++ b/dom/media/systemservices/objc_video_capture/device_info_objc.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CAPTURE_OBJC_DEVICE_INFO_OBJC_H_ +#define MODULES_VIDEO_CAPTURE_OBJC_DEVICE_INFO_OBJC_H_ + +#import <AVFoundation/AVFoundation.h> + +#include "modules/video_capture/video_capture_defines.h" +#include "device_info.h" + +@interface DeviceInfoIosObjC : NSObject { + NSArray* _observers; + NSLock* _lock; + webrtc::VideoCaptureModule::DeviceInfo* _owner; +} + ++ (int)captureDeviceCount; ++ (AVCaptureDevice*)captureDeviceForIndex:(int)index; ++ (AVCaptureDevice*)captureDeviceForUniqueId:(NSString*)uniqueId; ++ (NSString*)deviceNameForIndex:(int)index; ++ (NSString*)deviceUniqueIdForIndex:(int)index; ++ (NSString*)deviceNameForUniqueId:(NSString*)uniqueId; ++ (webrtc::VideoCaptureCapability)capabilityForPreset:(NSString*)preset; + +- (void)registerOwner:(webrtc::VideoCaptureModule::DeviceInfo*)owner; +- (void)configureObservers; + +@end + +#endif // MODULES_VIDEO_CAPTURE_OBJC_DEVICE_INFO_OBJC_H_ diff --git a/dom/media/systemservices/objc_video_capture/device_info_objc.mm b/dom/media/systemservices/objc_video_capture/device_info_objc.mm new file mode 100644 index 0000000000..6e9435daff --- /dev/null +++ b/dom/media/systemservices/objc_video_capture/device_info_objc.mm @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#if !defined(__has_feature) || !__has_feature(objc_arc) +# error "This file requires ARC support." +#endif + +#import <AVFoundation/AVFoundation.h> + +#import "device_info_objc.h" + +@implementation DeviceInfoIosObjC + +- (id)init { + self = [super init]; + if (nil != self) { + _lock = [[NSLock alloc] init]; + } + return self; +} + +- (void)dealloc { +} + +- (void)registerOwner:(webrtc::VideoCaptureModule::DeviceInfo*)owner { + [_lock lock]; + if (!_owner && owner) { + [self configureObservers]; + } else if (_owner && !owner) { + NSNotificationCenter* notificationCenter = [NSNotificationCenter defaultCenter]; + for (id observer in _observers) { + [notificationCenter removeObserver:observer]; + } + _observers = nil; + } + _owner = owner; + [_lock unlock]; +} + ++ (int)captureDeviceCount { + int cnt = 0; + @try { + for (AVCaptureDevice* device in [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]) { + if ([device isSuspended]) { + continue; + } + cnt++; + } + } @catch (NSException* exception) { + cnt = 0; + } + return cnt; +} + ++ (AVCaptureDevice*)captureDeviceForIndex:(int)index { + int cnt = 0; + @try { + for (AVCaptureDevice* device in [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]) { + if ([device isSuspended]) { + continue; + } + if (cnt == index) { + return device; + } + cnt++; + } + } @catch (NSException* exception) { + cnt = 0; + } + + return nil; +} + ++ (AVCaptureDevice*)captureDeviceForUniqueId:(NSString*)uniqueId { + for (AVCaptureDevice* device in [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]) { + if ([device isSuspended]) { + continue; + } + if ([uniqueId isEqual:device.uniqueID]) { + return device; + } + } + + return nil; +} + ++ (NSString*)deviceNameForIndex:(int)index { + return [DeviceInfoIosObjC captureDeviceForIndex:index].localizedName; +} + ++ (NSString*)deviceUniqueIdForIndex:(int)index { + return [DeviceInfoIosObjC captureDeviceForIndex:index].uniqueID; +} + ++ (NSString*)deviceNameForUniqueId:(NSString*)uniqueId { + return [[AVCaptureDevice deviceWithUniqueID:uniqueId] localizedName]; +} + ++ (webrtc::VideoCaptureCapability)capabilityForPreset:(NSString*)preset { + webrtc::VideoCaptureCapability capability; + + // TODO(tkchin): Maybe query AVCaptureDevice for supported formats, and + // then get the dimensions / frame rate from each supported format + if ([preset isEqualToString:AVCaptureSessionPreset352x288]) { + capability.width = 352; + capability.height = 288; + capability.maxFPS = 30; + capability.videoType = webrtc::VideoType::kNV12; + capability.interlaced = false; + } else if ([preset isEqualToString:AVCaptureSessionPreset640x480]) { + capability.width = 640; + capability.height = 480; + capability.maxFPS = 30; + capability.videoType = webrtc::VideoType::kNV12; + capability.interlaced = false; + } else if ([preset isEqualToString:AVCaptureSessionPreset1280x720]) { + capability.width = 1280; + capability.height = 720; + capability.maxFPS = 30; + capability.videoType = webrtc::VideoType::kNV12; + capability.interlaced = false; + } + + return capability; +} + +- (void)configureObservers { + // register device connected / disconnected event + NSNotificationCenter* notificationCenter = [NSNotificationCenter defaultCenter]; + + id deviceWasConnectedObserver = + [notificationCenter addObserverForName:AVCaptureDeviceWasConnectedNotification + object:nil + queue:[NSOperationQueue mainQueue] + usingBlock:^(NSNotification* note) { + [_lock lock]; + AVCaptureDevice* device = [note object]; + BOOL isVideoDevice = [device hasMediaType:AVMediaTypeVideo]; + if (isVideoDevice && _owner) _owner->DeviceChange(); + [_lock unlock]; + }]; + + id deviceWasDisconnectedObserver = + [notificationCenter addObserverForName:AVCaptureDeviceWasDisconnectedNotification + object:nil + queue:[NSOperationQueue mainQueue] + usingBlock:^(NSNotification* note) { + [_lock lock]; + AVCaptureDevice* device = [note object]; + BOOL isVideoDevice = [device hasMediaType:AVMediaTypeVideo]; + if (isVideoDevice && _owner) _owner->DeviceChange(); + [_lock unlock]; + }]; + + _observers = [[NSArray alloc] + initWithObjects:deviceWasConnectedObserver, deviceWasDisconnectedObserver, nil]; +} + +@end diff --git a/dom/media/systemservices/objc_video_capture/rtc_video_capture_objc.h b/dom/media/systemservices/objc_video_capture/rtc_video_capture_objc.h new file mode 100644 index 0000000000..9c6604ffe5 --- /dev/null +++ b/dom/media/systemservices/objc_video_capture/rtc_video_capture_objc.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CAPTURE_OBJC_RTC_VIDEO_CAPTURE_OBJC_H_ +#define MODULES_VIDEO_CAPTURE_OBJC_RTC_VIDEO_CAPTURE_OBJC_H_ + +#import <AVFoundation/AVFoundation.h> +#import <Foundation/Foundation.h> +#ifdef WEBRTC_IOS +# import <UIKit/UIKit.h> +#endif + +#include "video_capture.h" + +// The following class listens to a notification with name: +// 'StatusBarOrientationDidChange'. +// This notification must be posted in order for the capturer to reflect the +// orientation change in video w.r.t. the application orientation. +@interface RTCVideoCaptureIosObjC : NSObject <AVCaptureVideoDataOutputSampleBufferDelegate> + +@property webrtc::VideoRotation frameRotation; + +// custom initializer. Instance of VideoCaptureIos is needed +// for callback purposes. +// default init methods have been overridden to return nil. +- (id)initWithOwner:(webrtc::videocapturemodule::VideoCaptureIos*)owner; +- (BOOL)setCaptureDeviceByUniqueId:(NSString*)uniqueId; +- (BOOL)startCaptureWithCapability:(const webrtc::VideoCaptureCapability&)capability; +- (BOOL)stopCapture; + +@end +#endif // MODULES_VIDEO_CAPTURE_OBJC_RTC_VIDEO_CAPTURE_OBJC_H_ diff --git a/dom/media/systemservices/objc_video_capture/rtc_video_capture_objc.mm b/dom/media/systemservices/objc_video_capture/rtc_video_capture_objc.mm new file mode 100644 index 0000000000..0a36768fa8 --- /dev/null +++ b/dom/media/systemservices/objc_video_capture/rtc_video_capture_objc.mm @@ -0,0 +1,355 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#if !defined(__has_feature) || !__has_feature(objc_arc) +# error "This file requires ARC support." +#endif + +#import <AVFoundation/AVFoundation.h> +#ifdef WEBRTC_IOS +# import <UIKit/UIKit.h> +#endif + +#import "device_info_objc.h" +#import "rtc_video_capture_objc.h" + +#include "rtc_base/logging.h" + +using namespace webrtc; +using namespace webrtc::videocapturemodule; + +@interface RTCVideoCaptureIosObjC (hidden) +- (int)changeCaptureInputWithName:(NSString*)captureDeviceName; +@end + +@implementation RTCVideoCaptureIosObjC { + webrtc::videocapturemodule::VideoCaptureIos* _owner; + webrtc::VideoCaptureCapability _capability; + AVCaptureSession* _captureSession; + BOOL _orientationHasChanged; + AVCaptureConnection* _connection; + BOOL _captureChanging; // Guarded by _captureChangingCondition. + NSCondition* _captureChangingCondition; + dispatch_queue_t _frameQueue; +} + +@synthesize frameRotation = _framRotation; + +- (id)initWithOwner:(VideoCaptureIos*)owner { + if (self == [super init]) { + _owner = owner; + _captureSession = [[AVCaptureSession alloc] init]; +#if defined(WEBRTC_IOS) + _captureSession.usesApplicationAudioSession = NO; +#endif + _captureChanging = NO; + _captureChangingCondition = [[NSCondition alloc] init]; + + if (!_captureSession || !_captureChangingCondition) { + return nil; + } + + // create and configure a new output (using callbacks) + AVCaptureVideoDataOutput* captureOutput = [[AVCaptureVideoDataOutput alloc] init]; + NSString* key = (NSString*)kCVPixelBufferPixelFormatTypeKey; + + NSNumber* val = [NSNumber numberWithUnsignedInt:kCVPixelFormatType_422YpCbCr8]; + NSDictionary* videoSettings = [NSDictionary dictionaryWithObject:val forKey:key]; + captureOutput.videoSettings = videoSettings; + + // add new output + if ([_captureSession canAddOutput:captureOutput]) { + [_captureSession addOutput:captureOutput]; + } else { + RTC_LOG(LS_ERROR) << __FUNCTION__ << ": Could not add output to AVCaptureSession"; + } + +#ifdef WEBRTC_IOS + [[UIDevice currentDevice] beginGeneratingDeviceOrientationNotifications]; + + NSNotificationCenter* notify = [NSNotificationCenter defaultCenter]; + [notify addObserver:self + selector:@selector(onVideoError:) + name:AVCaptureSessionRuntimeErrorNotification + object:_captureSession]; + [notify addObserver:self + selector:@selector(deviceOrientationDidChange:) + name:UIDeviceOrientationDidChangeNotification + object:nil]; +#endif + } + + // Create a serial queue on which video capture will run. By setting the target, + // blocks should still run on DISPATH_QUEUE_PRIORITY_DEFAULT rather than creating + // a new thread. + _frameQueue = dispatch_queue_create("org.webrtc.videocapture", DISPATCH_QUEUE_SERIAL); + dispatch_set_target_queue(_frameQueue, + dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0)); + + return self; +} + +- (void)directOutputToSelf { + [[self currentOutput] setSampleBufferDelegate:self queue:_frameQueue]; +} + +- (void)directOutputToNil { + [[self currentOutput] setSampleBufferDelegate:nil queue:NULL]; +} + +- (void)deviceOrientationDidChange:(NSNotification*)notification { + _orientationHasChanged = YES; + [self setRelativeVideoOrientation]; +} + +- (void)dealloc { + [[NSNotificationCenter defaultCenter] removeObserver:self]; +} + +- (BOOL)setCaptureDeviceByUniqueId:(NSString*)uniqueId { + [self waitForCaptureChangeToFinish]; + // check to see if the camera is already set + if (_captureSession) { + NSArray* currentInputs = [NSArray arrayWithArray:[_captureSession inputs]]; + if ([currentInputs count] > 0) { + AVCaptureDeviceInput* currentInput = [currentInputs objectAtIndex:0]; + if ([uniqueId isEqualToString:[currentInput.device localizedName]]) { + return YES; + } + } + } + + return [self changeCaptureInputByUniqueId:uniqueId]; +} + +- (BOOL)startCaptureWithCapability:(const VideoCaptureCapability&)capability { + [self waitForCaptureChangeToFinish]; + if (!_captureSession) { + return NO; + } + + // check limits of the resolution + if (capability.maxFPS < 0 || capability.maxFPS > 60) { + return NO; + } + + if ([_captureSession canSetSessionPreset:AVCaptureSessionPreset1280x720]) { + if (capability.width > 1280 || capability.height > 720) { + return NO; + } + } else if ([_captureSession canSetSessionPreset:AVCaptureSessionPreset640x480]) { + if (capability.width > 640 || capability.height > 480) { + return NO; + } + } else if ([_captureSession canSetSessionPreset:AVCaptureSessionPreset352x288]) { + if (capability.width > 352 || capability.height > 288) { + return NO; + } + } else if (capability.width < 0 || capability.height < 0) { + return NO; + } + + _capability = capability; + + AVCaptureVideoDataOutput* currentOutput = [self currentOutput]; + if (!currentOutput) return NO; + + [self directOutputToSelf]; + + _orientationHasChanged = NO; + _captureChanging = YES; + dispatch_async(_frameQueue, ^{ + [self startCaptureInBackgroundWithOutput:currentOutput]; + }); + return YES; +} + +- (AVCaptureVideoDataOutput*)currentOutput { + return [[_captureSession outputs] firstObject]; +} + +- (void)startCaptureInBackgroundWithOutput:(AVCaptureVideoDataOutput*)currentOutput { + NSString* captureQuality = [NSString stringWithString:AVCaptureSessionPresetLow]; + if (_capability.width >= 1280 || _capability.height >= 720) { + captureQuality = [NSString stringWithString:AVCaptureSessionPreset1280x720]; + } else if (_capability.width >= 640 || _capability.height >= 480) { + captureQuality = [NSString stringWithString:AVCaptureSessionPreset640x480]; + } else if (_capability.width >= 352 || _capability.height >= 288) { + captureQuality = [NSString stringWithString:AVCaptureSessionPreset352x288]; + } + + // begin configuration for the AVCaptureSession + [_captureSession beginConfiguration]; + + // picture resolution + [_captureSession setSessionPreset:captureQuality]; + + _connection = [currentOutput connectionWithMediaType:AVMediaTypeVideo]; + [self setRelativeVideoOrientation]; + + // finished configuring, commit settings to AVCaptureSession. + [_captureSession commitConfiguration]; + + [_captureSession startRunning]; + [self signalCaptureChangeEnd]; +} + +- (void)setRelativeVideoOrientation { + if (!_connection.supportsVideoOrientation) { + return; + } +#ifndef WEBRTC_IOS + _connection.videoOrientation = AVCaptureVideoOrientationLandscapeRight; + return; +#else + switch ([UIDevice currentDevice].orientation) { + case UIDeviceOrientationPortrait: + _connection.videoOrientation = AVCaptureVideoOrientationPortrait; + break; + case UIDeviceOrientationPortraitUpsideDown: + _connection.videoOrientation = AVCaptureVideoOrientationPortraitUpsideDown; + break; + case UIDeviceOrientationLandscapeLeft: + _connection.videoOrientation = AVCaptureVideoOrientationLandscapeRight; + break; + case UIDeviceOrientationLandscapeRight: + _connection.videoOrientation = AVCaptureVideoOrientationLandscapeLeft; + break; + case UIDeviceOrientationFaceUp: + case UIDeviceOrientationFaceDown: + case UIDeviceOrientationUnknown: + if (!_orientationHasChanged) { + _connection.videoOrientation = AVCaptureVideoOrientationPortrait; + } + break; + } +#endif +} + +- (void)onVideoError:(NSNotification*)notification { + NSLog(@"onVideoError: %@", notification); + // TODO(sjlee): make the specific error handling with this notification. + RTC_LOG(LS_ERROR) << __FUNCTION__ << ": [AVCaptureSession startRunning] error."; +} + +- (BOOL)stopCapture { +#ifdef WEBRTC_IOS + [[UIDevice currentDevice] endGeneratingDeviceOrientationNotifications]; +#endif + _orientationHasChanged = NO; + [self waitForCaptureChangeToFinish]; + [self directOutputToNil]; + + if (!_captureSession) { + return NO; + } + + _captureChanging = YES; + [_captureSession stopRunning]; + + dispatch_sync(_frameQueue, ^{ + [self signalCaptureChangeEnd]; + }); + return YES; +} + +- (BOOL)changeCaptureInputByUniqueId:(NSString*)uniqueId { + [self waitForCaptureChangeToFinish]; + NSArray* currentInputs = [_captureSession inputs]; + // remove current input + if ([currentInputs count] > 0) { + AVCaptureInput* currentInput = (AVCaptureInput*)[currentInputs objectAtIndex:0]; + + [_captureSession removeInput:currentInput]; + } + + // Look for input device with the name requested (as our input param) + // get list of available capture devices + int captureDeviceCount = [DeviceInfoIosObjC captureDeviceCount]; + if (captureDeviceCount <= 0) { + return NO; + } + + AVCaptureDevice* captureDevice = [DeviceInfoIosObjC captureDeviceForUniqueId:uniqueId]; + + if (!captureDevice) { + return NO; + } + + // now create capture session input out of AVCaptureDevice + NSError* deviceError = nil; + AVCaptureDeviceInput* newCaptureInput = [AVCaptureDeviceInput deviceInputWithDevice:captureDevice + error:&deviceError]; + + if (!newCaptureInput) { + const char* errorMessage = [[deviceError localizedDescription] UTF8String]; + + RTC_LOG(LS_ERROR) << __FUNCTION__ << ": deviceInputWithDevice error:" << errorMessage; + + return NO; + } + + // try to add our new capture device to the capture session + [_captureSession beginConfiguration]; + + BOOL addedCaptureInput = NO; + if ([_captureSession canAddInput:newCaptureInput]) { + [_captureSession addInput:newCaptureInput]; + addedCaptureInput = YES; + } else { + addedCaptureInput = NO; + } + + [_captureSession commitConfiguration]; + + return addedCaptureInput; +} + +- (void)captureOutput:(AVCaptureOutput*)captureOutput + didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection*)connection { + const int kFlags = 0; + CVImageBufferRef videoFrame = CMSampleBufferGetImageBuffer(sampleBuffer); + + if (CVPixelBufferLockBaseAddress(videoFrame, kFlags) != kCVReturnSuccess) { + return; + } + + uint8_t* baseAddress = (uint8_t*)CVPixelBufferGetBaseAddress(videoFrame); + const size_t width = CVPixelBufferGetWidth(videoFrame); + const size_t height = CVPixelBufferGetHeight(videoFrame); + const size_t frameSize = width * height * 2; + + VideoCaptureCapability tempCaptureCapability; + tempCaptureCapability.width = width; + tempCaptureCapability.height = height; + tempCaptureCapability.maxFPS = _capability.maxFPS; + tempCaptureCapability.videoType = VideoType::kUYVY; + + _owner->IncomingFrame(baseAddress, frameSize, tempCaptureCapability, 0); + + CVPixelBufferUnlockBaseAddress(videoFrame, kFlags); +} + +- (void)signalCaptureChangeEnd { + [_captureChangingCondition lock]; + _captureChanging = NO; + [_captureChangingCondition signal]; + [_captureChangingCondition unlock]; +} + +- (void)waitForCaptureChangeToFinish { + [_captureChangingCondition lock]; + while (_captureChanging) { + [_captureChangingCondition wait]; + } + [_captureChangingCondition unlock]; +} +@end diff --git a/dom/media/systemservices/objc_video_capture/video_capture.h b/dom/media/systemservices/objc_video_capture/video_capture.h new file mode 100644 index 0000000000..b9f228f679 --- /dev/null +++ b/dom/media/systemservices/objc_video_capture/video_capture.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CAPTURE_OBJC_VIDEO_CAPTURE_H_ +#define MODULES_VIDEO_CAPTURE_OBJC_VIDEO_CAPTURE_H_ + +#include "modules/video_capture/video_capture_impl.h" +#include "api/scoped_refptr.h" + +@class RTCVideoCaptureIosObjC; + +namespace webrtc::videocapturemodule { +class VideoCaptureIos : public VideoCaptureImpl { + public: + VideoCaptureIos(); + virtual ~VideoCaptureIos(); + + static rtc::scoped_refptr<VideoCaptureModule> Create(const char* device_unique_id_utf8); + + // Implementation of VideoCaptureImpl. + int32_t StartCapture(const VideoCaptureCapability& capability) override; + int32_t StopCapture() override; + bool CaptureStarted() override; + int32_t CaptureSettings(VideoCaptureCapability& settings) override; + + private: + RTCVideoCaptureIosObjC* capture_device_; + bool is_capturing_; + VideoCaptureCapability capability_; +}; + +} // namespace webrtc::videocapturemodule + +#endif // MODULES_VIDEO_CAPTURE_OBJC_VIDEO_CAPTURE_H_ diff --git a/dom/media/systemservices/objc_video_capture/video_capture.mm b/dom/media/systemservices/objc_video_capture/video_capture.mm new file mode 100644 index 0000000000..63aef3204c --- /dev/null +++ b/dom/media/systemservices/objc_video_capture/video_capture.mm @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#if !defined(__has_feature) || !__has_feature(objc_arc) +# error "This file requires ARC support." +#endif + +#include "device_info_objc.h" +#include "rtc_video_capture_objc.h" +#include "rtc_base/ref_counted_object.h" +#include "api/scoped_refptr.h" +#include "video_capture_avfoundation.h" +#include "mozilla/StaticPrefs_media.h" + +using namespace mozilla; +using namespace webrtc; +using namespace videocapturemodule; + +rtc::scoped_refptr<VideoCaptureModule> VideoCaptureImpl::Create(const char* deviceUniqueIdUTF8) { + if (StaticPrefs::media_getusermedia_camera_macavf_enabled_AtStartup()) { + return VideoCaptureAvFoundation::Create(deviceUniqueIdUTF8); + } + return VideoCaptureIos::Create(deviceUniqueIdUTF8); +} + +VideoCaptureIos::VideoCaptureIos() : is_capturing_(false) { + capability_.width = kDefaultWidth; + capability_.height = kDefaultHeight; + capability_.maxFPS = kDefaultFrameRate; + capture_device_ = nil; +} + +VideoCaptureIos::~VideoCaptureIos() { + if (is_capturing_) { + [capture_device_ stopCapture]; + capture_device_ = nil; + } +} + +rtc::scoped_refptr<VideoCaptureModule> VideoCaptureIos::Create(const char* deviceUniqueIdUTF8) { + if (!deviceUniqueIdUTF8[0]) { + return NULL; + } + + rtc::scoped_refptr<VideoCaptureIos> capture_module(new rtc::RefCountedObject<VideoCaptureIos>()); + + const int32_t name_length = strlen(deviceUniqueIdUTF8); + if (name_length >= kVideoCaptureUniqueNameLength) return nullptr; + + capture_module->_deviceUniqueId = new char[name_length + 1]; + strncpy(capture_module->_deviceUniqueId, deviceUniqueIdUTF8, name_length + 1); + capture_module->_deviceUniqueId[name_length] = '\0'; + + capture_module->capture_device_ = + [[RTCVideoCaptureIosObjC alloc] initWithOwner:capture_module.get()]; + if (!capture_module->capture_device_) { + return nullptr; + } + + if (![capture_module->capture_device_ + setCaptureDeviceByUniqueId:[[NSString alloc] initWithCString:deviceUniqueIdUTF8 + encoding:NSUTF8StringEncoding]]) { + return nullptr; + } + return capture_module; +} + +int32_t VideoCaptureIos::StartCapture(const VideoCaptureCapability& capability) { + capability_ = capability; + + if (![capture_device_ startCaptureWithCapability:capability]) { + return -1; + } + + is_capturing_ = true; + + return 0; +} + +int32_t VideoCaptureIos::StopCapture() { + if (![capture_device_ stopCapture]) { + return -1; + } + + is_capturing_ = false; + return 0; +} + +bool VideoCaptureIos::CaptureStarted() { return is_capturing_; } + +int32_t VideoCaptureIos::CaptureSettings(VideoCaptureCapability& settings) { + settings = capability_; + settings.videoType = VideoType::kNV12; + return 0; +} diff --git a/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.h b/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.h new file mode 100644 index 0000000000..f5a45b4531 --- /dev/null +++ b/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.h @@ -0,0 +1,79 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_SYSTEMSERVICES_OBJC_VIDEO_CAPTURE_VIDEO_CAPTURE2_H_ +#define DOM_MEDIA_SYSTEMSERVICES_OBJC_VIDEO_CAPTURE_VIDEO_CAPTURE2_H_ + +#import "components/capturer/RTCCameraVideoCapturer.h" + +#include "api/scoped_refptr.h" +#include "api/sequence_checker.h" +#include "modules/video_capture/video_capture_impl.h" +#include "mozilla/Maybe.h" +#include "PerformanceRecorder.h" + +@class VideoCaptureAdapter; + +namespace webrtc::videocapturemodule { + +/** + * VideoCaptureImpl implementation of the libwebrtc ios/mac sdk camera backend. + * Single threaded except for OnFrame() that happens on a platform callback thread. + */ +class VideoCaptureAvFoundation : public VideoCaptureImpl { + public: + VideoCaptureAvFoundation(AVCaptureDevice* _Nonnull aDevice); + virtual ~VideoCaptureAvFoundation(); + + static rtc::scoped_refptr<VideoCaptureModule> Create(const char* _Nullable aDeviceUniqueIdUTF8); + + // Implementation of VideoCaptureImpl. Single threaded. + + // Starts capturing synchronously. Idempotent. If an existing capture is live and another + // capability is requested we'll restart the underlying backend with the new capability. + int32_t StartCapture(const VideoCaptureCapability& aCapability) MOZ_EXCLUDES(api_lock_) override; + // Stops capturing synchronously. Idempotent. + int32_t StopCapture() MOZ_EXCLUDES(api_lock_) override; + bool CaptureStarted() MOZ_EXCLUDES(api_lock_) override; + int32_t CaptureSettings(VideoCaptureCapability& aSettings) override; + + // Callback. This can be called on any thread. + int32_t OnFrame(__strong RTCVideoFrame* _Nonnull aFrame) MOZ_EXCLUDES(api_lock_); + + void SetTrackingId(uint32_t aTrackingIdProcId) MOZ_EXCLUDES(api_lock_) override; + + // Registers the current thread with the profiler if not already registered. + void MaybeRegisterCallbackThread(); + + private: + // Control thread checker. + SequenceChecker mChecker; + AVCaptureDevice* _Nonnull const mDevice RTC_GUARDED_BY(mChecker); + VideoCaptureAdapter* _Nonnull const mAdapter RTC_GUARDED_BY(mChecker); + RTCCameraVideoCapturer* _Nonnull const mCapturer RTC_GUARDED_BY(mChecker); + // If capture has started, this is the capability it was started for. Written on the mChecker + // thread only. + mozilla::Maybe<VideoCaptureCapability> mCapability MOZ_GUARDED_BY(api_lock_); + // The image type that mCapability maps to. Set in lockstep with mCapability. + mozilla::Maybe<mozilla::CaptureStage::ImageType> mImageType MOZ_GUARDED_BY(api_lock_); + // Id string uniquely identifying this capture source. Written on the mChecker thread only. + mozilla::Maybe<mozilla::TrackingId> mTrackingId MOZ_GUARDED_BY(api_lock_); + // Adds frame specific markers to the profiler while mTrackingId is set. Callback thread only. + mozilla::PerformanceRecorderMulti<mozilla::CaptureStage> mCaptureRecorder; + mozilla::PerformanceRecorderMulti<mozilla::CopyVideoStage> mConversionRecorder; + std::atomic<ProfilerThreadId> mCallbackThreadId; +}; + +} // namespace webrtc::videocapturemodule + +@interface VideoCaptureAdapter : NSObject <RTCVideoCapturerDelegate> { + webrtc::Mutex _mutex; + webrtc::videocapturemodule::VideoCaptureAvFoundation* _Nullable _capturer RTC_GUARDED_BY(_mutex); +} +- (void)setCapturer:(webrtc::videocapturemodule::VideoCaptureAvFoundation* _Nullable)capturer; +@end + +#endif diff --git a/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.mm b/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.mm new file mode 100644 index 0000000000..e5ca826fa4 --- /dev/null +++ b/dom/media/systemservices/objc_video_capture/video_capture_avfoundation.mm @@ -0,0 +1,306 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "video_capture_avfoundation.h" + +#import "api/video_frame_buffer/RTCNativeI420Buffer+Private.h" +#import "base/RTCI420Buffer.h" +#import "base/RTCVideoFrame.h" +#import "base/RTCVideoFrameBuffer.h" +#import "components/capturer/RTCCameraVideoCapturer.h" +#import "helpers/NSString+StdString.h" + +#include "api/scoped_refptr.h" +#include "api/video/video_rotation.h" +#include "CallbackThreadRegistry.h" +#include "device_info_avfoundation.h" +#include "modules/video_capture/video_capture_defines.h" +#include "mozilla/Assertions.h" +#include "mozilla/Monitor.h" +#include "mozilla/UniquePtr.h" +#include "rtc_base/time_utils.h" + +using namespace mozilla; +using namespace webrtc::videocapturemodule; + +namespace { +webrtc::VideoRotation ToNativeRotation(RTCVideoRotation aRotation) { + switch (aRotation) { + case RTCVideoRotation_0: + return webrtc::kVideoRotation_0; + case RTCVideoRotation_90: + return webrtc::kVideoRotation_90; + case RTCVideoRotation_180: + return webrtc::kVideoRotation_180; + case RTCVideoRotation_270: + return webrtc::kVideoRotation_270; + default: + MOZ_CRASH_UNSAFE_PRINTF("Unexpected rotation %d", static_cast<int>(aRotation)); + return webrtc::kVideoRotation_0; + } +} + +AVCaptureDeviceFormat* _Nullable FindFormat(AVCaptureDevice* _Nonnull aDevice, + webrtc::VideoCaptureCapability aCapability) { + for (AVCaptureDeviceFormat* format in [aDevice formats]) { + CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(format.formatDescription); + if (dimensions.width != aCapability.width) { + continue; + } + if (dimensions.height != aCapability.height) { + continue; + } + FourCharCode fourcc = CMFormatDescriptionGetMediaSubType(format.formatDescription); + if (aCapability.videoType != DeviceInfoAvFoundation::ConvertFourCCToVideoType(fourcc)) { + continue; + } + if ([format.videoSupportedFrameRateRanges + indexOfObjectPassingTest:^BOOL(AVFrameRateRange* _Nonnull obj, NSUInteger idx, + BOOL* _Nonnull stop) { + return static_cast<BOOL>(DeviceInfoAvFoundation::ConvertAVFrameRateToCapabilityFPS( + obj.maxFrameRate) == aCapability.maxFPS); + }] == NSNotFound) { + continue; + } + + return format; + } + return nullptr; +} +} // namespace + +@implementation VideoCaptureAdapter +- (void)setCapturer:(webrtc::videocapturemodule::VideoCaptureAvFoundation* _Nullable)capturer { + webrtc::MutexLock lock(&_mutex); + _capturer = capturer; +} + +- (void)capturer:(RTCVideoCapturer* _Nonnull)capturer + didCaptureVideoFrame:(RTCVideoFrame* _Nonnull)frame { + rtc::scoped_refptr<webrtc::videocapturemodule::VideoCaptureAvFoundation> cap; + { + webrtc::MutexLock lock(&_mutex); + cap = rtc::scoped_refptr(_capturer); + } + if (!cap) return; + cap->OnFrame(frame); +} +@end + +namespace webrtc::videocapturemodule { +VideoCaptureAvFoundation::VideoCaptureAvFoundation(AVCaptureDevice* _Nonnull aDevice) + : mDevice(aDevice), + mAdapter([[VideoCaptureAdapter alloc] init]), + mCapturer([[RTC_OBJC_TYPE(RTCCameraVideoCapturer) alloc] initWithDelegate:mAdapter]), + mCallbackThreadId() { + const char* uniqueId = [[aDevice uniqueID] UTF8String]; + size_t len = strlen(uniqueId); + _deviceUniqueId = new (std::nothrow) char[len + 1]; + if (_deviceUniqueId) { + memcpy(_deviceUniqueId, uniqueId, len + 1); + } +} + +VideoCaptureAvFoundation::~VideoCaptureAvFoundation() { + // Must block until capture has fully stopped, including async operations. + StopCapture(); +} + +/* static */ +rtc::scoped_refptr<VideoCaptureModule> VideoCaptureAvFoundation::Create( + const char* _Nullable aDeviceUniqueIdUTF8) { + std::string uniqueId(aDeviceUniqueIdUTF8); + for (AVCaptureDevice* device in [RTCCameraVideoCapturer captureDevices]) { + if ([NSString stdStringForString:device.uniqueID] == uniqueId) { + rtc::scoped_refptr<VideoCaptureModule> module( + new rtc::RefCountedObject<VideoCaptureAvFoundation>(device)); + return module; + } + } + return nullptr; +} + +int32_t VideoCaptureAvFoundation::StartCapture(const VideoCaptureCapability& aCapability) { + RTC_DCHECK_RUN_ON(&mChecker); + AVCaptureDeviceFormat* format = FindFormat(mDevice, aCapability); + if (!format) { + return -1; + } + + { + MutexLock lock(&api_lock_); + if (mCapability) { + if (mCapability->width == aCapability.width && mCapability->height == aCapability.height && + mCapability->maxFPS == aCapability.maxFPS && + mCapability->videoType == aCapability.videoType) { + return 0; + } + + api_lock_.Unlock(); + int32_t rv = StopCapture(); + api_lock_.Lock(); + + if (rv != 0) { + return rv; + } + } + } + + [mAdapter setCapturer:this]; + + { + Monitor monitor("VideoCaptureAVFoundation::StartCapture"); + Monitor* copyableMonitor = &monitor; + MonitorAutoLock lock(monitor); + __block Maybe<int32_t> rv; + + [mCapturer startCaptureWithDevice:mDevice + format:format + fps:aCapability.maxFPS + completionHandler:^(NSError* error) { + MonitorAutoLock lock2(*copyableMonitor); + MOZ_RELEASE_ASSERT(!rv); + rv = Some(error ? -1 : 0); + copyableMonitor->Notify(); + }]; + + while (!rv) { + monitor.Wait(); + } + + if (*rv != 0) { + return *rv; + } + } + + MutexLock lock(&api_lock_); + mCapability = Some(aCapability); + mImageType = Some([type = aCapability.videoType] { + switch (type) { + case webrtc::VideoType::kI420: + return CaptureStage::ImageType::I420; + case webrtc::VideoType::kYUY2: + return CaptureStage::ImageType::YUY2; + case webrtc::VideoType::kYV12: + case webrtc::VideoType::kIYUV: + return CaptureStage::ImageType::YV12; + case webrtc::VideoType::kUYVY: + return CaptureStage::ImageType::UYVY; + case webrtc::VideoType::kNV12: + return CaptureStage::ImageType::NV12; + case webrtc::VideoType::kNV21: + return CaptureStage::ImageType::NV21; + case webrtc::VideoType::kMJPEG: + return CaptureStage::ImageType::MJPEG; + case webrtc::VideoType::kRGB24: + case webrtc::VideoType::kBGR24: + case webrtc::VideoType::kABGR: + case webrtc::VideoType::kARGB: + case webrtc::VideoType::kARGB4444: + case webrtc::VideoType::kRGB565: + case webrtc::VideoType::kARGB1555: + case webrtc::VideoType::kBGRA: + case webrtc::VideoType::kUnknown: + // Unlikely, and not represented by CaptureStage::ImageType. + return CaptureStage::ImageType::Unknown; + } + return CaptureStage::ImageType::Unknown; + }()); + + return 0; +} + +int32_t VideoCaptureAvFoundation::StopCapture() { + RTC_DCHECK_RUN_ON(&mChecker); + { + MutexLock lock(&api_lock_); + if (!mCapability) { + return 0; + } + mCapability = Nothing(); + } + + Monitor monitor("VideoCaptureAVFoundation::StopCapture"); + Monitor* copyableMonitor = &monitor; + MonitorAutoLock lock(monitor); + __block bool done = false; + + [mCapturer stopCaptureWithCompletionHandler:^(void) { + MonitorAutoLock lock2(*copyableMonitor); + MOZ_RELEASE_ASSERT(!done); + done = true; + copyableMonitor->Notify(); + }]; + + while (!done) { + monitor.Wait(); + } + + [mAdapter setCapturer:nil]; + + return 0; +} + +bool VideoCaptureAvFoundation::CaptureStarted() { + RTC_DCHECK_RUN_ON(&mChecker); + MutexLock lock(&api_lock_); + return mCapability.isSome(); +} + +int32_t VideoCaptureAvFoundation::CaptureSettings(VideoCaptureCapability& aSettings) { + MOZ_CRASH("Unexpected call"); + return -1; +} + +int32_t VideoCaptureAvFoundation::OnFrame(__strong RTCVideoFrame* _Nonnull aFrame) { + MaybeRegisterCallbackThread(); + if (MutexLock lock(&api_lock_); MOZ_LIKELY(mTrackingId)) { + mCaptureRecorder.Start(0, "VideoCaptureAVFoundation"_ns, *mTrackingId, aFrame.width, + aFrame.height, mImageType.valueOr(CaptureStage::ImageType::Unknown)); + if (mCapability && mCapability->videoType != webrtc::VideoType::kI420) { + mConversionRecorder.Start(0, "VideoCaptureAVFoundation"_ns, *mTrackingId, aFrame.width, + aFrame.height); + } + } + + const int64_t timestamp_us = aFrame.timeStampNs / rtc::kNumNanosecsPerMicrosec; + RTCI420Buffer* buffer = [aFrame.buffer toI420]; + mConversionRecorder.Record(0); + // Accessing the (intended-to-be-private) native buffer directly is hacky but lets us skip two + // copies + rtc::scoped_refptr<webrtc::I420BufferInterface> nativeBuffer = buffer.nativeI420Buffer; + auto frame = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(nativeBuffer) + .set_rotation(ToNativeRotation(aFrame.rotation)) + .set_timestamp_us(timestamp_us) + .build(); + + MutexLock lock(&api_lock_); + int32_t rv = DeliverCapturedFrame(frame); + mCaptureRecorder.Record(0); + return rv; +} + +void VideoCaptureAvFoundation::SetTrackingId(uint32_t aTrackingIdProcId) { + RTC_DCHECK_RUN_ON(&mChecker); + MutexLock lock(&api_lock_); + if (NS_WARN_IF(mTrackingId.isSome())) { + // This capture instance must be shared across multiple camera requests. For now ignore other + // requests than the first. + return; + } + mTrackingId.emplace(TrackingId::Source::Camera, aTrackingIdProcId); +} + +void VideoCaptureAvFoundation::MaybeRegisterCallbackThread() { + ProfilerThreadId id = profiler_current_thread_id(); + if (MOZ_LIKELY(id == mCallbackThreadId)) { + return; + } + mCallbackThreadId = id; + CallbackThreadRegistry::Get()->Register(mCallbackThreadId, "VideoCaptureAVFoundationCallback"); +} +} // namespace webrtc::videocapturemodule diff --git a/dom/media/systemservices/video_engine/browser_capture_impl.h b/dom/media/systemservices/video_engine/browser_capture_impl.h new file mode 100644 index 0000000000..aeaae62202 --- /dev/null +++ b/dom/media/systemservices/video_engine/browser_capture_impl.h @@ -0,0 +1,78 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef WEBRTC_MODULES_BROWSER_CAPTURE_MAIN_SOURCE_BROWSER_CAPTURE_IMPL_H_ +#define WEBRTC_MODULES_BROWSER_CAPTURE_MAIN_SOURCE_BROWSER_CAPTURE_IMPL_H_ + +#include "webrtc/modules/video_capture/video_capture.h" + +using namespace webrtc::videocapturemodule; + +namespace webrtc { + +class BrowserDeviceInfoImpl : public VideoCaptureModule::DeviceInfo { + public: + virtual uint32_t NumberOfDevices() { return 1; } + + virtual int32_t Refresh() { return 0; } + + virtual int32_t GetDeviceName(uint32_t deviceNumber, char* deviceNameUTF8, + uint32_t deviceNameSize, + char* deviceUniqueIdUTF8, + uint32_t deviceUniqueIdUTF8Size, + char* productUniqueIdUTF8 = NULL, + uint32_t productUniqueIdUTF8Size = 0, + pid_t* pid = 0) { + deviceNameUTF8 = const_cast<char*>(kDeviceName); + deviceUniqueIdUTF8 = const_cast<char*>(kUniqueDeviceName); + productUniqueIdUTF8 = const_cast<char*>(kProductUniqueId); + return 1; + }; + + virtual int32_t NumberOfCapabilities(const char* deviceUniqueIdUTF8) { + return 0; + } + + virtual int32_t GetCapability(const char* deviceUniqueIdUTF8, + const uint32_t deviceCapabilityNumber, + VideoCaptureCapability& capability) { + return 0; + }; + + virtual int32_t GetOrientation(const char* deviceUniqueIdUTF8, + VideoRotation& orientation) { + return 0; + } + + virtual int32_t GetBestMatchedCapability( + const char* deviceUniqueIdUTF8, const VideoCaptureCapability& requested, + VideoCaptureCapability& resulting) { + return 0; + } + + virtual int32_t DisplayCaptureSettingsDialogBox( + const char* deviceUniqueIdUTF8, const char* dialogTitleUTF8, + void* parentWindow, uint32_t positionX, uint32_t positionY) { + return 0; + } + + BrowserDeviceInfoImpl() + : kDeviceName("browser"), + kUniqueDeviceName("browser"), + kProductUniqueId("browser") {} + + static BrowserDeviceInfoImpl* CreateDeviceInfo() { + return new BrowserDeviceInfoImpl(); + } + virtual ~BrowserDeviceInfoImpl() {} + + private: + const char* kDeviceName; + const char* kUniqueDeviceName; + const char* kProductUniqueId; +}; +} // namespace webrtc +#endif diff --git a/dom/media/systemservices/video_engine/desktop_capture_impl.cc b/dom/media/systemservices/video_engine/desktop_capture_impl.cc new file mode 100644 index 0000000000..2274a21e8a --- /dev/null +++ b/dom/media/systemservices/video_engine/desktop_capture_impl.cc @@ -0,0 +1,776 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video_engine/desktop_capture_impl.h" + +#include <cstdlib> +#include <memory> +#include <string> + +#include "CamerasTypes.h" +#include "VideoEngine.h" +#include "VideoUtils.h" +#include "api/video/i420_buffer.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "libyuv.h" // NOLINT +#include "modules/include/module_common_types.h" +#include "modules/video_capture/video_capture_config.h" +#include "modules/video_capture/video_capture_impl.h" +#include "system_wrappers/include/clock.h" +#include "rtc_base/logging.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/time_utils.h" +#include "rtc_base/trace_event.h" +#include "modules/desktop_capture/desktop_and_cursor_composer.h" +#include "modules/desktop_capture/desktop_frame.h" +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_capturer_differ_wrapper.h" +#include "modules/video_capture/video_capture.h" +#include "mozilla/StaticPrefs_media.h" +#include "mozilla/SyncRunnable.h" +#include "mozilla/TaskQueue.h" +#include "mozilla/TimeStamp.h" +#include "nsThreadUtils.h" +#include "tab_capturer.h" + +using mozilla::NewRunnableMethod; +using mozilla::TabCapturerWebrtc; +using mozilla::TimeDuration; +using mozilla::camera::CaptureDeviceType; +using mozilla::camera::CaptureEngine; + +static void CaptureFrameOnThread(nsITimer* aTimer, void* aClosure) { + static_cast<webrtc::DesktopCaptureImpl*>(aClosure)->CaptureFrameOnThread(); +} + +namespace webrtc { + +int32_t ScreenDeviceInfoImpl::Init() { + mDesktopDeviceInfo = + std::unique_ptr<DesktopDeviceInfo>(DesktopDeviceInfo::Create()); + return 0; +} + +int32_t ScreenDeviceInfoImpl::Refresh() { + mDesktopDeviceInfo->Refresh(); + return 0; +} + +uint32_t ScreenDeviceInfoImpl::NumberOfDevices() { + return mDesktopDeviceInfo->getDisplayDeviceCount(); +} + +int32_t ScreenDeviceInfoImpl::GetDeviceName( + uint32_t aDeviceNumber, char* aDeviceNameUTF8, uint32_t aDeviceNameUTF8Size, + char* aDeviceUniqueIdUTF8, uint32_t aDeviceUniqueIdUTF8Size, + char* aProductUniqueIdUTF8, uint32_t aProductUniqueIdUTF8Size, + pid_t* aPid) { + DesktopDisplayDevice desktopDisplayDevice; + + // always initialize output + if (aDeviceNameUTF8 && aDeviceNameUTF8Size > 0) { + memset(aDeviceNameUTF8, 0, aDeviceNameUTF8Size); + } + + if (aDeviceUniqueIdUTF8 && aDeviceUniqueIdUTF8Size > 0) { + memset(aDeviceUniqueIdUTF8, 0, aDeviceUniqueIdUTF8Size); + } + if (aProductUniqueIdUTF8 && aProductUniqueIdUTF8Size > 0) { + memset(aProductUniqueIdUTF8, 0, aProductUniqueIdUTF8Size); + } + + if (mDesktopDeviceInfo->getDesktopDisplayDeviceInfo( + aDeviceNumber, desktopDisplayDevice) == 0) { + size_t len; + + const char* deviceName = desktopDisplayDevice.getDeviceName(); + len = deviceName ? strlen(deviceName) : 0; + if (len && aDeviceNameUTF8 && len < aDeviceNameUTF8Size) { + memcpy(aDeviceNameUTF8, deviceName, len); + } + + const char* deviceUniqueId = desktopDisplayDevice.getUniqueIdName(); + len = deviceUniqueId ? strlen(deviceUniqueId) : 0; + if (len && aDeviceUniqueIdUTF8 && len < aDeviceUniqueIdUTF8Size) { + memcpy(aDeviceUniqueIdUTF8, deviceUniqueId, len); + } + } + + return 0; +} + +int32_t ScreenDeviceInfoImpl::DisplayCaptureSettingsDialogBox( + const char* aDeviceUniqueIdUTF8, const char* aDialogTitleUTF8, + void* aParentWindow, uint32_t aPositionX, uint32_t aPositionY) { + // no device properties to change + return 0; +} + +int32_t ScreenDeviceInfoImpl::NumberOfCapabilities( + const char* aDeviceUniqueIdUTF8) { + return 0; +} + +int32_t ScreenDeviceInfoImpl::GetCapability( + const char* aDeviceUniqueIdUTF8, uint32_t aDeviceCapabilityNumber, + VideoCaptureCapability& aCapability) { + return 0; +} + +int32_t ScreenDeviceInfoImpl::GetBestMatchedCapability( + const char* aDeviceUniqueIdUTF8, const VideoCaptureCapability& aRequested, + VideoCaptureCapability& aResulting) { + return 0; +} + +int32_t ScreenDeviceInfoImpl::GetOrientation(const char* aDeviceUniqueIdUTF8, + VideoRotation& aOrientation) { + return 0; +} + +VideoCaptureModule* DesktopCaptureImpl::Create(const int32_t aModuleId, + const char* aUniqueId, + const CaptureDeviceType aType) { + return new rtc::RefCountedObject<DesktopCaptureImpl>(aModuleId, aUniqueId, + aType); +} + +int32_t WindowDeviceInfoImpl::Init() { + mDesktopDeviceInfo = + std::unique_ptr<DesktopDeviceInfo>(DesktopDeviceInfo::Create()); + return 0; +} + +int32_t WindowDeviceInfoImpl::Refresh() { + mDesktopDeviceInfo->Refresh(); + return 0; +} + +uint32_t WindowDeviceInfoImpl::NumberOfDevices() { + return mDesktopDeviceInfo->getWindowCount(); +} + +int32_t WindowDeviceInfoImpl::GetDeviceName( + uint32_t aDeviceNumber, char* aDeviceNameUTF8, uint32_t aDeviceNameUTF8Size, + char* aDeviceUniqueIdUTF8, uint32_t aDeviceUniqueIdUTF8Size, + char* aProductUniqueIdUTF8, uint32_t aProductUniqueIdUTF8Size, + pid_t* aPid) { + DesktopDisplayDevice desktopDisplayDevice; + + // always initialize output + if (aDeviceNameUTF8 && aDeviceNameUTF8Size > 0) { + memset(aDeviceNameUTF8, 0, aDeviceNameUTF8Size); + } + if (aDeviceUniqueIdUTF8 && aDeviceUniqueIdUTF8Size > 0) { + memset(aDeviceUniqueIdUTF8, 0, aDeviceUniqueIdUTF8Size); + } + if (aProductUniqueIdUTF8 && aProductUniqueIdUTF8Size > 0) { + memset(aProductUniqueIdUTF8, 0, aProductUniqueIdUTF8Size); + } + + if (mDesktopDeviceInfo->getWindowInfo(aDeviceNumber, desktopDisplayDevice) == + 0) { + size_t len; + + const char* deviceName = desktopDisplayDevice.getDeviceName(); + len = deviceName ? strlen(deviceName) : 0; + if (len && aDeviceNameUTF8 && len < aDeviceNameUTF8Size) { + memcpy(aDeviceNameUTF8, deviceName, len); + } + + const char* deviceUniqueId = desktopDisplayDevice.getUniqueIdName(); + len = deviceUniqueId ? strlen(deviceUniqueId) : 0; + if (len && aDeviceUniqueIdUTF8 && len < aDeviceUniqueIdUTF8Size) { + memcpy(aDeviceUniqueIdUTF8, deviceUniqueId, len); + } + if (aPid) { + *aPid = desktopDisplayDevice.getPid(); + } + } + + return 0; +} + +int32_t WindowDeviceInfoImpl::DisplayCaptureSettingsDialogBox( + const char* aDeviceUniqueIdUTF8, const char* aDialogTitleUTF8, + void* aParentWindow, uint32_t aPositionX, uint32_t aPositionY) { + // no device properties to change + return 0; +} + +int32_t WindowDeviceInfoImpl::NumberOfCapabilities( + const char* aDeviceUniqueIdUTF8) { + return 0; +} + +int32_t WindowDeviceInfoImpl::GetCapability( + const char* aDeviceUniqueIdUTF8, uint32_t aDeviceCapabilityNumber, + VideoCaptureCapability& aCapability) { + return 0; +} + +int32_t WindowDeviceInfoImpl::GetBestMatchedCapability( + const char* aDeviceUniqueIdUTF8, const VideoCaptureCapability& aRequested, + VideoCaptureCapability& aResulting) { + return 0; +} + +int32_t WindowDeviceInfoImpl::GetOrientation(const char* aDeviceUniqueIdUTF8, + VideoRotation& aOrientation) { + return 0; +} + +int32_t BrowserDeviceInfoImpl::Init() { + mDesktopDeviceInfo = + std::unique_ptr<DesktopDeviceInfo>(DesktopDeviceInfo::Create()); + return 0; +} + +int32_t BrowserDeviceInfoImpl::Refresh() { + mDesktopDeviceInfo->Refresh(); + return 0; +} + +uint32_t BrowserDeviceInfoImpl::NumberOfDevices() { + return mDesktopDeviceInfo->getTabCount(); +} + +int32_t BrowserDeviceInfoImpl::GetDeviceName( + uint32_t aDeviceNumber, char* aDeviceNameUTF8, uint32_t aDeviceNameUTF8Size, + char* aDeviceUniqueIdUTF8, uint32_t aDeviceUniqueIdUTF8Size, + char* aProductUniqueIdUTF8, uint32_t aProductUniqueIdUTF8Size, + pid_t* aPid) { + DesktopTab desktopTab; + + // always initialize output + if (aDeviceNameUTF8 && aDeviceNameUTF8Size > 0) { + memset(aDeviceNameUTF8, 0, aDeviceNameUTF8Size); + } + if (aDeviceUniqueIdUTF8 && aDeviceUniqueIdUTF8Size > 0) { + memset(aDeviceUniqueIdUTF8, 0, aDeviceUniqueIdUTF8Size); + } + if (aProductUniqueIdUTF8 && aProductUniqueIdUTF8Size > 0) { + memset(aProductUniqueIdUTF8, 0, aProductUniqueIdUTF8Size); + } + + if (mDesktopDeviceInfo->getTabInfo(aDeviceNumber, desktopTab) == 0) { + size_t len; + + const char* deviceName = desktopTab.getTabName(); + len = deviceName ? strlen(deviceName) : 0; + if (len && aDeviceNameUTF8 && len < aDeviceNameUTF8Size) { + memcpy(aDeviceNameUTF8, deviceName, len); + } + + const char* deviceUniqueId = desktopTab.getUniqueIdName(); + len = deviceUniqueId ? strlen(deviceUniqueId) : 0; + if (len && aDeviceUniqueIdUTF8 && len < aDeviceUniqueIdUTF8Size) { + memcpy(aDeviceUniqueIdUTF8, deviceUniqueId, len); + } + } + + return 0; +} + +int32_t BrowserDeviceInfoImpl::DisplayCaptureSettingsDialogBox( + const char* aDeviceUniqueIdUTF8, const char* aDialogTitleUTF8, + void* aParentWindow, uint32_t aPositionX, uint32_t aPositionY) { + // no device properties to change + return 0; +} + +int32_t BrowserDeviceInfoImpl::NumberOfCapabilities( + const char* aDeviceUniqueIdUTF8) { + return 0; +} + +int32_t BrowserDeviceInfoImpl::GetCapability( + const char* aDeviceUniqueIdUTF8, uint32_t aDeviceCapabilityNumber, + VideoCaptureCapability& aCapability) { + return 0; +} + +int32_t BrowserDeviceInfoImpl::GetBestMatchedCapability( + const char* aDeviceUniqueIdUTF8, const VideoCaptureCapability& aRequested, + VideoCaptureCapability& aResulting) { + return 0; +} + +int32_t BrowserDeviceInfoImpl::GetOrientation(const char* aDeviceUniqueIdUTF8, + VideoRotation& aOrientation) { + return 0; +} + +std::shared_ptr<VideoCaptureModule::DeviceInfo> +DesktopCaptureImpl::CreateDeviceInfo(const int32_t aId, + const CaptureDeviceType aType) { + if (aType == CaptureDeviceType::Screen) { + auto screenInfo = std::make_shared<ScreenDeviceInfoImpl>(aId); + if (!screenInfo || screenInfo->Init() != 0) { + return nullptr; + } + return screenInfo; + } + if (aType == CaptureDeviceType::Window) { + auto windowInfo = std::make_shared<WindowDeviceInfoImpl>(aId); + if (!windowInfo || windowInfo->Init() != 0) { + return nullptr; + } + return windowInfo; + } + if (aType == CaptureDeviceType::Browser) { + auto browserInfo = std::make_shared<BrowserDeviceInfoImpl>(aId); + if (!browserInfo || browserInfo->Init() != 0) { + return nullptr; + } + return browserInfo; + } + return nullptr; +} + +const char* DesktopCaptureImpl::CurrentDeviceName() const { + return mDeviceUniqueId.c_str(); +} + +static DesktopCaptureOptions CreateDesktopCaptureOptions() { + DesktopCaptureOptions options; +// Help avoid an X11 deadlock, see bug 1456101. +#ifdef MOZ_X11 + MOZ_ALWAYS_SUCCEEDS(mozilla::SyncRunnable::DispatchToThread( + mozilla::GetMainThreadSerialEventTarget(), + NS_NewRunnableFunction(__func__, [&] { + options = DesktopCaptureOptions::CreateDefault(); + }))); +#else + options = DesktopCaptureOptions::CreateDefault(); +#endif + + // Leave desktop effects enabled during WebRTC captures. + options.set_disable_effects(false); + +#if defined(WEBRTC_WIN) + if (mozilla::StaticPrefs::media_webrtc_capture_allow_directx()) { + options.set_allow_directx_capturer(true); + options.set_allow_use_magnification_api(false); + } else { + options.set_allow_use_magnification_api(true); + } + options.set_allow_cropping_window_capturer(true); +# if defined(RTC_ENABLE_WIN_WGC) + if (mozilla::StaticPrefs::media_webrtc_capture_allow_wgc()) { + options.set_allow_wgc_capturer(true); + } +# endif +#endif + +#if defined(WEBRTC_MAC) + if (mozilla::StaticPrefs::media_webrtc_capture_allow_iosurface()) { + options.set_allow_iosurface(true); + } +#endif + +#if defined(WEBRTC_USE_PIPEWIRE) + if (mozilla::StaticPrefs::media_webrtc_capture_allow_pipewire()) { + options.set_allow_pipewire(true); + } +#endif + + return options; +} + +static std::unique_ptr<DesktopCapturer> CreateTabCapturer( + const DesktopCaptureOptions& options, DesktopCapturer::SourceId aSourceId, + nsCOMPtr<nsISerialEventTarget> aCaptureThread) { + std::unique_ptr<DesktopCapturer> capturer = + TabCapturerWebrtc::Create(aSourceId, std::move(aCaptureThread)); + if (capturer && options.detect_updated_region()) { + capturer.reset(new DesktopCapturerDifferWrapper(std::move(capturer))); + } + + return capturer; +} + +static bool UsePipewire() { +#if defined(WEBRTC_USE_PIPEWIRE) + return mozilla::StaticPrefs::media_webrtc_capture_allow_pipewire() && + webrtc::DesktopCapturer::IsRunningUnderWayland(); +#else + return false; +#endif +} + +static std::unique_ptr<DesktopCapturer> CreateDesktopCapturerAndThread( + CaptureDeviceType aDeviceType, DesktopCapturer::SourceId aSourceId, + nsIThread** aOutThread) { + DesktopCaptureOptions options = CreateDesktopCaptureOptions(); + std::unique_ptr<DesktopCapturer> capturer; + + auto ensureThread = [&]() { + if (*aOutThread) { + return *aOutThread; + } + + nsIThreadManager::ThreadCreationOptions threadOptions; +#if defined(XP_WIN) || defined(XP_MACOSX) + // Windows desktop capture needs a UI thread. + // Mac screen capture needs a thread with a CFRunLoop. + threadOptions.isUiThread = true; +#endif + NS_NewNamedThread("DesktopCapture", aOutThread, nullptr, threadOptions); + return *aOutThread; + }; + + if ((aDeviceType == CaptureDeviceType::Screen || + aDeviceType == CaptureDeviceType::Window) && + UsePipewire()) { + capturer = DesktopCapturer::CreateGenericCapturer(options); + if (!capturer) { + return capturer; + } + + capturer = std::make_unique<DesktopAndCursorComposer>(std::move(capturer), + options); + } else if (aDeviceType == CaptureDeviceType::Screen) { + capturer = DesktopCapturer::CreateScreenCapturer(options); + if (!capturer) { + return capturer; + } + + capturer->SelectSource(aSourceId); + + capturer = std::make_unique<DesktopAndCursorComposer>(std::move(capturer), + options); + } else if (aDeviceType == CaptureDeviceType::Window) { +#if defined(RTC_ENABLE_WIN_WGC) + options.set_allow_wgc_capturer_fallback(true); +#endif + capturer = DesktopCapturer::CreateWindowCapturer(options); + if (!capturer) { + return capturer; + } + + capturer->SelectSource(aSourceId); + + capturer = std::make_unique<DesktopAndCursorComposer>(std::move(capturer), + options); + } else if (aDeviceType == CaptureDeviceType::Browser) { + // XXX We don't capture cursors, so avoid the extra indirection layer. We + // could also pass null for the pMouseCursorMonitor. + capturer = CreateTabCapturer(options, aSourceId, ensureThread()); + } else { + MOZ_ASSERT(!capturer); + return capturer; + } + + MOZ_ASSERT(capturer); + ensureThread(); + + return capturer; +} + +DesktopCaptureImpl::DesktopCaptureImpl(const int32_t aId, const char* aUniqueId, + const CaptureDeviceType aType) + : mModuleId(aId), + mTrackingId(mozilla::TrackingId(CaptureEngineToTrackingSourceStr([&] { + switch (aType) { + case CaptureDeviceType::Screen: + return CaptureEngine::ScreenEngine; + case CaptureDeviceType::Window: + return CaptureEngine::WinEngine; + case CaptureDeviceType::Browser: + return CaptureEngine::BrowserEngine; + default: + return CaptureEngine::InvalidEngine; + } + }()), + aId)), + mDeviceUniqueId(aUniqueId), + mDeviceType(aType), + mControlThread(mozilla::GetCurrentSerialEventTarget()), + mNextFrameMinimumTime(Timestamp::Zero()), + mCallbacks("DesktopCaptureImpl::mCallbacks") {} + +DesktopCaptureImpl::~DesktopCaptureImpl() { + MOZ_ASSERT(!mCaptureThread); + MOZ_ASSERT(!mRequestedCapability); +} + +void DesktopCaptureImpl::RegisterCaptureDataCallback( + rtc::VideoSinkInterface<VideoFrame>* aDataCallback) { + auto callbacks = mCallbacks.Lock(); + callbacks->insert(aDataCallback); +} + +void DesktopCaptureImpl::DeRegisterCaptureDataCallback( + rtc::VideoSinkInterface<VideoFrame>* aDataCallback) { + auto callbacks = mCallbacks.Lock(); + auto it = callbacks->find(aDataCallback); + if (it != callbacks->end()) { + callbacks->erase(it); + } +} + +int32_t DesktopCaptureImpl::StopCaptureIfAllClientsClose() { + { + auto callbacks = mCallbacks.Lock(); + if (!callbacks->empty()) { + return 0; + } + } + return StopCapture(); +} + +int32_t DesktopCaptureImpl::SetCaptureRotation(VideoRotation aRotation) { + MOZ_ASSERT_UNREACHABLE("Unused"); + return -1; +} + +bool DesktopCaptureImpl::SetApplyRotation(bool aEnable) { return true; } + +int32_t DesktopCaptureImpl::StartCapture( + const VideoCaptureCapability& aCapability) { + RTC_DCHECK_RUN_ON(&mControlThreadChecker); + + if (mRequestedCapability) { + // Already initialized + MOZ_ASSERT(*mRequestedCapability == aCapability); + + return 0; + } + + MOZ_ASSERT(!mCaptureThread); + + DesktopCapturer::SourceId sourceId = std::stoi(mDeviceUniqueId); + std::unique_ptr capturer = CreateDesktopCapturerAndThread( + mDeviceType, sourceId, getter_AddRefs(mCaptureThread)); + + MOZ_ASSERT(!capturer == !mCaptureThread); + if (!capturer) { + return -1; + } + + mRequestedCapability = mozilla::Some(aCapability); + mCaptureThreadChecker.Detach(); + + MOZ_ALWAYS_SUCCEEDS(mCaptureThread->Dispatch(NS_NewRunnableFunction( + "DesktopCaptureImpl::InitOnThread", + [this, self = RefPtr(this), capturer = std::move(capturer), + maxFps = std::max(aCapability.maxFPS, 1)]() mutable { + InitOnThread(std::move(capturer), maxFps); + }))); + + return 0; +} + +bool DesktopCaptureImpl::FocusOnSelectedSource() { + RTC_DCHECK_RUN_ON(&mControlThreadChecker); + if (!mCaptureThread) { + MOZ_ASSERT_UNREACHABLE( + "FocusOnSelectedSource must be called after StartCapture"); + return false; + } + + bool success = false; + MOZ_ALWAYS_SUCCEEDS(mozilla::SyncRunnable::DispatchToThread( + mCaptureThread, NS_NewRunnableFunction(__func__, [&] { + RTC_DCHECK_RUN_ON(&mCaptureThreadChecker); + MOZ_ASSERT(mCapturer); + success = mCapturer && mCapturer->FocusOnSelectedSource(); + }))); + return success; +} + +int32_t DesktopCaptureImpl::StopCapture() { + RTC_DCHECK_RUN_ON(&mControlThreadChecker); + if (mRequestedCapability) { + // Sync-cancel the capture timer so no CaptureFrame calls will come in after + // we return. + MOZ_ALWAYS_SUCCEEDS(mozilla::SyncRunnable::DispatchToThread( + mCaptureThread, + NewRunnableMethod(__func__, this, + &DesktopCaptureImpl::ShutdownOnThread))); + + mRequestedCapability = mozilla::Nothing(); + } + + if (mCaptureThread) { + // CaptureThread shutdown. + mCaptureThread->AsyncShutdown(); + mCaptureThread = nullptr; + } + + return 0; +} + +bool DesktopCaptureImpl::CaptureStarted() { + MOZ_ASSERT_UNREACHABLE("Unused"); + return true; +} + +int32_t DesktopCaptureImpl::CaptureSettings(VideoCaptureCapability& aSettings) { + MOZ_ASSERT_UNREACHABLE("Unused"); + return -1; +} + +void DesktopCaptureImpl::OnCaptureResult(DesktopCapturer::Result aResult, + std::unique_ptr<DesktopFrame> aFrame) { + RTC_DCHECK_RUN_ON(&mCaptureThreadChecker); + if (!aFrame) { + return; + } + + const auto startProcessTime = Timestamp::Micros(rtc::TimeMicros()); + auto frameTime = startProcessTime; + if (auto diff = startProcessTime - mNextFrameMinimumTime; + diff < TimeDelta::Zero()) { + if (diff > TimeDelta::Millis(-1)) { + // Two consecutive frames within a millisecond is OK. It could happen due + // to timing. + frameTime = mNextFrameMinimumTime; + } else { + // Three consecutive frames within two milliseconds seems too much, drop + // one. + MOZ_ASSERT(diff >= TimeDelta::Millis(-2)); + RTC_LOG(LS_WARNING) << "DesktopCapture render time is getting too far " + "ahead. Framerate is unexpectedly high."; + return; + } + } + + uint8_t* videoFrame = aFrame->data(); + VideoCaptureCapability frameInfo; + frameInfo.width = aFrame->size().width(); + frameInfo.height = aFrame->size().height(); + frameInfo.videoType = VideoType::kARGB; + + size_t videoFrameLength = + frameInfo.width * frameInfo.height * DesktopFrame::kBytesPerPixel; + + const int32_t width = frameInfo.width; + const int32_t height = frameInfo.height; + + // Not encoded, convert to I420. + if (frameInfo.videoType != VideoType::kMJPEG && + CalcBufferSize(frameInfo.videoType, width, abs(height)) != + videoFrameLength) { + RTC_LOG(LS_ERROR) << "Wrong incoming frame length."; + return; + } + + int stride_y = width; + int stride_uv = (width + 1) / 2; + + // Setting absolute height (in case it was negative). + // In Windows, the image starts bottom left, instead of top left. + // Setting a negative source height, inverts the image (within LibYuv). + + mozilla::PerformanceRecorder<mozilla::CopyVideoStage> rec( + "DesktopCaptureImpl::ConvertToI420"_ns, mTrackingId, width, abs(height)); + // TODO(nisse): Use a pool? + rtc::scoped_refptr<I420Buffer> buffer = + I420Buffer::Create(width, abs(height), stride_y, stride_uv, stride_uv); + + const int conversionResult = libyuv::ConvertToI420( + videoFrame, videoFrameLength, buffer->MutableDataY(), buffer->StrideY(), + buffer->MutableDataU(), buffer->StrideU(), buffer->MutableDataV(), + buffer->StrideV(), 0, 0, // No Cropping + aFrame->stride() / DesktopFrame::kBytesPerPixel, height, width, height, + libyuv::kRotate0, ConvertVideoType(frameInfo.videoType)); + if (conversionResult != 0) { + RTC_LOG(LS_ERROR) << "Failed to convert capture frame from type " + << static_cast<int>(frameInfo.videoType) << "to I420."; + return; + } + rec.Record(); + + NotifyOnFrame(VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_us(frameTime.us()) + .build()); + + const TimeDelta processTime = + Timestamp::Micros(rtc::TimeMicros()) - startProcessTime; + + if (processTime > TimeDelta::Millis(10)) { + RTC_LOG(LS_WARNING) + << "Too long processing time of incoming frame with dimensions " + << width << "x" << height << ": " << processTime.ms() << " ms"; + } +} + +void DesktopCaptureImpl::NotifyOnFrame(const VideoFrame& aFrame) { + RTC_DCHECK_RUN_ON(&mCaptureThreadChecker); + MOZ_ASSERT(Timestamp::Millis(aFrame.render_time_ms()) > + mNextFrameMinimumTime); + // Set the next frame's minimum time to ensure two consecutive frames don't + // have an identical render time (which is in milliseconds). + mNextFrameMinimumTime = + Timestamp::Millis(aFrame.render_time_ms()) + TimeDelta::Millis(1); + auto callbacks = mCallbacks.Lock(); + for (auto* cb : *callbacks) { + cb->OnFrame(aFrame); + } +} + +void DesktopCaptureImpl::InitOnThread( + std::unique_ptr<DesktopCapturer> aCapturer, int aFramerate) { + RTC_DCHECK_RUN_ON(&mCaptureThreadChecker); + + mCapturer = std::move(aCapturer); + + // We need to call Start on the same thread we call CaptureFrame on. + mCapturer->Start(this); + + mCaptureTimer = NS_NewTimer(); + mRequestedCaptureInterval = mozilla::Some( + TimeDuration::FromSeconds(1. / static_cast<double>(aFramerate))); + + CaptureFrameOnThread(); +} + +void DesktopCaptureImpl::ShutdownOnThread() { + RTC_DCHECK_RUN_ON(&mCaptureThreadChecker); + if (mCaptureTimer) { + mCaptureTimer->Cancel(); + mCaptureTimer = nullptr; + } + + // DesktopCapturer dtor blocks until fully shut down. TabCapturerWebrtc needs + // the capture thread to be alive. + mCapturer = nullptr; + + mRequestedCaptureInterval = mozilla::Nothing(); +} + +void DesktopCaptureImpl::CaptureFrameOnThread() { + RTC_DCHECK_RUN_ON(&mCaptureThreadChecker); + + auto start = mozilla::TimeStamp::Now(); + mCapturer->CaptureFrame(); + auto end = mozilla::TimeStamp::Now(); + + // Calculate next capture time. + const auto duration = end - start; + const auto timeUntilRequestedCapture = *mRequestedCaptureInterval - duration; + + // Use at most x% CPU or limit framerate + constexpr float sleepTimeFactor = + (100.0f / kMaxDesktopCaptureCpuUsage) - 1.0f; + static_assert(sleepTimeFactor >= 0.0); + static_assert(sleepTimeFactor < 100.0); + const auto sleepTime = duration.MultDouble(sleepTimeFactor); + + mCaptureTimer->InitHighResolutionWithNamedFuncCallback( + &::CaptureFrameOnThread, this, + std::max(timeUntilRequestedCapture, sleepTime), nsITimer::TYPE_ONE_SHOT, + "DesktopCaptureImpl::mCaptureTimer"); +} + +} // namespace webrtc diff --git a/dom/media/systemservices/video_engine/desktop_capture_impl.h b/dom/media/systemservices/video_engine/desktop_capture_impl.h new file mode 100644 index 0000000000..64eadb1401 --- /dev/null +++ b/dom/media/systemservices/video_engine/desktop_capture_impl.h @@ -0,0 +1,246 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_DESKTOP_CAPTURE_MAIN_SOURCE_DESKTOP_CAPTURE_IMPL_H_ +#define WEBRTC_MODULES_DESKTOP_CAPTURE_MAIN_SOURCE_DESKTOP_CAPTURE_IMPL_H_ + +/* + * video_capture_impl.h + */ + +#include <memory> +#include <set> +#include <string> + +#include "api/sequence_checker.h" +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "modules/desktop_capture/desktop_capturer.h" +#include "modules/video_capture/video_capture.h" + +#include "desktop_device_info.h" +#include "mozilla/DataMutex.h" +#include "mozilla/Maybe.h" +#include "mozilla/TimeStamp.h" +#include "nsCOMPtr.h" +#include "PerformanceRecorder.h" + +class nsIThread; +class nsITimer; + +namespace mozilla::camera { +enum class CaptureDeviceType; +} + +namespace webrtc { + +class VideoCaptureEncodeInterface; + +// simulate deviceInfo interface for video engine, bridge screen/application and +// real screen/application device info + +class ScreenDeviceInfoImpl : public VideoCaptureModule::DeviceInfo { + public: + ScreenDeviceInfoImpl(int32_t aId) : mId(aId) {} + virtual ~ScreenDeviceInfoImpl() = default; + + int32_t Init(); + int32_t Refresh(); + + virtual uint32_t NumberOfDevices(); + virtual int32_t GetDeviceName(uint32_t aDeviceNumber, char* aDeviceNameUTF8, + uint32_t aDeviceNameUTF8Size, + char* aDeviceUniqueIdUTF8, + uint32_t aDeviceUniqueIdUTF8Size, + char* aProductUniqueIdUTF8, + uint32_t aProductUniqueIdUTF8Size, pid_t* aPid); + + virtual int32_t DisplayCaptureSettingsDialogBox( + const char* aDeviceUniqueIdUTF8, const char* aDialogTitleUTF8, + void* aParentWindow, uint32_t aPositionX, uint32_t aPositionY); + virtual int32_t NumberOfCapabilities(const char* aDeviceUniqueIdUTF8); + virtual int32_t GetCapability(const char* aDeviceUniqueIdUTF8, + uint32_t aDeviceCapabilityNumber, + VideoCaptureCapability& aCapability); + + virtual int32_t GetBestMatchedCapability( + const char* aDeviceUniqueIdUTF8, const VideoCaptureCapability& aRequested, + VideoCaptureCapability& aResulting); + virtual int32_t GetOrientation(const char* aDeviceUniqueIdUTF8, + VideoRotation& aOrientation); + + protected: + int32_t mId; + std::unique_ptr<DesktopDeviceInfo> mDesktopDeviceInfo; +}; + +class WindowDeviceInfoImpl : public VideoCaptureModule::DeviceInfo { + public: + WindowDeviceInfoImpl(int32_t aId) : mId(aId){}; + virtual ~WindowDeviceInfoImpl() = default; + + int32_t Init(); + int32_t Refresh(); + + virtual uint32_t NumberOfDevices(); + virtual int32_t GetDeviceName(uint32_t aDeviceNumber, char* aDeviceNameUTF8, + uint32_t aDeviceNameUTF8Size, + char* aDeviceUniqueIdUTF8, + uint32_t aDeviceUniqueIdUTF8Size, + char* aProductUniqueIdUTF8, + uint32_t aProductUniqueIdUTF8Size, pid_t* aPid); + + virtual int32_t DisplayCaptureSettingsDialogBox( + const char* aDeviceUniqueIdUTF8, const char* aDialogTitleUTF8, + void* aParentWindow, uint32_t aPositionX, uint32_t aPositionY); + virtual int32_t NumberOfCapabilities(const char* aDeviceUniqueIdUTF8); + virtual int32_t GetCapability(const char* aDeviceUniqueIdUTF8, + uint32_t aDeviceCapabilityNumber, + VideoCaptureCapability& aCapability); + + virtual int32_t GetBestMatchedCapability( + const char* aDeviceUniqueIdUTF8, const VideoCaptureCapability& aRequested, + VideoCaptureCapability& aResulting); + virtual int32_t GetOrientation(const char* aDeviceUniqueIdUTF8, + VideoRotation& aOrientation); + + protected: + int32_t mId; + std::unique_ptr<DesktopDeviceInfo> mDesktopDeviceInfo; +}; + +class BrowserDeviceInfoImpl : public VideoCaptureModule::DeviceInfo { + public: + BrowserDeviceInfoImpl(int32_t aId) : mId(aId){}; + virtual ~BrowserDeviceInfoImpl() = default; + + int32_t Init(); + int32_t Refresh(); + + virtual uint32_t NumberOfDevices(); + virtual int32_t GetDeviceName(uint32_t aDeviceNumber, char* aDeviceNameUTF8, + uint32_t aDeviceNameUTF8Size, + char* aDeviceUniqueIdUTF8, + uint32_t aDeviceUniqueIdUTF8Size, + char* aProductUniqueIdUTF8, + uint32_t aProductUniqueIdUTF8Size, pid_t* aPid); + + virtual int32_t DisplayCaptureSettingsDialogBox( + const char* aDeviceUniqueIdUTF8, const char* aDialogTitleUTF8, + void* aParentWindow, uint32_t aPositionX, uint32_t aPositionY); + virtual int32_t NumberOfCapabilities(const char* aDeviceUniqueIdUTF8); + virtual int32_t GetCapability(const char* aDeviceUniqueIdUTF8, + uint32_t aDeviceCapabilityNumber, + VideoCaptureCapability& aCapability); + + virtual int32_t GetBestMatchedCapability( + const char* aDeviceUniqueIdUTF8, const VideoCaptureCapability& aRequested, + VideoCaptureCapability& aResulting); + virtual int32_t GetOrientation(const char* aDeviceUniqueIdUTF8, + VideoRotation& aOrientation); + + protected: + int32_t mId; + std::unique_ptr<DesktopDeviceInfo> mDesktopDeviceInfo; +}; + +// Reuses the video engine pipeline for screen sharing. +// As with video, DesktopCaptureImpl is a proxy for screen sharing +// and follows the video pipeline design +class DesktopCaptureImpl : public DesktopCapturer::Callback, + public VideoCaptureModule { + public: + /* Create a screen capture modules object + */ + static VideoCaptureModule* Create( + const int32_t aModuleId, const char* aUniqueId, + const mozilla::camera::CaptureDeviceType aType); + + [[nodiscard]] static std::shared_ptr<VideoCaptureModule::DeviceInfo> + CreateDeviceInfo(const int32_t aId, + const mozilla::camera::CaptureDeviceType aType); + + // mControlThread only. + void RegisterCaptureDataCallback( + rtc::VideoSinkInterface<VideoFrame>* aCallback) override; + void RegisterCaptureDataCallback( + RawVideoSinkInterface* dataCallback) override {} + void DeRegisterCaptureDataCallback( + rtc::VideoSinkInterface<VideoFrame>* aCallback) override; + int32_t StopCaptureIfAllClientsClose() override; + + int32_t SetCaptureRotation(VideoRotation aRotation) override; + bool SetApplyRotation(bool aEnable) override; + bool GetApplyRotation() override { return true; } + + const char* CurrentDeviceName() const override; + + int32_t StartCapture(const VideoCaptureCapability& aCapability) override; + virtual bool FocusOnSelectedSource() override; + int32_t StopCapture() override; + bool CaptureStarted() override; + int32_t CaptureSettings(VideoCaptureCapability& aSettings) override; + + void CaptureFrameOnThread(); + + const int32_t mModuleId; + const mozilla::TrackingId mTrackingId; + const std::string mDeviceUniqueId; + const mozilla::camera::CaptureDeviceType mDeviceType; + + protected: + DesktopCaptureImpl(const int32_t aId, const char* aUniqueId, + const mozilla::camera::CaptureDeviceType aType); + virtual ~DesktopCaptureImpl(); + + private: + // Maximum CPU usage in %. + static constexpr uint32_t kMaxDesktopCaptureCpuUsage = 50; + void InitOnThread(std::unique_ptr<DesktopCapturer> aCapturer, int aFramerate); + void ShutdownOnThread(); + // DesktopCapturer::Callback interface. + void OnCaptureResult(DesktopCapturer::Result aResult, + std::unique_ptr<DesktopFrame> aFrame) override; + + // Notifies all mCallbacks of OnFrame(). mCaptureThread only. + void NotifyOnFrame(const VideoFrame& aFrame); + + // Control thread on which the public API is called. + const nsCOMPtr<nsISerialEventTarget> mControlThread; + // Set in StartCapture. + mozilla::Maybe<VideoCaptureCapability> mRequestedCapability + RTC_GUARDED_BY(mControlThreadChecker); + // The DesktopCapturer is created on mControlThread but assigned and accessed + // only on mCaptureThread. + std::unique_ptr<DesktopCapturer> mCapturer + RTC_GUARDED_BY(mCaptureThreadChecker); + // Dedicated thread that does the capturing. + nsCOMPtr<nsIThread> mCaptureThread RTC_GUARDED_BY(mControlThreadChecker); + // Checks that API methods are called on mControlThread. + webrtc::SequenceChecker mControlThreadChecker; + // Checks that frame delivery only happens on mCaptureThread. + webrtc::SequenceChecker mCaptureThreadChecker; + // Timer that triggers frame captures. Only used on mCaptureThread. + // TODO(Bug 1806646): Drive capture with vsync instead. + nsCOMPtr<nsITimer> mCaptureTimer RTC_GUARDED_BY(mCaptureThreadChecker); + // Interval between captured frames, based on the framerate in + // mRequestedCapability. mCaptureThread only. + mozilla::Maybe<mozilla::TimeDuration> mRequestedCaptureInterval + RTC_GUARDED_BY(mCaptureThreadChecker); + // Used to make sure incoming timestamp is increasing for every frame. + webrtc::Timestamp mNextFrameMinimumTime RTC_GUARDED_BY(mCaptureThreadChecker); + // Callbacks for captured frames. Mutated on mControlThread, callbacks happen + // on mCaptureThread. + mozilla::DataMutex<std::set<rtc::VideoSinkInterface<VideoFrame>*>> mCallbacks; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_DESKTOP_CAPTURE_MAIN_SOURCE_DESKTOP_CAPTURE_IMPL_H_ diff --git a/dom/media/systemservices/video_engine/desktop_device_info.cc b/dom/media/systemservices/video_engine/desktop_device_info.cc new file mode 100644 index 0000000000..b3632a509f --- /dev/null +++ b/dom/media/systemservices/video_engine/desktop_device_info.cc @@ -0,0 +1,488 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "desktop_device_info.h" +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_capturer.h" +#include "mozilla/Sprintf.h" +#include "mozilla/StaticPrefs_media.h" +#include "mozilla/SyncRunnable.h" +#include "mozilla/UniquePtr.h" +#include "nsIBrowserWindowTracker.h" +#include "nsImportModule.h" + +#include <cstddef> +#include <cstdlib> +#include <cstdio> +#include <cstring> +#include <memory> + +namespace webrtc { + +static inline void SetStringMember(char** aMember, const char* aValue) { + if (!aValue) { + return; + } + + if (*aMember) { + delete[] * aMember; + *aMember = nullptr; + } + + size_t nBufLen = strlen(aValue) + 1; + char* buffer = new char[nBufLen]; + memcpy(buffer, aValue, nBufLen - 1); + buffer[nBufLen - 1] = '\0'; + *aMember = buffer; +} + +DesktopDisplayDevice::DesktopDisplayDevice() { + mScreenId = kInvalidScreenId; + mDeviceUniqueIdUTF8 = nullptr; + mDeviceNameUTF8 = nullptr; + mPid = 0; +} + +DesktopDisplayDevice::~DesktopDisplayDevice() { + mScreenId = kInvalidScreenId; + + delete[] mDeviceUniqueIdUTF8; + delete[] mDeviceNameUTF8; + + mDeviceUniqueIdUTF8 = nullptr; + mDeviceNameUTF8 = nullptr; +} + +void DesktopDisplayDevice::setScreenId(const ScreenId aScreenId) { + mScreenId = aScreenId; +} + +void DesktopDisplayDevice::setDeviceName(const char* aDeviceNameUTF8) { + SetStringMember(&mDeviceNameUTF8, aDeviceNameUTF8); +} + +void DesktopDisplayDevice::setUniqueIdName(const char* aDeviceUniqueIdUTF8) { + SetStringMember(&mDeviceUniqueIdUTF8, aDeviceUniqueIdUTF8); +} + +void DesktopDisplayDevice::setPid(const int aPid) { mPid = aPid; } + +ScreenId DesktopDisplayDevice::getScreenId() { return mScreenId; } + +const char* DesktopDisplayDevice::getDeviceName() { return mDeviceNameUTF8; } + +const char* DesktopDisplayDevice::getUniqueIdName() { + return mDeviceUniqueIdUTF8; +} + +pid_t DesktopDisplayDevice::getPid() { return mPid; } + +DesktopDisplayDevice& DesktopDisplayDevice::operator=( + DesktopDisplayDevice& aOther) { + if (&aOther == this) { + return *this; + } + mScreenId = aOther.getScreenId(); + setUniqueIdName(aOther.getUniqueIdName()); + setDeviceName(aOther.getDeviceName()); + mPid = aOther.getPid(); + + return *this; +} + +DesktopTab::DesktopTab() { + mTabBrowserId = 0; + mTabNameUTF8 = nullptr; + mTabUniqueIdUTF8 = nullptr; + mTabCount = 0; +} + +DesktopTab::~DesktopTab() { + delete[] mTabNameUTF8; + delete[] mTabUniqueIdUTF8; + + mTabNameUTF8 = nullptr; + mTabUniqueIdUTF8 = nullptr; +} + +void DesktopTab::setTabBrowserId(uint64_t aTabBrowserId) { + mTabBrowserId = aTabBrowserId; +} + +void DesktopTab::setUniqueIdName(const char* aTabUniqueIdUTF8) { + SetStringMember(&mTabUniqueIdUTF8, aTabUniqueIdUTF8); +} + +void DesktopTab::setTabName(const char* aTabNameUTF8) { + SetStringMember(&mTabNameUTF8, aTabNameUTF8); +} + +void DesktopTab::setTabCount(const uint32_t aCount) { mTabCount = aCount; } + +uint64_t DesktopTab::getTabBrowserId() { return mTabBrowserId; } + +const char* DesktopTab::getUniqueIdName() { return mTabUniqueIdUTF8; } + +const char* DesktopTab::getTabName() { return mTabNameUTF8; } + +uint32_t DesktopTab::getTabCount() { return mTabCount; } + +DesktopTab& DesktopTab::operator=(DesktopTab& aOther) { + mTabBrowserId = aOther.getTabBrowserId(); + setUniqueIdName(aOther.getUniqueIdName()); + setTabName(aOther.getTabName()); + + return *this; +} + +class DesktopDeviceInfoImpl : public DesktopDeviceInfo { + public: + DesktopDeviceInfoImpl(); + ~DesktopDeviceInfoImpl(); + + int32_t Init() override; + int32_t Refresh() override; + int32_t getDisplayDeviceCount() override; + int32_t getDesktopDisplayDeviceInfo( + uint32_t aIndex, DesktopDisplayDevice& aDesktopDisplayDevice) override; + int32_t getWindowCount() override; + int32_t getWindowInfo(uint32_t aIndex, + DesktopDisplayDevice& aWindowDevice) override; + uint32_t getTabCount() override; + int32_t getTabInfo(uint32_t aIndex, DesktopTab& aDesktopTab) override; + + protected: + DesktopDisplayDeviceList mDesktopDisplayList; + DesktopDisplayDeviceList mDesktopWindowList; + DesktopTabList mDesktopTabList; + + void CleanUp(); + void CleanUpWindowList(); + void CleanUpTabList(); + void CleanUpScreenList(); + + void InitializeWindowList(); + virtual void InitializeTabList(); + void InitializeScreenList(); + + void RefreshWindowList(); + void RefreshTabList(); + void RefreshScreenList(); + + void DummyTabList(DesktopTabList& aList); +}; + +DesktopDeviceInfoImpl::DesktopDeviceInfoImpl() = default; + +DesktopDeviceInfoImpl::~DesktopDeviceInfoImpl() { CleanUp(); } + +int32_t DesktopDeviceInfoImpl::getDisplayDeviceCount() { + return static_cast<int32_t>(mDesktopDisplayList.size()); +} + +int32_t DesktopDeviceInfoImpl::getDesktopDisplayDeviceInfo( + uint32_t aIndex, DesktopDisplayDevice& aDesktopDisplayDevice) { + if (aIndex >= mDesktopDisplayList.size()) { + return -1; + } + + std::map<intptr_t, DesktopDisplayDevice*>::iterator iter = + mDesktopDisplayList.begin(); + std::advance(iter, aIndex); + DesktopDisplayDevice* desktopDisplayDevice = iter->second; + if (desktopDisplayDevice) { + aDesktopDisplayDevice = (*desktopDisplayDevice); + } + + return 0; +} + +int32_t DesktopDeviceInfoImpl::getWindowCount() { + return static_cast<int32_t>(mDesktopWindowList.size()); +} + +int32_t DesktopDeviceInfoImpl::getWindowInfo( + uint32_t aIndex, DesktopDisplayDevice& aWindowDevice) { + if (aIndex >= mDesktopWindowList.size()) { + return -1; + } + + std::map<intptr_t, DesktopDisplayDevice*>::iterator itr = + mDesktopWindowList.begin(); + std::advance(itr, aIndex); + DesktopDisplayDevice* window = itr->second; + if (!window) { + return -1; + } + + aWindowDevice = (*window); + return 0; +} + +uint32_t DesktopDeviceInfoImpl::getTabCount() { return mDesktopTabList.size(); } + +int32_t DesktopDeviceInfoImpl::getTabInfo(uint32_t aIndex, + DesktopTab& aDesktopTab) { + if (aIndex >= mDesktopTabList.size()) { + return -1; + } + + std::map<intptr_t, DesktopTab*>::iterator iter = mDesktopTabList.begin(); + std::advance(iter, aIndex); + DesktopTab* desktopTab = iter->second; + if (desktopTab) { + aDesktopTab = (*desktopTab); + } + + return 0; +} + +void DesktopDeviceInfoImpl::CleanUp() { + CleanUpScreenList(); + CleanUpWindowList(); + CleanUpTabList(); +} +int32_t DesktopDeviceInfoImpl::Init() { + InitializeScreenList(); + InitializeWindowList(); + InitializeTabList(); + + return 0; +} +int32_t DesktopDeviceInfoImpl::Refresh() { + RefreshScreenList(); + RefreshWindowList(); + RefreshTabList(); + + return 0; +} + +void DesktopDeviceInfoImpl::CleanUpWindowList() { + std::map<intptr_t, DesktopDisplayDevice*>::iterator iterWindow; + for (iterWindow = mDesktopWindowList.begin(); + iterWindow != mDesktopWindowList.end(); iterWindow++) { + DesktopDisplayDevice* aWindow = iterWindow->second; + delete aWindow; + iterWindow->second = nullptr; + } + mDesktopWindowList.clear(); +} + +void DesktopDeviceInfoImpl::InitializeWindowList() { + DesktopCaptureOptions options; + +// Wayland is special and we will not get any information about windows +// without going through xdg-desktop-portal. We will already have +// a screen placeholder so there is no reason to build windows list. +#if defined(WEBRTC_USE_PIPEWIRE) + if (mozilla::StaticPrefs::media_webrtc_capture_allow_pipewire() && + webrtc::DesktopCapturer::IsRunningUnderWayland()) { + return; + } +#endif + +// Help avoid an X11 deadlock, see bug 1456101. +#ifdef MOZ_X11 + MOZ_ALWAYS_SUCCEEDS(mozilla::SyncRunnable::DispatchToThread( + mozilla::GetMainThreadSerialEventTarget(), + NS_NewRunnableFunction(__func__, [&] { + options = DesktopCaptureOptions::CreateDefault(); + }))); +#else + options = DesktopCaptureOptions::CreateDefault(); +#endif + std::unique_ptr<DesktopCapturer> winCap = + DesktopCapturer::CreateWindowCapturer(options); + DesktopCapturer::SourceList list; + if (winCap && winCap->GetSourceList(&list)) { + DesktopCapturer::SourceList::iterator itr; + for (itr = list.begin(); itr != list.end(); itr++) { + DesktopDisplayDevice* winDevice = new DesktopDisplayDevice; + if (!winDevice) { + continue; + } + + winDevice->setScreenId(itr->id); + winDevice->setDeviceName(itr->title.c_str()); + winDevice->setPid(itr->pid); + + char idStr[BUFSIZ]; +#if WEBRTC_WIN + _snprintf_s(idStr, sizeof(idStr), sizeof(idStr) - 1, "%ld", + static_cast<long>(winDevice->getScreenId())); +#else + SprintfLiteral(idStr, "%ld", static_cast<long>(winDevice->getScreenId())); +#endif + winDevice->setUniqueIdName(idStr); + mDesktopWindowList[winDevice->getScreenId()] = winDevice; + } + } +} + +void DesktopDeviceInfoImpl::RefreshWindowList() { + CleanUpWindowList(); + InitializeWindowList(); +} + +void DesktopDeviceInfoImpl::CleanUpTabList() { + for (auto& iterTab : mDesktopTabList) { + DesktopTab* desktopTab = iterTab.second; + delete desktopTab; + iterTab.second = nullptr; + } + mDesktopTabList.clear(); +} + +void webrtc::DesktopDeviceInfoImpl::InitializeTabList() { + if (!mozilla::StaticPrefs::media_getusermedia_browser_enabled()) { + return; + } + + // This is a sync dispatch to main thread, which is unfortunate. To + // call JavaScript we have to be on main thread, but the remaining + // DesktopCapturer very much wants to be off main thread. This might + // be solvable by calling this method earlier on while we're still on + // main thread and plumbing the information down to here. + nsCOMPtr<nsIRunnable> runnable = NS_NewRunnableFunction(__func__, [&] { + nsresult rv; + nsCOMPtr<nsIBrowserWindowTracker> bwt = + do_ImportModule("resource:///modules/BrowserWindowTracker.jsm", + "BrowserWindowTracker", &rv); + if (NS_FAILED(rv)) { + return; + } + + nsTArray<RefPtr<nsIVisibleTab>> tabArray; + rv = bwt->GetAllVisibleTabs(tabArray); + if (NS_FAILED(rv)) { + return; + } + + for (const auto& browserTab : tabArray) { + nsString contentTitle; + browserTab->GetContentTitle(contentTitle); + int64_t browserId; + browserTab->GetBrowserId(&browserId); + + DesktopTab* desktopTab = new DesktopTab; + if (desktopTab) { + char* contentTitleUTF8 = ToNewUTF8String(contentTitle); + desktopTab->setTabBrowserId(browserId); + desktopTab->setTabName(contentTitleUTF8); + std::ostringstream uniqueId; + uniqueId << browserId; + desktopTab->setUniqueIdName(uniqueId.str().c_str()); + mDesktopTabList[static_cast<intptr_t>(desktopTab->getTabBrowserId())] = + desktopTab; + free(contentTitleUTF8); + } + } + }); + mozilla::SyncRunnable::DispatchToThread( + mozilla::GetMainThreadSerialEventTarget(), runnable); +} + +void DesktopDeviceInfoImpl::RefreshTabList() { + CleanUpTabList(); + InitializeTabList(); +} + +void DesktopDeviceInfoImpl::CleanUpScreenList() { + std::map<intptr_t, DesktopDisplayDevice*>::iterator iterDevice; + for (iterDevice = mDesktopDisplayList.begin(); + iterDevice != mDesktopDisplayList.end(); iterDevice++) { + DesktopDisplayDevice* desktopDisplayDevice = iterDevice->second; + delete desktopDisplayDevice; + iterDevice->second = nullptr; + } + mDesktopDisplayList.clear(); +} + +// With PipeWire we can't select which system resource is shared so +// we don't create a window/screen list. Instead we place these constants +// as window name/id so frontend code can identify PipeWire backend +// and does not try to create screen/window preview. + +#define PIPEWIRE_ID 0xaffffff +#define PIPEWIRE_NAME "####_PIPEWIRE_PORTAL_####" + +void DesktopDeviceInfoImpl::InitializeScreenList() { + DesktopCaptureOptions options; + +// Wayland is special and we will not get any information about screens +// without going through xdg-desktop-portal so we just need a screen +// placeholder. +#if defined(WEBRTC_USE_PIPEWIRE) + if (mozilla::StaticPrefs::media_webrtc_capture_allow_pipewire() && + webrtc::DesktopCapturer::IsRunningUnderWayland()) { + DesktopDisplayDevice* screenDevice = new DesktopDisplayDevice; + if (!screenDevice) { + return; + } + + screenDevice->setScreenId(PIPEWIRE_ID); + screenDevice->setDeviceName(PIPEWIRE_NAME); + + char idStr[BUFSIZ]; + SprintfLiteral(idStr, "%ld", + static_cast<long>(screenDevice->getScreenId())); + screenDevice->setUniqueIdName(idStr); + mDesktopDisplayList[screenDevice->getScreenId()] = screenDevice; + return; + } +#endif + +// Help avoid an X11 deadlock, see bug 1456101. +#ifdef MOZ_X11 + MOZ_ALWAYS_SUCCEEDS(mozilla::SyncRunnable::DispatchToThread( + mozilla::GetMainThreadSerialEventTarget(), + NS_NewRunnableFunction(__func__, [&] { + options = DesktopCaptureOptions::CreateDefault(); + }))); +#else + options = DesktopCaptureOptions::CreateDefault(); +#endif + std::unique_ptr<DesktopCapturer> screenCapturer = + DesktopCapturer::CreateScreenCapturer(options); + DesktopCapturer::SourceList list; + if (screenCapturer && screenCapturer->GetSourceList(&list)) { + DesktopCapturer::SourceList::iterator itr; + for (itr = list.begin(); itr != list.end(); itr++) { + DesktopDisplayDevice* screenDevice = new DesktopDisplayDevice; + screenDevice->setScreenId(itr->id); + if (list.size() == 1) { + screenDevice->setDeviceName("Primary Monitor"); + } else { + screenDevice->setDeviceName(itr->title.c_str()); + } + screenDevice->setPid(itr->pid); + + char idStr[BUFSIZ]; +#if WEBRTC_WIN + _snprintf_s(idStr, sizeof(idStr), sizeof(idStr) - 1, "%ld", + static_cast<long>(screenDevice->getScreenId())); +#else + SprintfLiteral(idStr, "%ld", + static_cast<long>(screenDevice->getScreenId())); +#endif + screenDevice->setUniqueIdName(idStr); + mDesktopDisplayList[screenDevice->getScreenId()] = screenDevice; + } + } +} + +void DesktopDeviceInfoImpl::RefreshScreenList() { + CleanUpScreenList(); + InitializeScreenList(); +} + +/* static */ +DesktopDeviceInfo* DesktopDeviceInfo::Create() { + auto info = mozilla::MakeUnique<DesktopDeviceInfoImpl>(); + if (info->Init() != 0) { + return nullptr; + } + return info.release(); +} +} // namespace webrtc diff --git a/dom/media/systemservices/video_engine/desktop_device_info.h b/dom/media/systemservices/video_engine/desktop_device_info.h new file mode 100644 index 0000000000..824792b3c0 --- /dev/null +++ b/dom/media/systemservices/video_engine/desktop_device_info.h @@ -0,0 +1,84 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef WEBRTC_MODULES_DESKTOP_CAPTURE_DEVICE_INFO_H_ +#define WEBRTC_MODULES_DESKTOP_CAPTURE_DEVICE_INFO_H_ + +#include <map> +#include "modules/desktop_capture/desktop_capture_types.h" + +namespace webrtc { + +class DesktopDisplayDevice { + public: + DesktopDisplayDevice(); + ~DesktopDisplayDevice(); + + void setScreenId(const ScreenId aScreenId); + void setDeviceName(const char* aDeviceNameUTF8); + void setUniqueIdName(const char* aDeviceUniqueIdUTF8); + void setPid(pid_t aPid); + + ScreenId getScreenId(); + const char* getDeviceName(); + const char* getUniqueIdName(); + pid_t getPid(); + + DesktopDisplayDevice& operator=(DesktopDisplayDevice& aOther); + + protected: + ScreenId mScreenId; + char* mDeviceNameUTF8; + char* mDeviceUniqueIdUTF8; + pid_t mPid; +}; + +using DesktopDisplayDeviceList = std::map<intptr_t, DesktopDisplayDevice*>; + +class DesktopTab { + public: + DesktopTab(); + ~DesktopTab(); + + void setTabBrowserId(uint64_t aTabBrowserId); + void setUniqueIdName(const char* aTabUniqueIdUTF8); + void setTabName(const char* aTabNameUTF8); + void setTabCount(const uint32_t aCount); + + uint64_t getTabBrowserId(); + const char* getUniqueIdName(); + const char* getTabName(); + uint32_t getTabCount(); + + DesktopTab& operator=(DesktopTab& aOther); + + protected: + uint64_t mTabBrowserId; + char* mTabNameUTF8; + char* mTabUniqueIdUTF8; + uint32_t mTabCount; +}; + +using DesktopTabList = std::map<intptr_t, DesktopTab*>; + +class DesktopDeviceInfo { + public: + virtual ~DesktopDeviceInfo() = default; + + virtual int32_t Init() = 0; + virtual int32_t Refresh() = 0; + virtual int32_t getDisplayDeviceCount() = 0; + virtual int32_t getDesktopDisplayDeviceInfo( + uint32_t aIndex, DesktopDisplayDevice& aDesktopDisplayDevice) = 0; + virtual int32_t getWindowCount() = 0; + virtual int32_t getWindowInfo(uint32_t aIndex, + DesktopDisplayDevice& aWindowDevice) = 0; + virtual uint32_t getTabCount() = 0; + virtual int32_t getTabInfo(uint32_t aIndex, DesktopTab& aDesktopTab) = 0; + + static DesktopDeviceInfo* Create(); +}; +}; // namespace webrtc + +#endif diff --git a/dom/media/systemservices/video_engine/platform_uithread.cc b/dom/media/systemservices/video_engine/platform_uithread.cc new file mode 100644 index 0000000000..701a989a18 --- /dev/null +++ b/dom/media/systemservices/video_engine/platform_uithread.cc @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#if defined(WEBRTC_WIN) + +# include "platform_uithread.h" + +namespace rtc { + +// timer id used in delayed callbacks +static const UINT_PTR kTimerId = 1; +static const wchar_t kThisProperty[] = L"ThreadWindowsUIPtr"; +static const wchar_t kThreadWindow[] = L"WebrtcWindowsUIThread"; + +PlatformUIThread::~PlatformUIThread() { + CritScope scoped_lock(&cs_); + switch (state_) { + case State::STARTED: { + MOZ_DIAGNOSTIC_ASSERT( + false, "PlatformUIThread must be stopped before destruction"); + break; + } + case State::STOPPED: + break; + case State::UNSTARTED: + break; + } +} + +bool PlatformUIThread::InternalInit() { + // Create an event window for use in generating callbacks to capture + // objects. + CritScope scoped_lock(&cs_); + switch (state_) { + // We have already started there is nothing todo. Should this be assert? + case State::STARTED: + break; + // Stop() has already been called so there is likewise nothing to do. + case State::STOPPED: + break; + // Stop() has not been called yet, setup the UI thread, and set our + // state to STARTED. + case State::UNSTARTED: { + WNDCLASSW wc; + HMODULE hModule = GetModuleHandle(NULL); + if (!GetClassInfoW(hModule, kThreadWindow, &wc)) { + ZeroMemory(&wc, sizeof(WNDCLASSW)); + wc.hInstance = hModule; + wc.lpfnWndProc = EventWindowProc; + wc.lpszClassName = kThreadWindow; + RegisterClassW(&wc); + } + hwnd_ = CreateWindowW(kThreadWindow, L"", 0, 0, 0, 0, 0, NULL, NULL, + hModule, NULL); + // Added in review of bug 1760843, follow up to remove 1767861 + MOZ_RELEASE_ASSERT(hwnd_); + // Expected to always work but if it doesn't we should still fulfill the + // contract of always running the process loop at least a single + // iteration. + // This could be rexamined in the future. + if (hwnd_) { + SetPropW(hwnd_, kThisProperty, this); + // state_ needs to be STARTED before we request the initial timer + state_ = State::STARTED; + if (timeout_) { + // if someone set the timer before we started + RequestCallbackTimer(timeout_); + } + } + break; + } + }; + return state_ == State::STARTED; +} + +bool PlatformUIThread::RequestCallbackTimer(unsigned int milliseconds) { + CritScope scoped_lock(&cs_); + + switch (state_) { + // InternalInit() has yet to run so we do not have a UI thread to use as a + // target of the timer. We should just remember what timer interval was + // requested and let InternalInit() call this function again when it is + // ready. + case State::UNSTARTED: { + timeout_ = milliseconds; + return false; + } + // We have already stopped, do not schedule a new timer. + case State::STOPPED: + return false; + case State::STARTED: { + if (timerid_) { + KillTimer(hwnd_, timerid_); + } + timeout_ = milliseconds; + timerid_ = SetTimer(hwnd_, kTimerId, milliseconds, NULL); + return !!timerid_; + } + } + // UNREACHABLE +} + +void PlatformUIThread::Stop() { + { + RTC_DCHECK_RUN_ON(&thread_checker_); + CritScope scoped_lock(&cs_); + // Shut down the dispatch loop and let the background thread exit. + if (timerid_) { + MOZ_ASSERT(hwnd_); + KillTimer(hwnd_, timerid_); + timerid_ = 0; + } + switch (state_) { + // If we haven't started yet there is nothing to do, we will go into + // the STOPPED state at the end of the function and InternalInit() + // will not move us to STARTED. + case State::UNSTARTED: + break; + // If we have started, that means that InternalInit() has run and the + // message wait loop has or will run. We need to signal it to stop. wich + // will allow PlatformThread::Stop to join that thread. + case State::STARTED: { + MOZ_ASSERT(hwnd_); + PostMessage(hwnd_, WM_CLOSE, 0, 0); + break; + } + // We have already stopped. There is nothing to do. + case State::STOPPED: + break; + } + // Always set our state to STOPPED + state_ = State::STOPPED; + } + monitor_thread_.Finalize(); +} + +void PlatformUIThread::Run() { + // InternalInit() will return false when the thread is already in shutdown. + // otherwise we must run until we get a Windows WM_QUIT msg. + const bool runUntilQuitMsg = InternalInit(); + // The interface contract of Start/Stop is that for a successful call to + // Start, there should be at least one call to the run function. + NativeEventCallback(); + while (runUntilQuitMsg) { + // Alertable sleep to receive WM_QUIT (following a WM_CLOSE triggering a + // WM_DESTROY) + if (MsgWaitForMultipleObjectsEx(0, nullptr, INFINITE, QS_ALLINPUT, + MWMO_ALERTABLE | MWMO_INPUTAVAILABLE) == + WAIT_OBJECT_0) { + MSG msg; + if (PeekMessage(&msg, NULL, 0, 0, PM_REMOVE)) { + if (msg.message == WM_QUIT) { + // THE ONLY WAY to exit the thread loop + break; + } + TranslateMessage(&msg); + DispatchMessage(&msg); + } + } + } +} + +void PlatformUIThread::NativeEventCallback() { native_event_callback_(); } + +/* static */ +LRESULT CALLBACK PlatformUIThread::EventWindowProc(HWND hwnd, UINT uMsg, + WPARAM wParam, + LPARAM lParam) { + if (uMsg == WM_DESTROY) { + RemovePropW(hwnd, kThisProperty); + PostQuitMessage(0); + return 0; + } + + PlatformUIThread* twui = + static_cast<PlatformUIThread*>(GetPropW(hwnd, kThisProperty)); + if (!twui) { + return DefWindowProc(hwnd, uMsg, wParam, lParam); + } + + if (uMsg == WM_TIMER && wParam == kTimerId) { + twui->NativeEventCallback(); + return 0; + } + + return DefWindowProc(hwnd, uMsg, wParam, lParam); +} + +} // namespace rtc + +#endif diff --git a/dom/media/systemservices/video_engine/platform_uithread.h b/dom/media/systemservices/video_engine/platform_uithread.h new file mode 100644 index 0000000000..9c213ca933 --- /dev/null +++ b/dom/media/systemservices/video_engine/platform_uithread.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_PLATFORM_UITHREAD_H_ +#define RTC_BASE_PLATFORM_UITHREAD_H_ + +#if defined(WEBRTC_WIN) +# include "Assertions.h" +# include "rtc_base/deprecated/recursive_critical_section.h" +# include "rtc_base/platform_thread.h" +# include "api/sequence_checker.h" +# include "ThreadSafety.h" + +namespace rtc { +/* + * Windows UI thread for screen capture + * Launches a thread which enters a message wait loop after calling the + * provided ThreadRunFunction once. A repeating timer event might be registered + * with a callback through the Win32 API. If so, that timer will cause WM_TIMER + * messages to appear in the threads message queue. This will wake the thread + * which will then first look to see if it received the WM_QUIT message, then + * it will pass any non WM_QUIT messages on to the registered message handlers + * (synchronously on the current thread). In the case oF WM_TIMER the + * registered handler calls the NativeEventCallback which is simply the + * ThreadRunFunction which was passed to the constructor. + * + * Shutdown of the message wait loop is triggered by sending a WM_CLOSE which + * will start tearing down the "window" which hosts the UI thread. This will + * cause a WM_DESTROY message to be received. Upon reception a WM_QUIT message + * is enqueued. When the message wait loop receives a WM_QUIT message it stops, + * thus allowing the thread to be joined. + * + * Note: that the only source of a WM_CLOSE should be PlatformUIThread::Stop. + * Note: because PlatformUIThread::Stop is called from a different thread than + * PlatformUIThread::Run, it is possible that Stop can race Run. + * + * After being stopped PlatformUIThread can not be started again. + * + */ + +class PlatformUIThread { + public: + PlatformUIThread(std::function<void()> func, const char* name, + ThreadAttributes attributes) + : name_(name), + native_event_callback_(std::move(func)), + monitor_thread_(PlatformThread::SpawnJoinable([this]() { Run(); }, name, + attributes)) {} + + virtual ~PlatformUIThread(); + + void Stop(); + + /** + * Request a recurring callback. + */ + bool RequestCallbackTimer(unsigned int milliseconds); + + protected: + void Run(); + + private: + static LRESULT CALLBACK EventWindowProc(HWND, UINT, WPARAM, LPARAM); + void NativeEventCallback(); + // Initialize the UI thread that is servicing the timer events + bool InternalInit(); + + // Needs to be initialized before monitor_thread_ as it takes a string view to + // name_ + std::string name_; + RecursiveCriticalSection cs_; + std::function<void()> native_event_callback_; + webrtc::SequenceChecker thread_checker_; + PlatformThread monitor_thread_; + HWND hwnd_ MOZ_GUARDED_BY(cs_) = nullptr; + UINT_PTR timerid_ MOZ_GUARDED_BY(cs_) = 0; + unsigned int timeout_ MOZ_GUARDED_BY(cs_) = 0; + enum class State { + UNSTARTED, + STARTED, + STOPPED, + }; + State state_ MOZ_GUARDED_BY(cs_) = State::UNSTARTED; +}; + +} // namespace rtc + +#endif +#endif // RTC_BASE_PLATFORM_UITHREAD_H_ diff --git a/dom/media/systemservices/video_engine/tab_capturer.cc b/dom/media/systemservices/video_engine/tab_capturer.cc new file mode 100644 index 0000000000..793d965028 --- /dev/null +++ b/dom/media/systemservices/video_engine/tab_capturer.cc @@ -0,0 +1,331 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "tab_capturer.h" + +#include "desktop_device_info.h" +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_frame.h" +#include "mozilla/Logging.h" +#include "mozilla/dom/BrowsingContext.h" +#include "mozilla/dom/ImageBitmap.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/PromiseNativeHandler.h" +#include "mozilla/dom/WindowGlobalParent.h" +#include "mozilla/gfx/2D.h" +#include "mozilla/SpinEventLoopUntil.h" +#include "mozilla/TaskQueue.h" +#include "nsThreadUtils.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +mozilla::LazyLogModule gTabShareLog("TabShare"); +#define LOG_FUNC_IMPL(level) \ + MOZ_LOG( \ + gTabShareLog, level, \ + ("TabCapturerWebrtc %p: %s id=%" PRIu64, this, __func__, mBrowserId)) +#define LOG_FUNC() LOG_FUNC_IMPL(LogLevel::Debug) +#define LOG_FUNCV() LOG_FUNC_IMPL(LogLevel::Verbose) + +using namespace mozilla::dom; + +namespace mozilla { + +class CaptureFrameRequest { + using CapturePromise = TabCapturerWebrtc::CapturePromise; + + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(CaptureFrameRequest) + + CaptureFrameRequest() : mCaptureTime(TimeStamp::Now()) {} + + operator MozPromiseRequestHolder<CapturePromise>&() { return mRequest; } + + void Complete() { mRequest.Complete(); } + void Disconnect() { mRequest.Disconnect(); } + bool Exists() { return mRequest.Exists(); } + + protected: + virtual ~CaptureFrameRequest() { MOZ_RELEASE_ASSERT(!Exists()); } + + public: + const TimeStamp mCaptureTime; + + private: + MozPromiseRequestHolder<CapturePromise> mRequest; +}; + +TabCapturerWebrtc::TabCapturerWebrtc( + SourceId aSourceId, nsCOMPtr<nsISerialEventTarget> aCaptureThread) + : mBrowserId(aSourceId), + mMainThreadWorker( + TaskQueue::Create(do_AddRef(GetMainThreadSerialEventTarget()), + "TabCapturerWebrtc::mMainThreadWorker")), + mCallbackWorker(TaskQueue::Create(aCaptureThread.forget(), + "TabCapturerWebrtc::mCallbackWorker")) { + RTC_DCHECK_RUN_ON(&mControlChecker); + MOZ_ASSERT(aSourceId != 0); + mCallbackChecker.Detach(); + + LOG_FUNC(); +} + +// static +std::unique_ptr<webrtc::DesktopCapturer> TabCapturerWebrtc::Create( + SourceId aSourceId, nsCOMPtr<nsISerialEventTarget> aCaptureThread) { + return std::unique_ptr<webrtc::DesktopCapturer>( + new TabCapturerWebrtc(aSourceId, std::move(aCaptureThread))); +} + +TabCapturerWebrtc::~TabCapturerWebrtc() { + RTC_DCHECK_RUN_ON(&mCallbackChecker); + LOG_FUNC(); + + // mMainThreadWorker handles frame capture requests async. Since we're in the + // dtor, no more frame capture requests can be made through CaptureFrame(). It + // can be shut down now. + mMainThreadWorker->BeginShutdown(); + + // There may still be async frame capture requests in flight, waiting to be + // reported to mCallback on mCallbackWorker. Disconnect them (must be done on + // mCallbackWorker) and shut down mCallbackWorker to ensure nothing more can + // get queued to it. + MOZ_ALWAYS_SUCCEEDS( + mCallbackWorker->Dispatch(NS_NewRunnableFunction(__func__, [this] { + RTC_DCHECK_RUN_ON(&mCallbackChecker); + for (const auto& req : mRequests) { + DisconnectRequest(req); + } + mCallbackWorker->BeginShutdown(); + }))); + + // Block until the workers have run all pending tasks. We must do this for two + // reasons: + // - All runnables dispatched to mMainThreadWorker and mCallbackWorker capture + // the raw pointer `this` as they rely on `this` outliving the worker + // TaskQueues. + // - mCallback is only guaranteed to outlive `this`. No calls can be made to + // it after the dtor is finished. + + // Spin the underlying thread of mCallbackWorker, which we are currently on, + // until it is empty. We have no other way of waiting for mCallbackWorker to + // become empty while blocking the current call. + SpinEventLoopUntil<ProcessFailureBehavior::IgnoreAndContinue>( + "~TabCapturerWebrtc"_ns, [&] { return mCallbackWorker->IsEmpty(); }); + + // No need to await shutdown since it was shut down synchronously above. + mMainThreadWorker->AwaitIdle(); +} + +bool TabCapturerWebrtc::GetSourceList( + webrtc::DesktopCapturer::SourceList* aSources) { + MOZ_LOG(gTabShareLog, LogLevel::Debug, + ("TabShare: GetSourceList, result %zu", aSources->size())); + // XXX UI + return true; +} + +bool TabCapturerWebrtc::SelectSource(webrtc::DesktopCapturer::SourceId) { + MOZ_ASSERT_UNREACHABLE("Source is passed through ctor for constness"); + return true; +} + +bool TabCapturerWebrtc::FocusOnSelectedSource() { return true; } + +void TabCapturerWebrtc::Start(webrtc::DesktopCapturer::Callback* aCallback) { + RTC_DCHECK_RUN_ON(&mCallbackChecker); + RTC_DCHECK(!mCallback); + RTC_DCHECK(aCallback); + + LOG_FUNC(); + + mCallback = aCallback; +} + +void TabCapturerWebrtc::CaptureFrame() { + RTC_DCHECK_RUN_ON(&mCallbackChecker); + LOG_FUNCV(); + if (mRequests.GetSize() > 2) { + // Allow two async capture requests in flight + OnCaptureFrameFailure(); + return; + } + + auto request = MakeRefPtr<CaptureFrameRequest>(); + InvokeAsync(mMainThreadWorker, __func__, [this] { return CaptureFrameNow(); }) + ->Then(mCallbackWorker, __func__, + [this, request](CapturePromise::ResolveOrRejectValue&& aValue) { + if (!CompleteRequest(request)) { + // Request was disconnected or overrun. Failure has already + // been reported to the callback elsewhere. + return; + } + + if (aValue.IsReject()) { + OnCaptureFrameFailure(); + return; + } + + OnCaptureFrameSuccess(std::move(aValue.ResolveValue())); + }) + ->Track(*request); + mRequests.PushFront(request.forget()); +} + +void TabCapturerWebrtc::OnCaptureFrameSuccess( + UniquePtr<dom::ImageBitmapCloneData> aData) { + RTC_DCHECK_RUN_ON(&mCallbackChecker); + MOZ_DIAGNOSTIC_ASSERT(aData); + LOG_FUNCV(); + webrtc::DesktopSize size(aData->mPictureRect.Width(), + aData->mPictureRect.Height()); + webrtc::DesktopRect rect = webrtc::DesktopRect::MakeSize(size); + std::unique_ptr<webrtc::DesktopFrame> frame( + new webrtc::BasicDesktopFrame(size)); + + gfx::DataSourceSurface::ScopedMap map(aData->mSurface, + gfx::DataSourceSurface::READ); + if (!map.IsMapped()) { + OnCaptureFrameFailure(); + return; + } + frame->CopyPixelsFrom(map.GetData(), map.GetStride(), rect); + + mCallback->OnCaptureResult(webrtc::DesktopCapturer::Result::SUCCESS, + std::move(frame)); +} + +void TabCapturerWebrtc::OnCaptureFrameFailure() { + RTC_DCHECK_RUN_ON(&mCallbackChecker); + LOG_FUNC(); + mCallback->OnCaptureResult(webrtc::DesktopCapturer::Result::ERROR_TEMPORARY, + nullptr); +} + +bool TabCapturerWebrtc::IsOccluded(const webrtc::DesktopVector& aPos) { + return false; +} + +class TabCapturedHandler final : public PromiseNativeHandler { + public: + NS_DECL_ISUPPORTS + + using CapturePromise = TabCapturerWebrtc::CapturePromise; + + static void Create(Promise* aPromise, + MozPromiseHolder<CapturePromise> aHolder) { + MOZ_ASSERT(aPromise); + MOZ_ASSERT(NS_IsMainThread()); + + RefPtr<TabCapturedHandler> handler = + new TabCapturedHandler(std::move(aHolder)); + aPromise->AppendNativeHandler(handler); + } + + void ResolvedCallback(JSContext* aCx, JS::Handle<JS::Value> aValue, + ErrorResult& aRv) override { + MOZ_ASSERT(NS_IsMainThread()); + if (NS_WARN_IF(!aValue.isObject())) { + mHolder.Reject(NS_ERROR_UNEXPECTED, __func__); + return; + } + + RefPtr<ImageBitmap> bitmap; + if (NS_WARN_IF(NS_FAILED( + UNWRAP_OBJECT(ImageBitmap, &aValue.toObject(), bitmap)))) { + mHolder.Reject(NS_ERROR_UNEXPECTED, __func__); + return; + } + + UniquePtr<ImageBitmapCloneData> data = bitmap->ToCloneData(); + if (!data) { + mHolder.Reject(NS_ERROR_UNEXPECTED, __func__); + return; + } + + mHolder.Resolve(std::move(data), __func__); + } + + void RejectedCallback(JSContext* aCx, JS::Handle<JS::Value> aValue, + ErrorResult& aRv) override { + MOZ_ASSERT(NS_IsMainThread()); + mHolder.Reject(aRv.StealNSResult(), __func__); + } + + private: + explicit TabCapturedHandler(MozPromiseHolder<CapturePromise> aHolder) + : mHolder(std::move(aHolder)) {} + + ~TabCapturedHandler() = default; + + MozPromiseHolder<CapturePromise> mHolder; +}; + +NS_IMPL_ISUPPORTS0(TabCapturedHandler) + +bool TabCapturerWebrtc::CompleteRequest(CaptureFrameRequest* aRequest) { + RTC_DCHECK_RUN_ON(&mCallbackChecker); + if (!aRequest->Exists()) { + // Request was disconnected or overrun. mCallback has already been notified. + return false; + } + while (CaptureFrameRequest* req = mRequests.Peek()) { + if (req->mCaptureTime > aRequest->mCaptureTime) { + break; + } + // Pop the request before calling the callback, in case it could mutate + // mRequests, now or in the future. + RefPtr<CaptureFrameRequest> dropMe = mRequests.Pop(); + req->Complete(); + if (req->mCaptureTime < aRequest->mCaptureTime) { + OnCaptureFrameFailure(); + } + } + MOZ_DIAGNOSTIC_ASSERT(!aRequest->Exists()); + return true; +} + +void TabCapturerWebrtc::DisconnectRequest(CaptureFrameRequest* aRequest) { + RTC_DCHECK_RUN_ON(&mCallbackChecker); + LOG_FUNCV(); + aRequest->Disconnect(); + OnCaptureFrameFailure(); +} + +auto TabCapturerWebrtc::CaptureFrameNow() -> RefPtr<CapturePromise> { + MOZ_ASSERT(mMainThreadWorker->IsOnCurrentThread()); + LOG_FUNCV(); + + WindowGlobalParent* wgp = nullptr; + RefPtr<BrowsingContext> context = + BrowsingContext::GetCurrentTopByBrowserId(mBrowserId); + if (context) { + wgp = context->Canonical()->GetCurrentWindowGlobal(); + } + if (!wgp) { + // If we can't access the window, we just won't capture anything + return CapturePromise::CreateAndReject(NS_ERROR_NOT_AVAILABLE, __func__); + } + + // XXX This would be more efficient if we used CrossProcessPaint directly and + // returned a surface. + RefPtr<Promise> promise = + wgp->DrawSnapshot(nullptr, 1.0, "white"_ns, false, IgnoreErrors()); + if (!promise) { + return CapturePromise::CreateAndReject(NS_ERROR_FAILURE, __func__); + } + + MozPromiseHolder<CapturePromise> holder; + RefPtr<CapturePromise> p = holder.Ensure(__func__); + TabCapturedHandler::Create(promise, std::move(holder)); + return p; +} + +} // namespace mozilla diff --git a/dom/media/systemservices/video_engine/tab_capturer.h b/dom/media/systemservices/video_engine/tab_capturer.h new file mode 100644 index 0000000000..92c4fa2ad1 --- /dev/null +++ b/dom/media/systemservices/video_engine/tab_capturer.h @@ -0,0 +1,88 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_TAB_CAPTURER_H_ +#define MODULES_DESKTOP_CAPTURE_TAB_CAPTURER_H_ + +#include "api/sequence_checker.h" +#include "modules/desktop_capture/desktop_capturer.h" +#include "mozilla/MozPromise.h" +#include "nsDeque.h" +#include "nsThreadUtils.h" + +namespace mozilla { +namespace dom { +struct ImageBitmapCloneData; +} // namespace dom + +class CaptureFrameRequest; +class TabCapturedHandler; +class TaskQueue; + +class TabCapturerWebrtc : public webrtc::DesktopCapturer { + protected: + TabCapturerWebrtc(SourceId aSourceId, + nsCOMPtr<nsISerialEventTarget> aCaptureThread); + ~TabCapturerWebrtc(); + + public: + friend class CaptureFrameRequest; + friend class TabCapturedHandler; + + static std::unique_ptr<webrtc::DesktopCapturer> Create( + SourceId aSourceId, nsCOMPtr<nsISerialEventTarget> aCaptureThread); + + TabCapturerWebrtc(const TabCapturerWebrtc&) = delete; + TabCapturerWebrtc& operator=(const TabCapturerWebrtc&) = delete; + + // DesktopCapturer interface. + void Start(Callback* aCallback) override; + void CaptureFrame() override; + bool GetSourceList(SourceList* aSources) override; + bool SelectSource(SourceId) override; + bool FocusOnSelectedSource() override; + bool IsOccluded(const webrtc::DesktopVector& aPos) override; + + private: + // Capture code + using CapturePromise = + MozPromise<UniquePtr<dom::ImageBitmapCloneData>, nsresult, true>; + RefPtr<CapturePromise> CaptureFrameNow(); + + // Helper that checks for overrun requests. Returns true if aRequest had not + // been dropped due to disconnection or overrun. + // Note that if this returns true, the caller takes the responsibility to call + // mCallback with a capture result for aRequest. + bool CompleteRequest(CaptureFrameRequest* aRequest); + + // Helper that disconnects the request, and notifies mCallback of a temporary + // failure. + void DisconnectRequest(CaptureFrameRequest* aRequest); + + // Handle the result from the async callback from CaptureFrameNow. + void OnCaptureFrameSuccess(UniquePtr<dom::ImageBitmapCloneData> aData); + void OnCaptureFrameFailure(); + + const uint64_t mBrowserId; + const RefPtr<TaskQueue> mMainThreadWorker; + const RefPtr<TaskQueue> mCallbackWorker; + webrtc::SequenceChecker mControlChecker; + webrtc::SequenceChecker mCallbackChecker; + // Set in Start() and guaranteed by the owner of this class to outlive us. + webrtc::DesktopCapturer::Callback* mCallback + RTC_GUARDED_BY(mCallbackChecker) = nullptr; + + // mCallbackWorker only + nsRefPtrDeque<CaptureFrameRequest> mRequests RTC_GUARDED_BY(mCallbackChecker); +}; + +} // namespace mozilla + +#endif // MODULES_DESKTOP_CAPTURE_TAB_CAPTURER_H_ |