diff options
Diffstat (limited to 'dom/media/systemservices/video_engine')
9 files changed, 2385 insertions, 0 deletions
diff --git a/dom/media/systemservices/video_engine/browser_capture_impl.h b/dom/media/systemservices/video_engine/browser_capture_impl.h new file mode 100644 index 0000000000..aeaae62202 --- /dev/null +++ b/dom/media/systemservices/video_engine/browser_capture_impl.h @@ -0,0 +1,78 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef WEBRTC_MODULES_BROWSER_CAPTURE_MAIN_SOURCE_BROWSER_CAPTURE_IMPL_H_ +#define WEBRTC_MODULES_BROWSER_CAPTURE_MAIN_SOURCE_BROWSER_CAPTURE_IMPL_H_ + +#include "webrtc/modules/video_capture/video_capture.h" + +using namespace webrtc::videocapturemodule; + +namespace webrtc { + +class BrowserDeviceInfoImpl : public VideoCaptureModule::DeviceInfo { + public: + virtual uint32_t NumberOfDevices() { return 1; } + + virtual int32_t Refresh() { return 0; } + + virtual int32_t GetDeviceName(uint32_t deviceNumber, char* deviceNameUTF8, + uint32_t deviceNameSize, + char* deviceUniqueIdUTF8, + uint32_t deviceUniqueIdUTF8Size, + char* productUniqueIdUTF8 = NULL, + uint32_t productUniqueIdUTF8Size = 0, + pid_t* pid = 0) { + deviceNameUTF8 = const_cast<char*>(kDeviceName); + deviceUniqueIdUTF8 = const_cast<char*>(kUniqueDeviceName); + productUniqueIdUTF8 = const_cast<char*>(kProductUniqueId); + return 1; + }; + + virtual int32_t NumberOfCapabilities(const char* deviceUniqueIdUTF8) { + return 0; + } + + virtual int32_t GetCapability(const char* deviceUniqueIdUTF8, + const uint32_t deviceCapabilityNumber, + VideoCaptureCapability& capability) { + return 0; + }; + + virtual int32_t GetOrientation(const char* deviceUniqueIdUTF8, + VideoRotation& orientation) { + return 0; + } + + virtual int32_t GetBestMatchedCapability( + const char* deviceUniqueIdUTF8, const VideoCaptureCapability& requested, + VideoCaptureCapability& resulting) { + return 0; + } + + virtual int32_t DisplayCaptureSettingsDialogBox( + const char* deviceUniqueIdUTF8, const char* dialogTitleUTF8, + void* parentWindow, uint32_t positionX, uint32_t positionY) { + return 0; + } + + BrowserDeviceInfoImpl() + : kDeviceName("browser"), + kUniqueDeviceName("browser"), + kProductUniqueId("browser") {} + + static BrowserDeviceInfoImpl* CreateDeviceInfo() { + return new BrowserDeviceInfoImpl(); + } + virtual ~BrowserDeviceInfoImpl() {} + + private: + const char* kDeviceName; + const char* kUniqueDeviceName; + const char* kProductUniqueId; +}; +} // namespace webrtc +#endif diff --git a/dom/media/systemservices/video_engine/desktop_capture_impl.cc b/dom/media/systemservices/video_engine/desktop_capture_impl.cc new file mode 100644 index 0000000000..2274a21e8a --- /dev/null +++ b/dom/media/systemservices/video_engine/desktop_capture_impl.cc @@ -0,0 +1,776 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video_engine/desktop_capture_impl.h" + +#include <cstdlib> +#include <memory> +#include <string> + +#include "CamerasTypes.h" +#include "VideoEngine.h" +#include "VideoUtils.h" +#include "api/video/i420_buffer.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "libyuv.h" // NOLINT +#include "modules/include/module_common_types.h" +#include "modules/video_capture/video_capture_config.h" +#include "modules/video_capture/video_capture_impl.h" +#include "system_wrappers/include/clock.h" +#include "rtc_base/logging.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/time_utils.h" +#include "rtc_base/trace_event.h" +#include "modules/desktop_capture/desktop_and_cursor_composer.h" +#include "modules/desktop_capture/desktop_frame.h" +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_capturer_differ_wrapper.h" +#include "modules/video_capture/video_capture.h" +#include "mozilla/StaticPrefs_media.h" +#include "mozilla/SyncRunnable.h" +#include "mozilla/TaskQueue.h" +#include "mozilla/TimeStamp.h" +#include "nsThreadUtils.h" +#include "tab_capturer.h" + +using mozilla::NewRunnableMethod; +using mozilla::TabCapturerWebrtc; +using mozilla::TimeDuration; +using mozilla::camera::CaptureDeviceType; +using mozilla::camera::CaptureEngine; + +static void CaptureFrameOnThread(nsITimer* aTimer, void* aClosure) { + static_cast<webrtc::DesktopCaptureImpl*>(aClosure)->CaptureFrameOnThread(); +} + +namespace webrtc { + +int32_t ScreenDeviceInfoImpl::Init() { + mDesktopDeviceInfo = + std::unique_ptr<DesktopDeviceInfo>(DesktopDeviceInfo::Create()); + return 0; +} + +int32_t ScreenDeviceInfoImpl::Refresh() { + mDesktopDeviceInfo->Refresh(); + return 0; +} + +uint32_t ScreenDeviceInfoImpl::NumberOfDevices() { + return mDesktopDeviceInfo->getDisplayDeviceCount(); +} + +int32_t ScreenDeviceInfoImpl::GetDeviceName( + uint32_t aDeviceNumber, char* aDeviceNameUTF8, uint32_t aDeviceNameUTF8Size, + char* aDeviceUniqueIdUTF8, uint32_t aDeviceUniqueIdUTF8Size, + char* aProductUniqueIdUTF8, uint32_t aProductUniqueIdUTF8Size, + pid_t* aPid) { + DesktopDisplayDevice desktopDisplayDevice; + + // always initialize output + if (aDeviceNameUTF8 && aDeviceNameUTF8Size > 0) { + memset(aDeviceNameUTF8, 0, aDeviceNameUTF8Size); + } + + if (aDeviceUniqueIdUTF8 && aDeviceUniqueIdUTF8Size > 0) { + memset(aDeviceUniqueIdUTF8, 0, aDeviceUniqueIdUTF8Size); + } + if (aProductUniqueIdUTF8 && aProductUniqueIdUTF8Size > 0) { + memset(aProductUniqueIdUTF8, 0, aProductUniqueIdUTF8Size); + } + + if (mDesktopDeviceInfo->getDesktopDisplayDeviceInfo( + aDeviceNumber, desktopDisplayDevice) == 0) { + size_t len; + + const char* deviceName = desktopDisplayDevice.getDeviceName(); + len = deviceName ? strlen(deviceName) : 0; + if (len && aDeviceNameUTF8 && len < aDeviceNameUTF8Size) { + memcpy(aDeviceNameUTF8, deviceName, len); + } + + const char* deviceUniqueId = desktopDisplayDevice.getUniqueIdName(); + len = deviceUniqueId ? strlen(deviceUniqueId) : 0; + if (len && aDeviceUniqueIdUTF8 && len < aDeviceUniqueIdUTF8Size) { + memcpy(aDeviceUniqueIdUTF8, deviceUniqueId, len); + } + } + + return 0; +} + +int32_t ScreenDeviceInfoImpl::DisplayCaptureSettingsDialogBox( + const char* aDeviceUniqueIdUTF8, const char* aDialogTitleUTF8, + void* aParentWindow, uint32_t aPositionX, uint32_t aPositionY) { + // no device properties to change + return 0; +} + +int32_t ScreenDeviceInfoImpl::NumberOfCapabilities( + const char* aDeviceUniqueIdUTF8) { + return 0; +} + +int32_t ScreenDeviceInfoImpl::GetCapability( + const char* aDeviceUniqueIdUTF8, uint32_t aDeviceCapabilityNumber, + VideoCaptureCapability& aCapability) { + return 0; +} + +int32_t ScreenDeviceInfoImpl::GetBestMatchedCapability( + const char* aDeviceUniqueIdUTF8, const VideoCaptureCapability& aRequested, + VideoCaptureCapability& aResulting) { + return 0; +} + +int32_t ScreenDeviceInfoImpl::GetOrientation(const char* aDeviceUniqueIdUTF8, + VideoRotation& aOrientation) { + return 0; +} + +VideoCaptureModule* DesktopCaptureImpl::Create(const int32_t aModuleId, + const char* aUniqueId, + const CaptureDeviceType aType) { + return new rtc::RefCountedObject<DesktopCaptureImpl>(aModuleId, aUniqueId, + aType); +} + +int32_t WindowDeviceInfoImpl::Init() { + mDesktopDeviceInfo = + std::unique_ptr<DesktopDeviceInfo>(DesktopDeviceInfo::Create()); + return 0; +} + +int32_t WindowDeviceInfoImpl::Refresh() { + mDesktopDeviceInfo->Refresh(); + return 0; +} + +uint32_t WindowDeviceInfoImpl::NumberOfDevices() { + return mDesktopDeviceInfo->getWindowCount(); +} + +int32_t WindowDeviceInfoImpl::GetDeviceName( + uint32_t aDeviceNumber, char* aDeviceNameUTF8, uint32_t aDeviceNameUTF8Size, + char* aDeviceUniqueIdUTF8, uint32_t aDeviceUniqueIdUTF8Size, + char* aProductUniqueIdUTF8, uint32_t aProductUniqueIdUTF8Size, + pid_t* aPid) { + DesktopDisplayDevice desktopDisplayDevice; + + // always initialize output + if (aDeviceNameUTF8 && aDeviceNameUTF8Size > 0) { + memset(aDeviceNameUTF8, 0, aDeviceNameUTF8Size); + } + if (aDeviceUniqueIdUTF8 && aDeviceUniqueIdUTF8Size > 0) { + memset(aDeviceUniqueIdUTF8, 0, aDeviceUniqueIdUTF8Size); + } + if (aProductUniqueIdUTF8 && aProductUniqueIdUTF8Size > 0) { + memset(aProductUniqueIdUTF8, 0, aProductUniqueIdUTF8Size); + } + + if (mDesktopDeviceInfo->getWindowInfo(aDeviceNumber, desktopDisplayDevice) == + 0) { + size_t len; + + const char* deviceName = desktopDisplayDevice.getDeviceName(); + len = deviceName ? strlen(deviceName) : 0; + if (len && aDeviceNameUTF8 && len < aDeviceNameUTF8Size) { + memcpy(aDeviceNameUTF8, deviceName, len); + } + + const char* deviceUniqueId = desktopDisplayDevice.getUniqueIdName(); + len = deviceUniqueId ? strlen(deviceUniqueId) : 0; + if (len && aDeviceUniqueIdUTF8 && len < aDeviceUniqueIdUTF8Size) { + memcpy(aDeviceUniqueIdUTF8, deviceUniqueId, len); + } + if (aPid) { + *aPid = desktopDisplayDevice.getPid(); + } + } + + return 0; +} + +int32_t WindowDeviceInfoImpl::DisplayCaptureSettingsDialogBox( + const char* aDeviceUniqueIdUTF8, const char* aDialogTitleUTF8, + void* aParentWindow, uint32_t aPositionX, uint32_t aPositionY) { + // no device properties to change + return 0; +} + +int32_t WindowDeviceInfoImpl::NumberOfCapabilities( + const char* aDeviceUniqueIdUTF8) { + return 0; +} + +int32_t WindowDeviceInfoImpl::GetCapability( + const char* aDeviceUniqueIdUTF8, uint32_t aDeviceCapabilityNumber, + VideoCaptureCapability& aCapability) { + return 0; +} + +int32_t WindowDeviceInfoImpl::GetBestMatchedCapability( + const char* aDeviceUniqueIdUTF8, const VideoCaptureCapability& aRequested, + VideoCaptureCapability& aResulting) { + return 0; +} + +int32_t WindowDeviceInfoImpl::GetOrientation(const char* aDeviceUniqueIdUTF8, + VideoRotation& aOrientation) { + return 0; +} + +int32_t BrowserDeviceInfoImpl::Init() { + mDesktopDeviceInfo = + std::unique_ptr<DesktopDeviceInfo>(DesktopDeviceInfo::Create()); + return 0; +} + +int32_t BrowserDeviceInfoImpl::Refresh() { + mDesktopDeviceInfo->Refresh(); + return 0; +} + +uint32_t BrowserDeviceInfoImpl::NumberOfDevices() { + return mDesktopDeviceInfo->getTabCount(); +} + +int32_t BrowserDeviceInfoImpl::GetDeviceName( + uint32_t aDeviceNumber, char* aDeviceNameUTF8, uint32_t aDeviceNameUTF8Size, + char* aDeviceUniqueIdUTF8, uint32_t aDeviceUniqueIdUTF8Size, + char* aProductUniqueIdUTF8, uint32_t aProductUniqueIdUTF8Size, + pid_t* aPid) { + DesktopTab desktopTab; + + // always initialize output + if (aDeviceNameUTF8 && aDeviceNameUTF8Size > 0) { + memset(aDeviceNameUTF8, 0, aDeviceNameUTF8Size); + } + if (aDeviceUniqueIdUTF8 && aDeviceUniqueIdUTF8Size > 0) { + memset(aDeviceUniqueIdUTF8, 0, aDeviceUniqueIdUTF8Size); + } + if (aProductUniqueIdUTF8 && aProductUniqueIdUTF8Size > 0) { + memset(aProductUniqueIdUTF8, 0, aProductUniqueIdUTF8Size); + } + + if (mDesktopDeviceInfo->getTabInfo(aDeviceNumber, desktopTab) == 0) { + size_t len; + + const char* deviceName = desktopTab.getTabName(); + len = deviceName ? strlen(deviceName) : 0; + if (len && aDeviceNameUTF8 && len < aDeviceNameUTF8Size) { + memcpy(aDeviceNameUTF8, deviceName, len); + } + + const char* deviceUniqueId = desktopTab.getUniqueIdName(); + len = deviceUniqueId ? strlen(deviceUniqueId) : 0; + if (len && aDeviceUniqueIdUTF8 && len < aDeviceUniqueIdUTF8Size) { + memcpy(aDeviceUniqueIdUTF8, deviceUniqueId, len); + } + } + + return 0; +} + +int32_t BrowserDeviceInfoImpl::DisplayCaptureSettingsDialogBox( + const char* aDeviceUniqueIdUTF8, const char* aDialogTitleUTF8, + void* aParentWindow, uint32_t aPositionX, uint32_t aPositionY) { + // no device properties to change + return 0; +} + +int32_t BrowserDeviceInfoImpl::NumberOfCapabilities( + const char* aDeviceUniqueIdUTF8) { + return 0; +} + +int32_t BrowserDeviceInfoImpl::GetCapability( + const char* aDeviceUniqueIdUTF8, uint32_t aDeviceCapabilityNumber, + VideoCaptureCapability& aCapability) { + return 0; +} + +int32_t BrowserDeviceInfoImpl::GetBestMatchedCapability( + const char* aDeviceUniqueIdUTF8, const VideoCaptureCapability& aRequested, + VideoCaptureCapability& aResulting) { + return 0; +} + +int32_t BrowserDeviceInfoImpl::GetOrientation(const char* aDeviceUniqueIdUTF8, + VideoRotation& aOrientation) { + return 0; +} + +std::shared_ptr<VideoCaptureModule::DeviceInfo> +DesktopCaptureImpl::CreateDeviceInfo(const int32_t aId, + const CaptureDeviceType aType) { + if (aType == CaptureDeviceType::Screen) { + auto screenInfo = std::make_shared<ScreenDeviceInfoImpl>(aId); + if (!screenInfo || screenInfo->Init() != 0) { + return nullptr; + } + return screenInfo; + } + if (aType == CaptureDeviceType::Window) { + auto windowInfo = std::make_shared<WindowDeviceInfoImpl>(aId); + if (!windowInfo || windowInfo->Init() != 0) { + return nullptr; + } + return windowInfo; + } + if (aType == CaptureDeviceType::Browser) { + auto browserInfo = std::make_shared<BrowserDeviceInfoImpl>(aId); + if (!browserInfo || browserInfo->Init() != 0) { + return nullptr; + } + return browserInfo; + } + return nullptr; +} + +const char* DesktopCaptureImpl::CurrentDeviceName() const { + return mDeviceUniqueId.c_str(); +} + +static DesktopCaptureOptions CreateDesktopCaptureOptions() { + DesktopCaptureOptions options; +// Help avoid an X11 deadlock, see bug 1456101. +#ifdef MOZ_X11 + MOZ_ALWAYS_SUCCEEDS(mozilla::SyncRunnable::DispatchToThread( + mozilla::GetMainThreadSerialEventTarget(), + NS_NewRunnableFunction(__func__, [&] { + options = DesktopCaptureOptions::CreateDefault(); + }))); +#else + options = DesktopCaptureOptions::CreateDefault(); +#endif + + // Leave desktop effects enabled during WebRTC captures. + options.set_disable_effects(false); + +#if defined(WEBRTC_WIN) + if (mozilla::StaticPrefs::media_webrtc_capture_allow_directx()) { + options.set_allow_directx_capturer(true); + options.set_allow_use_magnification_api(false); + } else { + options.set_allow_use_magnification_api(true); + } + options.set_allow_cropping_window_capturer(true); +# if defined(RTC_ENABLE_WIN_WGC) + if (mozilla::StaticPrefs::media_webrtc_capture_allow_wgc()) { + options.set_allow_wgc_capturer(true); + } +# endif +#endif + +#if defined(WEBRTC_MAC) + if (mozilla::StaticPrefs::media_webrtc_capture_allow_iosurface()) { + options.set_allow_iosurface(true); + } +#endif + +#if defined(WEBRTC_USE_PIPEWIRE) + if (mozilla::StaticPrefs::media_webrtc_capture_allow_pipewire()) { + options.set_allow_pipewire(true); + } +#endif + + return options; +} + +static std::unique_ptr<DesktopCapturer> CreateTabCapturer( + const DesktopCaptureOptions& options, DesktopCapturer::SourceId aSourceId, + nsCOMPtr<nsISerialEventTarget> aCaptureThread) { + std::unique_ptr<DesktopCapturer> capturer = + TabCapturerWebrtc::Create(aSourceId, std::move(aCaptureThread)); + if (capturer && options.detect_updated_region()) { + capturer.reset(new DesktopCapturerDifferWrapper(std::move(capturer))); + } + + return capturer; +} + +static bool UsePipewire() { +#if defined(WEBRTC_USE_PIPEWIRE) + return mozilla::StaticPrefs::media_webrtc_capture_allow_pipewire() && + webrtc::DesktopCapturer::IsRunningUnderWayland(); +#else + return false; +#endif +} + +static std::unique_ptr<DesktopCapturer> CreateDesktopCapturerAndThread( + CaptureDeviceType aDeviceType, DesktopCapturer::SourceId aSourceId, + nsIThread** aOutThread) { + DesktopCaptureOptions options = CreateDesktopCaptureOptions(); + std::unique_ptr<DesktopCapturer> capturer; + + auto ensureThread = [&]() { + if (*aOutThread) { + return *aOutThread; + } + + nsIThreadManager::ThreadCreationOptions threadOptions; +#if defined(XP_WIN) || defined(XP_MACOSX) + // Windows desktop capture needs a UI thread. + // Mac screen capture needs a thread with a CFRunLoop. + threadOptions.isUiThread = true; +#endif + NS_NewNamedThread("DesktopCapture", aOutThread, nullptr, threadOptions); + return *aOutThread; + }; + + if ((aDeviceType == CaptureDeviceType::Screen || + aDeviceType == CaptureDeviceType::Window) && + UsePipewire()) { + capturer = DesktopCapturer::CreateGenericCapturer(options); + if (!capturer) { + return capturer; + } + + capturer = std::make_unique<DesktopAndCursorComposer>(std::move(capturer), + options); + } else if (aDeviceType == CaptureDeviceType::Screen) { + capturer = DesktopCapturer::CreateScreenCapturer(options); + if (!capturer) { + return capturer; + } + + capturer->SelectSource(aSourceId); + + capturer = std::make_unique<DesktopAndCursorComposer>(std::move(capturer), + options); + } else if (aDeviceType == CaptureDeviceType::Window) { +#if defined(RTC_ENABLE_WIN_WGC) + options.set_allow_wgc_capturer_fallback(true); +#endif + capturer = DesktopCapturer::CreateWindowCapturer(options); + if (!capturer) { + return capturer; + } + + capturer->SelectSource(aSourceId); + + capturer = std::make_unique<DesktopAndCursorComposer>(std::move(capturer), + options); + } else if (aDeviceType == CaptureDeviceType::Browser) { + // XXX We don't capture cursors, so avoid the extra indirection layer. We + // could also pass null for the pMouseCursorMonitor. + capturer = CreateTabCapturer(options, aSourceId, ensureThread()); + } else { + MOZ_ASSERT(!capturer); + return capturer; + } + + MOZ_ASSERT(capturer); + ensureThread(); + + return capturer; +} + +DesktopCaptureImpl::DesktopCaptureImpl(const int32_t aId, const char* aUniqueId, + const CaptureDeviceType aType) + : mModuleId(aId), + mTrackingId(mozilla::TrackingId(CaptureEngineToTrackingSourceStr([&] { + switch (aType) { + case CaptureDeviceType::Screen: + return CaptureEngine::ScreenEngine; + case CaptureDeviceType::Window: + return CaptureEngine::WinEngine; + case CaptureDeviceType::Browser: + return CaptureEngine::BrowserEngine; + default: + return CaptureEngine::InvalidEngine; + } + }()), + aId)), + mDeviceUniqueId(aUniqueId), + mDeviceType(aType), + mControlThread(mozilla::GetCurrentSerialEventTarget()), + mNextFrameMinimumTime(Timestamp::Zero()), + mCallbacks("DesktopCaptureImpl::mCallbacks") {} + +DesktopCaptureImpl::~DesktopCaptureImpl() { + MOZ_ASSERT(!mCaptureThread); + MOZ_ASSERT(!mRequestedCapability); +} + +void DesktopCaptureImpl::RegisterCaptureDataCallback( + rtc::VideoSinkInterface<VideoFrame>* aDataCallback) { + auto callbacks = mCallbacks.Lock(); + callbacks->insert(aDataCallback); +} + +void DesktopCaptureImpl::DeRegisterCaptureDataCallback( + rtc::VideoSinkInterface<VideoFrame>* aDataCallback) { + auto callbacks = mCallbacks.Lock(); + auto it = callbacks->find(aDataCallback); + if (it != callbacks->end()) { + callbacks->erase(it); + } +} + +int32_t DesktopCaptureImpl::StopCaptureIfAllClientsClose() { + { + auto callbacks = mCallbacks.Lock(); + if (!callbacks->empty()) { + return 0; + } + } + return StopCapture(); +} + +int32_t DesktopCaptureImpl::SetCaptureRotation(VideoRotation aRotation) { + MOZ_ASSERT_UNREACHABLE("Unused"); + return -1; +} + +bool DesktopCaptureImpl::SetApplyRotation(bool aEnable) { return true; } + +int32_t DesktopCaptureImpl::StartCapture( + const VideoCaptureCapability& aCapability) { + RTC_DCHECK_RUN_ON(&mControlThreadChecker); + + if (mRequestedCapability) { + // Already initialized + MOZ_ASSERT(*mRequestedCapability == aCapability); + + return 0; + } + + MOZ_ASSERT(!mCaptureThread); + + DesktopCapturer::SourceId sourceId = std::stoi(mDeviceUniqueId); + std::unique_ptr capturer = CreateDesktopCapturerAndThread( + mDeviceType, sourceId, getter_AddRefs(mCaptureThread)); + + MOZ_ASSERT(!capturer == !mCaptureThread); + if (!capturer) { + return -1; + } + + mRequestedCapability = mozilla::Some(aCapability); + mCaptureThreadChecker.Detach(); + + MOZ_ALWAYS_SUCCEEDS(mCaptureThread->Dispatch(NS_NewRunnableFunction( + "DesktopCaptureImpl::InitOnThread", + [this, self = RefPtr(this), capturer = std::move(capturer), + maxFps = std::max(aCapability.maxFPS, 1)]() mutable { + InitOnThread(std::move(capturer), maxFps); + }))); + + return 0; +} + +bool DesktopCaptureImpl::FocusOnSelectedSource() { + RTC_DCHECK_RUN_ON(&mControlThreadChecker); + if (!mCaptureThread) { + MOZ_ASSERT_UNREACHABLE( + "FocusOnSelectedSource must be called after StartCapture"); + return false; + } + + bool success = false; + MOZ_ALWAYS_SUCCEEDS(mozilla::SyncRunnable::DispatchToThread( + mCaptureThread, NS_NewRunnableFunction(__func__, [&] { + RTC_DCHECK_RUN_ON(&mCaptureThreadChecker); + MOZ_ASSERT(mCapturer); + success = mCapturer && mCapturer->FocusOnSelectedSource(); + }))); + return success; +} + +int32_t DesktopCaptureImpl::StopCapture() { + RTC_DCHECK_RUN_ON(&mControlThreadChecker); + if (mRequestedCapability) { + // Sync-cancel the capture timer so no CaptureFrame calls will come in after + // we return. + MOZ_ALWAYS_SUCCEEDS(mozilla::SyncRunnable::DispatchToThread( + mCaptureThread, + NewRunnableMethod(__func__, this, + &DesktopCaptureImpl::ShutdownOnThread))); + + mRequestedCapability = mozilla::Nothing(); + } + + if (mCaptureThread) { + // CaptureThread shutdown. + mCaptureThread->AsyncShutdown(); + mCaptureThread = nullptr; + } + + return 0; +} + +bool DesktopCaptureImpl::CaptureStarted() { + MOZ_ASSERT_UNREACHABLE("Unused"); + return true; +} + +int32_t DesktopCaptureImpl::CaptureSettings(VideoCaptureCapability& aSettings) { + MOZ_ASSERT_UNREACHABLE("Unused"); + return -1; +} + +void DesktopCaptureImpl::OnCaptureResult(DesktopCapturer::Result aResult, + std::unique_ptr<DesktopFrame> aFrame) { + RTC_DCHECK_RUN_ON(&mCaptureThreadChecker); + if (!aFrame) { + return; + } + + const auto startProcessTime = Timestamp::Micros(rtc::TimeMicros()); + auto frameTime = startProcessTime; + if (auto diff = startProcessTime - mNextFrameMinimumTime; + diff < TimeDelta::Zero()) { + if (diff > TimeDelta::Millis(-1)) { + // Two consecutive frames within a millisecond is OK. It could happen due + // to timing. + frameTime = mNextFrameMinimumTime; + } else { + // Three consecutive frames within two milliseconds seems too much, drop + // one. + MOZ_ASSERT(diff >= TimeDelta::Millis(-2)); + RTC_LOG(LS_WARNING) << "DesktopCapture render time is getting too far " + "ahead. Framerate is unexpectedly high."; + return; + } + } + + uint8_t* videoFrame = aFrame->data(); + VideoCaptureCapability frameInfo; + frameInfo.width = aFrame->size().width(); + frameInfo.height = aFrame->size().height(); + frameInfo.videoType = VideoType::kARGB; + + size_t videoFrameLength = + frameInfo.width * frameInfo.height * DesktopFrame::kBytesPerPixel; + + const int32_t width = frameInfo.width; + const int32_t height = frameInfo.height; + + // Not encoded, convert to I420. + if (frameInfo.videoType != VideoType::kMJPEG && + CalcBufferSize(frameInfo.videoType, width, abs(height)) != + videoFrameLength) { + RTC_LOG(LS_ERROR) << "Wrong incoming frame length."; + return; + } + + int stride_y = width; + int stride_uv = (width + 1) / 2; + + // Setting absolute height (in case it was negative). + // In Windows, the image starts bottom left, instead of top left. + // Setting a negative source height, inverts the image (within LibYuv). + + mozilla::PerformanceRecorder<mozilla::CopyVideoStage> rec( + "DesktopCaptureImpl::ConvertToI420"_ns, mTrackingId, width, abs(height)); + // TODO(nisse): Use a pool? + rtc::scoped_refptr<I420Buffer> buffer = + I420Buffer::Create(width, abs(height), stride_y, stride_uv, stride_uv); + + const int conversionResult = libyuv::ConvertToI420( + videoFrame, videoFrameLength, buffer->MutableDataY(), buffer->StrideY(), + buffer->MutableDataU(), buffer->StrideU(), buffer->MutableDataV(), + buffer->StrideV(), 0, 0, // No Cropping + aFrame->stride() / DesktopFrame::kBytesPerPixel, height, width, height, + libyuv::kRotate0, ConvertVideoType(frameInfo.videoType)); + if (conversionResult != 0) { + RTC_LOG(LS_ERROR) << "Failed to convert capture frame from type " + << static_cast<int>(frameInfo.videoType) << "to I420."; + return; + } + rec.Record(); + + NotifyOnFrame(VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_us(frameTime.us()) + .build()); + + const TimeDelta processTime = + Timestamp::Micros(rtc::TimeMicros()) - startProcessTime; + + if (processTime > TimeDelta::Millis(10)) { + RTC_LOG(LS_WARNING) + << "Too long processing time of incoming frame with dimensions " + << width << "x" << height << ": " << processTime.ms() << " ms"; + } +} + +void DesktopCaptureImpl::NotifyOnFrame(const VideoFrame& aFrame) { + RTC_DCHECK_RUN_ON(&mCaptureThreadChecker); + MOZ_ASSERT(Timestamp::Millis(aFrame.render_time_ms()) > + mNextFrameMinimumTime); + // Set the next frame's minimum time to ensure two consecutive frames don't + // have an identical render time (which is in milliseconds). + mNextFrameMinimumTime = + Timestamp::Millis(aFrame.render_time_ms()) + TimeDelta::Millis(1); + auto callbacks = mCallbacks.Lock(); + for (auto* cb : *callbacks) { + cb->OnFrame(aFrame); + } +} + +void DesktopCaptureImpl::InitOnThread( + std::unique_ptr<DesktopCapturer> aCapturer, int aFramerate) { + RTC_DCHECK_RUN_ON(&mCaptureThreadChecker); + + mCapturer = std::move(aCapturer); + + // We need to call Start on the same thread we call CaptureFrame on. + mCapturer->Start(this); + + mCaptureTimer = NS_NewTimer(); + mRequestedCaptureInterval = mozilla::Some( + TimeDuration::FromSeconds(1. / static_cast<double>(aFramerate))); + + CaptureFrameOnThread(); +} + +void DesktopCaptureImpl::ShutdownOnThread() { + RTC_DCHECK_RUN_ON(&mCaptureThreadChecker); + if (mCaptureTimer) { + mCaptureTimer->Cancel(); + mCaptureTimer = nullptr; + } + + // DesktopCapturer dtor blocks until fully shut down. TabCapturerWebrtc needs + // the capture thread to be alive. + mCapturer = nullptr; + + mRequestedCaptureInterval = mozilla::Nothing(); +} + +void DesktopCaptureImpl::CaptureFrameOnThread() { + RTC_DCHECK_RUN_ON(&mCaptureThreadChecker); + + auto start = mozilla::TimeStamp::Now(); + mCapturer->CaptureFrame(); + auto end = mozilla::TimeStamp::Now(); + + // Calculate next capture time. + const auto duration = end - start; + const auto timeUntilRequestedCapture = *mRequestedCaptureInterval - duration; + + // Use at most x% CPU or limit framerate + constexpr float sleepTimeFactor = + (100.0f / kMaxDesktopCaptureCpuUsage) - 1.0f; + static_assert(sleepTimeFactor >= 0.0); + static_assert(sleepTimeFactor < 100.0); + const auto sleepTime = duration.MultDouble(sleepTimeFactor); + + mCaptureTimer->InitHighResolutionWithNamedFuncCallback( + &::CaptureFrameOnThread, this, + std::max(timeUntilRequestedCapture, sleepTime), nsITimer::TYPE_ONE_SHOT, + "DesktopCaptureImpl::mCaptureTimer"); +} + +} // namespace webrtc diff --git a/dom/media/systemservices/video_engine/desktop_capture_impl.h b/dom/media/systemservices/video_engine/desktop_capture_impl.h new file mode 100644 index 0000000000..64eadb1401 --- /dev/null +++ b/dom/media/systemservices/video_engine/desktop_capture_impl.h @@ -0,0 +1,246 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_DESKTOP_CAPTURE_MAIN_SOURCE_DESKTOP_CAPTURE_IMPL_H_ +#define WEBRTC_MODULES_DESKTOP_CAPTURE_MAIN_SOURCE_DESKTOP_CAPTURE_IMPL_H_ + +/* + * video_capture_impl.h + */ + +#include <memory> +#include <set> +#include <string> + +#include "api/sequence_checker.h" +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "modules/desktop_capture/desktop_capturer.h" +#include "modules/video_capture/video_capture.h" + +#include "desktop_device_info.h" +#include "mozilla/DataMutex.h" +#include "mozilla/Maybe.h" +#include "mozilla/TimeStamp.h" +#include "nsCOMPtr.h" +#include "PerformanceRecorder.h" + +class nsIThread; +class nsITimer; + +namespace mozilla::camera { +enum class CaptureDeviceType; +} + +namespace webrtc { + +class VideoCaptureEncodeInterface; + +// simulate deviceInfo interface for video engine, bridge screen/application and +// real screen/application device info + +class ScreenDeviceInfoImpl : public VideoCaptureModule::DeviceInfo { + public: + ScreenDeviceInfoImpl(int32_t aId) : mId(aId) {} + virtual ~ScreenDeviceInfoImpl() = default; + + int32_t Init(); + int32_t Refresh(); + + virtual uint32_t NumberOfDevices(); + virtual int32_t GetDeviceName(uint32_t aDeviceNumber, char* aDeviceNameUTF8, + uint32_t aDeviceNameUTF8Size, + char* aDeviceUniqueIdUTF8, + uint32_t aDeviceUniqueIdUTF8Size, + char* aProductUniqueIdUTF8, + uint32_t aProductUniqueIdUTF8Size, pid_t* aPid); + + virtual int32_t DisplayCaptureSettingsDialogBox( + const char* aDeviceUniqueIdUTF8, const char* aDialogTitleUTF8, + void* aParentWindow, uint32_t aPositionX, uint32_t aPositionY); + virtual int32_t NumberOfCapabilities(const char* aDeviceUniqueIdUTF8); + virtual int32_t GetCapability(const char* aDeviceUniqueIdUTF8, + uint32_t aDeviceCapabilityNumber, + VideoCaptureCapability& aCapability); + + virtual int32_t GetBestMatchedCapability( + const char* aDeviceUniqueIdUTF8, const VideoCaptureCapability& aRequested, + VideoCaptureCapability& aResulting); + virtual int32_t GetOrientation(const char* aDeviceUniqueIdUTF8, + VideoRotation& aOrientation); + + protected: + int32_t mId; + std::unique_ptr<DesktopDeviceInfo> mDesktopDeviceInfo; +}; + +class WindowDeviceInfoImpl : public VideoCaptureModule::DeviceInfo { + public: + WindowDeviceInfoImpl(int32_t aId) : mId(aId){}; + virtual ~WindowDeviceInfoImpl() = default; + + int32_t Init(); + int32_t Refresh(); + + virtual uint32_t NumberOfDevices(); + virtual int32_t GetDeviceName(uint32_t aDeviceNumber, char* aDeviceNameUTF8, + uint32_t aDeviceNameUTF8Size, + char* aDeviceUniqueIdUTF8, + uint32_t aDeviceUniqueIdUTF8Size, + char* aProductUniqueIdUTF8, + uint32_t aProductUniqueIdUTF8Size, pid_t* aPid); + + virtual int32_t DisplayCaptureSettingsDialogBox( + const char* aDeviceUniqueIdUTF8, const char* aDialogTitleUTF8, + void* aParentWindow, uint32_t aPositionX, uint32_t aPositionY); + virtual int32_t NumberOfCapabilities(const char* aDeviceUniqueIdUTF8); + virtual int32_t GetCapability(const char* aDeviceUniqueIdUTF8, + uint32_t aDeviceCapabilityNumber, + VideoCaptureCapability& aCapability); + + virtual int32_t GetBestMatchedCapability( + const char* aDeviceUniqueIdUTF8, const VideoCaptureCapability& aRequested, + VideoCaptureCapability& aResulting); + virtual int32_t GetOrientation(const char* aDeviceUniqueIdUTF8, + VideoRotation& aOrientation); + + protected: + int32_t mId; + std::unique_ptr<DesktopDeviceInfo> mDesktopDeviceInfo; +}; + +class BrowserDeviceInfoImpl : public VideoCaptureModule::DeviceInfo { + public: + BrowserDeviceInfoImpl(int32_t aId) : mId(aId){}; + virtual ~BrowserDeviceInfoImpl() = default; + + int32_t Init(); + int32_t Refresh(); + + virtual uint32_t NumberOfDevices(); + virtual int32_t GetDeviceName(uint32_t aDeviceNumber, char* aDeviceNameUTF8, + uint32_t aDeviceNameUTF8Size, + char* aDeviceUniqueIdUTF8, + uint32_t aDeviceUniqueIdUTF8Size, + char* aProductUniqueIdUTF8, + uint32_t aProductUniqueIdUTF8Size, pid_t* aPid); + + virtual int32_t DisplayCaptureSettingsDialogBox( + const char* aDeviceUniqueIdUTF8, const char* aDialogTitleUTF8, + void* aParentWindow, uint32_t aPositionX, uint32_t aPositionY); + virtual int32_t NumberOfCapabilities(const char* aDeviceUniqueIdUTF8); + virtual int32_t GetCapability(const char* aDeviceUniqueIdUTF8, + uint32_t aDeviceCapabilityNumber, + VideoCaptureCapability& aCapability); + + virtual int32_t GetBestMatchedCapability( + const char* aDeviceUniqueIdUTF8, const VideoCaptureCapability& aRequested, + VideoCaptureCapability& aResulting); + virtual int32_t GetOrientation(const char* aDeviceUniqueIdUTF8, + VideoRotation& aOrientation); + + protected: + int32_t mId; + std::unique_ptr<DesktopDeviceInfo> mDesktopDeviceInfo; +}; + +// Reuses the video engine pipeline for screen sharing. +// As with video, DesktopCaptureImpl is a proxy for screen sharing +// and follows the video pipeline design +class DesktopCaptureImpl : public DesktopCapturer::Callback, + public VideoCaptureModule { + public: + /* Create a screen capture modules object + */ + static VideoCaptureModule* Create( + const int32_t aModuleId, const char* aUniqueId, + const mozilla::camera::CaptureDeviceType aType); + + [[nodiscard]] static std::shared_ptr<VideoCaptureModule::DeviceInfo> + CreateDeviceInfo(const int32_t aId, + const mozilla::camera::CaptureDeviceType aType); + + // mControlThread only. + void RegisterCaptureDataCallback( + rtc::VideoSinkInterface<VideoFrame>* aCallback) override; + void RegisterCaptureDataCallback( + RawVideoSinkInterface* dataCallback) override {} + void DeRegisterCaptureDataCallback( + rtc::VideoSinkInterface<VideoFrame>* aCallback) override; + int32_t StopCaptureIfAllClientsClose() override; + + int32_t SetCaptureRotation(VideoRotation aRotation) override; + bool SetApplyRotation(bool aEnable) override; + bool GetApplyRotation() override { return true; } + + const char* CurrentDeviceName() const override; + + int32_t StartCapture(const VideoCaptureCapability& aCapability) override; + virtual bool FocusOnSelectedSource() override; + int32_t StopCapture() override; + bool CaptureStarted() override; + int32_t CaptureSettings(VideoCaptureCapability& aSettings) override; + + void CaptureFrameOnThread(); + + const int32_t mModuleId; + const mozilla::TrackingId mTrackingId; + const std::string mDeviceUniqueId; + const mozilla::camera::CaptureDeviceType mDeviceType; + + protected: + DesktopCaptureImpl(const int32_t aId, const char* aUniqueId, + const mozilla::camera::CaptureDeviceType aType); + virtual ~DesktopCaptureImpl(); + + private: + // Maximum CPU usage in %. + static constexpr uint32_t kMaxDesktopCaptureCpuUsage = 50; + void InitOnThread(std::unique_ptr<DesktopCapturer> aCapturer, int aFramerate); + void ShutdownOnThread(); + // DesktopCapturer::Callback interface. + void OnCaptureResult(DesktopCapturer::Result aResult, + std::unique_ptr<DesktopFrame> aFrame) override; + + // Notifies all mCallbacks of OnFrame(). mCaptureThread only. + void NotifyOnFrame(const VideoFrame& aFrame); + + // Control thread on which the public API is called. + const nsCOMPtr<nsISerialEventTarget> mControlThread; + // Set in StartCapture. + mozilla::Maybe<VideoCaptureCapability> mRequestedCapability + RTC_GUARDED_BY(mControlThreadChecker); + // The DesktopCapturer is created on mControlThread but assigned and accessed + // only on mCaptureThread. + std::unique_ptr<DesktopCapturer> mCapturer + RTC_GUARDED_BY(mCaptureThreadChecker); + // Dedicated thread that does the capturing. + nsCOMPtr<nsIThread> mCaptureThread RTC_GUARDED_BY(mControlThreadChecker); + // Checks that API methods are called on mControlThread. + webrtc::SequenceChecker mControlThreadChecker; + // Checks that frame delivery only happens on mCaptureThread. + webrtc::SequenceChecker mCaptureThreadChecker; + // Timer that triggers frame captures. Only used on mCaptureThread. + // TODO(Bug 1806646): Drive capture with vsync instead. + nsCOMPtr<nsITimer> mCaptureTimer RTC_GUARDED_BY(mCaptureThreadChecker); + // Interval between captured frames, based on the framerate in + // mRequestedCapability. mCaptureThread only. + mozilla::Maybe<mozilla::TimeDuration> mRequestedCaptureInterval + RTC_GUARDED_BY(mCaptureThreadChecker); + // Used to make sure incoming timestamp is increasing for every frame. + webrtc::Timestamp mNextFrameMinimumTime RTC_GUARDED_BY(mCaptureThreadChecker); + // Callbacks for captured frames. Mutated on mControlThread, callbacks happen + // on mCaptureThread. + mozilla::DataMutex<std::set<rtc::VideoSinkInterface<VideoFrame>*>> mCallbacks; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_DESKTOP_CAPTURE_MAIN_SOURCE_DESKTOP_CAPTURE_IMPL_H_ diff --git a/dom/media/systemservices/video_engine/desktop_device_info.cc b/dom/media/systemservices/video_engine/desktop_device_info.cc new file mode 100644 index 0000000000..b3632a509f --- /dev/null +++ b/dom/media/systemservices/video_engine/desktop_device_info.cc @@ -0,0 +1,488 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "desktop_device_info.h" +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_capturer.h" +#include "mozilla/Sprintf.h" +#include "mozilla/StaticPrefs_media.h" +#include "mozilla/SyncRunnable.h" +#include "mozilla/UniquePtr.h" +#include "nsIBrowserWindowTracker.h" +#include "nsImportModule.h" + +#include <cstddef> +#include <cstdlib> +#include <cstdio> +#include <cstring> +#include <memory> + +namespace webrtc { + +static inline void SetStringMember(char** aMember, const char* aValue) { + if (!aValue) { + return; + } + + if (*aMember) { + delete[] * aMember; + *aMember = nullptr; + } + + size_t nBufLen = strlen(aValue) + 1; + char* buffer = new char[nBufLen]; + memcpy(buffer, aValue, nBufLen - 1); + buffer[nBufLen - 1] = '\0'; + *aMember = buffer; +} + +DesktopDisplayDevice::DesktopDisplayDevice() { + mScreenId = kInvalidScreenId; + mDeviceUniqueIdUTF8 = nullptr; + mDeviceNameUTF8 = nullptr; + mPid = 0; +} + +DesktopDisplayDevice::~DesktopDisplayDevice() { + mScreenId = kInvalidScreenId; + + delete[] mDeviceUniqueIdUTF8; + delete[] mDeviceNameUTF8; + + mDeviceUniqueIdUTF8 = nullptr; + mDeviceNameUTF8 = nullptr; +} + +void DesktopDisplayDevice::setScreenId(const ScreenId aScreenId) { + mScreenId = aScreenId; +} + +void DesktopDisplayDevice::setDeviceName(const char* aDeviceNameUTF8) { + SetStringMember(&mDeviceNameUTF8, aDeviceNameUTF8); +} + +void DesktopDisplayDevice::setUniqueIdName(const char* aDeviceUniqueIdUTF8) { + SetStringMember(&mDeviceUniqueIdUTF8, aDeviceUniqueIdUTF8); +} + +void DesktopDisplayDevice::setPid(const int aPid) { mPid = aPid; } + +ScreenId DesktopDisplayDevice::getScreenId() { return mScreenId; } + +const char* DesktopDisplayDevice::getDeviceName() { return mDeviceNameUTF8; } + +const char* DesktopDisplayDevice::getUniqueIdName() { + return mDeviceUniqueIdUTF8; +} + +pid_t DesktopDisplayDevice::getPid() { return mPid; } + +DesktopDisplayDevice& DesktopDisplayDevice::operator=( + DesktopDisplayDevice& aOther) { + if (&aOther == this) { + return *this; + } + mScreenId = aOther.getScreenId(); + setUniqueIdName(aOther.getUniqueIdName()); + setDeviceName(aOther.getDeviceName()); + mPid = aOther.getPid(); + + return *this; +} + +DesktopTab::DesktopTab() { + mTabBrowserId = 0; + mTabNameUTF8 = nullptr; + mTabUniqueIdUTF8 = nullptr; + mTabCount = 0; +} + +DesktopTab::~DesktopTab() { + delete[] mTabNameUTF8; + delete[] mTabUniqueIdUTF8; + + mTabNameUTF8 = nullptr; + mTabUniqueIdUTF8 = nullptr; +} + +void DesktopTab::setTabBrowserId(uint64_t aTabBrowserId) { + mTabBrowserId = aTabBrowserId; +} + +void DesktopTab::setUniqueIdName(const char* aTabUniqueIdUTF8) { + SetStringMember(&mTabUniqueIdUTF8, aTabUniqueIdUTF8); +} + +void DesktopTab::setTabName(const char* aTabNameUTF8) { + SetStringMember(&mTabNameUTF8, aTabNameUTF8); +} + +void DesktopTab::setTabCount(const uint32_t aCount) { mTabCount = aCount; } + +uint64_t DesktopTab::getTabBrowserId() { return mTabBrowserId; } + +const char* DesktopTab::getUniqueIdName() { return mTabUniqueIdUTF8; } + +const char* DesktopTab::getTabName() { return mTabNameUTF8; } + +uint32_t DesktopTab::getTabCount() { return mTabCount; } + +DesktopTab& DesktopTab::operator=(DesktopTab& aOther) { + mTabBrowserId = aOther.getTabBrowserId(); + setUniqueIdName(aOther.getUniqueIdName()); + setTabName(aOther.getTabName()); + + return *this; +} + +class DesktopDeviceInfoImpl : public DesktopDeviceInfo { + public: + DesktopDeviceInfoImpl(); + ~DesktopDeviceInfoImpl(); + + int32_t Init() override; + int32_t Refresh() override; + int32_t getDisplayDeviceCount() override; + int32_t getDesktopDisplayDeviceInfo( + uint32_t aIndex, DesktopDisplayDevice& aDesktopDisplayDevice) override; + int32_t getWindowCount() override; + int32_t getWindowInfo(uint32_t aIndex, + DesktopDisplayDevice& aWindowDevice) override; + uint32_t getTabCount() override; + int32_t getTabInfo(uint32_t aIndex, DesktopTab& aDesktopTab) override; + + protected: + DesktopDisplayDeviceList mDesktopDisplayList; + DesktopDisplayDeviceList mDesktopWindowList; + DesktopTabList mDesktopTabList; + + void CleanUp(); + void CleanUpWindowList(); + void CleanUpTabList(); + void CleanUpScreenList(); + + void InitializeWindowList(); + virtual void InitializeTabList(); + void InitializeScreenList(); + + void RefreshWindowList(); + void RefreshTabList(); + void RefreshScreenList(); + + void DummyTabList(DesktopTabList& aList); +}; + +DesktopDeviceInfoImpl::DesktopDeviceInfoImpl() = default; + +DesktopDeviceInfoImpl::~DesktopDeviceInfoImpl() { CleanUp(); } + +int32_t DesktopDeviceInfoImpl::getDisplayDeviceCount() { + return static_cast<int32_t>(mDesktopDisplayList.size()); +} + +int32_t DesktopDeviceInfoImpl::getDesktopDisplayDeviceInfo( + uint32_t aIndex, DesktopDisplayDevice& aDesktopDisplayDevice) { + if (aIndex >= mDesktopDisplayList.size()) { + return -1; + } + + std::map<intptr_t, DesktopDisplayDevice*>::iterator iter = + mDesktopDisplayList.begin(); + std::advance(iter, aIndex); + DesktopDisplayDevice* desktopDisplayDevice = iter->second; + if (desktopDisplayDevice) { + aDesktopDisplayDevice = (*desktopDisplayDevice); + } + + return 0; +} + +int32_t DesktopDeviceInfoImpl::getWindowCount() { + return static_cast<int32_t>(mDesktopWindowList.size()); +} + +int32_t DesktopDeviceInfoImpl::getWindowInfo( + uint32_t aIndex, DesktopDisplayDevice& aWindowDevice) { + if (aIndex >= mDesktopWindowList.size()) { + return -1; + } + + std::map<intptr_t, DesktopDisplayDevice*>::iterator itr = + mDesktopWindowList.begin(); + std::advance(itr, aIndex); + DesktopDisplayDevice* window = itr->second; + if (!window) { + return -1; + } + + aWindowDevice = (*window); + return 0; +} + +uint32_t DesktopDeviceInfoImpl::getTabCount() { return mDesktopTabList.size(); } + +int32_t DesktopDeviceInfoImpl::getTabInfo(uint32_t aIndex, + DesktopTab& aDesktopTab) { + if (aIndex >= mDesktopTabList.size()) { + return -1; + } + + std::map<intptr_t, DesktopTab*>::iterator iter = mDesktopTabList.begin(); + std::advance(iter, aIndex); + DesktopTab* desktopTab = iter->second; + if (desktopTab) { + aDesktopTab = (*desktopTab); + } + + return 0; +} + +void DesktopDeviceInfoImpl::CleanUp() { + CleanUpScreenList(); + CleanUpWindowList(); + CleanUpTabList(); +} +int32_t DesktopDeviceInfoImpl::Init() { + InitializeScreenList(); + InitializeWindowList(); + InitializeTabList(); + + return 0; +} +int32_t DesktopDeviceInfoImpl::Refresh() { + RefreshScreenList(); + RefreshWindowList(); + RefreshTabList(); + + return 0; +} + +void DesktopDeviceInfoImpl::CleanUpWindowList() { + std::map<intptr_t, DesktopDisplayDevice*>::iterator iterWindow; + for (iterWindow = mDesktopWindowList.begin(); + iterWindow != mDesktopWindowList.end(); iterWindow++) { + DesktopDisplayDevice* aWindow = iterWindow->second; + delete aWindow; + iterWindow->second = nullptr; + } + mDesktopWindowList.clear(); +} + +void DesktopDeviceInfoImpl::InitializeWindowList() { + DesktopCaptureOptions options; + +// Wayland is special and we will not get any information about windows +// without going through xdg-desktop-portal. We will already have +// a screen placeholder so there is no reason to build windows list. +#if defined(WEBRTC_USE_PIPEWIRE) + if (mozilla::StaticPrefs::media_webrtc_capture_allow_pipewire() && + webrtc::DesktopCapturer::IsRunningUnderWayland()) { + return; + } +#endif + +// Help avoid an X11 deadlock, see bug 1456101. +#ifdef MOZ_X11 + MOZ_ALWAYS_SUCCEEDS(mozilla::SyncRunnable::DispatchToThread( + mozilla::GetMainThreadSerialEventTarget(), + NS_NewRunnableFunction(__func__, [&] { + options = DesktopCaptureOptions::CreateDefault(); + }))); +#else + options = DesktopCaptureOptions::CreateDefault(); +#endif + std::unique_ptr<DesktopCapturer> winCap = + DesktopCapturer::CreateWindowCapturer(options); + DesktopCapturer::SourceList list; + if (winCap && winCap->GetSourceList(&list)) { + DesktopCapturer::SourceList::iterator itr; + for (itr = list.begin(); itr != list.end(); itr++) { + DesktopDisplayDevice* winDevice = new DesktopDisplayDevice; + if (!winDevice) { + continue; + } + + winDevice->setScreenId(itr->id); + winDevice->setDeviceName(itr->title.c_str()); + winDevice->setPid(itr->pid); + + char idStr[BUFSIZ]; +#if WEBRTC_WIN + _snprintf_s(idStr, sizeof(idStr), sizeof(idStr) - 1, "%ld", + static_cast<long>(winDevice->getScreenId())); +#else + SprintfLiteral(idStr, "%ld", static_cast<long>(winDevice->getScreenId())); +#endif + winDevice->setUniqueIdName(idStr); + mDesktopWindowList[winDevice->getScreenId()] = winDevice; + } + } +} + +void DesktopDeviceInfoImpl::RefreshWindowList() { + CleanUpWindowList(); + InitializeWindowList(); +} + +void DesktopDeviceInfoImpl::CleanUpTabList() { + for (auto& iterTab : mDesktopTabList) { + DesktopTab* desktopTab = iterTab.second; + delete desktopTab; + iterTab.second = nullptr; + } + mDesktopTabList.clear(); +} + +void webrtc::DesktopDeviceInfoImpl::InitializeTabList() { + if (!mozilla::StaticPrefs::media_getusermedia_browser_enabled()) { + return; + } + + // This is a sync dispatch to main thread, which is unfortunate. To + // call JavaScript we have to be on main thread, but the remaining + // DesktopCapturer very much wants to be off main thread. This might + // be solvable by calling this method earlier on while we're still on + // main thread and plumbing the information down to here. + nsCOMPtr<nsIRunnable> runnable = NS_NewRunnableFunction(__func__, [&] { + nsresult rv; + nsCOMPtr<nsIBrowserWindowTracker> bwt = + do_ImportModule("resource:///modules/BrowserWindowTracker.jsm", + "BrowserWindowTracker", &rv); + if (NS_FAILED(rv)) { + return; + } + + nsTArray<RefPtr<nsIVisibleTab>> tabArray; + rv = bwt->GetAllVisibleTabs(tabArray); + if (NS_FAILED(rv)) { + return; + } + + for (const auto& browserTab : tabArray) { + nsString contentTitle; + browserTab->GetContentTitle(contentTitle); + int64_t browserId; + browserTab->GetBrowserId(&browserId); + + DesktopTab* desktopTab = new DesktopTab; + if (desktopTab) { + char* contentTitleUTF8 = ToNewUTF8String(contentTitle); + desktopTab->setTabBrowserId(browserId); + desktopTab->setTabName(contentTitleUTF8); + std::ostringstream uniqueId; + uniqueId << browserId; + desktopTab->setUniqueIdName(uniqueId.str().c_str()); + mDesktopTabList[static_cast<intptr_t>(desktopTab->getTabBrowserId())] = + desktopTab; + free(contentTitleUTF8); + } + } + }); + mozilla::SyncRunnable::DispatchToThread( + mozilla::GetMainThreadSerialEventTarget(), runnable); +} + +void DesktopDeviceInfoImpl::RefreshTabList() { + CleanUpTabList(); + InitializeTabList(); +} + +void DesktopDeviceInfoImpl::CleanUpScreenList() { + std::map<intptr_t, DesktopDisplayDevice*>::iterator iterDevice; + for (iterDevice = mDesktopDisplayList.begin(); + iterDevice != mDesktopDisplayList.end(); iterDevice++) { + DesktopDisplayDevice* desktopDisplayDevice = iterDevice->second; + delete desktopDisplayDevice; + iterDevice->second = nullptr; + } + mDesktopDisplayList.clear(); +} + +// With PipeWire we can't select which system resource is shared so +// we don't create a window/screen list. Instead we place these constants +// as window name/id so frontend code can identify PipeWire backend +// and does not try to create screen/window preview. + +#define PIPEWIRE_ID 0xaffffff +#define PIPEWIRE_NAME "####_PIPEWIRE_PORTAL_####" + +void DesktopDeviceInfoImpl::InitializeScreenList() { + DesktopCaptureOptions options; + +// Wayland is special and we will not get any information about screens +// without going through xdg-desktop-portal so we just need a screen +// placeholder. +#if defined(WEBRTC_USE_PIPEWIRE) + if (mozilla::StaticPrefs::media_webrtc_capture_allow_pipewire() && + webrtc::DesktopCapturer::IsRunningUnderWayland()) { + DesktopDisplayDevice* screenDevice = new DesktopDisplayDevice; + if (!screenDevice) { + return; + } + + screenDevice->setScreenId(PIPEWIRE_ID); + screenDevice->setDeviceName(PIPEWIRE_NAME); + + char idStr[BUFSIZ]; + SprintfLiteral(idStr, "%ld", + static_cast<long>(screenDevice->getScreenId())); + screenDevice->setUniqueIdName(idStr); + mDesktopDisplayList[screenDevice->getScreenId()] = screenDevice; + return; + } +#endif + +// Help avoid an X11 deadlock, see bug 1456101. +#ifdef MOZ_X11 + MOZ_ALWAYS_SUCCEEDS(mozilla::SyncRunnable::DispatchToThread( + mozilla::GetMainThreadSerialEventTarget(), + NS_NewRunnableFunction(__func__, [&] { + options = DesktopCaptureOptions::CreateDefault(); + }))); +#else + options = DesktopCaptureOptions::CreateDefault(); +#endif + std::unique_ptr<DesktopCapturer> screenCapturer = + DesktopCapturer::CreateScreenCapturer(options); + DesktopCapturer::SourceList list; + if (screenCapturer && screenCapturer->GetSourceList(&list)) { + DesktopCapturer::SourceList::iterator itr; + for (itr = list.begin(); itr != list.end(); itr++) { + DesktopDisplayDevice* screenDevice = new DesktopDisplayDevice; + screenDevice->setScreenId(itr->id); + if (list.size() == 1) { + screenDevice->setDeviceName("Primary Monitor"); + } else { + screenDevice->setDeviceName(itr->title.c_str()); + } + screenDevice->setPid(itr->pid); + + char idStr[BUFSIZ]; +#if WEBRTC_WIN + _snprintf_s(idStr, sizeof(idStr), sizeof(idStr) - 1, "%ld", + static_cast<long>(screenDevice->getScreenId())); +#else + SprintfLiteral(idStr, "%ld", + static_cast<long>(screenDevice->getScreenId())); +#endif + screenDevice->setUniqueIdName(idStr); + mDesktopDisplayList[screenDevice->getScreenId()] = screenDevice; + } + } +} + +void DesktopDeviceInfoImpl::RefreshScreenList() { + CleanUpScreenList(); + InitializeScreenList(); +} + +/* static */ +DesktopDeviceInfo* DesktopDeviceInfo::Create() { + auto info = mozilla::MakeUnique<DesktopDeviceInfoImpl>(); + if (info->Init() != 0) { + return nullptr; + } + return info.release(); +} +} // namespace webrtc diff --git a/dom/media/systemservices/video_engine/desktop_device_info.h b/dom/media/systemservices/video_engine/desktop_device_info.h new file mode 100644 index 0000000000..824792b3c0 --- /dev/null +++ b/dom/media/systemservices/video_engine/desktop_device_info.h @@ -0,0 +1,84 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef WEBRTC_MODULES_DESKTOP_CAPTURE_DEVICE_INFO_H_ +#define WEBRTC_MODULES_DESKTOP_CAPTURE_DEVICE_INFO_H_ + +#include <map> +#include "modules/desktop_capture/desktop_capture_types.h" + +namespace webrtc { + +class DesktopDisplayDevice { + public: + DesktopDisplayDevice(); + ~DesktopDisplayDevice(); + + void setScreenId(const ScreenId aScreenId); + void setDeviceName(const char* aDeviceNameUTF8); + void setUniqueIdName(const char* aDeviceUniqueIdUTF8); + void setPid(pid_t aPid); + + ScreenId getScreenId(); + const char* getDeviceName(); + const char* getUniqueIdName(); + pid_t getPid(); + + DesktopDisplayDevice& operator=(DesktopDisplayDevice& aOther); + + protected: + ScreenId mScreenId; + char* mDeviceNameUTF8; + char* mDeviceUniqueIdUTF8; + pid_t mPid; +}; + +using DesktopDisplayDeviceList = std::map<intptr_t, DesktopDisplayDevice*>; + +class DesktopTab { + public: + DesktopTab(); + ~DesktopTab(); + + void setTabBrowserId(uint64_t aTabBrowserId); + void setUniqueIdName(const char* aTabUniqueIdUTF8); + void setTabName(const char* aTabNameUTF8); + void setTabCount(const uint32_t aCount); + + uint64_t getTabBrowserId(); + const char* getUniqueIdName(); + const char* getTabName(); + uint32_t getTabCount(); + + DesktopTab& operator=(DesktopTab& aOther); + + protected: + uint64_t mTabBrowserId; + char* mTabNameUTF8; + char* mTabUniqueIdUTF8; + uint32_t mTabCount; +}; + +using DesktopTabList = std::map<intptr_t, DesktopTab*>; + +class DesktopDeviceInfo { + public: + virtual ~DesktopDeviceInfo() = default; + + virtual int32_t Init() = 0; + virtual int32_t Refresh() = 0; + virtual int32_t getDisplayDeviceCount() = 0; + virtual int32_t getDesktopDisplayDeviceInfo( + uint32_t aIndex, DesktopDisplayDevice& aDesktopDisplayDevice) = 0; + virtual int32_t getWindowCount() = 0; + virtual int32_t getWindowInfo(uint32_t aIndex, + DesktopDisplayDevice& aWindowDevice) = 0; + virtual uint32_t getTabCount() = 0; + virtual int32_t getTabInfo(uint32_t aIndex, DesktopTab& aDesktopTab) = 0; + + static DesktopDeviceInfo* Create(); +}; +}; // namespace webrtc + +#endif diff --git a/dom/media/systemservices/video_engine/platform_uithread.cc b/dom/media/systemservices/video_engine/platform_uithread.cc new file mode 100644 index 0000000000..701a989a18 --- /dev/null +++ b/dom/media/systemservices/video_engine/platform_uithread.cc @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#if defined(WEBRTC_WIN) + +# include "platform_uithread.h" + +namespace rtc { + +// timer id used in delayed callbacks +static const UINT_PTR kTimerId = 1; +static const wchar_t kThisProperty[] = L"ThreadWindowsUIPtr"; +static const wchar_t kThreadWindow[] = L"WebrtcWindowsUIThread"; + +PlatformUIThread::~PlatformUIThread() { + CritScope scoped_lock(&cs_); + switch (state_) { + case State::STARTED: { + MOZ_DIAGNOSTIC_ASSERT( + false, "PlatformUIThread must be stopped before destruction"); + break; + } + case State::STOPPED: + break; + case State::UNSTARTED: + break; + } +} + +bool PlatformUIThread::InternalInit() { + // Create an event window for use in generating callbacks to capture + // objects. + CritScope scoped_lock(&cs_); + switch (state_) { + // We have already started there is nothing todo. Should this be assert? + case State::STARTED: + break; + // Stop() has already been called so there is likewise nothing to do. + case State::STOPPED: + break; + // Stop() has not been called yet, setup the UI thread, and set our + // state to STARTED. + case State::UNSTARTED: { + WNDCLASSW wc; + HMODULE hModule = GetModuleHandle(NULL); + if (!GetClassInfoW(hModule, kThreadWindow, &wc)) { + ZeroMemory(&wc, sizeof(WNDCLASSW)); + wc.hInstance = hModule; + wc.lpfnWndProc = EventWindowProc; + wc.lpszClassName = kThreadWindow; + RegisterClassW(&wc); + } + hwnd_ = CreateWindowW(kThreadWindow, L"", 0, 0, 0, 0, 0, NULL, NULL, + hModule, NULL); + // Added in review of bug 1760843, follow up to remove 1767861 + MOZ_RELEASE_ASSERT(hwnd_); + // Expected to always work but if it doesn't we should still fulfill the + // contract of always running the process loop at least a single + // iteration. + // This could be rexamined in the future. + if (hwnd_) { + SetPropW(hwnd_, kThisProperty, this); + // state_ needs to be STARTED before we request the initial timer + state_ = State::STARTED; + if (timeout_) { + // if someone set the timer before we started + RequestCallbackTimer(timeout_); + } + } + break; + } + }; + return state_ == State::STARTED; +} + +bool PlatformUIThread::RequestCallbackTimer(unsigned int milliseconds) { + CritScope scoped_lock(&cs_); + + switch (state_) { + // InternalInit() has yet to run so we do not have a UI thread to use as a + // target of the timer. We should just remember what timer interval was + // requested and let InternalInit() call this function again when it is + // ready. + case State::UNSTARTED: { + timeout_ = milliseconds; + return false; + } + // We have already stopped, do not schedule a new timer. + case State::STOPPED: + return false; + case State::STARTED: { + if (timerid_) { + KillTimer(hwnd_, timerid_); + } + timeout_ = milliseconds; + timerid_ = SetTimer(hwnd_, kTimerId, milliseconds, NULL); + return !!timerid_; + } + } + // UNREACHABLE +} + +void PlatformUIThread::Stop() { + { + RTC_DCHECK_RUN_ON(&thread_checker_); + CritScope scoped_lock(&cs_); + // Shut down the dispatch loop and let the background thread exit. + if (timerid_) { + MOZ_ASSERT(hwnd_); + KillTimer(hwnd_, timerid_); + timerid_ = 0; + } + switch (state_) { + // If we haven't started yet there is nothing to do, we will go into + // the STOPPED state at the end of the function and InternalInit() + // will not move us to STARTED. + case State::UNSTARTED: + break; + // If we have started, that means that InternalInit() has run and the + // message wait loop has or will run. We need to signal it to stop. wich + // will allow PlatformThread::Stop to join that thread. + case State::STARTED: { + MOZ_ASSERT(hwnd_); + PostMessage(hwnd_, WM_CLOSE, 0, 0); + break; + } + // We have already stopped. There is nothing to do. + case State::STOPPED: + break; + } + // Always set our state to STOPPED + state_ = State::STOPPED; + } + monitor_thread_.Finalize(); +} + +void PlatformUIThread::Run() { + // InternalInit() will return false when the thread is already in shutdown. + // otherwise we must run until we get a Windows WM_QUIT msg. + const bool runUntilQuitMsg = InternalInit(); + // The interface contract of Start/Stop is that for a successful call to + // Start, there should be at least one call to the run function. + NativeEventCallback(); + while (runUntilQuitMsg) { + // Alertable sleep to receive WM_QUIT (following a WM_CLOSE triggering a + // WM_DESTROY) + if (MsgWaitForMultipleObjectsEx(0, nullptr, INFINITE, QS_ALLINPUT, + MWMO_ALERTABLE | MWMO_INPUTAVAILABLE) == + WAIT_OBJECT_0) { + MSG msg; + if (PeekMessage(&msg, NULL, 0, 0, PM_REMOVE)) { + if (msg.message == WM_QUIT) { + // THE ONLY WAY to exit the thread loop + break; + } + TranslateMessage(&msg); + DispatchMessage(&msg); + } + } + } +} + +void PlatformUIThread::NativeEventCallback() { native_event_callback_(); } + +/* static */ +LRESULT CALLBACK PlatformUIThread::EventWindowProc(HWND hwnd, UINT uMsg, + WPARAM wParam, + LPARAM lParam) { + if (uMsg == WM_DESTROY) { + RemovePropW(hwnd, kThisProperty); + PostQuitMessage(0); + return 0; + } + + PlatformUIThread* twui = + static_cast<PlatformUIThread*>(GetPropW(hwnd, kThisProperty)); + if (!twui) { + return DefWindowProc(hwnd, uMsg, wParam, lParam); + } + + if (uMsg == WM_TIMER && wParam == kTimerId) { + twui->NativeEventCallback(); + return 0; + } + + return DefWindowProc(hwnd, uMsg, wParam, lParam); +} + +} // namespace rtc + +#endif diff --git a/dom/media/systemservices/video_engine/platform_uithread.h b/dom/media/systemservices/video_engine/platform_uithread.h new file mode 100644 index 0000000000..9c213ca933 --- /dev/null +++ b/dom/media/systemservices/video_engine/platform_uithread.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_PLATFORM_UITHREAD_H_ +#define RTC_BASE_PLATFORM_UITHREAD_H_ + +#if defined(WEBRTC_WIN) +# include "Assertions.h" +# include "rtc_base/deprecated/recursive_critical_section.h" +# include "rtc_base/platform_thread.h" +# include "api/sequence_checker.h" +# include "ThreadSafety.h" + +namespace rtc { +/* + * Windows UI thread for screen capture + * Launches a thread which enters a message wait loop after calling the + * provided ThreadRunFunction once. A repeating timer event might be registered + * with a callback through the Win32 API. If so, that timer will cause WM_TIMER + * messages to appear in the threads message queue. This will wake the thread + * which will then first look to see if it received the WM_QUIT message, then + * it will pass any non WM_QUIT messages on to the registered message handlers + * (synchronously on the current thread). In the case oF WM_TIMER the + * registered handler calls the NativeEventCallback which is simply the + * ThreadRunFunction which was passed to the constructor. + * + * Shutdown of the message wait loop is triggered by sending a WM_CLOSE which + * will start tearing down the "window" which hosts the UI thread. This will + * cause a WM_DESTROY message to be received. Upon reception a WM_QUIT message + * is enqueued. When the message wait loop receives a WM_QUIT message it stops, + * thus allowing the thread to be joined. + * + * Note: that the only source of a WM_CLOSE should be PlatformUIThread::Stop. + * Note: because PlatformUIThread::Stop is called from a different thread than + * PlatformUIThread::Run, it is possible that Stop can race Run. + * + * After being stopped PlatformUIThread can not be started again. + * + */ + +class PlatformUIThread { + public: + PlatformUIThread(std::function<void()> func, const char* name, + ThreadAttributes attributes) + : name_(name), + native_event_callback_(std::move(func)), + monitor_thread_(PlatformThread::SpawnJoinable([this]() { Run(); }, name, + attributes)) {} + + virtual ~PlatformUIThread(); + + void Stop(); + + /** + * Request a recurring callback. + */ + bool RequestCallbackTimer(unsigned int milliseconds); + + protected: + void Run(); + + private: + static LRESULT CALLBACK EventWindowProc(HWND, UINT, WPARAM, LPARAM); + void NativeEventCallback(); + // Initialize the UI thread that is servicing the timer events + bool InternalInit(); + + // Needs to be initialized before monitor_thread_ as it takes a string view to + // name_ + std::string name_; + RecursiveCriticalSection cs_; + std::function<void()> native_event_callback_; + webrtc::SequenceChecker thread_checker_; + PlatformThread monitor_thread_; + HWND hwnd_ MOZ_GUARDED_BY(cs_) = nullptr; + UINT_PTR timerid_ MOZ_GUARDED_BY(cs_) = 0; + unsigned int timeout_ MOZ_GUARDED_BY(cs_) = 0; + enum class State { + UNSTARTED, + STARTED, + STOPPED, + }; + State state_ MOZ_GUARDED_BY(cs_) = State::UNSTARTED; +}; + +} // namespace rtc + +#endif +#endif // RTC_BASE_PLATFORM_UITHREAD_H_ diff --git a/dom/media/systemservices/video_engine/tab_capturer.cc b/dom/media/systemservices/video_engine/tab_capturer.cc new file mode 100644 index 0000000000..793d965028 --- /dev/null +++ b/dom/media/systemservices/video_engine/tab_capturer.cc @@ -0,0 +1,331 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "tab_capturer.h" + +#include "desktop_device_info.h" +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_frame.h" +#include "mozilla/Logging.h" +#include "mozilla/dom/BrowsingContext.h" +#include "mozilla/dom/ImageBitmap.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/PromiseNativeHandler.h" +#include "mozilla/dom/WindowGlobalParent.h" +#include "mozilla/gfx/2D.h" +#include "mozilla/SpinEventLoopUntil.h" +#include "mozilla/TaskQueue.h" +#include "nsThreadUtils.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +mozilla::LazyLogModule gTabShareLog("TabShare"); +#define LOG_FUNC_IMPL(level) \ + MOZ_LOG( \ + gTabShareLog, level, \ + ("TabCapturerWebrtc %p: %s id=%" PRIu64, this, __func__, mBrowserId)) +#define LOG_FUNC() LOG_FUNC_IMPL(LogLevel::Debug) +#define LOG_FUNCV() LOG_FUNC_IMPL(LogLevel::Verbose) + +using namespace mozilla::dom; + +namespace mozilla { + +class CaptureFrameRequest { + using CapturePromise = TabCapturerWebrtc::CapturePromise; + + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(CaptureFrameRequest) + + CaptureFrameRequest() : mCaptureTime(TimeStamp::Now()) {} + + operator MozPromiseRequestHolder<CapturePromise>&() { return mRequest; } + + void Complete() { mRequest.Complete(); } + void Disconnect() { mRequest.Disconnect(); } + bool Exists() { return mRequest.Exists(); } + + protected: + virtual ~CaptureFrameRequest() { MOZ_RELEASE_ASSERT(!Exists()); } + + public: + const TimeStamp mCaptureTime; + + private: + MozPromiseRequestHolder<CapturePromise> mRequest; +}; + +TabCapturerWebrtc::TabCapturerWebrtc( + SourceId aSourceId, nsCOMPtr<nsISerialEventTarget> aCaptureThread) + : mBrowserId(aSourceId), + mMainThreadWorker( + TaskQueue::Create(do_AddRef(GetMainThreadSerialEventTarget()), + "TabCapturerWebrtc::mMainThreadWorker")), + mCallbackWorker(TaskQueue::Create(aCaptureThread.forget(), + "TabCapturerWebrtc::mCallbackWorker")) { + RTC_DCHECK_RUN_ON(&mControlChecker); + MOZ_ASSERT(aSourceId != 0); + mCallbackChecker.Detach(); + + LOG_FUNC(); +} + +// static +std::unique_ptr<webrtc::DesktopCapturer> TabCapturerWebrtc::Create( + SourceId aSourceId, nsCOMPtr<nsISerialEventTarget> aCaptureThread) { + return std::unique_ptr<webrtc::DesktopCapturer>( + new TabCapturerWebrtc(aSourceId, std::move(aCaptureThread))); +} + +TabCapturerWebrtc::~TabCapturerWebrtc() { + RTC_DCHECK_RUN_ON(&mCallbackChecker); + LOG_FUNC(); + + // mMainThreadWorker handles frame capture requests async. Since we're in the + // dtor, no more frame capture requests can be made through CaptureFrame(). It + // can be shut down now. + mMainThreadWorker->BeginShutdown(); + + // There may still be async frame capture requests in flight, waiting to be + // reported to mCallback on mCallbackWorker. Disconnect them (must be done on + // mCallbackWorker) and shut down mCallbackWorker to ensure nothing more can + // get queued to it. + MOZ_ALWAYS_SUCCEEDS( + mCallbackWorker->Dispatch(NS_NewRunnableFunction(__func__, [this] { + RTC_DCHECK_RUN_ON(&mCallbackChecker); + for (const auto& req : mRequests) { + DisconnectRequest(req); + } + mCallbackWorker->BeginShutdown(); + }))); + + // Block until the workers have run all pending tasks. We must do this for two + // reasons: + // - All runnables dispatched to mMainThreadWorker and mCallbackWorker capture + // the raw pointer `this` as they rely on `this` outliving the worker + // TaskQueues. + // - mCallback is only guaranteed to outlive `this`. No calls can be made to + // it after the dtor is finished. + + // Spin the underlying thread of mCallbackWorker, which we are currently on, + // until it is empty. We have no other way of waiting for mCallbackWorker to + // become empty while blocking the current call. + SpinEventLoopUntil<ProcessFailureBehavior::IgnoreAndContinue>( + "~TabCapturerWebrtc"_ns, [&] { return mCallbackWorker->IsEmpty(); }); + + // No need to await shutdown since it was shut down synchronously above. + mMainThreadWorker->AwaitIdle(); +} + +bool TabCapturerWebrtc::GetSourceList( + webrtc::DesktopCapturer::SourceList* aSources) { + MOZ_LOG(gTabShareLog, LogLevel::Debug, + ("TabShare: GetSourceList, result %zu", aSources->size())); + // XXX UI + return true; +} + +bool TabCapturerWebrtc::SelectSource(webrtc::DesktopCapturer::SourceId) { + MOZ_ASSERT_UNREACHABLE("Source is passed through ctor for constness"); + return true; +} + +bool TabCapturerWebrtc::FocusOnSelectedSource() { return true; } + +void TabCapturerWebrtc::Start(webrtc::DesktopCapturer::Callback* aCallback) { + RTC_DCHECK_RUN_ON(&mCallbackChecker); + RTC_DCHECK(!mCallback); + RTC_DCHECK(aCallback); + + LOG_FUNC(); + + mCallback = aCallback; +} + +void TabCapturerWebrtc::CaptureFrame() { + RTC_DCHECK_RUN_ON(&mCallbackChecker); + LOG_FUNCV(); + if (mRequests.GetSize() > 2) { + // Allow two async capture requests in flight + OnCaptureFrameFailure(); + return; + } + + auto request = MakeRefPtr<CaptureFrameRequest>(); + InvokeAsync(mMainThreadWorker, __func__, [this] { return CaptureFrameNow(); }) + ->Then(mCallbackWorker, __func__, + [this, request](CapturePromise::ResolveOrRejectValue&& aValue) { + if (!CompleteRequest(request)) { + // Request was disconnected or overrun. Failure has already + // been reported to the callback elsewhere. + return; + } + + if (aValue.IsReject()) { + OnCaptureFrameFailure(); + return; + } + + OnCaptureFrameSuccess(std::move(aValue.ResolveValue())); + }) + ->Track(*request); + mRequests.PushFront(request.forget()); +} + +void TabCapturerWebrtc::OnCaptureFrameSuccess( + UniquePtr<dom::ImageBitmapCloneData> aData) { + RTC_DCHECK_RUN_ON(&mCallbackChecker); + MOZ_DIAGNOSTIC_ASSERT(aData); + LOG_FUNCV(); + webrtc::DesktopSize size(aData->mPictureRect.Width(), + aData->mPictureRect.Height()); + webrtc::DesktopRect rect = webrtc::DesktopRect::MakeSize(size); + std::unique_ptr<webrtc::DesktopFrame> frame( + new webrtc::BasicDesktopFrame(size)); + + gfx::DataSourceSurface::ScopedMap map(aData->mSurface, + gfx::DataSourceSurface::READ); + if (!map.IsMapped()) { + OnCaptureFrameFailure(); + return; + } + frame->CopyPixelsFrom(map.GetData(), map.GetStride(), rect); + + mCallback->OnCaptureResult(webrtc::DesktopCapturer::Result::SUCCESS, + std::move(frame)); +} + +void TabCapturerWebrtc::OnCaptureFrameFailure() { + RTC_DCHECK_RUN_ON(&mCallbackChecker); + LOG_FUNC(); + mCallback->OnCaptureResult(webrtc::DesktopCapturer::Result::ERROR_TEMPORARY, + nullptr); +} + +bool TabCapturerWebrtc::IsOccluded(const webrtc::DesktopVector& aPos) { + return false; +} + +class TabCapturedHandler final : public PromiseNativeHandler { + public: + NS_DECL_ISUPPORTS + + using CapturePromise = TabCapturerWebrtc::CapturePromise; + + static void Create(Promise* aPromise, + MozPromiseHolder<CapturePromise> aHolder) { + MOZ_ASSERT(aPromise); + MOZ_ASSERT(NS_IsMainThread()); + + RefPtr<TabCapturedHandler> handler = + new TabCapturedHandler(std::move(aHolder)); + aPromise->AppendNativeHandler(handler); + } + + void ResolvedCallback(JSContext* aCx, JS::Handle<JS::Value> aValue, + ErrorResult& aRv) override { + MOZ_ASSERT(NS_IsMainThread()); + if (NS_WARN_IF(!aValue.isObject())) { + mHolder.Reject(NS_ERROR_UNEXPECTED, __func__); + return; + } + + RefPtr<ImageBitmap> bitmap; + if (NS_WARN_IF(NS_FAILED( + UNWRAP_OBJECT(ImageBitmap, &aValue.toObject(), bitmap)))) { + mHolder.Reject(NS_ERROR_UNEXPECTED, __func__); + return; + } + + UniquePtr<ImageBitmapCloneData> data = bitmap->ToCloneData(); + if (!data) { + mHolder.Reject(NS_ERROR_UNEXPECTED, __func__); + return; + } + + mHolder.Resolve(std::move(data), __func__); + } + + void RejectedCallback(JSContext* aCx, JS::Handle<JS::Value> aValue, + ErrorResult& aRv) override { + MOZ_ASSERT(NS_IsMainThread()); + mHolder.Reject(aRv.StealNSResult(), __func__); + } + + private: + explicit TabCapturedHandler(MozPromiseHolder<CapturePromise> aHolder) + : mHolder(std::move(aHolder)) {} + + ~TabCapturedHandler() = default; + + MozPromiseHolder<CapturePromise> mHolder; +}; + +NS_IMPL_ISUPPORTS0(TabCapturedHandler) + +bool TabCapturerWebrtc::CompleteRequest(CaptureFrameRequest* aRequest) { + RTC_DCHECK_RUN_ON(&mCallbackChecker); + if (!aRequest->Exists()) { + // Request was disconnected or overrun. mCallback has already been notified. + return false; + } + while (CaptureFrameRequest* req = mRequests.Peek()) { + if (req->mCaptureTime > aRequest->mCaptureTime) { + break; + } + // Pop the request before calling the callback, in case it could mutate + // mRequests, now or in the future. + RefPtr<CaptureFrameRequest> dropMe = mRequests.Pop(); + req->Complete(); + if (req->mCaptureTime < aRequest->mCaptureTime) { + OnCaptureFrameFailure(); + } + } + MOZ_DIAGNOSTIC_ASSERT(!aRequest->Exists()); + return true; +} + +void TabCapturerWebrtc::DisconnectRequest(CaptureFrameRequest* aRequest) { + RTC_DCHECK_RUN_ON(&mCallbackChecker); + LOG_FUNCV(); + aRequest->Disconnect(); + OnCaptureFrameFailure(); +} + +auto TabCapturerWebrtc::CaptureFrameNow() -> RefPtr<CapturePromise> { + MOZ_ASSERT(mMainThreadWorker->IsOnCurrentThread()); + LOG_FUNCV(); + + WindowGlobalParent* wgp = nullptr; + RefPtr<BrowsingContext> context = + BrowsingContext::GetCurrentTopByBrowserId(mBrowserId); + if (context) { + wgp = context->Canonical()->GetCurrentWindowGlobal(); + } + if (!wgp) { + // If we can't access the window, we just won't capture anything + return CapturePromise::CreateAndReject(NS_ERROR_NOT_AVAILABLE, __func__); + } + + // XXX This would be more efficient if we used CrossProcessPaint directly and + // returned a surface. + RefPtr<Promise> promise = + wgp->DrawSnapshot(nullptr, 1.0, "white"_ns, false, IgnoreErrors()); + if (!promise) { + return CapturePromise::CreateAndReject(NS_ERROR_FAILURE, __func__); + } + + MozPromiseHolder<CapturePromise> holder; + RefPtr<CapturePromise> p = holder.Ensure(__func__); + TabCapturedHandler::Create(promise, std::move(holder)); + return p; +} + +} // namespace mozilla diff --git a/dom/media/systemservices/video_engine/tab_capturer.h b/dom/media/systemservices/video_engine/tab_capturer.h new file mode 100644 index 0000000000..92c4fa2ad1 --- /dev/null +++ b/dom/media/systemservices/video_engine/tab_capturer.h @@ -0,0 +1,88 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_TAB_CAPTURER_H_ +#define MODULES_DESKTOP_CAPTURE_TAB_CAPTURER_H_ + +#include "api/sequence_checker.h" +#include "modules/desktop_capture/desktop_capturer.h" +#include "mozilla/MozPromise.h" +#include "nsDeque.h" +#include "nsThreadUtils.h" + +namespace mozilla { +namespace dom { +struct ImageBitmapCloneData; +} // namespace dom + +class CaptureFrameRequest; +class TabCapturedHandler; +class TaskQueue; + +class TabCapturerWebrtc : public webrtc::DesktopCapturer { + protected: + TabCapturerWebrtc(SourceId aSourceId, + nsCOMPtr<nsISerialEventTarget> aCaptureThread); + ~TabCapturerWebrtc(); + + public: + friend class CaptureFrameRequest; + friend class TabCapturedHandler; + + static std::unique_ptr<webrtc::DesktopCapturer> Create( + SourceId aSourceId, nsCOMPtr<nsISerialEventTarget> aCaptureThread); + + TabCapturerWebrtc(const TabCapturerWebrtc&) = delete; + TabCapturerWebrtc& operator=(const TabCapturerWebrtc&) = delete; + + // DesktopCapturer interface. + void Start(Callback* aCallback) override; + void CaptureFrame() override; + bool GetSourceList(SourceList* aSources) override; + bool SelectSource(SourceId) override; + bool FocusOnSelectedSource() override; + bool IsOccluded(const webrtc::DesktopVector& aPos) override; + + private: + // Capture code + using CapturePromise = + MozPromise<UniquePtr<dom::ImageBitmapCloneData>, nsresult, true>; + RefPtr<CapturePromise> CaptureFrameNow(); + + // Helper that checks for overrun requests. Returns true if aRequest had not + // been dropped due to disconnection or overrun. + // Note that if this returns true, the caller takes the responsibility to call + // mCallback with a capture result for aRequest. + bool CompleteRequest(CaptureFrameRequest* aRequest); + + // Helper that disconnects the request, and notifies mCallback of a temporary + // failure. + void DisconnectRequest(CaptureFrameRequest* aRequest); + + // Handle the result from the async callback from CaptureFrameNow. + void OnCaptureFrameSuccess(UniquePtr<dom::ImageBitmapCloneData> aData); + void OnCaptureFrameFailure(); + + const uint64_t mBrowserId; + const RefPtr<TaskQueue> mMainThreadWorker; + const RefPtr<TaskQueue> mCallbackWorker; + webrtc::SequenceChecker mControlChecker; + webrtc::SequenceChecker mCallbackChecker; + // Set in Start() and guaranteed by the owner of this class to outlive us. + webrtc::DesktopCapturer::Callback* mCallback + RTC_GUARDED_BY(mCallbackChecker) = nullptr; + + // mCallbackWorker only + nsRefPtrDeque<CaptureFrameRequest> mRequests RTC_GUARDED_BY(mCallbackChecker); +}; + +} // namespace mozilla + +#endif // MODULES_DESKTOP_CAPTURE_TAB_CAPTURER_H_ |