diff options
Diffstat (limited to '')
46 files changed, 8204 insertions, 0 deletions
diff --git a/dom/media/ipc/MFMediaEngineChild.cpp b/dom/media/ipc/MFMediaEngineChild.cpp new file mode 100644 index 0000000000..77c7b63c2b --- /dev/null +++ b/dom/media/ipc/MFMediaEngineChild.cpp @@ -0,0 +1,336 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MFMediaEngineChild.h" + +#include "MFMediaEngineUtils.h" +#include "RemoteDecoderManagerChild.h" +#include "mozilla/WindowsVersion.h" + +namespace mozilla { + +#define CLOG(msg, ...) \ + MOZ_LOG(gMFMediaEngineLog, LogLevel::Debug, \ + ("MFMediaEngineChild=%p, Id=%" PRId64 ", " msg, this, this->Id(), \ + ##__VA_ARGS__)) + +#define WLOG(msg, ...) \ + MOZ_LOG(gMFMediaEngineLog, LogLevel::Debug, \ + ("MFMediaEngineWrapper=%p, Id=%" PRId64 ", " msg, this, this->Id(), \ + ##__VA_ARGS__)) + +#define WLOGV(msg, ...) \ + MOZ_LOG(gMFMediaEngineLog, LogLevel::Verbose, \ + ("MFMediaEngineWrapper=%p, Id=%" PRId64 ", " msg, this, this->Id(), \ + ##__VA_ARGS__)) + +using media::TimeUnit; + +MFMediaEngineChild::MFMediaEngineChild(MFMediaEngineWrapper* aOwner, + FrameStatistics* aFrameStats) + : mOwner(aOwner), + mManagerThread(RemoteDecoderManagerChild::GetManagerThread()), + mMediaEngineId(0 /* invalid id, will be initialized later */), + mFrameStats(WrapNotNull(aFrameStats)) {} + +RefPtr<GenericNonExclusivePromise> MFMediaEngineChild::Init( + bool aShouldPreload) { + if (!mManagerThread) { + return GenericNonExclusivePromise::CreateAndReject(NS_ERROR_FAILURE, + __func__); + } + + if (!IsWin10OrLater()) { + CLOG("Only support MF media engine playback on Windows 10+"); + return GenericNonExclusivePromise::CreateAndReject(NS_ERROR_FAILURE, + __func__); + } + + CLOG("Init"); + MOZ_ASSERT(mMediaEngineId == 0); + RefPtr<MFMediaEngineChild> self = this; + RemoteDecoderManagerChild::LaunchUtilityProcessIfNeeded( + RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM) + ->Then( + mManagerThread, __func__, + [self, this, aShouldPreload](bool) { + RefPtr<RemoteDecoderManagerChild> manager = + RemoteDecoderManagerChild::GetSingleton( + RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM); + if (!manager || !manager->CanSend()) { + CLOG("Manager not exists or can't send"); + mInitPromiseHolder.RejectIfExists(NS_ERROR_FAILURE, __func__); + return; + } + + mIPDLSelfRef = this; + Unused << manager->SendPMFMediaEngineConstructor(this); + MediaEngineInfoIPDL info(aShouldPreload); + SendInitMediaEngine(info) + ->Then( + mManagerThread, __func__, + [self, this](uint64_t aId) { + mInitEngineRequest.Complete(); + // Id 0 is used to indicate error. + if (aId == 0) { + CLOG("Failed to initialize MFMediaEngineChild"); + mInitPromiseHolder.RejectIfExists(NS_ERROR_FAILURE, + __func__); + return; + } + mMediaEngineId = aId; + CLOG("Initialized MFMediaEngineChild"); + mInitPromiseHolder.ResolveIfExists(true, __func__); + }, + [self, + this](const mozilla::ipc::ResponseRejectReason& aReason) { + mInitEngineRequest.Complete(); + CLOG( + "Failed to initialize MFMediaEngineChild due to " + "IPC failure"); + mInitPromiseHolder.RejectIfExists(NS_ERROR_FAILURE, + __func__); + }) + ->Track(mInitEngineRequest); + }, + [self, this](nsresult aResult) { + CLOG("SendInitMediaEngine Failed"); + self->mInitPromiseHolder.Reject(NS_ERROR_FAILURE, __func__); + }); + return mInitPromiseHolder.Ensure(__func__); +} + +mozilla::ipc::IPCResult MFMediaEngineChild::RecvRequestSample(TrackType aType, + bool aIsEnough) { + AssertOnManagerThread(); + if (!mOwner) { + return IPC_OK(); + } + if (aType == TrackType::kVideoTrack) { + mOwner->NotifyEvent(aIsEnough ? ExternalEngineEvent::VideoEnough + : ExternalEngineEvent::RequestForVideo); + } else if (aType == TrackType::kAudioTrack) { + mOwner->NotifyEvent(aIsEnough ? ExternalEngineEvent::AudioEnough + : ExternalEngineEvent::RequestForAudio); + } + return IPC_OK(); +} + +mozilla::ipc::IPCResult MFMediaEngineChild::RecvUpdateCurrentTime( + double aCurrentTimeInSecond) { + AssertOnManagerThread(); + if (mOwner) { + mOwner->UpdateCurrentTime(aCurrentTimeInSecond); + } + return IPC_OK(); +} + +mozilla::ipc::IPCResult MFMediaEngineChild::RecvNotifyEvent( + MFMediaEngineEvent aEvent) { + AssertOnManagerThread(); + switch (aEvent) { + case MF_MEDIA_ENGINE_EVENT_FIRSTFRAMEREADY: + mOwner->NotifyEvent(ExternalEngineEvent::LoadedFirstFrame); + break; + case MF_MEDIA_ENGINE_EVENT_LOADEDDATA: + mOwner->NotifyEvent(ExternalEngineEvent::LoadedData); + break; + case MF_MEDIA_ENGINE_EVENT_WAITING: + mOwner->NotifyEvent(ExternalEngineEvent::Waiting); + break; + case MF_MEDIA_ENGINE_EVENT_SEEKED: + mOwner->NotifyEvent(ExternalEngineEvent::Seeked); + break; + case MF_MEDIA_ENGINE_EVENT_BUFFERINGSTARTED: + mOwner->NotifyEvent(ExternalEngineEvent::BufferingStarted); + break; + case MF_MEDIA_ENGINE_EVENT_BUFFERINGENDED: + mOwner->NotifyEvent(ExternalEngineEvent::BufferingEnded); + break; + case MF_MEDIA_ENGINE_EVENT_ENDED: + mOwner->NotifyEvent(ExternalEngineEvent::Ended); + break; + case MF_MEDIA_ENGINE_EVENT_PLAYING: + mOwner->NotifyEvent(ExternalEngineEvent::Playing); + break; + default: + NS_WARNING( + nsPrintfCString("Unhandled event=%s", MediaEngineEventToStr(aEvent)) + .get()); + break; + } + return IPC_OK(); +} + +mozilla::ipc::IPCResult MFMediaEngineChild::RecvNotifyError( + const MediaResult& aError) { + AssertOnManagerThread(); + mOwner->NotifyError(aError); + return IPC_OK(); +} + +mozilla::ipc::IPCResult MFMediaEngineChild::RecvUpdateStatisticData( + const StatisticData& aData) { + AssertOnManagerThread(); + const uint64_t currentRenderedFrames = mFrameStats->GetPresentedFrames(); + // Media engine won't tell us that which stage those dropped frames happened, + // so we treat all of them as the frames dropped in the a/v sync stage (sink). + const uint64_t currentDroppedSinkFrames = mFrameStats->GetDroppedSinkFrames(); + MOZ_ASSERT(aData.renderedFrames() >= currentRenderedFrames); + MOZ_ASSERT(aData.droppedFrames() >= currentDroppedSinkFrames); + mFrameStats->Accumulate({0, 0, aData.renderedFrames() - currentRenderedFrames, + 0, aData.droppedFrames() - currentDroppedSinkFrames, + 0}); + CLOG("Update statictis data (rendered %" PRIu64 " -> %" PRIu64 + ", dropped %" PRIu64 " -> %" PRIu64 ")", + currentRenderedFrames, mFrameStats->GetPresentedFrames(), + currentDroppedSinkFrames, mFrameStats->GetDroppedSinkFrames()); + return IPC_OK(); +} + +void MFMediaEngineChild::OwnerDestroyed() { + Unused << ManagerThread()->Dispatch(NS_NewRunnableFunction( + "MFMediaEngineChild::OwnerDestroy", [self = RefPtr{this}, this] { + self->mOwner = nullptr; + // Ask to destroy IPDL. + if (CanSend()) { + MFMediaEngineChild::Send__delete__(this); + } + })); +} + +void MFMediaEngineChild::IPDLActorDestroyed() { + AssertOnManagerThread(); + mIPDLSelfRef = nullptr; +} + +void MFMediaEngineChild::Shutdown() { + AssertOnManagerThread(); + SendShutdown(); + mInitPromiseHolder.RejectIfExists(NS_ERROR_FAILURE, __func__); + mInitEngineRequest.DisconnectIfExists(); +} + +MFMediaEngineWrapper::MFMediaEngineWrapper(ExternalEngineStateMachine* aOwner, + FrameStatistics* aFrameStats) + : ExternalPlaybackEngine(aOwner), + mEngine(new MFMediaEngineChild(this, aFrameStats)), + mCurrentTimeInSecond(0.0) {} + +RefPtr<GenericNonExclusivePromise> MFMediaEngineWrapper::Init( + bool aShouldPreload) { + WLOG("Init"); + return mEngine->Init(aShouldPreload); +} + +MFMediaEngineWrapper::~MFMediaEngineWrapper() { mEngine->OwnerDestroyed(); } + +void MFMediaEngineWrapper::Play() { + WLOG("Play"); + MOZ_ASSERT(IsInited()); + Unused << ManagerThread()->Dispatch( + NS_NewRunnableFunction("MFMediaEngineWrapper::Play", + [engine = mEngine] { engine->SendPlay(); })); +} + +void MFMediaEngineWrapper::Pause() { + WLOG("Pause"); + MOZ_ASSERT(IsInited()); + Unused << ManagerThread()->Dispatch( + NS_NewRunnableFunction("MFMediaEngineWrapper::Pause", + [engine = mEngine] { engine->SendPause(); })); +} + +void MFMediaEngineWrapper::Seek(const TimeUnit& aTargetTime) { + auto currentTimeInSecond = aTargetTime.ToSeconds(); + mCurrentTimeInSecond = currentTimeInSecond; + WLOG("Seek to %f", currentTimeInSecond); + MOZ_ASSERT(IsInited()); + Unused << ManagerThread()->Dispatch(NS_NewRunnableFunction( + "MFMediaEngineWrapper::Seek", [engine = mEngine, currentTimeInSecond] { + engine->SendSeek(currentTimeInSecond); + })); +} + +void MFMediaEngineWrapper::Shutdown() { + WLOG("Shutdown"); + Unused << ManagerThread()->Dispatch( + NS_NewRunnableFunction("MFMediaEngineWrapper::Shutdown", + [engine = mEngine] { engine->Shutdown(); })); +} + +void MFMediaEngineWrapper::SetPlaybackRate(double aPlaybackRate) { + WLOG("Set playback rate %f", aPlaybackRate); + MOZ_ASSERT(IsInited()); + Unused << ManagerThread()->Dispatch( + NS_NewRunnableFunction("MFMediaEngineWrapper::SetPlaybackRate", + [engine = mEngine, aPlaybackRate] { + engine->SendSetPlaybackRate(aPlaybackRate); + })); +} + +void MFMediaEngineWrapper::SetVolume(double aVolume) { + WLOG("Set volume %f", aVolume); + MOZ_ASSERT(IsInited()); + Unused << ManagerThread()->Dispatch(NS_NewRunnableFunction( + "MFMediaEngineWrapper::SetVolume", + [engine = mEngine, aVolume] { engine->SendSetVolume(aVolume); })); +} + +void MFMediaEngineWrapper::SetLooping(bool aLooping) { + WLOG("Set looping %d", aLooping); + MOZ_ASSERT(IsInited()); + Unused << ManagerThread()->Dispatch(NS_NewRunnableFunction( + "MFMediaEngineWrapper::SetLooping", + [engine = mEngine, aLooping] { engine->SendSetLooping(aLooping); })); +} + +void MFMediaEngineWrapper::SetPreservesPitch(bool aPreservesPitch) { + // Media Engine doesn't support this. +} + +void MFMediaEngineWrapper::NotifyEndOfStream(TrackInfo::TrackType aType) { + WLOG("NotifyEndOfStream, type=%s", TrackTypeToStr(aType)); + MOZ_ASSERT(IsInited()); + Unused << ManagerThread()->Dispatch(NS_NewRunnableFunction( + "MFMediaEngineWrapper::NotifyEndOfStream", + [engine = mEngine, aType] { engine->SendNotifyEndOfStream(aType); })); +} + +void MFMediaEngineWrapper::SetMediaInfo(const MediaInfo& aInfo) { + WLOG("SetMediaInfo, hasAudio=%d, hasVideo=%d", aInfo.HasAudio(), + aInfo.HasVideo()); + MOZ_ASSERT(IsInited()); + Unused << ManagerThread()->Dispatch(NS_NewRunnableFunction( + "MFMediaEngineWrapper::SetMediaInfo", [engine = mEngine, aInfo] { + MediaInfoIPDL info(aInfo.HasAudio() ? Some(aInfo.mAudio) : Nothing(), + aInfo.HasVideo() ? Some(aInfo.mVideo) : Nothing()); + engine->SendNotifyMediaInfo(info); + })); +} + +TimeUnit MFMediaEngineWrapper::GetCurrentPosition() { + return TimeUnit::FromSeconds(mCurrentTimeInSecond); +} + +void MFMediaEngineWrapper::UpdateCurrentTime(double aCurrentTimeInSecond) { + AssertOnManagerThread(); + WLOGV("Update current time %f", aCurrentTimeInSecond); + mCurrentTimeInSecond = aCurrentTimeInSecond; + NotifyEvent(ExternalEngineEvent::Timeupdate); +} + +void MFMediaEngineWrapper::NotifyEvent(ExternalEngineEvent aEvent) { + AssertOnManagerThread(); + WLOGV("Received event %s", ExternalEngineEventToStr(aEvent)); + mOwner->NotifyEvent(aEvent); +} + +void MFMediaEngineWrapper::NotifyError(const MediaResult& aError) { + AssertOnManagerThread(); + WLOG("Received error: %s", aError.Description().get()); + mOwner->NotifyError(aError); +} + +} // namespace mozilla diff --git a/dom/media/ipc/MFMediaEngineChild.h b/dom/media/ipc/MFMediaEngineChild.h new file mode 100644 index 0000000000..0c7c512317 --- /dev/null +++ b/dom/media/ipc/MFMediaEngineChild.h @@ -0,0 +1,123 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_IPC_MFMEDIAENGINECHILD_H_ +#define DOM_MEDIA_IPC_MFMEDIAENGINECHILD_H_ + +#include "ExternalEngineStateMachine.h" +#include "MFMediaEngineUtils.h" +#include "TimeUnits.h" +#include "mozilla/Atomics.h" +#include "mozilla/PMFMediaEngineChild.h" +#include "mozilla/NotNull.h" + +namespace mozilla { + +class MFMediaEngineWrapper; + +/** + * MFMediaEngineChild is a wrapper class for a MediaEngine in the content + * process. It communicates with MFMediaEngineParent in the remote process by + * using IPDL interfaces to send commands to the MediaEngine. + * https://docs.microsoft.com/en-us/windows/win32/api/mfmediaengine/nn-mfmediaengine-imfmediaengine + */ +class MFMediaEngineChild final : public PMFMediaEngineChild { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MFMediaEngineChild); + + MFMediaEngineChild(MFMediaEngineWrapper* aOwner, + FrameStatistics* aFrameStats); + + void OwnerDestroyed(); + void IPDLActorDestroyed(); + + RefPtr<GenericNonExclusivePromise> Init(bool aShouldPreload); + void Shutdown(); + + // Methods for PMFMediaEngineChild + mozilla::ipc::IPCResult RecvRequestSample(TrackInfo::TrackType aType, + bool aIsEnough); + mozilla::ipc::IPCResult RecvUpdateCurrentTime(double aCurrentTimeInSecond); + mozilla::ipc::IPCResult RecvNotifyEvent(MFMediaEngineEvent aEvent); + mozilla::ipc::IPCResult RecvNotifyError(const MediaResult& aError); + mozilla::ipc::IPCResult RecvUpdateStatisticData(const StatisticData& aData); + + nsISerialEventTarget* ManagerThread() { return mManagerThread; } + void AssertOnManagerThread() const { + MOZ_ASSERT(mManagerThread->IsOnCurrentThread()); + } + + uint64_t Id() const { return mMediaEngineId; } + + private: + ~MFMediaEngineChild() = default; + + // Only modified on the manager thread. + MFMediaEngineWrapper* MOZ_NON_OWNING_REF mOwner; + + const nsCOMPtr<nsISerialEventTarget> mManagerThread; + + // This represents an unique Id to indentify the media engine in the remote + // process. Zero is used for the status before initializaing Id from the + // remote process. + // Modified on the manager thread, and read on other threads. + Atomic<uint64_t> mMediaEngineId; + + RefPtr<MFMediaEngineChild> mIPDLSelfRef; + + MozPromiseHolder<GenericNonExclusivePromise> mInitPromiseHolder; + MozPromiseRequestHolder<InitMediaEnginePromise> mInitEngineRequest; + + // This is guaranteed always being alive in our lifetime. + NotNull<FrameStatistics*> const MOZ_NON_OWNING_REF mFrameStats; +}; + +/** + * MFMediaEngineWrapper acts as an external playback engine, which is + * implemented by using the Media Foundation Media Engine. It holds hold an + * actor used to communicate with the engine in the remote process. Its methods + * are all thread-safe. + */ +class MFMediaEngineWrapper final : public ExternalPlaybackEngine { + public: + MFMediaEngineWrapper(ExternalEngineStateMachine* aOwner, + FrameStatistics* aFrameStats); + ~MFMediaEngineWrapper(); + + // Methods for ExternalPlaybackEngine + RefPtr<GenericNonExclusivePromise> Init(bool aShouldPreload) override; + void Play() override; + void Pause() override; + void Seek(const media::TimeUnit& aTargetTime) override; + void Shutdown() override; + void SetPlaybackRate(double aPlaybackRate) override; + void SetVolume(double aVolume) override; + void SetLooping(bool aLooping) override; + void SetPreservesPitch(bool aPreservesPitch) override; + media::TimeUnit GetCurrentPosition() override; + void NotifyEndOfStream(TrackInfo::TrackType aType) override; + uint64_t Id() const override { return mEngine->Id(); } + void SetMediaInfo(const MediaInfo& aInfo) override; + + nsISerialEventTarget* ManagerThread() { return mEngine->ManagerThread(); } + void AssertOnManagerThread() const { mEngine->AssertOnManagerThread(); } + + private: + friend class MFMediaEngineChild; + + bool IsInited() const { return mEngine->Id() != 0; } + void UpdateCurrentTime(double aCurrentTimeInSecond); + void NotifyEvent(ExternalEngineEvent aEvent); + void NotifyError(const MediaResult& aError); + + const RefPtr<MFMediaEngineChild> mEngine; + + // The current time which we receive from the MediaEngine or set by the state + // machine when seeking. + std::atomic<double> mCurrentTimeInSecond; +}; + +} // namespace mozilla + +#endif // DOM_MEDIA_IPC_MFMEDIAENGINECHILD_H_ diff --git a/dom/media/ipc/MFMediaEngineParent.cpp b/dom/media/ipc/MFMediaEngineParent.cpp new file mode 100644 index 0000000000..010b867895 --- /dev/null +++ b/dom/media/ipc/MFMediaEngineParent.cpp @@ -0,0 +1,638 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MFMediaEngineParent.h" + +#include <audiosessiontypes.h> +#include <intsafe.h> +#include <mfapi.h> + +#include "MFMediaEngineExtension.h" +#include "MFMediaEngineVideoStream.h" +#include "MFMediaEngineUtils.h" +#include "MFMediaEngineStream.h" +#include "MFMediaSource.h" +#include "RemoteDecoderManagerParent.h" +#include "WMF.h" +#include "mozilla/ClearOnShutdown.h" +#include "mozilla/RemoteDecodeUtils.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/StaticMutex.h" +#include "mozilla/StaticPrefs_media.h" +#include "mozilla/StaticPtr.h" +#include "mozilla/WindowsVersion.h" +#include "mozilla/gfx/DeviceManagerDx.h" + +namespace mozilla { + +#define LOG(msg, ...) \ + MOZ_LOG(gMFMediaEngineLog, LogLevel::Debug, \ + ("MFMediaEngineParent=%p, Id=%" PRId64 ", " msg, this, this->Id(), \ + ##__VA_ARGS__)) + +using MediaEngineMap = nsTHashMap<nsUint64HashKey, MFMediaEngineParent*>; +static StaticAutoPtr<MediaEngineMap> sMediaEngines; + +using Microsoft::WRL::ComPtr; +using Microsoft::WRL::MakeAndInitialize; + +StaticMutex sMediaEnginesLock; + +static void RegisterMediaEngine(MFMediaEngineParent* aMediaEngine) { + MOZ_ASSERT(aMediaEngine); + StaticMutexAutoLock lock(sMediaEnginesLock); + if (!sMediaEngines) { + sMediaEngines = new MediaEngineMap(); + } + sMediaEngines->InsertOrUpdate(aMediaEngine->Id(), aMediaEngine); +} + +static void UnregisterMedieEngine(MFMediaEngineParent* aMediaEngine) { + StaticMutexAutoLock lock(sMediaEnginesLock); + if (sMediaEngines) { + sMediaEngines->Remove(aMediaEngine->Id()); + } +} + +/* static */ +MFMediaEngineParent* MFMediaEngineParent::GetMediaEngineById(uint64_t aId) { + StaticMutexAutoLock lock(sMediaEnginesLock); + return sMediaEngines->Get(aId); +} + +MFMediaEngineParent::MFMediaEngineParent(RemoteDecoderManagerParent* aManager, + nsISerialEventTarget* aManagerThread) + : mMediaEngineId(++sMediaEngineIdx), + mManager(aManager), + mManagerThread(aManagerThread) { + MOZ_ASSERT(aManager); + MOZ_ASSERT(aManagerThread); + MOZ_ASSERT(mMediaEngineId != 0); + MOZ_ASSERT(XRE_IsUtilityProcess()); + MOZ_ASSERT(GetCurrentSandboxingKind() == + ipc::SandboxingKind::MF_MEDIA_ENGINE_CDM); + LOG("Created MFMediaEngineParent"); + RegisterMediaEngine(this); + mIPDLSelfRef = this; + CreateMediaEngine(); +} + +MFMediaEngineParent::~MFMediaEngineParent() { + LOG("Destoryed MFMediaEngineParent"); + DestroyEngineIfExists(); + UnregisterMedieEngine(this); +} + +void MFMediaEngineParent::DestroyEngineIfExists( + const Maybe<MediaResult>& aError) { + LOG("DestroyEngineIfExists, hasError=%d", aError.isSome()); + ENGINE_MARKER("MFMediaEngineParent::DestroyEngineIfExists"); + mMediaEngineNotify = nullptr; + mMediaEngineExtension = nullptr; + if (mMediaSource) { + mMediaSource->ShutdownTaskQueue(); + mMediaSource = nullptr; + } + if (mMediaEngine) { + mMediaEngine->Shutdown(); + mMediaEngine = nullptr; + } + mMediaEngineEventListener.DisconnectIfExists(); + mRequestSampleListener.DisconnectIfExists(); + if (mDXGIDeviceManager) { + mDXGIDeviceManager = nullptr; + wmf::MFUnlockDXGIDeviceManager(); + } + if (mVirtualVideoWindow) { + DestroyWindow(mVirtualVideoWindow); + } + if (aError) { + Unused << SendNotifyError(*aError); + } +} + +void MFMediaEngineParent::CreateMediaEngine() { + LOG("CreateMediaEngine"); + auto errorExit = MakeScopeExit([&] { + MediaResult error(NS_ERROR_DOM_MEDIA_FATAL_ERR, "Failed to create engine"); + DestroyEngineIfExists(Some(error)); + }); + + if (!wmf::MediaFoundationInitializer::HasInitialized()) { + NS_WARNING("Failed to initialize media foundation"); + return; + } + + InitializeDXGIDeviceManager(); + InitializeVirtualVideoWindow(); + + // Create an attribute and set mandatory information that are required for + // a media engine creation. + ComPtr<IMFAttributes> creationAttributes; + RETURN_VOID_IF_FAILED(wmf::MFCreateAttributes(&creationAttributes, 6)); + RETURN_VOID_IF_FAILED( + MakeAndInitialize<MFMediaEngineNotify>(&mMediaEngineNotify)); + mMediaEngineEventListener = mMediaEngineNotify->MediaEngineEvent().Connect( + mManagerThread, this, &MFMediaEngineParent::HandleMediaEngineEvent); + RETURN_VOID_IF_FAILED(creationAttributes->SetUnknown( + MF_MEDIA_ENGINE_CALLBACK, mMediaEngineNotify.Get())); + RETURN_VOID_IF_FAILED(creationAttributes->SetUINT32( + MF_MEDIA_ENGINE_AUDIO_CATEGORY, AudioCategory_Media)); + RETURN_VOID_IF_FAILED( + MakeAndInitialize<MFMediaEngineExtension>(&mMediaEngineExtension)); + RETURN_VOID_IF_FAILED(creationAttributes->SetUnknown( + MF_MEDIA_ENGINE_EXTENSION, mMediaEngineExtension.Get())); + // TODO : SET MF_MEDIA_ENGINE_CONTENT_PROTECTION_FLAGS + if (mDXGIDeviceManager) { + RETURN_VOID_IF_FAILED(creationAttributes->SetUnknown( + MF_MEDIA_ENGINE_DXGI_MANAGER, mDXGIDeviceManager.Get())); + } + if (mVirtualVideoWindow) { + RETURN_VOID_IF_FAILED(creationAttributes->SetUINT64( + MF_MEDIA_ENGINE_OPM_HWND, + reinterpret_cast<uint64_t>(mVirtualVideoWindow))); + } + + ComPtr<IMFMediaEngineClassFactory> factory; + RETURN_VOID_IF_FAILED(CoCreateInstance(CLSID_MFMediaEngineClassFactory, + nullptr, CLSCTX_INPROC_SERVER, + IID_PPV_ARGS(&factory))); + const bool isLowLatency = + StaticPrefs::media_wmf_low_latency_enabled() && + !StaticPrefs::media_wmf_low_latency_force_disabled(); + static const DWORD MF_MEDIA_ENGINE_DEFAULT = 0; + RETURN_VOID_IF_FAILED(factory->CreateInstance( + isLowLatency ? MF_MEDIA_ENGINE_REAL_TIME_MODE : MF_MEDIA_ENGINE_DEFAULT, + creationAttributes.Get(), &mMediaEngine)); + + // TODO : deal with encrypted content (set ContentProtectionManager and cdm + // proxy) + + LOG("Created media engine successfully"); + mIsCreatedMediaEngine = true; + ENGINE_MARKER("MFMediaEngineParent::CreatedMediaEngine"); + errorExit.release(); +} + +void MFMediaEngineParent::InitializeDXGIDeviceManager() { + auto* deviceManager = gfx::DeviceManagerDx::Get(); + if (!deviceManager) { + return; + } + RefPtr<ID3D11Device> d3d11Device = deviceManager->CreateMediaEngineDevice(); + if (!d3d11Device) { + return; + } + + auto errorExit = MakeScopeExit([&] { + mDXGIDeviceManager = nullptr; + wmf::MFUnlockDXGIDeviceManager(); + }); + UINT deviceResetToken; + RETURN_VOID_IF_FAILED( + wmf::MFLockDXGIDeviceManager(&deviceResetToken, &mDXGIDeviceManager)); + RETURN_VOID_IF_FAILED( + mDXGIDeviceManager->ResetDevice(d3d11Device.get(), deviceResetToken)); + LOG("Initialized DXGI manager"); + errorExit.release(); +} + +void MFMediaEngineParent::InitializeVirtualVideoWindow() { + static ATOM sVideoWindowClass = 0; + if (!sVideoWindowClass) { + WNDCLASS wnd{}; + wnd.lpszClassName = L"MFMediaEngine"; + wnd.hInstance = nullptr; + wnd.lpfnWndProc = DefWindowProc; + sVideoWindowClass = RegisterClass(&wnd); + } + if (!sVideoWindowClass) { + HRESULT hr = HRESULT_FROM_WIN32(GetLastError()); + LOG("Failed to register video window class: %lX", hr); + return; + } + mVirtualVideoWindow = + CreateWindowEx(WS_EX_NOPARENTNOTIFY | WS_EX_LAYERED | WS_EX_TRANSPARENT | + WS_EX_NOREDIRECTIONBITMAP, + reinterpret_cast<wchar_t*>(sVideoWindowClass), L"", + WS_POPUP | WS_DISABLED | WS_CLIPSIBLINGS, 0, 0, 1, 1, + nullptr, nullptr, nullptr, nullptr); + if (!mVirtualVideoWindow) { + HRESULT hr = HRESULT_FROM_WIN32(GetLastError()); + LOG("Failed to create virtual window: %lX", hr); + return; + } + LOG("Initialized virtual window"); +} + +#ifndef ENSURE_EVENT_DISPATCH_DURING_PLAYING +# define ENSURE_EVENT_DISPATCH_DURING_PLAYING(event) \ + do { \ + if (mMediaEngine->IsPaused()) { \ + LOG("Ignore incorrect '%s' during pausing!", event); \ + return; \ + } \ + } while (false) +#endif + +void MFMediaEngineParent::HandleMediaEngineEvent( + MFMediaEngineEventWrapper aEvent) { + AssertOnManagerThread(); + ENGINE_MARKER_TEXT( + "MFMediaEngineParent::HandleMediaEngineEvent", + nsPrintfCString("%s", MediaEngineEventToStr(aEvent.mEvent))); + switch (aEvent.mEvent) { + case MF_MEDIA_ENGINE_EVENT_ERROR: { + MOZ_ASSERT(aEvent.mParam1 && aEvent.mParam2); + auto error = static_cast<MF_MEDIA_ENGINE_ERR>(*aEvent.mParam1); + auto result = static_cast<HRESULT>(*aEvent.mParam2); + NotifyError(error, result); + break; + } + case MF_MEDIA_ENGINE_EVENT_FORMATCHANGE: + case MF_MEDIA_ENGINE_EVENT_FIRSTFRAMEREADY: { + if (mMediaEngine->HasVideo()) { + EnsureDcompSurfaceHandle(); + } + [[fallthrough]]; + } + case MF_MEDIA_ENGINE_EVENT_LOADEDDATA: + case MF_MEDIA_ENGINE_EVENT_WAITING: + case MF_MEDIA_ENGINE_EVENT_SEEKED: + case MF_MEDIA_ENGINE_EVENT_BUFFERINGSTARTED: + case MF_MEDIA_ENGINE_EVENT_BUFFERINGENDED: + Unused << SendNotifyEvent(aEvent.mEvent); + break; + case MF_MEDIA_ENGINE_EVENT_PLAYING: + ENSURE_EVENT_DISPATCH_DURING_PLAYING( + MediaEngineEventToStr(aEvent.mEvent)); + Unused << SendNotifyEvent(aEvent.mEvent); + break; + case MF_MEDIA_ENGINE_EVENT_ENDED: { + ENSURE_EVENT_DISPATCH_DURING_PLAYING( + MediaEngineEventToStr(aEvent.mEvent)); + Unused << SendNotifyEvent(aEvent.mEvent); + UpdateStatisticsData(); + break; + } + case MF_MEDIA_ENGINE_EVENT_TIMEUPDATE: { + auto currentTimeInSeconds = mMediaEngine->GetCurrentTime(); + Unused << SendUpdateCurrentTime(currentTimeInSeconds); + UpdateStatisticsData(); + break; + } + default: + LOG("Unhandled event=%s", MediaEngineEventToStr(aEvent.mEvent)); + break; + } +} + +void MFMediaEngineParent::NotifyError(MF_MEDIA_ENGINE_ERR aError, + HRESULT aResult) { + // TODO : handle HRESULT 0x8004CD12, DRM_E_TEE_INVALID_HWDRM_STATE, which can + // happen during OS sleep/resume, or moving video to different graphics + // adapters. + if (aError == MF_MEDIA_ENGINE_ERR_NOERROR) { + return; + } + LOG("Notify error '%s', hr=%lx", MFMediaEngineErrorToStr(aError), aResult); + ENGINE_MARKER_TEXT( + "MFMediaEngineParent::NotifyError", + nsPrintfCString("%s, hr=%lx", MFMediaEngineErrorToStr(aError), aResult)); + switch (aError) { + case MF_MEDIA_ENGINE_ERR_ABORTED: + case MF_MEDIA_ENGINE_ERR_NETWORK: + // We ignore these two because we fetch data by ourselves. + return; + case MF_MEDIA_ENGINE_ERR_DECODE: { + MediaResult error(NS_ERROR_DOM_MEDIA_DECODE_ERR, "Decoder error"); + Unused << SendNotifyError(error); + return; + } + case MF_MEDIA_ENGINE_ERR_SRC_NOT_SUPPORTED: { + MediaResult error(NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR, + "Source not supported"); + Unused << SendNotifyError(error); + return; + } + case MF_MEDIA_ENGINE_ERR_ENCRYPTED: { + MediaResult error(NS_ERROR_DOM_MEDIA_FATAL_ERR, "Encrypted error"); + Unused << SendNotifyError(error); + return; + } + default: + MOZ_ASSERT_UNREACHABLE("Unsupported error"); + return; + } +} + +MFMediaEngineStreamWrapper* MFMediaEngineParent::GetMediaEngineStream( + TrackType aType, const CreateDecoderParams& aParam) { + // Has been shutdowned. + if (!mMediaSource) { + return nullptr; + } + LOG("Create a media engine decoder for %s", TrackTypeToStr(aType)); + if (aType == TrackType::kAudioTrack) { + auto* stream = mMediaSource->GetAudioStream(); + return new MFMediaEngineStreamWrapper(stream, stream->GetTaskQueue(), + aParam); + } + MOZ_ASSERT(aType == TrackType::kVideoTrack); + auto* stream = mMediaSource->GetVideoStream(); + stream->AsVideoStream()->SetKnowsCompositor(aParam.mKnowsCompositor); + stream->AsVideoStream()->SetConfig(aParam.mConfig); + return new MFMediaEngineStreamWrapper(stream, stream->GetTaskQueue(), aParam); +} + +mozilla::ipc::IPCResult MFMediaEngineParent::RecvInitMediaEngine( + const MediaEngineInfoIPDL& aInfo, InitMediaEngineResolver&& aResolver) { + AssertOnManagerThread(); + if (!mIsCreatedMediaEngine) { + aResolver(0); + return IPC_OK(); + } + // Metadata preload is controlled by content process side before creating + // media engine. + if (aInfo.preload()) { + // TODO : really need this? + Unused << mMediaEngine->SetPreload(MF_MEDIA_ENGINE_PRELOAD_AUTOMATIC); + } + aResolver(mMediaEngineId); + return IPC_OK(); +} + +mozilla::ipc::IPCResult MFMediaEngineParent::RecvNotifyMediaInfo( + const MediaInfoIPDL& aInfo) { + AssertOnManagerThread(); + MOZ_ASSERT(mIsCreatedMediaEngine, "Hasn't created media engine?"); + MOZ_ASSERT(!mMediaSource); + + LOG("RecvNotifyMediaInfo"); + + auto errorExit = MakeScopeExit([&] { + MediaResult error(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "Failed to setup media source"); + DestroyEngineIfExists(Some(error)); + }); + + // Create media source and set it to the media engine. + NS_ENSURE_TRUE( + SUCCEEDED(MakeAndInitialize<MFMediaSource>( + &mMediaSource, aInfo.audioInfo(), aInfo.videoInfo(), mManagerThread)), + IPC_OK()); + mMediaEngineExtension->SetMediaSource(mMediaSource.Get()); + + // We use the source scheme in order to let the media engine to load our + // custom source. + NS_ENSURE_TRUE( + SUCCEEDED(mMediaEngine->SetSource(SysAllocString(L"MFRendererSrc"))), + IPC_OK()); + + LOG("Finished setup our custom media source to the media engine"); + ENGINE_MARKER_TEXT( + "MFMediaEngineParent,FinishedSetupMediaSource", + nsPrintfCString( + "audio=%s, video=%s", + aInfo.audioInfo() ? aInfo.audioInfo()->mMimeType.BeginReading() + : "none", + aInfo.videoInfo() ? aInfo.videoInfo()->mMimeType.BeginReading() + : "none")); + mRequestSampleListener = mMediaSource->RequestSampleEvent().Connect( + mManagerThread, this, &MFMediaEngineParent::HandleRequestSample); + + errorExit.release(); + return IPC_OK(); +} + +mozilla::ipc::IPCResult MFMediaEngineParent::RecvPlay() { + AssertOnManagerThread(); + if (!mMediaEngine) { + LOG("Engine has been shutdowned!"); + return IPC_OK(); + } + LOG("Play, expected playback rate %f, default playback rate=%f", + mPlaybackRate, mMediaEngine->GetDefaultPlaybackRate()); + ENGINE_MARKER("MFMediaEngineParent,Play"); + NS_ENSURE_TRUE(SUCCEEDED(mMediaEngine->Play()), IPC_OK()); + return IPC_OK(); +} + +mozilla::ipc::IPCResult MFMediaEngineParent::RecvPause() { + AssertOnManagerThread(); + if (!mMediaEngine) { + LOG("Engine has been shutdowned!"); + return IPC_OK(); + } + LOG("Pause"); + ENGINE_MARKER("MFMediaEngineParent,Pause"); + NS_ENSURE_TRUE(SUCCEEDED(mMediaEngine->Pause()), IPC_OK()); + return IPC_OK(); +} + +mozilla::ipc::IPCResult MFMediaEngineParent::RecvSeek( + double aTargetTimeInSecond) { + AssertOnManagerThread(); + if (!mMediaEngine) { + return IPC_OK(); + } + + // If the target time is already equal to the current time, the media engine + // doesn't perform seek internally so we won't be able to receive `seeked` + // event. Therefore, we simply return `seeked` here because we're already in + // the target time. + const auto currentTimeInSeconds = mMediaEngine->GetCurrentTime(); + if (currentTimeInSeconds == aTargetTimeInSecond) { + Unused << SendNotifyEvent(MF_MEDIA_ENGINE_EVENT_SEEKED); + return IPC_OK(); + } + + LOG("Seek to %f", aTargetTimeInSecond); + ENGINE_MARKER_TEXT("MFMediaEngineParent,Seek", + nsPrintfCString("%f", aTargetTimeInSecond)); + NS_ENSURE_TRUE(SUCCEEDED(mMediaEngine->SetCurrentTime(aTargetTimeInSecond)), + IPC_OK()); + + return IPC_OK(); +} + +mozilla::ipc::IPCResult MFMediaEngineParent::RecvSetVolume(double aVolume) { + AssertOnManagerThread(); + if (mMediaEngine) { + LOG("SetVolume=%f", aVolume); + ENGINE_MARKER_TEXT("MFMediaEngineParent,SetVolume", + nsPrintfCString("%f", aVolume)); + NS_ENSURE_TRUE(SUCCEEDED(mMediaEngine->SetVolume(aVolume)), IPC_OK()); + } + return IPC_OK(); +} + +mozilla::ipc::IPCResult MFMediaEngineParent::RecvSetPlaybackRate( + double aPlaybackRate) { + AssertOnManagerThread(); + if (aPlaybackRate <= 0) { + LOG("Not support zero or negative playback rate"); + return IPC_OK(); + } + LOG("SetPlaybackRate=%f", aPlaybackRate); + ENGINE_MARKER_TEXT("MFMediaEngineParent,SetPlaybackRate", + nsPrintfCString("%f", aPlaybackRate)); + mPlaybackRate = aPlaybackRate; + NS_ENSURE_TRUE(SUCCEEDED(mMediaEngine->SetPlaybackRate(mPlaybackRate)), + IPC_OK()); + // The media Engine uses the default playback rate to determine the playback + // rate when calling `play()`. So if we don't change default playback rate + // together, the playback rate would fallback to 1 after pausing or + // seeking, which would be different from our expected playback rate. + NS_ENSURE_TRUE(SUCCEEDED(mMediaEngine->SetDefaultPlaybackRate(mPlaybackRate)), + IPC_OK()); + return IPC_OK(); +} + +mozilla::ipc::IPCResult MFMediaEngineParent::RecvSetLooping(bool aLooping) { + AssertOnManagerThread(); + // We handle looping by seeking back to the head by ourselves, so we don't + // rely on the media engine for looping. + return IPC_OK(); +} + +mozilla::ipc::IPCResult MFMediaEngineParent::RecvNotifyEndOfStream( + TrackInfo::TrackType aType) { + AssertOnManagerThread(); + MOZ_ASSERT(mMediaSource); + LOG("NotifyEndOfStream, type=%s", TrackTypeToStr(aType)); + mMediaSource->NotifyEndOfStream(aType); + return IPC_OK(); +} + +mozilla::ipc::IPCResult MFMediaEngineParent::RecvShutdown() { + AssertOnManagerThread(); + LOG("Shutdown"); + ENGINE_MARKER("MFMediaEngineParent,Shutdown"); + DestroyEngineIfExists(); + return IPC_OK(); +} + +void MFMediaEngineParent::Destroy() { + AssertOnManagerThread(); + mIPDLSelfRef = nullptr; +} + +void MFMediaEngineParent::HandleRequestSample(const SampleRequest& aRequest) { + AssertOnManagerThread(); + MOZ_ASSERT(aRequest.mType == TrackInfo::TrackType::kAudioTrack || + aRequest.mType == TrackInfo::TrackType::kVideoTrack); + Unused << SendRequestSample(aRequest.mType, aRequest.mIsEnough); +} + +void MFMediaEngineParent::AssertOnManagerThread() const { + MOZ_ASSERT(mManagerThread->IsOnCurrentThread()); +} + +void MFMediaEngineParent::EnsureDcompSurfaceHandle() { + AssertOnManagerThread(); + MOZ_ASSERT(mMediaEngine); + MOZ_ASSERT(mMediaEngine->HasVideo()); + + ComPtr<IMFMediaEngineEx> mediaEngineEx; + RETURN_VOID_IF_FAILED(mMediaEngine.As(&mediaEngineEx)); + DWORD width, height; + RETURN_VOID_IF_FAILED(mMediaEngine->GetNativeVideoSize(&width, &height)); + if (width != mDisplayWidth || height != mDisplayHeight) { + // Update stream size before asking for a handle. If we don't update the + // size, media engine will create the dcomp surface in a wrong size. If + // the size isn't changed, then we don't need to recreate the surface. + mDisplayWidth = width; + mDisplayHeight = height; + RECT rect = {0, 0, (LONG)mDisplayWidth, (LONG)mDisplayHeight}; + RETURN_VOID_IF_FAILED(mediaEngineEx->UpdateVideoStream( + nullptr /* pSrc */, &rect, nullptr /* pBorderClr */)); + LOG("Updated video size for engine=[%lux%lu]", mDisplayWidth, + mDisplayHeight); + ENGINE_MARKER_TEXT( + "MFMediaEngineParent,UpdateVideoSize", + nsPrintfCString("%lux%lu", mDisplayWidth, mDisplayHeight)); + } + + if (!mIsEnableDcompMode) { + RETURN_VOID_IF_FAILED(mediaEngineEx->EnableWindowlessSwapchainMode(true)); + LOG("Enabled dcomp swap chain mode"); + mIsEnableDcompMode = true; + ENGINE_MARKER("MFMediaEngineParent,EnabledSwapChain"); + } + + HANDLE surfaceHandle = INVALID_HANDLE_VALUE; + RETURN_VOID_IF_FAILED(mediaEngineEx->GetVideoSwapchainHandle(&surfaceHandle)); + if (surfaceHandle && surfaceHandle != INVALID_HANDLE_VALUE) { + LOG("EnsureDcompSurfaceHandle, handle=%p, size=[%lux%lu]", surfaceHandle, + width, height); + mMediaSource->SetDCompSurfaceHandle(surfaceHandle); + } else { + NS_WARNING("SurfaceHandle is not ready yet"); + } +} + +void MFMediaEngineParent::UpdateStatisticsData() { + AssertOnManagerThread(); + + // Statistic data is only for video. + if (!mMediaEngine->HasVideo()) { + return; + } + + ComPtr<IMFMediaEngineEx> mediaEngineEx; + RETURN_VOID_IF_FAILED(mMediaEngine.As(&mediaEngineEx)); + + struct scopePropVariant : public PROPVARIANT { + scopePropVariant() { PropVariantInit(this); } + ~scopePropVariant() { PropVariantClear(this); } + scopePropVariant(scopePropVariant const&) = delete; + scopePropVariant& operator=(scopePropVariant const&) = delete; + }; + + // https://docs.microsoft.com/en-us/windows/win32/api/mfmediaengine/ne-mfmediaengine-mf_media_engine_statistic + scopePropVariant renderedFramesProp, droppedFramesProp; + RETURN_VOID_IF_FAILED(mediaEngineEx->GetStatistics( + MF_MEDIA_ENGINE_STATISTIC_FRAMES_RENDERED, &renderedFramesProp)); + RETURN_VOID_IF_FAILED(mediaEngineEx->GetStatistics( + MF_MEDIA_ENGINE_STATISTIC_FRAMES_DROPPED, &droppedFramesProp)); + + const unsigned long renderedFrames = renderedFramesProp.ulVal; + const unsigned long droppedFrames = droppedFramesProp.ulVal; + + // As the amount of rendered frame MUST increase monotonically. If the new + // statistic data show the decreasing, which means the media engine has reset + // the statistic data and started a new one. (That will happens after calling + // flush internally) + if (renderedFrames < mCurrentPlaybackStatisticData.renderedFrames()) { + mPrevPlaybackStatisticData = + StatisticData{mPrevPlaybackStatisticData.renderedFrames() + + mCurrentPlaybackStatisticData.renderedFrames(), + mPrevPlaybackStatisticData.droppedFrames() + + mCurrentPlaybackStatisticData.droppedFrames()}; + mCurrentPlaybackStatisticData = StatisticData{}; + } + + if (mCurrentPlaybackStatisticData.renderedFrames() != renderedFrames || + mCurrentPlaybackStatisticData.droppedFrames() != droppedFrames) { + mCurrentPlaybackStatisticData = + StatisticData{renderedFrames, droppedFrames}; + const uint64_t totalRenderedFrames = + mPrevPlaybackStatisticData.renderedFrames() + + mCurrentPlaybackStatisticData.renderedFrames(); + const uint64_t totalDroppedFrames = + mPrevPlaybackStatisticData.droppedFrames() + + mCurrentPlaybackStatisticData.droppedFrames(); + LOG("Update statistic data, rendered=%" PRIu64 ", dropped=%" PRIu64, + totalRenderedFrames, totalDroppedFrames); + Unused << SendUpdateStatisticData( + StatisticData{totalRenderedFrames, totalDroppedFrames}); + } +} + +#undef LOG +#undef RETURN_IF_FAILED +#undef ENSURE_EVENT_DISPATCH_DURING_PLAYING + +} // namespace mozilla diff --git a/dom/media/ipc/MFMediaEngineParent.h b/dom/media/ipc/MFMediaEngineParent.h new file mode 100644 index 0000000000..ab93ce16cb --- /dev/null +++ b/dom/media/ipc/MFMediaEngineParent.h @@ -0,0 +1,136 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_IPC_MFMEDIAENGINEPARENT_H_ +#define DOM_MEDIA_IPC_MFMEDIAENGINEPARENT_H_ + +#include <Mfidl.h> +#include <winnt.h> +#include <wrl.h> + +#include "MediaInfo.h" +#include "MFMediaEngineExtra.h" +#include "MFMediaEngineNotify.h" +#include "MFMediaEngineUtils.h" +#include "MFMediaSource.h" +#include "PlatformDecoderModule.h" +#include "mozilla/PMFMediaEngineParent.h" + +namespace mozilla { + +class MFMediaEngineExtension; +class MFMediaEngineStreamWrapper; +class MFMediaSource; +class RemoteDecoderManagerParent; + +/** + * MFMediaEngineParent is a wrapper class for a MediaEngine in the RDD process. + * It's responsible to create the media engine and its related classes, such as + * a custom media source, media engine extension, media engine notify...e.t.c + * It communicates with MFMediaEngineChild in the content process to receive + * commands and direct them to the media engine. + * https://docs.microsoft.com/en-us/windows/win32/api/mfmediaengine/nn-mfmediaengine-imfmediaengine + */ +class MFMediaEngineParent final : public PMFMediaEngineParent { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MFMediaEngineParent); + MFMediaEngineParent(RemoteDecoderManagerParent* aManager, + nsISerialEventTarget* aManagerThread); + + using TrackType = TrackInfo::TrackType; + + static MFMediaEngineParent* GetMediaEngineById(uint64_t aId); + + MFMediaEngineStreamWrapper* GetMediaEngineStream( + TrackType aType, const CreateDecoderParams& aParam); + + uint64_t Id() const { return mMediaEngineId; } + + // Methods for PMFMediaEngineParent + mozilla::ipc::IPCResult RecvInitMediaEngine( + const MediaEngineInfoIPDL& aInfo, InitMediaEngineResolver&& aResolver); + mozilla::ipc::IPCResult RecvNotifyMediaInfo(const MediaInfoIPDL& aInfo); + mozilla::ipc::IPCResult RecvPlay(); + mozilla::ipc::IPCResult RecvPause(); + mozilla::ipc::IPCResult RecvSeek(double aTargetTimeInSecond); + mozilla::ipc::IPCResult RecvSetVolume(double aVolume); + mozilla::ipc::IPCResult RecvSetPlaybackRate(double aPlaybackRate); + mozilla::ipc::IPCResult RecvSetLooping(bool aLooping); + mozilla::ipc::IPCResult RecvNotifyEndOfStream(TrackInfo::TrackType aType); + mozilla::ipc::IPCResult RecvShutdown(); + + void Destroy(); + + private: + ~MFMediaEngineParent(); + + void CreateMediaEngine(); + + void InitializeVirtualVideoWindow(); + void InitializeDXGIDeviceManager(); + + void AssertOnManagerThread() const; + + void HandleMediaEngineEvent(MFMediaEngineEventWrapper aEvent); + void HandleRequestSample(const SampleRequest& aRequest); + + void NotifyError(MF_MEDIA_ENGINE_ERR aError, HRESULT aResult = 0); + + void DestroyEngineIfExists(const Maybe<MediaResult>& aError = Nothing()); + + void EnsureDcompSurfaceHandle(); + + void UpdateStatisticsData(); + + // This generates unique id for each MFMediaEngineParent instance, and it + // would be increased monotonically. + static inline uint64_t sMediaEngineIdx = 0; + + const uint64_t mMediaEngineId; + + // The life cycle of this class is determined by the actor in the content + // process, we would hold a reference until the content actor asks us to + // destroy. + RefPtr<MFMediaEngineParent> mIPDLSelfRef; + + const RefPtr<RemoteDecoderManagerParent> mManager; + const RefPtr<nsISerialEventTarget> mManagerThread; + + // Required classes for working with the media engine. + Microsoft::WRL::ComPtr<IMFMediaEngine> mMediaEngine; + Microsoft::WRL::ComPtr<MFMediaEngineNotify> mMediaEngineNotify; + Microsoft::WRL::ComPtr<MFMediaEngineExtension> mMediaEngineExtension; + Microsoft::WRL::ComPtr<MFMediaSource> mMediaSource; + + MediaEventListener mMediaEngineEventListener; + MediaEventListener mRequestSampleListener; + bool mIsCreatedMediaEngine = false; + + // A fake window handle passed to MF-based rendering pipeline for OPM. + HWND mVirtualVideoWindow = nullptr; + + Microsoft::WRL::ComPtr<IMFDXGIDeviceManager> mDXGIDeviceManager; + + // These will be always zero for audio playback. + DWORD mDisplayWidth = 0; + DWORD mDisplayHeight = 0; + + // When it's true, the media engine will output decoded video frames to a + // shareable dcomp surface. + bool mIsEnableDcompMode = false; + + float mPlaybackRate = 1.0; + + // When flush happens inside the media engine, it will reset the statistic + // data. Therefore, whenever the statistic data gets reset, we will use + // `mCurrentPlaybackStatisticData` to track new data and store previous data + // to `mPrevPlaybackStatisticData`. The sum of these two data is the total + // statistic data for playback. + StatisticData mCurrentPlaybackStatisticData; + StatisticData mPrevPlaybackStatisticData; +}; + +} // namespace mozilla + +#endif // DOM_MEDIA_IPC_MFMEDIAENGINEPARENT_H_ diff --git a/dom/media/ipc/MFMediaEngineUtils.cpp b/dom/media/ipc/MFMediaEngineUtils.cpp new file mode 100644 index 0000000000..516a9feffd --- /dev/null +++ b/dom/media/ipc/MFMediaEngineUtils.cpp @@ -0,0 +1,157 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "MFMediaEngineUtils.h" + +#include "WMFUtils.h" + +namespace mozilla { + +#define ENUM_TO_STR(enumVal) \ + case enumVal: \ + return #enumVal + +#define ENUM_TO_STR2(guid, enumVal) \ + if (guid == enumVal) { \ + return #enumVal; \ + } + +const char* MediaEventTypeToStr(MediaEventType aType) { + switch (aType) { + ENUM_TO_STR(MESourceUnknown); + ENUM_TO_STR(MESourceStarted); + ENUM_TO_STR(MEStreamStarted); + ENUM_TO_STR(MESourceSeeked); + ENUM_TO_STR(MEStreamSeeked); + ENUM_TO_STR(MENewStream); + ENUM_TO_STR(MEUpdatedStream); + ENUM_TO_STR(MESourceStopped); + ENUM_TO_STR(MEStreamStopped); + ENUM_TO_STR(MESourcePaused); + ENUM_TO_STR(MEStreamPaused); + ENUM_TO_STR(MEEndOfPresentation); + ENUM_TO_STR(MEEndOfStream); + ENUM_TO_STR(MEMediaSample); + ENUM_TO_STR(MEStreamTick); + ENUM_TO_STR(MEStreamThinMode); + ENUM_TO_STR(MEStreamFormatChanged); + ENUM_TO_STR(MESourceRateChanged); + ENUM_TO_STR(MEEndOfPresentationSegment); + ENUM_TO_STR(MESourceCharacteristicsChanged); + ENUM_TO_STR(MESourceRateChangeRequested); + ENUM_TO_STR(MESourceMetadataChanged); + ENUM_TO_STR(MESequencerSourceTopologyUpdated); + default: + return "Unknown MediaEventType"; + } +} + +const char* MediaEngineEventToStr(MF_MEDIA_ENGINE_EVENT aEvent) { + switch (aEvent) { + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_LOADSTART); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_PROGRESS); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_SUSPEND); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_ABORT); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_ERROR); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_EMPTIED); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_STALLED); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_PLAY); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_PAUSE); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_LOADEDMETADATA); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_LOADEDDATA); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_WAITING); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_PLAYING); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_CANPLAY); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_CANPLAYTHROUGH); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_SEEKING); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_SEEKED); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_TIMEUPDATE); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_ENDED); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_RATECHANGE); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_DURATIONCHANGE); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_VOLUMECHANGE); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_FORMATCHANGE); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_PURGEQUEUEDEVENTS); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_TIMELINE_MARKER); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_BALANCECHANGE); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_DOWNLOADCOMPLETE); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_BUFFERINGSTARTED); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_BUFFERINGENDED); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_FRAMESTEPCOMPLETED); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_NOTIFYSTABLESTATE); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_FIRSTFRAMEREADY); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_TRACKSCHANGE); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_OPMINFO); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_RESOURCELOST); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_DELAYLOADEVENT_CHANGED); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_STREAMRENDERINGERROR); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_SUPPORTEDRATES_CHANGED); + ENUM_TO_STR(MF_MEDIA_ENGINE_EVENT_AUDIOENDPOINTCHANGE); + default: + return "Unknown MF_MEDIA_ENGINE_EVENT"; + } +} + +const char* MFMediaEngineErrorToStr(MFMediaEngineError aError) { + switch (aError) { + ENUM_TO_STR(MF_MEDIA_ENGINE_ERR_NOERROR); + ENUM_TO_STR(MF_MEDIA_ENGINE_ERR_ABORTED); + ENUM_TO_STR(MF_MEDIA_ENGINE_ERR_NETWORK); + ENUM_TO_STR(MF_MEDIA_ENGINE_ERR_DECODE); + ENUM_TO_STR(MF_MEDIA_ENGINE_ERR_SRC_NOT_SUPPORTED); + ENUM_TO_STR(MF_MEDIA_ENGINE_ERR_ENCRYPTED); + default: + return "Unknown MFMediaEngineError"; + } +} + +const char* GUIDToStr(GUID aGUID) { + ENUM_TO_STR2(aGUID, MFAudioFormat_MP3) + ENUM_TO_STR2(aGUID, MFAudioFormat_AAC) + ENUM_TO_STR2(aGUID, MFAudioFormat_Vorbis) + ENUM_TO_STR2(aGUID, MFAudioFormat_Opus) + ENUM_TO_STR2(aGUID, MFVideoFormat_H264) + ENUM_TO_STR2(aGUID, MFVideoFormat_VP80) + ENUM_TO_STR2(aGUID, MFVideoFormat_VP90) + ENUM_TO_STR2(aGUID, MFVideoFormat_AV1) + ENUM_TO_STR2(aGUID, MFMediaType_Audio) + return "Unknown GUID"; +} + +const char* MFVideoRotationFormatToStr(MFVideoRotationFormat aFormat) { + switch (aFormat) { + ENUM_TO_STR(MFVideoRotationFormat_0); + ENUM_TO_STR(MFVideoRotationFormat_90); + ENUM_TO_STR(MFVideoRotationFormat_180); + ENUM_TO_STR(MFVideoRotationFormat_270); + default: + return "Unknown MFVideoRotationFormat"; + } +} + +const char* MFVideoTransferFunctionToStr(MFVideoTransferFunction aFunc) { + switch (aFunc) { + ENUM_TO_STR(MFVideoTransFunc_Unknown); + ENUM_TO_STR(MFVideoTransFunc_709); + ENUM_TO_STR(MFVideoTransFunc_2020); + ENUM_TO_STR(MFVideoTransFunc_sRGB); + default: + return "Unsupported MFVideoTransferFunction"; + } +} + +const char* MFVideoPrimariesToStr(MFVideoPrimaries aPrimaries) { + switch (aPrimaries) { + ENUM_TO_STR(MFVideoPrimaries_Unknown); + ENUM_TO_STR(MFVideoPrimaries_BT709); + ENUM_TO_STR(MFVideoPrimaries_BT2020); + default: + return "Unsupported MFVideoPrimaries"; + } +} + +#undef ENUM_TO_STR +#undef ENUM_TO_STR2 + +} // namespace mozilla diff --git a/dom/media/ipc/MFMediaEngineUtils.h b/dom/media/ipc/MFMediaEngineUtils.h new file mode 100644 index 0000000000..5054fbb5c4 --- /dev/null +++ b/dom/media/ipc/MFMediaEngineUtils.h @@ -0,0 +1,88 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_IPC_MFMEDIAENGINEUTILS_H_ +#define DOM_MEDIA_IPC_MFMEDIAENGINEUTILS_H_ + +#include "MFMediaEngineExtra.h" +#include "ipc/EnumSerializer.h" +#include "mozilla/Logging.h" +#include "mozilla/ProfilerMarkerTypes.h" + +namespace mozilla { + +inline LazyLogModule gMFMediaEngineLog{"MFMediaEngine"}; + +// https://docs.microsoft.com/en-us/windows/win32/api/mfmediaengine/ne-mfmediaengine-mf_media_engine_event +using MFMediaEngineEvent = MF_MEDIA_ENGINE_EVENT; + +// https://docs.microsoft.com/en-us/windows/win32/api/mfmediaengine/ne-mfmediaengine-mf_media_engine_err +using MFMediaEngineError = MF_MEDIA_ENGINE_ERR; + +#define LOG_AND_WARNING(msg, ...) \ + do { \ + NS_WARNING(nsPrintfCString(msg, rv).get()); \ + MOZ_LOG(gMFMediaEngineLog, LogLevel::Debug, \ + ("%s:%d, " msg, __FILE__, __LINE__, ##__VA_ARGS__)); \ + } while (false) + +#ifndef RETURN_IF_FAILED +# define RETURN_IF_FAILED(x) \ + do { \ + HRESULT rv = x; \ + if (MOZ_UNLIKELY(FAILED(rv))) { \ + LOG_AND_WARNING("(" #x ") failed, rv=%lx", rv); \ + return rv; \ + } \ + } while (false) +#endif + +#ifndef RETURN_VOID_IF_FAILED +# define RETURN_VOID_IF_FAILED(x) \ + do { \ + HRESULT rv = x; \ + if (MOZ_UNLIKELY(FAILED(rv))) { \ + LOG_AND_WARNING("(" #x ") failed, rv=%lx", rv); \ + return; \ + } \ + } while (false) +#endif + +#define ENGINE_MARKER(markerName) \ + PROFILER_MARKER(markerName, MEDIA_PLAYBACK, {}, MediaEngineMarker, Id()) + +#define ENGINE_MARKER_TEXT(markerName, text) \ + PROFILER_MARKER(markerName, MEDIA_PLAYBACK, {}, MediaEngineTextMarker, Id(), \ + text) + +const char* MediaEventTypeToStr(MediaEventType aType); +const char* MediaEngineEventToStr(MF_MEDIA_ENGINE_EVENT aEvent); +const char* MFMediaEngineErrorToStr(MFMediaEngineError aError); +const char* GUIDToStr(GUID aGUID); +const char* MFVideoRotationFormatToStr(MFVideoRotationFormat aFormat); +const char* MFVideoTransferFunctionToStr(MFVideoTransferFunction aFunc); +const char* MFVideoPrimariesToStr(MFVideoPrimaries aPrimaries); + +} // namespace mozilla + +namespace IPC { + +template <> +struct ParamTraits<mozilla::MFMediaEngineError> + : public ContiguousEnumSerializerInclusive< + mozilla::MFMediaEngineError, + mozilla::MFMediaEngineError::MF_MEDIA_ENGINE_ERR_ABORTED, + mozilla::MFMediaEngineError::MF_MEDIA_ENGINE_ERR_ENCRYPTED> {}; + +template <> +struct ParamTraits<mozilla::MFMediaEngineEvent> + : public ContiguousEnumSerializerInclusive< + mozilla::MFMediaEngineEvent, + mozilla::MFMediaEngineEvent::MF_MEDIA_ENGINE_EVENT_LOADSTART, + mozilla::MFMediaEngineEvent:: + MF_MEDIA_ENGINE_EVENT_AUDIOENDPOINTCHANGE> {}; + +} // namespace IPC + +#endif // DOM_MEDIA_IPC_MFMEDIAENGINECHILD_H_ diff --git a/dom/media/ipc/MediaIPCUtils.h b/dom/media/ipc/MediaIPCUtils.h new file mode 100644 index 0000000000..274bf46a5b --- /dev/null +++ b/dom/media/ipc/MediaIPCUtils.h @@ -0,0 +1,350 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_media_MediaIPCUtils_h +#define mozilla_dom_media_MediaIPCUtils_h + +#include <type_traits> + +#include "DecoderDoctorDiagnostics.h" +#include "PerformanceRecorder.h" +#include "PlatformDecoderModule.h" +#include "ipc/EnumSerializer.h" +#include "mozilla/EnumSet.h" +#include "mozilla/GfxMessageUtils.h" +#include "mozilla/gfx/Rect.h" + +namespace IPC { +template <> +struct ParamTraits<mozilla::VideoInfo> { + typedef mozilla::VideoInfo paramType; + + static void Write(MessageWriter* aWriter, const paramType& aParam) { + // TrackInfo + WriteParam(aWriter, aParam.mMimeType); + + // VideoInfo + WriteParam(aWriter, aParam.mDisplay); + WriteParam(aWriter, aParam.mStereoMode); + WriteParam(aWriter, aParam.mImage); + WriteParam(aWriter, aParam.mImageRect); + WriteParam(aWriter, *aParam.mCodecSpecificConfig); + WriteParam(aWriter, *aParam.mExtraData); + WriteParam(aWriter, aParam.mRotation); + WriteParam(aWriter, aParam.mColorDepth); + WriteParam(aWriter, aParam.mColorSpace); + WriteParam(aWriter, aParam.mColorPrimaries); + WriteParam(aWriter, aParam.mTransferFunction); + WriteParam(aWriter, aParam.mColorRange); + WriteParam(aWriter, aParam.HasAlpha()); + } + + static bool Read(MessageReader* aReader, paramType* aResult) { + mozilla::gfx::IntRect imageRect; + bool alphaPresent; + if (ReadParam(aReader, &aResult->mMimeType) && + ReadParam(aReader, &aResult->mDisplay) && + ReadParam(aReader, &aResult->mStereoMode) && + ReadParam(aReader, &aResult->mImage) && + ReadParam(aReader, &aResult->mImageRect) && + ReadParam(aReader, aResult->mCodecSpecificConfig.get()) && + ReadParam(aReader, aResult->mExtraData.get()) && + ReadParam(aReader, &aResult->mRotation) && + ReadParam(aReader, &aResult->mColorDepth) && + ReadParam(aReader, &aResult->mColorSpace) && + ReadParam(aReader, &aResult->mColorPrimaries) && + ReadParam(aReader, &aResult->mTransferFunction) && + ReadParam(aReader, &aResult->mColorRange) && + ReadParam(aReader, &alphaPresent)) { + aResult->SetAlpha(alphaPresent); + return true; + } + return false; + } +}; + +template <> +struct ParamTraits<mozilla::TrackInfo::TrackType> + : public ContiguousEnumSerializerInclusive< + mozilla::TrackInfo::TrackType, + mozilla::TrackInfo::TrackType::kUndefinedTrack, + mozilla::TrackInfo::TrackType::kTextTrack> {}; + +template <> +struct ParamTraits<mozilla::VideoInfo::Rotation> + : public ContiguousEnumSerializerInclusive< + mozilla::VideoInfo::Rotation, mozilla::VideoInfo::Rotation::kDegree_0, + mozilla::VideoInfo::Rotation::kDegree_270> {}; + +template <> +struct ParamTraits<mozilla::MediaByteBuffer> + : public ParamTraits<nsTArray<uint8_t>> { + typedef mozilla::MediaByteBuffer paramType; +}; + +// Traits for AudioCodecSpecificVariant types. + +template <> +struct ParamTraits<mozilla::NoCodecSpecificData> + : public EmptyStructSerializer<mozilla::NoCodecSpecificData> {}; + +template <> +struct ParamTraits<mozilla::AudioCodecSpecificBinaryBlob> { + using paramType = mozilla::AudioCodecSpecificBinaryBlob; + + static void Write(MessageWriter* aWriter, const paramType& aParam) { + WriteParam(aWriter, *aParam.mBinaryBlob); + } + static bool Read(MessageReader* aReader, paramType* aResult) { + return ReadParam(aReader, aResult->mBinaryBlob.get()); + } +}; + +template <> +struct ParamTraits<mozilla::AacCodecSpecificData> { + using paramType = mozilla::AacCodecSpecificData; + + static void Write(MessageWriter* aWriter, const paramType& aParam) { + WriteParam(aWriter, *aParam.mEsDescriptorBinaryBlob); + WriteParam(aWriter, *aParam.mDecoderConfigDescriptorBinaryBlob); + } + static bool Read(MessageReader* aReader, paramType* aResult) { + return ReadParam(aReader, aResult->mEsDescriptorBinaryBlob.get()) && + ReadParam(aReader, + aResult->mDecoderConfigDescriptorBinaryBlob.get()); + } +}; + +template <> +struct ParamTraits<mozilla::FlacCodecSpecificData> { + using paramType = mozilla::FlacCodecSpecificData; + + static void Write(MessageWriter* aWriter, const paramType& aParam) { + WriteParam(aWriter, *aParam.mStreamInfoBinaryBlob); + } + static bool Read(MessageReader* aReader, paramType* aResult) { + return ReadParam(aReader, aResult->mStreamInfoBinaryBlob.get()); + } +}; + +template <> +struct ParamTraits<mozilla::Mp3CodecSpecificData> + : public PlainOldDataSerializer<mozilla::Mp3CodecSpecificData> {}; + +template <> +struct ParamTraits<mozilla::OpusCodecSpecificData> { + using paramType = mozilla::OpusCodecSpecificData; + + static void Write(MessageWriter* aWriter, const paramType& aParam) { + WriteParam(aWriter, aParam.mContainerCodecDelayMicroSeconds); + WriteParam(aWriter, *aParam.mHeadersBinaryBlob); + } + static bool Read(MessageReader* aReader, paramType* aResult) { + return ReadParam(aReader, &aResult->mContainerCodecDelayMicroSeconds) && + ReadParam(aReader, aResult->mHeadersBinaryBlob.get()); + } +}; + +template <> +struct ParamTraits<mozilla::VorbisCodecSpecificData> { + using paramType = mozilla::VorbisCodecSpecificData; + + static void Write(MessageWriter* aWriter, const paramType& aParam) { + WriteParam(aWriter, *aParam.mHeadersBinaryBlob); + } + static bool Read(MessageReader* aReader, paramType* aResult) { + return ReadParam(aReader, aResult->mHeadersBinaryBlob.get()); + } +}; + +template <> +struct ParamTraits<mozilla::WaveCodecSpecificData> + : public EmptyStructSerializer<mozilla::WaveCodecSpecificData> {}; + +// End traits for AudioCodecSpecificVariant types. + +template <> +struct ParamTraits<mozilla::AudioInfo> { + typedef mozilla::AudioInfo paramType; + + static void Write(MessageWriter* aWriter, const paramType& aParam) { + // TrackInfo + WriteParam(aWriter, aParam.mMimeType); + + // AudioInfo + WriteParam(aWriter, aParam.mRate); + WriteParam(aWriter, aParam.mChannels); + WriteParam(aWriter, aParam.mChannelMap); + WriteParam(aWriter, aParam.mBitDepth); + WriteParam(aWriter, aParam.mProfile); + WriteParam(aWriter, aParam.mExtendedProfile); + WriteParam(aWriter, aParam.mCodecSpecificConfig); + } + + static bool Read(MessageReader* aReader, paramType* aResult) { + if (ReadParam(aReader, &aResult->mMimeType) && + ReadParam(aReader, &aResult->mRate) && + ReadParam(aReader, &aResult->mChannels) && + ReadParam(aReader, &aResult->mChannelMap) && + ReadParam(aReader, &aResult->mBitDepth) && + ReadParam(aReader, &aResult->mProfile) && + ReadParam(aReader, &aResult->mExtendedProfile) && + ReadParam(aReader, &aResult->mCodecSpecificConfig)) { + return true; + } + return false; + } +}; + +template <> +struct ParamTraits<mozilla::MediaDataDecoder::ConversionRequired> + : public ContiguousEnumSerializerInclusive< + mozilla::MediaDataDecoder::ConversionRequired, + mozilla::MediaDataDecoder::ConversionRequired(0), + mozilla::MediaDataDecoder::ConversionRequired( + mozilla::MediaDataDecoder::ConversionRequired::kNeedAnnexB)> {}; + +template <> +struct ParamTraits<mozilla::media::TimeUnit> { + typedef mozilla::media::TimeUnit paramType; + + static void Write(MessageWriter* aWriter, const paramType& aParam) { + WriteParam(aWriter, aParam.IsValid()); + WriteParam(aWriter, aParam.IsValid() ? aParam.ToMicroseconds() : 0); + } + + static bool Read(MessageReader* aReader, paramType* aResult) { + bool valid; + int64_t value; + if (ReadParam(aReader, &valid) && ReadParam(aReader, &value)) { + if (!valid) { + *aResult = mozilla::media::TimeUnit::Invalid(); + } else { + *aResult = mozilla::media::TimeUnit::FromMicroseconds(value); + } + return true; + } + return false; + }; +}; + +template <> +struct ParamTraits<mozilla::media::TimeInterval> { + typedef mozilla::media::TimeInterval paramType; + + static void Write(MessageWriter* aWriter, const paramType& aParam) { + WriteParam(aWriter, aParam.mStart); + WriteParam(aWriter, aParam.mEnd); + WriteParam(aWriter, aParam.mFuzz); + } + + static bool Read(MessageReader* aReader, paramType* aResult) { + if (ReadParam(aReader, &aResult->mStart) && + ReadParam(aReader, &aResult->mEnd) && + ReadParam(aReader, &aResult->mFuzz)) { + return true; + } + return false; + } +}; + +template <> +struct ParamTraits<mozilla::MediaResult> { + typedef mozilla::MediaResult paramType; + + static void Write(MessageWriter* aWriter, const paramType& aParam) { + WriteParam(aWriter, aParam.Code()); + WriteParam(aWriter, aParam.Message()); + } + + static bool Read(MessageReader* aReader, paramType* aResult) { + nsresult result; + nsCString message; + if (ReadParam(aReader, &result) && ReadParam(aReader, &message)) { + *aResult = paramType(result, std::move(message)); + return true; + } + return false; + }; +}; + +template <> +struct ParamTraits<mozilla::DecoderDoctorDiagnostics> { + typedef mozilla::DecoderDoctorDiagnostics paramType; + + static void Write(MessageWriter* aWriter, const paramType& aParam) { + WriteParam(aWriter, aParam.mDiagnosticsType); + WriteParam(aWriter, aParam.mFormat); + WriteParam(aWriter, aParam.mFlags); + WriteParam(aWriter, aParam.mEvent); + } + + static bool Read(MessageReader* aReader, paramType* aResult) { + if (ReadParam(aReader, &aResult->mDiagnosticsType) && + ReadParam(aReader, &aResult->mFormat) && + ReadParam(aReader, &aResult->mFlags) && + ReadParam(aReader, &aResult->mEvent)) { + return true; + } + return false; + }; +}; + +template <> +struct ParamTraits<mozilla::DecoderDoctorDiagnostics::DiagnosticsType> + : public ContiguousEnumSerializerInclusive< + mozilla::DecoderDoctorDiagnostics::DiagnosticsType, + mozilla::DecoderDoctorDiagnostics::DiagnosticsType::eUnsaved, + mozilla::DecoderDoctorDiagnostics::DiagnosticsType::eDecodeWarning> { +}; + +template <> +struct ParamTraits<mozilla::DecoderDoctorEvent> { + typedef mozilla::DecoderDoctorEvent paramType; + + static void Write(MessageWriter* aWriter, const paramType& aParam) { + int domain = aParam.mDomain; + WriteParam(aWriter, domain); + WriteParam(aWriter, aParam.mResult); + } + + static bool Read(MessageReader* aReader, paramType* aResult) { + int domain = 0; + if (ReadParam(aReader, &domain) && ReadParam(aReader, &aResult->mResult)) { + aResult->mDomain = paramType::Domain(domain); + return true; + } + return false; + }; +}; + +template <> +struct ParamTraits<mozilla::TrackingId::Source> + : public ContiguousEnumSerializer< + mozilla::TrackingId::Source, + mozilla::TrackingId::Source::Unimplemented, + mozilla::TrackingId::Source::LAST> {}; + +template <> +struct ParamTraits<mozilla::TrackingId> { + typedef mozilla::TrackingId paramType; + + static void Write(MessageWriter* aWriter, const paramType& aParam) { + WriteParam(aWriter, aParam.mSource); + WriteParam(aWriter, aParam.mProcId); + WriteParam(aWriter, aParam.mUniqueInProcId); + } + + static bool Read(MessageReader* aReader, paramType* aResult) { + return ReadParam(aReader, &aResult->mSource) && + ReadParam(aReader, &aResult->mProcId) && + ReadParam(aReader, &aResult->mUniqueInProcId); + } +}; + +} // namespace IPC + +#endif // mozilla_dom_media_MediaIPCUtils_h diff --git a/dom/media/ipc/PMFMediaEngine.ipdl b/dom/media/ipc/PMFMediaEngine.ipdl new file mode 100644 index 0000000000..fd9b063625 --- /dev/null +++ b/dom/media/ipc/PMFMediaEngine.ipdl @@ -0,0 +1,61 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include "mozilla/dom/MediaIPCUtils.h"; + +include protocol PRemoteDecoderManager; + +using mozilla::AudioInfo from "MediaInfo.h"; +using mozilla::VideoInfo from "MediaInfo.h"; +using mozilla::MediaResult from "MediaResult.h"; +using mozilla::TrackInfo::TrackType from "MediaInfo.h"; +using mozilla::MFMediaEngineError from "MFMediaEngineUtils.h"; +using mozilla::MFMediaEngineEvent from "MFMediaEngineUtils.h"; + +namespace mozilla { + +struct MediaEngineInfoIPDL +{ + bool preload; +}; + +struct MediaInfoIPDL +{ + AudioInfo? audioInfo; + VideoInfo? videoInfo; +}; + +struct StatisticData +{ + uint64_t renderedFrames; + uint64_t droppedFrames; +}; + +[ManualDealloc] +async protocol PMFMediaEngine +{ + manager PRemoteDecoderManager; +parent: + // Return 0 if media engine can't be created. + async InitMediaEngine(MediaEngineInfoIPDL info) returns (uint64_t id); + async NotifyMediaInfo(MediaInfoIPDL info); + async Play(); + async Pause(); + async Seek(double targetTimeInSecond); + async SetVolume(double volume); + async SetPlaybackRate(double playbackRate); + async SetLooping(bool looping); + async NotifyEndOfStream(TrackType type); + async Shutdown(); + async __delete__(); + +child: + async NotifyEvent(MFMediaEngineEvent event); + async NotifyError(MediaResult error); + async UpdateCurrentTime(double currentTimeInSecond); + async RequestSample(TrackType type, bool isEnough); + async UpdateStatisticData(StatisticData data); +}; + +} // namespace mozilla diff --git a/dom/media/ipc/PMediaDecoderParams.ipdlh b/dom/media/ipc/PMediaDecoderParams.ipdlh new file mode 100644 index 0000000000..636b91bafa --- /dev/null +++ b/dom/media/ipc/PMediaDecoderParams.ipdlh @@ -0,0 +1,21 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include "mozilla/dom/MediaIPCUtils.h"; + +using mozilla::media::TimeUnit from "TimeUnits.h"; + +namespace mozilla { + +// used for both SendInput/RecvInput and ProcessDecodedData/RecvOutput +struct MediaDataIPDL +{ + int64_t offset; + TimeUnit time; + TimeUnit timecode; + TimeUnit duration; + bool keyframe; +}; + +} // namespace mozilla diff --git a/dom/media/ipc/PRDD.ipdl b/dom/media/ipc/PRDD.ipdl new file mode 100644 index 0000000000..c2635cb08d --- /dev/null +++ b/dom/media/ipc/PRDD.ipdl @@ -0,0 +1,109 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include GraphicsMessages; +include MemoryReportTypes; +include PrefsTypes; + +include protocol PProfiler; +include protocol PRemoteDecoderManager; +include protocol PVideoBridge; + +#if defined(MOZ_SANDBOX) && defined(MOZ_DEBUG) && defined(ENABLE_TESTS) +include protocol PSandboxTesting; +#endif + +include "mozilla/ipc/ByteBufUtils.h"; + +using mozilla::dom::NativeThreadId from "mozilla/dom/NativeThreadId.h"; +using mozilla::media::MediaCodecsSupported from "MediaCodecsSupport.h"; + +#if defined(XP_WIN) +[MoveOnly] using mozilla::UntrustedModulesData from "mozilla/UntrustedModulesData.h"; +[MoveOnly] using mozilla::ModulePaths from "mozilla/UntrustedModulesData.h"; +[MoveOnly] using mozilla::ModulesMapResult from "mozilla/UntrustedModulesData.h"; +#endif // defined(XP_WIN) + +namespace mozilla { + +// This protocol allows the UI process to talk to the RDD +// (RemoteDataDecoder) process. There is one instance of this protocol, +// with the RDDParent living on the main thread of the RDD process and +// the RDDChild living on the main thread of the UI process. +[ManualDealloc, NeedsOtherPid] +protocol PRDD +{ +parent: + + async Init(GfxVarUpdate[] vars, FileDescriptor? sandboxBroker, + bool canRecordReleaseTelemetry, + bool aIsReadyForBackgroundProcessing); + + async InitProfiler(Endpoint<PProfilerChild> endpoint); + + async NewContentRemoteDecoderManager( + Endpoint<PRemoteDecoderManagerParent> endpoint); + + async RequestMemoryReport(uint32_t generation, + bool anonymize, + bool minimizeMemoryUsage, + FileDescriptor? DMDFile) + returns (uint32_t aGeneration); + + async PreferenceUpdate(Pref pref); + + async UpdateVar(GfxVarUpdate var); + + async InitVideoBridge(Endpoint<PVideoBridgeChild> endpoint, + bool createHardwareDevice, + ContentDeviceData contentDeviceData); + +#if defined(XP_WIN) + async GetUntrustedModulesData() returns (UntrustedModulesData? data); + + /** + * This method is used to notifty a child process to start + * processing module loading events in UntrustedModulesProcessor. + * This should be called when the parent process has gone idle. + */ + async UnblockUntrustedModulesThread(); +#endif // defined(XP_WIN) + +#if defined(MOZ_SANDBOX) && defined(MOZ_DEBUG) && defined(ENABLE_TESTS) + async InitSandboxTesting(Endpoint<PSandboxTestingChild> aEndpoint); +#endif + + // Tells the RDD process to flush any pending telemetry. + // Used in tests and ping assembly. Buffer contains bincoded Rust structs. + // https://firefox-source-docs.mozilla.org/toolkit/components/glean/dev/ipc.html + async FlushFOGData() returns (ByteBuf buf); + + // Test-only method. + // Asks the RDD process to trigger test-only instrumentation. + // The unused returned value is to have a promise we can await. + async TestTriggerMetrics() returns (bool unused); + +child: + + async InitCrashReporter(NativeThreadId threadId); + + async AddMemoryReport(MemoryReport aReport); + +#if defined(XP_WIN) + async GetModulesTrust(ModulePaths aModPaths, bool aRunAtNormalPriority) + returns (ModulesMapResult? modMapResult); +#endif // defined(XP_WIN) + + // Update the cached list of codec supported following a check in the + // RDD parent. + async UpdateMediaCodecsSupported(MediaCodecsSupported aSupported); + + // Sent from time-to-time to limit the amount of telemetry vulnerable to loss + // Buffer contains bincoded Rust structs. + // https://firefox-source-docs.mozilla.org/toolkit/components/glean/dev/ipc.html + async FOGData(ByteBuf buf); +}; + +} // namespace mozilla diff --git a/dom/media/ipc/PRemoteDecoder.ipdl b/dom/media/ipc/PRemoteDecoder.ipdl new file mode 100644 index 0000000000..ce2bb0722b --- /dev/null +++ b/dom/media/ipc/PRemoteDecoder.ipdl @@ -0,0 +1,78 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +include "mozilla/dom/MediaIPCUtils.h"; + +include protocol PRemoteDecoderManager; + +using mozilla::MediaDataDecoder::ConversionRequired from "PlatformDecoderModule.h"; +using mozilla::TrackInfo::TrackType from "MediaInfo.h"; +using mozilla::layers::LayersBackend from "mozilla/layers/LayersTypes.h"; +using mozilla::MediaResult from "MediaResult.h"; +[RefCounted] using class mozilla::ArrayOfRemoteMediaRawData from "mozilla/RemoteMediaData.h"; +[RefCounted] using class mozilla::ArrayOfRemoteAudioData from "mozilla/RemoteMediaData.h"; +[RefCounted] using class mozilla::ArrayOfRemoteVideoData from "mozilla/RemoteMediaData.h"; +include PMediaDecoderParams; +include LayersSurfaces; + +namespace mozilla { + +union DecodedOutputIPDL +{ + ArrayOfRemoteAudioData; + ArrayOfRemoteVideoData; +}; + +struct InitCompletionIPDL +{ + TrackType type; + nsCString decoderDescription; + bool hardware; + nsCString hardwareReason; + ConversionRequired conversion; +}; + +union InitResultIPDL +{ + MediaResult; + InitCompletionIPDL; +}; + +union DecodeResultIPDL +{ + MediaResult; + DecodedOutputIPDL; +}; + +// This protocol provides a way to use MediaDataDecoder across processes. +// The parent side currently is only implemented to work with +// RemoteDecoderModule or WindowsMediaFoundation. +// The child side runs in the content process, and the parent side runs +// in the RDD process or the GPU process. We run a separate IPDL thread +// for both sides. +[ManualDealloc] +async protocol PRemoteDecoder +{ + manager PRemoteDecoderManager; +parent: + async Construct() returns (MediaResult result); + + async Init() returns (InitResultIPDL result); + + // Each output may include a SurfaceDescriptorGPUVideo that represents the decoded + // frame. This SurfaceDescriptor can be used on the Layers IPDL protocol, but + // must be released explicitly using DeallocateSurfaceDescriptorGPUVideo + // on the manager protocol. + async Decode(ArrayOfRemoteMediaRawData data) returns (DecodeResultIPDL result); + async Flush() returns (MediaResult error); + async Drain() returns (DecodeResultIPDL result); + async Shutdown() returns (bool unused); + // To clear the threshold, call with INT64_MIN. + async SetSeekThreshold(TimeUnit time); + + async __delete__(); +}; + +} // namespace mozilla diff --git a/dom/media/ipc/PRemoteDecoderManager.ipdl b/dom/media/ipc/PRemoteDecoderManager.ipdl new file mode 100644 index 0000000000..eac4900278 --- /dev/null +++ b/dom/media/ipc/PRemoteDecoderManager.ipdl @@ -0,0 +1,60 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifdef MOZ_WMF_MEDIA_ENGINE + include protocol PMFMediaEngine; +#endif +include protocol PTexture; +include protocol PRemoteDecoder; +include LayersSurfaces; +include PMediaDecoderParams; +include "mozilla/dom/MediaIPCUtils.h"; +include "mozilla/layers/LayersMessageUtils.h"; + +using mozilla::VideoInfo from "MediaInfo.h"; +using mozilla::AudioInfo from "MediaInfo.h"; +using struct mozilla::layers::TextureFactoryIdentifier from "mozilla/layers/CompositorTypes.h"; +using mozilla::CreateDecoderParams::OptionSet from "PlatformDecoderModule.h"; +using mozilla::DecoderDoctorDiagnostics from "DecoderDoctorDiagnostics.h"; +using mozilla::TrackingId from "PerformanceRecorder.h"; + +namespace mozilla { + +struct VideoDecoderInfoIPDL +{ + VideoInfo videoInfo; + float framerate; +}; + +union RemoteDecoderInfoIPDL +{ + AudioInfo; + VideoDecoderInfoIPDL; +}; + +[ManualDealloc, NeedsOtherPid] +sync protocol PRemoteDecoderManager +{ + manages PRemoteDecoder; +#ifdef MOZ_WMF_MEDIA_ENGINE + manages PMFMediaEngine; +#endif + +parent: +#ifdef MOZ_WMF_MEDIA_ENGINE + async PMFMediaEngine(); +#endif + async PRemoteDecoder(RemoteDecoderInfoIPDL info, + OptionSet options, + TextureFactoryIdentifier? identifier, + uint64_t? mediaEngineId, + TrackingId? trackingId); + + sync Readback(SurfaceDescriptorGPUVideo sd) returns (SurfaceDescriptor aResult); + + async DeallocateSurfaceDescriptorGPUVideo(SurfaceDescriptorGPUVideo sd); +}; + +} // namespace mozilla diff --git a/dom/media/ipc/RDDChild.cpp b/dom/media/ipc/RDDChild.cpp new file mode 100644 index 0000000000..b6e38ff828 --- /dev/null +++ b/dom/media/ipc/RDDChild.cpp @@ -0,0 +1,187 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "RDDChild.h" + +#include "mozilla/FOGIPC.h" +#include "mozilla/RDDProcessManager.h" +#include "mozilla/dom/ContentParent.h" +#include "mozilla/dom/MemoryReportRequest.h" +#include "mozilla/gfx/GPUProcessManager.h" +#include "mozilla/gfx/gfxVars.h" +#include "mozilla/ipc/CrashReporterHost.h" +#include "mozilla/ipc/Endpoint.h" + +#if defined(XP_LINUX) && defined(MOZ_SANDBOX) +# include "mozilla/SandboxBroker.h" +# include "mozilla/SandboxBrokerPolicyFactory.h" +#endif + +#include "mozilla/Telemetry.h" + +#if defined(XP_WIN) +# include "mozilla/WinDllServices.h" +#endif + +#include "ProfilerParent.h" +#include "RDDProcessHost.h" + +namespace mozilla { + +using namespace layers; +using namespace gfx; + +RDDChild::RDDChild(RDDProcessHost* aHost) : mHost(aHost) { + MOZ_COUNT_CTOR(RDDChild); +} + +RDDChild::~RDDChild() { MOZ_COUNT_DTOR(RDDChild); } + +bool RDDChild::Init() { + Maybe<FileDescriptor> brokerFd; + +#if defined(XP_LINUX) && defined(MOZ_SANDBOX) + auto policy = SandboxBrokerPolicyFactory::GetRDDPolicy(OtherPid()); + if (policy != nullptr) { + brokerFd = Some(FileDescriptor()); + mSandboxBroker = + SandboxBroker::Create(std::move(policy), OtherPid(), brokerFd.ref()); + // This is unlikely to fail and probably indicates OS resource + // exhaustion, but we can at least try to recover. + if (NS_WARN_IF(mSandboxBroker == nullptr)) { + return false; + } + MOZ_ASSERT(brokerFd.ref().IsValid()); + } +#endif // XP_LINUX && MOZ_SANDBOX + + nsTArray<GfxVarUpdate> updates = gfxVars::FetchNonDefaultVars(); + + bool isReadyForBackgroundProcessing = false; +#if defined(XP_WIN) + RefPtr<DllServices> dllSvc(DllServices::Get()); + isReadyForBackgroundProcessing = dllSvc->IsReadyForBackgroundProcessing(); +#endif + + SendInit(updates, brokerFd, Telemetry::CanRecordReleaseData(), + isReadyForBackgroundProcessing); + + Unused << SendInitProfiler(ProfilerParent::CreateForProcess(OtherPid())); + + gfxVars::AddReceiver(this); + auto* gpm = gfx::GPUProcessManager::Get(); + if (gpm) { + gpm->AddListener(this); + } + + return true; +} + +bool RDDChild::SendRequestMemoryReport(const uint32_t& aGeneration, + const bool& aAnonymize, + const bool& aMinimizeMemoryUsage, + const Maybe<FileDescriptor>& aDMDFile) { + mMemoryReportRequest = MakeUnique<MemoryReportRequestHost>(aGeneration); + + PRDDChild::SendRequestMemoryReport( + aGeneration, aAnonymize, aMinimizeMemoryUsage, aDMDFile, + [&](const uint32_t& aGeneration2) { + if (RDDProcessManager* rddpm = RDDProcessManager::Get()) { + if (RDDChild* child = rddpm->GetRDDChild()) { + if (child->mMemoryReportRequest) { + child->mMemoryReportRequest->Finish(aGeneration2); + child->mMemoryReportRequest = nullptr; + } + } + } + }, + [&](mozilla::ipc::ResponseRejectReason) { + if (RDDProcessManager* rddpm = RDDProcessManager::Get()) { + if (RDDChild* child = rddpm->GetRDDChild()) { + child->mMemoryReportRequest = nullptr; + } + } + }); + + return true; +} + +void RDDChild::OnCompositorUnexpectedShutdown() { + auto* rddm = RDDProcessManager::Get(); + if (rddm) { + rddm->CreateVideoBridge(); + } +} + +void RDDChild::OnVarChanged(const GfxVarUpdate& aVar) { SendUpdateVar(aVar); } + +mozilla::ipc::IPCResult RDDChild::RecvAddMemoryReport( + const MemoryReport& aReport) { + if (mMemoryReportRequest) { + mMemoryReportRequest->RecvReport(aReport); + } + return IPC_OK(); +} + +#if defined(XP_WIN) +mozilla::ipc::IPCResult RDDChild::RecvGetModulesTrust( + ModulePaths&& aModPaths, bool aRunAtNormalPriority, + GetModulesTrustResolver&& aResolver) { + RefPtr<DllServices> dllSvc(DllServices::Get()); + dllSvc->GetModulesTrust(std::move(aModPaths), aRunAtNormalPriority) + ->Then( + GetMainThreadSerialEventTarget(), __func__, + [aResolver](ModulesMapResult&& aResult) { + aResolver(Some(ModulesMapResult(std::move(aResult)))); + }, + [aResolver](nsresult aRv) { aResolver(Nothing()); }); + return IPC_OK(); +} +#endif // defined(XP_WIN) + +mozilla::ipc::IPCResult RDDChild::RecvUpdateMediaCodecsSupported( + const media::MediaCodecsSupported& aSupported) { + dom::ContentParent::BroadcastMediaCodecsSupportedUpdate( + RemoteDecodeIn::RddProcess, aSupported); + return IPC_OK(); +} + +mozilla::ipc::IPCResult RDDChild::RecvFOGData(ByteBuf&& aBuf) { + glean::FOGData(std::move(aBuf)); + return IPC_OK(); +} + +void RDDChild::ActorDestroy(ActorDestroyReason aWhy) { + if (aWhy == AbnormalShutdown) { + GenerateCrashReport(OtherPid()); + } + + auto* gpm = gfx::GPUProcessManager::Get(); + if (gpm) { + // Note: the manager could have shutdown already. + gpm->RemoveListener(this); + } + + gfxVars::RemoveReceiver(this); + mHost->OnChannelClosed(); +} + +class DeferredDeleteRDDChild : public Runnable { + public: + explicit DeferredDeleteRDDChild(UniquePtr<RDDChild>&& aChild) + : Runnable("gfx::DeferredDeleteRDDChild"), mChild(std::move(aChild)) {} + + NS_IMETHODIMP Run() override { return NS_OK; } + + private: + UniquePtr<RDDChild> mChild; +}; + +/* static */ +void RDDChild::Destroy(UniquePtr<RDDChild>&& aChild) { + NS_DispatchToMainThread(new DeferredDeleteRDDChild(std::move(aChild))); +} + +} // namespace mozilla diff --git a/dom/media/ipc/RDDChild.h b/dom/media/ipc/RDDChild.h new file mode 100644 index 0000000000..e18c9e4677 --- /dev/null +++ b/dom/media/ipc/RDDChild.h @@ -0,0 +1,70 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef _include_dom_media_ipc_RDDChild_h_ +#define _include_dom_media_ipc_RDDChild_h_ +#include "mozilla/PRDDChild.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/gfx/GPUProcessListener.h" +#include "mozilla/gfx/gfxVarReceiver.h" +#include "mozilla/ipc/CrashReporterHelper.h" + +namespace mozilla { + +#if defined(XP_LINUX) && defined(MOZ_SANDBOX) +class SandboxBroker; +#endif + +namespace dom { +class MemoryReportRequestHost; +} // namespace dom + +class RDDProcessHost; + +class RDDChild final : public PRDDChild, + public ipc::CrashReporterHelper<GeckoProcessType_RDD>, + public gfx::gfxVarReceiver, + public gfx::GPUProcessListener { + typedef mozilla::dom::MemoryReportRequestHost MemoryReportRequestHost; + + public: + explicit RDDChild(RDDProcessHost* aHost); + ~RDDChild(); + + bool Init(); + + void OnCompositorUnexpectedShutdown() override; + void OnVarChanged(const GfxVarUpdate& aVar) override; + + void ActorDestroy(ActorDestroyReason aWhy) override; + + mozilla::ipc::IPCResult RecvAddMemoryReport(const MemoryReport& aReport); +#if defined(XP_WIN) + mozilla::ipc::IPCResult RecvGetModulesTrust( + ModulePaths&& aModPaths, bool aRunAtNormalPriority, + GetModulesTrustResolver&& aResolver); +#endif // defined(XP_WIN) + mozilla::ipc::IPCResult RecvUpdateMediaCodecsSupported( + const media::MediaCodecsSupported& aSupported); + mozilla::ipc::IPCResult RecvFOGData(ByteBuf&& aBuf); + + bool SendRequestMemoryReport(const uint32_t& aGeneration, + const bool& aAnonymize, + const bool& aMinimizeMemoryUsage, + const Maybe<ipc::FileDescriptor>& aDMDFile); + + static void Destroy(UniquePtr<RDDChild>&& aChild); + + private: + RDDProcessHost* mHost; + UniquePtr<MemoryReportRequestHost> mMemoryReportRequest; +#if defined(XP_LINUX) && defined(MOZ_SANDBOX) + UniquePtr<SandboxBroker> mSandboxBroker; +#endif +}; + +} // namespace mozilla + +#endif // _include_dom_media_ipc_RDDChild_h_ diff --git a/dom/media/ipc/RDDParent.cpp b/dom/media/ipc/RDDParent.cpp new file mode 100644 index 0000000000..e928463a9d --- /dev/null +++ b/dom/media/ipc/RDDParent.cpp @@ -0,0 +1,331 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "RDDParent.h" + +#if defined(XP_WIN) +# include <dwrite.h> +# include <process.h> + +# include "WMF.h" +# include "WMFDecoderModule.h" +# include "mozilla/WinDllServices.h" +# include "mozilla/gfx/DeviceManagerDx.h" +#else +# include <unistd.h> +#endif + +#include "PDMFactory.h" +#include "gfxConfig.h" +#include "mozilla/Assertions.h" +#include "mozilla/FOGIPC.h" +#include "mozilla/Preferences.h" +#include "mozilla/RemoteDecoderManagerParent.h" +#include "mozilla/TimeStamp.h" +#include "mozilla/dom/MemoryReportRequest.h" +#include "mozilla/gfx/gfxVars.h" +#include "mozilla/glean/GleanMetrics.h" +#include "mozilla/ipc/CrashReporterClient.h" +#include "mozilla/ipc/ProcessChild.h" + +#if defined(XP_LINUX) && defined(MOZ_SANDBOX) +# include "mozilla/Sandbox.h" +#endif + +#include "ChildProfilerController.h" + +#if defined(XP_MACOSX) && defined(MOZ_SANDBOX) +# include "RDDProcessHost.h" +# include "mozilla/Sandbox.h" +# include "nsMacUtilsImpl.h" +#endif + +#include "mozilla/ipc/ProcessUtils.h" +#include "nsDebugImpl.h" +#include "nsIXULRuntime.h" +#include "nsThreadManager.h" + +#if defined(MOZ_SANDBOX) && defined(MOZ_DEBUG) && defined(ENABLE_TESTS) +# include "mozilla/SandboxTestingChild.h" +#endif + +namespace mozilla { + +using namespace ipc; +using namespace gfx; + +static RDDParent* sRDDParent; + +RDDParent::RDDParent() : mLaunchTime(TimeStamp::Now()) { sRDDParent = this; } + +RDDParent::~RDDParent() { sRDDParent = nullptr; } + +/* static */ +RDDParent* RDDParent::GetSingleton() { + MOZ_DIAGNOSTIC_ASSERT(sRDDParent); + return sRDDParent; +} + +bool RDDParent::Init(mozilla::ipc::UntypedEndpoint&& aEndpoint, + const char* aParentBuildID) { + // Initialize the thread manager before starting IPC. Otherwise, messages + // may be posted to the main thread and we won't be able to process them. + if (NS_WARN_IF(NS_FAILED(nsThreadManager::get().Init()))) { + return false; + } + + // Now it's safe to start IPC. + if (NS_WARN_IF(!aEndpoint.Bind(this))) { + return false; + } + + nsDebugImpl::SetMultiprocessMode("RDD"); + + // This must be checked before any IPDL message, which may hit sentinel + // errors due to parent and content processes having different + // versions. + MessageChannel* channel = GetIPCChannel(); + if (channel && !channel->SendBuildIDsMatchMessage(aParentBuildID)) { + // We need to quit this process if the buildID doesn't match the parent's. + // This can occur when an update occurred in the background. + ProcessChild::QuickExit(); + } + + // Init crash reporter support. + CrashReporterClient::InitSingleton(this); + + if (NS_FAILED(NS_InitMinimalXPCOM())) { + return false; + } + + gfxConfig::Init(); + gfxVars::Initialize(); +#ifdef XP_WIN + DeviceManagerDx::Init(); + auto rv = wmf::MediaFoundationInitializer::HasInitialized(); + if (!rv) { + NS_WARNING("Failed to init Media Foundation in the RDD process"); + } +#endif + + mozilla::ipc::SetThisProcessName("RDD Process"); + + return true; +} + +#if defined(XP_MACOSX) && defined(MOZ_SANDBOX) +extern "C" { +void CGSShutdownServerConnections(); +}; +#endif + +mozilla::ipc::IPCResult RDDParent::RecvInit( + nsTArray<GfxVarUpdate>&& vars, const Maybe<FileDescriptor>& aBrokerFd, + const bool& aCanRecordReleaseTelemetry, + const bool& aIsReadyForBackgroundProcessing) { + for (const auto& var : vars) { + gfxVars::ApplyUpdate(var); + } + + auto supported = PDMFactory::Supported(); + Unused << SendUpdateMediaCodecsSupported(supported); + +#if defined(MOZ_SANDBOX) +# if defined(XP_MACOSX) + // Close all current connections to the WindowServer. This ensures that the + // Activity Monitor will not label the content process as "Not responding" + // because it's not running a native event loop. See bug 1384336. + CGSShutdownServerConnections(); + +# elif defined(XP_LINUX) + int fd = -1; + if (aBrokerFd.isSome()) { + fd = aBrokerFd.value().ClonePlatformHandle().release(); + } + SetRemoteDataDecoderSandbox(fd); +# endif // XP_MACOSX/XP_LINUX +#endif // MOZ_SANDBOX + +#if defined(XP_WIN) + if (aCanRecordReleaseTelemetry) { + RefPtr<DllServices> dllSvc(DllServices::Get()); + dllSvc->StartUntrustedModulesProcessor(aIsReadyForBackgroundProcessing); + } +#endif // defined(XP_WIN) + return IPC_OK(); +} + +IPCResult RDDParent::RecvUpdateVar(const GfxVarUpdate& aUpdate) { +#if defined(XP_WIN) + auto scopeExit = MakeScopeExit( + [couldUseHWDecoder = gfx::gfxVars::CanUseHardwareVideoDecoding()] { + if (couldUseHWDecoder != gfx::gfxVars::CanUseHardwareVideoDecoding()) { + // The capabilities of the system may have changed, force a refresh by + // re-initializing the WMF PDM. + WMFDecoderModule::Init(); + Unused << RDDParent::GetSingleton()->SendUpdateMediaCodecsSupported( + PDMFactory::Supported(true /* force refresh */)); + } + }); +#endif + gfxVars::ApplyUpdate(aUpdate); + return IPC_OK(); +} + +mozilla::ipc::IPCResult RDDParent::RecvInitProfiler( + Endpoint<PProfilerChild>&& aEndpoint) { + mProfilerController = ChildProfilerController::Create(std::move(aEndpoint)); + return IPC_OK(); +} + +mozilla::ipc::IPCResult RDDParent::RecvNewContentRemoteDecoderManager( + Endpoint<PRemoteDecoderManagerParent>&& aEndpoint) { + if (!RemoteDecoderManagerParent::CreateForContent(std::move(aEndpoint))) { + return IPC_FAIL_NO_REASON(this); + } + return IPC_OK(); +} + +mozilla::ipc::IPCResult RDDParent::RecvInitVideoBridge( + Endpoint<PVideoBridgeChild>&& aEndpoint, const bool& aCreateHardwareDevice, + const ContentDeviceData& aContentDeviceData) { + if (!RemoteDecoderManagerParent::CreateVideoBridgeToOtherProcess( + std::move(aEndpoint))) { + return IPC_FAIL_NO_REASON(this); + } + + gfxConfig::Inherit( + { + Feature::HW_COMPOSITING, + Feature::D3D11_COMPOSITING, + Feature::OPENGL_COMPOSITING, + Feature::DIRECT2D, + }, + aContentDeviceData.prefs()); +#ifdef XP_WIN + if (gfxConfig::IsEnabled(Feature::D3D11_COMPOSITING)) { + auto* devmgr = DeviceManagerDx::Get(); + if (devmgr) { + devmgr->ImportDeviceInfo(aContentDeviceData.d3d11()); + if (aCreateHardwareDevice) { + devmgr->CreateContentDevices(); + } + } + } +#endif + + return IPC_OK(); +} + +mozilla::ipc::IPCResult RDDParent::RecvRequestMemoryReport( + const uint32_t& aGeneration, const bool& aAnonymize, + const bool& aMinimizeMemoryUsage, const Maybe<FileDescriptor>& aDMDFile, + const RequestMemoryReportResolver& aResolver) { + nsPrintfCString processName("RDD (pid %u)", (unsigned)getpid()); + + mozilla::dom::MemoryReportRequestClient::Start( + aGeneration, aAnonymize, aMinimizeMemoryUsage, aDMDFile, processName, + [&](const MemoryReport& aReport) { + Unused << GetSingleton()->SendAddMemoryReport(aReport); + }, + aResolver); + return IPC_OK(); +} + +#if defined(XP_WIN) +mozilla::ipc::IPCResult RDDParent::RecvGetUntrustedModulesData( + GetUntrustedModulesDataResolver&& aResolver) { + RefPtr<DllServices> dllSvc(DllServices::Get()); + dllSvc->GetUntrustedModulesData()->Then( + GetMainThreadSerialEventTarget(), __func__, + [aResolver](Maybe<UntrustedModulesData>&& aData) { + aResolver(std::move(aData)); + }, + [aResolver](nsresult aReason) { aResolver(Nothing()); }); + return IPC_OK(); +} + +mozilla::ipc::IPCResult RDDParent::RecvUnblockUntrustedModulesThread() { + if (nsCOMPtr<nsIObserverService> obs = + mozilla::services::GetObserverService()) { + obs->NotifyObservers(nullptr, "unblock-untrusted-modules-thread", nullptr); + } + return IPC_OK(); +} +#endif // defined(XP_WIN) + +mozilla::ipc::IPCResult RDDParent::RecvPreferenceUpdate(const Pref& aPref) { + Preferences::SetPreference(aPref); + return IPC_OK(); +} + +#if defined(MOZ_SANDBOX) && defined(MOZ_DEBUG) && defined(ENABLE_TESTS) +mozilla::ipc::IPCResult RDDParent::RecvInitSandboxTesting( + Endpoint<PSandboxTestingChild>&& aEndpoint) { + if (!SandboxTestingChild::Initialize(std::move(aEndpoint))) { + return IPC_FAIL( + this, "InitSandboxTesting failed to initialise the child process."); + } + return IPC_OK(); +} +#endif + +mozilla::ipc::IPCResult RDDParent::RecvFlushFOGData( + FlushFOGDataResolver&& aResolver) { + glean::FlushFOGData(std::move(aResolver)); + return IPC_OK(); +} + +mozilla::ipc::IPCResult RDDParent::RecvTestTriggerMetrics( + TestTriggerMetricsResolver&& aResolve) { + mozilla::glean::test_only_ipc::a_counter.Add(nsIXULRuntime::PROCESS_TYPE_RDD); + aResolve(true); + return IPC_OK(); +} + +void RDDParent::ActorDestroy(ActorDestroyReason aWhy) { + if (AbnormalShutdown == aWhy) { + NS_WARNING("Shutting down RDD process early due to a crash!"); + Telemetry::Accumulate(Telemetry::SUBPROCESS_ABNORMAL_ABORT, "rdd"_ns, 1); + ProcessChild::QuickExit(); + } + + // Send the last bits of Glean data over to the main process. + glean::FlushFOGData( + [](ByteBuf&& aBuf) { glean::SendFOGData(std::move(aBuf)); }); + +#ifndef NS_FREE_PERMANENT_DATA + // No point in going through XPCOM shutdown because we don't keep persistent + // state. + ProcessChild::QuickExit(); +#endif + + // Wait until all RemoteDecoderManagerParent have closed. + mShutdownBlockers.WaitUntilClear(10 * 1000 /* 10s timeout*/) + ->Then(GetCurrentSerialEventTarget(), __func__, [&]() { + +#if defined(XP_WIN) + RefPtr<DllServices> dllSvc(DllServices::Get()); + dllSvc->DisableFull(); +#endif // defined(XP_WIN) + + if (mProfilerController) { + mProfilerController->Shutdown(); + mProfilerController = nullptr; + } + + RemoteDecoderManagerParent::ShutdownVideoBridge(); + +#ifdef XP_WIN + DeviceManagerDx::Shutdown(); +#endif + gfxVars::Shutdown(); + gfxConfig::Shutdown(); + CrashReporterClient::DestroySingleton(); + XRE_ShutdownChildProcess(); + }); +} + +} // namespace mozilla diff --git a/dom/media/ipc/RDDParent.h b/dom/media/ipc/RDDParent.h new file mode 100644 index 0000000000..5c640623f6 --- /dev/null +++ b/dom/media/ipc/RDDParent.h @@ -0,0 +1,75 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef _include_dom_media_ipc_RDDParent_h__ +#define _include_dom_media_ipc_RDDParent_h__ +#include "mozilla/PRDDParent.h" + +#include "mozilla/RefPtr.h" +#include "mozilla/ipc/AsyncBlockers.h" + +namespace mozilla { + +class TimeStamp; +class ChildProfilerController; + +class RDDParent final : public PRDDParent { + public: + RDDParent(); + ~RDDParent(); + + static RDDParent* GetSingleton(); + + ipc::AsyncBlockers& AsyncShutdownService() { return mShutdownBlockers; } + + bool Init(mozilla::ipc::UntypedEndpoint&& aEndpoint, + const char* aParentBuildID); + + mozilla::ipc::IPCResult RecvInit(nsTArray<GfxVarUpdate>&& vars, + const Maybe<ipc::FileDescriptor>& aBrokerFd, + const bool& aCanRecordReleaseTelemetry, + const bool& aIsReadyForBackgroundProcessing); + mozilla::ipc::IPCResult RecvInitProfiler( + Endpoint<PProfilerChild>&& aEndpoint); + + mozilla::ipc::IPCResult RecvNewContentRemoteDecoderManager( + Endpoint<PRemoteDecoderManagerParent>&& aEndpoint); + mozilla::ipc::IPCResult RecvInitVideoBridge( + Endpoint<PVideoBridgeChild>&& aEndpoint, + const bool& aCreateHardwareDevice, + const ContentDeviceData& aContentDeviceData); + mozilla::ipc::IPCResult RecvRequestMemoryReport( + const uint32_t& generation, const bool& anonymize, + const bool& minimizeMemoryUsage, + const Maybe<ipc::FileDescriptor>& DMDFile, + const RequestMemoryReportResolver& aResolver); +#if defined(XP_WIN) + mozilla::ipc::IPCResult RecvGetUntrustedModulesData( + GetUntrustedModulesDataResolver&& aResolver); + mozilla::ipc::IPCResult RecvUnblockUntrustedModulesThread(); +#endif // defined(XP_WIN) + mozilla::ipc::IPCResult RecvPreferenceUpdate(const Pref& pref); + mozilla::ipc::IPCResult RecvUpdateVar(const GfxVarUpdate& pref); + +#if defined(MOZ_SANDBOX) && defined(MOZ_DEBUG) && defined(ENABLE_TESTS) + mozilla::ipc::IPCResult RecvInitSandboxTesting( + Endpoint<PSandboxTestingChild>&& aEndpoint); +#endif + void ActorDestroy(ActorDestroyReason aWhy) override; + + mozilla::ipc::IPCResult RecvFlushFOGData(FlushFOGDataResolver&& aResolver); + + mozilla::ipc::IPCResult RecvTestTriggerMetrics( + TestTriggerMetricsResolver&& aResolve); + + private: + const TimeStamp mLaunchTime; + RefPtr<ChildProfilerController> mProfilerController; + ipc::AsyncBlockers mShutdownBlockers; +}; + +} // namespace mozilla + +#endif // _include_dom_media_ipc_RDDParent_h__ diff --git a/dom/media/ipc/RDDProcessHost.cpp b/dom/media/ipc/RDDProcessHost.cpp new file mode 100644 index 0000000000..d4620c3e3a --- /dev/null +++ b/dom/media/ipc/RDDProcessHost.cpp @@ -0,0 +1,312 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "RDDProcessHost.h" + +#include "mozilla/dom/ContentParent.h" +#include "mozilla/ipc/ProcessUtils.h" +#include "RDDChild.h" +#include "chrome/common/process_watcher.h" +#include "mozilla/Preferences.h" +#include "mozilla/StaticPrefs_media.h" + +#if defined(XP_MACOSX) && defined(MOZ_SANDBOX) +# include "mozilla/Sandbox.h" +#endif + +namespace mozilla { + +using namespace ipc; + +#if defined(XP_MACOSX) && defined(MOZ_SANDBOX) +bool RDDProcessHost::sLaunchWithMacSandbox = false; +#endif + +RDDProcessHost::RDDProcessHost(Listener* aListener) + : GeckoChildProcessHost(GeckoProcessType_RDD), + mListener(aListener), + mLiveToken(new media::Refcountable<bool>(true)) { + MOZ_COUNT_CTOR(RDDProcessHost); + +#if defined(XP_MACOSX) && defined(MOZ_SANDBOX) + if (!sLaunchWithMacSandbox) { + sLaunchWithMacSandbox = (PR_GetEnv("MOZ_DISABLE_RDD_SANDBOX") == nullptr); + } + mDisableOSActivityMode = sLaunchWithMacSandbox; +#endif +} + +RDDProcessHost::~RDDProcessHost() { MOZ_COUNT_DTOR(RDDProcessHost); } + +bool RDDProcessHost::Launch(StringVector aExtraOpts) { + MOZ_ASSERT(NS_IsMainThread()); + + MOZ_ASSERT(mLaunchPhase == LaunchPhase::Unlaunched); + MOZ_ASSERT(!mRDDChild); + + mPrefSerializer = MakeUnique<ipc::SharedPreferenceSerializer>(); + if (!mPrefSerializer->SerializeToSharedMemory(GeckoProcessType_RDD, + /* remoteType */ ""_ns)) { + return false; + } + mPrefSerializer->AddSharedPrefCmdLineArgs(*this, aExtraOpts); + +#if defined(XP_WIN) && defined(MOZ_SANDBOX) + mSandboxLevel = Preferences::GetInt("security.sandbox.rdd.level"); +#endif + + mLaunchPhase = LaunchPhase::Waiting; + mLaunchTime = TimeStamp::Now(); + + int32_t timeoutMs = StaticPrefs::media_rdd_process_startup_timeout_ms(); + + // If one of the following environment variables are set we can + // effectively ignore the timeout - as we can guarantee the RDD + // process will be terminated + if (PR_GetEnv("MOZ_DEBUG_CHILD_PROCESS") || + PR_GetEnv("MOZ_DEBUG_CHILD_PAUSE")) { + timeoutMs = 0; + } + if (timeoutMs) { + // We queue a delayed task. If that task runs before the + // WhenProcessHandleReady promise gets resolved, we will abort the launch. + GetMainThreadSerialEventTarget()->DelayedDispatch( + NS_NewRunnableFunction( + "RDDProcessHost::Launchtimeout", + [this, liveToken = mLiveToken]() { + if (!*liveToken || mTimerChecked) { + // We have been deleted or the runnable has already started, we + // can abort. + return; + } + InitAfterConnect(false); + MOZ_ASSERT(mTimerChecked, + "InitAfterConnect must have acted on the promise"); + }), + timeoutMs); + } + + if (!GeckoChildProcessHost::AsyncLaunch(aExtraOpts)) { + mLaunchPhase = LaunchPhase::Complete; + mPrefSerializer = nullptr; + return false; + } + return true; +} + +RefPtr<GenericNonExclusivePromise> RDDProcessHost::LaunchPromise() { + MOZ_ASSERT(NS_IsMainThread()); + + if (mLaunchPromise) { + return mLaunchPromise; + } + mLaunchPromise = MakeRefPtr<GenericNonExclusivePromise::Private>(__func__); + WhenProcessHandleReady()->Then( + GetCurrentSerialEventTarget(), __func__, + [this, liveToken = mLiveToken]( + const ipc::ProcessHandlePromise::ResolveOrRejectValue& aResult) { + if (!*liveToken) { + // The RDDProcessHost got deleted. Abort. The promise would have + // already been rejected. + return; + } + if (mTimerChecked) { + // We hit the timeout earlier, abort. + return; + } + mTimerChecked = true; + if (aResult.IsReject()) { + RejectPromise(); + } + // If aResult.IsResolve() then we have succeeded in launching the + // RDD process. The promise will be resolved once the channel has + // connected (or failed to) later. + }); + return mLaunchPromise; +} + +void RDDProcessHost::OnChannelConnected(base::ProcessId peer_pid) { + MOZ_ASSERT(!NS_IsMainThread()); + + GeckoChildProcessHost::OnChannelConnected(peer_pid); + + NS_DispatchToMainThread(NS_NewRunnableFunction( + "RDDProcessHost::OnChannelConnected", [this, liveToken = mLiveToken]() { + if (*liveToken && mLaunchPhase == LaunchPhase::Waiting) { + InitAfterConnect(true); + } + })); +} + +void RDDProcessHost::OnChannelError() { + MOZ_ASSERT(!NS_IsMainThread()); + + GeckoChildProcessHost::OnChannelError(); + + NS_DispatchToMainThread(NS_NewRunnableFunction( + "RDDProcessHost::OnChannelError", [this, liveToken = mLiveToken]() { + if (*liveToken && mLaunchPhase == LaunchPhase::Waiting) { + InitAfterConnect(false); + } + })); +} + +static uint64_t sRDDProcessTokenCounter = 0; + +void RDDProcessHost::InitAfterConnect(bool aSucceeded) { + MOZ_ASSERT(NS_IsMainThread()); + + MOZ_ASSERT(mLaunchPhase == LaunchPhase::Waiting); + MOZ_ASSERT(!mRDDChild); + + mLaunchPhase = LaunchPhase::Complete; + + if (!aSucceeded) { + RejectPromise(); + return; + } + mProcessToken = ++sRDDProcessTokenCounter; + mRDDChild = MakeUnique<RDDChild>(this); + DebugOnly<bool> rv = TakeInitialEndpoint().Bind(mRDDChild.get()); + MOZ_ASSERT(rv); + + // Only clear mPrefSerializer in the success case to avoid a + // possible race in the case case of a timeout on Windows launch. + // See Bug 1555076 comment 7: + // https://bugzilla.mozilla.org/show_bug.cgi?id=1555076#c7 + mPrefSerializer = nullptr; + + if (!mRDDChild->Init()) { + // Can't just kill here because it will create a timing race that + // will crash the tab. We don't really want to crash the tab just + // because RDD linux sandbox failed to initialize. In this case, + // we'll close the child channel which will cause the RDD process + // to shutdown nicely avoiding the tab crash (which manifests as + // Bug 1535335). + mRDDChild->Close(); + RejectPromise(); + } else { + ResolvePromise(); + } +} + +void RDDProcessHost::Shutdown() { + MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(!mShutdownRequested); + + RejectPromise(); + + if (mRDDChild) { + // OnChannelClosed uses this to check if the shutdown was expected or + // unexpected. + mShutdownRequested = true; + + // The channel might already be closed if we got here unexpectedly. + if (!mChannelClosed) { + mRDDChild->Close(); + } + +#ifndef NS_FREE_PERMANENT_DATA + // No need to communicate shutdown, the RDD process doesn't need to + // communicate anything back. + KillHard("NormalShutdown"); +#endif + + // If we're shutting down unexpectedly, we're in the middle of handling an + // ActorDestroy for PRDDChild, which is still on the stack. We'll return + // back to OnChannelClosed. + // + // Otherwise, we'll wait for OnChannelClose to be called whenever PRDDChild + // acknowledges shutdown. + return; + } + + DestroyProcess(); +} + +void RDDProcessHost::OnChannelClosed() { + MOZ_ASSERT(NS_IsMainThread()); + + mChannelClosed = true; + RejectPromise(); + + if (!mShutdownRequested && mListener) { + // This is an unclean shutdown. Notify our listener that we're going away. + mListener->OnProcessUnexpectedShutdown(this); + } else { + DestroyProcess(); + } + + // Release the actor. + RDDChild::Destroy(std::move(mRDDChild)); +} + +void RDDProcessHost::KillHard(const char* aReason) { + MOZ_ASSERT(NS_IsMainThread()); + + ProcessHandle handle = GetChildProcessHandle(); + if (!base::KillProcess(handle, base::PROCESS_END_KILLED_BY_USER)) { + NS_WARNING("failed to kill subprocess!"); + } + + SetAlreadyDead(); +} + +uint64_t RDDProcessHost::GetProcessToken() const { + MOZ_ASSERT(NS_IsMainThread()); + return mProcessToken; +} + +void RDDProcessHost::DestroyProcess() { + MOZ_ASSERT(NS_IsMainThread()); + RejectPromise(); + + // Any pending tasks will be cancelled from now on. + *mLiveToken = false; + + NS_DispatchToMainThread( + NS_NewRunnableFunction("DestroyProcessRunnable", [this] { Destroy(); })); +} + +void RDDProcessHost::ResolvePromise() { + MOZ_ASSERT(NS_IsMainThread()); + + if (!mLaunchPromiseSettled) { + mLaunchPromise->Resolve(true, __func__); + mLaunchPromiseSettled = true; + } + // We have already acted on the promise; the timeout runnable no longer needs + // to interrupt anything. + mTimerChecked = true; +} + +void RDDProcessHost::RejectPromise() { + MOZ_ASSERT(NS_IsMainThread()); + + if (!mLaunchPromiseSettled) { + mLaunchPromise->Reject(NS_ERROR_FAILURE, __func__); + mLaunchPromiseSettled = true; + } + // We have already acted on the promise; the timeout runnable no longer needs + // to interrupt anything. + mTimerChecked = true; +} + +#if defined(XP_MACOSX) && defined(MOZ_SANDBOX) +bool RDDProcessHost::FillMacSandboxInfo(MacSandboxInfo& aInfo) { + GeckoChildProcessHost::FillMacSandboxInfo(aInfo); + if (!aInfo.shouldLog && PR_GetEnv("MOZ_SANDBOX_RDD_LOGGING")) { + aInfo.shouldLog = true; + } + return true; +} + +/* static */ +MacSandboxType RDDProcessHost::GetMacSandboxType() { + return MacSandboxType_RDD; +} +#endif + +} // namespace mozilla diff --git a/dom/media/ipc/RDDProcessHost.h b/dom/media/ipc/RDDProcessHost.h new file mode 100644 index 0000000000..68a949721a --- /dev/null +++ b/dom/media/ipc/RDDProcessHost.h @@ -0,0 +1,161 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _include_dom_media_ipc_RDDProcessHost_h_ +#define _include_dom_media_ipc_RDDProcessHost_h_ +#include "mozilla/UniquePtr.h" +#include "mozilla/ipc/GeckoChildProcessHost.h" +#include "mozilla/ipc/ProtocolUtils.h" +#include "mozilla/media/MediaUtils.h" + +namespace mozilla::ipc { +class SharedPreferenceSerializer; +} // namespace mozilla::ipc +class nsITimer; + +namespace mozilla { + +class RDDChild; + +// RDDProcessHost is the "parent process" container for a subprocess handle and +// IPC connection. It owns the parent process IPDL actor, which in this case, +// is a RDDChild. +// +// RDDProcessHosts are allocated and managed by RDDProcessManager. For all +// intents and purposes it is a singleton, though more than one may be allocated +// at a time due to its shutdown being asynchronous. +class RDDProcessHost final : public mozilla::ipc::GeckoChildProcessHost { + friend class RDDChild; + + public: + class Listener { + public: + // The RDDProcessHost has unexpectedly shutdown or had its connection + // severed. This is not called if an error occurs after calling + // Shutdown(). + virtual void OnProcessUnexpectedShutdown(RDDProcessHost* aHost) {} + }; + + explicit RDDProcessHost(Listener* listener); + + // Launch the subprocess asynchronously. On failure, false is returned. + // Otherwise, true is returned. If succeeded, a follow-up call should be made + // to LaunchPromise() which will return a promise that will be resolved once + // the RDD process has launched and a channel has been established. + // + // @param aExtraOpts (StringVector) + // Extra options to pass to the subprocess. + bool Launch(StringVector aExtraOpts); + + // Return a promise that will be resolved once the process has completed its + // launch. The promise will be immediately resolved if the launch has already + // succeeded. + RefPtr<GenericNonExclusivePromise> LaunchPromise(); + + // Inform the process that it should clean up its resources and shut + // down. This initiates an asynchronous shutdown sequence. After this + // method returns, it is safe for the caller to forget its pointer to + // the RDDProcessHost. + // + // After this returns, the attached Listener is no longer used. + void Shutdown(); + + // Return the actor for the top-level actor of the process. If the process + // has not connected yet, this returns null. + RDDChild* GetActor() const { + MOZ_ASSERT(NS_IsMainThread()); + return mRDDChild.get(); + } + + // Return a unique id for this process, guaranteed not to be shared with any + // past or future instance of RDDProcessHost. + uint64_t GetProcessToken() const; + + bool IsConnected() const { + MOZ_ASSERT(NS_IsMainThread()); + return !!mRDDChild; + } + + // Return the time stamp for when we tried to launch the RDD process. + // This is currently used for Telemetry so that we can determine how + // long RDD processes take to spin up. Note this doesn't denote a + // successful launch, just when we attempted launch. + TimeStamp GetLaunchTime() const { return mLaunchTime; } + + // Called on the IO thread. + void OnChannelConnected(base::ProcessId peer_pid) override; + void OnChannelError() override; + + void SetListener(Listener* aListener); + +#if defined(XP_MACOSX) && defined(MOZ_SANDBOX) + // Return the sandbox type to be used with this process type. + static MacSandboxType GetMacSandboxType(); +#endif + + private: + ~RDDProcessHost(); + + // Called on the main thread with true after a connection has been established + // or false if it failed (including if it failed before the timeout kicked in) + void InitAfterConnect(bool aSucceeded); + + // Called on the main thread when the mRDDChild actor is shutting down. + void OnChannelClosed(); + + // Kill the remote process, triggering IPC shutdown. + void KillHard(const char* aReason); + + void DestroyProcess(); + +#if defined(XP_MACOSX) && defined(MOZ_SANDBOX) + static bool sLaunchWithMacSandbox; + + // Sandbox the RDD process at launch for all instances + bool IsMacSandboxLaunchEnabled() override { return sLaunchWithMacSandbox; } + + // Override so we can turn on RDD process-specific sandbox logging + bool FillMacSandboxInfo(MacSandboxInfo& aInfo) override; +#endif + + DISALLOW_COPY_AND_ASSIGN(RDDProcessHost); + + Listener* const mListener; + + // All members below are only ever accessed on the main thread. + enum class LaunchPhase { Unlaunched, Waiting, Complete }; + LaunchPhase mLaunchPhase = LaunchPhase::Unlaunched; + + UniquePtr<RDDChild> mRDDChild; + uint64_t mProcessToken = 0; + + UniquePtr<ipc::SharedPreferenceSerializer> mPrefSerializer; + + bool mShutdownRequested = false; + bool mChannelClosed = false; + + TimeStamp mLaunchTime; + void RejectPromise(); + void ResolvePromise(); + + // Set to true on construction and to false just prior deletion. + // The RDDProcessHost isn't refcounted; so we can capture this by value in + // lambdas along with a strong reference to mLiveToken and check if that value + // is true before accessing "this". + // While a reference to mLiveToken can be taken on any thread; its value can + // only be read on the main thread. + const RefPtr<media::Refcountable<bool>> mLiveToken; + RefPtr<GenericNonExclusivePromise::Private> mLaunchPromise; + bool mLaunchPromiseSettled = false; + // Will be set to true if we've exceeded the allowed startup time or if the + // RDD process as successfully started. This is used to determine if the + // timeout runnable needs to execute code or not. + bool mTimerChecked = false; +}; + +} // namespace mozilla + +#endif // _include_dom_media_ipc_RDDProcessHost_h_ diff --git a/dom/media/ipc/RDDProcessImpl.cpp b/dom/media/ipc/RDDProcessImpl.cpp new file mode 100644 index 0000000000..ca7b7c65b4 --- /dev/null +++ b/dom/media/ipc/RDDProcessImpl.cpp @@ -0,0 +1,51 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "RDDProcessImpl.h" + +#include "mozilla/ipc/IOThreadChild.h" +#include "mozilla/GeckoArgs.h" + +#if defined(OS_WIN) && defined(MOZ_SANDBOX) +# include "mozilla/sandboxTarget.h" +#elif defined(__OpenBSD__) && defined(MOZ_SANDBOX) +# include "mozilla/SandboxSettings.h" +# include "prlink.h" +#endif + +namespace mozilla { + +using namespace ipc; + +RDDProcessImpl::~RDDProcessImpl() = default; + +bool RDDProcessImpl::Init(int aArgc, char* aArgv[]) { +#if defined(MOZ_SANDBOX) && defined(OS_WIN) + // Preload AV dlls so we can enable Binary Signature Policy + // to restrict further dll loads. + LoadLibraryW(L"mozavcodec.dll"); + LoadLibraryW(L"mozavutil.dll"); + mozilla::SandboxTarget::Instance()->StartSandbox(); +#elif defined(__OpenBSD__) && defined(MOZ_SANDBOX) + PR_LoadLibrary("libmozavcodec.so"); + PR_LoadLibrary("libmozavutil.so"); + StartOpenBSDSandbox(GeckoProcessType_RDD); +#endif + Maybe<const char*> parentBuildID = + geckoargs::sParentBuildID.Get(aArgc, aArgv); + if (parentBuildID.isNothing()) { + return false; + } + + if (!ProcessChild::InitPrefs(aArgc, aArgv)) { + return false; + } + + return mRDD.Init(TakeInitialEndpoint(), *parentBuildID); +} + +void RDDProcessImpl::CleanUp() { NS_ShutdownXPCOM(nullptr); } + +} // namespace mozilla diff --git a/dom/media/ipc/RDDProcessImpl.h b/dom/media/ipc/RDDProcessImpl.h new file mode 100644 index 0000000000..4911cc9d97 --- /dev/null +++ b/dom/media/ipc/RDDProcessImpl.h @@ -0,0 +1,39 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef _include_dom_media_ipc_RDDProcessImpl_h__ +#define _include_dom_media_ipc_RDDProcessImpl_h__ +#include "mozilla/ipc/ProcessChild.h" + +#if defined(XP_WIN) +# include "mozilla/mscom/ProcessRuntime.h" +#endif + +#include "RDDParent.h" + +namespace mozilla { + +// This class owns the subprocess instance of a PRDD - which in this case, +// is a RDDParent. It is instantiated as a singleton in XRE_InitChildProcess. +class RDDProcessImpl final : public ipc::ProcessChild { + public: + using ipc::ProcessChild::ProcessChild; + ~RDDProcessImpl(); + + bool Init(int aArgc, char* aArgv[]) override; + void CleanUp() override; + + private: + RDDParent mRDD; + +#if defined(XP_WIN) + // This object initializes and configures COM. + mozilla::mscom::ProcessRuntime mCOMRuntime; +#endif +}; + +} // namespace mozilla + +#endif // _include_dom_media_ipc_RDDProcessImpl_h__ diff --git a/dom/media/ipc/RDDProcessManager.cpp b/dom/media/ipc/RDDProcessManager.cpp new file mode 100644 index 0000000000..42aee9e3e3 --- /dev/null +++ b/dom/media/ipc/RDDProcessManager.cpp @@ -0,0 +1,407 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "RDDProcessManager.h" + +#include "RDDChild.h" +#include "RDDProcessHost.h" +#include "mozilla/MemoryReportingProcess.h" +#include "mozilla/Preferences.h" +#include "mozilla/RemoteDecoderManagerChild.h" +#include "mozilla/RemoteDecoderManagerParent.h" +#include "mozilla/StaticPrefs_media.h" +#include "mozilla/SyncRunnable.h" // for LaunchRDDProcess +#include "mozilla/dom/ContentParent.h" +#include "mozilla/gfx/GPUProcessManager.h" +#include "mozilla/ipc/Endpoint.h" +#include "mozilla/ipc/ProcessChild.h" +#include "mozilla/layers/CompositorThread.h" +#include "mozilla/layers/VideoBridgeParent.h" +#include "nsAppRunner.h" +#include "nsContentUtils.h" + +namespace mozilla { + +using namespace gfx; +using namespace layers; + +static StaticAutoPtr<RDDProcessManager> sRDDSingleton; + +static bool sXPCOMShutdown = false; + +bool RDDProcessManager::IsShutdown() const { + MOZ_ASSERT(NS_IsMainThread()); + return sXPCOMShutdown || !sRDDSingleton; +} + +RDDProcessManager* RDDProcessManager::Get() { return sRDDSingleton; } + +void RDDProcessManager::Initialize() { + MOZ_ASSERT(XRE_IsParentProcess()); + sRDDSingleton = new RDDProcessManager(); +} + +void RDDProcessManager::Shutdown() { sRDDSingleton = nullptr; } + +RDDProcessManager::RDDProcessManager() + : mObserver(new Observer(this)), mTaskFactory(this) { + MOZ_COUNT_CTOR(RDDProcessManager); + // Start listening for pref changes so we can + // forward them to the process once it is running. + nsContentUtils::RegisterShutdownObserver(mObserver); + Preferences::AddStrongObserver(mObserver, ""); +} + +RDDProcessManager::~RDDProcessManager() { + MOZ_COUNT_DTOR(RDDProcessManager); + MOZ_ASSERT(NS_IsMainThread()); + + // The RDD process should have already been shut down. + MOZ_ASSERT(!mProcess && !mRDDChild); +} + +NS_IMPL_ISUPPORTS(RDDProcessManager::Observer, nsIObserver); + +RDDProcessManager::Observer::Observer(RDDProcessManager* aManager) + : mManager(aManager) {} + +NS_IMETHODIMP +RDDProcessManager::Observer::Observe(nsISupports* aSubject, const char* aTopic, + const char16_t* aData) { + if (!strcmp(aTopic, NS_XPCOM_SHUTDOWN_OBSERVER_ID)) { + mManager->OnXPCOMShutdown(); + } else if (!strcmp(aTopic, "nsPref:changed")) { + mManager->OnPreferenceChange(aData); + } + return NS_OK; +} + +void RDDProcessManager::OnXPCOMShutdown() { + MOZ_ASSERT(NS_IsMainThread()); + sXPCOMShutdown = true; + nsContentUtils::UnregisterShutdownObserver(mObserver); + Preferences::RemoveObserver(mObserver, ""); + CleanShutdown(); +} + +void RDDProcessManager::OnPreferenceChange(const char16_t* aData) { + MOZ_ASSERT(NS_IsMainThread()); + if (!mProcess) { + // Process hasn't been launched yet + return; + } + + // We know prefs are ASCII here. + NS_LossyConvertUTF16toASCII strData(aData); + + mozilla::dom::Pref pref(strData, /* isLocked */ false, + /* isSanitized */ false, Nothing(), Nothing()); + + Preferences::GetPreference(&pref, GeckoProcessType_RDD, + /* remoteType */ ""_ns); + if (!!mRDDChild) { + MOZ_ASSERT(mQueuedPrefs.IsEmpty()); + mRDDChild->SendPreferenceUpdate(pref); + } else if (IsRDDProcessLaunching()) { + mQueuedPrefs.AppendElement(pref); + } +} + +RefPtr<GenericNonExclusivePromise> RDDProcessManager::LaunchRDDProcess() { + MOZ_ASSERT(NS_IsMainThread()); + + if (IsShutdown()) { + return GenericNonExclusivePromise::CreateAndReject(NS_ERROR_NOT_AVAILABLE, + __func__); + } + + if (mNumProcessAttempts && !StaticPrefs::media_rdd_retryonfailure_enabled()) { + // We failed to start the RDD process earlier, abort now. + return GenericNonExclusivePromise::CreateAndReject(NS_ERROR_NOT_AVAILABLE, + __func__); + } + + if (mLaunchRDDPromise && mProcess) { + return mLaunchRDDPromise; + } + + std::vector<std::string> extraArgs; + ipc::ProcessChild::AddPlatformBuildID(extraArgs); + + // The subprocess is launched asynchronously, so we + // wait for the promise to be resolved to acquire the IPDL actor. + mProcess = new RDDProcessHost(this); + if (!mProcess->Launch(extraArgs)) { + mNumProcessAttempts++; + DestroyProcess(); + return GenericNonExclusivePromise::CreateAndReject(NS_ERROR_NOT_AVAILABLE, + __func__); + } + + mLaunchRDDPromise = mProcess->LaunchPromise()->Then( + GetMainThreadSerialEventTarget(), __func__, + [this](bool) { + if (IsShutdown()) { + return GenericNonExclusivePromise::CreateAndReject( + NS_ERROR_NOT_AVAILABLE, __func__); + } + + if (IsRDDProcessDestroyed()) { + return GenericNonExclusivePromise::CreateAndReject( + NS_ERROR_NOT_AVAILABLE, __func__); + } + + mRDDChild = mProcess->GetActor(); + mProcessToken = mProcess->GetProcessToken(); + + // Flush any pref updates that happened during + // launch and weren't included in the blobs set + // up in LaunchRDDProcess. + for (const mozilla::dom::Pref& pref : mQueuedPrefs) { + Unused << NS_WARN_IF(!mRDDChild->SendPreferenceUpdate(pref)); + } + mQueuedPrefs.Clear(); + + CrashReporter::AnnotateCrashReport( + CrashReporter::Annotation::RDDProcessStatus, "Running"_ns); + + if (!CreateVideoBridge()) { + mNumProcessAttempts++; + DestroyProcess(); + return GenericNonExclusivePromise::CreateAndReject( + NS_ERROR_NOT_AVAILABLE, __func__); + } + return GenericNonExclusivePromise::CreateAndResolve(true, __func__); + }, + [this](nsresult aError) { + if (Get()) { + mNumProcessAttempts++; + DestroyProcess(); + } + return GenericNonExclusivePromise::CreateAndReject(aError, __func__); + }); + return mLaunchRDDPromise; +} + +auto RDDProcessManager::EnsureRDDProcessAndCreateBridge( + base::ProcessId aOtherProcess) -> RefPtr<EnsureRDDPromise> { + return InvokeAsync( + GetMainThreadSerialEventTarget(), __func__, + [aOtherProcess, this]() -> RefPtr<EnsureRDDPromise> { + return LaunchRDDProcess()->Then( + GetMainThreadSerialEventTarget(), __func__, + [aOtherProcess, this]() { + if (IsShutdown()) { + return EnsureRDDPromise::CreateAndReject(NS_ERROR_NOT_AVAILABLE, + __func__); + } + ipc::Endpoint<PRemoteDecoderManagerChild> endpoint; + if (!CreateContentBridge(aOtherProcess, &endpoint)) { + return EnsureRDDPromise::CreateAndReject(NS_ERROR_NOT_AVAILABLE, + __func__); + } + mNumProcessAttempts = 0; + return EnsureRDDPromise::CreateAndResolve(std::move(endpoint), + __func__); + }, + [](nsresult aError) { + return EnsureRDDPromise::CreateAndReject(aError, __func__); + }); + }); +} + +bool RDDProcessManager::IsRDDProcessLaunching() { + MOZ_ASSERT(NS_IsMainThread()); + return !!mProcess && !mRDDChild; +} + +bool RDDProcessManager::IsRDDProcessDestroyed() const { + MOZ_ASSERT(NS_IsMainThread()); + return !mRDDChild && !mProcess; +} + +void RDDProcessManager::OnProcessUnexpectedShutdown(RDDProcessHost* aHost) { + MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(mProcess && mProcess == aHost); + + mNumUnexpectedCrashes++; + + DestroyProcess(); +} + +void RDDProcessManager::NotifyRemoteActorDestroyed( + const uint64_t& aProcessToken) { + if (!NS_IsMainThread()) { + RefPtr<Runnable> task = mTaskFactory.NewRunnableMethod( + &RDDProcessManager::NotifyRemoteActorDestroyed, aProcessToken); + NS_DispatchToMainThread(task.forget()); + return; + } + + if (mProcessToken != aProcessToken) { + // This token is for an older process; we can safely ignore it. + return; + } + + // One of the bridged top-level actors for the RDD process has been + // prematurely terminated, and we're receiving a notification. This + // can happen if the ActorDestroy for a bridged protocol fires + // before the ActorDestroy for PRDDChild. + OnProcessUnexpectedShutdown(mProcess); +} + +void RDDProcessManager::CleanShutdown() { DestroyProcess(); } + +void RDDProcessManager::DestroyProcess() { + MOZ_ASSERT(NS_IsMainThread()); + if (!mProcess) { + return; + } + + mProcess->Shutdown(); + mProcessToken = 0; + mProcess = nullptr; + mRDDChild = nullptr; + mQueuedPrefs.Clear(); + + CrashReporter::AnnotateCrashReport( + CrashReporter::Annotation::RDDProcessStatus, "Destroyed"_ns); +} + +bool RDDProcessManager::CreateContentBridge( + base::ProcessId aOtherProcess, + ipc::Endpoint<PRemoteDecoderManagerChild>* aOutRemoteDecoderManager) { + MOZ_ASSERT(NS_IsMainThread()); + + if (IsRDDProcessDestroyed()) { + MOZ_LOG(sPDMLog, LogLevel::Debug, + ("RDD shutdown before creating content bridge")); + return false; + } + + ipc::Endpoint<PRemoteDecoderManagerParent> parentPipe; + ipc::Endpoint<PRemoteDecoderManagerChild> childPipe; + + nsresult rv = PRemoteDecoderManager::CreateEndpoints( + mRDDChild->OtherPid(), aOtherProcess, &parentPipe, &childPipe); + if (NS_FAILED(rv)) { + MOZ_LOG(sPDMLog, LogLevel::Debug, + ("Could not create content remote decoder: %d", int(rv))); + return false; + } + + mRDDChild->SendNewContentRemoteDecoderManager(std::move(parentPipe)); + + *aOutRemoteDecoderManager = std::move(childPipe); + return true; +} + +bool RDDProcessManager::CreateVideoBridge() { + MOZ_ASSERT(NS_IsMainThread()); + ipc::Endpoint<PVideoBridgeParent> parentPipe; + ipc::Endpoint<PVideoBridgeChild> childPipe; + + GPUProcessManager* gpuManager = GPUProcessManager::Get(); + base::ProcessId gpuProcessPid = + gpuManager ? gpuManager->GPUProcessPid() : base::kInvalidProcessId; + + // Build content device data first; this ensure that the GPU process is fully + // ready. + ContentDeviceData contentDeviceData; + gfxPlatform::GetPlatform()->BuildContentDeviceData(&contentDeviceData); + + // The child end is the producer of video frames; the parent end is the + // consumer. + base::ProcessId childPid = RDDProcessPid(); + base::ProcessId parentPid = gpuProcessPid != base::kInvalidProcessId + ? gpuProcessPid + : base::GetCurrentProcId(); + + nsresult rv = PVideoBridge::CreateEndpoints(parentPid, childPid, &parentPipe, + &childPipe); + if (NS_FAILED(rv)) { + MOZ_LOG(sPDMLog, LogLevel::Debug, + ("Could not create video bridge: %d", int(rv))); + return false; + } + + mRDDChild->SendInitVideoBridge(std::move(childPipe), + mNumUnexpectedCrashes == 0, contentDeviceData); + if (gpuProcessPid != base::kInvalidProcessId) { + gpuManager->InitVideoBridge(std::move(parentPipe), + VideoBridgeSource::RddProcess); + } else { + VideoBridgeParent::Open(std::move(parentPipe), + VideoBridgeSource::RddProcess); + } + + return true; +} + +base::ProcessId RDDProcessManager::RDDProcessPid() { + MOZ_ASSERT(NS_IsMainThread()); + base::ProcessId rddPid = + mRDDChild ? mRDDChild->OtherPid() : base::kInvalidProcessId; + return rddPid; +} + +class RDDMemoryReporter : public MemoryReportingProcess { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(RDDMemoryReporter, override) + + bool IsAlive() const override { return !!GetChild(); } + + bool SendRequestMemoryReport( + const uint32_t& aGeneration, const bool& aAnonymize, + const bool& aMinimizeMemoryUsage, + const Maybe<ipc::FileDescriptor>& aDMDFile) override { + RDDChild* child = GetChild(); + if (!child) { + return false; + } + + return child->SendRequestMemoryReport(aGeneration, aAnonymize, + aMinimizeMemoryUsage, aDMDFile); + } + + int32_t Pid() const override { + if (RDDChild* child = GetChild()) { + return (int32_t)child->OtherPid(); + } + return 0; + } + + private: + RDDChild* GetChild() const { + if (RDDProcessManager* rddpm = RDDProcessManager::Get()) { + if (RDDChild* child = rddpm->GetRDDChild()) { + return child; + } + } + return nullptr; + } + + protected: + ~RDDMemoryReporter() = default; +}; + +RefPtr<MemoryReportingProcess> RDDProcessManager::GetProcessMemoryReporter() { + if (!mProcess || !mProcess->IsConnected()) { + return nullptr; + } + return new RDDMemoryReporter(); +} + +RefPtr<PRDDChild::TestTriggerMetricsPromise> +RDDProcessManager::TestTriggerMetrics() { + if (!NS_WARN_IF(!mRDDChild)) { + return mRDDChild->SendTestTriggerMetrics(); + } + + return PRDDChild::TestTriggerMetricsPromise::CreateAndReject( + ipc::ResponseRejectReason::SendError, __func__); +} + +} // namespace mozilla diff --git a/dom/media/ipc/RDDProcessManager.h b/dom/media/ipc/RDDProcessManager.h new file mode 100644 index 0000000000..b762d739a9 --- /dev/null +++ b/dom/media/ipc/RDDProcessManager.h @@ -0,0 +1,127 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef _include_dom_media_ipc_RDDProcessManager_h_ +#define _include_dom_media_ipc_RDDProcessManager_h_ +#include "mozilla/MozPromise.h" +#include "mozilla/PRemoteDecoderManagerChild.h" +#include "mozilla/RDDProcessHost.h" +#include "mozilla/ipc/TaskFactory.h" +#include "mozilla/PRDDChild.h" +#include "nsIObserver.h" + +namespace mozilla { + +class MemoryReportingProcess; +class RDDChild; + +// The RDDProcessManager is a singleton responsible for creating RDD-bound +// objects that may live in another process. Currently, it provides access +// to the RDD process via ContentParent. +class RDDProcessManager final : public RDDProcessHost::Listener { + friend class RDDChild; + + public: + static void Initialize(); + static void Shutdown(); + static RDDProcessManager* Get(); + + ~RDDProcessManager(); + + using EnsureRDDPromise = + MozPromise<ipc::Endpoint<PRemoteDecoderManagerChild>, nsresult, true>; + // Launch a new RDD process asynchronously + RefPtr<GenericNonExclusivePromise> LaunchRDDProcess(); + // If not using a RDD process, launch a new RDD process asynchronously and + // create a RemoteDecoderManager bridge + RefPtr<EnsureRDDPromise> EnsureRDDProcessAndCreateBridge( + base::ProcessId aOtherProcess); + + void OnProcessUnexpectedShutdown(RDDProcessHost* aHost) override; + + // Notify the RDDProcessManager that a top-level PRDD protocol has been + // terminated. This may be called from any thread. + void NotifyRemoteActorDestroyed(const uint64_t& aProcessToken); + + // Returns -1 if there is no RDD process, or the platform pid for it. + base::ProcessId RDDProcessPid(); + + // If a RDD process is present, create a MemoryReportingProcess object. + // Otherwise, return null. + RefPtr<MemoryReportingProcess> GetProcessMemoryReporter(); + + // Returns access to the PRDD protocol if a RDD process is present. + RDDChild* GetRDDChild() { return mRDDChild; } + + // Returns whether or not a RDD process was ever launched. + bool AttemptedRDDProcess() const { return mNumProcessAttempts > 0; } + + // Returns the RDD Process + RDDProcessHost* Process() { return mProcess; } + + /* + * ** Test-only Method ** + * + * Trigger RDD-process test metric instrumentation. + */ + RefPtr<PRDDChild::TestTriggerMetricsPromise> TestTriggerMetrics(); + + private: + bool IsRDDProcessLaunching(); + bool IsRDDProcessDestroyed() const; + bool CreateVideoBridge(); + + // Called from our xpcom-shutdown observer. + void OnXPCOMShutdown(); + void OnPreferenceChange(const char16_t* aData); + + RDDProcessManager(); + + // Shutdown the RDD process. + void CleanShutdown(); + void DestroyProcess(); + + bool IsShutdown() const; + + DISALLOW_COPY_AND_ASSIGN(RDDProcessManager); + + class Observer final : public nsIObserver { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSIOBSERVER + explicit Observer(RDDProcessManager* aManager); + + protected: + ~Observer() = default; + + RDDProcessManager* mManager; + }; + friend class Observer; + + bool CreateContentBridge( + base::ProcessId aOtherProcess, + ipc::Endpoint<PRemoteDecoderManagerChild>* aOutRemoteDecoderManager); + + const RefPtr<Observer> mObserver; + ipc::TaskFactory<RDDProcessManager> mTaskFactory; + uint32_t mNumProcessAttempts = 0; + uint32_t mNumUnexpectedCrashes = 0; + + // Fields that are associated with the current RDD process. + RDDProcessHost* mProcess = nullptr; + uint64_t mProcessToken = 0; + RDDChild* mRDDChild = nullptr; + // Collects any pref changes that occur during process launch (after + // the initial map is passed in command-line arguments) to be sent + // when the process can receive IPC messages. + nsTArray<dom::Pref> mQueuedPrefs; + // Promise will be resolved when the RDD process has been fully started and + // VideoBridge configured. Only accessed on the main thread. + RefPtr<GenericNonExclusivePromise> mLaunchRDDPromise; +}; + +} // namespace mozilla + +#endif // _include_dom_media_ipc_RDDProcessManager_h_ diff --git a/dom/media/ipc/RemoteAudioDecoder.cpp b/dom/media/ipc/RemoteAudioDecoder.cpp new file mode 100644 index 0000000000..09b420261f --- /dev/null +++ b/dom/media/ipc/RemoteAudioDecoder.cpp @@ -0,0 +1,121 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "RemoteAudioDecoder.h" + +#include "MediaDataDecoderProxy.h" +#include "PDMFactory.h" +#include "RemoteDecoderManagerChild.h" +#include "RemoteDecoderManagerParent.h" +#include "mozilla/PodOperations.h" +#include "mozilla/StaticPrefs_media.h" + +namespace mozilla { + +RemoteAudioDecoderChild::RemoteAudioDecoderChild(RemoteDecodeIn aLocation) + : RemoteDecoderChild(aLocation) {} + +MediaResult RemoteAudioDecoderChild::ProcessOutput( + DecodedOutputIPDL&& aDecodedData) { + AssertOnManagerThread(); + + MOZ_ASSERT(aDecodedData.type() == DecodedOutputIPDL::TArrayOfRemoteAudioData); + RefPtr<ArrayOfRemoteAudioData> arrayData = + aDecodedData.get_ArrayOfRemoteAudioData(); + + for (size_t i = 0; i < arrayData->Count(); i++) { + RefPtr<AudioData> data = arrayData->ElementAt(i); + if (!data) { + // OOM + return MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__); + } + mDecodedData.AppendElement(data); + } + return NS_OK; +} + +MediaResult RemoteAudioDecoderChild::InitIPDL( + const AudioInfo& aAudioInfo, const CreateDecoderParams::OptionSet& aOptions, + const Maybe<uint64_t>& aMediaEngineId) { + RefPtr<RemoteDecoderManagerChild> manager = + RemoteDecoderManagerChild::GetSingleton(mLocation); + + // The manager isn't available because RemoteDecoderManagerChild has been + // initialized with null end points and we don't want to decode video on RDD + // process anymore. Return false here so that we can fallback to other PDMs. + if (!manager) { + return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("RemoteDecoderManager is not available.")); + } + + if (!manager->CanSend()) { + return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("RemoteDecoderManager unable to send.")); + } + + mIPDLSelfRef = this; + Unused << manager->SendPRemoteDecoderConstructor( + this, aAudioInfo, aOptions, Nothing(), aMediaEngineId, Nothing()); + return NS_OK; +} + +RemoteAudioDecoderParent::RemoteAudioDecoderParent( + RemoteDecoderManagerParent* aParent, const AudioInfo& aAudioInfo, + const CreateDecoderParams::OptionSet& aOptions, + nsISerialEventTarget* aManagerThread, TaskQueue* aDecodeTaskQueue, + Maybe<uint64_t> aMediaEngineId) + : RemoteDecoderParent(aParent, aOptions, aManagerThread, aDecodeTaskQueue, + aMediaEngineId, Nothing()), + mAudioInfo(aAudioInfo) {} + +IPCResult RemoteAudioDecoderParent::RecvConstruct( + ConstructResolver&& aResolver) { + auto params = CreateDecoderParams{mAudioInfo, mOptions, + CreateDecoderParams::NoWrapper(true), + mMediaEngineId, mTrackingId}; + + mParent->EnsurePDMFactory().CreateDecoder(params)->Then( + GetCurrentSerialEventTarget(), __func__, + [resolver = std::move(aResolver), self = RefPtr{this}]( + PlatformDecoderModule::CreateDecoderPromise::ResolveOrRejectValue&& + aValue) { + if (aValue.IsReject()) { + resolver(aValue.RejectValue()); + return; + } + MOZ_ASSERT(aValue.ResolveValue()); + self->mDecoder = + new MediaDataDecoderProxy(aValue.ResolveValue().forget(), + do_AddRef(self->mDecodeTaskQueue.get())); + resolver(NS_OK); + }); + + return IPC_OK(); +} + +MediaResult RemoteAudioDecoderParent::ProcessDecodedData( + MediaDataDecoder::DecodedData&& aData, DecodedOutputIPDL& aDecodedData) { + MOZ_ASSERT(OnManagerThread()); + + // Converted array to array of RefPtr<AudioData> + nsTArray<RefPtr<AudioData>> data(aData.Length()); + for (auto&& element : aData) { + MOZ_ASSERT(element->mType == MediaData::Type::AUDIO_DATA, + "Can only decode audio using RemoteAudioDecoderParent!"); + AudioData* audio = static_cast<AudioData*>(element.get()); + data.AppendElement(audio); + } + auto array = MakeRefPtr<ArrayOfRemoteAudioData>(); + if (!array->Fill(std::move(data), + [&](size_t aSize) { return AllocateBuffer(aSize); })) { + return MediaResult( + NS_ERROR_OUT_OF_MEMORY, + "Failed in RemoteAudioDecoderParent::ProcessDecodedData"); + } + aDecodedData = std::move(array); + return NS_OK; +} + +} // namespace mozilla diff --git a/dom/media/ipc/RemoteAudioDecoder.h b/dom/media/ipc/RemoteAudioDecoder.h new file mode 100644 index 0000000000..55c4e001d1 --- /dev/null +++ b/dom/media/ipc/RemoteAudioDecoder.h @@ -0,0 +1,53 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef include_dom_media_ipc_RemoteAudioDecoderChild_h +#define include_dom_media_ipc_RemoteAudioDecoderChild_h +#include "RemoteDecoderChild.h" +#include "RemoteDecoderParent.h" + +namespace mozilla { + +using mozilla::ipc::IPCResult; + +class RemoteAudioDecoderChild final : public RemoteDecoderChild { + public: + explicit RemoteAudioDecoderChild(RemoteDecodeIn aLocation); + + MOZ_IS_CLASS_INIT + MediaResult InitIPDL(const AudioInfo& aAudioInfo, + const CreateDecoderParams::OptionSet& aOptions, + const Maybe<uint64_t>& aMediaEngineId); + + MediaResult ProcessOutput(DecodedOutputIPDL&& aDecodedData) override; +}; + +class RemoteAudioDecoderParent final : public RemoteDecoderParent { + public: + RemoteAudioDecoderParent(RemoteDecoderManagerParent* aParent, + const AudioInfo& aAudioInfo, + const CreateDecoderParams::OptionSet& aOptions, + nsISerialEventTarget* aManagerThread, + TaskQueue* aDecodeTaskQueue, + Maybe<uint64_t> aMediaEngineId); + + protected: + IPCResult RecvConstruct(ConstructResolver&& aResolver) override; + MediaResult ProcessDecodedData(MediaDataDecoder::DecodedData&& aData, + DecodedOutputIPDL& aDecodedData) override; + + private: + // Can only be accessed from the manager thread + // Note: we can't keep a reference to the original AudioInfo here + // because unlike in typical MediaDataDecoder situations, we're being + // passed a deserialized AudioInfo from RecvPRemoteDecoderConstructor + // which is destroyed when RecvPRemoteDecoderConstructor returns. + const AudioInfo mAudioInfo; + const CreateDecoderParams::OptionSet mOptions; +}; + +} // namespace mozilla + +#endif // include_dom_media_ipc_RemoteAudioDecoderChild_h diff --git a/dom/media/ipc/RemoteDecodeUtils.cpp b/dom/media/ipc/RemoteDecodeUtils.cpp new file mode 100644 index 0000000000..15036a0370 --- /dev/null +++ b/dom/media/ipc/RemoteDecodeUtils.cpp @@ -0,0 +1,103 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "RemoteDecodeUtils.h" +#include "mozilla/ipc/UtilityProcessChild.h" + +namespace mozilla { + +using SandboxingKind = ipc::SandboxingKind; + +SandboxingKind GetCurrentSandboxingKind() { + MOZ_ASSERT(XRE_IsUtilityProcess()); + return ipc::UtilityProcessChild::GetSingleton()->mSandbox; +} + +SandboxingKind GetSandboxingKindFromLocation(RemoteDecodeIn aLocation) { + switch (aLocation) { + case RemoteDecodeIn::UtilityProcess_Generic: + return SandboxingKind::GENERIC_UTILITY; +#ifdef MOZ_APPLEMEDIA + case RemoteDecodeIn::UtilityProcess_AppleMedia: + return SandboxingKind::UTILITY_AUDIO_DECODING_APPLE_MEDIA; + break; +#endif +#ifdef XP_WIN + case RemoteDecodeIn::UtilityProcess_WMF: + return SandboxingKind::UTILITY_AUDIO_DECODING_WMF; +#endif +#ifdef MOZ_WMF_MEDIA_ENGINE + case RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM: + return SandboxingKind::MF_MEDIA_ENGINE_CDM; +#endif + default: + MOZ_ASSERT_UNREACHABLE("Unsupported RemoteDecodeIn"); + return SandboxingKind::COUNT; + } +} + +RemoteDecodeIn GetRemoteDecodeInFromKind(SandboxingKind aKind) { + switch (aKind) { + case SandboxingKind::GENERIC_UTILITY: + return RemoteDecodeIn::UtilityProcess_Generic; +#ifdef MOZ_APPLEMEDIA + case SandboxingKind::UTILITY_AUDIO_DECODING_APPLE_MEDIA: + return RemoteDecodeIn::UtilityProcess_AppleMedia; +#endif +#ifdef XP_WIN + case SandboxingKind::UTILITY_AUDIO_DECODING_WMF: + return RemoteDecodeIn::UtilityProcess_WMF; +#endif +#ifdef MOZ_WMF_MEDIA_ENGINE + case SandboxingKind::MF_MEDIA_ENGINE_CDM: + return RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM; +#endif + default: + MOZ_ASSERT_UNREACHABLE("Unsupported SandboxingKind"); + return RemoteDecodeIn::Unspecified; + } +} + +RemoteDecodeIn GetRemoteDecodeInFromVideoBridgeSource( + layers::VideoBridgeSource aSource) { + switch (aSource) { + case layers::VideoBridgeSource::RddProcess: + return RemoteDecodeIn::RddProcess; + case layers::VideoBridgeSource::GpuProcess: + return RemoteDecodeIn::GpuProcess; + case layers::VideoBridgeSource::MFMediaEngineCDMProcess: + return RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM; + default: + MOZ_ASSERT_UNREACHABLE("Unsupported VideoBridgeSource"); + return RemoteDecodeIn::Unspecified; + } +} + +const char* RemoteDecodeInToStr(RemoteDecodeIn aLocation) { + switch (aLocation) { + case RemoteDecodeIn::RddProcess: + return "RDD"; + case RemoteDecodeIn::GpuProcess: + return "GPU"; + case RemoteDecodeIn::UtilityProcess_Generic: + return "Utility Generic"; +#ifdef MOZ_APPLEMEDIA + case RemoteDecodeIn::UtilityProcess_AppleMedia: + return "Utility AppleMedia"; +#endif +#ifdef XP_WIN + case RemoteDecodeIn::UtilityProcess_WMF: + return "Utility WMF"; +#endif +#ifdef MOZ_WMF_MEDIA_ENGINE + case RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM: + return "Utility MF Media Engine CDM"; +#endif + default: + MOZ_ASSERT_UNREACHABLE("Unsupported RemoteDecodeIn"); + return "Unknown"; + } +} + +} // namespace mozilla diff --git a/dom/media/ipc/RemoteDecodeUtils.h b/dom/media/ipc/RemoteDecodeUtils.h new file mode 100644 index 0000000000..212250c089 --- /dev/null +++ b/dom/media/ipc/RemoteDecodeUtils.h @@ -0,0 +1,31 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_IPC_REMOTEDECODEUTILS_H_ +#define DOM_MEDIA_IPC_REMOTEDECODEUTILS_H_ + +#include "mozilla/Logging.h" +#include "mozilla/RemoteDecoderManagerChild.h" +#include "mozilla/ipc/UtilityProcessSandboxing.h" + +namespace mozilla { + +inline LazyLogModule gRemoteDecodeLog{"RemoteDecode"}; + +// Return the sandboxing kind of the current utility process. Should only be +// called on the utility process. +ipc::SandboxingKind GetCurrentSandboxingKind(); + +ipc::SandboxingKind GetSandboxingKindFromLocation(RemoteDecodeIn aLocation); + +RemoteDecodeIn GetRemoteDecodeInFromKind(ipc::SandboxingKind aKind); + +RemoteDecodeIn GetRemoteDecodeInFromVideoBridgeSource( + layers::VideoBridgeSource aSource); + +const char* RemoteDecodeInToStr(RemoteDecodeIn aLocation); + +} // namespace mozilla + +#endif // DOM_MEDIA_IPC_REMOTEDECODEUTILS_H_ diff --git a/dom/media/ipc/RemoteDecoderChild.cpp b/dom/media/ipc/RemoteDecoderChild.cpp new file mode 100644 index 0000000000..d20856f667 --- /dev/null +++ b/dom/media/ipc/RemoteDecoderChild.cpp @@ -0,0 +1,297 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "RemoteDecoderChild.h" + +#include "RemoteDecoderManagerChild.h" + +#include "mozilla/RemoteDecodeUtils.h" + +namespace mozilla { + +RemoteDecoderChild::RemoteDecoderChild(RemoteDecodeIn aLocation) + : ShmemRecycleAllocator(this), + mLocation(aLocation), + mThread(GetCurrentSerialEventTarget()) { + MOZ_DIAGNOSTIC_ASSERT( + RemoteDecoderManagerChild::GetManagerThread() && + RemoteDecoderManagerChild::GetManagerThread()->IsOnCurrentThread(), + "Must be created on the manager thread"); +} + +RemoteDecoderChild::~RemoteDecoderChild() = default; + +void RemoteDecoderChild::HandleRejectionError( + const ipc::ResponseRejectReason& aReason, + std::function<void(const MediaResult&)>&& aCallback) { + // If the channel goes down and CanSend() returns false, the IPDL promise will + // be rejected with SendError rather than ActorDestroyed. Both means the same + // thing and we can consider that the parent has crashed. The child can no + // longer be used. + // + + // The GPU/RDD process crashed. + if (mLocation == RemoteDecodeIn::GpuProcess) { + // The GPU process will get automatically restarted by the parent process. + // Once it has been restarted the ContentChild will receive the message and + // will call GetManager()->InitForGPUProcess. + // We defer reporting an error until we've recreated the RemoteDecoder + // manager so that it'll be safe for MediaFormatReader to recreate decoders + RefPtr<RemoteDecoderChild> self = this; + GetManager()->RunWhenGPUProcessRecreated(NS_NewRunnableFunction( + "RemoteDecoderChild::HandleRejectionError", + [self, callback = std::move(aCallback)]() { + MediaResult error( + NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR, + __func__); + callback(error); + })); + return; + } + + nsresult err = ((mLocation == RemoteDecodeIn::GpuProcess) || + (mLocation == RemoteDecodeIn::RddProcess)) + ? NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR + : NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_UTILITY_ERR; + // The RDD process is restarted on demand and asynchronously, we can + // immediately inform the caller that a new decoder is needed. The RDD will + // then be restarted during the new decoder creation by + aCallback(MediaResult(err, __func__)); +} + +// ActorDestroy is called if the channel goes down while waiting for a response. +void RemoteDecoderChild::ActorDestroy(ActorDestroyReason aWhy) { + mRemoteDecoderCrashed = (aWhy == AbnormalShutdown); + mDecodedData.Clear(); + CleanupShmemRecycleAllocator(); + RecordShutdownTelemetry(mRemoteDecoderCrashed); +} + +void RemoteDecoderChild::DestroyIPDL() { + AssertOnManagerThread(); + MOZ_DIAGNOSTIC_ASSERT(mInitPromise.IsEmpty() && mDecodePromise.IsEmpty() && + mDrainPromise.IsEmpty() && + mFlushPromise.IsEmpty() && + mShutdownPromise.IsEmpty(), + "All promises should have been rejected"); + if (CanSend()) { + PRemoteDecoderChild::Send__delete__(this); + } +} + +void RemoteDecoderChild::IPDLActorDestroyed() { mIPDLSelfRef = nullptr; } + +// MediaDataDecoder methods + +RefPtr<MediaDataDecoder::InitPromise> RemoteDecoderChild::Init() { + AssertOnManagerThread(); + + mRemoteDecoderCrashed = false; + + RefPtr<RemoteDecoderChild> self = this; + SendInit() + ->Then( + mThread, __func__, + [self, this](InitResultIPDL&& aResponse) { + mInitPromiseRequest.Complete(); + if (aResponse.type() == InitResultIPDL::TMediaResult) { + mInitPromise.Reject(aResponse.get_MediaResult(), __func__); + return; + } + const auto& initResponse = aResponse.get_InitCompletionIPDL(); + mDescription = initResponse.decoderDescription(); + mDescription.Append(" ("); + mDescription.Append(RemoteDecodeInToStr(GetManager()->Location())); + mDescription.Append(" remote)"); + + mIsHardwareAccelerated = initResponse.hardware(); + mHardwareAcceleratedReason = initResponse.hardwareReason(); + mConversion = initResponse.conversion(); + // Either the promise has not yet been resolved or the handler has + // been disconnected and we can't get here. + mInitPromise.Resolve(initResponse.type(), __func__); + }, + [self](const mozilla::ipc::ResponseRejectReason& aReason) { + self->mInitPromiseRequest.Complete(); + self->HandleRejectionError( + aReason, [self](const MediaResult& aError) { + self->mInitPromise.RejectIfExists(aError, __func__); + }); + }) + ->Track(mInitPromiseRequest); + + return mInitPromise.Ensure(__func__); +} + +RefPtr<MediaDataDecoder::DecodePromise> RemoteDecoderChild::Decode( + const nsTArray<RefPtr<MediaRawData>>& aSamples) { + AssertOnManagerThread(); + + if (mRemoteDecoderCrashed) { + nsresult err = + ((mLocation == RemoteDecodeIn::GpuProcess) || + (mLocation == RemoteDecodeIn::RddProcess)) + ? NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR + : NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_UTILITY_ERR; + return MediaDataDecoder::DecodePromise::CreateAndReject(err, __func__); + } + + auto samples = MakeRefPtr<ArrayOfRemoteMediaRawData>(); + if (!samples->Fill(aSamples, + [&](size_t aSize) { return AllocateBuffer(aSize); })) { + return MediaDataDecoder::DecodePromise::CreateAndReject( + NS_ERROR_OUT_OF_MEMORY, __func__); + } + SendDecode(samples)->Then( + mThread, __func__, + [self = RefPtr{this}, this]( + PRemoteDecoderChild::DecodePromise::ResolveOrRejectValue&& aValue) { + // We no longer need the samples as the data has been + // processed by the parent. + // If the parent died, the error being fatal will cause the + // decoder to be torn down and all shmem in the pool will be + // deallocated. + ReleaseAllBuffers(); + + if (aValue.IsReject()) { + HandleRejectionError( + aValue.RejectValue(), [self](const MediaResult& aError) { + self->mDecodePromise.RejectIfExists(aError, __func__); + }); + return; + } + MOZ_DIAGNOSTIC_ASSERT(CanSend(), + "The parent unexpectedly died, promise should " + "have been rejected first"); + if (mDecodePromise.IsEmpty()) { + // We got flushed. + return; + } + auto response = std::move(aValue.ResolveValue()); + if (response.type() == DecodeResultIPDL::TMediaResult && + NS_FAILED(response.get_MediaResult())) { + mDecodePromise.Reject(response.get_MediaResult(), __func__); + return; + } + if (response.type() == DecodeResultIPDL::TDecodedOutputIPDL) { + ProcessOutput(std::move(response.get_DecodedOutputIPDL())); + } + mDecodePromise.Resolve(std::move(mDecodedData), __func__); + mDecodedData = MediaDataDecoder::DecodedData(); + }); + + return mDecodePromise.Ensure(__func__); +} + +RefPtr<MediaDataDecoder::FlushPromise> RemoteDecoderChild::Flush() { + AssertOnManagerThread(); + mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + mDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + + RefPtr<RemoteDecoderChild> self = this; + SendFlush()->Then( + mThread, __func__, + [self](const MediaResult& aResult) { + if (NS_SUCCEEDED(aResult)) { + self->mFlushPromise.ResolveIfExists(true, __func__); + } else { + self->mFlushPromise.RejectIfExists(aResult, __func__); + } + }, + [self](const mozilla::ipc::ResponseRejectReason& aReason) { + self->HandleRejectionError(aReason, [self](const MediaResult& aError) { + self->mFlushPromise.RejectIfExists(aError, __func__); + }); + }); + return mFlushPromise.Ensure(__func__); +} + +RefPtr<MediaDataDecoder::DecodePromise> RemoteDecoderChild::Drain() { + AssertOnManagerThread(); + + RefPtr<RemoteDecoderChild> self = this; + SendDrain()->Then( + mThread, __func__, + [self, this](DecodeResultIPDL&& aResponse) { + if (mDrainPromise.IsEmpty()) { + // We got flushed. + return; + } + if (aResponse.type() == DecodeResultIPDL::TMediaResult && + NS_FAILED(aResponse.get_MediaResult())) { + mDrainPromise.Reject(aResponse.get_MediaResult(), __func__); + return; + } + MOZ_DIAGNOSTIC_ASSERT(CanSend(), + "The parent unexpectedly died, promise should " + "have been rejected first"); + if (aResponse.type() == DecodeResultIPDL::TDecodedOutputIPDL) { + ProcessOutput(std::move(aResponse.get_DecodedOutputIPDL())); + } + mDrainPromise.Resolve(std::move(mDecodedData), __func__); + mDecodedData = MediaDataDecoder::DecodedData(); + }, + [self](const mozilla::ipc::ResponseRejectReason& aReason) { + self->HandleRejectionError(aReason, [self](const MediaResult& aError) { + self->mDrainPromise.RejectIfExists(aError, __func__); + }); + }); + return mDrainPromise.Ensure(__func__); +} + +RefPtr<mozilla::ShutdownPromise> RemoteDecoderChild::Shutdown() { + AssertOnManagerThread(); + // Shutdown() can be called while an InitPromise is pending. + mInitPromiseRequest.DisconnectIfExists(); + mInitPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + mDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + mFlushPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + + RefPtr<RemoteDecoderChild> self = this; + SendShutdown()->Then( + mThread, __func__, + [self]( + PRemoteDecoderChild::ShutdownPromise::ResolveOrRejectValue&& aValue) { + self->mShutdownPromise.Resolve(aValue.IsResolve(), __func__); + }); + return mShutdownPromise.Ensure(__func__); +} + +bool RemoteDecoderChild::IsHardwareAccelerated( + nsACString& aFailureReason) const { + AssertOnManagerThread(); + aFailureReason = mHardwareAcceleratedReason; + return mIsHardwareAccelerated; +} + +nsCString RemoteDecoderChild::GetDescriptionName() const { + AssertOnManagerThread(); + return mDescription; +} + +void RemoteDecoderChild::SetSeekThreshold(const media::TimeUnit& aTime) { + AssertOnManagerThread(); + Unused << SendSetSeekThreshold(aTime); +} + +MediaDataDecoder::ConversionRequired RemoteDecoderChild::NeedsConversion() + const { + AssertOnManagerThread(); + return mConversion; +} + +void RemoteDecoderChild::AssertOnManagerThread() const { + MOZ_ASSERT(mThread->IsOnCurrentThread()); +} + +RemoteDecoderManagerChild* RemoteDecoderChild::GetManager() { + if (!CanSend()) { + return nullptr; + } + return static_cast<RemoteDecoderManagerChild*>(Manager()); +} + +} // namespace mozilla diff --git a/dom/media/ipc/RemoteDecoderChild.h b/dom/media/ipc/RemoteDecoderChild.h new file mode 100644 index 0000000000..b300f6226f --- /dev/null +++ b/dom/media/ipc/RemoteDecoderChild.h @@ -0,0 +1,86 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef include_dom_media_ipc_RemoteDecoderChild_h +#define include_dom_media_ipc_RemoteDecoderChild_h + +#include <functional> + +#include "mozilla/PRemoteDecoderChild.h" +#include "mozilla/RemoteDecoderManagerChild.h" +#include "mozilla/ShmemRecycleAllocator.h" + +namespace mozilla { + +class RemoteDecoderManagerChild; +using mozilla::MediaDataDecoder; +using mozilla::ipc::IPCResult; + +class RemoteDecoderChild : public ShmemRecycleAllocator<RemoteDecoderChild>, + public PRemoteDecoderChild { + friend class PRemoteDecoderChild; + + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(RemoteDecoderChild); + + explicit RemoteDecoderChild(RemoteDecodeIn aLocation); + + void ActorDestroy(ActorDestroyReason aWhy) override; + + // This interface closely mirrors the MediaDataDecoder plus a bit + // (DestroyIPDL) to allow proxying to a remote decoder in RemoteDecoderModule. + RefPtr<MediaDataDecoder::InitPromise> Init(); + RefPtr<MediaDataDecoder::DecodePromise> Decode( + const nsTArray<RefPtr<MediaRawData>>& aSamples); + RefPtr<MediaDataDecoder::DecodePromise> Drain(); + RefPtr<MediaDataDecoder::FlushPromise> Flush(); + RefPtr<mozilla::ShutdownPromise> Shutdown(); + bool IsHardwareAccelerated(nsACString& aFailureReason) const; + nsCString GetDescriptionName() const; + void SetSeekThreshold(const media::TimeUnit& aTime); + MediaDataDecoder::ConversionRequired NeedsConversion() const; + void DestroyIPDL(); + + // Called from IPDL when our actor has been destroyed + void IPDLActorDestroyed(); + + RemoteDecoderManagerChild* GetManager(); + + protected: + virtual ~RemoteDecoderChild(); + void AssertOnManagerThread() const; + + virtual MediaResult ProcessOutput(DecodedOutputIPDL&& aDecodedData) = 0; + virtual void RecordShutdownTelemetry(bool aForAbnormalShutdown) {} + + RefPtr<RemoteDecoderChild> mIPDLSelfRef; + MediaDataDecoder::DecodedData mDecodedData; + const RemoteDecodeIn mLocation; + + private: + const nsCOMPtr<nsISerialEventTarget> mThread; + + MozPromiseHolder<MediaDataDecoder::InitPromise> mInitPromise; + MozPromiseRequestHolder<PRemoteDecoderChild::InitPromise> mInitPromiseRequest; + MozPromiseHolder<MediaDataDecoder::DecodePromise> mDecodePromise; + MozPromiseHolder<MediaDataDecoder::DecodePromise> mDrainPromise; + MozPromiseHolder<MediaDataDecoder::FlushPromise> mFlushPromise; + MozPromiseHolder<mozilla::ShutdownPromise> mShutdownPromise; + + void HandleRejectionError( + const ipc::ResponseRejectReason& aReason, + std::function<void(const MediaResult&)>&& aCallback); + + nsCString mHardwareAcceleratedReason; + nsCString mDescription; + bool mIsHardwareAccelerated = false; + bool mRemoteDecoderCrashed = false; + MediaDataDecoder::ConversionRequired mConversion = + MediaDataDecoder::ConversionRequired::kNeedNone; +}; + +} // namespace mozilla + +#endif // include_dom_media_ipc_RemoteDecoderChild_h diff --git a/dom/media/ipc/RemoteDecoderManagerChild.cpp b/dom/media/ipc/RemoteDecoderManagerChild.cpp new file mode 100644 index 0000000000..83027791c5 --- /dev/null +++ b/dom/media/ipc/RemoteDecoderManagerChild.cpp @@ -0,0 +1,864 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "RemoteDecoderManagerChild.h" + +#include "PDMFactory.h" +#include "RemoteAudioDecoder.h" +#include "RemoteMediaDataDecoder.h" +#include "RemoteVideoDecoder.h" +#include "VideoUtils.h" +#include "mozilla/DataMutex.h" +#include "mozilla/RemoteDecodeUtils.h" +#include "mozilla/SyncRunnable.h" +#include "mozilla/dom/ContentChild.h" // for launching RDD w/ ContentChild +#include "mozilla/gfx/2D.h" +#include "mozilla/gfx/DataSurfaceHelpers.h" +#include "mozilla/ipc/BackgroundChild.h" +#include "mozilla/ipc/PBackgroundChild.h" +#include "mozilla/ipc/Endpoint.h" +#include "mozilla/layers/ISurfaceAllocator.h" +#include "mozilla/ipc/UtilityAudioDecoderChild.h" +#include "nsContentUtils.h" +#include "nsIObserver.h" +#include "mozilla/StaticPrefs_media.h" + +#ifdef MOZ_WMF_MEDIA_ENGINE +# include "MFMediaEngineChild.h" +#endif + +namespace mozilla { + +#define LOG(msg, ...) \ + MOZ_LOG(gRemoteDecodeLog, LogLevel::Debug, (msg, ##__VA_ARGS__)) + +using namespace layers; +using namespace gfx; + +// Used so that we only ever attempt to check if the RDD/GPU/Utility processes +// should be launched serially. Protects sLaunchPromise +StaticMutex sLaunchMutex; +static EnumeratedArray<RemoteDecodeIn, RemoteDecodeIn::SENTINEL, + StaticRefPtr<GenericNonExclusivePromise>> + sLaunchPromises MOZ_GUARDED_BY(sLaunchMutex); + +// Only modified on the main-thread, read on any thread. While it could be read +// on the main thread directly, for clarity we force access via the DataMutex +// wrapper. +static StaticDataMutex<StaticRefPtr<nsIThread>> + sRemoteDecoderManagerChildThread("sRemoteDecoderManagerChildThread"); + +// Only accessed from sRemoteDecoderManagerChildThread +static EnumeratedArray<RemoteDecodeIn, RemoteDecodeIn::SENTINEL, + StaticRefPtr<RemoteDecoderManagerChild>> + sRemoteDecoderManagerChildForProcesses; + +static UniquePtr<nsTArray<RefPtr<Runnable>>> sRecreateTasks; + +// Used for protecting codec support information collected from different remote +// processes. +StaticMutex sProcessSupportedMutex; +static EnumeratedArray<RemoteDecodeIn, RemoteDecodeIn::SENTINEL, + Maybe<media::MediaCodecsSupported>> + sProcessSupported MOZ_GUARDED_BY(sProcessSupportedMutex); + +class ShutdownObserver final : public nsIObserver { + public: + NS_DECL_ISUPPORTS + NS_DECL_NSIOBSERVER + + protected: + ~ShutdownObserver() = default; +}; +NS_IMPL_ISUPPORTS(ShutdownObserver, nsIObserver); + +NS_IMETHODIMP +ShutdownObserver::Observe(nsISupports* aSubject, const char* aTopic, + const char16_t* aData) { + MOZ_ASSERT(!strcmp(aTopic, NS_XPCOM_SHUTDOWN_OBSERVER_ID)); + RemoteDecoderManagerChild::Shutdown(); + return NS_OK; +} + +StaticRefPtr<ShutdownObserver> sObserver; + +/* static */ +void RemoteDecoderManagerChild::Init() { + MOZ_ASSERT(NS_IsMainThread()); + + auto remoteDecoderManagerThread = sRemoteDecoderManagerChildThread.Lock(); + if (!*remoteDecoderManagerThread) { + // We can't use a MediaThreadType::SUPERVISOR as the RemoteDecoderModule + // runs on it and dispatch synchronous tasks to the manager thread, should + // more than 4 concurrent videos being instantiated at the same time, we + // could end up in a deadlock. + RefPtr<nsIThread> childThread; + nsresult rv = NS_NewNamedThread( + "RemVidChild", getter_AddRefs(childThread), + NS_NewRunnableFunction( + "RemoteDecoderManagerChild::InitPBackground", []() { + ipc::PBackgroundChild* bgActor = + ipc::BackgroundChild::GetOrCreateForCurrentThread(); + NS_WARNING_ASSERTION(bgActor, + "Failed to start Background channel"); + Unused << bgActor; + })); + + NS_ENSURE_SUCCESS_VOID(rv); + *remoteDecoderManagerThread = childThread; + sRecreateTasks = MakeUnique<nsTArray<RefPtr<Runnable>>>(); + sObserver = new ShutdownObserver(); + nsContentUtils::RegisterShutdownObserver(sObserver); + } +} + +/* static */ +void RemoteDecoderManagerChild::InitForGPUProcess( + Endpoint<PRemoteDecoderManagerChild>&& aVideoManager) { + MOZ_ASSERT(NS_IsMainThread()); + + Init(); + + auto remoteDecoderManagerThread = sRemoteDecoderManagerChildThread.Lock(); + MOZ_ALWAYS_SUCCEEDS( + (*remoteDecoderManagerThread) + ->Dispatch(NewRunnableFunction( + "InitForContentRunnable", + &OpenRemoteDecoderManagerChildForProcess, + std::move(aVideoManager), RemoteDecodeIn::GpuProcess))); +} + +/* static */ +void RemoteDecoderManagerChild::Shutdown() { + MOZ_ASSERT(NS_IsMainThread()); + + if (sObserver) { + nsContentUtils::UnregisterShutdownObserver(sObserver); + sObserver = nullptr; + } + + nsCOMPtr<nsIThread> childThread; + { + auto remoteDecoderManagerThread = sRemoteDecoderManagerChildThread.Lock(); + childThread = remoteDecoderManagerThread->forget(); + } + if (childThread) { + MOZ_ALWAYS_SUCCEEDS(childThread->Dispatch(NS_NewRunnableFunction( + "dom::RemoteDecoderManagerChild::Shutdown", []() { + for (auto& p : sRemoteDecoderManagerChildForProcesses) { + if (p && p->CanSend()) { + p->Close(); + } + p = nullptr; + } + { + StaticMutexAutoLock lock(sLaunchMutex); + for (auto& p : sLaunchPromises) { + p = nullptr; + } + } + ipc::BackgroundChild::CloseForCurrentThread(); + }))); + childThread->Shutdown(); + sRecreateTasks = nullptr; + } +} + +void RemoteDecoderManagerChild::RunWhenGPUProcessRecreated( + already_AddRefed<Runnable> aTask) { + nsCOMPtr<nsISerialEventTarget> managerThread = GetManagerThread(); + if (!managerThread) { + // We've been shutdown, bail. + return; + } + MOZ_ASSERT(managerThread->IsOnCurrentThread()); + + // If we've already been recreated, then run the task immediately. + auto* manager = GetSingleton(RemoteDecodeIn::GpuProcess); + if (manager && manager != this && manager->CanSend()) { + RefPtr<Runnable> task = aTask; + task->Run(); + } else { + sRecreateTasks->AppendElement(aTask); + } +} + +/* static */ +RemoteDecoderManagerChild* RemoteDecoderManagerChild::GetSingleton( + RemoteDecodeIn aLocation) { + nsCOMPtr<nsISerialEventTarget> managerThread = GetManagerThread(); + if (!managerThread) { + // We've been shutdown, bail. + return nullptr; + } + MOZ_ASSERT(managerThread->IsOnCurrentThread()); + switch (aLocation) { + case RemoteDecodeIn::GpuProcess: + case RemoteDecodeIn::RddProcess: + case RemoteDecodeIn::UtilityProcess_Generic: + case RemoteDecodeIn::UtilityProcess_AppleMedia: + case RemoteDecodeIn::UtilityProcess_WMF: + case RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM: + return sRemoteDecoderManagerChildForProcesses[aLocation]; + default: + MOZ_CRASH("Unexpected RemoteDecode variant"); + return nullptr; + } +} + +/* static */ +nsISerialEventTarget* RemoteDecoderManagerChild::GetManagerThread() { + auto remoteDecoderManagerThread = sRemoteDecoderManagerChildThread.Lock(); + return *remoteDecoderManagerThread; +} + +/* static */ +bool RemoteDecoderManagerChild::Supports( + RemoteDecodeIn aLocation, const SupportDecoderParams& aParams, + DecoderDoctorDiagnostics* aDiagnostics) { + Maybe<media::MediaCodecsSupported> supported; + switch (aLocation) { + case RemoteDecodeIn::GpuProcess: + case RemoteDecodeIn::RddProcess: + case RemoteDecodeIn::UtilityProcess_AppleMedia: + case RemoteDecodeIn::UtilityProcess_Generic: + case RemoteDecodeIn::UtilityProcess_WMF: + case RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM: { + StaticMutexAutoLock lock(sProcessSupportedMutex); + supported = sProcessSupported[aLocation]; + break; + } + default: + return false; + } + if (!supported) { + // We haven't received the correct information yet from either the GPU or + // the RDD process nor the Utility process. + if (aLocation == RemoteDecodeIn::UtilityProcess_Generic || + aLocation == RemoteDecodeIn::UtilityProcess_AppleMedia || + aLocation == RemoteDecodeIn::UtilityProcess_WMF || + aLocation == RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM) { + LaunchUtilityProcessIfNeeded(aLocation); + } + if (aLocation == RemoteDecodeIn::RddProcess) { + // Ensure the RDD process got started. + // TODO: This can be removed once bug 1684991 is fixed. + LaunchRDDProcessIfNeeded(); + } + + // Assume the format is supported to prevent false negative, if the remote + // process supports that specific track type. + const bool isVideo = aParams.mConfig.IsVideo(); + const bool isAudio = aParams.mConfig.IsAudio(); + const auto trackSupport = GetTrackSupport(aLocation); + if (isVideo) { + return trackSupport.contains(TrackSupport::Video); + } + if (isAudio) { + return trackSupport.contains(TrackSupport::Audio); + } + MOZ_ASSERT_UNREACHABLE("Not audio and video?!"); + return false; + } + + // We can ignore the SupportDecoderParams argument for now as creation of the + // decoder will actually fail later and fallback PDMs will be tested on later. + return PDMFactory::SupportsMimeType(aParams.MimeType(), *supported, + aLocation) != + media::DecodeSupport::Unsupported; +} + +/* static */ +RefPtr<PlatformDecoderModule::CreateDecoderPromise> +RemoteDecoderManagerChild::CreateAudioDecoder( + const CreateDecoderParams& aParams, RemoteDecodeIn aLocation) { + nsCOMPtr<nsISerialEventTarget> managerThread = GetManagerThread(); + if (!managerThread) { + // We got shutdown. + return PlatformDecoderModule::CreateDecoderPromise::CreateAndReject( + NS_ERROR_DOM_MEDIA_CANCELED, __func__); + } + + if (!GetTrackSupport(aLocation).contains(TrackSupport::Audio)) { + return PlatformDecoderModule::CreateDecoderPromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_CANCELED, + nsPrintfCString("%s doesn't support audio decoding", + RemoteDecodeInToStr(aLocation)) + .get()), + __func__); + } + + RefPtr<GenericNonExclusivePromise> launchPromise; + if (StaticPrefs::media_utility_process_enabled() && + (aLocation == RemoteDecodeIn::UtilityProcess_Generic || + aLocation == RemoteDecodeIn::UtilityProcess_AppleMedia || + aLocation == RemoteDecodeIn::UtilityProcess_WMF)) { + launchPromise = LaunchUtilityProcessIfNeeded(aLocation); + } else if (aLocation == RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM) { + launchPromise = LaunchUtilityProcessIfNeeded(aLocation); + } else { + launchPromise = LaunchRDDProcessIfNeeded(); + } + LOG("Create audio decoder in %s", RemoteDecodeInToStr(aLocation)); + + return launchPromise->Then( + managerThread, __func__, + [params = CreateDecoderParamsForAsync(aParams), aLocation](bool) { + auto child = MakeRefPtr<RemoteAudioDecoderChild>(aLocation); + MediaResult result = child->InitIPDL( + params.AudioConfig(), params.mOptions, params.mMediaEngineId); + if (NS_FAILED(result)) { + return PlatformDecoderModule::CreateDecoderPromise::CreateAndReject( + result, __func__); + } + return Construct(std::move(child), aLocation); + }, + [aLocation](nsresult aResult) { + return PlatformDecoderModule::CreateDecoderPromise::CreateAndReject( + MediaResult(aResult, + aLocation == RemoteDecodeIn::GpuProcess + ? "Couldn't start GPU process" + : (aLocation == RemoteDecodeIn::RddProcess + ? "Couldn't start RDD process" + : "Couldn't start Utility process")), + __func__); + }); +} + +/* static */ +RefPtr<PlatformDecoderModule::CreateDecoderPromise> +RemoteDecoderManagerChild::CreateVideoDecoder( + const CreateDecoderParams& aParams, RemoteDecodeIn aLocation) { + nsCOMPtr<nsISerialEventTarget> managerThread = GetManagerThread(); + if (!managerThread) { + // We got shutdown. + return PlatformDecoderModule::CreateDecoderPromise::CreateAndReject( + NS_ERROR_DOM_MEDIA_CANCELED, __func__); + } + + if (!aParams.mKnowsCompositor && aLocation == RemoteDecodeIn::GpuProcess) { + // We don't have an image bridge; don't attempt to decode in the GPU + // process. + return PlatformDecoderModule::CreateDecoderPromise::CreateAndReject( + NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR, __func__); + } + + if (!GetTrackSupport(aLocation).contains(TrackSupport::Video)) { + return PlatformDecoderModule::CreateDecoderPromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_CANCELED, + nsPrintfCString("%s doesn't support video decoding", + RemoteDecodeInToStr(aLocation)) + .get()), + __func__); + } + + MOZ_ASSERT(aLocation != RemoteDecodeIn::Unspecified); + + RefPtr<GenericNonExclusivePromise> p; + if (aLocation == RemoteDecodeIn::GpuProcess) { + p = GenericNonExclusivePromise::CreateAndResolve(true, __func__); + } else if (aLocation == RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM) { + p = LaunchUtilityProcessIfNeeded(aLocation); + } else { + p = LaunchRDDProcessIfNeeded(); + } + LOG("Create video decoder in %s", RemoteDecodeInToStr(aLocation)); + + return p->Then( + managerThread, __func__, + [aLocation, params = CreateDecoderParamsForAsync(aParams)](bool) { + auto child = MakeRefPtr<RemoteVideoDecoderChild>(aLocation); + MediaResult result = child->InitIPDL( + params.VideoConfig(), params.mRate.mValue, params.mOptions, + params.mKnowsCompositor + ? Some(params.mKnowsCompositor->GetTextureFactoryIdentifier()) + : Nothing(), + params.mMediaEngineId, params.mTrackingId); + if (NS_FAILED(result)) { + return PlatformDecoderModule::CreateDecoderPromise::CreateAndReject( + result, __func__); + } + return Construct(std::move(child), aLocation); + }, + [](nsresult aResult) { + return PlatformDecoderModule::CreateDecoderPromise::CreateAndReject( + MediaResult(aResult, "Couldn't start RDD process"), __func__); + }); +} + +/* static */ +RefPtr<PlatformDecoderModule::CreateDecoderPromise> +RemoteDecoderManagerChild::Construct(RefPtr<RemoteDecoderChild>&& aChild, + RemoteDecodeIn aLocation) { + nsCOMPtr<nsISerialEventTarget> managerThread = GetManagerThread(); + if (!managerThread) { + // We got shutdown. + return PlatformDecoderModule::CreateDecoderPromise::CreateAndReject( + NS_ERROR_DOM_MEDIA_CANCELED, __func__); + } + MOZ_ASSERT(managerThread->IsOnCurrentThread()); + + RefPtr<PlatformDecoderModule::CreateDecoderPromise> p = + aChild->SendConstruct()->Then( + managerThread, __func__, + [child = std::move(aChild)](MediaResult aResult) { + if (NS_FAILED(aResult)) { + // We will never get to use this remote decoder, tear it down. + child->DestroyIPDL(); + return PlatformDecoderModule::CreateDecoderPromise:: + CreateAndReject(aResult, __func__); + } + return PlatformDecoderModule::CreateDecoderPromise:: + CreateAndResolve(MakeRefPtr<RemoteMediaDataDecoder>(child), + __func__); + }, + [aLocation](const mozilla::ipc::ResponseRejectReason& aReason) { + // The parent has died. + nsresult err = + ((aLocation == RemoteDecodeIn::GpuProcess) || + (aLocation == RemoteDecodeIn::RddProcess)) + ? NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_RDD_OR_GPU_ERR + : NS_ERROR_DOM_MEDIA_REMOTE_DECODER_CRASHED_UTILITY_ERR; + return PlatformDecoderModule::CreateDecoderPromise::CreateAndReject( + err, __func__); + }); + return p; +} + +/* static */ +RefPtr<GenericNonExclusivePromise> +RemoteDecoderManagerChild::LaunchRDDProcessIfNeeded() { + MOZ_DIAGNOSTIC_ASSERT(XRE_IsContentProcess(), + "Only supported from a content process."); + + nsCOMPtr<nsISerialEventTarget> managerThread = GetManagerThread(); + if (!managerThread) { + // We got shutdown. + return GenericNonExclusivePromise::CreateAndReject(NS_ERROR_FAILURE, + __func__); + } + + StaticMutexAutoLock lock(sLaunchMutex); + auto& rddLaunchPromise = sLaunchPromises[RemoteDecodeIn::RddProcess]; + if (rddLaunchPromise) { + return rddLaunchPromise; + } + + // We have a couple possible states here. We are in a content process + // and: + // 1) the RDD process has never been launched. RDD should be launched + // and the IPC connections setup. + // 2) the RDD process has been launched, but this particular content + // process has not setup (or has lost) its IPC connection. + // In the code below, we assume we need to launch the RDD process and + // setup the IPC connections. However, if the manager thread for + // RemoteDecoderManagerChild is available we do a quick check to see + // if we can send (meaning the IPC channel is open). If we can send, + // then no work is necessary. If we can't send, then we call + // LaunchRDDProcess which will launch RDD if necessary, and setup the + // IPC connections between *this* content process and the RDD process. + + RefPtr<GenericNonExclusivePromise> p = InvokeAsync( + managerThread, __func__, []() -> RefPtr<GenericNonExclusivePromise> { + auto* rps = GetSingleton(RemoteDecodeIn::RddProcess); + if (rps && rps->CanSend()) { + return GenericNonExclusivePromise::CreateAndResolve(true, __func__); + } + nsCOMPtr<nsISerialEventTarget> managerThread = GetManagerThread(); + ipc::PBackgroundChild* bgActor = + ipc::BackgroundChild::GetForCurrentThread(); + if (!managerThread || NS_WARN_IF(!bgActor)) { + return GenericNonExclusivePromise::CreateAndReject(NS_ERROR_FAILURE, + __func__); + } + + return bgActor->SendEnsureRDDProcessAndCreateBridge()->Then( + managerThread, __func__, + [](ipc::PBackgroundChild::EnsureRDDProcessAndCreateBridgePromise:: + ResolveOrRejectValue&& aResult) { + nsCOMPtr<nsISerialEventTarget> managerThread = GetManagerThread(); + if (!managerThread || aResult.IsReject()) { + // The parent process died or we got shutdown + return GenericNonExclusivePromise::CreateAndReject( + NS_ERROR_FAILURE, __func__); + } + nsresult rv = Get<0>(aResult.ResolveValue()); + if (NS_FAILED(rv)) { + return GenericNonExclusivePromise::CreateAndReject(rv, + __func__); + } + OpenRemoteDecoderManagerChildForProcess( + Get<1>(std::move(aResult.ResolveValue())), + RemoteDecodeIn::RddProcess); + return GenericNonExclusivePromise::CreateAndResolve(true, + __func__); + }); + }); + + // This should not be dispatched to a threadpool thread, so use managerThread + p = p->Then( + managerThread, __func__, + [](const GenericNonExclusivePromise::ResolveOrRejectValue& aResult) { + StaticMutexAutoLock lock(sLaunchMutex); + sLaunchPromises[RemoteDecodeIn::RddProcess] = nullptr; + return GenericNonExclusivePromise::CreateAndResolveOrReject(aResult, + __func__); + }); + + rddLaunchPromise = p; + return rddLaunchPromise; +} + +/* static */ +RefPtr<GenericNonExclusivePromise> +RemoteDecoderManagerChild::LaunchUtilityProcessIfNeeded( + RemoteDecodeIn aLocation) { + MOZ_DIAGNOSTIC_ASSERT(XRE_IsContentProcess(), + "Only supported from a content process."); + + nsCOMPtr<nsISerialEventTarget> managerThread = GetManagerThread(); + if (!managerThread) { + // We got shutdown. + return GenericNonExclusivePromise::CreateAndReject(NS_ERROR_FAILURE, + __func__); + } + + MOZ_ASSERT(aLocation == RemoteDecodeIn::UtilityProcess_Generic || + aLocation == RemoteDecodeIn::UtilityProcess_AppleMedia || + aLocation == RemoteDecodeIn::UtilityProcess_WMF || + aLocation == RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM); + StaticMutexAutoLock lock(sLaunchMutex); + auto& utilityLaunchPromise = sLaunchPromises[aLocation]; + + if (utilityLaunchPromise) { + return utilityLaunchPromise; + } + + // We have a couple possible states here. We are in a content process + // and: + // 1) the Utility process has never been launched. Utility should be launched + // and the IPC connections setup. + // 2) the Utility process has been launched, but this particular content + // process has not setup (or has lost) its IPC connection. + // In the code below, we assume we need to launch the Utility process and + // setup the IPC connections. However, if the manager thread for + // RemoteDecoderManagerChild is available we do a quick check to see + // if we can send (meaning the IPC channel is open). If we can send, + // then no work is necessary. If we can't send, then we call + // LaunchUtilityProcess which will launch Utility if necessary, and setup the + // IPC connections between *this* content process and the Utility process. + + RefPtr<GenericNonExclusivePromise> p = InvokeAsync( + managerThread, __func__, + [aLocation]() -> RefPtr<GenericNonExclusivePromise> { + auto* rps = GetSingleton(aLocation); + if (rps && rps->CanSend()) { + return GenericNonExclusivePromise::CreateAndResolve(true, __func__); + } + nsCOMPtr<nsISerialEventTarget> managerThread = GetManagerThread(); + ipc::PBackgroundChild* bgActor = + ipc::BackgroundChild::GetForCurrentThread(); + if (!managerThread || NS_WARN_IF(!bgActor)) { + return GenericNonExclusivePromise::CreateAndReject(NS_ERROR_FAILURE, + __func__); + } + + return bgActor->SendEnsureUtilityProcessAndCreateBridge(aLocation) + ->Then(managerThread, __func__, + [aLocation](ipc::PBackgroundChild:: + EnsureUtilityProcessAndCreateBridgePromise:: + ResolveOrRejectValue&& aResult) + -> RefPtr<GenericNonExclusivePromise> { + nsCOMPtr<nsISerialEventTarget> managerThread = + GetManagerThread(); + if (!managerThread || aResult.IsReject()) { + // The parent process died or we got shutdown + return GenericNonExclusivePromise::CreateAndReject( + NS_ERROR_FAILURE, __func__); + } + nsresult rv = Get<0>(aResult.ResolveValue()); + if (NS_FAILED(rv)) { + return GenericNonExclusivePromise::CreateAndReject( + rv, __func__); + } + OpenRemoteDecoderManagerChildForProcess( + Get<1>(std::move(aResult.ResolveValue())), aLocation); + return GenericNonExclusivePromise::CreateAndResolve( + true, __func__); + }); + }); + + // Let's make sure this promise is also run on the managerThread to avoid + // situations where it would be run on a threadpool thread. + // During bug 1794988 this was happening when enabling Utility for audio on + // Android when running the sequence of tests + // dom/media/test/test_access_control.html + // dom/media/test/test_arraybuffer.html + // + // We would have a launched utility process but the promises would not have + // been cleared, so any subsequent tentative to perform audio decoding would + // think the process is not yet ran and it would try to wait on the pending + // promises. + p = p->Then( + managerThread, __func__, + [aLocation]( + const GenericNonExclusivePromise::ResolveOrRejectValue& aResult) { + StaticMutexAutoLock lock(sLaunchMutex); + sLaunchPromises[aLocation] = nullptr; + return GenericNonExclusivePromise::CreateAndResolveOrReject(aResult, + __func__); + }); + utilityLaunchPromise = p; + return utilityLaunchPromise; +} + +/* static */ +TrackSupportSet RemoteDecoderManagerChild::GetTrackSupport( + RemoteDecodeIn aLocation) { + switch (aLocation) { + case RemoteDecodeIn::GpuProcess: { + return TrackSupportSet{TrackSupport::Video}; + } + case RemoteDecodeIn::RddProcess: { + TrackSupportSet s{TrackSupport::Video}; + // Only use RDD for audio decoding if we don't have the utility process. + if (!StaticPrefs::media_utility_process_enabled()) { + s += TrackSupport::Audio; + } + return s; + } + case RemoteDecodeIn::UtilityProcess_Generic: + case RemoteDecodeIn::UtilityProcess_AppleMedia: + case RemoteDecodeIn::UtilityProcess_WMF: + return StaticPrefs::media_utility_process_enabled() + ? TrackSupportSet{TrackSupport::Audio} + : TrackSupportSet{TrackSupport::None}; + case RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM: { + TrackSupportSet s{TrackSupport::None}; +#ifdef MOZ_WMF_MEDIA_ENGINE + // When we enable the media engine, it would need both tracks to + // synchronize the a/v playback. + if (StaticPrefs::media_wmf_media_engine_enabled()) { + s += TrackSupportSet{TrackSupport::Audio, TrackSupport::Video}; + } +#endif + return s; + } + default: + MOZ_ASSERT_UNREACHABLE("Undefined location!"); + } + return TrackSupportSet{TrackSupport::None}; +} + +PRemoteDecoderChild* RemoteDecoderManagerChild::AllocPRemoteDecoderChild( + const RemoteDecoderInfoIPDL& /* not used */, + const CreateDecoderParams::OptionSet& aOptions, + const Maybe<layers::TextureFactoryIdentifier>& aIdentifier, + const Maybe<uint64_t>& aMediaEngineId, + const Maybe<TrackingId>& aTrackingId) { + // RemoteDecoderModule is responsible for creating RemoteDecoderChild + // classes. + MOZ_ASSERT(false, + "RemoteDecoderManagerChild cannot create " + "RemoteDecoderChild classes"); + return nullptr; +} + +bool RemoteDecoderManagerChild::DeallocPRemoteDecoderChild( + PRemoteDecoderChild* actor) { + RemoteDecoderChild* child = static_cast<RemoteDecoderChild*>(actor); + child->IPDLActorDestroyed(); + return true; +} + +PMFMediaEngineChild* RemoteDecoderManagerChild::AllocPMFMediaEngineChild() { + MOZ_ASSERT_UNREACHABLE( + "RemoteDecoderManagerChild cannot create MFMediaEngineChild classes"); + return nullptr; +} + +bool RemoteDecoderManagerChild::DeallocPMFMediaEngineChild( + PMFMediaEngineChild* actor) { +#ifdef MOZ_WMF_MEDIA_ENGINE + MFMediaEngineChild* child = static_cast<MFMediaEngineChild*>(actor); + child->IPDLActorDestroyed(); +#endif + return true; +} + +RemoteDecoderManagerChild::RemoteDecoderManagerChild(RemoteDecodeIn aLocation) + : mLocation(aLocation) { + MOZ_ASSERT(mLocation == RemoteDecodeIn::GpuProcess || + mLocation == RemoteDecodeIn::RddProcess || + mLocation == RemoteDecodeIn::UtilityProcess_Generic || + mLocation == RemoteDecodeIn::UtilityProcess_AppleMedia || + mLocation == RemoteDecodeIn::UtilityProcess_WMF || + mLocation == RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM); +} + +/* static */ +void RemoteDecoderManagerChild::OpenRemoteDecoderManagerChildForProcess( + Endpoint<PRemoteDecoderManagerChild>&& aEndpoint, + RemoteDecodeIn aLocation) { + nsCOMPtr<nsISerialEventTarget> managerThread = GetManagerThread(); + if (!managerThread) { + // We've been shutdown, bail. + return; + } + MOZ_ASSERT(managerThread->IsOnCurrentThread()); + + // For GPU process, make sure we always dispatch everything in sRecreateTasks, + // even if we fail since this is as close to being recreated as we will ever + // be. + auto runRecreateTasksIfNeeded = MakeScopeExit([aLocation]() { + if (aLocation == RemoteDecodeIn::GpuProcess) { + for (Runnable* task : *sRecreateTasks) { + task->Run(); + } + sRecreateTasks->Clear(); + } + }); + + // Only create RemoteDecoderManagerChild, bind new endpoint and init + // ipdl if: + // 1) haven't init'd sRemoteDecoderManagerChildForProcesses[aLocation] + // or + // 2) if ActorDestroy was called meaning the other end of the ipc channel was + // torn down + // But for GPU process, we always recreate a new manager child. + MOZ_ASSERT(aLocation != RemoteDecodeIn::SENTINEL); + auto& remoteDecoderManagerChild = + sRemoteDecoderManagerChildForProcesses[aLocation]; + if (aLocation != RemoteDecodeIn::GpuProcess && remoteDecoderManagerChild && + remoteDecoderManagerChild->CanSend()) { + return; + } + remoteDecoderManagerChild = nullptr; + if (aEndpoint.IsValid()) { + RefPtr<RemoteDecoderManagerChild> manager = + new RemoteDecoderManagerChild(aLocation); + if (aEndpoint.Bind(manager)) { + remoteDecoderManagerChild = manager; + manager->InitIPDL(); + } + } +} + +void RemoteDecoderManagerChild::InitIPDL() { mIPDLSelfRef = this; } + +void RemoteDecoderManagerChild::ActorDealloc() { mIPDLSelfRef = nullptr; } + +bool RemoteDecoderManagerChild::DeallocShmem(mozilla::ipc::Shmem& aShmem) { + nsCOMPtr<nsISerialEventTarget> managerThread = GetManagerThread(); + if (!managerThread) { + return false; + } + if (!managerThread->IsOnCurrentThread()) { + MOZ_ALWAYS_SUCCEEDS(managerThread->Dispatch(NS_NewRunnableFunction( + "RemoteDecoderManagerChild::DeallocShmem", + [self = RefPtr{this}, shmem = aShmem]() mutable { + if (self->CanSend()) { + self->PRemoteDecoderManagerChild::DeallocShmem(shmem); + } + }))); + return true; + } + return PRemoteDecoderManagerChild::DeallocShmem(aShmem); +} + +struct SurfaceDescriptorUserData { + SurfaceDescriptorUserData(RemoteDecoderManagerChild* aAllocator, + SurfaceDescriptor& aSD) + : mAllocator(aAllocator), mSD(aSD) {} + ~SurfaceDescriptorUserData() { DestroySurfaceDescriptor(mAllocator, &mSD); } + + RefPtr<RemoteDecoderManagerChild> mAllocator; + SurfaceDescriptor mSD; +}; + +void DeleteSurfaceDescriptorUserData(void* aClosure) { + SurfaceDescriptorUserData* sd = + reinterpret_cast<SurfaceDescriptorUserData*>(aClosure); + delete sd; +} + +already_AddRefed<SourceSurface> RemoteDecoderManagerChild::Readback( + const SurfaceDescriptorGPUVideo& aSD) { + // We can't use NS_DISPATCH_SYNC here since that can spin the event + // loop while it waits. This function can be called from JS and we + // don't want that to happen. + nsCOMPtr<nsISerialEventTarget> managerThread = GetManagerThread(); + if (!managerThread) { + return nullptr; + } + + SurfaceDescriptor sd; + RefPtr<Runnable> task = + NS_NewRunnableFunction("RemoteDecoderManagerChild::Readback", [&]() { + if (CanSend()) { + SendReadback(aSD, &sd); + } + }); + SyncRunnable::DispatchToThread(managerThread, task); + + if (!IsSurfaceDescriptorValid(sd)) { + return nullptr; + } + + RefPtr<DataSourceSurface> source = GetSurfaceForDescriptor(sd); + if (!source) { + DestroySurfaceDescriptor(this, &sd); + NS_WARNING("Failed to map SurfaceDescriptor in Readback"); + return nullptr; + } + + static UserDataKey sSurfaceDescriptor; + source->AddUserData(&sSurfaceDescriptor, + new SurfaceDescriptorUserData(this, sd), + DeleteSurfaceDescriptorUserData); + + return source.forget(); +} + +void RemoteDecoderManagerChild::DeallocateSurfaceDescriptor( + const SurfaceDescriptorGPUVideo& aSD) { + nsCOMPtr<nsISerialEventTarget> managerThread = GetManagerThread(); + if (!managerThread) { + return; + } + MOZ_ALWAYS_SUCCEEDS(managerThread->Dispatch(NS_NewRunnableFunction( + "RemoteDecoderManagerChild::DeallocateSurfaceDescriptor", + [ref = RefPtr{this}, sd = aSD]() { + if (ref->CanSend()) { + ref->SendDeallocateSurfaceDescriptorGPUVideo(sd); + } + }))); +} + +void RemoteDecoderManagerChild::HandleFatalError(const char* aMsg) const { + dom::ContentChild::FatalErrorIfNotUsingGPUProcess(aMsg, OtherPid()); +} + +void RemoteDecoderManagerChild::SetSupported( + RemoteDecodeIn aLocation, const media::MediaCodecsSupported& aSupported) { + switch (aLocation) { + case RemoteDecodeIn::GpuProcess: + case RemoteDecodeIn::RddProcess: + case RemoteDecodeIn::UtilityProcess_AppleMedia: + case RemoteDecodeIn::UtilityProcess_Generic: + case RemoteDecodeIn::UtilityProcess_WMF: + case RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM: { + StaticMutexAutoLock lock(sProcessSupportedMutex); + sProcessSupported[aLocation] = Some(aSupported); + break; + } + default: + MOZ_CRASH("Not to be used for any other process"); + } +} + +#undef LOG + +} // namespace mozilla diff --git a/dom/media/ipc/RemoteDecoderManagerChild.h b/dom/media/ipc/RemoteDecoderManagerChild.h new file mode 100644 index 0000000000..bfdf80e23a --- /dev/null +++ b/dom/media/ipc/RemoteDecoderManagerChild.h @@ -0,0 +1,154 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef include_dom_media_ipc_RemoteDecoderManagerChild_h +#define include_dom_media_ipc_RemoteDecoderManagerChild_h + +#include "GPUVideoImage.h" +#include "PDMFactory.h" +#include "ipc/EnumSerializer.h" +#include "mozilla/EnumTypeTraits.h" +#include "mozilla/PRemoteDecoderManagerChild.h" +#include "mozilla/layers/VideoBridgeUtils.h" +#include "mozilla/ipc/UtilityProcessSandboxing.h" + +namespace mozilla { + +class PMFMediaEngineChild; +class RemoteDecoderChild; + +enum class RemoteDecodeIn { + Unspecified, + RddProcess, + GpuProcess, + UtilityProcess_Generic, + UtilityProcess_AppleMedia, + UtilityProcess_WMF, + UtilityProcess_MFMediaEngineCDM, + SENTINEL, +}; + +enum class TrackSupport { + None, + Audio, + Video, +}; +using TrackSupportSet = EnumSet<TrackSupport, uint8_t>; + +class RemoteDecoderManagerChild final + : public PRemoteDecoderManagerChild, + public mozilla::ipc::IShmemAllocator, + public mozilla::layers::IGPUVideoSurfaceManager { + friend class PRemoteDecoderManagerChild; + + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(RemoteDecoderManagerChild, override) + + // Can only be called from the manager thread + static RemoteDecoderManagerChild* GetSingleton(RemoteDecodeIn aLocation); + + static void Init(); + static void SetSupported(RemoteDecodeIn aLocation, + const media::MediaCodecsSupported& aSupported); + + // Can be called from any thread. + static bool Supports(RemoteDecodeIn aLocation, + const SupportDecoderParams& aParams, + DecoderDoctorDiagnostics* aDiagnostics); + static RefPtr<PlatformDecoderModule::CreateDecoderPromise> CreateAudioDecoder( + const CreateDecoderParams& aParams, RemoteDecodeIn aLocation); + static RefPtr<PlatformDecoderModule::CreateDecoderPromise> CreateVideoDecoder( + const CreateDecoderParams& aParams, RemoteDecodeIn aLocation); + + // Can be called from any thread. + static nsISerialEventTarget* GetManagerThread(); + + // Return the track support information based on the location of the remote + // process. Thread-safe. + static TrackSupportSet GetTrackSupport(RemoteDecodeIn aLocation); + + // Can be called from any thread, dispatches the request to the IPDL thread + // internally and will be ignored if the IPDL actor has been destroyed. + already_AddRefed<gfx::SourceSurface> Readback( + const SurfaceDescriptorGPUVideo& aSD) override; + void DeallocateSurfaceDescriptor( + const SurfaceDescriptorGPUVideo& aSD) override; + + bool AllocShmem(size_t aSize, mozilla::ipc::Shmem* aShmem) override { + return PRemoteDecoderManagerChild::AllocShmem(aSize, aShmem); + } + bool AllocUnsafeShmem(size_t aSize, mozilla::ipc::Shmem* aShmem) override { + return PRemoteDecoderManagerChild::AllocUnsafeShmem(aSize, aShmem); + } + + // Can be called from any thread, dispatches the request to the IPDL thread + // internally and will be ignored if the IPDL actor has been destroyed. + bool DeallocShmem(mozilla::ipc::Shmem& aShmem) override; + + // Main thread only + static void InitForGPUProcess( + Endpoint<PRemoteDecoderManagerChild>&& aVideoManager); + static void Shutdown(); + + // Run aTask (on the manager thread) when we next attempt to create a new + // manager (even if creation fails). Intended to be called from ActorDestroy + // when we get notified that the old manager is being destroyed. Can only be + // called from the manager thread. + void RunWhenGPUProcessRecreated(already_AddRefed<Runnable> aTask); + + RemoteDecodeIn Location() const { return mLocation; } + + // A thread-safe method to launch the utility process if it hasn't launched + // yet. + static RefPtr<GenericNonExclusivePromise> LaunchUtilityProcessIfNeeded( + RemoteDecodeIn aLocation); + + protected: + void InitIPDL(); + + void ActorDealloc() override; + + void HandleFatalError(const char* aMsg) const override; + + PRemoteDecoderChild* AllocPRemoteDecoderChild( + const RemoteDecoderInfoIPDL& aRemoteDecoderInfo, + const CreateDecoderParams::OptionSet& aOptions, + const Maybe<layers::TextureFactoryIdentifier>& aIdentifier, + const Maybe<uint64_t>& aMediaEngineId, + const Maybe<TrackingId>& aTrackingId); + bool DeallocPRemoteDecoderChild(PRemoteDecoderChild* actor); + + PMFMediaEngineChild* AllocPMFMediaEngineChild(); + bool DeallocPMFMediaEngineChild(PMFMediaEngineChild* actor); + + private: + explicit RemoteDecoderManagerChild(RemoteDecodeIn aLocation); + ~RemoteDecoderManagerChild() = default; + static RefPtr<PlatformDecoderModule::CreateDecoderPromise> Construct( + RefPtr<RemoteDecoderChild>&& aChild, RemoteDecodeIn aLocation); + + static void OpenRemoteDecoderManagerChildForProcess( + Endpoint<PRemoteDecoderManagerChild>&& aEndpoint, + RemoteDecodeIn aLocation); + + // A thread-safe method to launch the RDD process if it hasn't launched yet. + static RefPtr<GenericNonExclusivePromise> LaunchRDDProcessIfNeeded(); + + RefPtr<RemoteDecoderManagerChild> mIPDLSelfRef; + // The location for decoding, Rdd or Gpu process. + const RemoteDecodeIn mLocation; +}; + +} // namespace mozilla + +namespace IPC { +template <> +struct ParamTraits<mozilla::RemoteDecodeIn> + : public ContiguousEnumSerializer<mozilla::RemoteDecodeIn, + mozilla::RemoteDecodeIn::Unspecified, + mozilla::RemoteDecodeIn::SENTINEL> {}; +} // namespace IPC + +#endif // include_dom_media_ipc_RemoteDecoderManagerChild_h diff --git a/dom/media/ipc/RemoteDecoderManagerParent.cpp b/dom/media/ipc/RemoteDecoderManagerParent.cpp new file mode 100644 index 0000000000..428f69384b --- /dev/null +++ b/dom/media/ipc/RemoteDecoderManagerParent.cpp @@ -0,0 +1,335 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "RemoteDecoderManagerParent.h" + +#if XP_WIN +# include <objbase.h> +#endif + +#include "ImageContainer.h" +#include "PDMFactory.h" +#include "RemoteAudioDecoder.h" +#include "RemoteVideoDecoder.h" +#include "VideoUtils.h" // for MediaThreadType +#include "mozilla/RDDParent.h" +#include "mozilla/RemoteDecodeUtils.h" +#include "mozilla/ipc/UtilityProcessChild.h" +#include "mozilla/SyncRunnable.h" +#include "mozilla/gfx/GPUParent.h" +#include "mozilla/ipc/Endpoint.h" +#include "mozilla/layers/ImageDataSerializer.h" +#include "mozilla/layers/VideoBridgeChild.h" +#include "mozilla/layers/VideoBridgeParent.h" + +#ifdef MOZ_WMF_MEDIA_ENGINE +# include "MFMediaEngineParent.h" +#endif + +namespace mozilla { + +#define LOG(msg, ...) \ + MOZ_LOG(gRemoteDecodeLog, LogLevel::Debug, (msg, ##__VA_ARGS__)) + +using namespace ipc; +using namespace layers; +using namespace gfx; + +StaticRefPtr<TaskQueue> sRemoteDecoderManagerParentThread; + +void RemoteDecoderManagerParent::StoreImage( + const SurfaceDescriptorGPUVideo& aSD, Image* aImage, + TextureClient* aTexture) { + MOZ_ASSERT(OnManagerThread()); + mImageMap[static_cast<SurfaceDescriptorRemoteDecoder>(aSD).handle()] = aImage; + mTextureMap[static_cast<SurfaceDescriptorRemoteDecoder>(aSD).handle()] = + aTexture; +} + +class RemoteDecoderManagerThreadShutdownObserver : public nsIObserver { + virtual ~RemoteDecoderManagerThreadShutdownObserver() = default; + + public: + RemoteDecoderManagerThreadShutdownObserver() = default; + + NS_DECL_ISUPPORTS + + NS_IMETHOD Observe(nsISupports* aSubject, const char* aTopic, + const char16_t* aData) override { + MOZ_ASSERT(strcmp(aTopic, NS_XPCOM_SHUTDOWN_OBSERVER_ID) == 0); + + RemoteDecoderManagerParent::ShutdownVideoBridge(); + RemoteDecoderManagerParent::ShutdownThreads(); + return NS_OK; + } +}; +NS_IMPL_ISUPPORTS(RemoteDecoderManagerThreadShutdownObserver, nsIObserver); + +bool RemoteDecoderManagerParent::StartupThreads() { + MOZ_ASSERT(NS_IsMainThread()); + + if (sRemoteDecoderManagerParentThread) { + return true; + } + + nsCOMPtr<nsIObserverService> observerService = services::GetObserverService(); + if (!observerService) { + return false; + } + + sRemoteDecoderManagerParentThread = TaskQueue::Create( + GetMediaThreadPool(MediaThreadType::SUPERVISOR), "RemVidParent"); + if (XRE_IsGPUProcess()) { + MOZ_ALWAYS_SUCCEEDS( + sRemoteDecoderManagerParentThread->Dispatch(NS_NewRunnableFunction( + "RemoteDecoderManagerParent::StartupThreads", + []() { layers::VideoBridgeChild::StartupForGPUProcess(); }))); + } + + auto* obs = new RemoteDecoderManagerThreadShutdownObserver(); + observerService->AddObserver(obs, NS_XPCOM_SHUTDOWN_OBSERVER_ID, false); + return true; +} + +void RemoteDecoderManagerParent::ShutdownThreads() { + sRemoteDecoderManagerParentThread->BeginShutdown(); + sRemoteDecoderManagerParentThread->AwaitShutdownAndIdle(); + sRemoteDecoderManagerParentThread = nullptr; +} + +/* static */ +void RemoteDecoderManagerParent::ShutdownVideoBridge() { + if (sRemoteDecoderManagerParentThread) { + RefPtr<Runnable> task = NS_NewRunnableFunction( + "RemoteDecoderManagerParent::ShutdownVideoBridge", []() { + VideoBridgeParent::Shutdown(); + VideoBridgeChild::Shutdown(); + }); + SyncRunnable::DispatchToThread(sRemoteDecoderManagerParentThread, task); + } +} + +bool RemoteDecoderManagerParent::OnManagerThread() { + return sRemoteDecoderManagerParentThread->IsOnCurrentThread(); +} + +PDMFactory& RemoteDecoderManagerParent::EnsurePDMFactory() { + MOZ_ASSERT(OnManagerThread()); + if (!mPDMFactory) { + mPDMFactory = MakeRefPtr<PDMFactory>(); + } + return *mPDMFactory; +} + +bool RemoteDecoderManagerParent::CreateForContent( + Endpoint<PRemoteDecoderManagerParent>&& aEndpoint) { + MOZ_ASSERT(XRE_GetProcessType() == GeckoProcessType_RDD || + XRE_GetProcessType() == GeckoProcessType_Utility || + XRE_GetProcessType() == GeckoProcessType_GPU); + MOZ_ASSERT(NS_IsMainThread()); + + if (!StartupThreads()) { + return false; + } + + RefPtr<RemoteDecoderManagerParent> parent = + new RemoteDecoderManagerParent(sRemoteDecoderManagerParentThread); + + RefPtr<Runnable> task = + NewRunnableMethod<Endpoint<PRemoteDecoderManagerParent>&&>( + "dom::RemoteDecoderManagerParent::Open", parent, + &RemoteDecoderManagerParent::Open, std::move(aEndpoint)); + MOZ_ALWAYS_SUCCEEDS( + sRemoteDecoderManagerParentThread->Dispatch(task.forget())); + return true; +} + +bool RemoteDecoderManagerParent::CreateVideoBridgeToOtherProcess( + Endpoint<PVideoBridgeChild>&& aEndpoint) { + LOG("Create video bridge"); + // We never want to decode in the GPU process, but output + // frames to the parent process. + MOZ_ASSERT(XRE_GetProcessType() == GeckoProcessType_RDD || + XRE_GetProcessType() == GeckoProcessType_Utility); +#ifdef MOZ_WMF_MEDIA_ENGINE + MOZ_ASSERT_IF( + XRE_GetProcessType() == GeckoProcessType_Utility, + GetCurrentSandboxingKind() == SandboxingKind::MF_MEDIA_ENGINE_CDM); +#endif + MOZ_ASSERT(NS_IsMainThread()); + + if (!StartupThreads()) { + return false; + } + + RefPtr<Runnable> task = + NewRunnableFunction("gfx::VideoBridgeChild::Open", + &VideoBridgeChild::Open, std::move(aEndpoint)); + MOZ_ALWAYS_SUCCEEDS( + sRemoteDecoderManagerParentThread->Dispatch(task.forget())); + return true; +} + +RemoteDecoderManagerParent::RemoteDecoderManagerParent( + nsISerialEventTarget* aThread) + : mThread(aThread) { + MOZ_COUNT_CTOR(RemoteDecoderManagerParent); + auto& registrar = + XRE_IsGPUProcess() ? GPUParent::GetSingleton()->AsyncShutdownService() + : XRE_IsUtilityProcess() + ? UtilityProcessChild::GetSingleton()->AsyncShutdownService() + : RDDParent::GetSingleton()->AsyncShutdownService(); + registrar.Register(this); +} + +RemoteDecoderManagerParent::~RemoteDecoderManagerParent() { + MOZ_COUNT_DTOR(RemoteDecoderManagerParent); + auto& registrar = + XRE_IsGPUProcess() ? GPUParent::GetSingleton()->AsyncShutdownService() + : XRE_IsUtilityProcess() + ? UtilityProcessChild::GetSingleton()->AsyncShutdownService() + : RDDParent::GetSingleton()->AsyncShutdownService(); + registrar.Deregister(this); +} + +void RemoteDecoderManagerParent::ActorDestroy( + mozilla::ipc::IProtocol::ActorDestroyReason) { + mThread = nullptr; +} + +PRemoteDecoderParent* RemoteDecoderManagerParent::AllocPRemoteDecoderParent( + const RemoteDecoderInfoIPDL& aRemoteDecoderInfo, + const CreateDecoderParams::OptionSet& aOptions, + const Maybe<layers::TextureFactoryIdentifier>& aIdentifier, + const Maybe<uint64_t>& aMediaEngineId, + const Maybe<TrackingId>& aTrackingId) { + RefPtr<TaskQueue> decodeTaskQueue = + TaskQueue::Create(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER), + "RemoteVideoDecoderParent::mDecodeTaskQueue"); + + if (aRemoteDecoderInfo.type() == + RemoteDecoderInfoIPDL::TVideoDecoderInfoIPDL) { + const VideoDecoderInfoIPDL& decoderInfo = + aRemoteDecoderInfo.get_VideoDecoderInfoIPDL(); + return new RemoteVideoDecoderParent( + this, decoderInfo.videoInfo(), decoderInfo.framerate(), aOptions, + aIdentifier, sRemoteDecoderManagerParentThread, decodeTaskQueue, + aMediaEngineId, aTrackingId); + } + + if (aRemoteDecoderInfo.type() == RemoteDecoderInfoIPDL::TAudioInfo) { + return new RemoteAudioDecoderParent( + this, aRemoteDecoderInfo.get_AudioInfo(), aOptions, + sRemoteDecoderManagerParentThread, decodeTaskQueue, aMediaEngineId); + } + + MOZ_CRASH("unrecognized type of RemoteDecoderInfoIPDL union"); + return nullptr; +} + +bool RemoteDecoderManagerParent::DeallocPRemoteDecoderParent( + PRemoteDecoderParent* actor) { + RemoteDecoderParent* parent = static_cast<RemoteDecoderParent*>(actor); + parent->Destroy(); + return true; +} + +PMFMediaEngineParent* RemoteDecoderManagerParent::AllocPMFMediaEngineParent() { +#ifdef MOZ_WMF_MEDIA_ENGINE + return new MFMediaEngineParent(this, sRemoteDecoderManagerParentThread); +#else + return nullptr; +#endif +} + +bool RemoteDecoderManagerParent::DeallocPMFMediaEngineParent( + PMFMediaEngineParent* actor) { +#ifdef MOZ_WMF_MEDIA_ENGINE + MFMediaEngineParent* parent = static_cast<MFMediaEngineParent*>(actor); + parent->Destroy(); +#endif + return true; +} + +void RemoteDecoderManagerParent::Open( + Endpoint<PRemoteDecoderManagerParent>&& aEndpoint) { + if (!aEndpoint.Bind(this)) { + // We can't recover from this. + MOZ_CRASH("Failed to bind RemoteDecoderManagerParent to endpoint"); + } + AddRef(); +} + +void RemoteDecoderManagerParent::ActorDealloc() { Release(); } + +mozilla::ipc::IPCResult RemoteDecoderManagerParent::RecvReadback( + const SurfaceDescriptorGPUVideo& aSD, SurfaceDescriptor* aResult) { + const SurfaceDescriptorRemoteDecoder& sd = aSD; + RefPtr<Image> image = mImageMap[sd.handle()]; + if (!image) { + *aResult = null_t(); + return IPC_OK(); + } + + RefPtr<SourceSurface> source = image->GetAsSourceSurface(); + if (!source) { + *aResult = null_t(); + return IPC_OK(); + } + + SurfaceFormat format = source->GetFormat(); + IntSize size = source->GetSize(); + size_t length = ImageDataSerializer::ComputeRGBBufferSize(size, format); + + Shmem buffer; + if (!length || !AllocShmem(length, &buffer)) { + *aResult = null_t(); + return IPC_OK(); + } + + RefPtr<DrawTarget> dt = Factory::CreateDrawTargetForData( + gfx::BackendType::CAIRO, buffer.get<uint8_t>(), size, + ImageDataSerializer::ComputeRGBStride(format, size.width), format); + if (!dt) { + DeallocShmem(buffer); + *aResult = null_t(); + return IPC_OK(); + } + + dt->CopySurface(source, IntRect(0, 0, size.width, size.height), IntPoint()); + dt->Flush(); + + *aResult = SurfaceDescriptorBuffer(RGBDescriptor(size, format), + MemoryOrShmem(std::move(buffer))); + return IPC_OK(); +} + +mozilla::ipc::IPCResult +RemoteDecoderManagerParent::RecvDeallocateSurfaceDescriptorGPUVideo( + const SurfaceDescriptorGPUVideo& aSD) { + MOZ_ASSERT(OnManagerThread()); + const SurfaceDescriptorRemoteDecoder& sd = aSD; + mImageMap.erase(sd.handle()); + mTextureMap.erase(sd.handle()); + return IPC_OK(); +} + +void RemoteDecoderManagerParent::DeallocateSurfaceDescriptor( + const SurfaceDescriptorGPUVideo& aSD) { + if (!OnManagerThread()) { + MOZ_ALWAYS_SUCCEEDS( + sRemoteDecoderManagerParentThread->Dispatch(NS_NewRunnableFunction( + "RemoteDecoderManagerParent::DeallocateSurfaceDescriptor", + [ref = RefPtr{this}, sd = aSD]() { + ref->RecvDeallocateSurfaceDescriptorGPUVideo(sd); + }))); + } else { + RecvDeallocateSurfaceDescriptorGPUVideo(aSD); + } +} + +#undef LOG + +} // namespace mozilla diff --git a/dom/media/ipc/RemoteDecoderManagerParent.h b/dom/media/ipc/RemoteDecoderManagerParent.h new file mode 100644 index 0000000000..f88dadd996 --- /dev/null +++ b/dom/media/ipc/RemoteDecoderManagerParent.h @@ -0,0 +1,92 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef include_dom_media_ipc_RemoteDecoderManagerParent_h +#define include_dom_media_ipc_RemoteDecoderManagerParent_h + +#include "GPUVideoImage.h" +#include "mozilla/PRemoteDecoderManagerParent.h" +#include "mozilla/layers/VideoBridgeChild.h" + +namespace mozilla { + +class PDMFactory; +class PMFMediaEngineParent; + +class RemoteDecoderManagerParent final + : public PRemoteDecoderManagerParent, + public layers::IGPUVideoSurfaceManager { + friend class PRemoteDecoderManagerParent; + + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(RemoteDecoderManagerParent, override) + + static bool CreateForContent( + Endpoint<PRemoteDecoderManagerParent>&& aEndpoint); + + static bool CreateVideoBridgeToOtherProcess( + Endpoint<layers::PVideoBridgeChild>&& aEndpoint); + + // Must be called on manager thread. + // Store the image so that it can be used out of process. Will be released + // when DeallocateSurfaceDescriptor is called. + void StoreImage(const SurfaceDescriptorGPUVideo& aSD, layers::Image* aImage, + layers::TextureClient* aTexture); + + // IGPUVideoSurfaceManager methods + already_AddRefed<gfx::SourceSurface> Readback( + const SurfaceDescriptorGPUVideo& aSD) override { + MOZ_ASSERT_UNREACHABLE("Not usable from the parent"); + return nullptr; + } + void DeallocateSurfaceDescriptor( + const SurfaceDescriptorGPUVideo& aSD) override; + + static bool StartupThreads(); + static void ShutdownThreads(); + + static void ShutdownVideoBridge(); + + bool OnManagerThread(); + + // Can be called from manager thread only + PDMFactory& EnsurePDMFactory(); + + protected: + PRemoteDecoderParent* AllocPRemoteDecoderParent( + const RemoteDecoderInfoIPDL& aRemoteDecoderInfo, + const CreateDecoderParams::OptionSet& aOptions, + const Maybe<layers::TextureFactoryIdentifier>& aIdentifier, + const Maybe<uint64_t>& aMediaEngineId, + const Maybe<TrackingId>& aTrackingId); + bool DeallocPRemoteDecoderParent(PRemoteDecoderParent* actor); + + PMFMediaEngineParent* AllocPMFMediaEngineParent(); + bool DeallocPMFMediaEngineParent(PMFMediaEngineParent* actor); + + mozilla::ipc::IPCResult RecvReadback(const SurfaceDescriptorGPUVideo& aSD, + SurfaceDescriptor* aResult); + mozilla::ipc::IPCResult RecvDeallocateSurfaceDescriptorGPUVideo( + const SurfaceDescriptorGPUVideo& aSD); + + void ActorDestroy(mozilla::ipc::IProtocol::ActorDestroyReason) override; + void ActorDealloc() override; + + private: + explicit RemoteDecoderManagerParent(nsISerialEventTarget* aThread); + ~RemoteDecoderManagerParent(); + + void Open(Endpoint<PRemoteDecoderManagerParent>&& aEndpoint); + + std::map<uint64_t, RefPtr<layers::Image>> mImageMap; + std::map<uint64_t, RefPtr<layers::TextureClient>> mTextureMap; + + nsCOMPtr<nsISerialEventTarget> mThread; + RefPtr<PDMFactory> mPDMFactory; +}; + +} // namespace mozilla + +#endif // include_dom_media_ipc_RemoteDecoderManagerParent_h diff --git a/dom/media/ipc/RemoteDecoderModule.cpp b/dom/media/ipc/RemoteDecoderModule.cpp new file mode 100644 index 0000000000..2f0d6e1752 --- /dev/null +++ b/dom/media/ipc/RemoteDecoderModule.cpp @@ -0,0 +1,87 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "RemoteDecoderModule.h" + +#ifdef MOZ_AV1 +# include "AOMDecoder.h" +#endif +#include "OpusDecoder.h" +#include "RemoteAudioDecoder.h" +#include "RemoteDecoderManagerChild.h" +#include "RemoteMediaDataDecoder.h" +#include "RemoteVideoDecoder.h" +#include "VideoUtils.h" +#include "VorbisDecoder.h" +#include "WAVDecoder.h" +#include "gfxConfig.h" +#include "mozilla/RemoteDecodeUtils.h" + +namespace mozilla { + +using namespace ipc; +using namespace layers; + +already_AddRefed<PlatformDecoderModule> RemoteDecoderModule::Create( + RemoteDecodeIn aLocation) { + MOZ_ASSERT(!XRE_IsGPUProcess() && !XRE_IsRDDProcess(), + "Should not be created in GPU or RDD process."); + if (!XRE_IsContentProcess()) { + // For now, the RemoteDecoderModule is only available in the content + // process. + return nullptr; + } + return MakeAndAddRef<RemoteDecoderModule>(aLocation); +} + +RemoteDecoderModule::RemoteDecoderModule(RemoteDecodeIn aLocation) + : mLocation(aLocation) {} + +media::DecodeSupportSet RemoteDecoderModule::SupportsMimeType( + const nsACString& aMimeType, DecoderDoctorDiagnostics* aDiagnostics) const { + MOZ_CRASH("Deprecated: Use RemoteDecoderModule::Supports"); +} // namespace mozilla + +media::DecodeSupportSet RemoteDecoderModule::Supports( + const SupportDecoderParams& aParams, + DecoderDoctorDiagnostics* aDiagnostics) const { + bool supports = + RemoteDecoderManagerChild::Supports(mLocation, aParams, aDiagnostics); + // This should only be supported by mf media engine cdm process. + if (aParams.mMediaEngineId && + mLocation != RemoteDecodeIn::UtilityProcess_MFMediaEngineCDM) { + supports = false; + } + MOZ_LOG(sPDMLog, LogLevel::Debug, + ("Sandbox %s decoder %s requested type %s", + RemoteDecodeInToStr(mLocation), supports ? "supports" : "rejects", + aParams.MimeType().get())); + if (supports) { + // TODO: Note that we do not yet distinguish between SW/HW decode support. + // Will be done in bug 1754239. + return media::DecodeSupport::SoftwareDecode; + } + return media::DecodeSupport::Unsupported; +} + +RefPtr<RemoteDecoderModule::CreateDecoderPromise> +RemoteDecoderModule::AsyncCreateDecoder(const CreateDecoderParams& aParams) { + if (aParams.mConfig.IsAudio()) { + // OpusDataDecoder will check this option to provide the same info + // that IsDefaultPlaybackDeviceMono provides. We want to avoid calls + // to IsDefaultPlaybackDeviceMono on RDD because initializing audio + // backends on RDD will be blocked by the sandbox. + if (OpusDataDecoder::IsOpus(aParams.mConfig.mMimeType) && + IsDefaultPlaybackDeviceMono()) { + CreateDecoderParams params = aParams; + params.mOptions += CreateDecoderParams::Option::DefaultPlaybackDeviceMono; + return RemoteDecoderManagerChild::CreateAudioDecoder(params, mLocation); + } + return RemoteDecoderManagerChild::CreateAudioDecoder(aParams, mLocation); + } + return RemoteDecoderManagerChild::CreateVideoDecoder(aParams, mLocation); +} + +} // namespace mozilla diff --git a/dom/media/ipc/RemoteDecoderModule.h b/dom/media/ipc/RemoteDecoderModule.h new file mode 100644 index 0000000000..a11f73227e --- /dev/null +++ b/dom/media/ipc/RemoteDecoderModule.h @@ -0,0 +1,52 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef include_dom_media_ipc_RemoteDecoderModule_h +#define include_dom_media_ipc_RemoteDecoderModule_h +#include "PlatformDecoderModule.h" + +namespace mozilla { + +enum class RemoteDecodeIn; + +// A decoder module that proxies decoding to either GPU or RDD process. +class RemoteDecoderModule : public PlatformDecoderModule { + template <typename T, typename... Args> + friend already_AddRefed<T> MakeAndAddRef(Args&&...); + + public: + static already_AddRefed<PlatformDecoderModule> Create( + RemoteDecodeIn aLocation); + + media::DecodeSupportSet SupportsMimeType( + const nsACString& aMimeType, + DecoderDoctorDiagnostics* aDiagnostics) const override; + + media::DecodeSupportSet Supports( + const SupportDecoderParams& aParams, + DecoderDoctorDiagnostics* aDiagnostics) const override; + + RefPtr<CreateDecoderPromise> AsyncCreateDecoder( + const CreateDecoderParams& aParams) override; + + already_AddRefed<MediaDataDecoder> CreateVideoDecoder( + const CreateDecoderParams& aParams) override { + MOZ_CRASH("Not available"); + } + + already_AddRefed<MediaDataDecoder> CreateAudioDecoder( + const CreateDecoderParams& aParams) override { + MOZ_CRASH("Not available"); + } + + private: + explicit RemoteDecoderModule(RemoteDecodeIn aLocation); + + const RemoteDecodeIn mLocation; +}; + +} // namespace mozilla + +#endif // include_dom_media_ipc_RemoteDecoderModule_h diff --git a/dom/media/ipc/RemoteDecoderParent.cpp b/dom/media/ipc/RemoteDecoderParent.cpp new file mode 100644 index 0000000000..c03bd65a65 --- /dev/null +++ b/dom/media/ipc/RemoteDecoderParent.cpp @@ -0,0 +1,226 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "RemoteDecoderParent.h" + +#include "RemoteDecoderManagerParent.h" +#include "mozilla/Unused.h" + +namespace mozilla { + +RemoteDecoderParent::RemoteDecoderParent( + RemoteDecoderManagerParent* aParent, + const CreateDecoderParams::OptionSet& aOptions, + nsISerialEventTarget* aManagerThread, TaskQueue* aDecodeTaskQueue, + const Maybe<uint64_t>& aMediaEngineId, Maybe<TrackingId> aTrackingId) + : ShmemRecycleAllocator(this), + mParent(aParent), + mOptions(aOptions), + mDecodeTaskQueue(aDecodeTaskQueue), + mTrackingId(aTrackingId), + mMediaEngineId(aMediaEngineId), + mManagerThread(aManagerThread) { + MOZ_COUNT_CTOR(RemoteDecoderParent); + MOZ_ASSERT(OnManagerThread()); + // We hold a reference to ourselves to keep us alive until IPDL + // explictly destroys us. There may still be refs held by + // tasks, but no new ones should be added after we're + // destroyed. + mIPDLSelfRef = this; +} + +RemoteDecoderParent::~RemoteDecoderParent() { + MOZ_COUNT_DTOR(RemoteDecoderParent); +} + +void RemoteDecoderParent::Destroy() { + MOZ_ASSERT(OnManagerThread()); + mIPDLSelfRef = nullptr; +} + +mozilla::ipc::IPCResult RemoteDecoderParent::RecvInit( + InitResolver&& aResolver) { + MOZ_ASSERT(OnManagerThread()); + RefPtr<RemoteDecoderParent> self = this; + mDecoder->Init()->Then( + mManagerThread, __func__, + [self, resolver = std::move(aResolver)]( + MediaDataDecoder::InitPromise::ResolveOrRejectValue&& aValue) { + if (!self->CanRecv()) { + // The promise to the child would have already been rejected. + return; + } + if (aValue.IsReject()) { + resolver(aValue.RejectValue()); + return; + } + auto track = aValue.ResolveValue(); + MOZ_ASSERT(track == TrackInfo::kAudioTrack || + track == TrackInfo::kVideoTrack); + if (self->mDecoder) { + nsCString hardwareReason; + bool hardwareAccelerated = + self->mDecoder->IsHardwareAccelerated(hardwareReason); + resolver(InitCompletionIPDL{ + track, self->mDecoder->GetDescriptionName(), hardwareAccelerated, + hardwareReason, self->mDecoder->NeedsConversion()}); + } + }); + return IPC_OK(); +} + +void RemoteDecoderParent::DecodeNextSample( + const RefPtr<ArrayOfRemoteMediaRawData>& aData, size_t aIndex, + MediaDataDecoder::DecodedData&& aOutput, DecodeResolver&& aResolver) { + MOZ_ASSERT(OnManagerThread()); + + if (!CanRecv()) { + // Avoid unnecessarily creating shmem objects later. + return; + } + + if (!mDecoder) { + // We got shutdown or the child got destroyed. + aResolver(MediaResult(NS_ERROR_ABORT, __func__)); + return; + } + + if (aData->Count() == aIndex) { + DecodedOutputIPDL result; + MediaResult rv = ProcessDecodedData(std::move(aOutput), result); + if (NS_FAILED(rv)) { + aResolver(std::move(rv)); // Out of Memory. + } else { + aResolver(std::move(result)); + } + return; + } + + RefPtr<MediaRawData> rawData = aData->ElementAt(aIndex); + if (!rawData) { + // OOM + aResolver(MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__)); + return; + } + + mDecoder->Decode(rawData)->Then( + mManagerThread, __func__, + [self = RefPtr{this}, this, aData, aIndex, output = std::move(aOutput), + resolver = std::move(aResolver)]( + MediaDataDecoder::DecodePromise::ResolveOrRejectValue&& + aValue) mutable { + if (aValue.IsReject()) { + resolver(aValue.RejectValue()); + return; + } + + output.AppendElements(std::move(aValue.ResolveValue())); + + // Call again in case we have more data to decode. + DecodeNextSample(aData, aIndex + 1, std::move(output), + std::move(resolver)); + }); +} + +mozilla::ipc::IPCResult RemoteDecoderParent::RecvDecode( + ArrayOfRemoteMediaRawData* aData, DecodeResolver&& aResolver) { + MOZ_ASSERT(OnManagerThread()); + // If we are here, we know all previously returned DecodedOutputIPDL got + // used by the child. We can mark all previously sent ShmemBuffer as + // available again. + ReleaseAllBuffers(); + MediaDataDecoder::DecodedData output; + DecodeNextSample(aData, 0, std::move(output), std::move(aResolver)); + + return IPC_OK(); +} + +mozilla::ipc::IPCResult RemoteDecoderParent::RecvFlush( + FlushResolver&& aResolver) { + MOZ_ASSERT(OnManagerThread()); + RefPtr<RemoteDecoderParent> self = this; + mDecoder->Flush()->Then( + mManagerThread, __func__, + [self, resolver = std::move(aResolver)]( + MediaDataDecoder::FlushPromise::ResolveOrRejectValue&& aValue) { + self->ReleaseAllBuffers(); + if (aValue.IsReject()) { + resolver(aValue.RejectValue()); + } else { + resolver(MediaResult(NS_OK)); + } + }); + + return IPC_OK(); +} + +mozilla::ipc::IPCResult RemoteDecoderParent::RecvDrain( + DrainResolver&& aResolver) { + MOZ_ASSERT(OnManagerThread()); + RefPtr<RemoteDecoderParent> self = this; + mDecoder->Drain()->Then( + mManagerThread, __func__, + [self, this, resolver = std::move(aResolver)]( + MediaDataDecoder::DecodePromise::ResolveOrRejectValue&& aValue) { + ReleaseAllBuffers(); + if (!self->CanRecv()) { + // Avoid unnecessarily creating shmem objects later. + return; + } + if (aValue.IsReject()) { + resolver(aValue.RejectValue()); + return; + } + DecodedOutputIPDL output; + MediaResult rv = + ProcessDecodedData(std::move(aValue.ResolveValue()), output); + if (NS_FAILED(rv)) { + resolver(rv); + } else { + resolver(std::move(output)); + } + }); + return IPC_OK(); +} + +mozilla::ipc::IPCResult RemoteDecoderParent::RecvShutdown( + ShutdownResolver&& aResolver) { + MOZ_ASSERT(OnManagerThread()); + if (mDecoder) { + RefPtr<RemoteDecoderParent> self = this; + mDecoder->Shutdown()->Then( + mManagerThread, __func__, + [self, resolver = std::move(aResolver)]( + const ShutdownPromise::ResolveOrRejectValue& aValue) { + MOZ_ASSERT(aValue.IsResolve()); + self->ReleaseAllBuffers(); + resolver(true); + }); + } + mDecoder = nullptr; + return IPC_OK(); +} + +mozilla::ipc::IPCResult RemoteDecoderParent::RecvSetSeekThreshold( + const TimeUnit& aTime) { + MOZ_ASSERT(OnManagerThread()); + mDecoder->SetSeekThreshold(aTime); + return IPC_OK(); +} + +void RemoteDecoderParent::ActorDestroy(ActorDestroyReason aWhy) { + MOZ_ASSERT(OnManagerThread()); + if (mDecoder) { + mDecoder->Shutdown(); + mDecoder = nullptr; + } + CleanupShmemRecycleAllocator(); +} + +bool RemoteDecoderParent::OnManagerThread() { + return mParent->OnManagerThread(); +} + +} // namespace mozilla diff --git a/dom/media/ipc/RemoteDecoderParent.h b/dom/media/ipc/RemoteDecoderParent.h new file mode 100644 index 0000000000..002a93fa09 --- /dev/null +++ b/dom/media/ipc/RemoteDecoderParent.h @@ -0,0 +1,74 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef include_dom_media_ipc_RemoteDecoderParent_h +#define include_dom_media_ipc_RemoteDecoderParent_h + +#include "mozilla/PRemoteDecoderParent.h" +#include "mozilla/ShmemRecycleAllocator.h" + +namespace mozilla { + +class RemoteDecoderManagerParent; +using mozilla::ipc::IPCResult; + +class RemoteDecoderParent : public ShmemRecycleAllocator<RemoteDecoderParent>, + public PRemoteDecoderParent { + friend class PRemoteDecoderParent; + + public: + // We refcount this class since the task queue can have runnables + // that reference us. + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(RemoteDecoderParent) + + RemoteDecoderParent(RemoteDecoderManagerParent* aParent, + const CreateDecoderParams::OptionSet& aOptions, + nsISerialEventTarget* aManagerThread, + TaskQueue* aDecodeTaskQueue, + const Maybe<uint64_t>& aMediaEngineId, + Maybe<TrackingId> aTrackingId); + + void Destroy(); + + // PRemoteDecoderParent + virtual IPCResult RecvConstruct(ConstructResolver&& aResolver) = 0; + IPCResult RecvInit(InitResolver&& aResolver); + IPCResult RecvDecode(ArrayOfRemoteMediaRawData* aData, + DecodeResolver&& aResolver); + IPCResult RecvFlush(FlushResolver&& aResolver); + IPCResult RecvDrain(DrainResolver&& aResolver); + IPCResult RecvShutdown(ShutdownResolver&& aResolver); + IPCResult RecvSetSeekThreshold(const media::TimeUnit& aTime); + + void ActorDestroy(ActorDestroyReason aWhy) override; + + protected: + virtual ~RemoteDecoderParent(); + + bool OnManagerThread(); + + virtual MediaResult ProcessDecodedData(MediaDataDecoder::DecodedData&& aData, + DecodedOutputIPDL& aDecodedData) = 0; + + const RefPtr<RemoteDecoderManagerParent> mParent; + const CreateDecoderParams::OptionSet mOptions; + const RefPtr<TaskQueue> mDecodeTaskQueue; + RefPtr<MediaDataDecoder> mDecoder; + const Maybe<TrackingId> mTrackingId; + + // Only be used on Windows when the media engine playback is enabled. + const Maybe<uint64_t> mMediaEngineId; + + private: + void DecodeNextSample(const RefPtr<ArrayOfRemoteMediaRawData>& aData, + size_t aIndex, MediaDataDecoder::DecodedData&& aOutput, + DecodeResolver&& aResolver); + RefPtr<RemoteDecoderParent> mIPDLSelfRef; + const RefPtr<nsISerialEventTarget> mManagerThread; +}; + +} // namespace mozilla + +#endif // include_dom_media_ipc_RemoteDecoderParent_h diff --git a/dom/media/ipc/RemoteImageHolder.cpp b/dom/media/ipc/RemoteImageHolder.cpp new file mode 100644 index 0000000000..553e41fef6 --- /dev/null +++ b/dom/media/ipc/RemoteImageHolder.cpp @@ -0,0 +1,172 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "RemoteImageHolder.h" + +#include "GPUVideoImage.h" +#include "mozilla/PRemoteDecoderChild.h" +#include "mozilla/RemoteDecodeUtils.h" +#include "mozilla/RemoteDecoderManagerChild.h" +#include "mozilla/layers/ImageDataSerializer.h" +#include "mozilla/layers/VideoBridgeUtils.h" + +namespace mozilla { + +using namespace gfx; +using namespace layers; + +RemoteImageHolder::RemoteImageHolder() = default; +RemoteImageHolder::RemoteImageHolder(layers::IGPUVideoSurfaceManager* aManager, + layers::VideoBridgeSource aSource, + const gfx::IntSize& aSize, + const gfx::ColorDepth& aColorDepth, + const layers::SurfaceDescriptor& aSD) + : mSource(aSource), + mSize(aSize), + mColorDepth(aColorDepth), + mSD(Some(aSD)), + mManager(aManager) {} +RemoteImageHolder::RemoteImageHolder(RemoteImageHolder&& aOther) + : mSource(aOther.mSource), + mSize(aOther.mSize), + mColorDepth(aOther.mColorDepth), + mSD(std::move(aOther.mSD)), + mManager(aOther.mManager) { + aOther.mSD = Nothing(); +} + +already_AddRefed<Image> RemoteImageHolder::DeserializeImage( + layers::BufferRecycleBin* aBufferRecycleBin) { + MOZ_ASSERT(mSD && mSD->type() == SurfaceDescriptor::TSurfaceDescriptorBuffer); + const SurfaceDescriptorBuffer& sdBuffer = mSD->get_SurfaceDescriptorBuffer(); + MOZ_ASSERT(sdBuffer.desc().type() == BufferDescriptor::TYCbCrDescriptor); + if (sdBuffer.desc().type() != BufferDescriptor::TYCbCrDescriptor || + !aBufferRecycleBin) { + return nullptr; + } + const YCbCrDescriptor& descriptor = sdBuffer.desc().get_YCbCrDescriptor(); + + uint8_t* buffer = nullptr; + const MemoryOrShmem& memOrShmem = sdBuffer.data(); + switch (memOrShmem.type()) { + case MemoryOrShmem::Tuintptr_t: + buffer = reinterpret_cast<uint8_t*>(memOrShmem.get_uintptr_t()); + break; + case MemoryOrShmem::TShmem: + buffer = memOrShmem.get_Shmem().get<uint8_t>(); + break; + default: + MOZ_ASSERT(false, "Unknown MemoryOrShmem type"); + } + if (!buffer) { + return nullptr; + } + + PlanarYCbCrData pData; + pData.mYStride = descriptor.yStride(); + pData.mCbCrStride = descriptor.cbCrStride(); + // default mYSkip, mCbSkip, mCrSkip because not held in YCbCrDescriptor + pData.mYSkip = pData.mCbSkip = pData.mCrSkip = 0; + pData.mPictureRect = descriptor.display(); + pData.mStereoMode = descriptor.stereoMode(); + pData.mColorDepth = descriptor.colorDepth(); + pData.mYUVColorSpace = descriptor.yUVColorSpace(); + pData.mColorRange = descriptor.colorRange(); + pData.mChromaSubsampling = descriptor.chromaSubsampling(); + pData.mYChannel = ImageDataSerializer::GetYChannel(buffer, descriptor); + pData.mCbChannel = ImageDataSerializer::GetCbChannel(buffer, descriptor); + pData.mCrChannel = ImageDataSerializer::GetCrChannel(buffer, descriptor); + + // images coming from AOMDecoder are RecyclingPlanarYCbCrImages. + RefPtr<RecyclingPlanarYCbCrImage> image = + new RecyclingPlanarYCbCrImage(aBufferRecycleBin); + bool setData = image->CopyData(pData); + MOZ_ASSERT(setData); + + switch (memOrShmem.type()) { + case MemoryOrShmem::Tuintptr_t: + delete[] reinterpret_cast<uint8_t*>(memOrShmem.get_uintptr_t()); + break; + case MemoryOrShmem::TShmem: + // Memory buffer will be recycled by the parent automatically. + break; + default: + MOZ_ASSERT(false, "Unknown MemoryOrShmem type"); + } + + if (!setData) { + return nullptr; + } + + return image.forget(); +} + +already_AddRefed<layers::Image> RemoteImageHolder::TransferToImage( + layers::BufferRecycleBin* aBufferRecycleBin) { + if (IsEmpty()) { + return nullptr; + } + RefPtr<Image> image; + if (mSD->type() == SurfaceDescriptor::TSurfaceDescriptorBuffer) { + image = DeserializeImage(aBufferRecycleBin); + } else { + // The Image here creates a TextureData object that takes ownership + // of the SurfaceDescriptor, and is responsible for making sure that + // it gets deallocated. + SurfaceDescriptorRemoteDecoder remoteSD = + static_cast<const SurfaceDescriptorGPUVideo&>(*mSD); + remoteSD.source() = Some(mSource); + image = new GPUVideoImage(mManager, remoteSD, mSize, mColorDepth); + } + mSD = Nothing(); + mManager = nullptr; + + return image.forget(); +} + +RemoteImageHolder::~RemoteImageHolder() { + // GPU Images are held by the RemoteDecoderManagerParent, we didn't get to use + // this image holder (the decoder could have been flushed). We don't need to + // worry about Shmem based image as the Shmem will be automatically re-used + // once the decoder is used again. + if (!IsEmpty() && mManager && + mSD->type() != SurfaceDescriptor::TSurfaceDescriptorBuffer) { + SurfaceDescriptorRemoteDecoder remoteSD = + static_cast<const SurfaceDescriptorGPUVideo&>(*mSD); + mManager->DeallocateSurfaceDescriptor(remoteSD); + } +} + +/* static */ void ipc::IPDLParamTraits<RemoteImageHolder>::Write( + IPC::MessageWriter* aWriter, ipc::IProtocol* aActor, + RemoteImageHolder&& aParam) { + WriteIPDLParam(aWriter, aActor, aParam.mSource); + WriteIPDLParam(aWriter, aActor, aParam.mSize); + WriteIPDLParam(aWriter, aActor, aParam.mColorDepth); + WriteIPDLParam(aWriter, aActor, aParam.mSD); + // Empty this holder. + aParam.mSD = Nothing(); + aParam.mManager = nullptr; +} + +/* static */ bool ipc::IPDLParamTraits<RemoteImageHolder>::Read( + IPC::MessageReader* aReader, ipc::IProtocol* aActor, + RemoteImageHolder* aResult) { + if (!ReadIPDLParam(aReader, aActor, &aResult->mSource) || + !ReadIPDLParam(aReader, aActor, &aResult->mSize) || + !ReadIPDLParam(aReader, aActor, &aResult->mColorDepth) || + !ReadIPDLParam(aReader, aActor, &aResult->mSD)) { + return false; + } + + if (!aResult->IsEmpty()) { + aResult->mManager = RemoteDecoderManagerChild::GetSingleton( + GetRemoteDecodeInFromVideoBridgeSource(aResult->mSource)); + } + return true; +} + +} // namespace mozilla diff --git a/dom/media/ipc/RemoteImageHolder.h b/dom/media/ipc/RemoteImageHolder.h new file mode 100644 index 0000000000..b83293d0ed --- /dev/null +++ b/dom/media/ipc/RemoteImageHolder.h @@ -0,0 +1,66 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_media_RemoteImageHolder_h +#define mozilla_dom_media_RemoteImageHolder_h + +#include "MediaData.h" +#include "ipc/IPCMessageUtils.h" +#include "mozilla/Maybe.h" +#include "mozilla/RefPtr.h" +#include "mozilla/layers/LayersSurfaces.h" +#include "mozilla/layers/VideoBridgeUtils.h" + +namespace mozilla { +namespace layers { +class BufferRecycleBin; +class IGPUVideoSurfaceManager; +class SurfaceDescriptor; +} // namespace layers +class RemoteImageHolder final { + friend struct ipc::IPDLParamTraits<RemoteImageHolder>; + + public: + RemoteImageHolder(); + RemoteImageHolder(layers::IGPUVideoSurfaceManager* aManager, + layers::VideoBridgeSource aSource, + const gfx::IntSize& aSize, + const gfx::ColorDepth& aColorDepth, + const layers::SurfaceDescriptor& aSD); + RemoteImageHolder(RemoteImageHolder&& aOther); + // Ensure we never copy this object. + RemoteImageHolder(const RemoteImageHolder& aOther) = delete; + RemoteImageHolder& operator=(const RemoteImageHolder& aOther) = delete; + ~RemoteImageHolder(); + + bool IsEmpty() const { return mSD.isNothing(); } + // Move content of RemoteImageHolder into a usable Image. Ownership is + // transfered to that Image. + already_AddRefed<layers::Image> TransferToImage( + layers::BufferRecycleBin* aBufferRecycleBin = nullptr); + + private: + already_AddRefed<layers::Image> DeserializeImage( + layers::BufferRecycleBin* aBufferRecycleBin); + // We need a default for the default constructor, never used in practice. + layers::VideoBridgeSource mSource = layers::VideoBridgeSource::GpuProcess; + gfx::IntSize mSize; + gfx::ColorDepth mColorDepth = gfx::ColorDepth::COLOR_8; + Maybe<layers::SurfaceDescriptor> mSD; + RefPtr<layers::IGPUVideoSurfaceManager> mManager; +}; + +template <> +struct ipc::IPDLParamTraits<RemoteImageHolder> { + static void Write(IPC::MessageWriter* aWriter, IProtocol* aActor, + RemoteImageHolder&& aParam); + static bool Read(IPC::MessageReader* aReader, IProtocol* aActor, + RemoteImageHolder* aResult); +}; + +} // namespace mozilla + +#endif // mozilla_dom_media_RemoteImageHolder_h diff --git a/dom/media/ipc/RemoteMediaData.cpp b/dom/media/ipc/RemoteMediaData.cpp new file mode 100644 index 0000000000..e7ab4aeefe --- /dev/null +++ b/dom/media/ipc/RemoteMediaData.cpp @@ -0,0 +1,354 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "RemoteMediaData.h" + +#include "PerformanceRecorder.h" +#include "mozilla/CheckedInt.h" +#include "mozilla/dom/MediaIPCUtils.h" +#include "mozilla/ipc/Shmem.h" + +namespace mozilla { + +bool RemoteArrayOfByteBuffer::AllocateShmem( + size_t aSize, std::function<ShmemBuffer(size_t)>& aAllocator) { + ShmemBuffer buffer = aAllocator(aSize); + if (!buffer.Valid()) { + return false; + } + mBuffers.emplace(std::move(buffer.Get())); + return true; +} + +uint8_t* RemoteArrayOfByteBuffer::BuffersStartAddress() const { + MOZ_ASSERT(mBuffers); + return mBuffers->get<uint8_t>(); +} + +bool RemoteArrayOfByteBuffer::Check(size_t aOffset, size_t aSizeInBytes) const { + return mBuffers && mBuffers->IsReadable() && + detail::IsAddValid(aOffset, aSizeInBytes) && + aOffset + aSizeInBytes <= mBuffers->Size<uint8_t>(); +} + +void RemoteArrayOfByteBuffer::Write(size_t aOffset, const void* aSourceAddr, + size_t aSizeInBytes) { + if (!aSizeInBytes) { + return; + } + MOZ_DIAGNOSTIC_ASSERT(Check(aOffset, aSizeInBytes), + "Allocated Shmem is too small"); + memcpy(BuffersStartAddress() + aOffset, aSourceAddr, aSizeInBytes); +} + +RemoteArrayOfByteBuffer::RemoteArrayOfByteBuffer() = default; + +RemoteArrayOfByteBuffer::RemoteArrayOfByteBuffer( + const nsTArray<RefPtr<MediaByteBuffer>>& aArray, + std::function<ShmemBuffer(size_t)>& aAllocator) { + // Determine the total size we will need for this object. + size_t totalSize = 0; + for (const auto& buffer : aArray) { + if (buffer) { + totalSize += buffer->Length(); + } + } + if (totalSize) { + if (!AllocateShmem(totalSize, aAllocator)) { + return; + } + } + size_t offset = 0; + for (const auto& buffer : aArray) { + size_t sizeBuffer = buffer ? buffer->Length() : 0; + if (totalSize && sizeBuffer) { + Write(offset, buffer->Elements(), sizeBuffer); + } + mOffsets.AppendElement(OffsetEntry{offset, sizeBuffer}); + offset += sizeBuffer; + } + mIsValid = true; +} + +RemoteArrayOfByteBuffer& RemoteArrayOfByteBuffer::operator=( + RemoteArrayOfByteBuffer&& aOther) noexcept { + mIsValid = aOther.mIsValid; + mBuffers = std::move(aOther.mBuffers); + mOffsets = std::move(aOther.mOffsets); + aOther.mIsValid = false; + return *this; +} + +RemoteArrayOfByteBuffer::~RemoteArrayOfByteBuffer() = default; + +already_AddRefed<MediaByteBuffer> RemoteArrayOfByteBuffer::MediaByteBufferAt( + size_t aIndex) const { + MOZ_ASSERT(aIndex < Count()); + const OffsetEntry& entry = mOffsets[aIndex]; + if (!mBuffers || !Get<1>(entry)) { + // It's an empty one. + return nullptr; + } + size_t entrySize = Get<1>(entry); + if (!Check(Get<0>(entry), entrySize)) { + // This Shmem is corrupted and can't contain the data we are about to + // retrieve. We return an empty array instead of asserting to allow for + // recovery. + return nullptr; + } + RefPtr<MediaByteBuffer> buffer = new MediaByteBuffer(entrySize); + buffer->SetLength(entrySize); + memcpy(buffer->Elements(), mBuffers->get<uint8_t>() + Get<0>(entry), + entrySize); + return buffer.forget(); +} + +/*static */ void ipc::IPDLParamTraits<RemoteArrayOfByteBuffer>::Write( + IPC::MessageWriter* aWriter, ipc::IProtocol* aActor, + const RemoteArrayOfByteBuffer& aVar) { + WriteIPDLParam(aWriter, aActor, aVar.mIsValid); + // We need the following gymnastic as the Shmem transfered over IPC will be + // revoked. We must create a temporary one instead so that it can be recycled + // later back into the original ShmemPool. + if (aVar.mBuffers) { + WriteIPDLParam(aWriter, aActor, Some(ipc::Shmem(*aVar.mBuffers))); + } else { + WriteIPDLParam(aWriter, aActor, Maybe<ipc::Shmem>()); + } + WriteIPDLParam(aWriter, aActor, aVar.mOffsets); +} + +/* static */ bool ipc::IPDLParamTraits<RemoteArrayOfByteBuffer>::Read( + IPC::MessageReader* aReader, mozilla::ipc::IProtocol* aActor, + RemoteArrayOfByteBuffer* aVar) { + return ReadIPDLParam(aReader, aActor, &aVar->mIsValid) && + ReadIPDLParam(aReader, aActor, &aVar->mBuffers) && + ReadIPDLParam(aReader, aActor, &aVar->mOffsets); +} + +bool ArrayOfRemoteMediaRawData::Fill( + const nsTArray<RefPtr<MediaRawData>>& aData, + std::function<ShmemBuffer(size_t)>&& aAllocator) { + nsTArray<AlignedByteBuffer> dataBuffers(aData.Length()); + nsTArray<AlignedByteBuffer> alphaBuffers(aData.Length()); + nsTArray<RefPtr<MediaByteBuffer>> extraDataBuffers(aData.Length()); + int32_t height = 0; + for (auto&& entry : aData) { + dataBuffers.AppendElement(std::move(entry->mBuffer)); + alphaBuffers.AppendElement(std::move(entry->mAlphaBuffer)); + extraDataBuffers.AppendElement(std::move(entry->mExtraData)); + if (auto&& info = entry->mTrackInfo; info && info->GetAsVideoInfo()) { + height = info->GetAsVideoInfo()->mImage.height; + } + mSamples.AppendElement(RemoteMediaRawData{ + MediaDataIPDL(entry->mOffset, entry->mTime, entry->mTimecode, + entry->mDuration, entry->mKeyframe), + entry->mEOS, height, entry->mDiscardPadding, + entry->mOriginalPresentationWindow}); + } + PerformanceRecorder<PlaybackStage> perfRecorder(MediaStage::CopyDemuxedData, + height); + mBuffers = RemoteArrayOfByteBuffer(dataBuffers, aAllocator); + if (!mBuffers.IsValid()) { + return false; + } + mAlphaBuffers = RemoteArrayOfByteBuffer(alphaBuffers, aAllocator); + if (!mAlphaBuffers.IsValid()) { + return false; + } + mExtraDatas = RemoteArrayOfByteBuffer(extraDataBuffers, aAllocator); + if (!mExtraDatas.IsValid()) { + return false; + } + perfRecorder.Record(); + return true; +} + +already_AddRefed<MediaRawData> ArrayOfRemoteMediaRawData::ElementAt( + size_t aIndex) const { + if (!IsValid()) { + return nullptr; + } + MOZ_ASSERT(aIndex < Count()); + MOZ_DIAGNOSTIC_ASSERT(mBuffers.Count() == Count() && + mAlphaBuffers.Count() == Count() && + mExtraDatas.Count() == Count(), + "Something ain't right here"); + const auto& sample = mSamples[aIndex]; + PerformanceRecorder<PlaybackStage> perfRecorder(MediaStage::CopyDemuxedData, + sample.mHeight); + AlignedByteBuffer data = mBuffers.AlignedBufferAt<uint8_t>(aIndex); + if (mBuffers.SizeAt(aIndex) && !data) { + // OOM + return nullptr; + } + AlignedByteBuffer alphaData = mAlphaBuffers.AlignedBufferAt<uint8_t>(aIndex); + if (mAlphaBuffers.SizeAt(aIndex) && !alphaData) { + // OOM + return nullptr; + } + RefPtr<MediaRawData> rawData; + if (mAlphaBuffers.SizeAt(aIndex)) { + rawData = new MediaRawData(std::move(data), std::move(alphaData)); + } else { + rawData = new MediaRawData(std::move(data)); + } + rawData->mOffset = sample.mBase.offset(); + rawData->mTime = sample.mBase.time(); + rawData->mTimecode = sample.mBase.timecode(); + rawData->mDuration = sample.mBase.duration(); + rawData->mKeyframe = sample.mBase.keyframe(); + rawData->mEOS = sample.mEOS; + rawData->mDiscardPadding = sample.mDiscardPadding; + rawData->mExtraData = mExtraDatas.MediaByteBufferAt(aIndex); + perfRecorder.Record(); + return rawData.forget(); +} + +/*static */ void ipc::IPDLParamTraits<ArrayOfRemoteMediaRawData*>::Write( + IPC::MessageWriter* aWriter, ipc::IProtocol* aActor, + ArrayOfRemoteMediaRawData* aVar) { + WriteIPDLParam(aWriter, aActor, std::move(aVar->mSamples)); + WriteIPDLParam(aWriter, aActor, std::move(aVar->mBuffers)); + WriteIPDLParam(aWriter, aActor, std::move(aVar->mAlphaBuffers)); + WriteIPDLParam(aWriter, aActor, std::move(aVar->mExtraDatas)); +} + +/* static */ bool ipc::IPDLParamTraits<ArrayOfRemoteMediaRawData*>::Read( + IPC::MessageReader* aReader, mozilla::ipc::IProtocol* aActor, + RefPtr<ArrayOfRemoteMediaRawData>* aVar) { + auto array = MakeRefPtr<ArrayOfRemoteMediaRawData>(); + if (!ReadIPDLParam(aReader, aActor, &array->mSamples) || + !ReadIPDLParam(aReader, aActor, &array->mBuffers) || + !ReadIPDLParam(aReader, aActor, &array->mAlphaBuffers) || + !ReadIPDLParam(aReader, aActor, &array->mExtraDatas)) { + return false; + } + *aVar = std::move(array); + return true; +} + +/* static */ void +ipc::IPDLParamTraits<ArrayOfRemoteMediaRawData::RemoteMediaRawData>::Write( + IPC::MessageWriter* aWriter, ipc::IProtocol* aActor, + const paramType& aVar) { + WriteIPDLParam(aWriter, aActor, aVar.mBase); + WriteIPDLParam(aWriter, aActor, aVar.mEOS); + WriteIPDLParam(aWriter, aActor, aVar.mHeight); + WriteIPDLParam(aWriter, aActor, aVar.mDiscardPadding); + WriteIPDLParam(aWriter, aActor, aVar.mOriginalPresentationWindow); +} + +/* static */ bool +ipc::IPDLParamTraits<ArrayOfRemoteMediaRawData::RemoteMediaRawData>::Read( + IPC::MessageReader* aReader, ipc::IProtocol* aActor, paramType* aVar) { + MediaDataIPDL mBase; + return ReadIPDLParam(aReader, aActor, &aVar->mBase) && + ReadIPDLParam(aReader, aActor, &aVar->mEOS) && + ReadIPDLParam(aReader, aActor, &aVar->mHeight) && + ReadIPDLParam(aReader, aActor, &aVar->mDiscardPadding) && + ReadIPDLParam(aReader, aActor, &aVar->mOriginalPresentationWindow); +}; + +bool ArrayOfRemoteAudioData::Fill( + const nsTArray<RefPtr<AudioData>>& aData, + std::function<ShmemBuffer(size_t)>&& aAllocator) { + nsTArray<AlignedAudioBuffer> dataBuffers(aData.Length()); + for (auto&& entry : aData) { + dataBuffers.AppendElement(std::move(entry->mAudioData)); + mSamples.AppendElement(RemoteAudioData{ + MediaDataIPDL(entry->mOffset, entry->mTime, entry->mTimecode, + entry->mDuration, entry->mKeyframe), + entry->mChannels, entry->mRate, uint32_t(entry->mChannelMap), + entry->mOriginalTime, entry->mTrimWindow, entry->mFrames, + entry->mDataOffset}); + } + mBuffers = RemoteArrayOfByteBuffer(dataBuffers, aAllocator); + if (!mBuffers.IsValid()) { + return false; + } + return true; +} + +already_AddRefed<AudioData> ArrayOfRemoteAudioData::ElementAt( + size_t aIndex) const { + if (!IsValid()) { + return nullptr; + } + MOZ_ASSERT(aIndex < Count()); + MOZ_DIAGNOSTIC_ASSERT(mBuffers.Count() == Count(), + "Something ain't right here"); + const auto& sample = mSamples[aIndex]; + AlignedAudioBuffer data = mBuffers.AlignedBufferAt<AudioDataValue>(aIndex); + if (mBuffers.SizeAt(aIndex) && !data) { + // OOM + return nullptr; + } + auto audioData = MakeRefPtr<AudioData>( + sample.mBase.offset(), sample.mBase.time(), std::move(data), + sample.mChannels, sample.mRate, sample.mChannelMap); + // An AudioData's duration is set at construction time based on the size of + // the provided buffer. However, if a trim window is set, this value will be + // incorrect. We have to re-set it to what it actually was. + audioData->mDuration = sample.mBase.duration(); + audioData->mOriginalTime = sample.mOriginalTime; + audioData->mTrimWindow = sample.mTrimWindow; + audioData->mFrames = sample.mFrames; + audioData->mDataOffset = sample.mDataOffset; + return audioData.forget(); +} + +/*static */ void ipc::IPDLParamTraits<ArrayOfRemoteAudioData*>::Write( + IPC::MessageWriter* aWriter, ipc::IProtocol* aActor, + ArrayOfRemoteAudioData* aVar) { + WriteIPDLParam(aWriter, aActor, std::move(aVar->mSamples)); + WriteIPDLParam(aWriter, aActor, std::move(aVar->mBuffers)); +} + +/* static */ bool ipc::IPDLParamTraits<ArrayOfRemoteAudioData*>::Read( + IPC::MessageReader* aReader, mozilla::ipc::IProtocol* aActor, + RefPtr<ArrayOfRemoteAudioData>* aVar) { + auto array = MakeRefPtr<ArrayOfRemoteAudioData>(); + if (!ReadIPDLParam(aReader, aActor, &array->mSamples) || + !ReadIPDLParam(aReader, aActor, &array->mBuffers)) { + return false; + } + *aVar = std::move(array); + return true; +} + +/* static */ void +ipc::IPDLParamTraits<ArrayOfRemoteAudioData::RemoteAudioData>::Write( + IPC::MessageWriter* aWriter, ipc::IProtocol* aActor, + const paramType& aVar) { + WriteIPDLParam(aWriter, aActor, aVar.mBase); + WriteIPDLParam(aWriter, aActor, aVar.mChannels); + WriteIPDLParam(aWriter, aActor, aVar.mRate); + WriteIPDLParam(aWriter, aActor, aVar.mChannelMap); + WriteIPDLParam(aWriter, aActor, aVar.mOriginalTime); + WriteIPDLParam(aWriter, aActor, aVar.mTrimWindow); + WriteIPDLParam(aWriter, aActor, aVar.mFrames); + WriteIPDLParam(aWriter, aActor, aVar.mDataOffset); +} + +/* static */ bool +ipc::IPDLParamTraits<ArrayOfRemoteAudioData::RemoteAudioData>::Read( + IPC::MessageReader* aReader, ipc::IProtocol* aActor, paramType* aVar) { + MediaDataIPDL mBase; + if (!ReadIPDLParam(aReader, aActor, &aVar->mBase) || + !ReadIPDLParam(aReader, aActor, &aVar->mChannels) || + !ReadIPDLParam(aReader, aActor, &aVar->mRate) || + !ReadIPDLParam(aReader, aActor, &aVar->mChannelMap) || + !ReadIPDLParam(aReader, aActor, &aVar->mOriginalTime) || + !ReadIPDLParam(aReader, aActor, &aVar->mTrimWindow) || + !ReadIPDLParam(aReader, aActor, &aVar->mFrames) || + !ReadIPDLParam(aReader, aActor, &aVar->mDataOffset)) { + return false; + } + return true; +}; + +} // namespace mozilla diff --git a/dom/media/ipc/RemoteMediaData.h b/dom/media/ipc/RemoteMediaData.h new file mode 100644 index 0000000000..25b07a5ce2 --- /dev/null +++ b/dom/media/ipc/RemoteMediaData.h @@ -0,0 +1,394 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_dom_media_ipc_RemoteMediaData_h +#define mozilla_dom_media_ipc_RemoteMediaData_h + +#include <functional> + +#include "MediaData.h" +#include "PlatformDecoderModule.h" +#include "ipc/IPCMessageUtils.h" +#include "mozilla/GfxMessageUtils.h" +#include "mozilla/PMediaDecoderParams.h" +#include "mozilla/RemoteImageHolder.h" +#include "mozilla/ShmemPool.h" +#include "mozilla/gfx/Rect.h" + +namespace mozilla { + +class ShmemPool; + +namespace ipc { +class IProtocol; +class Shmem; +} // namespace ipc + +//----------------------------------------------------------------------------- +// Declaration of the IPDL type |struct RemoteVideoData| +// +// We can't use the generated binding in order to use move semantics properly +// (see bug 1664362) +class RemoteVideoData final { + private: + typedef mozilla::gfx::IntSize IntSize; + + public: + RemoteVideoData() = default; + + RemoteVideoData(const MediaDataIPDL& aBase, const IntSize& aDisplay, + RemoteImageHolder&& aImage, int32_t aFrameID) + : mBase(aBase), + mDisplay(aDisplay), + mImage(std::move(aImage)), + mFrameID(aFrameID) {} + + // This is equivalent to the old RemoteVideoDataIPDL object and is similar to + // the RemoteAudioDataIPDL object. To ensure style consistency we use the IPDL + // naming convention here. + MediaDataIPDL& base() { return mBase; } + const MediaDataIPDL& base() const { return mBase; } + + IntSize& display() { return mDisplay; } + const IntSize& display() const { return mDisplay; } + + RemoteImageHolder& image() { return mImage; } + const RemoteImageHolder& image() const { return mImage; } + + int32_t& frameID() { return mFrameID; } + const int32_t& frameID() const { return mFrameID; } + + private: + friend struct ipc::IPDLParamTraits<RemoteVideoData>; + MediaDataIPDL mBase; + IntSize mDisplay; + RemoteImageHolder mImage; + int32_t mFrameID; +}; + +// Until bug 1572054 is resolved, we can't move our objects when using IPDL's +// union or array. They are always copied. So we make the class refcounted to +// and always pass it by pointed to bypass the problem for now. +class ArrayOfRemoteVideoData final { + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ArrayOfRemoteVideoData) + public: + ArrayOfRemoteVideoData() = default; + ArrayOfRemoteVideoData(ArrayOfRemoteVideoData&& aOther) + : mArray(std::move(aOther.mArray)) {} + explicit ArrayOfRemoteVideoData(nsTArray<RemoteVideoData>&& aOther) + : mArray(std::move(aOther)) {} + ArrayOfRemoteVideoData(const ArrayOfRemoteVideoData& aOther) { + MOZ_CRASH("Should never be used but declared by generated IPDL binding"); + } + ArrayOfRemoteVideoData& operator=(ArrayOfRemoteVideoData&& aOther) noexcept { + if (this != &aOther) { + mArray = std::move(aOther.mArray); + } + return *this; + } + ArrayOfRemoteVideoData& operator=(nsTArray<RemoteVideoData>&& aOther) { + mArray = std::move(aOther); + return *this; + } + + void AppendElements(nsTArray<RemoteVideoData>&& aOther) { + mArray.AppendElements(std::move(aOther)); + } + void Append(RemoteVideoData&& aVideo) { + mArray.AppendElement(std::move(aVideo)); + } + const nsTArray<RemoteVideoData>& Array() const { return mArray; } + nsTArray<RemoteVideoData>& Array() { return mArray; } + + private: + ~ArrayOfRemoteVideoData() = default; + friend struct ipc::IPDLParamTraits<mozilla::ArrayOfRemoteVideoData*>; + nsTArray<RemoteVideoData> mArray; +}; + +/* The class will pack either an array of AlignedBuffer or MediaByteBuffer + * into a single Shmem objects. */ +class RemoteArrayOfByteBuffer { + public: + RemoteArrayOfByteBuffer(); + template <typename Type> + RemoteArrayOfByteBuffer(const nsTArray<AlignedBuffer<Type>>& aArray, + std::function<ShmemBuffer(size_t)>& aAllocator) { + // Determine the total size we will need for this object. + size_t totalSize = 0; + for (auto& buffer : aArray) { + totalSize += buffer.Size(); + } + if (totalSize) { + if (!AllocateShmem(totalSize, aAllocator)) { + return; + } + } + size_t offset = 0; + for (auto& buffer : aArray) { + if (totalSize && buffer && buffer.Size()) { + Write(offset, buffer.Data(), buffer.Size()); + } + mOffsets.AppendElement(OffsetEntry{offset, buffer.Size()}); + offset += buffer.Size(); + } + mIsValid = true; + } + + RemoteArrayOfByteBuffer(const nsTArray<RefPtr<MediaByteBuffer>>& aArray, + std::function<ShmemBuffer(size_t)>& aAllocator); + RemoteArrayOfByteBuffer& operator=(RemoteArrayOfByteBuffer&& aOther) noexcept; + + // Return the packed aIndexth buffer as an AlignedByteBuffer. + // The operation is fallible should an out of memory be encountered. The + // result should be tested accordingly. + template <typename Type> + AlignedBuffer<Type> AlignedBufferAt(size_t aIndex) const { + MOZ_ASSERT(aIndex < Count()); + const OffsetEntry& entry = mOffsets[aIndex]; + size_t entrySize = Get<1>(entry); + if (!mBuffers || !entrySize) { + // It's an empty one. + return AlignedBuffer<Type>(); + } + if (!Check(Get<0>(entry), entrySize)) { + // This Shmem is corrupted and can't contain the data we are about to + // retrieve. We return an empty array instead of asserting to allow for + // recovery. + return AlignedBuffer<Type>(); + } + if (0 != entrySize % sizeof(Type)) { + // There's an error, that entry can't represent this data. + return AlignedBuffer<Type>(); + } + return AlignedBuffer<Type>( + reinterpret_cast<Type*>(BuffersStartAddress() + Get<0>(entry)), + entrySize / sizeof(Type)); + } + + // Return the packed aIndexth buffer as aMediaByteBuffer. + // Will return nullptr if the packed buffer was originally empty. + already_AddRefed<MediaByteBuffer> MediaByteBufferAt(size_t aIndex) const; + // Return the size of the aIndexth buffer. + size_t SizeAt(size_t aIndex) const { return Get<1>(mOffsets[aIndex]); } + // Return false if an out of memory error was encountered during construction. + bool IsValid() const { return mIsValid; }; + // Return the number of buffers packed into this entity. + size_t Count() const { return mOffsets.Length(); } + virtual ~RemoteArrayOfByteBuffer(); + + private: + friend struct ipc::IPDLParamTraits<RemoteArrayOfByteBuffer>; + // Allocate shmem, false if an error occurred. + bool AllocateShmem(size_t aSize, + std::function<ShmemBuffer(size_t)>& aAllocator); + // The starting address of the Shmem + uint8_t* BuffersStartAddress() const; + // Check that the allocated Shmem can contain such range. + bool Check(size_t aOffset, size_t aSizeInBytes) const; + void Write(size_t aOffset, const void* aSourceAddr, size_t aSizeInBytes); + // Set to false is the buffer isn't initialized yet or a memory error occurred + // during construction. + bool mIsValid = false; + // The packed data. The Maybe will be empty if all buffers packed were + // orignally empty. + Maybe<ipc::Shmem> mBuffers; + // The offset to the start of the individual buffer and its size (all in + // bytes) + typedef Tuple<size_t, size_t> OffsetEntry; + nsTArray<OffsetEntry> mOffsets; +}; + +/* The class will pack an array of MediaRawData using at most three Shmem + * objects. Under the most common scenaria, only two Shmems will be used as + * there are few videos with an alpha channel in the wild. + * We unfortunately can't populate the array at construction nor present an + * interface similar to an actual nsTArray or the ArrayOfRemoteVideoData above + * as currently IPC serialization is always non-fallible. So we must create the + * object first, fill it to determine if we ran out of memory and then send the + * object over IPC. + */ +class ArrayOfRemoteMediaRawData { + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ArrayOfRemoteMediaRawData) + public: + // Fill the content, return false if an OOM occurred. + bool Fill(const nsTArray<RefPtr<MediaRawData>>& aData, + std::function<ShmemBuffer(size_t)>&& aAllocator); + + // Return the aIndexth MediaRawData or nullptr if a memory error occurred. + already_AddRefed<MediaRawData> ElementAt(size_t aIndex) const; + + // Return the number of MediaRawData stored in this container. + size_t Count() const { return mSamples.Length(); } + bool IsEmpty() const { return Count() == 0; } + bool IsValid() const { + return mBuffers.IsValid() && mAlphaBuffers.IsValid() && + mExtraDatas.IsValid(); + } + + struct RemoteMediaRawData { + MediaDataIPDL mBase; + bool mEOS; + // This will be zero for audio. + int32_t mHeight; + uint32_t mDiscardPadding; + Maybe<media::TimeInterval> mOriginalPresentationWindow; + }; + + private: + friend struct ipc::IPDLParamTraits<ArrayOfRemoteMediaRawData*>; + virtual ~ArrayOfRemoteMediaRawData() = default; + + nsTArray<RemoteMediaRawData> mSamples; + RemoteArrayOfByteBuffer mBuffers; + RemoteArrayOfByteBuffer mAlphaBuffers; + RemoteArrayOfByteBuffer mExtraDatas; +}; + +/* The class will pack an array of MediaAudioData using at most a single Shmem + * objects. + * We unfortunately can't populate the array at construction nor present an + * interface similar to an actual nsTArray or the ArrayOfRemoteVideoData above + * as currently IPC serialization is always non-fallible. So we must create the + * object first, fill it to determine if we ran out of memory and then send the + * object over IPC. + */ +class ArrayOfRemoteAudioData final { + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(ArrayOfRemoteAudioData) + public: + // Fill the content, return false if an OOM occurred. + bool Fill(const nsTArray<RefPtr<AudioData>>& aData, + std::function<ShmemBuffer(size_t)>&& aAllocator); + + // Return the aIndexth MediaRawData or nullptr if a memory error occurred. + already_AddRefed<AudioData> ElementAt(size_t aIndex) const; + + // Return the number of MediaRawData stored in this container. + size_t Count() const { return mSamples.Length(); } + bool IsEmpty() const { return Count() == 0; } + bool IsValid() const { return mBuffers.IsValid(); } + + struct RemoteAudioData { + friend struct ipc::IPDLParamTraits<RemoteVideoData>; + MediaDataIPDL mBase; + uint32_t mChannels; + uint32_t mRate; + uint32_t mChannelMap; + media::TimeUnit mOriginalTime; + Maybe<media::TimeInterval> mTrimWindow; + uint32_t mFrames; + size_t mDataOffset; + }; + + private: + friend struct ipc::IPDLParamTraits<ArrayOfRemoteAudioData*>; + ~ArrayOfRemoteAudioData() = default; + + nsTArray<RemoteAudioData> mSamples; + RemoteArrayOfByteBuffer mBuffers; +}; + +namespace ipc { + +template <> +struct IPDLParamTraits<RemoteVideoData> { + typedef RemoteVideoData paramType; + static void Write(IPC::MessageWriter* aWriter, ipc::IProtocol* aActor, + paramType&& aVar) { + WriteIPDLParam(aWriter, aActor, std::move(aVar.mBase)); + WriteIPDLParam(aWriter, aActor, std::move(aVar.mDisplay)); + WriteIPDLParam(aWriter, aActor, std::move(aVar.mImage)); + aWriter->WriteBytes(&aVar.mFrameID, 4); + } + + static bool Read(IPC::MessageReader* aReader, mozilla::ipc::IProtocol* aActor, + paramType* aVar) { + if (!ReadIPDLParam(aReader, aActor, &aVar->mBase) || + !ReadIPDLParam(aReader, aActor, &aVar->mDisplay) || + !ReadIPDLParam(aReader, aActor, &aVar->mImage) || + !aReader->ReadBytesInto(&aVar->mFrameID, 4)) { + return false; + } + return true; + } +}; + +template <> +struct IPDLParamTraits<ArrayOfRemoteVideoData*> { + typedef ArrayOfRemoteVideoData paramType; + static void Write(IPC::MessageWriter* aWriter, + mozilla::ipc::IProtocol* aActor, paramType* aVar) { + WriteIPDLParam(aWriter, aActor, std::move(aVar->mArray)); + } + + static bool Read(IPC::MessageReader* aReader, ipc::IProtocol* aActor, + RefPtr<paramType>* aVar) { + nsTArray<RemoteVideoData> array; + if (!ReadIPDLParam(aReader, aActor, &array)) { + return false; + } + auto results = MakeRefPtr<ArrayOfRemoteVideoData>(std::move(array)); + *aVar = std::move(results); + return true; + } +}; + +template <> +struct IPDLParamTraits<RemoteArrayOfByteBuffer> { + typedef RemoteArrayOfByteBuffer paramType; + // We do not want to move the RemoteArrayOfByteBuffer as we want to recycle + // the shmem it contains for another time. + static void Write(IPC::MessageWriter* aWriter, ipc::IProtocol* aActor, + const paramType& aVar); + + static bool Read(IPC::MessageReader* aReader, ipc::IProtocol* aActor, + paramType* aVar); +}; + +template <> +struct IPDLParamTraits<ArrayOfRemoteMediaRawData::RemoteMediaRawData> { + typedef ArrayOfRemoteMediaRawData::RemoteMediaRawData paramType; + static void Write(IPC::MessageWriter* aWriter, ipc::IProtocol* aActor, + const paramType& aVar); + + static bool Read(IPC::MessageReader* aReader, ipc::IProtocol* aActor, + paramType* aVar); +}; + +template <> +struct IPDLParamTraits<ArrayOfRemoteMediaRawData*> { + typedef ArrayOfRemoteMediaRawData paramType; + static void Write(IPC::MessageWriter* aWriter, ipc::IProtocol* aActor, + paramType* aVar); + + static bool Read(IPC::MessageReader* aReader, ipc::IProtocol* aActor, + RefPtr<paramType>* aVar); +}; + +template <> +struct IPDLParamTraits<ArrayOfRemoteAudioData::RemoteAudioData> { + typedef ArrayOfRemoteAudioData::RemoteAudioData paramType; + static void Write(IPC::MessageWriter* aWriter, ipc::IProtocol* aActor, + const paramType& aVar); + + static bool Read(IPC::MessageReader* aReader, ipc::IProtocol* aActor, + paramType* aVar); +}; + +template <> +struct IPDLParamTraits<ArrayOfRemoteAudioData*> { + typedef ArrayOfRemoteAudioData paramType; + static void Write(IPC::MessageWriter* aWriter, ipc::IProtocol* aActor, + paramType* aVar); + + static bool Read(IPC::MessageReader* aReader, ipc::IProtocol* aActor, + RefPtr<paramType>* aVar); +}; +} // namespace ipc + +} // namespace mozilla + +#endif // mozilla_dom_media_ipc_RemoteMediaData_h diff --git a/dom/media/ipc/RemoteMediaDataDecoder.cpp b/dom/media/ipc/RemoteMediaDataDecoder.cpp new file mode 100644 index 0000000000..740f38721c --- /dev/null +++ b/dom/media/ipc/RemoteMediaDataDecoder.cpp @@ -0,0 +1,140 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "RemoteMediaDataDecoder.h" + +#include "RemoteDecoderChild.h" +#include "RemoteDecoderManagerChild.h" + +namespace mozilla { + +RemoteMediaDataDecoder::RemoteMediaDataDecoder(RemoteDecoderChild* aChild) + : mChild(aChild) {} + +RemoteMediaDataDecoder::~RemoteMediaDataDecoder() { + if (mChild) { + // Shutdown didn't get called. This can happen if the creation of the + // decoder got interrupted while pending. + nsCOMPtr<nsISerialEventTarget> thread = + RemoteDecoderManagerChild::GetManagerThread(); + MOZ_ASSERT(thread); + thread->Dispatch(NS_NewRunnableFunction( + "RemoteMediaDataDecoderShutdown", [child = std::move(mChild), thread] { + child->Shutdown()->Then( + thread, __func__, + [child](const ShutdownPromise::ResolveOrRejectValue& aValue) { + child->DestroyIPDL(); + }); + })); + } +} + +RefPtr<MediaDataDecoder::InitPromise> RemoteMediaDataDecoder::Init() { + RefPtr<RemoteMediaDataDecoder> self = this; + return InvokeAsync(RemoteDecoderManagerChild::GetManagerThread(), __func__, + [self]() { return self->mChild->Init(); }) + ->Then( + RemoteDecoderManagerChild::GetManagerThread(), __func__, + [self, this](TrackType aTrack) { + // If shutdown has started in the meantime shutdown promise may + // be resloved before this task. In this case mChild will be null + // and the init promise has to be canceled. + if (!mChild) { + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED, + __func__); + } + mDescription = mChild->GetDescriptionName(); + mIsHardwareAccelerated = + mChild->IsHardwareAccelerated(mHardwareAcceleratedReason); + mConversion = mChild->NeedsConversion(); + return InitPromise::CreateAndResolve(aTrack, __func__); + }, + [self](const MediaResult& aError) { + return InitPromise::CreateAndReject(aError, __func__); + }); +} + +RefPtr<MediaDataDecoder::DecodePromise> RemoteMediaDataDecoder::Decode( + MediaRawData* aSample) { + RefPtr<RemoteMediaDataDecoder> self = this; + RefPtr<MediaRawData> sample = aSample; + return InvokeAsync( + RemoteDecoderManagerChild::GetManagerThread(), __func__, + [self, sample]() { + return self->mChild->Decode(nsTArray<RefPtr<MediaRawData>>{sample}); + }); +} + +RefPtr<MediaDataDecoder::DecodePromise> RemoteMediaDataDecoder::DecodeBatch( + nsTArray<RefPtr<MediaRawData>>&& aSamples) { + RefPtr<RemoteMediaDataDecoder> self = this; + return InvokeAsync(RemoteDecoderManagerChild::GetManagerThread(), __func__, + [self, samples = std::move(aSamples)]() { + return self->mChild->Decode(samples); + }); +} + +RefPtr<MediaDataDecoder::FlushPromise> RemoteMediaDataDecoder::Flush() { + RefPtr<RemoteMediaDataDecoder> self = this; + return InvokeAsync(RemoteDecoderManagerChild::GetManagerThread(), __func__, + [self]() { return self->mChild->Flush(); }); +} + +RefPtr<MediaDataDecoder::DecodePromise> RemoteMediaDataDecoder::Drain() { + RefPtr<RemoteMediaDataDecoder> self = this; + return InvokeAsync(RemoteDecoderManagerChild::GetManagerThread(), __func__, + [self]() { return self->mChild->Drain(); }); +} + +RefPtr<ShutdownPromise> RemoteMediaDataDecoder::Shutdown() { + RefPtr<RemoteMediaDataDecoder> self = this; + return InvokeAsync( + RemoteDecoderManagerChild::GetManagerThread(), __func__, [self]() { + RefPtr<ShutdownPromise> p = self->mChild->Shutdown(); + + // We're about to be destroyed and drop our ref to + // *DecoderChild. Make sure we put a ref into the + // task queue for the *DecoderChild thread to keep + // it alive until we send the delete message. + p->Then(RemoteDecoderManagerChild::GetManagerThread(), __func__, + [child = std::move(self->mChild)]( + const ShutdownPromise::ResolveOrRejectValue& aValue) { + MOZ_ASSERT(aValue.IsResolve()); + child->DestroyIPDL(); + return ShutdownPromise::CreateAndResolveOrReject(aValue, + __func__); + }); + return p; + }); +} + +bool RemoteMediaDataDecoder::IsHardwareAccelerated( + nsACString& aFailureReason) const { + aFailureReason = mHardwareAcceleratedReason; + return mIsHardwareAccelerated; +} + +void RemoteMediaDataDecoder::SetSeekThreshold(const media::TimeUnit& aTime) { + RefPtr<RemoteMediaDataDecoder> self = this; + media::TimeUnit time = aTime; + RemoteDecoderManagerChild::GetManagerThread()->Dispatch( + NS_NewRunnableFunction("dom::RemoteMediaDataDecoder::SetSeekThreshold", + [=]() { + MOZ_ASSERT(self->mChild); + self->mChild->SetSeekThreshold(time); + }), + NS_DISPATCH_NORMAL); +} + +MediaDataDecoder::ConversionRequired RemoteMediaDataDecoder::NeedsConversion() + const { + return mConversion; +} + +nsCString RemoteMediaDataDecoder::GetDescriptionName() const { + return mDescription; +} + +} // namespace mozilla diff --git a/dom/media/ipc/RemoteMediaDataDecoder.h b/dom/media/ipc/RemoteMediaDataDecoder.h new file mode 100644 index 0000000000..6e5ff8c406 --- /dev/null +++ b/dom/media/ipc/RemoteMediaDataDecoder.h @@ -0,0 +1,62 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef include_dom_media_ipc_RemoteMediaDataDecoder_h +#define include_dom_media_ipc_RemoteMediaDataDecoder_h +#include "PlatformDecoderModule.h" + +#include "MediaData.h" + +namespace mozilla { + +class RemoteDecoderChild; +class RemoteDecoderManagerChild; +class RemoteMediaDataDecoder; + +DDLoggedTypeCustomNameAndBase(RemoteMediaDataDecoder, RemoteMediaDataDecoder, + MediaDataDecoder); + +// A MediaDataDecoder implementation that proxies through IPDL +// to a 'real' decoder in the GPU or RDD process. +// All requests get forwarded to a *DecoderChild instance that +// operates solely on the provided manager and abstract manager threads. +class RemoteMediaDataDecoder + : public MediaDataDecoder, + public DecoderDoctorLifeLogger<RemoteMediaDataDecoder> { + public: + explicit RemoteMediaDataDecoder(RemoteDecoderChild* aChild); + + // MediaDataDecoder + RefPtr<InitPromise> Init() override; + RefPtr<DecodePromise> Decode(MediaRawData* aSample) override; + bool CanDecodeBatch() const override { return true; } + RefPtr<DecodePromise> DecodeBatch( + nsTArray<RefPtr<MediaRawData>>&& aSamples) override; + RefPtr<DecodePromise> Drain() override; + RefPtr<FlushPromise> Flush() override; + RefPtr<ShutdownPromise> Shutdown() override; + bool IsHardwareAccelerated(nsACString& aFailureReason) const override; + void SetSeekThreshold(const media::TimeUnit& aTime) override; + nsCString GetDescriptionName() const override; + ConversionRequired NeedsConversion() const override; + + private: + ~RemoteMediaDataDecoder(); + + // Only ever written to from the reader task queue (during the constructor and + // destructor when we can guarantee no other threads are accessing it). Only + // read from the manager thread. + RefPtr<RemoteDecoderChild> mChild; + // Only ever written/modified during decoder initialisation. + // As such can be accessed from any threads after that. + nsCString mDescription = "RemoteMediaDataDecoder"_ns; + bool mIsHardwareAccelerated = false; + nsCString mHardwareAcceleratedReason; + ConversionRequired mConversion = ConversionRequired::kNeedNone; +}; + +} // namespace mozilla + +#endif // include_dom_media_ipc_RemoteMediaDataDecoder_h diff --git a/dom/media/ipc/RemoteVideoDecoder.cpp b/dom/media/ipc/RemoteVideoDecoder.cpp new file mode 100644 index 0000000000..ed46103eff --- /dev/null +++ b/dom/media/ipc/RemoteVideoDecoder.cpp @@ -0,0 +1,296 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include "RemoteVideoDecoder.h" + +#include "mozilla/layers/ImageDataSerializer.h" + +#ifdef MOZ_AV1 +# include "AOMDecoder.h" +# include "DAV1DDecoder.h" +#endif +#ifdef XP_WIN +# include "WMFDecoderModule.h" +#endif +#include "GPUVideoImage.h" +#include "ImageContainer.h" // for PlanarYCbCrData and BufferRecycleBin +#include "MediaDataDecoderProxy.h" +#include "MediaInfo.h" +#include "PDMFactory.h" +#include "RemoteDecoderManagerParent.h" +#include "RemoteImageHolder.h" +#include "mozilla/StaticPrefs_media.h" +#include "mozilla/Telemetry.h" +#include "mozilla/layers/ImageClient.h" +#include "mozilla/layers/TextureClient.h" +#include "mozilla/layers/VideoBridgeChild.h" + +namespace mozilla { + +using namespace layers; // for PlanarYCbCrData and BufferRecycleBin +using namespace ipc; +using namespace gfx; + +layers::TextureForwarder* KnowsCompositorVideo::GetTextureForwarder() { + auto* vbc = VideoBridgeChild::GetSingleton(); + return (vbc && vbc->CanSend()) ? vbc : nullptr; +} +layers::LayersIPCActor* KnowsCompositorVideo::GetLayersIPCActor() { + return GetTextureForwarder(); +} + +/* static */ already_AddRefed<KnowsCompositorVideo> +KnowsCompositorVideo::TryCreateForIdentifier( + const layers::TextureFactoryIdentifier& aIdentifier) { + VideoBridgeChild* child = VideoBridgeChild::GetSingleton(); + if (!child) { + return nullptr; + } + + RefPtr<KnowsCompositorVideo> knowsCompositor = new KnowsCompositorVideo(); + knowsCompositor->IdentifyTextureHost(aIdentifier); + return knowsCompositor.forget(); +} + +RemoteVideoDecoderChild::RemoteVideoDecoderChild(RemoteDecodeIn aLocation) + : RemoteDecoderChild(aLocation), mBufferRecycleBin(new BufferRecycleBin) {} + +MediaResult RemoteVideoDecoderChild::ProcessOutput( + DecodedOutputIPDL&& aDecodedData) { + AssertOnManagerThread(); + MOZ_ASSERT(aDecodedData.type() == DecodedOutputIPDL::TArrayOfRemoteVideoData); + + nsTArray<RemoteVideoData>& arrayData = + aDecodedData.get_ArrayOfRemoteVideoData()->Array(); + + for (auto&& data : arrayData) { + if (data.image().IsEmpty()) { + // This is a NullData object. + mDecodedData.AppendElement(MakeRefPtr<NullData>( + data.base().offset(), data.base().time(), data.base().duration())); + continue; + } + RefPtr<Image> image = data.image().TransferToImage(mBufferRecycleBin); + + RefPtr<VideoData> video = VideoData::CreateFromImage( + data.display(), data.base().offset(), data.base().time(), + data.base().duration(), image, data.base().keyframe(), + data.base().timecode()); + + if (!video) { + // OOM + return MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__); + } + mDecodedData.AppendElement(std::move(video)); + } + return NS_OK; +} + +MediaResult RemoteVideoDecoderChild::InitIPDL( + const VideoInfo& aVideoInfo, float aFramerate, + const CreateDecoderParams::OptionSet& aOptions, + Maybe<layers::TextureFactoryIdentifier> aIdentifier, + const Maybe<uint64_t>& aMediaEngineId, + const Maybe<TrackingId>& aTrackingId) { + MOZ_ASSERT_IF(mLocation == RemoteDecodeIn::GpuProcess, aIdentifier); + + RefPtr<RemoteDecoderManagerChild> manager = + RemoteDecoderManagerChild::GetSingleton(mLocation); + + // The manager isn't available because RemoteDecoderManagerChild has been + // initialized with null end points and we don't want to decode video on RDD + // process anymore. Return false here so that we can fallback to other PDMs. + if (!manager) { + return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("RemoteDecoderManager is not available.")); + } + + if (!manager->CanSend()) { + if (mLocation == RemoteDecodeIn::GpuProcess) { + // The manager doesn't support sending messages because we've just crashed + // and are working on reinitialization. Don't initialize mIPDLSelfRef and + // leave us in an error state. We'll then immediately reject the promise + // when Init() is called and the caller can try again. Hopefully by then + // the new manager is ready, or we've notified the caller of it being no + // longer available. If not, then the cycle repeats until we're ready. + return NS_OK; + } + + return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("RemoteDecoderManager unable to send.")); + } + + mIPDLSelfRef = this; + VideoDecoderInfoIPDL decoderInfo(aVideoInfo, aFramerate); + Unused << manager->SendPRemoteDecoderConstructor( + this, decoderInfo, aOptions, aIdentifier, aMediaEngineId, aTrackingId); + + return NS_OK; +} + +RemoteVideoDecoderParent::RemoteVideoDecoderParent( + RemoteDecoderManagerParent* aParent, const VideoInfo& aVideoInfo, + float aFramerate, const CreateDecoderParams::OptionSet& aOptions, + const Maybe<layers::TextureFactoryIdentifier>& aIdentifier, + nsISerialEventTarget* aManagerThread, TaskQueue* aDecodeTaskQueue, + const Maybe<uint64_t>& aMediaEngineId, Maybe<TrackingId> aTrackingId) + : RemoteDecoderParent(aParent, aOptions, aManagerThread, aDecodeTaskQueue, + aMediaEngineId, std::move(aTrackingId)), + mVideoInfo(aVideoInfo), + mFramerate(aFramerate) { + if (aIdentifier) { + // Check to see if we have a direct PVideoBridge connection to the + // destination process specified in aIdentifier, and create a + // KnowsCompositor representing that connection if so. If this fails, then + // we fall back to returning the decoded frames directly via Output(). + mKnowsCompositor = + KnowsCompositorVideo::TryCreateForIdentifier(*aIdentifier); + } +} + +IPCResult RemoteVideoDecoderParent::RecvConstruct( + ConstructResolver&& aResolver) { + auto imageContainer = MakeRefPtr<layers::ImageContainer>(); + if (mKnowsCompositor && XRE_IsRDDProcess()) { + // Ensure to allocate recycle allocator + imageContainer->EnsureRecycleAllocatorForRDD(mKnowsCompositor); + } + auto params = CreateDecoderParams{ + mVideoInfo, mKnowsCompositor, + imageContainer, CreateDecoderParams::VideoFrameRate(mFramerate), + mOptions, CreateDecoderParams::NoWrapper(true), + mMediaEngineId, mTrackingId, + }; + + mParent->EnsurePDMFactory().CreateDecoder(params)->Then( + GetCurrentSerialEventTarget(), __func__, + [resolver = std::move(aResolver), self = RefPtr{this}]( + PlatformDecoderModule::CreateDecoderPromise::ResolveOrRejectValue&& + aValue) { + if (aValue.IsReject()) { + resolver(aValue.RejectValue()); + return; + } + MOZ_ASSERT(aValue.ResolveValue()); + self->mDecoder = + new MediaDataDecoderProxy(aValue.ResolveValue().forget(), + do_AddRef(self->mDecodeTaskQueue.get())); + resolver(NS_OK); + }); + return IPC_OK(); +} + +MediaResult RemoteVideoDecoderParent::ProcessDecodedData( + MediaDataDecoder::DecodedData&& aData, DecodedOutputIPDL& aDecodedData) { + MOZ_ASSERT(OnManagerThread()); + + // If the video decoder bridge has shut down, stop. + if (mKnowsCompositor && !mKnowsCompositor->GetTextureForwarder()) { + aDecodedData = MakeRefPtr<ArrayOfRemoteVideoData>(); + return NS_OK; + } + + nsTArray<RemoteVideoData> array; + + for (const auto& data : aData) { + MOZ_ASSERT(data->mType == MediaData::Type::VIDEO_DATA || + data->mType == MediaData::Type::NULL_DATA, + "Can only decode videos using RemoteDecoderParent!"); + if (data->mType == MediaData::Type::NULL_DATA) { + RemoteVideoData output( + MediaDataIPDL(data->mOffset, data->mTime, data->mTimecode, + data->mDuration, data->mKeyframe), + IntSize(), RemoteImageHolder(), -1); + + array.AppendElement(std::move(output)); + continue; + } + VideoData* video = static_cast<VideoData*>(data.get()); + + MOZ_ASSERT(video->mImage, + "Decoded video must output a layer::Image to " + "be used with RemoteDecoderParent"); + + RefPtr<TextureClient> texture; + SurfaceDescriptor sd; + IntSize size; + bool needStorage = false; + + if (mKnowsCompositor) { + texture = video->mImage->GetTextureClient(mKnowsCompositor); + + if (!texture) { + texture = ImageClient::CreateTextureClientForImage(video->mImage, + mKnowsCompositor); + } + + if (texture) { + if (!texture->IsAddedToCompositableClient()) { + texture->InitIPDLActor(mKnowsCompositor); + texture->SetAddedToCompositableClient(); + } + needStorage = true; + SurfaceDescriptorRemoteDecoder remoteSD; + texture->GetSurfaceDescriptorRemoteDecoder(&remoteSD); + sd = remoteSD; + size = texture->GetSize(); + } + } + + // If failed to create a GPU accelerated surface descriptor, fall back to + // copying frames via shmem. + if (!IsSurfaceDescriptorValid(sd)) { + needStorage = false; + PlanarYCbCrImage* image = video->mImage->AsPlanarYCbCrImage(); + if (!image) { + return MediaResult(NS_ERROR_UNEXPECTED, + "Expected Planar YCbCr image in " + "RemoteVideoDecoderParent::ProcessDecodedData"); + } + + SurfaceDescriptorBuffer sdBuffer; + nsresult rv = image->BuildSurfaceDescriptorBuffer( + sdBuffer, [&](uint32_t aBufferSize) { + ShmemBuffer buffer = AllocateBuffer(aBufferSize); + if (buffer.Valid()) { + return MemoryOrShmem(std::move(buffer.Get())); + } + return MemoryOrShmem(); + }); + NS_ENSURE_SUCCESS(rv, rv); + + sd = sdBuffer; + size = image->GetSize(); + } + + if (needStorage) { + MOZ_ASSERT(sd.type() != SurfaceDescriptor::TSurfaceDescriptorBuffer); + mParent->StoreImage(static_cast<const SurfaceDescriptorGPUVideo&>(sd), + video->mImage, texture); + } + + RemoteVideoData output( + MediaDataIPDL(data->mOffset, data->mTime, data->mTimecode, + data->mDuration, data->mKeyframe), + video->mDisplay, + RemoteImageHolder( + mParent, + XRE_IsGPUProcess() + ? VideoBridgeSource::GpuProcess + : (XRE_IsRDDProcess() + ? VideoBridgeSource::RddProcess + : VideoBridgeSource::MFMediaEngineCDMProcess), + size, video->mImage->GetColorDepth(), sd), + video->mFrameID); + + array.AppendElement(std::move(output)); + } + + aDecodedData = MakeRefPtr<ArrayOfRemoteVideoData>(std::move(array)); + + return NS_OK; +} + +} // namespace mozilla diff --git a/dom/media/ipc/RemoteVideoDecoder.h b/dom/media/ipc/RemoteVideoDecoder.h new file mode 100644 index 0000000000..160d5cc4ba --- /dev/null +++ b/dom/media/ipc/RemoteVideoDecoder.h @@ -0,0 +1,80 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef include_dom_media_ipc_RemoteVideoDecoderChild_h +#define include_dom_media_ipc_RemoteVideoDecoderChild_h +#include "RemoteDecoderChild.h" +#include "RemoteDecoderManagerChild.h" +#include "RemoteDecoderParent.h" + +namespace mozilla::layers { +class BufferRecycleBin; +} // namespace mozilla::layers + +namespace mozilla { + +class KnowsCompositorVideo : public layers::KnowsCompositor { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(KnowsCompositorVideo, override) + + layers::TextureForwarder* GetTextureForwarder() override; + layers::LayersIPCActor* GetLayersIPCActor() override; + + static already_AddRefed<KnowsCompositorVideo> TryCreateForIdentifier( + const layers::TextureFactoryIdentifier& aIdentifier); + + private: + KnowsCompositorVideo() = default; + virtual ~KnowsCompositorVideo() = default; +}; + +using mozilla::ipc::IPCResult; + +class RemoteVideoDecoderChild : public RemoteDecoderChild { + public: + explicit RemoteVideoDecoderChild(RemoteDecodeIn aLocation); + + MOZ_IS_CLASS_INIT MediaResult + InitIPDL(const VideoInfo& aVideoInfo, float aFramerate, + const CreateDecoderParams::OptionSet& aOptions, + mozilla::Maybe<layers::TextureFactoryIdentifier> aIdentifier, + const Maybe<uint64_t>& aMediaEngineId, + const Maybe<TrackingId>& aTrackingId); + + MediaResult ProcessOutput(DecodedOutputIPDL&& aDecodedData) override; + + private: + RefPtr<mozilla::layers::BufferRecycleBin> mBufferRecycleBin; +}; + +class RemoteVideoDecoderParent final : public RemoteDecoderParent { + public: + RemoteVideoDecoderParent( + RemoteDecoderManagerParent* aParent, const VideoInfo& aVideoInfo, + float aFramerate, const CreateDecoderParams::OptionSet& aOptions, + const Maybe<layers::TextureFactoryIdentifier>& aIdentifier, + nsISerialEventTarget* aManagerThread, TaskQueue* aDecodeTaskQueue, + const Maybe<uint64_t>& aMediaEngineId, Maybe<TrackingId> aTrackingId); + + protected: + IPCResult RecvConstruct(ConstructResolver&& aResolver) override; + + MediaResult ProcessDecodedData(MediaDataDecoder::DecodedData&& aData, + DecodedOutputIPDL& aDecodedData) override; + + private: + // Can only be accessed from the manager thread + // Note: we can't keep a reference to the original VideoInfo here + // because unlike in typical MediaDataDecoder situations, we're being + // passed a deserialized VideoInfo from RecvPRemoteDecoderConstructor + // which is destroyed when RecvPRemoteDecoderConstructor returns. + const VideoInfo mVideoInfo; + const float mFramerate; + RefPtr<KnowsCompositorVideo> mKnowsCompositor; +}; + +} // namespace mozilla + +#endif // include_dom_media_ipc_RemoteVideoDecoderChild_h diff --git a/dom/media/ipc/ShmemRecycleAllocator.h b/dom/media/ipc/ShmemRecycleAllocator.h new file mode 100644 index 0000000000..7207e27014 --- /dev/null +++ b/dom/media/ipc/ShmemRecycleAllocator.h @@ -0,0 +1,60 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef include_dom_media_ipc_ShmemRecycleAllocator_h +#define include_dom_media_ipc_ShmemRecycleAllocator_h + +#include "mozilla/ShmemPool.h" + +namespace mozilla { + +template <class T> +class ShmemRecycleAllocator { + public: + explicit ShmemRecycleAllocator(T* aActor) + : mActor(aActor), mPool(1, ShmemPool::PoolType::DynamicPool) {} + ShmemBuffer AllocateBuffer(size_t aSize, + ShmemPool::AllocationPolicy aPolicy = + ShmemPool::AllocationPolicy::Unsafe) { + ShmemBuffer buffer = mPool.Get(mActor, aSize, aPolicy); + if (!buffer.Valid()) { + return buffer; + } + MOZ_DIAGNOSTIC_ASSERT(aSize <= buffer.Get().Size<uint8_t>()); + mUsedShmems.AppendElement(buffer.Get()); + mNeedCleanup = true; + return buffer; + } + + void ReleaseBuffer(ShmemBuffer&& aBuffer) { mPool.Put(std::move(aBuffer)); } + + void ReleaseAllBuffers() { + for (auto&& mem : mUsedShmems) { + ReleaseBuffer(ShmemBuffer(mem.Get())); + } + mUsedShmems.Clear(); + } + + void CleanupShmemRecycleAllocator() { + ReleaseAllBuffers(); + mPool.Cleanup(mActor); + mNeedCleanup = false; + } + + ~ShmemRecycleAllocator() { + MOZ_DIAGNOSTIC_ASSERT(mUsedShmems.IsEmpty() && !mNeedCleanup, + "Shmems not all deallocated"); + } + + private: + T* const mActor; + ShmemPool mPool; + AutoTArray<ShmemBuffer, 4> mUsedShmems; + bool mNeedCleanup = false; +}; + +} // namespace mozilla + +#endif // include_dom_media_ipc_ShmemRecycleAllocator_h diff --git a/dom/media/ipc/moz.build b/dom/media/ipc/moz.build new file mode 100644 index 0000000000..53424922f7 --- /dev/null +++ b/dom/media/ipc/moz.build @@ -0,0 +1,88 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +IPDL_SOURCES += [ + "PMediaDecoderParams.ipdlh", + "PRemoteDecoder.ipdl", +] + +PREPROCESSED_IPDL_SOURCES += [ + "PRDD.ipdl", + "PRemoteDecoderManager.ipdl", +] + +EXPORTS.mozilla += [ + "RDDChild.h", + "RDDParent.h", + "RDDProcessHost.h", + "RDDProcessImpl.h", + "RDDProcessManager.h", + "RemoteDecoderChild.h", + "RemoteDecoderManagerChild.h", + "RemoteDecoderManagerParent.h", + "RemoteDecoderModule.h", + "RemoteDecoderParent.h", + "RemoteDecodeUtils.h", + "RemoteImageHolder.h", + "RemoteMediaData.h", + "RemoteMediaDataDecoder.h", + "ShmemRecycleAllocator.h", +] + +EXPORTS.mozilla.dom += [ + "MediaIPCUtils.h", +] + +SOURCES += [ + "RDDChild.cpp", + "RDDParent.cpp", + "RDDProcessHost.cpp", + "RDDProcessImpl.cpp", + "RDDProcessManager.cpp", + "RemoteAudioDecoder.cpp", + "RemoteDecoderChild.cpp", + "RemoteDecoderManagerChild.cpp", + "RemoteDecoderManagerParent.cpp", + "RemoteDecoderModule.cpp", + "RemoteDecoderParent.cpp", + "RemoteDecodeUtils.cpp", + "RemoteImageHolder.cpp", + "RemoteMediaData.cpp", + "RemoteMediaDataDecoder.cpp", + "RemoteVideoDecoder.cpp", +] + +if CONFIG["MOZ_WMF_MEDIA_ENGINE"]: + IPDL_SOURCES += [ + "PMFMediaEngine.ipdl", + ] + SOURCES += [ + "MFMediaEngineChild.cpp", + "MFMediaEngineParent.cpp", + "MFMediaEngineUtils.cpp", + ] + EXPORTS.mozilla += [ + "MFMediaEngineChild.h", + "MFMediaEngineParent.h", + "MFMediaEngineUtils.h", + ] + LOCAL_INCLUDES += [ + "../platforms/wmf", + ] + +# so we can include nsMacUtilsImpl.h in RDDParent.cpp for sandboxing +LOCAL_INCLUDES += [ + "/xpcom/base", +] + +include("/ipc/chromium/chromium-config.mozbuild") + +# Add libFuzzer configuration directives +include("/tools/fuzzing/libfuzzer-config.mozbuild") + + +FINAL_LIBRARY = "xul" |