From 43a97878ce14b72f0981164f87f2e35e14151312 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 11:22:09 +0200 Subject: Adding upstream version 110.0.1. Signed-off-by: Daniel Baumann --- dom/media/platforms/omx/OmxCoreLibLinker.cpp | 113 +++ dom/media/platforms/omx/OmxCoreLibLinker.h | 36 + dom/media/platforms/omx/OmxDataDecoder.cpp | 1001 ++++++++++++++++++++++ dom/media/platforms/omx/OmxDataDecoder.h | 220 +++++ dom/media/platforms/omx/OmxDecoderModule.cpp | 59 ++ dom/media/platforms/omx/OmxDecoderModule.h | 33 + dom/media/platforms/omx/OmxFunctionList.h | 13 + dom/media/platforms/omx/OmxPlatformLayer.cpp | 307 +++++++ dom/media/platforms/omx/OmxPlatformLayer.h | 103 +++ dom/media/platforms/omx/OmxPromiseLayer.cpp | 355 ++++++++ dom/media/platforms/omx/OmxPromiseLayer.h | 243 ++++++ dom/media/platforms/omx/PureOmxPlatformLayer.cpp | 405 +++++++++ dom/media/platforms/omx/PureOmxPlatformLayer.h | 110 +++ dom/media/platforms/omx/moz.build | 36 + 14 files changed, 3034 insertions(+) create mode 100644 dom/media/platforms/omx/OmxCoreLibLinker.cpp create mode 100644 dom/media/platforms/omx/OmxCoreLibLinker.h create mode 100644 dom/media/platforms/omx/OmxDataDecoder.cpp create mode 100644 dom/media/platforms/omx/OmxDataDecoder.h create mode 100644 dom/media/platforms/omx/OmxDecoderModule.cpp create mode 100644 dom/media/platforms/omx/OmxDecoderModule.h create mode 100644 dom/media/platforms/omx/OmxFunctionList.h create mode 100644 dom/media/platforms/omx/OmxPlatformLayer.cpp create mode 100644 dom/media/platforms/omx/OmxPlatformLayer.h create mode 100644 dom/media/platforms/omx/OmxPromiseLayer.cpp create mode 100644 dom/media/platforms/omx/OmxPromiseLayer.h create mode 100644 dom/media/platforms/omx/PureOmxPlatformLayer.cpp create mode 100644 dom/media/platforms/omx/PureOmxPlatformLayer.h create mode 100644 dom/media/platforms/omx/moz.build (limited to 'dom/media/platforms/omx') diff --git a/dom/media/platforms/omx/OmxCoreLibLinker.cpp b/dom/media/platforms/omx/OmxCoreLibLinker.cpp new file mode 100644 index 0000000000..a0ee61ec42 --- /dev/null +++ b/dom/media/platforms/omx/OmxCoreLibLinker.cpp @@ -0,0 +1,113 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "OmxCoreLibLinker.h" +#include "mozilla/ArrayUtils.h" +#include "mozilla/Preferences.h" +#include "MainThreadUtils.h" +#include "prlink.h" +#include "PlatformDecoderModule.h" + +#ifdef LOG +# undef LOG +#endif + +#define LOG(arg, ...) \ + MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, \ + ("OmxCoreLibLinker::%s: " arg, __func__, ##__VA_ARGS__)) + +namespace mozilla { + +OmxCoreLibLinker::LinkStatus OmxCoreLibLinker::sLinkStatus = LinkStatus_INIT; + +const char* OmxCoreLibLinker::sLibNames[] = { + "libopenmaxil.so", // Raspberry Pi + "libomxr_core.so", // Renesas R-Car, RZ/G + "libomxil-bellagio.so.0", // Bellagio: An OSS implementation of OpenMAX IL +}; + +PRLibrary* OmxCoreLibLinker::sLinkedLib = nullptr; +const char* OmxCoreLibLinker::sLibName = nullptr; + +#define OMX_FUNC(func) void (*func)(); +#include "OmxFunctionList.h" +#undef OMX_FUNC + +bool OmxCoreLibLinker::TryLinkingLibrary(const char* libName) { + PRLibSpec lspec; + lspec.type = PR_LibSpec_Pathname; + lspec.value.pathname = libName; + sLinkedLib = PR_LoadLibraryWithFlags(lspec, PR_LD_NOW | PR_LD_LOCAL); + if (sLinkedLib) { + if (Bind(libName)) { + sLibName = libName; + LOG("Succeeded to load %s", libName); + return true; + } else { + LOG("Failed to link %s", libName); + } + Unlink(); + } + return false; +} + +/* static */ +bool OmxCoreLibLinker::Link() { + LOG(""); + + if (sLinkStatus) { + return sLinkStatus == LinkStatus_SUCCEEDED; + } + + MOZ_ASSERT(NS_IsMainThread()); + + nsAutoCString libPath; + nsresult rv = Preferences::GetCString("media.omx.core-lib-path", libPath); + if (NS_SUCCEEDED(rv) && !libPath.IsEmpty()) { + if (TryLinkingLibrary(libPath.Data())) { + sLinkStatus = LinkStatus_SUCCEEDED; + return true; + } + } + + // try known paths + for (size_t i = 0; i < ArrayLength(sLibNames); i++) { + if (TryLinkingLibrary(sLibNames[i])) { + sLinkStatus = LinkStatus_SUCCEEDED; + return true; + } + } + sLinkStatus = LinkStatus_FAILED; + return false; +} + +/* static */ +bool OmxCoreLibLinker::Bind(const char* aLibName) { +#define OMX_FUNC(func) \ + { \ + if (!(func = (typeof(func))PR_FindSymbol(sLinkedLib, #func))) { \ + LOG("Couldn't load function " #func " from %s.", aLibName); \ + return false; \ + } \ + } +#include "OmxFunctionList.h" +#undef OMX_FUNC + return true; +} + +/* static */ +void OmxCoreLibLinker::Unlink() { + LOG(""); + + if (sLinkedLib) { + PR_UnloadLibrary(sLinkedLib); + sLinkedLib = nullptr; + sLibName = nullptr; + sLinkStatus = LinkStatus_INIT; + } +} + +} // namespace mozilla diff --git a/dom/media/platforms/omx/OmxCoreLibLinker.h b/dom/media/platforms/omx/OmxCoreLibLinker.h new file mode 100644 index 0000000000..aaf1bf92de --- /dev/null +++ b/dom/media/platforms/omx/OmxCoreLibLinker.h @@ -0,0 +1,36 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef __OmxCoreLibLinker_h__ +#define __OmxCoreLibLinker_h__ + +struct PRLibrary; + +namespace mozilla { + +class OmxCoreLibLinker { + public: + static bool Link(); + static void Unlink(); + + private: + static PRLibrary* sLinkedLib; + static const char* sLibName; + static const char* sLibNames[]; + + static bool TryLinkingLibrary(const char* libName); + static bool Bind(const char* aLibName); + + static enum LinkStatus { + LinkStatus_INIT = 0, + LinkStatus_FAILED, + LinkStatus_SUCCEEDED + } sLinkStatus; +}; + +} // namespace mozilla + +#endif // __OmxCoreLibLinker_h__ diff --git a/dom/media/platforms/omx/OmxDataDecoder.cpp b/dom/media/platforms/omx/OmxDataDecoder.cpp new file mode 100644 index 0000000000..3224721878 --- /dev/null +++ b/dom/media/platforms/omx/OmxDataDecoder.cpp @@ -0,0 +1,1001 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "OmxDataDecoder.h" + +#include "OMX_Audio.h" +#include "OMX_Component.h" +#include "OMX_Types.h" +#include "OmxPlatformLayer.h" +#include "mozilla/IntegerPrintfMacros.h" + +#ifdef LOG +# undef LOG +# undef LOGL +#endif + +#define LOG(arg, ...) \ + DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, __func__, \ + ##__VA_ARGS__) + +#define LOGL(arg, ...) \ + DDMOZ_LOGEX(self.get(), sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, \ + __func__, ##__VA_ARGS__) + +#define CHECK_OMX_ERR(err) \ + if (err != OMX_ErrorNone) { \ + NotifyError(err, __func__); \ + return; \ + } + +namespace mozilla { + +using namespace gfx; + +static const char* StateTypeToStr(OMX_STATETYPE aType) { + MOZ_ASSERT(aType == OMX_StateLoaded || aType == OMX_StateIdle || + aType == OMX_StateExecuting || aType == OMX_StatePause || + aType == OMX_StateWaitForResources || aType == OMX_StateInvalid); + + switch (aType) { + case OMX_StateLoaded: + return "OMX_StateLoaded"; + case OMX_StateIdle: + return "OMX_StateIdle"; + case OMX_StateExecuting: + return "OMX_StateExecuting"; + case OMX_StatePause: + return "OMX_StatePause"; + case OMX_StateWaitForResources: + return "OMX_StateWaitForResources"; + case OMX_StateInvalid: + return "OMX_StateInvalid"; + default: + return "Unknown"; + } +} + +// A helper class to retrieve AudioData or VideoData. +class MediaDataHelper { + protected: + virtual ~MediaDataHelper() = default; + + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaDataHelper) + + MediaDataHelper(const TrackInfo* aTrackInfo, + layers::ImageContainer* aImageContainer, + OmxPromiseLayer* aOmxLayer); + + already_AddRefed GetMediaData(BufferData* aBufferData, + bool& aPlatformDepenentData); + + protected: + already_AddRefed CreateAudioData(BufferData* aBufferData); + + already_AddRefed CreateYUV420VideoData(BufferData* aBufferData); + + const TrackInfo* mTrackInfo; + + OMX_PARAM_PORTDEFINITIONTYPE mOutputPortDef; + + // audio output + MediaQueue mAudioQueue; + + AudioCompactor mAudioCompactor; + + // video output + RefPtr mImageContainer; +}; + +OmxDataDecoder::OmxDataDecoder(const TrackInfo& aTrackInfo, + layers::ImageContainer* aImageContainer, + Maybe aTrackingId) + : mOmxTaskQueue( + CreateMediaDecodeTaskQueue("OmxDataDecoder::mOmxTaskQueue")), + mImageContainer(aImageContainer), + mWatchManager(this, mOmxTaskQueue), + mOmxState(OMX_STATETYPE::OMX_StateInvalid, "OmxDataDecoder::mOmxState"), + mTrackInfo(aTrackInfo.Clone()), + mFlushing(false), + mShuttingDown(false), + mCheckingInputExhausted(false), + mPortSettingsChanged(-1, "OmxDataDecoder::mPortSettingsChanged"), + mTrackingId(std::move(aTrackingId)) { + LOG(""); + mOmxLayer = new OmxPromiseLayer(mOmxTaskQueue, this, aImageContainer); +} + +OmxDataDecoder::~OmxDataDecoder() { LOG(""); } + +void OmxDataDecoder::InitializationTask() { + mWatchManager.Watch(mOmxState, &OmxDataDecoder::OmxStateRunner); + mWatchManager.Watch(mPortSettingsChanged, + &OmxDataDecoder::PortSettingsChanged); +} + +void OmxDataDecoder::EndOfStream() { + LOG(""); + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + + RefPtr self = this; + mOmxLayer->SendCommand(OMX_CommandFlush, OMX_ALL, nullptr) + ->Then(mOmxTaskQueue, __func__, + [self, this](OmxCommandPromise::ResolveOrRejectValue&& aValue) { + mDrainPromise.ResolveIfExists(std::move(mDecodedData), __func__); + mDecodedData = DecodedData(); + }); +} + +RefPtr OmxDataDecoder::Init() { + LOG(""); + + mThread = GetCurrentSerialEventTarget(); + RefPtr self = this; + return InvokeAsync(mOmxTaskQueue, __func__, [self, this]() { + InitializationTask(); + + RefPtr p = mInitPromise.Ensure(__func__); + mOmxLayer->Init(mTrackInfo.get()) + ->Then( + mOmxTaskQueue, __func__, + [self, this]() { + // Omx state should be OMX_StateIdle. + mOmxState = mOmxLayer->GetState(); + MOZ_ASSERT(mOmxState != OMX_StateIdle); + }, + [self, this]() { + RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + }); + return p; + }); +} + +RefPtr OmxDataDecoder::Decode( + MediaRawData* aSample) { + LOG("sample %p", aSample); + MOZ_ASSERT(mThread->IsOnCurrentThread()); + MOZ_ASSERT(mInitPromise.IsEmpty()); + + RefPtr self = this; + RefPtr sample = aSample; + return InvokeAsync(mOmxTaskQueue, __func__, [self, this, sample]() { + RefPtr p = mDecodePromise.Ensure(__func__); + + mTrackingId.apply([&](const auto& aId) { + MediaInfoFlag flag = MediaInfoFlag::None; + flag |= (sample->mKeyframe ? MediaInfoFlag::KeyFrame + : MediaInfoFlag::NonKeyFrame); + + mPerformanceRecorder.Start(sample->mTimecode.ToMicroseconds(), + "OmxDataDecoder"_ns, aId, flag); + }); + mMediaRawDatas.AppendElement(std::move(sample)); + + // Start to fill/empty buffers. + if (mOmxState == OMX_StateIdle || mOmxState == OMX_StateExecuting) { + FillAndEmptyBuffers(); + } + return p; + }); +} + +RefPtr OmxDataDecoder::Flush() { + LOG(""); + MOZ_ASSERT(mThread->IsOnCurrentThread()); + + mFlushing = true; + + return InvokeAsync(mOmxTaskQueue, this, __func__, &OmxDataDecoder::DoFlush); +} + +RefPtr OmxDataDecoder::Drain() { + LOG(""); + MOZ_ASSERT(mThread->IsOnCurrentThread()); + + RefPtr self = this; + return InvokeAsync(mOmxTaskQueue, __func__, [self]() { + RefPtr p = self->mDrainPromise.Ensure(__func__); + self->SendEosBuffer(); + return p; + }); +} + +RefPtr OmxDataDecoder::Shutdown() { + LOG(""); + // mThread may not be set if Init hasn't been called first. + MOZ_ASSERT(!mThread || mThread->IsOnCurrentThread()); + + mShuttingDown = true; + + return InvokeAsync(mOmxTaskQueue, this, __func__, + &OmxDataDecoder::DoAsyncShutdown); +} + +RefPtr OmxDataDecoder::DoAsyncShutdown() { + LOG(""); + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + MOZ_ASSERT(!mFlushing); + + mWatchManager.Unwatch(mOmxState, &OmxDataDecoder::OmxStateRunner); + mWatchManager.Unwatch(mPortSettingsChanged, + &OmxDataDecoder::PortSettingsChanged); + + // Flush to all ports, so all buffers can be returned from component. + RefPtr self = this; + mOmxLayer->SendCommand(OMX_CommandFlush, OMX_ALL, nullptr) + ->Then( + mOmxTaskQueue, __func__, + [self]() -> RefPtr { + LOGL("DoAsyncShutdown: flush complete"); + return self->mOmxLayer->SendCommand(OMX_CommandStateSet, + OMX_StateIdle, nullptr); + }, + [self](const OmxCommandFailureHolder& aError) { + self->mOmxLayer->Shutdown(); + return OmxCommandPromise::CreateAndReject(aError, __func__); + }) + ->Then( + mOmxTaskQueue, __func__, + [self]() -> RefPtr { + RefPtr p = self->mOmxLayer->SendCommand( + OMX_CommandStateSet, OMX_StateLoaded, nullptr); + + // According to spec 3.1.1.2.2.1: + // OMX_StateLoaded needs to be sent before releasing buffers. + // And state transition from OMX_StateIdle to OMX_StateLoaded + // is completed when all of the buffers have been removed + // from the component. + // Here the buffer promises are not resolved due to displaying + // in layer, it needs to wait before the layer returns the + // buffers. + LOGL("DoAsyncShutdown: releasing buffers..."); + self->ReleaseBuffers(OMX_DirInput); + self->ReleaseBuffers(OMX_DirOutput); + + return p; + }, + [self](const OmxCommandFailureHolder& aError) { + self->mOmxLayer->Shutdown(); + return OmxCommandPromise::CreateAndReject(aError, __func__); + }) + ->Then( + mOmxTaskQueue, __func__, + [self]() -> RefPtr { + LOGL( + "DoAsyncShutdown: OMX_StateLoaded, it is safe to shutdown omx"); + self->mOmxLayer->Shutdown(); + self->mWatchManager.Shutdown(); + self->mOmxLayer = nullptr; + self->mMediaDataHelper = nullptr; + self->mShuttingDown = false; + return ShutdownPromise::CreateAndResolve(true, __func__); + }, + [self]() -> RefPtr { + self->mOmxLayer->Shutdown(); + self->mWatchManager.Shutdown(); + self->mOmxLayer = nullptr; + self->mMediaDataHelper = nullptr; + return ShutdownPromise::CreateAndReject(false, __func__); + }) + ->Then( + mThread, __func__, + [self]() { + self->mOmxTaskQueue->BeginShutdown(); + self->mOmxTaskQueue->AwaitShutdownAndIdle(); + self->mShutdownPromise.Resolve(true, __func__); + }, + [self]() { + self->mOmxTaskQueue->BeginShutdown(); + self->mOmxTaskQueue->AwaitShutdownAndIdle(); + self->mShutdownPromise.Resolve(true, __func__); + }); + return mShutdownPromise.Ensure(__func__); +} + +void OmxDataDecoder::FillBufferDone(BufferData* aData) { + MOZ_ASSERT(!aData || aData->mStatus == BufferData::BufferStatus::OMX_CLIENT); + + // Don't output sample when flush or shutting down, especially for video + // decoded frame. Because video decoded frame can have a promise in + // BufferData waiting for layer to resolve it via recycle callback, if other + // module doesn't send it to layer, it will cause a unresolved promise and + // waiting for resolve infinitely. + if (mFlushing || mShuttingDown) { + LOG("mFlush or mShuttingDown, drop data"); + aData->mStatus = BufferData::BufferStatus::FREE; + return; + } + + if (aData->mBuffer->nFlags & OMX_BUFFERFLAG_EOS) { + // Reach eos, it's an empty data so it doesn't need to output. + EndOfStream(); + aData->mStatus = BufferData::BufferStatus::FREE; + } else { + Output(aData); + FillAndEmptyBuffers(); + } +} + +void OmxDataDecoder::Output(BufferData* aData) { + if (!mMediaDataHelper) { + mMediaDataHelper = + new MediaDataHelper(mTrackInfo.get(), mImageContainer, mOmxLayer); + } + + bool isPlatformData = false; + RefPtr data = + mMediaDataHelper->GetMediaData(aData, isPlatformData); + if (!data) { + aData->mStatus = BufferData::BufferStatus::FREE; + return; + } + + if (isPlatformData) { + // If the MediaData is platform dependnet data, it's mostly a kind of + // limited resource, so we use promise to notify when the resource is free. + aData->mStatus = BufferData::BufferStatus::OMX_CLIENT_OUTPUT; + + MOZ_RELEASE_ASSERT(aData->mPromise.IsEmpty()); + RefPtr p = aData->mPromise.Ensure(__func__); + + RefPtr self = this; + RefPtr buffer = aData; + p->Then( + mOmxTaskQueue, __func__, + [self, buffer]() { + MOZ_RELEASE_ASSERT(buffer->mStatus == + BufferData::BufferStatus::OMX_CLIENT_OUTPUT); + buffer->mStatus = BufferData::BufferStatus::FREE; + self->FillAndEmptyBuffers(); + }, + [buffer]() { + MOZ_RELEASE_ASSERT(buffer->mStatus == + BufferData::BufferStatus::OMX_CLIENT_OUTPUT); + buffer->mStatus = BufferData::BufferStatus::FREE; + }); + } else { + aData->mStatus = BufferData::BufferStatus::FREE; + } + + if (mTrackInfo->IsVideo()) { + mPerformanceRecorder.Record( + aData->mRawData->mTimecode.ToMicroseconds(), [&](DecodeStage& aStage) { + const auto& image = data->As()->mImage; + aStage.SetResolution(image->GetSize().Width(), + image->GetSize().Height()); + aStage.SetImageFormat(DecodeStage::YUV420P); + aStage.SetColorDepth(image->GetColorDepth()); + }); + } + + mDecodedData.AppendElement(std::move(data)); +} + +void OmxDataDecoder::FillBufferFailure(OmxBufferFailureHolder aFailureHolder) { + NotifyError(aFailureHolder.mError, __func__); +} + +void OmxDataDecoder::EmptyBufferDone(BufferData* aData) { + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + MOZ_ASSERT(!aData || aData->mStatus == BufferData::BufferStatus::OMX_CLIENT); + + // Nothing to do when status of input buffer is OMX_CLIENT. + aData->mStatus = BufferData::BufferStatus::FREE; + FillAndEmptyBuffers(); + + // There is no way to know if component gets enough raw samples to generate + // output, especially for video decoding. So here it needs to request raw + // samples aggressively. + if (!mCheckingInputExhausted && !mMediaRawDatas.Length()) { + mCheckingInputExhausted = true; + + RefPtr self = this; + nsCOMPtr r = NS_NewRunnableFunction( + "OmxDataDecoder::EmptyBufferDone", [self, this]() { + mCheckingInputExhausted = false; + + if (mMediaRawDatas.Length()) { + return; + } + + mDecodePromise.ResolveIfExists(std::move(mDecodedData), __func__); + mDecodedData = DecodedData(); + }); + + nsresult rv = mOmxTaskQueue->Dispatch(r.forget()); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; + } +} + +void OmxDataDecoder::EmptyBufferFailure(OmxBufferFailureHolder aFailureHolder) { + NotifyError(aFailureHolder.mError, __func__); +} + +void OmxDataDecoder::NotifyError(OMX_ERRORTYPE aOmxError, const char* aLine, + const MediaResult& aError) { + LOG("NotifyError %d (%s) at %s", static_cast(aOmxError), + aError.ErrorName().get(), aLine); + mDecodedData = DecodedData(); + mDecodePromise.RejectIfExists(aError, __func__); + mDrainPromise.RejectIfExists(aError, __func__); + mFlushPromise.RejectIfExists(aError, __func__); +} + +void OmxDataDecoder::FillAndEmptyBuffers() { + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + MOZ_ASSERT(mOmxState == OMX_StateExecuting); + + // During the port setting changed, it is forbidden to do any buffer + // operation. + if (mPortSettingsChanged != -1 || mShuttingDown || mFlushing) { + return; + } + + // Trigger input port. + while (!!mMediaRawDatas.Length()) { + // input buffer must be used by component if there is data available. + RefPtr inbuf = FindAvailableBuffer(OMX_DirInput); + if (!inbuf) { + LOG("no input buffer!"); + break; + } + + RefPtr data = mMediaRawDatas[0]; + // Buffer size should large enough for raw data. + MOZ_RELEASE_ASSERT(inbuf->mBuffer->nAllocLen >= data->Size()); + + memcpy(inbuf->mBuffer->pBuffer, data->Data(), data->Size()); + inbuf->mBuffer->nFilledLen = data->Size(); + inbuf->mBuffer->nOffset = 0; + inbuf->mBuffer->nFlags = inbuf->mBuffer->nAllocLen > data->Size() + ? OMX_BUFFERFLAG_ENDOFFRAME + : 0; + inbuf->mBuffer->nTimeStamp = data->mTime.ToMicroseconds(); + if (data->Size()) { + inbuf->mRawData = mMediaRawDatas[0]; + } else { + LOG("send EOS buffer"); + inbuf->mBuffer->nFlags |= OMX_BUFFERFLAG_EOS; + } + + LOG("feed sample %p to omx component, len %ld, flag %lX", data.get(), + inbuf->mBuffer->nFilledLen, inbuf->mBuffer->nFlags); + mOmxLayer->EmptyBuffer(inbuf)->Then(mOmxTaskQueue, __func__, this, + &OmxDataDecoder::EmptyBufferDone, + &OmxDataDecoder::EmptyBufferFailure); + mMediaRawDatas.RemoveElementAt(0); + } + + // Trigger output port. + while (true) { + RefPtr outbuf = FindAvailableBuffer(OMX_DirOutput); + if (!outbuf) { + break; + } + + mOmxLayer->FillBuffer(outbuf)->Then(mOmxTaskQueue, __func__, this, + &OmxDataDecoder::FillBufferDone, + &OmxDataDecoder::FillBufferFailure); + } +} + +OmxPromiseLayer::BufferData* OmxDataDecoder::FindAvailableBuffer( + OMX_DIRTYPE aType) { + BUFFERLIST* buffers = GetBuffers(aType); + + for (uint32_t i = 0; i < buffers->Length(); i++) { + BufferData* buf = buffers->ElementAt(i); + if (buf->mStatus == BufferData::BufferStatus::FREE) { + return buf; + } + } + + return nullptr; +} + +nsresult OmxDataDecoder::AllocateBuffers(OMX_DIRTYPE aType) { + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + + return mOmxLayer->AllocateOmxBuffer(aType, GetBuffers(aType)); +} + +nsresult OmxDataDecoder::ReleaseBuffers(OMX_DIRTYPE aType) { + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + + return mOmxLayer->ReleaseOmxBuffer(aType, GetBuffers(aType)); +} + +nsTArray>* OmxDataDecoder::GetBuffers( + OMX_DIRTYPE aType) { + MOZ_ASSERT(aType == OMX_DIRTYPE::OMX_DirInput || + aType == OMX_DIRTYPE::OMX_DirOutput); + + if (aType == OMX_DIRTYPE::OMX_DirInput) { + return &mInPortBuffers; + } + return &mOutPortBuffers; +} + +void OmxDataDecoder::ResolveInitPromise(const char* aMethodName) { + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + LOG("called from %s", aMethodName); + mInitPromise.ResolveIfExists(mTrackInfo->GetType(), aMethodName); +} + +void OmxDataDecoder::RejectInitPromise(MediaResult aError, + const char* aMethodName) { + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + mInitPromise.RejectIfExists(aError, aMethodName); +} + +void OmxDataDecoder::OmxStateRunner() { + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + LOG("OMX state: %s", StateTypeToStr(mOmxState)); + + // TODO: maybe it'd be better to use promise CompletionPromise() to replace + // this state machine. + if (mOmxState == OMX_StateLoaded) { + ConfigCodec(); + + // Send OpenMax state command to OMX_StateIdle. + RefPtr self = this; + mOmxLayer->SendCommand(OMX_CommandStateSet, OMX_StateIdle, nullptr) + ->Then( + mOmxTaskQueue, __func__, + [self]() { + // Current state should be OMX_StateIdle. + self->mOmxState = self->mOmxLayer->GetState(); + MOZ_ASSERT(self->mOmxState == OMX_StateIdle); + }, + [self]() { + self->RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + }); + + // Allocate input and output buffers. + OMX_DIRTYPE types[] = {OMX_DIRTYPE::OMX_DirInput, + OMX_DIRTYPE::OMX_DirOutput}; + for (const auto id : types) { + if (NS_FAILED(AllocateBuffers(id))) { + LOG("Failed to allocate buffer on port %d", id); + RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + break; + } + } + } else if (mOmxState == OMX_StateIdle) { + RefPtr self = this; + mOmxLayer->SendCommand(OMX_CommandStateSet, OMX_StateExecuting, nullptr) + ->Then( + mOmxTaskQueue, __func__, + [self]() { + self->mOmxState = self->mOmxLayer->GetState(); + MOZ_ASSERT(self->mOmxState == OMX_StateExecuting); + + self->ResolveInitPromise(__func__); + }, + [self]() { + self->RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); + }); + } else if (mOmxState == OMX_StateExecuting) { + // Configure codec once it gets OMX_StateExecuting state. + FillCodecConfigDataToOmx(); + } else { + MOZ_ASSERT(0); + } +} + +void OmxDataDecoder::ConfigCodec() { + OMX_ERRORTYPE err = mOmxLayer->Config(); + CHECK_OMX_ERR(err); +} + +void OmxDataDecoder::FillCodecConfigDataToOmx() { + // Codec configure data should be the first sample running on Omx TaskQueue. + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + MOZ_ASSERT(!mMediaRawDatas.Length()); + MOZ_ASSERT(mOmxState == OMX_StateIdle || mOmxState == OMX_StateExecuting); + + RefPtr inbuf = FindAvailableBuffer(OMX_DirInput); + RefPtr csc; + if (mTrackInfo->IsAudio()) { + // It would be nice to instead use more specific information here, but + // we force a byte buffer for now since this handles arbitrary codecs. + // TODO(bug 1768566): implement further type checking for codec data. + csc = ForceGetAudioCodecSpecificBlob( + mTrackInfo->GetAsAudioInfo()->mCodecSpecificConfig); + } else if (mTrackInfo->IsVideo()) { + csc = mTrackInfo->GetAsVideoInfo()->mExtraData; + } + + MOZ_RELEASE_ASSERT(csc); + + // Some codecs like h264, its codec specific data is at the first packet, not + // in container. + if (csc->Length()) { + // Buffer size should large enough for raw data. + MOZ_RELEASE_ASSERT(inbuf->mBuffer->nAllocLen >= csc->Length()); + + memcpy(inbuf->mBuffer->pBuffer, csc->Elements(), csc->Length()); + inbuf->mBuffer->nFilledLen = csc->Length(); + inbuf->mBuffer->nOffset = 0; + inbuf->mBuffer->nFlags = + (OMX_BUFFERFLAG_ENDOFFRAME | OMX_BUFFERFLAG_CODECCONFIG); + + LOG("Feed codec configure data to OMX component"); + mOmxLayer->EmptyBuffer(inbuf)->Then(mOmxTaskQueue, __func__, this, + &OmxDataDecoder::EmptyBufferDone, + &OmxDataDecoder::EmptyBufferFailure); + } +} + +bool OmxDataDecoder::Event(OMX_EVENTTYPE aEvent, OMX_U32 aData1, + OMX_U32 aData2) { + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + + if (mOmxLayer->Event(aEvent, aData1, aData2)) { + return true; + } + + switch (aEvent) { + case OMX_EventPortSettingsChanged: { + // Don't always disable port. See bug 1235340. + if (aData2 == 0 || aData2 == OMX_IndexParamPortDefinition) { + // According to spec: "To prevent the loss of any input data, the + // component issuing the OMX_EventPortSettingsChanged event on its input + // port should buffer all input port data that arrives between the + // emission of the OMX_EventPortSettingsChanged event and the arrival of + // the command to disable the input port." + // + // So client needs to disable port and reallocate buffers. + MOZ_ASSERT(mPortSettingsChanged == -1); + mPortSettingsChanged = aData1; + } + LOG("Got OMX_EventPortSettingsChanged event"); + break; + } + default: { + // Got error during decoding, send msg to MFR skipping to next key frame. + if (aEvent == OMX_EventError && mOmxState == OMX_StateExecuting) { + NotifyError((OMX_ERRORTYPE)aData1, __func__, + MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, __func__)); + return true; + } + LOG("WARNING: got none handle event: %d, aData1: %ld, aData2: %ld", + aEvent, aData1, aData2); + return false; + } + } + + return true; +} + +bool OmxDataDecoder::BuffersCanBeReleased(OMX_DIRTYPE aType) { + BUFFERLIST* buffers = GetBuffers(aType); + uint32_t len = buffers->Length(); + for (uint32_t i = 0; i < len; i++) { + BufferData::BufferStatus buf_status = buffers->ElementAt(i)->mStatus; + if (buf_status == BufferData::BufferStatus::OMX_COMPONENT || + buf_status == BufferData::BufferStatus::OMX_CLIENT_OUTPUT) { + return false; + } + } + return true; +} + +OMX_DIRTYPE +OmxDataDecoder::GetPortDirection(uint32_t aPortIndex) { + OMX_PARAM_PORTDEFINITIONTYPE def; + InitOmxParameter(&def); + def.nPortIndex = mPortSettingsChanged; + + OMX_ERRORTYPE err = + mOmxLayer->GetParameter(OMX_IndexParamPortDefinition, &def, sizeof(def)); + if (err != OMX_ErrorNone) { + return OMX_DirMax; + } + return def.eDir; +} + +RefPtr +OmxDataDecoder::CollectBufferPromises(OMX_DIRTYPE aType) { + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + + nsTArray> promises; + OMX_DIRTYPE types[] = {OMX_DIRTYPE::OMX_DirInput, OMX_DIRTYPE::OMX_DirOutput}; + for (const auto type : types) { + if ((aType == type) || (aType == OMX_DirMax)) { + // find the buffer which has promise. + BUFFERLIST* buffers = GetBuffers(type); + + for (uint32_t i = 0; i < buffers->Length(); i++) { + BufferData* buf = buffers->ElementAt(i); + if (!buf->mPromise.IsEmpty()) { + // OmxBufferPromise is not exclusive, it can be multiple "Then"s, so + // it is safe to call "Ensure" here. + promises.AppendElement(buf->mPromise.Ensure(__func__)); + } + } + } + } + + LOG("CollectBufferPromises: type %d, total %zu promiese", aType, + promises.Length()); + if (promises.Length()) { + return OmxBufferPromise::All(mOmxTaskQueue, promises); + } + + return OmxBufferPromise::AllPromiseType::CreateAndResolve( + nsTArray(), __func__); +} + +void OmxDataDecoder::PortSettingsChanged() { + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + + if (mPortSettingsChanged == -1 || + mOmxState == OMX_STATETYPE::OMX_StateInvalid) { + return; + } + + // The PortSettingsChanged algorithm: + // + // 1. disable port. + // 2. wait for port buffers return to client and then release these buffers. + // 3. enable port. + // 4. allocate port buffers. + // + + // Disable port. Get port definition if the target port is enable. + OMX_PARAM_PORTDEFINITIONTYPE def; + InitOmxParameter(&def); + def.nPortIndex = mPortSettingsChanged; + + OMX_ERRORTYPE err = + mOmxLayer->GetParameter(OMX_IndexParamPortDefinition, &def, sizeof(def)); + CHECK_OMX_ERR(err); + + RefPtr self = this; + if (def.bEnabled) { + // 1. disable port. + LOG("PortSettingsChanged: disable port %lu", def.nPortIndex); + mOmxLayer + ->SendCommand(OMX_CommandPortDisable, mPortSettingsChanged, nullptr) + ->Then( + mOmxTaskQueue, __func__, + [self, def]() -> RefPtr { + // 3. enable port. + // Send enable port command. + RefPtr p = self->mOmxLayer->SendCommand( + OMX_CommandPortEnable, self->mPortSettingsChanged, nullptr); + + // 4. allocate port buffers. + // Allocate new port buffers. + nsresult rv = self->AllocateBuffers(def.eDir); + if (NS_FAILED(rv)) { + self->NotifyError(OMX_ErrorUndefined, __func__); + } + + return p; + }, + [self](const OmxCommandFailureHolder& aError) { + self->NotifyError(OMX_ErrorUndefined, __func__); + return OmxCommandPromise::CreateAndReject(aError, __func__); + }) + ->Then( + mOmxTaskQueue, __func__, + [self]() { + LOGL("PortSettingsChanged: port settings changed complete"); + // finish port setting changed. + self->mPortSettingsChanged = -1; + self->FillAndEmptyBuffers(); + }, + [self]() { self->NotifyError(OMX_ErrorUndefined, __func__); }); + + // 2. wait for port buffers return to client and then release these buffers. + // + // Port buffers will be returned to client soon once OMX_CommandPortDisable + // command is sent. Then releasing these buffers. + CollectBufferPromises(def.eDir)->Then( + mOmxTaskQueue, __func__, + [self, def]() { + MOZ_ASSERT(self->BuffersCanBeReleased(def.eDir)); + nsresult rv = self->ReleaseBuffers(def.eDir); + if (NS_FAILED(rv)) { + MOZ_RELEASE_ASSERT(0); + self->NotifyError(OMX_ErrorUndefined, __func__); + } + }, + [self]() { self->NotifyError(OMX_ErrorUndefined, __func__); }); + } +} + +void OmxDataDecoder::SendEosBuffer() { + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + + // There is no 'Drain' API in OpenMax, so it needs to wait for output sample + // with EOS flag. However, MediaRawData doesn't provide EOS information, + // so here it generates an empty BufferData with eos OMX_BUFFERFLAG_EOS in + // queue. This behaviour should be compliant with spec, I think... + RefPtr eos_data = new MediaRawData(); + mMediaRawDatas.AppendElement(eos_data); + FillAndEmptyBuffers(); +} + +RefPtr OmxDataDecoder::DoFlush() { + MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); + + mDecodedData = DecodedData(); + mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + mDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + mPerformanceRecorder.Record(std::numeric_limits::max()); + + RefPtr p = mFlushPromise.Ensure(__func__); + + // 1. Call OMX command OMX_CommandFlush in Omx TaskQueue. + // 2. Remove all elements in mMediaRawDatas when flush is completed. + mOmxLayer->SendCommand(OMX_CommandFlush, OMX_ALL, nullptr) + ->Then(mOmxTaskQueue, __func__, this, &OmxDataDecoder::FlushComplete, + &OmxDataDecoder::FlushFailure); + + return p; +} + +void OmxDataDecoder::FlushComplete(OMX_COMMANDTYPE aCommandType) { + mMediaRawDatas.Clear(); + mFlushing = false; + + LOG("Flush complete"); + mFlushPromise.ResolveIfExists(true, __func__); +} + +void OmxDataDecoder::FlushFailure(OmxCommandFailureHolder aFailureHolder) { + mFlushing = false; + mFlushPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); +} + +MediaDataHelper::MediaDataHelper(const TrackInfo* aTrackInfo, + layers::ImageContainer* aImageContainer, + OmxPromiseLayer* aOmxLayer) + : mTrackInfo(aTrackInfo), + mAudioCompactor(mAudioQueue), + mImageContainer(aImageContainer) { + InitOmxParameter(&mOutputPortDef); + mOutputPortDef.nPortIndex = aOmxLayer->OutputPortIndex(); + aOmxLayer->GetParameter(OMX_IndexParamPortDefinition, &mOutputPortDef, + sizeof(mOutputPortDef)); +} + +already_AddRefed MediaDataHelper::GetMediaData( + BufferData* aBufferData, bool& aPlatformDepenentData) { + aPlatformDepenentData = false; + RefPtr data; + + if (mTrackInfo->IsAudio()) { + if (!aBufferData->mBuffer->nFilledLen) { + return nullptr; + } + data = CreateAudioData(aBufferData); + } else if (mTrackInfo->IsVideo()) { + data = aBufferData->GetPlatformMediaData(); + if (data) { + aPlatformDepenentData = true; + } else { + if (!aBufferData->mBuffer->nFilledLen) { + return nullptr; + } + // Get YUV VideoData, it uses more CPU, in most cases, on software codec. + data = CreateYUV420VideoData(aBufferData); + } + + // Update video time code, duration... from the raw data. + VideoData* video(data->As()); + if (aBufferData->mRawData) { + video->mTime = aBufferData->mRawData->mTime; + video->mTimecode = aBufferData->mRawData->mTimecode; + video->mOffset = aBufferData->mRawData->mOffset; + video->mDuration = aBufferData->mRawData->mDuration; + video->mKeyframe = aBufferData->mRawData->mKeyframe; + } + } + + return data.forget(); +} + +already_AddRefed MediaDataHelper::CreateAudioData( + BufferData* aBufferData) { + RefPtr audio; + OMX_BUFFERHEADERTYPE* buf = aBufferData->mBuffer; + const AudioInfo* info = mTrackInfo->GetAsAudioInfo(); + if (buf->nFilledLen) { + uint64_t offset = 0; + uint32_t frames = buf->nFilledLen / (2 * info->mChannels); + if (aBufferData->mRawData) { + offset = aBufferData->mRawData->mOffset; + } + typedef AudioCompactor::NativeCopy OmxCopy; + mAudioCompactor.Push( + offset, buf->nTimeStamp, info->mRate, frames, info->mChannels, + OmxCopy(buf->pBuffer + buf->nOffset, buf->nFilledLen, info->mChannels)); + audio = mAudioQueue.PopFront(); + } + + return audio.forget(); +} + +already_AddRefed MediaDataHelper::CreateYUV420VideoData( + BufferData* aBufferData) { + uint8_t* yuv420p_buffer = (uint8_t*)aBufferData->mBuffer->pBuffer; + int32_t stride = mOutputPortDef.format.video.nStride; + int32_t slice_height = mOutputPortDef.format.video.nSliceHeight; + int32_t width = mTrackInfo->GetAsVideoInfo()->mImage.width; + int32_t height = mTrackInfo->GetAsVideoInfo()->mImage.height; + + // TODO: convert other formats to YUV420. + if (mOutputPortDef.format.video.eColorFormat != + OMX_COLOR_FormatYUV420Planar) { + return nullptr; + } + + size_t yuv420p_y_size = stride * slice_height; + size_t yuv420p_u_size = ((stride + 1) / 2) * ((slice_height + 1) / 2); + uint8_t* yuv420p_y = yuv420p_buffer; + uint8_t* yuv420p_u = yuv420p_y + yuv420p_y_size; + uint8_t* yuv420p_v = yuv420p_u + yuv420p_u_size; + + VideoData::YCbCrBuffer b; + b.mPlanes[0].mData = yuv420p_y; + b.mPlanes[0].mWidth = width; + b.mPlanes[0].mHeight = height; + b.mPlanes[0].mStride = stride; + b.mPlanes[0].mSkip = 0; + + b.mPlanes[1].mData = yuv420p_u; + b.mPlanes[1].mWidth = (width + 1) / 2; + b.mPlanes[1].mHeight = (height + 1) / 2; + b.mPlanes[1].mStride = (stride + 1) / 2; + b.mPlanes[1].mSkip = 0; + + b.mPlanes[2].mData = yuv420p_v; + b.mPlanes[2].mWidth = (width + 1) / 2; + b.mPlanes[2].mHeight = (height + 1) / 2; + b.mPlanes[2].mStride = (stride + 1) / 2; + b.mPlanes[2].mSkip = 0; + + b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT; + + VideoInfo info(*mTrackInfo->GetAsVideoInfo()); + + auto maybeColorSpace = info.mColorSpace; + if (!maybeColorSpace) { + maybeColorSpace = Some(DefaultColorSpace({width, height})); + } + b.mYUVColorSpace = *maybeColorSpace; + + auto maybeColorPrimaries = info.mColorPrimaries; + if (!maybeColorPrimaries) { + maybeColorPrimaries = Some(gfx::ColorSpace2::BT709); + } + b.mColorPrimaries = *maybeColorPrimaries; + + RefPtr data = VideoData::CreateAndCopyData( + info, mImageContainer, + 0, // Filled later by caller. + media::TimeUnit::Zero(), // Filled later by caller. + media::TimeUnit::FromMicroseconds(1), // We don't know the duration. + b, + 0, // Filled later by caller. + media::TimeUnit::FromMicroseconds(-1), info.ImageRect(), nullptr); + + MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, + ("YUV420 VideoData: disp width %d, height %d, pic width %d, height " + "%d, time %lld", + info.mDisplay.width, info.mDisplay.height, info.mImage.width, + info.mImage.height, aBufferData->mBuffer->nTimeStamp)); + + return data.forget(); +} + +} // namespace mozilla diff --git a/dom/media/platforms/omx/OmxDataDecoder.h b/dom/media/platforms/omx/OmxDataDecoder.h new file mode 100644 index 0000000000..b9e5ab27df --- /dev/null +++ b/dom/media/platforms/omx/OmxDataDecoder.h @@ -0,0 +1,220 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(OmxDataDecoder_h_) +# define OmxDataDecoder_h_ + +# include "AudioCompactor.h" +# include "ImageContainer.h" +# include "MediaInfo.h" +# include "OMX_Component.h" +# include "OmxPromiseLayer.h" +# include "PerformanceRecorder.h" +# include "PlatformDecoderModule.h" +# include "mozilla/Monitor.h" +# include "mozilla/StateWatching.h" + +namespace mozilla { + +class MediaDataHelper; + +typedef OmxPromiseLayer::OmxCommandPromise OmxCommandPromise; +typedef OmxPromiseLayer::OmxBufferPromise OmxBufferPromise; +typedef OmxPromiseLayer::OmxBufferFailureHolder OmxBufferFailureHolder; +typedef OmxPromiseLayer::OmxCommandFailureHolder OmxCommandFailureHolder; +typedef OmxPromiseLayer::BufferData BufferData; +typedef OmxPromiseLayer::BUFFERLIST BUFFERLIST; + +DDLoggedTypeDeclNameAndBase(OmxDataDecoder, MediaDataDecoder); + +/* OmxDataDecoder is the major class which performs followings: + * 1. Translate PDM function into OMX commands. + * 2. Keeping the buffers between client and component. + * 3. Manage the OMX state. + * + * From the definition in OpenMax spec. "2.2.1", there are 3 major roles in + * OpenMax IL. + * + * IL client: + * "The IL client may be a layer below the GUI application, such as GStreamer, + * or may be several layers below the GUI layer." + * + * OmxDataDecoder acts as the IL client. + * + * OpenMAX IL component: + * "A component that is intended to wrap functionality that is required in the + * target system." + * + * OmxPromiseLayer acts as the OpenMAX IL component. + * + * OpenMAX IL core: + * "Platform-specific code that has the functionality necessary to locate and + * then load an OpenMAX IL component into main memory." + * + * OmxPlatformLayer acts as the OpenMAX IL core. + */ +class OmxDataDecoder : public MediaDataDecoder, + public DecoderDoctorLifeLogger { + protected: + virtual ~OmxDataDecoder(); + + public: + OmxDataDecoder(const TrackInfo& aTrackInfo, + layers::ImageContainer* aImageContainer, + Maybe aTrackingId); + + RefPtr Init() override; + RefPtr Decode(MediaRawData* aSample) override; + RefPtr Drain() override; + RefPtr Flush() override; + RefPtr Shutdown() override; + + nsCString GetDescriptionName() const override { return "omx decoder"_ns; } + + ConversionRequired NeedsConversion() const override { + return ConversionRequired::kNeedAnnexB; + } + + // Return true if event is handled. + bool Event(OMX_EVENTTYPE aEvent, OMX_U32 aData1, OMX_U32 aData2); + + protected: + void InitializationTask(); + + void ResolveInitPromise(const char* aMethodName); + + void RejectInitPromise(MediaResult aError, const char* aMethodName); + + void OmxStateRunner(); + + void FillAndEmptyBuffers(); + + void FillBufferDone(BufferData* aData); + + void FillBufferFailure(OmxBufferFailureHolder aFailureHolder); + + void EmptyBufferDone(BufferData* aData); + + void EmptyBufferFailure(OmxBufferFailureHolder aFailureHolder); + + void NotifyError( + OMX_ERRORTYPE aOmxError, const char* aLine, + const MediaResult& aError = MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR)); + + // Configure audio/video codec. + // Some codec may just ignore this and rely on codec specific data in + // FillCodecConfigDataToOmx(). + void ConfigCodec(); + + // Sending codec specific data to OMX component. OMX component could send a + // OMX_EventPortSettingsChanged back to client. And then client needs to + // disable port and reallocate buffer. + void FillCodecConfigDataToOmx(); + + void SendEosBuffer(); + + void EndOfStream(); + + // It could be called after codec specific data is sent and component found + // the port format is changed due to different codec specific. + void PortSettingsChanged(); + + void Output(BufferData* aData); + + // Buffer can be released if its status is not OMX_COMPONENT or + // OMX_CLIENT_OUTPUT. + bool BuffersCanBeReleased(OMX_DIRTYPE aType); + + OMX_DIRTYPE GetPortDirection(uint32_t aPortIndex); + + RefPtr DoAsyncShutdown(); + + RefPtr DoFlush(); + + void FlushComplete(OMX_COMMANDTYPE aCommandType); + + void FlushFailure(OmxCommandFailureHolder aFailureHolder); + + BUFFERLIST* GetBuffers(OMX_DIRTYPE aType); + + nsresult AllocateBuffers(OMX_DIRTYPE aType); + + nsresult ReleaseBuffers(OMX_DIRTYPE aType); + + BufferData* FindAvailableBuffer(OMX_DIRTYPE aType); + + // aType could be OMX_DirMax for all types. + RefPtr + CollectBufferPromises(OMX_DIRTYPE aType); + + // The Omx TaskQueue. + RefPtr mOmxTaskQueue; + + nsCOMPtr mThread; + RefPtr mImageContainer; + + WatchManager mWatchManager; + + // It is accessed in omx TaskQueue. + Watchable mOmxState; + + RefPtr mOmxLayer; + + UniquePtr mTrackInfo; + + // It is accessed in both omx and reader TaskQueue. + Atomic mFlushing; + + // It is accessed in Omx/reader TaskQueue. + Atomic mShuttingDown; + + // It is accessed in Omx TaskQeueu. + bool mCheckingInputExhausted; + + // It is accessed in OMX TaskQueue. + MozPromiseHolder mInitPromise; + MozPromiseHolder mDecodePromise; + MozPromiseHolder mDrainPromise; + MozPromiseHolder mFlushPromise; + MozPromiseHolder mShutdownPromise; + // Where decoded samples will be stored until the decode promise is resolved. + DecodedData mDecodedData; + + void CompleteDrain(); + + // It is written in Omx TaskQueue. Read in Omx TaskQueue. + // It value means the port index which port settings is changed. + // -1 means no port setting changed. + // + // Note: when port setting changed, there should be no buffer operations + // via EmptyBuffer or FillBuffer. + Watchable mPortSettingsChanged; + + // It is access in Omx TaskQueue. + nsTArray> mMediaRawDatas; + + BUFFERLIST mInPortBuffers; + + BUFFERLIST mOutPortBuffers; + + RefPtr mMediaDataHelper; + + const Maybe mTrackingId; + + // Accessed on Omx TaskQueue + PerformanceRecorderMulti mPerformanceRecorder; +}; + +template +void InitOmxParameter(T* aParam) { + PodZero(aParam); + aParam->nSize = sizeof(T); + aParam->nVersion.s.nVersionMajor = 1; +} + +} // namespace mozilla + +#endif /* OmxDataDecoder_h_ */ diff --git a/dom/media/platforms/omx/OmxDecoderModule.cpp b/dom/media/platforms/omx/OmxDecoderModule.cpp new file mode 100644 index 0000000000..2267e66cc3 --- /dev/null +++ b/dom/media/platforms/omx/OmxDecoderModule.cpp @@ -0,0 +1,59 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "OmxDecoderModule.h" + +#include "OmxDataDecoder.h" +#include "OmxPlatformLayer.h" + +#ifdef MOZ_OMX +# include "PureOmxPlatformLayer.h" +#endif + +namespace mozilla { + +/* static */ +bool OmxDecoderModule::Init() { +#ifdef MOZ_OMX + return PureOmxPlatformLayer::Init(); +#endif + return false; +} + +OmxDecoderModule* OmxDecoderModule::Create() { +#ifdef MOZ_OMX + if (Init()) { + return new OmxDecoderModule(); + } +#endif + return nullptr; +} + +already_AddRefed OmxDecoderModule::CreateVideoDecoder( + const CreateDecoderParams& aParams) { + RefPtr decoder = new OmxDataDecoder( + aParams.mConfig, aParams.mImageContainer, aParams.mTrackingId); + return decoder.forget(); +} + +already_AddRefed OmxDecoderModule::CreateAudioDecoder( + const CreateDecoderParams& aParams) { + RefPtr decoder = + new OmxDataDecoder(aParams.mConfig, nullptr, aParams.mTrackingId); + return decoder.forget(); +} + +media::DecodeSupportSet OmxDecoderModule::SupportsMimeType( + const nsACString& aMimeType, DecoderDoctorDiagnostics* aDiagnostics) const { + if (OmxPlatformLayer::SupportsMimeType(aMimeType)) { + // TODO: Note that we do not yet distinguish between SW/HW decode support. + // Will be done in bug 1754239. + return media::DecodeSupport::SoftwareDecode; + } + return media::DecodeSupport::Unsupported; +} + +} // namespace mozilla diff --git a/dom/media/platforms/omx/OmxDecoderModule.h b/dom/media/platforms/omx/OmxDecoderModule.h new file mode 100644 index 0000000000..04fa809e54 --- /dev/null +++ b/dom/media/platforms/omx/OmxDecoderModule.h @@ -0,0 +1,33 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(OmxDecoderModule_h_) +# define OmxDecoderModule_h_ + +# include "PlatformDecoderModule.h" + +namespace mozilla { + +class OmxDecoderModule : public PlatformDecoderModule { + public: + // Called on main thread. + static bool Init(); + static OmxDecoderModule* Create(); + + already_AddRefed CreateVideoDecoder( + const CreateDecoderParams& aParams) override; + + already_AddRefed CreateAudioDecoder( + const CreateDecoderParams& aParams) override; + + media::DecodeSupportSet SupportsMimeType( + const nsACString& aMimeType, + DecoderDoctorDiagnostics* aDiagnostics) const override; +}; + +} // namespace mozilla + +#endif // OmxDecoderModule_h_ diff --git a/dom/media/platforms/omx/OmxFunctionList.h b/dom/media/platforms/omx/OmxFunctionList.h new file mode 100644 index 0000000000..e1e92bfe65 --- /dev/null +++ b/dom/media/platforms/omx/OmxFunctionList.h @@ -0,0 +1,13 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +OMX_FUNC(OMX_Init) +OMX_FUNC(OMX_Deinit) +OMX_FUNC(OMX_GetHandle) +OMX_FUNC(OMX_FreeHandle) +OMX_FUNC(OMX_ComponentNameEnum) +OMX_FUNC(OMX_GetComponentsOfRole) +OMX_FUNC(OMX_GetRolesOfComponent) +OMX_FUNC(OMX_SetupTunnel) +OMX_FUNC(OMX_GetContentPipe) diff --git a/dom/media/platforms/omx/OmxPlatformLayer.cpp b/dom/media/platforms/omx/OmxPlatformLayer.cpp new file mode 100644 index 0000000000..dc3ccc0979 --- /dev/null +++ b/dom/media/platforms/omx/OmxPlatformLayer.cpp @@ -0,0 +1,307 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "OmxPlatformLayer.h" + +#include "OmxDataDecoder.h" +#include "OMX_Component.h" +#include "OMX_VideoExt.h" // For VP8. + +#ifdef MOZ_OMX +# include "PureOmxPlatformLayer.h" +#endif + +#include "VPXDecoder.h" + +#ifdef LOG +# undef LOG +#endif + +#define LOG(arg, ...) \ + MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, \ + ("OmxPlatformLayer -- %s: " arg, __func__, ##__VA_ARGS__)) + +#define RETURN_IF_ERR(err) \ + if (err != OMX_ErrorNone) { \ + LOG("error: 0x%08x", err); \ + return err; \ + } + +// Common OMX decoder configuration code. +namespace mozilla { + +// This helper class encapsulates the details of component parameters setting +// for different OMX audio & video codecs. +template +class OmxConfig { + public: + virtual ~OmxConfig() = default; + // Subclasses should implement this method to configure the codec. + virtual OMX_ERRORTYPE Apply(OmxPlatformLayer& aOmx, + const ParamType& aParam) = 0; +}; + +typedef OmxConfig OmxAudioConfig; +typedef OmxConfig OmxVideoConfig; + +template +UniquePtr ConfigForMime(const nsACString&); + +static OMX_ERRORTYPE ConfigAudioOutputPort(OmxPlatformLayer& aOmx, + const AudioInfo& aInfo) { + OMX_PARAM_PORTDEFINITIONTYPE def; + InitOmxParameter(&def); + def.nPortIndex = aOmx.OutputPortIndex(); + OMX_ERRORTYPE err = + aOmx.GetParameter(OMX_IndexParamPortDefinition, &def, sizeof(def)); + RETURN_IF_ERR(err); + + def.format.audio.eEncoding = OMX_AUDIO_CodingPCM; + err = aOmx.SetParameter(OMX_IndexParamPortDefinition, &def, sizeof(def)); + RETURN_IF_ERR(err); + + OMX_AUDIO_PARAM_PCMMODETYPE pcmParams; + InitOmxParameter(&pcmParams); + pcmParams.nPortIndex = def.nPortIndex; + err = + aOmx.GetParameter(OMX_IndexParamAudioPcm, &pcmParams, sizeof(pcmParams)); + RETURN_IF_ERR(err); + + pcmParams.nChannels = aInfo.mChannels; + pcmParams.eNumData = OMX_NumericalDataSigned; + pcmParams.bInterleaved = OMX_TRUE; + pcmParams.nBitPerSample = 16; + pcmParams.nSamplingRate = aInfo.mRate; + pcmParams.ePCMMode = OMX_AUDIO_PCMModeLinear; + err = + aOmx.SetParameter(OMX_IndexParamAudioPcm, &pcmParams, sizeof(pcmParams)); + RETURN_IF_ERR(err); + + LOG("Config OMX_IndexParamAudioPcm, channel %lu, sample rate %lu", + pcmParams.nChannels, pcmParams.nSamplingRate); + + return OMX_ErrorNone; +} + +class OmxAacConfig : public OmxAudioConfig { + public: + OMX_ERRORTYPE Apply(OmxPlatformLayer& aOmx, const AudioInfo& aInfo) override { + OMX_AUDIO_PARAM_AACPROFILETYPE aacProfile; + InitOmxParameter(&aacProfile); + aacProfile.nPortIndex = aOmx.InputPortIndex(); + OMX_ERRORTYPE err = aOmx.GetParameter(OMX_IndexParamAudioAac, &aacProfile, + sizeof(aacProfile)); + RETURN_IF_ERR(err); + + aacProfile.nChannels = aInfo.mChannels; + aacProfile.nSampleRate = aInfo.mRate; + aacProfile.eAACProfile = + static_cast(aInfo.mProfile); + err = aOmx.SetParameter(OMX_IndexParamAudioAac, &aacProfile, + sizeof(aacProfile)); + RETURN_IF_ERR(err); + + LOG("Config OMX_IndexParamAudioAac, channel %lu, sample rate %lu, profile " + "%d", + aacProfile.nChannels, aacProfile.nSampleRate, aacProfile.eAACProfile); + + return ConfigAudioOutputPort(aOmx, aInfo); + } +}; + +class OmxMp3Config : public OmxAudioConfig { + public: + OMX_ERRORTYPE Apply(OmxPlatformLayer& aOmx, const AudioInfo& aInfo) override { + OMX_AUDIO_PARAM_MP3TYPE mp3Param; + InitOmxParameter(&mp3Param); + mp3Param.nPortIndex = aOmx.InputPortIndex(); + OMX_ERRORTYPE err = + aOmx.GetParameter(OMX_IndexParamAudioMp3, &mp3Param, sizeof(mp3Param)); + RETURN_IF_ERR(err); + + mp3Param.nChannels = aInfo.mChannels; + mp3Param.nSampleRate = aInfo.mRate; + err = + aOmx.SetParameter(OMX_IndexParamAudioMp3, &mp3Param, sizeof(mp3Param)); + RETURN_IF_ERR(err); + + LOG("Config OMX_IndexParamAudioMp3, channel %lu, sample rate %lu", + mp3Param.nChannels, mp3Param.nSampleRate); + + return ConfigAudioOutputPort(aOmx, aInfo); + } +}; + +enum OmxAmrSampleRate { + kNarrowBand = 8000, + kWideBand = 16000, +}; + +template +class OmxAmrConfig : public OmxAudioConfig { + public: + OMX_ERRORTYPE Apply(OmxPlatformLayer& aOmx, const AudioInfo& aInfo) override { + OMX_AUDIO_PARAM_AMRTYPE def; + InitOmxParameter(&def); + def.nPortIndex = aOmx.InputPortIndex(); + OMX_ERRORTYPE err = + aOmx.GetParameter(OMX_IndexParamAudioAmr, &def, sizeof(def)); + RETURN_IF_ERR(err); + + def.eAMRFrameFormat = OMX_AUDIO_AMRFrameFormatFSF; + err = aOmx.SetParameter(OMX_IndexParamAudioAmr, &def, sizeof(def)); + RETURN_IF_ERR(err); + + MOZ_ASSERT(aInfo.mChannels == 1); + MOZ_ASSERT(aInfo.mRate == R); + + return ConfigAudioOutputPort(aOmx, aInfo); + } +}; + +template <> +UniquePtr ConfigForMime(const nsACString& aMimeType) { + UniquePtr conf; + + if (OmxPlatformLayer::SupportsMimeType(aMimeType)) { + if (aMimeType.EqualsLiteral("audio/mp4a-latm")) { + conf.reset(new OmxAacConfig()); + } else if (aMimeType.EqualsLiteral("audio/mp3") || + aMimeType.EqualsLiteral("audio/mpeg")) { + conf.reset(new OmxMp3Config()); + } else if (aMimeType.EqualsLiteral("audio/3gpp")) { + conf.reset(new OmxAmrConfig()); + } else if (aMimeType.EqualsLiteral("audio/amr-wb")) { + conf.reset(new OmxAmrConfig()); + } + } + return conf; +} + +// There should be a better way to calculate it. +#define MIN_VIDEO_INPUT_BUFFER_SIZE 64 * 1024 + +class OmxCommonVideoConfig : public OmxVideoConfig { + public: + explicit OmxCommonVideoConfig() : OmxVideoConfig() {} + + OMX_ERRORTYPE Apply(OmxPlatformLayer& aOmx, const VideoInfo& aInfo) override { + OMX_ERRORTYPE err = OMX_ErrorNone; + OMX_PARAM_PORTDEFINITIONTYPE def; + + // Set up in/out port definition. + nsTArray ports; + aOmx.GetPortIndices(ports); + for (auto idx : ports) { + InitOmxParameter(&def); + def.nPortIndex = idx; + err = aOmx.GetParameter(OMX_IndexParamPortDefinition, &def, sizeof(def)); + RETURN_IF_ERR(err); + + def.format.video.nFrameWidth = aInfo.mDisplay.width; + def.format.video.nFrameHeight = aInfo.mDisplay.height; + def.format.video.nStride = aInfo.mImage.width; + def.format.video.nSliceHeight = aInfo.mImage.height; + + if (def.eDir == OMX_DirInput) { + def.format.video.eCompressionFormat = aOmx.CompressionFormat(); + def.format.video.eColorFormat = OMX_COLOR_FormatUnused; + if (def.nBufferSize < MIN_VIDEO_INPUT_BUFFER_SIZE) { + def.nBufferSize = aInfo.mImage.width * aInfo.mImage.height; + LOG("Change input buffer size to %lu", def.nBufferSize); + } + } else { + def.format.video.eCompressionFormat = OMX_VIDEO_CodingUnused; + } + + err = aOmx.SetParameter(OMX_IndexParamPortDefinition, &def, sizeof(def)); + } + return err; + } +}; + +template <> +UniquePtr ConfigForMime(const nsACString& aMimeType) { + UniquePtr conf; + + if (OmxPlatformLayer::SupportsMimeType(aMimeType)) { + conf.reset(new OmxCommonVideoConfig()); + } + return conf; +} + +OMX_ERRORTYPE +OmxPlatformLayer::Config() { + MOZ_ASSERT(mInfo); + + OMX_PORT_PARAM_TYPE portParam; + InitOmxParameter(&portParam); + if (mInfo->IsAudio()) { + GetParameter(OMX_IndexParamAudioInit, &portParam, sizeof(portParam)); + mStartPortNumber = portParam.nStartPortNumber; + UniquePtr conf( + ConfigForMime(mInfo->mMimeType)); + MOZ_RELEASE_ASSERT(conf.get()); + return conf->Apply(*this, *(mInfo->GetAsAudioInfo())); + } else if (mInfo->IsVideo()) { + GetParameter(OMX_IndexParamVideoInit, &portParam, sizeof(portParam)); + UniquePtr conf( + ConfigForMime(mInfo->mMimeType)); + MOZ_RELEASE_ASSERT(conf.get()); + return conf->Apply(*this, *(mInfo->GetAsVideoInfo())); + } else { + MOZ_ASSERT_UNREACHABLE("non-AV data (text?) is not supported."); + return OMX_ErrorNotImplemented; + } +} + +OMX_VIDEO_CODINGTYPE +OmxPlatformLayer::CompressionFormat() { + MOZ_ASSERT(mInfo); + + if (mInfo->mMimeType.EqualsLiteral("video/avc")) { + return OMX_VIDEO_CodingAVC; + } else if (mInfo->mMimeType.EqualsLiteral("video/mp4v-es") || + mInfo->mMimeType.EqualsLiteral("video/mp4")) { + return OMX_VIDEO_CodingMPEG4; + } else if (mInfo->mMimeType.EqualsLiteral("video/3gpp")) { + return OMX_VIDEO_CodingH263; + } else if (VPXDecoder::IsVP8(mInfo->mMimeType)) { + return static_cast(OMX_VIDEO_CodingVP8); + } else { + MOZ_ASSERT_UNREACHABLE("Unsupported compression format"); + return OMX_VIDEO_CodingUnused; + } +} + +// Implementations for different platforms will be defined in their own files. +#if defined(MOZ_OMX) + +bool OmxPlatformLayer::SupportsMimeType(const nsACString& aMimeType) { + return PureOmxPlatformLayer::SupportsMimeType(aMimeType); +} + +OmxPlatformLayer* OmxPlatformLayer::Create( + OmxDataDecoder* aDataDecoder, OmxPromiseLayer* aPromiseLayer, + TaskQueue* aTaskQueue, layers::ImageContainer* aImageContainer) { + return new PureOmxPlatformLayer(aDataDecoder, aPromiseLayer, aTaskQueue, + aImageContainer); +} + +#else // For platforms without OMX IL support. + +bool OmxPlatformLayer::SupportsMimeType(const nsACString& aMimeType) { + return false; +} + +OmxPlatformLayer* OmxPlatformLayer::Create( + OmxDataDecoder* aDataDecoder, OmxPromiseLayer* aPromiseLayer, + TaskQueue* aTaskQueue, layers::ImageContainer* aImageContainer) { + return nullptr; +} + +#endif +} // namespace mozilla diff --git a/dom/media/platforms/omx/OmxPlatformLayer.h b/dom/media/platforms/omx/OmxPlatformLayer.h new file mode 100644 index 0000000000..53f3dccbca --- /dev/null +++ b/dom/media/platforms/omx/OmxPlatformLayer.h @@ -0,0 +1,103 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(OmxPlatformLayer_h_) +# define OmxPlatformLayer_h_ + +# include "OMX_Core.h" +# include "OMX_Types.h" +# include "OMX_Video.h" + +# include "nsStringFwd.h" +# include "OmxPromiseLayer.h" + +namespace mozilla { + +class TaskQueue; +class TrackInfo; + +/* + * This class the the abstract layer of the platform OpenMax IL implementation. + * + * For some platform like andoird, it exposures its OpenMax IL via IOMX which + * is definitions are different comparing to standard. + * For other platforms like Raspberry Pi, it will be easy to implement this + * layer with the standard OpenMax IL api. + */ +class OmxPlatformLayer { + public: + typedef OmxPromiseLayer::BUFFERLIST BUFFERLIST; + typedef OmxPromiseLayer::BufferData BufferData; + + virtual OMX_ERRORTYPE InitOmxToStateLoaded(const TrackInfo* aInfo) = 0; + + OMX_ERRORTYPE Config(); + + virtual OMX_ERRORTYPE EmptyThisBuffer(BufferData* aData) = 0; + + virtual OMX_ERRORTYPE FillThisBuffer(BufferData* aData) = 0; + + virtual OMX_ERRORTYPE SendCommand(OMX_COMMANDTYPE aCmd, OMX_U32 aParam1, + OMX_PTR aCmdData) = 0; + + // Buffer could be platform dependent. Therefore, derived class needs to + // implement its owned buffer allocate/release API according to its platform + // type. + virtual nsresult AllocateOmxBuffer(OMX_DIRTYPE aType, + BUFFERLIST* aBufferList) = 0; + + virtual nsresult ReleaseOmxBuffer(OMX_DIRTYPE aType, + BUFFERLIST* aBufferList) = 0; + + virtual OMX_ERRORTYPE GetState(OMX_STATETYPE* aType) = 0; + + virtual OMX_ERRORTYPE GetParameter(OMX_INDEXTYPE aParamIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize) = 0; + + virtual OMX_ERRORTYPE SetParameter(OMX_INDEXTYPE nIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize) = 0; + + virtual nsresult Shutdown() = 0; + + virtual ~OmxPlatformLayer() = default; + + // For decoders, input port index is start port number and output port is + // next. See OpenMAX IL spec v1.1.2 section 8.6.1 & 8.8.1. + OMX_U32 InputPortIndex() { return mStartPortNumber; } + + OMX_U32 OutputPortIndex() { return mStartPortNumber + 1; } + + void GetPortIndices(nsTArray& aPortIndex) { + aPortIndex.AppendElement(InputPortIndex()); + aPortIndex.AppendElement(OutputPortIndex()); + } + + virtual OMX_VIDEO_CODINGTYPE CompressionFormat(); + + // Check if the platform implementation supports given MIME type. + static bool SupportsMimeType(const nsACString& aMimeType); + + // Hide the details of creating implementation objects for different + // platforms. + static OmxPlatformLayer* Create(OmxDataDecoder* aDataDecoder, + OmxPromiseLayer* aPromiseLayer, + TaskQueue* aTaskQueue, + layers::ImageContainer* aImageContainer); + + protected: + OmxPlatformLayer() : mInfo(nullptr), mStartPortNumber(0) {} + + // The pointee is held by |OmxDataDecoder::mTrackInfo| and will outlive this + // pointer. + const TrackInfo* mInfo; + OMX_U32 mStartPortNumber; +}; + +} // namespace mozilla + +#endif // OmxPlatformLayer_h_ diff --git a/dom/media/platforms/omx/OmxPromiseLayer.cpp b/dom/media/platforms/omx/OmxPromiseLayer.cpp new file mode 100644 index 0000000000..74d1fe15f8 --- /dev/null +++ b/dom/media/platforms/omx/OmxPromiseLayer.cpp @@ -0,0 +1,355 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "OmxPromiseLayer.h" + +#include "ImageContainer.h" + +#include "OmxDataDecoder.h" +#include "OmxPlatformLayer.h" + +#ifdef LOG +# undef LOG +#endif + +#define LOG(arg, ...) \ + MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, \ + ("OmxPromiseLayer(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) + +namespace mozilla { + +OmxPromiseLayer::OmxPromiseLayer(TaskQueue* aTaskQueue, + OmxDataDecoder* aDataDecoder, + layers::ImageContainer* aImageContainer) + : mTaskQueue(aTaskQueue) { + mPlatformLayer.reset(OmxPlatformLayer::Create(aDataDecoder, this, aTaskQueue, + aImageContainer)); + MOZ_ASSERT(!!mPlatformLayer); +} + +RefPtr OmxPromiseLayer::Init( + const TrackInfo* aInfo) { + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + + OMX_ERRORTYPE err = mPlatformLayer->InitOmxToStateLoaded(aInfo); + if (err != OMX_ErrorNone) { + OmxCommandFailureHolder failure(OMX_ErrorUndefined, OMX_CommandStateSet); + return OmxCommandPromise::CreateAndReject(failure, __func__); + } + + OMX_STATETYPE state = GetState(); + if (state == OMX_StateLoaded) { + return OmxCommandPromise::CreateAndResolve(OMX_CommandStateSet, __func__); + } + if (state == OMX_StateIdle) { + return SendCommand(OMX_CommandStateSet, OMX_StateIdle, nullptr); + } + + OmxCommandFailureHolder failure(OMX_ErrorUndefined, OMX_CommandStateSet); + return OmxCommandPromise::CreateAndReject(failure, __func__); +} + +OMX_ERRORTYPE +OmxPromiseLayer::Config() { + MOZ_ASSERT(GetState() == OMX_StateLoaded); + + return mPlatformLayer->Config(); +} + +RefPtr OmxPromiseLayer::FillBuffer( + BufferData* aData) { + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + LOG("buffer %p", aData->mBuffer); + + RefPtr p = aData->mPromise.Ensure(__func__); + + OMX_ERRORTYPE err = mPlatformLayer->FillThisBuffer(aData); + + if (err != OMX_ErrorNone) { + OmxBufferFailureHolder failure(err, aData); + aData->mPromise.Reject(std::move(failure), __func__); + } else { + aData->mStatus = BufferData::BufferStatus::OMX_COMPONENT; + GetBufferHolders(OMX_DirOutput)->AppendElement(aData); + } + + return p; +} + +RefPtr OmxPromiseLayer::EmptyBuffer( + BufferData* aData) { + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + LOG("buffer %p, size %lu", aData->mBuffer, aData->mBuffer->nFilledLen); + + RefPtr p = aData->mPromise.Ensure(__func__); + + OMX_ERRORTYPE err = mPlatformLayer->EmptyThisBuffer(aData); + + if (err != OMX_ErrorNone) { + OmxBufferFailureHolder failure(err, aData); + aData->mPromise.Reject(std::move(failure), __func__); + } else { + if (aData->mRawData) { + mRawDatas.AppendElement(std::move(aData->mRawData)); + } + aData->mStatus = BufferData::BufferStatus::OMX_COMPONENT; + GetBufferHolders(OMX_DirInput)->AppendElement(aData); + } + + return p; +} + +OmxPromiseLayer::BUFFERLIST* OmxPromiseLayer::GetBufferHolders( + OMX_DIRTYPE aType) { + MOZ_ASSERT(aType == OMX_DirInput || aType == OMX_DirOutput); + + if (aType == OMX_DirInput) { + return &mInbufferHolders; + } + + return &mOutbufferHolders; +} + +already_AddRefed OmxPromiseLayer::FindAndRemoveRawData( + OMX_TICKS aTimecode) { + for (auto raw : mRawDatas) { + if (raw->mTime.ToMicroseconds() == aTimecode) { + mRawDatas.RemoveElement(raw); + return raw.forget(); + } + } + return nullptr; +} + +already_AddRefed OmxPromiseLayer::FindAndRemoveBufferHolder( + OMX_DIRTYPE aType, BufferData::BufferID aId) { + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + + RefPtr holder; + BUFFERLIST* holders = GetBufferHolders(aType); + + for (uint32_t i = 0; i < holders->Length(); i++) { + if (holders->ElementAt(i)->ID() == aId) { + holder = holders->ElementAt(i); + holders->RemoveElementAt(i); + return holder.forget(); + } + } + + return nullptr; +} + +already_AddRefed OmxPromiseLayer::FindBufferById( + OMX_DIRTYPE aType, BufferData::BufferID aId) { + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + + RefPtr holder; + BUFFERLIST* holders = GetBufferHolders(aType); + + for (uint32_t i = 0; i < holders->Length(); i++) { + if (holders->ElementAt(i)->ID() == aId) { + holder = holders->ElementAt(i); + return holder.forget(); + } + } + + return nullptr; +} + +void OmxPromiseLayer::EmptyFillBufferDone(OMX_DIRTYPE aType, + BufferData* aData) { + if (aData) { + LOG("type %d, buffer %p", aType, aData->mBuffer); + if (aType == OMX_DirOutput) { + aData->mRawData = nullptr; + aData->mRawData = FindAndRemoveRawData(aData->mBuffer->nTimeStamp); + } + aData->mStatus = BufferData::BufferStatus::OMX_CLIENT; + aData->mPromise.Resolve(aData, __func__); + } else { + LOG("type %d, no buffer", aType); + } +} + +void OmxPromiseLayer::EmptyFillBufferDone(OMX_DIRTYPE aType, + BufferData::BufferID aID) { + RefPtr holder = FindAndRemoveBufferHolder(aType, aID); + EmptyFillBufferDone(aType, holder); +} + +RefPtr OmxPromiseLayer::SendCommand( + OMX_COMMANDTYPE aCmd, OMX_U32 aParam1, OMX_PTR aCmdData) { + if (aCmd == OMX_CommandFlush) { + // It doesn't support another flush commands before previous one is + // completed. + MOZ_RELEASE_ASSERT(!mFlushCommands.Length()); + + // Some coomponents don't send event with OMX_ALL, they send flush complete + // event with input port and another event for output port. + // In prupose of better compatibility, we interpret the OMX_ALL to + // OMX_DirInput and OMX_DirOutput flush separately. + OMX_DIRTYPE types[] = {OMX_DIRTYPE::OMX_DirInput, + OMX_DIRTYPE::OMX_DirOutput}; + for (const auto type : types) { + if ((aParam1 == type) || (aParam1 == OMX_ALL)) { + mFlushCommands.AppendElement(FlushCommand({type, aCmdData})); + } + + if (type == OMX_DirInput) { + // Clear all buffered raw data. + mRawDatas.Clear(); + } + } + + // Don't overlay more than one flush command, some components can't overlay + // flush commands. So here we send another flush after receiving the + // previous flush completed event. + if (mFlushCommands.Length()) { + OMX_ERRORTYPE err = mPlatformLayer->SendCommand( + OMX_CommandFlush, mFlushCommands.ElementAt(0).type, + mFlushCommands.ElementAt(0).cmd); + if (err != OMX_ErrorNone) { + OmxCommandFailureHolder failure(OMX_ErrorNotReady, OMX_CommandFlush); + return OmxCommandPromise::CreateAndReject(failure, __func__); + } + } else { + LOG("OMX_CommandFlush parameter error"); + OmxCommandFailureHolder failure(OMX_ErrorNotReady, OMX_CommandFlush); + return OmxCommandPromise::CreateAndReject(failure, __func__); + } + } else { + OMX_ERRORTYPE err = mPlatformLayer->SendCommand(aCmd, aParam1, aCmdData); + if (err != OMX_ErrorNone) { + OmxCommandFailureHolder failure(OMX_ErrorNotReady, aCmd); + return OmxCommandPromise::CreateAndReject(failure, __func__); + } + } + + RefPtr p; + if (aCmd == OMX_CommandStateSet) { + p = mCommandStatePromise.Ensure(__func__); + } else if (aCmd == OMX_CommandFlush) { + p = mFlushPromise.Ensure(__func__); + } else if (aCmd == OMX_CommandPortEnable) { + p = mPortEnablePromise.Ensure(__func__); + } else if (aCmd == OMX_CommandPortDisable) { + p = mPortDisablePromise.Ensure(__func__); + } else { + LOG("error unsupport command"); + MOZ_ASSERT(0); + } + + return p; +} + +bool OmxPromiseLayer::Event(OMX_EVENTTYPE aEvent, OMX_U32 aData1, + OMX_U32 aData2) { + OMX_COMMANDTYPE cmd = (OMX_COMMANDTYPE)aData1; + switch (aEvent) { + case OMX_EventCmdComplete: { + if (cmd == OMX_CommandStateSet) { + mCommandStatePromise.Resolve(OMX_CommandStateSet, __func__); + } else if (cmd == OMX_CommandFlush) { + MOZ_RELEASE_ASSERT(mFlushCommands.ElementAt(0).type == aData2); + LOG("OMX_CommandFlush completed port type %lu", aData2); + mFlushCommands.RemoveElementAt(0); + + // Sending next flush command. + if (mFlushCommands.Length()) { + OMX_ERRORTYPE err = mPlatformLayer->SendCommand( + OMX_CommandFlush, mFlushCommands.ElementAt(0).type, + mFlushCommands.ElementAt(0).cmd); + if (err != OMX_ErrorNone) { + OmxCommandFailureHolder failure(OMX_ErrorNotReady, + OMX_CommandFlush); + mFlushPromise.Reject(failure, __func__); + } + } else { + mFlushPromise.Resolve(OMX_CommandFlush, __func__); + } + } else if (cmd == OMX_CommandPortDisable) { + mPortDisablePromise.Resolve(OMX_CommandPortDisable, __func__); + } else if (cmd == OMX_CommandPortEnable) { + mPortEnablePromise.Resolve(OMX_CommandPortEnable, __func__); + } + break; + } + case OMX_EventError: { + if (cmd == OMX_CommandStateSet) { + OmxCommandFailureHolder failure(OMX_ErrorUndefined, + OMX_CommandStateSet); + mCommandStatePromise.Reject(failure, __func__); + } else if (cmd == OMX_CommandFlush) { + OmxCommandFailureHolder failure(OMX_ErrorUndefined, OMX_CommandFlush); + mFlushPromise.Reject(failure, __func__); + } else if (cmd == OMX_CommandPortDisable) { + OmxCommandFailureHolder failure(OMX_ErrorUndefined, + OMX_CommandPortDisable); + mPortDisablePromise.Reject(failure, __func__); + } else if (cmd == OMX_CommandPortEnable) { + OmxCommandFailureHolder failure(OMX_ErrorUndefined, + OMX_CommandPortEnable); + mPortEnablePromise.Reject(failure, __func__); + } else { + return false; + } + break; + } + default: { + return false; + } + } + return true; +} + +nsresult OmxPromiseLayer::AllocateOmxBuffer(OMX_DIRTYPE aType, + BUFFERLIST* aBuffers) { + return mPlatformLayer->AllocateOmxBuffer(aType, aBuffers); +} + +nsresult OmxPromiseLayer::ReleaseOmxBuffer(OMX_DIRTYPE aType, + BUFFERLIST* aBuffers) { + return mPlatformLayer->ReleaseOmxBuffer(aType, aBuffers); +} + +OMX_STATETYPE +OmxPromiseLayer::GetState() { + OMX_STATETYPE state; + OMX_ERRORTYPE err = mPlatformLayer->GetState(&state); + return err == OMX_ErrorNone ? state : OMX_StateInvalid; +} + +OMX_ERRORTYPE +OmxPromiseLayer::GetParameter(OMX_INDEXTYPE aParamIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize) { + return mPlatformLayer->GetParameter(aParamIndex, aComponentParameterStructure, + aComponentParameterSize); +} + +OMX_ERRORTYPE +OmxPromiseLayer::SetParameter(OMX_INDEXTYPE aParamIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize) { + return mPlatformLayer->SetParameter(aParamIndex, aComponentParameterStructure, + aComponentParameterSize); +} + +OMX_U32 +OmxPromiseLayer::InputPortIndex() { return mPlatformLayer->InputPortIndex(); } + +OMX_U32 +OmxPromiseLayer::OutputPortIndex() { return mPlatformLayer->OutputPortIndex(); } + +nsresult OmxPromiseLayer::Shutdown() { + LOG(""); + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + MOZ_ASSERT(!GetBufferHolders(OMX_DirInput)->Length()); + MOZ_ASSERT(!GetBufferHolders(OMX_DirOutput)->Length()); + return mPlatformLayer->Shutdown(); +} + +} // namespace mozilla diff --git a/dom/media/platforms/omx/OmxPromiseLayer.h b/dom/media/platforms/omx/OmxPromiseLayer.h new file mode 100644 index 0000000000..82a1b3c268 --- /dev/null +++ b/dom/media/platforms/omx/OmxPromiseLayer.h @@ -0,0 +1,243 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(OmxPromiseLayer_h_) +# define OmxPromiseLayer_h_ + +# include "mozilla/MozPromise.h" +# include "mozilla/TaskQueue.h" + +# include "OMX_Core.h" +# include "OMX_Types.h" + +namespace mozilla { + +namespace layers { +class ImageContainer; +} + +class MediaData; +class MediaRawData; +class OmxDataDecoder; +class OmxPlatformLayer; +class TrackInfo; + +/* This class acts as a middle layer between OmxDataDecoder and the underlying + * OmxPlatformLayer. + * + * This class has two purposes: + * 1. Using promise instead of OpenMax async callback function. + * For example, OmxCommandPromise is used for OpenMax IL SendCommand. + * 2. Manage the buffer exchanged between client and component. + * Because omx buffer works crossing threads, so each omx buffer has its own + * promise, it is defined in BufferData. + * + * All of functions and members in this class should be run in the same + * TaskQueue. + */ +class OmxPromiseLayer { + protected: + virtual ~OmxPromiseLayer() = default; + + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(OmxPromiseLayer) + + OmxPromiseLayer(TaskQueue* aTaskQueue, OmxDataDecoder* aDataDecoder, + layers::ImageContainer* aImageContainer); + + class BufferData; + + typedef nsTArray> BUFFERLIST; + + class OmxBufferFailureHolder { + public: + OmxBufferFailureHolder(OMX_ERRORTYPE aError, BufferData* aBuffer) + : mError(aError), mBuffer(aBuffer) {} + + OMX_ERRORTYPE mError; + BufferData* mBuffer; + }; + + typedef MozPromise + OmxBufferPromise; + + class OmxCommandFailureHolder { + public: + OmxCommandFailureHolder(OMX_ERRORTYPE aErrorType, + OMX_COMMANDTYPE aCommandType) + : mErrorType(aErrorType), mCommandType(aCommandType) {} + + OMX_ERRORTYPE mErrorType; + OMX_COMMANDTYPE mCommandType; + }; + + typedef MozPromise + OmxCommandPromise; + + typedef MozPromise + OmxPortConfigPromise; + + // TODO: maybe a generic promise is good enough for this case? + RefPtr Init(const TrackInfo* aInfo); + + OMX_ERRORTYPE Config(); + + RefPtr FillBuffer(BufferData* aData); + + RefPtr EmptyBuffer(BufferData* aData); + + RefPtr SendCommand(OMX_COMMANDTYPE aCmd, OMX_U32 aParam1, + OMX_PTR aCmdData); + + nsresult AllocateOmxBuffer(OMX_DIRTYPE aType, BUFFERLIST* aBuffers); + + nsresult ReleaseOmxBuffer(OMX_DIRTYPE aType, BUFFERLIST* aBuffers); + + OMX_STATETYPE GetState(); + + OMX_ERRORTYPE GetParameter(OMX_INDEXTYPE aParamIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize); + + OMX_ERRORTYPE SetParameter(OMX_INDEXTYPE nIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize); + + OMX_U32 InputPortIndex(); + + OMX_U32 OutputPortIndex(); + + nsresult Shutdown(); + + // BufferData maintains the status of OMX buffer (OMX_BUFFERHEADERTYPE). + // mStatus tracks the buffer owner. + // And a promise because OMX buffer working among different threads. + class BufferData { + protected: + virtual ~BufferData() = default; + + public: + explicit BufferData(OMX_BUFFERHEADERTYPE* aBuffer) + : mEos(false), mStatus(BufferStatus::FREE), mBuffer(aBuffer) {} + + typedef void* BufferID; + + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(BufferData) + + // In most cases, the ID of this buffer is the pointer address of mBuffer. + // However, on some platforms it may be another value. + virtual BufferID ID() { return mBuffer; } + + // Return the platform dependent MediaData(). + // For example, it returns the MediaData with Gralloc texture. + // If it returns nullptr, then caller uses the normal way to + // create MediaData(). + virtual already_AddRefed GetPlatformMediaData() { + return nullptr; + } + + // The buffer could be used by several objects. And only one object owns the + // buffer the same time. + // FREE: + // nobody uses it. + // + // OMX_COMPONENT: + // buffer is used by OMX component (OmxPlatformLayer). + // + // OMX_CLIENT: + // buffer is used by client which is wait for audio/video playing + // (OmxDataDecoder) + // + // OMX_CLIENT_OUTPUT: + // used by client to output decoded data (for example, Gecko layer in + // this case) + // + // For output port buffer, the status transition is: + // FREE -> OMX_COMPONENT -> OMX_CLIENT -> OMX_CLIENT_OUTPUT -> FREE + // + // For input port buffer, the status transition is: + // FREE -> OMX_COMPONENT -> OMX_CLIENT -> FREE + // + enum BufferStatus { + FREE, + OMX_COMPONENT, + OMX_CLIENT, + OMX_CLIENT_OUTPUT, + INVALID + }; + + bool mEos; + + // The raw keeps in OmxPromiseLayer after EmptyBuffer and then passing to + // output decoded buffer in EmptyFillBufferDone. It is used to keep the + // records of the original data from demuxer, like duration, stream + // offset...etc. + RefPtr mRawData; + + // Because OMX buffer works across threads, so it uses a promise + // for each buffer when the buffer is used by Omx component. + MozPromiseHolder mPromise; + BufferStatus mStatus; + OMX_BUFFERHEADERTYPE* mBuffer; + }; + + void EmptyFillBufferDone(OMX_DIRTYPE aType, BufferData::BufferID aID); + + void EmptyFillBufferDone(OMX_DIRTYPE aType, BufferData* aData); + + already_AddRefed FindBufferById(OMX_DIRTYPE aType, + BufferData::BufferID aId); + + already_AddRefed FindAndRemoveBufferHolder( + OMX_DIRTYPE aType, BufferData::BufferID aId); + + // Return true if event is handled. + bool Event(OMX_EVENTTYPE aEvent, OMX_U32 aData1, OMX_U32 aData2); + + protected: + struct FlushCommand { + OMX_DIRTYPE type; + OMX_PTR cmd; + }; + + BUFFERLIST* GetBufferHolders(OMX_DIRTYPE aType); + + already_AddRefed FindAndRemoveRawData(OMX_TICKS aTimecode); + + RefPtr mTaskQueue; + + MozPromiseHolder mCommandStatePromise; + + MozPromiseHolder mPortDisablePromise; + + MozPromiseHolder mPortEnablePromise; + + MozPromiseHolder mFlushPromise; + + nsTArray mFlushCommands; + + UniquePtr mPlatformLayer; + + private: + // Elements are added to holders when FillBuffer() or FillBuffer(). And + // removing element when the promise is resolved. Buffers in these lists + // should NOT be used by other component; for example, output it to audio + // output. These lists should be empty when engine is about to shutdown. + // + // Note: + // There bufferlist should not be used by other class directly. + BUFFERLIST mInbufferHolders; + + BUFFERLIST mOutbufferHolders; + + nsTArray> mRawDatas; +}; + +} // namespace mozilla + +#endif /* OmxPromiseLayer_h_ */ diff --git a/dom/media/platforms/omx/PureOmxPlatformLayer.cpp b/dom/media/platforms/omx/PureOmxPlatformLayer.cpp new file mode 100644 index 0000000000..dd6b259ac6 --- /dev/null +++ b/dom/media/platforms/omx/PureOmxPlatformLayer.cpp @@ -0,0 +1,405 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "OmxDataDecoder.h" +#include "OmxPromiseLayer.h" +#include "PureOmxPlatformLayer.h" +#include "OmxCoreLibLinker.h" + +#ifdef LOG +# undef LOG +#endif + +#define LOG(arg, ...) \ + MOZ_LOG( \ + sPDMLog, mozilla::LogLevel::Debug, \ + ("PureOmxPlatformLayer(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) +#define LOG_BUF(arg, ...) \ + MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, \ + ("PureOmxBufferData(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) + +namespace mozilla { + +#define OMX_FUNC(func) extern typeof(func)* func; +#include "OmxFunctionList.h" +#undef OMX_FUNC + +PureOmxBufferData::PureOmxBufferData( + const PureOmxPlatformLayer& aPlatformLayer, + const OMX_PARAM_PORTDEFINITIONTYPE& aPortDef) + : BufferData(nullptr), mPlatformLayer(aPlatformLayer), mPortDef(aPortDef) { + LOG_BUF(""); + + if (ShouldUseEGLImage()) { + // TODO + LOG_BUF( + "OMX_UseEGLImage() seems available but using it isn't implemented " + "yet."); + } + + OMX_ERRORTYPE err; + err = OMX_AllocateBuffer(mPlatformLayer.GetComponent(), &mBuffer, + mPortDef.nPortIndex, this, mPortDef.nBufferSize); + if (err != OMX_ErrorNone) { + LOG_BUF("Failed to allocate the buffer!: 0x%08x", err); + } +} + +PureOmxBufferData::~PureOmxBufferData() { + LOG_BUF(""); + ReleaseBuffer(); +} + +void PureOmxBufferData::ReleaseBuffer() { + LOG_BUF(""); + + if (mBuffer) { + OMX_ERRORTYPE err; + err = OMX_FreeBuffer(mPlatformLayer.GetComponent(), mPortDef.nPortIndex, + mBuffer); + if (err != OMX_ErrorNone) { + LOG_BUF("Failed to free the buffer!: 0x%08x", err); + } + mBuffer = nullptr; + } +} + +bool PureOmxBufferData::ShouldUseEGLImage() { + OMX_ERRORTYPE err; + err = OMX_UseEGLImage(mPlatformLayer.GetComponent(), nullptr, + mPortDef.nPortIndex, nullptr, nullptr); + return (err != OMX_ErrorNotImplemented); +} + +/* static */ +bool PureOmxPlatformLayer::Init(void) { + if (!OmxCoreLibLinker::Link()) { + return false; + } + + OMX_ERRORTYPE err = OMX_Init(); + if (err != OMX_ErrorNone) { + MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, + ("PureOmxPlatformLayer::%s: Failed to initialize OMXCore: 0x%08x", + __func__, err)); + return false; + } + + return true; +} + +/* static */ +OMX_CALLBACKTYPE PureOmxPlatformLayer::sCallbacks = { + EventHandler, EmptyBufferDone, FillBufferDone}; + +PureOmxPlatformLayer::PureOmxPlatformLayer( + OmxDataDecoder* aDataDecoder, OmxPromiseLayer* aPromiseLayer, + TaskQueue* aTaskQueue, layers::ImageContainer* aImageContainer) + : mComponent(nullptr), + mDataDecoder(aDataDecoder), + mPromiseLayer(aPromiseLayer), + mTaskQueue(aTaskQueue), + mImageContainer(aImageContainer) { + LOG(""); +} + +PureOmxPlatformLayer::~PureOmxPlatformLayer() { LOG(""); } + +OMX_ERRORTYPE +PureOmxPlatformLayer::InitOmxToStateLoaded(const TrackInfo* aInfo) { + LOG(""); + + if (!aInfo) { + return OMX_ErrorUndefined; + } + mInfo = aInfo; + + return CreateComponent(); +} + +OMX_ERRORTYPE +PureOmxPlatformLayer::EmptyThisBuffer(BufferData* aData) { + LOG(""); + return OMX_EmptyThisBuffer(mComponent, aData->mBuffer); +} + +OMX_ERRORTYPE +PureOmxPlatformLayer::FillThisBuffer(BufferData* aData) { + LOG(""); + return OMX_FillThisBuffer(mComponent, aData->mBuffer); +} + +OMX_ERRORTYPE +PureOmxPlatformLayer::SendCommand(OMX_COMMANDTYPE aCmd, OMX_U32 aParam1, + OMX_PTR aCmdData) { + LOG("aCmd: 0x%08x", aCmd); + if (!mComponent) { + return OMX_ErrorUndefined; + } + return OMX_SendCommand(mComponent, aCmd, aParam1, aCmdData); +} + +nsresult PureOmxPlatformLayer::FindPortDefinition( + OMX_DIRTYPE aType, OMX_PARAM_PORTDEFINITIONTYPE& portDef) { + nsTArray portIndex; + GetPortIndices(portIndex); + for (auto idx : portIndex) { + InitOmxParameter(&portDef); + portDef.nPortIndex = idx; + + OMX_ERRORTYPE err; + err = GetParameter(OMX_IndexParamPortDefinition, &portDef, + sizeof(OMX_PARAM_PORTDEFINITIONTYPE)); + if (err != OMX_ErrorNone) { + return NS_ERROR_FAILURE; + } else if (portDef.eDir == aType) { + LOG("Found OMX_IndexParamPortDefinition: port: %d, type: %d", + portDef.nPortIndex, portDef.eDir); + return NS_OK; + } + } + return NS_ERROR_FAILURE; +} + +nsresult PureOmxPlatformLayer::AllocateOmxBuffer(OMX_DIRTYPE aType, + BUFFERLIST* aBufferList) { + LOG("aType: %d", aType); + + OMX_PARAM_PORTDEFINITIONTYPE portDef; + nsresult result = FindPortDefinition(aType, portDef); + if (result != NS_OK) { + return result; + } + + LOG("nBufferCountActual: %d, nBufferSize: %d", portDef.nBufferCountActual, + portDef.nBufferSize); + + for (OMX_U32 i = 0; i < portDef.nBufferCountActual; ++i) { + RefPtr buffer = new PureOmxBufferData(*this, portDef); + aBufferList->AppendElement(buffer); + } + + return NS_OK; +} + +nsresult PureOmxPlatformLayer::ReleaseOmxBuffer(OMX_DIRTYPE aType, + BUFFERLIST* aBufferList) { + LOG("aType: 0x%08x", aType); + + uint32_t len = aBufferList->Length(); + for (uint32_t i = 0; i < len; i++) { + PureOmxBufferData* buffer = + static_cast(aBufferList->ElementAt(i).get()); + + // All raw OpenMAX buffers have to be released here to flush + // OMX_CommandStateSet for switching the state to OMX_StateLoaded. + // See OmxDataDecoder::DoAsyncShutdown() for more detail. + buffer->ReleaseBuffer(); + } + aBufferList->Clear(); + + return NS_OK; +} + +OMX_ERRORTYPE +PureOmxPlatformLayer::GetState(OMX_STATETYPE* aType) { + LOG(""); + + if (mComponent) { + return OMX_GetState(mComponent, aType); + } + + return OMX_ErrorUndefined; +} + +OMX_ERRORTYPE +PureOmxPlatformLayer::GetParameter(OMX_INDEXTYPE aParamIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize) { + LOG("aParamIndex: 0x%08x", aParamIndex); + + if (!mComponent) { + return OMX_ErrorUndefined; + } + + return OMX_GetParameter(mComponent, aParamIndex, + aComponentParameterStructure); +} + +OMX_ERRORTYPE +PureOmxPlatformLayer::SetParameter(OMX_INDEXTYPE aParamIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize) { + LOG("aParamIndex: 0x%08x", aParamIndex); + + if (!mComponent) { + return OMX_ErrorUndefined; + } + + return OMX_SetParameter(mComponent, aParamIndex, + aComponentParameterStructure); +} + +nsresult PureOmxPlatformLayer::Shutdown() { + LOG(""); + if (mComponent) { + OMX_FreeHandle(mComponent); + mComponent = nullptr; + } + mPromiseLayer = nullptr; + mDataDecoder = nullptr; + return NS_OK; +} + +/* static */ +OMX_ERRORTYPE PureOmxPlatformLayer::EventHandler(OMX_HANDLETYPE hComponent, + OMX_PTR pAppData, + OMX_EVENTTYPE eEventType, + OMX_U32 nData1, OMX_U32 nData2, + OMX_PTR pEventData) { + PureOmxPlatformLayer* self = static_cast(pAppData); + nsCOMPtr r = NS_NewRunnableFunction( + "mozilla::PureOmxPlatformLayer::EventHandler", + [self, eEventType, nData1, nData2, pEventData]() { + self->EventHandler(eEventType, nData1, nData2, pEventData); + }); + nsresult rv = self->mTaskQueue->Dispatch(r.forget()); + return NS_SUCCEEDED(rv) ? OMX_ErrorNone : OMX_ErrorUndefined; +} + +/* static */ +OMX_ERRORTYPE PureOmxPlatformLayer::EmptyBufferDone( + OMX_HANDLETYPE hComponent, OMX_IN OMX_PTR pAppData, + OMX_IN OMX_BUFFERHEADERTYPE* pBuffer) { + PureOmxPlatformLayer* self = static_cast(pAppData); + nsCOMPtr r = NS_NewRunnableFunction( + "mozilla::PureOmxPlatformLayer::EmptyBufferDone", + [self, pBuffer]() { self->EmptyBufferDone(pBuffer); }); + nsresult rv = self->mTaskQueue->Dispatch(r.forget()); + return NS_SUCCEEDED(rv) ? OMX_ErrorNone : OMX_ErrorUndefined; +} + +/* static */ +OMX_ERRORTYPE PureOmxPlatformLayer::FillBufferDone( + OMX_OUT OMX_HANDLETYPE hComponent, OMX_OUT OMX_PTR pAppData, + OMX_OUT OMX_BUFFERHEADERTYPE* pBuffer) { + PureOmxPlatformLayer* self = static_cast(pAppData); + nsCOMPtr r = NS_NewRunnableFunction( + "mozilla::PureOmxPlatformLayer::FillBufferDone", + [self, pBuffer]() { self->FillBufferDone(pBuffer); }); + nsresult rv = self->mTaskQueue->Dispatch(r.forget()); + return NS_SUCCEEDED(rv) ? OMX_ErrorNone : OMX_ErrorUndefined; +} + +OMX_ERRORTYPE +PureOmxPlatformLayer::EventHandler(OMX_EVENTTYPE eEventType, OMX_U32 nData1, + OMX_U32 nData2, OMX_PTR pEventData) { + bool handled = mPromiseLayer->Event(eEventType, nData1, nData2); + LOG("eEventType: 0x%08x, handled: %d", eEventType, handled); + return OMX_ErrorNone; +} + +OMX_ERRORTYPE +PureOmxPlatformLayer::EmptyBufferDone(OMX_IN OMX_BUFFERHEADERTYPE* pBuffer) { + PureOmxBufferData* buffer = + static_cast(pBuffer->pAppPrivate); + OMX_DIRTYPE portDirection = buffer->GetPortDirection(); + LOG("PortDirection: %d", portDirection); + mPromiseLayer->EmptyFillBufferDone(portDirection, buffer); + return OMX_ErrorNone; +} + +OMX_ERRORTYPE +PureOmxPlatformLayer::FillBufferDone(OMX_OUT OMX_BUFFERHEADERTYPE* pBuffer) { + PureOmxBufferData* buffer = + static_cast(pBuffer->pAppPrivate); + OMX_DIRTYPE portDirection = buffer->GetPortDirection(); + LOG("PortDirection: %d", portDirection); + mPromiseLayer->EmptyFillBufferDone(portDirection, buffer); + return OMX_ErrorNone; +} + +bool PureOmxPlatformLayer::SupportsMimeType(const nsACString& aMimeType) { + return FindStandardComponent(aMimeType, nullptr); +} + +static bool GetStandardComponentRole(const nsACString& aMimeType, + nsACString& aRole) { + if (aMimeType.EqualsLiteral("video/avc") || + aMimeType.EqualsLiteral("video/mp4") || + aMimeType.EqualsLiteral("video/mp4v-es")) { + aRole.Assign("video_decoder.avc"); + return true; + } else if (aMimeType.EqualsLiteral("audio/mp4a-latm") || + aMimeType.EqualsLiteral("audio/mp4") || + aMimeType.EqualsLiteral("audio/aac")) { + aRole.Assign("audio_decoder.aac"); + return true; + } + return false; +} + +/* static */ +bool PureOmxPlatformLayer::FindStandardComponent(const nsACString& aMimeType, + nsACString* aComponentName) { + nsAutoCString role; + if (!GetStandardComponentRole(aMimeType, role)) return false; + + OMX_U32 nComponents = 0; + OMX_ERRORTYPE err; + err = OMX_GetComponentsOfRole(const_cast(role.Data()), + &nComponents, nullptr); + if (err != OMX_ErrorNone || nComponents <= 0) { + return false; + } + if (!aComponentName) { + return true; + } + + // TODO: + // Only the first component will be used. + // We should detect the most preferred component. + OMX_U8* componentNames[1]; + UniquePtr componentName; + componentName = MakeUniqueFallible(OMX_MAX_STRINGNAME_SIZE); + componentNames[0] = componentName.get(); + nComponents = 1; + err = OMX_GetComponentsOfRole(const_cast(role.Data()), + &nComponents, componentNames); + if (err == OMX_ErrorNone) { + MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, + ("PureOmxPlatformLayer::%s: A component has been found for %s: %s", + __func__, aMimeType.Data(), componentNames[0])); + aComponentName->Assign(reinterpret_cast(componentNames[0])); + } + + return err == OMX_ErrorNone; +} + +OMX_ERRORTYPE +PureOmxPlatformLayer::CreateComponent(const nsACString* aComponentName) { + nsAutoCString componentName; + if (aComponentName) { + componentName = *aComponentName; + } else if (!FindStandardComponent(mInfo->mMimeType, &componentName)) { + return OMX_ErrorComponentNotFound; + } + + OMX_ERRORTYPE err; + err = OMX_GetHandle(&mComponent, const_cast(componentName.Data()), + this, &sCallbacks); + + const char* mime = mInfo->mMimeType.Data(); + if (err == OMX_ErrorNone) { + LOG("Succeeded to create the component for %s", mime); + } else { + LOG("Failed to create the component for %s: 0x%08x", mime, err); + } + + return err; +} + +} // namespace mozilla diff --git a/dom/media/platforms/omx/PureOmxPlatformLayer.h b/dom/media/platforms/omx/PureOmxPlatformLayer.h new file mode 100644 index 0000000000..95c95b8781 --- /dev/null +++ b/dom/media/platforms/omx/PureOmxPlatformLayer.h @@ -0,0 +1,110 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#if !defined(PureOmxPlatformLayer_h_) +# define PureOmxPlatformLayer_h_ + +# include "OmxPlatformLayer.h" + +namespace mozilla { + +class PureOmxPlatformLayer; + +class PureOmxBufferData : public OmxPromiseLayer::BufferData { + protected: + virtual ~PureOmxBufferData(); + + public: + PureOmxBufferData(const PureOmxPlatformLayer& aPlatformLayer, + const OMX_PARAM_PORTDEFINITIONTYPE& aPortDef); + + void ReleaseBuffer(); + OMX_DIRTYPE GetPortDirection() const { return mPortDef.eDir; }; + + protected: + bool ShouldUseEGLImage(); + + const PureOmxPlatformLayer& mPlatformLayer; + const OMX_PARAM_PORTDEFINITIONTYPE mPortDef; +}; + +class PureOmxPlatformLayer : public OmxPlatformLayer { + public: + static bool Init(void); + + static bool SupportsMimeType(const nsACString& aMimeType); + + PureOmxPlatformLayer(OmxDataDecoder* aDataDecoder, + OmxPromiseLayer* aPromiseLayer, TaskQueue* aTaskQueue, + layers::ImageContainer* aImageContainer); + + virtual ~PureOmxPlatformLayer(); + + OMX_ERRORTYPE InitOmxToStateLoaded(const TrackInfo* aInfo) override; + + OMX_ERRORTYPE EmptyThisBuffer(BufferData* aData) override; + + OMX_ERRORTYPE FillThisBuffer(BufferData* aData) override; + + OMX_ERRORTYPE SendCommand(OMX_COMMANDTYPE aCmd, OMX_U32 aParam1, + OMX_PTR aCmdData) override; + + nsresult AllocateOmxBuffer(OMX_DIRTYPE aType, + BUFFERLIST* aBufferList) override; + + nsresult ReleaseOmxBuffer(OMX_DIRTYPE aType, + BUFFERLIST* aBufferList) override; + + OMX_ERRORTYPE GetState(OMX_STATETYPE* aType) override; + + OMX_ERRORTYPE GetParameter(OMX_INDEXTYPE aParamIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize) override; + + OMX_ERRORTYPE SetParameter(OMX_INDEXTYPE aParamIndex, + OMX_PTR aComponentParameterStructure, + OMX_U32 aComponentParameterSize) override; + + nsresult Shutdown() override; + + OMX_HANDLETYPE GetComponent() const { return mComponent; }; + + static OMX_ERRORTYPE EventHandler(OMX_HANDLETYPE hComponent, OMX_PTR pAppData, + OMX_EVENTTYPE eEventType, OMX_U32 nData1, + OMX_U32 nData2, OMX_PTR pEventData); + static OMX_ERRORTYPE EmptyBufferDone(OMX_HANDLETYPE hComponent, + OMX_IN OMX_PTR pAppData, + OMX_IN OMX_BUFFERHEADERTYPE* pBuffer); + static OMX_ERRORTYPE FillBufferDone(OMX_OUT OMX_HANDLETYPE hComponent, + OMX_OUT OMX_PTR pAppData, + OMX_OUT OMX_BUFFERHEADERTYPE* pBuffer); + + protected: + static bool FindStandardComponent(const nsACString& aMimeType, + nsACString* aComponentName); + + OMX_ERRORTYPE CreateComponent(const nsACString* aComponentName = nullptr); + nsresult FindPortDefinition(OMX_DIRTYPE aType, + OMX_PARAM_PORTDEFINITIONTYPE& portDef); + + OMX_ERRORTYPE EventHandler(OMX_EVENTTYPE eEventType, OMX_U32 nData1, + OMX_U32 nData2, OMX_PTR pEventData); + OMX_ERRORTYPE EmptyBufferDone(OMX_IN OMX_BUFFERHEADERTYPE* pBuffer); + OMX_ERRORTYPE FillBufferDone(OMX_OUT OMX_BUFFERHEADERTYPE* pBuffer); + + protected: + static OMX_CALLBACKTYPE sCallbacks; + + OMX_HANDLETYPE mComponent; + RefPtr mDataDecoder; + RefPtr mPromiseLayer; + RefPtr mTaskQueue; + RefPtr mImageContainer; +}; + +} // namespace mozilla + +#endif // PureOmxPlatformLayer_h_ diff --git a/dom/media/platforms/omx/moz.build b/dom/media/platforms/omx/moz.build new file mode 100644 index 0000000000..c8a79cabb4 --- /dev/null +++ b/dom/media/platforms/omx/moz.build @@ -0,0 +1,36 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXPORTS += [ + "OmxDecoderModule.h", +] + +UNIFIED_SOURCES += [ + "OmxDataDecoder.cpp", + "OmxDecoderModule.cpp", + "OmxPlatformLayer.cpp", + "OmxPromiseLayer.cpp", +] + +LOCAL_INCLUDES += [ + "/media/openmax_il/il112", +] + +include("/ipc/chromium/chromium-config.mozbuild") + +if CONFIG["MOZ_OMX"]: + UNIFIED_SOURCES += [ + "PureOmxPlatformLayer.cpp", + ] + +FINAL_LIBRARY = "xul" + +# Avoid warnings from third-party code that we can not modify. +if CONFIG["CC_TYPE"] == "clang-cl": + CXXFLAGS += ["-Wno-invalid-source-encoding"] + +# Add libFuzzer configuration directives +include("/tools/fuzzing/libfuzzer-config.mozbuild") -- cgit v1.2.3