/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim:set ts=2 sw=2 sts=2 et cindent: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "OmxDataDecoder.h" #include "OMX_Audio.h" #include "OMX_Component.h" #include "OMX_Types.h" #include "OmxPlatformLayer.h" #include "mozilla/IntegerPrintfMacros.h" #ifdef LOG # undef LOG # undef LOGL #endif #define LOG(arg, ...) \ DDMOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, __func__, \ ##__VA_ARGS__) #define LOGL(arg, ...) \ DDMOZ_LOGEX(self.get(), sPDMLog, mozilla::LogLevel::Debug, "::%s: " arg, \ __func__, ##__VA_ARGS__) #define CHECK_OMX_ERR(err) \ if (err != OMX_ErrorNone) { \ NotifyError(err, __func__); \ return; \ } namespace mozilla { using namespace gfx; static const char* StateTypeToStr(OMX_STATETYPE aType) { MOZ_ASSERT(aType == OMX_StateLoaded || aType == OMX_StateIdle || aType == OMX_StateExecuting || aType == OMX_StatePause || aType == OMX_StateWaitForResources || aType == OMX_StateInvalid); switch (aType) { case OMX_StateLoaded: return "OMX_StateLoaded"; case OMX_StateIdle: return "OMX_StateIdle"; case OMX_StateExecuting: return "OMX_StateExecuting"; case OMX_StatePause: return "OMX_StatePause"; case OMX_StateWaitForResources: return "OMX_StateWaitForResources"; case OMX_StateInvalid: return "OMX_StateInvalid"; default: return "Unknown"; } } // A helper class to retrieve AudioData or VideoData. class MediaDataHelper { protected: virtual ~MediaDataHelper() = default; public: NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaDataHelper) MediaDataHelper(const TrackInfo* aTrackInfo, layers::ImageContainer* aImageContainer, OmxPromiseLayer* aOmxLayer); already_AddRefed GetMediaData(BufferData* aBufferData, bool& aPlatformDepenentData); protected: already_AddRefed CreateAudioData(BufferData* aBufferData); already_AddRefed CreateYUV420VideoData(BufferData* aBufferData); const TrackInfo* mTrackInfo; OMX_PARAM_PORTDEFINITIONTYPE mOutputPortDef; // audio output MediaQueue mAudioQueue; AudioCompactor mAudioCompactor; // video output RefPtr mImageContainer; }; OmxDataDecoder::OmxDataDecoder(const TrackInfo& aTrackInfo, layers::ImageContainer* aImageContainer, Maybe aTrackingId) : mOmxTaskQueue( CreateMediaDecodeTaskQueue("OmxDataDecoder::mOmxTaskQueue")), mImageContainer(aImageContainer), mWatchManager(this, mOmxTaskQueue), mOmxState(OMX_STATETYPE::OMX_StateInvalid, "OmxDataDecoder::mOmxState"), mTrackInfo(aTrackInfo.Clone()), mFlushing(false), mShuttingDown(false), mCheckingInputExhausted(false), mPortSettingsChanged(-1, "OmxDataDecoder::mPortSettingsChanged"), mTrackingId(std::move(aTrackingId)) { LOG(""); mOmxLayer = new OmxPromiseLayer(mOmxTaskQueue, this, aImageContainer); } OmxDataDecoder::~OmxDataDecoder() { LOG(""); } void OmxDataDecoder::InitializationTask() { mWatchManager.Watch(mOmxState, &OmxDataDecoder::OmxStateRunner); mWatchManager.Watch(mPortSettingsChanged, &OmxDataDecoder::PortSettingsChanged); } void OmxDataDecoder::EndOfStream() { LOG(""); MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); RefPtr self = this; mOmxLayer->SendCommand(OMX_CommandFlush, OMX_ALL, nullptr) ->Then(mOmxTaskQueue, __func__, [self, this](OmxCommandPromise::ResolveOrRejectValue&& aValue) { mDrainPromise.ResolveIfExists(std::move(mDecodedData), __func__); mDecodedData = DecodedData(); }); } RefPtr OmxDataDecoder::Init() { LOG(""); mThread = GetCurrentSerialEventTarget(); RefPtr self = this; return InvokeAsync(mOmxTaskQueue, __func__, [self, this]() { InitializationTask(); RefPtr p = mInitPromise.Ensure(__func__); mOmxLayer->Init(mTrackInfo.get()) ->Then( mOmxTaskQueue, __func__, [self, this]() { // Omx state should be OMX_StateIdle. mOmxState = mOmxLayer->GetState(); MOZ_ASSERT(mOmxState != OMX_StateIdle); }, [self, this]() { RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); }); return p; }); } RefPtr OmxDataDecoder::Decode( MediaRawData* aSample) { LOG("sample %p", aSample); MOZ_ASSERT(mThread->IsOnCurrentThread()); MOZ_ASSERT(mInitPromise.IsEmpty()); RefPtr self = this; RefPtr sample = aSample; return InvokeAsync(mOmxTaskQueue, __func__, [self, this, sample]() { RefPtr p = mDecodePromise.Ensure(__func__); mTrackingId.apply([&](const auto& aId) { MediaInfoFlag flag = MediaInfoFlag::None; flag |= (sample->mKeyframe ? MediaInfoFlag::KeyFrame : MediaInfoFlag::NonKeyFrame); mPerformanceRecorder.Start(sample->mTimecode.ToMicroseconds(), "OmxDataDecoder"_ns, aId, flag); }); mMediaRawDatas.AppendElement(std::move(sample)); // Start to fill/empty buffers. if (mOmxState == OMX_StateIdle || mOmxState == OMX_StateExecuting) { FillAndEmptyBuffers(); } return p; }); } RefPtr OmxDataDecoder::Flush() { LOG(""); MOZ_ASSERT(mThread->IsOnCurrentThread()); mFlushing = true; return InvokeAsync(mOmxTaskQueue, this, __func__, &OmxDataDecoder::DoFlush); } RefPtr OmxDataDecoder::Drain() { LOG(""); MOZ_ASSERT(mThread->IsOnCurrentThread()); RefPtr self = this; return InvokeAsync(mOmxTaskQueue, __func__, [self]() { RefPtr p = self->mDrainPromise.Ensure(__func__); self->SendEosBuffer(); return p; }); } RefPtr OmxDataDecoder::Shutdown() { LOG(""); // mThread may not be set if Init hasn't been called first. MOZ_ASSERT(!mThread || mThread->IsOnCurrentThread()); mShuttingDown = true; return InvokeAsync(mOmxTaskQueue, this, __func__, &OmxDataDecoder::DoAsyncShutdown); } RefPtr OmxDataDecoder::DoAsyncShutdown() { LOG(""); MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); MOZ_ASSERT(!mFlushing); mWatchManager.Unwatch(mOmxState, &OmxDataDecoder::OmxStateRunner); mWatchManager.Unwatch(mPortSettingsChanged, &OmxDataDecoder::PortSettingsChanged); // Flush to all ports, so all buffers can be returned from component. RefPtr self = this; mOmxLayer->SendCommand(OMX_CommandFlush, OMX_ALL, nullptr) ->Then( mOmxTaskQueue, __func__, [self]() -> RefPtr { LOGL("DoAsyncShutdown: flush complete"); return self->mOmxLayer->SendCommand(OMX_CommandStateSet, OMX_StateIdle, nullptr); }, [self](const OmxCommandFailureHolder& aError) { self->mOmxLayer->Shutdown(); return OmxCommandPromise::CreateAndReject(aError, __func__); }) ->Then( mOmxTaskQueue, __func__, [self]() -> RefPtr { RefPtr p = self->mOmxLayer->SendCommand( OMX_CommandStateSet, OMX_StateLoaded, nullptr); // According to spec 3.1.1.2.2.1: // OMX_StateLoaded needs to be sent before releasing buffers. // And state transition from OMX_StateIdle to OMX_StateLoaded // is completed when all of the buffers have been removed // from the component. // Here the buffer promises are not resolved due to displaying // in layer, it needs to wait before the layer returns the // buffers. LOGL("DoAsyncShutdown: releasing buffers..."); self->ReleaseBuffers(OMX_DirInput); self->ReleaseBuffers(OMX_DirOutput); return p; }, [self](const OmxCommandFailureHolder& aError) { self->mOmxLayer->Shutdown(); return OmxCommandPromise::CreateAndReject(aError, __func__); }) ->Then( mOmxTaskQueue, __func__, [self]() -> RefPtr { LOGL( "DoAsyncShutdown: OMX_StateLoaded, it is safe to shutdown omx"); self->mOmxLayer->Shutdown(); self->mWatchManager.Shutdown(); self->mOmxLayer = nullptr; self->mMediaDataHelper = nullptr; self->mShuttingDown = false; return ShutdownPromise::CreateAndResolve(true, __func__); }, [self]() -> RefPtr { self->mOmxLayer->Shutdown(); self->mWatchManager.Shutdown(); self->mOmxLayer = nullptr; self->mMediaDataHelper = nullptr; return ShutdownPromise::CreateAndReject(false, __func__); }) ->Then( mThread, __func__, [self]() { self->mOmxTaskQueue->BeginShutdown(); self->mOmxTaskQueue->AwaitShutdownAndIdle(); self->mShutdownPromise.Resolve(true, __func__); }, [self]() { self->mOmxTaskQueue->BeginShutdown(); self->mOmxTaskQueue->AwaitShutdownAndIdle(); self->mShutdownPromise.Resolve(true, __func__); }); return mShutdownPromise.Ensure(__func__); } void OmxDataDecoder::FillBufferDone(BufferData* aData) { MOZ_ASSERT(!aData || aData->mStatus == BufferData::BufferStatus::OMX_CLIENT); // Don't output sample when flush or shutting down, especially for video // decoded frame. Because video decoded frame can have a promise in // BufferData waiting for layer to resolve it via recycle callback, if other // module doesn't send it to layer, it will cause a unresolved promise and // waiting for resolve infinitely. if (mFlushing || mShuttingDown) { LOG("mFlush or mShuttingDown, drop data"); aData->mStatus = BufferData::BufferStatus::FREE; return; } if (aData->mBuffer->nFlags & OMX_BUFFERFLAG_EOS) { // Reach eos, it's an empty data so it doesn't need to output. EndOfStream(); aData->mStatus = BufferData::BufferStatus::FREE; } else { Output(aData); FillAndEmptyBuffers(); } } void OmxDataDecoder::Output(BufferData* aData) { if (!mMediaDataHelper) { mMediaDataHelper = new MediaDataHelper(mTrackInfo.get(), mImageContainer, mOmxLayer); } bool isPlatformData = false; RefPtr data = mMediaDataHelper->GetMediaData(aData, isPlatformData); if (!data) { aData->mStatus = BufferData::BufferStatus::FREE; return; } if (isPlatformData) { // If the MediaData is platform dependnet data, it's mostly a kind of // limited resource, so we use promise to notify when the resource is free. aData->mStatus = BufferData::BufferStatus::OMX_CLIENT_OUTPUT; MOZ_RELEASE_ASSERT(aData->mPromise.IsEmpty()); RefPtr p = aData->mPromise.Ensure(__func__); RefPtr self = this; RefPtr buffer = aData; p->Then( mOmxTaskQueue, __func__, [self, buffer]() { MOZ_RELEASE_ASSERT(buffer->mStatus == BufferData::BufferStatus::OMX_CLIENT_OUTPUT); buffer->mStatus = BufferData::BufferStatus::FREE; self->FillAndEmptyBuffers(); }, [buffer]() { MOZ_RELEASE_ASSERT(buffer->mStatus == BufferData::BufferStatus::OMX_CLIENT_OUTPUT); buffer->mStatus = BufferData::BufferStatus::FREE; }); } else { aData->mStatus = BufferData::BufferStatus::FREE; } if (mTrackInfo->IsVideo()) { mPerformanceRecorder.Record( aData->mRawData->mTimecode.ToMicroseconds(), [&](DecodeStage& aStage) { const auto& image = data->As()->mImage; aStage.SetResolution(image->GetSize().Width(), image->GetSize().Height()); aStage.SetImageFormat(DecodeStage::YUV420P); aStage.SetColorDepth(image->GetColorDepth()); }); } mDecodedData.AppendElement(std::move(data)); } void OmxDataDecoder::FillBufferFailure(OmxBufferFailureHolder aFailureHolder) { NotifyError(aFailureHolder.mError, __func__); } void OmxDataDecoder::EmptyBufferDone(BufferData* aData) { MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); MOZ_ASSERT(!aData || aData->mStatus == BufferData::BufferStatus::OMX_CLIENT); // Nothing to do when status of input buffer is OMX_CLIENT. aData->mStatus = BufferData::BufferStatus::FREE; FillAndEmptyBuffers(); // There is no way to know if component gets enough raw samples to generate // output, especially for video decoding. So here it needs to request raw // samples aggressively. if (!mCheckingInputExhausted && !mMediaRawDatas.Length()) { mCheckingInputExhausted = true; RefPtr self = this; nsCOMPtr r = NS_NewRunnableFunction( "OmxDataDecoder::EmptyBufferDone", [self, this]() { mCheckingInputExhausted = false; if (mMediaRawDatas.Length()) { return; } mDecodePromise.ResolveIfExists(std::move(mDecodedData), __func__); mDecodedData = DecodedData(); }); nsresult rv = mOmxTaskQueue->Dispatch(r.forget()); MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); Unused << rv; } } void OmxDataDecoder::EmptyBufferFailure(OmxBufferFailureHolder aFailureHolder) { NotifyError(aFailureHolder.mError, __func__); } void OmxDataDecoder::NotifyError(OMX_ERRORTYPE aOmxError, const char* aLine, const MediaResult& aError) { LOG("NotifyError %d (%s) at %s", static_cast(aOmxError), aError.ErrorName().get(), aLine); mDecodedData = DecodedData(); mDecodePromise.RejectIfExists(aError, __func__); mDrainPromise.RejectIfExists(aError, __func__); mFlushPromise.RejectIfExists(aError, __func__); } void OmxDataDecoder::FillAndEmptyBuffers() { MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); MOZ_ASSERT(mOmxState == OMX_StateExecuting); // During the port setting changed, it is forbidden to do any buffer // operation. if (mPortSettingsChanged != -1 || mShuttingDown || mFlushing) { return; } // Trigger input port. while (!!mMediaRawDatas.Length()) { // input buffer must be used by component if there is data available. RefPtr inbuf = FindAvailableBuffer(OMX_DirInput); if (!inbuf) { LOG("no input buffer!"); break; } RefPtr data = mMediaRawDatas[0]; // Buffer size should large enough for raw data. MOZ_RELEASE_ASSERT(inbuf->mBuffer->nAllocLen >= data->Size()); memcpy(inbuf->mBuffer->pBuffer, data->Data(), data->Size()); inbuf->mBuffer->nFilledLen = data->Size(); inbuf->mBuffer->nOffset = 0; inbuf->mBuffer->nFlags = inbuf->mBuffer->nAllocLen > data->Size() ? OMX_BUFFERFLAG_ENDOFFRAME : 0; inbuf->mBuffer->nTimeStamp = data->mTime.ToMicroseconds(); if (data->Size()) { inbuf->mRawData = mMediaRawDatas[0]; } else { LOG("send EOS buffer"); inbuf->mBuffer->nFlags |= OMX_BUFFERFLAG_EOS; } LOG("feed sample %p to omx component, len %ld, flag %lX", data.get(), inbuf->mBuffer->nFilledLen, inbuf->mBuffer->nFlags); mOmxLayer->EmptyBuffer(inbuf)->Then(mOmxTaskQueue, __func__, this, &OmxDataDecoder::EmptyBufferDone, &OmxDataDecoder::EmptyBufferFailure); mMediaRawDatas.RemoveElementAt(0); } // Trigger output port. while (true) { RefPtr outbuf = FindAvailableBuffer(OMX_DirOutput); if (!outbuf) { break; } mOmxLayer->FillBuffer(outbuf)->Then(mOmxTaskQueue, __func__, this, &OmxDataDecoder::FillBufferDone, &OmxDataDecoder::FillBufferFailure); } } OmxPromiseLayer::BufferData* OmxDataDecoder::FindAvailableBuffer( OMX_DIRTYPE aType) { BUFFERLIST* buffers = GetBuffers(aType); for (uint32_t i = 0; i < buffers->Length(); i++) { BufferData* buf = buffers->ElementAt(i); if (buf->mStatus == BufferData::BufferStatus::FREE) { return buf; } } return nullptr; } nsresult OmxDataDecoder::AllocateBuffers(OMX_DIRTYPE aType) { MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); return mOmxLayer->AllocateOmxBuffer(aType, GetBuffers(aType)); } nsresult OmxDataDecoder::ReleaseBuffers(OMX_DIRTYPE aType) { MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); return mOmxLayer->ReleaseOmxBuffer(aType, GetBuffers(aType)); } nsTArray>* OmxDataDecoder::GetBuffers( OMX_DIRTYPE aType) { MOZ_ASSERT(aType == OMX_DIRTYPE::OMX_DirInput || aType == OMX_DIRTYPE::OMX_DirOutput); if (aType == OMX_DIRTYPE::OMX_DirInput) { return &mInPortBuffers; } return &mOutPortBuffers; } void OmxDataDecoder::ResolveInitPromise(StaticString aMethodName) { MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); LOG("called from %s", aMethodName.get()); mInitPromise.ResolveIfExists(mTrackInfo->GetType(), aMethodName); } void OmxDataDecoder::RejectInitPromise(MediaResult aError, StaticString aMethodName) { MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); mInitPromise.RejectIfExists(aError, aMethodName); } void OmxDataDecoder::OmxStateRunner() { MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); LOG("OMX state: %s", StateTypeToStr(mOmxState)); // TODO: maybe it'd be better to use promise CompletionPromise() to replace // this state machine. if (mOmxState == OMX_StateLoaded) { ConfigCodec(); // Send OpenMax state command to OMX_StateIdle. RefPtr self = this; mOmxLayer->SendCommand(OMX_CommandStateSet, OMX_StateIdle, nullptr) ->Then( mOmxTaskQueue, __func__, [self]() { // Current state should be OMX_StateIdle. self->mOmxState = self->mOmxLayer->GetState(); MOZ_ASSERT(self->mOmxState == OMX_StateIdle); }, [self]() { self->RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); }); // Allocate input and output buffers. OMX_DIRTYPE types[] = {OMX_DIRTYPE::OMX_DirInput, OMX_DIRTYPE::OMX_DirOutput}; for (const auto id : types) { if (NS_FAILED(AllocateBuffers(id))) { LOG("Failed to allocate buffer on port %d", id); RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); break; } } } else if (mOmxState == OMX_StateIdle) { RefPtr self = this; mOmxLayer->SendCommand(OMX_CommandStateSet, OMX_StateExecuting, nullptr) ->Then( mOmxTaskQueue, __func__, [self]() { self->mOmxState = self->mOmxLayer->GetState(); MOZ_ASSERT(self->mOmxState == OMX_StateExecuting); self->ResolveInitPromise(__func__); }, [self]() { self->RejectInitPromise(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); }); } else if (mOmxState == OMX_StateExecuting) { // Configure codec once it gets OMX_StateExecuting state. FillCodecConfigDataToOmx(); } else { MOZ_ASSERT(0); } } void OmxDataDecoder::ConfigCodec() { OMX_ERRORTYPE err = mOmxLayer->Config(); CHECK_OMX_ERR(err); } void OmxDataDecoder::FillCodecConfigDataToOmx() { // Codec configure data should be the first sample running on Omx TaskQueue. MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); MOZ_ASSERT(!mMediaRawDatas.Length()); MOZ_ASSERT(mOmxState == OMX_StateIdle || mOmxState == OMX_StateExecuting); RefPtr inbuf = FindAvailableBuffer(OMX_DirInput); RefPtr csc; if (mTrackInfo->IsAudio()) { // It would be nice to instead use more specific information here, but // we force a byte buffer for now since this handles arbitrary codecs. // TODO(bug 1768566): implement further type checking for codec data. csc = ForceGetAudioCodecSpecificBlob( mTrackInfo->GetAsAudioInfo()->mCodecSpecificConfig); } else if (mTrackInfo->IsVideo()) { csc = mTrackInfo->GetAsVideoInfo()->mExtraData; } MOZ_RELEASE_ASSERT(csc); // Some codecs like h264, its codec specific data is at the first packet, not // in container. if (csc->Length()) { // Buffer size should large enough for raw data. MOZ_RELEASE_ASSERT(inbuf->mBuffer->nAllocLen >= csc->Length()); memcpy(inbuf->mBuffer->pBuffer, csc->Elements(), csc->Length()); inbuf->mBuffer->nFilledLen = csc->Length(); inbuf->mBuffer->nOffset = 0; inbuf->mBuffer->nFlags = (OMX_BUFFERFLAG_ENDOFFRAME | OMX_BUFFERFLAG_CODECCONFIG); LOG("Feed codec configure data to OMX component"); mOmxLayer->EmptyBuffer(inbuf)->Then(mOmxTaskQueue, __func__, this, &OmxDataDecoder::EmptyBufferDone, &OmxDataDecoder::EmptyBufferFailure); } } bool OmxDataDecoder::Event(OMX_EVENTTYPE aEvent, OMX_U32 aData1, OMX_U32 aData2) { MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); if (mOmxLayer->Event(aEvent, aData1, aData2)) { return true; } switch (aEvent) { case OMX_EventPortSettingsChanged: { // Don't always disable port. See bug 1235340. if (aData2 == 0 || aData2 == OMX_IndexParamPortDefinition) { // According to spec: "To prevent the loss of any input data, the // component issuing the OMX_EventPortSettingsChanged event on its input // port should buffer all input port data that arrives between the // emission of the OMX_EventPortSettingsChanged event and the arrival of // the command to disable the input port." // // So client needs to disable port and reallocate buffers. MOZ_ASSERT(mPortSettingsChanged == -1); mPortSettingsChanged = aData1; } LOG("Got OMX_EventPortSettingsChanged event"); break; } default: { // Got error during decoding, send msg to MFR skipping to next key frame. if (aEvent == OMX_EventError && mOmxState == OMX_StateExecuting) { NotifyError((OMX_ERRORTYPE)aData1, __func__, MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, __func__)); return true; } LOG("WARNING: got none handle event: %d, aData1: %ld, aData2: %ld", aEvent, aData1, aData2); return false; } } return true; } bool OmxDataDecoder::BuffersCanBeReleased(OMX_DIRTYPE aType) { BUFFERLIST* buffers = GetBuffers(aType); uint32_t len = buffers->Length(); for (uint32_t i = 0; i < len; i++) { BufferData::BufferStatus buf_status = buffers->ElementAt(i)->mStatus; if (buf_status == BufferData::BufferStatus::OMX_COMPONENT || buf_status == BufferData::BufferStatus::OMX_CLIENT_OUTPUT) { return false; } } return true; } OMX_DIRTYPE OmxDataDecoder::GetPortDirection(uint32_t aPortIndex) { OMX_PARAM_PORTDEFINITIONTYPE def; InitOmxParameter(&def); def.nPortIndex = mPortSettingsChanged; OMX_ERRORTYPE err = mOmxLayer->GetParameter(OMX_IndexParamPortDefinition, &def, sizeof(def)); if (err != OMX_ErrorNone) { return OMX_DirMax; } return def.eDir; } RefPtr OmxDataDecoder::CollectBufferPromises(OMX_DIRTYPE aType) { MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); nsTArray> promises; OMX_DIRTYPE types[] = {OMX_DIRTYPE::OMX_DirInput, OMX_DIRTYPE::OMX_DirOutput}; for (const auto type : types) { if ((aType == type) || (aType == OMX_DirMax)) { // find the buffer which has promise. BUFFERLIST* buffers = GetBuffers(type); for (uint32_t i = 0; i < buffers->Length(); i++) { BufferData* buf = buffers->ElementAt(i); if (!buf->mPromise.IsEmpty()) { // OmxBufferPromise is not exclusive, it can be multiple "Then"s, so // it is safe to call "Ensure" here. promises.AppendElement(buf->mPromise.Ensure(__func__)); } } } } LOG("CollectBufferPromises: type %d, total %zu promiese", aType, promises.Length()); if (promises.Length()) { return OmxBufferPromise::All(mOmxTaskQueue, promises); } return OmxBufferPromise::AllPromiseType::CreateAndResolve( nsTArray(), __func__); } void OmxDataDecoder::PortSettingsChanged() { MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); if (mPortSettingsChanged == -1 || mOmxState == OMX_STATETYPE::OMX_StateInvalid) { return; } // The PortSettingsChanged algorithm: // // 1. disable port. // 2. wait for port buffers return to client and then release these buffers. // 3. enable port. // 4. allocate port buffers. // // Disable port. Get port definition if the target port is enable. OMX_PARAM_PORTDEFINITIONTYPE def; InitOmxParameter(&def); def.nPortIndex = mPortSettingsChanged; OMX_ERRORTYPE err = mOmxLayer->GetParameter(OMX_IndexParamPortDefinition, &def, sizeof(def)); CHECK_OMX_ERR(err); RefPtr self = this; if (def.bEnabled) { // 1. disable port. LOG("PortSettingsChanged: disable port %lu", def.nPortIndex); mOmxLayer ->SendCommand(OMX_CommandPortDisable, mPortSettingsChanged, nullptr) ->Then( mOmxTaskQueue, __func__, [self, def]() -> RefPtr { // 3. enable port. // Send enable port command. RefPtr p = self->mOmxLayer->SendCommand( OMX_CommandPortEnable, self->mPortSettingsChanged, nullptr); // 4. allocate port buffers. // Allocate new port buffers. nsresult rv = self->AllocateBuffers(def.eDir); if (NS_FAILED(rv)) { self->NotifyError(OMX_ErrorUndefined, __func__); } return p; }, [self](const OmxCommandFailureHolder& aError) { self->NotifyError(OMX_ErrorUndefined, __func__); return OmxCommandPromise::CreateAndReject(aError, __func__); }) ->Then( mOmxTaskQueue, __func__, [self]() { LOGL("PortSettingsChanged: port settings changed complete"); // finish port setting changed. self->mPortSettingsChanged = -1; self->FillAndEmptyBuffers(); }, [self]() { self->NotifyError(OMX_ErrorUndefined, __func__); }); // 2. wait for port buffers return to client and then release these buffers. // // Port buffers will be returned to client soon once OMX_CommandPortDisable // command is sent. Then releasing these buffers. CollectBufferPromises(def.eDir)->Then( mOmxTaskQueue, __func__, [self, def]() { MOZ_ASSERT(self->BuffersCanBeReleased(def.eDir)); nsresult rv = self->ReleaseBuffers(def.eDir); if (NS_FAILED(rv)) { MOZ_RELEASE_ASSERT(0); self->NotifyError(OMX_ErrorUndefined, __func__); } }, [self]() { self->NotifyError(OMX_ErrorUndefined, __func__); }); } } void OmxDataDecoder::SendEosBuffer() { MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); // There is no 'Drain' API in OpenMax, so it needs to wait for output sample // with EOS flag. However, MediaRawData doesn't provide EOS information, // so here it generates an empty BufferData with eos OMX_BUFFERFLAG_EOS in // queue. This behaviour should be compliant with spec, I think... RefPtr eos_data = new MediaRawData(); mMediaRawDatas.AppendElement(eos_data); FillAndEmptyBuffers(); } RefPtr OmxDataDecoder::DoFlush() { MOZ_ASSERT(mOmxTaskQueue->IsCurrentThreadIn()); mDecodedData = DecodedData(); mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); mDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); mPerformanceRecorder.Record(std::numeric_limits::max()); RefPtr p = mFlushPromise.Ensure(__func__); // 1. Call OMX command OMX_CommandFlush in Omx TaskQueue. // 2. Remove all elements in mMediaRawDatas when flush is completed. mOmxLayer->SendCommand(OMX_CommandFlush, OMX_ALL, nullptr) ->Then(mOmxTaskQueue, __func__, this, &OmxDataDecoder::FlushComplete, &OmxDataDecoder::FlushFailure); return p; } void OmxDataDecoder::FlushComplete(OMX_COMMANDTYPE aCommandType) { mMediaRawDatas.Clear(); mFlushing = false; LOG("Flush complete"); mFlushPromise.ResolveIfExists(true, __func__); } void OmxDataDecoder::FlushFailure(OmxCommandFailureHolder aFailureHolder) { mFlushing = false; mFlushPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__); } MediaDataHelper::MediaDataHelper(const TrackInfo* aTrackInfo, layers::ImageContainer* aImageContainer, OmxPromiseLayer* aOmxLayer) : mTrackInfo(aTrackInfo), mAudioCompactor(mAudioQueue), mImageContainer(aImageContainer) { InitOmxParameter(&mOutputPortDef); mOutputPortDef.nPortIndex = aOmxLayer->OutputPortIndex(); aOmxLayer->GetParameter(OMX_IndexParamPortDefinition, &mOutputPortDef, sizeof(mOutputPortDef)); } already_AddRefed MediaDataHelper::GetMediaData( BufferData* aBufferData, bool& aPlatformDepenentData) { aPlatformDepenentData = false; RefPtr data; if (mTrackInfo->IsAudio()) { if (!aBufferData->mBuffer->nFilledLen) { return nullptr; } data = CreateAudioData(aBufferData); } else if (mTrackInfo->IsVideo()) { data = aBufferData->GetPlatformMediaData(); if (data) { aPlatformDepenentData = true; } else { if (!aBufferData->mBuffer->nFilledLen) { return nullptr; } // Get YUV VideoData, it uses more CPU, in most cases, on software codec. data = CreateYUV420VideoData(aBufferData); } // Update video time code, duration... from the raw data. VideoData* video(data->As()); if (aBufferData->mRawData) { video->mTime = aBufferData->mRawData->mTime; video->mTimecode = aBufferData->mRawData->mTimecode; video->mOffset = aBufferData->mRawData->mOffset; video->mDuration = aBufferData->mRawData->mDuration; video->mKeyframe = aBufferData->mRawData->mKeyframe; } } return data.forget(); } already_AddRefed MediaDataHelper::CreateAudioData( BufferData* aBufferData) { RefPtr audio; OMX_BUFFERHEADERTYPE* buf = aBufferData->mBuffer; const AudioInfo* info = mTrackInfo->GetAsAudioInfo(); if (buf->nFilledLen) { uint64_t offset = 0; uint32_t frames = buf->nFilledLen / (2 * info->mChannels); if (aBufferData->mRawData) { offset = aBufferData->mRawData->mOffset; } typedef AudioCompactor::NativeCopy OmxCopy; mAudioCompactor.Push( offset, buf->nTimeStamp, info->mRate, frames, info->mChannels, OmxCopy(buf->pBuffer + buf->nOffset, buf->nFilledLen, info->mChannels)); audio = mAudioQueue.PopFront(); } return audio.forget(); } already_AddRefed MediaDataHelper::CreateYUV420VideoData( BufferData* aBufferData) { uint8_t* yuv420p_buffer = (uint8_t*)aBufferData->mBuffer->pBuffer; int32_t stride = mOutputPortDef.format.video.nStride; int32_t slice_height = mOutputPortDef.format.video.nSliceHeight; int32_t width = mTrackInfo->GetAsVideoInfo()->mImage.width; int32_t height = mTrackInfo->GetAsVideoInfo()->mImage.height; // TODO: convert other formats to YUV420. if (mOutputPortDef.format.video.eColorFormat != OMX_COLOR_FormatYUV420Planar) { return nullptr; } size_t yuv420p_y_size = stride * slice_height; size_t yuv420p_u_size = ((stride + 1) / 2) * ((slice_height + 1) / 2); uint8_t* yuv420p_y = yuv420p_buffer; uint8_t* yuv420p_u = yuv420p_y + yuv420p_y_size; uint8_t* yuv420p_v = yuv420p_u + yuv420p_u_size; VideoData::YCbCrBuffer b; b.mPlanes[0].mData = yuv420p_y; b.mPlanes[0].mWidth = width; b.mPlanes[0].mHeight = height; b.mPlanes[0].mStride = stride; b.mPlanes[0].mSkip = 0; b.mPlanes[1].mData = yuv420p_u; b.mPlanes[1].mWidth = (width + 1) / 2; b.mPlanes[1].mHeight = (height + 1) / 2; b.mPlanes[1].mStride = (stride + 1) / 2; b.mPlanes[1].mSkip = 0; b.mPlanes[2].mData = yuv420p_v; b.mPlanes[2].mWidth = (width + 1) / 2; b.mPlanes[2].mHeight = (height + 1) / 2; b.mPlanes[2].mStride = (stride + 1) / 2; b.mPlanes[2].mSkip = 0; b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT; VideoInfo info(*mTrackInfo->GetAsVideoInfo()); auto maybeColorSpace = info.mColorSpace; if (!maybeColorSpace) { maybeColorSpace = Some(DefaultColorSpace({width, height})); } b.mYUVColorSpace = *maybeColorSpace; auto maybeColorPrimaries = info.mColorPrimaries; if (!maybeColorPrimaries) { maybeColorPrimaries = Some(gfx::ColorSpace2::BT709); } b.mColorPrimaries = *maybeColorPrimaries; Result, MediaResult> result = VideoData::CreateAndCopyData( info, mImageContainer, 0, // Filled later by caller. media::TimeUnit::Zero(), // Filled later by caller. media::TimeUnit::FromMicroseconds(1), // We don't know the duration. b, false, // Filled later by caller. media::TimeUnit::FromMicroseconds(-1), info.ImageRect(), nullptr); if (result.isErr()) { MediaResult r = result.unwrapErr(); MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, ("Failed to create a YUV420 VideoData - %s: %s", r.ErrorName().get(), r.Message().get())); return nullptr; } RefPtr data = result.unwrap(); MOZ_ASSERT(data); MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, ("YUV420 VideoData: disp width %d, height %d, pic width %d, height " "%d, time %lld", info.mDisplay.width, info.mDisplay.height, info.mImage.width, info.mImage.height, aBufferData->mBuffer->nTimeStamp)); return data.forget(); } } // namespace mozilla