diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 19:33:14 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 19:33:14 +0000 |
commit | 36d22d82aa202bb199967e9512281e9a53db42c9 (patch) | |
tree | 105e8c98ddea1c1e4784a60a5a6410fa416be2de /dom/media/platforms/android | |
parent | Initial commit. (diff) | |
download | firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.tar.xz firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.zip |
Adding upstream version 115.7.0esr.upstream/115.7.0esr
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'dom/media/platforms/android')
-rw-r--r-- | dom/media/platforms/android/AndroidDataEncoder.cpp | 534 | ||||
-rw-r--r-- | dom/media/platforms/android/AndroidDataEncoder.h | 110 | ||||
-rw-r--r-- | dom/media/platforms/android/AndroidDecoderModule.cpp | 243 | ||||
-rw-r--r-- | dom/media/platforms/android/AndroidDecoderModule.h | 60 | ||||
-rw-r--r-- | dom/media/platforms/android/AndroidEncoderModule.cpp | 56 | ||||
-rw-r--r-- | dom/media/platforms/android/AndroidEncoderModule.h | 23 | ||||
-rw-r--r-- | dom/media/platforms/android/JavaCallbacksSupport.h | 73 | ||||
-rw-r--r-- | dom/media/platforms/android/RemoteDataDecoder.cpp | 1136 | ||||
-rw-r--r-- | dom/media/platforms/android/RemoteDataDecoder.h | 112 |
9 files changed, 2347 insertions, 0 deletions
diff --git a/dom/media/platforms/android/AndroidDataEncoder.cpp b/dom/media/platforms/android/AndroidDataEncoder.cpp new file mode 100644 index 0000000000..f2768af09e --- /dev/null +++ b/dom/media/platforms/android/AndroidDataEncoder.cpp @@ -0,0 +1,534 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "AndroidDataEncoder.h" + +#include "AnnexB.h" +#include "H264.h" +#include "MediaData.h" +#include "MediaInfo.h" +#include "SimpleMap.h" + +#include "ImageContainer.h" +#include "mozilla/Logging.h" +#include "mozilla/ResultVariant.h" + +#include "nsMimeTypes.h" + +#include "libyuv.h" + +namespace mozilla { +using media::TimeUnit; + +extern LazyLogModule sPEMLog; +#define AND_ENC_LOG(arg, ...) \ + MOZ_LOG(sPEMLog, mozilla::LogLevel::Debug, \ + ("AndroidDataEncoder(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) +#define AND_ENC_LOGE(arg, ...) \ + MOZ_LOG(sPEMLog, mozilla::LogLevel::Error, \ + ("AndroidDataEncoder(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) + +#define REJECT_IF_ERROR() \ + do { \ + if (mError) { \ + auto error = mError.value(); \ + mError.reset(); \ + return EncodePromise::CreateAndReject(std::move(error), __func__); \ + } \ + } while (0) + +template <typename ConfigType> +RefPtr<MediaDataEncoder::InitPromise> AndroidDataEncoder<ConfigType>::Init() { + // Sanity-check the input size for Android software encoder fails to do it. + if (mConfig.mSize.width == 0 || mConfig.mSize.height == 0) { + return InitPromise::CreateAndReject(NS_ERROR_ILLEGAL_VALUE, __func__); + } + + return InvokeAsync(mTaskQueue, this, __func__, + &AndroidDataEncoder<ConfigType>::ProcessInit); +} + +static const char* MimeTypeOf(MediaDataEncoder::CodecType aCodec) { + switch (aCodec) { + case MediaDataEncoder::CodecType::H264: + return "video/avc"; + case MediaDataEncoder::CodecType::VP8: + return "video/x-vnd.on2.vp8"; + case MediaDataEncoder::CodecType::VP9: + return "video/x-vnd.on2.vp9"; + default: + return ""; + } +} + +using FormatResult = Result<java::sdk::MediaFormat::LocalRef, MediaResult>; + +template <typename ConfigType> +FormatResult ToMediaFormat(const ConfigType& aConfig) { + nsresult rv = NS_OK; + java::sdk::MediaFormat::LocalRef format; + rv = java::sdk::MediaFormat::CreateVideoFormat(MimeTypeOf(aConfig.mCodecType), + aConfig.mSize.width, + aConfig.mSize.height, &format); + NS_ENSURE_SUCCESS( + rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "fail to create Java MediaFormat object"))); + + rv = + format->SetInteger(java::sdk::MediaFormat::KEY_BITRATE_MODE, 2 /* CBR */); + NS_ENSURE_SUCCESS(rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "fail to set bitrate mode"))); + + rv = format->SetInteger(java::sdk::MediaFormat::KEY_BIT_RATE, + aConfig.mBitsPerSec); + NS_ENSURE_SUCCESS(rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "fail to set bitrate"))); + + // COLOR_FormatYUV420SemiPlanar(NV12) is the most widely supported + // format. + rv = format->SetInteger(java::sdk::MediaFormat::KEY_COLOR_FORMAT, 0x15); + NS_ENSURE_SUCCESS(rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "fail to set color format"))); + + rv = format->SetInteger(java::sdk::MediaFormat::KEY_FRAME_RATE, + aConfig.mFramerate); + NS_ENSURE_SUCCESS(rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "fail to set frame rate"))); + + // Ensure interval >= 1. A negative value means no key frames are + // requested after the first frame. A zero value means a stream + // containing all key frames is requested. + int32_t intervalInSec = + std::max<size_t>(1, aConfig.mKeyframeInterval / aConfig.mFramerate); + rv = format->SetInteger(java::sdk::MediaFormat::KEY_I_FRAME_INTERVAL, + intervalInSec); + NS_ENSURE_SUCCESS(rv, + FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "fail to set I-frame interval"))); + + return format; +} + +template <typename ConfigType> +RefPtr<MediaDataEncoder::InitPromise> +AndroidDataEncoder<ConfigType>::ProcessInit() { + AssertOnTaskQueue(); + MOZ_ASSERT(!mJavaEncoder); + + java::sdk::MediaCodec::BufferInfo::LocalRef bufferInfo; + if (NS_FAILED(java::sdk::MediaCodec::BufferInfo::New(&bufferInfo)) || + !bufferInfo) { + return InitPromise::CreateAndReject(NS_ERROR_OUT_OF_MEMORY, __func__); + } + mInputBufferInfo = bufferInfo; + + FormatResult result = ToMediaFormat<ConfigType>(mConfig); + if (result.isErr()) { + return InitPromise::CreateAndReject(result.unwrapErr(), __func__); + } + mFormat = result.unwrap(); + + // Register native methods. + JavaCallbacksSupport::Init(); + + mJavaCallbacks = java::CodecProxy::NativeCallbacks::New(); + if (!mJavaCallbacks) { + return InitPromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "cannot create Java callback object"), + __func__); + } + JavaCallbacksSupport::AttachNative( + mJavaCallbacks, mozilla::MakeUnique<CallbacksSupport>(this)); + + mJavaEncoder = java::CodecProxy::Create(true /* encoder */, mFormat, nullptr, + mJavaCallbacks, u""_ns); + if (!mJavaEncoder) { + return InitPromise::CreateAndReject( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + "cannot create Java encoder object"), + __func__); + } + + mIsHardwareAccelerated = mJavaEncoder->IsHardwareAccelerated(); + mDrainState = DrainState::DRAINABLE; + + return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__); +} + +template <typename ConfigType> +RefPtr<MediaDataEncoder::EncodePromise> AndroidDataEncoder<ConfigType>::Encode( + const MediaData* aSample) { + RefPtr<AndroidDataEncoder> self = this; + MOZ_ASSERT(aSample != nullptr); + + RefPtr<const MediaData> sample(aSample); + return InvokeAsync(mTaskQueue, __func__, [self, sample]() { + return self->ProcessEncode(std::move(sample)); + }); +} + +static jni::ByteBuffer::LocalRef ConvertI420ToNV12Buffer( + RefPtr<const VideoData> aSample, RefPtr<MediaByteBuffer>& aYUVBuffer, + int aStride, int aYPlaneHeight) { + const layers::PlanarYCbCrImage* image = aSample->mImage->AsPlanarYCbCrImage(); + MOZ_ASSERT(image); + const layers::PlanarYCbCrData* yuv = image->GetData(); + auto ySize = yuv->YDataSize(); + auto cbcrSize = yuv->CbCrDataSize(); + // If we have a stride or height passed in from the Codec we need to use + // those. + auto yStride = aStride != 0 ? aStride : yuv->mYStride; + auto height = aYPlaneHeight != 0 ? aYPlaneHeight : ySize.height; + size_t yLength = yStride * height; + size_t length = + yLength + yStride * (cbcrSize.height - 1) + cbcrSize.width * 2; + + if (!aYUVBuffer || aYUVBuffer->Capacity() < length) { + aYUVBuffer = MakeRefPtr<MediaByteBuffer>(length); + aYUVBuffer->SetLength(length); + } else { + MOZ_ASSERT(aYUVBuffer->Length() >= length); + } + + if (libyuv::I420ToNV12(yuv->mYChannel, yuv->mYStride, yuv->mCbChannel, + yuv->mCbCrStride, yuv->mCrChannel, yuv->mCbCrStride, + aYUVBuffer->Elements(), yStride, + aYUVBuffer->Elements() + yLength, yStride, ySize.width, + ySize.height) != 0) { + return nullptr; + } + + return jni::ByteBuffer::New(aYUVBuffer->Elements(), aYUVBuffer->Length()); +} + +template <typename ConfigType> +RefPtr<MediaDataEncoder::EncodePromise> +AndroidDataEncoder<ConfigType>::ProcessEncode(RefPtr<const MediaData> aSample) { + AssertOnTaskQueue(); + + REJECT_IF_ERROR(); + + RefPtr<const VideoData> sample(aSample->As<const VideoData>()); + MOZ_ASSERT(sample); + + // Bug 1789846: Check with the Encoder if MediaCodec has a stride or height + // value to use. + jni::ByteBuffer::LocalRef buffer = ConvertI420ToNV12Buffer( + sample, mYUVBuffer, mJavaEncoder->GetInputFormatStride(), + mJavaEncoder->GetInputFormatYPlaneHeight()); + if (!buffer) { + return EncodePromise::CreateAndReject(NS_ERROR_ILLEGAL_INPUT, __func__); + } + + if (aSample->mKeyframe) { + mInputBufferInfo->Set(0, mYUVBuffer->Length(), + aSample->mTime.ToMicroseconds(), + java::sdk::MediaCodec::BUFFER_FLAG_SYNC_FRAME); + } else { + mInputBufferInfo->Set(0, mYUVBuffer->Length(), + aSample->mTime.ToMicroseconds(), 0); + } + + mJavaEncoder->Input(buffer, mInputBufferInfo, nullptr); + + if (mEncodedData.Length() > 0) { + EncodedData pending = std::move(mEncodedData); + return EncodePromise::CreateAndResolve(std::move(pending), __func__); + } else { + return EncodePromise::CreateAndResolve(EncodedData(), __func__); + } +} + +class AutoRelease final { + public: + AutoRelease(java::CodecProxy::Param aEncoder, java::Sample::Param aSample) + : mEncoder(aEncoder), mSample(aSample) {} + + ~AutoRelease() { mEncoder->ReleaseOutput(mSample, false); } + + private: + java::CodecProxy::GlobalRef mEncoder; + java::Sample::GlobalRef mSample; +}; + +static RefPtr<MediaByteBuffer> ExtractCodecConfig( + java::SampleBuffer::Param aBuffer, const int32_t aOffset, + const int32_t aSize, const bool aAsAnnexB) { + auto annexB = MakeRefPtr<MediaByteBuffer>(aSize); + annexB->SetLength(aSize); + jni::ByteBuffer::LocalRef dest = + jni::ByteBuffer::New(annexB->Elements(), aSize); + aBuffer->WriteToByteBuffer(dest, aOffset, aSize); + if (aAsAnnexB) { + return annexB; + } + // Convert to avcC. + nsTArray<AnnexB::NALEntry> paramSets; + AnnexB::ParseNALEntries( + Span<const uint8_t>(annexB->Elements(), annexB->Length()), paramSets); + + auto avcc = MakeRefPtr<MediaByteBuffer>(); + AnnexB::NALEntry& sps = paramSets.ElementAt(0); + AnnexB::NALEntry& pps = paramSets.ElementAt(1); + const uint8_t* spsPtr = annexB->Elements() + sps.mOffset; + H264::WriteExtraData( + avcc, spsPtr[1], spsPtr[2], spsPtr[3], + Span<const uint8_t>(spsPtr, sps.mSize), + Span<const uint8_t>(annexB->Elements() + pps.mOffset, pps.mSize)); + return avcc; +} + +template <typename ConfigType> +void AndroidDataEncoder<ConfigType>::ProcessOutput( + java::Sample::GlobalRef&& aSample, + java::SampleBuffer::GlobalRef&& aBuffer) { + if (!mTaskQueue->IsCurrentThreadIn()) { + nsresult rv = + mTaskQueue->Dispatch(NewRunnableMethod<java::Sample::GlobalRef&&, + java::SampleBuffer::GlobalRef&&>( + "AndroidDataEncoder::ProcessOutput", this, + &AndroidDataEncoder::ProcessOutput, std::move(aSample), + std::move(aBuffer))); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; + return; + } + AssertOnTaskQueue(); + + if (!mJavaEncoder) { + return; + } + + AutoRelease releaseSample(mJavaEncoder, aSample); + + java::sdk::MediaCodec::BufferInfo::LocalRef info = aSample->Info(); + MOZ_ASSERT(info); + + int32_t flags; + bool ok = NS_SUCCEEDED(info->Flags(&flags)); + bool isEOS = !!(flags & java::sdk::MediaCodec::BUFFER_FLAG_END_OF_STREAM); + + int32_t offset; + ok &= NS_SUCCEEDED(info->Offset(&offset)); + + int32_t size; + ok &= NS_SUCCEEDED(info->Size(&size)); + + int64_t presentationTimeUs; + ok &= NS_SUCCEEDED(info->PresentationTimeUs(&presentationTimeUs)); + + if (!ok) { + return; + } + + if (size > 0) { + if ((flags & java::sdk::MediaCodec::BUFFER_FLAG_CODEC_CONFIG) != 0) { + mConfigData = ExtractCodecConfig(aBuffer, offset, size, + mConfig.mUsage == Usage::Realtime); + return; + } + RefPtr<MediaRawData> output = + GetOutputData(aBuffer, offset, size, + !!(flags & java::sdk::MediaCodec::BUFFER_FLAG_KEY_FRAME)); + output->mEOS = isEOS; + output->mTime = media::TimeUnit::FromMicroseconds(presentationTimeUs); + mEncodedData.AppendElement(std::move(output)); + } + + if (isEOS) { + mDrainState = DrainState::DRAINED; + } + if (!mDrainPromise.IsEmpty()) { + EncodedData pending = std::move(mEncodedData); + mDrainPromise.Resolve(std::move(pending), __func__); + } +} + +template <typename ConfigType> +RefPtr<MediaRawData> AndroidDataEncoder<ConfigType>::GetOutputData( + java::SampleBuffer::Param aBuffer, const int32_t aOffset, + const int32_t aSize, const bool aIsKeyFrame) { + // Copy frame data from Java buffer. + auto output = MakeRefPtr<MediaRawData>(); + UniquePtr<MediaRawDataWriter> writer(output->CreateWriter()); + if (!writer->SetSize(aSize)) { + AND_ENC_LOGE("fail to allocate output buffer"); + return nullptr; + } + + jni::ByteBuffer::LocalRef buf = jni::ByteBuffer::New(writer->Data(), aSize); + aBuffer->WriteToByteBuffer(buf, aOffset, aSize); + output->mKeyframe = aIsKeyFrame; + + return output; +} + +// AVC/H.264 frame can be in avcC or Annex B and needs extra convertion steps. +template <> +RefPtr<MediaRawData> +AndroidDataEncoder<MediaDataEncoder::H264Config>::GetOutputData( + java::SampleBuffer::Param aBuffer, const int32_t aOffset, + const int32_t aSize, const bool aIsKeyFrame) { + auto output = MakeRefPtr<MediaRawData>(); + + size_t prependSize = 0; + RefPtr<MediaByteBuffer> avccHeader; + if (aIsKeyFrame && mConfigData) { + if (mConfig.mUsage == Usage::Realtime) { + prependSize = mConfigData->Length(); + } else { + avccHeader = mConfigData; + } + } + + UniquePtr<MediaRawDataWriter> writer(output->CreateWriter()); + if (!writer->SetSize(prependSize + aSize)) { + AND_ENC_LOGE("fail to allocate output buffer"); + return nullptr; + } + + if (prependSize > 0) { + PodCopy(writer->Data(), mConfigData->Elements(), prependSize); + } + + jni::ByteBuffer::LocalRef buf = + jni::ByteBuffer::New(writer->Data() + prependSize, aSize); + aBuffer->WriteToByteBuffer(buf, aOffset, aSize); + + if (mConfig.mUsage != Usage::Realtime && + !AnnexB::ConvertSampleToAVCC(output, avccHeader)) { + AND_ENC_LOGE("fail to convert annex-b sample to AVCC"); + return nullptr; + } + + output->mKeyframe = aIsKeyFrame; + + return output; +} + +template <typename ConfigType> +RefPtr<MediaDataEncoder::EncodePromise> +AndroidDataEncoder<ConfigType>::Drain() { + return InvokeAsync(mTaskQueue, this, __func__, + &AndroidDataEncoder<ConfigType>::ProcessDrain); +} + +template <typename ConfigType> +RefPtr<MediaDataEncoder::EncodePromise> +AndroidDataEncoder<ConfigType>::ProcessDrain() { + AssertOnTaskQueue(); + MOZ_ASSERT(mJavaEncoder); + MOZ_ASSERT(mDrainPromise.IsEmpty()); + + REJECT_IF_ERROR(); + + switch (mDrainState) { + case DrainState::DRAINABLE: + mInputBufferInfo->Set(0, 0, -1, + java::sdk::MediaCodec::BUFFER_FLAG_END_OF_STREAM); + mJavaEncoder->Input(nullptr, mInputBufferInfo, nullptr); + mDrainState = DrainState::DRAINING; + [[fallthrough]]; + case DrainState::DRAINING: + if (mEncodedData.IsEmpty()) { + return mDrainPromise.Ensure(__func__); // Pending promise. + } + [[fallthrough]]; + case DrainState::DRAINED: + if (mEncodedData.Length() > 0) { + EncodedData pending = std::move(mEncodedData); + return EncodePromise::CreateAndResolve(std::move(pending), __func__); + } else { + return EncodePromise::CreateAndResolve(EncodedData(), __func__); + } + } +} + +template <typename ConfigType> +RefPtr<ShutdownPromise> AndroidDataEncoder<ConfigType>::Shutdown() { + return InvokeAsync(mTaskQueue, this, __func__, + &AndroidDataEncoder<ConfigType>::ProcessShutdown); +} + +template <typename ConfigType> +RefPtr<ShutdownPromise> AndroidDataEncoder<ConfigType>::ProcessShutdown() { + AssertOnTaskQueue(); + if (mJavaEncoder) { + mJavaEncoder->Release(); + mJavaEncoder = nullptr; + } + + if (mJavaCallbacks) { + JavaCallbacksSupport::GetNative(mJavaCallbacks)->Cancel(); + JavaCallbacksSupport::DisposeNative(mJavaCallbacks); + mJavaCallbacks = nullptr; + } + + mFormat = nullptr; + + return ShutdownPromise::CreateAndResolve(true, __func__); +} + +template <typename ConfigType> +RefPtr<GenericPromise> AndroidDataEncoder<ConfigType>::SetBitrate( + const MediaDataEncoder::Rate aBitsPerSec) { + RefPtr<AndroidDataEncoder> self(this); + return InvokeAsync(mTaskQueue, __func__, [self, aBitsPerSec]() { + self->mJavaEncoder->SetBitrate(aBitsPerSec); + return GenericPromise::CreateAndResolve(true, __func__); + }); + + return nullptr; +} + +template <typename ConfigType> +void AndroidDataEncoder<ConfigType>::Error(const MediaResult& aError) { + if (!mTaskQueue->IsCurrentThreadIn()) { + nsresult rv = mTaskQueue->Dispatch(NewRunnableMethod<MediaResult>( + "AndroidDataEncoder::Error", this, + &AndroidDataEncoder<ConfigType>::Error, aError)); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; + return; + } + AssertOnTaskQueue(); + + mError = Some(aError); +} + +template <typename ConfigType> +void AndroidDataEncoder<ConfigType>::CallbacksSupport::HandleInput( + int64_t aTimestamp, bool aProcessed) {} + +template <typename ConfigType> +void AndroidDataEncoder<ConfigType>::CallbacksSupport::HandleOutput( + java::Sample::Param aSample, java::SampleBuffer::Param aBuffer) { + MutexAutoLock lock(mMutex); + if (mEncoder) { + mEncoder->ProcessOutput(std::move(aSample), std::move(aBuffer)); + } +} + +template <typename ConfigType> +void AndroidDataEncoder<ConfigType>::CallbacksSupport:: + HandleOutputFormatChanged(java::sdk::MediaFormat::Param aFormat) {} + +template <typename ConfigType> +void AndroidDataEncoder<ConfigType>::CallbacksSupport::HandleError( + const MediaResult& aError) { + MutexAutoLock lock(mMutex); + if (mEncoder) { + mEncoder->Error(aError); + } +} + +// Force compiler to generate code. +template class AndroidDataEncoder<MediaDataEncoder::H264Config>; +template class AndroidDataEncoder<MediaDataEncoder::VP8Config>; +template class AndroidDataEncoder<MediaDataEncoder::VP9Config>; +} // namespace mozilla + +#undef AND_ENC_LOG +#undef AND_ENC_LOGE diff --git a/dom/media/platforms/android/AndroidDataEncoder.h b/dom/media/platforms/android/AndroidDataEncoder.h new file mode 100644 index 0000000000..b7752e2a43 --- /dev/null +++ b/dom/media/platforms/android/AndroidDataEncoder.h @@ -0,0 +1,110 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_PLATFORMS_ANDROID_ANDROIDDATAENCODER_H_ +#define DOM_MEDIA_PLATFORMS_ANDROID_ANDROIDDATAENCODER_H_ + +#include "MediaData.h" +#include "PlatformEncoderModule.h" +#include "TimeUnits.h" + +#include "JavaCallbacksSupport.h" + +#include "mozilla/Maybe.h" +#include "mozilla/Monitor.h" +#include "mozilla/Mutex.h" + +namespace mozilla { + +template <typename ConfigType> +class AndroidDataEncoder final : public MediaDataEncoder { + public: + AndroidDataEncoder(const ConfigType& aConfig, RefPtr<TaskQueue> aTaskQueue) + : mConfig(aConfig), mTaskQueue(aTaskQueue) { + MOZ_ASSERT(mConfig.mSize.width > 0 && mConfig.mSize.height > 0); + MOZ_ASSERT(mTaskQueue); + } + RefPtr<InitPromise> Init() override; + RefPtr<EncodePromise> Encode(const MediaData* aSample) override; + RefPtr<EncodePromise> Drain() override; + RefPtr<ShutdownPromise> Shutdown() override; + RefPtr<GenericPromise> SetBitrate(const Rate aBitsPerSec) override; + + nsCString GetDescriptionName() const override { return "Android Encoder"_ns; } + + private: + class CallbacksSupport final : public JavaCallbacksSupport { + public: + explicit CallbacksSupport(AndroidDataEncoder* aEncoder) + : mMutex("AndroidDataEncoder::CallbacksSupport") { + MutexAutoLock lock(mMutex); + mEncoder = aEncoder; + } + + ~CallbacksSupport() { + MutexAutoLock lock(mMutex); + mEncoder = nullptr; + } + + void HandleInput(int64_t aTimestamp, bool aProcessed) override; + void HandleOutput(java::Sample::Param aSample, + java::SampleBuffer::Param aBuffer) override; + void HandleOutputFormatChanged( + java::sdk::MediaFormat::Param aFormat) override; + void HandleError(const MediaResult& aError) override; + + private: + Mutex mMutex; + AndroidDataEncoder* mEncoder MOZ_GUARDED_BY(mMutex); + }; + friend class CallbacksSupport; + + // Methods only called on mTaskQueue. + RefPtr<InitPromise> ProcessInit(); + RefPtr<EncodePromise> ProcessEncode(RefPtr<const MediaData> aSample); + RefPtr<EncodePromise> ProcessDrain(); + RefPtr<ShutdownPromise> ProcessShutdown(); + void ProcessInput(); + void ProcessOutput(java::Sample::GlobalRef&& aSample, + java::SampleBuffer::GlobalRef&& aBuffer); + RefPtr<MediaRawData> GetOutputData(java::SampleBuffer::Param aBuffer, + const int32_t aOffset, const int32_t aSize, + const bool aIsKeyFrame); + void Error(const MediaResult& aError); + + void AssertOnTaskQueue() const { + MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn()); + } + + const ConfigType mConfig; + + RefPtr<TaskQueue> mTaskQueue; + + // Can be accessed on any thread, but only written on during init. + bool mIsHardwareAccelerated = false; + + java::CodecProxy::GlobalRef mJavaEncoder; + java::CodecProxy::NativeCallbacks::GlobalRef mJavaCallbacks; + java::sdk::MediaFormat::GlobalRef mFormat; + // Preallocated Java object used as a reusable storage for input buffer + // information. Contents must be changed only on mTaskQueue. + java::sdk::MediaCodec::BufferInfo::GlobalRef mInputBufferInfo; + + MozPromiseHolder<EncodePromise> mDrainPromise; + + // Accessed on mTaskqueue only. + RefPtr<MediaByteBuffer> mYUVBuffer; + EncodedData mEncodedData; + // SPS/PPS NALUs for realtime usage, avcC otherwise. + RefPtr<MediaByteBuffer> mConfigData; + + enum class DrainState { DRAINED, DRAINABLE, DRAINING }; + DrainState mDrainState; + + Maybe<MediaResult> mError; +}; + +} // namespace mozilla + +#endif diff --git a/dom/media/platforms/android/AndroidDecoderModule.cpp b/dom/media/platforms/android/AndroidDecoderModule.cpp new file mode 100644 index 0000000000..af7691168b --- /dev/null +++ b/dom/media/platforms/android/AndroidDecoderModule.cpp @@ -0,0 +1,243 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <jni.h> + +#ifdef MOZ_AV1 +# include "AOMDecoder.h" +#endif +#include "MediaInfo.h" +#include "OpusDecoder.h" +#include "RemoteDataDecoder.h" +#include "TheoraDecoder.h" +#include "VPXDecoder.h" +#include "VorbisDecoder.h" +#include "mozilla/ClearOnShutdown.h" +#include "mozilla/Components.h" +#include "mozilla/StaticPrefs_media.h" +#include "mozilla/gfx/gfxVars.h" +#include "mozilla/java/HardwareCodecCapabilityUtilsWrappers.h" +#include "nsIGfxInfo.h" +#include "nsPromiseFlatString.h" +#include "prlog.h" + +#undef LOG +#define LOG(arg, ...) \ + MOZ_LOG( \ + sAndroidDecoderModuleLog, mozilla::LogLevel::Debug, \ + ("AndroidDecoderModule(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) +#define SLOG(arg, ...) \ + MOZ_LOG(sAndroidDecoderModuleLog, mozilla::LogLevel::Debug, \ + ("%s: " arg, __func__, ##__VA_ARGS__)) + +using namespace mozilla; + +namespace mozilla { + +mozilla::LazyLogModule sAndroidDecoderModuleLog("AndroidDecoderModule"); + +nsCString TranslateMimeType(const nsACString& aMimeType) { + if (VPXDecoder::IsVPX(aMimeType, VPXDecoder::VP8)) { + static constexpr auto vp8 = "video/x-vnd.on2.vp8"_ns; + return vp8; + } + if (VPXDecoder::IsVPX(aMimeType, VPXDecoder::VP9)) { + static constexpr auto vp9 = "video/x-vnd.on2.vp9"_ns; + return vp9; + } + if (aMimeType.EqualsLiteral("video/av1")) { + static constexpr auto av1 = "video/av01"_ns; + return av1; + } + return nsCString(aMimeType); +} + +AndroidDecoderModule::AndroidDecoderModule(CDMProxy* aProxy) { + mProxy = static_cast<MediaDrmCDMProxy*>(aProxy); +} + +StaticAutoPtr<nsTArray<nsCString>> AndroidDecoderModule::sSupportedMimeTypes; + +media::DecodeSupportSet AndroidDecoderModule::SupportsMimeType( + const nsACString& aMimeType) { + if (jni::GetAPIVersion() < 16) { + return media::DecodeSupport::Unsupported; + } + + if (aMimeType.EqualsLiteral("video/mp4") || + aMimeType.EqualsLiteral("video/avc")) { + // TODO: Note that we do not yet distinguish between SW/HW decode support. + // Will be done in bug 1754239. + return media::DecodeSupport::SoftwareDecode; + } + + // When checking "audio/x-wav", CreateDecoder can cause a JNI ERROR by + // Accessing a stale local reference leading to a SIGSEGV crash. + // To avoid this we check for wav types here. + if (aMimeType.EqualsLiteral("audio/x-wav") || + aMimeType.EqualsLiteral("audio/wave; codecs=1") || + aMimeType.EqualsLiteral("audio/wave; codecs=3") || + aMimeType.EqualsLiteral("audio/wave; codecs=6") || + aMimeType.EqualsLiteral("audio/wave; codecs=7") || + aMimeType.EqualsLiteral("audio/wave; codecs=65534")) { + return media::DecodeSupport::Unsupported; + } + + if ((VPXDecoder::IsVPX(aMimeType, VPXDecoder::VP8) && + !gfx::gfxVars::UseVP8HwDecode()) || + (VPXDecoder::IsVPX(aMimeType, VPXDecoder::VP9) && + !gfx::gfxVars::UseVP9HwDecode())) { + return media::DecodeSupport::Unsupported; + } + + // Prefer the gecko decoder for opus and vorbis; stagefright crashes + // on content demuxed from mp4. + // Not all android devices support FLAC even when they say they do. + if (OpusDataDecoder::IsOpus(aMimeType) || + VorbisDataDecoder::IsVorbis(aMimeType) || + aMimeType.EqualsLiteral("audio/flac")) { + SLOG("Rejecting audio of type %s", aMimeType.Data()); + return media::DecodeSupport::Unsupported; + } + + // Prefer the gecko decoder for Theora. + // Not all android devices support Theora even when they say they do. + if (TheoraDecoder::IsTheora(aMimeType)) { + SLOG("Rejecting video of type %s", aMimeType.Data()); + return media::DecodeSupport::Unsupported; + } + + if (aMimeType.EqualsLiteral("audio/mpeg") && + StaticPrefs::media_ffvpx_mp3_enabled()) { + // Prefer the ffvpx mp3 software decoder if available. + return media::DecodeSupport::Unsupported; + } + + if (sSupportedMimeTypes) { + if (sSupportedMimeTypes->Contains(TranslateMimeType(aMimeType))) { + // TODO: Note that we do not yet distinguish between SW/HW decode support. + // Will be done in bug 1754239. + return media::DecodeSupport::SoftwareDecode; + } + } + + return media::DecodeSupport::Unsupported; +} + +nsTArray<nsCString> AndroidDecoderModule::GetSupportedMimeTypes() { + mozilla::jni::ObjectArray::LocalRef supportedTypes = mozilla::java:: + HardwareCodecCapabilityUtils::GetDecoderSupportedMimeTypes(); + + nsTArray<nsCString> st = nsTArray<nsCString>(); + for (size_t i = 0; i < supportedTypes->Length(); i++) { + st.AppendElement( + jni::String::LocalRef(supportedTypes->GetElement(i))->ToCString()); + } + + return st; +} + +void AndroidDecoderModule::SetSupportedMimeTypes( + nsTArray<nsCString>&& aSupportedTypes) { + if (!sSupportedMimeTypes) { + sSupportedMimeTypes = new nsTArray<nsCString>(std::move(aSupportedTypes)); + ClearOnShutdown(&sSupportedMimeTypes); + } +} + +media::DecodeSupportSet AndroidDecoderModule::SupportsMimeType( + const nsACString& aMimeType, DecoderDoctorDiagnostics* aDiagnostics) const { + return AndroidDecoderModule::SupportsMimeType(aMimeType); +} + +bool AndroidDecoderModule::SupportsColorDepth( + gfx::ColorDepth aColorDepth, DecoderDoctorDiagnostics* aDiagnostics) const { + // 10-bit support is codec dependent so this is not entirely accurate. + // Supports() will correct it. + return aColorDepth == gfx::ColorDepth::COLOR_8 || + aColorDepth == gfx::ColorDepth::COLOR_10; +} + +// Further check is needed because the base class uses the inaccurate +// SupportsColorDepth(). +media::DecodeSupportSet AndroidDecoderModule::Supports( + const SupportDecoderParams& aParams, + DecoderDoctorDiagnostics* aDiagnostics) const { + media::DecodeSupportSet support = + PlatformDecoderModule::Supports(aParams, aDiagnostics); + + // Short-circuit. + if (support == media::DecodeSupport::Unsupported) { + return support; + } + +#ifdef MOZ_AV1 + // For AV1, only allow HW decoder. + if (AOMDecoder::IsAV1(aParams.MimeType()) && + (!StaticPrefs::media_av1_enabled() || + !support.contains(media::DecodeSupport::HardwareDecode))) { + return media::DecodeSupport::Unsupported; + } +#endif + + // Check 10-bit video. + const TrackInfo& trackInfo = aParams.mConfig; + const VideoInfo* videoInfo = trackInfo.GetAsVideoInfo(); + if (!videoInfo || videoInfo->mColorDepth != gfx::ColorDepth::COLOR_10) { + return support; + } + + return java::HardwareCodecCapabilityUtils::Decodes10Bit( + TranslateMimeType(aParams.MimeType())) + ? support + : media::DecodeSupport::Unsupported; +} + +already_AddRefed<MediaDataDecoder> AndroidDecoderModule::CreateVideoDecoder( + const CreateDecoderParams& aParams) { + // Temporary - forces use of VPXDecoder when alpha is present. + // Bug 1263836 will handle alpha scenario once implemented. It will shift + // the check for alpha to PDMFactory but not itself remove the need for a + // check. + if (aParams.VideoConfig().HasAlpha()) { + return nullptr; + } + + nsString drmStubId; + if (mProxy) { + drmStubId = mProxy->GetMediaDrmStubId(); + } + + RefPtr<MediaDataDecoder> decoder = + RemoteDataDecoder::CreateVideoDecoder(aParams, drmStubId, mProxy); + return decoder.forget(); +} + +already_AddRefed<MediaDataDecoder> AndroidDecoderModule::CreateAudioDecoder( + const CreateDecoderParams& aParams) { + const AudioInfo& config = aParams.AudioConfig(); + if (config.mBitDepth != 16) { + // We only handle 16-bit audio. + return nullptr; + } + + LOG("CreateAudioFormat with mimeType=%s, mRate=%d, channels=%d", + config.mMimeType.Data(), config.mRate, config.mChannels); + + nsString drmStubId; + if (mProxy) { + drmStubId = mProxy->GetMediaDrmStubId(); + } + RefPtr<MediaDataDecoder> decoder = + RemoteDataDecoder::CreateAudioDecoder(aParams, drmStubId, mProxy); + return decoder.forget(); +} + +/* static */ +already_AddRefed<PlatformDecoderModule> AndroidDecoderModule::Create( + CDMProxy* aProxy) { + return MakeAndAddRef<AndroidDecoderModule>(aProxy); +} + +} // namespace mozilla diff --git a/dom/media/platforms/android/AndroidDecoderModule.h b/dom/media/platforms/android/AndroidDecoderModule.h new file mode 100644 index 0000000000..406fe09f31 --- /dev/null +++ b/dom/media/platforms/android/AndroidDecoderModule.h @@ -0,0 +1,60 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef AndroidDecoderModule_h_ +#define AndroidDecoderModule_h_ + +#include "PlatformDecoderModule.h" +#include "mozilla/MediaDrmCDMProxy.h" +#include "mozilla/StaticPtr.h" // for StaticAutoPtr + +namespace mozilla { + +class AndroidDecoderModule : public PlatformDecoderModule { + template <typename T, typename... Args> + friend already_AddRefed<T> MakeAndAddRef(Args&&...); + + public: + static already_AddRefed<PlatformDecoderModule> Create( + CDMProxy* aProxy = nullptr); + + already_AddRefed<MediaDataDecoder> CreateVideoDecoder( + const CreateDecoderParams& aParams) override; + + already_AddRefed<MediaDataDecoder> CreateAudioDecoder( + const CreateDecoderParams& aParams) override; + + media::DecodeSupportSet SupportsMimeType( + const nsACString& aMimeType, + DecoderDoctorDiagnostics* aDiagnostics) const override; + + static media::DecodeSupportSet SupportsMimeType(const nsACString& aMimeType); + + static nsTArray<nsCString> GetSupportedMimeTypes(); + + static void SetSupportedMimeTypes(nsTArray<nsCString>&& aSupportedTypes); + + media::DecodeSupportSet Supports( + const SupportDecoderParams& aParams, + DecoderDoctorDiagnostics* aDiagnostics) const override; + + protected: + bool SupportsColorDepth( + gfx::ColorDepth aColorDepth, + DecoderDoctorDiagnostics* aDiagnostics) const override; + + private: + explicit AndroidDecoderModule(CDMProxy* aProxy = nullptr); + virtual ~AndroidDecoderModule() = default; + RefPtr<MediaDrmCDMProxy> mProxy; + static StaticAutoPtr<nsTArray<nsCString>> sSupportedMimeTypes; +}; + +extern LazyLogModule sAndroidDecoderModuleLog; + +nsCString TranslateMimeType(const nsACString& aMimeType); + +} // namespace mozilla + +#endif diff --git a/dom/media/platforms/android/AndroidEncoderModule.cpp b/dom/media/platforms/android/AndroidEncoderModule.cpp new file mode 100644 index 0000000000..4fd85e3182 --- /dev/null +++ b/dom/media/platforms/android/AndroidEncoderModule.cpp @@ -0,0 +1,56 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "AndroidEncoderModule.h" + +#include "AndroidDataEncoder.h" +#include "MP4Decoder.h" +#include "VPXDecoder.h" + +#include "mozilla/Logging.h" + +namespace mozilla { +extern LazyLogModule sPEMLog; +#define AND_PEM_LOG(arg, ...) \ + MOZ_LOG( \ + sPEMLog, mozilla::LogLevel::Debug, \ + ("AndroidEncoderModule(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) + +bool AndroidEncoderModule::SupportsMimeType(const nsACString& aMimeType) const { + return (MP4Decoder::IsH264(aMimeType) && + java::HardwareCodecCapabilityUtils::HasHWH264(true /* encoder */)) || + (VPXDecoder::IsVP8(aMimeType) && + java::HardwareCodecCapabilityUtils::HasHWVP8(true /* encoder */)) || + (VPXDecoder::IsVP9(aMimeType) && + java::HardwareCodecCapabilityUtils::HasHWVP9(true /* encoder */)); +} + +already_AddRefed<MediaDataEncoder> AndroidEncoderModule::CreateVideoEncoder( + const CreateEncoderParams& aParams, const bool aHardwareNotAllowed) const { + // TODO: extend AndroidDataEncoder and Java codec to accept this option. + MOZ_ASSERT(!aHardwareNotAllowed); + + RefPtr<MediaDataEncoder> encoder; + switch (CreateEncoderParams::CodecTypeForMime(aParams.mConfig.mMimeType)) { + case MediaDataEncoder::CodecType::H264: + return MakeRefPtr<AndroidDataEncoder<MediaDataEncoder::H264Config>>( + aParams.ToH264Config(), aParams.mTaskQueue) + .forget(); + case MediaDataEncoder::CodecType::VP8: + return MakeRefPtr<AndroidDataEncoder<MediaDataEncoder::VP8Config>>( + aParams.ToVP8Config(), aParams.mTaskQueue) + .forget(); + case MediaDataEncoder::CodecType::VP9: + return MakeRefPtr<AndroidDataEncoder<MediaDataEncoder::VP9Config>>( + aParams.ToVP9Config(), aParams.mTaskQueue) + .forget(); + default: + AND_PEM_LOG("Unsupported MIME type:%s", aParams.mConfig.mMimeType.get()); + return nullptr; + } +} + +} // namespace mozilla + +#undef AND_PEM_LOG diff --git a/dom/media/platforms/android/AndroidEncoderModule.h b/dom/media/platforms/android/AndroidEncoderModule.h new file mode 100644 index 0000000000..e19b472a0e --- /dev/null +++ b/dom/media/platforms/android/AndroidEncoderModule.h @@ -0,0 +1,23 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef DOM_MEDIA_PLATFORMS_ANDROID_ANDROIDENCODERMODULE_H_ +#define DOM_MEDIA_PLATFORMS_ANDROID_ANDROIDENCODERMODULE_H_ + +#include "PlatformEncoderModule.h" + +namespace mozilla { + +class AndroidEncoderModule final : public PlatformEncoderModule { + public: + bool SupportsMimeType(const nsACString& aMimeType) const override; + + already_AddRefed<MediaDataEncoder> CreateVideoEncoder( + const CreateEncoderParams& aParams, + const bool aHardwareNotAllowed) const override; +}; + +} // namespace mozilla + +#endif diff --git a/dom/media/platforms/android/JavaCallbacksSupport.h b/dom/media/platforms/android/JavaCallbacksSupport.h new file mode 100644 index 0000000000..e79d796209 --- /dev/null +++ b/dom/media/platforms/android/JavaCallbacksSupport.h @@ -0,0 +1,73 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef JavaCallbacksSupport_h_ +#define JavaCallbacksSupport_h_ + +#include "MediaResult.h" +#include "MediaCodec.h" +#include "mozilla/java/CodecProxyNatives.h" +#include "mozilla/java/SampleBufferWrappers.h" +#include "mozilla/java/SampleWrappers.h" + +namespace mozilla { + +class JavaCallbacksSupport + : public java::CodecProxy::NativeCallbacks::Natives<JavaCallbacksSupport> { + public: + typedef java::CodecProxy::NativeCallbacks::Natives<JavaCallbacksSupport> Base; + using Base::AttachNative; + using Base::DisposeNative; + using Base::GetNative; + + JavaCallbacksSupport() : mCanceled(false) {} + + virtual ~JavaCallbacksSupport() {} + + virtual void HandleInput(int64_t aTimestamp, bool aProcessed) = 0; + + void OnInputStatus(jlong aTimestamp, bool aProcessed) { + if (!mCanceled) { + HandleInput(aTimestamp, aProcessed); + } + } + + virtual void HandleOutput(java::Sample::Param aSample, + java::SampleBuffer::Param aBuffer) = 0; + + void OnOutput(jni::Object::Param aSample, jni::Object::Param aBuffer) { + if (!mCanceled) { + HandleOutput(java::Sample::Ref::From(aSample), + java::SampleBuffer::Ref::From(aBuffer)); + } + } + + virtual void HandleOutputFormatChanged( + java::sdk::MediaFormat::Param aFormat){}; + + void OnOutputFormatChanged(jni::Object::Param aFormat) { + if (!mCanceled) { + HandleOutputFormatChanged(java::sdk::MediaFormat::Ref::From(aFormat)); + } + } + + virtual void HandleError(const MediaResult& aError) = 0; + + void OnError(bool aIsFatal) { + if (!mCanceled) { + HandleError(aIsFatal + ? MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__) + : MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, __func__)); + } + } + + void Cancel() { mCanceled = true; } + + private: + Atomic<bool> mCanceled; +}; + +} // namespace mozilla + +#endif diff --git a/dom/media/platforms/android/RemoteDataDecoder.cpp b/dom/media/platforms/android/RemoteDataDecoder.cpp new file mode 100644 index 0000000000..5233f9ac90 --- /dev/null +++ b/dom/media/platforms/android/RemoteDataDecoder.cpp @@ -0,0 +1,1136 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "RemoteDataDecoder.h" + +#include <jni.h> + +#include "AndroidBridge.h" +#include "AndroidBuild.h" +#include "AndroidDecoderModule.h" +#include "EMEDecoderModule.h" +#include "GLImages.h" +#include "JavaCallbacksSupport.h" +#include "MediaCodec.h" +#include "MediaData.h" +#include "MediaInfo.h" +#include "PerformanceRecorder.h" +#include "SimpleMap.h" +#include "VPXDecoder.h" +#include "VideoUtils.h" +#include "mozilla/gfx/Matrix.h" +#include "mozilla/gfx/Types.h" +#include "mozilla/java/CodecProxyWrappers.h" +#include "mozilla/java/GeckoSurfaceWrappers.h" +#include "mozilla/java/SampleBufferWrappers.h" +#include "mozilla/java/SampleWrappers.h" +#include "mozilla/java/SurfaceAllocatorWrappers.h" +#include "mozilla/Maybe.h" +#include "nsPromiseFlatString.h" +#include "nsThreadUtils.h" +#include "prlog.h" + +#undef LOG +#define LOG(arg, ...) \ + MOZ_LOG(sAndroidDecoderModuleLog, mozilla::LogLevel::Debug, \ + ("RemoteDataDecoder(%p)::%s: " arg, this, __func__, ##__VA_ARGS__)) + +using namespace mozilla; +using namespace mozilla::gl; +using media::TimeUnit; + +namespace mozilla { + +// Hold a reference to the output buffer until we're ready to release it back to +// the Java codec (for rendering or not). +class RenderOrReleaseOutput { + public: + RenderOrReleaseOutput(java::CodecProxy::Param aCodec, + java::Sample::Param aSample) + : mCodec(aCodec), mSample(aSample) {} + + virtual ~RenderOrReleaseOutput() { ReleaseOutput(false); } + + protected: + void ReleaseOutput(bool aToRender) { + if (mCodec && mSample) { + mCodec->ReleaseOutput(mSample, aToRender); + mCodec = nullptr; + mSample = nullptr; + } + } + + private: + java::CodecProxy::GlobalRef mCodec; + java::Sample::GlobalRef mSample; +}; + +class RemoteVideoDecoder final : public RemoteDataDecoder { + public: + // Render the output to the surface when the frame is sent + // to compositor, or release it if not presented. + class CompositeListener + : private RenderOrReleaseOutput, + public layers::SurfaceTextureImage::SetCurrentCallback { + public: + CompositeListener(java::CodecProxy::Param aCodec, + java::Sample::Param aSample) + : RenderOrReleaseOutput(aCodec, aSample) {} + + void operator()(void) override { ReleaseOutput(true); } + }; + + class InputInfo { + public: + InputInfo() = default; + + InputInfo(const int64_t aDurationUs, const gfx::IntSize& aImageSize, + const gfx::IntSize& aDisplaySize) + : mDurationUs(aDurationUs), + mImageSize(aImageSize), + mDisplaySize(aDisplaySize) {} + + int64_t mDurationUs = {}; + gfx::IntSize mImageSize = {}; + gfx::IntSize mDisplaySize = {}; + }; + + class CallbacksSupport final : public JavaCallbacksSupport { + public: + explicit CallbacksSupport(RemoteVideoDecoder* aDecoder) + : mDecoder(aDecoder) {} + + void HandleInput(int64_t aTimestamp, bool aProcessed) override { + mDecoder->UpdateInputStatus(aTimestamp, aProcessed); + } + + void HandleOutput(java::Sample::Param aSample, + java::SampleBuffer::Param aBuffer) override { + MOZ_ASSERT(!aBuffer, "Video sample should be bufferless"); + // aSample will be implicitly converted into a GlobalRef. + mDecoder->ProcessOutput(aSample); + } + + void HandleOutputFormatChanged( + java::sdk::MediaFormat::Param aFormat) override { + int32_t colorFormat = 0; + aFormat->GetInteger(java::sdk::MediaFormat::KEY_COLOR_FORMAT, + &colorFormat); + if (colorFormat == 0) { + mDecoder->Error( + MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("Invalid color format:%d", colorFormat))); + return; + } + + Maybe<int32_t> colorRange; + { + int32_t range = 0; + if (NS_SUCCEEDED(aFormat->GetInteger( + java::sdk::MediaFormat::KEY_COLOR_RANGE, &range))) { + colorRange.emplace(range); + } + } + + Maybe<int32_t> colorSpace; + { + int32_t space = 0; + if (NS_SUCCEEDED(aFormat->GetInteger( + java::sdk::MediaFormat::KEY_COLOR_STANDARD, &space))) { + colorSpace.emplace(space); + } + } + + mDecoder->ProcessOutputFormatChange(colorFormat, colorRange, colorSpace); + } + + void HandleError(const MediaResult& aError) override { + mDecoder->Error(aError); + } + + friend class RemoteDataDecoder; + + private: + RemoteVideoDecoder* mDecoder; + }; + + RemoteVideoDecoder(const VideoInfo& aConfig, + java::sdk::MediaFormat::Param aFormat, + const nsString& aDrmStubId, Maybe<TrackingId> aTrackingId) + : RemoteDataDecoder(MediaData::Type::VIDEO_DATA, aConfig.mMimeType, + aFormat, aDrmStubId), + mConfig(aConfig), + mTrackingId(std::move(aTrackingId)) {} + + ~RemoteVideoDecoder() { + if (mSurface) { + java::SurfaceAllocator::DisposeSurface(mSurface); + } + } + + RefPtr<InitPromise> Init() override { + mThread = GetCurrentSerialEventTarget(); + java::sdk::MediaCodec::BufferInfo::LocalRef bufferInfo; + if (NS_FAILED(java::sdk::MediaCodec::BufferInfo::New(&bufferInfo)) || + !bufferInfo) { + return InitPromise::CreateAndReject(NS_ERROR_OUT_OF_MEMORY, __func__); + } + mInputBufferInfo = bufferInfo; + + mSurface = + java::GeckoSurface::LocalRef(java::SurfaceAllocator::AcquireSurface( + mConfig.mImage.width, mConfig.mImage.height, false)); + if (!mSurface) { + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, + __func__); + } + + mSurfaceHandle = mSurface->GetHandle(); + + // Register native methods. + JavaCallbacksSupport::Init(); + + mJavaCallbacks = java::CodecProxy::NativeCallbacks::New(); + if (!mJavaCallbacks) { + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, + __func__); + } + JavaCallbacksSupport::AttachNative( + mJavaCallbacks, mozilla::MakeUnique<CallbacksSupport>(this)); + + mJavaDecoder = java::CodecProxy::Create( + false, // false indicates to create a decoder and true denotes encoder + mFormat, mSurface, mJavaCallbacks, mDrmStubId); + if (mJavaDecoder == nullptr) { + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, + __func__); + } + mIsCodecSupportAdaptivePlayback = + mJavaDecoder->IsAdaptivePlaybackSupported(); + mIsHardwareAccelerated = mJavaDecoder->IsHardwareAccelerated(); + + // On some devices we have observed that the transform obtained from + // SurfaceTexture.getTransformMatrix() is incorrect for surfaces produced by + // a MediaCodec. We therefore override the transform to be a simple y-flip + // to ensure it is rendered correctly. + const auto hardware = java::sdk::Build::HARDWARE()->ToString(); + if (hardware.EqualsASCII("mt6735") || hardware.EqualsASCII("kirin980")) { + mTransformOverride = Some( + gfx::Matrix4x4::Scaling(1.0, -1.0, 1.0).PostTranslate(0.0, 1.0, 0.0)); + } + + mMediaInfoFlag = MediaInfoFlag::None; + mMediaInfoFlag |= mIsHardwareAccelerated ? MediaInfoFlag::HardwareDecoding + : MediaInfoFlag::SoftwareDecoding; + if (mMimeType.EqualsLiteral("video/mp4") || + mMimeType.EqualsLiteral("video/avc")) { + mMediaInfoFlag |= MediaInfoFlag::VIDEO_H264; + } else if (mMimeType.EqualsLiteral("video/vp8")) { + mMediaInfoFlag |= MediaInfoFlag::VIDEO_VP8; + } else if (mMimeType.EqualsLiteral("video/vp9")) { + mMediaInfoFlag |= MediaInfoFlag::VIDEO_VP9; + } else if (mMimeType.EqualsLiteral("video/av1")) { + mMediaInfoFlag |= MediaInfoFlag::VIDEO_AV1; + } + return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__); + } + + RefPtr<MediaDataDecoder::FlushPromise> Flush() override { + AssertOnThread(); + mInputInfos.Clear(); + mSeekTarget.reset(); + mLatestOutputTime.reset(); + mPerformanceRecorder.Record(std::numeric_limits<int64_t>::max()); + return RemoteDataDecoder::Flush(); + } + + nsCString GetCodecName() const override { + if (mMediaInfoFlag & MediaInfoFlag::VIDEO_H264) { + return "h264"_ns; + } else if (mMediaInfoFlag & MediaInfoFlag::VIDEO_VP8) { + return "vp8"_ns; + } else if (mMediaInfoFlag & MediaInfoFlag::VIDEO_VP9) { + return "vp9"_ns; + } else if (mMediaInfoFlag & MediaInfoFlag::VIDEO_AV1) { + return "av1"_ns; + } else { + return "unknown"_ns; + } + } + + RefPtr<MediaDataDecoder::DecodePromise> Decode( + MediaRawData* aSample) override { + AssertOnThread(); + + if (NeedsNewDecoder()) { + return DecodePromise::CreateAndReject(NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER, + __func__); + } + + const VideoInfo* config = + aSample->mTrackInfo ? aSample->mTrackInfo->GetAsVideoInfo() : &mConfig; + MOZ_ASSERT(config); + + mTrackingId.apply([&](const auto& aId) { + MediaInfoFlag flag = mMediaInfoFlag; + flag |= (aSample->mKeyframe ? MediaInfoFlag::KeyFrame + : MediaInfoFlag::NonKeyFrame); + mPerformanceRecorder.Start(aSample->mTime.ToMicroseconds(), + "AndroidDecoder"_ns, aId, flag); + }); + + InputInfo info(aSample->mDuration.ToMicroseconds(), config->mImage, + config->mDisplay); + mInputInfos.Insert(aSample->mTime.ToMicroseconds(), info); + return RemoteDataDecoder::Decode(aSample); + } + + bool SupportDecoderRecycling() const override { + return mIsCodecSupportAdaptivePlayback; + } + + void SetSeekThreshold(const TimeUnit& aTime) override { + auto setter = [self = RefPtr{this}, aTime] { + if (aTime.IsValid()) { + self->mSeekTarget = Some(aTime); + } else { + self->mSeekTarget.reset(); + } + }; + if (mThread->IsOnCurrentThread()) { + setter(); + } else { + nsCOMPtr<nsIRunnable> runnable = NS_NewRunnableFunction( + "RemoteVideoDecoder::SetSeekThreshold", std::move(setter)); + nsresult rv = mThread->Dispatch(runnable.forget()); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; + } + } + + bool IsUsefulData(const RefPtr<MediaData>& aSample) override { + AssertOnThread(); + + if (mLatestOutputTime && aSample->mTime < mLatestOutputTime.value()) { + return false; + } + + const TimeUnit endTime = aSample->GetEndTime(); + if (mSeekTarget && endTime <= mSeekTarget.value()) { + return false; + } + + mSeekTarget.reset(); + mLatestOutputTime = Some(endTime); + return true; + } + + bool IsHardwareAccelerated(nsACString& aFailureReason) const override { + return mIsHardwareAccelerated; + } + + ConversionRequired NeedsConversion() const override { + return ConversionRequired::kNeedAnnexB; + } + + private: + // Param and LocalRef are only valid for the duration of a JNI method call. + // Use GlobalRef as the parameter type to keep the Java object referenced + // until running. + void ProcessOutput(java::Sample::GlobalRef&& aSample) { + if (!mThread->IsOnCurrentThread()) { + nsresult rv = + mThread->Dispatch(NewRunnableMethod<java::Sample::GlobalRef&&>( + "RemoteVideoDecoder::ProcessOutput", this, + &RemoteVideoDecoder::ProcessOutput, std::move(aSample))); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; + return; + } + + AssertOnThread(); + if (GetState() == State::SHUTDOWN) { + aSample->Dispose(); + return; + } + + UniquePtr<layers::SurfaceTextureImage::SetCurrentCallback> releaseSample( + new CompositeListener(mJavaDecoder, aSample)); + + // If our output surface has been released (due to the GPU process crashing) + // then request a new decoder, which will in turn allocate a new + // Surface. This is usually be handled by the Error() callback, but on some + // devices (or at least on the emulator) the java decoder does not raise an + // error when the Surface is released. So we raise this error here as well. + if (NeedsNewDecoder()) { + Error(MediaResult(NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER, + RESULT_DETAIL("VideoCallBack::HandleOutput"))); + return; + } + + java::sdk::MediaCodec::BufferInfo::LocalRef info = aSample->Info(); + MOZ_ASSERT(info); + + int32_t flags; + bool ok = NS_SUCCEEDED(info->Flags(&flags)); + + int32_t offset; + ok &= NS_SUCCEEDED(info->Offset(&offset)); + + int32_t size; + ok &= NS_SUCCEEDED(info->Size(&size)); + + int64_t presentationTimeUs; + ok &= NS_SUCCEEDED(info->PresentationTimeUs(&presentationTimeUs)); + + if (!ok) { + Error(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("VideoCallBack::HandleOutput"))); + return; + } + + InputInfo inputInfo; + ok = mInputInfos.Find(presentationTimeUs, inputInfo); + bool isEOS = !!(flags & java::sdk::MediaCodec::BUFFER_FLAG_END_OF_STREAM); + if (!ok && !isEOS) { + // Ignore output with no corresponding input. + return; + } + + if (ok && (size > 0 || presentationTimeUs >= 0)) { + RefPtr<layers::Image> img = new layers::SurfaceTextureImage( + mSurfaceHandle, inputInfo.mImageSize, false /* NOT continuous */, + gl::OriginPos::BottomLeft, mConfig.HasAlpha(), mTransformOverride); + img->AsSurfaceTextureImage()->RegisterSetCurrentCallback( + std::move(releaseSample)); + + RefPtr<VideoData> v = VideoData::CreateFromImage( + inputInfo.mDisplaySize, offset, + TimeUnit::FromMicroseconds(presentationTimeUs), + TimeUnit::FromMicroseconds(inputInfo.mDurationUs), img.forget(), + !!(flags & java::sdk::MediaCodec::BUFFER_FLAG_SYNC_FRAME), + TimeUnit::FromMicroseconds(presentationTimeUs)); + + mPerformanceRecorder.Record(presentationTimeUs, [&](DecodeStage& aStage) { + using Cap = java::sdk::MediaCodecInfo::CodecCapabilities; + using Fmt = java::sdk::MediaFormat; + mColorFormat.apply([&](int32_t aFormat) { + switch (aFormat) { + case Cap::COLOR_Format32bitABGR8888: + case Cap::COLOR_Format32bitARGB8888: + case Cap::COLOR_Format32bitBGRA8888: + case Cap::COLOR_FormatRGBAFlexible: + aStage.SetImageFormat(DecodeStage::RGBA32); + break; + case Cap::COLOR_Format24bitBGR888: + case Cap::COLOR_Format24bitRGB888: + case Cap::COLOR_FormatRGBFlexible: + aStage.SetImageFormat(DecodeStage::RGB24); + break; + case Cap::COLOR_FormatYUV411Planar: + case Cap::COLOR_FormatYUV411PackedPlanar: + case Cap::COLOR_FormatYUV420Planar: + case Cap::COLOR_FormatYUV420PackedPlanar: + case Cap::COLOR_FormatYUV420Flexible: + aStage.SetImageFormat(DecodeStage::YUV420P); + break; + case Cap::COLOR_FormatYUV420SemiPlanar: + case Cap::COLOR_FormatYUV420PackedSemiPlanar: + case Cap::COLOR_QCOM_FormatYUV420SemiPlanar: + case Cap::COLOR_TI_FormatYUV420PackedSemiPlanar: + aStage.SetImageFormat(DecodeStage::NV12); + break; + case Cap::COLOR_FormatYCbYCr: + case Cap::COLOR_FormatYCrYCb: + case Cap::COLOR_FormatCbYCrY: + case Cap::COLOR_FormatCrYCbY: + case Cap::COLOR_FormatYUV422Planar: + case Cap::COLOR_FormatYUV422PackedPlanar: + case Cap::COLOR_FormatYUV422Flexible: + aStage.SetImageFormat(DecodeStage::YUV422P); + break; + case Cap::COLOR_FormatYUV444Interleaved: + case Cap::COLOR_FormatYUV444Flexible: + aStage.SetImageFormat(DecodeStage::YUV444P); + break; + case Cap::COLOR_FormatSurface: + aStage.SetImageFormat(DecodeStage::ANDROID_SURFACE); + break; + /* Added in API level 33 + case Cap::COLOR_FormatYUVP010: + aStage.SetImageFormat(DecodeStage::P010); + break; + */ + default: + NS_WARNING(nsPrintfCString("Unhandled color format %d (0x%08x)", + aFormat, aFormat) + .get()); + } + }); + mColorRange.apply([&](int32_t aRange) { + switch (aRange) { + case Fmt::COLOR_RANGE_FULL: + aStage.SetColorRange(gfx::ColorRange::FULL); + break; + case Fmt::COLOR_RANGE_LIMITED: + aStage.SetColorRange(gfx::ColorRange::LIMITED); + break; + default: + NS_WARNING(nsPrintfCString("Unhandled color range %d (0x%08x)", + aRange, aRange) + .get()); + } + }); + mColorSpace.apply([&](int32_t aSpace) { + switch (aSpace) { + case Fmt::COLOR_STANDARD_BT2020: + aStage.SetYUVColorSpace(gfx::YUVColorSpace::BT2020); + break; + case Fmt::COLOR_STANDARD_BT601_NTSC: + case Fmt::COLOR_STANDARD_BT601_PAL: + aStage.SetYUVColorSpace(gfx::YUVColorSpace::BT601); + break; + case Fmt::COLOR_STANDARD_BT709: + aStage.SetYUVColorSpace(gfx::YUVColorSpace::BT709); + break; + default: + NS_WARNING(nsPrintfCString("Unhandled color space %d (0x%08x)", + aSpace, aSpace) + .get()); + } + }); + aStage.SetResolution(v->mImage->GetSize().Width(), + v->mImage->GetSize().Height()); + }); + + RemoteDataDecoder::UpdateOutputStatus(std::move(v)); + } + + if (isEOS) { + DrainComplete(); + } + } + + void ProcessOutputFormatChange(int32_t aColorFormat, + Maybe<int32_t> aColorRange, + Maybe<int32_t> aColorSpace) { + if (!mThread->IsOnCurrentThread()) { + nsresult rv = mThread->Dispatch( + NewRunnableMethod<int32_t, Maybe<int32_t>, Maybe<int32_t>>( + "RemoteVideoDecoder::ProcessOutputFormatChange", this, + &RemoteVideoDecoder::ProcessOutputFormatChange, aColorFormat, + aColorRange, aColorSpace)); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; + return; + } + + AssertOnThread(); + + mColorFormat = Some(aColorFormat); + mColorRange = aColorRange; + mColorSpace = aColorSpace; + } + + bool NeedsNewDecoder() const override { + return !mSurface || mSurface->IsReleased(); + } + + const VideoInfo mConfig; + java::GeckoSurface::GlobalRef mSurface; + AndroidSurfaceTextureHandle mSurfaceHandle{}; + // Used to override the SurfaceTexture transform on some devices where the + // decoder provides a buggy value. + Maybe<gfx::Matrix4x4> mTransformOverride; + // Only accessed on reader's task queue. + bool mIsCodecSupportAdaptivePlayback = false; + // Can be accessed on any thread, but only written on during init. + bool mIsHardwareAccelerated = false; + // Accessed on mThread and reader's thread. SimpleMap however is + // thread-safe, so it's okay to do so. + SimpleMap<InputInfo> mInputInfos; + // Only accessed on mThread. + Maybe<TimeUnit> mSeekTarget; + Maybe<TimeUnit> mLatestOutputTime; + Maybe<int32_t> mColorFormat; + Maybe<int32_t> mColorRange; + Maybe<int32_t> mColorSpace; + // Only accessed on mThread. + // Tracking id for the performance recorder. + const Maybe<TrackingId> mTrackingId; + // Can be accessed on any thread, but only written during init. + // Pre-filled decode info used by the performance recorder. + MediaInfoFlag mMediaInfoFlag = {}; + // Only accessed on mThread. + // Records decode performance to the profiler. + PerformanceRecorderMulti<DecodeStage> mPerformanceRecorder; +}; + +class RemoteAudioDecoder final : public RemoteDataDecoder { + public: + RemoteAudioDecoder(const AudioInfo& aConfig, + java::sdk::MediaFormat::Param aFormat, + const nsString& aDrmStubId) + : RemoteDataDecoder(MediaData::Type::AUDIO_DATA, aConfig.mMimeType, + aFormat, aDrmStubId), + mOutputChannels(AssertedCast<int32_t>(aConfig.mChannels)), + mOutputSampleRate(AssertedCast<int32_t>(aConfig.mRate)) { + JNIEnv* const env = jni::GetEnvForThread(); + + bool formatHasCSD = false; + NS_ENSURE_SUCCESS_VOID(aFormat->ContainsKey(u"csd-0"_ns, &formatHasCSD)); + + // It would be nice to instead use more specific information here, but + // we force a byte buffer for now since this handles arbitrary codecs. + // TODO(bug 1768564): implement further type checking for codec data. + RefPtr<MediaByteBuffer> audioCodecSpecificBinaryBlob = + ForceGetAudioCodecSpecificBlob(aConfig.mCodecSpecificConfig); + if (!formatHasCSD && audioCodecSpecificBinaryBlob->Length() >= 2) { + jni::ByteBuffer::LocalRef buffer(env); + buffer = jni::ByteBuffer::New(audioCodecSpecificBinaryBlob->Elements(), + audioCodecSpecificBinaryBlob->Length()); + NS_ENSURE_SUCCESS_VOID(aFormat->SetByteBuffer(u"csd-0"_ns, buffer)); + } + } + + RefPtr<InitPromise> Init() override { + mThread = GetCurrentSerialEventTarget(); + java::sdk::MediaCodec::BufferInfo::LocalRef bufferInfo; + if (NS_FAILED(java::sdk::MediaCodec::BufferInfo::New(&bufferInfo)) || + !bufferInfo) { + return InitPromise::CreateAndReject(NS_ERROR_OUT_OF_MEMORY, __func__); + } + mInputBufferInfo = bufferInfo; + + // Register native methods. + JavaCallbacksSupport::Init(); + + mJavaCallbacks = java::CodecProxy::NativeCallbacks::New(); + if (!mJavaCallbacks) { + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, + __func__); + } + JavaCallbacksSupport::AttachNative( + mJavaCallbacks, mozilla::MakeUnique<CallbacksSupport>(this)); + + mJavaDecoder = java::CodecProxy::Create(false, mFormat, nullptr, + mJavaCallbacks, mDrmStubId); + if (mJavaDecoder == nullptr) { + return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR, + __func__); + } + + return InitPromise::CreateAndResolve(TrackInfo::kAudioTrack, __func__); + } + + nsCString GetCodecName() const override { + if (mMimeType.EqualsLiteral("audio/mp4a-latm")) { + return "aac"_ns; + } + return "unknown"_ns; + } + + RefPtr<FlushPromise> Flush() override { + AssertOnThread(); + mFirstDemuxedSampleTime.reset(); + return RemoteDataDecoder::Flush(); + } + + RefPtr<DecodePromise> Decode(MediaRawData* aSample) override { + AssertOnThread(); + if (!mFirstDemuxedSampleTime) { + MOZ_ASSERT(aSample->mTime.IsValid()); + mFirstDemuxedSampleTime.emplace(aSample->mTime); + } + return RemoteDataDecoder::Decode(aSample); + } + + private: + class CallbacksSupport final : public JavaCallbacksSupport { + public: + explicit CallbacksSupport(RemoteAudioDecoder* aDecoder) + : mDecoder(aDecoder) {} + + void HandleInput(int64_t aTimestamp, bool aProcessed) override { + mDecoder->UpdateInputStatus(aTimestamp, aProcessed); + } + + void HandleOutput(java::Sample::Param aSample, + java::SampleBuffer::Param aBuffer) override { + MOZ_ASSERT(aBuffer, "Audio sample should have buffer"); + // aSample will be implicitly converted into a GlobalRef. + mDecoder->ProcessOutput(aSample, aBuffer); + } + + void HandleOutputFormatChanged( + java::sdk::MediaFormat::Param aFormat) override { + int32_t outputChannels = 0; + aFormat->GetInteger(u"channel-count"_ns, &outputChannels); + AudioConfig::ChannelLayout layout(outputChannels); + if (!layout.IsValid()) { + mDecoder->Error(MediaResult( + NS_ERROR_DOM_MEDIA_FATAL_ERR, + RESULT_DETAIL("Invalid channel layout:%d", outputChannels))); + return; + } + + int32_t sampleRate = 0; + aFormat->GetInteger(u"sample-rate"_ns, &sampleRate); + LOG("Audio output format changed: channels:%d sample rate:%d", + outputChannels, sampleRate); + + mDecoder->ProcessOutputFormatChange(outputChannels, sampleRate); + } + + void HandleError(const MediaResult& aError) override { + mDecoder->Error(aError); + } + + private: + RemoteAudioDecoder* mDecoder; + }; + + bool IsSampleTimeSmallerThanFirstDemuxedSampleTime(int64_t aTime) const { + return mFirstDemuxedSampleTime->ToMicroseconds() > aTime; + } + + bool ShouldDiscardSample(int64_t aSession) const { + AssertOnThread(); + // HandleOutput() runs on Android binder thread pool and could be preempted + // by RemoteDateDecoder task queue. That means ProcessOutput() could be + // scheduled after Shutdown() or Flush(). We won't need the + // sample which is returned after calling Shutdown() and Flush(). We can + // check mFirstDemuxedSampleTime to know whether the Flush() has been + // called, becasue it would be reset in Flush(). + return GetState() == State::SHUTDOWN || !mFirstDemuxedSampleTime || + mSession != aSession; + } + + // Param and LocalRef are only valid for the duration of a JNI method call. + // Use GlobalRef as the parameter type to keep the Java object referenced + // until running. + void ProcessOutput(java::Sample::GlobalRef&& aSample, + java::SampleBuffer::GlobalRef&& aBuffer) { + if (!mThread->IsOnCurrentThread()) { + nsresult rv = + mThread->Dispatch(NewRunnableMethod<java::Sample::GlobalRef&&, + java::SampleBuffer::GlobalRef&&>( + "RemoteAudioDecoder::ProcessOutput", this, + &RemoteAudioDecoder::ProcessOutput, std::move(aSample), + std::move(aBuffer))); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; + return; + } + + AssertOnThread(); + + if (ShouldDiscardSample(aSample->Session()) || !aBuffer->IsValid()) { + aSample->Dispose(); + return; + } + + RenderOrReleaseOutput autoRelease(mJavaDecoder, aSample); + + java::sdk::MediaCodec::BufferInfo::LocalRef info = aSample->Info(); + MOZ_ASSERT(info); + + int32_t flags = 0; + bool ok = NS_SUCCEEDED(info->Flags(&flags)); + bool isEOS = !!(flags & java::sdk::MediaCodec::BUFFER_FLAG_END_OF_STREAM); + + int32_t offset; + ok &= NS_SUCCEEDED(info->Offset(&offset)); + + int64_t presentationTimeUs; + ok &= NS_SUCCEEDED(info->PresentationTimeUs(&presentationTimeUs)); + + int32_t size; + ok &= NS_SUCCEEDED(info->Size(&size)); + + if (!ok || + (IsSampleTimeSmallerThanFirstDemuxedSampleTime(presentationTimeUs) && + !isEOS)) { + Error(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__)); + return; + } + + if (size > 0) { +#ifdef MOZ_SAMPLE_TYPE_S16 + const int32_t numSamples = size / 2; +#else +# error We only support 16-bit integer PCM +#endif + + AlignedAudioBuffer audio(numSamples); + if (!audio) { + Error(MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__)); + return; + } + + jni::ByteBuffer::LocalRef dest = jni::ByteBuffer::New(audio.get(), size); + aBuffer->WriteToByteBuffer(dest, offset, size); + + RefPtr<AudioData> data = + new AudioData(0, TimeUnit::FromMicroseconds(presentationTimeUs), + std::move(audio), mOutputChannels, mOutputSampleRate); + + UpdateOutputStatus(std::move(data)); + } + + if (isEOS) { + DrainComplete(); + } + } + + void ProcessOutputFormatChange(int32_t aChannels, int32_t aSampleRate) { + if (!mThread->IsOnCurrentThread()) { + nsresult rv = mThread->Dispatch(NewRunnableMethod<int32_t, int32_t>( + "RemoteAudioDecoder::ProcessOutputFormatChange", this, + &RemoteAudioDecoder::ProcessOutputFormatChange, aChannels, + aSampleRate)); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; + return; + } + + AssertOnThread(); + + mOutputChannels = aChannels; + mOutputSampleRate = aSampleRate; + } + + int32_t mOutputChannels{}; + int32_t mOutputSampleRate{}; + Maybe<TimeUnit> mFirstDemuxedSampleTime; +}; + +already_AddRefed<MediaDataDecoder> RemoteDataDecoder::CreateAudioDecoder( + const CreateDecoderParams& aParams, const nsString& aDrmStubId, + CDMProxy* aProxy) { + const AudioInfo& config = aParams.AudioConfig(); + java::sdk::MediaFormat::LocalRef format; + NS_ENSURE_SUCCESS( + java::sdk::MediaFormat::CreateAudioFormat(config.mMimeType, config.mRate, + config.mChannels, &format), + nullptr); + + RefPtr<MediaDataDecoder> decoder = + new RemoteAudioDecoder(config, format, aDrmStubId); + if (aProxy) { + decoder = new EMEMediaDataDecoderProxy(aParams, decoder.forget(), aProxy); + } + return decoder.forget(); +} + +already_AddRefed<MediaDataDecoder> RemoteDataDecoder::CreateVideoDecoder( + const CreateDecoderParams& aParams, const nsString& aDrmStubId, + CDMProxy* aProxy) { + const VideoInfo& config = aParams.VideoConfig(); + java::sdk::MediaFormat::LocalRef format; + NS_ENSURE_SUCCESS(java::sdk::MediaFormat::CreateVideoFormat( + TranslateMimeType(config.mMimeType), + config.mImage.width, config.mImage.height, &format), + nullptr); + + RefPtr<MediaDataDecoder> decoder = + new RemoteVideoDecoder(config, format, aDrmStubId, aParams.mTrackingId); + if (aProxy) { + decoder = new EMEMediaDataDecoderProxy(aParams, decoder.forget(), aProxy); + } + return decoder.forget(); +} + +RemoteDataDecoder::RemoteDataDecoder(MediaData::Type aType, + const nsACString& aMimeType, + java::sdk::MediaFormat::Param aFormat, + const nsString& aDrmStubId) + : mType(aType), + mMimeType(aMimeType), + mFormat(aFormat), + mDrmStubId(aDrmStubId), + mSession(0), + mNumPendingInputs(0) {} + +RefPtr<MediaDataDecoder::FlushPromise> RemoteDataDecoder::Flush() { + AssertOnThread(); + MOZ_ASSERT(GetState() != State::SHUTDOWN); + + mDecodedData = DecodedData(); + UpdatePendingInputStatus(PendingOp::CLEAR); + mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + mDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__); + SetState(State::DRAINED); + mJavaDecoder->Flush(); + return FlushPromise::CreateAndResolve(true, __func__); +} + +RefPtr<MediaDataDecoder::DecodePromise> RemoteDataDecoder::Drain() { + AssertOnThread(); + if (GetState() == State::SHUTDOWN) { + return DecodePromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED, + __func__); + } + RefPtr<DecodePromise> p = mDrainPromise.Ensure(__func__); + if (GetState() == State::DRAINED) { + // There's no operation to perform other than returning any already + // decoded data. + ReturnDecodedData(); + return p; + } + + if (GetState() == State::DRAINING) { + // Draining operation already pending, let it complete its course. + return p; + } + + SetState(State::DRAINING); + mInputBufferInfo->Set(0, 0, -1, + java::sdk::MediaCodec::BUFFER_FLAG_END_OF_STREAM); + mSession = mJavaDecoder->Input(nullptr, mInputBufferInfo, nullptr); + return p; +} + +RefPtr<ShutdownPromise> RemoteDataDecoder::Shutdown() { + LOG(""); + AssertOnThread(); + SetState(State::SHUTDOWN); + if (mJavaDecoder) { + mJavaDecoder->Release(); + mJavaDecoder = nullptr; + } + + if (mJavaCallbacks) { + JavaCallbacksSupport::GetNative(mJavaCallbacks)->Cancel(); + JavaCallbacksSupport::DisposeNative(mJavaCallbacks); + mJavaCallbacks = nullptr; + } + + mFormat = nullptr; + + return ShutdownPromise::CreateAndResolve(true, __func__); +} + +using CryptoInfoResult = + Result<java::sdk::MediaCodec::CryptoInfo::LocalRef, nsresult>; + +static CryptoInfoResult GetCryptoInfoFromSample(const MediaRawData* aSample) { + const auto& cryptoObj = aSample->mCrypto; + java::sdk::MediaCodec::CryptoInfo::LocalRef cryptoInfo; + + if (!cryptoObj.IsEncrypted()) { + return CryptoInfoResult(cryptoInfo); + } + + static bool supportsCBCS = java::CodecProxy::SupportsCBCS(); + if (cryptoObj.mCryptoScheme == CryptoScheme::Cbcs && !supportsCBCS) { + return CryptoInfoResult(NS_ERROR_DOM_MEDIA_NOT_SUPPORTED_ERR); + } + + nsresult rv = java::sdk::MediaCodec::CryptoInfo::New(&cryptoInfo); + NS_ENSURE_SUCCESS(rv, CryptoInfoResult(rv)); + + uint32_t numSubSamples = std::min<uint32_t>( + cryptoObj.mPlainSizes.Length(), cryptoObj.mEncryptedSizes.Length()); + + uint32_t totalSubSamplesSize = 0; + for (const auto& size : cryptoObj.mPlainSizes) { + totalSubSamplesSize += size; + } + for (const auto& size : cryptoObj.mEncryptedSizes) { + totalSubSamplesSize += size; + } + + // Deep copy the plain sizes so we can modify them. + nsTArray<uint32_t> plainSizes = cryptoObj.mPlainSizes.Clone(); + uint32_t codecSpecificDataSize = aSample->Size() - totalSubSamplesSize; + // Size of codec specific data("CSD") for Android java::sdk::MediaCodec usage + // should be included in the 1st plain size if it exists. + if (codecSpecificDataSize > 0 && !plainSizes.IsEmpty()) { + // This shouldn't overflow as the the plain size should be UINT16_MAX at + // most, and the CSD should never be that large. Checked int acts like a + // diagnostic assert here to help catch if we ever have insane inputs. + CheckedUint32 newLeadingPlainSize{plainSizes[0]}; + newLeadingPlainSize += codecSpecificDataSize; + plainSizes[0] = newLeadingPlainSize.value(); + } + + static const int kExpectedIVLength = 16; + nsTArray<uint8_t> tempIV(kExpectedIVLength); + jint mode; + switch (cryptoObj.mCryptoScheme) { + case CryptoScheme::None: + mode = java::sdk::MediaCodec::CRYPTO_MODE_UNENCRYPTED; + MOZ_ASSERT(cryptoObj.mIV.Length() <= kExpectedIVLength); + tempIV.AppendElements(cryptoObj.mIV); + break; + case CryptoScheme::Cenc: + mode = java::sdk::MediaCodec::CRYPTO_MODE_AES_CTR; + MOZ_ASSERT(cryptoObj.mIV.Length() <= kExpectedIVLength); + tempIV.AppendElements(cryptoObj.mIV); + break; + case CryptoScheme::Cbcs: + mode = java::sdk::MediaCodec::CRYPTO_MODE_AES_CBC; + MOZ_ASSERT(cryptoObj.mConstantIV.Length() <= kExpectedIVLength); + tempIV.AppendElements(cryptoObj.mConstantIV); + break; + } + auto tempIVLength = tempIV.Length(); + for (size_t i = tempIVLength; i < kExpectedIVLength; i++) { + // Padding with 0 + tempIV.AppendElement(0); + } + + MOZ_ASSERT(numSubSamples <= INT32_MAX); + cryptoInfo->Set(static_cast<int32_t>(numSubSamples), + mozilla::jni::IntArray::From(plainSizes), + mozilla::jni::IntArray::From(cryptoObj.mEncryptedSizes), + mozilla::jni::ByteArray::From(cryptoObj.mKeyId), + mozilla::jni::ByteArray::From(tempIV), mode); + if (mode == java::sdk::MediaCodec::CRYPTO_MODE_AES_CBC) { + java::CodecProxy::SetCryptoPatternIfNeeded( + cryptoInfo, cryptoObj.mCryptByteBlock, cryptoObj.mSkipByteBlock); + } + + return CryptoInfoResult(cryptoInfo); +} + +RefPtr<MediaDataDecoder::DecodePromise> RemoteDataDecoder::Decode( + MediaRawData* aSample) { + AssertOnThread(); + MOZ_ASSERT(GetState() != State::SHUTDOWN); + MOZ_ASSERT(aSample != nullptr); + jni::ByteBuffer::LocalRef bytes = jni::ByteBuffer::New( + const_cast<uint8_t*>(aSample->Data()), aSample->Size()); + + SetState(State::DRAINABLE); + MOZ_ASSERT(aSample->Size() <= INT32_MAX); + mInputBufferInfo->Set(0, static_cast<int32_t>(aSample->Size()), + aSample->mTime.ToMicroseconds(), 0); + CryptoInfoResult crypto = GetCryptoInfoFromSample(aSample); + if (crypto.isErr()) { + return DecodePromise::CreateAndReject( + MediaResult(crypto.unwrapErr(), __func__), __func__); + } + int64_t session = + mJavaDecoder->Input(bytes, mInputBufferInfo, crypto.unwrap()); + if (session == java::CodecProxy::INVALID_SESSION) { + return DecodePromise::CreateAndReject( + MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__), __func__); + } + mSession = session; + return mDecodePromise.Ensure(__func__); +} + +void RemoteDataDecoder::UpdatePendingInputStatus(PendingOp aOp) { + AssertOnThread(); + switch (aOp) { + case PendingOp::INCREASE: + mNumPendingInputs++; + break; + case PendingOp::DECREASE: + mNumPendingInputs--; + break; + case PendingOp::CLEAR: + mNumPendingInputs = 0; + break; + } +} + +void RemoteDataDecoder::UpdateInputStatus(int64_t aTimestamp, bool aProcessed) { + if (!mThread->IsOnCurrentThread()) { + nsresult rv = mThread->Dispatch(NewRunnableMethod<int64_t, bool>( + "RemoteDataDecoder::UpdateInputStatus", this, + &RemoteDataDecoder::UpdateInputStatus, aTimestamp, aProcessed)); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; + return; + } + AssertOnThread(); + if (GetState() == State::SHUTDOWN) { + return; + } + + if (!aProcessed) { + UpdatePendingInputStatus(PendingOp::INCREASE); + } else if (HasPendingInputs()) { + UpdatePendingInputStatus(PendingOp::DECREASE); + } + + if (!HasPendingInputs() || // Input has been processed, request the next one. + !mDecodedData.IsEmpty()) { // Previous output arrived before Decode(). + ReturnDecodedData(); + } +} + +void RemoteDataDecoder::UpdateOutputStatus(RefPtr<MediaData>&& aSample) { + AssertOnThread(); + if (GetState() == State::SHUTDOWN) { + return; + } + if (IsUsefulData(aSample)) { + mDecodedData.AppendElement(std::move(aSample)); + } + ReturnDecodedData(); +} + +void RemoteDataDecoder::ReturnDecodedData() { + AssertOnThread(); + MOZ_ASSERT(GetState() != State::SHUTDOWN); + + // We only want to clear mDecodedData when we have resolved the promises. + if (!mDecodePromise.IsEmpty()) { + mDecodePromise.Resolve(std::move(mDecodedData), __func__); + mDecodedData = DecodedData(); + } else if (!mDrainPromise.IsEmpty() && + (!mDecodedData.IsEmpty() || GetState() == State::DRAINED)) { + mDrainPromise.Resolve(std::move(mDecodedData), __func__); + mDecodedData = DecodedData(); + } +} + +void RemoteDataDecoder::DrainComplete() { + if (!mThread->IsOnCurrentThread()) { + nsresult rv = mThread->Dispatch( + NewRunnableMethod("RemoteDataDecoder::DrainComplete", this, + &RemoteDataDecoder::DrainComplete)); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; + return; + } + AssertOnThread(); + if (GetState() == State::SHUTDOWN) { + return; + } + SetState(State::DRAINED); + ReturnDecodedData(); +} + +void RemoteDataDecoder::Error(const MediaResult& aError) { + if (!mThread->IsOnCurrentThread()) { + nsresult rv = mThread->Dispatch(NewRunnableMethod<MediaResult>( + "RemoteDataDecoder::Error", this, &RemoteDataDecoder::Error, aError)); + MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv)); + Unused << rv; + return; + } + AssertOnThread(); + if (GetState() == State::SHUTDOWN) { + return; + } + + // If we know we need a new decoder (eg because RemoteVideoDecoder's mSurface + // has been released due to a GPU process crash) then override the error to + // request a new decoder. + const MediaResult& error = + NeedsNewDecoder() + ? MediaResult(NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER, __func__) + : aError; + + mDecodePromise.RejectIfExists(error, __func__); + mDrainPromise.RejectIfExists(error, __func__); +} + +} // namespace mozilla +#undef LOG diff --git a/dom/media/platforms/android/RemoteDataDecoder.h b/dom/media/platforms/android/RemoteDataDecoder.h new file mode 100644 index 0000000000..5dfda4a7f5 --- /dev/null +++ b/dom/media/platforms/android/RemoteDataDecoder.h @@ -0,0 +1,112 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef RemoteDataDecoder_h_ +#define RemoteDataDecoder_h_ + +#include "AndroidDecoderModule.h" +#include "SurfaceTexture.h" +#include "TimeUnits.h" +#include "mozilla/Monitor.h" +#include "mozilla/java/CodecProxyWrappers.h" + +namespace mozilla { + +DDLoggedTypeDeclNameAndBase(RemoteDataDecoder, MediaDataDecoder); + +class RemoteDataDecoder : public MediaDataDecoder, + public DecoderDoctorLifeLogger<RemoteDataDecoder> { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(RemoteDataDecoder, final); + + static already_AddRefed<MediaDataDecoder> CreateAudioDecoder( + const CreateDecoderParams& aParams, const nsString& aDrmStubId, + CDMProxy* aProxy); + + static already_AddRefed<MediaDataDecoder> CreateVideoDecoder( + const CreateDecoderParams& aParams, const nsString& aDrmStubId, + CDMProxy* aProxy); + + RefPtr<DecodePromise> Decode(MediaRawData* aSample) override; + RefPtr<DecodePromise> Drain() override; + RefPtr<FlushPromise> Flush() override; + RefPtr<ShutdownPromise> Shutdown() override; + nsCString GetDescriptionName() const override { + return "android decoder (remote)"_ns; + } + + protected: + virtual ~RemoteDataDecoder() = default; + RemoteDataDecoder(MediaData::Type aType, const nsACString& aMimeType, + java::sdk::MediaFormat::Param aFormat, + const nsString& aDrmStubId); + + // Methods only called on mThread. + void UpdateInputStatus(int64_t aTimestamp, bool aProcessed); + void UpdateOutputStatus(RefPtr<MediaData>&& aSample); + void ReturnDecodedData(); + void DrainComplete(); + void Error(const MediaResult& aError); + void AssertOnThread() const { + // mThread may not be set if Init hasn't been called first. + MOZ_ASSERT(!mThread || mThread->IsOnCurrentThread()); + } + + enum class State { DRAINED, DRAINABLE, DRAINING, SHUTDOWN }; + void SetState(State aState) { + AssertOnThread(); + mState = aState; + } + State GetState() const { + AssertOnThread(); + return mState; + } + + // Whether the sample will be used. + virtual bool IsUsefulData(const RefPtr<MediaData>& aSample) { return true; } + + MediaData::Type mType; + + nsAutoCString mMimeType; + java::sdk::MediaFormat::GlobalRef mFormat; + + java::CodecProxy::GlobalRef mJavaDecoder; + java::CodecProxy::NativeCallbacks::GlobalRef mJavaCallbacks; + nsString mDrmStubId; + + nsCOMPtr<nsISerialEventTarget> mThread; + + // Preallocated Java object used as a reusable storage for input buffer + // information. Contents must be changed only on mThread. + java::sdk::MediaCodec::BufferInfo::GlobalRef mInputBufferInfo; + + // Session ID attached to samples. It is returned by CodecProxy::Input(). + // Accessed on mThread only. + int64_t mSession; + + private: + enum class PendingOp { INCREASE, DECREASE, CLEAR }; + void UpdatePendingInputStatus(PendingOp aOp); + size_t HasPendingInputs() { + AssertOnThread(); + return mNumPendingInputs > 0; + } + + // Returns true if we are in a state which requires a new decoder to be + // created. In this case all errors will be reported as + // NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER to avoid reporting errors as fatal when + // they can be fixed with a new decoder. + virtual bool NeedsNewDecoder() const { return false; } + + // The following members must only be accessed on mThread. + MozPromiseHolder<DecodePromise> mDecodePromise; + MozPromiseHolder<DecodePromise> mDrainPromise; + DecodedData mDecodedData; + State mState = State::DRAINED; + size_t mNumPendingInputs; +}; + +} // namespace mozilla + +#endif |