summaryrefslogtreecommitdiffstats
path: root/dom/media/platforms/android
diff options
context:
space:
mode:
Diffstat (limited to 'dom/media/platforms/android')
-rw-r--r--dom/media/platforms/android/AndroidDataEncoder.cpp470
-rw-r--r--dom/media/platforms/android/AndroidDataEncoder.h101
-rw-r--r--dom/media/platforms/android/AndroidDecoderModule.cpp168
-rw-r--r--dom/media/platforms/android/AndroidDecoderModule.h44
-rw-r--r--dom/media/platforms/android/AndroidEncoderModule.cpp32
-rw-r--r--dom/media/platforms/android/AndroidEncoderModule.h22
-rw-r--r--dom/media/platforms/android/JavaCallbacksSupport.h73
-rw-r--r--dom/media/platforms/android/RemoteDataDecoder.cpp849
-rw-r--r--dom/media/platforms/android/RemoteDataDecoder.h105
9 files changed, 1864 insertions, 0 deletions
diff --git a/dom/media/platforms/android/AndroidDataEncoder.cpp b/dom/media/platforms/android/AndroidDataEncoder.cpp
new file mode 100644
index 0000000000..bf9bae2cc5
--- /dev/null
+++ b/dom/media/platforms/android/AndroidDataEncoder.cpp
@@ -0,0 +1,470 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "AndroidDataEncoder.h"
+
+#include "AnnexB.h"
+#include "MediaData.h"
+#include "MediaInfo.h"
+#include "SimpleMap.h"
+
+#include "ImageContainer.h"
+#include "mozilla/Logging.h"
+#include "mozilla/ResultVariant.h"
+
+#include "nsMimeTypes.h"
+
+#include "libyuv.h"
+
+namespace mozilla {
+using media::TimeUnit;
+
+extern LazyLogModule sPEMLog;
+#define AND_ENC_LOG(arg, ...) \
+ MOZ_LOG(sPEMLog, mozilla::LogLevel::Debug, \
+ ("AndroidDataEncoder(%p)::%s: " arg, this, __func__, ##__VA_ARGS__))
+#define AND_ENC_LOGE(arg, ...) \
+ MOZ_LOG(sPEMLog, mozilla::LogLevel::Error, \
+ ("AndroidDataEncoder(%p)::%s: " arg, this, __func__, ##__VA_ARGS__))
+
+#define REJECT_IF_ERROR() \
+ do { \
+ if (mError) { \
+ auto error = mError.value(); \
+ mError.reset(); \
+ return EncodePromise::CreateAndReject(std::move(error), __func__); \
+ } \
+ } while (0)
+
+RefPtr<MediaDataEncoder::InitPromise> AndroidDataEncoder::Init() {
+ // Sanity-check the input size for Android software encoder fails to do it.
+ if (mConfig.mSize.width == 0 || mConfig.mSize.height == 0) {
+ return InitPromise::CreateAndReject(NS_ERROR_ILLEGAL_VALUE, __func__);
+ }
+
+ return InvokeAsync(mTaskQueue, this, __func__,
+ &AndroidDataEncoder::ProcessInit);
+}
+
+static const char* MimeTypeOf(MediaDataEncoder::CodecType aCodec) {
+ switch (aCodec) {
+ case MediaDataEncoder::CodecType::H264:
+ return "video/avc";
+ default:
+ return "";
+ }
+}
+
+using FormatResult = Result<java::sdk::MediaFormat::LocalRef, MediaResult>;
+
+FormatResult ToMediaFormat(const AndroidDataEncoder::Config& aConfig) {
+ if (!aConfig.mCodecSpecific) {
+ return FormatResult(
+ MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ "Android video encoder requires I-frame inverval"));
+ }
+
+ nsresult rv = NS_OK;
+ java::sdk::MediaFormat::LocalRef format;
+ rv = java::sdk::MediaFormat::CreateVideoFormat(MimeTypeOf(aConfig.mCodecType),
+ aConfig.mSize.width,
+ aConfig.mSize.height, &format);
+ NS_ENSURE_SUCCESS(
+ rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ "fail to create Java MediaFormat object")));
+
+ rv =
+ format->SetInteger(java::sdk::MediaFormat::KEY_BITRATE_MODE, 2 /* CBR */);
+ NS_ENSURE_SUCCESS(rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ "fail to set bitrate mode")));
+
+ rv = format->SetInteger(java::sdk::MediaFormat::KEY_BIT_RATE,
+ aConfig.mBitsPerSec);
+ NS_ENSURE_SUCCESS(rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ "fail to set bitrate")));
+
+ // COLOR_FormatYUV420SemiPlanar(NV12) is the most widely supported
+ // format.
+ rv = format->SetInteger(java::sdk::MediaFormat::KEY_COLOR_FORMAT, 0x15);
+ NS_ENSURE_SUCCESS(rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ "fail to set color format")));
+
+ rv = format->SetInteger(java::sdk::MediaFormat::KEY_FRAME_RATE,
+ aConfig.mFramerate);
+ NS_ENSURE_SUCCESS(rv, FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ "fail to set frame rate")));
+
+ // Ensure interval >= 1. A negative value means no key frames are
+ // requested after the first frame. A zero value means a stream
+ // containing all key frames is requested.
+ int32_t intervalInSec = std::max<size_t>(
+ 1, aConfig.mCodecSpecific.value().mKeyframeInterval / aConfig.mFramerate);
+ rv = format->SetInteger(java::sdk::MediaFormat::KEY_I_FRAME_INTERVAL,
+ intervalInSec);
+ NS_ENSURE_SUCCESS(rv,
+ FormatResult(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ "fail to set I-frame interval")));
+
+ return format;
+}
+
+RefPtr<MediaDataEncoder::InitPromise> AndroidDataEncoder::ProcessInit() {
+ AssertOnTaskQueue();
+ MOZ_ASSERT(!mJavaEncoder);
+
+ java::sdk::BufferInfo::LocalRef bufferInfo;
+ if (NS_FAILED(java::sdk::BufferInfo::New(&bufferInfo)) || !bufferInfo) {
+ return InitPromise::CreateAndReject(NS_ERROR_OUT_OF_MEMORY, __func__);
+ }
+ mInputBufferInfo = bufferInfo;
+
+ FormatResult result = ToMediaFormat(mConfig);
+ if (result.isErr()) {
+ return InitPromise::CreateAndReject(result.unwrapErr(), __func__);
+ }
+ mFormat = result.unwrap();
+
+ // Register native methods.
+ JavaCallbacksSupport::Init();
+
+ mJavaCallbacks = java::CodecProxy::NativeCallbacks::New();
+ if (!mJavaCallbacks) {
+ return InitPromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ "cannot create Java callback object"),
+ __func__);
+ }
+ JavaCallbacksSupport::AttachNative(
+ mJavaCallbacks, mozilla::MakeUnique<CallbacksSupport>(this));
+
+ mJavaEncoder = java::CodecProxy::Create(true /* encoder */, mFormat, nullptr,
+ mJavaCallbacks, u""_ns);
+ if (!mJavaEncoder) {
+ return InitPromise::CreateAndReject(
+ MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ "cannot create Java encoder object"),
+ __func__);
+ }
+
+ mIsHardwareAccelerated = mJavaEncoder->IsHardwareAccelerated();
+ mDrainState = DrainState::DRAINABLE;
+
+ return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__);
+}
+
+RefPtr<MediaDataEncoder::EncodePromise> AndroidDataEncoder::Encode(
+ const MediaData* aSample) {
+ RefPtr<AndroidDataEncoder> self = this;
+ MOZ_ASSERT(aSample != nullptr);
+
+ RefPtr<const MediaData> sample(aSample);
+ return InvokeAsync(mTaskQueue, __func__, [self, sample]() {
+ return self->ProcessEncode(std::move(sample));
+ });
+}
+
+static jni::ByteBuffer::LocalRef ConvertI420ToNV12Buffer(
+ RefPtr<const VideoData> aSample, RefPtr<MediaByteBuffer>& aYUVBuffer) {
+ const PlanarYCbCrImage* image = aSample->mImage->AsPlanarYCbCrImage();
+ MOZ_ASSERT(image);
+ const PlanarYCbCrData* yuv = image->GetData();
+ size_t ySize = yuv->mYStride * yuv->mYSize.height;
+ size_t size = ySize + (yuv->mCbCrStride * yuv->mCbCrSize.height * 2);
+ if (!aYUVBuffer || aYUVBuffer->Capacity() < size) {
+ aYUVBuffer = MakeRefPtr<MediaByteBuffer>(size);
+ aYUVBuffer->SetLength(size);
+ } else {
+ MOZ_ASSERT(aYUVBuffer->Length() >= size);
+ }
+
+ if (libyuv::I420ToNV12(yuv->mYChannel, yuv->mYStride, yuv->mCbChannel,
+ yuv->mCbCrStride, yuv->mCrChannel, yuv->mCbCrStride,
+ aYUVBuffer->Elements(), yuv->mYStride,
+ aYUVBuffer->Elements() + ySize, yuv->mCbCrStride * 2,
+ yuv->mYSize.width, yuv->mYSize.height) != 0) {
+ return nullptr;
+ }
+
+ return jni::ByteBuffer::New(aYUVBuffer->Elements(), aYUVBuffer->Length());
+}
+
+RefPtr<MediaDataEncoder::EncodePromise> AndroidDataEncoder::ProcessEncode(
+ RefPtr<const MediaData> aSample) {
+ AssertOnTaskQueue();
+
+ REJECT_IF_ERROR();
+
+ RefPtr<const VideoData> sample(aSample->As<const VideoData>());
+ MOZ_ASSERT(sample);
+
+ jni::ByteBuffer::LocalRef buffer =
+ ConvertI420ToNV12Buffer(sample, mYUVBuffer);
+ if (!buffer) {
+ return EncodePromise::CreateAndReject(NS_ERROR_ILLEGAL_INPUT, __func__);
+ }
+
+ if (aSample->mKeyframe) {
+ mInputBufferInfo->Set(0, mYUVBuffer->Length(),
+ aSample->mTime.ToMicroseconds(),
+ java::sdk::MediaCodec::BUFFER_FLAG_SYNC_FRAME);
+ } else {
+ mInputBufferInfo->Set(0, mYUVBuffer->Length(),
+ aSample->mTime.ToMicroseconds(), 0);
+ }
+
+ mJavaEncoder->Input(buffer, mInputBufferInfo, nullptr);
+
+ if (mEncodedData.Length() > 0) {
+ EncodedData pending = std::move(mEncodedData);
+ return EncodePromise::CreateAndResolve(std::move(pending), __func__);
+ } else {
+ return EncodePromise::CreateAndResolve(EncodedData(), __func__);
+ }
+}
+
+class AutoRelease final {
+ public:
+ AutoRelease(java::CodecProxy::Param aEncoder, java::Sample::Param aSample)
+ : mEncoder(aEncoder), mSample(aSample) {}
+
+ ~AutoRelease() { mEncoder->ReleaseOutput(mSample, false); }
+
+ private:
+ java::CodecProxy::GlobalRef mEncoder;
+ java::Sample::GlobalRef mSample;
+};
+
+static RefPtr<MediaByteBuffer> ExtractCodecConfig(
+ java::SampleBuffer::Param aBuffer, const int32_t aOffset,
+ const int32_t aSize, const bool aAsAnnexB) {
+ auto annexB = MakeRefPtr<MediaByteBuffer>(aSize);
+ annexB->SetLength(aSize);
+ jni::ByteBuffer::LocalRef dest =
+ jni::ByteBuffer::New(annexB->Elements(), aSize);
+ aBuffer->WriteToByteBuffer(dest, aOffset, aSize);
+ if (aAsAnnexB) {
+ return annexB;
+ }
+ // Convert to avcC.
+ nsTArray<AnnexB::NALEntry> paramSets;
+ AnnexB::ParseNALEntries(
+ Span<const uint8_t>(annexB->Elements(), annexB->Length()), paramSets);
+
+ auto avcc = MakeRefPtr<MediaByteBuffer>();
+ AnnexB::NALEntry& sps = paramSets.ElementAt(0);
+ AnnexB::NALEntry& pps = paramSets.ElementAt(1);
+ const uint8_t* spsPtr = annexB->Elements() + sps.mOffset;
+ H264::WriteExtraData(
+ avcc, spsPtr[1], spsPtr[2], spsPtr[3],
+ Span<const uint8_t>(spsPtr, sps.mSize),
+ Span<const uint8_t>(annexB->Elements() + pps.mOffset, pps.mSize));
+ return avcc;
+}
+
+void AndroidDataEncoder::ProcessOutput(
+ java::Sample::GlobalRef&& aSample,
+ java::SampleBuffer::GlobalRef&& aBuffer) {
+ if (!mTaskQueue->IsCurrentThreadIn()) {
+ nsresult rv =
+ mTaskQueue->Dispatch(NewRunnableMethod<java::Sample::GlobalRef&&,
+ java::SampleBuffer::GlobalRef&&>(
+ "AndroidDataEncoder::ProcessOutput", this,
+ &AndroidDataEncoder::ProcessOutput, std::move(aSample),
+ std::move(aBuffer)));
+ MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+ Unused << rv;
+ return;
+ }
+ AssertOnTaskQueue();
+
+ if (!mJavaEncoder) {
+ return;
+ }
+
+ AutoRelease releaseSample(mJavaEncoder, aSample);
+
+ java::sdk::BufferInfo::LocalRef info = aSample->Info();
+ MOZ_ASSERT(info);
+
+ int32_t flags;
+ bool ok = NS_SUCCEEDED(info->Flags(&flags));
+ bool isEOS = !!(flags & java::sdk::MediaCodec::BUFFER_FLAG_END_OF_STREAM);
+
+ int32_t offset;
+ ok &= NS_SUCCEEDED(info->Offset(&offset));
+
+ int32_t size;
+ ok &= NS_SUCCEEDED(info->Size(&size));
+
+ int64_t presentationTimeUs;
+ ok &= NS_SUCCEEDED(info->PresentationTimeUs(&presentationTimeUs));
+
+ if (!ok) {
+ return;
+ }
+
+ if (size > 0) {
+ if ((flags & java::sdk::MediaCodec::BUFFER_FLAG_CODEC_CONFIG) != 0) {
+ mConfigData = ExtractCodecConfig(aBuffer, offset, size,
+ mConfig.mUsage == Usage::Realtime);
+ return;
+ }
+ RefPtr<MediaRawData> output =
+ GetOutputData(aBuffer, offset, size,
+ !!(flags & java::sdk::MediaCodec::BUFFER_FLAG_KEY_FRAME));
+ output->mEOS = isEOS;
+ output->mTime = media::TimeUnit::FromMicroseconds(presentationTimeUs);
+ mEncodedData.AppendElement(std::move(output));
+ }
+
+ if (isEOS) {
+ mDrainState = DrainState::DRAINED;
+ }
+ if (!mDrainPromise.IsEmpty()) {
+ EncodedData pending = std::move(mEncodedData);
+ mDrainPromise.Resolve(std::move(pending), __func__);
+ }
+}
+
+RefPtr<MediaRawData> AndroidDataEncoder::GetOutputData(
+ java::SampleBuffer::Param aBuffer, const int32_t aOffset,
+ const int32_t aSize, const bool aIsKeyFrame) {
+ auto output = MakeRefPtr<MediaRawData>();
+
+ size_t prependSize = 0;
+ RefPtr<MediaByteBuffer> avccHeader;
+ if (aIsKeyFrame && mConfigData) {
+ if (mConfig.mUsage == Usage::Realtime) {
+ prependSize = mConfigData->Length();
+ } else {
+ avccHeader = mConfigData;
+ }
+ }
+
+ UniquePtr<MediaRawDataWriter> writer(output->CreateWriter());
+ if (!writer->SetSize(prependSize + aSize)) {
+ AND_ENC_LOGE("fail to allocate output buffer");
+ return nullptr;
+ }
+
+ if (prependSize > 0) {
+ PodCopy(writer->Data(), mConfigData->Elements(), prependSize);
+ }
+
+ jni::ByteBuffer::LocalRef buf =
+ jni::ByteBuffer::New(writer->Data() + prependSize, aSize);
+ aBuffer->WriteToByteBuffer(buf, aOffset, aSize);
+
+ if (mConfig.mUsage != Usage::Realtime &&
+ !AnnexB::ConvertSampleToAVCC(output, avccHeader)) {
+ AND_ENC_LOGE("fail to convert annex-b sample to AVCC");
+ return nullptr;
+ }
+
+ output->mKeyframe = aIsKeyFrame;
+
+ return output;
+}
+
+RefPtr<MediaDataEncoder::EncodePromise> AndroidDataEncoder::Drain() {
+ return InvokeAsync(mTaskQueue, this, __func__,
+ &AndroidDataEncoder::ProcessDrain);
+}
+
+RefPtr<MediaDataEncoder::EncodePromise> AndroidDataEncoder::ProcessDrain() {
+ AssertOnTaskQueue();
+ MOZ_ASSERT(mJavaEncoder);
+ MOZ_ASSERT(mDrainPromise.IsEmpty());
+
+ REJECT_IF_ERROR();
+
+ switch (mDrainState) {
+ case DrainState::DRAINABLE:
+ mInputBufferInfo->Set(0, 0, -1,
+ java::sdk::MediaCodec::BUFFER_FLAG_END_OF_STREAM);
+ mJavaEncoder->Input(nullptr, mInputBufferInfo, nullptr);
+ mDrainState = DrainState::DRAINING;
+ [[fallthrough]];
+ case DrainState::DRAINING:
+ if (mEncodedData.IsEmpty()) {
+ return mDrainPromise.Ensure(__func__); // Pending promise.
+ }
+ [[fallthrough]];
+ case DrainState::DRAINED:
+ if (mEncodedData.Length() > 0) {
+ EncodedData pending = std::move(mEncodedData);
+ return EncodePromise::CreateAndResolve(std::move(pending), __func__);
+ } else {
+ return EncodePromise::CreateAndResolve(EncodedData(), __func__);
+ }
+ }
+}
+
+RefPtr<ShutdownPromise> AndroidDataEncoder::Shutdown() {
+ return InvokeAsync(mTaskQueue, this, __func__,
+ &AndroidDataEncoder::ProcessShutdown);
+}
+
+RefPtr<ShutdownPromise> AndroidDataEncoder::ProcessShutdown() {
+ AssertOnTaskQueue();
+ if (mJavaEncoder) {
+ mJavaEncoder->Release();
+ mJavaEncoder = nullptr;
+ }
+
+ if (mJavaCallbacks) {
+ JavaCallbacksSupport::GetNative(mJavaCallbacks)->Cancel();
+ JavaCallbacksSupport::DisposeNative(mJavaCallbacks);
+ mJavaCallbacks = nullptr;
+ }
+
+ mFormat = nullptr;
+
+ return ShutdownPromise::CreateAndResolve(true, __func__);
+}
+
+RefPtr<GenericPromise> AndroidDataEncoder::SetBitrate(
+ const MediaDataEncoder::Rate aBitsPerSec) {
+ RefPtr<AndroidDataEncoder> self(this);
+ return InvokeAsync(mTaskQueue, __func__, [self, aBitsPerSec]() {
+ self->mJavaEncoder->SetBitrate(aBitsPerSec);
+ return GenericPromise::CreateAndResolve(true, __func__);
+ });
+
+ return nullptr;
+}
+
+void AndroidDataEncoder::Error(const MediaResult& aError) {
+ if (!mTaskQueue->IsCurrentThreadIn()) {
+ nsresult rv = mTaskQueue->Dispatch(NewRunnableMethod<MediaResult>(
+ "AndroidDataEncoder::Error", this, &AndroidDataEncoder::Error, aError));
+ MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+ Unused << rv;
+ return;
+ }
+ AssertOnTaskQueue();
+
+ mError = Some(aError);
+}
+
+void AndroidDataEncoder::CallbacksSupport::HandleInput(int64_t aTimestamp,
+ bool aProcessed) {}
+
+void AndroidDataEncoder::CallbacksSupport::HandleOutput(
+ java::Sample::Param aSample, java::SampleBuffer::Param aBuffer) {
+ mEncoder->ProcessOutput(std::move(aSample), std::move(aBuffer));
+}
+
+void AndroidDataEncoder::CallbacksSupport::HandleOutputFormatChanged(
+ java::sdk::MediaFormat::Param aFormat) {}
+
+void AndroidDataEncoder::CallbacksSupport::HandleError(
+ const MediaResult& aError) {
+ mEncoder->Error(aError);
+}
+
+} // namespace mozilla
+
+#undef AND_ENC_LOG
+#undef AND_ENC_LOGE
diff --git a/dom/media/platforms/android/AndroidDataEncoder.h b/dom/media/platforms/android/AndroidDataEncoder.h
new file mode 100644
index 0000000000..ad4e5a8ee9
--- /dev/null
+++ b/dom/media/platforms/android/AndroidDataEncoder.h
@@ -0,0 +1,101 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef DOM_MEDIA_PLATFORMS_ANDROID_ANDROIDDATAENCODER_H_
+#define DOM_MEDIA_PLATFORMS_ANDROID_ANDROIDDATAENCODER_H_
+
+#include "MediaData.h"
+#include "PlatformEncoderModule.h"
+#include "TimeUnits.h"
+
+#include "JavaCallbacksSupport.h"
+
+#include "mozilla/Maybe.h"
+#include "mozilla/Monitor.h"
+
+namespace mozilla {
+
+class AndroidDataEncoder final : public MediaDataEncoder {
+ public:
+ using Config = H264Config;
+
+ AndroidDataEncoder(const Config& aConfig, RefPtr<TaskQueue> aTaskQueue)
+ : mConfig(aConfig), mTaskQueue(aTaskQueue) {
+ MOZ_ASSERT(mConfig.mSize.width > 0 && mConfig.mSize.height > 0);
+ MOZ_ASSERT(mTaskQueue);
+ }
+ RefPtr<InitPromise> Init() override;
+ RefPtr<EncodePromise> Encode(const MediaData* aSample) override;
+ RefPtr<EncodePromise> Drain() override;
+ RefPtr<ShutdownPromise> Shutdown() override;
+ RefPtr<GenericPromise> SetBitrate(const Rate aBitsPerSec) override;
+
+ nsCString GetDescriptionName() const override { return "Android Encoder"_ns; }
+
+ private:
+ class CallbacksSupport final : public JavaCallbacksSupport {
+ public:
+ explicit CallbacksSupport(AndroidDataEncoder* aEncoder)
+ : mEncoder(aEncoder) {}
+
+ void HandleInput(int64_t aTimestamp, bool aProcessed) override;
+ void HandleOutput(java::Sample::Param aSample,
+ java::SampleBuffer::Param aBuffer) override;
+ void HandleOutputFormatChanged(
+ java::sdk::MediaFormat::Param aFormat) override;
+ void HandleError(const MediaResult& aError) override;
+
+ private:
+ AndroidDataEncoder* mEncoder;
+ };
+ friend class CallbacksSupport;
+
+ // Methods only called on mTaskQueue.
+ RefPtr<InitPromise> ProcessInit();
+ RefPtr<EncodePromise> ProcessEncode(RefPtr<const MediaData> aSample);
+ RefPtr<EncodePromise> ProcessDrain();
+ RefPtr<ShutdownPromise> ProcessShutdown();
+ void ProcessInput();
+ void ProcessOutput(java::Sample::GlobalRef&& aSample,
+ java::SampleBuffer::GlobalRef&& aBuffer);
+ RefPtr<MediaRawData> GetOutputData(java::SampleBuffer::Param aBuffer,
+ const int32_t aOffset, const int32_t aSize,
+ const bool aIsKeyFrame);
+ void Error(const MediaResult& aError);
+
+ void AssertOnTaskQueue() const {
+ MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
+ }
+
+ Config mConfig;
+
+ RefPtr<TaskQueue> mTaskQueue;
+
+ // Can be accessed on any thread, but only written on during init.
+ bool mIsHardwareAccelerated = false;
+
+ java::CodecProxy::GlobalRef mJavaEncoder;
+ java::CodecProxy::NativeCallbacks::GlobalRef mJavaCallbacks;
+ java::sdk::MediaFormat::GlobalRef mFormat;
+ // Preallocated Java object used as a reusable storage for input buffer
+ // information. Contents must be changed only on mTaskQueue.
+ java::sdk::BufferInfo::GlobalRef mInputBufferInfo;
+
+ MozPromiseHolder<EncodePromise> mDrainPromise;
+
+ // Accessed on mTaskqueue only.
+ RefPtr<MediaByteBuffer> mYUVBuffer;
+ EncodedData mEncodedData;
+ // SPS/PPS NALUs for realtime usage, avcC otherwise.
+ RefPtr<MediaByteBuffer> mConfigData;
+
+ enum class DrainState { DRAINED, DRAINABLE, DRAINING };
+ DrainState mDrainState;
+
+ Maybe<MediaResult> mError;
+};
+
+} // namespace mozilla
+
+#endif
diff --git a/dom/media/platforms/android/AndroidDecoderModule.cpp b/dom/media/platforms/android/AndroidDecoderModule.cpp
new file mode 100644
index 0000000000..804f0455fb
--- /dev/null
+++ b/dom/media/platforms/android/AndroidDecoderModule.cpp
@@ -0,0 +1,168 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include <jni.h>
+
+#include "MediaInfo.h"
+#include "OpusDecoder.h"
+#include "RemoteDataDecoder.h"
+#include "TheoraDecoder.h"
+#include "VPXDecoder.h"
+#include "VorbisDecoder.h"
+#include "mozilla/StaticPrefs_media.h"
+#include "mozilla/java/HardwareCodecCapabilityUtilsWrappers.h"
+#include "nsIGfxInfo.h"
+#include "nsPromiseFlatString.h"
+#include "prlog.h"
+
+#undef LOG
+#define LOG(arg, ...) \
+ MOZ_LOG( \
+ sAndroidDecoderModuleLog, mozilla::LogLevel::Debug, \
+ ("AndroidDecoderModule(%p)::%s: " arg, this, __func__, ##__VA_ARGS__))
+#define SLOG(arg, ...) \
+ MOZ_LOG(sAndroidDecoderModuleLog, mozilla::LogLevel::Debug, \
+ ("%s: " arg, __func__, ##__VA_ARGS__))
+
+using namespace mozilla;
+using media::TimeUnit;
+
+namespace mozilla {
+
+mozilla::LazyLogModule sAndroidDecoderModuleLog("AndroidDecoderModule");
+
+const nsCString TranslateMimeType(const nsACString& aMimeType) {
+ if (VPXDecoder::IsVPX(aMimeType, VPXDecoder::VP8)) {
+ static constexpr auto vp8 = "video/x-vnd.on2.vp8"_ns;
+ return vp8;
+ } else if (VPXDecoder::IsVPX(aMimeType, VPXDecoder::VP9)) {
+ static constexpr auto vp9 = "video/x-vnd.on2.vp9"_ns;
+ return vp9;
+ }
+ return nsCString(aMimeType);
+}
+
+static bool GetFeatureStatus(int32_t aFeature) {
+ nsCOMPtr<nsIGfxInfo> gfxInfo = services::GetGfxInfo();
+ int32_t status = nsIGfxInfo::FEATURE_STATUS_UNKNOWN;
+ nsCString discardFailureId;
+ if (!gfxInfo || NS_FAILED(gfxInfo->GetFeatureStatus(
+ aFeature, discardFailureId, &status))) {
+ return false;
+ }
+ return status == nsIGfxInfo::FEATURE_STATUS_OK;
+};
+
+AndroidDecoderModule::AndroidDecoderModule(CDMProxy* aProxy) {
+ mProxy = static_cast<MediaDrmCDMProxy*>(aProxy);
+}
+
+bool AndroidDecoderModule::SupportsMimeType(const nsACString& aMimeType) {
+ if (jni::GetAPIVersion() < 16) {
+ return false;
+ }
+
+ if (aMimeType.EqualsLiteral("video/mp4") ||
+ aMimeType.EqualsLiteral("video/avc")) {
+ return true;
+ }
+
+ // When checking "audio/x-wav", CreateDecoder can cause a JNI ERROR by
+ // Accessing a stale local reference leading to a SIGSEGV crash.
+ // To avoid this we check for wav types here.
+ if (aMimeType.EqualsLiteral("audio/x-wav") ||
+ aMimeType.EqualsLiteral("audio/wave; codecs=1") ||
+ aMimeType.EqualsLiteral("audio/wave; codecs=3") ||
+ aMimeType.EqualsLiteral("audio/wave; codecs=6") ||
+ aMimeType.EqualsLiteral("audio/wave; codecs=7") ||
+ aMimeType.EqualsLiteral("audio/wave; codecs=65534")) {
+ return false;
+ }
+
+ if ((VPXDecoder::IsVPX(aMimeType, VPXDecoder::VP8) &&
+ !GetFeatureStatus(nsIGfxInfo::FEATURE_VP8_HW_DECODE)) ||
+ (VPXDecoder::IsVPX(aMimeType, VPXDecoder::VP9) &&
+ !GetFeatureStatus(nsIGfxInfo::FEATURE_VP9_HW_DECODE))) {
+ return false;
+ }
+
+ // Prefer the gecko decoder for opus and vorbis; stagefright crashes
+ // on content demuxed from mp4.
+ // Not all android devices support FLAC even when they say they do.
+ if (OpusDataDecoder::IsOpus(aMimeType) ||
+ VorbisDataDecoder::IsVorbis(aMimeType) ||
+ aMimeType.EqualsLiteral("audio/flac")) {
+ SLOG("Rejecting audio of type %s", aMimeType.Data());
+ return false;
+ }
+
+ // Prefer the gecko decoder for Theora.
+ // Not all android devices support Theora even when they say they do.
+ if (TheoraDecoder::IsTheora(aMimeType)) {
+ SLOG("Rejecting video of type %s", aMimeType.Data());
+ return false;
+ }
+
+ if (aMimeType.EqualsLiteral("audio/mpeg") &&
+ StaticPrefs::media_ffvpx_mp3_enabled()) {
+ // Prefer the ffvpx mp3 software decoder if available.
+ return false;
+ }
+
+ return java::HardwareCodecCapabilityUtils::FindDecoderCodecInfoForMimeType(
+ TranslateMimeType(aMimeType));
+}
+
+bool AndroidDecoderModule::SupportsMimeType(
+ const nsACString& aMimeType, DecoderDoctorDiagnostics* aDiagnostics) const {
+ return AndroidDecoderModule::SupportsMimeType(aMimeType);
+}
+
+already_AddRefed<MediaDataDecoder> AndroidDecoderModule::CreateVideoDecoder(
+ const CreateDecoderParams& aParams) {
+ // Temporary - forces use of VPXDecoder when alpha is present.
+ // Bug 1263836 will handle alpha scenario once implemented. It will shift
+ // the check for alpha to PDMFactory but not itself remove the need for a
+ // check.
+ if (aParams.VideoConfig().HasAlpha()) {
+ return nullptr;
+ }
+
+ nsString drmStubId;
+ if (mProxy) {
+ drmStubId = mProxy->GetMediaDrmStubId();
+ }
+
+ RefPtr<MediaDataDecoder> decoder =
+ RemoteDataDecoder::CreateVideoDecoder(aParams, drmStubId, mProxy);
+ return decoder.forget();
+}
+
+already_AddRefed<MediaDataDecoder> AndroidDecoderModule::CreateAudioDecoder(
+ const CreateDecoderParams& aParams) {
+ const AudioInfo& config = aParams.AudioConfig();
+ if (config.mBitDepth != 16) {
+ // We only handle 16-bit audio.
+ return nullptr;
+ }
+
+ LOG("CreateAudioFormat with mimeType=%s, mRate=%d, channels=%d",
+ config.mMimeType.Data(), config.mRate, config.mChannels);
+
+ nsString drmStubId;
+ if (mProxy) {
+ drmStubId = mProxy->GetMediaDrmStubId();
+ }
+ RefPtr<MediaDataDecoder> decoder =
+ RemoteDataDecoder::CreateAudioDecoder(aParams, drmStubId, mProxy);
+ return decoder.forget();
+}
+
+/* static */
+already_AddRefed<PlatformDecoderModule> AndroidDecoderModule::Create(
+ CDMProxy* aProxy) {
+ return MakeAndAddRef<AndroidDecoderModule>(aProxy);
+}
+
+} // namespace mozilla
diff --git a/dom/media/platforms/android/AndroidDecoderModule.h b/dom/media/platforms/android/AndroidDecoderModule.h
new file mode 100644
index 0000000000..b3a16a0743
--- /dev/null
+++ b/dom/media/platforms/android/AndroidDecoderModule.h
@@ -0,0 +1,44 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef AndroidDecoderModule_h_
+#define AndroidDecoderModule_h_
+
+#include "PlatformDecoderModule.h"
+#include "mozilla/MediaDrmCDMProxy.h"
+
+namespace mozilla {
+
+class AndroidDecoderModule : public PlatformDecoderModule {
+ template <typename T, typename... Args>
+ friend already_AddRefed<T> MakeAndAddRef(Args&&...);
+
+ public:
+ static already_AddRefed<PlatformDecoderModule> Create(
+ CDMProxy* aProxy = nullptr);
+
+ already_AddRefed<MediaDataDecoder> CreateVideoDecoder(
+ const CreateDecoderParams& aParams) override;
+
+ already_AddRefed<MediaDataDecoder> CreateAudioDecoder(
+ const CreateDecoderParams& aParams) override;
+
+ bool SupportsMimeType(const nsACString& aMimeType,
+ DecoderDoctorDiagnostics* aDiagnostics) const override;
+
+ static bool SupportsMimeType(const nsACString& aMimeType);
+
+ private:
+ explicit AndroidDecoderModule(CDMProxy* aProxy = nullptr);
+ virtual ~AndroidDecoderModule() = default;
+ RefPtr<MediaDrmCDMProxy> mProxy;
+};
+
+extern LazyLogModule sAndroidDecoderModuleLog;
+
+const nsCString TranslateMimeType(const nsACString& aMimeType);
+
+} // namespace mozilla
+
+#endif
diff --git a/dom/media/platforms/android/AndroidEncoderModule.cpp b/dom/media/platforms/android/AndroidEncoderModule.cpp
new file mode 100644
index 0000000000..0ca3efbc14
--- /dev/null
+++ b/dom/media/platforms/android/AndroidEncoderModule.cpp
@@ -0,0 +1,32 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "AndroidEncoderModule.h"
+
+#include "AndroidDataEncoder.h"
+#include "MP4Decoder.h"
+
+#include "mozilla/Logging.h"
+
+namespace mozilla {
+extern LazyLogModule sPEMLog;
+#define AND_PEM_LOG(arg, ...) \
+ MOZ_LOG( \
+ sPEMLog, mozilla::LogLevel::Debug, \
+ ("AndroidEncoderModule(%p)::%s: " arg, this, __func__, ##__VA_ARGS__))
+
+bool AndroidEncoderModule::SupportsMimeType(const nsACString& aMimeType) const {
+ return MP4Decoder::IsH264(aMimeType);
+}
+
+already_AddRefed<MediaDataEncoder> AndroidEncoderModule::CreateVideoEncoder(
+ const CreateEncoderParams& aParams) const {
+ RefPtr<MediaDataEncoder> encoder =
+ new AndroidDataEncoder(aParams.ToH264Config(), aParams.mTaskQueue);
+ return encoder.forget();
+}
+
+} // namespace mozilla
+
+#undef AND_PEM_LOG
diff --git a/dom/media/platforms/android/AndroidEncoderModule.h b/dom/media/platforms/android/AndroidEncoderModule.h
new file mode 100644
index 0000000000..2593f75043
--- /dev/null
+++ b/dom/media/platforms/android/AndroidEncoderModule.h
@@ -0,0 +1,22 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef DOM_MEDIA_PLATFORMS_ANDROID_ANDROIDENCODERMODULE_H_
+#define DOM_MEDIA_PLATFORMS_ANDROID_ANDROIDENCODERMODULE_H_
+
+#include "PlatformEncoderModule.h"
+
+namespace mozilla {
+
+class AndroidEncoderModule final : public PlatformEncoderModule {
+ public:
+ bool SupportsMimeType(const nsACString& aMimeType) const override;
+
+ already_AddRefed<MediaDataEncoder> CreateVideoEncoder(
+ const CreateEncoderParams& aParams) const override;
+};
+
+} // namespace mozilla
+
+#endif
diff --git a/dom/media/platforms/android/JavaCallbacksSupport.h b/dom/media/platforms/android/JavaCallbacksSupport.h
new file mode 100644
index 0000000000..e79d796209
--- /dev/null
+++ b/dom/media/platforms/android/JavaCallbacksSupport.h
@@ -0,0 +1,73 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef JavaCallbacksSupport_h_
+#define JavaCallbacksSupport_h_
+
+#include "MediaResult.h"
+#include "MediaCodec.h"
+#include "mozilla/java/CodecProxyNatives.h"
+#include "mozilla/java/SampleBufferWrappers.h"
+#include "mozilla/java/SampleWrappers.h"
+
+namespace mozilla {
+
+class JavaCallbacksSupport
+ : public java::CodecProxy::NativeCallbacks::Natives<JavaCallbacksSupport> {
+ public:
+ typedef java::CodecProxy::NativeCallbacks::Natives<JavaCallbacksSupport> Base;
+ using Base::AttachNative;
+ using Base::DisposeNative;
+ using Base::GetNative;
+
+ JavaCallbacksSupport() : mCanceled(false) {}
+
+ virtual ~JavaCallbacksSupport() {}
+
+ virtual void HandleInput(int64_t aTimestamp, bool aProcessed) = 0;
+
+ void OnInputStatus(jlong aTimestamp, bool aProcessed) {
+ if (!mCanceled) {
+ HandleInput(aTimestamp, aProcessed);
+ }
+ }
+
+ virtual void HandleOutput(java::Sample::Param aSample,
+ java::SampleBuffer::Param aBuffer) = 0;
+
+ void OnOutput(jni::Object::Param aSample, jni::Object::Param aBuffer) {
+ if (!mCanceled) {
+ HandleOutput(java::Sample::Ref::From(aSample),
+ java::SampleBuffer::Ref::From(aBuffer));
+ }
+ }
+
+ virtual void HandleOutputFormatChanged(
+ java::sdk::MediaFormat::Param aFormat){};
+
+ void OnOutputFormatChanged(jni::Object::Param aFormat) {
+ if (!mCanceled) {
+ HandleOutputFormatChanged(java::sdk::MediaFormat::Ref::From(aFormat));
+ }
+ }
+
+ virtual void HandleError(const MediaResult& aError) = 0;
+
+ void OnError(bool aIsFatal) {
+ if (!mCanceled) {
+ HandleError(aIsFatal
+ ? MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__)
+ : MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR, __func__));
+ }
+ }
+
+ void Cancel() { mCanceled = true; }
+
+ private:
+ Atomic<bool> mCanceled;
+};
+
+} // namespace mozilla
+
+#endif
diff --git a/dom/media/platforms/android/RemoteDataDecoder.cpp b/dom/media/platforms/android/RemoteDataDecoder.cpp
new file mode 100644
index 0000000000..14c3266c52
--- /dev/null
+++ b/dom/media/platforms/android/RemoteDataDecoder.cpp
@@ -0,0 +1,849 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "RemoteDataDecoder.h"
+
+#include <jni.h>
+
+#include "AndroidBridge.h"
+#include "AndroidDecoderModule.h"
+#include "EMEDecoderModule.h"
+#include "GLImages.h"
+#include "JavaCallbacksSupport.h"
+#include "MediaData.h"
+#include "MediaInfo.h"
+#include "SimpleMap.h"
+#include "VPXDecoder.h"
+#include "VideoUtils.h"
+#include "mozilla/java/CodecProxyWrappers.h"
+#include "mozilla/java/GeckoSurfaceWrappers.h"
+#include "mozilla/java/SampleBufferWrappers.h"
+#include "mozilla/java/SampleWrappers.h"
+#include "mozilla/java/SurfaceAllocatorWrappers.h"
+#include "nsPromiseFlatString.h"
+#include "nsThreadUtils.h"
+#include "prlog.h"
+
+#undef LOG
+#define LOG(arg, ...) \
+ MOZ_LOG(sAndroidDecoderModuleLog, mozilla::LogLevel::Debug, \
+ ("RemoteDataDecoder(%p)::%s: " arg, this, __func__, ##__VA_ARGS__))
+
+using namespace mozilla;
+using namespace mozilla::gl;
+using media::TimeUnit;
+
+namespace mozilla {
+
+// Hold a reference to the output buffer until we're ready to release it back to
+// the Java codec (for rendering or not).
+class RenderOrReleaseOutput {
+ public:
+ RenderOrReleaseOutput(java::CodecProxy::Param aCodec,
+ java::Sample::Param aSample)
+ : mCodec(aCodec), mSample(aSample) {}
+
+ virtual ~RenderOrReleaseOutput() { ReleaseOutput(false); }
+
+ protected:
+ void ReleaseOutput(bool aToRender) {
+ if (mCodec && mSample) {
+ mCodec->ReleaseOutput(mSample, aToRender);
+ mCodec = nullptr;
+ mSample = nullptr;
+ }
+ }
+
+ private:
+ java::CodecProxy::GlobalRef mCodec;
+ java::Sample::GlobalRef mSample;
+};
+
+class RemoteVideoDecoder : public RemoteDataDecoder {
+ public:
+ // Render the output to the surface when the frame is sent
+ // to compositor, or release it if not presented.
+ class CompositeListener
+ : private RenderOrReleaseOutput,
+ public layers::SurfaceTextureImage::SetCurrentCallback {
+ public:
+ CompositeListener(java::CodecProxy::Param aCodec,
+ java::Sample::Param aSample)
+ : RenderOrReleaseOutput(aCodec, aSample) {}
+
+ void operator()(void) override { ReleaseOutput(true); }
+ };
+
+ class InputInfo {
+ public:
+ InputInfo() {}
+
+ InputInfo(const int64_t aDurationUs, const gfx::IntSize& aImageSize,
+ const gfx::IntSize& aDisplaySize)
+ : mDurationUs(aDurationUs),
+ mImageSize(aImageSize),
+ mDisplaySize(aDisplaySize) {}
+
+ int64_t mDurationUs;
+ gfx::IntSize mImageSize;
+ gfx::IntSize mDisplaySize;
+ };
+
+ class CallbacksSupport final : public JavaCallbacksSupport {
+ public:
+ explicit CallbacksSupport(RemoteVideoDecoder* aDecoder)
+ : mDecoder(aDecoder) {}
+
+ void HandleInput(int64_t aTimestamp, bool aProcessed) override {
+ mDecoder->UpdateInputStatus(aTimestamp, aProcessed);
+ }
+
+ void HandleOutput(java::Sample::Param aSample,
+ java::SampleBuffer::Param aBuffer) override {
+ MOZ_ASSERT(!aBuffer, "Video sample should be bufferless");
+ // aSample will be implicitly converted into a GlobalRef.
+ mDecoder->ProcessOutput(std::move(aSample));
+ }
+
+ void HandleError(const MediaResult& aError) override {
+ mDecoder->Error(aError);
+ }
+
+ friend class RemoteDataDecoder;
+
+ private:
+ RemoteVideoDecoder* mDecoder;
+ };
+
+ RemoteVideoDecoder(const VideoInfo& aConfig,
+ java::sdk::MediaFormat::Param aFormat,
+ const nsString& aDrmStubId)
+ : RemoteDataDecoder(MediaData::Type::VIDEO_DATA, aConfig.mMimeType,
+ aFormat, aDrmStubId),
+ mConfig(aConfig) {}
+
+ ~RemoteVideoDecoder() {
+ if (mSurface) {
+ java::SurfaceAllocator::DisposeSurface(mSurface);
+ }
+ }
+
+ RefPtr<InitPromise> Init() override {
+ mThread = GetCurrentSerialEventTarget();
+ java::sdk::BufferInfo::LocalRef bufferInfo;
+ if (NS_FAILED(java::sdk::BufferInfo::New(&bufferInfo)) || !bufferInfo) {
+ return InitPromise::CreateAndReject(NS_ERROR_OUT_OF_MEMORY, __func__);
+ }
+ mInputBufferInfo = bufferInfo;
+
+ mSurface =
+ java::GeckoSurface::LocalRef(java::SurfaceAllocator::AcquireSurface(
+ mConfig.mImage.width, mConfig.mImage.height, false));
+ if (!mSurface) {
+ return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ __func__);
+ }
+
+ mSurfaceHandle = mSurface->GetHandle();
+
+ // Register native methods.
+ JavaCallbacksSupport::Init();
+
+ mJavaCallbacks = java::CodecProxy::NativeCallbacks::New();
+ if (!mJavaCallbacks) {
+ return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ __func__);
+ }
+ JavaCallbacksSupport::AttachNative(
+ mJavaCallbacks, mozilla::MakeUnique<CallbacksSupport>(this));
+
+ mJavaDecoder = java::CodecProxy::Create(
+ false, // false indicates to create a decoder and true denotes encoder
+ mFormat, mSurface, mJavaCallbacks, mDrmStubId);
+ if (mJavaDecoder == nullptr) {
+ return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ __func__);
+ }
+ mIsCodecSupportAdaptivePlayback =
+ mJavaDecoder->IsAdaptivePlaybackSupported();
+ mIsHardwareAccelerated = mJavaDecoder->IsHardwareAccelerated();
+ return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__);
+ }
+
+ RefPtr<MediaDataDecoder::FlushPromise> Flush() override {
+ AssertOnThread();
+ mInputInfos.Clear();
+ mSeekTarget.reset();
+ mLatestOutputTime.reset();
+ return RemoteDataDecoder::Flush();
+ }
+
+ RefPtr<MediaDataDecoder::DecodePromise> Decode(
+ MediaRawData* aSample) override {
+ AssertOnThread();
+
+ const VideoInfo* config =
+ aSample->mTrackInfo ? aSample->mTrackInfo->GetAsVideoInfo() : &mConfig;
+ MOZ_ASSERT(config);
+
+ InputInfo info(aSample->mDuration.ToMicroseconds(), config->mImage,
+ config->mDisplay);
+ mInputInfos.Insert(aSample->mTime.ToMicroseconds(), info);
+ return RemoteDataDecoder::Decode(aSample);
+ }
+
+ bool SupportDecoderRecycling() const override {
+ return mIsCodecSupportAdaptivePlayback;
+ }
+
+ void SetSeekThreshold(const TimeUnit& aTime) override {
+ auto setter = [self = RefPtr{this}, aTime] {
+ if (aTime.IsValid()) {
+ self->mSeekTarget = Some(aTime);
+ } else {
+ self->mSeekTarget.reset();
+ }
+ };
+ if (mThread->IsOnCurrentThread()) {
+ setter();
+ } else {
+ nsCOMPtr<nsIRunnable> runnable = NS_NewRunnableFunction(
+ "RemoteVideoDecoder::SetSeekThreshold", std::move(setter));
+ nsresult rv = mThread->Dispatch(runnable.forget());
+ MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+ Unused << rv;
+ }
+ }
+
+ bool IsUsefulData(const RefPtr<MediaData>& aSample) override {
+ AssertOnThread();
+
+ if (mLatestOutputTime && aSample->mTime < mLatestOutputTime.value()) {
+ return false;
+ }
+
+ const TimeUnit endTime = aSample->GetEndTime();
+ if (mSeekTarget && endTime <= mSeekTarget.value()) {
+ return false;
+ }
+
+ mSeekTarget.reset();
+ mLatestOutputTime = Some(endTime);
+ return true;
+ }
+
+ bool IsHardwareAccelerated(nsACString& aFailureReason) const override {
+ return mIsHardwareAccelerated;
+ }
+
+ ConversionRequired NeedsConversion() const override {
+ return ConversionRequired::kNeedAnnexB;
+ }
+
+ private:
+ // Param and LocalRef are only valid for the duration of a JNI method call.
+ // Use GlobalRef as the parameter type to keep the Java object referenced
+ // until running.
+ void ProcessOutput(java::Sample::GlobalRef&& aSample) {
+ if (!mThread->IsOnCurrentThread()) {
+ nsresult rv =
+ mThread->Dispatch(NewRunnableMethod<java::Sample::GlobalRef&&>(
+ "RemoteVideoDecoder::ProcessOutput", this,
+ &RemoteVideoDecoder::ProcessOutput, std::move(aSample)));
+ MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+ Unused << rv;
+ return;
+ }
+
+ AssertOnThread();
+ if (GetState() == State::SHUTDOWN) {
+ aSample->Dispose();
+ return;
+ }
+
+ UniquePtr<layers::SurfaceTextureImage::SetCurrentCallback> releaseSample(
+ new CompositeListener(mJavaDecoder, aSample));
+
+ java::sdk::BufferInfo::LocalRef info = aSample->Info();
+ MOZ_ASSERT(info);
+
+ int32_t flags;
+ bool ok = NS_SUCCEEDED(info->Flags(&flags));
+
+ int32_t offset;
+ ok &= NS_SUCCEEDED(info->Offset(&offset));
+
+ int32_t size;
+ ok &= NS_SUCCEEDED(info->Size(&size));
+
+ int64_t presentationTimeUs;
+ ok &= NS_SUCCEEDED(info->PresentationTimeUs(&presentationTimeUs));
+
+ if (!ok) {
+ Error(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ RESULT_DETAIL("VideoCallBack::HandleOutput")));
+ return;
+ }
+
+ InputInfo inputInfo;
+ ok = mInputInfos.Find(presentationTimeUs, inputInfo);
+ bool isEOS = !!(flags & java::sdk::MediaCodec::BUFFER_FLAG_END_OF_STREAM);
+ if (!ok && !isEOS) {
+ // Ignore output with no corresponding input.
+ return;
+ }
+
+ if (ok && (size > 0 || presentationTimeUs >= 0)) {
+ RefPtr<layers::Image> img = new layers::SurfaceTextureImage(
+ mSurfaceHandle, inputInfo.mImageSize, false /* NOT continuous */,
+ gl::OriginPos::BottomLeft, mConfig.HasAlpha());
+ img->AsSurfaceTextureImage()->RegisterSetCurrentCallback(
+ std::move(releaseSample));
+
+ RefPtr<VideoData> v = VideoData::CreateFromImage(
+ inputInfo.mDisplaySize, offset,
+ TimeUnit::FromMicroseconds(presentationTimeUs),
+ TimeUnit::FromMicroseconds(inputInfo.mDurationUs), img.forget(),
+ !!(flags & java::sdk::MediaCodec::BUFFER_FLAG_SYNC_FRAME),
+ TimeUnit::FromMicroseconds(presentationTimeUs));
+
+ RemoteDataDecoder::UpdateOutputStatus(std::move(v));
+ }
+
+ if (isEOS) {
+ DrainComplete();
+ }
+ }
+
+ const VideoInfo mConfig;
+ java::GeckoSurface::GlobalRef mSurface;
+ AndroidSurfaceTextureHandle mSurfaceHandle;
+ // Only accessed on reader's task queue.
+ bool mIsCodecSupportAdaptivePlayback = false;
+ // Can be accessed on any thread, but only written on during init.
+ bool mIsHardwareAccelerated = false;
+ // Accessed on mThread and reader's thread. SimpleMap however is
+ // thread-safe, so it's okay to do so.
+ SimpleMap<InputInfo> mInputInfos;
+ // Only accessed on mThread.
+ Maybe<TimeUnit> mSeekTarget;
+ Maybe<TimeUnit> mLatestOutputTime;
+};
+
+class RemoteAudioDecoder : public RemoteDataDecoder {
+ public:
+ RemoteAudioDecoder(const AudioInfo& aConfig,
+ java::sdk::MediaFormat::Param aFormat,
+ const nsString& aDrmStubId)
+ : RemoteDataDecoder(MediaData::Type::AUDIO_DATA, aConfig.mMimeType,
+ aFormat, aDrmStubId) {
+ JNIEnv* const env = jni::GetEnvForThread();
+
+ bool formatHasCSD = false;
+ NS_ENSURE_SUCCESS_VOID(aFormat->ContainsKey(u"csd-0"_ns, &formatHasCSD));
+
+ if (!formatHasCSD && aConfig.mCodecSpecificConfig->Length() >= 2) {
+ jni::ByteBuffer::LocalRef buffer(env);
+ buffer = jni::ByteBuffer::New(aConfig.mCodecSpecificConfig->Elements(),
+ aConfig.mCodecSpecificConfig->Length());
+ NS_ENSURE_SUCCESS_VOID(aFormat->SetByteBuffer(u"csd-0"_ns, buffer));
+ }
+ }
+
+ RefPtr<InitPromise> Init() override {
+ mThread = GetCurrentSerialEventTarget();
+ java::sdk::BufferInfo::LocalRef bufferInfo;
+ if (NS_FAILED(java::sdk::BufferInfo::New(&bufferInfo)) || !bufferInfo) {
+ return InitPromise::CreateAndReject(NS_ERROR_OUT_OF_MEMORY, __func__);
+ }
+ mInputBufferInfo = bufferInfo;
+
+ // Register native methods.
+ JavaCallbacksSupport::Init();
+
+ mJavaCallbacks = java::CodecProxy::NativeCallbacks::New();
+ if (!mJavaCallbacks) {
+ return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ __func__);
+ }
+ JavaCallbacksSupport::AttachNative(
+ mJavaCallbacks, mozilla::MakeUnique<CallbacksSupport>(this));
+
+ mJavaDecoder = java::CodecProxy::Create(false, mFormat, nullptr,
+ mJavaCallbacks, mDrmStubId);
+ if (mJavaDecoder == nullptr) {
+ return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ __func__);
+ }
+
+ return InitPromise::CreateAndResolve(TrackInfo::kAudioTrack, __func__);
+ }
+
+ RefPtr<FlushPromise> Flush() override {
+ AssertOnThread();
+ mFirstDemuxedSampleTime.reset();
+ return RemoteDataDecoder::Flush();
+ }
+
+ RefPtr<DecodePromise> Decode(MediaRawData* aSample) override {
+ AssertOnThread();
+ if (!mFirstDemuxedSampleTime) {
+ MOZ_ASSERT(aSample->mTime.IsValid());
+ mFirstDemuxedSampleTime.emplace(aSample->mTime);
+ }
+ return RemoteDataDecoder::Decode(aSample);
+ }
+
+ private:
+ class CallbacksSupport final : public JavaCallbacksSupport {
+ public:
+ explicit CallbacksSupport(RemoteAudioDecoder* aDecoder)
+ : mDecoder(aDecoder) {}
+
+ void HandleInput(int64_t aTimestamp, bool aProcessed) override {
+ mDecoder->UpdateInputStatus(aTimestamp, aProcessed);
+ }
+
+ void HandleOutput(java::Sample::Param aSample,
+ java::SampleBuffer::Param aBuffer) override {
+ MOZ_ASSERT(aBuffer, "Audio sample should have buffer");
+ // aSample will be implicitly converted into a GlobalRef.
+ mDecoder->ProcessOutput(std::move(aSample), std::move(aBuffer));
+ }
+
+ void HandleOutputFormatChanged(
+ java::sdk::MediaFormat::Param aFormat) override {
+ int32_t outputChannels = 0;
+ aFormat->GetInteger(u"channel-count"_ns, &outputChannels);
+ AudioConfig::ChannelLayout layout(outputChannels);
+ if (!layout.IsValid()) {
+ mDecoder->Error(MediaResult(
+ NS_ERROR_DOM_MEDIA_FATAL_ERR,
+ RESULT_DETAIL("Invalid channel layout:%d", outputChannels)));
+ return;
+ }
+
+ int32_t sampleRate = 0;
+ aFormat->GetInteger(u"sample-rate"_ns, &sampleRate);
+ LOG("Audio output format changed: channels:%d sample rate:%d",
+ outputChannels, sampleRate);
+
+ mDecoder->ProcessOutputFormatChange(outputChannels, sampleRate);
+ }
+
+ void HandleError(const MediaResult& aError) override {
+ mDecoder->Error(aError);
+ }
+
+ private:
+ RemoteAudioDecoder* mDecoder;
+ };
+
+ bool IsSampleTimeSmallerThanFirstDemuxedSampleTime(int64_t aTime) const {
+ return mFirstDemuxedSampleTime->ToMicroseconds() > aTime;
+ }
+
+ bool ShouldDiscardSample(int64_t aSession) const {
+ AssertOnThread();
+ // HandleOutput() runs on Android binder thread pool and could be preempted
+ // by RemoteDateDecoder task queue. That means ProcessOutput() could be
+ // scheduled after Shutdown() or Flush(). We won't need the
+ // sample which is returned after calling Shutdown() and Flush(). We can
+ // check mFirstDemuxedSampleTime to know whether the Flush() has been
+ // called, becasue it would be reset in Flush().
+ return GetState() == State::SHUTDOWN || !mFirstDemuxedSampleTime ||
+ mSession != aSession;
+ }
+
+ // Param and LocalRef are only valid for the duration of a JNI method call.
+ // Use GlobalRef as the parameter type to keep the Java object referenced
+ // until running.
+ void ProcessOutput(java::Sample::GlobalRef&& aSample,
+ java::SampleBuffer::GlobalRef&& aBuffer) {
+ if (!mThread->IsOnCurrentThread()) {
+ nsresult rv =
+ mThread->Dispatch(NewRunnableMethod<java::Sample::GlobalRef&&,
+ java::SampleBuffer::GlobalRef&&>(
+ "RemoteAudioDecoder::ProcessOutput", this,
+ &RemoteAudioDecoder::ProcessOutput, std::move(aSample),
+ std::move(aBuffer)));
+ MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+ Unused << rv;
+ return;
+ }
+
+ AssertOnThread();
+
+ if (ShouldDiscardSample(aSample->Session()) || !aBuffer->IsValid()) {
+ aSample->Dispose();
+ return;
+ }
+
+ RenderOrReleaseOutput autoRelease(mJavaDecoder, aSample);
+
+ java::sdk::BufferInfo::LocalRef info = aSample->Info();
+ MOZ_ASSERT(info);
+
+ int32_t flags = 0;
+ bool ok = NS_SUCCEEDED(info->Flags(&flags));
+ bool isEOS = !!(flags & java::sdk::MediaCodec::BUFFER_FLAG_END_OF_STREAM);
+
+ int32_t offset;
+ ok &= NS_SUCCEEDED(info->Offset(&offset));
+
+ int64_t presentationTimeUs;
+ ok &= NS_SUCCEEDED(info->PresentationTimeUs(&presentationTimeUs));
+
+ int32_t size;
+ ok &= NS_SUCCEEDED(info->Size(&size));
+
+ if (!ok ||
+ (IsSampleTimeSmallerThanFirstDemuxedSampleTime(presentationTimeUs) &&
+ !isEOS)) {
+ Error(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__));
+ return;
+ }
+
+ if (size > 0) {
+#ifdef MOZ_SAMPLE_TYPE_S16
+ const int32_t numSamples = size / 2;
+#else
+# error We only support 16-bit integer PCM
+#endif
+
+ AlignedAudioBuffer audio(numSamples);
+ if (!audio) {
+ Error(MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__));
+ return;
+ }
+
+ jni::ByteBuffer::LocalRef dest = jni::ByteBuffer::New(audio.get(), size);
+ aBuffer->WriteToByteBuffer(dest, offset, size);
+
+ RefPtr<AudioData> data =
+ new AudioData(0, TimeUnit::FromMicroseconds(presentationTimeUs),
+ std::move(audio), mOutputChannels, mOutputSampleRate);
+
+ UpdateOutputStatus(std::move(data));
+ }
+
+ if (isEOS) {
+ DrainComplete();
+ }
+ }
+
+ void ProcessOutputFormatChange(int32_t aChannels, int32_t aSampleRate) {
+ if (!mThread->IsOnCurrentThread()) {
+ nsresult rv = mThread->Dispatch(NewRunnableMethod<int32_t, int32_t>(
+ "RemoteAudioDecoder::ProcessOutputFormatChange", this,
+ &RemoteAudioDecoder::ProcessOutputFormatChange, aChannels,
+ aSampleRate));
+ MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+ Unused << rv;
+ return;
+ }
+
+ AssertOnThread();
+
+ mOutputChannels = aChannels;
+ mOutputSampleRate = aSampleRate;
+ }
+
+ int32_t mOutputChannels;
+ int32_t mOutputSampleRate;
+ Maybe<TimeUnit> mFirstDemuxedSampleTime;
+};
+
+already_AddRefed<MediaDataDecoder> RemoteDataDecoder::CreateAudioDecoder(
+ const CreateDecoderParams& aParams, const nsString& aDrmStubId,
+ CDMProxy* aProxy) {
+ const AudioInfo& config = aParams.AudioConfig();
+ java::sdk::MediaFormat::LocalRef format;
+ NS_ENSURE_SUCCESS(
+ java::sdk::MediaFormat::CreateAudioFormat(config.mMimeType, config.mRate,
+ config.mChannels, &format),
+ nullptr);
+
+ RefPtr<MediaDataDecoder> decoder =
+ new RemoteAudioDecoder(config, format, aDrmStubId);
+ if (aProxy) {
+ decoder = new EMEMediaDataDecoderProxy(aParams, decoder.forget(), aProxy);
+ }
+ return decoder.forget();
+}
+
+already_AddRefed<MediaDataDecoder> RemoteDataDecoder::CreateVideoDecoder(
+ const CreateDecoderParams& aParams, const nsString& aDrmStubId,
+ CDMProxy* aProxy) {
+ const VideoInfo& config = aParams.VideoConfig();
+ java::sdk::MediaFormat::LocalRef format;
+ NS_ENSURE_SUCCESS(java::sdk::MediaFormat::CreateVideoFormat(
+ TranslateMimeType(config.mMimeType),
+ config.mImage.width, config.mImage.height, &format),
+ nullptr);
+
+ RefPtr<MediaDataDecoder> decoder =
+ new RemoteVideoDecoder(config, format, aDrmStubId);
+ if (aProxy) {
+ decoder = new EMEMediaDataDecoderProxy(aParams, decoder.forget(), aProxy);
+ }
+ return decoder.forget();
+}
+
+RemoteDataDecoder::RemoteDataDecoder(MediaData::Type aType,
+ const nsACString& aMimeType,
+ java::sdk::MediaFormat::Param aFormat,
+ const nsString& aDrmStubId)
+ : mType(aType),
+ mMimeType(aMimeType),
+ mFormat(aFormat),
+ mDrmStubId(aDrmStubId),
+ mSession(0),
+ mNumPendingInputs(0) {}
+
+RefPtr<MediaDataDecoder::FlushPromise> RemoteDataDecoder::Flush() {
+ AssertOnThread();
+ MOZ_ASSERT(GetState() != State::SHUTDOWN);
+
+ mDecodedData = DecodedData();
+ UpdatePendingInputStatus(PendingOp::CLEAR);
+ mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ mDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+ SetState(State::DRAINED);
+ mJavaDecoder->Flush();
+ return FlushPromise::CreateAndResolve(true, __func__);
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> RemoteDataDecoder::Drain() {
+ AssertOnThread();
+ if (GetState() == State::SHUTDOWN) {
+ return DecodePromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
+ __func__);
+ }
+ RefPtr<DecodePromise> p = mDrainPromise.Ensure(__func__);
+ if (GetState() == State::DRAINED) {
+ // There's no operation to perform other than returning any already
+ // decoded data.
+ ReturnDecodedData();
+ return p;
+ }
+
+ if (GetState() == State::DRAINING) {
+ // Draining operation already pending, let it complete its course.
+ return p;
+ }
+
+ SetState(State::DRAINING);
+ mInputBufferInfo->Set(0, 0, -1,
+ java::sdk::MediaCodec::BUFFER_FLAG_END_OF_STREAM);
+ mSession = mJavaDecoder->Input(nullptr, mInputBufferInfo, nullptr);
+ return p;
+}
+
+RefPtr<ShutdownPromise> RemoteDataDecoder::Shutdown() {
+ LOG("");
+ AssertOnThread();
+ SetState(State::SHUTDOWN);
+ if (mJavaDecoder) {
+ mJavaDecoder->Release();
+ mJavaDecoder = nullptr;
+ }
+
+ if (mJavaCallbacks) {
+ JavaCallbacksSupport::GetNative(mJavaCallbacks)->Cancel();
+ JavaCallbacksSupport::DisposeNative(mJavaCallbacks);
+ mJavaCallbacks = nullptr;
+ }
+
+ mFormat = nullptr;
+
+ return ShutdownPromise::CreateAndResolve(true, __func__);
+}
+
+static java::sdk::CryptoInfo::LocalRef GetCryptoInfoFromSample(
+ const MediaRawData* aSample) {
+ auto& cryptoObj = aSample->mCrypto;
+
+ if (!cryptoObj.IsEncrypted()) {
+ return nullptr;
+ }
+
+ java::sdk::CryptoInfo::LocalRef cryptoInfo;
+ nsresult rv = java::sdk::CryptoInfo::New(&cryptoInfo);
+ NS_ENSURE_SUCCESS(rv, nullptr);
+
+ uint32_t numSubSamples = std::min<uint32_t>(
+ cryptoObj.mPlainSizes.Length(), cryptoObj.mEncryptedSizes.Length());
+
+ uint32_t totalSubSamplesSize = 0;
+ for (auto& size : cryptoObj.mPlainSizes) {
+ totalSubSamplesSize += size;
+ }
+ for (auto& size : cryptoObj.mEncryptedSizes) {
+ totalSubSamplesSize += size;
+ }
+
+ // Deep copy the plain sizes so we can modify them.
+ nsTArray<uint32_t> plainSizes = cryptoObj.mPlainSizes.Clone();
+ uint32_t codecSpecificDataSize = aSample->Size() - totalSubSamplesSize;
+ // Size of codec specific data("CSD") for Android java::sdk::MediaCodec usage
+ // should be included in the 1st plain size if it exists.
+ if (!plainSizes.IsEmpty()) {
+ // This shouldn't overflow as the the plain size should be UINT16_MAX at
+ // most, and the CSD should never be that large. Checked int acts like a
+ // diagnostic assert here to help catch if we ever have insane inputs.
+ CheckedUint32 newLeadingPlainSize{plainSizes[0]};
+ newLeadingPlainSize += codecSpecificDataSize;
+ plainSizes[0] = newLeadingPlainSize.value();
+ }
+
+ static const int kExpectedIVLength = 16;
+ auto tempIV(cryptoObj.mIV);
+ auto tempIVLength = tempIV.Length();
+ MOZ_ASSERT(tempIVLength <= kExpectedIVLength);
+ for (size_t i = tempIVLength; i < kExpectedIVLength; i++) {
+ // Padding with 0
+ tempIV.AppendElement(0);
+ }
+
+ auto numBytesOfPlainData = mozilla::jni::IntArray::New(
+ reinterpret_cast<const int32_t*>(&plainSizes[0]), plainSizes.Length());
+
+ auto numBytesOfEncryptedData = mozilla::jni::IntArray::New(
+ reinterpret_cast<const int32_t*>(&cryptoObj.mEncryptedSizes[0]),
+ cryptoObj.mEncryptedSizes.Length());
+
+ auto iv = mozilla::jni::ByteArray::New(reinterpret_cast<int8_t*>(&tempIV[0]),
+ tempIV.Length());
+ auto keyId = mozilla::jni::ByteArray::New(
+ reinterpret_cast<const int8_t*>(&cryptoObj.mKeyId[0]),
+ cryptoObj.mKeyId.Length());
+ cryptoInfo->Set(numSubSamples, numBytesOfPlainData, numBytesOfEncryptedData,
+ keyId, iv, java::sdk::MediaCodec::CRYPTO_MODE_AES_CTR);
+
+ return cryptoInfo;
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> RemoteDataDecoder::Decode(
+ MediaRawData* aSample) {
+ AssertOnThread();
+ MOZ_ASSERT(GetState() != State::SHUTDOWN);
+ MOZ_ASSERT(aSample != nullptr);
+ jni::ByteBuffer::LocalRef bytes = jni::ByteBuffer::New(
+ const_cast<uint8_t*>(aSample->Data()), aSample->Size());
+
+ SetState(State::DRAINABLE);
+ mInputBufferInfo->Set(0, aSample->Size(), aSample->mTime.ToMicroseconds(), 0);
+ int64_t session = mJavaDecoder->Input(bytes, mInputBufferInfo,
+ GetCryptoInfoFromSample(aSample));
+ if (session == java::CodecProxy::INVALID_SESSION) {
+ return DecodePromise::CreateAndReject(
+ MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__), __func__);
+ }
+ mSession = session;
+ return mDecodePromise.Ensure(__func__);
+}
+
+void RemoteDataDecoder::UpdatePendingInputStatus(PendingOp aOp) {
+ AssertOnThread();
+ switch (aOp) {
+ case PendingOp::INCREASE:
+ mNumPendingInputs++;
+ break;
+ case PendingOp::DECREASE:
+ mNumPendingInputs--;
+ break;
+ case PendingOp::CLEAR:
+ mNumPendingInputs = 0;
+ break;
+ }
+}
+
+void RemoteDataDecoder::UpdateInputStatus(int64_t aTimestamp, bool aProcessed) {
+ if (!mThread->IsOnCurrentThread()) {
+ nsresult rv = mThread->Dispatch(NewRunnableMethod<int64_t, bool>(
+ "RemoteDataDecoder::UpdateInputStatus", this,
+ &RemoteDataDecoder::UpdateInputStatus, aTimestamp, aProcessed));
+ MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+ Unused << rv;
+ return;
+ }
+ AssertOnThread();
+ if (GetState() == State::SHUTDOWN) {
+ return;
+ }
+
+ if (!aProcessed) {
+ UpdatePendingInputStatus(PendingOp::INCREASE);
+ } else if (HasPendingInputs()) {
+ UpdatePendingInputStatus(PendingOp::DECREASE);
+ }
+
+ if (!HasPendingInputs() || // Input has been processed, request the next one.
+ !mDecodedData.IsEmpty()) { // Previous output arrived before Decode().
+ ReturnDecodedData();
+ }
+}
+
+void RemoteDataDecoder::UpdateOutputStatus(RefPtr<MediaData>&& aSample) {
+ AssertOnThread();
+ if (GetState() == State::SHUTDOWN) {
+ return;
+ }
+ if (IsUsefulData(aSample)) {
+ mDecodedData.AppendElement(std::move(aSample));
+ }
+ ReturnDecodedData();
+}
+
+void RemoteDataDecoder::ReturnDecodedData() {
+ AssertOnThread();
+ MOZ_ASSERT(GetState() != State::SHUTDOWN);
+
+ // We only want to clear mDecodedData when we have resolved the promises.
+ if (!mDecodePromise.IsEmpty()) {
+ mDecodePromise.Resolve(std::move(mDecodedData), __func__);
+ mDecodedData = DecodedData();
+ } else if (!mDrainPromise.IsEmpty() &&
+ (!mDecodedData.IsEmpty() || GetState() == State::DRAINED)) {
+ mDrainPromise.Resolve(std::move(mDecodedData), __func__);
+ mDecodedData = DecodedData();
+ }
+}
+
+void RemoteDataDecoder::DrainComplete() {
+ if (!mThread->IsOnCurrentThread()) {
+ nsresult rv = mThread->Dispatch(
+ NewRunnableMethod("RemoteDataDecoder::DrainComplete", this,
+ &RemoteDataDecoder::DrainComplete));
+ MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+ Unused << rv;
+ return;
+ }
+ AssertOnThread();
+ if (GetState() == State::SHUTDOWN) {
+ return;
+ }
+ SetState(State::DRAINED);
+ ReturnDecodedData();
+}
+
+void RemoteDataDecoder::Error(const MediaResult& aError) {
+ if (!mThread->IsOnCurrentThread()) {
+ nsresult rv = mThread->Dispatch(NewRunnableMethod<MediaResult>(
+ "RemoteDataDecoder::Error", this, &RemoteDataDecoder::Error, aError));
+ MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+ Unused << rv;
+ return;
+ }
+ AssertOnThread();
+ if (GetState() == State::SHUTDOWN) {
+ return;
+ }
+ mDecodePromise.RejectIfExists(aError, __func__);
+ mDrainPromise.RejectIfExists(aError, __func__);
+}
+
+} // namespace mozilla
+#undef LOG
diff --git a/dom/media/platforms/android/RemoteDataDecoder.h b/dom/media/platforms/android/RemoteDataDecoder.h
new file mode 100644
index 0000000000..b4c353f4af
--- /dev/null
+++ b/dom/media/platforms/android/RemoteDataDecoder.h
@@ -0,0 +1,105 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef RemoteDataDecoder_h_
+#define RemoteDataDecoder_h_
+
+#include "AndroidDecoderModule.h"
+#include "SurfaceTexture.h"
+#include "TimeUnits.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/Monitor.h"
+#include "mozilla/java/CodecProxyWrappers.h"
+
+namespace mozilla {
+
+DDLoggedTypeDeclNameAndBase(RemoteDataDecoder, MediaDataDecoder);
+
+class RemoteDataDecoder : public MediaDataDecoder,
+ public DecoderDoctorLifeLogger<RemoteDataDecoder> {
+ public:
+ static already_AddRefed<MediaDataDecoder> CreateAudioDecoder(
+ const CreateDecoderParams& aParams, const nsString& aDrmStubId,
+ CDMProxy* aProxy);
+
+ static already_AddRefed<MediaDataDecoder> CreateVideoDecoder(
+ const CreateDecoderParams& aParams, const nsString& aDrmStubId,
+ CDMProxy* aProxy);
+
+ RefPtr<DecodePromise> Decode(MediaRawData* aSample) override;
+ RefPtr<DecodePromise> Drain() override;
+ RefPtr<FlushPromise> Flush() override;
+ RefPtr<ShutdownPromise> Shutdown() override;
+ nsCString GetDescriptionName() const override {
+ return "android decoder (remote)"_ns;
+ }
+
+ protected:
+ virtual ~RemoteDataDecoder() {}
+ RemoteDataDecoder(MediaData::Type aType, const nsACString& aMimeType,
+ java::sdk::MediaFormat::Param aFormat,
+ const nsString& aDrmStubId);
+
+ // Methods only called on mThread.
+ void UpdateInputStatus(int64_t aTimestamp, bool aProcessed);
+ void UpdateOutputStatus(RefPtr<MediaData>&& aSample);
+ void ReturnDecodedData();
+ void DrainComplete();
+ void Error(const MediaResult& aError);
+ void AssertOnThread() const {
+ // mThread may not be set if Init hasn't been called first.
+ MOZ_ASSERT(!mThread || mThread->IsOnCurrentThread());
+ }
+
+ enum class State { DRAINED, DRAINABLE, DRAINING, SHUTDOWN };
+ void SetState(State aState) {
+ AssertOnThread();
+ mState = aState;
+ }
+ State GetState() const {
+ AssertOnThread();
+ return mState;
+ }
+
+ // Whether the sample will be used.
+ virtual bool IsUsefulData(const RefPtr<MediaData>& aSample) { return true; }
+
+ MediaData::Type mType;
+
+ nsAutoCString mMimeType;
+ java::sdk::MediaFormat::GlobalRef mFormat;
+
+ java::CodecProxy::GlobalRef mJavaDecoder;
+ java::CodecProxy::NativeCallbacks::GlobalRef mJavaCallbacks;
+ nsString mDrmStubId;
+
+ nsCOMPtr<nsISerialEventTarget> mThread;
+
+ // Preallocated Java object used as a reusable storage for input buffer
+ // information. Contents must be changed only on mThread.
+ java::sdk::BufferInfo::GlobalRef mInputBufferInfo;
+
+ // Session ID attached to samples. It is returned by CodecProxy::Input().
+ // Accessed on mThread only.
+ int64_t mSession;
+
+ private:
+ enum class PendingOp { INCREASE, DECREASE, CLEAR };
+ void UpdatePendingInputStatus(PendingOp aOp);
+ size_t HasPendingInputs() {
+ AssertOnThread();
+ return mNumPendingInputs > 0;
+ }
+
+ // The following members must only be accessed on mThread.
+ MozPromiseHolder<DecodePromise> mDecodePromise;
+ MozPromiseHolder<DecodePromise> mDrainPromise;
+ DecodedData mDecodedData;
+ State mState = State::DRAINED;
+ size_t mNumPendingInputs;
+};
+
+} // namespace mozilla
+
+#endif
generated by cgit v1.2.3 (git 2.39.1) at 2025-02-25 14:39:28 +0000