1 files changed, 489 insertions, 0 deletions
diff --git a/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.cpp b/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.cpp
new file mode 100644
index 0000000000..47798fafb0
--- /dev/null
+++ b/dom/media/platforms/agnostic/gmp/GMPVideoDecoder.cpp
@@ -0,0 +1,489 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et cindent: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "GMPVideoDecoder.h"
+#include "GMPDecoderModule.h"
+#include "GMPVideoHost.h"
+#include "GMPLog.h"
+#include "MediaData.h"
+#include "mozilla/EndianUtils.h"
+#include "mozilla/StaticPrefs_media.h"
+#include "nsServiceManagerUtils.h"
+#include "AnnexB.h"
+#include "H264.h"
+#include "MP4Decoder.h"
+#include "prsystem.h"
+#include "VPXDecoder.h"
+#include "VideoUtils.h"
+
+namespace mozilla {
+
+#if defined(DEBUG)
+static bool IsOnGMPThread() {
+  nsCOMPtr<mozIGeckoMediaPluginService> mps =
+      do_GetService("@mozilla.org/gecko-media-plugin-service;1");
+  MOZ_ASSERT(mps);
+
+  nsCOMPtr<nsIThread> gmpThread;
+  nsresult rv = mps->GetThread(getter_AddRefs(gmpThread));
+  MOZ_ASSERT(NS_SUCCEEDED(rv) && gmpThread);
+  return gmpThread->IsOnCurrentThread();
+}
+#endif
+
+GMPVideoDecoderParams::GMPVideoDecoderParams(const CreateDecoderParams& aParams)
+    : mConfig(aParams.VideoConfig()),
+      mImageContainer(aParams.mImageContainer),
+      mCrashHelper(aParams.mCrashHelper),
+      mKnowsCompositor(aParams.mKnowsCompositor),
+      mTrackingId(aParams.mTrackingId) {}
+
+nsCString GMPVideoDecoder::GetCodecName() const {
+  if (MP4Decoder::IsH264(mConfig.mMimeType)) {
+    return "h264"_ns;
+  } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) {
+    return "vp8"_ns;
+  } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) {
+    return "vp9"_ns;
+  }
+  return "unknown"_ns;
+}
+
+void GMPVideoDecoder::Decoded(GMPVideoi420Frame* aDecodedFrame) {
+  GMP_LOG_DEBUG("GMPVideoDecoder::Decoded");
+
+  GMPUniquePtr<GMPVideoi420Frame> decodedFrame(aDecodedFrame);
+  MOZ_ASSERT(IsOnGMPThread());
+
+  VideoData::YCbCrBuffer b;
+  for (int i = 0; i < kGMPNumOfPlanes; ++i) {
+    b.mPlanes[i].mData = decodedFrame->Buffer(GMPPlaneType(i));
+    b.mPlanes[i].mStride = decodedFrame->Stride(GMPPlaneType(i));
+    if (i == kGMPYPlane) {
+      b.mPlanes[i].mWidth = decodedFrame->Width();
+      b.mPlanes[i].mHeight = decodedFrame->Height();
+    } else {
+      b.mPlanes[i].mWidth = (decodedFrame->Width() + 1) / 2;
+      b.mPlanes[i].mHeight = (decodedFrame->Height() + 1) / 2;
+    }
+    b.mPlanes[i].mSkip = 0;
+  }
+
+  b.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT;
+  b.mYUVColorSpace =
+      DefaultColorSpace({decodedFrame->Width(), decodedFrame->Height()});
+
+  UniquePtr<SampleMetadata> sampleData;
+  if (auto entryHandle = mSamples.Lookup(decodedFrame->Timestamp())) {
+    sampleData = std::move(entryHandle.Data());
+    entryHandle.Remove();
+  } else {
+    GMP_LOG_DEBUG(
+        "GMPVideoDecoder::Decoded(this=%p) missing sample metadata for "
+        "time %" PRIu64,
+        this, decodedFrame->Timestamp());
+    if (mSamples.IsEmpty()) {
+      // If we have no remaining samples in the table, then we have processed
+      // all outstanding decode requests.
+      ProcessReorderQueue(mDecodePromise, __func__);
+    }
+    return;
+  }
+
+  MOZ_ASSERT(sampleData);
+
+  gfx::IntRect pictureRegion(0, 0, decodedFrame->Width(),
+                             decodedFrame->Height());
+  RefPtr<VideoData> v = VideoData::CreateAndCopyData(
+      mConfig, mImageContainer, sampleData->mOffset,
+      media::TimeUnit::FromMicroseconds(decodedFrame->UpdatedTimestamp()),
+      media::TimeUnit::FromMicroseconds(decodedFrame->Duration()), b,
+      sampleData->mKeyframe, media::TimeUnit::FromMicroseconds(-1),
+      pictureRegion, mKnowsCompositor);
+  RefPtr<GMPVideoDecoder> self = this;
+  if (v) {
+    mPerformanceRecorder.Record(static_cast<int64_t>(decodedFrame->Timestamp()),
+                                [&](DecodeStage& aStage) {
+                                  aStage.SetImageFormat(DecodeStage::YUV420P);
+                                  aStage.SetResolution(decodedFrame->Width(),
+                                                       decodedFrame->Height());
+                                  aStage.SetYUVColorSpace(b.mYUVColorSpace);
+                                  aStage.SetColorDepth(b.mColorDepth);
+                                  aStage.SetColorRange(b.mColorRange);
+                                });
+
+    if (mReorderFrames) {
+      mReorderQueue.Push(std::move(v));
+    } else {
+      mUnorderedData.AppendElement(std::move(v));
+    }
+
+    if (mSamples.IsEmpty()) {
+      // If we have no remaining samples in the table, then we have processed
+      // all outstanding decode requests.
+      ProcessReorderQueue(mDecodePromise, __func__);
+    }
+  } else {
+    mReorderQueue.Clear();
+    mUnorderedData.Clear();
+    mSamples.Clear();
+    mDecodePromise.RejectIfExists(
+        MediaResult(NS_ERROR_OUT_OF_MEMORY,
+                    RESULT_DETAIL("CallBack::CreateAndCopyData")),
+        __func__);
+  }
+}
+
+void GMPVideoDecoder::ReceivedDecodedReferenceFrame(const uint64_t aPictureId) {
+  GMP_LOG_DEBUG("GMPVideoDecoder::ReceivedDecodedReferenceFrame");
+  MOZ_ASSERT(IsOnGMPThread());
+}
+
+void GMPVideoDecoder::ReceivedDecodedFrame(const uint64_t aPictureId) {
+  GMP_LOG_DEBUG("GMPVideoDecoder::ReceivedDecodedFrame");
+  MOZ_ASSERT(IsOnGMPThread());
+}
+
+void GMPVideoDecoder::InputDataExhausted() {
+  GMP_LOG_DEBUG("GMPVideoDecoder::InputDataExhausted");
+  MOZ_ASSERT(IsOnGMPThread());
+  mSamples.Clear();
+  ProcessReorderQueue(mDecodePromise, __func__);
+}
+
+void GMPVideoDecoder::DrainComplete() {
+  GMP_LOG_DEBUG("GMPVideoDecoder::DrainComplete");
+  MOZ_ASSERT(IsOnGMPThread());
+  mSamples.Clear();
+
+  if (mDrainPromise.IsEmpty()) {
+    return;
+  }
+
+  DecodedData results;
+  if (mReorderFrames) {
+    results.SetCapacity(mReorderQueue.Length());
+    while (!mReorderQueue.IsEmpty()) {
+      results.AppendElement(mReorderQueue.Pop());
+    }
+  } else {
+    results = std::move(mUnorderedData);
+  }
+
+  mDrainPromise.Resolve(std::move(results), __func__);
+}
+
+void GMPVideoDecoder::ResetComplete() {
+  GMP_LOG_DEBUG("GMPVideoDecoder::ResetComplete");
+  MOZ_ASSERT(IsOnGMPThread());
+  mPerformanceRecorder.Record(std::numeric_limits<int64_t>::max());
+  mFlushPromise.ResolveIfExists(true, __func__);
+}
+
+void GMPVideoDecoder::Error(GMPErr aErr) {
+  GMP_LOG_DEBUG("GMPVideoDecoder::Error");
+  MOZ_ASSERT(IsOnGMPThread());
+  auto error = MediaResult(aErr == GMPDecodeErr ? NS_ERROR_DOM_MEDIA_DECODE_ERR
+                                                : NS_ERROR_DOM_MEDIA_FATAL_ERR,
+                           RESULT_DETAIL("GMPErr:%x", aErr));
+  mDecodePromise.RejectIfExists(error, __func__);
+  mDrainPromise.RejectIfExists(error, __func__);
+  mFlushPromise.RejectIfExists(error, __func__);
+}
+
+void GMPVideoDecoder::Terminated() {
+  GMP_LOG_DEBUG("GMPVideoDecoder::Terminated");
+  MOZ_ASSERT(IsOnGMPThread());
+  Error(GMPErr::GMPAbortedErr);
+}
+
+void GMPVideoDecoder::ProcessReorderQueue(
+    MozPromiseHolder<DecodePromise>& aPromise, const char* aMethodName) {
+  if (aPromise.IsEmpty()) {
+    return;
+  }
+
+  if (!mReorderFrames) {
+    aPromise.Resolve(std::move(mUnorderedData), aMethodName);
+    return;
+  }
+
+  DecodedData results;
+  size_t availableFrames = mReorderQueue.Length();
+  if (availableFrames > mMaxRefFrames) {
+    size_t resolvedFrames = availableFrames - mMaxRefFrames;
+    results.SetCapacity(resolvedFrames);
+    do {
+      results.AppendElement(mReorderQueue.Pop());
+    } while (--resolvedFrames > 0);
+  }
+
+  aPromise.Resolve(std::move(results), aMethodName);
+}
+
+GMPVideoDecoder::GMPVideoDecoder(const GMPVideoDecoderParams& aParams)
+    : mConfig(aParams.mConfig),
+      mGMP(nullptr),
+      mHost(nullptr),
+      mConvertNALUnitLengths(false),
+      mCrashHelper(aParams.mCrashHelper),
+      mImageContainer(aParams.mImageContainer),
+      mKnowsCompositor(aParams.mKnowsCompositor),
+      mTrackingId(aParams.mTrackingId),
+      mCanDecodeBatch(StaticPrefs::media_gmp_decoder_decode_batch()),
+      mReorderFrames(StaticPrefs::media_gmp_decoder_reorder_frames()) {}
+
+void GMPVideoDecoder::InitTags(nsTArray<nsCString>& aTags) {
+  if (MP4Decoder::IsH264(mConfig.mMimeType)) {
+    aTags.AppendElement("h264"_ns);
+  } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) {
+    aTags.AppendElement("vp8"_ns);
+  } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) {
+    aTags.AppendElement("vp9"_ns);
+  }
+}
+
+nsCString GMPVideoDecoder::GetNodeId() { return SHARED_GMP_DECODING_NODE_ID; }
+
+GMPUniquePtr<GMPVideoEncodedFrame> GMPVideoDecoder::CreateFrame(
+    MediaRawData* aSample) {
+  GMPVideoFrame* ftmp = nullptr;
+  GMPErr err = mHost->CreateFrame(kGMPEncodedVideoFrame, &ftmp);
+  if (GMP_FAILED(err)) {
+    return nullptr;
+  }
+
+  GMPUniquePtr<GMPVideoEncodedFrame> frame(
+      static_cast<GMPVideoEncodedFrame*>(ftmp));
+  err = frame->CreateEmptyFrame(aSample->Size());
+  if (GMP_FAILED(err)) {
+    return nullptr;
+  }
+
+  memcpy(frame->Buffer(), aSample->Data(), frame->Size());
+
+  // Convert 4-byte NAL unit lengths to host-endian 4-byte buffer lengths to
+  // suit the GMP API.
+  if (mConvertNALUnitLengths) {
+    const int kNALLengthSize = 4;
+    uint8_t* buf = frame->Buffer();
+    while (buf < frame->Buffer() + frame->Size() - kNALLengthSize) {
+      uint32_t length = BigEndian::readUint32(buf) + kNALLengthSize;
+      *reinterpret_cast<uint32_t*>(buf) = length;
+      buf += length;
+    }
+  }
+
+  frame->SetBufferType(GMP_BufferLength32);
+
+  frame->SetEncodedWidth(mConfig.mDisplay.width);
+  frame->SetEncodedHeight(mConfig.mDisplay.height);
+  frame->SetTimeStamp(aSample->mTime.ToMicroseconds());
+  frame->SetCompleteFrame(true);
+  frame->SetDuration(aSample->mDuration.ToMicroseconds());
+  frame->SetFrameType(aSample->mKeyframe ? kGMPKeyFrame : kGMPDeltaFrame);
+
+  return frame;
+}
+
+const VideoInfo& GMPVideoDecoder::GetConfig() const { return mConfig; }
+
+void GMPVideoDecoder::GMPInitDone(GMPVideoDecoderProxy* aGMP,
+                                  GMPVideoHost* aHost) {
+  MOZ_ASSERT(IsOnGMPThread());
+
+  if (!aGMP) {
+    mInitPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
+    return;
+  }
+  MOZ_ASSERT(aHost);
+
+  if (mInitPromise.IsEmpty()) {
+    // GMP must have been shutdown while we were waiting for Init operation
+    // to complete.
+    aGMP->Close();
+    return;
+  }
+
+  bool isOpenH264 = aGMP->GetPluginType() == GMPPluginType::OpenH264;
+
+  GMPVideoCodec codec;
+  memset(&codec, 0, sizeof(codec));
+
+  codec.mGMPApiVersion = kGMPVersion34;
+  nsTArray<uint8_t> codecSpecific;
+  if (MP4Decoder::IsH264(mConfig.mMimeType)) {
+    codec.mCodecType = kGMPVideoCodecH264;
+    codecSpecific.AppendElement(0);  // mPacketizationMode.
+    codecSpecific.AppendElements(mConfig.mExtraData->Elements(),
+                                 mConfig.mExtraData->Length());
+    // OpenH264 expects pseudo-AVCC, but others must be passed
+    // AnnexB for H264.
+    mConvertToAnnexB = !isOpenH264;
+    mMaxRefFrames = H264::ComputeMaxRefFrames(mConfig.mExtraData);
+  } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) {
+    codec.mCodecType = kGMPVideoCodecVP8;
+  } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) {
+    codec.mCodecType = kGMPVideoCodecVP9;
+  } else {
+    // Unrecognized mime type
+    aGMP->Close();
+    mInitPromise.Reject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
+    return;
+  }
+  codec.mWidth = mConfig.mImage.width;
+  codec.mHeight = mConfig.mImage.height;
+  codec.mUseThreadedDecode = StaticPrefs::media_gmp_decoder_multithreaded();
+  codec.mLogLevel = GetGMPLibraryLogLevel();
+
+  nsresult rv =
+      aGMP->InitDecode(codec, codecSpecific, this, PR_GetNumberOfProcessors());
+  if (NS_FAILED(rv)) {
+    aGMP->Close();
+    mInitPromise.Reject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
+    return;
+  }
+
+  mGMP = aGMP;
+  mHost = aHost;
+
+  // GMP implementations have interpreted the meaning of GMP_BufferLength32
+  // differently.  The OpenH264 GMP expects GMP_BufferLength32 to behave as
+  // specified in the GMP API, where each buffer is prefixed by a 32-bit
+  // host-endian buffer length that includes the size of the buffer length
+  // field.  Other existing GMPs currently expect GMP_BufferLength32 (when
+  // combined with kGMPVideoCodecH264) to mean "like AVCC but restricted to
+  // 4-byte NAL lengths" (i.e. buffer lengths are specified in big-endian
+  // and do not include the length of the buffer length field.
+  mConvertNALUnitLengths = isOpenH264;
+
+  mInitPromise.Resolve(TrackInfo::kVideoTrack, __func__);
+}
+
+RefPtr<MediaDataDecoder::InitPromise> GMPVideoDecoder::Init() {
+  MOZ_ASSERT(IsOnGMPThread());
+
+  mMPS = do_GetService("@mozilla.org/gecko-media-plugin-service;1");
+  MOZ_ASSERT(mMPS);
+
+  RefPtr<InitPromise> promise(mInitPromise.Ensure(__func__));
+
+  nsTArray<nsCString> tags;
+  InitTags(tags);
+  UniquePtr<GetGMPVideoDecoderCallback> callback(new GMPInitDoneCallback(this));
+  if (NS_FAILED(mMPS->GetGMPVideoDecoder(mCrashHelper, &tags, GetNodeId(),
+                                         std::move(callback)))) {
+    mInitPromise.Reject(NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);
+  }
+
+  return promise;
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> GMPVideoDecoder::Decode(
+    MediaRawData* aSample) {
+  MOZ_ASSERT(IsOnGMPThread());
+
+  RefPtr<MediaRawData> sample(aSample);
+  if (!mGMP) {
+    return DecodePromise::CreateAndReject(
+        MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
+                    RESULT_DETAIL("mGMP not initialized")),
+        __func__);
+  }
+
+  if (mTrackingId) {
+    MediaInfoFlag flag = MediaInfoFlag::None;
+    flag |= (aSample->mKeyframe ? MediaInfoFlag::KeyFrame
+                                : MediaInfoFlag::NonKeyFrame);
+    if (mGMP->GetPluginType() == GMPPluginType::OpenH264) {
+      flag |= MediaInfoFlag::SoftwareDecoding;
+    }
+    if (MP4Decoder::IsH264(mConfig.mMimeType)) {
+      flag |= MediaInfoFlag::VIDEO_H264;
+    } else if (VPXDecoder::IsVP8(mConfig.mMimeType)) {
+      flag |= MediaInfoFlag::VIDEO_VP8;
+    } else if (VPXDecoder::IsVP9(mConfig.mMimeType)) {
+      flag |= MediaInfoFlag::VIDEO_VP9;
+    }
+    mPerformanceRecorder.Start(aSample->mTime.ToMicroseconds(),
+                               "GMPVideoDecoder"_ns, *mTrackingId, flag);
+  }
+
+  GMPUniquePtr<GMPVideoEncodedFrame> frame = CreateFrame(sample);
+  if (!frame) {
+    return DecodePromise::CreateAndReject(
+        MediaResult(NS_ERROR_OUT_OF_MEMORY,
+                    RESULT_DETAIL("CreateFrame returned null")),
+        __func__);
+  }
+
+  uint64_t frameTimestamp = frame->TimeStamp();
+  RefPtr<DecodePromise> p = mDecodePromise.Ensure(__func__);
+  nsTArray<uint8_t> info;  // No codec specific per-frame info to pass.
+  nsresult rv = mGMP->Decode(std::move(frame), false, info, 0);
+  if (NS_FAILED(rv)) {
+    mDecodePromise.Reject(MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
+                                      RESULT_DETAIL("mGMP->Decode:%" PRIx32,
+                                                    static_cast<uint32_t>(rv))),
+                          __func__);
+  }
+
+  // If we have multiple outstanding frames, we need to track which offset
+  // belongs to which frame. During seek, it is possible to get the same frame
+  // requested twice, if the old frame is still outstanding. We will simply drop
+  // the extra decoded frame and request more input if the last outstanding.
+  mSamples.WithEntryHandle(frameTimestamp, [&](auto entryHandle) {
+    auto sampleData = MakeUnique<SampleMetadata>(sample);
+    entryHandle.InsertOrUpdate(std::move(sampleData));
+  });
+
+  return p;
+}
+
+RefPtr<MediaDataDecoder::FlushPromise> GMPVideoDecoder::Flush() {
+  MOZ_ASSERT(IsOnGMPThread());
+
+  mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+  mDrainPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+
+  RefPtr<FlushPromise> p = mFlushPromise.Ensure(__func__);
+  if (!mGMP || NS_FAILED(mGMP->Reset())) {
+    // Abort the flush.
+    mPerformanceRecorder.Record(std::numeric_limits<int64_t>::max());
+    mFlushPromise.Resolve(true, __func__);
+  }
+  return p;
+}
+
+RefPtr<MediaDataDecoder::DecodePromise> GMPVideoDecoder::Drain() {
+  MOZ_ASSERT(IsOnGMPThread());
+
+  MOZ_ASSERT(mDecodePromise.IsEmpty(), "Must wait for decoding to complete");
+
+  RefPtr<DecodePromise> p = mDrainPromise.Ensure(__func__);
+  if (!mGMP || NS_FAILED(mGMP->Drain())) {
+    mDrainPromise.Resolve(DecodedData(), __func__);
+  }
+
+  return p;
+}
+
+RefPtr<ShutdownPromise> GMPVideoDecoder::Shutdown() {
+  mInitPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+  mFlushPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
+
+  // Note that this *may* be called from the proxy thread also.
+  // TODO: If that's the case, then this code is racy.
+  if (!mGMP) {
+    return ShutdownPromise::CreateAndResolve(true, __func__);
+  }
+  // Note this unblocks flush and drain operations waiting for callbacks.
+  mGMP->Close();
+  mGMP = nullptr;
+  return ShutdownPromise::CreateAndResolve(true, __func__);
+}
+
+}  // namespace mozilla