From 6bf0a5cb5034a7e684dcc3500e841785237ce2dd Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 19:32:43 +0200 Subject: Adding upstream version 1:115.7.0. Signed-off-by: Daniel Baumann --- .../webrtc/libwebrtcglue/WebrtcGmpVideoCodec.h | 507 +++++++++++++++++++++ 1 file changed, 507 insertions(+) create mode 100644 dom/media/webrtc/libwebrtcglue/WebrtcGmpVideoCodec.h (limited to 'dom/media/webrtc/libwebrtcglue/WebrtcGmpVideoCodec.h') diff --git a/dom/media/webrtc/libwebrtcglue/WebrtcGmpVideoCodec.h b/dom/media/webrtc/libwebrtcglue/WebrtcGmpVideoCodec.h new file mode 100644 index 0000000000..ef3e1bd5af --- /dev/null +++ b/dom/media/webrtc/libwebrtcglue/WebrtcGmpVideoCodec.h @@ -0,0 +1,507 @@ +/* + * Copyright (c) 2012, The WebRTC project authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of Google nor the names of its contributors may + * be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef WEBRTCGMPVIDEOCODEC_H_ +#define WEBRTCGMPVIDEOCODEC_H_ + +#include +#include + +#include "nsThreadUtils.h" +#include "mozilla/Monitor.h" +#include "mozilla/Mutex.h" +#include "mozilla/Telemetry.h" + +#include "mozIGeckoMediaPluginService.h" +#include "MediaConduitInterface.h" +#include "AudioConduit.h" +#include "PerformanceRecorder.h" +#include "VideoConduit.h" +#include "api/video/video_frame_type.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "common_video/h264/h264_bitstream_parser.h" + +#include "gmp-video-host.h" +#include "GMPVideoDecoderProxy.h" +#include "GMPVideoEncoderProxy.h" + +#include "jsapi/PeerConnectionImpl.h" + +namespace mozilla { + +class GmpInitDoneRunnable : public Runnable { + public: + explicit GmpInitDoneRunnable(std::string aPCHandle) + : Runnable("GmpInitDoneRunnable"), + mResult(WEBRTC_VIDEO_CODEC_OK), + mPCHandle(std::move(aPCHandle)) {} + + NS_IMETHOD Run() override { + Telemetry::Accumulate(Telemetry::WEBRTC_GMP_INIT_SUCCESS, + mResult == WEBRTC_VIDEO_CODEC_OK); + if (mResult == WEBRTC_VIDEO_CODEC_OK) { + // Might be useful to notify the PeerConnection about successful init + // someday. + return NS_OK; + } + + PeerConnectionWrapper wrapper(mPCHandle); + if (wrapper.impl()) { + wrapper.impl()->OnMediaError(mError); + } + return NS_OK; + } + + void Dispatch(int32_t aResult, const std::string& aError = "") { + mResult = aResult; + mError = aError; + nsCOMPtr mainThread(do_GetMainThread()); + if (mainThread) { + // For some reason, the compiler on CI is treating |this| as a const + // pointer, despite the fact that we're in a non-const function. And, + // interestingly enough, correcting this doesn't require a const_cast. + mainThread->Dispatch(do_AddRef(static_cast(this)), + NS_DISPATCH_NORMAL); + } + } + + int32_t Result() { return mResult; } + + private: + int32_t mResult; + const std::string mPCHandle; + std::string mError; +}; + +// Hold a frame for later decode +class GMPDecodeData { + public: + GMPDecodeData(const webrtc::EncodedImage& aInputImage, bool aMissingFrames, + int64_t aRenderTimeMs) + : mImage(aInputImage), + mMissingFrames(aMissingFrames), + mRenderTimeMs(aRenderTimeMs) { + // We want to use this for queuing, and the calling code recycles the + // buffer on return from Decode() + MOZ_RELEASE_ASSERT(aInputImage.size() < + (std::numeric_limits::max() >> 1)); + } + + ~GMPDecodeData() = default; + + const webrtc::EncodedImage mImage; + const bool mMissingFrames; + const int64_t mRenderTimeMs; +}; + +class RefCountedWebrtcVideoEncoder { + public: + NS_INLINE_DECL_PURE_VIRTUAL_REFCOUNTING + + // Implement sort of WebrtcVideoEncoder interface and support refcounting. + // (We cannot use |Release|, since that's needed for nsRefPtr) + virtual int32_t InitEncode( + const webrtc::VideoCodec* aCodecSettings, + const webrtc::VideoEncoder::Settings& aSettings) = 0; + + virtual int32_t Encode( + const webrtc::VideoFrame& aInputImage, + const std::vector* aFrameTypes) = 0; + + virtual int32_t RegisterEncodeCompleteCallback( + webrtc::EncodedImageCallback* aCallback) = 0; + + virtual int32_t Shutdown() = 0; + + virtual int32_t SetRates( + const webrtc::VideoEncoder::RateControlParameters& aParameters) = 0; + + virtual MediaEventSource* InitPluginEvent() = 0; + + virtual MediaEventSource* ReleasePluginEvent() = 0; + + virtual WebrtcVideoEncoder::EncoderInfo GetEncoderInfo() const = 0; + + protected: + virtual ~RefCountedWebrtcVideoEncoder() = default; +}; + +class WebrtcGmpVideoEncoder : public GMPVideoEncoderCallbackProxy, + public RefCountedWebrtcVideoEncoder { + public: + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(WebrtcGmpVideoEncoder, final); + + WebrtcGmpVideoEncoder(const webrtc::SdpVideoFormat& aFormat, + std::string aPCHandle); + + // Implement VideoEncoder interface, sort of. + // (We cannot use |Release|, since that's needed for nsRefPtr) + int32_t InitEncode(const webrtc::VideoCodec* aCodecSettings, + const webrtc::VideoEncoder::Settings& aSettings) override; + + int32_t Encode( + const webrtc::VideoFrame& aInputImage, + const std::vector* aFrameTypes) override; + + int32_t RegisterEncodeCompleteCallback( + webrtc::EncodedImageCallback* aCallback) override; + + int32_t Shutdown() override; + + int32_t SetRates( + const webrtc::VideoEncoder::RateControlParameters& aParameters) override; + + WebrtcVideoEncoder::EncoderInfo GetEncoderInfo() const override; + + MediaEventSource* InitPluginEvent() override { + return &mInitPluginEvent; + } + + MediaEventSource* ReleasePluginEvent() override { + return &mReleasePluginEvent; + } + + // GMPVideoEncoderCallback virtual functions. + virtual void Terminated() override; + + virtual void Encoded(GMPVideoEncodedFrame* aEncodedFrame, + const nsTArray& aCodecSpecificInfo) override; + + virtual void Error(GMPErr aError) override {} + + private: + virtual ~WebrtcGmpVideoEncoder(); + + static void InitEncode_g(const RefPtr& aThis, + const GMPVideoCodec& aCodecParams, + int32_t aNumberOfCores, uint32_t aMaxPayloadSize, + const RefPtr& aInitDone); + int32_t GmpInitDone(GMPVideoEncoderProxy* aGMP, GMPVideoHost* aHost, + const GMPVideoCodec& aCodecParams, + std::string* aErrorOut); + int32_t GmpInitDone(GMPVideoEncoderProxy* aGMP, GMPVideoHost* aHost, + std::string* aErrorOut); + int32_t InitEncoderForSize(unsigned short aWidth, unsigned short aHeight, + std::string* aErrorOut); + static void ReleaseGmp_g(const RefPtr& aEncoder); + void Close_g(); + + class InitDoneCallback : public GetGMPVideoEncoderCallback { + public: + InitDoneCallback(const RefPtr& aEncoder, + const RefPtr& aInitDone, + const GMPVideoCodec& aCodecParams) + : mEncoder(aEncoder), + mInitDone(aInitDone), + mCodecParams(aCodecParams) {} + + virtual void Done(GMPVideoEncoderProxy* aGMP, + GMPVideoHost* aHost) override { + std::string errorOut; + int32_t result = + mEncoder->GmpInitDone(aGMP, aHost, mCodecParams, &errorOut); + + mInitDone->Dispatch(result, errorOut); + } + + private: + const RefPtr mEncoder; + const RefPtr mInitDone; + const GMPVideoCodec mCodecParams; + }; + + static void Encode_g(const RefPtr& aEncoder, + webrtc::VideoFrame aInputImage, + std::vector aFrameTypes); + void RegetEncoderForResolutionChange( + uint32_t aWidth, uint32_t aHeight, + const RefPtr& aInitDone); + + class InitDoneForResolutionChangeCallback + : public GetGMPVideoEncoderCallback { + public: + InitDoneForResolutionChangeCallback( + const RefPtr& aEncoder, + const RefPtr& aInitDone, uint32_t aWidth, + uint32_t aHeight) + : mEncoder(aEncoder), + mInitDone(aInitDone), + mWidth(aWidth), + mHeight(aHeight) {} + + virtual void Done(GMPVideoEncoderProxy* aGMP, + GMPVideoHost* aHost) override { + std::string errorOut; + int32_t result = mEncoder->GmpInitDone(aGMP, aHost, &errorOut); + if (result != WEBRTC_VIDEO_CODEC_OK) { + mInitDone->Dispatch(result, errorOut); + return; + } + + result = mEncoder->InitEncoderForSize(mWidth, mHeight, &errorOut); + mInitDone->Dispatch(result, errorOut); + } + + private: + const RefPtr mEncoder; + const RefPtr mInitDone; + const uint32_t mWidth; + const uint32_t mHeight; + }; + + static int32_t SetRates_g(RefPtr aThis, + uint32_t aNewBitRateKbps, Maybe aFrameRate); + + nsCOMPtr mMPS; + nsCOMPtr mGMPThread; + GMPVideoEncoderProxy* mGMP; + // Used to handle a race where Release() is called while init is in progress + bool mInitting; + GMPVideoHost* mHost; + GMPVideoCodec mCodecParams; + uint32_t mMaxPayloadSize; + const webrtc::SdpVideoFormat::Parameters mFormatParams; + webrtc::CodecSpecificInfo mCodecSpecificInfo; + webrtc::H264BitstreamParser mH264BitstreamParser; + // Protects mCallback + Mutex mCallbackMutex MOZ_UNANNOTATED; + webrtc::EncodedImageCallback* mCallback; + Maybe mCachedPluginId; + const std::string mPCHandle; + + struct InputImageData { + int64_t timestamp_us; + }; + // Map rtp time -> input image data + DataMutex> mInputImageMap; + + MediaEventProducer mInitPluginEvent; + MediaEventProducer mReleasePluginEvent; +}; + +// Basically a strong ref to a RefCountedWebrtcVideoEncoder, that also +// translates from Release() to RefCountedWebrtcVideoEncoder::Shutdown(), +// since we need RefCountedWebrtcVideoEncoder::Release() for managing the +// refcount. The webrtc.org code gets one of these, so it doesn't unilaterally +// delete the "real" encoder. +class WebrtcVideoEncoderProxy : public WebrtcVideoEncoder { + public: + explicit WebrtcVideoEncoderProxy( + RefPtr aEncoder) + : mEncoderImpl(std::move(aEncoder)) {} + + virtual ~WebrtcVideoEncoderProxy() { + RegisterEncodeCompleteCallback(nullptr); + } + + MediaEventSource* InitPluginEvent() override { + return mEncoderImpl->InitPluginEvent(); + } + + MediaEventSource* ReleasePluginEvent() override { + return mEncoderImpl->ReleasePluginEvent(); + } + + int32_t InitEncode(const webrtc::VideoCodec* aCodecSettings, + const WebrtcVideoEncoder::Settings& aSettings) override { + return mEncoderImpl->InitEncode(aCodecSettings, aSettings); + } + + int32_t Encode( + const webrtc::VideoFrame& aInputImage, + const std::vector* aFrameTypes) override { + return mEncoderImpl->Encode(aInputImage, aFrameTypes); + } + + int32_t RegisterEncodeCompleteCallback( + webrtc::EncodedImageCallback* aCallback) override { + return mEncoderImpl->RegisterEncodeCompleteCallback(aCallback); + } + + int32_t Release() override { return mEncoderImpl->Shutdown(); } + + void SetRates(const RateControlParameters& aParameters) override { + mEncoderImpl->SetRates(aParameters); + } + + EncoderInfo GetEncoderInfo() const override { + return mEncoderImpl->GetEncoderInfo(); + } + + private: + const RefPtr mEncoderImpl; +}; + +class WebrtcGmpVideoDecoder : public GMPVideoDecoderCallbackProxy { + public: + WebrtcGmpVideoDecoder(std::string aPCHandle, TrackingId aTrackingId); + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(WebrtcGmpVideoDecoder, final); + + // Implement VideoEncoder interface, sort of. + // (We cannot use |Release|, since that's needed for nsRefPtr) + virtual bool Configure(const webrtc::VideoDecoder::Settings& settings); + virtual int32_t Decode(const webrtc::EncodedImage& aInputImage, + bool aMissingFrames, int64_t aRenderTimeMs); + virtual int32_t RegisterDecodeCompleteCallback( + webrtc::DecodedImageCallback* aCallback); + + virtual int32_t ReleaseGmp(); + + MediaEventSource* InitPluginEvent() { return &mInitPluginEvent; } + + MediaEventSource* ReleasePluginEvent() { + return &mReleasePluginEvent; + } + + // GMPVideoDecoderCallbackProxy + virtual void Terminated() override; + + virtual void Decoded(GMPVideoi420Frame* aDecodedFrame) override; + + virtual void ReceivedDecodedReferenceFrame( + const uint64_t aPictureId) override { + MOZ_CRASH(); + } + + virtual void ReceivedDecodedFrame(const uint64_t aPictureId) override { + MOZ_CRASH(); + } + + virtual void InputDataExhausted() override {} + + virtual void DrainComplete() override {} + + virtual void ResetComplete() override {} + + virtual void Error(GMPErr aError) override { mDecoderStatus = aError; } + + private: + virtual ~WebrtcGmpVideoDecoder(); + + static void Configure_g(const RefPtr& aThis, + const webrtc::VideoDecoder::Settings& settings, + const RefPtr& aInitDone); + int32_t GmpInitDone(GMPVideoDecoderProxy* aGMP, GMPVideoHost* aHost, + std::string* aErrorOut); + static void ReleaseGmp_g(const RefPtr& aDecoder); + void Close_g(); + + class InitDoneCallback : public GetGMPVideoDecoderCallback { + public: + explicit InitDoneCallback(const RefPtr& aDecoder, + const RefPtr& aInitDone) + : mDecoder(aDecoder), mInitDone(aInitDone) {} + + virtual void Done(GMPVideoDecoderProxy* aGMP, + GMPVideoHost* aHost) override { + std::string errorOut; + int32_t result = mDecoder->GmpInitDone(aGMP, aHost, &errorOut); + + mInitDone->Dispatch(result, errorOut); + } + + private: + const RefPtr mDecoder; + const RefPtr mInitDone; + }; + + static void Decode_g(const RefPtr& aThis, + UniquePtr&& aDecodeData); + + nsCOMPtr mMPS; + nsCOMPtr mGMPThread; + GMPVideoDecoderProxy* mGMP; // Addref is held for us + // Used to handle a race where Release() is called while init is in progress + bool mInitting; + // Frames queued for decode while mInitting is true + nsTArray> mQueuedFrames; + GMPVideoHost* mHost; + // Protects mCallback + Mutex mCallbackMutex MOZ_UNANNOTATED; + webrtc::DecodedImageCallback* mCallback; + Maybe mCachedPluginId; + Atomic mDecoderStatus; + const std::string mPCHandle; + const TrackingId mTrackingId; + PerformanceRecorderMulti mPerformanceRecorder; + + MediaEventProducer mInitPluginEvent; + MediaEventProducer mReleasePluginEvent; +}; + +// Basically a strong ref to a WebrtcGmpVideoDecoder, that also translates +// from Release() to WebrtcGmpVideoDecoder::ReleaseGmp(), since we need +// WebrtcGmpVideoDecoder::Release() for managing the refcount. +// The webrtc.org code gets one of these, so it doesn't unilaterally delete +// the "real" encoder. +class WebrtcVideoDecoderProxy : public WebrtcVideoDecoder { + public: + explicit WebrtcVideoDecoderProxy(std::string aPCHandle, + TrackingId aTrackingId) + : mDecoderImpl(new WebrtcGmpVideoDecoder(std::move(aPCHandle), + std::move(aTrackingId))) {} + + virtual ~WebrtcVideoDecoderProxy() { + RegisterDecodeCompleteCallback(nullptr); + } + + MediaEventSource* InitPluginEvent() override { + return mDecoderImpl->InitPluginEvent(); + } + + MediaEventSource* ReleasePluginEvent() override { + return mDecoderImpl->ReleasePluginEvent(); + } + + bool Configure(const Settings& settings) override { + return mDecoderImpl->Configure(settings); + } + + int32_t Decode(const webrtc::EncodedImage& aInputImage, bool aMissingFrames, + int64_t aRenderTimeMs) override { + return mDecoderImpl->Decode(aInputImage, aMissingFrames, aRenderTimeMs); + } + + int32_t RegisterDecodeCompleteCallback( + webrtc::DecodedImageCallback* aCallback) override { + return mDecoderImpl->RegisterDecodeCompleteCallback(aCallback); + } + + int32_t Release() override { return mDecoderImpl->ReleaseGmp(); } + + private: + const RefPtr mDecoderImpl; +}; + +} // namespace mozilla + +#endif -- cgit v1.2.3