/* * Copyright (c) 2012, The WebRTC project authors. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * * Neither the name of Google nor the names of its contributors may * be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef WEBRTCGMPVIDEOCODEC_H_ #define WEBRTCGMPVIDEOCODEC_H_ #include #include #include "nsThreadUtils.h" #include "mozilla/Monitor.h" #include "mozilla/Mutex.h" #include "mozilla/Telemetry.h" #include "mozIGeckoMediaPluginService.h" #include "MediaConduitInterface.h" #include "AudioConduit.h" #include "PerformanceRecorder.h" #include "VideoConduit.h" #include "api/video/video_frame_type.h" #include "modules/video_coding/include/video_codec_interface.h" #include "common_video/h264/h264_bitstream_parser.h" #include "gmp-video-host.h" #include "GMPVideoDecoderProxy.h" #include "GMPVideoEncoderProxy.h" #include "jsapi/PeerConnectionImpl.h" namespace mozilla { class GmpInitDoneRunnable : public Runnable { public: explicit GmpInitDoneRunnable(std::string aPCHandle) : Runnable("GmpInitDoneRunnable"), mResult(WEBRTC_VIDEO_CODEC_OK), mPCHandle(std::move(aPCHandle)) {} NS_IMETHOD Run() override { Telemetry::Accumulate(Telemetry::WEBRTC_GMP_INIT_SUCCESS, mResult == WEBRTC_VIDEO_CODEC_OK); if (mResult == WEBRTC_VIDEO_CODEC_OK) { // Might be useful to notify the PeerConnection about successful init // someday. return NS_OK; } PeerConnectionWrapper wrapper(mPCHandle); if (wrapper.impl()) { wrapper.impl()->OnMediaError(mError); } return NS_OK; } void Dispatch(int32_t aResult, const std::string& aError = "") { mResult = aResult; mError = aError; nsCOMPtr mainThread(do_GetMainThread()); if (mainThread) { // For some reason, the compiler on CI is treating |this| as a const // pointer, despite the fact that we're in a non-const function. And, // interestingly enough, correcting this doesn't require a const_cast. mainThread->Dispatch(do_AddRef(static_cast(this)), NS_DISPATCH_NORMAL); } } int32_t Result() { return mResult; } private: int32_t mResult; const std::string mPCHandle; std::string mError; }; // Hold a frame for later decode class GMPDecodeData { public: GMPDecodeData(const webrtc::EncodedImage& aInputImage, bool aMissingFrames, int64_t aRenderTimeMs) : mImage(aInputImage), mMissingFrames(aMissingFrames), mRenderTimeMs(aRenderTimeMs) { // We want to use this for queuing, and the calling code recycles the // buffer on return from Decode() MOZ_RELEASE_ASSERT(aInputImage.size() < (std::numeric_limits::max() >> 1)); } ~GMPDecodeData() = default; const webrtc::EncodedImage mImage; const bool mMissingFrames; const int64_t mRenderTimeMs; }; class RefCountedWebrtcVideoEncoder { public: NS_INLINE_DECL_PURE_VIRTUAL_REFCOUNTING // Implement sort of WebrtcVideoEncoder interface and support refcounting. // (We cannot use |Release|, since that's needed for nsRefPtr) virtual int32_t InitEncode( const webrtc::VideoCodec* aCodecSettings, const webrtc::VideoEncoder::Settings& aSettings) = 0; virtual int32_t Encode( const webrtc::VideoFrame& aInputImage, const std::vector* aFrameTypes) = 0; virtual int32_t RegisterEncodeCompleteCallback( webrtc::EncodedImageCallback* aCallback) = 0; virtual int32_t Shutdown() = 0; virtual int32_t SetRates( const webrtc::VideoEncoder::RateControlParameters& aParameters) = 0; virtual MediaEventSource* InitPluginEvent() = 0; virtual MediaEventSource* ReleasePluginEvent() = 0; virtual WebrtcVideoEncoder::EncoderInfo GetEncoderInfo() const = 0; protected: virtual ~RefCountedWebrtcVideoEncoder() = default; }; class WebrtcGmpVideoEncoder : public GMPVideoEncoderCallbackProxy, public RefCountedWebrtcVideoEncoder { public: NS_INLINE_DECL_THREADSAFE_REFCOUNTING(WebrtcGmpVideoEncoder, final); WebrtcGmpVideoEncoder(const webrtc::SdpVideoFormat& aFormat, std::string aPCHandle); // Implement VideoEncoder interface, sort of. // (We cannot use |Release|, since that's needed for nsRefPtr) int32_t InitEncode(const webrtc::VideoCodec* aCodecSettings, const webrtc::VideoEncoder::Settings& aSettings) override; int32_t Encode( const webrtc::VideoFrame& aInputImage, const std::vector* aFrameTypes) override; int32_t RegisterEncodeCompleteCallback( webrtc::EncodedImageCallback* aCallback) override; int32_t Shutdown() override; int32_t SetRates( const webrtc::VideoEncoder::RateControlParameters& aParameters) override; WebrtcVideoEncoder::EncoderInfo GetEncoderInfo() const override; MediaEventSource* InitPluginEvent() override { return &mInitPluginEvent; } MediaEventSource* ReleasePluginEvent() override { return &mReleasePluginEvent; } // GMPVideoEncoderCallback virtual functions. virtual void Terminated() override; virtual void Encoded(GMPVideoEncodedFrame* aEncodedFrame, const nsTArray& aCodecSpecificInfo) override; virtual void Error(GMPErr aError) override {} private: virtual ~WebrtcGmpVideoEncoder(); static void InitEncode_g(const RefPtr& aThis, const GMPVideoCodec& aCodecParams, int32_t aNumberOfCores, uint32_t aMaxPayloadSize, const RefPtr& aInitDone); int32_t GmpInitDone(GMPVideoEncoderProxy* aGMP, GMPVideoHost* aHost, const GMPVideoCodec& aCodecParams, std::string* aErrorOut); int32_t GmpInitDone(GMPVideoEncoderProxy* aGMP, GMPVideoHost* aHost, std::string* aErrorOut); int32_t InitEncoderForSize(unsigned short aWidth, unsigned short aHeight, std::string* aErrorOut); static void ReleaseGmp_g(const RefPtr& aEncoder); void Close_g(); class InitDoneCallback : public GetGMPVideoEncoderCallback { public: InitDoneCallback(const RefPtr& aEncoder, const RefPtr& aInitDone, const GMPVideoCodec& aCodecParams) : mEncoder(aEncoder), mInitDone(aInitDone), mCodecParams(aCodecParams) {} virtual void Done(GMPVideoEncoderProxy* aGMP, GMPVideoHost* aHost) override { std::string errorOut; int32_t result = mEncoder->GmpInitDone(aGMP, aHost, mCodecParams, &errorOut); mInitDone->Dispatch(result, errorOut); } private: const RefPtr mEncoder; const RefPtr mInitDone; const GMPVideoCodec mCodecParams; }; static void Encode_g(const RefPtr& aEncoder, webrtc::VideoFrame aInputImage, std::vector aFrameTypes); void RegetEncoderForResolutionChange( uint32_t aWidth, uint32_t aHeight, const RefPtr& aInitDone); class InitDoneForResolutionChangeCallback : public GetGMPVideoEncoderCallback { public: InitDoneForResolutionChangeCallback( const RefPtr& aEncoder, const RefPtr& aInitDone, uint32_t aWidth, uint32_t aHeight) : mEncoder(aEncoder), mInitDone(aInitDone), mWidth(aWidth), mHeight(aHeight) {} virtual void Done(GMPVideoEncoderProxy* aGMP, GMPVideoHost* aHost) override { std::string errorOut; int32_t result = mEncoder->GmpInitDone(aGMP, aHost, &errorOut); if (result != WEBRTC_VIDEO_CODEC_OK) { mInitDone->Dispatch(result, errorOut); return; } result = mEncoder->InitEncoderForSize(mWidth, mHeight, &errorOut); mInitDone->Dispatch(result, errorOut); } private: const RefPtr mEncoder; const RefPtr mInitDone; const uint32_t mWidth; const uint32_t mHeight; }; static int32_t SetRates_g(RefPtr aThis, uint32_t aNewBitRateKbps, Maybe aFrameRate); nsCOMPtr mMPS; nsCOMPtr mGMPThread; GMPVideoEncoderProxy* mGMP; // Used to handle a race where Release() is called while init is in progress bool mInitting; GMPVideoHost* mHost; GMPVideoCodec mCodecParams; uint32_t mMaxPayloadSize; const webrtc::SdpVideoFormat::Parameters mFormatParams; webrtc::CodecSpecificInfo mCodecSpecificInfo; webrtc::H264BitstreamParser mH264BitstreamParser; // Protects mCallback Mutex mCallbackMutex MOZ_UNANNOTATED; webrtc::EncodedImageCallback* mCallback; Maybe mCachedPluginId; const std::string mPCHandle; struct InputImageData { int64_t timestamp_us; }; // Map rtp time -> input image data DataMutex> mInputImageMap; MediaEventProducer mInitPluginEvent; MediaEventProducer mReleasePluginEvent; }; // Basically a strong ref to a RefCountedWebrtcVideoEncoder, that also // translates from Release() to RefCountedWebrtcVideoEncoder::Shutdown(), // since we need RefCountedWebrtcVideoEncoder::Release() for managing the // refcount. The webrtc.org code gets one of these, so it doesn't unilaterally // delete the "real" encoder. class WebrtcVideoEncoderProxy : public WebrtcVideoEncoder { public: explicit WebrtcVideoEncoderProxy( RefPtr aEncoder) : mEncoderImpl(std::move(aEncoder)) {} virtual ~WebrtcVideoEncoderProxy() { RegisterEncodeCompleteCallback(nullptr); } MediaEventSource* InitPluginEvent() override { return mEncoderImpl->InitPluginEvent(); } MediaEventSource* ReleasePluginEvent() override { return mEncoderImpl->ReleasePluginEvent(); } int32_t InitEncode(const webrtc::VideoCodec* aCodecSettings, const WebrtcVideoEncoder::Settings& aSettings) override { return mEncoderImpl->InitEncode(aCodecSettings, aSettings); } int32_t Encode( const webrtc::VideoFrame& aInputImage, const std::vector* aFrameTypes) override { return mEncoderImpl->Encode(aInputImage, aFrameTypes); } int32_t RegisterEncodeCompleteCallback( webrtc::EncodedImageCallback* aCallback) override { return mEncoderImpl->RegisterEncodeCompleteCallback(aCallback); } int32_t Release() override { return mEncoderImpl->Shutdown(); } void SetRates(const RateControlParameters& aParameters) override { mEncoderImpl->SetRates(aParameters); } EncoderInfo GetEncoderInfo() const override { return mEncoderImpl->GetEncoderInfo(); } private: const RefPtr mEncoderImpl; }; class WebrtcGmpVideoDecoder : public GMPVideoDecoderCallbackProxy { public: WebrtcGmpVideoDecoder(std::string aPCHandle, TrackingId aTrackingId); NS_INLINE_DECL_THREADSAFE_REFCOUNTING(WebrtcGmpVideoDecoder, final); // Implement VideoEncoder interface, sort of. // (We cannot use |Release|, since that's needed for nsRefPtr) virtual bool Configure(const webrtc::VideoDecoder::Settings& settings); virtual int32_t Decode(const webrtc::EncodedImage& aInputImage, bool aMissingFrames, int64_t aRenderTimeMs); virtual int32_t RegisterDecodeCompleteCallback( webrtc::DecodedImageCallback* aCallback); virtual int32_t ReleaseGmp(); MediaEventSource* InitPluginEvent() { return &mInitPluginEvent; } MediaEventSource* ReleasePluginEvent() { return &mReleasePluginEvent; } // GMPVideoDecoderCallbackProxy virtual void Terminated() override; virtual void Decoded(GMPVideoi420Frame* aDecodedFrame) override; virtual void ReceivedDecodedReferenceFrame( const uint64_t aPictureId) override { MOZ_CRASH(); } virtual void ReceivedDecodedFrame(const uint64_t aPictureId) override { MOZ_CRASH(); } virtual void InputDataExhausted() override {} virtual void DrainComplete() override {} virtual void ResetComplete() override {} virtual void Error(GMPErr aError) override { mDecoderStatus = aError; } private: virtual ~WebrtcGmpVideoDecoder(); static void Configure_g(const RefPtr& aThis, const webrtc::VideoDecoder::Settings& settings, const RefPtr& aInitDone); int32_t GmpInitDone(GMPVideoDecoderProxy* aGMP, GMPVideoHost* aHost, std::string* aErrorOut); static void ReleaseGmp_g(const RefPtr& aDecoder); void Close_g(); class InitDoneCallback : public GetGMPVideoDecoderCallback { public: explicit InitDoneCallback(const RefPtr& aDecoder, const RefPtr& aInitDone) : mDecoder(aDecoder), mInitDone(aInitDone) {} virtual void Done(GMPVideoDecoderProxy* aGMP, GMPVideoHost* aHost) override { std::string errorOut; int32_t result = mDecoder->GmpInitDone(aGMP, aHost, &errorOut); mInitDone->Dispatch(result, errorOut); } private: const RefPtr mDecoder; const RefPtr mInitDone; }; static void Decode_g(const RefPtr& aThis, UniquePtr&& aDecodeData); nsCOMPtr mMPS; nsCOMPtr mGMPThread; GMPVideoDecoderProxy* mGMP; // Addref is held for us // Used to handle a race where Release() is called while init is in progress bool mInitting; // Frames queued for decode while mInitting is true nsTArray> mQueuedFrames; GMPVideoHost* mHost; // Protects mCallback Mutex mCallbackMutex MOZ_UNANNOTATED; webrtc::DecodedImageCallback* mCallback; Maybe mCachedPluginId; Atomic mDecoderStatus; const std::string mPCHandle; const TrackingId mTrackingId; PerformanceRecorderMulti mPerformanceRecorder; MediaEventProducer mInitPluginEvent; MediaEventProducer mReleasePluginEvent; }; // Basically a strong ref to a WebrtcGmpVideoDecoder, that also translates // from Release() to WebrtcGmpVideoDecoder::ReleaseGmp(), since we need // WebrtcGmpVideoDecoder::Release() for managing the refcount. // The webrtc.org code gets one of these, so it doesn't unilaterally delete // the "real" encoder. class WebrtcVideoDecoderProxy : public WebrtcVideoDecoder { public: explicit WebrtcVideoDecoderProxy(std::string aPCHandle, TrackingId aTrackingId) : mDecoderImpl(new WebrtcGmpVideoDecoder(std::move(aPCHandle), std::move(aTrackingId))) {} virtual ~WebrtcVideoDecoderProxy() { RegisterDecodeCompleteCallback(nullptr); } MediaEventSource* InitPluginEvent() override { return mDecoderImpl->InitPluginEvent(); } MediaEventSource* ReleasePluginEvent() override { return mDecoderImpl->ReleasePluginEvent(); } bool Configure(const Settings& settings) override { return mDecoderImpl->Configure(settings); } int32_t Decode(const webrtc::EncodedImage& aInputImage, bool aMissingFrames, int64_t aRenderTimeMs) override { return mDecoderImpl->Decode(aInputImage, aMissingFrames, aRenderTimeMs); } int32_t RegisterDecodeCompleteCallback( webrtc::DecodedImageCallback* aCallback) override { return mDecoderImpl->RegisterDecodeCompleteCallback(aCallback); } int32_t Release() override { return mDecoderImpl->ReleaseGmp(); } private: const RefPtr mDecoderImpl; }; } // namespace mozilla #endif