From 6bf0a5cb5034a7e684dcc3500e841785237ce2dd Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 19:32:43 +0200 Subject: Adding upstream version 1:115.7.0. Signed-off-by: Daniel Baumann --- dom/media/webrtc/libwebrtcglue/VideoConduit.h | 494 ++++++++++++++++++++++++++ 1 file changed, 494 insertions(+) create mode 100644 dom/media/webrtc/libwebrtcglue/VideoConduit.h (limited to 'dom/media/webrtc/libwebrtcglue/VideoConduit.h') diff --git a/dom/media/webrtc/libwebrtcglue/VideoConduit.h b/dom/media/webrtc/libwebrtcglue/VideoConduit.h new file mode 100644 index 0000000000..f46f1f962c --- /dev/null +++ b/dom/media/webrtc/libwebrtcglue/VideoConduit.h @@ -0,0 +1,494 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef VIDEO_SESSION_H_ +#define VIDEO_SESSION_H_ + +#include "mozilla/Atomics.h" +#include "mozilla/Attributes.h" +#include "mozilla/DataMutex.h" +#include "mozilla/ReentrantMonitor.h" +#include "mozilla/SharedThreadPool.h" +#include "mozilla/StateMirroring.h" +#include "mozilla/UniquePtr.h" +#include "nsITimer.h" + +#include "MediaConduitInterface.h" +#include "common/MediaEngineWrapper.h" +#include "RtpRtcpConfig.h" +#include "RunningStat.h" +#include "transport/runnable_utils.h" + +// conflicts with #include of scoped_ptr.h +#undef FF +// Video Engine Includes +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/sdp_video_format.h" +#include "call/call_basic_stats.h" +#include "common_video/include/video_frame_buffer_pool.h" +#include "media/base/video_broadcaster.h" +#include +#include +/** This file hosts several structures identifying different aspects + * of a RTP Session. + */ + +namespace mozilla { + +// Convert (SI) kilobits/sec to (SI) bits/sec +#define KBPS(kbps) kbps * 1000 + +const int kViEMinCodecBitrate_bps = KBPS(30); +const unsigned int kVideoMtu = 1200; +const int kQpMax = 56; + +template +T MinIgnoreZero(const T& a, const T& b) { + return std::min(a ? a : b, b ? b : a); +} + +class VideoStreamFactory; +class WebrtcAudioConduit; + +// Interface of external video encoder for WebRTC. +class WebrtcVideoEncoder : public VideoEncoder, public webrtc::VideoEncoder {}; + +// Interface of external video decoder for WebRTC. +class WebrtcVideoDecoder : public VideoDecoder, public webrtc::VideoDecoder {}; + +/** + * Concrete class for Video session. Hooks up + * - media-source and target to external transport + */ +class WebrtcVideoConduit + : public VideoSessionConduit, + public webrtc::RtcpEventObserver, + public rtc::VideoSinkInterface, + public rtc::VideoSourceInterface { + public: + // Returns true when both encoder and decoder are HW accelerated. + static bool HasH264Hardware(); + + Maybe ActiveSendPayloadType() const override; + Maybe ActiveRecvPayloadType() const override; + + /** + * Function to attach Renderer end-point for the Media-Video conduit. + * @param aRenderer : Reference to the concrete mozilla Video renderer + * implementation Note: Multiple invocations of this API shall remove an + * existing renderer and attaches the new to the Conduit. + */ + MediaConduitErrorCode AttachRenderer( + RefPtr aVideoRenderer) override; + void DetachRenderer() override; + + Maybe RtpSendBaseSeqFor(uint32_t aSsrc) const override; + + const dom::RTCStatsTimestampMaker& GetTimestampMaker() const override; + + void StopTransmitting(); + void StartTransmitting(); + void StopReceiving(); + void StartReceiving(); + + /** + * Function to deliver a capture video frame for encoding and transport. + * If the frame's timestamp is 0, it will be automatically generated. + * + * NOTE: ConfigureSendMediaCodec() must be called before this function can + * be invoked. This ensures the inserted video-frames can be + * transmitted by the conduit. + */ + MediaConduitErrorCode SendVideoFrame(webrtc::VideoFrame aFrame) override; + + bool SendRtp(const uint8_t* aData, size_t aLength, + const webrtc::PacketOptions& aOptions) override; + bool SendSenderRtcp(const uint8_t* aData, size_t aLength) override; + bool SendReceiverRtcp(const uint8_t* aData, size_t aLength) override; + + /* + * webrtc:VideoSinkInterface implementation + * ------------------------------- + */ + void OnFrame(const webrtc::VideoFrame& frame) override; + + /* + * webrtc:VideoSourceInterface implementation + * ------------------------------- + */ + void AddOrUpdateSink(rtc::VideoSinkInterface* sink, + const rtc::VideoSinkWants& wants) override; + void RemoveSink(rtc::VideoSinkInterface* sink) override; + + bool HasCodecPluginID(uint64_t aPluginID) const override; + + RefPtr Shutdown() override; + + bool Denoising() const { return mDenoising; } + + uint8_t SpatialLayers() const { return mSpatialLayers; } + + uint8_t TemporalLayers() const { return mTemporalLayers; } + + webrtc::VideoCodecMode CodecMode() const; + + WebrtcVideoConduit(RefPtr aCall, + nsCOMPtr aStsThread, + Options aOptions, std::string aPCHandle, + const TrackingId& aRecvTrackingId); + virtual ~WebrtcVideoConduit(); + + // Call thread. + void InitControl(VideoConduitControlInterface* aControl) override; + + // Called when a parameter in mControl has changed. Call thread. + void OnControlConfigChange(); + + // Necessary Init steps on main thread. + MediaConduitErrorCode Init(); + + Ssrcs GetLocalSSRCs() const override; + Maybe GetAssociatedLocalRtxSSRC(Ssrc aSsrc) const override; + Maybe GetRemoteSSRC() const override; + + // Call thread. + void UnsetRemoteSSRC(uint32_t aSsrc) override; + + static unsigned ToLibwebrtcMaxFramerate(const Maybe& aMaxFramerate); + + private: + void NotifyUnsetCurrentRemoteSSRC(); + void SetRemoteSSRCConfig(uint32_t aSsrc, uint32_t aRtxSsrc); + void SetRemoteSSRCAndRestartAsNeeded(uint32_t aSsrc, uint32_t aRtxSsrc); + rtc::RefCountedObject* + CreateVideoStreamFactory(); + + public: + // Creating a recv stream or a send stream requires a local ssrc to be + // configured. This method will generate one if needed. + void EnsureLocalSSRC(); + // Creating a recv stream requires a remote ssrc to be configured. This method + // will generate one if needed. + void EnsureRemoteSSRC(); + + Maybe GetReceiverStats() + const override; + Maybe GetSenderStats() const override; + Maybe GetCallStats() const override; + + bool AddFrameHistory(dom::Sequence* + outHistories) const override; + + void SetJitterBufferTarget(DOMHighResTimeStamp aTargetMs) override; + + uint64_t MozVideoLatencyAvg(); + + void DisableSsrcChanges() override { + MOZ_ASSERT(mCallThread->IsOnCurrentThread()); + mAllowSsrcChange = false; + } + + void CollectTelemetryData() override; + + void OnRtpReceived(webrtc::RtpPacketReceived&& aPacket, + webrtc::RTPHeader&& aHeader); + void OnRtcpReceived(MediaPacket&& aPacket); + + void OnRtcpBye() override; + void OnRtcpTimeout() override; + + void SetTransportActive(bool aActive) override; + + MediaEventSourceExc& SenderRtpSendEvent() override { + return mSenderRtpSendEvent; + } + MediaEventSourceExc& SenderRtcpSendEvent() override { + return mSenderRtcpSendEvent; + } + MediaEventSourceExc& ReceiverRtcpSendEvent() override { + return mReceiverRtcpSendEvent; + } + void ConnectReceiverRtpEvent( + MediaEventSourceExc& aEvent) + override { + mReceiverRtpEventListener = + aEvent.Connect(mCallThread, this, &WebrtcVideoConduit::OnRtpReceived); + } + void ConnectReceiverRtcpEvent( + MediaEventSourceExc& aEvent) override { + mReceiverRtcpEventListener = + aEvent.Connect(mCallThread, this, &WebrtcVideoConduit::OnRtcpReceived); + } + void ConnectSenderRtcpEvent( + MediaEventSourceExc& aEvent) override { + mSenderRtcpEventListener = + aEvent.Connect(mCallThread, this, &WebrtcVideoConduit::OnRtcpReceived); + } + + std::vector GetUpstreamRtpSources() const override; + + private: + // Don't allow copying/assigning. + WebrtcVideoConduit(const WebrtcVideoConduit&) = delete; + void operator=(const WebrtcVideoConduit&) = delete; + + // Utility function to dump recv codec database + void DumpCodecDB() const; + + // Video Latency Test averaging filter + void VideoLatencyUpdate(uint64_t aNewSample); + + void CreateSendStream(); + void DeleteSendStream(); + void CreateRecvStream(); + void DeleteRecvStream(); + + void DeliverPacket(rtc::CopyOnWriteBuffer packet, PacketType type) override; + + MediaEventSource& RtcpByeEvent() override { return mRtcpByeEvent; } + MediaEventSource& RtcpTimeoutEvent() override { + return mRtcpTimeoutEvent; + } + MediaEventSource& RtpPacketEvent() override { return mRtpPacketEvent; } + + bool RequiresNewSendStream(const VideoCodecConfig& newConfig) const; + + mutable mozilla::ReentrantMonitor mRendererMonitor MOZ_UNANNOTATED; + + // Accessed on any thread under mRendererMonitor. + RefPtr mRenderer; + + // Accessed on any thread under mRendererMonitor. + unsigned short mReceivingWidth = 0; + + // Accessed on any thread under mRendererMonitor. + unsigned short mReceivingHeight = 0; + + // Call worker thread. All access to mCall->Call() happens here. + const nsCOMPtr mCallThread; + + // Socket transport service thread that runs stats queries against us. Any + // thread. + const nsCOMPtr mStsThread; + + // Thread on which we are fed video frames. Set lazily on first call to + // SendVideoFrame(). + nsCOMPtr mFrameSendingThread; + + struct Control { + // Mirrors that map to VideoConduitControlInterface for control. Call thread + // only. + Mirror mReceiving; + Mirror mTransmitting; + Mirror mLocalSsrcs; + Mirror mLocalRtxSsrcs; + Mirror mLocalCname; + Mirror mMid; + Mirror mRemoteSsrc; + Mirror mRemoteRtxSsrc; + Mirror mSyncGroup; + Mirror mLocalRecvRtpExtensions; + Mirror mLocalSendRtpExtensions; + Mirror> mSendCodec; + Mirror> mSendRtpRtcpConfig; + Mirror> mRecvCodecs; + Mirror> mRecvRtpRtcpConfig; + Mirror mCodecMode; + + // For caching mRemoteSsrc and mRemoteRtxSsrc, since another caller may + // change the remote ssrc in the stream config directly. + Ssrc mConfiguredRemoteSsrc = 0; + Ssrc mConfiguredRemoteRtxSsrc = 0; + // For tracking changes to mSendCodec and mSendRtpRtcpConfig. + Maybe mConfiguredSendCodec; + Maybe mConfiguredSendRtpRtcpConfig; + // For tracking changes to mRecvCodecs and mRecvRtpRtcpConfig. + std::vector mConfiguredRecvCodecs; + Maybe mConfiguredRecvRtpRtcpConfig; + + Control() = delete; + explicit Control(const RefPtr& aCallThread); + } mControl; + + // WatchManager allowing Mirrors and other watch targets to trigger functions + // that will update the webrtc.org configuration. + WatchManager mWatchManager; + + mutable Mutex mMutex MOZ_UNANNOTATED; + + // Decoder factory used by mRecvStream when it needs new decoders. This is + // not shared broader like some state in the WebrtcCallWrapper because it + // handles CodecPluginID plumbing tied to this VideoConduit. + const UniquePtr mDecoderFactory; + + // Encoder factory used by mSendStream when it needs new encoders. This is + // not shared broader like some state in the WebrtcCallWrapper because it + // handles CodecPluginID plumbing tied to this VideoConduit. + const UniquePtr mEncoderFactory; + + // Our own record of the sinks added to mVideoBroadcaster so we can support + // dispatching updates to sinks from off-Call-thread. Call thread only. + AutoTArray*, 1> mRegisteredSinks; + + // Broadcaster that distributes our frames to all registered sinks. + // Threadsafe. + rtc::VideoBroadcaster mVideoBroadcaster; + + // Buffer pool used for scaling frames. + // Accessed on the frame-feeding thread only. + webrtc::VideoFrameBufferPool mBufferPool; + + // Engine state we are concerned with. Written on the Call thread and read + // anywhere. + mozilla::Atomic + mEngineTransmitting; // If true ==> Transmit Subsystem is up and running + mozilla::Atomic + mEngineReceiving; // if true ==> Receive Subsystem up and running + + // Written only on the Call thread. Guarded by mMutex, except for reads on the + // Call thread. + Maybe mCurSendCodecConfig; + + // Bookkeeping of stats for telemetry. Call thread only. + RunningStat mSendFramerate; + RunningStat mSendBitrate; + RunningStat mRecvFramerate; + RunningStat mRecvBitrate; + + // Must call webrtc::Call::DestroyVideoReceive/SendStream to delete this. + // Written only on the Call thread. Guarded by mMutex, except for reads on the + // Call thread. + webrtc::VideoReceiveStreamInterface* mRecvStream = nullptr; + + // Must call webrtc::Call::DestroyVideoReceive/SendStream to delete this. + webrtc::VideoSendStream* mSendStream = nullptr; + + // Written on the frame feeding thread. + // Guarded by mMutex, except for reads on the frame feeding thread. + unsigned short mLastWidth = 0; + + // Written on the frame feeding thread. + // Guarded by mMutex, except for reads on the frame feeding thread. + unsigned short mLastHeight = 0; + + // Written on the frame feeding thread, the timestamp of the last frame on the + // send side, in microseconds. This is a local timestamp using the system + // clock with a unspecified epoch (Like mozilla::TimeStamp). + // Guarded by mMutex. + Maybe mLastTimestampSendUs; + + // Written on the frame receive thread, the rtp timestamp of the last frame + // on the receive side, in 90kHz base. This comes from the RTP packet. + // Guarded by mMutex. + Maybe mLastRTPTimestampReceive; + + // Accessed from any thread under mRendererMonitor. + uint64_t mVideoLatencyAvg = 0; + + const bool mVideoLatencyTestEnable; + + // All in bps. + const int mMinBitrate; + const int mStartBitrate; + const int mPrefMaxBitrate; + const int mMinBitrateEstimate; + + // Max bitrate in bps as provided by negotiation. Call thread only. + int mNegotiatedMaxBitrate = 0; + + // Set to true to force denoising on. + const bool mDenoising; + + // Set to true to ignore sink wants (scaling due to bwe and cpu usage). + const bool mLockScaling; + + const uint8_t mSpatialLayers; + const uint8_t mTemporalLayers; + + static const unsigned int sAlphaNum = 7; + static const unsigned int sAlphaDen = 8; + static const unsigned int sRoundingPadding = 1024; + + // Target jitter buffer to be applied to the receive stream in milliseconds. + uint16_t mJitterBufferTargetMs = 0; + + // WEBRTC.ORG Call API + // Const so can be accessed on any thread. All methods are called on the Call + // thread. + const RefPtr mCall; + + // Set up in the ctor and then not touched. Called through by the streams on + // any thread. Safe since we own and control the lifetime of the streams. + WebrtcSendTransport mSendTransport; + WebrtcReceiveTransport mRecvTransport; + + // Written only on the Call thread. Guarded by mMutex, except for reads on the + // Call thread. Typical non-Call thread access is on the frame delivery + // thread. + webrtc::VideoSendStream::Config mSendStreamConfig; + + // Call thread only. + webrtc::VideoEncoderConfig mEncoderConfig; + + // Written only on the Call thread. Guarded by mMutex, except for reads on the + // Call thread. Calls can happen under mMutex on any thread. + DataMutex>> + mVideoStreamFactory; + + // Call thread only. + webrtc::VideoReceiveStreamInterface::Config mRecvStreamConfig; + + // Are SSRC changes without signaling allowed or not. + // Call thread only. + bool mAllowSsrcChange = true; + + // Accessed during configuration/signaling (Call thread), and on the frame + // delivery thread for frame history tracking. Set only on the Call thread. + Atomic mRecvSSRC = + Atomic(0); // this can change during a stream! + + // Accessed from both the STS and frame delivery thread for frame history + // tracking. Set when receiving packets. + Atomic mRemoteSendSSRC = + Atomic(0); // this can change during a stream! + + // Main thread only + nsTArray mSendCodecPluginIDs; + // Main thread only + nsTArray mRecvCodecPluginIDs; + + // Main thread only + MediaEventListener mSendPluginCreated; + MediaEventListener mSendPluginReleased; + MediaEventListener mRecvPluginCreated; + MediaEventListener mRecvPluginReleased; + + // Call thread only. ssrc -> base_seq + std::map mRtpSendBaseSeqs; + // libwebrtc network thread only. ssrc -> base_seq. + // To track changes needed to mRtpSendBaseSeqs. + std::map mRtpSendBaseSeqs_n; + + // Tracking the attributes of received frames over time + // Protected by mRendererMonitor + dom::RTCVideoFrameHistoryInternal mReceivedFrameHistory; + + // Thread safe + Atomic mTransportActive = Atomic(false); + MediaEventProducer mRtcpByeEvent; + MediaEventProducer mRtcpTimeoutEvent; + MediaEventProducer mRtpPacketEvent; + MediaEventProducerExc mSenderRtpSendEvent; + MediaEventProducerExc mSenderRtcpSendEvent; + MediaEventProducerExc mReceiverRtcpSendEvent; + + // Assigned and revoked on mStsThread. Listeners for receiving packets. + MediaEventListener mSenderRtcpEventListener; // Rtp-transmitting pipeline + MediaEventListener mReceiverRtcpEventListener; // Rtp-receiving pipeline + MediaEventListener mReceiverRtpEventListener; // Rtp-receiving pipeline +}; +} // namespace mozilla + +#endif -- cgit v1.2.3