/* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ #ifndef VIDEO_SESSION_H_ #define VIDEO_SESSION_H_ #include "mozilla/Atomics.h" #include "mozilla/Attributes.h" #include "mozilla/DataMutex.h" #include "mozilla/ReentrantMonitor.h" #include "mozilla/SharedThreadPool.h" #include "mozilla/StateMirroring.h" #include "mozilla/UniquePtr.h" #include "nsITimer.h" #include "MediaConduitInterface.h" #include "common/MediaEngineWrapper.h" #include "RtpRtcpConfig.h" #include "RunningStat.h" #include "transport/runnable_utils.h" // conflicts with #include of scoped_ptr.h #undef FF // Video Engine Includes #include "api/video_codecs/video_decoder.h" #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/sdp_video_format.h" #include "call/call_basic_stats.h" #include "common_video/include/video_frame_buffer_pool.h" #include "media/base/video_broadcaster.h" #include #include /** This file hosts several structures identifying different aspects * of a RTP Session. */ namespace mozilla { // Convert (SI) kilobits/sec to (SI) bits/sec #define KBPS(kbps) kbps * 1000 const int kViEMinCodecBitrate_bps = KBPS(30); const unsigned int kVideoMtu = 1200; const int kQpMax = 56; template T MinIgnoreZero(const T& a, const T& b) { return std::min(a ? a : b, b ? b : a); } class VideoStreamFactory; class WebrtcAudioConduit; // Interface of external video encoder for WebRTC. class WebrtcVideoEncoder : public VideoEncoder, public webrtc::VideoEncoder {}; // Interface of external video decoder for WebRTC. class WebrtcVideoDecoder : public VideoDecoder, public webrtc::VideoDecoder {}; /** * Concrete class for Video session. Hooks up * - media-source and target to external transport */ class WebrtcVideoConduit : public VideoSessionConduit, public webrtc::RtcpEventObserver, public rtc::VideoSinkInterface, public rtc::VideoSourceInterface { public: // Returns true when both encoder and decoder are HW accelerated. static bool HasH264Hardware(); Maybe ActiveSendPayloadType() const override; Maybe ActiveRecvPayloadType() const override; /** * Function to attach Renderer end-point for the Media-Video conduit. * @param aRenderer : Reference to the concrete mozilla Video renderer * implementation Note: Multiple invocations of this API shall remove an * existing renderer and attaches the new to the Conduit. */ MediaConduitErrorCode AttachRenderer( RefPtr aVideoRenderer) override; void DetachRenderer() override; Maybe RtpSendBaseSeqFor(uint32_t aSsrc) const override; const dom::RTCStatsTimestampMaker& GetTimestampMaker() const override; void StopTransmitting(); void StartTransmitting(); void StopReceiving(); void StartReceiving(); /** * Function to deliver a capture video frame for encoding and transport. * If the frame's timestamp is 0, it will be automatically generated. * * NOTE: ConfigureSendMediaCodec() must be called before this function can * be invoked. This ensures the inserted video-frames can be * transmitted by the conduit. */ MediaConduitErrorCode SendVideoFrame(webrtc::VideoFrame aFrame) override; bool SendRtp(const uint8_t* aData, size_t aLength, const webrtc::PacketOptions& aOptions) override; bool SendSenderRtcp(const uint8_t* aData, size_t aLength) override; bool SendReceiverRtcp(const uint8_t* aData, size_t aLength) override; /* * webrtc:VideoSinkInterface implementation * ------------------------------- */ void OnFrame(const webrtc::VideoFrame& frame) override; /* * webrtc:VideoSourceInterface implementation * ------------------------------- */ void AddOrUpdateSink(rtc::VideoSinkInterface* sink, const rtc::VideoSinkWants& wants) override; void RemoveSink(rtc::VideoSinkInterface* sink) override; bool HasCodecPluginID(uint64_t aPluginID) const override; RefPtr Shutdown() override; bool Denoising() const { return mDenoising; } uint8_t SpatialLayers() const { return mSpatialLayers; } uint8_t TemporalLayers() const { return mTemporalLayers; } webrtc::VideoCodecMode CodecMode() const; WebrtcVideoConduit(RefPtr aCall, nsCOMPtr aStsThread, Options aOptions, std::string aPCHandle, const TrackingId& aRecvTrackingId); virtual ~WebrtcVideoConduit(); // Call thread. void InitControl(VideoConduitControlInterface* aControl) override; // Called when a parameter in mControl has changed. Call thread. void OnControlConfigChange(); // Necessary Init steps on main thread. MediaConduitErrorCode Init(); Ssrcs GetLocalSSRCs() const override; Maybe GetAssociatedLocalRtxSSRC(Ssrc aSsrc) const override; Maybe GetRemoteSSRC() const override; // Call thread. void UnsetRemoteSSRC(uint32_t aSsrc) override; static unsigned ToLibwebrtcMaxFramerate(const Maybe& aMaxFramerate); private: void NotifyUnsetCurrentRemoteSSRC(); void SetRemoteSSRCConfig(uint32_t aSsrc, uint32_t aRtxSsrc); void SetRemoteSSRCAndRestartAsNeeded(uint32_t aSsrc, uint32_t aRtxSsrc); rtc::RefCountedObject* CreateVideoStreamFactory(); public: // Creating a recv stream or a send stream requires a local ssrc to be // configured. This method will generate one if needed. void EnsureLocalSSRC(); // Creating a recv stream requires a remote ssrc to be configured. This method // will generate one if needed. void EnsureRemoteSSRC(); Maybe GetReceiverStats() const override; Maybe GetSenderStats() const override; Maybe GetCallStats() const override; bool AddFrameHistory(dom::Sequence* outHistories) const override; void SetJitterBufferTarget(DOMHighResTimeStamp aTargetMs) override; uint64_t MozVideoLatencyAvg(); void DisableSsrcChanges() override { MOZ_ASSERT(mCallThread->IsOnCurrentThread()); mAllowSsrcChange = false; } void CollectTelemetryData() override; void OnRtpReceived(webrtc::RtpPacketReceived&& aPacket, webrtc::RTPHeader&& aHeader); void OnRtcpReceived(MediaPacket&& aPacket); void OnRtcpBye() override; void OnRtcpTimeout() override; void SetTransportActive(bool aActive) override; MediaEventSourceExc& SenderRtpSendEvent() override { return mSenderRtpSendEvent; } MediaEventSourceExc& SenderRtcpSendEvent() override { return mSenderRtcpSendEvent; } MediaEventSourceExc& ReceiverRtcpSendEvent() override { return mReceiverRtcpSendEvent; } void ConnectReceiverRtpEvent( MediaEventSourceExc& aEvent) override { mReceiverRtpEventListener = aEvent.Connect(mCallThread, this, &WebrtcVideoConduit::OnRtpReceived); } void ConnectReceiverRtcpEvent( MediaEventSourceExc& aEvent) override { mReceiverRtcpEventListener = aEvent.Connect(mCallThread, this, &WebrtcVideoConduit::OnRtcpReceived); } void ConnectSenderRtcpEvent( MediaEventSourceExc& aEvent) override { mSenderRtcpEventListener = aEvent.Connect(mCallThread, this, &WebrtcVideoConduit::OnRtcpReceived); } std::vector GetUpstreamRtpSources() const override; private: // Don't allow copying/assigning. WebrtcVideoConduit(const WebrtcVideoConduit&) = delete; void operator=(const WebrtcVideoConduit&) = delete; // Utility function to dump recv codec database void DumpCodecDB() const; // Video Latency Test averaging filter void VideoLatencyUpdate(uint64_t aNewSample); void CreateSendStream(); void DeleteSendStream(); void CreateRecvStream(); void DeleteRecvStream(); void DeliverPacket(rtc::CopyOnWriteBuffer packet, PacketType type) override; MediaEventSource& RtcpByeEvent() override { return mRtcpByeEvent; } MediaEventSource& RtcpTimeoutEvent() override { return mRtcpTimeoutEvent; } MediaEventSource& RtpPacketEvent() override { return mRtpPacketEvent; } bool RequiresNewSendStream(const VideoCodecConfig& newConfig) const; mutable mozilla::ReentrantMonitor mRendererMonitor MOZ_UNANNOTATED; // Accessed on any thread under mRendererMonitor. RefPtr mRenderer; // Accessed on any thread under mRendererMonitor. unsigned short mReceivingWidth = 0; // Accessed on any thread under mRendererMonitor. unsigned short mReceivingHeight = 0; // Call worker thread. All access to mCall->Call() happens here. const nsCOMPtr mCallThread; // Socket transport service thread that runs stats queries against us. Any // thread. const nsCOMPtr mStsThread; // Thread on which we are fed video frames. Set lazily on first call to // SendVideoFrame(). nsCOMPtr mFrameSendingThread; struct Control { // Mirrors that map to VideoConduitControlInterface for control. Call thread // only. Mirror mReceiving; Mirror mTransmitting; Mirror mLocalSsrcs; Mirror mLocalRtxSsrcs; Mirror mLocalCname; Mirror mMid; Mirror mRemoteSsrc; Mirror mRemoteRtxSsrc; Mirror mSyncGroup; Mirror mLocalRecvRtpExtensions; Mirror mLocalSendRtpExtensions; Mirror> mSendCodec; Mirror> mSendRtpRtcpConfig; Mirror> mRecvCodecs; Mirror> mRecvRtpRtcpConfig; Mirror mCodecMode; // For caching mRemoteSsrc and mRemoteRtxSsrc, since another caller may // change the remote ssrc in the stream config directly. Ssrc mConfiguredRemoteSsrc = 0; Ssrc mConfiguredRemoteRtxSsrc = 0; // For tracking changes to mSendCodec and mSendRtpRtcpConfig. Maybe mConfiguredSendCodec; Maybe mConfiguredSendRtpRtcpConfig; // For tracking changes to mRecvCodecs and mRecvRtpRtcpConfig. std::vector mConfiguredRecvCodecs; Maybe mConfiguredRecvRtpRtcpConfig; Control() = delete; explicit Control(const RefPtr& aCallThread); } mControl; // WatchManager allowing Mirrors and other watch targets to trigger functions // that will update the webrtc.org configuration. WatchManager mWatchManager; mutable Mutex mMutex MOZ_UNANNOTATED; // Decoder factory used by mRecvStream when it needs new decoders. This is // not shared broader like some state in the WebrtcCallWrapper because it // handles CodecPluginID plumbing tied to this VideoConduit. const UniquePtr mDecoderFactory; // Encoder factory used by mSendStream when it needs new encoders. This is // not shared broader like some state in the WebrtcCallWrapper because it // handles CodecPluginID plumbing tied to this VideoConduit. const UniquePtr mEncoderFactory; // Our own record of the sinks added to mVideoBroadcaster so we can support // dispatching updates to sinks from off-Call-thread. Call thread only. AutoTArray*, 1> mRegisteredSinks; // Broadcaster that distributes our frames to all registered sinks. // Threadsafe. rtc::VideoBroadcaster mVideoBroadcaster; // Buffer pool used for scaling frames. // Accessed on the frame-feeding thread only. webrtc::VideoFrameBufferPool mBufferPool; // Engine state we are concerned with. Written on the Call thread and read // anywhere. mozilla::Atomic mEngineTransmitting; // If true ==> Transmit Subsystem is up and running mozilla::Atomic mEngineReceiving; // if true ==> Receive Subsystem up and running // Written only on the Call thread. Guarded by mMutex, except for reads on the // Call thread. Maybe mCurSendCodecConfig; // Bookkeeping of stats for telemetry. Call thread only. RunningStat mSendFramerate; RunningStat mSendBitrate; RunningStat mRecvFramerate; RunningStat mRecvBitrate; // Must call webrtc::Call::DestroyVideoReceive/SendStream to delete this. // Written only on the Call thread. Guarded by mMutex, except for reads on the // Call thread. webrtc::VideoReceiveStreamInterface* mRecvStream = nullptr; // Must call webrtc::Call::DestroyVideoReceive/SendStream to delete this. webrtc::VideoSendStream* mSendStream = nullptr; // Written on the frame feeding thread. // Guarded by mMutex, except for reads on the frame feeding thread. unsigned short mLastWidth = 0; // Written on the frame feeding thread. // Guarded by mMutex, except for reads on the frame feeding thread. unsigned short mLastHeight = 0; // Written on the frame feeding thread, the timestamp of the last frame on the // send side, in microseconds. This is a local timestamp using the system // clock with a unspecified epoch (Like mozilla::TimeStamp). // Guarded by mMutex. Maybe mLastTimestampSendUs; // Written on the frame receive thread, the rtp timestamp of the last frame // on the receive side, in 90kHz base. This comes from the RTP packet. // Guarded by mMutex. Maybe mLastRTPTimestampReceive; // Accessed from any thread under mRendererMonitor. uint64_t mVideoLatencyAvg = 0; const bool mVideoLatencyTestEnable; // All in bps. const int mMinBitrate; const int mStartBitrate; const int mPrefMaxBitrate; const int mMinBitrateEstimate; // Max bitrate in bps as provided by negotiation. Call thread only. int mNegotiatedMaxBitrate = 0; // Set to true to force denoising on. const bool mDenoising; // Set to true to ignore sink wants (scaling due to bwe and cpu usage). const bool mLockScaling; const uint8_t mSpatialLayers; const uint8_t mTemporalLayers; static const unsigned int sAlphaNum = 7; static const unsigned int sAlphaDen = 8; static const unsigned int sRoundingPadding = 1024; // Target jitter buffer to be applied to the receive stream in milliseconds. uint16_t mJitterBufferTargetMs = 0; // WEBRTC.ORG Call API // Const so can be accessed on any thread. All methods are called on the Call // thread. const RefPtr mCall; // Set up in the ctor and then not touched. Called through by the streams on // any thread. Safe since we own and control the lifetime of the streams. WebrtcSendTransport mSendTransport; WebrtcReceiveTransport mRecvTransport; // Written only on the Call thread. Guarded by mMutex, except for reads on the // Call thread. Typical non-Call thread access is on the frame delivery // thread. webrtc::VideoSendStream::Config mSendStreamConfig; // Call thread only. webrtc::VideoEncoderConfig mEncoderConfig; // Written only on the Call thread. Guarded by mMutex, except for reads on the // Call thread. Calls can happen under mMutex on any thread. DataMutex>> mVideoStreamFactory; // Call thread only. webrtc::VideoReceiveStreamInterface::Config mRecvStreamConfig; // Are SSRC changes without signaling allowed or not. // Call thread only. bool mAllowSsrcChange = true; // Accessed during configuration/signaling (Call thread), and on the frame // delivery thread for frame history tracking. Set only on the Call thread. Atomic mRecvSSRC = Atomic(0); // this can change during a stream! // Accessed from both the STS and frame delivery thread for frame history // tracking. Set when receiving packets. Atomic mRemoteSendSSRC = Atomic(0); // this can change during a stream! // Main thread only nsTArray mSendCodecPluginIDs; // Main thread only nsTArray mRecvCodecPluginIDs; // Main thread only MediaEventListener mSendPluginCreated; MediaEventListener mSendPluginReleased; MediaEventListener mRecvPluginCreated; MediaEventListener mRecvPluginReleased; // Call thread only. ssrc -> base_seq std::map mRtpSendBaseSeqs; // libwebrtc network thread only. ssrc -> base_seq. // To track changes needed to mRtpSendBaseSeqs. std::map mRtpSendBaseSeqs_n; // Tracking the attributes of received frames over time // Protected by mRendererMonitor dom::RTCVideoFrameHistoryInternal mReceivedFrameHistory; // Thread safe Atomic mTransportActive = Atomic(false); MediaEventProducer mRtcpByeEvent; MediaEventProducer mRtcpTimeoutEvent; MediaEventProducer mRtpPacketEvent; MediaEventProducerExc mSenderRtpSendEvent; MediaEventProducerExc mSenderRtcpSendEvent; MediaEventProducerExc mReceiverRtcpSendEvent; // Assigned and revoked on mStsThread. Listeners for receiving packets. MediaEventListener mSenderRtcpEventListener; // Rtp-transmitting pipeline MediaEventListener mReceiverRtcpEventListener; // Rtp-receiving pipeline MediaEventListener mReceiverRtpEventListener; // Rtp-receiving pipeline }; } // namespace mozilla #endif