diff options
Diffstat (limited to 'dom/media/webrtc/libwebrtcglue/VideoConduit.h')
| -rw-r--r-- | dom/media/webrtc/libwebrtcglue/VideoConduit.h | 494 |
1 files changed, 494 insertions, 0 deletions
diff --git a/dom/media/webrtc/libwebrtcglue/VideoConduit.h b/dom/media/webrtc/libwebrtcglue/VideoConduit.h new file mode 100644 index 0000000000..f46f1f962c --- /dev/null +++ b/dom/media/webrtc/libwebrtcglue/VideoConduit.h @@ -0,0 +1,494 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef VIDEO_SESSION_H_ +#define VIDEO_SESSION_H_ + +#include "mozilla/Atomics.h" +#include "mozilla/Attributes.h" +#include "mozilla/DataMutex.h" +#include "mozilla/ReentrantMonitor.h" +#include "mozilla/SharedThreadPool.h" +#include "mozilla/StateMirroring.h" +#include "mozilla/UniquePtr.h" +#include "nsITimer.h" + +#include "MediaConduitInterface.h" +#include "common/MediaEngineWrapper.h" +#include "RtpRtcpConfig.h" +#include "RunningStat.h" +#include "transport/runnable_utils.h" + +// conflicts with #include of scoped_ptr.h +#undef FF +// Video Engine Includes +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/sdp_video_format.h" +#include "call/call_basic_stats.h" +#include "common_video/include/video_frame_buffer_pool.h" +#include "media/base/video_broadcaster.h" +#include <functional> +#include <memory> +/** This file hosts several structures identifying different aspects + * of a RTP Session. + */ + +namespace mozilla { + +// Convert (SI) kilobits/sec to (SI) bits/sec +#define KBPS(kbps) kbps * 1000 + +const int kViEMinCodecBitrate_bps = KBPS(30); +const unsigned int kVideoMtu = 1200; +const int kQpMax = 56; + +template <typename T> +T MinIgnoreZero(const T& a, const T& b) { + return std::min(a ? a : b, b ? b : a); +} + +class VideoStreamFactory; +class WebrtcAudioConduit; + +// Interface of external video encoder for WebRTC. +class WebrtcVideoEncoder : public VideoEncoder, public webrtc::VideoEncoder {}; + +// Interface of external video decoder for WebRTC. +class WebrtcVideoDecoder : public VideoDecoder, public webrtc::VideoDecoder {}; + +/** + * Concrete class for Video session. Hooks up + * - media-source and target to external transport + */ +class WebrtcVideoConduit + : public VideoSessionConduit, + public webrtc::RtcpEventObserver, + public rtc::VideoSinkInterface<webrtc::VideoFrame>, + public rtc::VideoSourceInterface<webrtc::VideoFrame> { + public: + // Returns true when both encoder and decoder are HW accelerated. + static bool HasH264Hardware(); + + Maybe<int> ActiveSendPayloadType() const override; + Maybe<int> ActiveRecvPayloadType() const override; + + /** + * Function to attach Renderer end-point for the Media-Video conduit. + * @param aRenderer : Reference to the concrete mozilla Video renderer + * implementation Note: Multiple invocations of this API shall remove an + * existing renderer and attaches the new to the Conduit. + */ + MediaConduitErrorCode AttachRenderer( + RefPtr<mozilla::VideoRenderer> aVideoRenderer) override; + void DetachRenderer() override; + + Maybe<uint16_t> RtpSendBaseSeqFor(uint32_t aSsrc) const override; + + const dom::RTCStatsTimestampMaker& GetTimestampMaker() const override; + + void StopTransmitting(); + void StartTransmitting(); + void StopReceiving(); + void StartReceiving(); + + /** + * Function to deliver a capture video frame for encoding and transport. + * If the frame's timestamp is 0, it will be automatically generated. + * + * NOTE: ConfigureSendMediaCodec() must be called before this function can + * be invoked. This ensures the inserted video-frames can be + * transmitted by the conduit. + */ + MediaConduitErrorCode SendVideoFrame(webrtc::VideoFrame aFrame) override; + + bool SendRtp(const uint8_t* aData, size_t aLength, + const webrtc::PacketOptions& aOptions) override; + bool SendSenderRtcp(const uint8_t* aData, size_t aLength) override; + bool SendReceiverRtcp(const uint8_t* aData, size_t aLength) override; + + /* + * webrtc:VideoSinkInterface implementation + * ------------------------------- + */ + void OnFrame(const webrtc::VideoFrame& frame) override; + + /* + * webrtc:VideoSourceInterface implementation + * ------------------------------- + */ + void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink, + const rtc::VideoSinkWants& wants) override; + void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override; + + bool HasCodecPluginID(uint64_t aPluginID) const override; + + RefPtr<GenericPromise> Shutdown() override; + + bool Denoising() const { return mDenoising; } + + uint8_t SpatialLayers() const { return mSpatialLayers; } + + uint8_t TemporalLayers() const { return mTemporalLayers; } + + webrtc::VideoCodecMode CodecMode() const; + + WebrtcVideoConduit(RefPtr<WebrtcCallWrapper> aCall, + nsCOMPtr<nsISerialEventTarget> aStsThread, + Options aOptions, std::string aPCHandle, + const TrackingId& aRecvTrackingId); + virtual ~WebrtcVideoConduit(); + + // Call thread. + void InitControl(VideoConduitControlInterface* aControl) override; + + // Called when a parameter in mControl has changed. Call thread. + void OnControlConfigChange(); + + // Necessary Init steps on main thread. + MediaConduitErrorCode Init(); + + Ssrcs GetLocalSSRCs() const override; + Maybe<Ssrc> GetAssociatedLocalRtxSSRC(Ssrc aSsrc) const override; + Maybe<Ssrc> GetRemoteSSRC() const override; + + // Call thread. + void UnsetRemoteSSRC(uint32_t aSsrc) override; + + static unsigned ToLibwebrtcMaxFramerate(const Maybe<double>& aMaxFramerate); + + private: + void NotifyUnsetCurrentRemoteSSRC(); + void SetRemoteSSRCConfig(uint32_t aSsrc, uint32_t aRtxSsrc); + void SetRemoteSSRCAndRestartAsNeeded(uint32_t aSsrc, uint32_t aRtxSsrc); + rtc::RefCountedObject<mozilla::VideoStreamFactory>* + CreateVideoStreamFactory(); + + public: + // Creating a recv stream or a send stream requires a local ssrc to be + // configured. This method will generate one if needed. + void EnsureLocalSSRC(); + // Creating a recv stream requires a remote ssrc to be configured. This method + // will generate one if needed. + void EnsureRemoteSSRC(); + + Maybe<webrtc::VideoReceiveStreamInterface::Stats> GetReceiverStats() + const override; + Maybe<webrtc::VideoSendStream::Stats> GetSenderStats() const override; + Maybe<webrtc::CallBasicStats> GetCallStats() const override; + + bool AddFrameHistory(dom::Sequence<dom::RTCVideoFrameHistoryInternal>* + outHistories) const override; + + void SetJitterBufferTarget(DOMHighResTimeStamp aTargetMs) override; + + uint64_t MozVideoLatencyAvg(); + + void DisableSsrcChanges() override { + MOZ_ASSERT(mCallThread->IsOnCurrentThread()); + mAllowSsrcChange = false; + } + + void CollectTelemetryData() override; + + void OnRtpReceived(webrtc::RtpPacketReceived&& aPacket, + webrtc::RTPHeader&& aHeader); + void OnRtcpReceived(MediaPacket&& aPacket); + + void OnRtcpBye() override; + void OnRtcpTimeout() override; + + void SetTransportActive(bool aActive) override; + + MediaEventSourceExc<MediaPacket>& SenderRtpSendEvent() override { + return mSenderRtpSendEvent; + } + MediaEventSourceExc<MediaPacket>& SenderRtcpSendEvent() override { + return mSenderRtcpSendEvent; + } + MediaEventSourceExc<MediaPacket>& ReceiverRtcpSendEvent() override { + return mReceiverRtcpSendEvent; + } + void ConnectReceiverRtpEvent( + MediaEventSourceExc<webrtc::RtpPacketReceived, webrtc::RTPHeader>& aEvent) + override { + mReceiverRtpEventListener = + aEvent.Connect(mCallThread, this, &WebrtcVideoConduit::OnRtpReceived); + } + void ConnectReceiverRtcpEvent( + MediaEventSourceExc<MediaPacket>& aEvent) override { + mReceiverRtcpEventListener = + aEvent.Connect(mCallThread, this, &WebrtcVideoConduit::OnRtcpReceived); + } + void ConnectSenderRtcpEvent( + MediaEventSourceExc<MediaPacket>& aEvent) override { + mSenderRtcpEventListener = + aEvent.Connect(mCallThread, this, &WebrtcVideoConduit::OnRtcpReceived); + } + + std::vector<webrtc::RtpSource> GetUpstreamRtpSources() const override; + + private: + // Don't allow copying/assigning. + WebrtcVideoConduit(const WebrtcVideoConduit&) = delete; + void operator=(const WebrtcVideoConduit&) = delete; + + // Utility function to dump recv codec database + void DumpCodecDB() const; + + // Video Latency Test averaging filter + void VideoLatencyUpdate(uint64_t aNewSample); + + void CreateSendStream(); + void DeleteSendStream(); + void CreateRecvStream(); + void DeleteRecvStream(); + + void DeliverPacket(rtc::CopyOnWriteBuffer packet, PacketType type) override; + + MediaEventSource<void>& RtcpByeEvent() override { return mRtcpByeEvent; } + MediaEventSource<void>& RtcpTimeoutEvent() override { + return mRtcpTimeoutEvent; + } + MediaEventSource<void>& RtpPacketEvent() override { return mRtpPacketEvent; } + + bool RequiresNewSendStream(const VideoCodecConfig& newConfig) const; + + mutable mozilla::ReentrantMonitor mRendererMonitor MOZ_UNANNOTATED; + + // Accessed on any thread under mRendererMonitor. + RefPtr<mozilla::VideoRenderer> mRenderer; + + // Accessed on any thread under mRendererMonitor. + unsigned short mReceivingWidth = 0; + + // Accessed on any thread under mRendererMonitor. + unsigned short mReceivingHeight = 0; + + // Call worker thread. All access to mCall->Call() happens here. + const nsCOMPtr<nsISerialEventTarget> mCallThread; + + // Socket transport service thread that runs stats queries against us. Any + // thread. + const nsCOMPtr<nsISerialEventTarget> mStsThread; + + // Thread on which we are fed video frames. Set lazily on first call to + // SendVideoFrame(). + nsCOMPtr<nsISerialEventTarget> mFrameSendingThread; + + struct Control { + // Mirrors that map to VideoConduitControlInterface for control. Call thread + // only. + Mirror<bool> mReceiving; + Mirror<bool> mTransmitting; + Mirror<Ssrcs> mLocalSsrcs; + Mirror<Ssrcs> mLocalRtxSsrcs; + Mirror<std::string> mLocalCname; + Mirror<std::string> mMid; + Mirror<Ssrc> mRemoteSsrc; + Mirror<Ssrc> mRemoteRtxSsrc; + Mirror<std::string> mSyncGroup; + Mirror<RtpExtList> mLocalRecvRtpExtensions; + Mirror<RtpExtList> mLocalSendRtpExtensions; + Mirror<Maybe<VideoCodecConfig>> mSendCodec; + Mirror<Maybe<RtpRtcpConfig>> mSendRtpRtcpConfig; + Mirror<std::vector<VideoCodecConfig>> mRecvCodecs; + Mirror<Maybe<RtpRtcpConfig>> mRecvRtpRtcpConfig; + Mirror<webrtc::VideoCodecMode> mCodecMode; + + // For caching mRemoteSsrc and mRemoteRtxSsrc, since another caller may + // change the remote ssrc in the stream config directly. + Ssrc mConfiguredRemoteSsrc = 0; + Ssrc mConfiguredRemoteRtxSsrc = 0; + // For tracking changes to mSendCodec and mSendRtpRtcpConfig. + Maybe<VideoCodecConfig> mConfiguredSendCodec; + Maybe<RtpRtcpConfig> mConfiguredSendRtpRtcpConfig; + // For tracking changes to mRecvCodecs and mRecvRtpRtcpConfig. + std::vector<VideoCodecConfig> mConfiguredRecvCodecs; + Maybe<RtpRtcpConfig> mConfiguredRecvRtpRtcpConfig; + + Control() = delete; + explicit Control(const RefPtr<AbstractThread>& aCallThread); + } mControl; + + // WatchManager allowing Mirrors and other watch targets to trigger functions + // that will update the webrtc.org configuration. + WatchManager<WebrtcVideoConduit> mWatchManager; + + mutable Mutex mMutex MOZ_UNANNOTATED; + + // Decoder factory used by mRecvStream when it needs new decoders. This is + // not shared broader like some state in the WebrtcCallWrapper because it + // handles CodecPluginID plumbing tied to this VideoConduit. + const UniquePtr<WebrtcVideoDecoderFactory> mDecoderFactory; + + // Encoder factory used by mSendStream when it needs new encoders. This is + // not shared broader like some state in the WebrtcCallWrapper because it + // handles CodecPluginID plumbing tied to this VideoConduit. + const UniquePtr<WebrtcVideoEncoderFactory> mEncoderFactory; + + // Our own record of the sinks added to mVideoBroadcaster so we can support + // dispatching updates to sinks from off-Call-thread. Call thread only. + AutoTArray<rtc::VideoSinkInterface<webrtc::VideoFrame>*, 1> mRegisteredSinks; + + // Broadcaster that distributes our frames to all registered sinks. + // Threadsafe. + rtc::VideoBroadcaster mVideoBroadcaster; + + // Buffer pool used for scaling frames. + // Accessed on the frame-feeding thread only. + webrtc::VideoFrameBufferPool mBufferPool; + + // Engine state we are concerned with. Written on the Call thread and read + // anywhere. + mozilla::Atomic<bool> + mEngineTransmitting; // If true ==> Transmit Subsystem is up and running + mozilla::Atomic<bool> + mEngineReceiving; // if true ==> Receive Subsystem up and running + + // Written only on the Call thread. Guarded by mMutex, except for reads on the + // Call thread. + Maybe<VideoCodecConfig> mCurSendCodecConfig; + + // Bookkeeping of stats for telemetry. Call thread only. + RunningStat mSendFramerate; + RunningStat mSendBitrate; + RunningStat mRecvFramerate; + RunningStat mRecvBitrate; + + // Must call webrtc::Call::DestroyVideoReceive/SendStream to delete this. + // Written only on the Call thread. Guarded by mMutex, except for reads on the + // Call thread. + webrtc::VideoReceiveStreamInterface* mRecvStream = nullptr; + + // Must call webrtc::Call::DestroyVideoReceive/SendStream to delete this. + webrtc::VideoSendStream* mSendStream = nullptr; + + // Written on the frame feeding thread. + // Guarded by mMutex, except for reads on the frame feeding thread. + unsigned short mLastWidth = 0; + + // Written on the frame feeding thread. + // Guarded by mMutex, except for reads on the frame feeding thread. + unsigned short mLastHeight = 0; + + // Written on the frame feeding thread, the timestamp of the last frame on the + // send side, in microseconds. This is a local timestamp using the system + // clock with a unspecified epoch (Like mozilla::TimeStamp). + // Guarded by mMutex. + Maybe<uint64_t> mLastTimestampSendUs; + + // Written on the frame receive thread, the rtp timestamp of the last frame + // on the receive side, in 90kHz base. This comes from the RTP packet. + // Guarded by mMutex. + Maybe<uint32_t> mLastRTPTimestampReceive; + + // Accessed from any thread under mRendererMonitor. + uint64_t mVideoLatencyAvg = 0; + + const bool mVideoLatencyTestEnable; + + // All in bps. + const int mMinBitrate; + const int mStartBitrate; + const int mPrefMaxBitrate; + const int mMinBitrateEstimate; + + // Max bitrate in bps as provided by negotiation. Call thread only. + int mNegotiatedMaxBitrate = 0; + + // Set to true to force denoising on. + const bool mDenoising; + + // Set to true to ignore sink wants (scaling due to bwe and cpu usage). + const bool mLockScaling; + + const uint8_t mSpatialLayers; + const uint8_t mTemporalLayers; + + static const unsigned int sAlphaNum = 7; + static const unsigned int sAlphaDen = 8; + static const unsigned int sRoundingPadding = 1024; + + // Target jitter buffer to be applied to the receive stream in milliseconds. + uint16_t mJitterBufferTargetMs = 0; + + // WEBRTC.ORG Call API + // Const so can be accessed on any thread. All methods are called on the Call + // thread. + const RefPtr<WebrtcCallWrapper> mCall; + + // Set up in the ctor and then not touched. Called through by the streams on + // any thread. Safe since we own and control the lifetime of the streams. + WebrtcSendTransport mSendTransport; + WebrtcReceiveTransport mRecvTransport; + + // Written only on the Call thread. Guarded by mMutex, except for reads on the + // Call thread. Typical non-Call thread access is on the frame delivery + // thread. + webrtc::VideoSendStream::Config mSendStreamConfig; + + // Call thread only. + webrtc::VideoEncoderConfig mEncoderConfig; + + // Written only on the Call thread. Guarded by mMutex, except for reads on the + // Call thread. Calls can happen under mMutex on any thread. + DataMutex<RefPtr<rtc::RefCountedObject<VideoStreamFactory>>> + mVideoStreamFactory; + + // Call thread only. + webrtc::VideoReceiveStreamInterface::Config mRecvStreamConfig; + + // Are SSRC changes without signaling allowed or not. + // Call thread only. + bool mAllowSsrcChange = true; + + // Accessed during configuration/signaling (Call thread), and on the frame + // delivery thread for frame history tracking. Set only on the Call thread. + Atomic<uint32_t> mRecvSSRC = + Atomic<uint32_t>(0); // this can change during a stream! + + // Accessed from both the STS and frame delivery thread for frame history + // tracking. Set when receiving packets. + Atomic<uint32_t> mRemoteSendSSRC = + Atomic<uint32_t>(0); // this can change during a stream! + + // Main thread only + nsTArray<uint64_t> mSendCodecPluginIDs; + // Main thread only + nsTArray<uint64_t> mRecvCodecPluginIDs; + + // Main thread only + MediaEventListener mSendPluginCreated; + MediaEventListener mSendPluginReleased; + MediaEventListener mRecvPluginCreated; + MediaEventListener mRecvPluginReleased; + + // Call thread only. ssrc -> base_seq + std::map<uint32_t, uint16_t> mRtpSendBaseSeqs; + // libwebrtc network thread only. ssrc -> base_seq. + // To track changes needed to mRtpSendBaseSeqs. + std::map<uint32_t, uint16_t> mRtpSendBaseSeqs_n; + + // Tracking the attributes of received frames over time + // Protected by mRendererMonitor + dom::RTCVideoFrameHistoryInternal mReceivedFrameHistory; + + // Thread safe + Atomic<bool> mTransportActive = Atomic<bool>(false); + MediaEventProducer<void> mRtcpByeEvent; + MediaEventProducer<void> mRtcpTimeoutEvent; + MediaEventProducer<void> mRtpPacketEvent; + MediaEventProducerExc<MediaPacket> mSenderRtpSendEvent; + MediaEventProducerExc<MediaPacket> mSenderRtcpSendEvent; + MediaEventProducerExc<MediaPacket> mReceiverRtcpSendEvent; + + // Assigned and revoked on mStsThread. Listeners for receiving packets. + MediaEventListener mSenderRtcpEventListener; // Rtp-transmitting pipeline + MediaEventListener mReceiverRtcpEventListener; // Rtp-receiving pipeline + MediaEventListener mReceiverRtpEventListener; // Rtp-receiving pipeline +}; +} // namespace mozilla + +#endif |