/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ // Original author: ekr@rtfm.com #ifndef mediapipeline_h__ #define mediapipeline_h__ #include #include "transport/sigslot.h" #include "transport/transportlayer.h" // For TransportLayer::State #include "libwebrtcglue/MediaConduitInterface.h" #include "mozilla/ReentrantMonitor.h" #include "mozilla/Atomics.h" #include "transport/SrtpFlow.h" // For SRTP_MAX_EXPANSION #include "transport/mediapacket.h" #include "transport/runnable_utils.h" #include "AudioPacketizer.h" #include "MediaPipelineFilter.h" #include "MediaSegment.h" #include "jsapi/PacketDumper.h" #include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h" // Should come from MediaEngine.h, but that's a pain to include here // because of the MOZILLA_EXTERNAL_LINKAGE stuff. #define WEBRTC_MAX_SAMPLE_RATE 48000 class nsIPrincipal; namespace mozilla { class AudioProxyThread; class MediaInputPort; class MediaPipelineFilter; class MediaTransportHandler; class PeerIdentity; class ProcessedMediaTrack; class SourceMediaTrack; class VideoFrameConverter; namespace dom { class MediaStreamTrack; struct RTCRTPContributingSourceStats; } // namespace dom // A class that represents the pipeline of audio and video // The dataflow looks like: // // TRANSMIT // CaptureDevice -> stream -> [us] -> conduit -> [us] -> transport -> network // // RECEIVE // network -> transport -> [us] -> conduit -> [us] -> stream -> Playout // // The boxes labeled [us] are just bridge logic implemented in this class // // We have to deal with a number of threads: // // GSM: // * Assembles the pipeline // SocketTransportService // * Receives notification that ICE and DTLS have completed // * Processes incoming network data and passes it to the conduit // * Processes outgoing RTP and RTCP // MediaTrackGraph // * Receives outgoing data from the MediaTrackGraph // * Receives pull requests for more data from the // MediaTrackGraph // One or another GIPS threads // * Receives RTCP messages to send to the other side // * Processes video frames GIPS wants to render // // For a transmitting conduit, "output" is RTP and "input" is RTCP. // For a receiving conduit, "input" is RTP and "output" is RTCP. // class MediaPipeline : public sigslot::has_slots<> { public: enum class DirectionType { TRANSMIT, RECEIVE }; MediaPipeline(const std::string& aPc, RefPtr aTransportHandler, DirectionType aDirection, RefPtr aMainThread, RefPtr aStsThread, RefPtr aConduit); virtual void Start() = 0; virtual RefPtr Stop() = 0; virtual void DetachMedia() {} void SetLevel(size_t aLevel) { mLevel = aLevel; } // Must be called on the main thread. void Shutdown_m(); void UpdateTransport_m(const std::string& aTransportId, UniquePtr&& aFilter); void UpdateTransport_s(const std::string& aTransportId, UniquePtr&& aFilter); // Used only for testing; adds RTP header extension for RTP Stream Id with // the given id. void AddRIDExtension_m(size_t aExtensionId); void AddRIDExtension_s(size_t aExtensionId); // Used only for testing; installs a MediaPipelineFilter that filters // everything but the given RID void AddRIDFilter_m(const std::string& aRid); void AddRIDFilter_s(const std::string& aRid); virtual DirectionType Direction() const { return mDirection; } size_t Level() const { return mLevel; } virtual bool IsVideo() const = 0; class RtpCSRCStats { public: // Gets an expiration time for CRC info given a reference time, // this reference time would normally be the time of calling. // This value can then be used to check if a RtpCSRCStats // has expired via Expired(...) static DOMHighResTimeStamp GetExpiryFromTime( const DOMHighResTimeStamp aTime); RtpCSRCStats(const uint32_t aCsrc, const DOMHighResTimeStamp aTime); ~RtpCSRCStats() = default; // Initialize a webidl representation suitable for adding to a report. // This assumes that the webidl object is empty. // @param aWebidlObj the webidl binding object to popluate // @param aInboundRtpStreamId the associated RTCInboundRTPStreamStats.id void GetWebidlInstance(dom::RTCRTPContributingSourceStats& aWebidlObj, const nsString& aInboundRtpStreamId) const; void SetTimestamp(const DOMHighResTimeStamp aTime) { mTimestamp = aTime; } // Check if the RtpCSRCStats has expired, checks against a // given expiration time. bool Expired(const DOMHighResTimeStamp aExpiry) const { return mTimestamp < aExpiry; } private: static const double constexpr EXPIRY_TIME_MILLISECONDS = 10 * 1000; const uint32_t mCsrc; DOMHighResTimeStamp mTimestamp; }; // Gets the gathered contributing source stats for the last expiration period. // @param aId the stream id to use for populating inboundRtpStreamId field // @param aArr the array to append the stats objects to void GetContributingSourceStats( const nsString& aInboundRtpStreamId, FallibleTArray& aArr) const; int32_t RtpPacketsSent() const { return mRtpPacketsSent; } int64_t RtpBytesSent() const { return mRtpBytesSent; } int32_t RtcpPacketsSent() const { return mRtcpPacketsSent; } int32_t RtpPacketsReceived() const { return mRtpPacketsReceived; } int64_t RtpBytesReceived() const { return mRtpBytesReceived; } int32_t RtcpPacketsReceived() const { return mRtcpPacketsReceived; } // Gets the current time as a DOMHighResTimeStamp DOMHighResTimeStamp GetNow() const; MediaSessionConduit* Conduit() const { return mConduit; } // Thread counting NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaPipeline) // Separate class to allow ref counting class PipelineTransport : public TransportInterface { public: // Implement the TransportInterface functions explicit PipelineTransport(RefPtr aStsThread) : mPipeline(nullptr), mStsThread(std::move(aStsThread)) {} void Attach(MediaPipeline* pipeline) { mPipeline = pipeline; } void Detach() { mPipeline = nullptr; } MediaPipeline* Pipeline() const { return mPipeline; } virtual nsresult SendRtpPacket(const uint8_t* aData, size_t aLen) override; virtual nsresult SendRtcpPacket(const uint8_t* aData, size_t aLen) override; private: void SendRtpRtcpPacket_s(MediaPacket&& aPacket); // Creates a cycle, which we break with Detach RefPtr mPipeline; const RefPtr mStsThread; }; protected: virtual ~MediaPipeline(); friend class PipelineTransport; // The transport is ready virtual void TransportReady_s() {} void IncrementRtpPacketsSent(int aBytes); void IncrementRtcpPacketsSent(); void IncrementRtpPacketsReceived(int aBytes); virtual void OnRtpPacketReceived() {} void IncrementRtcpPacketsReceived(); virtual void SendPacket(MediaPacket&& packet); // Process slots on transports void RtpStateChange(const std::string& aTransportId, TransportLayer::State); void RtcpStateChange(const std::string& aTransportId, TransportLayer::State); virtual void CheckTransportStates(); void PacketReceived(const std::string& aTransportId, const MediaPacket& packet); void AlpnNegotiated(const std::string& aAlpn, bool aPrivacyRequested); void RtpPacketReceived(const MediaPacket& packet); void RtcpPacketReceived(const MediaPacket& packet); void EncryptedPacketSending(const std::string& aTransportId, const MediaPacket& aPacket); void SetDescription_s(const std::string& description); // Called when ALPN is negotiated and is requesting privacy, so receive // pipelines do not enter data into the graph under a content principal. virtual void MakePrincipalPrivate_s() {} const DirectionType mDirection; Atomic mLevel; std::string mTransportId; const RefPtr mTransportHandler; RefPtr mConduit; // Our conduit. Written on the main // thread. Read on STS thread. TransportLayer::State mRtpState = TransportLayer::TS_NONE; TransportLayer::State mRtcpState = TransportLayer::TS_NONE; bool mSignalsConnected = false; // Pointers to the threads we need. Initialized at creation // and used all over the place. const RefPtr mMainThread; const RefPtr mStsThread; // Created in c'tor. Referenced by the conduit. const RefPtr mTransport; // Only safe to access from STS thread. int32_t mRtpPacketsSent; int32_t mRtcpPacketsSent; int32_t mRtpPacketsReceived; int32_t mRtcpPacketsReceived; int64_t mRtpBytesSent; int64_t mRtpBytesReceived; // Only safe to access from STS thread. std::map mCsrcStats; // Written in c'tor. Read on STS thread. const std::string mPc; std::string mDescription; // Written in c'tor, all following accesses are on the STS thread. UniquePtr mFilter; const UniquePtr mRtpParser; UniquePtr mPacketDumper; private: bool IsRtp(const unsigned char* aData, size_t aLen) const; // Must be called on the STS thread. Must be called after DetachMedia(). void DetachTransport_s(); }; // A specialization of pipeline for reading from an input device // and transmitting to the network. class MediaPipelineTransmit : public MediaPipeline { public: // Set aRtcpTransport to nullptr to use rtcp-mux MediaPipelineTransmit(const std::string& aPc, RefPtr aTransportHandler, RefPtr aMainThread, RefPtr aStsThread, bool aIsVideo, RefPtr aConduit); bool Transmitting() const; void Start() override; RefPtr Stop() override; // written and used from MainThread bool IsVideo() const override; // When the principal of the domtrack changes, it calls through to here // so that we can determine whether to enable track transmission. // `track` has to be null or equal `mDomTrack` for us to apply the update. virtual void UpdateSinkIdentity_m(const dom::MediaStreamTrack* aTrack, nsIPrincipal* aPrincipal, const PeerIdentity* aSinkIdentity); // Called on the main thread. void DetachMedia() override; // Override MediaPipeline::TransportReady_s. void TransportReady_s() override; // Replace a track with a different one. nsresult SetTrack(RefPtr aDomTrack); // Used to correlate stats RefPtr GetTrack() const; // Set the track whose data we will transmit. For internal and test use. void SetSendTrack(RefPtr aSendTrack); // Separate classes to allow ref counting class PipelineListener; class VideoFrameFeeder; protected: ~MediaPipelineTransmit(); void SetDescription(); private: void AsyncStart(const RefPtr& aPromise); const bool mIsVideo; const RefPtr mListener; const RefPtr mFeeder; RefPtr mAudioProcessing; RefPtr mConverter; RefPtr mDomTrack; // Input port connecting mDomTrack's MediaTrack to mSendTrack. RefPtr mSendPort; // MediaTrack that we send over the network. This allows changing mDomTrack. RefPtr mSendTrack; // True if we're actively transmitting data to the network. Main thread only. bool mTransmitting; // When AsyncStart() is used this flag helps to avoid unexpected starts. One // case is that a start has already been scheduled. A second case is that a // start has already taken place (from JS for example). A third case is that // a stop has taken place so we want to cancel the start. Main thread only. bool mAsyncStartRequested; }; // A specialization of pipeline for reading from the network and // rendering media. class MediaPipelineReceive : public MediaPipeline { public: // Set aRtcpTransport to nullptr to use rtcp-mux MediaPipelineReceive(const std::string& aPc, RefPtr aTransportHandler, RefPtr aMainThread, RefPtr aStsThread, RefPtr aConduit); protected: ~MediaPipelineReceive(); }; // A specialization of pipeline for reading from the network and // rendering audio. class MediaPipelineReceiveAudio : public MediaPipelineReceive { public: MediaPipelineReceiveAudio(const std::string& aPc, RefPtr aTransportHandler, RefPtr aMainThread, RefPtr aStsThread, RefPtr aConduit, const RefPtr& aTrack, const PrincipalHandle& aPrincipalHandle); void DetachMedia() override; bool IsVideo() const override { return false; } void MakePrincipalPrivate_s() override; void Start() override; RefPtr Stop() override; void OnRtpPacketReceived() override; private: // Separate class to allow ref counting class PipelineListener; const RefPtr mListener; }; // A specialization of pipeline for reading from the network and // rendering video. class MediaPipelineReceiveVideo : public MediaPipelineReceive { public: MediaPipelineReceiveVideo(const std::string& aPc, RefPtr aTransportHandler, RefPtr aMainThread, RefPtr aStsThread, RefPtr aConduit, const RefPtr& aTrack, const PrincipalHandle& aPrincipalHandle); // Called on the main thread. void DetachMedia() override; bool IsVideo() const override { return true; } void MakePrincipalPrivate_s() override; void Start() override; RefPtr Stop() override; void OnRtpPacketReceived() override; private: class PipelineRenderer; friend class PipelineRenderer; // Separate class to allow ref counting class PipelineListener; const RefPtr mRenderer; const RefPtr mListener; }; } // namespace mozilla #endif