diff options
Diffstat (limited to 'dom/media/webrtc/libwebrtcglue/MediaConduitInterface.h')
| -rw-r--r-- | dom/media/webrtc/libwebrtcglue/MediaConduitInterface.h | 495 |
1 files changed, 495 insertions, 0 deletions
diff --git a/dom/media/webrtc/libwebrtcglue/MediaConduitInterface.h b/dom/media/webrtc/libwebrtcglue/MediaConduitInterface.h new file mode 100644 index 0000000000..86e148dc5c --- /dev/null +++ b/dom/media/webrtc/libwebrtcglue/MediaConduitInterface.h @@ -0,0 +1,495 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MEDIA_CONDUIT_ABSTRACTION_ +#define MEDIA_CONDUIT_ABSTRACTION_ + +#include <vector> +#include <functional> +#include <map> + +#include "CodecConfig.h" +#include "ImageContainer.h" +#include "jsapi/RTCStatsReport.h" +#include "MediaConduitErrors.h" +#include "mozilla/media/MediaUtils.h" +#include "mozilla/MozPromise.h" +#include "WebrtcVideoCodecFactory.h" +#include "nsTArray.h" +#include "mozilla/dom/RTCRtpSourcesBinding.h" +#include "PerformanceRecorder.h" +#include "transport/mediapacket.h" +#include "MediaConduitControl.h" + +// libwebrtc includes +#include "api/audio/audio_frame.h" +#include "api/call/transport.h" +#include "api/rtp_headers.h" +#include "api/rtp_parameters.h" +#include "api/transport/rtp/rtp_source.h" +#include "api/video/video_frame_buffer.h" +#include "call/audio_receive_stream.h" +#include "call/audio_send_stream.h" +#include "call/call_basic_stats.h" +#include "call/video_receive_stream.h" +#include "call/video_send_stream.h" +#include "rtc_base/copy_on_write_buffer.h" + +namespace webrtc { +class RtpPacketReceived; +class VideoFrame; +} // namespace webrtc + +namespace mozilla { +namespace dom { +struct RTCRtpSourceEntry; +} + +namespace dom { +struct RTCRtpSourceEntry; +} + +enum class MediaSessionConduitLocalDirection : int { kSend, kRecv }; + +class VideoSessionConduit; +class AudioSessionConduit; +class WebrtcCallWrapper; + +/** + * 1. Abstract renderer for video data + * 2. This class acts as abstract interface between the video-engine and + * video-engine agnostic renderer implementation. + * 3. Concrete implementation of this interface is responsible for + * processing and/or rendering the obtained raw video frame to appropriate + * output , say, <video> + */ +class VideoRenderer { + protected: + virtual ~VideoRenderer() {} + + public: + /** + * Callback Function reportng any change in the video-frame dimensions + * @param width: current width of the video @ decoder + * @param height: current height of the video @ decoder + */ + virtual void FrameSizeChange(unsigned int width, unsigned int height) = 0; + + /** + * Callback Function reporting decoded frame for processing. + * @param buffer: reference to decoded video frame + * @param buffer_size: size of the decoded frame + * @param time_stamp: Decoder timestamp, typically 90KHz as per RTP + * @render_time: Wall-clock time at the decoder for synchronization + * purposes in milliseconds + * NOTE: If decoded video frame is passed through buffer , it is the + * responsibility of the concrete implementations of this class to own copy + * of the frame if needed for time longer than scope of this callback. + * Such implementations should be quick in processing the frames and return + * immediately. + */ + virtual void RenderVideoFrame(const webrtc::VideoFrameBuffer& buffer, + uint32_t time_stamp, int64_t render_time) = 0; + + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoRenderer) +}; + +/** + * Generic Interface for representing Audio/Video Session + * MediaSession conduit is identified by 2 main components + * 1. Attached Transport Interface (through events) for inbound and outbound RTP + * transport + * 2. Attached Renderer Interface for rendering media data off the network + * This class hides specifics of Media-Engine implementation from the consumers + * of this interface. + * Also provides codec configuration API for the media sent and recevied + */ +class MediaSessionConduit { + protected: + virtual ~MediaSessionConduit() {} + + public: + enum Type { AUDIO, VIDEO }; + enum class PacketType { RTP, RTCP }; + + static std::string LocalDirectionToString( + const MediaSessionConduitLocalDirection aDirection) { + return aDirection == MediaSessionConduitLocalDirection::kSend ? "send" + : "receive"; + } + + virtual Type type() const = 0; + + // Call thread only + virtual Maybe<int> ActiveSendPayloadType() const = 0; + virtual Maybe<int> ActiveRecvPayloadType() const = 0; + + // Whether transport is currently sending and receiving packets + virtual void SetTransportActive(bool aActive) = 0; + + // Sending packets + virtual MediaEventSourceExc<MediaPacket>& SenderRtpSendEvent() = 0; + virtual MediaEventSourceExc<MediaPacket>& SenderRtcpSendEvent() = 0; + virtual MediaEventSourceExc<MediaPacket>& ReceiverRtcpSendEvent() = 0; + + // Receiving packets... + // from an rtp-receiving pipeline + virtual void ConnectReceiverRtpEvent( + MediaEventSourceExc<webrtc::RtpPacketReceived, webrtc::RTPHeader>& + aEvent) = 0; + // from an rtp-receiving pipeline + virtual void ConnectReceiverRtcpEvent( + MediaEventSourceExc<MediaPacket>& aEvent) = 0; + // from an rtp-transmitting pipeline + virtual void ConnectSenderRtcpEvent( + MediaEventSourceExc<MediaPacket>& aEvent) = 0; + + // Sts thread only. + virtual Maybe<uint16_t> RtpSendBaseSeqFor(uint32_t aSsrc) const = 0; + + // Any thread. + virtual const dom::RTCStatsTimestampMaker& GetTimestampMaker() const = 0; + + virtual Ssrcs GetLocalSSRCs() const = 0; + + virtual Maybe<Ssrc> GetRemoteSSRC() const = 0; + virtual void UnsetRemoteSSRC(Ssrc aSsrc) = 0; + + virtual void DisableSsrcChanges() = 0; + + virtual bool HasCodecPluginID(uint64_t aPluginID) const = 0; + + // Stuff for driving mute/unmute events + virtual MediaEventSource<void>& RtcpByeEvent() = 0; + virtual MediaEventSource<void>& RtcpTimeoutEvent() = 0; + virtual MediaEventSource<void>& RtpPacketEvent() = 0; + + virtual bool SendRtp(const uint8_t* aData, size_t aLength, + const webrtc::PacketOptions& aOptions) = 0; + virtual bool SendSenderRtcp(const uint8_t* aData, size_t aLength) = 0; + virtual bool SendReceiverRtcp(const uint8_t* aData, size_t aLength) = 0; + + virtual void DeliverPacket(rtc::CopyOnWriteBuffer packet, + PacketType type) = 0; + + virtual RefPtr<GenericPromise> Shutdown() = 0; + + virtual Maybe<RefPtr<AudioSessionConduit>> AsAudioSessionConduit() = 0; + virtual Maybe<RefPtr<VideoSessionConduit>> AsVideoSessionConduit() = 0; + + virtual Maybe<webrtc::CallBasicStats> GetCallStats() const = 0; + + NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaSessionConduit) + + void GetRtpSources(nsTArray<dom::RTCRtpSourceEntry>& outSources) const; + + virtual void SetJitterBufferTarget(DOMHighResTimeStamp aTargetMs) = 0; + + // test-only: inserts fake CSRCs and audio level data. + // NB: fake data is only valid during the current main thread task. + void InsertAudioLevelForContributingSource(const uint32_t aCsrcSource, + const int64_t aTimestamp, + const uint32_t aRtpTimestamp, + const bool aHasAudioLevel, + const uint8_t aAudioLevel); + + protected: + virtual std::vector<webrtc::RtpSource> GetUpstreamRtpSources() const = 0; + + private: + void UpdateRtpSources(const std::vector<webrtc::RtpSource>& aSources) const; + + // Marks the cache as having been updated in the current task, and keeps it + // stable until the current task is finished. + void OnSourcesUpdated() const; + + // Accessed only on main thread. This exists for a couple of reasons: + // 1. The webrtc spec says that source stats are updated using a queued task; + // libwebrtc's internal representation of these stats is updated without + // any task queueing, which means we need a mainthread-only cache. + // 2. libwebrtc uses its own clock that is not consistent with the one we + // need to use for stats (the so-called JS timestamps), which means we need + // to adjust the timestamps. Since timestamp adjustment is inexact and will + // not necessarily yield exactly the same result if performed again later, we + // need to avoid performing it more than once for each entry, which means we + // need to remember both the JS timestamp (in dom::RTCRtpSourceEntry) and the + // libwebrtc timestamp (in SourceKey::mLibwebrtcTimestampMs). + class SourceKey { + public: + explicit SourceKey(const webrtc::RtpSource& aSource) + : SourceKey(aSource.timestamp_ms(), aSource.source_id()) {} + + SourceKey(uint32_t aTimestamp, uint32_t aSrc) + : mLibwebrtcTimestampMs(aTimestamp), mSrc(aSrc) {} + + // TODO: Once we support = default for this in our toolchain, do so + auto operator>(const SourceKey& aRhs) const { + if (mLibwebrtcTimestampMs == aRhs.mLibwebrtcTimestampMs) { + return mSrc > aRhs.mSrc; + } + return mLibwebrtcTimestampMs > aRhs.mLibwebrtcTimestampMs; + } + + private: + uint32_t mLibwebrtcTimestampMs; + uint32_t mSrc; + }; + mutable std::map<SourceKey, dom::RTCRtpSourceEntry, std::greater<SourceKey>> + mSourcesCache; + // Accessed only on main thread. A flag saying whether mSourcesCache needs + // updating. Ensures that get*Sources() appear stable from javascript + // throughout a main thread task, even though we don't follow the spec to the + // letter (dispatch a task to update the sources). + mutable bool mSourcesUpdateNeeded = true; +}; + +class WebrtcSendTransport : public webrtc::Transport { + // WeakRef to the owning conduit + MediaSessionConduit* mConduit; + + public: + explicit WebrtcSendTransport(MediaSessionConduit* aConduit) + : mConduit(aConduit) {} + bool SendRtp(const uint8_t* aPacket, size_t aLength, + const webrtc::PacketOptions& aOptions) override { + return mConduit->SendRtp(aPacket, aLength, aOptions); + } + bool SendRtcp(const uint8_t* aPacket, size_t aLength) override { + return mConduit->SendSenderRtcp(aPacket, aLength); + } +}; + +class WebrtcReceiveTransport : public webrtc::Transport { + // WeakRef to the owning conduit + MediaSessionConduit* mConduit; + + public: + explicit WebrtcReceiveTransport(MediaSessionConduit* aConduit) + : mConduit(aConduit) {} + bool SendRtp(const uint8_t* aPacket, size_t aLength, + const webrtc::PacketOptions& aOptions) override { + MOZ_CRASH("Unexpected RTP packet"); + } + bool SendRtcp(const uint8_t* aPacket, size_t aLength) override { + return mConduit->SendReceiverRtcp(aPacket, aLength); + } +}; + +// Abstract base classes for external encoder/decoder. + +// Interface to help signal PluginIDs +class CodecPluginID { + public: + virtual MediaEventSource<uint64_t>* InitPluginEvent() { return nullptr; } + virtual MediaEventSource<uint64_t>* ReleasePluginEvent() { return nullptr; } + virtual ~CodecPluginID() {} +}; + +class VideoEncoder : public CodecPluginID { + public: + virtual ~VideoEncoder() {} +}; + +class VideoDecoder : public CodecPluginID { + public: + virtual ~VideoDecoder() {} +}; + +/** + * MediaSessionConduit for video + * Refer to the comments on MediaSessionConduit above for overall + * information + */ +class VideoSessionConduit : public MediaSessionConduit { + public: + struct Options { + bool mVideoLatencyTestEnable = false; + // All in bps. + int mMinBitrate = 0; + int mStartBitrate = 0; + int mPrefMaxBitrate = 0; + int mMinBitrateEstimate = 0; + bool mDenoising = false; + bool mLockScaling = false; + uint8_t mSpatialLayers = 1; + uint8_t mTemporalLayers = 1; + }; + + /** + * Factory function to create and initialize a Video Conduit Session + * @param webrtc::Call instance shared by paired audio and video + * media conduits + * @param aOptions are a number of options, typically from prefs, used to + * configure the created VideoConduit. + * @param aPCHandle is a string representing the RTCPeerConnection that is + * creating this VideoConduit. This is used when reporting GMP plugin + * crashes. + * @result Concrete VideoSessionConduitObject or nullptr in the case + * of failure + */ + static RefPtr<VideoSessionConduit> Create( + RefPtr<WebrtcCallWrapper> aCall, + nsCOMPtr<nsISerialEventTarget> aStsThread, Options aOptions, + std::string aPCHandle, const TrackingId& aRecvTrackingId); + + enum FrameRequestType { + FrameRequestNone, + FrameRequestFir, + FrameRequestPli, + FrameRequestUnknown + }; + + VideoSessionConduit() + : mFrameRequestMethod(FrameRequestNone), + mUsingNackBasic(false), + mUsingTmmbr(false), + mUsingFEC(false) {} + + virtual ~VideoSessionConduit() {} + + Type type() const override { return VIDEO; } + + Maybe<RefPtr<AudioSessionConduit>> AsAudioSessionConduit() override { + return Nothing(); + } + + Maybe<RefPtr<VideoSessionConduit>> AsVideoSessionConduit() override { + return Some(RefPtr<VideoSessionConduit>(this)); + } + + /** + * Hooks up mControl Mirrors with aControl Canonicals, and sets up + * mWatchManager to react on Mirror changes. + */ + virtual void InitControl(VideoConduitControlInterface* aControl) = 0; + + /** + * Function to attach Renderer end-point of the Media-Video conduit. + * @param aRenderer : Reference to the concrete Video renderer implementation + * Note: Multiple invocations of this API shall remove an existing renderer + * and attaches the new to the Conduit. + */ + virtual MediaConduitErrorCode AttachRenderer( + RefPtr<mozilla::VideoRenderer> aRenderer) = 0; + virtual void DetachRenderer() = 0; + + /** + * Function to deliver a capture video frame for encoding and transport. + * If the frame's timestamp is 0, it will be automatcally generated. + * + * NOTE: ConfigureSendMediaCodec() must be called before this function can + * be invoked. This ensures the inserted video-frames can be + * transmitted by the conduit. + */ + virtual MediaConduitErrorCode SendVideoFrame(webrtc::VideoFrame aFrame) = 0; + + /** + * These methods allow unit tests to double-check that the + * rtcp-fb settings are as expected. + */ + FrameRequestType FrameRequestMethod() const { return mFrameRequestMethod; } + + bool UsingNackBasic() const { return mUsingNackBasic; } + + bool UsingTmmbr() const { return mUsingTmmbr; } + + bool UsingFEC() const { return mUsingFEC; } + + virtual Maybe<webrtc::VideoReceiveStreamInterface::Stats> GetReceiverStats() + const = 0; + virtual Maybe<webrtc::VideoSendStream::Stats> GetSenderStats() const = 0; + + virtual void CollectTelemetryData() = 0; + + virtual bool AddFrameHistory( + dom::Sequence<dom::RTCVideoFrameHistoryInternal>* outHistories) const = 0; + + virtual Maybe<Ssrc> GetAssociatedLocalRtxSSRC(Ssrc aSsrc) const = 0; + + protected: + /* RTCP feedback settings, for unit testing purposes */ + FrameRequestType mFrameRequestMethod; + bool mUsingNackBasic; + bool mUsingTmmbr; + bool mUsingFEC; +}; + +/** + * MediaSessionConduit for audio + * Refer to the comments on MediaSessionConduit above for overall + * information + */ +class AudioSessionConduit : public MediaSessionConduit { + public: + /** + * Factory function to create and initialize an Audio Conduit Session + * @param webrtc::Call instance shared by paired audio and video + * media conduits + * @result Concrete AudioSessionConduitObject or nullptr in the case + * of failure + */ + static RefPtr<AudioSessionConduit> Create( + RefPtr<WebrtcCallWrapper> aCall, + nsCOMPtr<nsISerialEventTarget> aStsThread); + + virtual ~AudioSessionConduit() {} + + Type type() const override { return AUDIO; } + + Maybe<RefPtr<AudioSessionConduit>> AsAudioSessionConduit() override { + return Some(this); + } + + Maybe<RefPtr<VideoSessionConduit>> AsVideoSessionConduit() override { + return Nothing(); + } + + /** + * Hooks up mControl Mirrors with aControl Canonicals, and sets up + * mWatchManager to react on Mirror changes. + */ + virtual void InitControl(AudioConduitControlInterface* aControl) = 0; + + /** + * Function to deliver externally captured audio sample for encoding and + * transport + * @param frame [in]: AudioFrame in upstream's format for forwarding to the + * send stream. Ownership is passed along. + * NOTE: ConfigureSendMediaCodec() SHOULD be called before this function can + * be invoked. This ensures the inserted audio-samples can be transmitted by + * the conduit. + */ + virtual MediaConduitErrorCode SendAudioFrame( + std::unique_ptr<webrtc::AudioFrame> frame) = 0; + + /** + * Function to grab a decoded audio-sample from the media engine for + * rendering / playout of length 10 milliseconds. + * + * @param samplingFreqHz [in]: Frequency of the sampling for playback in + * Hertz (16000, 32000,..) + * @param frame [in/out]: Pointer to an AudioFrame to which audio data will be + * copied + * NOTE: This function should be invoked every 10 milliseconds for the best + * performance + * NOTE: ConfigureRecvMediaCodec() SHOULD be called before this function can + * be invoked + * This ensures the decoded samples are ready for reading and playout is + * enabled. + */ + virtual MediaConduitErrorCode GetAudioFrame(int32_t samplingFreqHz, + webrtc::AudioFrame* frame) = 0; + + /** + * Checks if given sampling frequency is supported + * @param freq: Sampling rate (in Hz) to check + */ + virtual bool IsSamplingFreqSupported(int freq) const = 0; + + virtual Maybe<webrtc::AudioReceiveStreamInterface::Stats> GetReceiverStats() + const = 0; + virtual Maybe<webrtc::AudioSendStream::Stats> GetSenderStats() const = 0; +}; +} // namespace mozilla +#endif |