path: root/dom/media/webrtc/libwebrtcglue/VideoConduit.h

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef VIDEO_SESSION_H_
#define VIDEO_SESSION_H_

#include "mozilla/Atomics.h"
#include "mozilla/Attributes.h"
#include "mozilla/DataMutex.h"
#include "mozilla/ReentrantMonitor.h"
#include "mozilla/SharedThreadPool.h"
#include "mozilla/StateMirroring.h"
#include "mozilla/UniquePtr.h"
#include "nsITimer.h"

#include "MediaConduitInterface.h"
#include "RtpRtcpConfig.h"
#include "RunningStat.h"
#include "transport/runnable_utils.h"

// conflicts with #include of scoped_ptr.h
#undef FF
// Video Engine Includes
#include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/sdp_video_format.h"
#include "call/call_basic_stats.h"
#include "common_video/include/video_frame_buffer_pool.h"
#include "media/base/video_broadcaster.h"
#include <functional>
#include <memory>
/** This file hosts several structures identifying different aspects
 * of a RTP Session.
 */

namespace mozilla {

// Convert (SI) kilobits/sec to (SI) bits/sec
#define KBPS(kbps) kbps * 1000

const int kViEMinCodecBitrate_bps = KBPS(30);
const unsigned int kVideoMtu = 1200;
const int kQpMax = 56;

template <typename T>
T MinIgnoreZero(const T& a, const T& b) {
  return std::min(a ? a : b, b ? b : a);
}

class VideoStreamFactory;
class WebrtcAudioConduit;

// Interface of external video encoder for WebRTC.
class WebrtcVideoEncoder : public VideoEncoder, public webrtc::VideoEncoder {};

// Interface of external video decoder for WebRTC.
class WebrtcVideoDecoder : public VideoDecoder, public webrtc::VideoDecoder {};

/**
 * Concrete class for Video session. Hooks up
 *  - media-source and target to external transport
 */
class WebrtcVideoConduit
    : public VideoSessionConduit,
      public webrtc::RtcpEventObserver,
      public rtc::VideoSinkInterface<webrtc::VideoFrame>,
      public rtc::VideoSourceInterface<webrtc::VideoFrame> {
 public:
  // Returns true when both encoder and decoder are HW accelerated.
  static bool HasH264Hardware();

  Maybe<int> ActiveSendPayloadType() const override;
  Maybe<int> ActiveRecvPayloadType() const override;

  /**
   * Function to attach Renderer end-point for the Media-Video conduit.
   * @param aRenderer : Reference to the concrete mozilla Video renderer
   * implementation Note: Multiple invocations of this API shall remove an
   * existing renderer and attaches the new to the Conduit.
   */
  MediaConduitErrorCode AttachRenderer(
      RefPtr<mozilla::VideoRenderer> aVideoRenderer) override;
  void DetachRenderer() override;

  Maybe<uint16_t> RtpSendBaseSeqFor(uint32_t aSsrc) const override;

  const dom::RTCStatsTimestampMaker& GetTimestampMaker() const override;

  void StopTransmitting();
  void StartTransmitting();
  void StopReceiving();
  void StartReceiving();

  /**
   * Function to deliver a capture video frame for encoding and transport.
   * If the frame's timestamp is 0, it will be automatically generated.
   *
   * NOTE: ConfigureSendMediaCodec() must be called before this function can
   *       be invoked. This ensures the inserted video-frames can be
   *       transmitted by the conduit.
   */
  MediaConduitErrorCode SendVideoFrame(webrtc::VideoFrame aFrame) override;

  bool SendRtp(const uint8_t* aData, size_t aLength,
               const webrtc::PacketOptions& aOptions) override;
  bool SendSenderRtcp(const uint8_t* aData, size_t aLength) override;
  bool SendReceiverRtcp(const uint8_t* aData, size_t aLength) override;

  /*
   * webrtc:VideoSinkInterface implementation
   * -------------------------------
   */
  void OnFrame(const webrtc::VideoFrame& frame) override;

  /*
   * webrtc:VideoSourceInterface implementation
   * -------------------------------
   */
  void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink,
                       const rtc::VideoSinkWants& wants) override;
  void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override;

  bool HasCodecPluginID(uint64_t aPluginID) const override;

  RefPtr<GenericPromise> Shutdown() override;

  bool Denoising() const { return mDenoising; }

  uint8_t SpatialLayers() const { return mSpatialLayers; }

  uint8_t TemporalLayers() const { return mTemporalLayers; }

  webrtc::VideoCodecMode CodecMode() const;

  WebrtcVideoConduit(RefPtr<WebrtcCallWrapper> aCall,
                     nsCOMPtr<nsISerialEventTarget> aStsThread,
                     Options aOptions, std::string aPCHandle,
                     const TrackingId& aRecvTrackingId);
  virtual ~WebrtcVideoConduit();

  // Call thread.
  void InitControl(VideoConduitControlInterface* aControl) override;

  // Called when a parameter in mControl has changed. Call thread.
  void OnControlConfigChange();

  // Necessary Init steps on main thread.
  MediaConduitErrorCode Init();

  Ssrcs GetLocalSSRCs() const override;
  Maybe<Ssrc> GetAssociatedLocalRtxSSRC(Ssrc aSsrc) const override;
  Maybe<Ssrc> GetRemoteSSRC() const override;

  Maybe<VideoSessionConduit::Resolution> GetLastResolution() const override;

  // Call thread.
  void UnsetRemoteSSRC(uint32_t aSsrc) override;

  static unsigned ToLibwebrtcMaxFramerate(const Maybe<double>& aMaxFramerate);

 private:
  void NotifyUnsetCurrentRemoteSSRC();
  void SetRemoteSSRCConfig(uint32_t aSsrc, uint32_t aRtxSsrc);
  void SetRemoteSSRCAndRestartAsNeeded(uint32_t aSsrc, uint32_t aRtxSsrc);
  rtc::RefCountedObject<mozilla::VideoStreamFactory>*
  CreateVideoStreamFactory();

 public:
  // Creating a recv stream or a send stream requires a local ssrc to be
  // configured. This method will generate one if needed.
  void EnsureLocalSSRC();
  // Creating a recv stream requires a remote ssrc to be configured. This method
  // will generate one if needed.
  void EnsureRemoteSSRC();

  Maybe<webrtc::VideoReceiveStreamInterface::Stats> GetReceiverStats()
      const override;
  Maybe<webrtc::VideoSendStream::Stats> GetSenderStats() const override;
  Maybe<webrtc::CallBasicStats> GetCallStats() const override;

  bool AddFrameHistory(dom::Sequence<dom::RTCVideoFrameHistoryInternal>*
                           outHistories) const override;

  void SetJitterBufferTarget(DOMHighResTimeStamp aTargetMs) override;

  uint64_t MozVideoLatencyAvg();

  void DisableSsrcChanges() override {
    MOZ_ASSERT(mCallThread->IsOnCurrentThread());
    mAllowSsrcChange = false;
  }

  void CollectTelemetryData() override;

  void OnRtpReceived(webrtc::RtpPacketReceived&& aPacket,
                     webrtc::RTPHeader&& aHeader);

  void OnRtcpBye() override;
  void OnRtcpTimeout() override;

  void SetTransportActive(bool aActive) override;

  MediaEventSourceExc<MediaPacket>& SenderRtpSendEvent() override {
    return mSenderRtpSendEvent;
  }
  MediaEventSourceExc<MediaPacket>& SenderRtcpSendEvent() override {
    return mSenderRtcpSendEvent;
  }
  MediaEventSourceExc<MediaPacket>& ReceiverRtcpSendEvent() override {
    return mReceiverRtcpSendEvent;
  }
  void ConnectReceiverRtpEvent(
      MediaEventSourceExc<webrtc::RtpPacketReceived, webrtc::RTPHeader>& aEvent)
      override {
    mReceiverRtpEventListener =
        aEvent.Connect(mCallThread, this, &WebrtcVideoConduit::OnRtpReceived);
  }

  std::vector<webrtc::RtpSource> GetUpstreamRtpSources() const override;

  void RequestKeyFrame(FrameTransformerProxy* aProxy) override;
  void GenerateKeyFrame(const Maybe<std::string>& aRid,
                        FrameTransformerProxy* aProxy) override;

 private:
  // Don't allow copying/assigning.
  WebrtcVideoConduit(const WebrtcVideoConduit&) = delete;
  void operator=(const WebrtcVideoConduit&) = delete;

  // Utility function to dump recv codec database
  void DumpCodecDB() const;

  // Video Latency Test averaging filter
  void VideoLatencyUpdate(uint64_t aNewSample);

  void CreateSendStream();
  void DeleteSendStream();
  void CreateRecvStream();
  void DeleteRecvStream();

  void DeliverPacket(rtc::CopyOnWriteBuffer packet, PacketType type) override;

  MediaEventSource<void>& RtcpByeEvent() override { return mRtcpByeEvent; }
  MediaEventSource<void>& RtcpTimeoutEvent() override {
    return mRtcpTimeoutEvent;
  }
  MediaEventSource<void>& RtpPacketEvent() override { return mRtpPacketEvent; }

  bool RequiresNewSendStream(const VideoCodecConfig& newConfig) const;

  mutable mozilla::ReentrantMonitor mRendererMonitor MOZ_UNANNOTATED;

  // Accessed on any thread under mRendererMonitor.
  RefPtr<mozilla::VideoRenderer> mRenderer;

  // Accessed on any thread under mRendererMonitor.
  unsigned short mReceivingWidth = 0;

  // Accessed on any thread under mRendererMonitor.
  unsigned short mReceivingHeight = 0;

  // Call worker thread. All access to mCall->Call() happens here.
  const nsCOMPtr<nsISerialEventTarget> mCallThread;

  // Socket transport service thread that runs stats queries against us. Any
  // thread.
  const nsCOMPtr<nsISerialEventTarget> mStsThread;

  // Thread on which we are fed video frames. Set lazily on first call to
  // SendVideoFrame().
  nsCOMPtr<nsISerialEventTarget> mFrameSendingThread;

  struct Control {
    // Mirrors that map to VideoConduitControlInterface for control. Call thread
    // only.
    Mirror<bool> mReceiving;
    Mirror<bool> mTransmitting;
    Mirror<Ssrcs> mLocalSsrcs;
    Mirror<Ssrcs> mLocalRtxSsrcs;
    Mirror<std::string> mLocalCname;
    Mirror<std::string> mMid;
    Mirror<Ssrc> mRemoteSsrc;
    Mirror<Ssrc> mRemoteRtxSsrc;
    Mirror<std::string> mSyncGroup;
    Mirror<RtpExtList> mLocalRecvRtpExtensions;
    Mirror<RtpExtList> mLocalSendRtpExtensions;
    Mirror<Maybe<VideoCodecConfig>> mSendCodec;
    Mirror<Maybe<RtpRtcpConfig>> mSendRtpRtcpConfig;
    Mirror<std::vector<VideoCodecConfig>> mRecvCodecs;
    Mirror<Maybe<RtpRtcpConfig>> mRecvRtpRtcpConfig;
    Mirror<webrtc::VideoCodecMode> mCodecMode;
    Mirror<RefPtr<FrameTransformerProxy>> mFrameTransformerProxySend;
    Mirror<RefPtr<FrameTransformerProxy>> mFrameTransformerProxyRecv;

    // For caching mRemoteSsrc and mRemoteRtxSsrc, since another caller may
    // change the remote ssrc in the stream config directly.
    Ssrc mConfiguredRemoteSsrc = 0;
    Ssrc mConfiguredRemoteRtxSsrc = 0;
    // For tracking changes to mSendCodec and mSendRtpRtcpConfig.
    Maybe<VideoCodecConfig> mConfiguredSendCodec;
    Maybe<RtpRtcpConfig> mConfiguredSendRtpRtcpConfig;
    // For tracking changes to mRecvCodecs and mRecvRtpRtcpConfig.
    std::vector<VideoCodecConfig> mConfiguredRecvCodecs;
    Maybe<RtpRtcpConfig> mConfiguredRecvRtpRtcpConfig;

    // For change tracking. Callthread only.
    RefPtr<FrameTransformerProxy> mConfiguredFrameTransformerProxySend;
    RefPtr<FrameTransformerProxy> mConfiguredFrameTransformerProxyRecv;

    Control() = delete;
    explicit Control(const RefPtr<AbstractThread>& aCallThread);
  } mControl;

  // WatchManager allowing Mirrors and other watch targets to trigger functions
  // that will update the webrtc.org configuration.
  WatchManager<WebrtcVideoConduit> mWatchManager;

  mutable Mutex mMutex MOZ_UNANNOTATED;

  // Decoder factory used by mRecvStream when it needs new decoders. This is
  // not shared broader like some state in the WebrtcCallWrapper because it
  // handles CodecPluginID plumbing tied to this VideoConduit.
  const UniquePtr<WebrtcVideoDecoderFactory> mDecoderFactory;

  // Encoder factory used by mSendStream when it needs new encoders. This is
  // not shared broader like some state in the WebrtcCallWrapper because it
  // handles CodecPluginID plumbing tied to this VideoConduit.
  const UniquePtr<WebrtcVideoEncoderFactory> mEncoderFactory;

  // Our own record of the sinks added to mVideoBroadcaster so we can support
  // dispatching updates to sinks from off-Call-thread. Call thread only.
  AutoTArray<rtc::VideoSinkInterface<webrtc::VideoFrame>*, 1> mRegisteredSinks;

  // Broadcaster that distributes our frames to all registered sinks.
  // Threadsafe.
  rtc::VideoBroadcaster mVideoBroadcaster;

  // Buffer pool used for scaling frames.
  // Accessed on the frame-feeding thread only.
  webrtc::VideoFrameBufferPool mBufferPool;

  // Engine state we are concerned with. Written on the Call thread and read
  // anywhere.
  mozilla::Atomic<bool>
      mEngineTransmitting;  // If true ==> Transmit Subsystem is up and running
  mozilla::Atomic<bool>
      mEngineReceiving;  // if true ==> Receive Subsystem up and running

  // Written only on the Call thread. Guarded by mMutex, except for reads on the
  // Call thread.
  Maybe<VideoCodecConfig> mCurSendCodecConfig;

  // Bookkeeping of stats for telemetry. Call thread only.
  RunningStat mSendFramerate;
  RunningStat mSendBitrate;
  RunningStat mRecvFramerate;
  RunningStat mRecvBitrate;

  // Must call webrtc::Call::DestroyVideoReceive/SendStream to delete this.
  // Written only on the Call thread. Guarded by mMutex, except for reads on the
  // Call thread.
  webrtc::VideoReceiveStreamInterface* mRecvStream = nullptr;

  // Must call webrtc::Call::DestroyVideoReceive/SendStream to delete this.
  webrtc::VideoSendStream* mSendStream = nullptr;

  // Written on the frame feeding thread.
  // Guarded by mMutex, except for reads on the frame feeding thread.
  unsigned short mLastWidth = 0;

  // Written on the frame feeding thread.
  // Guarded by mMutex, except for reads on the frame feeding thread.
  unsigned short mLastHeight = 0;

  // Written on the frame feeding thread, the timestamp of the last frame on the
  // send side, in microseconds. This is a local timestamp using the system
  // clock with a unspecified epoch (Like mozilla::TimeStamp).
  // Guarded by mMutex.
  Maybe<uint64_t> mLastTimestampSendUs;

  // Written on the frame receive thread, the rtp timestamp of the last frame
  // on the receive side, in 90kHz base. This comes from the RTP packet.
  // Guarded by mMutex.
  Maybe<uint32_t> mLastRTPTimestampReceive;

  // Accessed from any thread under mRendererMonitor.
  uint64_t mVideoLatencyAvg = 0;

  const bool mVideoLatencyTestEnable;

  // All in bps.
  const int mMinBitrate;
  const int mStartBitrate;
  const int mPrefMaxBitrate;
  const int mMinBitrateEstimate;

  // Max bitrate in bps as provided by negotiation. Call thread only.
  int mNegotiatedMaxBitrate = 0;

  // Set to true to force denoising on.
  const bool mDenoising;

  // Set to true to ignore sink wants (scaling due to bwe and cpu usage).
  const bool mLockScaling;

  const uint8_t mSpatialLayers;
  const uint8_t mTemporalLayers;

  static const unsigned int sAlphaNum = 7;
  static const unsigned int sAlphaDen = 8;
  static const unsigned int sRoundingPadding = 1024;

  // Target jitter buffer to be applied to the receive stream in milliseconds.
  uint16_t mJitterBufferTargetMs = 0;

  // WEBRTC.ORG Call API
  // Const so can be accessed on any thread. All methods are called on the Call
  // thread.
  const RefPtr<WebrtcCallWrapper> mCall;

  // Set up in the ctor and then not touched. Called through by the streams on
  // any thread. Safe since we own and control the lifetime of the streams.
  WebrtcSendTransport mSendTransport;
  WebrtcReceiveTransport mRecvTransport;

  // Written only on the Call thread. Guarded by mMutex, except for reads on the
  // Call thread. Typical non-Call thread access is on the frame delivery
  // thread.
  webrtc::VideoSendStream::Config mSendStreamConfig;

  // Call thread only.
  webrtc::VideoEncoderConfig mEncoderConfig;

  // Written only on the Call thread. Guarded by mMutex, except for reads on the
  // Call thread. Calls can happen under mMutex on any thread.
  DataMutex<RefPtr<rtc::RefCountedObject<VideoStreamFactory>>>
      mVideoStreamFactory;

  // Call thread only.
  webrtc::VideoReceiveStreamInterface::Config mRecvStreamConfig;

  // Are SSRC changes without signaling allowed or not.
  // Call thread only.
  bool mAllowSsrcChange = true;

  // Accessed during configuration/signaling (Call thread), and on the frame
  // delivery thread for frame history tracking. Set only on the Call thread.
  Atomic<uint32_t> mRecvSSRC =
      Atomic<uint32_t>(0);  // this can change during a stream!

  // Accessed from both the STS and frame delivery thread for frame history
  // tracking. Set when receiving packets.
  Atomic<uint32_t> mRemoteSendSSRC =
      Atomic<uint32_t>(0);  // this can change during a stream!

  // Main thread only
  nsTArray<uint64_t> mSendCodecPluginIDs;
  // Main thread only
  nsTArray<uint64_t> mRecvCodecPluginIDs;

  // Main thread only
  MediaEventListener mSendPluginCreated;
  MediaEventListener mSendPluginReleased;
  MediaEventListener mRecvPluginCreated;
  MediaEventListener mRecvPluginReleased;

  // Call thread only. ssrc -> base_seq
  std::map<uint32_t, uint16_t> mRtpSendBaseSeqs;
  // libwebrtc network thread only. ssrc -> base_seq.
  // To track changes needed to mRtpSendBaseSeqs.
  std::map<uint32_t, uint16_t> mRtpSendBaseSeqs_n;

  // Tracking the attributes of received frames over time
  // Protected by mRendererMonitor
  dom::RTCVideoFrameHistoryInternal mReceivedFrameHistory;

  // Written only on the main thread.  Guarded by mMutex, except for
  // reads on the main thread.
  std::vector<webrtc::RtpSource> mRtpSources;

  // Thread safe
  Atomic<bool> mTransportActive = Atomic<bool>(false);
  MediaEventProducer<void> mRtcpByeEvent;
  MediaEventProducer<void> mRtcpTimeoutEvent;
  MediaEventProducer<void> mRtpPacketEvent;
  MediaEventProducerExc<MediaPacket> mSenderRtpSendEvent;
  MediaEventProducerExc<MediaPacket> mSenderRtcpSendEvent;
  MediaEventProducerExc<MediaPacket> mReceiverRtcpSendEvent;

  // Assigned and revoked on mStsThread. Listeners for receiving packets.
  MediaEventListener mReceiverRtpEventListener;  // Rtp-receiving pipeline
};
}  // namespace mozilla

#endif