summaryrefslogtreecommitdiffstats
path: root/dom/media/webrtc/libwebrtcglue/VideoConduit.h
blob: bb15cd35a9b117202818e242e3ba071c1e6d2166 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef VIDEO_SESSION_H_
#define VIDEO_SESSION_H_

#include "mozilla/Atomics.h"
#include "mozilla/Attributes.h"
#include "mozilla/DataMutex.h"
#include "mozilla/ReentrantMonitor.h"
#include "mozilla/SharedThreadPool.h"
#include "mozilla/StateMirroring.h"
#include "mozilla/UniquePtr.h"
#include "nsITimer.h"

#include "MediaConduitInterface.h"
#include "RtpRtcpConfig.h"
#include "RunningStat.h"
#include "transport/runnable_utils.h"

// conflicts with #include of scoped_ptr.h
#undef FF
// Video Engine Includes
#include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/sdp_video_format.h"
#include "call/call_basic_stats.h"
#include "common_video/include/video_frame_buffer_pool.h"
#include "media/base/video_broadcaster.h"
#include <functional>
#include <memory>
/** This file hosts several structures identifying different aspects
 * of a RTP Session.
 */

namespace mozilla {

// Convert (SI) kilobits/sec to (SI) bits/sec
#define KBPS(kbps) kbps * 1000

const int kViEMinCodecBitrate_bps = KBPS(30);
const unsigned int kVideoMtu = 1200;
const int kQpMax = 56;

template <typename T>
T MinIgnoreZero(const T& a, const T& b) {
  return std::min(a ? a : b, b ? b : a);
}

class VideoStreamFactory;
class WebrtcAudioConduit;

// Interface of external video encoder for WebRTC.
class WebrtcVideoEncoder : public VideoEncoder, public webrtc::VideoEncoder {};

// Interface of external video decoder for WebRTC.
class WebrtcVideoDecoder : public VideoDecoder, public webrtc::VideoDecoder {};

/**
 * Concrete class for Video session. Hooks up
 *  - media-source and target to external transport
 */
class WebrtcVideoConduit
    : public VideoSessionConduit,
      public webrtc::RtcpEventObserver,
      public rtc::VideoSinkInterface<webrtc::VideoFrame>,
      public rtc::VideoSourceInterface<webrtc::VideoFrame> {
 public:
  // Returns true when both encoder and decoder are HW accelerated.
  static bool HasH264Hardware();

  Maybe<int> ActiveSendPayloadType() const override;
  Maybe<int> ActiveRecvPayloadType() const override;

  /**
   * Function to attach Renderer end-point for the Media-Video conduit.
   * @param aRenderer : Reference to the concrete mozilla Video renderer
   * implementation Note: Multiple invocations of this API shall remove an
   * existing renderer and attaches the new to the Conduit.
   */
  MediaConduitErrorCode AttachRenderer(
      RefPtr<mozilla::VideoRenderer> aVideoRenderer) override;
  void DetachRenderer() override;

  Maybe<uint16_t> RtpSendBaseSeqFor(uint32_t aSsrc) const override;

  const dom::RTCStatsTimestampMaker& GetTimestampMaker() const override;

  void StopTransmitting();
  void StartTransmitting();
  void StopReceiving();
  void StartReceiving();

  /**
   * Function to deliver a capture video frame for encoding and transport.
   * If the frame's timestamp is 0, it will be automatically generated.
   *
   * NOTE: ConfigureSendMediaCodec() must be called before this function can
   *       be invoked. This ensures the inserted video-frames can be
   *       transmitted by the conduit.
   */
  MediaConduitErrorCode SendVideoFrame(webrtc::VideoFrame aFrame) override;

  bool SendRtp(const uint8_t* aData, size_t aLength,
               const webrtc::PacketOptions& aOptions) override;
  bool SendSenderRtcp(const uint8_t* aData, size_t aLength) override;
  bool SendReceiverRtcp(const uint8_t* aData, size_t aLength) override;

  /*
   * webrtc:VideoSinkInterface implementation
   * -------------------------------
   */
  void OnFrame(const webrtc::VideoFrame& frame) override;

  /*
   * webrtc:VideoSourceInterface implementation
   * -------------------------------
   */
  void AddOrUpdateSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink,
                       const rtc::VideoSinkWants& wants) override;
  void RemoveSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override;

  bool HasCodecPluginID(uint64_t aPluginID) const override;

  RefPtr<GenericPromise> Shutdown() override;

  bool Denoising() const { return mDenoising; }

  uint8_t SpatialLayers() const { return mSpatialLayers; }

  uint8_t TemporalLayers() const { return mTemporalLayers; }

  webrtc::VideoCodecMode CodecMode() const;

  WebrtcVideoConduit(RefPtr<WebrtcCallWrapper> aCall,
                     nsCOMPtr<nsISerialEventTarget> aStsThread,
                     Options aOptions, std::string aPCHandle,
                     const TrackingId& aRecvTrackingId);
  virtual ~WebrtcVideoConduit();

  // Call thread.
  void InitControl(VideoConduitControlInterface* aControl) override;

  // Called when a parameter in mControl has changed. Call thread.
  void OnControlConfigChange();

  // Necessary Init steps on main thread.
  MediaConduitErrorCode Init();

  Ssrcs GetLocalSSRCs() const override;
  Maybe<Ssrc> GetAssociatedLocalRtxSSRC(Ssrc aSsrc) const override;
  Maybe<Ssrc> GetRemoteSSRC() const override;

  Maybe<VideoSessionConduit::Resolution> GetLastResolution() const override;

  // Call thread.
  void UnsetRemoteSSRC(uint32_t aSsrc) override;

  static unsigned ToLibwebrtcMaxFramerate(const Maybe<double>& aMaxFramerate);

 private:
  void NotifyUnsetCurrentRemoteSSRC();
  void SetRemoteSSRCConfig(uint32_t aSsrc, uint32_t aRtxSsrc);
  void SetRemoteSSRCAndRestartAsNeeded(uint32_t aSsrc, uint32_t aRtxSsrc);
  rtc::RefCountedObject<mozilla::VideoStreamFactory>*
  CreateVideoStreamFactory();

 public:
  // Creating a recv stream or a send stream requires a local ssrc to be
  // configured. This method will generate one if needed.
  void EnsureLocalSSRC();
  // Creating a recv stream requires a remote ssrc to be configured. This method
  // will generate one if needed.
  void EnsureRemoteSSRC();

  Maybe<webrtc::VideoReceiveStreamInterface::Stats> GetReceiverStats()
      const override;
  Maybe<webrtc::VideoSendStream::Stats> GetSenderStats() const override;
  Maybe<webrtc::CallBasicStats> GetCallStats() const override;

  bool AddFrameHistory(dom::Sequence<dom::RTCVideoFrameHistoryInternal>*
                           outHistories) const override;

  void SetJitterBufferTarget(DOMHighResTimeStamp aTargetMs) override;

  uint64_t MozVideoLatencyAvg();

  void DisableSsrcChanges() override {
    MOZ_ASSERT(mCallThread->IsOnCurrentThread());
    mAllowSsrcChange = false;
  }

  void CollectTelemetryData() override;

  void OnRtpReceived(webrtc::RtpPacketReceived&& aPacket,
                     webrtc::RTPHeader&& aHeader);

  void OnRtcpBye() override;
  void OnRtcpTimeout() override;

  void SetTransportActive(bool aActive) override;

  MediaEventSourceExc<MediaPacket>& SenderRtpSendEvent() override {
    return mSenderRtpSendEvent;
  }
  MediaEventSourceExc<MediaPacket>& SenderRtcpSendEvent() override {
    return mSenderRtcpSendEvent;
  }
  MediaEventSourceExc<MediaPacket>& ReceiverRtcpSendEvent() override {
    return mReceiverRtcpSendEvent;
  }
  void ConnectReceiverRtpEvent(
      MediaEventSourceExc<webrtc::RtpPacketReceived, webrtc::RTPHeader>& aEvent)
      override {
    mReceiverRtpEventListener =
        aEvent.Connect(mCallThread, this, &WebrtcVideoConduit::OnRtpReceived);
  }

  std::vector<webrtc::RtpSource> GetUpstreamRtpSources() const override;

  void RequestKeyFrame(FrameTransformerProxy* aProxy) override;
  void GenerateKeyFrame(const Maybe<std::string>& aRid,
                        FrameTransformerProxy* aProxy) override;

 private:
  // Don't allow copying/assigning.
  WebrtcVideoConduit(const WebrtcVideoConduit&) = delete;
  void operator=(const WebrtcVideoConduit&) = delete;

  // Utility function to dump recv codec database
  void DumpCodecDB() const;

  // Video Latency Test averaging filter
  void VideoLatencyUpdate(uint64_t aNewSample);

  void CreateSendStream();
  void DeleteSendStream();
  void CreateRecvStream();
  void DeleteRecvStream();

  void DeliverPacket(rtc::CopyOnWriteBuffer packet, PacketType type) override;

  MediaEventSource<void>& RtcpByeEvent() override { return mRtcpByeEvent; }
  MediaEventSource<void>& RtcpTimeoutEvent() override {
    return mRtcpTimeoutEvent;
  }
  MediaEventSource<void>& RtpPacketEvent() override { return mRtpPacketEvent; }

  bool RequiresNewSendStream(const VideoCodecConfig& newConfig) const;

  mutable mozilla::ReentrantMonitor mRendererMonitor MOZ_UNANNOTATED;

  // Accessed on any thread under mRendererMonitor.
  RefPtr<mozilla::VideoRenderer> mRenderer;

  // Accessed on any thread under mRendererMonitor.
  unsigned short mReceivingWidth = 0;

  // Accessed on any thread under mRendererMonitor.
  unsigned short mReceivingHeight = 0;

  // Call worker thread. All access to mCall->Call() happens here.
  const nsCOMPtr<nsISerialEventTarget> mCallThread;

  // Socket transport service thread that runs stats queries against us. Any
  // thread.
  const nsCOMPtr<nsISerialEventTarget> mStsThread;

  // Thread on which we are fed video frames. Set lazily on first call to
  // SendVideoFrame().
  nsCOMPtr<nsISerialEventTarget> mFrameSendingThread;

  struct Control {
    // Mirrors that map to VideoConduitControlInterface for control. Call thread
    // only.
    Mirror<bool> mReceiving;
    Mirror<bool> mTransmitting;
    Mirror<Ssrcs> mLocalSsrcs;
    Mirror<Ssrcs> mLocalRtxSsrcs;
    Mirror<std::string> mLocalCname;
    Mirror<std::string> mMid;
    Mirror<Ssrc> mRemoteSsrc;
    Mirror<Ssrc> mRemoteRtxSsrc;
    Mirror<std::string> mSyncGroup;
    Mirror<RtpExtList> mLocalRecvRtpExtensions;
    Mirror<RtpExtList> mLocalSendRtpExtensions;
    Mirror<Maybe<VideoCodecConfig>> mSendCodec;
    Mirror<Maybe<RtpRtcpConfig>> mSendRtpRtcpConfig;
    Mirror<std::vector<VideoCodecConfig>> mRecvCodecs;
    Mirror<Maybe<RtpRtcpConfig>> mRecvRtpRtcpConfig;
    Mirror<webrtc::VideoCodecMode> mCodecMode;
    Mirror<RefPtr<FrameTransformerProxy>> mFrameTransformerProxySend;
    Mirror<RefPtr<FrameTransformerProxy>> mFrameTransformerProxyRecv;

    // For caching mRemoteSsrc and mRemoteRtxSsrc, since another caller may
    // change the remote ssrc in the stream config directly.
    Ssrc mConfiguredRemoteSsrc = 0;
    Ssrc mConfiguredRemoteRtxSsrc = 0;
    // For tracking changes to mSendCodec and mSendRtpRtcpConfig.
    Maybe<VideoCodecConfig> mConfiguredSendCodec;
    Maybe<RtpRtcpConfig> mConfiguredSendRtpRtcpConfig;
    // For tracking changes to mRecvCodecs and mRecvRtpRtcpConfig.
    std::vector<VideoCodecConfig> mConfiguredRecvCodecs;
    Maybe<RtpRtcpConfig> mConfiguredRecvRtpRtcpConfig;

    // For change tracking. Callthread only.
    RefPtr<FrameTransformerProxy> mConfiguredFrameTransformerProxySend;
    RefPtr<FrameTransformerProxy> mConfiguredFrameTransformerProxyRecv;

    Control() = delete;
    explicit Control(const RefPtr<AbstractThread>& aCallThread);
  } mControl;

  // WatchManager allowing Mirrors and other watch targets to trigger functions
  // that will update the webrtc.org configuration.
  WatchManager<WebrtcVideoConduit> mWatchManager;

  mutable Mutex mMutex MOZ_UNANNOTATED;

  // Decoder factory used by mRecvStream when it needs new decoders. This is
  // not shared broader like some state in the WebrtcCallWrapper because it
  // handles CodecPluginID plumbing tied to this VideoConduit.
  const UniquePtr<WebrtcVideoDecoderFactory> mDecoderFactory;

  // Encoder factory used by mSendStream when it needs new encoders. This is
  // not shared broader like some state in the WebrtcCallWrapper because it
  // handles CodecPluginID plumbing tied to this VideoConduit.
  const UniquePtr<WebrtcVideoEncoderFactory> mEncoderFactory;

  // Our own record of the sinks added to mVideoBroadcaster so we can support
  // dispatching updates to sinks from off-Call-thread. Call thread only.
  AutoTArray<rtc::VideoSinkInterface<webrtc::VideoFrame>*, 1> mRegisteredSinks;

  // Broadcaster that distributes our frames to all registered sinks.
  // Threadsafe.
  rtc::VideoBroadcaster mVideoBroadcaster;

  // Buffer pool used for scaling frames.
  // Accessed on the frame-feeding thread only.
  webrtc::VideoFrameBufferPool mBufferPool;

  // Engine state we are concerned with. Written on the Call thread and read
  // anywhere.
  mozilla::Atomic<bool>
      mEngineTransmitting;  // If true ==> Transmit Subsystem is up and running
  mozilla::Atomic<bool>
      mEngineReceiving;  // if true ==> Receive Subsystem up and running

  // Written only on the Call thread. Guarded by mMutex, except for reads on the
  // Call thread.
  Maybe<VideoCodecConfig> mCurSendCodecConfig;

  // Bookkeeping of stats for telemetry. Call thread only.
  RunningStat mSendFramerate;
  RunningStat mSendBitrate;
  RunningStat mRecvFramerate;
  RunningStat mRecvBitrate;

  // Must call webrtc::Call::DestroyVideoReceive/SendStream to delete this.
  // Written only on the Call thread. Guarded by mMutex, except for reads on the
  // Call thread.
  webrtc::VideoReceiveStreamInterface* mRecvStream = nullptr;

  // Must call webrtc::Call::DestroyVideoReceive/SendStream to delete this.
  webrtc::VideoSendStream* mSendStream = nullptr;

  // Written on the frame feeding thread.
  // Guarded by mMutex, except for reads on the frame feeding thread.
  unsigned short mLastWidth = 0;

  // Written on the frame feeding thread.
  // Guarded by mMutex, except for reads on the frame feeding thread.
  unsigned short mLastHeight = 0;

  // Written on the frame feeding thread, the timestamp of the last frame on the
  // send side, in microseconds. This is a local timestamp using the system
  // clock with a unspecified epoch (Like mozilla::TimeStamp).
  // Guarded by mMutex.
  Maybe<uint64_t> mLastTimestampSendUs;

  // Written on the frame receive thread, the rtp timestamp of the last frame
  // on the receive side, in 90kHz base. This comes from the RTP packet.
  // Guarded by mMutex.
  Maybe<uint32_t> mLastRTPTimestampReceive;

  // Accessed from any thread under mRendererMonitor.
  uint64_t mVideoLatencyAvg = 0;

  const bool mVideoLatencyTestEnable;

  // All in bps.
  const int mMinBitrate;
  const int mStartBitrate;
  const int mPrefMaxBitrate;
  const int mMinBitrateEstimate;

  // Max bitrate in bps as provided by negotiation. Call thread only.
  int mNegotiatedMaxBitrate = 0;

  // Set to true to force denoising on.
  const bool mDenoising;

  // Set to true to ignore sink wants (scaling due to bwe and cpu usage).
  const bool mLockScaling;

  const uint8_t mSpatialLayers;
  const uint8_t mTemporalLayers;

  static const unsigned int sAlphaNum = 7;
  static const unsigned int sAlphaDen = 8;
  static const unsigned int sRoundingPadding = 1024;

  // Target jitter buffer to be applied to the receive stream in milliseconds.
  uint16_t mJitterBufferTargetMs = 0;

  // WEBRTC.ORG Call API
  // Const so can be accessed on any thread. All methods are called on the Call
  // thread.
  const RefPtr<WebrtcCallWrapper> mCall;

  // Set up in the ctor and then not touched. Called through by the streams on
  // any thread. Safe since we own and control the lifetime of the streams.
  WebrtcSendTransport mSendTransport;
  WebrtcReceiveTransport mRecvTransport;

  // Written only on the Call thread. Guarded by mMutex, except for reads on the
  // Call thread. Typical non-Call thread access is on the frame delivery
  // thread.
  webrtc::VideoSendStream::Config mSendStreamConfig;

  // Call thread only.
  webrtc::VideoEncoderConfig mEncoderConfig;

  // Written only on the Call thread. Guarded by mMutex, except for reads on the
  // Call thread. Calls can happen under mMutex on any thread.
  DataMutex<RefPtr<rtc::RefCountedObject<VideoStreamFactory>>>
      mVideoStreamFactory;

  // Call thread only.
  webrtc::VideoReceiveStreamInterface::Config mRecvStreamConfig;

  // Are SSRC changes without signaling allowed or not.
  // Call thread only.
  bool mAllowSsrcChange = true;

  // Accessed during configuration/signaling (Call thread), and on the frame
  // delivery thread for frame history tracking. Set only on the Call thread.
  Atomic<uint32_t> mRecvSSRC =
      Atomic<uint32_t>(0);  // this can change during a stream!

  // Accessed from both the STS and frame delivery thread for frame history
  // tracking. Set when receiving packets.
  Atomic<uint32_t> mRemoteSendSSRC =
      Atomic<uint32_t>(0);  // this can change during a stream!

  // Main thread only
  nsTArray<uint64_t> mSendCodecPluginIDs;
  // Main thread only
  nsTArray<uint64_t> mRecvCodecPluginIDs;

  // Main thread only
  MediaEventListener mSendPluginCreated;
  MediaEventListener mSendPluginReleased;
  MediaEventListener mRecvPluginCreated;
  MediaEventListener mRecvPluginReleased;

  // Call thread only. ssrc -> base_seq
  std::map<uint32_t, uint16_t> mRtpSendBaseSeqs;
  // libwebrtc network thread only. ssrc -> base_seq.
  // To track changes needed to mRtpSendBaseSeqs.
  std::map<uint32_t, uint16_t> mRtpSendBaseSeqs_n;

  // Tracking the attributes of received frames over time
  // Protected by mRendererMonitor
  dom::RTCVideoFrameHistoryInternal mReceivedFrameHistory;

  // Written only on the main thread.  Guarded by mMutex, except for
  // reads on the main thread.
  std::vector<webrtc::RtpSource> mRtpSources;

  // Thread safe
  Atomic<bool> mTransportActive = Atomic<bool>(false);
  MediaEventProducer<void> mRtcpByeEvent;
  MediaEventProducer<void> mRtcpTimeoutEvent;
  MediaEventProducer<void> mRtpPacketEvent;
  MediaEventProducerExc<MediaPacket> mSenderRtpSendEvent;
  MediaEventProducerExc<MediaPacket> mSenderRtcpSendEvent;
  MediaEventProducerExc<MediaPacket> mReceiverRtcpSendEvent;

  // Assigned and revoked on mStsThread. Listeners for receiving packets.
  MediaEventListener mReceiverRtpEventListener;  // Rtp-receiving pipeline
};
}  // namespace mozilla

#endif