summaryrefslogtreecommitdiffstats
path: root/third_party/libwebrtc/media/base/media_channel.h
blob: 138d28ae4c4b2a5a1cd523ddd30cc68e8bae66a4 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
/*
 *  Copyright (c) 2004 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#ifndef MEDIA_BASE_MEDIA_CHANNEL_H_
#define MEDIA_BASE_MEDIA_CHANNEL_H_

#include <map>
#include <memory>
#include <string>
#include <utility>
#include <vector>

#include "absl/types/optional.h"
#include "api/audio_codecs/audio_encoder.h"
#include "api/audio_options.h"
#include "api/call/audio_sink.h"
#include "api/crypto/frame_decryptor_interface.h"
#include "api/crypto/frame_encryptor_interface.h"
#include "api/frame_transformer_interface.h"
#include "api/media_stream_interface.h"
#include "api/rtc_error.h"
#include "api/rtp_parameters.h"
#include "api/rtp_sender_interface.h"
#include "api/task_queue/pending_task_safety_flag.h"
#include "api/transport/data_channel_transport_interface.h"
#include "api/transport/rtp/rtp_source.h"
#include "api/units/time_delta.h"
#include "api/video/video_content_type.h"
#include "api/video/video_sink_interface.h"
#include "api/video/video_source_interface.h"
#include "api/video/video_timing.h"
#include "api/video_codecs/scalability_mode.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "call/video_receive_stream.h"
#include "common_video/include/quality_limitation_reason.h"
#include "media/base/codec.h"
#include "media/base/delayable.h"
#include "media/base/media_constants.h"
#include "media/base/stream_params.h"
#include "modules/audio_processing/include/audio_processing_statistics.h"
#include "modules/rtp_rtcp/include/report_block_data.h"
#include "modules/rtp_rtcp/source/rtp_packet_received.h"
#include "rtc_base/async_packet_socket.h"
#include "rtc_base/buffer.h"
#include "rtc_base/copy_on_write_buffer.h"
#include "rtc_base/dscp.h"
#include "rtc_base/logging.h"
#include "rtc_base/network_route.h"
#include "rtc_base/socket.h"
#include "rtc_base/string_encode.h"
#include "rtc_base/strings/string_builder.h"
#include "video/config/video_encoder_config.h"

namespace rtc {
class Timing;
}

namespace webrtc {
class VideoFrame;
}  // namespace webrtc

namespace cricket {

class AudioSource;
class VideoCapturer;
struct RtpHeader;
struct VideoFormat;
class VideoMediaSendChannelInterface;
class VideoMediaReceiveChannelInterface;
class VoiceMediaSendChannelInterface;
class VoiceMediaReceiveChannelInterface;

const int kScreencastDefaultFps = 5;

template <class T>
static std::string ToStringIfSet(const char* key,
                                 const absl::optional<T>& val) {
  std::string str;
  if (val) {
    str = key;
    str += ": ";
    str += val ? rtc::ToString(*val) : "";
    str += ", ";
  }
  return str;
}

template <class T>
static std::string VectorToString(const std::vector<T>& vals) {
  rtc::StringBuilder ost;  // no-presubmit-check TODO(webrtc:8982)
  ost << "[";
  for (size_t i = 0; i < vals.size(); ++i) {
    if (i > 0) {
      ost << ", ";
    }
    ost << vals[i].ToString();
  }
  ost << "]";
  return ost.Release();
}

// Options that can be applied to a VideoMediaChannel or a VideoMediaEngine.
// Used to be flags, but that makes it hard to selectively apply options.
// We are moving all of the setting of options to structs like this,
// but some things currently still use flags.
struct VideoOptions {
  VideoOptions();
  ~VideoOptions();

  void SetAll(const VideoOptions& change) {
    SetFrom(&video_noise_reduction, change.video_noise_reduction);
    SetFrom(&screencast_min_bitrate_kbps, change.screencast_min_bitrate_kbps);
    SetFrom(&is_screencast, change.is_screencast);
  }

  bool operator==(const VideoOptions& o) const {
    return video_noise_reduction == o.video_noise_reduction &&
           screencast_min_bitrate_kbps == o.screencast_min_bitrate_kbps &&
           is_screencast == o.is_screencast;
  }
  bool operator!=(const VideoOptions& o) const { return !(*this == o); }

  std::string ToString() const {
    rtc::StringBuilder ost;
    ost << "VideoOptions {";
    ost << ToStringIfSet("noise reduction", video_noise_reduction);
    ost << ToStringIfSet("screencast min bitrate kbps",
                         screencast_min_bitrate_kbps);
    ost << ToStringIfSet("is_screencast ", is_screencast);
    ost << "}";
    return ost.Release();
  }

  // Enable denoising? This flag comes from the getUserMedia
  // constraint 'googNoiseReduction', and WebRtcVideoEngine passes it
  // on to the codec options. Disabled by default.
  absl::optional<bool> video_noise_reduction;
  // Force screencast to use a minimum bitrate. This flag comes from
  // the PeerConnection constraint 'googScreencastMinBitrate'. It is
  // copied to the encoder config by WebRtcVideoChannel.
  // TODO(https://crbug.com/1315155): Remove the ability to set it in Chromium
  // and delete this flag (it should default to 100 kbps).
  absl::optional<int> screencast_min_bitrate_kbps;
  // Set by screencast sources. Implies selection of encoding settings
  // suitable for screencast. Most likely not the right way to do
  // things, e.g., screencast of a text document and screencast of a
  // youtube video have different needs.
  absl::optional<bool> is_screencast;
  webrtc::VideoTrackInterface::ContentHint content_hint;

 private:
  template <typename T>
  static void SetFrom(absl::optional<T>* s, const absl::optional<T>& o) {
    if (o) {
      *s = o;
    }
  }
};

class MediaChannelNetworkInterface {
 public:
  enum SocketType { ST_RTP, ST_RTCP };
  virtual bool SendPacket(rtc::CopyOnWriteBuffer* packet,
                          const rtc::PacketOptions& options) = 0;
  virtual bool SendRtcp(rtc::CopyOnWriteBuffer* packet,
                        const rtc::PacketOptions& options) = 0;
  virtual int SetOption(SocketType type,
                        rtc::Socket::Option opt,
                        int option) = 0;
  virtual ~MediaChannelNetworkInterface() {}
};

// Functions shared across all MediaChannel interfaces.
// Because there are implementation types that implement multiple
// interfaces, this is not a base class (no diamond inheritance).
template <class T>
class MediaBaseChannelInterface {
 public:
  virtual ~MediaBaseChannelInterface() = default;
  virtual cricket::MediaType media_type() const = 0;

  // Networking functions. We assume that both the send channel and the
  // receive channel send RTP packets (RTCP packets in the case of a receive
  // channel).

  // Called on the network when an RTP packet is received.
  virtual void OnPacketReceived(const webrtc::RtpPacketReceived& packet) = 0;
  // Called on the network thread after a transport has finished sending a
  // packet.
  virtual void OnPacketSent(const rtc::SentPacket& sent_packet) = 0;
  // Called when the socket's ability to send has changed.
  virtual void OnReadyToSend(bool ready) = 0;
  // Called when the network route used for sending packets changed.
  virtual void OnNetworkRouteChanged(
      absl::string_view transport_name,
      const rtc::NetworkRoute& network_route) = 0;

  // Corresponds to the SDP attribute extmap-allow-mixed, see RFC8285.
  // Set to true if it's allowed to mix one- and two-byte RTP header extensions
  // in the same stream. The setter and getter must only be called from
  // worker_thread.
  virtual void SetExtmapAllowMixed(bool extmap_allow_mixed) = 0;
  virtual bool ExtmapAllowMixed() const = 0;
};

class MediaSendChannelInterface
    : public MediaBaseChannelInterface<MediaSendChannelInterface> {
 public:
  virtual ~MediaSendChannelInterface() = default;

  virtual VideoMediaSendChannelInterface* AsVideoSendChannel() = 0;

  virtual VoiceMediaSendChannelInterface* AsVoiceSendChannel() = 0;

  // Creates a new outgoing media stream with SSRCs and CNAME as described
  // by sp.
  virtual bool AddSendStream(const StreamParams& sp) = 0;
  // Removes an outgoing media stream.
  // SSRC must be the first SSRC of the media stream if the stream uses
  // multiple SSRCs. In the case of an ssrc of 0, the possibly cached
  // StreamParams is removed.
  virtual bool RemoveSendStream(uint32_t ssrc) = 0;
  // Set the frame encryptor to use on all outgoing frames. This is optional.
  // This pointers lifetime is managed by the set of RtpSender it is attached
  // to.
  virtual void SetFrameEncryptor(
      uint32_t ssrc,
      rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor) = 0;

  virtual webrtc::RTCError SetRtpSendParameters(
      uint32_t ssrc,
      const webrtc::RtpParameters& parameters,
      webrtc::SetParametersCallback callback = nullptr) = 0;

  virtual void SetEncoderToPacketizerFrameTransformer(
      uint32_t ssrc,
      rtc::scoped_refptr<webrtc::FrameTransformerInterface>
          frame_transformer) = 0;

  // note: The encoder_selector object must remain valid for the lifetime of the
  // MediaChannel, unless replaced.
  virtual void SetEncoderSelector(
      uint32_t ssrc,
      webrtc::VideoEncoderFactory::EncoderSelectorInterface* encoder_selector) {
  }
  virtual webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const = 0;
};

class MediaReceiveChannelInterface
    : public MediaBaseChannelInterface<MediaReceiveChannelInterface>,
      public Delayable {
 public:
  virtual ~MediaReceiveChannelInterface() = default;

  virtual VideoMediaReceiveChannelInterface* AsVideoReceiveChannel() {
    RTC_CHECK_NOTREACHED();
    return nullptr;
  }
  virtual VoiceMediaReceiveChannelInterface* AsVoiceReceiveChannel() {
    RTC_CHECK_NOTREACHED();
    return nullptr;
  }
  // Creates a new incoming media stream with SSRCs, CNAME as described
  // by sp. In the case of a sp without SSRCs, the unsignaled sp is cached
  // to be used later for unsignaled streams received.
  virtual bool AddRecvStream(const StreamParams& sp) = 0;
  // Removes an incoming media stream.
  // ssrc must be the first SSRC of the media stream if the stream uses
  // multiple SSRCs.
  virtual bool RemoveRecvStream(uint32_t ssrc) = 0;
  // Resets any cached StreamParams for an unsignaled RecvStream, and removes
  // any existing unsignaled streams.
  virtual void ResetUnsignaledRecvStream() = 0;
  // Gets the current unsignaled receive stream's SSRC, if there is one.
  virtual absl::optional<uint32_t> GetUnsignaledSsrc() const = 0;
  // This is currently a workaround because of the demuxer state being managed
  // across two separate threads. Once the state is consistently managed on
  // the same thread (network), this workaround can be removed.
  // These two notifications inform the media channel when the transport's
  // demuxer criteria is being updated.
  // * OnDemuxerCriteriaUpdatePending() happens on the same thread that the
  //   channel's streams are added and removed (worker thread).
  // * OnDemuxerCriteriaUpdateComplete() happens on the same thread.
  // Because the demuxer is updated asynchronously, there is a window of time
  // where packets are arriving to the channel for streams that have already
  // been removed on the worker thread. It is important NOT to treat these as
  // new unsignalled ssrcs.
  virtual void OnDemuxerCriteriaUpdatePending() = 0;
  virtual void OnDemuxerCriteriaUpdateComplete() = 0;
  // Set the frame decryptor to use on all incoming frames. This is optional.
  // This pointers lifetimes is managed by the set of RtpReceivers it is
  // attached to.
  virtual void SetFrameDecryptor(
      uint32_t ssrc,
      rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor) = 0;

  virtual void SetDepacketizerToDecoderFrameTransformer(
      uint32_t ssrc,
      rtc::scoped_refptr<webrtc::FrameTransformerInterface>
          frame_transformer) = 0;
};

// The stats information is structured as follows:
// Media are represented by either MediaSenderInfo or MediaReceiverInfo.
// Media contains a vector of SSRC infos that are exclusively used by this
// media. (SSRCs shared between media streams can't be represented.)

// Information about an SSRC.
// This data may be locally recorded, or received in an RTCP SR or RR.
struct SsrcSenderInfo {
  uint32_t ssrc = 0;
  double timestamp = 0.0;  // NTP timestamp, represented as seconds since epoch.
};

struct SsrcReceiverInfo {
  uint32_t ssrc = 0;
  double timestamp = 0.0;
};

struct MediaSenderInfo {
  MediaSenderInfo();
  ~MediaSenderInfo();
  void add_ssrc(const SsrcSenderInfo& stat) { local_stats.push_back(stat); }
  // Temporary utility function for call sites that only provide SSRC.
  // As more info is added into SsrcSenderInfo, this function should go away.
  void add_ssrc(uint32_t ssrc) {
    SsrcSenderInfo stat;
    stat.ssrc = ssrc;
    add_ssrc(stat);
  }
  // Utility accessor for clients that are only interested in ssrc numbers.
  std::vector<uint32_t> ssrcs() const {
    std::vector<uint32_t> retval;
    for (std::vector<SsrcSenderInfo>::const_iterator it = local_stats.begin();
         it != local_stats.end(); ++it) {
      retval.push_back(it->ssrc);
    }
    return retval;
  }
  // Returns true if the media has been connected.
  bool connected() const { return local_stats.size() > 0; }
  // Utility accessor for clients that make the assumption only one ssrc
  // exists per media.
  // This will eventually go away.
  // Call sites that compare this to zero should use connected() instead.
  // https://bugs.webrtc.org/8694
  uint32_t ssrc() const {
    if (connected()) {
      return local_stats[0].ssrc;
    } else {
      return 0;
    }
  }
  // https://w3c.github.io/webrtc-stats/#dom-rtcsentrtpstreamstats-bytessent
  int64_t payload_bytes_sent = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-headerbytessent
  int64_t header_and_padding_bytes_sent = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-retransmittedbytessent
  uint64_t retransmitted_bytes_sent = 0;
  int packets_sent = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-retransmittedpacketssent
  uint64_t retransmitted_packets_sent = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-nackcount
  uint32_t nacks_rcvd = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-targetbitrate
  absl::optional<double> target_bitrate;
  int packets_lost = 0;
  float fraction_lost = 0.0f;
  int64_t rtt_ms = 0;
  std::string codec_name;
  absl::optional<int> codec_payload_type;
  std::vector<SsrcSenderInfo> local_stats;
  std::vector<SsrcReceiverInfo> remote_stats;
  // A snapshot of the most recent Report Block with additional data of interest
  // to statistics. Used to implement RTCRemoteInboundRtpStreamStats. Within
  // this list, the ReportBlockData::RTCPReportBlock::source_ssrc(), which is
  // the SSRC of the corresponding outbound RTP stream, is unique.
  std::vector<webrtc::ReportBlockData> report_block_datas;
  absl::optional<bool> active;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalpacketsenddelay
  webrtc::TimeDelta total_packet_send_delay = webrtc::TimeDelta::Zero();
};

struct MediaReceiverInfo {
  MediaReceiverInfo();
  ~MediaReceiverInfo();

  void add_ssrc(const SsrcReceiverInfo& stat) { local_stats.push_back(stat); }
  // Temporary utility function for call sites that only provide SSRC.
  // As more info is added into SsrcSenderInfo, this function should go away.
  void add_ssrc(uint32_t ssrc) {
    SsrcReceiverInfo stat;
    stat.ssrc = ssrc;
    add_ssrc(stat);
  }
  std::vector<uint32_t> ssrcs() const {
    std::vector<uint32_t> retval;
    for (std::vector<SsrcReceiverInfo>::const_iterator it = local_stats.begin();
         it != local_stats.end(); ++it) {
      retval.push_back(it->ssrc);
    }
    return retval;
  }
  // Returns true if the media has been connected.
  bool connected() const { return local_stats.size() > 0; }
  // Utility accessor for clients that make the assumption only one ssrc
  // exists per media.
  // This will eventually go away.
  // Call sites that compare this to zero should use connected();
  // https://bugs.webrtc.org/8694
  uint32_t ssrc() const {
    if (connected()) {
      return local_stats[0].ssrc;
    } else {
      return 0;
    }
  }

  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-bytesreceived
  int64_t payload_bytes_rcvd = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-headerbytesreceived
  int64_t header_and_padding_bytes_rcvd = 0;
  int packets_rcvd = 0;
  int packets_lost = 0;
  absl::optional<uint32_t> nacks_sent;
  // Jitter (network-related) latency (cumulative).
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-jitterbufferdelay
  double jitter_buffer_delay_seconds = 0.0;
  // Target delay for the jitter buffer (cumulative).
  // TODO(crbug.com/webrtc/14244): This metric is only implemented for
  // audio, it should be implemented for video as well.
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-jitterbuffertargetdelay
  absl::optional<double> jitter_buffer_target_delay_seconds;
  // Minimum obtainable delay for the jitter buffer (cumulative).
  // TODO(crbug.com/webrtc/14244): This metric is only implemented for
  // audio, it should be implemented for video as well.
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-jitterbufferminimumdelay
  absl::optional<double> jitter_buffer_minimum_delay_seconds;
  // Number of observations for cumulative jitter latency.
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-jitterbufferemittedcount
  uint64_t jitter_buffer_emitted_count = 0;
  // The timestamp at which the last packet was received, i.e. the time of the
  // local clock when it was received - not the RTP timestamp of that packet.
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-lastpacketreceivedtimestamp
  absl::optional<int64_t> last_packet_received_timestamp_ms;
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-estimatedplayouttimestamp
  absl::optional<int64_t> estimated_playout_ntp_timestamp_ms;
  std::string codec_name;
  absl::optional<int> codec_payload_type;
  std::vector<SsrcReceiverInfo> local_stats;
  std::vector<SsrcSenderInfo> remote_stats;
};

struct VoiceSenderInfo : public MediaSenderInfo {
  VoiceSenderInfo();
  ~VoiceSenderInfo();
  int jitter_ms = 0;
  // Current audio level, expressed linearly [0,32767].
  int audio_level = 0;
  // See description of "totalAudioEnergy" in the WebRTC stats spec:
  // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy
  double total_input_energy = 0.0;
  double total_input_duration = 0.0;
  webrtc::ANAStats ana_statistics;
  webrtc::AudioProcessingStats apm_statistics;
};

struct VoiceReceiverInfo : public MediaReceiverInfo {
  VoiceReceiverInfo();
  ~VoiceReceiverInfo();
  int jitter_ms = 0;
  int jitter_buffer_ms = 0;
  int jitter_buffer_preferred_ms = 0;
  int delay_estimate_ms = 0;
  int audio_level = 0;
  // Stats below correspond to similarly-named fields in the WebRTC stats spec.
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats
  double total_output_energy = 0.0;
  uint64_t total_samples_received = 0;
  double total_output_duration = 0.0;
  uint64_t concealed_samples = 0;
  uint64_t silent_concealed_samples = 0;
  uint64_t concealment_events = 0;
  uint64_t inserted_samples_for_deceleration = 0;
  uint64_t removed_samples_for_acceleration = 0;
  uint64_t fec_packets_received = 0;
  uint64_t fec_packets_discarded = 0;
  // Stats below correspond to similarly-named fields in the WebRTC stats spec.
  // https://w3c.github.io/webrtc-stats/#dom-rtcreceivedrtpstreamstats
  uint64_t packets_discarded = 0;
  // Stats below DO NOT correspond directly to anything in the WebRTC stats
  // fraction of synthesized audio inserted through expansion.
  float expand_rate = 0.0f;
  // fraction of synthesized speech inserted through expansion.
  float speech_expand_rate = 0.0f;
  // fraction of data out of secondary decoding, including FEC and RED.
  float secondary_decoded_rate = 0.0f;
  // Fraction of secondary data, including FEC and RED, that is discarded.
  // Discarding of secondary data can be caused by the reception of the primary
  // data, obsoleting the secondary data. It can also be caused by early
  // or late arrival of secondary data. This metric is the percentage of
  // discarded secondary data since last query of receiver info.
  float secondary_discarded_rate = 0.0f;
  // Fraction of data removed through time compression.
  float accelerate_rate = 0.0f;
  // Fraction of data inserted through time stretching.
  float preemptive_expand_rate = 0.0f;
  int decoding_calls_to_silence_generator = 0;
  int decoding_calls_to_neteq = 0;
  int decoding_normal = 0;
  // TODO(alexnarest): Consider decoding_neteq_plc for consistency
  int decoding_plc = 0;
  int decoding_codec_plc = 0;
  int decoding_cng = 0;
  int decoding_plc_cng = 0;
  int decoding_muted_output = 0;
  // Estimated capture start time in NTP time in ms.
  int64_t capture_start_ntp_time_ms = -1;
  // Count of the number of buffer flushes.
  uint64_t jitter_buffer_flushes = 0;
  // Number of samples expanded due to delayed packets.
  uint64_t delayed_packet_outage_samples = 0;
  // Arrival delay of received audio packets.
  double relative_packet_arrival_delay_seconds = 0.0;
  // Count and total duration of audio interruptions (loss-concealement periods
  // longer than 150 ms).
  int32_t interruption_count = 0;
  int32_t total_interruption_duration_ms = 0;
  // Remote outbound stats derived by the received RTCP sender reports.
  // https://w3c.github.io/webrtc-stats/#remoteoutboundrtpstats-dict*
  absl::optional<int64_t> last_sender_report_timestamp_ms;
  absl::optional<int64_t> last_sender_report_remote_timestamp_ms;
  uint32_t sender_reports_packets_sent = 0;
  uint64_t sender_reports_bytes_sent = 0;
  uint64_t sender_reports_reports_count = 0;
  absl::optional<webrtc::TimeDelta> round_trip_time;
  webrtc::TimeDelta total_round_trip_time = webrtc::TimeDelta::Zero();
  int round_trip_time_measurements = 0;
};

struct VideoSenderInfo : public MediaSenderInfo {
  VideoSenderInfo();
  ~VideoSenderInfo();
  std::vector<SsrcGroup> ssrc_groups;
  std::string encoder_implementation_name;
  int firs_rcvd = 0;
  int plis_rcvd = 0;
  int send_frame_width = 0;
  int send_frame_height = 0;
  int frames = 0;
  double framerate_input = 0;
  int framerate_sent = 0;
  int aggregated_framerate_sent = 0;
  int nominal_bitrate = 0;
  int adapt_reason = 0;
  int adapt_changes = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-qualitylimitationreason
  webrtc::QualityLimitationReason quality_limitation_reason =
      webrtc::QualityLimitationReason::kNone;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-qualitylimitationdurations
  std::map<webrtc::QualityLimitationReason, int64_t>
      quality_limitation_durations_ms;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-qualitylimitationresolutionchanges
  uint32_t quality_limitation_resolution_changes = 0;
  int avg_encode_ms = 0;
  int encode_usage_percent = 0;
  uint32_t frames_encoded = 0;
  uint32_t key_frames_encoded = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalencodetime
  uint64_t total_encode_time_ms = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalencodedbytestarget
  uint64_t total_encoded_bytes_target = 0;
  bool has_entered_low_resolution = false;
  absl::optional<uint64_t> qp_sum;
  webrtc::VideoContentType content_type = webrtc::VideoContentType::UNSPECIFIED;
  uint32_t frames_sent = 0;
  // https://w3c.github.io/webrtc-stats/#dom-rtcvideosenderstats-hugeframessent
  uint32_t huge_frames_sent = 0;
  uint32_t aggregated_huge_frames_sent = 0;
  absl::optional<std::string> rid;
  absl::optional<bool> power_efficient_encoder;
  absl::optional<webrtc::ScalabilityMode> scalability_mode;
};

struct VideoReceiverInfo : public MediaReceiverInfo {
  VideoReceiverInfo();
  ~VideoReceiverInfo();
  std::vector<SsrcGroup> ssrc_groups;
  std::string decoder_implementation_name;
  absl::optional<bool> power_efficient_decoder;
  int packets_concealed = 0;
  int firs_sent = 0;
  int plis_sent = 0;
  int frame_width = 0;
  int frame_height = 0;
  int framerate_rcvd = 0;
  int framerate_decoded = 0;
  int framerate_output = 0;
  // Framerate as sent to the renderer.
  int framerate_render_input = 0;
  // Framerate that the renderer reports.
  int framerate_render_output = 0;
  uint32_t frames_received = 0;
  uint32_t frames_dropped = 0;
  uint32_t frames_decoded = 0;
  uint32_t key_frames_decoded = 0;
  uint32_t frames_rendered = 0;
  absl::optional<uint64_t> qp_sum;
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totaldecodetime
  webrtc::TimeDelta total_decode_time = webrtc::TimeDelta::Zero();
  // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totalprocessingdelay
  webrtc::TimeDelta total_processing_delay = webrtc::TimeDelta::Zero();
  webrtc::TimeDelta total_assembly_time = webrtc::TimeDelta::Zero();
  uint32_t frames_assembled_from_multiple_packets = 0;
  double total_inter_frame_delay = 0;
  double total_squared_inter_frame_delay = 0;
  int64_t interframe_delay_max_ms = -1;
  uint32_t freeze_count = 0;
  uint32_t pause_count = 0;
  uint32_t total_freezes_duration_ms = 0;
  uint32_t total_pauses_duration_ms = 0;
  uint32_t jitter_ms = 0;

  webrtc::VideoContentType content_type = webrtc::VideoContentType::UNSPECIFIED;

  // All stats below are gathered per-VideoReceiver, but some will be correlated
  // across MediaStreamTracks.  NOTE(hta): when sinking stats into per-SSRC
  // structures, reflect this in the new layout.

  // Current frame decode latency.
  int decode_ms = 0;
  // Maximum observed frame decode latency.
  int max_decode_ms = 0;
  // Jitter (network-related) latency.
  int jitter_buffer_ms = 0;
  // Requested minimum playout latency.
  int min_playout_delay_ms = 0;
  // Requested latency to account for rendering delay.
  int render_delay_ms = 0;
  // Target overall delay: network+decode+render, accounting for
  // min_playout_delay_ms.
  int target_delay_ms = 0;
  // Current overall delay, possibly ramping towards target_delay_ms.
  int current_delay_ms = 0;

  // Estimated capture start time in NTP time in ms.
  int64_t capture_start_ntp_time_ms = -1;

  // First frame received to first frame decoded latency.
  int64_t first_frame_received_to_decoded_ms = -1;

  // Timing frame info: all important timestamps for a full lifetime of a
  // single 'timing frame'.
  absl::optional<webrtc::TimingFrameInfo> timing_frame_info;
};

struct BandwidthEstimationInfo {
  int available_send_bandwidth = 0;
  int available_recv_bandwidth = 0;
  int target_enc_bitrate = 0;
  int actual_enc_bitrate = 0;
  int retransmit_bitrate = 0;
  int transmit_bitrate = 0;
  int64_t bucket_delay = 0;
};

// Maps from payload type to `RtpCodecParameters`.
typedef std::map<int, webrtc::RtpCodecParameters> RtpCodecParametersMap;

// Stats returned from VoiceMediaSendChannel.GetStats()
struct VoiceMediaSendInfo {
  VoiceMediaSendInfo();
  ~VoiceMediaSendInfo();
  void Clear() {
    senders.clear();
    send_codecs.clear();
  }
  std::vector<VoiceSenderInfo> senders;
  RtpCodecParametersMap send_codecs;
};

// Stats returned from VoiceMediaReceiveChannel.GetStats()
struct VoiceMediaReceiveInfo {
  VoiceMediaReceiveInfo();
  ~VoiceMediaReceiveInfo();
  void Clear() {
    receivers.clear();
    receive_codecs.clear();
  }
  std::vector<VoiceReceiverInfo> receivers;
  RtpCodecParametersMap receive_codecs;
  int32_t device_underrun_count = 0;
};

// Combined VoiceMediaSendInfo and VoiceMediaReceiveInfo
// Returned from Transceiver.getStats()
struct VoiceMediaInfo {
  VoiceMediaInfo();
  VoiceMediaInfo(VoiceMediaSendInfo&& send, VoiceMediaReceiveInfo&& receive)
      : senders(std::move(send.senders)),
        receivers(std::move(receive.receivers)),
        send_codecs(std::move(send.send_codecs)),
        receive_codecs(std::move(receive.receive_codecs)),
        device_underrun_count(receive.device_underrun_count) {}
  ~VoiceMediaInfo();
  void Clear() {
    senders.clear();
    receivers.clear();
    send_codecs.clear();
    receive_codecs.clear();
  }
  std::vector<VoiceSenderInfo> senders;
  std::vector<VoiceReceiverInfo> receivers;
  RtpCodecParametersMap send_codecs;
  RtpCodecParametersMap receive_codecs;
  int32_t device_underrun_count = 0;
};

// Stats for a VideoMediaSendChannel
struct VideoMediaSendInfo {
  VideoMediaSendInfo();
  ~VideoMediaSendInfo();
  void Clear() {
    senders.clear();
    aggregated_senders.clear();
    send_codecs.clear();
  }
  // Each sender info represents one "outbound-rtp" stream.In non - simulcast,
  // this means one info per RtpSender but if simulcast is used this means
  // one info per simulcast layer.
  std::vector<VideoSenderInfo> senders;
  // Used for legacy getStats() API's "ssrc" stats and modern getStats() API's
  // "track" stats. If simulcast is used, instead of having one sender info per
  // simulcast layer, the metrics of all layers of an RtpSender are aggregated
  // into a single sender info per RtpSender.
  std::vector<VideoSenderInfo> aggregated_senders;
  RtpCodecParametersMap send_codecs;
};

// Stats for a VideoMediaReceiveChannel
struct VideoMediaReceiveInfo {
  VideoMediaReceiveInfo();
  ~VideoMediaReceiveInfo();
  void Clear() {
    receivers.clear();
    receive_codecs.clear();
  }
  std::vector<VideoReceiverInfo> receivers;
  RtpCodecParametersMap receive_codecs;
};

// Combined VideoMediaSenderInfo and VideoMediaReceiverInfo.
// Returned from channel.GetStats()
struct VideoMediaInfo {
  VideoMediaInfo();
  VideoMediaInfo(VideoMediaSendInfo&& send, VideoMediaReceiveInfo&& receive)
      : senders(std::move(send.senders)),
        aggregated_senders(std::move(send.aggregated_senders)),
        receivers(std::move(receive.receivers)),
        send_codecs(std::move(send.send_codecs)),
        receive_codecs(std::move(receive.receive_codecs)) {}
  ~VideoMediaInfo();
  void Clear() {
    senders.clear();
    aggregated_senders.clear();
    receivers.clear();
    send_codecs.clear();
    receive_codecs.clear();
  }
  // Each sender info represents one "outbound-rtp" stream.In non - simulcast,
  // this means one info per RtpSender but if simulcast is used this means
  // one info per simulcast layer.
  std::vector<VideoSenderInfo> senders;
  // Used for legacy getStats() API's "ssrc" stats and modern getStats() API's
  // "track" stats. If simulcast is used, instead of having one sender info per
  // simulcast layer, the metrics of all layers of an RtpSender are aggregated
  // into a single sender info per RtpSender.
  std::vector<VideoSenderInfo> aggregated_senders;
  std::vector<VideoReceiverInfo> receivers;
  RtpCodecParametersMap send_codecs;
  RtpCodecParametersMap receive_codecs;
};

struct RtcpParameters {
  bool reduced_size = false;
  bool remote_estimate = false;
};

template <class Codec>
struct RtpParameters {
  virtual ~RtpParameters() = default;

  std::vector<Codec> codecs;
  std::vector<webrtc::RtpExtension> extensions;
  // For a send stream this is true if we've neogtiated a send direction,
  // for a receive stream this is true if we've negotiated a receive direction.
  bool is_stream_active = true;

  // TODO(pthatcher): Add streams.
  RtcpParameters rtcp;

  std::string ToString() const {
    rtc::StringBuilder ost;
    ost << "{";
    const char* separator = "";
    for (const auto& entry : ToStringMap()) {
      ost << separator << entry.first << ": " << entry.second;
      separator = ", ";
    }
    ost << "}";
    return ost.Release();
  }

 protected:
  virtual std::map<std::string, std::string> ToStringMap() const {
    return {{"codecs", VectorToString(codecs)},
            {"extensions", VectorToString(extensions)}};
  }
};

// TODO(deadbeef): Rename to RtpSenderParameters, since they're intended to
// encapsulate all the parameters needed for an RtpSender.
template <class Codec>
struct RtpSendParameters : RtpParameters<Codec> {
  int max_bandwidth_bps = -1;
  // This is the value to be sent in the MID RTP header extension (if the header
  // extension in included in the list of extensions).
  std::string mid;
  bool extmap_allow_mixed = false;

 protected:
  std::map<std::string, std::string> ToStringMap() const override {
    auto params = RtpParameters<Codec>::ToStringMap();
    params["max_bandwidth_bps"] = rtc::ToString(max_bandwidth_bps);
    params["mid"] = (mid.empty() ? "<not set>" : mid);
    params["extmap-allow-mixed"] = extmap_allow_mixed ? "true" : "false";
    return params;
  }
};

struct AudioSendParameters : RtpSendParameters<AudioCodec> {
  AudioSendParameters();
  ~AudioSendParameters() override;
  AudioOptions options;

 protected:
  std::map<std::string, std::string> ToStringMap() const override;
};

struct AudioRecvParameters : RtpParameters<AudioCodec> {};

class VoiceMediaSendChannelInterface : public MediaSendChannelInterface {
 public:
  virtual bool SetSendParameters(const AudioSendParameters& params) = 0;
  // Starts or stops sending (and potentially capture) of local audio.
  virtual void SetSend(bool send) = 0;
  // Configure stream for sending.
  virtual bool SetAudioSend(uint32_t ssrc,
                            bool enable,
                            const AudioOptions* options,
                            AudioSource* source) = 0;
  // Returns if the telephone-event has been negotiated.
  virtual bool CanInsertDtmf() = 0;
  // Send a DTMF `event`. The DTMF out-of-band signal will be used.
  // The `ssrc` should be either 0 or a valid send stream ssrc.
  // The valid value for the `event` are 0 to 15 which corresponding to
  // DTMF event 0-9, *, #, A-D.
  virtual bool InsertDtmf(uint32_t ssrc, int event, int duration) = 0;
  virtual bool GetStats(VoiceMediaSendInfo* stats) = 0;
};

class VoiceMediaReceiveChannelInterface : public MediaReceiveChannelInterface {
 public:
  virtual bool SetRecvParameters(const AudioRecvParameters& params) = 0;
  // Get the receive parameters for the incoming stream identified by `ssrc`.
  virtual webrtc::RtpParameters GetRtpReceiveParameters(
      uint32_t ssrc) const = 0;
  virtual std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const = 0;
  // Retrieve the receive parameters for the default receive
  // stream, which is used when SSRCs are not signaled.
  virtual webrtc::RtpParameters GetDefaultRtpReceiveParameters() const = 0;
  // Starts or stops playout of received audio.
  virtual void SetPlayout(bool playout) = 0;
  // Set speaker output volume of the specified ssrc.
  virtual bool SetOutputVolume(uint32_t ssrc, double volume) = 0;
  // Set speaker output volume for future unsignaled streams.
  virtual bool SetDefaultOutputVolume(double volume) = 0;
  virtual void SetRawAudioSink(
      uint32_t ssrc,
      std::unique_ptr<webrtc::AudioSinkInterface> sink) = 0;
  virtual void SetDefaultRawAudioSink(
      std::unique_ptr<webrtc::AudioSinkInterface> sink) = 0;
  virtual bool GetStats(VoiceMediaReceiveInfo* stats, bool reset_legacy) = 0;
};

// TODO(deadbeef): Rename to VideoSenderParameters, since they're intended to
// encapsulate all the parameters needed for a video RtpSender.
struct VideoSendParameters : RtpSendParameters<VideoCodec> {
  VideoSendParameters();
  ~VideoSendParameters() override;
  // Use conference mode? This flag comes from the remote
  // description's SDP line 'a=x-google-flag:conference', copied over
  // by VideoChannel::SetRemoteContent_w, and ultimately used by
  // conference mode screencast logic in
  // WebRtcVideoChannel::WebRtcVideoSendStream::CreateVideoEncoderConfig.
  // The special screencast behaviour is disabled by default.
  bool conference_mode = false;

 protected:
  std::map<std::string, std::string> ToStringMap() const override;
};

// TODO(deadbeef): Rename to VideoReceiverParameters, since they're intended to
// encapsulate all the parameters needed for a video RtpReceiver.
struct VideoRecvParameters : RtpParameters<VideoCodec> {};

class VideoMediaSendChannelInterface : public MediaSendChannelInterface {
 public:
  virtual bool SetSendParameters(const VideoSendParameters& params) = 0;
  // Gets the currently set codecs/payload types to be used for outgoing media.
  virtual bool GetSendCodec(VideoCodec* send_codec) = 0;
  // Starts or stops transmission (and potentially capture) of local video.
  virtual bool SetSend(bool send) = 0;
  // Configure stream for sending and register a source.
  // The `ssrc` must correspond to a registered send stream.
  virtual bool SetVideoSend(
      uint32_t ssrc,
      const VideoOptions* options,
      rtc::VideoSourceInterface<webrtc::VideoFrame>* source) = 0;
  // Cause generation of a keyframe for `ssrc` on a sending channel.
  virtual void GenerateSendKeyFrame(uint32_t ssrc,
                                    const std::vector<std::string>& rids) = 0;
  // Enable network condition based codec switching.
  virtual void SetVideoCodecSwitchingEnabled(bool enabled) = 0;
  virtual bool GetStats(VideoMediaSendInfo* stats) = 0;
  virtual void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) = 0;
};

class VideoMediaReceiveChannelInterface : public MediaReceiveChannelInterface {
 public:
  virtual bool SetRecvParameters(const VideoRecvParameters& params) = 0;
  // Get the receive parameters for the incoming stream identified by `ssrc`.
  virtual webrtc::RtpParameters GetRtpReceiveParameters(
      uint32_t ssrc) const = 0;
  // Retrieve the receive parameters for the default receive
  // stream, which is used when SSRCs are not signaled.
  virtual webrtc::RtpParameters GetDefaultRtpReceiveParameters() const = 0;
  // Sets the sink object to be used for the specified stream.
  virtual bool SetSink(uint32_t ssrc,
                       rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) = 0;
  // The sink is used for the 'default' stream.
  virtual void SetDefaultSink(
      rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) = 0;
  // Request generation of a keyframe for `ssrc` on a receiving channel via
  // RTCP feedback.
  virtual void RequestRecvKeyFrame(uint32_t ssrc) = 0;

  virtual std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const = 0;
  // Set recordable encoded frame callback for `ssrc`
  virtual void SetRecordableEncodedFrameCallback(
      uint32_t ssrc,
      std::function<void(const webrtc::RecordableEncodedFrame&)> callback) = 0;
  // Clear recordable encoded frame callback for `ssrc`
  virtual void ClearRecordableEncodedFrameCallback(uint32_t ssrc) = 0;
  virtual bool GetStats(VideoMediaReceiveInfo* stats) = 0;
};

// Info about data received in DataMediaChannel.  For use in
// DataMediaChannel::SignalDataReceived and in all of the signals that
// signal fires, on up the chain.
struct ReceiveDataParams {
  // The in-packet stream indentifier.
  // SCTP data channels use SIDs.
  int sid = 0;
  // The type of message (binary, text, or control).
  webrtc::DataMessageType type = webrtc::DataMessageType::kText;
  // A per-stream value incremented per packet in the stream.
  int seq_num = 0;
};

enum SendDataResult { SDR_SUCCESS, SDR_ERROR, SDR_BLOCK };

}  // namespace cricket

#endif  // MEDIA_BASE_MEDIA_CHANNEL_H_