1 files changed, 450 insertions, 0 deletions
diff --git a/third_party/libwebrtc/video/rtp_video_stream_receiver2.h b/third_party/libwebrtc/video/rtp_video_stream_receiver2.h
new file mode 100644
index 0000000000..10329005ba
--- /dev/null
+++ b/third_party/libwebrtc/video/rtp_video_stream_receiver2.h
@@ -0,0 +1,450 @@
+/*
+ *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_
+#define VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/crypto/frame_decryptor_interface.h"
+#include "api/sequence_checker.h"
+#include "api/units/timestamp.h"
+#include "api/video/color_space.h"
+#include "api/video/video_codec_type.h"
+#include "call/rtp_packet_sink_interface.h"
+#include "call/syncable.h"
+#include "call/video_receive_stream.h"
+#include "modules/rtp_rtcp/include/receive_statistics.h"
+#include "modules/rtp_rtcp/include/recovered_packet_receiver.h"
+#include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h"
+#include "modules/rtp_rtcp/include/rtp_header_extension_map.h"
+#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h"
+#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h"
+#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
+#include "modules/rtp_rtcp/source/rtp_packet_received.h"
+#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h"
+#include "modules/rtp_rtcp/source/rtp_rtcp_interface.h"
+#include "modules/rtp_rtcp/source/rtp_video_header.h"
+#include "modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h"
+#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h"
+#include "modules/video_coding/h264_sps_pps_tracker.h"
+#include "modules/video_coding/loss_notification_controller.h"
+#include "modules/video_coding/nack_requester.h"
+#include "modules/video_coding/packet_buffer.h"
+#include "modules/video_coding/rtp_frame_reference_finder.h"
+#include "rtc_base/experiments/field_trial_parser.h"
+#include "rtc_base/numerics/sequence_number_unwrapper.h"
+#include "rtc_base/system/no_unique_address.h"
+#include "rtc_base/thread_annotations.h"
+#include "video/buffered_frame_decryptor.h"
+#include "video/unique_timestamp_counter.h"
+#include "video/video_stream_buffer_controller.h"
+
+namespace webrtc {
+
+class NackRequester;
+class PacketRouter;
+class ReceiveStatistics;
+class RtcpRttStats;
+class RtpPacketReceived;
+class Transport;
+class UlpfecReceiver;
+
+class RtpVideoStreamReceiver2 : public LossNotificationSender,
+                                public RecoveredPacketReceiver,
+                                public RtpPacketSinkInterface,
+                                public KeyFrameRequestSender,
+                                public NackSender,
+                                public OnDecryptedFrameCallback,
+                                public OnDecryptionStatusChangeCallback,
+                                public RtpVideoFrameReceiver {
+ public:
+  // A complete frame is a frame which has received all its packets and all its
+  // references are known.
+  class OnCompleteFrameCallback {
+   public:
+    virtual ~OnCompleteFrameCallback() {}
+    virtual void OnCompleteFrame(std::unique_ptr<EncodedFrame> frame) = 0;
+  };
+
+  RtpVideoStreamReceiver2(
+      TaskQueueBase* current_queue,
+      Clock* clock,
+      Transport* transport,
+      RtcpRttStats* rtt_stats,
+      // The packet router is optional; if provided, the RtpRtcp module for this
+      // stream is registered as a candidate for sending REMB and transport
+      // feedback.
+      PacketRouter* packet_router,
+      const VideoReceiveStreamInterface::Config* config,
+      ReceiveStatistics* rtp_receive_statistics,
+      RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer,
+      RtcpCnameCallback* rtcp_cname_callback,
+      NackPeriodicProcessor* nack_periodic_processor,
+      VideoStreamBufferControllerStatsObserver* vcm_receive_statistics,
+      // The KeyFrameRequestSender is optional; if not provided, key frame
+      // requests are sent via the internal RtpRtcp module.
+      OnCompleteFrameCallback* complete_frame_callback,
+      rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
+      rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
+      const FieldTrialsView& field_trials,
+      RtcEventLog* event_log);
+  ~RtpVideoStreamReceiver2() override;
+
+  void AddReceiveCodec(uint8_t payload_type,
+                       VideoCodecType video_codec,
+                       const std::map<std::string, std::string>& codec_params,
+                       bool raw_payload);
+  void RemoveReceiveCodec(uint8_t payload_type);
+
+  // Clears state for all receive codecs added via `AddReceiveCodec`.
+  void RemoveReceiveCodecs();
+
+  void StartReceive();
+  void StopReceive();
+
+  // Produces the transport-related timestamps; current_delay_ms is left unset.
+  absl::optional<Syncable::Info> GetSyncInfo() const;
+
+  bool DeliverRtcp(const uint8_t* rtcp_packet, size_t rtcp_packet_length);
+
+  void FrameContinuous(int64_t seq_num);
+
+  void FrameDecoded(int64_t seq_num);
+
+  void SignalNetworkState(NetworkState state);
+
+  // Returns number of different frames seen.
+  int GetUniqueFramesSeen() const {
+    RTC_DCHECK_RUN_ON(&packet_sequence_checker_);
+    return frame_counter_.GetUniqueSeen();
+  }
+
+  // Implements RtpPacketSinkInterface.
+  void OnRtpPacket(const RtpPacketReceived& packet) override;
+
+  // Public only for tests.
+  void OnReceivedPayloadData(rtc::CopyOnWriteBuffer codec_payload,
+                             const RtpPacketReceived& rtp_packet,
+                             const RTPVideoHeader& video);
+
+  // Implements RecoveredPacketReceiver.
+  void OnRecoveredPacket(const RtpPacketReceived& packet) override;
+
+  // Send an RTCP keyframe request.
+  void RequestKeyFrame() override;
+
+  // Implements NackSender.
+  void SendNack(const std::vector<uint16_t>& sequence_numbers,
+                bool buffering_allowed) override;
+
+  // Implements LossNotificationSender.
+  void SendLossNotification(uint16_t last_decoded_seq_num,
+                            uint16_t last_received_seq_num,
+                            bool decodability_flag,
+                            bool buffering_allowed) override;
+
+  // Returns true if a decryptor is attached and frames can be decrypted.
+  // Updated by OnDecryptionStatusChangeCallback. Note this refers to Frame
+  // Decryption not SRTP.
+  bool IsDecryptable() const;
+
+  // Implements OnDecryptedFrameCallback.
+  void OnDecryptedFrame(std::unique_ptr<RtpFrameObject> frame) override;
+
+  // Implements OnDecryptionStatusChangeCallback.
+  void OnDecryptionStatusChange(
+      FrameDecryptorInterface::Status status) override;
+
+  // Optionally set a frame decryptor after a stream has started. This will not
+  // reset the decoder state.
+  void SetFrameDecryptor(
+      rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor);
+
+  // Sets a frame transformer after a stream has started, if no transformer
+  // has previously been set. Does not reset the decoder state.
+  void SetDepacketizerToDecoderFrameTransformer(
+      rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
+
+  // Called by VideoReceiveStreamInterface when stats are updated.
+  void UpdateRtt(int64_t max_rtt_ms);
+
+  // Called when the local_ssrc is changed to match with a sender.
+  void OnLocalSsrcChange(uint32_t local_ssrc);
+
+  // Forwards the call to set rtcp_sender_ to the RTCP mode of the rtcp sender.
+  void SetRtcpMode(RtcpMode mode);
+
+  void SetReferenceTimeReport(bool enabled);
+
+  // Sets or clears the callback sink that gets called for RTP packets. Used for
+  // packet handlers such as FlexFec. Must be called on the packet delivery
+  // thread (same context as `OnRtpPacket` is called on).
+  // TODO(bugs.webrtc.org/11993): Packet delivery thread today means `worker
+  // thread` but will be `network thread`.
+  void SetPacketSink(RtpPacketSinkInterface* packet_sink);
+
+  // Turns on/off loss notifications. Must be called on the packet delivery
+  // thread.
+  void SetLossNotificationEnabled(bool enabled);
+
+  void SetNackHistory(TimeDelta history);
+
+  int ulpfec_payload_type() const;
+  int red_payload_type() const;
+  void SetProtectionPayloadTypes(int red_payload_type, int ulpfec_payload_type);
+
+  absl::optional<int64_t> LastReceivedPacketMs() const;
+  absl::optional<uint32_t> LastReceivedFrameRtpTimestamp() const;
+  absl::optional<int64_t> LastReceivedKeyframePacketMs() const;
+
+  // Mozilla modification: VideoReceiveStream2 and friends do not surface RTCP
+  // stats at all, and even on the most recent libwebrtc code there does not
+  // seem to be any support for these stats right now. So, we hack this in.
+  void RemoteRTCPSenderInfo(uint32_t* packet_count, uint32_t* octet_count,
+                            int64_t* ntp_timestamp_ms,
+                            int64_t* remote_ntp_timestamp_ms) const;
+
+ private:
+  // Implements RtpVideoFrameReceiver.
+  void ManageFrame(std::unique_ptr<RtpFrameObject> frame) override;
+
+  void OnCompleteFrames(RtpFrameReferenceFinder::ReturnVector frame)
+      RTC_RUN_ON(packet_sequence_checker_);
+
+  // Used for buffering RTCP feedback messages and sending them all together.
+  // Note:
+  // 1. Key frame requests and NACKs are mutually exclusive, with the
+  //    former taking precedence over the latter.
+  // 2. Loss notifications are orthogonal to either. (That is, may be sent
+  //    alongside either.)
+  class RtcpFeedbackBuffer : public KeyFrameRequestSender,
+                             public NackSender,
+                             public LossNotificationSender {
+   public:
+    RtcpFeedbackBuffer(KeyFrameRequestSender* key_frame_request_sender,
+                       NackSender* nack_sender,
+                       LossNotificationSender* loss_notification_sender);
+
+    ~RtcpFeedbackBuffer() override = default;
+
+    // KeyFrameRequestSender implementation.
+    void RequestKeyFrame() override;
+
+    // NackSender implementation.
+    void SendNack(const std::vector<uint16_t>& sequence_numbers,
+                  bool buffering_allowed) override;
+
+    // LossNotificationSender implementation.
+    void SendLossNotification(uint16_t last_decoded_seq_num,
+                              uint16_t last_received_seq_num,
+                              bool decodability_flag,
+                              bool buffering_allowed) override;
+
+    // Send all RTCP feedback messages buffered thus far.
+    void SendBufferedRtcpFeedback();
+
+    void ClearLossNotificationState();
+
+   private:
+    // LNTF-related state.
+    struct LossNotificationState {
+      LossNotificationState(uint16_t last_decoded_seq_num,
+                            uint16_t last_received_seq_num,
+                            bool decodability_flag)
+          : last_decoded_seq_num(last_decoded_seq_num),
+            last_received_seq_num(last_received_seq_num),
+            decodability_flag(decodability_flag) {}
+
+      uint16_t last_decoded_seq_num;
+      uint16_t last_received_seq_num;
+      bool decodability_flag;
+    };
+
+    RTC_NO_UNIQUE_ADDRESS SequenceChecker packet_sequence_checker_;
+    KeyFrameRequestSender* const key_frame_request_sender_;
+    NackSender* const nack_sender_;
+    LossNotificationSender* const loss_notification_sender_;
+
+    // Key-frame-request-related state.
+    bool request_key_frame_ RTC_GUARDED_BY(packet_sequence_checker_);
+
+    // NACK-related state.
+    std::vector<uint16_t> nack_sequence_numbers_
+        RTC_GUARDED_BY(packet_sequence_checker_);
+
+    absl::optional<LossNotificationState> lntf_state_
+        RTC_GUARDED_BY(packet_sequence_checker_);
+  };
+  enum ParseGenericDependenciesResult {
+    kDropPacket,
+    kHasGenericDescriptor,
+    kNoGenericDescriptor
+  };
+
+  // Entry point doing non-stats work for a received packet. Called
+  // for the same packet both before and after RED decapsulation.
+  void ReceivePacket(const RtpPacketReceived& packet)
+      RTC_RUN_ON(packet_sequence_checker_);
+
+  // Parses and handles RED headers.
+  // This function assumes that it's being called from only one thread.
+  void ParseAndHandleEncapsulatingHeader(const RtpPacketReceived& packet)
+      RTC_RUN_ON(packet_sequence_checker_);
+  void NotifyReceiverOfEmptyPacket(uint16_t seq_num)
+      RTC_RUN_ON(packet_sequence_checker_);
+  bool IsRedEnabled() const;
+  void InsertSpsPpsIntoTracker(uint8_t payload_type)
+      RTC_RUN_ON(packet_sequence_checker_);
+  void OnInsertedPacket(video_coding::PacketBuffer::InsertResult result)
+      RTC_RUN_ON(packet_sequence_checker_);
+  ParseGenericDependenciesResult ParseGenericDependenciesExtension(
+      const RtpPacketReceived& rtp_packet,
+      RTPVideoHeader* video_header) RTC_RUN_ON(packet_sequence_checker_);
+  void OnAssembledFrame(std::unique_ptr<RtpFrameObject> frame)
+      RTC_RUN_ON(packet_sequence_checker_);
+  void UpdatePacketReceiveTimestamps(const RtpPacketReceived& packet,
+                                     bool is_keyframe)
+      RTC_RUN_ON(packet_sequence_checker_);
+
+  const FieldTrialsView& field_trials_;
+  TaskQueueBase* const worker_queue_;
+  Clock* const clock_;
+  // Ownership of this object lies with VideoReceiveStreamInterface, which owns
+  // `this`.
+  const VideoReceiveStreamInterface::Config& config_;
+  PacketRouter* const packet_router_;
+
+  RemoteNtpTimeEstimator ntp_estimator_;
+
+  // Set by the field trial WebRTC-ForcePlayoutDelay to override any playout
+  // delay that is specified in the received packets.
+  FieldTrialOptional<int> forced_playout_delay_max_ms_;
+  FieldTrialOptional<int> forced_playout_delay_min_ms_;
+  ReceiveStatistics* const rtp_receive_statistics_;
+  std::unique_ptr<UlpfecReceiver> ulpfec_receiver_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+  int red_payload_type_ RTC_GUARDED_BY(packet_sequence_checker_);
+
+  RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_task_checker_;
+  // TODO(bugs.webrtc.org/11993): This checker conceptually represents
+  // operations that belong to the network thread. The Call class is currently
+  // moving towards handling network packets on the network thread and while
+  // that work is ongoing, this checker may in practice represent the worker
+  // thread, but still serves as a mechanism of grouping together concepts
+  // that belong to the network thread. Once the packets are fully delivered
+  // on the network thread, this comment will be deleted.
+  RTC_NO_UNIQUE_ADDRESS SequenceChecker packet_sequence_checker_;
+  RtpPacketSinkInterface* packet_sink_ RTC_GUARDED_BY(packet_sequence_checker_);
+  bool receiving_ RTC_GUARDED_BY(packet_sequence_checker_);
+  int64_t last_packet_log_ms_ RTC_GUARDED_BY(packet_sequence_checker_);
+
+  const std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_;
+
+  NackPeriodicProcessor* const nack_periodic_processor_;
+  OnCompleteFrameCallback* complete_frame_callback_;
+  const KeyFrameReqMethod keyframe_request_method_;
+
+  RtcpFeedbackBuffer rtcp_feedback_buffer_;
+  // TODO(tommi): Consider absl::optional<NackRequester> instead of unique_ptr
+  // since nack is usually configured.
+  std::unique_ptr<NackRequester> nack_module_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+  std::unique_ptr<LossNotificationController> loss_notification_controller_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+
+  VideoStreamBufferControllerStatsObserver* const vcm_receive_statistics_;
+  video_coding::PacketBuffer packet_buffer_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+  UniqueTimestampCounter frame_counter_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+  SeqNumUnwrapper<uint16_t> frame_id_unwrapper_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+
+  // Video structure provided in the dependency descriptor in a first packet
+  // of a key frame. It is required to parse dependency descriptor in the
+  // following delta packets.
+  std::unique_ptr<FrameDependencyStructure> video_structure_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+  // Frame id of the last frame with the attached video structure.
+  // absl::nullopt when `video_structure_ == nullptr`;
+  absl::optional<int64_t> video_structure_frame_id_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+  Timestamp last_logged_failed_to_parse_dd_
+      RTC_GUARDED_BY(packet_sequence_checker_) = Timestamp::MinusInfinity();
+
+  std::unique_ptr<RtpFrameReferenceFinder> reference_finder_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+  absl::optional<VideoCodecType> current_codec_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+  uint32_t last_assembled_frame_rtp_timestamp_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+
+  std::map<int64_t, uint16_t> last_seq_num_for_pic_id_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+  video_coding::H264SpsPpsTracker tracker_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+
+  // Maps payload id to the depacketizer.
+  std::map<uint8_t, std::unique_ptr<VideoRtpDepacketizer>> payload_type_map_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+
+  // TODO(johan): Remove pt_codec_params_ once
+  // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved.
+  // Maps a payload type to a map of out-of-band supplied codec parameters.
+  std::map<uint8_t, std::map<std::string, std::string>> pt_codec_params_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+  int16_t last_payload_type_ RTC_GUARDED_BY(packet_sequence_checker_) = -1;
+
+  bool has_received_frame_ RTC_GUARDED_BY(packet_sequence_checker_);
+
+  absl::optional<uint32_t> last_received_rtp_timestamp_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+  absl::optional<uint32_t> last_received_keyframe_rtp_timestamp_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+  absl::optional<Timestamp> last_received_rtp_system_time_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+  absl::optional<Timestamp> last_received_keyframe_rtp_system_time_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+
+  // Handles incoming encrypted frames and forwards them to the
+  // rtp_reference_finder if they are decryptable.
+  std::unique_ptr<BufferedFrameDecryptor> buffered_frame_decryptor_
+      RTC_PT_GUARDED_BY(packet_sequence_checker_);
+  bool frames_decryptable_ RTC_GUARDED_BY(worker_task_checker_);
+  absl::optional<ColorSpace> last_color_space_;
+
+  AbsoluteCaptureTimeInterpolator absolute_capture_time_interpolator_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+
+  CaptureClockOffsetUpdater capture_clock_offset_updater_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+
+  int64_t last_completed_picture_id_ = 0;
+
+  rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate>
+      frame_transformer_delegate_;
+
+  SeqNumUnwrapper<uint16_t> rtp_seq_num_unwrapper_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+  std::map<int64_t, RtpPacketInfo> packet_infos_
+      RTC_GUARDED_BY(packet_sequence_checker_);
+
+  Timestamp next_keyframe_request_for_missing_video_structure_ =
+      Timestamp::MinusInfinity();
+};
+
+}  // namespace webrtc
+
+#endif  // VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_