diff options
Diffstat (limited to 'third_party/libwebrtc/video/rtp_video_stream_receiver2.h')
-rw-r--r-- | third_party/libwebrtc/video/rtp_video_stream_receiver2.h | 455 |
1 files changed, 455 insertions, 0 deletions
diff --git a/third_party/libwebrtc/video/rtp_video_stream_receiver2.h b/third_party/libwebrtc/video/rtp_video_stream_receiver2.h new file mode 100644 index 0000000000..931525a054 --- /dev/null +++ b/third_party/libwebrtc/video/rtp_video_stream_receiver2.h @@ -0,0 +1,455 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_ +#define VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_ + +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/crypto/frame_decryptor_interface.h" +#include "api/sequence_checker.h" +#include "api/units/timestamp.h" +#include "api/video/color_space.h" +#include "api/video/video_codec_type.h" +#include "call/rtp_packet_sink_interface.h" +#include "call/syncable.h" +#include "call/video_receive_stream.h" +#include "modules/rtp_rtcp/include/receive_statistics.h" +#include "modules/rtp_rtcp/include/recovered_packet_receiver.h" +#include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" +#include "modules/rtp_rtcp/include/rtp_header_extension_map.h" +#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h" +#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h" +#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" +#include "modules/rtp_rtcp/source/rtp_video_header.h" +#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h" +#include "modules/video_coding/h264_sps_pps_tracker.h" +#include "modules/video_coding/loss_notification_controller.h" +#include "modules/video_coding/nack_requester.h" +#include "modules/video_coding/packet_buffer.h" +#include "modules/video_coding/rtp_frame_reference_finder.h" +#include "rtc_base/experiments/field_trial_parser.h" +#include "rtc_base/numerics/sequence_number_unwrapper.h" +#include "rtc_base/system/no_unique_address.h" +#include "rtc_base/thread_annotations.h" +#include "video/buffered_frame_decryptor.h" +#include "video/rtp_video_stream_receiver_frame_transformer_delegate.h" +#include "video/unique_timestamp_counter.h" + +namespace webrtc { + +class NackRequester; +class PacketRouter; +class ReceiveStatistics; +class RtcpRttStats; +class RtpPacketReceived; +class Transport; +class UlpfecReceiver; + +class RtpVideoStreamReceiver2 : public LossNotificationSender, + public RecoveredPacketReceiver, + public RtpPacketSinkInterface, + public KeyFrameRequestSender, + public NackSender, + public OnDecryptedFrameCallback, + public OnDecryptionStatusChangeCallback, + public RtpVideoFrameReceiver { + public: + // A complete frame is a frame which has received all its packets and all its + // references are known. + class OnCompleteFrameCallback { + public: + virtual ~OnCompleteFrameCallback() {} + virtual void OnCompleteFrame(std::unique_ptr<EncodedFrame> frame) = 0; + }; + + RtpVideoStreamReceiver2( + TaskQueueBase* current_queue, + Clock* clock, + Transport* transport, + RtcpRttStats* rtt_stats, + // The packet router is optional; if provided, the RtpRtcp module for this + // stream is registered as a candidate for sending REMB and transport + // feedback. + PacketRouter* packet_router, + const VideoReceiveStreamInterface::Config* config, + ReceiveStatistics* rtp_receive_statistics, + RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer, + RtcpCnameCallback* rtcp_cname_callback, + NackPeriodicProcessor* nack_periodic_processor, + VCMReceiveStatisticsCallback* vcm_receive_statistics, + // The KeyFrameRequestSender is optional; if not provided, key frame + // requests are sent via the internal RtpRtcp module. + OnCompleteFrameCallback* complete_frame_callback, + rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor, + rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, + const FieldTrialsView& field_trials, + RtcEventLog* event_log); + ~RtpVideoStreamReceiver2() override; + + void AddReceiveCodec(uint8_t payload_type, + VideoCodecType video_codec, + const std::map<std::string, std::string>& codec_params, + bool raw_payload); + void RemoveReceiveCodec(uint8_t payload_type); + + // Clears state for all receive codecs added via `AddReceiveCodec`. + void RemoveReceiveCodecs(); + + void StartReceive(); + void StopReceive(); + + // Produces the transport-related timestamps; current_delay_ms is left unset. + absl::optional<Syncable::Info> GetSyncInfo() const; + + bool DeliverRtcp(const uint8_t* rtcp_packet, size_t rtcp_packet_length); + + void FrameContinuous(int64_t seq_num); + + void FrameDecoded(int64_t seq_num); + + void SignalNetworkState(NetworkState state); + + // Returns number of different frames seen. + int GetUniqueFramesSeen() const { + RTC_DCHECK_RUN_ON(&packet_sequence_checker_); + return frame_counter_.GetUniqueSeen(); + } + + // Implements RtpPacketSinkInterface. + void OnRtpPacket(const RtpPacketReceived& packet) override; + + // Public only for tests. + void OnReceivedPayloadData(rtc::CopyOnWriteBuffer codec_payload, + const RtpPacketReceived& rtp_packet, + const RTPVideoHeader& video); + + // Implements RecoveredPacketReceiver. + void OnRecoveredPacket(const RtpPacketReceived& packet) override; + + // Send an RTCP keyframe request. + void RequestKeyFrame() override; + + // Implements NackSender. + void SendNack(const std::vector<uint16_t>& sequence_numbers, + bool buffering_allowed) override; + + // Implements LossNotificationSender. + void SendLossNotification(uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag, + bool buffering_allowed) override; + + // Returns true if a decryptor is attached and frames can be decrypted. + // Updated by OnDecryptionStatusChangeCallback. Note this refers to Frame + // Decryption not SRTP. + bool IsDecryptable() const; + + // Implements OnDecryptedFrameCallback. + void OnDecryptedFrame(std::unique_ptr<RtpFrameObject> frame) override; + + // Implements OnDecryptionStatusChangeCallback. + void OnDecryptionStatusChange( + FrameDecryptorInterface::Status status) override; + + // Optionally set a frame decryptor after a stream has started. This will not + // reset the decoder state. + void SetFrameDecryptor( + rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor); + + // Sets a frame transformer after a stream has started, if no transformer + // has previously been set. Does not reset the decoder state. + void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr<FrameTransformerInterface> frame_transformer); + + // Updates the rtp header extensions at runtime. Must be called on the + // `packet_sequence_checker_` thread. + void SetRtpExtensions(const std::vector<RtpExtension>& extensions); + const RtpHeaderExtensionMap& GetRtpExtensions() const; + + // Called by VideoReceiveStreamInterface when stats are updated. + void UpdateRtt(int64_t max_rtt_ms); + + // Called when the local_ssrc is changed to match with a sender. + void OnLocalSsrcChange(uint32_t local_ssrc); + + // Forwards the call to set rtcp_sender_ to the RTCP mode of the rtcp sender. + void SetRtcpMode(RtcpMode mode); + + void SetReferenceTimeReport(bool enabled); + + // Sets or clears the callback sink that gets called for RTP packets. Used for + // packet handlers such as FlexFec. Must be called on the packet delivery + // thread (same context as `OnRtpPacket` is called on). + // TODO(bugs.webrtc.org/11993): Packet delivery thread today means `worker + // thread` but will be `network thread`. + void SetPacketSink(RtpPacketSinkInterface* packet_sink); + + // Turns on/off loss notifications. Must be called on the packet delivery + // thread. + void SetLossNotificationEnabled(bool enabled); + + void SetNackHistory(TimeDelta history); + + int ulpfec_payload_type() const; + int red_payload_type() const; + void SetProtectionPayloadTypes(int red_payload_type, int ulpfec_payload_type); + + absl::optional<int64_t> LastReceivedPacketMs() const; + absl::optional<int64_t> LastReceivedKeyframePacketMs() const; + + // Mozilla modification: VideoReceiveStream2 and friends do not surface RTCP + // stats at all, and even on the most recent libwebrtc code there does not + // seem to be any support for these stats right now. So, we hack this in. + void RemoteRTCPSenderInfo(uint32_t* packet_count, uint32_t* octet_count, + int64_t* ntp_timestamp_ms, + int64_t* remote_ntp_timestamp_ms) const; + + private: + // Implements RtpVideoFrameReceiver. + void ManageFrame(std::unique_ptr<RtpFrameObject> frame) override; + + void OnCompleteFrames(RtpFrameReferenceFinder::ReturnVector frame) + RTC_RUN_ON(packet_sequence_checker_); + + // Used for buffering RTCP feedback messages and sending them all together. + // Note: + // 1. Key frame requests and NACKs are mutually exclusive, with the + // former taking precedence over the latter. + // 2. Loss notifications are orthogonal to either. (That is, may be sent + // alongside either.) + class RtcpFeedbackBuffer : public KeyFrameRequestSender, + public NackSender, + public LossNotificationSender { + public: + RtcpFeedbackBuffer(KeyFrameRequestSender* key_frame_request_sender, + NackSender* nack_sender, + LossNotificationSender* loss_notification_sender); + + ~RtcpFeedbackBuffer() override = default; + + // KeyFrameRequestSender implementation. + void RequestKeyFrame() override; + + // NackSender implementation. + void SendNack(const std::vector<uint16_t>& sequence_numbers, + bool buffering_allowed) override; + + // LossNotificationSender implementation. + void SendLossNotification(uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag, + bool buffering_allowed) override; + + // Send all RTCP feedback messages buffered thus far. + void SendBufferedRtcpFeedback(); + + void ClearLossNotificationState(); + + private: + // LNTF-related state. + struct LossNotificationState { + LossNotificationState(uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag) + : last_decoded_seq_num(last_decoded_seq_num), + last_received_seq_num(last_received_seq_num), + decodability_flag(decodability_flag) {} + + uint16_t last_decoded_seq_num; + uint16_t last_received_seq_num; + bool decodability_flag; + }; + + RTC_NO_UNIQUE_ADDRESS SequenceChecker packet_sequence_checker_; + KeyFrameRequestSender* const key_frame_request_sender_; + NackSender* const nack_sender_; + LossNotificationSender* const loss_notification_sender_; + + // Key-frame-request-related state. + bool request_key_frame_ RTC_GUARDED_BY(packet_sequence_checker_); + + // NACK-related state. + std::vector<uint16_t> nack_sequence_numbers_ + RTC_GUARDED_BY(packet_sequence_checker_); + + absl::optional<LossNotificationState> lntf_state_ + RTC_GUARDED_BY(packet_sequence_checker_); + }; + enum ParseGenericDependenciesResult { + kDropPacket, + kHasGenericDescriptor, + kNoGenericDescriptor + }; + + // Entry point doing non-stats work for a received packet. Called + // for the same packet both before and after RED decapsulation. + void ReceivePacket(const RtpPacketReceived& packet) + RTC_RUN_ON(packet_sequence_checker_); + + // Parses and handles RED headers. + // This function assumes that it's being called from only one thread. + void ParseAndHandleEncapsulatingHeader(const RtpPacketReceived& packet) + RTC_RUN_ON(packet_sequence_checker_); + void NotifyReceiverOfEmptyPacket(uint16_t seq_num) + RTC_RUN_ON(packet_sequence_checker_); + bool IsRedEnabled() const; + void InsertSpsPpsIntoTracker(uint8_t payload_type) + RTC_RUN_ON(packet_sequence_checker_); + void OnInsertedPacket(video_coding::PacketBuffer::InsertResult result) + RTC_RUN_ON(packet_sequence_checker_); + ParseGenericDependenciesResult ParseGenericDependenciesExtension( + const RtpPacketReceived& rtp_packet, + RTPVideoHeader* video_header) RTC_RUN_ON(packet_sequence_checker_); + void OnAssembledFrame(std::unique_ptr<RtpFrameObject> frame) + RTC_RUN_ON(packet_sequence_checker_); + void UpdatePacketReceiveTimestamps(const RtpPacketReceived& packet, + bool is_keyframe) + RTC_RUN_ON(packet_sequence_checker_); + + const FieldTrialsView& field_trials_; + TaskQueueBase* const worker_queue_; + Clock* const clock_; + // Ownership of this object lies with VideoReceiveStreamInterface, which owns + // `this`. + const VideoReceiveStreamInterface::Config& config_; + PacketRouter* const packet_router_; + + RemoteNtpTimeEstimator ntp_estimator_; + + RtpHeaderExtensionMap rtp_header_extensions_ + RTC_GUARDED_BY(packet_sequence_checker_); + // Set by the field trial WebRTC-ForcePlayoutDelay to override any playout + // delay that is specified in the received packets. + FieldTrialOptional<int> forced_playout_delay_max_ms_; + FieldTrialOptional<int> forced_playout_delay_min_ms_; + ReceiveStatistics* const rtp_receive_statistics_; + std::unique_ptr<UlpfecReceiver> ulpfec_receiver_ + RTC_GUARDED_BY(packet_sequence_checker_); + int red_payload_type_ RTC_GUARDED_BY(packet_sequence_checker_); + + RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_task_checker_; + // TODO(bugs.webrtc.org/11993): This checker conceptually represents + // operations that belong to the network thread. The Call class is currently + // moving towards handling network packets on the network thread and while + // that work is ongoing, this checker may in practice represent the worker + // thread, but still serves as a mechanism of grouping together concepts + // that belong to the network thread. Once the packets are fully delivered + // on the network thread, this comment will be deleted. + RTC_NO_UNIQUE_ADDRESS SequenceChecker packet_sequence_checker_; + RtpPacketSinkInterface* packet_sink_ RTC_GUARDED_BY(packet_sequence_checker_); + bool receiving_ RTC_GUARDED_BY(packet_sequence_checker_); + int64_t last_packet_log_ms_ RTC_GUARDED_BY(packet_sequence_checker_); + + const std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_; + + NackPeriodicProcessor* const nack_periodic_processor_; + OnCompleteFrameCallback* complete_frame_callback_; + const KeyFrameReqMethod keyframe_request_method_; + + RtcpFeedbackBuffer rtcp_feedback_buffer_; + // TODO(tommi): Consider absl::optional<NackRequester> instead of unique_ptr + // since nack is usually configured. + std::unique_ptr<NackRequester> nack_module_ + RTC_GUARDED_BY(packet_sequence_checker_); + std::unique_ptr<LossNotificationController> loss_notification_controller_ + RTC_GUARDED_BY(packet_sequence_checker_); + + VCMReceiveStatisticsCallback* const vcm_receive_statistics_; + video_coding::PacketBuffer packet_buffer_ + RTC_GUARDED_BY(packet_sequence_checker_); + UniqueTimestampCounter frame_counter_ + RTC_GUARDED_BY(packet_sequence_checker_); + SeqNumUnwrapper<uint16_t> frame_id_unwrapper_ + RTC_GUARDED_BY(packet_sequence_checker_); + + // Video structure provided in the dependency descriptor in a first packet + // of a key frame. It is required to parse dependency descriptor in the + // following delta packets. + std::unique_ptr<FrameDependencyStructure> video_structure_ + RTC_GUARDED_BY(packet_sequence_checker_); + // Frame id of the last frame with the attached video structure. + // absl::nullopt when `video_structure_ == nullptr`; + absl::optional<int64_t> video_structure_frame_id_ + RTC_GUARDED_BY(packet_sequence_checker_); + Timestamp last_logged_failed_to_parse_dd_ + RTC_GUARDED_BY(packet_sequence_checker_) = Timestamp::MinusInfinity(); + + std::unique_ptr<RtpFrameReferenceFinder> reference_finder_ + RTC_GUARDED_BY(packet_sequence_checker_); + absl::optional<VideoCodecType> current_codec_ + RTC_GUARDED_BY(packet_sequence_checker_); + uint32_t last_assembled_frame_rtp_timestamp_ + RTC_GUARDED_BY(packet_sequence_checker_); + + std::map<int64_t, uint16_t> last_seq_num_for_pic_id_ + RTC_GUARDED_BY(packet_sequence_checker_); + video_coding::H264SpsPpsTracker tracker_ + RTC_GUARDED_BY(packet_sequence_checker_); + + // Maps payload id to the depacketizer. + std::map<uint8_t, std::unique_ptr<VideoRtpDepacketizer>> payload_type_map_ + RTC_GUARDED_BY(packet_sequence_checker_); + + // TODO(johan): Remove pt_codec_params_ once + // https://bugs.chromium.org/p/webrtc/issues/detail?id=6883 is resolved. + // Maps a payload type to a map of out-of-band supplied codec parameters. + std::map<uint8_t, std::map<std::string, std::string>> pt_codec_params_ + RTC_GUARDED_BY(packet_sequence_checker_); + int16_t last_payload_type_ RTC_GUARDED_BY(packet_sequence_checker_) = -1; + + bool has_received_frame_ RTC_GUARDED_BY(packet_sequence_checker_); + + absl::optional<uint32_t> last_received_rtp_timestamp_ + RTC_GUARDED_BY(packet_sequence_checker_); + absl::optional<uint32_t> last_received_keyframe_rtp_timestamp_ + RTC_GUARDED_BY(packet_sequence_checker_); + absl::optional<Timestamp> last_received_rtp_system_time_ + RTC_GUARDED_BY(packet_sequence_checker_); + absl::optional<Timestamp> last_received_keyframe_rtp_system_time_ + RTC_GUARDED_BY(packet_sequence_checker_); + + // Handles incoming encrypted frames and forwards them to the + // rtp_reference_finder if they are decryptable. + std::unique_ptr<BufferedFrameDecryptor> buffered_frame_decryptor_ + RTC_PT_GUARDED_BY(packet_sequence_checker_); + bool frames_decryptable_ RTC_GUARDED_BY(worker_task_checker_); + absl::optional<ColorSpace> last_color_space_; + + AbsoluteCaptureTimeInterpolator absolute_capture_time_interpolator_ + RTC_GUARDED_BY(packet_sequence_checker_); + + CaptureClockOffsetUpdater capture_clock_offset_updater_ + RTC_GUARDED_BY(packet_sequence_checker_); + + int64_t last_completed_picture_id_ = 0; + + rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate> + frame_transformer_delegate_; + + SeqNumUnwrapper<uint16_t> rtp_seq_num_unwrapper_ + RTC_GUARDED_BY(packet_sequence_checker_); + std::map<int64_t, RtpPacketInfo> packet_infos_ + RTC_GUARDED_BY(packet_sequence_checker_); + + Timestamp next_keyframe_request_for_missing_video_structure_ = + Timestamp::MinusInfinity(); +}; + +} // namespace webrtc + +#endif // VIDEO_RTP_VIDEO_STREAM_RECEIVER2_H_ |