From: "Byron Campen [:bwc]" Date: Fri, 19 Feb 2021 15:56:00 -0600 Subject: Bug 1654112 - Get RTCP BYE and RTP timeout handling working again (from Bug 1595479) r=mjf,dminor Differential Revision: https://phabricator.services.mozilla.com/D106145 Mercurial Revision: https://hg.mozilla.org/mozilla-central/rev/d0b311007c033e83824f5f6996a70ab9e870f31f --- audio/audio_receive_stream.cc | 5 ++++- audio/channel_receive.cc | 13 +++++++++---- audio/channel_receive.h | 3 ++- call/audio_receive_stream.h | 3 +++ call/video_receive_stream.cc | 2 ++ call/video_receive_stream.h | 3 +++ modules/rtp_rtcp/include/rtp_rtcp_defines.h | 8 ++++++++ modules/rtp_rtcp/source/rtcp_receiver.cc | 18 ++++++++++++++++-- modules/rtp_rtcp/source/rtcp_receiver.h | 1 + modules/rtp_rtcp/source/rtp_rtcp_interface.h | 3 +++ video/rtp_video_stream_receiver2.cc | 7 +++++-- 11 files changed, 56 insertions(+), 10 deletions(-) diff --git a/audio/audio_receive_stream.cc b/audio/audio_receive_stream.cc index 0bb1168384..7063f40186 100644 --- a/audio/audio_receive_stream.cc +++ b/audio/audio_receive_stream.cc @@ -47,6 +47,8 @@ std::string AudioReceiveStreamInterface::Config::Rtp::ToString() const { } } ss << ']'; + ss << ", rtcp_event_observer: " + << (rtcp_event_observer ? "(rtcp_event_observer)" : "nullptr"); ss << '}'; return ss.str(); } @@ -81,7 +83,8 @@ std::unique_ptr CreateChannelReceive( config.jitter_buffer_fast_accelerate, config.jitter_buffer_min_delay_ms, config.enable_non_sender_rtt, config.decoder_factory, config.codec_pair_id, std::move(config.frame_decryptor), - config.crypto_options, std::move(config.frame_transformer)); + config.crypto_options, std::move(config.frame_transformer), + config.rtp.rtcp_event_observer); } } // namespace diff --git a/audio/channel_receive.cc b/audio/channel_receive.cc index b95d98c20c..50bc94fe1f 100644 --- a/audio/channel_receive.cc +++ b/audio/channel_receive.cc @@ -102,7 +102,8 @@ class ChannelReceive : public ChannelReceiveInterface, absl::optional codec_pair_id, rtc::scoped_refptr frame_decryptor, const webrtc::CryptoOptions& crypto_options, - rtc::scoped_refptr frame_transformer); + rtc::scoped_refptr frame_transformer, + RtcpEventObserver* rtcp_event_observer); ~ChannelReceive() override; void SetSink(AudioSinkInterface* sink) override; @@ -541,7 +542,8 @@ ChannelReceive::ChannelReceive( absl::optional codec_pair_id, rtc::scoped_refptr frame_decryptor, const webrtc::CryptoOptions& crypto_options, - rtc::scoped_refptr frame_transformer) + rtc::scoped_refptr frame_transformer, + RtcpEventObserver* rtcp_event_observer) : worker_thread_(TaskQueueBase::Current()), event_log_(rtc_event_log), rtp_receive_statistics_(ReceiveStatistics::Create(clock)), @@ -586,6 +588,7 @@ ChannelReceive::ChannelReceive( configuration.local_media_ssrc = local_ssrc; configuration.rtcp_packet_type_counter_observer = this; configuration.non_sender_rtt_measurement = enable_non_sender_rtt; + configuration.rtcp_event_observer = rtcp_event_observer; if (frame_transformer) InitFrameTransformerDelegate(std::move(frame_transformer)); @@ -1119,13 +1122,15 @@ std::unique_ptr CreateChannelReceive( absl::optional codec_pair_id, rtc::scoped_refptr frame_decryptor, const webrtc::CryptoOptions& crypto_options, - rtc::scoped_refptr frame_transformer) { + rtc::scoped_refptr frame_transformer, + RtcpEventObserver* rtcp_event_observer) { return std::make_unique( clock, neteq_factory, audio_device_module, rtcp_send_transport, rtc_event_log, local_ssrc, remote_ssrc, jitter_buffer_max_packets, jitter_buffer_fast_playout, jitter_buffer_min_delay_ms, enable_non_sender_rtt, decoder_factory, codec_pair_id, - std::move(frame_decryptor), crypto_options, std::move(frame_transformer)); + std::move(frame_decryptor), crypto_options, std::move(frame_transformer), + rtcp_event_observer); } } // namespace voe diff --git a/audio/channel_receive.h b/audio/channel_receive.h index b47a4b5b97..dd3ca1af83 100644 --- a/audio/channel_receive.h +++ b/audio/channel_receive.h @@ -186,7 +186,8 @@ std::unique_ptr CreateChannelReceive( absl::optional codec_pair_id, rtc::scoped_refptr frame_decryptor, const webrtc::CryptoOptions& crypto_options, - rtc::scoped_refptr frame_transformer); + rtc::scoped_refptr frame_transformer, + RtcpEventObserver* rtcp_event_observer); } // namespace voe } // namespace webrtc diff --git a/call/audio_receive_stream.h b/call/audio_receive_stream.h index 1228861c42..6fc93b2d9a 100644 --- a/call/audio_receive_stream.h +++ b/call/audio_receive_stream.h @@ -19,6 +19,7 @@ #include "absl/types/optional.h" #include "api/audio_codecs/audio_decoder_factory.h" #include "api/call/transport.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "api/crypto/crypto_options.h" #include "api/rtp_parameters.h" #include "call/receive_stream.h" @@ -117,6 +118,8 @@ class AudioReceiveStreamInterface : public MediaReceiveStreamInterface { // See NackConfig for description. NackConfig nack; + + RtcpEventObserver* rtcp_event_observer = nullptr; } rtp; // Receive-side RTT. diff --git a/call/video_receive_stream.cc b/call/video_receive_stream.cc index 87df97cbdd..838dfcf135 100644 --- a/call/video_receive_stream.cc +++ b/call/video_receive_stream.cc @@ -153,6 +153,8 @@ std::string VideoReceiveStreamInterface::Config::Rtp::ToString() const { ss << ", "; } ss << ']'; + ss << ", rtcp_event_observer: " + << (rtcp_event_observer ? "(rtcp_event_observer)" : "nullptr"); ss << '}'; return ss.str(); } diff --git a/call/video_receive_stream.h b/call/video_receive_stream.h index cda8b1f6af..eeb7d14cc3 100644 --- a/call/video_receive_stream.h +++ b/call/video_receive_stream.h @@ -19,6 +19,7 @@ #include #include "api/call/transport.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "api/crypto/crypto_options.h" #include "api/rtp_headers.h" #include "api/rtp_parameters.h" @@ -234,6 +235,8 @@ class VideoReceiveStreamInterface : public MediaReceiveStreamInterface { // meta data is expected to be present in generic frame descriptor // RTP header extension). std::set raw_payload_types; + + RtcpEventObserver* rtcp_event_observer = nullptr; } rtp; // Transport for outgoing packets (RTCP). diff --git a/modules/rtp_rtcp/include/rtp_rtcp_defines.h b/modules/rtp_rtcp/include/rtp_rtcp_defines.h index 43bba3e57a..882f861d0b 100644 --- a/modules/rtp_rtcp/include/rtp_rtcp_defines.h +++ b/modules/rtp_rtcp/include/rtp_rtcp_defines.h @@ -211,6 +211,14 @@ class RtcpBandwidthObserver { virtual ~RtcpBandwidthObserver() {} }; +class RtcpEventObserver { + public: + virtual void OnRtcpBye() = 0; + virtual void OnRtcpTimeout() = 0; + + virtual ~RtcpEventObserver() {} +}; + // NOTE! `kNumMediaTypes` must be kept in sync with RtpPacketMediaType! static constexpr size_t kNumMediaTypes = 5; enum class RtpPacketMediaType : size_t { diff --git a/modules/rtp_rtcp/source/rtcp_receiver.cc b/modules/rtp_rtcp/source/rtcp_receiver.cc index 68171d1c2a..69d62ead5a 100644 --- a/modules/rtp_rtcp/source/rtcp_receiver.cc +++ b/modules/rtp_rtcp/source/rtcp_receiver.cc @@ -145,6 +145,7 @@ RTCPReceiver::RTCPReceiver(const RtpRtcpInterface::Configuration& config, rtp_rtcp_(owner), registered_ssrcs_(false, config), rtcp_bandwidth_observer_(config.bandwidth_callback), + rtcp_event_observer_(config.rtcp_event_observer), rtcp_intra_frame_observer_(config.intra_frame_callback), rtcp_loss_notification_observer_(config.rtcp_loss_notification_observer), network_state_estimate_observer_(config.network_state_estimate_observer), @@ -178,6 +179,7 @@ RTCPReceiver::RTCPReceiver(const RtpRtcpInterface::Configuration& config, rtp_rtcp_(owner), registered_ssrcs_(true, config), rtcp_bandwidth_observer_(config.bandwidth_callback), + rtcp_event_observer_(config.rtcp_event_observer), rtcp_intra_frame_observer_(config.intra_frame_callback), rtcp_loss_notification_observer_(config.rtcp_loss_notification_observer), network_state_estimate_observer_(config.network_state_estimate_observer), @@ -848,6 +850,10 @@ void RTCPReceiver::HandleBye(const CommonHeader& rtcp_block) { return; } + if (rtcp_event_observer_) { + rtcp_event_observer_->OnRtcpBye(); + } + // Clear our lists. rtts_.erase(bye.sender_ssrc()); EraseIf(received_report_blocks_, [&](const auto& elem) { @@ -1265,12 +1271,20 @@ std::vector RTCPReceiver::TmmbrReceived() { } bool RTCPReceiver::RtcpRrTimeoutLocked(Timestamp now) { - return ResetTimestampIfExpired(now, last_received_rb_, report_interval_); + bool result = ResetTimestampIfExpired(now, last_received_rb_, report_interval_); + if (result && rtcp_event_observer_) { + rtcp_event_observer_->OnRtcpTimeout(); + } + return result; } bool RTCPReceiver::RtcpRrSequenceNumberTimeoutLocked(Timestamp now) { - return ResetTimestampIfExpired(now, last_increased_sequence_number_, + bool result = ResetTimestampIfExpired(now, last_increased_sequence_number_, report_interval_); + if (result && rtcp_event_observer_) { + rtcp_event_observer_->OnRtcpTimeout(); + } + return result; } } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtcp_receiver.h b/modules/rtp_rtcp/source/rtcp_receiver.h index 6912912cfc..a05a69059a 100644 --- a/modules/rtp_rtcp/source/rtcp_receiver.h +++ b/modules/rtp_rtcp/source/rtcp_receiver.h @@ -385,6 +385,7 @@ class RTCPReceiver final { RegisteredSsrcs registered_ssrcs_; RtcpBandwidthObserver* const rtcp_bandwidth_observer_; + RtcpEventObserver* const rtcp_event_observer_; RtcpIntraFrameObserver* const rtcp_intra_frame_observer_; RtcpLossNotificationObserver* const rtcp_loss_notification_observer_; NetworkStateEstimateObserver* const network_state_estimate_observer_; diff --git a/modules/rtp_rtcp/source/rtp_rtcp_interface.h b/modules/rtp_rtcp/source/rtp_rtcp_interface.h index c6854937cb..b988c7805d 100644 --- a/modules/rtp_rtcp/source/rtp_rtcp_interface.h +++ b/modules/rtp_rtcp/source/rtp_rtcp_interface.h @@ -73,6 +73,9 @@ class RtpRtcpInterface : public RtcpFeedbackSenderInterface { // stream. RtcpBandwidthObserver* bandwidth_callback = nullptr; + // Called when we receive a RTCP bye or timeout + RtcpEventObserver* rtcp_event_observer = nullptr; + NetworkStateEstimateObserver* network_state_estimate_observer = nullptr; TransportFeedbackObserver* transport_feedback_callback = nullptr; VideoBitrateAllocationObserver* bitrate_allocation_observer = nullptr; diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc index eed9770d93..c7b5e7bc7c 100644 --- a/video/rtp_video_stream_receiver2.cc +++ b/video/rtp_video_stream_receiver2.cc @@ -83,7 +83,8 @@ std::unique_ptr CreateRtpRtcpModule( RtcpCnameCallback* rtcp_cname_callback, bool non_sender_rtt_measurement, uint32_t local_ssrc, - RtcEventLog* rtc_event_log) { + RtcEventLog* rtc_event_log, + RtcpEventObserver* rtcp_event_observer) { RtpRtcpInterface::Configuration configuration; configuration.clock = clock; configuration.audio = false; @@ -95,6 +96,7 @@ std::unique_ptr CreateRtpRtcpModule( rtcp_packet_type_counter_observer; configuration.rtcp_cname_callback = rtcp_cname_callback; configuration.local_media_ssrc = local_ssrc; + configuration.rtcp_event_observer = rtcp_event_observer; configuration.non_sender_rtt_measurement = non_sender_rtt_measurement; configuration.event_log = rtc_event_log; @@ -276,7 +278,8 @@ RtpVideoStreamReceiver2::RtpVideoStreamReceiver2( rtcp_cname_callback, config_.rtp.rtcp_xr.receiver_reference_time_report, config_.rtp.local_ssrc, - event_log)), + event_log, + config_.rtp.rtcp_event_observer)), nack_periodic_processor_(nack_periodic_processor), complete_frame_callback_(complete_frame_callback), keyframe_request_method_(config_.rtp.keyframe_method), -- 2.34.1