/* * Copyright 2018 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #ifndef VIDEO_VIDEO_SEND_STREAM_IMPL_H_ #define VIDEO_VIDEO_SEND_STREAM_IMPL_H_ #include #include #include #include #include #include #include #include "absl/types/optional.h" #include "api/field_trials_view.h" #include "api/metronome/metronome.h" #include "api/task_queue/pending_task_safety_flag.h" #include "api/task_queue/task_queue_base.h" #include "api/video/encoded_image.h" #include "api/video/video_bitrate_allocation.h" #include "api/video_codecs/video_encoder.h" #include "call/bitrate_allocator.h" #include "call/rtp_config.h" #include "call/rtp_transport_controller_send_interface.h" #include "call/rtp_video_sender_interface.h" #include "call/video_send_stream.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/video_coding/include/video_codec_interface.h" #include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/system/no_unique_address.h" #include "rtc_base/task_utils/repeating_task.h" #include "rtc_base/thread_annotations.h" #include "video/config/video_encoder_config.h" #include "video/encoder_rtcp_feedback.h" #include "video/send_delay_stats.h" #include "video/send_statistics_proxy.h" #include "video/video_stream_encoder_interface.h" namespace webrtc { namespace test { class VideoSendStreamPeer; } // namespace test namespace internal { // Pacing buffer config; overridden by ALR config if provided. struct PacingConfig { explicit PacingConfig(const FieldTrialsView& field_trials); PacingConfig(const PacingConfig&); PacingConfig& operator=(const PacingConfig&) = default; ~PacingConfig(); FieldTrialParameter pacing_factor; FieldTrialParameter max_pacing_delay; }; // VideoSendStreamImpl implements webrtc::VideoSendStream. // It is created and destroyed on `worker queue`. The intent is to // An encoder may deliver frames through the EncodedImageCallback on an // arbitrary thread. class VideoSendStreamImpl : public webrtc::VideoSendStream, public webrtc::BitrateAllocatorObserver, public VideoStreamEncoderInterface::EncoderSink { public: using RtpStateMap = std::map; using RtpPayloadStateMap = std::map; VideoSendStreamImpl(Clock* clock, int num_cpu_cores, TaskQueueFactory* task_queue_factory, RtcpRttStats* call_stats, RtpTransportControllerSendInterface* transport, Metronome* metronome, BitrateAllocatorInterface* bitrate_allocator, SendDelayStats* send_delay_stats, RtcEventLog* event_log, VideoSendStream::Config config, VideoEncoderConfig encoder_config, const RtpStateMap& suspended_ssrcs, const RtpPayloadStateMap& suspended_payload_states, std::unique_ptr fec_controller, const FieldTrialsView& field_trials, std::unique_ptr video_stream_encoder_for_test = nullptr); ~VideoSendStreamImpl() override; void DeliverRtcp(const uint8_t* packet, size_t length); // webrtc::VideoSendStream implementation. void Start() override; void Stop() override; bool started() override; void AddAdaptationResource(rtc::scoped_refptr resource) override; std::vector> GetAdaptationResources() override; void SetSource(rtc::VideoSourceInterface* source, const DegradationPreference& degradation_preference) override; void ReconfigureVideoEncoder(VideoEncoderConfig config) override; void ReconfigureVideoEncoder(VideoEncoderConfig config, SetParametersCallback callback) override; Stats GetStats() override; void StopPermanentlyAndGetRtpStates(RtpStateMap* rtp_state_map, RtpPayloadStateMap* payload_state_map); void GenerateKeyFrame(const std::vector& rids) override; // TODO(holmer): Move these to RtpTransportControllerSend. std::map GetRtpStates() const; std::map GetRtpPayloadStates() const; const absl::optional& configured_pacing_factor() const { return configured_pacing_factor_; } private: friend class test::VideoSendStreamPeer; class OnSendPacketObserver : public SendPacketObserver { public: OnSendPacketObserver(SendStatisticsProxy* stats_proxy, SendDelayStats* send_delay_stats) : stats_proxy_(*stats_proxy), send_delay_stats_(*send_delay_stats) {} void OnSendPacket(absl::optional packet_id, Timestamp capture_time, uint32_t ssrc) override { stats_proxy_.OnSendPacket(ssrc, capture_time); if (packet_id.has_value()) { send_delay_stats_.OnSendPacket(*packet_id, capture_time, ssrc); } } private: SendStatisticsProxy& stats_proxy_; SendDelayStats& send_delay_stats_; }; absl::optional GetPacingFactorOverride() const; // Implements BitrateAllocatorObserver. uint32_t OnBitrateUpdated(BitrateAllocationUpdate update) override; // Implements VideoStreamEncoderInterface::EncoderSink void OnEncoderConfigurationChanged( std::vector streams, bool is_svc, VideoEncoderConfig::ContentType content_type, int min_transmit_bitrate_bps) override; void OnBitrateAllocationUpdated( const VideoBitrateAllocation& allocation) override; void OnVideoLayersAllocationUpdated( VideoLayersAllocation allocation) override; // Implements EncodedImageCallback. The implementation routes encoded frames // to the `payload_router_` and `config.pre_encode_callback` if set. // Called on an arbitrary encoder callback thread. EncodedImageCallback::Result OnEncodedImage( const EncodedImage& encoded_image, const CodecSpecificInfo* codec_specific_info) override; // Implements EncodedImageCallback. void OnDroppedFrame(EncodedImageCallback::DropReason reason) override; // Starts monitoring and sends a keyframe. void StartupVideoSendStream(); // Removes the bitrate observer, stops monitoring and notifies the video // encoder of the bitrate update. void StopVideoSendStream() RTC_RUN_ON(thread_checker_); void ConfigureProtection(); void ConfigureSsrcs(); void SignalEncoderTimedOut(); void SignalEncoderActive(); // A video send stream is running if VideoSendStream::Start has been invoked // and there is an active encoding. bool IsRunning() const; MediaStreamAllocationConfig GetAllocationConfig() const RTC_RUN_ON(thread_checker_); RTC_NO_UNIQUE_ADDRESS SequenceChecker thread_checker_; RtpTransportControllerSendInterface* const transport_; SendStatisticsProxy stats_proxy_; OnSendPacketObserver send_packet_observer_; const VideoSendStream::Config config_; const VideoEncoderConfig::ContentType content_type_; std::unique_ptr video_stream_encoder_; EncoderRtcpFeedback encoder_feedback_; RtpVideoSenderInterface* const rtp_video_sender_; bool running_ RTC_GUARDED_BY(thread_checker_) = false; Clock* const clock_; const bool has_alr_probing_; const PacingConfig pacing_config_; TaskQueueBase* const worker_queue_; RepeatingTaskHandle check_encoder_activity_task_ RTC_GUARDED_BY(thread_checker_); std::atomic_bool activity_; bool timed_out_ RTC_GUARDED_BY(thread_checker_); BitrateAllocatorInterface* const bitrate_allocator_; bool has_active_encodings_ RTC_GUARDED_BY(thread_checker_); bool disable_padding_ RTC_GUARDED_BY(thread_checker_); int max_padding_bitrate_ RTC_GUARDED_BY(thread_checker_); int encoder_min_bitrate_bps_ RTC_GUARDED_BY(thread_checker_); uint32_t encoder_max_bitrate_bps_ RTC_GUARDED_BY(thread_checker_); uint32_t encoder_target_rate_bps_ RTC_GUARDED_BY(thread_checker_); double encoder_bitrate_priority_ RTC_GUARDED_BY(thread_checker_); const int encoder_av1_priority_bitrate_override_bps_ RTC_GUARDED_BY(thread_checker_); ScopedTaskSafety worker_queue_safety_; // Context for the most recent and last sent video bitrate allocation. Used to // throttle sending of similar bitrate allocations. struct VbaSendContext { VideoBitrateAllocation last_sent_allocation; absl::optional throttled_allocation; int64_t last_send_time_ms; }; absl::optional video_bitrate_allocation_context_ RTC_GUARDED_BY(thread_checker_); const absl::optional configured_pacing_factor_; }; } // namespace internal } // namespace webrtc #endif // VIDEO_VIDEO_SEND_STREAM_IMPL_H_