diff options
Diffstat (limited to 'third_party/libwebrtc/media/engine/webrtc_video_engine.h')
| -rw-r--r-- | third_party/libwebrtc/media/engine/webrtc_video_engine.h | 906 |
1 files changed, 906 insertions, 0 deletions
diff --git a/third_party/libwebrtc/media/engine/webrtc_video_engine.h b/third_party/libwebrtc/media/engine/webrtc_video_engine.h new file mode 100644 index 0000000000..e4b1b2765b --- /dev/null +++ b/third_party/libwebrtc/media/engine/webrtc_video_engine.h @@ -0,0 +1,906 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_ +#define MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_ + +#include <stddef.h> + +#include <cstdint> +#include <functional> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <utility> +#include <vector> + +#include "absl/functional/any_invocable.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/call/transport.h" +#include "api/crypto/crypto_options.h" +#include "api/crypto/frame_decryptor_interface.h" +#include "api/crypto/frame_encryptor_interface.h" +#include "api/field_trials_view.h" +#include "api/frame_transformer_interface.h" +#include "api/rtc_error.h" +#include "api/rtp_headers.h" +#include "api/rtp_parameters.h" +#include "api/rtp_sender_interface.h" +#include "api/scoped_refptr.h" +#include "api/sequence_checker.h" +#include "api/task_queue/pending_task_safety_flag.h" +#include "api/task_queue/task_queue_base.h" +#include "api/transport/bitrate_settings.h" +#include "api/transport/field_trial_based_config.h" +#include "api/transport/rtp/rtp_source.h" +#include "api/video/recordable_encoded_frame.h" +#include "api/video/video_bitrate_allocator_factory.h" +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "api/video/video_source_interface.h" +#include "api/video/video_stream_encoder_settings.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "call/call.h" +#include "call/flexfec_receive_stream.h" +#include "call/rtp_config.h" +#include "call/video_receive_stream.h" +#include "call/video_send_stream.h" +#include "media/base/codec.h" +#include "media/base/media_channel.h" +#include "media/base/media_channel_impl.h" +#include "media/base/media_config.h" +#include "media/base/media_engine.h" +#include "media/base/stream_params.h" +#include "modules/rtp_rtcp/include/rtp_header_extension_map.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "rtc_base/network/sent_packet.h" +#include "rtc_base/network_route.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/system/no_unique_address.h" +#include "rtc_base/thread_annotations.h" +#include "video/config/video_encoder_config.h" + +namespace webrtc { +class VideoDecoderFactory; +class VideoEncoderFactory; +} // namespace webrtc + +namespace cricket { + +// Public for testing. +// Inputs StreamStats for all types of substreams (kMedia, kRtx, kFlexfec) and +// merges any non-kMedia substream stats object into its referenced kMedia-type +// substream. The resulting substreams are all kMedia. This means, for example, +// that packet and byte counters of RTX and FlexFEC streams are accounted for in +// the relevant RTP media stream's stats. This makes the resulting StreamStats +// objects ready to be turned into "outbound-rtp" stats objects for GetStats() +// which does not create separate stream stats objects for complementary +// streams. +std::map<uint32_t, webrtc::VideoSendStream::StreamStats> +MergeInfoAboutOutboundRtpSubstreamsForTesting( + const std::map<uint32_t, webrtc::VideoSendStream::StreamStats>& substreams); + +// WebRtcVideoEngine is used for the new native WebRTC Video API (webrtc:1667). +class WebRtcVideoEngine : public VideoEngineInterface { + public: + // These video codec factories represents all video codecs, i.e. both software + // and external hardware codecs. + WebRtcVideoEngine( + std::unique_ptr<webrtc::VideoEncoderFactory> video_encoder_factory, + std::unique_ptr<webrtc::VideoDecoderFactory> video_decoder_factory, + const webrtc::FieldTrialsView& trials); + + ~WebRtcVideoEngine() override; + + std::unique_ptr<VideoMediaSendChannelInterface> CreateSendChannel( + webrtc::Call* call, + const MediaConfig& config, + const VideoOptions& options, + const webrtc::CryptoOptions& crypto_options, + webrtc::VideoBitrateAllocatorFactory* video_bitrate_allocator_factory) + override; + std::unique_ptr<VideoMediaReceiveChannelInterface> CreateReceiveChannel( + webrtc::Call* call, + const MediaConfig& config, + const VideoOptions& options, + const webrtc::CryptoOptions& crypto_options) override; + + std::vector<VideoCodec> send_codecs() const override { + return send_codecs(true); + } + std::vector<VideoCodec> recv_codecs() const override { + return recv_codecs(true); + } + std::vector<VideoCodec> send_codecs(bool include_rtx) const override; + std::vector<VideoCodec> recv_codecs(bool include_rtx) const override; + std::vector<webrtc::RtpHeaderExtensionCapability> GetRtpHeaderExtensions() + const override; + + private: + const std::unique_ptr<webrtc::VideoDecoderFactory> decoder_factory_; + const std::unique_ptr<webrtc::VideoEncoderFactory> encoder_factory_; + const std::unique_ptr<webrtc::VideoBitrateAllocatorFactory> + bitrate_allocator_factory_; + const webrtc::FieldTrialsView& trials_; +}; + +struct VideoCodecSettings { + explicit VideoCodecSettings(const VideoCodec& codec); + + // Checks if all members of |*this| are equal to the corresponding members + // of `other`. + bool operator==(const VideoCodecSettings& other) const; + bool operator!=(const VideoCodecSettings& other) const; + + // Checks if all members of `a`, except `flexfec_payload_type`, are equal + // to the corresponding members of `b`. + static bool EqualsDisregardingFlexfec(const VideoCodecSettings& a, + const VideoCodecSettings& b); + + VideoCodec codec; + webrtc::UlpfecConfig ulpfec; + int flexfec_payload_type; // -1 if absent. + int rtx_payload_type; // -1 if absent. + absl::optional<int> rtx_time; +}; + +class WebRtcVideoSendChannel : public MediaChannelUtil, + public VideoMediaSendChannelInterface, + public webrtc::EncoderSwitchRequestCallback { + public: + WebRtcVideoSendChannel( + webrtc::Call* call, + const MediaConfig& config, + const VideoOptions& options, + const webrtc::CryptoOptions& crypto_options, + webrtc::VideoEncoderFactory* encoder_factory, + webrtc::VideoDecoderFactory* decoder_factory, + webrtc::VideoBitrateAllocatorFactory* bitrate_allocator_factory); + ~WebRtcVideoSendChannel() override; + + MediaType media_type() const override { return MEDIA_TYPE_VIDEO; } + // Type manipulations + VideoMediaSendChannelInterface* AsVideoSendChannel() override { return this; } + VoiceMediaSendChannelInterface* AsVoiceSendChannel() override { + RTC_CHECK_NOTREACHED(); + return nullptr; + } + // Functions imported from MediaChannelUtil + bool HasNetworkInterface() const override { + return MediaChannelUtil::HasNetworkInterface(); + } + void SetExtmapAllowMixed(bool extmap_allow_mixed) override { + MediaChannelUtil::SetExtmapAllowMixed(extmap_allow_mixed); + } + bool ExtmapAllowMixed() const override { + return MediaChannelUtil::ExtmapAllowMixed(); + } + + // Common functions between sender and receiver + void SetInterface(MediaChannelNetworkInterface* iface) override; + // VideoMediaSendChannelInterface implementation + bool SetSenderParameters(const VideoSenderParameters& params) override; + webrtc::RTCError SetRtpSendParameters( + uint32_t ssrc, + const webrtc::RtpParameters& parameters, + webrtc::SetParametersCallback callback) override; + webrtc::RtpParameters GetRtpSendParameters(uint32_t ssrc) const override; + absl::optional<Codec> GetSendCodec() const override; + bool SetSend(bool send) override; + bool SetVideoSend( + uint32_t ssrc, + const VideoOptions* options, + rtc::VideoSourceInterface<webrtc::VideoFrame>* source) override; + bool AddSendStream(const StreamParams& sp) override; + bool RemoveSendStream(uint32_t ssrc) override; + void FillBitrateInfo(BandwidthEstimationInfo* bwe_info) override; + bool GetStats(VideoMediaSendInfo* info) override; + + void OnPacketSent(const rtc::SentPacket& sent_packet) override; + void OnReadyToSend(bool ready) override; + void OnNetworkRouteChanged(absl::string_view transport_name, + const rtc::NetworkRoute& network_route) override; + + // Set a frame encryptor to a particular ssrc that will intercept all + // outgoing video frames and attempt to encrypt them and forward the result + // to the packetizer. + void SetFrameEncryptor(uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameEncryptorInterface> + frame_encryptor) override; + + // note: The encoder_selector object must remain valid for the lifetime of the + // MediaChannel, unless replaced. + void SetEncoderSelector(uint32_t ssrc, + webrtc::VideoEncoderFactory::EncoderSelectorInterface* + encoder_selector) override; + + void SetSendCodecChangedCallback( + absl::AnyInvocable<void()> callback) override { + send_codec_changed_callback_ = std::move(callback); + } + + void SetSsrcListChangedCallback( + absl::AnyInvocable<void(const std::set<uint32_t>&)> callback) override { + ssrc_list_changed_callback_ = std::move(callback); + } + + // Implemented for VideoMediaChannelTest. + bool sending() const { + RTC_DCHECK_RUN_ON(&thread_checker_); + return sending_; + } + + // AdaptReason is used for expressing why a WebRtcVideoSendStream request + // a lower input frame size than the currently configured camera input frame + // size. There can be more than one reason OR:ed together. + enum AdaptReason { + ADAPTREASON_NONE = 0, + ADAPTREASON_CPU = 1, + ADAPTREASON_BANDWIDTH = 2, + }; + + // TODO(webrtc:14852): Update downstream projects to use + // cricket::kDefaultVideoMaxQpVpx/H26x and remove. + static constexpr int kDefaultQpMax = 56; + + // Implements webrtc::EncoderSwitchRequestCallback. + void RequestEncoderFallback() override; + void RequestEncoderSwitch(const webrtc::SdpVideoFormat& format, + bool allow_default_fallback) override; + + void GenerateSendKeyFrame(uint32_t ssrc, + const std::vector<std::string>& rids) override; + + void SetEncoderToPacketizerFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) + override; + // Information queries to support SetReceiverFeedbackParameters + webrtc::RtcpMode SendCodecRtcpMode() const override { + RTC_DCHECK_RUN_ON(&thread_checker_); + return send_params_.rtcp.reduced_size ? webrtc::RtcpMode::kReducedSize + : webrtc::RtcpMode::kCompound; + } + + bool SendCodecHasLntf() const override { + RTC_DCHECK_RUN_ON(&thread_checker_); + if (!send_codec()) { + return false; + } + return HasLntf(send_codec()->codec); + } + bool SendCodecHasNack() const override { + RTC_DCHECK_RUN_ON(&thread_checker_); + if (!send_codec()) { + return false; + } + return HasNack(send_codec()->codec); + } + absl::optional<int> SendCodecRtxTime() const override { + RTC_DCHECK_RUN_ON(&thread_checker_); + if (!send_codec()) { + return absl::nullopt; + } + return send_codec()->rtx_time; + } + + private: + struct ChangedSenderParameters { + // These optionals are unset if not changed. + absl::optional<VideoCodecSettings> send_codec; + absl::optional<std::vector<VideoCodecSettings>> negotiated_codecs; + absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions; + absl::optional<std::string> mid; + absl::optional<bool> extmap_allow_mixed; + absl::optional<int> max_bandwidth_bps; + absl::optional<bool> conference_mode; + absl::optional<webrtc::RtcpMode> rtcp_mode; + }; + + bool GetChangedSenderParameters(const VideoSenderParameters& params, + ChangedSenderParameters* changed_params) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + bool ApplyChangedParams(const ChangedSenderParameters& changed_params); + bool ValidateSendSsrcAvailability(const StreamParams& sp) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + + // Populates `rtx_associated_payload_types`, `raw_payload_types` and + // `decoders` based on codec settings provided by `recv_codecs`. + // `recv_codecs` must be non-empty and all other parameters must be empty. + static void ExtractCodecInformation( + rtc::ArrayView<const VideoCodecSettings> recv_codecs, + std::map<int, int>& rtx_associated_payload_types, + std::set<int>& raw_payload_types, + std::vector<webrtc::VideoReceiveStreamInterface::Decoder>& decoders); + + // Wrapper for the sender part. + class WebRtcVideoSendStream { + public: + WebRtcVideoSendStream( + webrtc::Call* call, + const StreamParams& sp, + webrtc::VideoSendStream::Config config, + const VideoOptions& options, + bool enable_cpu_overuse_detection, + int max_bitrate_bps, + const absl::optional<VideoCodecSettings>& codec_settings, + const absl::optional<std::vector<webrtc::RtpExtension>>& rtp_extensions, + const VideoSenderParameters& send_params); + ~WebRtcVideoSendStream(); + + void SetSenderParameters(const ChangedSenderParameters& send_params); + webrtc::RTCError SetRtpParameters(const webrtc::RtpParameters& parameters, + webrtc::SetParametersCallback callback); + webrtc::RtpParameters GetRtpParameters() const; + + void SetFrameEncryptor( + rtc::scoped_refptr<webrtc::FrameEncryptorInterface> frame_encryptor); + + bool SetVideoSend(const VideoOptions* options, + rtc::VideoSourceInterface<webrtc::VideoFrame>* source); + + // note: The encoder_selector object must remain valid for the lifetime of + // the MediaChannel, unless replaced. + void SetEncoderSelector( + webrtc::VideoEncoderFactory::EncoderSelectorInterface* + encoder_selector); + + void SetSend(bool send); + + const std::vector<uint32_t>& GetSsrcs() const; + // Returns per ssrc VideoSenderInfos. Useful for simulcast scenario. + std::vector<VideoSenderInfo> GetPerLayerVideoSenderInfos(bool log_stats); + // Aggregates per ssrc VideoSenderInfos to single VideoSenderInfo for + // legacy reasons. Used in old GetStats API and track stats. + VideoSenderInfo GetAggregatedVideoSenderInfo( + const std::vector<VideoSenderInfo>& infos) const; + void FillBitrateInfo(BandwidthEstimationInfo* bwe_info); + + void SetEncoderToPacketizerFrameTransformer( + rtc::scoped_refptr<webrtc::FrameTransformerInterface> + frame_transformer); + void GenerateKeyFrame(const std::vector<std::string>& rids); + + private: + // Parameters needed to reconstruct the underlying stream. + // webrtc::VideoSendStream doesn't support setting a lot of options on the + // fly, so when those need to be changed we tear down and reconstruct with + // similar parameters depending on which options changed etc. + struct VideoSendStreamParameters { + VideoSendStreamParameters( + webrtc::VideoSendStream::Config config, + const VideoOptions& options, + int max_bitrate_bps, + const absl::optional<VideoCodecSettings>& codec_settings); + webrtc::VideoSendStream::Config config; + VideoOptions options; + int max_bitrate_bps; + bool conference_mode; + absl::optional<VideoCodecSettings> codec_settings; + // Sent resolutions + bitrates etc. by the underlying VideoSendStream, + // typically changes when setting a new resolution or reconfiguring + // bitrates. + webrtc::VideoEncoderConfig encoder_config; + }; + + rtc::scoped_refptr<webrtc::VideoEncoderConfig::EncoderSpecificSettings> + ConfigureVideoEncoderSettings(const VideoCodec& codec); + void SetCodec(const VideoCodecSettings& codec); + void RecreateWebRtcStream(); + webrtc::VideoEncoderConfig CreateVideoEncoderConfig( + const VideoCodec& codec) const; + void ReconfigureEncoder(webrtc::SetParametersCallback callback); + + // Calls Start or Stop according to whether or not `sending_` is true, + // and whether or not the encoding in `rtp_parameters_` is active. + void UpdateSendState(); + + webrtc::DegradationPreference GetDegradationPreference() const + RTC_EXCLUSIVE_LOCKS_REQUIRED(&thread_checker_); + + RTC_NO_UNIQUE_ADDRESS webrtc::SequenceChecker thread_checker_; + webrtc::TaskQueueBase* const worker_thread_; + const std::vector<uint32_t> ssrcs_ RTC_GUARDED_BY(&thread_checker_); + const std::vector<SsrcGroup> ssrc_groups_ RTC_GUARDED_BY(&thread_checker_); + webrtc::Call* const call_; + const bool enable_cpu_overuse_detection_; + rtc::VideoSourceInterface<webrtc::VideoFrame>* source_ + RTC_GUARDED_BY(&thread_checker_); + + webrtc::VideoSendStream* stream_ RTC_GUARDED_BY(&thread_checker_); + + // Contains settings that are the same for all streams in the MediaChannel, + // such as codecs, header extensions, and the global bitrate limit for the + // entire channel. + VideoSendStreamParameters parameters_ RTC_GUARDED_BY(&thread_checker_); + // Contains settings that are unique for each stream, such as max_bitrate. + // Does *not* contain codecs, however. + // TODO(skvlad): Move ssrcs_ and ssrc_groups_ into rtp_parameters_. + // TODO(skvlad): Combine parameters_ and rtp_parameters_ once we have only + // one stream per MediaChannel. + webrtc::RtpParameters rtp_parameters_ RTC_GUARDED_BY(&thread_checker_); + + bool sending_ RTC_GUARDED_BY(&thread_checker_); + + // TODO(asapersson): investigate why setting + // DegrationPreferences::MAINTAIN_RESOLUTION isn't sufficient to disable + // downscaling everywhere in the pipeline. + const bool disable_automatic_resize_; + }; + + void Construct(webrtc::Call* call, WebRtcVideoEngine* engine); + + // Get all codecs that are compatible with the receiver. + std::vector<VideoCodecSettings> SelectSendVideoCodecs( + const std::vector<VideoCodecSettings>& remote_mapped_codecs) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + + void FillSenderStats(VideoMediaSendInfo* info, bool log_stats) + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + void FillBandwidthEstimationStats(const webrtc::Call::Stats& stats, + VideoMediaInfo* info) + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + void FillSendCodecStats(VideoMediaSendInfo* video_media_info) + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + + // Accessor function for send_codec_. Introduced in order to ensure + // that a receive channel does not touch the send codec directly. + // Can go away once these are different classes. + // TODO(bugs.webrtc.org/13931): Remove this function + absl::optional<VideoCodecSettings>& send_codec() { return send_codec_; } + const absl::optional<VideoCodecSettings>& send_codec() const { + return send_codec_; + } + webrtc::TaskQueueBase* const worker_thread_; + webrtc::ScopedTaskSafety task_safety_; + RTC_NO_UNIQUE_ADDRESS webrtc::SequenceChecker network_thread_checker_{ + webrtc::SequenceChecker::kDetached}; + RTC_NO_UNIQUE_ADDRESS webrtc::SequenceChecker thread_checker_; + + uint32_t rtcp_receiver_report_ssrc_ RTC_GUARDED_BY(thread_checker_); + bool sending_ RTC_GUARDED_BY(thread_checker_); + bool receiving_ RTC_GUARDED_BY(&thread_checker_); + webrtc::Call* const call_; + + rtc::VideoSinkInterface<webrtc::VideoFrame>* default_sink_ + RTC_GUARDED_BY(thread_checker_); + + // Delay for unsignaled streams, which may be set before the stream exists. + int default_recv_base_minimum_delay_ms_ RTC_GUARDED_BY(thread_checker_) = 0; + + const MediaConfig::Video video_config_ RTC_GUARDED_BY(thread_checker_); + + // Using primary-ssrc (first ssrc) as key. + std::map<uint32_t, WebRtcVideoSendStream*> send_streams_ + RTC_GUARDED_BY(thread_checker_); + // When the channel and demuxer get reconfigured, there is a window of time + // where we have to be prepared for packets arriving based on the old demuxer + // criteria because the streams live on the worker thread and the demuxer + // lives on the network thread. Because packets are posted from the network + // thread to the worker thread, they can still be in-flight when streams are + // reconfgured. This can happen when `demuxer_criteria_id_` and + // `demuxer_criteria_completed_id_` don't match. During this time, we do not + // want to create unsignalled receive streams and should instead drop the + // packets. E.g: + // * If RemoveRecvStream(old_ssrc) was recently called, there may be packets + // in-flight for that ssrc. This happens when a receiver becomes inactive. + // * If we go from one to many m= sections, the demuxer may change from + // forwarding all packets to only forwarding the configured ssrcs, so there + // is a risk of receiving ssrcs for other, recently added m= sections. + uint32_t demuxer_criteria_id_ RTC_GUARDED_BY(thread_checker_) = 0; + uint32_t demuxer_criteria_completed_id_ RTC_GUARDED_BY(thread_checker_) = 0; + absl::optional<int64_t> last_unsignalled_ssrc_creation_time_ms_ + RTC_GUARDED_BY(thread_checker_); + std::set<uint32_t> send_ssrcs_ RTC_GUARDED_BY(thread_checker_); + std::set<uint32_t> receive_ssrcs_ RTC_GUARDED_BY(thread_checker_); + + absl::optional<VideoCodecSettings> send_codec_ + RTC_GUARDED_BY(thread_checker_); + std::vector<VideoCodecSettings> negotiated_codecs_ + RTC_GUARDED_BY(thread_checker_); + + std::vector<webrtc::RtpExtension> send_rtp_extensions_ + RTC_GUARDED_BY(thread_checker_); + + webrtc::VideoEncoderFactory* const encoder_factory_ + RTC_GUARDED_BY(thread_checker_); + webrtc::VideoDecoderFactory* const decoder_factory_ + RTC_GUARDED_BY(thread_checker_); + webrtc::VideoBitrateAllocatorFactory* const bitrate_allocator_factory_ + RTC_GUARDED_BY(thread_checker_); + std::vector<VideoCodecSettings> recv_codecs_ RTC_GUARDED_BY(thread_checker_); + webrtc::RtpHeaderExtensionMap recv_rtp_extension_map_ + RTC_GUARDED_BY(thread_checker_); + std::vector<webrtc::RtpExtension> recv_rtp_extensions_ + RTC_GUARDED_BY(thread_checker_); + // See reason for keeping track of the FlexFEC payload type separately in + // comment in WebRtcVideoChannel::ChangedReceiverParameters. + int recv_flexfec_payload_type_ RTC_GUARDED_BY(thread_checker_); + webrtc::BitrateConstraints bitrate_config_ RTC_GUARDED_BY(thread_checker_); + // TODO(deadbeef): Don't duplicate information between + // send_params/recv_params, rtp_extensions, options, etc. + VideoSenderParameters send_params_ RTC_GUARDED_BY(thread_checker_); + VideoOptions default_send_options_ RTC_GUARDED_BY(thread_checker_); + VideoReceiverParameters recv_params_ RTC_GUARDED_BY(thread_checker_); + int64_t last_send_stats_log_ms_ RTC_GUARDED_BY(thread_checker_); + int64_t last_receive_stats_log_ms_ RTC_GUARDED_BY(thread_checker_); + const bool discard_unknown_ssrc_packets_ RTC_GUARDED_BY(thread_checker_); + // This is a stream param that comes from the remote description, but wasn't + // signaled with any a=ssrc lines. It holds information that was signaled + // before the unsignaled receive stream is created when the first packet is + // received. + StreamParams unsignaled_stream_params_ RTC_GUARDED_BY(thread_checker_); + // Per peer connection crypto options that last for the lifetime of the peer + // connection. + const webrtc::CryptoOptions crypto_options_ RTC_GUARDED_BY(thread_checker_); + + // Optional frame transformer set on unsignaled streams. + rtc::scoped_refptr<webrtc::FrameTransformerInterface> + unsignaled_frame_transformer_ RTC_GUARDED_BY(thread_checker_); + + // RTP parameters that need to be set when creating a video receive stream. + // Only used in Receiver mode - in Both mode, it reads those things from the + // codec. + webrtc::VideoReceiveStreamInterface::Config::Rtp rtp_config_; + + // Callback invoked whenever the send codec changes. + // TODO(bugs.webrtc.org/13931): Remove again when coupling isn't needed. + absl::AnyInvocable<void()> send_codec_changed_callback_; + // Callback invoked whenever the list of SSRCs changes. + absl::AnyInvocable<void(const std::set<uint32_t>&)> + ssrc_list_changed_callback_; +}; + +class WebRtcVideoReceiveChannel : public MediaChannelUtil, + public VideoMediaReceiveChannelInterface { + public: + WebRtcVideoReceiveChannel(webrtc::Call* call, + const MediaConfig& config, + const VideoOptions& options, + const webrtc::CryptoOptions& crypto_options, + webrtc::VideoDecoderFactory* decoder_factory); + ~WebRtcVideoReceiveChannel() override; + + public: + MediaType media_type() const override { return MEDIA_TYPE_VIDEO; } + VideoMediaReceiveChannelInterface* AsVideoReceiveChannel() override { + return this; + } + VoiceMediaReceiveChannelInterface* AsVoiceReceiveChannel() override { + RTC_CHECK_NOTREACHED(); + return nullptr; + } + + // Common functions between sender and receiver + void SetInterface(MediaChannelNetworkInterface* iface) override; + // VideoMediaReceiveChannelInterface implementation + bool SetReceiverParameters(const VideoReceiverParameters& params) override; + webrtc::RtpParameters GetRtpReceiverParameters(uint32_t ssrc) const override; + webrtc::RtpParameters GetDefaultRtpReceiveParameters() const override; + void SetReceive(bool receive) override; + bool AddRecvStream(const StreamParams& sp) override; + bool AddDefaultRecvStreamForTesting(const StreamParams& sp) override { + // Invokes private AddRecvStream variant function + return AddRecvStream(sp, true); + } + bool RemoveRecvStream(uint32_t ssrc) override; + void ResetUnsignaledRecvStream() override; + absl::optional<uint32_t> GetUnsignaledSsrc() const override; + void OnDemuxerCriteriaUpdatePending() override; + void OnDemuxerCriteriaUpdateComplete() override; + bool SetSink(uint32_t ssrc, + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override; + void SetDefaultSink( + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override; + bool GetStats(VideoMediaReceiveInfo* info) override; + void OnPacketReceived(const webrtc::RtpPacketReceived& packet) override; + bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) override; + + absl::optional<int> GetBaseMinimumPlayoutDelayMs( + uint32_t ssrc) const override; + + // Choose one of the available SSRCs (or default if none) as the current + // receiver report SSRC. + void ChooseReceiverReportSsrc(const std::set<uint32_t>& choices) override; + + // E2E Encrypted Video Frame API + // Set a frame decryptor to a particular ssrc that will intercept all + // incoming video frames and attempt to decrypt them before forwarding the + // result. + void SetFrameDecryptor(uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameDecryptorInterface> + frame_decryptor) override; + void SetRecordableEncodedFrameCallback( + uint32_t ssrc, + std::function<void(const webrtc::RecordableEncodedFrame&)> callback) + override; + void ClearRecordableEncodedFrameCallback(uint32_t ssrc) override; + void RequestRecvKeyFrame(uint32_t ssrc) override; + void SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) + override; + std::vector<webrtc::RtpSource> GetSources(uint32_t ssrc) const override; + + void SetReceiverFeedbackParameters(bool lntf_enabled, + bool nack_enabled, + webrtc::RtcpMode rtcp_mode, + absl::optional<int> rtx_time) override; + + private: + class WebRtcVideoReceiveStream; + struct ChangedReceiverParameters { + // These optionals are unset if not changed. + absl::optional<std::vector<VideoCodecSettings>> codec_settings; + absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions; + // Keep track of the FlexFEC payload type separately from `codec_settings`. + // This allows us to recreate the FlexfecReceiveStream separately from the + // VideoReceiveStreamInterface when the FlexFEC payload type is changed. + absl::optional<int> flexfec_payload_type; + }; + + // Finds VideoReceiveStreamInterface corresponding to ssrc. Aware of + // unsignalled ssrc handling. + WebRtcVideoReceiveStream* FindReceiveStream(uint32_t ssrc) + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + + void ProcessReceivedPacket(webrtc::RtpPacketReceived packet) + RTC_RUN_ON(thread_checker_); + + // Expected to be invoked once per packet that belongs to this channel that + // can not be demuxed. + // Returns true if a new default stream has been created. + bool MaybeCreateDefaultReceiveStream( + const webrtc::RtpPacketReceived& parsed_packet) + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + void ReCreateDefaultReceiveStream(uint32_t ssrc, + absl::optional<uint32_t> rtx_ssrc); + // Add a receive stream. Used for testing. + bool AddRecvStream(const StreamParams& sp, bool default_stream); + + void ConfigureReceiverRtp( + webrtc::VideoReceiveStreamInterface::Config* config, + webrtc::FlexfecReceiveStream::Config* flexfec_config, + const StreamParams& sp) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + bool ValidateReceiveSsrcAvailability(const StreamParams& sp) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + void DeleteReceiveStream(WebRtcVideoReceiveStream* stream) + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + + // Called when the local ssrc changes. Sets `rtcp_receiver_report_ssrc_` and + // updates the receive streams. + void SetReceiverReportSsrc(uint32_t ssrc) RTC_RUN_ON(&thread_checker_); + + // Wrapper for the receiver part, contains configs etc. that are needed to + // reconstruct the underlying VideoReceiveStreamInterface. + class WebRtcVideoReceiveStream + : public rtc::VideoSinkInterface<webrtc::VideoFrame> { + public: + WebRtcVideoReceiveStream( + webrtc::Call* call, + const StreamParams& sp, + webrtc::VideoReceiveStreamInterface::Config config, + bool default_stream, + const std::vector<VideoCodecSettings>& recv_codecs, + const webrtc::FlexfecReceiveStream::Config& flexfec_config); + ~WebRtcVideoReceiveStream(); + + webrtc::VideoReceiveStreamInterface& stream(); + // Return value may be nullptr. + webrtc::FlexfecReceiveStream* flexfec_stream(); + + const std::vector<uint32_t>& GetSsrcs() const; + + std::vector<webrtc::RtpSource> GetSources(); + + // Does not return codecs, nor header extensions, they are filled by the + // owning WebRtcVideoChannel. + webrtc::RtpParameters GetRtpParameters() const; + + // TODO(deadbeef): Move these feedback parameters into the recv parameters. + void SetFeedbackParameters(bool lntf_enabled, + bool nack_enabled, + webrtc::RtcpMode rtcp_mode, + absl::optional<int> rtx_time); + void SetReceiverParameters(const ChangedReceiverParameters& recv_params); + + void OnFrame(const webrtc::VideoFrame& frame) override; + bool IsDefaultStream() const; + + void SetFrameDecryptor( + rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor); + + bool SetBaseMinimumPlayoutDelayMs(int delay_ms); + + int GetBaseMinimumPlayoutDelayMs() const; + + void SetSink(rtc::VideoSinkInterface<webrtc::VideoFrame>* sink); + + VideoReceiverInfo GetVideoReceiverInfo(bool log_stats); + + void SetRecordableEncodedFrameCallback( + std::function<void(const webrtc::RecordableEncodedFrame&)> callback); + void ClearRecordableEncodedFrameCallback(); + void GenerateKeyFrame(); + + void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr<webrtc::FrameTransformerInterface> + frame_transformer); + + void SetLocalSsrc(uint32_t local_ssrc); + void UpdateRtxSsrc(uint32_t ssrc); + void StartReceiveStream(); + void StopReceiveStream(); + + private: + // Attempts to reconfigure an already existing `flexfec_stream_`, create + // one if the configuration is now complete or remove a flexfec stream + // when disabled. + void SetFlexFecPayload(int payload_type); + + void RecreateReceiveStream(); + void CreateReceiveStream(); + + // Applies a new receive codecs configration to `config_`. Returns true + // if the internal stream needs to be reconstructed, or false if no changes + // were applied. + bool ReconfigureCodecs(const std::vector<VideoCodecSettings>& recv_codecs); + + webrtc::Call* const call_; + const StreamParams stream_params_; + + // Both `stream_` and `flexfec_stream_` are managed by `this`. They are + // destroyed by calling call_->DestroyVideoReceiveStream and + // call_->DestroyFlexfecReceiveStream, respectively. + webrtc::VideoReceiveStreamInterface* stream_; + const bool default_stream_; + webrtc::VideoReceiveStreamInterface::Config config_; + webrtc::FlexfecReceiveStream::Config flexfec_config_; + webrtc::FlexfecReceiveStream* flexfec_stream_; + + webrtc::Mutex sink_lock_; + rtc::VideoSinkInterface<webrtc::VideoFrame>* sink_ + RTC_GUARDED_BY(sink_lock_); + int64_t first_frame_timestamp_ RTC_GUARDED_BY(sink_lock_); + // Start NTP time is estimated as current remote NTP time (estimated from + // RTCP) minus the elapsed time, as soon as remote NTP time is available. + int64_t estimated_remote_start_ntp_time_ms_ RTC_GUARDED_BY(sink_lock_); + + RTC_NO_UNIQUE_ADDRESS webrtc::SequenceChecker thread_checker_; + bool receiving_ RTC_GUARDED_BY(&thread_checker_); + }; + bool GetChangedReceiverParameters(const VideoReceiverParameters& params, + ChangedReceiverParameters* changed_params) + const RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + + std::map<uint32_t, WebRtcVideoReceiveStream*> receive_streams_ + RTC_GUARDED_BY(thread_checker_); + void FillReceiverStats(VideoMediaReceiveInfo* info, bool log_stats) + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + void FillReceiveCodecStats(VideoMediaReceiveInfo* video_media_info) + RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_); + + StreamParams unsignaled_stream_params() { + RTC_DCHECK_RUN_ON(&thread_checker_); + return unsignaled_stream_params_; + } + // Variables. + webrtc::TaskQueueBase* const worker_thread_; + webrtc::ScopedTaskSafety task_safety_; + RTC_NO_UNIQUE_ADDRESS webrtc::SequenceChecker network_thread_checker_{ + webrtc::SequenceChecker::kDetached}; + RTC_NO_UNIQUE_ADDRESS webrtc::SequenceChecker thread_checker_; + + uint32_t rtcp_receiver_report_ssrc_ RTC_GUARDED_BY(thread_checker_); + bool receiving_ RTC_GUARDED_BY(&thread_checker_); + webrtc::Call* const call_; + + rtc::VideoSinkInterface<webrtc::VideoFrame>* default_sink_ + RTC_GUARDED_BY(thread_checker_); + + // Delay for unsignaled streams, which may be set before the stream exists. + int default_recv_base_minimum_delay_ms_ RTC_GUARDED_BY(thread_checker_) = 0; + + const MediaConfig::Video video_config_ RTC_GUARDED_BY(thread_checker_); + + // When the channel and demuxer get reconfigured, there is a window of time + // where we have to be prepared for packets arriving based on the old demuxer + // criteria because the streams live on the worker thread and the demuxer + // lives on the network thread. Because packets are posted from the network + // thread to the worker thread, they can still be in-flight when streams are + // reconfgured. This can happen when `demuxer_criteria_id_` and + // `demuxer_criteria_completed_id_` don't match. During this time, we do not + // want to create unsignalled receive streams and should instead drop the + // packets. E.g: + // * If RemoveRecvStream(old_ssrc) was recently called, there may be packets + // in-flight for that ssrc. This happens when a receiver becomes inactive. + // * If we go from one to many m= sections, the demuxer may change from + // forwarding all packets to only forwarding the configured ssrcs, so there + // is a risk of receiving ssrcs for other, recently added m= sections. + uint32_t demuxer_criteria_id_ RTC_GUARDED_BY(thread_checker_) = 0; + uint32_t demuxer_criteria_completed_id_ RTC_GUARDED_BY(thread_checker_) = 0; + absl::optional<int64_t> last_unsignalled_ssrc_creation_time_ms_ + RTC_GUARDED_BY(thread_checker_); + std::set<uint32_t> send_ssrcs_ RTC_GUARDED_BY(thread_checker_); + std::set<uint32_t> receive_ssrcs_ RTC_GUARDED_BY(thread_checker_); + + absl::optional<VideoCodecSettings> send_codec_ + RTC_GUARDED_BY(thread_checker_); + std::vector<VideoCodecSettings> negotiated_codecs_ + RTC_GUARDED_BY(thread_checker_); + + std::vector<webrtc::RtpExtension> send_rtp_extensions_ + RTC_GUARDED_BY(thread_checker_); + + webrtc::VideoDecoderFactory* const decoder_factory_ + RTC_GUARDED_BY(thread_checker_); + std::vector<VideoCodecSettings> recv_codecs_ RTC_GUARDED_BY(thread_checker_); + webrtc::RtpHeaderExtensionMap recv_rtp_extension_map_ + RTC_GUARDED_BY(thread_checker_); + std::vector<webrtc::RtpExtension> recv_rtp_extensions_ + RTC_GUARDED_BY(thread_checker_); + // See reason for keeping track of the FlexFEC payload type separately in + // comment in WebRtcVideoChannel::ChangedReceiverParameters. + int recv_flexfec_payload_type_ RTC_GUARDED_BY(thread_checker_); + webrtc::BitrateConstraints bitrate_config_ RTC_GUARDED_BY(thread_checker_); + // TODO(deadbeef): Don't duplicate information between + // send_params/recv_params, rtp_extensions, options, etc. + VideoSenderParameters send_params_ RTC_GUARDED_BY(thread_checker_); + VideoOptions default_send_options_ RTC_GUARDED_BY(thread_checker_); + VideoReceiverParameters recv_params_ RTC_GUARDED_BY(thread_checker_); + int64_t last_receive_stats_log_ms_ RTC_GUARDED_BY(thread_checker_); + const bool discard_unknown_ssrc_packets_ RTC_GUARDED_BY(thread_checker_); + // This is a stream param that comes from the remote description, but wasn't + // signaled with any a=ssrc lines. It holds information that was signaled + // before the unsignaled receive stream is created when the first packet is + // received. + StreamParams unsignaled_stream_params_ RTC_GUARDED_BY(thread_checker_); + // Per peer connection crypto options that last for the lifetime of the peer + // connection. + const webrtc::CryptoOptions crypto_options_ RTC_GUARDED_BY(thread_checker_); + + // Optional frame transformer set on unsignaled streams. + rtc::scoped_refptr<webrtc::FrameTransformerInterface> + unsignaled_frame_transformer_ RTC_GUARDED_BY(thread_checker_); + + // RTP parameters that need to be set when creating a video receive stream. + // Only used in Receiver mode - in Both mode, it reads those things from the + // codec. + webrtc::VideoReceiveStreamInterface::Config::Rtp rtp_config_; + + // Callback invoked whenever the send codec changes. + // TODO(bugs.webrtc.org/13931): Remove again when coupling isn't needed. + absl::AnyInvocable<void()> send_codec_changed_callback_; + // Callback invoked whenever the list of SSRCs changes. + absl::AnyInvocable<void(const std::set<uint32_t>&)> + ssrc_list_changed_callback_; + + const int receive_buffer_size_; +}; + +// Keeping the old name "WebRtcVideoChannel" around because some external +// customers are using cricket::WebRtcVideoChannel::AdaptReason +// TODO(bugs.webrtc.org/15216): Move this enum to an interface class and +// delete this workaround. +class WebRtcVideoChannel : public WebRtcVideoSendChannel { + public: + // Make all the values of AdaptReason available as + // WebRtcVideoChannel::ADAPT_xxx. + using WebRtcVideoSendChannel::AdaptReason; +}; + +} // namespace cricket + +#endif // MEDIA_ENGINE_WEBRTC_VIDEO_ENGINE_H_ |