/* * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "video/video_stream_encoder.h" #include #include #include #include #include "api/video/i420_buffer.h" #include "common_video/include/video_bitrate_allocator.h" #include "common_video/include/video_frame.h" #include "modules/pacing/paced_sender.h" #include "modules/video_coding/codecs/vp8/temporal_layers.h" #include "modules/video_coding/include/video_codec_initializer.h" #include "modules/video_coding/include/video_coding.h" #include "modules/video_coding/include/video_coding_defines.h" #include "rtc_base/arraysize.h" #include "rtc_base/checks.h" #include "rtc_base/location.h" #include "rtc_base/logging.h" #include "rtc_base/timeutils.h" #include "rtc_base/trace_event.h" #include "video/overuse_frame_detector.h" #include "video/send_statistics_proxy.h" namespace webrtc { namespace { // Time interval for logging frame counts. const int64_t kFrameLogIntervalMs = 60000; const int kMinFramerateFps = 2; const int kMaxFramerateFps = 120; // The maximum number of frames to drop at beginning of stream // to try and achieve desired bitrate. const int kMaxInitialFramedrop = 4; uint32_t MaximumFrameSizeForBitrate(uint32_t kbps) { if (kbps > 0) { if (kbps < 300 /* qvga */) { return 320 * 240; } else if (kbps < 500 /* vga */) { return 640 * 480; } } return std::numeric_limits::max(); } // Initial limits for kBalanced degradation preference. int MinFps(int pixels) { if (pixels <= 320 * 240) { return 7; } else if (pixels <= 480 * 270) { return 10; } else if (pixels <= 640 * 480) { return 15; } else { return std::numeric_limits::max(); } } int MaxFps(int pixels) { if (pixels <= 320 * 240) { return 10; } else if (pixels <= 480 * 270) { return 15; } else { return std::numeric_limits::max(); } } bool IsResolutionScalingEnabled( VideoSendStream::DegradationPreference degradation_preference) { return degradation_preference == VideoSendStream::DegradationPreference::kMaintainFramerate || degradation_preference == VideoSendStream::DegradationPreference::kBalanced; } bool IsFramerateScalingEnabled( VideoSendStream::DegradationPreference degradation_preference) { return degradation_preference == VideoSendStream::DegradationPreference::kMaintainResolution || degradation_preference == VideoSendStream::DegradationPreference::kBalanced; } } // namespace class VideoStreamEncoder::ConfigureEncoderTask : public rtc::QueuedTask { public: ConfigureEncoderTask(VideoStreamEncoder* video_stream_encoder, VideoEncoderConfig config, size_t max_data_payload_length, bool nack_enabled) : video_stream_encoder_(video_stream_encoder), config_(std::move(config)), max_data_payload_length_(max_data_payload_length), nack_enabled_(nack_enabled) {} private: bool Run() override { video_stream_encoder_->ConfigureEncoderOnTaskQueue( std::move(config_), max_data_payload_length_, nack_enabled_); return true; } VideoStreamEncoder* const video_stream_encoder_; VideoEncoderConfig config_; size_t max_data_payload_length_; bool nack_enabled_; }; class VideoStreamEncoder::EncodeTask : public rtc::QueuedTask { public: EncodeTask(const VideoFrame& frame, VideoStreamEncoder* video_stream_encoder, int64_t time_when_posted_us, bool log_stats) : frame_(frame), video_stream_encoder_(video_stream_encoder), time_when_posted_us_(time_when_posted_us), log_stats_(log_stats) { ++video_stream_encoder_->posted_frames_waiting_for_encode_; } private: bool Run() override { RTC_DCHECK_RUN_ON(&video_stream_encoder_->encoder_queue_); video_stream_encoder_->stats_proxy_->OnIncomingFrame(frame_.width(), frame_.height()); ++video_stream_encoder_->captured_frame_count_; const int posted_frames_waiting_for_encode = video_stream_encoder_->posted_frames_waiting_for_encode_.fetch_sub(1); RTC_DCHECK_GT(posted_frames_waiting_for_encode, 0); if (posted_frames_waiting_for_encode == 1) { video_stream_encoder_->EncodeVideoFrame(frame_, time_when_posted_us_); } else { // There is a newer frame in flight. Do not encode this frame. RTC_LOG(LS_VERBOSE) << "Incoming frame dropped due to that the encoder is blocked."; ++video_stream_encoder_->dropped_frame_count_; video_stream_encoder_->stats_proxy_->OnFrameDroppedInEncoderQueue(); } if (log_stats_) { RTC_LOG(LS_INFO) << "Number of frames: captured " << video_stream_encoder_->captured_frame_count_ << ", dropped (due to encoder blocked) " << video_stream_encoder_->dropped_frame_count_ << ", interval_ms " << kFrameLogIntervalMs; video_stream_encoder_->captured_frame_count_ = 0; video_stream_encoder_->dropped_frame_count_ = 0; } return true; } VideoFrame frame_; VideoStreamEncoder* const video_stream_encoder_; const int64_t time_when_posted_us_; const bool log_stats_; }; // VideoSourceProxy is responsible ensuring thread safety between calls to // VideoStreamEncoder::SetSource that will happen on libjingle's worker thread // when a video capturer is connected to the encoder and the encoder task queue // (encoder_queue_) where the encoder reports its VideoSinkWants. class VideoStreamEncoder::VideoSourceProxy { public: explicit VideoSourceProxy(VideoStreamEncoder* video_stream_encoder) : video_stream_encoder_(video_stream_encoder), degradation_preference_( VideoSendStream::DegradationPreference::kDegradationDisabled), source_(nullptr) {} void SetSource( rtc::VideoSourceInterface* source, const VideoSendStream::DegradationPreference& degradation_preference) { // Called on libjingle's worker thread. RTC_DCHECK_CALLED_SEQUENTIALLY(&main_checker_); rtc::VideoSourceInterface* old_source = nullptr; rtc::VideoSinkWants wants; { rtc::CritScope lock(&crit_); degradation_preference_ = degradation_preference; old_source = source_; source_ = source; wants = GetActiveSinkWantsInternal(); } if (old_source != source && old_source != nullptr) { old_source->RemoveSink(video_stream_encoder_); } if (!source) { return; } source->AddOrUpdateSink(video_stream_encoder_, wants); } void SetWantsRotationApplied(bool rotation_applied) { rtc::CritScope lock(&crit_); sink_wants_.rotation_applied = rotation_applied; if (source_) source_->AddOrUpdateSink(video_stream_encoder_, sink_wants_); } rtc::VideoSinkWants GetActiveSinkWants() { rtc::CritScope lock(&crit_); return GetActiveSinkWantsInternal(); } void ResetPixelFpsCount() { rtc::CritScope lock(&crit_); sink_wants_.max_pixel_count = std::numeric_limits::max(); sink_wants_.target_pixel_count.reset(); sink_wants_.max_framerate_fps = std::numeric_limits::max(); if (source_) source_->AddOrUpdateSink(video_stream_encoder_, sink_wants_); } bool RequestResolutionLowerThan(int pixel_count, int min_pixels_per_frame, bool* min_pixels_reached) { // Called on the encoder task queue. rtc::CritScope lock(&crit_); if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { // This can happen since |degradation_preference_| is set on libjingle's // worker thread but the adaptation is done on the encoder task queue. return false; } // The input video frame size will have a resolution less than or equal to // |max_pixel_count| depending on how the source can scale the frame size. const int pixels_wanted = (pixel_count * 3) / 5; if (pixels_wanted >= sink_wants_.max_pixel_count) { return false; } if (pixels_wanted < min_pixels_per_frame) { *min_pixels_reached = true; return false; } RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: " << pixels_wanted; sink_wants_.max_pixel_count = pixels_wanted; sink_wants_.target_pixel_count = rtc::Optional(); source_->AddOrUpdateSink(video_stream_encoder_, GetActiveSinkWantsInternal()); return true; } int RequestFramerateLowerThan(int fps) { // Called on the encoder task queue. // The input video frame rate will be scaled down to 2/3, rounding down. int framerate_wanted = (fps * 2) / 3; return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1; } bool RequestHigherResolutionThan(int pixel_count) { // Called on the encoder task queue. rtc::CritScope lock(&crit_); if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { // This can happen since |degradation_preference_| is set on libjingle's // worker thread but the adaptation is done on the encoder task queue. return false; } int max_pixels_wanted = pixel_count; if (max_pixels_wanted != std::numeric_limits::max()) max_pixels_wanted = pixel_count * 4; if (max_pixels_wanted <= sink_wants_.max_pixel_count) return false; sink_wants_.max_pixel_count = max_pixels_wanted; if (max_pixels_wanted == std::numeric_limits::max()) { // Remove any constraints. sink_wants_.target_pixel_count.reset(); } else { // On step down we request at most 3/5 the pixel count of the previous // resolution, so in order to take "one step up" we request a resolution // as close as possible to 5/3 of the current resolution. The actual pixel // count selected depends on the capabilities of the source. In order to // not take a too large step up, we cap the requested pixel count to be at // most four time the current number of pixels. sink_wants_.target_pixel_count = rtc::Optional((pixel_count * 5) / 3); } RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: " << max_pixels_wanted; source_->AddOrUpdateSink(video_stream_encoder_, GetActiveSinkWantsInternal()); return true; } // Request upgrade in framerate. Returns the new requested frame, or -1 if // no change requested. Note that maxint may be returned if limits due to // adaptation requests are removed completely. In that case, consider // |max_framerate_| to be the current limit (assuming the capturer complies). int RequestHigherFramerateThan(int fps) { // Called on the encoder task queue. // The input frame rate will be scaled up to the last step, with rounding. int framerate_wanted = fps; if (fps != std::numeric_limits::max()) framerate_wanted = (fps * 3) / 2; return IncreaseFramerate(framerate_wanted) ? framerate_wanted : -1; } bool RestrictFramerate(int fps) { // Called on the encoder task queue. rtc::CritScope lock(&crit_); if (!source_ || !IsFramerateScalingEnabled(degradation_preference_)) return false; const int fps_wanted = std::max(kMinFramerateFps, fps); if (fps_wanted >= sink_wants_.max_framerate_fps) return false; RTC_LOG(LS_INFO) << "Scaling down framerate: " << fps_wanted; sink_wants_.max_framerate_fps = fps_wanted; source_->AddOrUpdateSink(video_stream_encoder_, GetActiveSinkWantsInternal()); return true; } bool IncreaseFramerate(int fps) { // Called on the encoder task queue. rtc::CritScope lock(&crit_); if (!source_ || !IsFramerateScalingEnabled(degradation_preference_)) return false; const int fps_wanted = std::max(kMinFramerateFps, fps); if (fps_wanted <= sink_wants_.max_framerate_fps) return false; RTC_LOG(LS_INFO) << "Scaling up framerate: " << fps_wanted; sink_wants_.max_framerate_fps = fps_wanted; source_->AddOrUpdateSink(video_stream_encoder_, GetActiveSinkWantsInternal()); return true; } private: rtc::VideoSinkWants GetActiveSinkWantsInternal() RTC_EXCLUSIVE_LOCKS_REQUIRED(&crit_) { rtc::VideoSinkWants wants = sink_wants_; // Clear any constraints from the current sink wants that don't apply to // the used degradation_preference. switch (degradation_preference_) { case VideoSendStream::DegradationPreference::kBalanced: break; case VideoSendStream::DegradationPreference::kMaintainFramerate: wants.max_framerate_fps = std::numeric_limits::max(); break; case VideoSendStream::DegradationPreference::kMaintainResolution: wants.max_pixel_count = std::numeric_limits::max(); wants.target_pixel_count.reset(); break; case VideoSendStream::DegradationPreference::kDegradationDisabled: wants.max_pixel_count = std::numeric_limits::max(); wants.target_pixel_count.reset(); wants.max_framerate_fps = std::numeric_limits::max(); } return wants; } rtc::CriticalSection crit_; rtc::SequencedTaskChecker main_checker_; VideoStreamEncoder* const video_stream_encoder_; rtc::VideoSinkWants sink_wants_ RTC_GUARDED_BY(&crit_); VideoSendStream::DegradationPreference degradation_preference_ RTC_GUARDED_BY(&crit_); rtc::VideoSourceInterface* source_ RTC_GUARDED_BY(&crit_); RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy); }; VideoStreamEncoder::VideoStreamEncoder( uint32_t number_of_cores, SendStatisticsProxy* stats_proxy, const VideoSendStream::Config::EncoderSettings& settings, rtc::VideoSinkInterface* pre_encode_callback, EncodedFrameObserver* encoder_timing, std::unique_ptr overuse_detector) : shutdown_event_(true /* manual_reset */, false), number_of_cores_(number_of_cores), initial_rampup_(0), source_proxy_(new VideoSourceProxy(this)), sink_(nullptr), settings_(settings), codec_type_(PayloadStringToCodecType(settings.payload_name)), video_sender_(Clock::GetRealTimeClock(), this), overuse_detector_( overuse_detector.get() ? overuse_detector.release() : new OveruseFrameDetector( GetCpuOveruseOptions(settings.full_overuse_time), this, encoder_timing, stats_proxy)), stats_proxy_(stats_proxy), pre_encode_callback_(pre_encode_callback), max_framerate_(-1), pending_encoder_reconfiguration_(false), encoder_start_bitrate_bps_(0), max_data_payload_length_(0), nack_enabled_(false), last_observed_bitrate_bps_(0), encoder_paused_and_dropped_frame_(false), clock_(Clock::GetRealTimeClock()), degradation_preference_( VideoSendStream::DegradationPreference::kDegradationDisabled), posted_frames_waiting_for_encode_(0), last_captured_timestamp_(0), delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - clock_->TimeInMilliseconds()), last_frame_log_ms_(clock_->TimeInMilliseconds()), captured_frame_count_(0), dropped_frame_count_(0), bitrate_observer_(nullptr), encoder_queue_("EncoderQueue") { RTC_DCHECK(stats_proxy); encoder_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&encoder_queue_); overuse_detector_->StartCheckForOveruse(); video_sender_.RegisterExternalEncoder( settings_.encoder, settings_.payload_type, settings_.internal_source); }); } VideoStreamEncoder::~VideoStreamEncoder() { RTC_DCHECK_RUN_ON(&thread_checker_); RTC_DCHECK(shutdown_event_.Wait(0)) << "Must call ::Stop() before destruction."; } // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle // pipelining encoders better (multiple input frames before something comes // out). This should effectively turn off CPU adaptations for systems that // remotely cope with the load right now. CpuOveruseOptions VideoStreamEncoder::GetCpuOveruseOptions( bool full_overuse_time) { CpuOveruseOptions options; if (full_overuse_time) { options.low_encode_usage_threshold_percent = 150; options.high_encode_usage_threshold_percent = 200; } return options; } void VideoStreamEncoder::Stop() { RTC_DCHECK_RUN_ON(&thread_checker_); source_proxy_->SetSource(nullptr, VideoSendStream::DegradationPreference()); encoder_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&encoder_queue_); overuse_detector_->StopCheckForOveruse(); rate_allocator_.reset(); bitrate_observer_ = nullptr; video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type, false); quality_scaler_ = nullptr; shutdown_event_.Set(); }); shutdown_event_.Wait(rtc::Event::kForever); } void VideoStreamEncoder::SetBitrateObserver( VideoBitrateAllocationObserver* bitrate_observer) { RTC_DCHECK_RUN_ON(&thread_checker_); encoder_queue_.PostTask([this, bitrate_observer] { RTC_DCHECK_RUN_ON(&encoder_queue_); RTC_DCHECK(!bitrate_observer_); bitrate_observer_ = bitrate_observer; }); } void VideoStreamEncoder::SetSource( rtc::VideoSourceInterface* source, const VideoSendStream::DegradationPreference& degradation_preference) { RTC_DCHECK_RUN_ON(&thread_checker_); source_proxy_->SetSource(source, degradation_preference); encoder_queue_.PostTask([this, degradation_preference] { RTC_DCHECK_RUN_ON(&encoder_queue_); if (degradation_preference_ != degradation_preference) { // Reset adaptation state, so that we're not tricked into thinking there's // an already pending request of the same type. last_adaptation_request_.reset(); if (degradation_preference == VideoSendStream::DegradationPreference::kBalanced || degradation_preference_ == VideoSendStream::DegradationPreference::kBalanced) { // TODO(asapersson): Consider removing |adapt_counters_| map and use one // AdaptCounter for all modes. source_proxy_->ResetPixelFpsCount(); adapt_counters_.clear(); } } degradation_preference_ = degradation_preference; bool allow_scaling = IsResolutionScalingEnabled(degradation_preference_); initial_rampup_ = allow_scaling ? 0 : kMaxInitialFramedrop; ConfigureQualityScaler(); if (!IsFramerateScalingEnabled(degradation_preference) && max_framerate_ != -1) { // If frame rate scaling is no longer allowed, remove any potential // allowance for longer frame intervals. overuse_detector_->OnTargetFramerateUpdated(max_framerate_); } }); } void VideoStreamEncoder::SetSink(EncoderSink* sink, bool rotation_applied) { source_proxy_->SetWantsRotationApplied(rotation_applied); encoder_queue_.PostTask([this, sink] { RTC_DCHECK_RUN_ON(&encoder_queue_); sink_ = sink; }); } void VideoStreamEncoder::SetStartBitrate(int start_bitrate_bps) { encoder_queue_.PostTask([this, start_bitrate_bps] { RTC_DCHECK_RUN_ON(&encoder_queue_); encoder_start_bitrate_bps_ = start_bitrate_bps; }); } void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config, size_t max_data_payload_length, bool nack_enabled) { encoder_queue_.PostTask( std::unique_ptr(new ConfigureEncoderTask( this, std::move(config), max_data_payload_length, nack_enabled))); } void VideoStreamEncoder::ConfigureEncoderOnTaskQueue( VideoEncoderConfig config, size_t max_data_payload_length, bool nack_enabled) { RTC_DCHECK_RUN_ON(&encoder_queue_); RTC_DCHECK(sink_); RTC_LOG(LS_INFO) << "ConfigureEncoder requested."; max_data_payload_length_ = max_data_payload_length; nack_enabled_ = nack_enabled; encoder_config_ = std::move(config); pending_encoder_reconfiguration_ = true; // Reconfigure the encoder now if the encoder has an internal source or // if the frame resolution is known. Otherwise, the reconfiguration is // deferred until the next frame to minimize the number of reconfigurations. // The codec configuration depends on incoming video frame size. if (last_frame_info_) { ReconfigureEncoder(); } else if (settings_.internal_source) { last_frame_info_ = rtc::Optional(VideoFrameInfo(176, 144, false)); ReconfigureEncoder(); } } void VideoStreamEncoder::ReconfigureEncoder() { RTC_DCHECK_RUN_ON(&encoder_queue_); RTC_DCHECK(pending_encoder_reconfiguration_); std::vector streams = encoder_config_.video_stream_factory->CreateEncoderStreams( last_frame_info_->width, last_frame_info_->height, encoder_config_); // TODO(ilnik): If configured resolution is significantly less than provided, // e.g. because there are not enough SSRCs for all simulcast streams, // signal new resolutions via SinkWants to video source. // Stream dimensions may be not equal to given because of a simulcast // restrictions. auto highest_stream = std::max_element( streams.begin(), streams.end(), [](const webrtc::VideoStream& a, const webrtc::VideoStream& b) { return std::tie(a.width, a.height) < std::tie(b.width, b.height); }); int highest_stream_width = static_cast(highest_stream->width); int highest_stream_height = static_cast(highest_stream->height); // Dimension may be reduced to be, e.g. divisible by 4. RTC_CHECK_GE(last_frame_info_->width, highest_stream_width); RTC_CHECK_GE(last_frame_info_->height, highest_stream_height); crop_width_ = last_frame_info_->width - highest_stream_width; crop_height_ = last_frame_info_->height - highest_stream_height; VideoCodec codec; if (!VideoCodecInitializer::SetupCodec(encoder_config_, settings_, streams, nack_enabled_, &codec, &rate_allocator_)) { RTC_LOG(LS_ERROR) << "Failed to create encoder configuration."; } codec.startBitrate = std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate); codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate); codec.expect_encode_from_texture = last_frame_info_->is_texture; max_framerate_ = codec.maxFramerate; RTC_DCHECK_LE(max_framerate_, kMaxFramerateFps); bool success = video_sender_.RegisterSendCodec( &codec, number_of_cores_, static_cast(max_data_payload_length_)) == VCM_OK; if (!success) { RTC_LOG(LS_ERROR) << "Failed to configure encoder."; rate_allocator_.reset(); } video_sender_.UpdateChannelParemeters(rate_allocator_.get(), bitrate_observer_); // Get the current actual framerate, as measured by the stats proxy. This is // used to get the correct bitrate layer allocation. int current_framerate = stats_proxy_->GetSendFrameRate(); if (current_framerate == 0) current_framerate = codec.maxFramerate; stats_proxy_->OnEncoderReconfigured( encoder_config_, rate_allocator_.get() ? rate_allocator_->GetPreferredBitrateBps(current_framerate) : codec.maxBitrate); pending_encoder_reconfiguration_ = false; sink_->OnEncoderConfigurationChanged( std::move(streams), encoder_config_.min_transmit_bitrate_bps); // Get the current target framerate, ie the maximum framerate as specified by // the current codec configuration, or any limit imposed by cpu adaption in // maintain-resolution or balanced mode. This is used to make sure overuse // detection doesn't needlessly trigger in low and/or variable framerate // scenarios. int target_framerate = std::min( max_framerate_, source_proxy_->GetActiveSinkWants().max_framerate_fps); overuse_detector_->OnTargetFramerateUpdated(target_framerate); ConfigureQualityScaler(); } void VideoStreamEncoder::ConfigureQualityScaler() { RTC_DCHECK_RUN_ON(&encoder_queue_); const auto scaling_settings = settings_.encoder->GetScalingSettings(); const bool quality_scaling_allowed = IsResolutionScalingEnabled(degradation_preference_) && scaling_settings.enabled; if (quality_scaling_allowed) { if (quality_scaler_.get() == nullptr) { // Quality scaler has not already been configured. // Drop frames and scale down until desired quality is achieved. if (scaling_settings.thresholds) { quality_scaler_.reset( new QualityScaler(this, *(scaling_settings.thresholds))); } else { quality_scaler_.reset(new QualityScaler(this, codec_type_)); } } } else { quality_scaler_.reset(nullptr); initial_rampup_ = kMaxInitialFramedrop; } stats_proxy_->SetAdaptationStats(GetActiveCounts(kCpu), GetActiveCounts(kQuality)); } void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) { RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); VideoFrame incoming_frame = video_frame; // Local time in webrtc time base. int64_t current_time_us = clock_->TimeInMicroseconds(); int64_t current_time_ms = current_time_us / rtc::kNumMicrosecsPerMillisec; // In some cases, e.g., when the frame from decoder is fed to encoder, // the timestamp may be set to the future. As the encoding pipeline assumes // capture time to be less than present time, we should reset the capture // timestamps here. Otherwise there may be issues with RTP send stream. if (incoming_frame.timestamp_us() > current_time_us) incoming_frame.set_timestamp_us(current_time_us); // Capture time may come from clock with an offset and drift from clock_. int64_t capture_ntp_time_ms; if (video_frame.ntp_time_ms() > 0) { capture_ntp_time_ms = video_frame.ntp_time_ms(); } else if (video_frame.render_time_ms() != 0) { capture_ntp_time_ms = video_frame.render_time_ms() + delta_ntp_internal_ms_; } else { capture_ntp_time_ms = current_time_ms + delta_ntp_internal_ms_; } incoming_frame.set_ntp_time_ms(capture_ntp_time_ms); // Convert NTP time, in ms, to RTP timestamp. const int kMsToRtpTimestamp = 90; incoming_frame.set_timestamp( kMsToRtpTimestamp * static_cast(incoming_frame.ntp_time_ms())); if (incoming_frame.ntp_time_ms() <= last_captured_timestamp_) { // We don't allow the same capture time for two frames, drop this one. RTC_LOG(LS_WARNING) << "Same/old NTP timestamp (" << incoming_frame.ntp_time_ms() << " <= " << last_captured_timestamp_ << ") for incoming frame. Dropping."; return; } bool log_stats = false; if (current_time_ms - last_frame_log_ms_ > kFrameLogIntervalMs) { last_frame_log_ms_ = current_time_ms; log_stats = true; } last_captured_timestamp_ = incoming_frame.ntp_time_ms(); encoder_queue_.PostTask(std::unique_ptr(new EncodeTask( incoming_frame, this, rtc::TimeMicros(), log_stats))); } void VideoStreamEncoder::OnDiscardedFrame() { stats_proxy_->OnFrameDroppedBySource(); } bool VideoStreamEncoder::EncoderPaused() const { RTC_DCHECK_RUN_ON(&encoder_queue_); // Pause video if paused by caller or as long as the network is down or the // pacer queue has grown too large in buffered mode. // If the pacer queue has grown too large or the network is down, // last_observed_bitrate_bps_ will be 0. return last_observed_bitrate_bps_ == 0; } void VideoStreamEncoder::TraceFrameDropStart() { RTC_DCHECK_RUN_ON(&encoder_queue_); // Start trace event only on the first frame after encoder is paused. if (!encoder_paused_and_dropped_frame_) { TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this); } encoder_paused_and_dropped_frame_ = true; } void VideoStreamEncoder::TraceFrameDropEnd() { RTC_DCHECK_RUN_ON(&encoder_queue_); // End trace event on first frame after encoder resumes, if frame was dropped. if (encoder_paused_and_dropped_frame_) { TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this); } encoder_paused_and_dropped_frame_ = false; } void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame, int64_t time_when_posted_us) { RTC_DCHECK_RUN_ON(&encoder_queue_); if (pre_encode_callback_) pre_encode_callback_->OnFrame(video_frame); if (!last_frame_info_ || video_frame.width() != last_frame_info_->width || video_frame.height() != last_frame_info_->height || video_frame.is_texture() != last_frame_info_->is_texture) { pending_encoder_reconfiguration_ = true; last_frame_info_ = rtc::Optional(VideoFrameInfo( video_frame.width(), video_frame.height(), video_frame.is_texture())); RTC_LOG(LS_INFO) << "Video frame parameters changed: dimensions=" << last_frame_info_->width << "x" << last_frame_info_->height << ", texture=" << last_frame_info_->is_texture << "."; } if (initial_rampup_ < kMaxInitialFramedrop && video_frame.size() > MaximumFrameSizeForBitrate(encoder_start_bitrate_bps_ / 1000)) { RTC_LOG(LS_INFO) << "Dropping frame. Too large for target bitrate."; AdaptDown(kQuality); ++initial_rampup_; return; } initial_rampup_ = kMaxInitialFramedrop; int64_t now_ms = clock_->TimeInMilliseconds(); if (pending_encoder_reconfiguration_) { ReconfigureEncoder(); last_parameters_update_ms_.emplace(now_ms); } else if (!last_parameters_update_ms_ || now_ms - *last_parameters_update_ms_ >= vcm::VCMProcessTimer::kDefaultProcessIntervalMs) { video_sender_.UpdateChannelParemeters(rate_allocator_.get(), bitrate_observer_); last_parameters_update_ms_.emplace(now_ms); } if (EncoderPaused()) { TraceFrameDropStart(); return; } TraceFrameDropEnd(); VideoFrame out_frame(video_frame); // Crop frame if needed. if (crop_width_ > 0 || crop_height_ > 0) { int cropped_width = video_frame.width() - crop_width_; int cropped_height = video_frame.height() - crop_height_; rtc::scoped_refptr cropped_buffer = I420Buffer::Create(cropped_width, cropped_height); // TODO(ilnik): Remove scaling if cropping is too big, as it should never // happen after SinkWants signaled correctly from ReconfigureEncoder. if (crop_width_ < 4 && crop_height_ < 4) { cropped_buffer->CropAndScaleFrom( *video_frame.video_frame_buffer()->ToI420(), crop_width_ / 2, crop_height_ / 2, cropped_width, cropped_height); } else { cropped_buffer->ScaleFrom( *video_frame.video_frame_buffer()->ToI420().get()); } out_frame = VideoFrame(cropped_buffer, video_frame.timestamp(), video_frame.render_time_ms(), video_frame.rotation()); out_frame.set_ntp_time_ms(video_frame.ntp_time_ms()); } TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), "Encode"); overuse_detector_->FrameCaptured(out_frame, time_when_posted_us); video_sender_.AddVideoFrame(out_frame, nullptr); } void VideoStreamEncoder::SendKeyFrame() { if (!encoder_queue_.IsCurrent()) { encoder_queue_.PostTask([this] { SendKeyFrame(); }); return; } RTC_DCHECK_RUN_ON(&encoder_queue_); video_sender_.IntraFrameRequest(0); } EncodedImageCallback::Result VideoStreamEncoder::OnEncodedImage( const EncodedImage& encoded_image, const CodecSpecificInfo* codec_specific_info, const RTPFragmentationHeader* fragmentation) { // Encoded is called on whatever thread the real encoder implementation run // on. In the case of hardware encoders, there might be several encoders // running in parallel on different threads. stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info); EncodedImageCallback::Result result = sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation); int64_t time_sent_us = rtc::TimeMicros(); uint32_t timestamp = encoded_image._timeStamp; const int qp = encoded_image.qp_; encoder_queue_.PostTask([this, timestamp, time_sent_us, qp] { RTC_DCHECK_RUN_ON(&encoder_queue_); overuse_detector_->FrameSent(timestamp, time_sent_us); if (quality_scaler_ && qp >= 0) quality_scaler_->ReportQP(qp); }); return result; } void VideoStreamEncoder::OnDroppedFrame(DropReason reason) { switch (reason) { case DropReason::kDroppedByMediaOptimizations: stats_proxy_->OnFrameDroppedByMediaOptimizations(); encoder_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&encoder_queue_); if (quality_scaler_) quality_scaler_->ReportDroppedFrame(); }); break; case DropReason::kDroppedByEncoder: stats_proxy_->OnFrameDroppedByEncoder(); break; } } void VideoStreamEncoder::OnReceivedIntraFrameRequest(size_t stream_index) { if (!encoder_queue_.IsCurrent()) { encoder_queue_.PostTask( [this, stream_index] { OnReceivedIntraFrameRequest(stream_index); }); return; } RTC_DCHECK_RUN_ON(&encoder_queue_); // Key frame request from remote side, signal to VCM. TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); video_sender_.IntraFrameRequest(stream_index); } void VideoStreamEncoder::OnBitrateUpdated(uint32_t bitrate_bps, uint8_t fraction_lost, int64_t round_trip_time_ms) { if (!encoder_queue_.IsCurrent()) { encoder_queue_.PostTask( [this, bitrate_bps, fraction_lost, round_trip_time_ms] { OnBitrateUpdated(bitrate_bps, fraction_lost, round_trip_time_ms); }); return; } RTC_DCHECK_RUN_ON(&encoder_queue_); RTC_DCHECK(sink_) << "sink_ must be set before the encoder is active."; RTC_LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << bitrate_bps << " packet loss " << static_cast(fraction_lost) << " rtt " << round_trip_time_ms; video_sender_.SetChannelParameters(bitrate_bps, fraction_lost, round_trip_time_ms, rate_allocator_.get(), bitrate_observer_); encoder_start_bitrate_bps_ = bitrate_bps != 0 ? bitrate_bps : encoder_start_bitrate_bps_; bool video_is_suspended = bitrate_bps == 0; bool video_suspension_changed = video_is_suspended != EncoderPaused(); last_observed_bitrate_bps_ = bitrate_bps; if (video_suspension_changed) { RTC_LOG(LS_INFO) << "Video suspend state changed to: " << (video_is_suspended ? "suspended" : "not suspended"); stats_proxy_->OnSuspendChange(video_is_suspended); } } void VideoStreamEncoder::AdaptDown(AdaptReason reason) { RTC_DCHECK_RUN_ON(&encoder_queue_); AdaptationRequest adaptation_request = { last_frame_info_->pixel_count(), stats_proxy_->GetStats().input_frame_rate, AdaptationRequest::Mode::kAdaptDown}; bool downgrade_requested = last_adaptation_request_ && last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown; switch (degradation_preference_) { case VideoSendStream::DegradationPreference::kBalanced: break; case VideoSendStream::DegradationPreference::kMaintainFramerate: if (downgrade_requested && adaptation_request.input_pixel_count_ >= last_adaptation_request_->input_pixel_count_) { // Don't request lower resolution if the current resolution is not // lower than the last time we asked for the resolution to be lowered. return; } break; case VideoSendStream::DegradationPreference::kMaintainResolution: if (adaptation_request.framerate_fps_ <= 0 || (downgrade_requested && adaptation_request.framerate_fps_ < kMinFramerateFps)) { // If no input fps estimate available, can't determine how to scale down // framerate. Otherwise, don't request lower framerate if we don't have // a valid frame rate. Since framerate, unlike resolution, is a measure // we have to estimate, and can fluctuate naturally over time, don't // make the same kind of limitations as for resolution, but trust the // overuse detector to not trigger too often. return; } break; case VideoSendStream::DegradationPreference::kDegradationDisabled: return; } if (reason == kCpu) { if (GetConstAdaptCounter().ResolutionCount(kCpu) >= kMaxCpuResolutionDowngrades || GetConstAdaptCounter().FramerateCount(kCpu) >= kMaxCpuFramerateDowngrades) { return; } } switch (degradation_preference_) { case VideoSendStream::DegradationPreference::kBalanced: { // Try scale down framerate, if lower. int fps = MinFps(last_frame_info_->pixel_count()); if (source_proxy_->RestrictFramerate(fps)) { GetAdaptCounter().IncrementFramerate(reason); break; } // Scale down resolution. FALLTHROUGH(); } case VideoSendStream::DegradationPreference::kMaintainFramerate: { // Scale down resolution. bool min_pixels_reached = false; if (!source_proxy_->RequestResolutionLowerThan( adaptation_request.input_pixel_count_, settings_.encoder->GetScalingSettings().min_pixels_per_frame, &min_pixels_reached)) { if (min_pixels_reached) stats_proxy_->OnMinPixelLimitReached(); return; } GetAdaptCounter().IncrementResolution(reason); break; } case VideoSendStream::DegradationPreference::kMaintainResolution: { // Scale down framerate. const int requested_framerate = source_proxy_->RequestFramerateLowerThan( adaptation_request.framerate_fps_); if (requested_framerate == -1) return; RTC_DCHECK_NE(max_framerate_, -1); overuse_detector_->OnTargetFramerateUpdated( std::min(max_framerate_, requested_framerate)); GetAdaptCounter().IncrementFramerate(reason); break; } case VideoSendStream::DegradationPreference::kDegradationDisabled: RTC_NOTREACHED(); } last_adaptation_request_.emplace(adaptation_request); UpdateAdaptationStats(reason); RTC_LOG(LS_INFO) << GetConstAdaptCounter().ToString(); } void VideoStreamEncoder::AdaptUp(AdaptReason reason) { RTC_DCHECK_RUN_ON(&encoder_queue_); const AdaptCounter& adapt_counter = GetConstAdaptCounter(); int num_downgrades = adapt_counter.TotalCount(reason); if (num_downgrades == 0) return; RTC_DCHECK_GT(num_downgrades, 0); AdaptationRequest adaptation_request = { last_frame_info_->pixel_count(), stats_proxy_->GetStats().input_frame_rate, AdaptationRequest::Mode::kAdaptUp}; bool adapt_up_requested = last_adaptation_request_ && last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp; if (degradation_preference_ == VideoSendStream::DegradationPreference::kMaintainFramerate) { if (adapt_up_requested && adaptation_request.input_pixel_count_ <= last_adaptation_request_->input_pixel_count_) { // Don't request higher resolution if the current resolution is not // higher than the last time we asked for the resolution to be higher. return; } } switch (degradation_preference_) { case VideoSendStream::DegradationPreference::kBalanced: { // Try scale up framerate, if higher. int fps = MaxFps(last_frame_info_->pixel_count()); if (source_proxy_->IncreaseFramerate(fps)) { GetAdaptCounter().DecrementFramerate(reason, fps); // Reset framerate in case of fewer fps steps down than up. if (adapt_counter.FramerateCount() == 0 && fps != std::numeric_limits::max()) { RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; source_proxy_->IncreaseFramerate(std::numeric_limits::max()); } break; } // Scale up resolution. FALLTHROUGH(); } case VideoSendStream::DegradationPreference::kMaintainFramerate: { // Scale up resolution. int pixel_count = adaptation_request.input_pixel_count_; if (adapt_counter.ResolutionCount() == 1) { RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting."; pixel_count = std::numeric_limits::max(); } if (!source_proxy_->RequestHigherResolutionThan(pixel_count)) return; GetAdaptCounter().DecrementResolution(reason); break; } case VideoSendStream::DegradationPreference::kMaintainResolution: { // Scale up framerate. int fps = adaptation_request.framerate_fps_; if (adapt_counter.FramerateCount() == 1) { RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; fps = std::numeric_limits::max(); } const int requested_framerate = source_proxy_->RequestHigherFramerateThan(fps); if (requested_framerate == -1) { overuse_detector_->OnTargetFramerateUpdated(max_framerate_); return; } overuse_detector_->OnTargetFramerateUpdated( std::min(max_framerate_, requested_framerate)); GetAdaptCounter().DecrementFramerate(reason); break; } case VideoSendStream::DegradationPreference::kDegradationDisabled: return; } last_adaptation_request_.emplace(adaptation_request); UpdateAdaptationStats(reason); RTC_LOG(LS_INFO) << adapt_counter.ToString(); } void VideoStreamEncoder::UpdateAdaptationStats(AdaptReason reason) { switch (reason) { case kCpu: stats_proxy_->OnCpuAdaptationChanged(GetActiveCounts(kCpu), GetActiveCounts(kQuality)); break; case kQuality: stats_proxy_->OnQualityAdaptationChanged(GetActiveCounts(kCpu), GetActiveCounts(kQuality)); break; } } VideoStreamEncoder::AdaptCounts VideoStreamEncoder::GetActiveCounts( AdaptReason reason) { VideoStreamEncoder::AdaptCounts counts = GetConstAdaptCounter().Counts(reason); switch (reason) { case kCpu: if (!IsFramerateScalingEnabled(degradation_preference_)) counts.fps = -1; if (!IsResolutionScalingEnabled(degradation_preference_)) counts.resolution = -1; break; case kQuality: if (!IsFramerateScalingEnabled(degradation_preference_) || !quality_scaler_) { counts.fps = -1; } if (!IsResolutionScalingEnabled(degradation_preference_) || !quality_scaler_) { counts.resolution = -1; } break; } return counts; } VideoStreamEncoder::AdaptCounter& VideoStreamEncoder::GetAdaptCounter() { return adapt_counters_[degradation_preference_]; } const VideoStreamEncoder::AdaptCounter& VideoStreamEncoder::GetConstAdaptCounter() { return adapt_counters_[degradation_preference_]; } // Class holding adaptation information. VideoStreamEncoder::AdaptCounter::AdaptCounter() { fps_counters_.resize(kScaleReasonSize); resolution_counters_.resize(kScaleReasonSize); static_assert(kScaleReasonSize == 2, "Update MoveCount."); } VideoStreamEncoder::AdaptCounter::~AdaptCounter() {} std::string VideoStreamEncoder::AdaptCounter::ToString() const { std::stringstream ss; ss << "Downgrade counts: fps: {" << ToString(fps_counters_); ss << "}, resolution: {" << ToString(resolution_counters_) << "}"; return ss.str(); } VideoStreamEncoder::AdaptCounts VideoStreamEncoder::AdaptCounter::Counts( int reason) const { AdaptCounts counts; counts.fps = fps_counters_[reason]; counts.resolution = resolution_counters_[reason]; return counts; } void VideoStreamEncoder::AdaptCounter::IncrementFramerate(int reason) { ++(fps_counters_[reason]); } void VideoStreamEncoder::AdaptCounter::IncrementResolution(int reason) { ++(resolution_counters_[reason]); } void VideoStreamEncoder::AdaptCounter::DecrementFramerate(int reason) { if (fps_counters_[reason] == 0) { // Balanced mode: Adapt up is in a different order, switch reason. // E.g. framerate adapt down: quality (2), framerate adapt up: cpu (3). // 1. Down resolution (cpu): res={quality:0,cpu:1}, fps={quality:0,cpu:0} // 2. Down fps (quality): res={quality:0,cpu:1}, fps={quality:1,cpu:0} // 3. Up fps (cpu): res={quality:1,cpu:0}, fps={quality:0,cpu:0} // 4. Up resolution (quality): res={quality:0,cpu:0}, fps={quality:0,cpu:0} RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded."; MoveCount(&resolution_counters_, reason); MoveCount(&fps_counters_, (reason + 1) % kScaleReasonSize); } --(fps_counters_[reason]); RTC_DCHECK_GE(fps_counters_[reason], 0); } void VideoStreamEncoder::AdaptCounter::DecrementResolution(int reason) { if (resolution_counters_[reason] == 0) { // Balanced mode: Adapt up is in a different order, switch reason. RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded."; MoveCount(&fps_counters_, reason); MoveCount(&resolution_counters_, (reason + 1) % kScaleReasonSize); } --(resolution_counters_[reason]); RTC_DCHECK_GE(resolution_counters_[reason], 0); } void VideoStreamEncoder::AdaptCounter::DecrementFramerate(int reason, int cur_fps) { DecrementFramerate(reason); // Reset if at max fps (i.e. in case of fewer steps up than down). if (cur_fps == std::numeric_limits::max()) std::fill(fps_counters_.begin(), fps_counters_.end(), 0); } int VideoStreamEncoder::AdaptCounter::FramerateCount() const { return Count(fps_counters_); } int VideoStreamEncoder::AdaptCounter::ResolutionCount() const { return Count(resolution_counters_); } int VideoStreamEncoder::AdaptCounter::FramerateCount(int reason) const { return fps_counters_[reason]; } int VideoStreamEncoder::AdaptCounter::ResolutionCount(int reason) const { return resolution_counters_[reason]; } int VideoStreamEncoder::AdaptCounter::TotalCount(int reason) const { return FramerateCount(reason) + ResolutionCount(reason); } int VideoStreamEncoder::AdaptCounter::Count( const std::vector& counters) const { return std::accumulate(counters.begin(), counters.end(), 0); } void VideoStreamEncoder::AdaptCounter::MoveCount(std::vector* counters, int from_reason) { int to_reason = (from_reason + 1) % kScaleReasonSize; ++((*counters)[to_reason]); --((*counters)[from_reason]); } std::string VideoStreamEncoder::AdaptCounter::ToString( const std::vector& counters) const { std::stringstream ss; for (size_t reason = 0; reason < kScaleReasonSize; ++reason) { ss << (reason ? " cpu" : "quality") << ":" << counters[reason]; } return ss.str(); } } // namespace webrtc