From 6bf0a5cb5034a7e684dcc3500e841785237ce2dd Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 19:32:43 +0200 Subject: Adding upstream version 1:115.7.0. Signed-off-by: Daniel Baumann --- .../video_encoder_software_fallback_wrapper.cc | 519 +++++++++++++++++++++ 1 file changed, 519 insertions(+) create mode 100644 third_party/libwebrtc/api/video_codecs/video_encoder_software_fallback_wrapper.cc (limited to 'third_party/libwebrtc/api/video_codecs/video_encoder_software_fallback_wrapper.cc') diff --git a/third_party/libwebrtc/api/video_codecs/video_encoder_software_fallback_wrapper.cc b/third_party/libwebrtc/api/video_codecs/video_encoder_software_fallback_wrapper.cc new file mode 100644 index 0000000000..39c52a0081 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_encoder_software_fallback_wrapper.cc @@ -0,0 +1,519 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/video_encoder_software_fallback_wrapper.h" + +#include + +#include +#include +#include +#include + +#include "absl/strings/match.h" +#include "absl/types/optional.h" +#include "api/fec_controller_override.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_bitrate_allocation.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "media/base/video_common.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/utility/simulcast_utility.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { + +namespace { + +// If forced fallback is allowed, either: +// +// 1) The forced fallback is requested if the resolution is less than or equal +// to `max_pixels_`. The resolution is allowed to be scaled down to +// `min_pixels_`. +// +// 2) The forced fallback is requested if temporal support is preferred and the +// SW fallback supports temporal layers while the HW encoder does not. + +struct ForcedFallbackParams { + public: + bool SupportsResolutionBasedSwitch(const VideoCodec& codec) const { + return enable_resolution_based_switch && + codec.codecType == kVideoCodecVP8 && + codec.numberOfSimulcastStreams <= 1 && + codec.width * codec.height <= max_pixels; + } + + bool SupportsTemporalBasedSwitch(const VideoCodec& codec) const { + return enable_temporal_based_switch && + SimulcastUtility::NumberOfTemporalLayers(codec, 0) != 1; + } + + bool enable_temporal_based_switch = false; + bool enable_resolution_based_switch = false; + int min_pixels = 320 * 180; + int max_pixels = 320 * 240; +}; + +const char kVp8ForceFallbackEncoderFieldTrial[] = + "WebRTC-VP8-Forced-Fallback-Encoder-v2"; + +absl::optional ParseFallbackParamsFromFieldTrials( + const VideoEncoder& main_encoder) { + const std::string field_trial = + webrtc::field_trial::FindFullName(kVp8ForceFallbackEncoderFieldTrial); + if (!absl::StartsWith(field_trial, "Enabled")) { + return absl::nullopt; + } + + int max_pixels_lower_bound = + main_encoder.GetEncoderInfo().scaling_settings.min_pixels_per_frame - 1; + + ForcedFallbackParams params; + params.enable_resolution_based_switch = true; + + int min_bps = 0; + if (sscanf(field_trial.c_str(), "Enabled-%d,%d,%d", ¶ms.min_pixels, + ¶ms.max_pixels, &min_bps) != 3) { + RTC_LOG(LS_WARNING) + << "Invalid number of forced fallback parameters provided."; + return absl::nullopt; + } else if (params.min_pixels <= 0 || + params.max_pixels < max_pixels_lower_bound || + params.max_pixels < params.min_pixels || min_bps <= 0) { + RTC_LOG(LS_WARNING) << "Invalid forced fallback parameter value provided."; + return absl::nullopt; + } + + return params; +} + +absl::optional GetForcedFallbackParams( + bool prefer_temporal_support, + const VideoEncoder& main_encoder) { + absl::optional params = + ParseFallbackParamsFromFieldTrials(main_encoder); + if (prefer_temporal_support) { + if (!params.has_value()) { + params.emplace(); + } + params->enable_temporal_based_switch = prefer_temporal_support; + } + return params; +} + +class VideoEncoderSoftwareFallbackWrapper final : public VideoEncoder { + public: + VideoEncoderSoftwareFallbackWrapper( + std::unique_ptr sw_encoder, + std::unique_ptr hw_encoder, + bool prefer_temporal_support); + ~VideoEncoderSoftwareFallbackWrapper() override; + + void SetFecControllerOverride( + FecControllerOverride* fec_controller_override) override; + + int32_t InitEncode(const VideoCodec* codec_settings, + const VideoEncoder::Settings& settings) override; + + int32_t RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) override; + + int32_t Release() override; + + int32_t Encode(const VideoFrame& frame, + const std::vector* frame_types) override; + + void OnPacketLossRateUpdate(float packet_loss_rate) override; + + void OnRttUpdate(int64_t rtt_ms) override; + + void OnLossNotification(const LossNotification& loss_notification) override; + + void SetRates(const RateControlParameters& parameters) override; + + EncoderInfo GetEncoderInfo() const override; + + private: + bool InitFallbackEncoder(bool is_forced); + bool TryInitForcedFallbackEncoder(); + bool IsFallbackActive() const; + + VideoEncoder* current_encoder() { + switch (encoder_state_) { + case EncoderState::kUninitialized: + RTC_LOG(LS_WARNING) + << "Trying to access encoder in uninitialized fallback wrapper."; + // Return main encoder to preserve previous behavior. + [[fallthrough]]; + case EncoderState::kMainEncoderUsed: + return encoder_.get(); + case EncoderState::kFallbackDueToFailure: + case EncoderState::kForcedFallback: + return fallback_encoder_.get(); + } + RTC_CHECK_NOTREACHED(); + } + + // Updates encoder with last observed parameters, such as callbacks, rates, + // etc. + void PrimeEncoder(VideoEncoder* encoder) const; + + // Settings used in the last InitEncode call and used if a dynamic fallback to + // software is required. + VideoCodec codec_settings_; + absl::optional encoder_settings_; + + // The last rate control settings, if set. + absl::optional rate_control_parameters_; + + // The last channel parameters set. + absl::optional packet_loss_; + absl::optional rtt_; + absl::optional loss_notification_; + + enum class EncoderState { + kUninitialized, + kMainEncoderUsed, + kFallbackDueToFailure, + kForcedFallback + }; + + EncoderState encoder_state_; + const std::unique_ptr encoder_; + const std::unique_ptr fallback_encoder_; + + EncodedImageCallback* callback_; + + const absl::optional fallback_params_; + int32_t EncodeWithMainEncoder(const VideoFrame& frame, + const std::vector* frame_types); +}; + +VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper( + std::unique_ptr sw_encoder, + std::unique_ptr hw_encoder, + bool prefer_temporal_support) + : encoder_state_(EncoderState::kUninitialized), + encoder_(std::move(hw_encoder)), + fallback_encoder_(std::move(sw_encoder)), + callback_(nullptr), + fallback_params_( + GetForcedFallbackParams(prefer_temporal_support, *encoder_)) { + RTC_DCHECK(fallback_encoder_); +} + +VideoEncoderSoftwareFallbackWrapper::~VideoEncoderSoftwareFallbackWrapper() = + default; + +void VideoEncoderSoftwareFallbackWrapper::PrimeEncoder( + VideoEncoder* encoder) const { + RTC_DCHECK(encoder); + // Replay callback, rates, and channel parameters. + if (callback_) { + encoder->RegisterEncodeCompleteCallback(callback_); + } + if (rate_control_parameters_) { + encoder->SetRates(*rate_control_parameters_); + } + if (rtt_.has_value()) { + encoder->OnRttUpdate(rtt_.value()); + } + if (packet_loss_.has_value()) { + encoder->OnPacketLossRateUpdate(packet_loss_.value()); + } + + if (loss_notification_.has_value()) { + encoder->OnLossNotification(loss_notification_.value()); + } +} + +bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder(bool is_forced) { + RTC_LOG(LS_WARNING) << "Encoder falling back to software encoding."; + + RTC_DCHECK(encoder_settings_.has_value()); + const int ret = fallback_encoder_->InitEncode(&codec_settings_, + encoder_settings_.value()); + + if (ret != WEBRTC_VIDEO_CODEC_OK) { + RTC_LOG(LS_ERROR) << "Failed to initialize software-encoder fallback."; + fallback_encoder_->Release(); + return false; + } + + if (encoder_state_ == EncoderState::kMainEncoderUsed) { + // Since we're switching to the fallback encoder, Release the real encoder. + // It may be re-initialized via InitEncode later, and it will continue to + // get Set calls for rates and channel parameters in the meantime. + encoder_->Release(); + } + + if (is_forced) { + encoder_state_ = EncoderState::kForcedFallback; + } else { + encoder_state_ = EncoderState::kFallbackDueToFailure; + } + + return true; +} + +void VideoEncoderSoftwareFallbackWrapper::SetFecControllerOverride( + FecControllerOverride* fec_controller_override) { + // It is important that only one of those would ever interact with the + // `fec_controller_override` at a given time. This is the responsibility + // of `this` to maintain. + + encoder_->SetFecControllerOverride(fec_controller_override); + fallback_encoder_->SetFecControllerOverride(fec_controller_override); +} + +int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode( + const VideoCodec* codec_settings, + const VideoEncoder::Settings& settings) { + // Store settings, in case we need to dynamically switch to the fallback + // encoder after a failed Encode call. + codec_settings_ = *codec_settings; + encoder_settings_ = settings; + // Clear stored rate/channel parameters. + rate_control_parameters_ = absl::nullopt; + + RTC_DCHECK_EQ(encoder_state_, EncoderState::kUninitialized) + << "InitEncode() should never be called on an active instance!"; + + // Try to init forced software codec if it should be used. + if (TryInitForcedFallbackEncoder()) { + PrimeEncoder(current_encoder()); + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t ret = encoder_->InitEncode(codec_settings, settings); + if (ret == WEBRTC_VIDEO_CODEC_OK) { + encoder_state_ = EncoderState::kMainEncoderUsed; + PrimeEncoder(current_encoder()); + return ret; + } + + // Try to instantiate software codec. + if (InitFallbackEncoder(/*is_forced=*/false)) { + PrimeEncoder(current_encoder()); + return WEBRTC_VIDEO_CODEC_OK; + } + + // Software encoder failed too, use original return code. + encoder_state_ = EncoderState::kUninitialized; + return ret; +} + +int32_t VideoEncoderSoftwareFallbackWrapper::RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) { + callback_ = callback; + return current_encoder()->RegisterEncodeCompleteCallback(callback); +} + +int32_t VideoEncoderSoftwareFallbackWrapper::Release() { + if (encoder_state_ == EncoderState::kUninitialized) { + return WEBRTC_VIDEO_CODEC_OK; + } + int32_t ret = current_encoder()->Release(); + encoder_state_ = EncoderState::kUninitialized; + return ret; +} + +int32_t VideoEncoderSoftwareFallbackWrapper::Encode( + const VideoFrame& frame, + const std::vector* frame_types) { + switch (encoder_state_) { + case EncoderState::kUninitialized: + return WEBRTC_VIDEO_CODEC_ERROR; + case EncoderState::kMainEncoderUsed: { + return EncodeWithMainEncoder(frame, frame_types); + } + case EncoderState::kFallbackDueToFailure: + case EncoderState::kForcedFallback: + return fallback_encoder_->Encode(frame, frame_types); + } + RTC_CHECK_NOTREACHED(); +} + +int32_t VideoEncoderSoftwareFallbackWrapper::EncodeWithMainEncoder( + const VideoFrame& frame, + const std::vector* frame_types) { + int32_t ret = encoder_->Encode(frame, frame_types); + // If requested, try a software fallback. + bool fallback_requested = (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); + if (fallback_requested && InitFallbackEncoder(/*is_forced=*/false)) { + // Start using the fallback with this frame. + PrimeEncoder(current_encoder()); + if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative && + fallback_encoder_->GetEncoderInfo().supports_native_handle) { + return fallback_encoder_->Encode(frame, frame_types); + } else { + RTC_LOG(LS_INFO) << "Fallback encoder does not support native handle - " + "converting frame to I420"; + rtc::scoped_refptr src_buffer = + frame.video_frame_buffer()->ToI420(); + if (!src_buffer) { + RTC_LOG(LS_ERROR) << "Failed to convert from to I420"; + return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } + rtc::scoped_refptr dst_buffer = + src_buffer->Scale(codec_settings_.width, codec_settings_.height); + if (!dst_buffer) { + RTC_LOG(LS_ERROR) << "Failed to scale video frame."; + return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } + VideoFrame scaled_frame = frame; + scaled_frame.set_video_frame_buffer(dst_buffer); + scaled_frame.set_update_rect(VideoFrame::UpdateRect{ + 0, 0, scaled_frame.width(), scaled_frame.height()}); + return fallback_encoder_->Encode(scaled_frame, frame_types); + } + } + // Fallback encoder failed too, return original error code. + return ret; +} + +void VideoEncoderSoftwareFallbackWrapper::SetRates( + const RateControlParameters& parameters) { + rate_control_parameters_ = parameters; + return current_encoder()->SetRates(parameters); +} + +void VideoEncoderSoftwareFallbackWrapper::OnPacketLossRateUpdate( + float packet_loss_rate) { + packet_loss_ = packet_loss_rate; + current_encoder()->OnPacketLossRateUpdate(packet_loss_rate); +} + +void VideoEncoderSoftwareFallbackWrapper::OnRttUpdate(int64_t rtt_ms) { + rtt_ = rtt_ms; + current_encoder()->OnRttUpdate(rtt_ms); +} + +void VideoEncoderSoftwareFallbackWrapper::OnLossNotification( + const LossNotification& loss_notification) { + loss_notification_ = loss_notification; + current_encoder()->OnLossNotification(loss_notification); +} + +VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo() + const { + EncoderInfo fallback_encoder_info = fallback_encoder_->GetEncoderInfo(); + EncoderInfo default_encoder_info = encoder_->GetEncoderInfo(); + + EncoderInfo info = + IsFallbackActive() ? fallback_encoder_info : default_encoder_info; + + info.requested_resolution_alignment = cricket::LeastCommonMultiple( + fallback_encoder_info.requested_resolution_alignment, + default_encoder_info.requested_resolution_alignment); + info.apply_alignment_to_all_simulcast_layers = + fallback_encoder_info.apply_alignment_to_all_simulcast_layers || + default_encoder_info.apply_alignment_to_all_simulcast_layers; + + if (fallback_params_.has_value()) { + const auto settings = (encoder_state_ == EncoderState::kForcedFallback) + ? fallback_encoder_info.scaling_settings + : default_encoder_info.scaling_settings; + info.scaling_settings = + settings.thresholds + ? VideoEncoder::ScalingSettings(settings.thresholds->low, + settings.thresholds->high, + fallback_params_->min_pixels) + : VideoEncoder::ScalingSettings::kOff; + } else { + info.scaling_settings = default_encoder_info.scaling_settings; + } + + return info; +} + +bool VideoEncoderSoftwareFallbackWrapper::IsFallbackActive() const { + return encoder_state_ == EncoderState::kForcedFallback || + encoder_state_ == EncoderState::kFallbackDueToFailure; +} + +bool VideoEncoderSoftwareFallbackWrapper::TryInitForcedFallbackEncoder() { + if (!fallback_params_) { + return false; + } + + RTC_DCHECK_EQ(encoder_state_, EncoderState::kUninitialized); + + if (fallback_params_->SupportsResolutionBasedSwitch(codec_settings_)) { + // Settings valid, try to instantiate software codec. + RTC_LOG(LS_INFO) << "Request forced SW encoder fallback: " + << codec_settings_.width << "x" << codec_settings_.height; + return InitFallbackEncoder(/*is_forced=*/true); + } + + if (fallback_params_->SupportsTemporalBasedSwitch(codec_settings_)) { + // First init main encoder to see if that supports temporal layers. + if (encoder_->InitEncode(&codec_settings_, encoder_settings_.value()) == + WEBRTC_VIDEO_CODEC_OK) { + encoder_state_ = EncoderState::kMainEncoderUsed; + } + + if (encoder_state_ == EncoderState::kMainEncoderUsed && + encoder_->GetEncoderInfo().fps_allocation[0].size() != 1) { + // Primary encoder already supports temporal layers, use that instead. + return true; + } + + // Try to initialize fallback and check if it supports temporal layers. + if (fallback_encoder_->InitEncode(&codec_settings_, + encoder_settings_.value()) == + WEBRTC_VIDEO_CODEC_OK) { + if (fallback_encoder_->GetEncoderInfo().fps_allocation[0].size() != 1) { + // Fallback encoder available and supports temporal layers, use it! + if (encoder_state_ == EncoderState::kMainEncoderUsed) { + // Main encoder initialized but does not support temporal layers, + // release it again. + encoder_->Release(); + } + encoder_state_ = EncoderState::kForcedFallback; + RTC_LOG(LS_INFO) + << "Forced switch to SW encoder due to temporal support."; + return true; + } else { + // Fallback encoder intialization succeeded, but it does not support + // temporal layers either - release it. + fallback_encoder_->Release(); + } + } + + if (encoder_state_ == EncoderState::kMainEncoderUsed) { + // Main encoder already initialized - make use of it. + RTC_LOG(LS_INFO) + << "Cannot fall back for temporal support since fallback that " + "supports is not available. Using main encoder instead."; + return true; + } + } + + // Neither forced fallback mode supported. + return false; +} + +} // namespace + +std::unique_ptr CreateVideoEncoderSoftwareFallbackWrapper( + std::unique_ptr sw_fallback_encoder, + std::unique_ptr hw_encoder, + bool prefer_temporal_support) { + return std::make_unique( + std::move(sw_fallback_encoder), std::move(hw_encoder), + prefer_temporal_support); +} + +} // namespace webrtc -- cgit v1.2.3