diff options
Diffstat (limited to '')
| -rw-r--r-- | third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc | 403 |
1 files changed, 403 insertions, 0 deletions
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc new file mode 100644 index 0000000000..e814ba88b7 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc @@ -0,0 +1,403 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h" + +#include <cmath> +#include <memory> +#include <utility> + +#include "absl/strings/string_view.h" +#include "api/video/video_codec_type.h" +#include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/svc/scalability_mode_util.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using EmulatedSFUConfigMap = + ::webrtc::webrtc_pc_e2e::QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap; + +constexpr size_t kMaxFrameInPipelineCount = 1000; +constexpr double kNoMultiplier = 1.0; +constexpr double kEps = 1e-6; + +std::pair<uint32_t, uint32_t> GetMinMaxBitratesBps(const VideoCodec& codec, + size_t spatial_idx) { + uint32_t min_bitrate = codec.minBitrate; + uint32_t max_bitrate = codec.maxBitrate; + if (spatial_idx < codec.numberOfSimulcastStreams && + codec.codecType != VideoCodecType::kVideoCodecVP9) { + min_bitrate = + std::max(min_bitrate, codec.simulcastStream[spatial_idx].minBitrate); + max_bitrate = + std::min(max_bitrate, codec.simulcastStream[spatial_idx].maxBitrate); + } + if (codec.codecType == VideoCodecType::kVideoCodecVP9 && + spatial_idx < codec.VP9().numberOfSpatialLayers) { + min_bitrate = + std::max(min_bitrate, codec.spatialLayers[spatial_idx].minBitrate); + max_bitrate = + std::min(max_bitrate, codec.spatialLayers[spatial_idx].maxBitrate); + } + RTC_DCHECK_GT(max_bitrate, min_bitrate); + return {min_bitrate * 1000, max_bitrate * 1000}; +} + +} // namespace + +QualityAnalyzingVideoEncoder::QualityAnalyzingVideoEncoder( + absl::string_view peer_name, + std::unique_ptr<VideoEncoder> delegate, + double bitrate_multiplier, + EmulatedSFUConfigMap stream_to_sfu_config, + EncodedImageDataInjector* injector, + VideoQualityAnalyzerInterface* analyzer) + : peer_name_(peer_name), + delegate_(std::move(delegate)), + bitrate_multiplier_(bitrate_multiplier), + stream_to_sfu_config_(std::move(stream_to_sfu_config)), + injector_(injector), + analyzer_(analyzer), + mode_(SimulcastMode::kNormal), + delegate_callback_(nullptr) {} +QualityAnalyzingVideoEncoder::~QualityAnalyzingVideoEncoder() = default; + +void QualityAnalyzingVideoEncoder::SetFecControllerOverride( + FecControllerOverride* fec_controller_override) { + // Ignored. +} + +int32_t QualityAnalyzingVideoEncoder::InitEncode( + const VideoCodec* codec_settings, + const Settings& settings) { + MutexLock lock(&mutex_); + codec_settings_ = *codec_settings; + mode_ = SimulcastMode::kNormal; + absl::optional<InterLayerPredMode> inter_layer_pred_mode; + if (codec_settings->GetScalabilityMode().has_value()) { + inter_layer_pred_mode = ScalabilityModeToInterLayerPredMode( + *codec_settings->GetScalabilityMode()); + } else if (codec_settings->codecType == kVideoCodecVP9) { + if (codec_settings->VP9().numberOfSpatialLayers > 1) { + inter_layer_pred_mode = codec_settings->VP9().interLayerPred; + } + } + if (inter_layer_pred_mode.has_value()) { + switch (*inter_layer_pred_mode) { + case InterLayerPredMode::kOn: + mode_ = SimulcastMode::kSVC; + break; + case InterLayerPredMode::kOnKeyPic: + mode_ = SimulcastMode::kKSVC; + break; + case InterLayerPredMode::kOff: + mode_ = SimulcastMode::kSimulcast; + break; + default: + RTC_DCHECK_NOTREACHED() + << "Unknown InterLayerPredMode value " << *inter_layer_pred_mode; + break; + } + } + if (codec_settings->numberOfSimulcastStreams > 1) { + mode_ = SimulcastMode::kSimulcast; + } + return delegate_->InitEncode(codec_settings, settings); +} + +int32_t QualityAnalyzingVideoEncoder::RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) { + // We need to get a lock here because delegate_callback can be hypothetically + // accessed from different thread (encoder one) concurrently. + MutexLock lock(&mutex_); + delegate_callback_ = callback; + return delegate_->RegisterEncodeCompleteCallback(this); +} + +int32_t QualityAnalyzingVideoEncoder::Release() { + // Release encoder first. During release process it can still encode some + // frames, so we don't take a lock to prevent deadlock. + int32_t result = delegate_->Release(); + + MutexLock lock(&mutex_); + delegate_callback_ = nullptr; + return result; +} + +int32_t QualityAnalyzingVideoEncoder::Encode( + const VideoFrame& frame, + const std::vector<VideoFrameType>* frame_types) { + { + MutexLock lock(&mutex_); + // Store id to be able to retrieve it in analyzing callback. + timestamp_to_frame_id_list_.push_back({frame.timestamp(), frame.id()}); + // If this list is growing, it means that we are not receiving new encoded + // images from encoder. So it should be a bug in setup on in the encoder. + RTC_DCHECK_LT(timestamp_to_frame_id_list_.size(), kMaxFrameInPipelineCount); + } + analyzer_->OnFramePreEncode(peer_name_, frame); + int32_t result = delegate_->Encode(frame, frame_types); + if (result != WEBRTC_VIDEO_CODEC_OK) { + // If origin encoder failed, then cleanup data for this frame. + { + MutexLock lock(&mutex_); + // The timestamp-frame_id pair can be not the last one, so we need to + // find it first and then remove. We will search from the end, because + // usually it will be the last or close to the last one. + auto it = timestamp_to_frame_id_list_.end(); + while (it != timestamp_to_frame_id_list_.begin()) { + --it; + if (it->first == frame.timestamp()) { + timestamp_to_frame_id_list_.erase(it); + break; + } + } + } + analyzer_->OnEncoderError(peer_name_, frame, result); + } + return result; +} + +void QualityAnalyzingVideoEncoder::SetRates( + const VideoEncoder::RateControlParameters& parameters) { + RTC_DCHECK_GT(bitrate_multiplier_, 0.0); + if (fabs(bitrate_multiplier_ - kNoMultiplier) < kEps) { + { + MutexLock lock(&mutex_); + bitrate_allocation_ = parameters.bitrate; + } + return delegate_->SetRates(parameters); + } + + RateControlParameters adjusted_params = parameters; + { + MutexLock lock(&mutex_); + // Simulating encoder overshooting target bitrate, by configuring actual + // encoder too high. Take care not to adjust past limits of config, + // otherwise encoders may crash on DCHECK. + VideoBitrateAllocation multiplied_allocation; + for (size_t si = 0; si < kMaxSpatialLayers; ++si) { + const uint32_t spatial_layer_bitrate_bps = + parameters.bitrate.GetSpatialLayerSum(si); + if (spatial_layer_bitrate_bps == 0) { + continue; + } + + uint32_t min_bitrate_bps; + uint32_t max_bitrate_bps; + std::tie(min_bitrate_bps, max_bitrate_bps) = + GetMinMaxBitratesBps(codec_settings_, si); + double bitrate_multiplier = bitrate_multiplier_; + const uint32_t corrected_bitrate = rtc::checked_cast<uint32_t>( + bitrate_multiplier * spatial_layer_bitrate_bps); + if (corrected_bitrate < min_bitrate_bps) { + bitrate_multiplier = min_bitrate_bps / spatial_layer_bitrate_bps; + } else if (corrected_bitrate > max_bitrate_bps) { + bitrate_multiplier = max_bitrate_bps / spatial_layer_bitrate_bps; + } + + for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) { + if (parameters.bitrate.HasBitrate(si, ti)) { + multiplied_allocation.SetBitrate( + si, ti, + rtc::checked_cast<uint32_t>( + bitrate_multiplier * parameters.bitrate.GetBitrate(si, ti))); + } + } + } + + adjusted_params.bitrate = multiplied_allocation; + bitrate_allocation_ = adjusted_params.bitrate; + } + return delegate_->SetRates(adjusted_params); +} + +VideoEncoder::EncoderInfo QualityAnalyzingVideoEncoder::GetEncoderInfo() const { + return delegate_->GetEncoderInfo(); +} + +// It is assumed, that encoded callback will be always invoked with encoded +// images that correspond to the frames in the same sequence, that frames +// arrived. In other words, assume we have frames F1, F2 and F3 and they have +// corresponding encoded images I1, I2 and I3. In such case if we will call +// encode first with F1, then with F2 and then with F3, then encoder callback +// will be called first with all spatial layers for F1 (I1), then F2 (I2) and +// then F3 (I3). +// +// Basing on it we will use a list of timestamp-frame_id pairs like this: +// 1. If current encoded image timestamp is equals to timestamp in the front +// pair - pick frame id from that pair +// 2. If current encoded image timestamp isn't equals to timestamp in the front +// pair - remove the front pair and got to the step 1. +EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info) { + uint16_t frame_id; + bool discard = false; + uint32_t target_encode_bitrate = 0; + std::string codec_name; + { + MutexLock lock(&mutex_); + std::pair<uint32_t, uint16_t> timestamp_frame_id; + while (!timestamp_to_frame_id_list_.empty()) { + timestamp_frame_id = timestamp_to_frame_id_list_.front(); + if (timestamp_frame_id.first == encoded_image.Timestamp()) { + break; + } + timestamp_to_frame_id_list_.pop_front(); + } + + // After the loop the first element should point to current `encoded_image` + // frame id. We don't remove it from the list, because there may be + // multiple spatial layers for this frame, so encoder can produce more + // encoded images with this timestamp. The first element will be removed + // when the next frame would be encoded and EncodedImageCallback would be + // called with the next timestamp. + + if (timestamp_to_frame_id_list_.empty()) { + // Ensure, that we have info about this frame. It can happen that for some + // reasons encoder response, that he failed to decode, when we were + // posting frame to it, but then call the callback for this frame. + RTC_LOG(LS_ERROR) << "QualityAnalyzingVideoEncoder::OnEncodedImage: No " + "frame id for encoded_image.Timestamp()=" + << encoded_image.Timestamp(); + return EncodedImageCallback::Result( + EncodedImageCallback::Result::Error::OK); + } + frame_id = timestamp_frame_id.second; + + discard = ShouldDiscard(frame_id, encoded_image); + if (!discard) { + target_encode_bitrate = bitrate_allocation_.GetSpatialLayerSum( + encoded_image.SpatialIndex().value_or(0)); + } + codec_name = + std::string(CodecTypeToPayloadString(codec_settings_.codecType)) + "_" + + delegate_->GetEncoderInfo().implementation_name; + } + + VideoQualityAnalyzerInterface::EncoderStats stats; + stats.encoder_name = codec_name; + stats.target_encode_bitrate = target_encode_bitrate; + stats.qp = encoded_image.qp_; + analyzer_->OnFrameEncoded(peer_name_, frame_id, encoded_image, stats, + discard); + + // Image data injector injects frame id and discard flag into provided + // EncodedImage and returns the image with a) modified original buffer (in + // such case the current owner of the buffer will be responsible for deleting + // it) or b) a new buffer (in such case injector will be responsible for + // deleting it). + const EncodedImage& image = + injector_->InjectData(frame_id, discard, encoded_image); + { + MutexLock lock(&mutex_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->OnEncodedImage(image, codec_specific_info); + } +} + +void QualityAnalyzingVideoEncoder::OnDroppedFrame( + EncodedImageCallback::DropReason reason) { + MutexLock lock(&mutex_); + analyzer_->OnFrameDropped(peer_name_, reason); + RTC_DCHECK(delegate_callback_); + delegate_callback_->OnDroppedFrame(reason); +} + +bool QualityAnalyzingVideoEncoder::ShouldDiscard( + uint16_t frame_id, + const EncodedImage& encoded_image) { + std::string stream_label = analyzer_->GetStreamLabel(frame_id); + EmulatedSFUConfigMap::mapped_type emulated_sfu_config = + stream_to_sfu_config_[stream_label]; + + if (!emulated_sfu_config) + return false; + + int cur_spatial_index = encoded_image.SpatialIndex().value_or(0); + int cur_temporal_index = encoded_image.TemporalIndex().value_or(0); + + if (emulated_sfu_config->target_temporal_index && + cur_temporal_index > *emulated_sfu_config->target_temporal_index) + return true; + + if (emulated_sfu_config->target_layer_index) { + switch (mode_) { + case SimulcastMode::kSimulcast: + // In simulcast mode only encoded images with required spatial index are + // interested, so all others have to be discarded. + return cur_spatial_index != *emulated_sfu_config->target_layer_index; + case SimulcastMode::kSVC: + // In SVC mode encoded images with spatial indexes that are equal or + // less than required one are interesting, so all above have to be + // discarded. + return cur_spatial_index > *emulated_sfu_config->target_layer_index; + case SimulcastMode::kKSVC: + // In KSVC mode for key frame encoded images with spatial indexes that + // are equal or less than required one are interesting, so all above + // have to be discarded. For other frames only required spatial index + // is interesting, so all others except the ones depending on the + // keyframes can be discarded. There's no good test for that, so we keep + // all of temporal layer 0 for now. + if (encoded_image._frameType == VideoFrameType::kVideoFrameKey || + cur_temporal_index == 0) + return cur_spatial_index > *emulated_sfu_config->target_layer_index; + return cur_spatial_index != *emulated_sfu_config->target_layer_index; + case SimulcastMode::kNormal: + RTC_DCHECK_NOTREACHED() << "Analyzing encoder is in kNormal mode, but " + "target_layer_index is set"; + } + } + return false; +} + +QualityAnalyzingVideoEncoderFactory::QualityAnalyzingVideoEncoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoEncoderFactory> delegate, + double bitrate_multiplier, + EmulatedSFUConfigMap stream_to_sfu_config, + EncodedImageDataInjector* injector, + VideoQualityAnalyzerInterface* analyzer) + : peer_name_(peer_name), + delegate_(std::move(delegate)), + bitrate_multiplier_(bitrate_multiplier), + stream_to_sfu_config_(std::move(stream_to_sfu_config)), + injector_(injector), + analyzer_(analyzer) {} +QualityAnalyzingVideoEncoderFactory::~QualityAnalyzingVideoEncoderFactory() = + default; + +std::vector<SdpVideoFormat> +QualityAnalyzingVideoEncoderFactory::GetSupportedFormats() const { + return delegate_->GetSupportedFormats(); +} + +VideoEncoderFactory::CodecSupport +QualityAnalyzingVideoEncoderFactory::QueryCodecSupport( + const SdpVideoFormat& format, + absl::optional<std::string> scalability_mode) const { + return delegate_->QueryCodecSupport(format, scalability_mode); +} + +std::unique_ptr<VideoEncoder> +QualityAnalyzingVideoEncoderFactory::CreateVideoEncoder( + const SdpVideoFormat& format) { + return std::make_unique<QualityAnalyzingVideoEncoder>( + peer_name_, delegate_->CreateVideoEncoder(format), bitrate_multiplier_, + stream_to_sfu_config_, injector_, analyzer_); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc |