diff options
Diffstat (limited to 'third_party/libwebrtc/video/video_quality_observer2.cc')
| -rw-r--r-- | third_party/libwebrtc/video/video_quality_observer2.cc | 296 |
1 files changed, 296 insertions, 0 deletions
diff --git a/third_party/libwebrtc/video/video_quality_observer2.cc b/third_party/libwebrtc/video/video_quality_observer2.cc new file mode 100644 index 0000000000..0afc2f5235 --- /dev/null +++ b/third_party/libwebrtc/video/video_quality_observer2.cc @@ -0,0 +1,296 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/video_quality_observer2.h" + +#include <algorithm> +#include <cmath> +#include <cstdint> +#include <string> + +#include "rtc_base/logging.h" +#include "rtc_base/strings/string_builder.h" +#include "system_wrappers/include/metrics.h" +#include "video/video_receive_stream2.h" + +namespace webrtc { +namespace internal { +const uint32_t VideoQualityObserver::kMinFrameSamplesToDetectFreeze = 5; +const uint32_t VideoQualityObserver::kMinIncreaseForFreezeMs = 150; +const uint32_t VideoQualityObserver::kAvgInterframeDelaysWindowSizeFrames = 30; + +namespace { +constexpr int kMinVideoDurationMs = 3000; +constexpr int kMinRequiredSamples = 1; +constexpr int kPixelsInHighResolution = + 960 * 540; // CPU-adapted HD still counts. +constexpr int kPixelsInMediumResolution = 640 * 360; +constexpr int kBlockyQpThresholdVp8 = 70; +constexpr int kBlockyQpThresholdVp9 = 180; +constexpr int kMaxNumCachedBlockyFrames = 100; +// TODO(ilnik): Add H264/HEVC thresholds. +} // namespace + +VideoQualityObserver::VideoQualityObserver() + : last_frame_rendered_ms_(-1), + num_frames_rendered_(0), + first_frame_rendered_ms_(-1), + last_frame_pixels_(0), + is_last_frame_blocky_(false), + last_unfreeze_time_ms_(0), + render_interframe_delays_(kAvgInterframeDelaysWindowSizeFrames), + sum_squared_interframe_delays_secs_(0.0), + time_in_resolution_ms_(3, 0), + current_resolution_(Resolution::Low), + num_resolution_downgrades_(0), + time_in_blocky_video_ms_(0), + is_paused_(false) {} + +void VideoQualityObserver::UpdateHistograms(bool screenshare) { + // TODO(bugs.webrtc.org/11489): Called on the decoder thread - which _might_ + // be the same as the construction thread. + + // Don't report anything on an empty video stream. + if (num_frames_rendered_ == 0) { + return; + } + + char log_stream_buf[2 * 1024]; + rtc::SimpleStringBuilder log_stream(log_stream_buf); + + if (last_frame_rendered_ms_ > last_unfreeze_time_ms_) { + smooth_playback_durations_.Add(last_frame_rendered_ms_ - + last_unfreeze_time_ms_); + } + + std::string uma_prefix = + screenshare ? "WebRTC.Video.Screenshare" : "WebRTC.Video"; + + auto mean_time_between_freezes = + smooth_playback_durations_.Avg(kMinRequiredSamples); + if (mean_time_between_freezes) { + RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanTimeBetweenFreezesMs", + *mean_time_between_freezes); + log_stream << uma_prefix << ".MeanTimeBetweenFreezesMs " + << *mean_time_between_freezes << "\n"; + } + auto avg_freeze_length = freezes_durations_.Avg(kMinRequiredSamples); + if (avg_freeze_length) { + RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanFreezeDurationMs", + *avg_freeze_length); + log_stream << uma_prefix << ".MeanFreezeDurationMs " << *avg_freeze_length + << "\n"; + } + + int64_t video_duration_ms = + last_frame_rendered_ms_ - first_frame_rendered_ms_; + + if (video_duration_ms >= kMinVideoDurationMs) { + int time_spent_in_hd_percentage = static_cast<int>( + time_in_resolution_ms_[Resolution::High] * 100 / video_duration_ms); + RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInHdPercentage", + time_spent_in_hd_percentage); + log_stream << uma_prefix << ".TimeInHdPercentage " + << time_spent_in_hd_percentage << "\n"; + + int time_with_blocky_video_percentage = + static_cast<int>(time_in_blocky_video_ms_ * 100 / video_duration_ms); + RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInBlockyVideoPercentage", + time_with_blocky_video_percentage); + log_stream << uma_prefix << ".TimeInBlockyVideoPercentage " + << time_with_blocky_video_percentage << "\n"; + + int num_resolution_downgrades_per_minute = + num_resolution_downgrades_ * 60000 / video_duration_ms; + if (!screenshare) { + RTC_HISTOGRAM_COUNTS_SPARSE_100( + uma_prefix + ".NumberResolutionDownswitchesPerMinute", + num_resolution_downgrades_per_minute); + log_stream << uma_prefix << ".NumberResolutionDownswitchesPerMinute " + << num_resolution_downgrades_per_minute << "\n"; + } + + int num_freezes_per_minute = + freezes_durations_.NumSamples() * 60000 / video_duration_ms; + RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".NumberFreezesPerMinute", + num_freezes_per_minute); + log_stream << uma_prefix << ".NumberFreezesPerMinute " + << num_freezes_per_minute << "\n"; + + if (sum_squared_interframe_delays_secs_ > 0.0) { + int harmonic_framerate_fps = std::round( + video_duration_ms / (1000 * sum_squared_interframe_delays_secs_)); + RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".HarmonicFrameRate", + harmonic_framerate_fps); + log_stream << uma_prefix << ".HarmonicFrameRate " + << harmonic_framerate_fps << "\n"; + } + } + RTC_LOG(LS_INFO) << log_stream.str(); +} + +void VideoQualityObserver::OnRenderedFrame( + const VideoFrameMetaData& frame_meta) { + RTC_DCHECK_LE(last_frame_rendered_ms_, frame_meta.decode_timestamp.ms()); + RTC_DCHECK_LE(last_unfreeze_time_ms_, frame_meta.decode_timestamp.ms()); + + if (num_frames_rendered_ == 0) { + first_frame_rendered_ms_ = last_unfreeze_time_ms_ = + frame_meta.decode_timestamp.ms(); + } + + auto blocky_frame_it = blocky_frames_.find(frame_meta.rtp_timestamp); + + if (num_frames_rendered_ > 0) { + // Process inter-frame delay. + const int64_t interframe_delay_ms = + frame_meta.decode_timestamp.ms() - last_frame_rendered_ms_; + const double interframe_delays_secs = interframe_delay_ms / 1000.0; + + // Sum of squared inter frame intervals is used to calculate the harmonic + // frame rate metric. The metric aims to reflect overall experience related + // to smoothness of video playback and includes both freezes and pauses. + sum_squared_interframe_delays_secs_ += + interframe_delays_secs * interframe_delays_secs; + + if (!is_paused_) { + render_interframe_delays_.AddSample(interframe_delay_ms); + + bool was_freeze = false; + if (render_interframe_delays_.Size() >= kMinFrameSamplesToDetectFreeze) { + const absl::optional<int64_t> avg_interframe_delay = + render_interframe_delays_.GetAverageRoundedDown(); + RTC_DCHECK(avg_interframe_delay); + was_freeze = interframe_delay_ms >= + std::max(3 * *avg_interframe_delay, + *avg_interframe_delay + kMinIncreaseForFreezeMs); + } + + if (was_freeze) { + freezes_durations_.Add(interframe_delay_ms); + smooth_playback_durations_.Add(last_frame_rendered_ms_ - + last_unfreeze_time_ms_); + last_unfreeze_time_ms_ = frame_meta.decode_timestamp.ms(); + } else { + // Count spatial metrics if there were no freeze. + time_in_resolution_ms_[current_resolution_] += interframe_delay_ms; + + if (is_last_frame_blocky_) { + time_in_blocky_video_ms_ += interframe_delay_ms; + } + } + } + } + + if (is_paused_) { + // If the stream was paused since the previous frame, do not count the + // pause toward smooth playback. Explicitly count the part before it and + // start the new smooth playback interval from this frame. + is_paused_ = false; + if (last_frame_rendered_ms_ > last_unfreeze_time_ms_) { + smooth_playback_durations_.Add(last_frame_rendered_ms_ - + last_unfreeze_time_ms_); + } + last_unfreeze_time_ms_ = frame_meta.decode_timestamp.ms(); + + if (num_frames_rendered_ > 0) { + pauses_durations_.Add(frame_meta.decode_timestamp.ms() - + last_frame_rendered_ms_); + } + } + + int64_t pixels = frame_meta.width * frame_meta.height; + if (pixels >= kPixelsInHighResolution) { + current_resolution_ = Resolution::High; + } else if (pixels >= kPixelsInMediumResolution) { + current_resolution_ = Resolution::Medium; + } else { + current_resolution_ = Resolution::Low; + } + + if (pixels < last_frame_pixels_) { + ++num_resolution_downgrades_; + } + + last_frame_pixels_ = pixels; + last_frame_rendered_ms_ = frame_meta.decode_timestamp.ms(); + + is_last_frame_blocky_ = blocky_frame_it != blocky_frames_.end(); + if (is_last_frame_blocky_) { + blocky_frames_.erase(blocky_frames_.begin(), ++blocky_frame_it); + } + + ++num_frames_rendered_; +} + +void VideoQualityObserver::OnDecodedFrame(uint32_t rtp_frame_timestamp, + absl::optional<uint8_t> qp, + VideoCodecType codec) { + if (!qp) + return; + + absl::optional<int> qp_blocky_threshold; + // TODO(ilnik): add other codec types when we have QP for them. + switch (codec) { + case kVideoCodecVP8: + qp_blocky_threshold = kBlockyQpThresholdVp8; + break; + case kVideoCodecVP9: + qp_blocky_threshold = kBlockyQpThresholdVp9; + break; + default: + qp_blocky_threshold = absl::nullopt; + } + + RTC_DCHECK(blocky_frames_.find(rtp_frame_timestamp) == blocky_frames_.end()); + + if (qp_blocky_threshold && *qp > *qp_blocky_threshold) { + // Cache blocky frame. Its duration will be calculated in render callback. + if (blocky_frames_.size() > kMaxNumCachedBlockyFrames) { + RTC_LOG(LS_WARNING) << "Overflow of blocky frames cache."; + blocky_frames_.erase( + blocky_frames_.begin(), + std::next(blocky_frames_.begin(), kMaxNumCachedBlockyFrames / 2)); + } + + blocky_frames_.insert(rtp_frame_timestamp); + } +} + +void VideoQualityObserver::OnStreamInactive() { + is_paused_ = true; +} + +uint32_t VideoQualityObserver::NumFreezes() const { + return freezes_durations_.NumSamples(); +} + +uint32_t VideoQualityObserver::NumPauses() const { + return pauses_durations_.NumSamples(); +} + +uint32_t VideoQualityObserver::TotalFreezesDurationMs() const { + return freezes_durations_.Sum(kMinRequiredSamples).value_or(0); +} + +uint32_t VideoQualityObserver::TotalPausesDurationMs() const { + return pauses_durations_.Sum(kMinRequiredSamples).value_or(0); +} + +uint32_t VideoQualityObserver::TotalFramesDurationMs() const { + return last_frame_rendered_ms_ - first_frame_rendered_ms_; +} + +double VideoQualityObserver::SumSquaredFrameDurationsSec() const { + return sum_squared_interframe_delays_secs_; +} + +} // namespace internal +} // namespace webrtc |