diff options
Diffstat (limited to 'third_party/libwebrtc/test/pc/e2e/analyzer')
55 files changed, 14377 insertions, 0 deletions
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc new file mode 100644 index 0000000000..98d0c533c2 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h" + +#include "api/stats/rtc_stats.h" +#include "api/stats/rtcstats_objects.h" +#include "api/test/metrics/metric.h" +#include "api/test/track_id_stream_info_map.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "test/pc/e2e/metric_metadata_keys.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Unit; + +DefaultAudioQualityAnalyzer::DefaultAudioQualityAnalyzer( + test::MetricsLogger* const metrics_logger) + : metrics_logger_(metrics_logger) { + RTC_CHECK(metrics_logger_); +} + +void DefaultAudioQualityAnalyzer::Start(std::string test_case_name, + TrackIdStreamInfoMap* analyzer_helper) { + test_case_name_ = std::move(test_case_name); + analyzer_helper_ = analyzer_helper; +} + +void DefaultAudioQualityAnalyzer::OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) { + auto stats = report->GetStatsOfType<RTCInboundRTPStreamStats>(); + + for (auto& stat : stats) { + if (!stat->kind.is_defined() || + !(*stat->kind == RTCMediaStreamTrackKind::kAudio)) { + continue; + } + + StatsSample sample; + sample.total_samples_received = + stat->total_samples_received.ValueOrDefault(0ul); + sample.concealed_samples = stat->concealed_samples.ValueOrDefault(0ul); + sample.removed_samples_for_acceleration = + stat->removed_samples_for_acceleration.ValueOrDefault(0ul); + sample.inserted_samples_for_deceleration = + stat->inserted_samples_for_deceleration.ValueOrDefault(0ul); + sample.silent_concealed_samples = + stat->silent_concealed_samples.ValueOrDefault(0ul); + sample.jitter_buffer_delay = + TimeDelta::Seconds(stat->jitter_buffer_delay.ValueOrDefault(0.)); + sample.jitter_buffer_target_delay = + TimeDelta::Seconds(stat->jitter_buffer_target_delay.ValueOrDefault(0.)); + sample.jitter_buffer_emitted_count = + stat->jitter_buffer_emitted_count.ValueOrDefault(0ul); + + TrackIdStreamInfoMap::StreamInfo stream_info = + analyzer_helper_->GetStreamInfoFromTrackId(*stat->track_identifier); + + MutexLock lock(&lock_); + stream_info_.emplace(stream_info.stream_label, stream_info); + StatsSample prev_sample = last_stats_sample_[stream_info.stream_label]; + RTC_CHECK_GE(sample.total_samples_received, + prev_sample.total_samples_received); + double total_samples_diff = static_cast<double>( + sample.total_samples_received - prev_sample.total_samples_received); + if (total_samples_diff == 0) { + return; + } + + AudioStreamStats& audio_stream_stats = + streams_stats_[stream_info.stream_label]; + audio_stream_stats.expand_rate.AddSample( + (sample.concealed_samples - prev_sample.concealed_samples) / + total_samples_diff); + audio_stream_stats.accelerate_rate.AddSample( + (sample.removed_samples_for_acceleration - + prev_sample.removed_samples_for_acceleration) / + total_samples_diff); + audio_stream_stats.preemptive_rate.AddSample( + (sample.inserted_samples_for_deceleration - + prev_sample.inserted_samples_for_deceleration) / + total_samples_diff); + + int64_t speech_concealed_samples = + sample.concealed_samples - sample.silent_concealed_samples; + int64_t prev_speech_concealed_samples = + prev_sample.concealed_samples - prev_sample.silent_concealed_samples; + audio_stream_stats.speech_expand_rate.AddSample( + (speech_concealed_samples - prev_speech_concealed_samples) / + total_samples_diff); + + int64_t jitter_buffer_emitted_count_diff = + sample.jitter_buffer_emitted_count - + prev_sample.jitter_buffer_emitted_count; + if (jitter_buffer_emitted_count_diff > 0) { + TimeDelta jitter_buffer_delay_diff = + sample.jitter_buffer_delay - prev_sample.jitter_buffer_delay; + TimeDelta jitter_buffer_target_delay_diff = + sample.jitter_buffer_target_delay - + prev_sample.jitter_buffer_target_delay; + audio_stream_stats.average_jitter_buffer_delay_ms.AddSample( + jitter_buffer_delay_diff.ms<double>() / + jitter_buffer_emitted_count_diff); + audio_stream_stats.preferred_buffer_size_ms.AddSample( + jitter_buffer_target_delay_diff.ms<double>() / + jitter_buffer_emitted_count_diff); + } + + last_stats_sample_[stream_info.stream_label] = sample; + } +} + +std::string DefaultAudioQualityAnalyzer::GetTestCaseName( + const std::string& stream_label) const { + return test_case_name_ + "/" + stream_label; +} + +void DefaultAudioQualityAnalyzer::Stop() { + MutexLock lock(&lock_); + for (auto& item : streams_stats_) { + const TrackIdStreamInfoMap::StreamInfo& stream_info = + stream_info_[item.first]; + // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey. + std::map<std::string, std::string> metric_metadata{ + {MetricMetadataKey::kAudioStreamMetadataKey, item.first}, + {MetricMetadataKey::kPeerMetadataKey, stream_info.receiver_peer}, + {MetricMetadataKey::kReceiverMetadataKey, stream_info.receiver_peer}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_case_name_}}; + + metrics_logger_->LogMetric("expand_rate", GetTestCaseName(item.first), + item.second.expand_rate, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter, + metric_metadata); + metrics_logger_->LogMetric("accelerate_rate", GetTestCaseName(item.first), + item.second.accelerate_rate, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter, + metric_metadata); + metrics_logger_->LogMetric("preemptive_rate", GetTestCaseName(item.first), + item.second.preemptive_rate, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter, + metric_metadata); + metrics_logger_->LogMetric( + "speech_expand_rate", GetTestCaseName(item.first), + item.second.speech_expand_rate, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "average_jitter_buffer_delay_ms", GetTestCaseName(item.first), + item.second.average_jitter_buffer_delay_ms, Unit::kMilliseconds, + ImprovementDirection::kNeitherIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "preferred_buffer_size_ms", GetTestCaseName(item.first), + item.second.preferred_buffer_size_ms, Unit::kMilliseconds, + ImprovementDirection::kNeitherIsBetter, metric_metadata); + } +} + +std::map<std::string, AudioStreamStats> +DefaultAudioQualityAnalyzer::GetAudioStreamsStats() const { + MutexLock lock(&lock_); + return streams_stats_; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h b/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h new file mode 100644 index 0000000000..9e427afed8 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_AUDIO_DEFAULT_AUDIO_QUALITY_ANALYZER_H_ +#define TEST_PC_E2E_ANALYZER_AUDIO_DEFAULT_AUDIO_QUALITY_ANALYZER_H_ + +#include <map> +#include <string> + +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/audio_quality_analyzer_interface.h" +#include "api/test/metrics/metrics_logger.h" +#include "api/test/track_id_stream_info_map.h" +#include "api/units/time_delta.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +struct AudioStreamStats { + SamplesStatsCounter expand_rate; + SamplesStatsCounter accelerate_rate; + SamplesStatsCounter preemptive_rate; + SamplesStatsCounter speech_expand_rate; + SamplesStatsCounter average_jitter_buffer_delay_ms; + SamplesStatsCounter preferred_buffer_size_ms; +}; + +class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface { + public: + explicit DefaultAudioQualityAnalyzer( + test::MetricsLogger* const metrics_logger); + + void Start(std::string test_case_name, + TrackIdStreamInfoMap* analyzer_helper) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override; + void Stop() override; + + // Returns audio quality stats per stream label. + std::map<std::string, AudioStreamStats> GetAudioStreamsStats() const; + + private: + struct StatsSample { + uint64_t total_samples_received = 0; + uint64_t concealed_samples = 0; + uint64_t removed_samples_for_acceleration = 0; + uint64_t inserted_samples_for_deceleration = 0; + uint64_t silent_concealed_samples = 0; + TimeDelta jitter_buffer_delay = TimeDelta::Zero(); + TimeDelta jitter_buffer_target_delay = TimeDelta::Zero(); + uint64_t jitter_buffer_emitted_count = 0; + }; + + std::string GetTestCaseName(const std::string& stream_label) const; + + test::MetricsLogger* const metrics_logger_; + + std::string test_case_name_; + TrackIdStreamInfoMap* analyzer_helper_; + + mutable Mutex lock_; + std::map<std::string, AudioStreamStats> streams_stats_ RTC_GUARDED_BY(lock_); + std::map<std::string, TrackIdStreamInfoMap::StreamInfo> stream_info_ + RTC_GUARDED_BY(lock_); + std::map<std::string, StatsSample> last_stats_sample_ RTC_GUARDED_BY(lock_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_AUDIO_DEFAULT_AUDIO_QUALITY_ANALYZER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/BUILD.gn b/third_party/libwebrtc/test/pc/e2e/analyzer/video/BUILD.gn new file mode 100644 index 0000000000..cbb4c078f3 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/BUILD.gn @@ -0,0 +1,573 @@ +# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../../../webrtc.gni") + +if (!build_with_chromium) { + group("video_analyzer") { + testonly = true + + deps = [ + ":analyzing_video_sinks_helper", + ":default_video_quality_analyzer_internal", + ":encoded_image_data_injector_api", + ":example_video_quality_analyzer", + ":multi_reader_queue", + ":quality_analyzing_video_decoder", + ":quality_analyzing_video_encoder", + ":simulcast_dummy_buffer_helper", + ":single_process_encoded_image_data_injector", + ":video_dumping", + ":video_frame_tracking_id_injector", + ":video_quality_metrics_reporter", + ] + if (rtc_include_tests) { + deps += [ + ":analyzing_video_sink", + ":video_quality_analyzer_injection_helper", + ] + } + } + + if (rtc_include_tests) { + group("video_analyzer_unittests") { + testonly = true + + deps = [ + ":analyzing_video_sink_test", + ":analyzing_video_sinks_helper_test", + ":default_video_quality_analyzer_frames_comparator_test", + ":default_video_quality_analyzer_metric_names_test", + ":default_video_quality_analyzer_stream_state_test", + ":default_video_quality_analyzer_test", + ":multi_reader_queue_test", + ":names_collection_test", + ":simulcast_dummy_buffer_helper_test", + ":single_process_encoded_image_data_injector_unittest", + ":video_dumping_test", + ":video_frame_tracking_id_injector_unittest", + ] + } + } +} + +rtc_library("video_dumping") { + testonly = true + sources = [ + "video_dumping.cc", + "video_dumping.h", + ] + deps = [ + "../../../..:video_test_support", + "../../../../../api/test/video:video_frame_writer", + "../../../../../api/video:video_frame", + "../../../../../rtc_base:logging", + "../../../../../system_wrappers", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +rtc_library("encoded_image_data_injector_api") { + testonly = true + sources = [ "encoded_image_data_injector.h" ] + + deps = [ "../../../../../api/video:encoded_image" ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("single_process_encoded_image_data_injector") { + testonly = true + sources = [ + "single_process_encoded_image_data_injector.cc", + "single_process_encoded_image_data_injector.h", + ] + + deps = [ + ":encoded_image_data_injector_api", + "../../../../../api/video:encoded_image", + "../../../../../rtc_base:checks", + "../../../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/memory" ] +} + +rtc_library("video_frame_tracking_id_injector") { + testonly = true + sources = [ + "video_frame_tracking_id_injector.cc", + "video_frame_tracking_id_injector.h", + ] + + deps = [ + ":encoded_image_data_injector_api", + "../../../../../api/video:encoded_image", + "../../../../../rtc_base:checks", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/memory" ] +} + +rtc_library("simulcast_dummy_buffer_helper") { + testonly = true + sources = [ + "simulcast_dummy_buffer_helper.cc", + "simulcast_dummy_buffer_helper.h", + ] + deps = [ "../../../../../api/video:video_frame" ] +} + +rtc_library("quality_analyzing_video_decoder") { + testonly = true + sources = [ + "quality_analyzing_video_decoder.cc", + "quality_analyzing_video_decoder.h", + ] + deps = [ + ":encoded_image_data_injector_api", + ":simulcast_dummy_buffer_helper", + "../../../../../api:video_quality_analyzer_api", + "../../../../../api/video:encoded_image", + "../../../../../api/video:video_frame", + "../../../../../api/video_codecs:video_codecs_api", + "../../../../../modules/video_coding:video_codec_interface", + "../../../../../rtc_base:logging", + "../../../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("quality_analyzing_video_encoder") { + testonly = true + sources = [ + "quality_analyzing_video_encoder.cc", + "quality_analyzing_video_encoder.h", + ] + deps = [ + ":encoded_image_data_injector_api", + "../../../../../api:video_quality_analyzer_api", + "../../../../../api/test/pclf:media_configuration", + "../../../../../api/video:video_frame", + "../../../../../api/video_codecs:video_codecs_api", + "../../../../../modules/video_coding:video_codec_interface", + "../../../../../modules/video_coding/svc:scalability_mode_util", + "../../../../../rtc_base:logging", + "../../../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +rtc_library("analyzing_video_sinks_helper") { + testonly = true + sources = [ + "analyzing_video_sinks_helper.cc", + "analyzing_video_sinks_helper.h", + ] + deps = [ + "../../../../../api/test/pclf:media_configuration", + "../../../../../api/test/video:video_frame_writer", + "../../../../../rtc_base:macromagic", + "../../../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("example_video_quality_analyzer") { + testonly = true + sources = [ + "example_video_quality_analyzer.cc", + "example_video_quality_analyzer.h", + ] + + deps = [ + "../../../../../api:array_view", + "../../../../../api:video_quality_analyzer_api", + "../../../../../api/video:encoded_image", + "../../../../../api/video:video_frame", + "../../../../../rtc_base:logging", + "../../../../../rtc_base/synchronization:mutex", + ] +} + +# This target contains implementation details of DefaultVideoQualityAnalyzer, +# so headers exported by it shouldn't be used in other places. +rtc_library("default_video_quality_analyzer_internal") { + visibility = [ + ":default_video_quality_analyzer", + ":default_video_quality_analyzer_frames_comparator_test", + ":default_video_quality_analyzer_stream_state_test", + ":names_collection_test", + ":video_analyzer", + ] + + testonly = true + sources = [ + "default_video_quality_analyzer_cpu_measurer.cc", + "default_video_quality_analyzer_cpu_measurer.h", + "default_video_quality_analyzer_frame_in_flight.cc", + "default_video_quality_analyzer_frame_in_flight.h", + "default_video_quality_analyzer_frames_comparator.cc", + "default_video_quality_analyzer_frames_comparator.h", + "default_video_quality_analyzer_internal_shared_objects.cc", + "default_video_quality_analyzer_internal_shared_objects.h", + "default_video_quality_analyzer_stream_state.cc", + "default_video_quality_analyzer_stream_state.h", + "names_collection.cc", + "names_collection.h", + ] + + deps = [ + ":default_video_quality_analyzer_shared", + ":multi_reader_queue", + "../..:metric_metadata_keys", + "../../../../../api:array_view", + "../../../../../api:scoped_refptr", + "../../../../../api/numerics", + "../../../../../api/units:data_size", + "../../../../../api/units:timestamp", + "../../../../../api/video:video_frame", + "../../../../../api/video:video_frame_type", + "../../../../../common_video", + "../../../../../rtc_base:checks", + "../../../../../rtc_base:platform_thread", + "../../../../../rtc_base:rtc_base_tests_utils", + "../../../../../rtc_base:rtc_event", + "../../../../../rtc_base:stringutils", + "../../../../../rtc_base:timeutils", + "../../../../../rtc_base/synchronization:mutex", + "../../../../../rtc_tools:video_quality_analysis", + "../../../../../system_wrappers", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings:strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("multi_reader_queue") { + testonly = true + sources = [ "multi_reader_queue.h" ] + deps = [ "../../../../../rtc_base:checks" ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("video_quality_metrics_reporter") { + testonly = true + sources = [ + "video_quality_metrics_reporter.cc", + "video_quality_metrics_reporter.h", + ] + deps = [ + "../..:metric_metadata_keys", + "../../../../../api:peer_connection_quality_test_fixture_api", + "../../../../../api:rtc_stats_api", + "../../../../../api:track_id_stream_info_map", + "../../../../../api/numerics", + "../../../../../api/test/metrics:metric", + "../../../../../api/test/metrics:metrics_logger", + "../../../../../api/units:data_rate", + "../../../../../api/units:data_size", + "../../../../../api/units:time_delta", + "../../../../../api/units:timestamp", + "../../../../../rtc_base:checks", + "../../../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +rtc_library("default_video_quality_analyzer") { + testonly = true + sources = [ + "default_video_quality_analyzer.cc", + "default_video_quality_analyzer.h", + ] + + deps = [ + ":default_video_quality_analyzer_internal", + ":default_video_quality_analyzer_shared", + "../..:metric_metadata_keys", + "../../../../../api:array_view", + "../../../../../api:video_quality_analyzer_api", + "../../../../../api/numerics", + "../../../../../api/test/metrics:metric", + "../../../../../api/test/metrics:metrics_logger", + "../../../../../api/units:data_size", + "../../../../../api/units:time_delta", + "../../../../../api/units:timestamp", + "../../../../../api/video:encoded_image", + "../../../../../api/video:video_frame", + "../../../../../rtc_base:checks", + "../../../../../rtc_base:logging", + "../../../../../rtc_base:macromagic", + "../../../../../rtc_base:stringutils", + "../../../../../rtc_base/synchronization:mutex", + "../../../../../system_wrappers", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("default_video_quality_analyzer_shared") { + testonly = true + sources = [ + "default_video_quality_analyzer_shared_objects.cc", + "default_video_quality_analyzer_shared_objects.h", + ] + + deps = [ + "../../../../../api/numerics", + "../../../../../api/units:timestamp", + "../../../../../rtc_base:checks", + "../../../../../rtc_base:stringutils", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("analyzing_video_sink") { + testonly = true + sources = [ + "analyzing_video_sink.cc", + "analyzing_video_sink.h", + ] + deps = [ + ":analyzing_video_sinks_helper", + ":simulcast_dummy_buffer_helper", + ":video_dumping", + "../../../..:fixed_fps_video_frame_writer_adapter", + "../../../..:test_renderer", + "../../../../../api:video_quality_analyzer_api", + "../../../../../api/numerics", + "../../../../../api/test/pclf:media_configuration", + "../../../../../api/test/video:video_frame_writer", + "../../../../../api/units:timestamp", + "../../../../../api/video:video_frame", + "../../../../../rtc_base:checks", + "../../../../../rtc_base:logging", + "../../../../../rtc_base:macromagic", + "../../../../../rtc_base/synchronization:mutex", + "../../../../../system_wrappers", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory:memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("video_quality_analyzer_injection_helper") { + testonly = true + sources = [ + "video_quality_analyzer_injection_helper.cc", + "video_quality_analyzer_injection_helper.h", + ] + deps = [ + ":analyzing_video_sink", + ":analyzing_video_sinks_helper", + ":encoded_image_data_injector_api", + ":quality_analyzing_video_decoder", + ":quality_analyzing_video_encoder", + ":simulcast_dummy_buffer_helper", + ":video_dumping", + "../../../..:fixed_fps_video_frame_writer_adapter", + "../../../..:test_renderer", + "../../../..:video_test_common", + "../../../..:video_test_support", + "../../../../../api:array_view", + "../../../../../api:stats_observer_interface", + "../../../../../api:video_quality_analyzer_api", + "../../../../../api/test/pclf:media_configuration", + "../../../../../api/video:video_frame", + "../../../../../api/video_codecs:video_codecs_api", + "../../../../../rtc_base:checks", + "../../../../../rtc_base:logging", + "../../../../../rtc_base:stringutils", + "../../../../../rtc_base/synchronization:mutex", + "../../../../../system_wrappers", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + ] +} + +if (rtc_include_tests) { + rtc_library("simulcast_dummy_buffer_helper_test") { + testonly = true + sources = [ "simulcast_dummy_buffer_helper_test.cc" ] + deps = [ + ":simulcast_dummy_buffer_helper", + "../../../..:test_support", + "../../../../../api/video:video_frame", + "../../../../../rtc_base:random", + ] + } + + rtc_library("analyzing_video_sink_test") { + testonly = true + sources = [ "analyzing_video_sink_test.cc" ] + deps = [ + ":analyzing_video_sink", + ":example_video_quality_analyzer", + "../../../..:fileutils", + "../../../..:test_support", + "../../../..:video_test_support", + "../../../../../api:create_frame_generator", + "../../../../../api:frame_generator_api", + "../../../../../api:scoped_refptr", + "../../../../../api/test/pclf:media_configuration", + "../../../../../api/units:time_delta", + "../../../../../api/units:timestamp", + "../../../../../api/video:video_frame", + "../../../../../common_video", + "../../../../../rtc_base:timeutils", + "../../../../../system_wrappers", + "../../../../time_controller", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + + rtc_library("analyzing_video_sinks_helper_test") { + testonly = true + sources = [ "analyzing_video_sinks_helper_test.cc" ] + deps = [ + ":analyzing_video_sinks_helper", + "../../../..:test_support", + "../../../../../api/test/pclf:media_configuration", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_library("default_video_quality_analyzer_frames_comparator_test") { + testonly = true + sources = [ "default_video_quality_analyzer_frames_comparator_test.cc" ] + deps = [ + ":default_video_quality_analyzer_internal", + ":default_video_quality_analyzer_shared", + "../../../..:test_support", + "../../../../../api:create_frame_generator", + "../../../../../api/units:timestamp", + "../../../../../rtc_base:stringutils", + "../../../../../system_wrappers", + ] + } + + rtc_library("names_collection_test") { + testonly = true + sources = [ "names_collection_test.cc" ] + deps = [ + ":default_video_quality_analyzer_internal", + "../../../..:test_support", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings:strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + + rtc_library("multi_reader_queue_test") { + testonly = true + sources = [ "multi_reader_queue_test.cc" ] + deps = [ + ":multi_reader_queue", + "../../../..:test_support", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_library("default_video_quality_analyzer_stream_state_test") { + testonly = true + sources = [ "default_video_quality_analyzer_stream_state_test.cc" ] + deps = [ + ":default_video_quality_analyzer_internal", + "../../../..:test_support", + "../../../../../api/units:timestamp", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_library("default_video_quality_analyzer_test") { + testonly = true + sources = [ "default_video_quality_analyzer_test.cc" ] + deps = [ + ":default_video_quality_analyzer", + ":default_video_quality_analyzer_shared", + "../../../..:test_support", + "../../../../../api:create_frame_generator", + "../../../../../api:rtp_packet_info", + "../../../../../api/test/metrics:global_metrics_logger_and_exporter", + "../../../../../api/video:encoded_image", + "../../../../../api/video:video_frame", + "../../../../../common_video", + "../../../../../rtc_base:stringutils", + "../../../../../rtc_tools:video_quality_analysis", + "../../../../../system_wrappers", + ] + } + + rtc_library("default_video_quality_analyzer_metric_names_test") { + testonly = true + sources = [ "default_video_quality_analyzer_metric_names_test.cc" ] + deps = [ + ":default_video_quality_analyzer", + "../../../..:test_support", + "../../../../../api:create_frame_generator", + "../../../../../api:rtp_packet_info", + "../../../../../api/test/metrics:metric", + "../../../../../api/test/metrics:metrics_logger", + "../../../../../api/test/metrics:stdout_metrics_exporter", + "../../../../../api/video:encoded_image", + "../../../../../api/video:video_frame", + "../../../../../common_video", + "../../../../../rtc_tools:video_quality_analysis", + "../../../../../system_wrappers", + ] + } + + rtc_library("video_dumping_test") { + testonly = true + sources = [ "video_dumping_test.cc" ] + deps = [ + ":video_dumping", + "../../../..:fileutils", + "../../../..:test_support", + "../../../..:video_test_support", + "../../../../../api:scoped_refptr", + "../../../../../api/video:video_frame", + "../../../../../rtc_base:random", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_library("single_process_encoded_image_data_injector_unittest") { + testonly = true + sources = [ "single_process_encoded_image_data_injector_unittest.cc" ] + deps = [ + ":single_process_encoded_image_data_injector", + "../../../..:test_support", + "../../../../../api/video:encoded_image", + "../../../../../rtc_base:buffer", + ] + } + + rtc_library("video_frame_tracking_id_injector_unittest") { + testonly = true + sources = [ "video_frame_tracking_id_injector_unittest.cc" ] + deps = [ + ":video_frame_tracking_id_injector", + "../../../..:test_support", + "../../../../../api/video:encoded_image", + "../../../../../rtc_base:buffer", + ] + } +} diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.cc new file mode 100644 index 0000000000..fb221e6797 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.cc @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/analyzing_video_sink.h" + +#include <memory> +#include <set> +#include <utility> + +#include "absl/memory/memory.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/video/video_frame_writer.h" +#include "api/units/timestamp.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/synchronization/mutex.h" +#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h" +#include "test/pc/e2e/analyzer/video/video_dumping.h" +#include "test/testsupport/fixed_fps_video_frame_writer_adapter.h" +#include "test/video_renderer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +AnalyzingVideoSink::AnalyzingVideoSink(absl::string_view peer_name, + Clock* clock, + VideoQualityAnalyzerInterface& analyzer, + AnalyzingVideoSinksHelper& sinks_helper, + const VideoSubscription& subscription, + bool report_infra_stats) + : peer_name_(peer_name), + report_infra_stats_(report_infra_stats), + clock_(clock), + analyzer_(&analyzer), + sinks_helper_(&sinks_helper), + subscription_(subscription) {} + +void AnalyzingVideoSink::UpdateSubscription( + const VideoSubscription& subscription) { + // For peers with changed resolutions we need to close current writers and + // open new ones. This is done by removing existing sinks, which will force + // creation of the new sinks when next frame will be received. + std::set<test::VideoFrameWriter*> writers_to_close; + { + MutexLock lock(&mutex_); + subscription_ = subscription; + for (auto it = stream_sinks_.cbegin(); it != stream_sinks_.cend();) { + absl::optional<VideoResolution> new_requested_resolution = + subscription_.GetResolutionForPeer(it->second.sender_peer_name); + if (!new_requested_resolution.has_value() || + (*new_requested_resolution != it->second.resolution)) { + RTC_LOG(LS_INFO) << peer_name_ << ": Subscribed resolution for stream " + << it->first << " from " << it->second.sender_peer_name + << " was updated from " + << it->second.resolution.ToString() << " to " + << new_requested_resolution->ToString() + << ". Repopulating all video sinks and recreating " + << "requested video writers"; + writers_to_close.insert(it->second.video_frame_writer); + it = stream_sinks_.erase(it); + } else { + ++it; + } + } + } + sinks_helper_->CloseAndRemoveVideoWriters(writers_to_close); +} + +void AnalyzingVideoSink::OnFrame(const VideoFrame& frame) { + if (IsDummyFrame(frame)) { + // This is dummy frame, so we don't need to process it further. + return; + } + + if (frame.id() == VideoFrame::kNotSetId) { + // If frame ID is unknown we can't get required render resolution, so pass + // to the analyzer in the actual resolution of the frame. + AnalyzeFrame(frame); + } else { + std::string stream_label = analyzer_->GetStreamLabel(frame.id()); + MutexLock lock(&mutex_); + Timestamp processing_started = clock_->CurrentTime(); + SinksDescriptor* sinks_descriptor = PopulateSinks(stream_label); + RTC_CHECK(sinks_descriptor != nullptr); + + VideoFrame scaled_frame = + ScaleVideoFrame(frame, sinks_descriptor->resolution); + AnalyzeFrame(scaled_frame); + for (auto& sink : sinks_descriptor->sinks) { + sink->OnFrame(scaled_frame); + } + Timestamp processing_finished = clock_->CurrentTime(); + + if (report_infra_stats_) { + stats_.analyzing_sink_processing_time_ms.AddSample( + (processing_finished - processing_started).ms<double>()); + } + } +} + +AnalyzingVideoSink::Stats AnalyzingVideoSink::stats() const { + MutexLock lock(&mutex_); + return stats_; +} + +VideoFrame AnalyzingVideoSink::ScaleVideoFrame( + const VideoFrame& frame, + const VideoResolution& required_resolution) { + Timestamp processing_started = clock_->CurrentTime(); + if (required_resolution.width() == static_cast<size_t>(frame.width()) && + required_resolution.height() == static_cast<size_t>(frame.height())) { + if (report_infra_stats_) { + stats_.scaling_tims_ms.AddSample( + (clock_->CurrentTime() - processing_started).ms<double>()); + } + return frame; + } + + // We allow some difference in the aspect ration because when decoder + // downscales video stream it may round up some dimensions to make them even, + // ex: 960x540 -> 480x270 -> 240x136 instead of 240x135. + RTC_CHECK_LE(std::abs(static_cast<double>(required_resolution.width()) / + required_resolution.height() - + static_cast<double>(frame.width()) / frame.height()), + 0.1) + << peer_name_ + << ": Received frame has too different aspect ratio compared to " + << "requested video resolution: required resolution=" + << required_resolution.ToString() + << "; actual resolution=" << frame.width() << "x" << frame.height(); + + rtc::scoped_refptr<I420Buffer> scaled_buffer(I420Buffer::Create( + required_resolution.width(), required_resolution.height())); + scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420()); + + VideoFrame scaled_frame = frame; + scaled_frame.set_video_frame_buffer(scaled_buffer); + if (report_infra_stats_) { + stats_.scaling_tims_ms.AddSample( + (clock_->CurrentTime() - processing_started).ms<double>()); + } + return scaled_frame; +} + +void AnalyzingVideoSink::AnalyzeFrame(const VideoFrame& frame) { + VideoFrame frame_copy = frame; + frame_copy.set_video_frame_buffer( + I420Buffer::Copy(*frame.video_frame_buffer()->ToI420())); + analyzer_->OnFrameRendered(peer_name_, frame_copy); +} + +AnalyzingVideoSink::SinksDescriptor* AnalyzingVideoSink::PopulateSinks( + absl::string_view stream_label) { + // Fast pass: sinks already exists. + auto sinks_it = stream_sinks_.find(std::string(stream_label)); + if (sinks_it != stream_sinks_.end()) { + return &sinks_it->second; + } + + // Slow pass: we need to create and save sinks + absl::optional<std::pair<std::string, VideoConfig>> peer_and_config = + sinks_helper_->GetPeerAndConfig(stream_label); + RTC_CHECK(peer_and_config.has_value()) + << "No video config for stream " << stream_label; + const std::string& sender_peer_name = peer_and_config->first; + const VideoConfig& config = peer_and_config->second; + + absl::optional<VideoResolution> resolution = + subscription_.GetResolutionForPeer(sender_peer_name); + if (!resolution.has_value()) { + RTC_LOG(LS_ERROR) << peer_name_ << " received stream " << stream_label + << " from " << sender_peer_name + << " for which they were not subscribed"; + resolution = config.GetResolution(); + } + if (!resolution->IsRegular()) { + RTC_LOG(LS_ERROR) << peer_name_ << " received stream " << stream_label + << " from " << sender_peer_name + << " for which resolution wasn't resolved"; + resolution = config.GetResolution(); + } + + RTC_CHECK(resolution.has_value()); + + SinksDescriptor sinks_descriptor(sender_peer_name, *resolution); + if (config.output_dump_options.has_value()) { + std::unique_ptr<test::VideoFrameWriter> writer = + config.output_dump_options->CreateOutputDumpVideoFrameWriter( + stream_label, peer_name_, *resolution); + if (config.output_dump_use_fixed_framerate) { + writer = std::make_unique<test::FixedFpsVideoFrameWriterAdapter>( + resolution->fps(), clock_, std::move(writer)); + } + sinks_descriptor.sinks.push_back(std::make_unique<VideoWriter>( + writer.get(), config.output_dump_options->sampling_modulo())); + sinks_descriptor.video_frame_writer = + sinks_helper_->AddVideoWriter(std::move(writer)); + } + if (config.show_on_screen) { + sinks_descriptor.sinks.push_back( + absl::WrapUnique(test::VideoRenderer::Create( + (*config.stream_label + "-render").c_str(), resolution->width(), + resolution->height()))); + } + return &stream_sinks_.emplace(stream_label, std::move(sinks_descriptor)) + .first->second; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.h new file mode 100644 index 0000000000..1834bbe469 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINK_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINK_H_ + +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/video/video_frame_writer.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// A sink to inject video quality analyzer as a sink into WebRTC. +class AnalyzingVideoSink : public rtc::VideoSinkInterface<VideoFrame> { + public: + struct Stats { + // Time required to scale video frame to the requested rendered resolution. + // Collected only for frames with ID set and iff `report_infra_stats` is + // true. + SamplesStatsCounter scaling_tims_ms; + // Time required to process single video frame. Collected only for frames + // with ID set and iff `report_infra_stats` is true. + SamplesStatsCounter analyzing_sink_processing_time_ms; + }; + + AnalyzingVideoSink(absl::string_view peer_name, + Clock* clock, + VideoQualityAnalyzerInterface& analyzer, + AnalyzingVideoSinksHelper& sinks_helper, + const VideoSubscription& subscription, + bool report_infra_stats); + + // Updates subscription used by this peer to render received video. + void UpdateSubscription(const VideoSubscription& subscription); + + void OnFrame(const VideoFrame& frame) override; + + Stats stats() const; + + private: + struct SinksDescriptor { + SinksDescriptor(absl::string_view sender_peer_name, + const VideoResolution& resolution) + : sender_peer_name(sender_peer_name), resolution(resolution) {} + + // Required to be able to resolve resolutions on new subscription and + // understand if we need to recreate `video_frame_writer` and `sinks`. + std::string sender_peer_name; + // Resolution which was used to create `video_frame_writer` and `sinks`. + VideoResolution resolution; + + // Is set if dumping of output video was requested; + test::VideoFrameWriter* video_frame_writer = nullptr; + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> sinks; + }; + + // Scales video frame to `required_resolution` if necessary. Crashes if video + // frame and `required_resolution` have different aspect ratio. + VideoFrame ScaleVideoFrame(const VideoFrame& frame, + const VideoResolution& required_resolution) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + // Creates full copy of the frame to free any frame owned internal buffers + // and passes created copy to analyzer. Uses `I420Buffer` to represent + // frame content. + void AnalyzeFrame(const VideoFrame& frame); + // Populates sink for specified stream and caches them in `stream_sinks_`. + SinksDescriptor* PopulateSinks(absl::string_view stream_label) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + const std::string peer_name_; + const bool report_infra_stats_; + Clock* const clock_; + VideoQualityAnalyzerInterface* const analyzer_; + AnalyzingVideoSinksHelper* const sinks_helper_; + + mutable Mutex mutex_; + VideoSubscription subscription_ RTC_GUARDED_BY(mutex_); + std::map<std::string, SinksDescriptor> stream_sinks_ RTC_GUARDED_BY(mutex_); + Stats stats_ RTC_GUARDED_BY(mutex_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINK_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink_test.cc new file mode 100644 index 0000000000..6cd89551ea --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink_test.cc @@ -0,0 +1,598 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/analyzing_video_sink.h" + +#include <stdio.h> + +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "api/test/create_frame_generator.h" +#include "api/test/frame_generator_interface.h" +#include "api/test/pclf/media_configuration.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/clock.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/pc/e2e/analyzer/video/example_video_quality_analyzer.h" +#include "test/testsupport/file_utils.h" +#include "test/testsupport/frame_reader.h" +#include "test/time_controller/simulated_time_controller.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::testing::ElementsAreArray; +using ::testing::Eq; +using ::testing::Ge; +using ::testing::Test; + +// Remove files and directories in a directory non-recursively. +void CleanDir(absl::string_view dir, size_t expected_output_files_count) { + absl::optional<std::vector<std::string>> dir_content = + test::ReadDirectory(dir); + if (expected_output_files_count == 0) { + ASSERT_TRUE(!dir_content.has_value() || dir_content->empty()) + << "Empty directory is expected"; + } else { + ASSERT_TRUE(dir_content.has_value()) << "Test directory is empty!"; + EXPECT_EQ(dir_content->size(), expected_output_files_count); + for (const auto& entry : *dir_content) { + if (test::DirExists(entry)) { + EXPECT_TRUE(test::RemoveDir(entry)) + << "Failed to remove sub directory: " << entry; + } else if (test::FileExists(entry)) { + EXPECT_TRUE(test::RemoveFile(entry)) + << "Failed to remove file: " << entry; + } else { + FAIL() << "Can't remove unknown file type: " << entry; + } + } + } + EXPECT_TRUE(test::RemoveDir(dir)) << "Failed to remove directory: " << dir; +} + +VideoFrame CreateFrame(test::FrameGeneratorInterface& frame_generator) { + test::FrameGeneratorInterface::VideoFrameData frame_data = + frame_generator.NextFrame(); + return VideoFrame::Builder() + .set_video_frame_buffer(frame_data.buffer) + .set_update_rect(frame_data.update_rect) + .build(); +} + +std::unique_ptr<test::FrameGeneratorInterface> CreateFrameGenerator( + size_t width, + size_t height) { + return test::CreateSquareFrameGenerator(width, height, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); +} + +void AssertFrameIdsAre(const std::string& filename, + std::vector<std::string> expected_ids) { + FILE* file = fopen(filename.c_str(), "r"); + ASSERT_TRUE(file != nullptr) << "Failed to open frame ids file: " << filename; + std::vector<std::string> actual_ids; + char buffer[8]; + while (fgets(buffer, sizeof buffer, file) != nullptr) { + std::string current_id(buffer); + EXPECT_GE(current_id.size(), 2lu) + << "Found invalid frame id: [" << current_id << "]"; + if (current_id.size() < 2) { + continue; + } + // Trim "\n" at the end. + actual_ids.push_back(current_id.substr(0, current_id.size() - 1)); + } + fclose(file); + EXPECT_THAT(actual_ids, ElementsAreArray(expected_ids)); +} + +class AnalyzingVideoSinkTest : public Test { + protected: + ~AnalyzingVideoSinkTest() override = default; + + void SetUp() override { + // Create an empty temporary directory for this test. + test_directory_ = test::JoinFilename( + test::OutputPath(), + "TestDir_AnalyzingVideoSinkTest_" + + std::string( + testing::UnitTest::GetInstance()->current_test_info()->name())); + test::CreateDir(test_directory_); + } + + void TearDown() override { + CleanDir(test_directory_, expected_output_files_count_); + } + + void ExpectOutputFilesCount(size_t count) { + expected_output_files_count_ = count; + } + + std::string test_directory_; + size_t expected_output_files_count_ = 0; +}; + +TEST_F(AnalyzingVideoSinkTest, VideoFramesAreDumpedCorrectly) { + VideoSubscription subscription; + subscription.SubscribeToPeer( + "alice", VideoResolution(/*width=*/640, /*height=*/360, /*fps=*/30)); + VideoConfig video_config("alice_video", /*width=*/1280, /*height=*/720, + /*fps=*/30); + video_config.output_dump_options = VideoDumpOptions(test_directory_); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/1280, /*height=*/720); + VideoFrame frame = CreateFrame(*frame_generator); + frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame)); + + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription, /*report_infra_stats=*/false); + sink.OnFrame(frame); + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(1))); + + auto frame_reader = test::CreateY4mFrameReader( + test::JoinFilename(test_directory_, "alice_video_bob_640x360_30.y4m")); + EXPECT_THAT(frame_reader->num_frames(), Eq(1)); + rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame(); + rtc::scoped_refptr<I420BufferInterface> expected_frame = + frame.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Actual should be downscaled version of expected. + EXPECT_GT(ssim, 0.98); + EXPECT_GT(psnr, 38); + + ExpectOutputFilesCount(1); +} + +TEST_F(AnalyzingVideoSinkTest, + FallbackOnConfigResolutionIfNoSubscriptionProvided) { + VideoSubscription subscription; + VideoConfig video_config("alice_video", /*width=*/320, /*height=*/240, + /*fps=*/30); + video_config.output_dump_options = VideoDumpOptions(test_directory_); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/320, /*height=*/240); + VideoFrame frame = CreateFrame(*frame_generator); + frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame)); + + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription, /*report_infra_stats=*/false); + sink.OnFrame(frame); + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(1))); + + auto frame_reader = test::CreateY4mFrameReader( + test::JoinFilename(test_directory_, "alice_video_bob_320x240_30.y4m")); + EXPECT_THAT(frame_reader->num_frames(), Eq(1)); + rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame(); + rtc::scoped_refptr<I420BufferInterface> expected_frame = + frame.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Frames should be equal. + EXPECT_DOUBLE_EQ(ssim, 1.00); + EXPECT_DOUBLE_EQ(psnr, 48); + + ExpectOutputFilesCount(1); +} + +TEST_F(AnalyzingVideoSinkTest, + FallbackOnConfigResolutionIfNoSubscriptionIsNotResolved) { + VideoSubscription subscription; + subscription.SubscribeToAllPeers( + VideoResolution(VideoResolution::Spec::kMaxFromSender)); + VideoConfig video_config("alice_video", /*width=*/320, /*height=*/240, + /*fps=*/30); + video_config.output_dump_options = VideoDumpOptions(test_directory_); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/320, /*height=*/240); + VideoFrame frame = CreateFrame(*frame_generator); + frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame)); + + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription, /*report_infra_stats=*/false); + sink.OnFrame(frame); + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(1))); + + auto frame_reader = test::CreateY4mFrameReader( + test::JoinFilename(test_directory_, "alice_video_bob_320x240_30.y4m")); + EXPECT_THAT(frame_reader->num_frames(), Eq(1)); + rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame(); + rtc::scoped_refptr<I420BufferInterface> expected_frame = + frame.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Frames should be equal. + EXPECT_DOUBLE_EQ(ssim, 1.00); + EXPECT_DOUBLE_EQ(psnr, 48); + + ExpectOutputFilesCount(1); +} + +TEST_F(AnalyzingVideoSinkTest, + VideoFramesAreDumpedCorrectlyWhenSubscriptionChanged) { + VideoSubscription subscription_before; + subscription_before.SubscribeToPeer( + "alice", VideoResolution(/*width=*/1280, /*height=*/720, /*fps=*/30)); + VideoSubscription subscription_after; + subscription_after.SubscribeToPeer( + "alice", VideoResolution(/*width=*/640, /*height=*/360, /*fps=*/30)); + VideoConfig video_config("alice_video", /*width=*/1280, /*height=*/720, + /*fps=*/30); + video_config.output_dump_options = VideoDumpOptions(test_directory_); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/1280, /*height=*/720); + VideoFrame frame_before = CreateFrame(*frame_generator); + frame_before.set_id( + analyzer.OnFrameCaptured("alice", "alice_video", frame_before)); + VideoFrame frame_after = CreateFrame(*frame_generator); + frame_after.set_id( + analyzer.OnFrameCaptured("alice", "alice_video", frame_after)); + + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription_before, /*report_infra_stats=*/false); + sink.OnFrame(frame_before); + + sink.UpdateSubscription(subscription_after); + sink.OnFrame(frame_after); + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(2))); + + { + auto frame_reader = test::CreateY4mFrameReader( + test::JoinFilename(test_directory_, "alice_video_bob_1280x720_30.y4m")); + EXPECT_THAT(frame_reader->num_frames(), Eq(1)); + rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame(); + rtc::scoped_refptr<I420BufferInterface> expected_frame = + frame_before.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Frames should be equal. + EXPECT_DOUBLE_EQ(ssim, 1.00); + EXPECT_DOUBLE_EQ(psnr, 48); + } + { + auto frame_reader = test::CreateY4mFrameReader( + test::JoinFilename(test_directory_, "alice_video_bob_640x360_30.y4m")); + EXPECT_THAT(frame_reader->num_frames(), Eq(1)); + rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame(); + rtc::scoped_refptr<I420BufferInterface> expected_frame = + frame_after.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Actual should be downscaled version of expected. + EXPECT_GT(ssim, 0.98); + EXPECT_GT(psnr, 38); + } + + ExpectOutputFilesCount(2); +} + +TEST_F(AnalyzingVideoSinkTest, + VideoFramesAreDumpedCorrectlyWhenSubscriptionChangedOnTheSameOne) { + VideoSubscription subscription_before; + subscription_before.SubscribeToPeer( + "alice", VideoResolution(/*width=*/640, /*height=*/360, /*fps=*/30)); + VideoSubscription subscription_after; + subscription_after.SubscribeToPeer( + "alice", VideoResolution(/*width=*/640, /*height=*/360, /*fps=*/30)); + VideoConfig video_config("alice_video", /*width=*/640, /*height=*/360, + /*fps=*/30); + video_config.output_dump_options = VideoDumpOptions(test_directory_); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/640, /*height=*/360); + VideoFrame frame_before = CreateFrame(*frame_generator); + frame_before.set_id( + analyzer.OnFrameCaptured("alice", "alice_video", frame_before)); + VideoFrame frame_after = CreateFrame(*frame_generator); + frame_after.set_id( + analyzer.OnFrameCaptured("alice", "alice_video", frame_after)); + + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription_before, /*report_infra_stats=*/false); + sink.OnFrame(frame_before); + + sink.UpdateSubscription(subscription_after); + sink.OnFrame(frame_after); + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(2))); + + { + auto frame_reader = test::CreateY4mFrameReader( + test::JoinFilename(test_directory_, "alice_video_bob_640x360_30.y4m")); + EXPECT_THAT(frame_reader->num_frames(), Eq(2)); + // Read the first frame. + rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame(); + rtc::scoped_refptr<I420BufferInterface> expected_frame = + frame_before.video_frame_buffer()->ToI420(); + // Frames should be equal. + EXPECT_DOUBLE_EQ(I420SSIM(*expected_frame, *actual_frame), 1.00); + EXPECT_DOUBLE_EQ(I420PSNR(*expected_frame, *actual_frame), 48); + // Read the second frame. + actual_frame = frame_reader->PullFrame(); + expected_frame = frame_after.video_frame_buffer()->ToI420(); + // Frames should be equal. + EXPECT_DOUBLE_EQ(I420SSIM(*expected_frame, *actual_frame), 1.00); + EXPECT_DOUBLE_EQ(I420PSNR(*expected_frame, *actual_frame), 48); + } + + ExpectOutputFilesCount(1); +} + +TEST_F(AnalyzingVideoSinkTest, SmallDiviationsInAspectRationAreAllowed) { + VideoSubscription subscription; + subscription.SubscribeToPeer( + "alice", VideoResolution(/*width=*/480, /*height=*/270, /*fps=*/30)); + VideoConfig video_config("alice_video", /*width=*/480, /*height=*/270, + /*fps=*/30); + video_config.output_dump_options = VideoDumpOptions(test_directory_); + + ExampleVideoQualityAnalyzer analyzer; + // Generator produces downscaled frames with a bit different aspect ration. + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/240, /*height=*/136); + VideoFrame frame = CreateFrame(*frame_generator); + frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame)); + + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription, /*report_infra_stats=*/false); + sink.OnFrame(frame); + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(1))); + + { + auto frame_reader = test::CreateY4mFrameReader( + test::JoinFilename(test_directory_, "alice_video_bob_480x270_30.y4m")); + EXPECT_THAT(frame_reader->num_frames(), Eq(1)); + // Read the first frame. + rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame(); + rtc::scoped_refptr<I420BufferInterface> expected_frame = + frame.video_frame_buffer()->ToI420(); + // Actual frame is upscaled version of the expected. But because rendered + // resolution is equal to the actual frame size we need to upscale expected + // during comparison and then they have to be the same. + EXPECT_DOUBLE_EQ(I420SSIM(*actual_frame, *expected_frame), 1); + EXPECT_DOUBLE_EQ(I420PSNR(*actual_frame, *expected_frame), 48); + } + + ExpectOutputFilesCount(1); +} + +TEST_F(AnalyzingVideoSinkTest, VideoFramesIdsAreDumpedWhenRequested) { + VideoSubscription subscription; + subscription.SubscribeToPeer( + "alice", VideoResolution(/*width=*/320, /*height=*/240, /*fps=*/30)); + VideoConfig video_config("alice_video", /*width=*/320, /*height=*/240, + /*fps=*/30); + video_config.output_dump_options = + VideoDumpOptions(test_directory_, /*export_frame_ids=*/true); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/320, /*height=*/240); + + std::vector<std::string> expected_frame_ids; + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription, /*report_infra_stats=*/false); + for (int i = 0; i < 10; ++i) { + VideoFrame frame = CreateFrame(*frame_generator); + frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame)); + expected_frame_ids.push_back(std::to_string(frame.id())); + sink.OnFrame(frame); + } + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(10))); + + AssertFrameIdsAre( + test::JoinFilename(test_directory_, + "alice_video_bob_320x240_30.frame_ids.txt"), + expected_frame_ids); + + ExpectOutputFilesCount(2); +} + +TEST_F(AnalyzingVideoSinkTest, + VideoFramesAndIdsAreDumpedWithFixedFpsWhenRequested) { + GlobalSimulatedTimeController simulated_time(Timestamp::Seconds(100000)); + + VideoSubscription subscription; + subscription.SubscribeToPeer( + "alice", VideoResolution(/*width=*/320, /*height=*/240, /*fps=*/10)); + VideoConfig video_config("alice_video", /*width=*/320, /*height=*/240, + /*fps=*/10); + video_config.output_dump_options = + VideoDumpOptions(test_directory_, /*export_frame_ids=*/true); + video_config.output_dump_use_fixed_framerate = true; + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/320, /*height=*/240); + + VideoFrame frame1 = CreateFrame(*frame_generator); + frame1.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame1)); + VideoFrame frame2 = CreateFrame(*frame_generator); + frame2.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame2)); + + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", simulated_time.GetClock(), analyzer, helper, + subscription, /*report_infra_stats=*/false); + sink.OnFrame(frame1); + // Advance almost 1 second, so the first frame has to be repeated 9 time + // more. + simulated_time.AdvanceTime(TimeDelta::Millis(990)); + sink.OnFrame(frame2); + simulated_time.AdvanceTime(TimeDelta::Millis(100)); + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(2))); + + auto frame_reader = test::CreateY4mFrameReader( + test::JoinFilename(test_directory_, "alice_video_bob_320x240_10.y4m")); + EXPECT_THAT(frame_reader->num_frames(), Eq(11)); + for (int i = 0; i < 10; ++i) { + rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame(); + rtc::scoped_refptr<I420BufferInterface> expected_frame = + frame1.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Frames should be equal. + EXPECT_DOUBLE_EQ(ssim, 1.00); + EXPECT_DOUBLE_EQ(psnr, 48); + } + rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame(); + rtc::scoped_refptr<I420BufferInterface> expected_frame = + frame2.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Frames should be equal. + EXPECT_DOUBLE_EQ(ssim, 1.00); + EXPECT_DOUBLE_EQ(psnr, 48); + + AssertFrameIdsAre( + test::JoinFilename(test_directory_, + "alice_video_bob_320x240_10.frame_ids.txt"), + {std::to_string(frame1.id()), std::to_string(frame1.id()), + std::to_string(frame1.id()), std::to_string(frame1.id()), + std::to_string(frame1.id()), std::to_string(frame1.id()), + std::to_string(frame1.id()), std::to_string(frame1.id()), + std::to_string(frame1.id()), std::to_string(frame1.id()), + std::to_string(frame2.id())}); + + ExpectOutputFilesCount(2); +} + +TEST_F(AnalyzingVideoSinkTest, InfraMetricsCollectedWhenRequested) { + VideoSubscription subscription; + subscription.SubscribeToPeer( + "alice", VideoResolution(/*width=*/1280, /*height=*/720, /*fps=*/30)); + VideoConfig video_config("alice_video", /*width=*/640, /*height=*/360, + /*fps=*/30); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/640, /*height=*/360); + VideoFrame frame = CreateFrame(*frame_generator); + frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame)); + + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription, /*report_infra_stats=*/true); + sink.OnFrame(frame); + + AnalyzingVideoSink::Stats stats = sink.stats(); + EXPECT_THAT(stats.scaling_tims_ms.NumSamples(), Eq(1)); + EXPECT_THAT(stats.scaling_tims_ms.GetAverage(), Ge(0)); + EXPECT_THAT(stats.analyzing_sink_processing_time_ms.NumSamples(), Eq(1)); + EXPECT_THAT(stats.analyzing_sink_processing_time_ms.GetAverage(), + Ge(stats.scaling_tims_ms.GetAverage())); + + ExpectOutputFilesCount(0); +} + +TEST_F(AnalyzingVideoSinkTest, InfraMetricsNotCollectedWhenNotRequested) { + VideoSubscription subscription; + subscription.SubscribeToPeer( + "alice", VideoResolution(/*width=*/1280, /*height=*/720, /*fps=*/30)); + VideoConfig video_config("alice_video", /*width=*/640, /*height=*/360, + /*fps=*/30); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/640, /*height=*/360); + VideoFrame frame = CreateFrame(*frame_generator); + frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame)); + + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription, /*report_infra_stats=*/false); + sink.OnFrame(frame); + + AnalyzingVideoSink::Stats stats = sink.stats(); + EXPECT_THAT(stats.scaling_tims_ms.NumSamples(), Eq(0)); + EXPECT_THAT(stats.analyzing_sink_processing_time_ms.NumSamples(), Eq(0)); + + ExpectOutputFilesCount(0); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.cc new file mode 100644 index 0000000000..70dc4b00b5 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.cc @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h" + +#include <memory> +#include <set> +#include <string> +#include <utility> + +#include "absl/strings/string_view.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/video/video_frame_writer.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +void AnalyzingVideoSinksHelper::AddConfig(absl::string_view sender_peer_name, + VideoConfig config) { + MutexLock lock(&mutex_); + auto it = video_configs_.find(*config.stream_label); + if (it == video_configs_.end()) { + std::string stream_label = *config.stream_label; + video_configs_.emplace( + std::move(stream_label), + std::pair{std::string(sender_peer_name), std::move(config)}); + } else { + it->second = std::pair{std::string(sender_peer_name), std::move(config)}; + } +} + +absl::optional<std::pair<std::string, VideoConfig>> +AnalyzingVideoSinksHelper::GetPeerAndConfig(absl::string_view stream_label) { + MutexLock lock(&mutex_); + auto it = video_configs_.find(std::string(stream_label)); + if (it == video_configs_.end()) { + return absl::nullopt; + } + return it->second; +} + +void AnalyzingVideoSinksHelper::RemoveConfig(absl::string_view stream_label) { + MutexLock lock(&mutex_); + video_configs_.erase(std::string(stream_label)); +} + +test::VideoFrameWriter* AnalyzingVideoSinksHelper::AddVideoWriter( + std::unique_ptr<test::VideoFrameWriter> video_writer) { + MutexLock lock(&mutex_); + test::VideoFrameWriter* out = video_writer.get(); + video_writers_.push_back(std::move(video_writer)); + return out; +} + +void AnalyzingVideoSinksHelper::CloseAndRemoveVideoWriters( + std::set<test::VideoFrameWriter*> writers_to_close) { + MutexLock lock(&mutex_); + for (auto it = video_writers_.cbegin(); it != video_writers_.cend();) { + if (writers_to_close.find(it->get()) != writers_to_close.end()) { + (*it)->Close(); + it = video_writers_.erase(it); + } else { + ++it; + } + } +} + +void AnalyzingVideoSinksHelper::Clear() { + MutexLock lock(&mutex_); + video_configs_.clear(); + for (const auto& video_writer : video_writers_) { + video_writer->Close(); + } + video_writers_.clear(); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h new file mode 100644 index 0000000000..5f38c5a40e --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINKS_HELPER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINKS_HELPER_H_ + +#include <list> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <utility> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/video/video_frame_writer.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Registry of known video configs and video writers. +// This class is thread safe. +class AnalyzingVideoSinksHelper { + public: + // Adds config in the registry. If config with such stream label was + // registered before, the new value will override the old one. + void AddConfig(absl::string_view sender_peer_name, VideoConfig config); + absl::optional<std::pair<std::string, VideoConfig>> GetPeerAndConfig( + absl::string_view stream_label); + // Removes video config for specified stream label. If there are no know video + // config for such stream label - does nothing. + void RemoveConfig(absl::string_view stream_label); + + // Takes ownership of the provided video writer. All video writers owned by + // this class will be closed during `AnalyzingVideoSinksHelper` destruction + // and guaranteed to be alive either until explicitly removed by + // `CloseAndRemoveVideoWriters` or until `AnalyzingVideoSinksHelper` is + // destroyed. + // + // Returns pointer to the added writer. Ownership is maintained by + // `AnalyzingVideoSinksHelper`. + test::VideoFrameWriter* AddVideoWriter( + std::unique_ptr<test::VideoFrameWriter> video_writer); + // For each provided `writers_to_close`, if it is known, will close and + // destroy it, otherwise does nothing with it. + void CloseAndRemoveVideoWriters( + std::set<test::VideoFrameWriter*> writers_to_close); + + // Removes all added configs and close and removes all added writers. + void Clear(); + + private: + Mutex mutex_; + std::map<std::string, std::pair<std::string, VideoConfig>> video_configs_ + RTC_GUARDED_BY(mutex_); + std::list<std::unique_ptr<test::VideoFrameWriter>> video_writers_ + RTC_GUARDED_BY(mutex_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINKS_HELPER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper_test.cc new file mode 100644 index 0000000000..1a820a5229 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper_test.cc @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h" + +#include <memory> +#include <string> +#include <utility> + +#include "absl/types/optional.h" +#include "api/test/pclf/media_configuration.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::testing::Eq; + +// Asserts equality of the main fields of the video config. We don't compare +// the full config due to the lack of equality definition for a lot of subtypes. +void AssertConfigsAreEquals(const VideoConfig& actual, + const VideoConfig& expected) { + EXPECT_THAT(actual.stream_label, Eq(expected.stream_label)); + EXPECT_THAT(actual.width, Eq(expected.width)); + EXPECT_THAT(actual.height, Eq(expected.height)); + EXPECT_THAT(actual.fps, Eq(expected.fps)); +} + +TEST(AnalyzingVideoSinksHelperTest, ConfigsCanBeAdded) { + VideoConfig config("alice_video", /*width=*/1280, /*height=*/720, /*fps=*/30); + + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", config); + + absl::optional<std::pair<std::string, VideoConfig>> registred_config = + helper.GetPeerAndConfig("alice_video"); + ASSERT_TRUE(registred_config.has_value()); + EXPECT_THAT(registred_config->first, Eq("alice")); + AssertConfigsAreEquals(registred_config->second, config); +} + +TEST(AnalyzingVideoSinksHelperTest, AddingForExistingLabelWillOverwriteValue) { + VideoConfig config_before("alice_video", /*width=*/1280, /*height=*/720, + /*fps=*/30); + VideoConfig config_after("alice_video", /*width=*/640, /*height=*/360, + /*fps=*/15); + + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", config_before); + + absl::optional<std::pair<std::string, VideoConfig>> registred_config = + helper.GetPeerAndConfig("alice_video"); + ASSERT_TRUE(registred_config.has_value()); + EXPECT_THAT(registred_config->first, Eq("alice")); + AssertConfigsAreEquals(registred_config->second, config_before); + + helper.AddConfig("alice", config_after); + + registred_config = helper.GetPeerAndConfig("alice_video"); + ASSERT_TRUE(registred_config.has_value()); + EXPECT_THAT(registred_config->first, Eq("alice")); + AssertConfigsAreEquals(registred_config->second, config_after); +} + +TEST(AnalyzingVideoSinksHelperTest, ConfigsCanBeRemoved) { + VideoConfig config("alice_video", /*width=*/1280, /*height=*/720, /*fps=*/30); + + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", config); + + ASSERT_TRUE(helper.GetPeerAndConfig("alice_video").has_value()); + + helper.RemoveConfig("alice_video"); + ASSERT_FALSE(helper.GetPeerAndConfig("alice_video").has_value()); +} + +TEST(AnalyzingVideoSinksHelperTest, RemoveOfNonExistingConfigDontCrash) { + AnalyzingVideoSinksHelper helper; + helper.RemoveConfig("alice_video"); +} + +TEST(AnalyzingVideoSinksHelperTest, ClearRemovesAllConfigs) { + VideoConfig config1("alice_video", /*width=*/640, /*height=*/360, /*fps=*/30); + VideoConfig config2("bob_video", /*width=*/640, /*height=*/360, /*fps=*/30); + + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", config1); + helper.AddConfig("bob", config2); + + ASSERT_TRUE(helper.GetPeerAndConfig("alice_video").has_value()); + ASSERT_TRUE(helper.GetPeerAndConfig("bob_video").has_value()); + + helper.Clear(); + ASSERT_FALSE(helper.GetPeerAndConfig("alice_video").has_value()); + ASSERT_FALSE(helper.GetPeerAndConfig("bob_video").has_value()); +} + +struct TestVideoFrameWriterFactory { + int closed_writers_count = 0; + int deleted_writers_count = 0; + + std::unique_ptr<test::VideoFrameWriter> CreateWriter() { + return std::make_unique<TestVideoFrameWriter>(this); + } + + private: + class TestVideoFrameWriter : public test::VideoFrameWriter { + public: + explicit TestVideoFrameWriter(TestVideoFrameWriterFactory* factory) + : factory_(factory) {} + ~TestVideoFrameWriter() override { factory_->deleted_writers_count++; } + + bool WriteFrame(const VideoFrame& frame) override { return true; } + + void Close() override { factory_->closed_writers_count++; } + + private: + TestVideoFrameWriterFactory* factory_; + }; +}; + +TEST(AnalyzingVideoSinksHelperTest, RemovingWritersCloseAndDestroyAllOfThem) { + TestVideoFrameWriterFactory factory; + + AnalyzingVideoSinksHelper helper; + test::VideoFrameWriter* writer1 = + helper.AddVideoWriter(factory.CreateWriter()); + test::VideoFrameWriter* writer2 = + helper.AddVideoWriter(factory.CreateWriter()); + + helper.CloseAndRemoveVideoWriters({writer1, writer2}); + + EXPECT_THAT(factory.closed_writers_count, Eq(2)); + EXPECT_THAT(factory.deleted_writers_count, Eq(2)); +} + +TEST(AnalyzingVideoSinksHelperTest, ClearCloseAndDestroyAllWriters) { + TestVideoFrameWriterFactory factory; + + AnalyzingVideoSinksHelper helper; + helper.AddVideoWriter(factory.CreateWriter()); + helper.AddVideoWriter(factory.CreateWriter()); + + helper.Clear(); + + EXPECT_THAT(factory.closed_writers_count, Eq(2)); + EXPECT_THAT(factory.deleted_writers_count, Eq(2)); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc new file mode 100644 index 0000000000..59144589fc --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -0,0 +1,1228 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" + +#include <algorithm> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <utility> +#include <vector> + +#include "api/array_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metric.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "api/video/video_frame.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/strings/string_builder.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h" +#include "test/pc/e2e/metric_metadata_keys.h" + +namespace webrtc { +namespace { + +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Unit; +using ::webrtc::webrtc_pc_e2e::MetricMetadataKey; + +constexpr int kBitsInByte = 8; +constexpr absl::string_view kSkipRenderedFrameReasonProcessed = "processed"; +constexpr absl::string_view kSkipRenderedFrameReasonRendered = "rendered"; +constexpr absl::string_view kSkipRenderedFrameReasonDropped = + "considered dropped"; + +void LogFrameCounters(const std::string& name, const FrameCounters& counters) { + RTC_LOG(LS_INFO) << "[" << name + << "] Captured : " << counters.captured; + RTC_LOG(LS_INFO) << "[" << name + << "] Pre encoded : " << counters.pre_encoded; + RTC_LOG(LS_INFO) << "[" << name + << "] Encoded : " << counters.encoded; + RTC_LOG(LS_INFO) << "[" << name + << "] Received : " << counters.received; + RTC_LOG(LS_INFO) << "[" << name + << "] Decoded : " << counters.decoded; + RTC_LOG(LS_INFO) << "[" << name + << "] Rendered : " << counters.rendered; + RTC_LOG(LS_INFO) << "[" << name + << "] Dropped : " << counters.dropped; + RTC_LOG(LS_INFO) << "[" << name + << "] Failed to decode : " << counters.failed_to_decode; +} + +void LogStreamInternalStats(const std::string& name, + const StreamStats& stats, + Timestamp start_time) { + for (const auto& entry : stats.dropped_by_phase) { + RTC_LOG(LS_INFO) << "[" << name << "] Dropped at " << ToString(entry.first) + << ": " << entry.second; + } + Timestamp first_encoded_frame_time = Timestamp::PlusInfinity(); + for (const StreamCodecInfo& encoder : stats.encoders) { + RTC_DCHECK(encoder.switched_on_at.IsFinite()); + RTC_DCHECK(encoder.switched_from_at.IsFinite()); + if (first_encoded_frame_time.IsInfinite()) { + first_encoded_frame_time = encoder.switched_on_at; + } + RTC_LOG(LS_INFO) + << "[" << name << "] Used encoder: \"" << encoder.codec_name + << "\" used from (frame_id=" << encoder.first_frame_id + << "; from_stream_start=" + << (encoder.switched_on_at - stats.stream_started_time).ms() + << "ms, from_call_start=" << (encoder.switched_on_at - start_time).ms() + << "ms) until (frame_id=" << encoder.last_frame_id + << "; from_stream_start=" + << (encoder.switched_from_at - stats.stream_started_time).ms() + << "ms, from_call_start=" + << (encoder.switched_from_at - start_time).ms() << "ms)"; + } + for (const StreamCodecInfo& decoder : stats.decoders) { + RTC_DCHECK(decoder.switched_on_at.IsFinite()); + RTC_DCHECK(decoder.switched_from_at.IsFinite()); + RTC_LOG(LS_INFO) + << "[" << name << "] Used decoder: \"" << decoder.codec_name + << "\" used from (frame_id=" << decoder.first_frame_id + << "; from_stream_start=" + << (decoder.switched_on_at - stats.stream_started_time).ms() + << "ms, from_call_start=" << (decoder.switched_on_at - start_time).ms() + << "ms) until (frame_id=" << decoder.last_frame_id + << "; from_stream_start=" + << (decoder.switched_from_at - stats.stream_started_time).ms() + << "ms, from_call_start=" + << (decoder.switched_from_at - start_time).ms() << "ms)"; + } +} + +template <typename T> +absl::optional<T> MaybeGetValue(const std::map<size_t, T>& map, size_t key) { + auto it = map.find(key); + if (it == map.end()) { + return absl::nullopt; + } + return it->second; +} + +SamplesStatsCounter::StatsSample StatsSample(double value, + Timestamp sampling_time) { + return SamplesStatsCounter::StatsSample{value, sampling_time}; +} + +} // namespace + +DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer( + webrtc::Clock* clock, + test::MetricsLogger* metrics_logger, + DefaultVideoQualityAnalyzerOptions options) + : options_(options), + clock_(clock), + metrics_logger_(metrics_logger), + frames_comparator_(clock, cpu_measurer_, options) { + RTC_CHECK(metrics_logger_); +} + +DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() { + Stop(); +} + +void DefaultVideoQualityAnalyzer::Start( + std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count) { + test_label_ = std::move(test_case_name); + frames_comparator_.Start(max_threads_count); + { + MutexLock lock(&mutex_); + peers_ = std::make_unique<NamesCollection>(peer_names); + RTC_CHECK(start_time_.IsMinusInfinity()); + + RTC_CHECK_EQ(state_, State::kNew) + << "DefaultVideoQualityAnalyzer is already started"; + state_ = State::kActive; + start_time_ = Now(); + } +} + +uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured( + absl::string_view peer_name, + const std::string& stream_label, + const webrtc::VideoFrame& frame) { + // `next_frame_id` is atomic, so we needn't lock here. + Timestamp captured_time = Now(); + Timestamp start_time = Timestamp::MinusInfinity(); + size_t peer_index = -1; + size_t peers_count = -1; + size_t stream_index; + uint16_t frame_id = VideoFrame::kNotSetId; + { + MutexLock lock(&mutex_); + frame_id = GetNextFrameId(); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + // Create a local copy of `start_time_`, peer's index and total peers count + // to access it without holding a `mutex_` during access to + // `frames_comparator_`. + start_time = start_time_; + peer_index = peers_->index(peer_name); + peers_count = peers_->size(); + stream_index = streams_.AddIfAbsent(stream_label); + } + // Ensure stats for this stream exists. + frames_comparator_.EnsureStatsForStream(stream_index, peer_index, peers_count, + captured_time, start_time); + { + MutexLock lock(&mutex_); + stream_to_sender_[stream_index] = peer_index; + frame_counters_.captured++; + for (size_t i : peers_->GetAllIndexes()) { + if (i != peer_index || options_.enable_receive_own_stream) { + InternalStatsKey key(stream_index, peer_index, i); + stream_frame_counters_[key].captured++; + } + } + + std::set<size_t> frame_receivers_indexes = peers_->GetPresentIndexes(); + if (!options_.enable_receive_own_stream) { + frame_receivers_indexes.erase(peer_index); + } + + auto state_it = stream_states_.find(stream_index); + if (state_it == stream_states_.end()) { + stream_states_.emplace( + stream_index, + StreamState(peer_index, frame_receivers_indexes, captured_time)); + } + StreamState* state = &stream_states_.at(stream_index); + state->PushBack(frame_id); + // Update frames in flight info. + auto it = captured_frames_in_flight_.find(frame_id); + if (it != captured_frames_in_flight_.end()) { + // If we overflow uint16_t and hit previous frame id and this frame is + // still in flight, it means that this stream wasn't rendered for long + // time and we need to process existing frame as dropped. + for (size_t i : peers_->GetPresentIndexes()) { + if (i == peer_index && !options_.enable_receive_own_stream) { + continue; + } + + uint16_t oldest_frame_id = state->PopFront(i); + RTC_DCHECK_EQ(frame_id, oldest_frame_id); + frame_counters_.dropped++; + InternalStatsKey key(stream_index, peer_index, i); + stream_frame_counters_.at(key).dropped++; + + analyzer_stats_.frames_in_flight_left_count.AddSample( + StatsSample(captured_frames_in_flight_.size(), Now())); + frames_comparator_.AddComparison( + InternalStatsKey(stream_index, peer_index, i), + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, FrameComparisonType::kDroppedFrame, + it->second.GetStatsForPeer(i)); + } + + captured_frames_in_flight_.erase(it); + } + captured_frames_in_flight_.emplace( + frame_id, FrameInFlight(stream_index, frame, captured_time, + std::move(frame_receivers_indexes))); + // Set frame id on local copy of the frame + captured_frames_in_flight_.at(frame_id).SetFrameId(frame_id); + + // Update history stream<->frame mapping + for (auto it = stream_to_frame_id_history_.begin(); + it != stream_to_frame_id_history_.end(); ++it) { + it->second.erase(frame_id); + } + stream_to_frame_id_history_[stream_index].insert(frame_id); + stream_to_frame_id_full_history_[stream_index].push_back(frame_id); + + // If state has too many frames that are in flight => remove the oldest + // queued frame in order to avoid to use too much memory. + if (state->GetAliveFramesCount() > + options_.max_frames_in_flight_per_stream_count) { + uint16_t frame_id_to_remove = state->MarkNextAliveFrameAsDead(); + auto it = captured_frames_in_flight_.find(frame_id_to_remove); + RTC_CHECK(it != captured_frames_in_flight_.end()) + << "Frame with ID " << frame_id_to_remove + << " is expected to be in flight, but hasn't been found in " + << "|captured_frames_in_flight_|"; + bool is_removed = it->second.RemoveFrame(); + RTC_DCHECK(is_removed) + << "Invalid stream state: alive frame is removed already"; + } + if (options_.report_infra_metrics) { + analyzer_stats_.on_frame_captured_processing_time_ms.AddSample( + (Now() - captured_time).ms<double>()); + } + } + return frame_id; +} + +void DefaultVideoQualityAnalyzer::OnFramePreEncode( + absl::string_view peer_name, + const webrtc::VideoFrame& frame) { + Timestamp processing_started = Now(); + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + auto it = captured_frames_in_flight_.find(frame.id()); + RTC_CHECK(it != captured_frames_in_flight_.end()) + << "Frame id=" << frame.id() << " not found"; + FrameInFlight& frame_in_flight = it->second; + frame_counters_.pre_encoded++; + size_t peer_index = peers_->index(peer_name); + for (size_t i : peers_->GetAllIndexes()) { + if (i != peer_index || options_.enable_receive_own_stream) { + InternalStatsKey key(frame_in_flight.stream(), peer_index, i); + stream_frame_counters_.at(key).pre_encoded++; + } + } + frame_in_flight.SetPreEncodeTime(Now()); + + if (options_.report_infra_metrics) { + analyzer_stats_.on_frame_pre_encode_processing_time_ms.AddSample( + (Now() - processing_started).ms<double>()); + } +} + +void DefaultVideoQualityAnalyzer::OnFrameEncoded( + absl::string_view peer_name, + uint16_t frame_id, + const webrtc::EncodedImage& encoded_image, + const EncoderStats& stats, + bool discarded) { + if (discarded) + return; + + Timestamp processing_started = Now(); + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + auto it = captured_frames_in_flight_.find(frame_id); + if (it == captured_frames_in_flight_.end()) { + RTC_LOG(LS_WARNING) + << "The encoding of video frame with id [" << frame_id << "] for peer [" + << peer_name << "] finished after all receivers rendered this frame or " + << "were removed. It can be OK for simulcast/SVC if higher quality " + << "stream is not required or the last receiver was unregistered " + << "between encoding of different layers, but it may indicate an ERROR " + << "for singlecast or if it happens often."; + return; + } + FrameInFlight& frame_in_flight = it->second; + // For SVC we can receive multiple encoded images for one frame, so to cover + // all cases we have to pick the last encode time. + if (!frame_in_flight.HasEncodedTime()) { + // Increase counters only when we meet this frame first time. + frame_counters_.encoded++; + size_t peer_index = peers_->index(peer_name); + for (size_t i : peers_->GetAllIndexes()) { + if (i != peer_index || options_.enable_receive_own_stream) { + InternalStatsKey key(frame_in_flight.stream(), peer_index, i); + stream_frame_counters_.at(key).encoded++; + } + } + } + Timestamp now = Now(); + StreamCodecInfo used_encoder; + used_encoder.codec_name = stats.encoder_name; + used_encoder.first_frame_id = frame_id; + used_encoder.last_frame_id = frame_id; + used_encoder.switched_on_at = now; + used_encoder.switched_from_at = now; + frame_in_flight.OnFrameEncoded( + now, encoded_image._frameType, DataSize::Bytes(encoded_image.size()), + stats.target_encode_bitrate, encoded_image.SpatialIndex().value_or(0), + stats.qp, used_encoder); + + if (options_.report_infra_metrics) { + analyzer_stats_.on_frame_encoded_processing_time_ms.AddSample( + (Now() - processing_started).ms<double>()); + } +} + +void DefaultVideoQualityAnalyzer::OnFrameDropped( + absl::string_view peer_name, + webrtc::EncodedImageCallback::DropReason reason) { + // Here we do nothing, because we will see this drop on renderer side. +} + +void DefaultVideoQualityAnalyzer::OnFramePreDecode( + absl::string_view peer_name, + uint16_t frame_id, + const webrtc::EncodedImage& input_image) { + Timestamp processing_started = Now(); + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + size_t peer_index = peers_->index(peer_name); + + if (frame_id == VideoFrame::kNotSetId) { + frame_counters_.received++; + unknown_sender_frame_counters_[std::string(peer_name)].received++; + return; + } + + auto it = captured_frames_in_flight_.find(frame_id); + if (it == captured_frames_in_flight_.end() || + it->second.HasReceivedTime(peer_index)) { + // It means this frame was predecoded before, so we can skip it. It may + // happen when we have multiple simulcast streams in one track and received + // the same picture from two different streams because SFU can't reliably + // correlate two simulcast streams and started relaying the second stream + // from the same frame it has relayed right before for the first stream. + return; + } + + frame_counters_.received++; + InternalStatsKey key(it->second.stream(), + stream_to_sender_.at(it->second.stream()), peer_index); + stream_frame_counters_.at(key).received++; + // Determine the time of the last received packet of this video frame. + RTC_DCHECK(!input_image.PacketInfos().empty()); + Timestamp last_receive_time = + std::max_element(input_image.PacketInfos().cbegin(), + input_image.PacketInfos().cend(), + [](const RtpPacketInfo& a, const RtpPacketInfo& b) { + return a.receive_time() < b.receive_time(); + }) + ->receive_time(); + it->second.OnFramePreDecode(peer_index, + /*received_time=*/last_receive_time, + /*decode_start_time=*/Now(), + input_image._frameType, + DataSize::Bytes(input_image.size())); + + if (options_.report_infra_metrics) { + analyzer_stats_.on_frame_pre_decode_processing_time_ms.AddSample( + (Now() - processing_started).ms<double>()); + } +} + +void DefaultVideoQualityAnalyzer::OnFrameDecoded( + absl::string_view peer_name, + const webrtc::VideoFrame& frame, + const DecoderStats& stats) { + Timestamp processing_started = Now(); + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + size_t peer_index = peers_->index(peer_name); + + if (frame.id() == VideoFrame::kNotSetId) { + frame_counters_.decoded++; + unknown_sender_frame_counters_[std::string(peer_name)].decoded++; + return; + } + + auto it = captured_frames_in_flight_.find(frame.id()); + if (it == captured_frames_in_flight_.end() || + it->second.HasDecodeEndTime(peer_index)) { + // It means this frame was decoded before, so we can skip it. It may happen + // when we have multiple simulcast streams in one track and received + // the same frame from two different streams because SFU can't reliably + // correlate two simulcast streams and started relaying the second stream + // from the same frame it has relayed right before for the first stream. + return; + } + frame_counters_.decoded++; + InternalStatsKey key(it->second.stream(), + stream_to_sender_.at(it->second.stream()), peer_index); + stream_frame_counters_.at(key).decoded++; + Timestamp now = Now(); + StreamCodecInfo used_decoder; + used_decoder.codec_name = stats.decoder_name; + used_decoder.first_frame_id = frame.id(); + used_decoder.last_frame_id = frame.id(); + used_decoder.switched_on_at = now; + used_decoder.switched_from_at = now; + it->second.OnFrameDecoded(peer_index, now, frame.width(), frame.height(), + used_decoder); + + if (options_.report_infra_metrics) { + analyzer_stats_.on_frame_decoded_processing_time_ms.AddSample( + (Now() - processing_started).ms<double>()); + } +} + +void DefaultVideoQualityAnalyzer::OnFrameRendered( + absl::string_view peer_name, + const webrtc::VideoFrame& frame) { + Timestamp processing_started = Now(); + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + size_t peer_index = peers_->index(peer_name); + + if (frame.id() == VideoFrame::kNotSetId) { + frame_counters_.rendered++; + unknown_sender_frame_counters_[std::string(peer_name)].rendered++; + return; + } + + auto frame_it = captured_frames_in_flight_.find(frame.id()); + if (frame_it == captured_frames_in_flight_.end() || + frame_it->second.HasRenderedTime(peer_index) || + frame_it->second.IsDropped(peer_index)) { + // It means this frame was rendered or dropped before, so we can skip it. + // It may happen when we have multiple simulcast streams in one track and + // received the same frame from two different streams because SFU can't + // reliably correlate two simulcast streams and started relaying the second + // stream from the same frame it has relayed right before for the first + // stream. + absl::string_view reason = kSkipRenderedFrameReasonProcessed; + if (frame_it != captured_frames_in_flight_.end()) { + if (frame_it->second.HasRenderedTime(peer_index)) { + reason = kSkipRenderedFrameReasonRendered; + } else if (frame_it->second.IsDropped(peer_index)) { + reason = kSkipRenderedFrameReasonDropped; + } + } + RTC_LOG(LS_WARNING) + << "Peer " << peer_name + << "; Received frame out of order: received frame with id " + << frame.id() << " which was " << reason << " before"; + return; + } + + // Find corresponding captured frame. + FrameInFlight* frame_in_flight = &frame_it->second; + absl::optional<VideoFrame> captured_frame = frame_in_flight->frame(); + + const size_t stream_index = frame_in_flight->stream(); + StreamState* state = &stream_states_.at(stream_index); + const InternalStatsKey stats_key(stream_index, state->sender(), peer_index); + + // Update frames counters. + frame_counters_.rendered++; + stream_frame_counters_.at(stats_key).rendered++; + + // Update current frame stats. + frame_in_flight->OnFrameRendered(peer_index, Now()); + + // After we received frame here we need to check if there are any dropped + // frames between this one and last one, that was rendered for this video + // stream. + int dropped_count = 0; + while (!state->IsEmpty(peer_index) && + state->Front(peer_index) != frame.id()) { + dropped_count++; + uint16_t dropped_frame_id = state->PopFront(peer_index); + // Frame with id `dropped_frame_id` was dropped. We need: + // 1. Update global and stream frame counters + // 2. Extract corresponding frame from `captured_frames_in_flight_` + // 3. Send extracted frame to comparison with dropped=true + // 4. Cleanup dropped frame + frame_counters_.dropped++; + stream_frame_counters_.at(stats_key).dropped++; + + auto dropped_frame_it = captured_frames_in_flight_.find(dropped_frame_id); + RTC_DCHECK(dropped_frame_it != captured_frames_in_flight_.end()); + dropped_frame_it->second.MarkDropped(peer_index); + + analyzer_stats_.frames_in_flight_left_count.AddSample( + StatsSample(captured_frames_in_flight_.size(), Now())); + frames_comparator_.AddComparison( + stats_key, /*captured=*/absl::nullopt, /*rendered=*/absl::nullopt, + FrameComparisonType::kDroppedFrame, + dropped_frame_it->second.GetStatsForPeer(peer_index)); + + if (dropped_frame_it->second.HaveAllPeersReceived()) { + captured_frames_in_flight_.erase(dropped_frame_it); + } + } + RTC_DCHECK(!state->IsEmpty(peer_index)); + state->PopFront(peer_index); + + if (state->last_rendered_frame_time(peer_index)) { + frame_in_flight->SetPrevFrameRenderedTime( + peer_index, state->last_rendered_frame_time(peer_index).value()); + } + state->SetLastRenderedFrameTime(peer_index, + frame_in_flight->rendered_time(peer_index)); + analyzer_stats_.frames_in_flight_left_count.AddSample( + StatsSample(captured_frames_in_flight_.size(), Now())); + frames_comparator_.AddComparison( + stats_key, dropped_count, captured_frame, /*rendered=*/frame, + FrameComparisonType::kRegular, + frame_in_flight->GetStatsForPeer(peer_index)); + + if (frame_it->second.HaveAllPeersReceived()) { + captured_frames_in_flight_.erase(frame_it); + } + + if (options_.report_infra_metrics) { + analyzer_stats_.on_frame_rendered_processing_time_ms.AddSample( + (Now() - processing_started).ms<double>()); + } +} + +void DefaultVideoQualityAnalyzer::OnEncoderError( + absl::string_view peer_name, + const webrtc::VideoFrame& frame, + int32_t error_code) { + RTC_LOG(LS_ERROR) << "Encoder error for frame.id=" << frame.id() + << ", code=" << error_code; +} + +void DefaultVideoQualityAnalyzer::OnDecoderError(absl::string_view peer_name, + uint16_t frame_id, + int32_t error_code, + const DecoderStats& stats) { + RTC_LOG(LS_ERROR) << "Decoder error for frame_id=" << frame_id + << ", code=" << error_code; + + Timestamp processing_started = Now(); + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + size_t peer_index = peers_->index(peer_name); + + if (frame_id == VideoFrame::kNotSetId) { + frame_counters_.failed_to_decode++; + unknown_sender_frame_counters_[std::string(peer_name)].failed_to_decode++; + return; + } + + auto it = captured_frames_in_flight_.find(frame_id); + if (it == captured_frames_in_flight_.end() || + it->second.HasDecodeEndTime(peer_index)) { + // It means this frame was decoded before, so we can skip it. It may happen + // when we have multiple simulcast streams in one track and received + // the same frame from two different streams because SFU can't reliably + // correlate two simulcast streams and started relaying the second stream + // from the same frame it has relayed right before for the first stream. + return; + } + frame_counters_.failed_to_decode++; + InternalStatsKey key(it->second.stream(), + stream_to_sender_.at(it->second.stream()), peer_index); + stream_frame_counters_.at(key).failed_to_decode++; + Timestamp now = Now(); + StreamCodecInfo used_decoder; + used_decoder.codec_name = stats.decoder_name; + used_decoder.first_frame_id = frame_id; + used_decoder.last_frame_id = frame_id; + used_decoder.switched_on_at = now; + used_decoder.switched_from_at = now; + it->second.OnDecoderError(peer_index, used_decoder); + + if (options_.report_infra_metrics) { + analyzer_stats_.on_decoder_error_processing_time_ms.AddSample( + (Now() - processing_started).ms<double>()); + } +} + +void DefaultVideoQualityAnalyzer::RegisterParticipantInCall( + absl::string_view peer_name) { + MutexLock lock(&mutex_); + RTC_CHECK(!peers_->HasName(peer_name)); + size_t new_peer_index = peers_->AddIfAbsent(peer_name); + + // Ensure stats for receiving (for frames from other peers to this one) + // streams exists. Since in flight frames will be sent to the new peer + // as well. Sending stats (from this peer to others) will be added by + // DefaultVideoQualityAnalyzer::OnFrameCaptured. + std::vector<std::pair<InternalStatsKey, Timestamp>> stream_started_time; + for (auto [stream_index, sender_peer_index] : stream_to_sender_) { + InternalStatsKey key(stream_index, sender_peer_index, new_peer_index); + + // To initiate `FrameCounters` for the stream we should pick frame + // counters with the same stream index and the same sender's peer index + // and any receiver's peer index and copy from its sender side + // counters. + FrameCounters counters; + for (size_t i : peers_->GetPresentIndexes()) { + InternalStatsKey prototype_key(stream_index, sender_peer_index, i); + auto it = stream_frame_counters_.find(prototype_key); + if (it != stream_frame_counters_.end()) { + counters.captured = it->second.captured; + counters.pre_encoded = it->second.pre_encoded; + counters.encoded = it->second.encoded; + break; + } + } + // It may happen if we had only one peer before this method was invoked, + // then `counters` will be empty. In such case empty `counters` are ok. + stream_frame_counters_.insert({key, std::move(counters)}); + + stream_started_time.push_back( + {key, stream_states_.at(stream_index).stream_started_time()}); + } + frames_comparator_.RegisterParticipantInCall(stream_started_time, + start_time_); + // Ensure, that frames states are handled correctly + // (e.g. dropped frames tracking). + for (auto& [stream_index, stream_state] : stream_states_) { + stream_state.AddPeer(new_peer_index); + } + // Register new peer for every frame in flight. + // It is guaranteed, that no garbage FrameInFlight objects will stay in + // memory because of adding new peer. Even if the new peer won't receive the + // frame, the frame will be removed by OnFrameRendered after next frame comes + // for the new peer. It is important because FrameInFlight is a large object. + for (auto& [frame_id, frame_in_flight] : captured_frames_in_flight_) { + frame_in_flight.AddExpectedReceiver(new_peer_index); + } +} + +void DefaultVideoQualityAnalyzer::UnregisterParticipantInCall( + absl::string_view peer_name) { + MutexLock lock(&mutex_); + RTC_CHECK(peers_->HasName(peer_name)); + absl::optional<size_t> peer_index = peers_->RemoveIfPresent(peer_name); + RTC_CHECK(peer_index.has_value()); + + for (auto& [stream_index, stream_state] : stream_states_) { + if (!options_.enable_receive_own_stream && + peer_index == stream_state.sender()) { + continue; + } + + AddExistingFramesInFlightForStreamToComparator(stream_index, stream_state, + *peer_index); + + stream_state.RemovePeer(*peer_index); + } + + // Remove peer from every frame in flight. If we removed that last expected + // receiver for the frame, then we should removed this frame if it was + // already encoded. If frame wasn't encoded, it still will be used by sender + // side pipeline, so we can't delete it yet. + for (auto it = captured_frames_in_flight_.begin(); + it != captured_frames_in_flight_.end();) { + FrameInFlight& frame_in_flight = it->second; + frame_in_flight.RemoveExpectedReceiver(*peer_index); + // If frame was fully sent and all receivers received it, then erase it. + // It may happen that when we remove FrameInFlight only some Simulcast/SVC + // layers were encoded and frame has encoded time, but more layers might be + // encoded after removal. In such case it's safe to still remove a frame, + // because OnFrameEncoded method will correctly handle the case when there + // is no FrameInFlight for the received encoded image. + if (frame_in_flight.HasEncodedTime() && + frame_in_flight.HaveAllPeersReceived()) { + it = captured_frames_in_flight_.erase(it); + } else { + it++; + } + } +} + +void DefaultVideoQualityAnalyzer::Stop() { + std::map<InternalStatsKey, Timestamp> last_rendered_frame_times; + { + MutexLock lock(&mutex_); + if (state_ == State::kStopped) { + return; + } + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + state_ = State::kStopped; + + // Add the amount of frames in flight to the analyzer stats before all left + // frames in flight will be sent to the `frames_compartor_`. + analyzer_stats_.frames_in_flight_left_count.AddSample( + StatsSample(captured_frames_in_flight_.size(), Now())); + + for (auto& state_entry : stream_states_) { + const size_t stream_index = state_entry.first; + StreamState& stream_state = state_entry.second; + + // Populate `last_rendered_frame_times` map for all peers that were met in + // call, not only for the currently presented ones. + for (size_t peer_index : peers_->GetAllIndexes()) { + if (peer_index == stream_state.sender() && + !options_.enable_receive_own_stream) { + continue; + } + + InternalStatsKey stats_key(stream_index, stream_state.sender(), + peer_index); + + // If there are no freezes in the call we have to report + // time_between_freezes_ms as call duration and in such case + // `stream_last_freeze_end_time` for this stream will be `start_time_`. + // If there is freeze, then we need add time from last rendered frame + // to last freeze end as time between freezes. + if (stream_state.last_rendered_frame_time(peer_index)) { + last_rendered_frame_times.emplace( + stats_key, + stream_state.last_rendered_frame_time(peer_index).value()); + } + } + + // Push left frame in flight for analysis for the peers that are still in + // the call. + for (size_t peer_index : peers_->GetPresentIndexes()) { + if (peer_index == stream_state.sender() && + !options_.enable_receive_own_stream) { + continue; + } + + AddExistingFramesInFlightForStreamToComparator( + stream_index, stream_state, peer_index); + } + } + } + frames_comparator_.Stop(last_rendered_frame_times); + + // Perform final Metrics update. On this place analyzer is stopped and no one + // holds any locks. + { + MutexLock lock(&mutex_); + FramesComparatorStats frames_comparator_stats = + frames_comparator_.frames_comparator_stats(); + analyzer_stats_.comparisons_queue_size = + std::move(frames_comparator_stats.comparisons_queue_size); + analyzer_stats_.comparisons_done = frames_comparator_stats.comparisons_done; + analyzer_stats_.cpu_overloaded_comparisons_done = + frames_comparator_stats.cpu_overloaded_comparisons_done; + analyzer_stats_.memory_overloaded_comparisons_done = + frames_comparator_stats.memory_overloaded_comparisons_done; + } + ReportResults(); +} + +std::string DefaultVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) { + MutexLock lock1(&mutex_); + auto it = captured_frames_in_flight_.find(frame_id); + if (it != captured_frames_in_flight_.end()) { + return streams_.name(it->second.stream()); + } + for (auto hist_it = stream_to_frame_id_history_.begin(); + hist_it != stream_to_frame_id_history_.end(); ++hist_it) { + auto hist_set_it = hist_it->second.find(frame_id); + if (hist_set_it != hist_it->second.end()) { + return streams_.name(hist_it->first); + } + } + RTC_CHECK(false) << "Unknown frame_id=" << frame_id; +} + +std::set<StatsKey> DefaultVideoQualityAnalyzer::GetKnownVideoStreams() const { + MutexLock lock(&mutex_); + std::set<StatsKey> out; + for (auto& item : frames_comparator_.stream_stats()) { + RTC_LOG(LS_INFO) << item.first.ToString() << " ==> " + << ToStatsKey(item.first).ToString(); + out.insert(ToStatsKey(item.first)); + } + return out; +} + +VideoStreamsInfo DefaultVideoQualityAnalyzer::GetKnownStreams() const { + MutexLock lock(&mutex_); + std::map<std::string, std::string> stream_to_sender; + std::map<std::string, std::set<std::string>> sender_to_streams; + std::map<std::string, std::set<std::string>> stream_to_receivers; + + for (auto& item : frames_comparator_.stream_stats()) { + const std::string& stream_label = streams_.name(item.first.stream); + const std::string& sender = peers_->name(item.first.sender); + const std::string& receiver = peers_->name(item.first.receiver); + RTC_LOG(LS_INFO) << item.first.ToString() << " ==> " + << "stream=" << stream_label << "; sender=" << sender + << "; receiver=" << receiver; + stream_to_sender.emplace(stream_label, sender); + auto streams_it = sender_to_streams.find(sender); + if (streams_it != sender_to_streams.end()) { + streams_it->second.emplace(stream_label); + } else { + sender_to_streams.emplace(sender, std::set<std::string>{stream_label}); + } + auto receivers_it = stream_to_receivers.find(stream_label); + if (receivers_it != stream_to_receivers.end()) { + receivers_it->second.emplace(receiver); + } else { + stream_to_receivers.emplace(stream_label, + std::set<std::string>{receiver}); + } + } + + return VideoStreamsInfo(std::move(stream_to_sender), + std::move(sender_to_streams), + std::move(stream_to_receivers)); +} + +FrameCounters DefaultVideoQualityAnalyzer::GetGlobalCounters() const { + MutexLock lock(&mutex_); + return frame_counters_; +} + +std::map<std::string, FrameCounters> +DefaultVideoQualityAnalyzer::GetUnknownSenderFrameCounters() const { + MutexLock lock(&mutex_); + return unknown_sender_frame_counters_; +} + +std::map<StatsKey, FrameCounters> +DefaultVideoQualityAnalyzer::GetPerStreamCounters() const { + MutexLock lock(&mutex_); + std::map<StatsKey, FrameCounters> out; + for (auto& item : stream_frame_counters_) { + out.emplace(ToStatsKey(item.first), item.second); + } + return out; +} + +std::map<StatsKey, StreamStats> DefaultVideoQualityAnalyzer::GetStats() const { + MutexLock lock1(&mutex_); + std::map<StatsKey, StreamStats> out; + for (auto& item : frames_comparator_.stream_stats()) { + out.emplace(ToStatsKey(item.first), item.second); + } + return out; +} + +AnalyzerStats DefaultVideoQualityAnalyzer::GetAnalyzerStats() const { + MutexLock lock(&mutex_); + return analyzer_stats_; +} + +uint16_t DefaultVideoQualityAnalyzer::GetNextFrameId() { + uint16_t frame_id = next_frame_id_++; + if (next_frame_id_ == VideoFrame::kNotSetId) { + next_frame_id_ = 1; + } + return frame_id; +} + +void DefaultVideoQualityAnalyzer:: + AddExistingFramesInFlightForStreamToComparator(size_t stream_index, + StreamState& stream_state, + size_t peer_index) { + InternalStatsKey stats_key(stream_index, stream_state.sender(), peer_index); + + // Add frames in flight for this stream into frames comparator. + // Frames in flight were not rendered, so they won't affect stream's + // last rendered frame time. + while (!stream_state.IsEmpty(peer_index)) { + uint16_t frame_id = stream_state.PopFront(peer_index); + auto it = captured_frames_in_flight_.find(frame_id); + RTC_DCHECK(it != captured_frames_in_flight_.end()); + FrameInFlight& frame = it->second; + + frames_comparator_.AddComparison(stats_key, /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, + frame.GetStatsForPeer(peer_index)); + } +} + +void DefaultVideoQualityAnalyzer::ReportResults() { + MutexLock lock(&mutex_); + for (auto& item : frames_comparator_.stream_stats()) { + ReportResults(item.first, item.second, + stream_frame_counters_.at(item.first)); + } + // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey. + metrics_logger_->LogSingleValueMetric( + "cpu_usage_%", test_label_, GetCpuUsagePercent(), Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter, + {{MetricMetadataKey::kExperimentalTestNameMetadataKey, test_label_}}); + LogFrameCounters("Global", frame_counters_); + if (!unknown_sender_frame_counters_.empty()) { + RTC_LOG(LS_INFO) << "Received frame counters with unknown frame id:"; + for (const auto& [peer_name, frame_counters] : + unknown_sender_frame_counters_) { + LogFrameCounters(peer_name, frame_counters); + } + } + RTC_LOG(LS_INFO) << "Received frame counters per stream:"; + for (const auto& [stats_key, stream_stats] : + frames_comparator_.stream_stats()) { + LogFrameCounters(ToStatsKey(stats_key).ToString(), + stream_frame_counters_.at(stats_key)); + LogStreamInternalStats(ToStatsKey(stats_key).ToString(), stream_stats, + start_time_); + } + if (!analyzer_stats_.comparisons_queue_size.IsEmpty()) { + RTC_LOG(LS_INFO) << "comparisons_queue_size min=" + << analyzer_stats_.comparisons_queue_size.GetMin() + << "; max=" + << analyzer_stats_.comparisons_queue_size.GetMax() + << "; 99%=" + << analyzer_stats_.comparisons_queue_size.GetPercentile( + 0.99); + } + RTC_LOG(LS_INFO) << "comparisons_done=" << analyzer_stats_.comparisons_done; + RTC_LOG(LS_INFO) << "cpu_overloaded_comparisons_done=" + << analyzer_stats_.cpu_overloaded_comparisons_done; + RTC_LOG(LS_INFO) << "memory_overloaded_comparisons_done=" + << analyzer_stats_.memory_overloaded_comparisons_done; + if (options_.report_infra_metrics) { + metrics_logger_->LogMetric("comparisons_queue_size", test_label_, + analyzer_stats_.comparisons_queue_size, + Unit::kCount, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric("frames_in_flight_left_count", test_label_, + analyzer_stats_.frames_in_flight_left_count, + Unit::kCount, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogSingleValueMetric( + "comparisons_done", test_label_, analyzer_stats_.comparisons_done, + Unit::kCount, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "cpu_overloaded_comparisons_done", test_label_, + analyzer_stats_.cpu_overloaded_comparisons_done, Unit::kCount, + ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "memory_overloaded_comparisons_done", test_label_, + analyzer_stats_.memory_overloaded_comparisons_done, Unit::kCount, + ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "test_duration", test_label_, (Now() - start_time_).ms(), + Unit::kMilliseconds, ImprovementDirection::kNeitherIsBetter); + + metrics_logger_->LogMetric( + "on_frame_captured_processing_time_ms", test_label_, + analyzer_stats_.on_frame_captured_processing_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric( + "on_frame_pre_encode_processing_time_ms", test_label_, + analyzer_stats_.on_frame_pre_encode_processing_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric( + "on_frame_encoded_processing_time_ms", test_label_, + analyzer_stats_.on_frame_encoded_processing_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric( + "on_frame_pre_decode_processing_time_ms", test_label_, + analyzer_stats_.on_frame_pre_decode_processing_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric( + "on_frame_decoded_processing_time_ms", test_label_, + analyzer_stats_.on_frame_decoded_processing_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric( + "on_frame_rendered_processing_time_ms", test_label_, + analyzer_stats_.on_frame_rendered_processing_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric( + "on_decoder_error_processing_time_ms", test_label_, + analyzer_stats_.on_decoder_error_processing_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); + } +} + +void DefaultVideoQualityAnalyzer::ReportResults( + const InternalStatsKey& key, + const StreamStats& stats, + const FrameCounters& frame_counters) { + TimeDelta test_duration = Now() - start_time_; + std::string test_case_name = GetTestCaseName(ToMetricName(key)); + // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey. + std::map<std::string, std::string> metric_metadata{ + {MetricMetadataKey::kPeerMetadataKey, peers_->name(key.sender)}, + {MetricMetadataKey::kVideoStreamMetadataKey, streams_.name(key.stream)}, + {MetricMetadataKey::kSenderMetadataKey, peers_->name(key.sender)}, + {MetricMetadataKey::kReceiverMetadataKey, peers_->name(key.receiver)}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_label_}}; + + double sum_squared_interframe_delays_secs = 0; + Timestamp video_start_time = Timestamp::PlusInfinity(); + Timestamp video_end_time = Timestamp::MinusInfinity(); + for (const SamplesStatsCounter::StatsSample& sample : + stats.time_between_rendered_frames_ms.GetTimedSamples()) { + double interframe_delay_ms = sample.value; + const double interframe_delays_secs = interframe_delay_ms / 1000.0; + // Sum of squared inter frame intervals is used to calculate the harmonic + // frame rate metric. The metric aims to reflect overall experience related + // to smoothness of video playback and includes both freezes and pauses. + sum_squared_interframe_delays_secs += + interframe_delays_secs * interframe_delays_secs; + if (sample.time < video_start_time) { + video_start_time = sample.time; + } + if (sample.time > video_end_time) { + video_end_time = sample.time; + } + } + double harmonic_framerate_fps = 0; + TimeDelta video_duration = video_end_time - video_start_time; + if (sum_squared_interframe_delays_secs > 0.0 && video_duration.IsFinite()) { + harmonic_framerate_fps = + video_duration.seconds<double>() / sum_squared_interframe_delays_secs; + } + + metrics_logger_->LogMetric( + "psnr_dB", test_case_name, stats.psnr, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "ssim", test_case_name, stats.ssim, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogMetric("transport_time", test_case_name, + stats.transport_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, + metric_metadata); + metrics_logger_->LogMetric( + "total_delay_incl_transport", test_case_name, + stats.total_delay_incl_transport_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "time_between_rendered_frames", test_case_name, + stats.time_between_rendered_frames_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "harmonic_framerate", test_case_name, harmonic_framerate_fps, + Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "encode_frame_rate", test_case_name, + stats.encode_frame_rate.IsEmpty() + ? 0 + : stats.encode_frame_rate.GetEventsPerSecond(), + Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "encode_time", test_case_name, stats.encode_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogMetric("time_between_freezes", test_case_name, + stats.time_between_freezes_ms, Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + metric_metadata); + metrics_logger_->LogMetric("freeze_time_ms", test_case_name, + stats.freeze_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, + metric_metadata); + metrics_logger_->LogMetric( + "pixels_per_frame", test_case_name, stats.resolution_of_decoded_frame, + Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "min_psnr_dB", test_case_name, + stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "decode_time", test_case_name, stats.decode_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "receive_to_render_time", test_case_name, stats.receive_to_render_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter, + metric_metadata); + metrics_logger_->LogSingleValueMetric( + "dropped_frames", test_case_name, frame_counters.dropped, Unit::kCount, + ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "frames_in_flight", test_case_name, + frame_counters.captured - frame_counters.rendered - + frame_counters.dropped, + Unit::kCount, ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "rendered_frames", test_case_name, frame_counters.rendered, Unit::kCount, + ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "max_skipped", test_case_name, stats.skipped_between_rendered, + Unit::kCount, ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "target_encode_bitrate", test_case_name, + stats.target_encode_bitrate / 1000, Unit::kKilobitsPerSecond, + ImprovementDirection::kNeitherIsBetter, metric_metadata); + for (const auto& [spatial_layer, qp] : stats.spatial_layers_qp) { + std::map<std::string, std::string> qp_metadata = metric_metadata; + qp_metadata[MetricMetadataKey::kSpatialLayerMetadataKey] = + std::to_string(spatial_layer); + metrics_logger_->LogMetric("qp_sl" + std::to_string(spatial_layer), + test_case_name, qp, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter, + std::move(qp_metadata)); + } + metrics_logger_->LogSingleValueMetric( + "actual_encode_bitrate", test_case_name, + static_cast<double>(stats.total_encoded_images_payload) / + test_duration.seconds<double>() * kBitsInByte / 1000, + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter, + metric_metadata); + + if (options_.report_detailed_frame_stats) { + metrics_logger_->LogSingleValueMetric( + "capture_frame_rate", test_case_name, + stats.capture_frame_rate.IsEmpty() + ? 0 + : stats.capture_frame_rate.GetEventsPerSecond(), + Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "num_encoded_frames", test_case_name, frame_counters.encoded, + Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "num_decoded_frames", test_case_name, frame_counters.decoded, + Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "num_send_key_frames", test_case_name, stats.num_send_key_frames, + Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "num_recv_key_frames", test_case_name, stats.num_recv_key_frames, + Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); + + metrics_logger_->LogMetric("recv_key_frame_size_bytes", test_case_name, + stats.recv_key_frame_size_bytes, Unit::kCount, + ImprovementDirection::kBiggerIsBetter, + metric_metadata); + metrics_logger_->LogMetric("recv_delta_frame_size_bytes", test_case_name, + stats.recv_delta_frame_size_bytes, Unit::kCount, + ImprovementDirection::kBiggerIsBetter, + metric_metadata); + } +} + +std::string DefaultVideoQualityAnalyzer::GetTestCaseName( + const std::string& stream_label) const { + return test_label_ + "/" + stream_label; +} + +Timestamp DefaultVideoQualityAnalyzer::Now() { + return clock_->CurrentTime(); +} + +StatsKey DefaultVideoQualityAnalyzer::ToStatsKey( + const InternalStatsKey& key) const { + return StatsKey(streams_.name(key.stream), peers_->name(key.receiver)); +} + +std::string DefaultVideoQualityAnalyzer::ToMetricName( + const InternalStatsKey& key) const { + const std::string& stream_label = streams_.name(key.stream); + if (peers_->GetKnownSize() <= 2 && key.sender != key.receiver) { + // TODO(titovartem): remove this special case. + return stream_label; + } + rtc::StringBuilder out; + out << stream_label << "_" << peers_->name(key.sender) << "_" + << peers_->name(key.receiver); + return out.str(); +} + +double DefaultVideoQualityAnalyzer::GetCpuUsagePercent() { + return cpu_measurer_.GetCpuUsagePercent(); +} + +std::map<std::string, std::vector<uint16_t>> +DefaultVideoQualityAnalyzer::GetStreamFrames() const { + MutexLock lock(&mutex_); + std::map<std::string, std::vector<uint16_t>> out; + for (auto entry_it : stream_to_frame_id_full_history_) { + out.insert({streams_.name(entry_it.first), entry_it.second}); + } + return out; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h new file mode 100644 index 0000000000..b67e5a0147 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_H_ + +#include <atomic> +#include <cstdint> +#include <deque> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <vector> + +#include "api/array_view.h" +#include "api/test/metrics/metrics_logger.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h" +#include "test/pc/e2e/analyzer/video/names_collection.h" + +namespace webrtc { + +class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { + public: + DefaultVideoQualityAnalyzer(webrtc::Clock* clock, + test::MetricsLogger* metrics_logger, + DefaultVideoQualityAnalyzerOptions options = {}); + ~DefaultVideoQualityAnalyzer() override; + + void Start(std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count) override; + uint16_t OnFrameCaptured(absl::string_view peer_name, + const std::string& stream_label, + const VideoFrame& frame) override; + void OnFramePreEncode(absl::string_view peer_name, + const VideoFrame& frame) override; + void OnFrameEncoded(absl::string_view peer_name, + uint16_t frame_id, + const EncodedImage& encoded_image, + const EncoderStats& stats, + bool discarded) override; + void OnFrameDropped(absl::string_view peer_name, + EncodedImageCallback::DropReason reason) override; + void OnFramePreDecode(absl::string_view peer_name, + uint16_t frame_id, + const EncodedImage& input_image) override; + void OnFrameDecoded(absl::string_view peer_name, + const VideoFrame& frame, + const DecoderStats& stats) override; + void OnFrameRendered(absl::string_view peer_name, + const VideoFrame& frame) override; + void OnEncoderError(absl::string_view peer_name, + const VideoFrame& frame, + int32_t error_code) override; + void OnDecoderError(absl::string_view peer_name, + uint16_t frame_id, + int32_t error_code, + const DecoderStats& stats) override; + + void RegisterParticipantInCall(absl::string_view peer_name) override; + void UnregisterParticipantInCall(absl::string_view peer_name) override; + + void Stop() override; + std::string GetStreamLabel(uint16_t frame_id) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override {} + + // Returns set of stream labels, that were met during test call. + std::set<StatsKey> GetKnownVideoStreams() const; + VideoStreamsInfo GetKnownStreams() const; + FrameCounters GetGlobalCounters() const; + // Returns frame counter for frames received without frame id set. + std::map<std::string, FrameCounters> GetUnknownSenderFrameCounters() const; + // Returns frame counter per stream label. Valid stream labels can be obtained + // by calling GetKnownVideoStreams() + std::map<StatsKey, FrameCounters> GetPerStreamCounters() const; + // Returns video quality stats per stream label. Valid stream labels can be + // obtained by calling GetKnownVideoStreams() + std::map<StatsKey, StreamStats> GetStats() const; + AnalyzerStats GetAnalyzerStats() const; + double GetCpuUsagePercent(); + + // Returns mapping from the stream label to the history of frames that were + // met in this stream in the order as they were captured. + std::map<std::string, std::vector<uint16_t>> GetStreamFrames() const; + + private: + enum State { kNew, kActive, kStopped }; + + // Returns next frame id to use. Frame ID can't be `VideoFrame::kNotSetId`, + // because this value is reserved by `VideoFrame` as "ID not set". + uint16_t GetNextFrameId() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + void AddExistingFramesInFlightForStreamToComparator(size_t stream_index, + StreamState& stream_state, + size_t peer_index) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Report results for all metrics for all streams. + void ReportResults(); + void ReportResults(const InternalStatsKey& key, + const StreamStats& stats, + const FrameCounters& frame_counters) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + // Returns name of current test case for reporting. + std::string GetTestCaseName(const std::string& stream_label) const; + Timestamp Now(); + StatsKey ToStatsKey(const InternalStatsKey& key) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + // Returns string representation of stats key for metrics naming. Used for + // backward compatibility by metrics naming for 2 peers cases. + std::string ToMetricName(const InternalStatsKey& key) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + static const uint16_t kStartingFrameId = 1; + + const DefaultVideoQualityAnalyzerOptions options_; + webrtc::Clock* const clock_; + test::MetricsLogger* const metrics_logger_; + + std::string test_label_; + + mutable Mutex mutex_; + uint16_t next_frame_id_ RTC_GUARDED_BY(mutex_) = kStartingFrameId; + std::unique_ptr<NamesCollection> peers_ RTC_GUARDED_BY(mutex_); + State state_ RTC_GUARDED_BY(mutex_) = State::kNew; + Timestamp start_time_ RTC_GUARDED_BY(mutex_) = Timestamp::MinusInfinity(); + // Mapping from stream label to unique size_t value to use in stats and avoid + // extra string copying. + NamesCollection streams_ RTC_GUARDED_BY(mutex_); + // Frames that were captured by all streams and still aren't rendered on + // receivers or deemed dropped. Frame with id X can be removed from this map + // if: + // 1. The frame with id X was received in OnFrameRendered by all expected + // receivers. + // 2. The frame with id Y > X was received in OnFrameRendered by all expected + // receivers. + // 3. Next available frame id for newly captured frame is X + // 4. There too many frames in flight for current video stream and X is the + // oldest frame id in this stream. In such case only the frame content + // will be removed, but the map entry will be preserved. + std::map<uint16_t, FrameInFlight> captured_frames_in_flight_ + RTC_GUARDED_BY(mutex_); + // Global frames count for all video streams. + FrameCounters frame_counters_ RTC_GUARDED_BY(mutex_); + // Frame counters for received frames without video frame id set. + // Map from peer name to the frame counters. + std::map<std::string, FrameCounters> unknown_sender_frame_counters_ + RTC_GUARDED_BY(mutex_); + // Frame counters per each stream per each receiver. + std::map<InternalStatsKey, FrameCounters> stream_frame_counters_ + RTC_GUARDED_BY(mutex_); + // Map from stream index in `streams_` to its StreamState. + std::map<size_t, StreamState> stream_states_ RTC_GUARDED_BY(mutex_); + // Map from stream index in `streams_` to sender peer index in `peers_`. + std::map<size_t, size_t> stream_to_sender_ RTC_GUARDED_BY(mutex_); + + // Stores history mapping between stream index in `streams_` and frame ids. + // Updated when frame id overlap. It required to properly return stream label + // after 1st frame from simulcast streams was already rendered and last is + // still encoding. + std::map<size_t, std::set<uint16_t>> stream_to_frame_id_history_ + RTC_GUARDED_BY(mutex_); + // Map from stream index to the list of frames as they were met in the stream. + std::map<size_t, std::vector<uint16_t>> stream_to_frame_id_full_history_ + RTC_GUARDED_BY(mutex_); + AnalyzerStats analyzer_stats_ RTC_GUARDED_BY(mutex_); + + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer_; + DefaultVideoQualityAnalyzerFramesComparator frames_comparator_; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.cc new file mode 100644 index 0000000000..847c9f09a6 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.cc @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h" + +#include "rtc_base/cpu_time.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/system_time.h" + +namespace webrtc { + +void DefaultVideoQualityAnalyzerCpuMeasurer::StartMeasuringCpuProcessTime() { + MutexLock lock(&mutex_); + cpu_time_ -= rtc::GetProcessCpuTimeNanos(); + wallclock_time_ -= rtc::SystemTimeNanos(); +} + +void DefaultVideoQualityAnalyzerCpuMeasurer::StopMeasuringCpuProcessTime() { + MutexLock lock(&mutex_); + cpu_time_ += rtc::GetProcessCpuTimeNanos(); + wallclock_time_ += rtc::SystemTimeNanos(); +} + +void DefaultVideoQualityAnalyzerCpuMeasurer::StartExcludingCpuThreadTime() { + MutexLock lock(&mutex_); + cpu_time_ += rtc::GetThreadCpuTimeNanos(); +} + +void DefaultVideoQualityAnalyzerCpuMeasurer::StopExcludingCpuThreadTime() { + MutexLock lock(&mutex_); + cpu_time_ -= rtc::GetThreadCpuTimeNanos(); +} + +double DefaultVideoQualityAnalyzerCpuMeasurer::GetCpuUsagePercent() { + MutexLock lock(&mutex_); + return static_cast<double>(cpu_time_) / wallclock_time_ * 100.0; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h new file mode 100644 index 0000000000..dd9fa07af2 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_CPU_MEASURER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_CPU_MEASURER_H_ + +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { + +// This class is thread safe. +class DefaultVideoQualityAnalyzerCpuMeasurer { + public: + double GetCpuUsagePercent(); + + void StartMeasuringCpuProcessTime(); + void StopMeasuringCpuProcessTime(); + void StartExcludingCpuThreadTime(); + void StopExcludingCpuThreadTime(); + + private: + Mutex mutex_; + int64_t cpu_time_ RTC_GUARDED_BY(mutex_) = 0; + int64_t wallclock_time_ RTC_GUARDED_BY(mutex_) = 0; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_CPU_MEASURER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc new file mode 100644 index 0000000000..df34dadaf0 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h" + +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_type.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" + +namespace webrtc { +namespace { + +template <typename T> +absl::optional<T> MaybeGetValue(const std::map<size_t, T>& map, size_t key) { + auto it = map.find(key); + if (it == map.end()) { + return absl::nullopt; + } + return it->second; +} + +} // namespace + +FrameInFlight::FrameInFlight(size_t stream, + VideoFrame frame, + Timestamp captured_time, + std::set<size_t> expected_receivers) + : stream_(stream), + expected_receivers_(std::move(expected_receivers)), + frame_(std::move(frame)), + captured_time_(captured_time) {} + +bool FrameInFlight::RemoveFrame() { + if (!frame_) { + return false; + } + frame_ = absl::nullopt; + return true; +} + +void FrameInFlight::SetFrameId(uint16_t id) { + if (frame_) { + frame_->set_id(id); + } + frame_id_ = id; +} + +std::vector<size_t> FrameInFlight::GetPeersWhichDidntReceive() const { + std::vector<size_t> out; + for (size_t peer : expected_receivers_) { + auto it = receiver_stats_.find(peer); + if (it == receiver_stats_.end() || + (!it->second.dropped && it->second.rendered_time.IsInfinite())) { + out.push_back(peer); + } + } + return out; +} + +bool FrameInFlight::HaveAllPeersReceived() const { + for (size_t peer : expected_receivers_) { + auto it = receiver_stats_.find(peer); + if (it == receiver_stats_.end()) { + return false; + } + + if (!it->second.dropped && it->second.rendered_time.IsInfinite()) { + return false; + } + } + return true; +} + +void FrameInFlight::OnFrameEncoded(webrtc::Timestamp time, + VideoFrameType frame_type, + DataSize encoded_image_size, + uint32_t target_encode_bitrate, + int spatial_layer, + int qp, + StreamCodecInfo used_encoder) { + encoded_time_ = time; + frame_type_ = frame_type; + encoded_image_size_ = encoded_image_size; + target_encode_bitrate_ += target_encode_bitrate; + spatial_layers_qp_[spatial_layer].AddSample(SamplesStatsCounter::StatsSample{ + .value = static_cast<double>(qp), .time = time}); + // Update used encoder info. If simulcast/SVC is used, this method can + // be called multiple times, in such case we should preserve the value + // of `used_encoder_.switched_on_at` from the first invocation as the + // smallest one. + Timestamp encoder_switched_on_at = used_encoder_.has_value() + ? used_encoder_->switched_on_at + : Timestamp::PlusInfinity(); + RTC_DCHECK(used_encoder.switched_on_at.IsFinite()); + RTC_DCHECK(used_encoder.switched_from_at.IsFinite()); + used_encoder_ = used_encoder; + if (encoder_switched_on_at < used_encoder_->switched_on_at) { + used_encoder_->switched_on_at = encoder_switched_on_at; + } +} + +void FrameInFlight::OnFramePreDecode(size_t peer, + webrtc::Timestamp received_time, + webrtc::Timestamp decode_start_time, + VideoFrameType frame_type, + DataSize encoded_image_size) { + receiver_stats_[peer].received_time = received_time; + receiver_stats_[peer].decode_start_time = decode_start_time; + receiver_stats_[peer].frame_type = frame_type; + receiver_stats_[peer].encoded_image_size = encoded_image_size; +} + +bool FrameInFlight::HasReceivedTime(size_t peer) const { + auto it = receiver_stats_.find(peer); + if (it == receiver_stats_.end()) { + return false; + } + return it->second.received_time.IsFinite(); +} + +void FrameInFlight::OnFrameDecoded(size_t peer, + webrtc::Timestamp time, + int width, + int height, + const StreamCodecInfo& used_decoder) { + receiver_stats_[peer].decode_end_time = time; + receiver_stats_[peer].used_decoder = used_decoder; + receiver_stats_[peer].decoded_frame_width = width; + receiver_stats_[peer].decoded_frame_height = height; +} + +void FrameInFlight::OnDecoderError(size_t peer, + const StreamCodecInfo& used_decoder) { + receiver_stats_[peer].decoder_failed = true; + receiver_stats_[peer].used_decoder = used_decoder; +} + +bool FrameInFlight::HasDecodeEndTime(size_t peer) const { + auto it = receiver_stats_.find(peer); + if (it == receiver_stats_.end()) { + return false; + } + return it->second.decode_end_time.IsFinite(); +} + +void FrameInFlight::OnFrameRendered(size_t peer, webrtc::Timestamp time) { + receiver_stats_[peer].rendered_time = time; +} + +bool FrameInFlight::HasRenderedTime(size_t peer) const { + auto it = receiver_stats_.find(peer); + if (it == receiver_stats_.end()) { + return false; + } + return it->second.rendered_time.IsFinite(); +} + +bool FrameInFlight::IsDropped(size_t peer) const { + auto it = receiver_stats_.find(peer); + if (it == receiver_stats_.end()) { + return false; + } + return it->second.dropped; +} + +FrameStats FrameInFlight::GetStatsForPeer(size_t peer) const { + RTC_DCHECK_NE(frame_id_, VideoFrame::kNotSetId) + << "Frame id isn't initialized"; + FrameStats stats(frame_id_, captured_time_); + stats.pre_encode_time = pre_encode_time_; + stats.encoded_time = encoded_time_; + stats.target_encode_bitrate = target_encode_bitrate_; + stats.encoded_frame_type = frame_type_; + stats.encoded_image_size = encoded_image_size_; + stats.used_encoder = used_encoder_; + stats.spatial_layers_qp = spatial_layers_qp_; + + absl::optional<ReceiverFrameStats> receiver_stats = + MaybeGetValue<ReceiverFrameStats>(receiver_stats_, peer); + if (receiver_stats.has_value()) { + stats.received_time = receiver_stats->received_time; + stats.decode_start_time = receiver_stats->decode_start_time; + stats.decode_end_time = receiver_stats->decode_end_time; + stats.rendered_time = receiver_stats->rendered_time; + stats.prev_frame_rendered_time = receiver_stats->prev_frame_rendered_time; + stats.decoded_frame_width = receiver_stats->decoded_frame_width; + stats.decoded_frame_height = receiver_stats->decoded_frame_height; + stats.used_decoder = receiver_stats->used_decoder; + stats.pre_decoded_frame_type = receiver_stats->frame_type; + stats.pre_decoded_image_size = receiver_stats->encoded_image_size; + stats.decoder_failed = receiver_stats->decoder_failed; + } + return stats; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h new file mode 100644 index 0000000000..52a526d09b --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAME_IN_FLIGHT_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAME_IN_FLIGHT_H_ + +#include <map> +#include <set> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_type.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" + +namespace webrtc { + +struct ReceiverFrameStats { + // Time when last packet of a frame was received. + Timestamp received_time = Timestamp::MinusInfinity(); + Timestamp decode_start_time = Timestamp::MinusInfinity(); + Timestamp decode_end_time = Timestamp::MinusInfinity(); + Timestamp rendered_time = Timestamp::MinusInfinity(); + Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity(); + + // Type and encoded size of received frame. + VideoFrameType frame_type = VideoFrameType::kEmptyFrame; + DataSize encoded_image_size = DataSize::Bytes(0); + + absl::optional<int> decoded_frame_width = absl::nullopt; + absl::optional<int> decoded_frame_height = absl::nullopt; + + // Can be not set if frame was dropped in the network. + absl::optional<StreamCodecInfo> used_decoder = absl::nullopt; + + bool dropped = false; + bool decoder_failed = false; +}; + +// Represents a frame which was sent by sender and is currently on the way to +// multiple receivers. Some receivers may receive this frame and some don't. +// +// Contains all statistic associated with the frame and gathered in multiple +// points of the video pipeline. +// +// Internally may store the copy of the source frame which was sent. In such +// case this frame is "alive". +class FrameInFlight { + public: + FrameInFlight(size_t stream, + VideoFrame frame, + Timestamp captured_time, + std::set<size_t> expected_receivers); + + size_t stream() const { return stream_; } + // Returns internal copy of source `VideoFrame` or `absl::nullopt` if it was + // removed before. + const absl::optional<VideoFrame>& frame() const { return frame_; } + // Removes internal copy of the source `VideoFrame` to free up extra memory. + // Returns was frame removed or not. + bool RemoveFrame(); + void SetFrameId(uint16_t id); + + void AddExpectedReceiver(size_t peer) { expected_receivers_.insert(peer); } + + void RemoveExpectedReceiver(size_t peer) { expected_receivers_.erase(peer); } + + std::vector<size_t> GetPeersWhichDidntReceive() const; + + // Returns if all peers which were expected to receive this frame actually + // received it or not. + bool HaveAllPeersReceived() const; + + void SetPreEncodeTime(webrtc::Timestamp time) { pre_encode_time_ = time; } + + void OnFrameEncoded(webrtc::Timestamp time, + VideoFrameType frame_type, + DataSize encoded_image_size, + uint32_t target_encode_bitrate, + int spatial_layer, + int qp, + StreamCodecInfo used_encoder); + + bool HasEncodedTime() const { return encoded_time_.IsFinite(); } + + void OnFramePreDecode(size_t peer, + webrtc::Timestamp received_time, + webrtc::Timestamp decode_start_time, + VideoFrameType frame_type, + DataSize encoded_image_size); + + bool HasReceivedTime(size_t peer) const; + + void OnFrameDecoded(size_t peer, + webrtc::Timestamp time, + int width, + int height, + const StreamCodecInfo& used_decoder); + void OnDecoderError(size_t peer, const StreamCodecInfo& used_decoder); + + bool HasDecodeEndTime(size_t peer) const; + + void OnFrameRendered(size_t peer, webrtc::Timestamp time); + + bool HasRenderedTime(size_t peer) const; + + // Crash if rendered time is not set for specified `peer`. + webrtc::Timestamp rendered_time(size_t peer) const { + return receiver_stats_.at(peer).rendered_time; + } + + // Marks that frame was dropped and wasn't seen by particular `peer`. + void MarkDropped(size_t peer) { receiver_stats_[peer].dropped = true; } + bool IsDropped(size_t peer) const; + + void SetPrevFrameRenderedTime(size_t peer, webrtc::Timestamp time) { + receiver_stats_[peer].prev_frame_rendered_time = time; + } + + FrameStats GetStatsForPeer(size_t peer) const; + + private: + const size_t stream_; + // Set of peer's indexes who are expected to receive this frame. This is not + // the set of peer's indexes that received the frame. For example, if peer A + // was among expected receivers, it received frame and then left the call, A + // will be removed from this set, but the Stats for peer A still will be + // preserved in the FrameInFlight. + // + // This set is used to determine if this frame is expected to be received by + // any peer or can be safely deleted. It is responsibility of the user of this + // object to decide when it should be deleted. + std::set<size_t> expected_receivers_; + absl::optional<VideoFrame> frame_; + // Store frame id separately because `frame_` can be removed when we have too + // much memory consuption. + uint16_t frame_id_ = VideoFrame::kNotSetId; + + // Frame events timestamp. + Timestamp captured_time_; + Timestamp pre_encode_time_ = Timestamp::MinusInfinity(); + Timestamp encoded_time_ = Timestamp::MinusInfinity(); + // Type and encoded size of sent frame. + VideoFrameType frame_type_ = VideoFrameType::kEmptyFrame; + DataSize encoded_image_size_ = DataSize::Bytes(0); + uint32_t target_encode_bitrate_ = 0; + // Sender side qp values per spatial layer. In case when spatial layer is not + // set for `webrtc::EncodedImage`, 0 is used as default. + std::map<int, SamplesStatsCounter> spatial_layers_qp_; + // Can be not set if frame was dropped by encoder. + absl::optional<StreamCodecInfo> used_encoder_ = absl::nullopt; + // Map from the receiver peer's index to frame stats for that peer. + std::map<size_t, ReceiverFrameStats> receiver_stats_; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAME_IN_FLIGHT_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc new file mode 100644 index 0000000000..cbc0b7e8f3 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc @@ -0,0 +1,575 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" + +#include <algorithm> +#include <map> +#include <string> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/scoped_refptr.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame_type.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_base/checks.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_tools/frame_analyzer/video_geometry_aligner.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" +#include "test/pc/e2e/metric_metadata_keys.h" + +namespace webrtc { +namespace { + +using ::webrtc::webrtc_pc_e2e::SampleMetadataKey; + +constexpr TimeDelta kFreezeThreshold = TimeDelta::Millis(150); +constexpr int kMaxActiveComparisons = 10; + +SamplesStatsCounter::StatsSample StatsSample( + double value, + Timestamp sampling_time, + std::map<std::string, std::string> metadata) { + return SamplesStatsCounter::StatsSample{value, sampling_time, + std::move(metadata)}; +} + +SamplesStatsCounter::StatsSample StatsSample( + TimeDelta duration, + Timestamp sampling_time, + std::map<std::string, std::string> metadata) { + return SamplesStatsCounter::StatsSample{duration.ms<double>(), sampling_time, + std::move(metadata)}; +} + +FrameComparison ValidateFrameComparison(FrameComparison comparison) { + RTC_DCHECK(comparison.frame_stats.captured_time.IsFinite()) + << "Any comparison has to have finite captured_time"; + switch (comparison.type) { + case FrameComparisonType::kRegular: + // Regular comparison has to have all FrameStats filled in. + RTC_DCHECK(comparison.captured.has_value() || + comparison.overload_reason != OverloadReason::kNone) + << "Regular comparison has to have captured frame if it's not " + << "overloaded comparison"; + RTC_DCHECK(comparison.rendered.has_value() || + comparison.overload_reason != OverloadReason::kNone) + << "rendered frame has to be presented if it's not overloaded " + << "comparison"; + RTC_DCHECK(comparison.frame_stats.pre_encode_time.IsFinite()) + << "Regular comparison has to have finite pre_encode_time"; + RTC_DCHECK(comparison.frame_stats.encoded_time.IsFinite()) + << "Regular comparison has to have finite encoded_time"; + RTC_DCHECK(comparison.frame_stats.received_time.IsFinite()) + << "Regular comparison has to have finite received_time"; + RTC_DCHECK(comparison.frame_stats.decode_start_time.IsFinite()) + << "Regular comparison has to have finite decode_start_time"; + RTC_DCHECK(comparison.frame_stats.decode_end_time.IsFinite()) + << "Regular comparison has to have finite decode_end_time"; + RTC_DCHECK(comparison.frame_stats.rendered_time.IsFinite()) + << "Regular comparison has to have finite rendered_time"; + RTC_DCHECK(comparison.frame_stats.decoded_frame_width.has_value()) + << "Regular comparison has to have decoded_frame_width"; + RTC_DCHECK(comparison.frame_stats.decoded_frame_height.has_value()) + << "Regular comparison has to have decoded_frame_height"; + RTC_DCHECK(comparison.frame_stats.used_encoder.has_value()) + << "Regular comparison has to have used_encoder"; + RTC_DCHECK(comparison.frame_stats.used_decoder.has_value()) + << "Regular comparison has to have used_decoder"; + RTC_DCHECK(!comparison.frame_stats.decoder_failed) + << "Regular comparison can't have decoder failure"; + break; + case FrameComparisonType::kDroppedFrame: + // Frame can be dropped before encoder, by encoder, inside network or + // after decoder. + RTC_DCHECK(!comparison.captured.has_value()) + << "Dropped frame comparison can't have captured frame"; + RTC_DCHECK(!comparison.rendered.has_value()) + << "Dropped frame comparison can't have rendered frame"; + + if (comparison.frame_stats.encoded_time.IsFinite()) { + RTC_DCHECK(comparison.frame_stats.used_encoder.has_value()) + << "Dropped frame comparison has to have used_encoder when " + << "encoded_time is set"; + RTC_DCHECK(comparison.frame_stats.pre_encode_time.IsFinite()) + << "Dropped frame comparison has to have finite pre_encode_time " + << "when encoded_time is finite."; + } + + if (comparison.frame_stats.decode_end_time.IsFinite() || + comparison.frame_stats.decoder_failed) { + RTC_DCHECK(comparison.frame_stats.received_time.IsFinite()) + << "Dropped frame comparison has to have received_time when " + << "decode_end_time is set or decoder_failed is true"; + RTC_DCHECK(comparison.frame_stats.decode_start_time.IsFinite()) + << "Dropped frame comparison has to have decode_start_time when " + << "decode_end_time is set or decoder_failed is true"; + RTC_DCHECK(comparison.frame_stats.used_decoder.has_value()) + << "Dropped frame comparison has to have used_decoder when " + << "decode_end_time is set or decoder_failed is true"; + } else if (comparison.frame_stats.decode_end_time.IsFinite()) { + RTC_DCHECK(comparison.frame_stats.decoded_frame_width.has_value()) + << "Dropped frame comparison has to have decoded_frame_width when " + << "decode_end_time is set"; + RTC_DCHECK(comparison.frame_stats.decoded_frame_height.has_value()) + << "Dropped frame comparison has to have decoded_frame_height when " + << "decode_end_time is set"; + } + RTC_DCHECK(!comparison.frame_stats.rendered_time.IsFinite()) + << "Dropped frame comparison can't have rendered_time"; + break; + case FrameComparisonType::kFrameInFlight: + // Frame in flight comparison may miss almost any FrameStats, but if + // stats for stage X are set, then stats for stage X - 1 also has to be + // set. Also these frames were never rendered. + RTC_DCHECK(!comparison.captured.has_value()) + << "Frame in flight comparison can't have captured frame"; + RTC_DCHECK(!comparison.rendered.has_value()) + << "Frame in flight comparison can't have rendered frame"; + RTC_DCHECK(!comparison.frame_stats.rendered_time.IsFinite()) + << "Frame in flight comparison can't have rendered_time"; + + if (comparison.frame_stats.decode_end_time.IsFinite() || + comparison.frame_stats.decoder_failed) { + RTC_DCHECK(comparison.frame_stats.used_decoder.has_value()) + << "Frame in flight comparison has to have used_decoder when " + << "decode_end_time is set or decoder_failed is true."; + RTC_DCHECK(comparison.frame_stats.decode_start_time.IsFinite()) + << "Frame in flight comparison has to have finite " + << "decode_start_time when decode_end_time is finite or " + << "decoder_failed is true."; + } + if (comparison.frame_stats.decode_end_time.IsFinite()) { + RTC_DCHECK(comparison.frame_stats.decoded_frame_width.has_value()) + << "Frame in flight comparison has to have decoded_frame_width " + << "when decode_end_time is set."; + RTC_DCHECK(comparison.frame_stats.decoded_frame_height.has_value()) + << "Frame in flight comparison has to have decoded_frame_height " + << "when decode_end_time is set."; + } + if (comparison.frame_stats.decode_start_time.IsFinite()) { + RTC_DCHECK(comparison.frame_stats.received_time.IsFinite()) + << "Frame in flight comparison has to have finite received_time " + << "when decode_start_time is finite."; + } + if (comparison.frame_stats.received_time.IsFinite()) { + RTC_DCHECK(comparison.frame_stats.encoded_time.IsFinite()) + << "Frame in flight comparison has to have finite encoded_time " + << "when received_time is finite."; + } + if (comparison.frame_stats.encoded_time.IsFinite()) { + RTC_DCHECK(comparison.frame_stats.used_encoder.has_value()) + << "Frame in flight comparison has to have used_encoder when " + << "encoded_time is set"; + RTC_DCHECK(comparison.frame_stats.pre_encode_time.IsFinite()) + << "Frame in flight comparison has to have finite pre_encode_time " + << "when encoded_time is finite."; + } + break; + } + return comparison; +} + +} // namespace + +void DefaultVideoQualityAnalyzerFramesComparator::Start(int max_threads_count) { + for (int i = 0; i < max_threads_count; i++) { + thread_pool_.push_back(rtc::PlatformThread::SpawnJoinable( + [this] { ProcessComparisons(); }, + "DefaultVideoQualityAnalyzerFramesComparator-" + std::to_string(i))); + } + { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kNew) << "Frames comparator is already started"; + state_ = State::kActive; + } + cpu_measurer_.StartMeasuringCpuProcessTime(); +} + +void DefaultVideoQualityAnalyzerFramesComparator::Stop( + const std::map<InternalStatsKey, Timestamp>& last_rendered_frame_times) { + { + MutexLock lock(&mutex_); + if (state_ == State::kStopped) { + return; + } + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + state_ = State::kStopped; + } + cpu_measurer_.StopMeasuringCpuProcessTime(); + comparison_available_event_.Set(); + thread_pool_.clear(); + + { + MutexLock lock(&mutex_); + // Perform final Metrics update. On this place analyzer is stopped and no + // one holds any locks. + + // Time between freezes. + // Count time since the last freeze to the end of the call as time + // between freezes. + for (auto& entry : last_rendered_frame_times) { + const InternalStatsKey& stats_key = entry.first; + const Timestamp& last_rendered_frame_time = entry.second; + + // If there are no freezes in the call we have to report + // time_between_freezes_ms as call duration and in such case + // `last_rendered_frame_time` for this stream will be stream start time. + // If there is freeze, then we need add time from last rendered frame + // to last freeze end as time between freezes. + stream_stats_.at(stats_key).time_between_freezes_ms.AddSample(StatsSample( + last_rendered_frame_time - stream_last_freeze_end_time_.at(stats_key), + Now(), /*metadata=*/{})); + } + + // Freeze Time: + // If there were no freezes on a video stream, add only one sample with + // value 0 (0ms freezes time). + for (auto& [key, stream_stats] : stream_stats_) { + if (stream_stats.freeze_time_ms.IsEmpty()) { + stream_stats.freeze_time_ms.AddSample(0); + } + } + } +} + +void DefaultVideoQualityAnalyzerFramesComparator::EnsureStatsForStream( + size_t stream_index, + size_t sender_peer_index, + size_t peers_count, + Timestamp captured_time, + Timestamp start_time) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + + for (size_t i = 0; i < peers_count; ++i) { + if (i == sender_peer_index && !options_.enable_receive_own_stream) { + continue; + } + InternalStatsKey stats_key(stream_index, sender_peer_index, i); + if (stream_stats_.find(stats_key) == stream_stats_.end()) { + stream_stats_.insert({stats_key, StreamStats(captured_time)}); + // Assume that the first freeze was before first stream frame captured. + // This way time before the first freeze would be counted as time + // between freezes. + stream_last_freeze_end_time_.insert({stats_key, start_time}); + } else { + // When we see some `stream_label` for the first time we need to create + // stream stats object for it and set up some states, but we need to do + // it only once and for all receivers, so on the next frame on the same + // `stream_label` we can be sure, that it's already done and we needn't + // to scan though all peers again. + break; + } + } +} + +void DefaultVideoQualityAnalyzerFramesComparator::RegisterParticipantInCall( + rtc::ArrayView<std::pair<InternalStatsKey, Timestamp>> stream_started_time, + Timestamp start_time) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + + for (const std::pair<InternalStatsKey, Timestamp>& pair : + stream_started_time) { + stream_stats_.insert({pair.first, StreamStats(pair.second)}); + stream_last_freeze_end_time_.insert({pair.first, start_time}); + } +} + +void DefaultVideoQualityAnalyzerFramesComparator::AddComparison( + InternalStatsKey stats_key, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + AddComparisonInternal(std::move(stats_key), std::move(captured), + std::move(rendered), type, std::move(frame_stats)); +} + +void DefaultVideoQualityAnalyzerFramesComparator::AddComparison( + InternalStatsKey stats_key, + int skipped_between_rendered, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + stream_stats_.at(stats_key).skipped_between_rendered.AddSample( + StatsSample(skipped_between_rendered, Now(), + /*metadata=*/ + {{SampleMetadataKey::kFrameIdMetadataKey, + std::to_string(frame_stats.frame_id)}})); + AddComparisonInternal(std::move(stats_key), std::move(captured), + std::move(rendered), type, std::move(frame_stats)); +} + +void DefaultVideoQualityAnalyzerFramesComparator::AddComparisonInternal( + InternalStatsKey stats_key, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats) { + cpu_measurer_.StartExcludingCpuThreadTime(); + frames_comparator_stats_.comparisons_queue_size.AddSample( + StatsSample(comparisons_.size(), Now(), /*metadata=*/{})); + // If there too many computations waiting in the queue, we won't provide + // frames itself to make future computations lighter. + if (comparisons_.size() >= kMaxActiveComparisons) { + comparisons_.emplace_back(ValidateFrameComparison( + FrameComparison(std::move(stats_key), /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, type, + std::move(frame_stats), OverloadReason::kCpu))); + } else { + OverloadReason overload_reason = OverloadReason::kNone; + if (!captured && type == FrameComparisonType::kRegular) { + overload_reason = OverloadReason::kMemory; + } + comparisons_.emplace_back(ValidateFrameComparison(FrameComparison( + std::move(stats_key), std::move(captured), std::move(rendered), type, + std::move(frame_stats), overload_reason))); + } + comparison_available_event_.Set(); + cpu_measurer_.StopExcludingCpuThreadTime(); +} + +void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparisons() { + while (true) { + // Try to pick next comparison to perform from the queue. + absl::optional<FrameComparison> comparison = absl::nullopt; + bool more_new_comparisons_expected; + { + MutexLock lock(&mutex_); + if (!comparisons_.empty()) { + comparison = comparisons_.front(); + comparisons_.pop_front(); + if (!comparisons_.empty()) { + comparison_available_event_.Set(); + } + } + // If state is stopped => no new frame comparisons are expected. + more_new_comparisons_expected = state_ != State::kStopped; + } + if (!comparison) { + if (!more_new_comparisons_expected) { + comparison_available_event_.Set(); + return; + } + comparison_available_event_.Wait(TimeDelta::Seconds(1)); + continue; + } + + cpu_measurer_.StartExcludingCpuThreadTime(); + ProcessComparison(comparison.value()); + cpu_measurer_.StopExcludingCpuThreadTime(); + } +} + +void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison( + const FrameComparison& comparison) { + // Comparison is checked to be valid before adding, so we can use this + // assumptions during computations. + + // Perform expensive psnr and ssim calculations while not holding lock. + double psnr = -1.0; + double ssim = -1.0; + if ((options_.compute_psnr || options_.compute_ssim) && + comparison.captured.has_value() && comparison.rendered.has_value()) { + rtc::scoped_refptr<I420BufferInterface> reference_buffer = + comparison.captured->video_frame_buffer()->ToI420(); + rtc::scoped_refptr<I420BufferInterface> test_buffer = + comparison.rendered->video_frame_buffer()->ToI420(); + if (options_.adjust_cropping_before_comparing_frames) { + test_buffer = ScaleVideoFrameBuffer( + *test_buffer, reference_buffer->width(), reference_buffer->height()); + reference_buffer = test::AdjustCropping(reference_buffer, test_buffer); + } + if (options_.compute_psnr) { + psnr = options_.use_weighted_psnr + ? I420WeightedPSNR(*reference_buffer, *test_buffer) + : I420PSNR(*reference_buffer, *test_buffer); + } + if (options_.compute_ssim) { + ssim = I420SSIM(*reference_buffer, *test_buffer); + } + } + + const FrameStats& frame_stats = comparison.frame_stats; + + MutexLock lock(&mutex_); + auto stats_it = stream_stats_.find(comparison.stats_key); + RTC_CHECK(stats_it != stream_stats_.end()) << comparison.stats_key.ToString(); + StreamStats* stats = &stats_it->second; + + frames_comparator_stats_.comparisons_done++; + if (comparison.overload_reason == OverloadReason::kCpu) { + frames_comparator_stats_.cpu_overloaded_comparisons_done++; + } else if (comparison.overload_reason == OverloadReason::kMemory) { + frames_comparator_stats_.memory_overloaded_comparisons_done++; + } + + std::map<std::string, std::string> metadata; + metadata.emplace(SampleMetadataKey::kFrameIdMetadataKey, + std::to_string(frame_stats.frame_id)); + + if (psnr > 0) { + stats->psnr.AddSample( + StatsSample(psnr, frame_stats.rendered_time, metadata)); + } + if (ssim > 0) { + stats->ssim.AddSample( + StatsSample(ssim, frame_stats.received_time, metadata)); + } + stats->capture_frame_rate.AddEvent(frame_stats.captured_time); + + // Compute dropped phase for dropped frame + if (comparison.type == FrameComparisonType::kDroppedFrame) { + FrameDropPhase dropped_phase; + if (frame_stats.decode_end_time.IsFinite()) { + dropped_phase = FrameDropPhase::kAfterDecoder; + } else if (frame_stats.decode_start_time.IsFinite()) { + dropped_phase = FrameDropPhase::kByDecoder; + } else if (frame_stats.encoded_time.IsFinite()) { + dropped_phase = FrameDropPhase::kTransport; + } else if (frame_stats.pre_encode_time.IsFinite()) { + dropped_phase = FrameDropPhase::kByEncoder; + } else { + dropped_phase = FrameDropPhase::kBeforeEncoder; + } + stats->dropped_by_phase[dropped_phase]++; + } + + if (frame_stats.encoded_time.IsFinite()) { + stats->encode_time_ms.AddSample( + StatsSample(frame_stats.encoded_time - frame_stats.pre_encode_time, + frame_stats.encoded_time, metadata)); + stats->encode_frame_rate.AddEvent(frame_stats.encoded_time); + stats->total_encoded_images_payload += + frame_stats.encoded_image_size.bytes(); + stats->target_encode_bitrate.AddSample(StatsSample( + frame_stats.target_encode_bitrate, frame_stats.encoded_time, metadata)); + for (const auto& [spatial_layer, qp_values] : + frame_stats.spatial_layers_qp) { + for (SamplesStatsCounter::StatsSample qp : qp_values.GetTimedSamples()) { + qp.metadata = metadata; + stats->spatial_layers_qp[spatial_layer].AddSample(std::move(qp)); + } + } + + // Stats sliced on encoded frame type. + if (frame_stats.encoded_frame_type == VideoFrameType::kVideoFrameKey) { + ++stats->num_send_key_frames; + } + } + // Next stats can be calculated only if frame was received on remote side. + if (comparison.type != FrameComparisonType::kDroppedFrame || + comparison.frame_stats.decoder_failed) { + if (frame_stats.rendered_time.IsFinite()) { + stats->total_delay_incl_transport_ms.AddSample( + StatsSample(frame_stats.rendered_time - frame_stats.captured_time, + frame_stats.received_time, metadata)); + stats->receive_to_render_time_ms.AddSample( + StatsSample(frame_stats.rendered_time - frame_stats.received_time, + frame_stats.rendered_time, metadata)); + } + if (frame_stats.decode_start_time.IsFinite()) { + stats->transport_time_ms.AddSample( + StatsSample(frame_stats.decode_start_time - frame_stats.encoded_time, + frame_stats.decode_start_time, metadata)); + + // Stats sliced on decoded frame type. + if (frame_stats.pre_decoded_frame_type == + VideoFrameType::kVideoFrameKey) { + ++stats->num_recv_key_frames; + stats->recv_key_frame_size_bytes.AddSample( + StatsSample(frame_stats.pre_decoded_image_size.bytes(), + frame_stats.decode_start_time, metadata)); + } else if (frame_stats.pre_decoded_frame_type == + VideoFrameType::kVideoFrameDelta) { + stats->recv_delta_frame_size_bytes.AddSample( + StatsSample(frame_stats.pre_decoded_image_size.bytes(), + frame_stats.decode_start_time, metadata)); + } + } + if (frame_stats.decode_end_time.IsFinite()) { + stats->decode_time_ms.AddSample(StatsSample( + frame_stats.decode_end_time - frame_stats.decode_start_time, + frame_stats.decode_end_time, metadata)); + stats->resolution_of_decoded_frame.AddSample( + StatsSample(*comparison.frame_stats.decoded_frame_width * + *comparison.frame_stats.decoded_frame_height, + frame_stats.decode_end_time, metadata)); + } + + if (frame_stats.prev_frame_rendered_time.IsFinite() && + frame_stats.rendered_time.IsFinite()) { + TimeDelta time_between_rendered_frames = + frame_stats.rendered_time - frame_stats.prev_frame_rendered_time; + stats->time_between_rendered_frames_ms.AddSample(StatsSample( + time_between_rendered_frames, frame_stats.rendered_time, metadata)); + TimeDelta average_time_between_rendered_frames = TimeDelta::Millis( + stats->time_between_rendered_frames_ms.GetAverage()); + if (time_between_rendered_frames > + std::max(kFreezeThreshold + average_time_between_rendered_frames, + 3 * average_time_between_rendered_frames)) { + stats->freeze_time_ms.AddSample(StatsSample( + time_between_rendered_frames, frame_stats.rendered_time, metadata)); + auto freeze_end_it = + stream_last_freeze_end_time_.find(comparison.stats_key); + RTC_DCHECK(freeze_end_it != stream_last_freeze_end_time_.end()); + stats->time_between_freezes_ms.AddSample(StatsSample( + frame_stats.prev_frame_rendered_time - freeze_end_it->second, + frame_stats.rendered_time, metadata)); + freeze_end_it->second = frame_stats.rendered_time; + } + } + } + // Compute stream codec info. + if (frame_stats.used_encoder.has_value()) { + if (stats->encoders.empty() || stats->encoders.back().codec_name != + frame_stats.used_encoder->codec_name) { + stats->encoders.push_back(*frame_stats.used_encoder); + } + stats->encoders.back().last_frame_id = + frame_stats.used_encoder->last_frame_id; + stats->encoders.back().switched_from_at = + frame_stats.used_encoder->switched_from_at; + } + + if (frame_stats.used_decoder.has_value()) { + if (stats->decoders.empty() || stats->decoders.back().codec_name != + frame_stats.used_decoder->codec_name) { + stats->decoders.push_back(*frame_stats.used_decoder); + } + stats->decoders.back().last_frame_id = + frame_stats.used_decoder->last_frame_id; + stats->decoders.back().switched_from_at = + frame_stats.used_decoder->switched_from_at; + } +} + +Timestamp DefaultVideoQualityAnalyzerFramesComparator::Now() { + return clock_->CurrentTime(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h new file mode 100644 index 0000000000..006c3eb9bf --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_ + +#include <deque> +#include <map> +#include <utility> +#include <vector> + +#include "api/array_view.h" +#include "rtc_base/event.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/synchronization/mutex.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +namespace webrtc { + +struct FramesComparatorStats { + // Size of analyzer internal comparisons queue, measured when new element + // id added to the queue. + SamplesStatsCounter comparisons_queue_size; + // Number of performed comparisons of 2 video frames from captured and + // rendered streams. + int64_t comparisons_done = 0; + // Number of cpu overloaded comparisons. Comparison is cpu overloaded if it is + // queued when there are too many not processed comparisons in the queue. + // Overloaded comparison doesn't include metrics like SSIM and PSNR that + // require heavy computations. + int64_t cpu_overloaded_comparisons_done = 0; + // Number of memory overloaded comparisons. Comparison is memory overloaded if + // it is queued when its captured frame was already removed due to high memory + // usage for that video stream. + int64_t memory_overloaded_comparisons_done = 0; +}; + +// Performs comparisons of added frames and tracks frames related statistics. +// This class is thread safe. +class DefaultVideoQualityAnalyzerFramesComparator { + public: + // Creates frames comparator. + // Frames comparator doesn't use `options.enable_receive_own_stream` for any + // purposes, because it's unrelated to its functionality. + DefaultVideoQualityAnalyzerFramesComparator( + webrtc::Clock* clock, + DefaultVideoQualityAnalyzerCpuMeasurer& cpu_measurer, + DefaultVideoQualityAnalyzerOptions options = {}) + : options_(options), clock_(clock), cpu_measurer_(cpu_measurer) {} + ~DefaultVideoQualityAnalyzerFramesComparator() { Stop({}); } + + // Starts frames comparator. This method must be invoked before calling + // any other method on this object. + void Start(int max_threads_count); + // Stops frames comparator. This method will block until all added frame + // comparisons will be processed. After `Stop()` is invoked no more new + // comparisons can be added to this frames comparator. + // + // `last_rendered_frame_time` contains timestamps of last rendered frame for + // each (stream, sender, receiver) tuple to properly update time between + // freezes: it has include time from the last freeze until and of call. + void Stop( + const std::map<InternalStatsKey, Timestamp>& last_rendered_frame_times); + + // Ensures that stream `stream_index` has stats objects created for all + // potential receivers. This method must be called before adding any + // frames comparison for that stream. + void EnsureStatsForStream(size_t stream_index, + size_t sender_peer_index, + size_t peers_count, + Timestamp captured_time, + Timestamp start_time); + // Ensures that newly added participant will have stream stats objects created + // for all streams which they can receive. This method must be called before + // any frames comparison will be added for the newly added participant. + // + // `stream_started_time` - start time of each stream for which stats object + // has to be created. + // `start_time` - call start time. + void RegisterParticipantInCall( + rtc::ArrayView<std::pair<InternalStatsKey, Timestamp>> + stream_started_time, + Timestamp start_time); + + // `captured` - video frame captured by sender to use for PSNR/SSIM + // computation. If `type` is `FrameComparisonType::kRegular` and + // `captured` is `absl::nullopt` comparison is assumed to be overloaded + // due to memory constraints. + // `rendered` - video frame rendered by receiver to use for PSNR/SSIM + // computation. Required only if `type` is + // `FrameComparisonType::kRegular`, but can still be omitted if + // `captured` is `absl::nullopt`. + void AddComparison(InternalStatsKey stats_key, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats); + // `skipped_between_rendered` - amount of frames dropped on this stream before + // last received frame and current frame. + void AddComparison(InternalStatsKey stats_key, + int skipped_between_rendered, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats); + + std::map<InternalStatsKey, StreamStats> stream_stats() const { + MutexLock lock(&mutex_); + return stream_stats_; + } + FramesComparatorStats frames_comparator_stats() const { + MutexLock lock(&mutex_); + return frames_comparator_stats_; + } + + private: + enum State { kNew, kActive, kStopped }; + + void AddComparisonInternal(InternalStatsKey stats_key, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + void ProcessComparisons(); + void ProcessComparison(const FrameComparison& comparison); + Timestamp Now(); + + const DefaultVideoQualityAnalyzerOptions options_; + webrtc::Clock* const clock_; + DefaultVideoQualityAnalyzerCpuMeasurer& cpu_measurer_; + + mutable Mutex mutex_; + State state_ RTC_GUARDED_BY(mutex_) = State::kNew; + std::map<InternalStatsKey, StreamStats> stream_stats_ RTC_GUARDED_BY(mutex_); + std::map<InternalStatsKey, Timestamp> stream_last_freeze_end_time_ + RTC_GUARDED_BY(mutex_); + std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(mutex_); + FramesComparatorStats frames_comparator_stats_ RTC_GUARDED_BY(mutex_); + + std::vector<rtc::PlatformThread> thread_pool_; + rtc::Event comparison_available_event_; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc new file mode 100644 index 0000000000..8d3cd47ed6 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc @@ -0,0 +1,1648 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" + +#include <map> +#include <string> +#include <vector> + +#include "api/test/create_frame_generator.h" +#include "api/units/timestamp.h" +#include "rtc_base/strings/string_builder.h" +#include "system_wrappers/include/clock.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +namespace webrtc { +namespace { + +using ::testing::Contains; +using ::testing::DoubleEq; +using ::testing::Each; +using ::testing::Eq; +using ::testing::IsEmpty; +using ::testing::Pair; +using ::testing::SizeIs; + +using StatsSample = ::webrtc::SamplesStatsCounter::StatsSample; + +constexpr int kMaxFramesInFlightPerStream = 10; + +DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() { + DefaultVideoQualityAnalyzerOptions options; + options.compute_psnr = false; + options.compute_ssim = false; + options.adjust_cropping_before_comparing_frames = false; + options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream; + return options; +} + +VideoFrame CreateFrame(uint16_t frame_id, + int width, + int height, + Timestamp timestamp) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(width, height, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + test::FrameGeneratorInterface::VideoFrameData frame_data = + frame_generator->NextFrame(); + return VideoFrame::Builder() + .set_id(frame_id) + .set_video_frame_buffer(frame_data.buffer) + .set_update_rect(frame_data.update_rect) + .set_timestamp_us(timestamp.us()) + .build(); +} + +StreamCodecInfo Vp8CodecForOneFrame(uint16_t frame_id, Timestamp time) { + StreamCodecInfo info; + info.codec_name = "VP8"; + info.first_frame_id = frame_id; + info.last_frame_id = frame_id; + info.switched_on_at = time; + info.switched_from_at = time; + return info; +} + +FrameStats FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + uint16_t frame_id, + Timestamp captured_time) { + FrameStats frame_stats(frame_id, captured_time); + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + // Decode time is in microseconds. + frame_stats.decode_end_time = captured_time + TimeDelta::Micros(40010); + frame_stats.rendered_time = captured_time + TimeDelta::Millis(60); + frame_stats.used_encoder = Vp8CodecForOneFrame(1, frame_stats.encoded_time); + frame_stats.used_decoder = + Vp8CodecForOneFrame(1, frame_stats.decode_end_time); + frame_stats.decoded_frame_width = 10; + frame_stats.decoded_frame_height = 10; + return frame_stats; +} + +FrameStats ShiftStatsOn(const FrameStats& stats, TimeDelta delta) { + FrameStats frame_stats(stats.frame_id, stats.captured_time + delta); + frame_stats.pre_encode_time = stats.pre_encode_time + delta; + frame_stats.encoded_time = stats.encoded_time + delta; + frame_stats.received_time = stats.received_time + delta; + frame_stats.decode_start_time = stats.decode_start_time + delta; + frame_stats.decode_end_time = stats.decode_end_time + delta; + frame_stats.rendered_time = stats.rendered_time + delta; + + frame_stats.used_encoder = stats.used_encoder; + frame_stats.used_decoder = stats.used_decoder; + frame_stats.decoded_frame_width = stats.decoded_frame_width; + frame_stats.decoded_frame_height = stats.decoded_frame_height; + + return frame_stats; +} + +SamplesStatsCounter StatsCounter( + const std::vector<std::pair<double, Timestamp>>& samples) { + SamplesStatsCounter counter; + for (const std::pair<double, Timestamp>& sample : samples) { + counter.AddSample(SamplesStatsCounter::StatsSample{.value = sample.first, + .time = sample.second}); + } + return counter; +} + +double GetFirstOrDie(const SamplesStatsCounter& counter) { + EXPECT_FALSE(counter.IsEmpty()) << "Counter has to be not empty"; + return counter.GetSamples()[0]; +} + +void AssertFirstMetadataHasField(const SamplesStatsCounter& counter, + const std::string& field_name, + const std::string& field_value) { + EXPECT_FALSE(counter.IsEmpty()) << "Coutner has to be not empty"; + EXPECT_THAT(counter.GetTimedSamples()[0].metadata, + Contains(Pair(field_name, field_value))); +} + +std::string ToString(const SamplesStatsCounter& counter) { + rtc::StringBuilder out; + for (const StatsSample& s : counter.GetTimedSamples()) { + out << "{ time_ms=" << s.time.ms() << "; value=" << s.value << "}, "; + } + return out.str(); +} + +void ExpectEmpty(const SamplesStatsCounter& counter) { + EXPECT_TRUE(counter.IsEmpty()) + << "Expected empty SamplesStatsCounter, but got " << ToString(counter); +} + +void ExpectEmpty(const SamplesRateCounter& counter) { + EXPECT_TRUE(counter.IsEmpty()) + << "Expected empty SamplesRateCounter, but got " + << counter.GetEventsPerSecond(); +} + +void ExpectSizeAndAllElementsAre(const SamplesStatsCounter& counter, + int size, + double value) { + EXPECT_EQ(counter.NumSamples(), size); + EXPECT_THAT(counter.GetSamples(), Each(DoubleEq(value))); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + StatsPresentedAfterAddingOneComparison) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest()); + + Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + size_t peers_count = 2; + InternalStatsKey stats_key(stream, sender, receiver); + + FrameStats frame_stats = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + /*frame_id=*/1, stream_start_time); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, peers_count, + stream_start_time, stream_start_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kRegular, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + std::map<InternalStatsKey, StreamStats> stats = comparator.stream_stats(); + ExpectSizeAndAllElementsAre(stats.at(stats_key).transport_time_ms, /*size=*/1, + /*value=*/20.0); + ExpectSizeAndAllElementsAre(stats.at(stats_key).total_delay_incl_transport_ms, + /*size=*/1, /*value=*/60.0); + ExpectSizeAndAllElementsAre(stats.at(stats_key).encode_time_ms, /*size=*/1, + /*value=*/10.0); + ExpectSizeAndAllElementsAre(stats.at(stats_key).decode_time_ms, /*size=*/1, + /*value=*/0.01); + ExpectSizeAndAllElementsAre(stats.at(stats_key).receive_to_render_time_ms, + /*size=*/1, /*value=*/30.0); + ExpectSizeAndAllElementsAre(stats.at(stats_key).resolution_of_decoded_frame, + /*size=*/1, /*value=*/100.0); +} + +TEST( + DefaultVideoQualityAnalyzerFramesComparatorTest, + MultiFrameStatsPresentedWithMetadataAfterAddingTwoComparisonWith10msDelay) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest()); + + Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + size_t peers_count = 2; + InternalStatsKey stats_key(stream, sender, receiver); + + FrameStats frame_stats1 = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + /*frame_id=*/1, stream_start_time); + FrameStats frame_stats2 = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + /*frame_id=*/2, stream_start_time + TimeDelta::Millis(15)); + frame_stats2.prev_frame_rendered_time = frame_stats1.rendered_time; + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, peers_count, + stream_start_time, stream_start_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kRegular, frame_stats1); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kRegular, frame_stats2); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + std::map<InternalStatsKey, StreamStats> stats = comparator.stream_stats(); + ExpectSizeAndAllElementsAre( + stats.at(stats_key).time_between_rendered_frames_ms, /*size=*/1, + /*value=*/15.0); + AssertFirstMetadataHasField( + stats.at(stats_key).time_between_rendered_frames_ms, "frame_id", "2"); + EXPECT_DOUBLE_EQ(stats.at(stats_key).encode_frame_rate.GetEventsPerSecond(), + 2.0 / 15 * 1000) + << "There should be 2 events with interval of 15 ms"; +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + FrameInFlightStatsAreHandledCorrectly) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest()); + + Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + size_t peers_count = 2; + InternalStatsKey stats_key(stream, sender, receiver); + + // There are 7 different timings inside frame stats: captured, pre_encode, + // encoded, received, decode_start, decode_end, rendered. captured is always + // set and received is set together with decode_start. So we create 6 + // different frame stats with interval of 15 ms, where for each stat next + // timings will be set + // * 1st - captured + // * 2nd - captured, pre_encode + // * 3rd - captured, pre_encode, encoded + // * 4th - captured, pre_encode, encoded, received, decode_start + // * 5th - captured, pre_encode, encoded, received, decode_start, decode_end + // * 6th - all of them set + std::vector<FrameStats> stats; + // 1st stat + FrameStats frame_stats(/*frame_id=*/1, stream_start_time); + stats.push_back(frame_stats); + // 2nd stat + frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.frame_id = 2; + frame_stats.pre_encode_time = + frame_stats.captured_time + TimeDelta::Millis(10); + stats.push_back(frame_stats); + // 3rd stat + frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.frame_id = 3; + frame_stats.encoded_time = frame_stats.captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = Vp8CodecForOneFrame(1, frame_stats.encoded_time); + stats.push_back(frame_stats); + // 4th stat + frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.frame_id = 4; + frame_stats.received_time = frame_stats.captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = + frame_stats.captured_time + TimeDelta::Millis(40); + stats.push_back(frame_stats); + // 5th stat + frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.frame_id = 5; + frame_stats.decode_end_time = + frame_stats.captured_time + TimeDelta::Millis(50); + frame_stats.used_decoder = + Vp8CodecForOneFrame(1, frame_stats.decode_end_time); + frame_stats.decoded_frame_width = 10; + frame_stats.decoded_frame_height = 10; + stats.push_back(frame_stats); + // 6th stat + frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.frame_id = 6; + frame_stats.rendered_time = frame_stats.captured_time + TimeDelta::Millis(60); + stats.push_back(frame_stats); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, peers_count, + stream_start_time, stream_start_time); + for (size_t i = 0; i < stats.size() - 1; ++i) { + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, stats[i]); + } + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kRegular, + stats[stats.size() - 1]); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats result_stats = comparator.stream_stats().at(stats_key); + + EXPECT_DOUBLE_EQ(result_stats.transport_time_ms.GetAverage(), 20.0) + << ToString(result_stats.transport_time_ms); + EXPECT_EQ(result_stats.transport_time_ms.NumSamples(), 3); + + EXPECT_DOUBLE_EQ(result_stats.total_delay_incl_transport_ms.GetAverage(), + 60.0) + << ToString(result_stats.total_delay_incl_transport_ms); + EXPECT_EQ(result_stats.total_delay_incl_transport_ms.NumSamples(), 1); + + EXPECT_DOUBLE_EQ(result_stats.encode_time_ms.GetAverage(), 10) + << ToString(result_stats.encode_time_ms); + EXPECT_EQ(result_stats.encode_time_ms.NumSamples(), 4); + + EXPECT_DOUBLE_EQ(result_stats.decode_time_ms.GetAverage(), 10) + << ToString(result_stats.decode_time_ms); + EXPECT_EQ(result_stats.decode_time_ms.NumSamples(), 2); + + EXPECT_DOUBLE_EQ(result_stats.receive_to_render_time_ms.GetAverage(), 30) + << ToString(result_stats.receive_to_render_time_ms); + EXPECT_EQ(result_stats.receive_to_render_time_ms.NumSamples(), 1); + + EXPECT_DOUBLE_EQ(result_stats.resolution_of_decoded_frame.GetAverage(), 100) + << ToString(result_stats.resolution_of_decoded_frame); + EXPECT_EQ(result_stats.resolution_of_decoded_frame.NumSamples(), 2); + + EXPECT_DOUBLE_EQ(result_stats.encode_frame_rate.GetEventsPerSecond(), + 4.0 / 45 * 1000) + << "There should be 4 events with interval of 15 ms"; +} + +// Tests to validate that stats for each possible input frame are computed +// correctly. +// Frame in flight start +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + CapturedOnlyInFlightFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectEmpty(stats.encode_time_ms); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectEmpty(stats.target_encode_bitrate); + EXPECT_THAT(stats.spatial_layers_qp, IsEmpty()); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 0); + EXPECT_EQ(stats.num_send_key_frames, 0); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_THAT(stats.encoders, IsEmpty()); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + PreEncodedInFlightFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectEmpty(stats.encode_time_ms); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectEmpty(stats.target_encode_bitrate); + EXPECT_THAT(stats.spatial_layers_qp, IsEmpty()); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 0); + EXPECT_EQ(stats.num_send_key_frames, 0); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_THAT(stats.encoders, IsEmpty()); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + EncodedInFlightKeyFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 1); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + EncodedInFlightDeltaFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameDelta; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 0); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + PreDecodedInFlightKeyFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + // Frame pre decoded + frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.pre_decoded_image_size = DataSize::Bytes(500); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectSizeAndAllElementsAre(stats.transport_time_ms, /*size=*/1, + /*value=*/20.0); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1, + /*value=*/500.0); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 1); + EXPECT_EQ(stats.num_recv_key_frames, 1); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + DecodedInFlightKeyFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + // Frame pre decoded + frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.pre_decoded_image_size = DataSize::Bytes(500); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + // Frame decoded + frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50); + frame_stats.decoded_frame_width = 200; + frame_stats.decoded_frame_height = 100; + + frame_stats.used_decoder = + Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectSizeAndAllElementsAre(stats.transport_time_ms, /*size=*/1, + /*value=*/20.0); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectSizeAndAllElementsAre(stats.decode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + EXPECT_GE(GetFirstOrDie(stats.resolution_of_decoded_frame), 200 * 100.0); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1, + /*value=*/500.0); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 1); + EXPECT_EQ(stats.num_recv_key_frames, 1); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_EQ(stats.decoders, + std::vector<StreamCodecInfo>{*frame_stats.used_decoder}); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + DecoderFailureOnInFlightKeyFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + // Frame pre decoded + frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.pre_decoded_image_size = DataSize::Bytes(500); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + // Frame decoded + frame_stats.decoder_failed = true; + frame_stats.used_decoder = + Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectSizeAndAllElementsAre(stats.transport_time_ms, /*size=*/1, + /*value=*/20.0); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1, + /*value=*/500.0); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 1); + EXPECT_EQ(stats.num_recv_key_frames, 1); + // All frame in flight are not considered as dropped. + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_EQ(stats.decoders, + std::vector<StreamCodecInfo>{*frame_stats.used_decoder}); +} +// Frame in flight end + +// Dropped frame start +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + CapturedOnlyDroppedFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kDroppedFrame, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectEmpty(stats.encode_time_ms); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectEmpty(stats.target_encode_bitrate); + EXPECT_THAT(stats.spatial_layers_qp, IsEmpty()); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 0); + EXPECT_EQ(stats.num_send_key_frames, 0); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 1}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_THAT(stats.encoders, IsEmpty()); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + PreEncodedDroppedFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kDroppedFrame, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectEmpty(stats.encode_time_ms); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectEmpty(stats.target_encode_bitrate); + EXPECT_THAT(stats.spatial_layers_qp, IsEmpty()); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 0); + EXPECT_EQ(stats.num_send_key_frames, 0); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 1}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_THAT(stats.encoders, IsEmpty()); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + EncodedDroppedKeyFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kDroppedFrame, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 1); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 1}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + EncodedDroppedDeltaFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameDelta; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kDroppedFrame, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 0); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 1}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + PreDecodedDroppedKeyFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + // Frame pre decoded + frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.pre_decoded_image_size = DataSize::Bytes(500); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kDroppedFrame, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 1); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 1}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + DecodedDroppedKeyFrameAccountedInStats) { + // We don't really drop frames after decoder, so it's a bit unclear what is + // correct way to account such frames in stats, so this test just fixes some + // current way. + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + // Frame pre decoded + frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.pre_decoded_image_size = DataSize::Bytes(500); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + // Frame decoded + frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50); + frame_stats.used_decoder = + Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time); + frame_stats.decoded_frame_width = 200; + frame_stats.decoded_frame_height = 100; + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kDroppedFrame, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 1); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 1}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_EQ(stats.decoders, + std::vector<StreamCodecInfo>{*frame_stats.used_decoder}); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + DecoderFailedDroppedKeyFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + // Frame pre decoded + frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.pre_decoded_image_size = DataSize::Bytes(500); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + // Frame decoded + frame_stats.decoder_failed = true; + frame_stats.used_decoder = + Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kDroppedFrame, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectSizeAndAllElementsAre(stats.transport_time_ms, /*size=*/1, + /*value=*/20.0); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1, + /*value=*/500.0); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 1); + EXPECT_EQ(stats.num_recv_key_frames, 1); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 1}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_EQ(stats.decoders, + std::vector<StreamCodecInfo>{*frame_stats.used_decoder}); +} +// Dropped frame end + +// Regular frame start +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + RenderedKeyFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + VideoFrame frame = + CreateFrame(frame_id, /*width=*/320, /*height=*/180, captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + // Frame pre decoded + frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.pre_decoded_image_size = DataSize::Bytes(500); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + // Frame decoded + frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50); + frame_stats.used_decoder = + Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time); + frame_stats.decoded_frame_width = 200; + frame_stats.decoded_frame_height = 100; + // Frame rendered + frame_stats.rendered_time = captured_time + TimeDelta::Millis(60); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/frame, + /*rendered=*/frame, FrameComparisonType::kRegular, + frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + EXPECT_GE(GetFirstOrDie(stats.psnr), 20); + EXPECT_GE(GetFirstOrDie(stats.ssim), 0.5); + ExpectSizeAndAllElementsAre(stats.transport_time_ms, /*size=*/1, + /*value=*/20.0); + EXPECT_GE(GetFirstOrDie(stats.total_delay_incl_transport_ms), 60.0); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + EXPECT_GE(GetFirstOrDie(stats.decode_time_ms), 10.0); + EXPECT_GE(GetFirstOrDie(stats.receive_to_render_time_ms), 30.0); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + EXPECT_GE(GetFirstOrDie(stats.resolution_of_decoded_frame), 200 * 100.0); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1, + /*value=*/500.0); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 1); + EXPECT_EQ(stats.num_recv_key_frames, 1); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_EQ(stats.decoders, + std::vector<StreamCodecInfo>{*frame_stats.used_decoder}); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, AllStatsHaveMetadataSet) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + VideoFrame frame = + CreateFrame(frame_id, /*width=*/320, /*height=*/180, captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + // Frame pre decoded + frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.pre_decoded_image_size = DataSize::Bytes(500); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + // Frame decoded + frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50); + frame_stats.used_decoder = + Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time); + // Frame rendered + frame_stats.rendered_time = captured_time + TimeDelta::Millis(60); + frame_stats.decoded_frame_width = 200; + frame_stats.decoded_frame_height = 100; + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/frame, + /*rendered=*/frame, FrameComparisonType::kRegular, + frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + AssertFirstMetadataHasField(stats.psnr, "frame_id", "1"); + AssertFirstMetadataHasField(stats.ssim, "frame_id", "1"); + AssertFirstMetadataHasField(stats.transport_time_ms, "frame_id", "1"); + AssertFirstMetadataHasField(stats.total_delay_incl_transport_ms, "frame_id", + "1"); + AssertFirstMetadataHasField(stats.encode_time_ms, "frame_id", "1"); + AssertFirstMetadataHasField(stats.decode_time_ms, "frame_id", "1"); + AssertFirstMetadataHasField(stats.receive_to_render_time_ms, "frame_id", "1"); + AssertFirstMetadataHasField(stats.resolution_of_decoded_frame, "frame_id", + "1"); + AssertFirstMetadataHasField(stats.target_encode_bitrate, "frame_id", "1"); + AssertFirstMetadataHasField(stats.spatial_layers_qp[0], "frame_id", "1"); + AssertFirstMetadataHasField(stats.recv_key_frame_size_bytes, "frame_id", "1"); + + ExpectEmpty(stats.recv_delta_frame_size_bytes); +} +// Regular frame end + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + FreezeStatsPresentedWithMetadataAfterAddFrameWithSkippedAndDelay) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest()); + + Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + size_t peers_count = 2; + InternalStatsKey stats_key(stream, sender, receiver); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, peers_count, + stream_start_time, stream_start_time); + + // Add 5 frames which were rendered with 30 fps (~30ms between frames) + // Frame ids are in [1..5] and last frame is with 120ms offset from first. + Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity(); + for (int i = 0; i < 5; ++i) { + FrameStats frame_stats = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + /*frame_id=*/i + 1, stream_start_time + TimeDelta::Millis(30 * i)); + frame_stats.prev_frame_rendered_time = prev_frame_rendered_time; + prev_frame_rendered_time = frame_stats.rendered_time; + + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kRegular, frame_stats); + } + + // Next frame was rendered with 4 frames skipped and delay 300ms after last + // frame. + FrameStats freeze_frame_stats = + FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + /*frame_id=*/10, stream_start_time + TimeDelta::Millis(120 + 300)); + freeze_frame_stats.prev_frame_rendered_time = prev_frame_rendered_time; + + comparator.AddComparison(stats_key, + /*skipped_between_rendered=*/4, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kRegular, freeze_frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + StreamStats stats = comparator.stream_stats().at(stats_key); + ASSERT_THAT(GetFirstOrDie(stats.skipped_between_rendered), Eq(4)); + AssertFirstMetadataHasField(stats.skipped_between_rendered, "frame_id", "10"); + ASSERT_THAT(GetFirstOrDie(stats.freeze_time_ms), Eq(300)); + AssertFirstMetadataHasField(stats.freeze_time_ms, "frame_id", "10"); + // 180ms is time from the stream start to the rendered time of the last frame + // among first 5 frames which were received before freeze. + ASSERT_THAT(GetFirstOrDie(stats.time_between_freezes_ms), Eq(180)); + AssertFirstMetadataHasField(stats.time_between_freezes_ms, "frame_id", "10"); +} +// Stats validation tests end. + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.cc new file mode 100644 index 0000000000..16f49ef154 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.cc @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" + +#include "api/video/video_frame.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { + +std::string InternalStatsKey::ToString() const { + rtc::StringBuilder out; + out << "stream=" << stream << "_sender=" << sender + << "_receiver=" << receiver; + return out.str(); +} + +bool operator<(const InternalStatsKey& a, const InternalStatsKey& b) { + if (a.stream != b.stream) { + return a.stream < b.stream; + } + if (a.sender != b.sender) { + return a.sender < b.sender; + } + return a.receiver < b.receiver; +} + +bool operator==(const InternalStatsKey& a, const InternalStatsKey& b) { + return a.stream == b.stream && a.sender == b.sender && + a.receiver == b.receiver; +} + +FrameComparison::FrameComparison(InternalStatsKey stats_key, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats, + OverloadReason overload_reason) + : stats_key(std::move(stats_key)), + captured(std::move(captured)), + rendered(std::move(rendered)), + type(type), + frame_stats(std::move(frame_stats)), + overload_reason(overload_reason) {} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h new file mode 100644 index 0000000000..10f1314f46 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_INTERNAL_SHARED_OBJECTS_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_INTERNAL_SHARED_OBJECTS_H_ + +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_type.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +namespace webrtc { + +struct InternalStatsKey { + InternalStatsKey(size_t stream, size_t sender, size_t receiver) + : stream(stream), sender(sender), receiver(receiver) {} + + std::string ToString() const; + + size_t stream; + size_t sender; + size_t receiver; +}; + +// Required to use InternalStatsKey as std::map key. +bool operator<(const InternalStatsKey& a, const InternalStatsKey& b); +bool operator==(const InternalStatsKey& a, const InternalStatsKey& b); + +// Final stats computed for frame after it went through the whole video +// pipeline from capturing to rendering or dropping. +struct FrameStats { + FrameStats(uint16_t frame_id, Timestamp captured_time) + : frame_id(frame_id), captured_time(captured_time) {} + + uint16_t frame_id; + // Frame events timestamp. + Timestamp captured_time; + Timestamp pre_encode_time = Timestamp::MinusInfinity(); + Timestamp encoded_time = Timestamp::MinusInfinity(); + // Time when last packet of a frame was received. + Timestamp received_time = Timestamp::MinusInfinity(); + Timestamp decode_start_time = Timestamp::MinusInfinity(); + Timestamp decode_end_time = Timestamp::MinusInfinity(); + Timestamp rendered_time = Timestamp::MinusInfinity(); + Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity(); + + VideoFrameType encoded_frame_type = VideoFrameType::kEmptyFrame; + DataSize encoded_image_size = DataSize::Bytes(0); + VideoFrameType pre_decoded_frame_type = VideoFrameType::kEmptyFrame; + DataSize pre_decoded_image_size = DataSize::Bytes(0); + uint32_t target_encode_bitrate = 0; + // Sender side qp values per spatial layer. In case when spatial layer is not + // set for `webrtc::EncodedImage`, 0 is used as default. + std::map<int, SamplesStatsCounter> spatial_layers_qp; + + absl::optional<int> decoded_frame_width = absl::nullopt; + absl::optional<int> decoded_frame_height = absl::nullopt; + + // Can be not set if frame was dropped by encoder. + absl::optional<StreamCodecInfo> used_encoder = absl::nullopt; + // Can be not set if frame was dropped in the network. + absl::optional<StreamCodecInfo> used_decoder = absl::nullopt; + + bool decoder_failed = false; +}; + +// Describes why comparison was done in overloaded mode (without calculating +// PSNR and SSIM). +enum class OverloadReason { + kNone, + // Not enough CPU to process all incoming comparisons. + kCpu, + // Not enough memory to store captured frames for all comparisons. + kMemory +}; + +enum class FrameComparisonType { + // Comparison for captured and rendered frame. + kRegular, + // Comparison for captured frame that is known to be dropped somewhere in + // video pipeline. + kDroppedFrame, + // Comparison for captured frame that was still in the video pipeline when + // test was stopped. It's unknown is this frame dropped or would it be + // delivered if test continue. + kFrameInFlight +}; + +// Represents comparison between two VideoFrames. Contains video frames itself +// and stats. Can be one of two types: +// 1. Normal - in this case `captured` is presented and either `rendered` is +// presented and `dropped` is false, either `rendered` is omitted and +// `dropped` is true. +// 2. Overloaded - in this case both `captured` and `rendered` are omitted +// because there were too many comparisons in the queue. `dropped` can be +// true or false showing was frame dropped or not. +struct FrameComparison { + FrameComparison(InternalStatsKey stats_key, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats, + OverloadReason overload_reason); + + InternalStatsKey stats_key; + // Frames can be omitted if there too many computations waiting in the + // queue. + absl::optional<VideoFrame> captured; + absl::optional<VideoFrame> rendered; + FrameComparisonType type; + FrameStats frame_stats; + OverloadReason overload_reason; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_INTERNAL_SHARED_OBJECTS_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc new file mode 100644 index 0000000000..f5029ac956 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc @@ -0,0 +1,682 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <memory> +#include <string> +#include <vector> + +#include "api/rtp_packet_info.h" +#include "api/rtp_packet_infos.h" +#include "api/test/create_frame_generator.h" +#include "api/test/metrics/metric.h" +#include "api/test/metrics/metrics_logger.h" +#include "api/test/metrics/stdout_metrics_exporter.h" +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_tools/frame_analyzer/video_geometry_aligner.h" +#include "system_wrappers/include/sleep.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" + +namespace webrtc { +namespace { + +using ::testing::Contains; +using ::testing::SizeIs; +using ::testing::UnorderedElementsAre; + +using ::webrtc::test::DefaultMetricsLogger; +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Metric; +using ::webrtc::test::MetricsExporter; +using ::webrtc::test::StdoutMetricsExporter; +using ::webrtc::test::Unit; + +constexpr int kAnalyzerMaxThreadsCount = 1; +constexpr int kMaxFramesInFlightPerStream = 10; +constexpr int kFrameWidth = 320; +constexpr int kFrameHeight = 240; + +DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() { + DefaultVideoQualityAnalyzerOptions options; + options.compute_psnr = true; + options.compute_ssim = true; + options.adjust_cropping_before_comparing_frames = false; + options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream; + options.report_detailed_frame_stats = true; + return options; +} + +VideoFrame NextFrame(test::FrameGeneratorInterface* frame_generator, + int64_t timestamp_us) { + test::FrameGeneratorInterface::VideoFrameData frame_data = + frame_generator->NextFrame(); + return VideoFrame::Builder() + .set_video_frame_buffer(frame_data.buffer) + .set_update_rect(frame_data.update_rect) + .set_timestamp_us(timestamp_us) + .build(); +} + +EncodedImage FakeEncode(const VideoFrame& frame) { + EncodedImage image; + std::vector<RtpPacketInfo> packet_infos; + packet_infos.push_back(RtpPacketInfo( + /*ssrc=*/1, + /*csrcs=*/{}, + /*rtp_timestamp=*/frame.timestamp(), + /*receive_time=*/Timestamp::Micros(frame.timestamp_us() + 10000))); + image.SetPacketInfos(RtpPacketInfos(packet_infos)); + return image; +} + +VideoFrame DeepCopy(const VideoFrame& frame) { + VideoFrame copy = frame; + copy.set_video_frame_buffer( + I420Buffer::Copy(*frame.video_frame_buffer()->ToI420())); + return copy; +} + +void PassFramesThroughAnalyzer(DefaultVideoQualityAnalyzer& analyzer, + absl::string_view sender, + absl::string_view stream_label, + std::vector<absl::string_view> receivers, + int frames_count, + test::FrameGeneratorInterface& frame_generator, + int interframe_delay_ms = 0) { + for (int i = 0; i < frames_count; ++i) { + VideoFrame frame = NextFrame(&frame_generator, /*timestamp_us=*/1); + uint16_t frame_id = + analyzer.OnFrameCaptured(sender, std::string(stream_label), frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode(sender, frame); + analyzer.OnFrameEncoded(sender, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + for (absl::string_view receiver : receivers) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(receiver, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(receiver, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(receiver, received_frame); + } + if (i < frames_count - 1 && interframe_delay_ms > 0) { + SleepMs(interframe_delay_ms); + } + } +} + +// Metric fields to assert on +struct MetricValidationInfo { + std::string test_case; + std::string name; + Unit unit; + ImprovementDirection improvement_direction; +}; + +bool operator==(const MetricValidationInfo& a, const MetricValidationInfo& b) { + return a.name == b.name && a.test_case == b.test_case && a.unit == b.unit && + a.improvement_direction == b.improvement_direction; +} + +std::ostream& operator<<(std::ostream& os, const MetricValidationInfo& m) { + os << "{ test_case=" << m.test_case << "; name=" << m.name + << "; unit=" << test::ToString(m.unit) + << "; improvement_direction=" << test::ToString(m.improvement_direction) + << " }"; + return os; +} + +std::vector<MetricValidationInfo> ToValidationInfo( + const std::vector<Metric>& metrics) { + std::vector<MetricValidationInfo> out; + for (const Metric& m : metrics) { + out.push_back( + MetricValidationInfo{.test_case = m.test_case, + .name = m.name, + .unit = m.unit, + .improvement_direction = m.improvement_direction}); + } + return out; +} + +std::vector<std::string> ToTestCases(const std::vector<Metric>& metrics) { + std::vector<std::string> out; + for (const Metric& m : metrics) { + out.push_back(m.test_case); + } + return out; +} + +TEST(DefaultVideoQualityAnalyzerMetricNamesTest, MetricNamesForP2PAreCorrect) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultMetricsLogger metrics_logger(Clock::GetRealTimeClock()); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + &metrics_logger, options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"}, + /*frames_count=*/5, *frame_generator, + /*interframe_delay_ms=*/50); + analyzer.Stop(); + + std::vector<MetricValidationInfo> metrics = + ToValidationInfo(metrics_logger.GetCollectedMetrics()); + EXPECT_THAT( + metrics, + UnorderedElementsAre( + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "ssim", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "transport_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "total_delay_incl_transport", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "time_between_rendered_frames", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "harmonic_framerate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "encode_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "encode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "time_between_freezes", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "freeze_time_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "pixels_per_frame", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "min_psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "decode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "receive_to_render_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "dropped_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "frames_in_flight", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "rendered_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "max_skipped", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "target_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "qp_sl0", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "actual_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "capture_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "num_encoded_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "num_decoded_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "num_send_key_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "num_recv_key_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "recv_key_frame_size_bytes", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "recv_delta_frame_size_bytes", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{.test_case = "test_case", + .name = "cpu_usage_%", + .unit = Unit::kUnitless, + .improvement_direction = + ImprovementDirection::kSmallerIsBetter})); +} + +TEST(DefaultVideoQualityAnalyzerMetricNamesTest, + MetricNamesFor3PeersAreCorrect) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultMetricsLogger metrics_logger(Clock::GetRealTimeClock()); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + &metrics_logger, options); + analyzer.Start("test_case", + std::vector<std::string>{"alice", "bob", "charlie"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"bob", "charlie"}, + /*frames_count=*/5, *frame_generator, + /*interframe_delay_ms=*/50); + analyzer.Stop(); + + std::vector<MetricValidationInfo> metrics = + ToValidationInfo(metrics_logger.GetCollectedMetrics()); + EXPECT_THAT( + metrics, + UnorderedElementsAre( + // Bob + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "ssim", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "transport_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "total_delay_incl_transport", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "time_between_rendered_frames", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "harmonic_framerate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "encode_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "encode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "time_between_freezes", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "freeze_time_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "pixels_per_frame", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "min_psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "decode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "receive_to_render_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "dropped_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "frames_in_flight", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "rendered_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "max_skipped", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "target_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "qp_sl0", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "actual_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "capture_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "num_encoded_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "num_decoded_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "num_send_key_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "num_recv_key_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "recv_key_frame_size_bytes", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "recv_delta_frame_size_bytes", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + + // Charlie + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "ssim", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "transport_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "total_delay_incl_transport", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "time_between_rendered_frames", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "harmonic_framerate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "encode_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "encode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "time_between_freezes", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "freeze_time_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "pixels_per_frame", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "min_psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "decode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "receive_to_render_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "dropped_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "frames_in_flight", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "rendered_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "max_skipped", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "target_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "qp_sl0", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "actual_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "capture_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "num_encoded_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "num_decoded_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "num_send_key_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "num_recv_key_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "recv_key_frame_size_bytes", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "recv_delta_frame_size_bytes", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{.test_case = "test_case", + .name = "cpu_usage_%", + .unit = Unit::kUnitless, + .improvement_direction = + ImprovementDirection::kSmallerIsBetter})); +} + +TEST(DefaultVideoQualityAnalyzerMetricNamesTest, + TestCaseFor3PeerIsTheSameAfterAllPeersLeft) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultMetricsLogger metrics_logger(Clock::GetRealTimeClock()); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + &metrics_logger, options); + analyzer.Start("test_case", + std::vector<std::string>{"alice", "bob", "charlie"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"bob", "charlie"}, + /*frames_count=*/5, *frame_generator, + /*interframe_delay_ms=*/50); + analyzer.UnregisterParticipantInCall("alice"); + analyzer.UnregisterParticipantInCall("bob"); + analyzer.UnregisterParticipantInCall("charlie"); + analyzer.Stop(); + + std::vector<std::string> metrics = + ToTestCases(metrics_logger.GetCollectedMetrics()); + EXPECT_THAT(metrics, SizeIs(57)); + EXPECT_THAT(metrics, Contains("test_case/alice_video_alice_bob").Times(28)); + EXPECT_THAT(metrics, + Contains("test_case/alice_video_alice_charlie").Times(28)); + EXPECT_THAT(metrics, Contains("test_case").Times(1)); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc new file mode 100644 index 0000000000..79b9286e2d --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +#include <algorithm> +#include <iterator> +#include <ostream> +#include <string> + +#include "api/units/timestamp.h" +#include "rtc_base/checks.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { +namespace { + +constexpr int kMicrosPerSecond = 1000000; + +} // namespace + +std::string StreamCodecInfo::ToString() const { + rtc::StringBuilder out; + out << "{codec_name=" << codec_name << "; first_frame_id=" << first_frame_id + << "; last_frame_id=" << last_frame_id + << "; switched_on_at=" << webrtc::ToString(switched_on_at) + << "; switched_from_at=" << webrtc::ToString(switched_from_at) << " }"; + return out.str(); +} + +std::ostream& operator<<(std::ostream& os, const StreamCodecInfo& state) { + return os << state.ToString(); +} + +rtc::StringBuilder& operator<<(rtc::StringBuilder& sb, + const StreamCodecInfo& state) { + return sb << state.ToString(); +} + +bool operator==(const StreamCodecInfo& a, const StreamCodecInfo& b) { + return a.codec_name == b.codec_name && a.first_frame_id == b.first_frame_id && + a.last_frame_id == b.last_frame_id && + a.switched_on_at == b.switched_on_at && + a.switched_from_at == b.switched_from_at; +} + +std::string ToString(FrameDropPhase phase) { + switch (phase) { + case FrameDropPhase::kBeforeEncoder: + return "kBeforeEncoder"; + case FrameDropPhase::kByEncoder: + return "kByEncoder"; + case FrameDropPhase::kTransport: + return "kTransport"; + case FrameDropPhase::kByDecoder: + return "kByDecoder"; + case FrameDropPhase::kAfterDecoder: + return "kAfterDecoder"; + case FrameDropPhase::kLastValue: + return "kLastValue"; + } +} + +std::ostream& operator<<(std::ostream& os, FrameDropPhase phase) { + return os << ToString(phase); +} +rtc::StringBuilder& operator<<(rtc::StringBuilder& sb, FrameDropPhase phase) { + return sb << ToString(phase); +} + +void SamplesRateCounter::AddEvent(Timestamp event_time) { + if (event_first_time_.IsMinusInfinity()) { + event_first_time_ = event_time; + } + event_last_time_ = event_time; + events_count_++; +} + +double SamplesRateCounter::GetEventsPerSecond() const { + RTC_DCHECK(!IsEmpty()); + // Divide on us and multiply on kMicrosPerSecond to correctly process cases + // where there were too small amount of events, so difference is less then 1 + // sec. We can use us here, because Timestamp has us resolution. + return static_cast<double>(events_count_) / + (event_last_time_ - event_first_time_).us() * kMicrosPerSecond; +} + +StreamStats::StreamStats(Timestamp stream_started_time) + : stream_started_time(stream_started_time) { + for (int i = static_cast<int>(FrameDropPhase::kBeforeEncoder); + i < static_cast<int>(FrameDropPhase::kLastValue); ++i) { + dropped_by_phase.emplace(static_cast<FrameDropPhase>(i), 0); + } +} + +std::string StatsKey::ToString() const { + rtc::StringBuilder out; + out << stream_label << "_" << receiver; + return out.str(); +} + +bool operator<(const StatsKey& a, const StatsKey& b) { + if (a.stream_label != b.stream_label) { + return a.stream_label < b.stream_label; + } + return a.receiver < b.receiver; +} + +bool operator==(const StatsKey& a, const StatsKey& b) { + return a.stream_label == b.stream_label && a.receiver == b.receiver; +} + +VideoStreamsInfo::VideoStreamsInfo( + std::map<std::string, std::string> stream_to_sender, + std::map<std::string, std::set<std::string>> sender_to_streams, + std::map<std::string, std::set<std::string>> stream_to_receivers) + : stream_to_sender_(std::move(stream_to_sender)), + sender_to_streams_(std::move(sender_to_streams)), + stream_to_receivers_(std::move(stream_to_receivers)) {} + +std::set<StatsKey> VideoStreamsInfo::GetStatsKeys() const { + std::set<StatsKey> out; + for (const std::string& stream_label : GetStreams()) { + for (const std::string& receiver : GetReceivers(stream_label)) { + out.insert(StatsKey(stream_label, receiver)); + } + } + return out; +} + +std::set<std::string> VideoStreamsInfo::GetStreams() const { + std::set<std::string> out; + std::transform(stream_to_sender_.begin(), stream_to_sender_.end(), + std::inserter(out, out.end()), + [](auto map_entry) { return map_entry.first; }); + return out; +} + +std::set<std::string> VideoStreamsInfo::GetStreams( + absl::string_view sender_name) const { + auto it = sender_to_streams_.find(std::string(sender_name)); + if (it == sender_to_streams_.end()) { + return {}; + } + return it->second; +} + +absl::optional<std::string> VideoStreamsInfo::GetSender( + absl::string_view stream_label) const { + auto it = stream_to_sender_.find(std::string(stream_label)); + if (it == stream_to_sender_.end()) { + return absl::nullopt; + } + return it->second; +} + +std::set<std::string> VideoStreamsInfo::GetReceivers( + absl::string_view stream_label) const { + auto it = stream_to_receivers_.find(std::string(stream_label)); + if (it == stream_to_receivers_.end()) { + return {}; + } + return it->second; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h new file mode 100644 index 0000000000..175f777b68 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h @@ -0,0 +1,284 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_SHARED_OBJECTS_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_SHARED_OBJECTS_H_ + +#include <cstdint> +#include <map> +#include <memory> +#include <ostream> +#include <set> +#include <string> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/units/timestamp.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { + +// WebRTC will request a key frame after 3 seconds if no frames were received. +// We assume max frame rate ~60 fps, so 270 frames will cover max freeze without +// key frame request. +constexpr size_t kDefaultMaxFramesInFlightPerStream = 270; + +class SamplesRateCounter { + public: + void AddEvent(Timestamp event_time); + + bool IsEmpty() const { return event_first_time_ == event_last_time_; } + + double GetEventsPerSecond() const; + + private: + Timestamp event_first_time_ = Timestamp::MinusInfinity(); + Timestamp event_last_time_ = Timestamp::MinusInfinity(); + int64_t events_count_ = 0; +}; + +struct FrameCounters { + // Count of frames, that were passed into WebRTC pipeline by video stream + // source. + int64_t captured = 0; + // Count of frames that reached video encoder. + int64_t pre_encoded = 0; + // Count of encoded images that were produced by encoder for all requested + // spatial layers and simulcast streams. + int64_t encoded = 0; + // Count of encoded images received in decoder for all requested spatial + // layers and simulcast streams. + int64_t received = 0; + // Count of frames that were produced by decoder. + int64_t decoded = 0; + // Count of frames that went out from WebRTC pipeline to video sink. + int64_t rendered = 0; + // Count of frames that were dropped in any point between capturing and + // rendering. + int64_t dropped = 0; + // Count of frames for which decoder returned error when they were sent for + // decoding. + int64_t failed_to_decode = 0; +}; + +// Contains information about the codec that was used for encoding or decoding +// the stream. +struct StreamCodecInfo { + // Codec implementation name. + std::string codec_name; + // Id of the first frame for which this codec was used. + uint16_t first_frame_id; + // Id of the last frame for which this codec was used. + uint16_t last_frame_id; + // Timestamp when the first frame was handled by the encode/decoder. + Timestamp switched_on_at = Timestamp::PlusInfinity(); + // Timestamp when this codec was used last time. + Timestamp switched_from_at = Timestamp::PlusInfinity(); + + std::string ToString() const; +}; + +std::ostream& operator<<(std::ostream& os, const StreamCodecInfo& state); +rtc::StringBuilder& operator<<(rtc::StringBuilder& sb, + const StreamCodecInfo& state); +bool operator==(const StreamCodecInfo& a, const StreamCodecInfo& b); + +// Represents phases where video frame can be dropped and such drop will be +// detected by analyzer. +enum class FrameDropPhase : int { + kBeforeEncoder, + kByEncoder, + kTransport, + kByDecoder, + kAfterDecoder, + // kLastValue must be the last value in this enumeration. + kLastValue +}; + +std::string ToString(FrameDropPhase phase); +std::ostream& operator<<(std::ostream& os, FrameDropPhase phase); +rtc::StringBuilder& operator<<(rtc::StringBuilder& sb, FrameDropPhase phase); + +struct StreamStats { + explicit StreamStats(Timestamp stream_started_time); + + // The time when the first frame of this stream was captured. + Timestamp stream_started_time; + + // Spatial quality metrics. + SamplesStatsCounter psnr; + SamplesStatsCounter ssim; + + // Time from frame encoded (time point on exit from encoder) to the + // encoded image received in decoder (time point on entrance to decoder). + SamplesStatsCounter transport_time_ms; + // Time from frame was captured on device to time frame was displayed on + // device. + SamplesStatsCounter total_delay_incl_transport_ms; + // Time between frames out from renderer. + SamplesStatsCounter time_between_rendered_frames_ms; + SamplesRateCounter capture_frame_rate; + SamplesRateCounter encode_frame_rate; + SamplesStatsCounter encode_time_ms; + SamplesStatsCounter decode_time_ms; + // Time from last packet of frame is received until it's sent to the renderer. + SamplesStatsCounter receive_to_render_time_ms; + // Max frames skipped between two nearest. + SamplesStatsCounter skipped_between_rendered; + // In the next 2 metrics freeze is a pause that is longer, than maximum: + // 1. 150ms + // 2. 3 * average time between two sequential frames. + // Item 1 will cover high fps video and is a duration, that is noticeable by + // human eye. Item 2 will cover low fps video like screen sharing. + // Freeze duration. + SamplesStatsCounter freeze_time_ms; + // Mean time between one freeze end and next freeze start. + SamplesStatsCounter time_between_freezes_ms; + SamplesStatsCounter resolution_of_decoded_frame; + SamplesStatsCounter target_encode_bitrate; + // Sender side qp values per spatial layer. In case when spatial layer is not + // set for `webrtc::EncodedImage`, 0 is used as default. + std::map<int, SamplesStatsCounter> spatial_layers_qp; + + int64_t total_encoded_images_payload = 0; + // Counters on which phase how many frames were dropped. + std::map<FrameDropPhase, int64_t> dropped_by_phase; + + // Frame count metrics. + int64_t num_send_key_frames = 0; + int64_t num_recv_key_frames = 0; + + // Encoded frame size (in bytes) metrics. + SamplesStatsCounter recv_key_frame_size_bytes; + SamplesStatsCounter recv_delta_frame_size_bytes; + + // Vector of encoders used for this stream by sending client. + std::vector<StreamCodecInfo> encoders; + // Vectors of decoders used for this stream by receiving client. + std::vector<StreamCodecInfo> decoders; +}; + +struct AnalyzerStats { + // Size of analyzer internal comparisons queue, measured when new element + // id added to the queue. + SamplesStatsCounter comparisons_queue_size; + // Number of performed comparisons of 2 video frames from captured and + // rendered streams. + int64_t comparisons_done = 0; + // Number of cpu overloaded comparisons. Comparison is cpu overloaded if it is + // queued when there are too many not processed comparisons in the queue. + // Overloaded comparison doesn't include metrics like SSIM and PSNR that + // require heavy computations. + int64_t cpu_overloaded_comparisons_done = 0; + // Number of memory overloaded comparisons. Comparison is memory overloaded if + // it is queued when its captured frame was already removed due to high memory + // usage for that video stream. + int64_t memory_overloaded_comparisons_done = 0; + // Count of frames in flight in analyzer measured when new comparison is added + // and after analyzer was stopped. + SamplesStatsCounter frames_in_flight_left_count; + + // Next metrics are collected and reported iff + // `DefaultVideoQualityAnalyzerOptions::report_infra_metrics` is true. + SamplesStatsCounter on_frame_captured_processing_time_ms; + SamplesStatsCounter on_frame_pre_encode_processing_time_ms; + SamplesStatsCounter on_frame_encoded_processing_time_ms; + SamplesStatsCounter on_frame_pre_decode_processing_time_ms; + SamplesStatsCounter on_frame_decoded_processing_time_ms; + SamplesStatsCounter on_frame_rendered_processing_time_ms; + SamplesStatsCounter on_decoder_error_processing_time_ms; +}; + +struct StatsKey { + StatsKey(std::string stream_label, std::string receiver) + : stream_label(std::move(stream_label)), receiver(std::move(receiver)) {} + + std::string ToString() const; + + // Label of video stream to which stats belongs to. + std::string stream_label; + // Name of the peer on which stream was received. + std::string receiver; +}; + +// Required to use StatsKey as std::map key. +bool operator<(const StatsKey& a, const StatsKey& b); +bool operator==(const StatsKey& a, const StatsKey& b); + +// Contains all metadata related to the video streams that were seen by the +// video analyzer. +class VideoStreamsInfo { + public: + std::set<StatsKey> GetStatsKeys() const; + + // Returns all stream labels that are known to the video analyzer. + std::set<std::string> GetStreams() const; + + // Returns set of the stream for specified `sender_name`. If sender didn't + // send any streams or `sender_name` isn't known to the video analyzer + // empty set will be returned. + std::set<std::string> GetStreams(absl::string_view sender_name) const; + + // Returns sender name for specified `stream_label`. Returns `absl::nullopt` + // if provided `stream_label` isn't known to the video analyzer. + absl::optional<std::string> GetSender(absl::string_view stream_label) const; + + // Returns set of the receivers for specified `stream_label`. If stream wasn't + // received by any peer or `stream_label` isn't known to the video analyzer + // empty set will be returned. + std::set<std::string> GetReceivers(absl::string_view stream_label) const; + + protected: + friend class DefaultVideoQualityAnalyzer; + VideoStreamsInfo( + std::map<std::string, std::string> stream_to_sender, + std::map<std::string, std::set<std::string>> sender_to_streams, + std::map<std::string, std::set<std::string>> stream_to_receivers); + + private: + std::map<std::string, std::string> stream_to_sender_; + std::map<std::string, std::set<std::string>> sender_to_streams_; + std::map<std::string, std::set<std::string>> stream_to_receivers_; +}; + +struct DefaultVideoQualityAnalyzerOptions { + // Tells DefaultVideoQualityAnalyzer if heavy metrics have to be computed. + bool compute_psnr = true; + bool compute_ssim = true; + // If true, weights the luma plane more than the chroma planes in the PSNR. + bool use_weighted_psnr = false; + // Tells DefaultVideoQualityAnalyzer if detailed frame stats should be + // reported. + bool report_detailed_frame_stats = false; + // Tells DefaultVideoQualityAnalyzer if infra metrics related to the + // performance and stability of the analyzer itself should be reported. + bool report_infra_metrics = false; + // If true DefaultVideoQualityAnalyzer will try to adjust frames before + // computing PSNR and SSIM for them. In some cases picture may be shifted by + // a few pixels after the encode/decode step. Those difference is invisible + // for a human eye, but it affects the metrics. So the adjustment is used to + // get metrics that are closer to how human perceive the video. This feature + // significantly slows down the comparison, so turn it on only when it is + // needed. + bool adjust_cropping_before_comparing_frames = false; + // Amount of frames that are queued in the DefaultVideoQualityAnalyzer from + // the point they were captured to the point they were rendered on all + // receivers per stream. + size_t max_frames_in_flight_per_stream_count = + kDefaultMaxFramesInFlightPerStream; + // If true, the analyzer will expect peers to receive their own video streams. + bool enable_receive_own_stream = false; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_SHARED_OBJECTS_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.cc new file mode 100644 index 0000000000..d59ef12c63 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.cc @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h" + +#include <map> +#include <set> + +#include "absl/types/optional.h" +#include "api/units/timestamp.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +template <typename T> +absl::optional<T> MaybeGetValue(const std::map<size_t, T>& map, size_t key) { + auto it = map.find(key); + if (it == map.end()) { + return absl::nullopt; + } + return it->second; +} + +} // namespace + +StreamState::StreamState(size_t sender, + std::set<size_t> receivers, + Timestamp stream_started_time) + : sender_(sender), + stream_started_time_(stream_started_time), + receivers_(receivers), + frame_ids_(std::move(receivers)) { + frame_ids_.AddReader(kAliveFramesQueueIndex); + RTC_CHECK_NE(sender_, kAliveFramesQueueIndex); + for (size_t receiver : receivers_) { + RTC_CHECK_NE(receiver, kAliveFramesQueueIndex); + } +} + +uint16_t StreamState::PopFront(size_t peer) { + RTC_CHECK_NE(peer, kAliveFramesQueueIndex); + absl::optional<uint16_t> frame_id = frame_ids_.PopFront(peer); + RTC_DCHECK(frame_id.has_value()); + + // If alive's frame queue is longer than all others, than also pop frame from + // it, because that frame is received by all receivers. + size_t alive_size = frame_ids_.size(kAliveFramesQueueIndex); + size_t other_size = GetLongestReceiverQueue(); + // Pops frame from alive queue if alive's queue is the longest one. + if (alive_size > other_size) { + absl::optional<uint16_t> alive_frame_id = + frame_ids_.PopFront(kAliveFramesQueueIndex); + RTC_DCHECK(alive_frame_id.has_value()); + RTC_DCHECK_EQ(frame_id.value(), alive_frame_id.value()); + } + + return frame_id.value(); +} + +void StreamState::AddPeer(size_t peer) { + RTC_CHECK_NE(peer, kAliveFramesQueueIndex); + frame_ids_.AddReader(peer, kAliveFramesQueueIndex); + receivers_.insert(peer); +} + +void StreamState::RemovePeer(size_t peer) { + RTC_CHECK_NE(peer, kAliveFramesQueueIndex); + frame_ids_.RemoveReader(peer); + receivers_.erase(peer); + + // If we removed the last receiver for the alive frames, we need to pop them + // from the queue, because now they received by all receivers. + size_t alive_size = frame_ids_.size(kAliveFramesQueueIndex); + size_t other_size = GetLongestReceiverQueue(); + while (alive_size > other_size) { + frame_ids_.PopFront(kAliveFramesQueueIndex); + alive_size--; + } +} + +uint16_t StreamState::MarkNextAliveFrameAsDead() { + absl::optional<uint16_t> frame_id = + frame_ids_.PopFront(kAliveFramesQueueIndex); + RTC_DCHECK(frame_id.has_value()); + return frame_id.value(); +} + +void StreamState::SetLastRenderedFrameTime(size_t peer, Timestamp time) { + auto it = last_rendered_frame_time_.find(peer); + if (it == last_rendered_frame_time_.end()) { + last_rendered_frame_time_.insert({peer, time}); + } else { + it->second = time; + } +} + +absl::optional<Timestamp> StreamState::last_rendered_frame_time( + size_t peer) const { + return MaybeGetValue(last_rendered_frame_time_, peer); +} + +size_t StreamState::GetLongestReceiverQueue() const { + size_t max = 0; + for (size_t receiver : receivers_) { + size_t cur_size = frame_ids_.size(receiver); + if (cur_size > max) { + max = cur_size; + } + } + return max; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h new file mode 100644 index 0000000000..829a79c7bf --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_STREAM_STATE_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_STREAM_STATE_H_ + +#include <limits> +#include <map> +#include <set> + +#include "absl/types/optional.h" +#include "api/units/timestamp.h" +#include "test/pc/e2e/analyzer/video/multi_reader_queue.h" + +namespace webrtc { + +// Represents a current state of video stream inside +// DefaultVideoQualityAnalyzer. +// +// Maintains the sequence of frames for each video stream and keeps track about +// which frames were seen by each of the possible stream receiver. +// +// Keeps information about which frames are alive and which are dead. Frame is +// alive if it contains VideoFrame payload for corresponding FrameInFlight +// object inside DefaultVideoQualityAnalyzer, otherwise frame is considered +// dead. +// +// Supports peer indexes from 0 to max(size_t) - 1. +class StreamState { + public: + StreamState(size_t sender, + std::set<size_t> receivers, + Timestamp stream_started_time); + + size_t sender() const { return sender_; } + Timestamp stream_started_time() const { return stream_started_time_; } + + void PushBack(uint16_t frame_id) { frame_ids_.PushBack(frame_id); } + // Crash if state is empty. + uint16_t PopFront(size_t peer); + bool IsEmpty(size_t peer) const { return frame_ids_.IsEmpty(peer); } + // Crash if state is empty. + uint16_t Front(size_t peer) const { return frame_ids_.Front(peer).value(); } + + // Adds a new peer to the state. All currently alive frames will be expected + // to be received by the newly added peer. + void AddPeer(size_t peer); + + // Removes peer from the state. Frames that were expected to be received by + // this peer will be removed from it. On the other hand last rendered frame + // time for the removed peer will be preserved, because + // DefaultVideoQualityAnalyzer still may request it for stats processing. + void RemovePeer(size_t peer); + + size_t GetAliveFramesCount() const { + return frame_ids_.size(kAliveFramesQueueIndex); + } + uint16_t MarkNextAliveFrameAsDead(); + + void SetLastRenderedFrameTime(size_t peer, Timestamp time); + absl::optional<Timestamp> last_rendered_frame_time(size_t peer) const; + + private: + // Index of the `frame_ids_` queue which is used to track alive frames for + // this stream. + static constexpr size_t kAliveFramesQueueIndex = + std::numeric_limits<size_t>::max(); + + size_t GetLongestReceiverQueue() const; + + // Index of the owner. Owner's queue in `frame_ids_` will keep alive frames. + const size_t sender_; + const Timestamp stream_started_time_; + std::set<size_t> receivers_; + // To correctly determine dropped frames we have to know sequence of frames + // in each stream so we will keep a list of frame ids inside the stream. + // This list is represented by multi head queue of frame ids with separate + // head for each receiver. When the frame is rendered, we will pop ids from + // the corresponding head until id will match with rendered one. All ids + // before matched one can be considered as dropped: + // + // | frame_id1 |->| frame_id2 |->| frame_id3 |->| frame_id4 | + // + // If we received frame with id frame_id3, then we will pop frame_id1 and + // frame_id2 and consider those frames as dropped and then compare received + // frame with the one from `FrameInFlight` with id frame_id3. + MultiReaderQueue<uint16_t> frame_ids_; + std::map<size_t, Timestamp> last_rendered_frame_time_; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_STREAM_STATE_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state_test.cc new file mode 100644 index 0000000000..01a6aab28a --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state_test.cc @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h" + +#include <set> + +#include "api/units/timestamp.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +TEST(StreamStateTest, PopFrontAndFrontIndependentForEachPeer) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + state.PushBack(/*frame_id=*/2); + + EXPECT_EQ(state.Front(/*peer=*/1), 1); + EXPECT_EQ(state.PopFront(/*peer=*/1), 1); + EXPECT_EQ(state.Front(/*peer=*/1), 2); + EXPECT_EQ(state.PopFront(/*peer=*/1), 2); + EXPECT_EQ(state.Front(/*peer=*/2), 1); + EXPECT_EQ(state.PopFront(/*peer=*/2), 1); + EXPECT_EQ(state.Front(/*peer=*/2), 2); + EXPECT_EQ(state.PopFront(/*peer=*/2), 2); +} + +TEST(StreamStateTest, IsEmpty) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + + EXPECT_FALSE(state.IsEmpty(/*peer=*/1)); + + state.PopFront(/*peer=*/1); + + EXPECT_TRUE(state.IsEmpty(/*peer=*/1)); +} + +TEST(StreamStateTest, PopFrontForOnlyOnePeerDontChangeAliveFramesCount) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + state.PushBack(/*frame_id=*/2); + + EXPECT_EQ(state.GetAliveFramesCount(), 2lu); + + state.PopFront(/*peer=*/1); + state.PopFront(/*peer=*/1); + + EXPECT_EQ(state.GetAliveFramesCount(), 2lu); +} + +TEST(StreamStateTest, PopFrontForAllPeersReducesAliveFramesCount) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + state.PushBack(/*frame_id=*/2); + + EXPECT_EQ(state.GetAliveFramesCount(), 2lu); + + state.PopFront(/*peer=*/1); + state.PopFront(/*peer=*/2); + + EXPECT_EQ(state.GetAliveFramesCount(), 1lu); +} + +TEST(StreamStateTest, RemovePeerForLastExpectedReceiverUpdatesAliveFrames) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + state.PushBack(/*frame_id=*/2); + + state.PopFront(/*peer=*/1); + + EXPECT_EQ(state.GetAliveFramesCount(), 2lu); + + state.RemovePeer(/*peer=*/2); + + EXPECT_EQ(state.GetAliveFramesCount(), 1lu); +} + +TEST(StreamStateTest, MarkNextAliveFrameAsDeadDecreseAliveFramesCount) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + state.PushBack(/*frame_id=*/2); + + EXPECT_EQ(state.GetAliveFramesCount(), 2lu); + + state.MarkNextAliveFrameAsDead(); + + EXPECT_EQ(state.GetAliveFramesCount(), 1lu); +} + +TEST(StreamStateTest, MarkNextAliveFrameAsDeadDoesntAffectFrontFrameForPeer) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + state.PushBack(/*frame_id=*/2); + + EXPECT_EQ(state.Front(/*peer=*/1), 1); + + state.MarkNextAliveFrameAsDead(); + + EXPECT_EQ(state.Front(/*peer=*/1), 1); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc new file mode 100644 index 0000000000..fc970e1ea2 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc @@ -0,0 +1,2204 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" + +#include <algorithm> +#include <map> +#include <memory> +#include <vector> + +#include "api/rtp_packet_info.h" +#include "api/rtp_packet_infos.h" +#include "api/test/create_frame_generator.h" +#include "api/test/metrics/global_metrics_logger_and_exporter.h" +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_tools/frame_analyzer/video_geometry_aligner.h" +#include "system_wrappers/include/sleep.h" +#include "test/gtest.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +namespace webrtc { +namespace { + +using ::testing::TestWithParam; +using ::testing::ValuesIn; + +using StatsSample = ::webrtc::SamplesStatsCounter::StatsSample; + +constexpr int kAnalyzerMaxThreadsCount = 1; +constexpr int kMaxFramesInFlightPerStream = 10; +constexpr int kFrameWidth = 320; +constexpr int kFrameHeight = 240; +constexpr double kMaxSsim = 1; +constexpr char kStreamLabel[] = "video-stream"; +constexpr char kSenderPeerName[] = "alice"; +constexpr char kReceiverPeerName[] = "bob"; + +DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() { + DefaultVideoQualityAnalyzerOptions options; + options.compute_psnr = false; + options.compute_ssim = false; + options.adjust_cropping_before_comparing_frames = false; + options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream; + return options; +} + +VideoFrame NextFrame(test::FrameGeneratorInterface* frame_generator, + int64_t timestamp_us) { + test::FrameGeneratorInterface::VideoFrameData frame_data = + frame_generator->NextFrame(); + return VideoFrame::Builder() + .set_video_frame_buffer(frame_data.buffer) + .set_update_rect(frame_data.update_rect) + .set_timestamp_us(timestamp_us) + .build(); +} + +EncodedImage FakeEncode(const VideoFrame& frame) { + EncodedImage image; + std::vector<RtpPacketInfo> packet_infos; + packet_infos.push_back(RtpPacketInfo( + /*ssrc=*/1, + /*csrcs=*/{}, + /*rtp_timestamp=*/frame.timestamp(), + /*receive_time=*/Timestamp::Micros(frame.timestamp_us() + 10000))); + image.SetPacketInfos(RtpPacketInfos(packet_infos)); + return image; +} + +VideoFrame DeepCopy(const VideoFrame& frame) { + VideoFrame copy = frame; + copy.set_video_frame_buffer( + I420Buffer::Copy(*frame.video_frame_buffer()->ToI420())); + return copy; +} + +std::vector<StatsSample> GetSortedSamples(const SamplesStatsCounter& counter) { + rtc::ArrayView<const StatsSample> view = counter.GetTimedSamples(); + std::vector<StatsSample> out(view.begin(), view.end()); + std::sort(out.begin(), out.end(), + [](const StatsSample& a, const StatsSample& b) { + return a.time < b.time; + }); + return out; +} + +std::string ToString(const std::vector<StatsSample>& values) { + rtc::StringBuilder out; + for (const auto& v : values) { + out << "{ time_ms=" << v.time.ms() << "; value=" << v.value << "}, "; + } + return out.str(); +} + +void FakeCPULoad() { + std::vector<int> temp(1000000); + for (size_t i = 0; i < temp.size(); ++i) { + temp[i] = rand(); + } + std::sort(temp.begin(), temp.end()); + ASSERT_TRUE(std::is_sorted(temp.begin(), temp.end())); +} + +void PassFramesThroughAnalyzer(DefaultVideoQualityAnalyzer& analyzer, + absl::string_view sender, + absl::string_view stream_label, + std::vector<absl::string_view> receivers, + int frames_count, + test::FrameGeneratorInterface& frame_generator, + int interframe_delay_ms = 0) { + for (int i = 0; i < frames_count; ++i) { + VideoFrame frame = NextFrame(&frame_generator, /*timestamp_us=*/1); + uint16_t frame_id = + analyzer.OnFrameCaptured(sender, std::string(stream_label), frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode(sender, frame); + analyzer.OnFrameEncoded(sender, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + for (absl::string_view receiver : receivers) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(receiver, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(receiver, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(receiver, received_frame); + } + if (i < frames_count - 1 && interframe_delay_ms > 0) { + SleepMs(interframe_delay_ms); + } + } +} + +TEST(DefaultVideoQualityAnalyzerTest, + MemoryOverloadedAndThenAllFramesReceived) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + for (int i = 0; i < kMaxFramesInFlightPerStream * 2; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + for (const uint16_t& frame_id : frames_order) { + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, + kMaxFramesInFlightPerStream); + EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream * 2); + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream * 2); + EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream * 2); + EXPECT_EQ(frame_counters.dropped, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, + FillMaxMemoryReceiveAllMemoryOverloadedAndThenAllFramesReceived) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + // Feel analyzer's memory up to limit + for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + // Receive all frames. + for (const uint16_t& frame_id : frames_order) { + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + frames_order.clear(); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + + // Overload analyzer's memory up to limit + for (int i = 0; i < 2 * kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + // Receive all frames. + for (const uint16_t& frame_id : frames_order) { + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, + kMaxFramesInFlightPerStream); + EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream * 3); + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream * 3); + EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream * 3); + EXPECT_EQ(frame_counters.dropped, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, + MemoryOverloadedHalfDroppedAndThenHalfFramesReceived) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + for (int i = 0; i < kMaxFramesInFlightPerStream * 2; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + for (size_t i = kMaxFramesInFlightPerStream; i < frames_order.size(); ++i) { + uint16_t frame_id = frames_order.at(i); + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream * 2); + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream * 2); + EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream); + EXPECT_EQ(frame_counters.dropped, kMaxFramesInFlightPerStream); +} + +TEST(DefaultVideoQualityAnalyzerTest, NormalScenario) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + for (size_t i = 1; i < frames_order.size(); i += 2) { + uint16_t frame_id = frames_order.at(i); + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream); + + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream); + EXPECT_EQ(frame_counters.received, kMaxFramesInFlightPerStream / 2); + EXPECT_EQ(frame_counters.decoded, kMaxFramesInFlightPerStream / 2); + EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream / 2); + EXPECT_EQ(frame_counters.dropped, kMaxFramesInFlightPerStream / 2); +} + +TEST(DefaultVideoQualityAnalyzerTest, OneFrameReceivedTwice) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + VideoFrame captured_frame = NextFrame(frame_generator.get(), 0); + captured_frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, captured_frame)); + analyzer.OnFramePreEncode(kSenderPeerName, captured_frame); + analyzer.OnFrameEncoded(kSenderPeerName, captured_frame.id(), + FakeEncode(captured_frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + VideoFrame received_frame = DeepCopy(captured_frame); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + + received_frame = DeepCopy(captured_frame); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(stats.comparisons_done, 1); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 1); + EXPECT_EQ(frame_counters.received, 1); + EXPECT_EQ(frame_counters.decoded, 1); + EXPECT_EQ(frame_counters.rendered, 1); + EXPECT_EQ(frame_counters.dropped, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, NormalScenario2Receivers) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + constexpr char kAlice[] = "alice"; + constexpr char kBob[] = "bob"; + constexpr char kCharlie[] = "charlie"; + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie}, + kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id(analyzer.OnFrameCaptured(kAlice, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kAlice, frame); + SleepMs(20); + analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + SleepMs(50); + + for (size_t i = 1; i < frames_order.size(); i += 2) { + uint16_t frame_id = frames_order.at(i); + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kBob, received_frame.id(), + FakeEncode(received_frame)); + SleepMs(30); + analyzer.OnFrameDecoded(kBob, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + SleepMs(10); + analyzer.OnFrameRendered(kBob, received_frame); + } + + for (size_t i = 1; i < frames_order.size(); i += 2) { + uint16_t frame_id = frames_order.at(i); + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kCharlie, received_frame.id(), + FakeEncode(received_frame)); + SleepMs(40); + analyzer.OnFrameDecoded(kCharlie, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + SleepMs(5); + analyzer.OnFrameRendered(kCharlie, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(analyzer_stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(analyzer_stats.comparisons_done, kMaxFramesInFlightPerStream * 2); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream); + EXPECT_EQ(frame_counters.received, kMaxFramesInFlightPerStream); + EXPECT_EQ(frame_counters.decoded, kMaxFramesInFlightPerStream); + EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream); + EXPECT_EQ(frame_counters.dropped, kMaxFramesInFlightPerStream); + + VideoStreamsInfo streams_info = analyzer.GetKnownStreams(); + EXPECT_EQ(streams_info.GetStreams(), std::set<std::string>{kStreamLabel}); + EXPECT_EQ(streams_info.GetStreams(kAlice), + std::set<std::string>{kStreamLabel}); + EXPECT_EQ(streams_info.GetSender(kStreamLabel), kAlice); + EXPECT_EQ(streams_info.GetReceivers(kStreamLabel), + (std::set<std::string>{kBob, kCharlie})); + + EXPECT_EQ(streams_info.GetStatsKeys().size(), 2lu); + for (auto stream_key : streams_info.GetStatsKeys()) { + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(stream_key); + // On some devices the pipeline can be too slow, so we actually can't + // force real constraints here. Lets just check, that at least 1 + // frame passed whole pipeline. + EXPECT_GE(stream_conters.captured, 10); + EXPECT_GE(stream_conters.pre_encoded, 10); + EXPECT_GE(stream_conters.encoded, 10); + EXPECT_GE(stream_conters.received, 5); + EXPECT_GE(stream_conters.decoded, 5); + EXPECT_GE(stream_conters.rendered, 5); + EXPECT_GE(stream_conters.dropped, 5); + } + + std::map<StatsKey, StreamStats> stats = analyzer.GetStats(); + const StatsKey kAliceBobStats(kStreamLabel, kBob); + const StatsKey kAliceCharlieStats(kStreamLabel, kCharlie); + EXPECT_EQ(stats.size(), 2lu); + { + auto it = stats.find(kAliceBobStats); + EXPECT_FALSE(it == stats.end()); + ASSERT_FALSE(it->second.encode_time_ms.IsEmpty()); + EXPECT_GE(it->second.encode_time_ms.GetMin(), 20); + ASSERT_FALSE(it->second.decode_time_ms.IsEmpty()); + EXPECT_GE(it->second.decode_time_ms.GetMin(), 30); + ASSERT_FALSE(it->second.resolution_of_decoded_frame.IsEmpty()); + EXPECT_GE(it->second.resolution_of_decoded_frame.GetMin(), + kFrameWidth * kFrameHeight - 1); + EXPECT_LE(it->second.resolution_of_decoded_frame.GetMax(), + kFrameWidth * kFrameHeight + 1); + } + { + auto it = stats.find(kAliceCharlieStats); + EXPECT_FALSE(it == stats.end()); + ASSERT_FALSE(it->second.encode_time_ms.IsEmpty()); + EXPECT_GE(it->second.encode_time_ms.GetMin(), 20); + ASSERT_FALSE(it->second.decode_time_ms.IsEmpty()); + EXPECT_GE(it->second.decode_time_ms.GetMin(), 30); + ASSERT_FALSE(it->second.resolution_of_decoded_frame.IsEmpty()); + EXPECT_GE(it->second.resolution_of_decoded_frame.GetMin(), + kFrameWidth * kFrameHeight - 1); + EXPECT_LE(it->second.resolution_of_decoded_frame.GetMax(), + kFrameWidth * kFrameHeight + 1); + } +} + +// Test the case which can happen when SFU is switching from one layer to +// another, so the same frame can be received twice by the same peer. +TEST(DefaultVideoQualityAnalyzerTest, + OneFrameReceivedTwiceBySamePeerWith2Receivers) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + constexpr char kAlice[] = "alice"; + constexpr char kBob[] = "bob"; + constexpr char kCharlie[] = "charlie"; + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie}, + kAnalyzerMaxThreadsCount); + + VideoFrame captured_frame = NextFrame(frame_generator.get(), 0); + captured_frame.set_id( + analyzer.OnFrameCaptured(kAlice, kStreamLabel, captured_frame)); + analyzer.OnFramePreEncode(kAlice, captured_frame); + analyzer.OnFrameEncoded(kAlice, captured_frame.id(), + FakeEncode(captured_frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + VideoFrame received_frame = DeepCopy(captured_frame); + analyzer.OnFramePreDecode(kBob, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kBob, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kBob, received_frame); + + received_frame = DeepCopy(captured_frame); + analyzer.OnFramePreDecode(kBob, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kBob, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kBob, received_frame); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + // We have 2 comparisons here because 1 for the frame received by Bob and + // 1 for the frame in flight from Alice to Charlie. + EXPECT_EQ(stats.comparisons_done, 2); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 1); + EXPECT_EQ(frame_counters.received, 1); + EXPECT_EQ(frame_counters.decoded, 1); + EXPECT_EQ(frame_counters.rendered, 1); + EXPECT_EQ(frame_counters.dropped, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, HeavyQualityMetricsFromEqualFrames) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions analyzer_options; + analyzer_options.compute_psnr = true; + analyzer_options.compute_ssim = true; + analyzer_options.adjust_cropping_before_comparing_frames = false; + analyzer_options.max_frames_in_flight_per_stream_count = + kMaxFramesInFlightPerStream; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + analyzer_options); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. Heavy metrics + // computation is turned on, so giving some extra time to be sure that + // computatio have ended. + SleepMs(500); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream); + + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + std::map<StatsKey, StreamStats> stream_stats = analyzer.GetStats(); + const StatsKey kAliceBobStats(kStreamLabel, kReceiverPeerName); + EXPECT_EQ(stream_stats.size(), 1lu); + + auto it = stream_stats.find(kAliceBobStats); + EXPECT_GE(it->second.psnr.GetMin(), kPerfectPSNR); + EXPECT_GE(it->second.ssim.GetMin(), kMaxSsim); +} + +TEST(DefaultVideoQualityAnalyzerTest, + HeavyQualityMetricsFromShiftedFramesWithAdjustment) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions analyzer_options; + analyzer_options.compute_psnr = true; + analyzer_options.compute_ssim = true; + analyzer_options.adjust_cropping_before_comparing_frames = true; + analyzer_options.max_frames_in_flight_per_stream_count = + kMaxFramesInFlightPerStream; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + analyzer_options); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + + VideoFrame received_frame = frame; + // Shift frame by a few pixels. + test::CropRegion crop_region{0, 1, 3, 0}; + rtc::scoped_refptr<VideoFrameBuffer> cropped_buffer = + CropAndZoom(crop_region, received_frame.video_frame_buffer()->ToI420()); + received_frame.set_video_frame_buffer(cropped_buffer); + + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. Heavy metrics + // computation is turned on, so giving some extra time to be sure that + // computatio have ended. + SleepMs(500); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream); + + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + std::map<StatsKey, StreamStats> stream_stats = analyzer.GetStats(); + const StatsKey kAliceBobStats(kStreamLabel, kReceiverPeerName); + EXPECT_EQ(stream_stats.size(), 1lu); + + auto it = stream_stats.find(kAliceBobStats); + EXPECT_GE(it->second.psnr.GetMin(), kPerfectPSNR); + EXPECT_GE(it->second.ssim.GetMin(), kMaxSsim); +} + +TEST(DefaultVideoQualityAnalyzerTest, CpuUsage) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + // Windows CPU clock has low accuracy. We need to fake some additional load to + // be sure that the clock ticks (https://crbug.com/webrtc/12249). + FakeCPULoad(); + + for (size_t i = 1; i < frames_order.size(); i += 2) { + uint16_t frame_id = frames_order.at(i); + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + double cpu_usage = analyzer.GetCpuUsagePercent(); + ASSERT_GT(cpu_usage, 0); + + SleepMs(100); + analyzer.Stop(); + + EXPECT_EQ(analyzer.GetCpuUsagePercent(), cpu_usage); +} + +TEST(DefaultVideoQualityAnalyzerTest, RuntimeParticipantsAdding) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + constexpr char kAlice[] = "alice"; + constexpr char kBob[] = "bob"; + constexpr char kCharlie[] = "charlie"; + constexpr char kKatie[] = "katie"; + + constexpr int kFramesCount = 9; + constexpr int kOneThirdFrames = kFramesCount / 3; + constexpr int kTwoThirdFrames = 2 * kOneThirdFrames; + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", {}, kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + analyzer.RegisterParticipantInCall(kAlice); + analyzer.RegisterParticipantInCall(kBob); + + // Alice is sending frames. + for (int i = 0; i < kFramesCount; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id(analyzer.OnFrameCaptured(kAlice, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kAlice, frame); + analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + // Bob receives one third of the sent frames. + for (int i = 0; i < kOneThirdFrames; ++i) { + uint16_t frame_id = frames_order.at(i); + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kBob, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kBob, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kBob, received_frame); + } + + analyzer.RegisterParticipantInCall(kCharlie); + analyzer.RegisterParticipantInCall(kKatie); + + // New participants were dynamically added. Bob and Charlie receive second + // third of the sent frames. Katie drops the frames. + for (int i = kOneThirdFrames; i < kTwoThirdFrames; ++i) { + uint16_t frame_id = frames_order.at(i); + VideoFrame bob_received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kBob, bob_received_frame.id(), + FakeEncode(bob_received_frame)); + analyzer.OnFrameDecoded(kBob, bob_received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kBob, bob_received_frame); + + VideoFrame charlie_received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kCharlie, charlie_received_frame.id(), + FakeEncode(charlie_received_frame)); + analyzer.OnFrameDecoded(kCharlie, charlie_received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kCharlie, charlie_received_frame); + } + + // Bob, Charlie and Katie receive the rest of the sent frames. + for (int i = kTwoThirdFrames; i < kFramesCount; ++i) { + uint16_t frame_id = frames_order.at(i); + VideoFrame bob_received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kBob, bob_received_frame.id(), + FakeEncode(bob_received_frame)); + analyzer.OnFrameDecoded(kBob, bob_received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kBob, bob_received_frame); + + VideoFrame charlie_received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kCharlie, charlie_received_frame.id(), + FakeEncode(charlie_received_frame)); + analyzer.OnFrameDecoded(kCharlie, charlie_received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kCharlie, charlie_received_frame); + + VideoFrame katie_received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kKatie, katie_received_frame.id(), + FakeEncode(katie_received_frame)); + analyzer.OnFrameDecoded(kKatie, katie_received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kKatie, katie_received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(stats.comparisons_done, kFramesCount + 2 * kTwoThirdFrames); + + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, kFramesCount); + EXPECT_EQ(frame_counters.received, 2 * kFramesCount); + EXPECT_EQ(frame_counters.decoded, 2 * kFramesCount); + EXPECT_EQ(frame_counters.rendered, 2 * kFramesCount); + EXPECT_EQ(frame_counters.dropped, kOneThirdFrames); + + const StatsKey kAliceBobStats(kStreamLabel, kBob); + const StatsKey kAliceCharlieStats(kStreamLabel, kCharlie); + const StatsKey kAliceKatieStats(kStreamLabel, kKatie); + EXPECT_EQ(analyzer.GetKnownStreams().GetStatsKeys(), + (std::set<StatsKey>{kAliceBobStats, kAliceCharlieStats, + kAliceKatieStats})); + { + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(kAliceBobStats); + EXPECT_EQ(stream_conters.captured, kFramesCount); + EXPECT_EQ(stream_conters.pre_encoded, kFramesCount); + EXPECT_EQ(stream_conters.encoded, kFramesCount); + EXPECT_EQ(stream_conters.received, kFramesCount); + EXPECT_EQ(stream_conters.decoded, kFramesCount); + EXPECT_EQ(stream_conters.rendered, kFramesCount); + } + { + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(kAliceCharlieStats); + EXPECT_EQ(stream_conters.captured, kFramesCount); + EXPECT_EQ(stream_conters.pre_encoded, kFramesCount); + EXPECT_EQ(stream_conters.encoded, kFramesCount); + EXPECT_EQ(stream_conters.received, kTwoThirdFrames); + EXPECT_EQ(stream_conters.decoded, kTwoThirdFrames); + EXPECT_EQ(stream_conters.rendered, kTwoThirdFrames); + } + { + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(kAliceKatieStats); + EXPECT_EQ(stream_conters.captured, kFramesCount); + EXPECT_EQ(stream_conters.pre_encoded, kFramesCount); + EXPECT_EQ(stream_conters.encoded, kFramesCount); + EXPECT_EQ(stream_conters.received, kOneThirdFrames); + EXPECT_EQ(stream_conters.decoded, kOneThirdFrames); + EXPECT_EQ(stream_conters.rendered, kOneThirdFrames); + } +} + +TEST(DefaultVideoQualityAnalyzerTest, + SimulcastFrameWasFullyReceivedByAllPeersBeforeEncodeFinish) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + constexpr char kAlice[] = "alice"; + constexpr char kBob[] = "bob"; + constexpr char kCharlie[] = "charlie"; + analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), 1); + + frame.set_id(analyzer.OnFrameCaptured(kAlice, kStreamLabel, frame)); + analyzer.OnFramePreEncode(kAlice, frame); + // Encode 1st simulcast layer + analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + // Receive by Bob + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kBob, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kBob, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kBob, received_frame); + // Receive by Charlie + received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kCharlie, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kCharlie, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kCharlie, received_frame); + + // Encode 2nd simulcast layer + analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.comparisons_done, 2); + + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 1); + EXPECT_EQ(frame_counters.rendered, 2); +} + +TEST(DefaultVideoQualityAnalyzerTest, + FrameCanBeReceivedBySenderAfterItWasReceivedByReceiver) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::vector<VideoFrame> frames; + for (int i = 0; i < 3; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), 1); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames.push_back(frame); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + // Receive by 2nd peer. + for (VideoFrame& frame : frames) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Check that we still have that frame in flight. + AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats(); + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(analyzer_stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 3) + << "Expected that frame is still in flight, " + << "because it wasn't received by sender" + << ToString(frames_in_flight_sizes); + + // Receive by sender + for (VideoFrame& frame : frames) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kSenderPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kSenderPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kSenderPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + analyzer_stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(analyzer_stats.comparisons_done, 6); + + frames_in_flight_sizes = + GetSortedSamples(analyzer_stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 3); + EXPECT_EQ(frame_counters.rendered, 6); + + EXPECT_EQ(analyzer.GetStats().size(), 2lu); + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kReceiverPeerName)); + EXPECT_EQ(stream_conters.captured, 3); + EXPECT_EQ(stream_conters.pre_encoded, 3); + EXPECT_EQ(stream_conters.encoded, 3); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 3); + EXPECT_EQ(stream_conters.rendered, 3); + } + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kSenderPeerName)); + EXPECT_EQ(stream_conters.captured, 3); + EXPECT_EQ(stream_conters.pre_encoded, 3); + EXPECT_EQ(stream_conters.encoded, 3); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 3); + EXPECT_EQ(stream_conters.rendered, 3); + } +} + +TEST(DefaultVideoQualityAnalyzerTest, + FrameCanBeReceivedByReceiverAfterItWasReceivedBySender) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::vector<VideoFrame> frames; + for (int i = 0; i < 3; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), 1); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames.push_back(frame); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + // Receive by sender + for (VideoFrame& frame : frames) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kSenderPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kSenderPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kSenderPeerName, received_frame); + } + + // Check that we still have that frame in flight. + AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats(); + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(analyzer_stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 3) + << "Expected that frame is still in flight, " + << "because it wasn't received by sender" + << ToString(frames_in_flight_sizes); + + // Receive by 2nd peer. + for (VideoFrame& frame : frames) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + analyzer_stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(analyzer_stats.comparisons_done, 6); + + frames_in_flight_sizes = + GetSortedSamples(analyzer_stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 3); + EXPECT_EQ(frame_counters.rendered, 6); + + EXPECT_EQ(analyzer.GetStats().size(), 2lu); + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kReceiverPeerName)); + EXPECT_EQ(stream_conters.captured, 3); + EXPECT_EQ(stream_conters.pre_encoded, 3); + EXPECT_EQ(stream_conters.encoded, 3); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 3); + EXPECT_EQ(stream_conters.rendered, 3); + } + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kSenderPeerName)); + EXPECT_EQ(stream_conters.captured, 3); + EXPECT_EQ(stream_conters.pre_encoded, 3); + EXPECT_EQ(stream_conters.encoded, 3); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 3); + EXPECT_EQ(stream_conters.rendered, 3); + } +} + +TEST(DefaultVideoQualityAnalyzerTest, CodecTrackedCorrectly) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + VideoQualityAnalyzerInterface::EncoderStats encoder_stats; + std::vector<std::string> codec_names = {"codec_1", "codec_2"}; + std::vector<VideoFrame> frames; + // Send 3 frame for each codec. + for (size_t i = 0; i < codec_names.size(); ++i) { + for (size_t j = 0; j < 3; ++j) { + VideoFrame frame = NextFrame(frame_generator.get(), 3 * i + j); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + encoder_stats.encoder_name = codec_names[i]; + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + encoder_stats, false); + frames.push_back(std::move(frame)); + } + } + + // Receive 3 frame for each codec. + VideoQualityAnalyzerInterface::DecoderStats decoder_stats; + for (size_t i = 0; i < codec_names.size(); ++i) { + for (size_t j = 0; j < 3; ++j) { + VideoFrame received_frame = DeepCopy(frames[3 * i + j]); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + decoder_stats.decoder_name = codec_names[i]; + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, decoder_stats); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + std::map<StatsKey, StreamStats> stats = analyzer.GetStats(); + ASSERT_EQ(stats.size(), 1lu); + const StreamStats& stream_stats = + stats.at(StatsKey(kStreamLabel, kReceiverPeerName)); + ASSERT_EQ(stream_stats.encoders.size(), 2lu); + EXPECT_EQ(stream_stats.encoders[0].codec_name, codec_names[0]); + EXPECT_EQ(stream_stats.encoders[0].first_frame_id, frames[0].id()); + EXPECT_EQ(stream_stats.encoders[0].last_frame_id, frames[2].id()); + EXPECT_EQ(stream_stats.encoders[1].codec_name, codec_names[1]); + EXPECT_EQ(stream_stats.encoders[1].first_frame_id, frames[3].id()); + EXPECT_EQ(stream_stats.encoders[1].last_frame_id, frames[5].id()); + + ASSERT_EQ(stream_stats.decoders.size(), 2lu); + EXPECT_EQ(stream_stats.decoders[0].codec_name, codec_names[0]); + EXPECT_EQ(stream_stats.decoders[0].first_frame_id, frames[0].id()); + EXPECT_EQ(stream_stats.decoders[0].last_frame_id, frames[2].id()); + EXPECT_EQ(stream_stats.decoders[1].codec_name, codec_names[1]); + EXPECT_EQ(stream_stats.decoders[1].first_frame_id, frames[3].id()); + EXPECT_EQ(stream_stats.decoders[1].last_frame_id, frames[5].id()); +} + +TEST(DefaultVideoQualityAnalyzerTest, + FramesInFlightAreCorrectlySentToTheComparatorAfterStop) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + // There are 7 different timings inside frame stats: captured, pre_encode, + // encoded, received, decode_start, decode_end, rendered. captured is always + // set and received is set together with decode_start. So we create 6 + // different frames, where for each frame next timings will be set + // * 1st - all of them set + // * 2nd - captured, pre_encode, encoded, received, decode_start, decode_end + // * 3rd - captured, pre_encode, encoded, received, decode_start + // * 4th - captured, pre_encode, encoded + // * 5th - captured, pre_encode + // * 6th - captured + std::vector<VideoFrame> frames; + // Sender side actions + for (int i = 0; i < 6; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), 1); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames.push_back(frame); + } + for (int i = 0; i < 5; ++i) { + analyzer.OnFramePreEncode(kSenderPeerName, frames[i]); + } + for (int i = 0; i < 4; ++i) { + analyzer.OnFrameEncoded( + kSenderPeerName, frames[i].id(), FakeEncode(frames[i]), + VideoQualityAnalyzerInterface::EncoderStats(), false); + } + + // Receiver side actions + for (int i = 0; i < 3; ++i) { + analyzer.OnFramePreDecode(kReceiverPeerName, frames[i].id(), + FakeEncode(frames[i])); + } + for (int i = 0; i < 2; ++i) { + analyzer.OnFrameDecoded(kReceiverPeerName, DeepCopy(frames[i]), + VideoQualityAnalyzerInterface::DecoderStats()); + } + for (int i = 0; i < 1; ++i) { + analyzer.OnFrameRendered(kReceiverPeerName, DeepCopy(frames[i])); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(analyzer_stats.comparisons_done, 6); + + // The last frames in flight size has to reflect the amount of frame in flight + // before all of them were sent to the comparison when Stop() was invoked. + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(analyzer_stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 5) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 6); + EXPECT_EQ(frame_counters.pre_encoded, 5); + EXPECT_EQ(frame_counters.encoded, 4); + EXPECT_EQ(frame_counters.received, 3); + EXPECT_EQ(frame_counters.decoded, 2); + EXPECT_EQ(frame_counters.rendered, 1); + + EXPECT_EQ(analyzer.GetStats().size(), 1lu); + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kReceiverPeerName)); + EXPECT_EQ(stream_conters.captured, 6); + EXPECT_EQ(stream_conters.pre_encoded, 5); + EXPECT_EQ(stream_conters.encoded, 4); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 2); + EXPECT_EQ(stream_conters.rendered, 1); + } +} + +TEST( + DefaultVideoQualityAnalyzerTest, + FramesInFlightAreCorrectlySentToTheComparatorAfterStopForSenderAndReceiver) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + // There are 7 different timings inside frame stats: captured, pre_encode, + // encoded, received, decode_start, decode_end, rendered. captured is always + // set and received is set together with decode_start. So we create 6 + // different frames, where for each frame next timings will be set + // * 1st - all of them set + // * 2nd - captured, pre_encode, encoded, received, decode_start, decode_end + // * 3rd - captured, pre_encode, encoded, received, decode_start + // * 4th - captured, pre_encode, encoded + // * 5th - captured, pre_encode + // * 6th - captured + std::vector<VideoFrame> frames; + // Sender side actions + for (int i = 0; i < 6; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), 1); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames.push_back(frame); + } + for (int i = 0; i < 5; ++i) { + analyzer.OnFramePreEncode(kSenderPeerName, frames[i]); + } + for (int i = 0; i < 4; ++i) { + analyzer.OnFrameEncoded( + kSenderPeerName, frames[i].id(), FakeEncode(frames[i]), + VideoQualityAnalyzerInterface::EncoderStats(), false); + } + + // Receiver side actions + for (int i = 0; i < 3; ++i) { + analyzer.OnFramePreDecode(kSenderPeerName, frames[i].id(), + FakeEncode(frames[i])); + analyzer.OnFramePreDecode(kReceiverPeerName, frames[i].id(), + FakeEncode(frames[i])); + } + for (int i = 0; i < 2; ++i) { + analyzer.OnFrameDecoded(kSenderPeerName, DeepCopy(frames[i]), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameDecoded(kReceiverPeerName, DeepCopy(frames[i]), + VideoQualityAnalyzerInterface::DecoderStats()); + } + for (int i = 0; i < 1; ++i) { + analyzer.OnFrameRendered(kSenderPeerName, DeepCopy(frames[i])); + analyzer.OnFrameRendered(kReceiverPeerName, DeepCopy(frames[i])); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(analyzer_stats.comparisons_done, 12); + + // The last frames in flight size has to reflect the amount of frame in flight + // before all of them were sent to the comparison when Stop() was invoked. + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(analyzer_stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 5) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 6); + EXPECT_EQ(frame_counters.pre_encoded, 5); + EXPECT_EQ(frame_counters.encoded, 4); + EXPECT_EQ(frame_counters.received, 6); + EXPECT_EQ(frame_counters.decoded, 4); + EXPECT_EQ(frame_counters.rendered, 2); + + EXPECT_EQ(analyzer.GetStats().size(), 2lu); + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kReceiverPeerName)); + EXPECT_EQ(stream_conters.captured, 6); + EXPECT_EQ(stream_conters.pre_encoded, 5); + EXPECT_EQ(stream_conters.encoded, 4); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 2); + EXPECT_EQ(stream_conters.rendered, 1); + } + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kSenderPeerName)); + EXPECT_EQ(stream_conters.captured, 6); + EXPECT_EQ(stream_conters.pre_encoded, 5); + EXPECT_EQ(stream_conters.encoded, 4); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 2); + EXPECT_EQ(stream_conters.rendered, 1); + } +} + +TEST(DefaultVideoQualityAnalyzerTest, GetStreamFrames) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + // The order in which peers captured frames and passed them to analyzer. + std::vector<std::string> frame_capturers_sequence{ + "alice", "alice", "bob", "bob", "bob", + "bob", "bob", "alice", "alice", "alice", + }; + + std::map<std::string, std::vector<uint16_t>> stream_to_frame_ids; + stream_to_frame_ids.emplace("alice_video", std::vector<uint16_t>{}); + stream_to_frame_ids.emplace("bob_video", std::vector<uint16_t>{}); + + std::vector<VideoFrame> frames; + for (const std::string& sender : frame_capturers_sequence) { + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = + analyzer.OnFrameCaptured(sender, sender + "_video", frame); + frame.set_id(frame_id); + stream_to_frame_ids.find(sender + "_video")->second.push_back(frame_id); + frames.push_back(frame); + analyzer.OnFramePreEncode(sender, frame); + analyzer.OnFrameEncoded(sender, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + // We don't need to receive frames for stats to be gathered correctly. + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + EXPECT_EQ(analyzer.GetStreamFrames(), stream_to_frame_ids); +} + +TEST(DefaultVideoQualityAnalyzerTest, ReceiverReceivedFramesWhenSenderRemoved) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + analyzer.UnregisterParticipantInCall("alice"); + + analyzer.OnFramePreDecode("bob", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("bob", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("bob", DeepCopy(frame)); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 1); + EXPECT_EQ(stream_conters.decoded, 1); + EXPECT_EQ(stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, + ReceiverReceivedFramesWhenSenderRemovedWithSelfview) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + analyzer.UnregisterParticipantInCall("alice"); + + analyzer.OnFramePreDecode("bob", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("bob", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("bob", DeepCopy(frame)); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 1); + EXPECT_EQ(stream_conters.decoded, 1); + EXPECT_EQ(stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, + SenderReceivedFramesWhenReceiverRemovedWithSelfview) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + analyzer.UnregisterParticipantInCall("bob"); + + analyzer.OnFramePreDecode("alice", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("alice", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("alice", DeepCopy(frame)); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "alice")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 1); + EXPECT_EQ(stream_conters.decoded, 1); + EXPECT_EQ(stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, + SenderAndReceiverReceivedFramesWhenReceiverRemovedWithSelfview) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + analyzer.OnFramePreDecode("bob", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("bob", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("bob", DeepCopy(frame)); + + analyzer.UnregisterParticipantInCall("bob"); + + analyzer.OnFramePreDecode("alice", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("alice", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("alice", DeepCopy(frame)); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters alice_alice_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "alice")); + EXPECT_EQ(alice_alice_stream_conters.captured, 1); + EXPECT_EQ(alice_alice_stream_conters.pre_encoded, 1); + EXPECT_EQ(alice_alice_stream_conters.encoded, 1); + EXPECT_EQ(alice_alice_stream_conters.received, 1); + EXPECT_EQ(alice_alice_stream_conters.decoded, 1); + EXPECT_EQ(alice_alice_stream_conters.rendered, 1); + + FrameCounters alice_bob_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(alice_bob_stream_conters.captured, 1); + EXPECT_EQ(alice_bob_stream_conters.pre_encoded, 1); + EXPECT_EQ(alice_bob_stream_conters.encoded, 1); + EXPECT_EQ(alice_bob_stream_conters.received, 1); + EXPECT_EQ(alice_bob_stream_conters.decoded, 1); + EXPECT_EQ(alice_bob_stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeCapturing2ndFrame) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"}, + /*frames_count=*/1, *frame_generator); + analyzer.UnregisterParticipantInCall("bob"); + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {}, + /*frames_count=*/1, *frame_generator); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 2); + EXPECT_EQ(global_stream_conters.pre_encoded, 2); + EXPECT_EQ(global_stream_conters.encoded, 2); + EXPECT_EQ(global_stream_conters.received, 1); + EXPECT_EQ(global_stream_conters.decoded, 1); + EXPECT_EQ(global_stream_conters.rendered, 1); + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 2); + EXPECT_EQ(stream_conters.pre_encoded, 2); + EXPECT_EQ(stream_conters.encoded, 2); + EXPECT_EQ(stream_conters.received, 1); + EXPECT_EQ(stream_conters.decoded, 1); + EXPECT_EQ(stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforePreEncoded) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.UnregisterParticipantInCall("bob"); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 1); + EXPECT_EQ(global_stream_conters.pre_encoded, 1); + EXPECT_EQ(global_stream_conters.encoded, 1); + EXPECT_EQ(global_stream_conters.received, 0); + EXPECT_EQ(global_stream_conters.decoded, 0); + EXPECT_EQ(global_stream_conters.rendered, 0); + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 0); + EXPECT_EQ(stream_conters.decoded, 0); + EXPECT_EQ(stream_conters.rendered, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeEncoded) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.UnregisterParticipantInCall("bob"); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 1); + EXPECT_EQ(global_stream_conters.pre_encoded, 1); + EXPECT_EQ(global_stream_conters.encoded, 1); + EXPECT_EQ(global_stream_conters.received, 0); + EXPECT_EQ(global_stream_conters.decoded, 0); + EXPECT_EQ(global_stream_conters.rendered, 0); + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 0); + EXPECT_EQ(stream_conters.decoded, 0); + EXPECT_EQ(stream_conters.rendered, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, + ReceiverRemovedBetweenSimulcastLayersEncoded) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + // 1st simulcast layer encoded + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + analyzer.UnregisterParticipantInCall("bob"); + // 2nd simulcast layer encoded + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 1); + EXPECT_EQ(global_stream_conters.pre_encoded, 1); + EXPECT_EQ(global_stream_conters.encoded, 1); + EXPECT_EQ(global_stream_conters.received, 0); + EXPECT_EQ(global_stream_conters.decoded, 0); + EXPECT_EQ(global_stream_conters.rendered, 0); + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 0); + EXPECT_EQ(stream_conters.decoded, 0); + EXPECT_EQ(stream_conters.rendered, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, UnregisterOneAndRegisterAnother) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", + std::vector<std::string>{"alice", "bob", "charlie"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"bob", "charlie"}, + /*frames_count=*/2, *frame_generator); + analyzer.UnregisterParticipantInCall("bob"); + analyzer.RegisterParticipantInCall("david"); + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"charlie", "david"}, + /*frames_count=*/4, *frame_generator); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 6); + EXPECT_EQ(global_stream_conters.pre_encoded, 6); + EXPECT_EQ(global_stream_conters.encoded, 6); + EXPECT_EQ(global_stream_conters.received, 12); + EXPECT_EQ(global_stream_conters.decoded, 12); + EXPECT_EQ(global_stream_conters.rendered, 12); + FrameCounters alice_bob_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(alice_bob_stream_conters.captured, 6); + EXPECT_EQ(alice_bob_stream_conters.pre_encoded, 6); + EXPECT_EQ(alice_bob_stream_conters.encoded, 6); + EXPECT_EQ(alice_bob_stream_conters.received, 2); + EXPECT_EQ(alice_bob_stream_conters.decoded, 2); + EXPECT_EQ(alice_bob_stream_conters.rendered, 2); + FrameCounters alice_charlie_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "charlie")); + EXPECT_EQ(alice_charlie_stream_conters.captured, 6); + EXPECT_EQ(alice_charlie_stream_conters.pre_encoded, 6); + EXPECT_EQ(alice_charlie_stream_conters.encoded, 6); + EXPECT_EQ(alice_charlie_stream_conters.received, 6); + EXPECT_EQ(alice_charlie_stream_conters.decoded, 6); + EXPECT_EQ(alice_charlie_stream_conters.rendered, 6); + FrameCounters alice_david_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "david")); + EXPECT_EQ(alice_david_stream_conters.captured, 6); + EXPECT_EQ(alice_david_stream_conters.pre_encoded, 6); + EXPECT_EQ(alice_david_stream_conters.encoded, 6); + EXPECT_EQ(alice_david_stream_conters.received, 4); + EXPECT_EQ(alice_david_stream_conters.decoded, 4); + EXPECT_EQ(alice_david_stream_conters.rendered, 4); +} + +TEST(DefaultVideoQualityAnalyzerTest, + UnregisterOneAndRegisterAnotherRegisterBack) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", + std::vector<std::string>{"alice", "bob", "charlie"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"bob", "charlie"}, + /*frames_count=*/2, *frame_generator); + analyzer.UnregisterParticipantInCall("bob"); + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"charlie"}, + /*frames_count=*/4, *frame_generator); + analyzer.RegisterParticipantInCall("bob"); + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"bob", "charlie"}, + /*frames_count=*/6, *frame_generator); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 12); + EXPECT_EQ(global_stream_conters.pre_encoded, 12); + EXPECT_EQ(global_stream_conters.encoded, 12); + EXPECT_EQ(global_stream_conters.received, 20); + EXPECT_EQ(global_stream_conters.decoded, 20); + EXPECT_EQ(global_stream_conters.rendered, 20); + FrameCounters alice_bob_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(alice_bob_stream_conters.captured, 12); + EXPECT_EQ(alice_bob_stream_conters.pre_encoded, 12); + EXPECT_EQ(alice_bob_stream_conters.encoded, 12); + EXPECT_EQ(alice_bob_stream_conters.received, 8); + EXPECT_EQ(alice_bob_stream_conters.decoded, 8); + EXPECT_EQ(alice_bob_stream_conters.rendered, 8); + FrameCounters alice_charlie_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "charlie")); + EXPECT_EQ(alice_charlie_stream_conters.captured, 12); + EXPECT_EQ(alice_charlie_stream_conters.pre_encoded, 12); + EXPECT_EQ(alice_charlie_stream_conters.encoded, 12); + EXPECT_EQ(alice_charlie_stream_conters.received, 12); + EXPECT_EQ(alice_charlie_stream_conters.decoded, 12); + EXPECT_EQ(alice_charlie_stream_conters.rendered, 12); +} + +TEST(DefaultVideoQualityAnalyzerTest, + FramesInFlightAreAccountedForUnregisterPeers) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + // Add one frame in flight which has encode time >= 10ms. + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + SleepMs(10); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + analyzer.UnregisterParticipantInCall("bob"); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + StreamStats stats = analyzer.GetStats().at(StatsKey("alice_video", "bob")); + ASSERT_EQ(stats.encode_time_ms.NumSamples(), 1); + EXPECT_GE(stats.encode_time_ms.GetAverage(), 10); +} + +TEST(DefaultVideoQualityAnalyzerTest, InfraMetricsAreReportedWhenRequested) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.report_infra_metrics = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"}, + /*frames_count=*/1, *frame_generator); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.on_frame_captured_processing_time_ms.NumSamples(), 1); + EXPECT_EQ(stats.on_frame_pre_encode_processing_time_ms.NumSamples(), 1); + EXPECT_EQ(stats.on_frame_encoded_processing_time_ms.NumSamples(), 1); + EXPECT_EQ(stats.on_frame_pre_decode_processing_time_ms.NumSamples(), 1); + EXPECT_EQ(stats.on_frame_decoded_processing_time_ms.NumSamples(), 1); + EXPECT_EQ(stats.on_frame_rendered_processing_time_ms.NumSamples(), 1); + EXPECT_EQ(stats.on_decoder_error_processing_time_ms.NumSamples(), 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, InfraMetricsNotCollectedByDefault) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.report_infra_metrics = false; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"}, + /*frames_count=*/1, *frame_generator); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.on_frame_captured_processing_time_ms.NumSamples(), 0); + EXPECT_EQ(stats.on_frame_pre_encode_processing_time_ms.NumSamples(), 0); + EXPECT_EQ(stats.on_frame_encoded_processing_time_ms.NumSamples(), 0); + EXPECT_EQ(stats.on_frame_pre_decode_processing_time_ms.NumSamples(), 0); + EXPECT_EQ(stats.on_frame_decoded_processing_time_ms.NumSamples(), 0); + EXPECT_EQ(stats.on_frame_rendered_processing_time_ms.NumSamples(), 0); + EXPECT_EQ(stats.on_decoder_error_processing_time_ms.NumSamples(), 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, + FrameDroppedByDecoderIsAccountedCorrectly) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.report_infra_metrics = false; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame to_be_dropped_frame = + NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = + analyzer.OnFrameCaptured("alice", "alice_video", to_be_dropped_frame); + to_be_dropped_frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", to_be_dropped_frame); + analyzer.OnFrameEncoded("alice", to_be_dropped_frame.id(), + FakeEncode(to_be_dropped_frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + VideoFrame received_to_be_dropped_frame = DeepCopy(to_be_dropped_frame); + analyzer.OnFramePreDecode("bob", received_to_be_dropped_frame.id(), + FakeEncode(received_to_be_dropped_frame)); + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"}, + /*frames_count=*/1, *frame_generator); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + StreamStats stats = analyzer.GetStats().at(StatsKey("alice_video", "bob")); + ASSERT_EQ(stats.dropped_by_phase[FrameDropPhase::kByDecoder], 1); +} + +class DefaultVideoQualityAnalyzerTimeBetweenFreezesTest + : public TestWithParam<bool> {}; + +TEST_P(DefaultVideoQualityAnalyzerTimeBetweenFreezesTest, + TimeBetweenFreezesIsEqualToStreamDurationWhenThereAreNoFeeezes) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"}, + /*frames_count=*/5, *frame_generator, + /*interframe_delay_ms=*/50); + if (GetParam()) { + analyzer.UnregisterParticipantInCall("bob"); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(50); + analyzer.Stop(); + + StreamStats stats = analyzer.GetStats().at(StatsKey("alice_video", "bob")); + ASSERT_EQ(stats.time_between_freezes_ms.NumSamples(), 1); + EXPECT_GE(stats.time_between_freezes_ms.GetAverage(), 200); +} + +INSTANTIATE_TEST_SUITE_P(WithRegisteredAndUnregisteredPeerAtTheEndOfTheCall, + DefaultVideoQualityAnalyzerTimeBetweenFreezesTest, + ValuesIn({true, false})); + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h new file mode 100644 index 0000000000..384e901462 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_ + +#include <cstdint> +#include <utility> + +#include "absl/types/optional.h" +#include "api/video/encoded_image.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Injects frame id into EncodedImage on encoder side +class EncodedImageDataInjector { + public: + virtual ~EncodedImageDataInjector() = default; + + // Return encoded image with specified `id` and `discard` flag injected into + // its payload. `discard` flag mean does analyzing decoder should discard this + // encoded image because it belongs to unnecessary simulcast stream or spatial + // layer. + virtual EncodedImage InjectData(uint16_t id, + bool discard, + const EncodedImage& source) = 0; +}; + +struct EncodedImageExtractionResult { + absl::optional<uint16_t> id; + EncodedImage image; + // Is true if encoded image should be discarded. It is used to filter out + // unnecessary spatial layers and simulcast streams. + bool discard; +}; + +// Extracts frame id from EncodedImage on decoder side. +class EncodedImageDataExtractor { + public: + virtual ~EncodedImageDataExtractor() = default; + + // Invoked by framework before any image will come to the extractor. + // `expected_receivers_count` is the expected amount of receivers for each + // encoded image. + virtual void Start(int expected_receivers_count) = 0; + + // Invoked by framework when it is required to add one more receiver for + // frames. Will be invoked before that receiver will start receive data. + virtual void AddParticipantInCall() = 0; + + // Invoked by framework when it is required to remove receiver for frames. + // Will be invoked after that receiver will stop receiving data. + virtual void RemoveParticipantInCall() = 0; + + // Returns encoded image id, extracted from payload and also encoded image + // with its original payload. For concatenated spatial layers it should be the + // same id. + virtual EncodedImageExtractionResult ExtractData( + const EncodedImage& source) = 0; +}; + +class EncodedImageDataPropagator : public EncodedImageDataInjector, + public EncodedImageDataExtractor { + public: + ~EncodedImageDataPropagator() override = default; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc new file mode 100644 index 0000000000..da9c53beb9 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/example_video_quality_analyzer.h" + +#include "api/array_view.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +ExampleVideoQualityAnalyzer::ExampleVideoQualityAnalyzer() = default; +ExampleVideoQualityAnalyzer::~ExampleVideoQualityAnalyzer() = default; + +void ExampleVideoQualityAnalyzer::Start( + std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count) {} + +uint16_t ExampleVideoQualityAnalyzer::OnFrameCaptured( + absl::string_view peer_name, + const std::string& stream_label, + const webrtc::VideoFrame& frame) { + MutexLock lock(&lock_); + uint16_t frame_id = next_frame_id_++; + if (frame_id == VideoFrame::kNotSetId) { + frame_id = next_frame_id_++; + } + auto it = frames_in_flight_.find(frame_id); + if (it == frames_in_flight_.end()) { + frames_in_flight_.insert(frame_id); + frames_to_stream_label_.insert({frame_id, stream_label}); + } else { + RTC_LOG(LS_WARNING) << "Meet new frame with the same id: " << frame_id + << ". Assumes old one as dropped"; + // We needn't insert frame to frames_in_flight_, because it is already + // there. + ++frames_dropped_; + auto stream_it = frames_to_stream_label_.find(frame_id); + RTC_CHECK(stream_it != frames_to_stream_label_.end()); + stream_it->second = stream_label; + } + ++frames_captured_; + return frame_id; +} + +void ExampleVideoQualityAnalyzer::OnFramePreEncode( + absl::string_view peer_name, + const webrtc::VideoFrame& frame) { + MutexLock lock(&lock_); + ++frames_pre_encoded_; +} + +void ExampleVideoQualityAnalyzer::OnFrameEncoded( + absl::string_view peer_name, + uint16_t frame_id, + const webrtc::EncodedImage& encoded_image, + const EncoderStats& stats, + bool discarded) { + MutexLock lock(&lock_); + ++frames_encoded_; +} + +void ExampleVideoQualityAnalyzer::OnFrameDropped( + absl::string_view peer_name, + webrtc::EncodedImageCallback::DropReason reason) { + RTC_LOG(LS_INFO) << "Frame dropped by encoder"; + MutexLock lock(&lock_); + ++frames_dropped_; +} + +void ExampleVideoQualityAnalyzer::OnFramePreDecode( + absl::string_view peer_name, + uint16_t frame_id, + const webrtc::EncodedImage& encoded_image) { + MutexLock lock(&lock_); + ++frames_received_; +} + +void ExampleVideoQualityAnalyzer::OnFrameDecoded( + absl::string_view peer_name, + const webrtc::VideoFrame& frame, + const DecoderStats& stats) { + MutexLock lock(&lock_); + ++frames_decoded_; +} + +void ExampleVideoQualityAnalyzer::OnFrameRendered( + absl::string_view peer_name, + const webrtc::VideoFrame& frame) { + MutexLock lock(&lock_); + frames_in_flight_.erase(frame.id()); + ++frames_rendered_; +} + +void ExampleVideoQualityAnalyzer::OnEncoderError( + absl::string_view peer_name, + const webrtc::VideoFrame& frame, + int32_t error_code) { + RTC_LOG(LS_ERROR) << "Failed to encode frame " << frame.id() + << ". Code: " << error_code; +} + +void ExampleVideoQualityAnalyzer::OnDecoderError(absl::string_view peer_name, + uint16_t frame_id, + int32_t error_code, + const DecoderStats& stats) { + RTC_LOG(LS_ERROR) << "Failed to decode frame " << frame_id + << ". Code: " << error_code; +} + +void ExampleVideoQualityAnalyzer::Stop() { + MutexLock lock(&lock_); + RTC_LOG(LS_INFO) << "There are " << frames_in_flight_.size() + << " frames in flight, assuming all of them are dropped"; + frames_dropped_ += frames_in_flight_.size(); +} + +std::string ExampleVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) { + MutexLock lock(&lock_); + auto it = frames_to_stream_label_.find(frame_id); + RTC_DCHECK(it != frames_to_stream_label_.end()) + << "Unknown frame_id=" << frame_id; + return it->second; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_captured() const { + MutexLock lock(&lock_); + return frames_captured_; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_pre_encoded() const { + MutexLock lock(&lock_); + return frames_pre_encoded_; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_encoded() const { + MutexLock lock(&lock_); + return frames_encoded_; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_received() const { + MutexLock lock(&lock_); + return frames_received_; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_decoded() const { + MutexLock lock(&lock_); + return frames_decoded_; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_rendered() const { + MutexLock lock(&lock_); + return frames_rendered_; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_dropped() const { + MutexLock lock(&lock_); + return frames_dropped_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h new file mode 100644 index 0000000000..af4868a961 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_EXAMPLE_VIDEO_QUALITY_ANALYZER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_EXAMPLE_VIDEO_QUALITY_ANALYZER_H_ + +#include <atomic> +#include <map> +#include <set> +#include <string> + +#include "api/array_view.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { + +// This class is an example implementation of +// webrtc::VideoQualityAnalyzerInterface and calculates simple metrics +// just to demonstration purposes. Assumed to be used in the single process +// test cases, where both peers are in the same process. +class ExampleVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { + public: + ExampleVideoQualityAnalyzer(); + ~ExampleVideoQualityAnalyzer() override; + + void Start(std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count) override; + uint16_t OnFrameCaptured(absl::string_view peer_name, + const std::string& stream_label, + const VideoFrame& frame) override; + void OnFramePreEncode(absl::string_view peer_name, + const VideoFrame& frame) override; + void OnFrameEncoded(absl::string_view peer_name, + uint16_t frame_id, + const EncodedImage& encoded_image, + const EncoderStats& stats, + bool discarded) override; + void OnFrameDropped(absl::string_view peer_name, + EncodedImageCallback::DropReason reason) override; + void OnFramePreDecode(absl::string_view peer_name, + uint16_t frame_id, + const EncodedImage& encoded_image) override; + void OnFrameDecoded(absl::string_view peer_name, + const VideoFrame& frame, + const DecoderStats& stats) override; + void OnFrameRendered(absl::string_view peer_name, + const VideoFrame& frame) override; + void OnEncoderError(absl::string_view peer_name, + const VideoFrame& frame, + int32_t error_code) override; + void OnDecoderError(absl::string_view peer_name, + uint16_t frame_id, + int32_t error_code, + const DecoderStats& stats) override; + void Stop() override; + std::string GetStreamLabel(uint16_t frame_id) override; + + uint64_t frames_captured() const; + uint64_t frames_pre_encoded() const; + uint64_t frames_encoded() const; + uint64_t frames_received() const; + uint64_t frames_decoded() const; + uint64_t frames_rendered() const; + uint64_t frames_dropped() const; + + private: + // When peer A captured the frame it will come into analyzer's OnFrameCaptured + // and will be stored in frames_in_flight_. It will be removed from there + // when it will be received in peer B, so we need to guard it with lock. + // Also because analyzer will serve for all video streams it can be called + // from different threads inside one peer. + mutable Mutex lock_; + // Stores frame ids, that are currently going from one peer to another. We + // need to keep them to correctly determine dropped frames and also correctly + // process frame id overlap. + std::set<uint16_t> frames_in_flight_ RTC_GUARDED_BY(lock_); + std::map<uint16_t, std::string> frames_to_stream_label_ RTC_GUARDED_BY(lock_); + uint16_t next_frame_id_ RTC_GUARDED_BY(lock_) = 1; + uint64_t frames_captured_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_pre_encoded_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_encoded_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_received_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_decoded_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_rendered_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_dropped_ RTC_GUARDED_BY(lock_) = 0; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_EXAMPLE_VIDEO_QUALITY_ANALYZER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue.h new file mode 100644 index 0000000000..39d26b42bc --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue.h @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_MULTI_READER_QUEUE_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_MULTI_READER_QUEUE_H_ + +#include <deque> +#include <memory> +#include <set> +#include <unordered_map> + +#include "absl/types/optional.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +// Represents the queue which can be read by multiple readers. Each reader reads +// from its own queue head. When an element is added it will become visible for +// all readers. When an element will be removed by all the readers, the element +// will be removed from the queue. +template <typename T> +class MultiReaderQueue { + public: + // Creates queue with exactly `readers_count` readers named from 0 to + // `readers_count - 1`. + explicit MultiReaderQueue(size_t readers_count) { + for (size_t i = 0; i < readers_count; ++i) { + heads_[i] = 0; + } + } + // Creates queue with specified readers. + explicit MultiReaderQueue(std::set<size_t> readers) { + for (size_t reader : readers) { + heads_[reader] = 0; + } + } + + // Adds a new `reader`, initializing its reading position (the reader's head) + // equal to the one of `reader_to_copy`. + // Complexity O(MultiReaderQueue::size(reader_to_copy)). + void AddReader(size_t reader, size_t reader_to_copy) { + size_t pos = GetHeadPositionOrDie(reader_to_copy); + + auto it = heads_.find(reader); + RTC_CHECK(it == heads_.end()) + << "Reader " << reader << " is already in the queue"; + heads_[reader] = heads_[reader_to_copy]; + for (size_t i = pos; i < queue_.size(); ++i) { + in_queues_[i]++; + } + } + + // Adds a new `reader`, initializing its reading position equal to first + // element in the queue. + // Complexity O(MultiReaderQueue::size()). + void AddReader(size_t reader) { + auto it = heads_.find(reader); + RTC_CHECK(it == heads_.end()) + << "Reader " << reader << " is already in the queue"; + heads_[reader] = removed_elements_count_; + for (size_t i = 0; i < queue_.size(); ++i) { + in_queues_[i]++; + } + } + + // Removes specified `reader` from the queue. + // Complexity O(MultiReaderQueue::size(reader)). + void RemoveReader(size_t reader) { + size_t pos = GetHeadPositionOrDie(reader); + for (size_t i = pos; i < queue_.size(); ++i) { + in_queues_[i]--; + } + while (!in_queues_.empty() && in_queues_[0] == 0) { + PopFront(); + } + heads_.erase(reader); + } + + // Add value to the end of the queue. Complexity O(1). + void PushBack(T value) { + queue_.push_back(value); + in_queues_.push_back(heads_.size()); + } + + // Extract element from specified head. Complexity O(1). + absl::optional<T> PopFront(size_t reader) { + size_t pos = GetHeadPositionOrDie(reader); + if (pos >= queue_.size()) { + return absl::nullopt; + } + + T out = queue_[pos]; + + in_queues_[pos]--; + heads_[reader]++; + + if (in_queues_[pos] == 0) { + RTC_DCHECK_EQ(pos, 0); + PopFront(); + } + return out; + } + + // Returns element at specified head. Complexity O(1). + absl::optional<T> Front(size_t reader) const { + size_t pos = GetHeadPositionOrDie(reader); + if (pos >= queue_.size()) { + return absl::nullopt; + } + return queue_[pos]; + } + + // Returns true if for specified head there are no more elements in the queue + // or false otherwise. Complexity O(1). + bool IsEmpty(size_t reader) const { + size_t pos = GetHeadPositionOrDie(reader); + return pos >= queue_.size(); + } + + // Returns size of the longest queue between all readers. + // Complexity O(1). + size_t size() const { return queue_.size(); } + + // Returns size of the specified queue. Complexity O(1). + size_t size(size_t reader) const { + size_t pos = GetHeadPositionOrDie(reader); + return queue_.size() - pos; + } + + // Complexity O(1). + size_t readers_count() const { return heads_.size(); } + + private: + size_t GetHeadPositionOrDie(size_t reader) const { + auto it = heads_.find(reader); + RTC_CHECK(it != heads_.end()) << "No queue for reader " << reader; + return it->second - removed_elements_count_; + } + + void PopFront() { + RTC_DCHECK(!queue_.empty()); + RTC_DCHECK_EQ(in_queues_[0], 0); + queue_.pop_front(); + in_queues_.pop_front(); + removed_elements_count_++; + } + + // Number of the elements that were removed from the queue. It is used to + // subtract from each head to compute the right index inside `queue_`; + size_t removed_elements_count_ = 0; + std::deque<T> queue_; + // In how may queues the element at index `i` is. An element can be removed + // from the front if and only if it is in 0 queues. + std::deque<size_t> in_queues_; + // Map from the reader to the head position in the queue. + std::unordered_map<size_t, size_t> heads_; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_MULTI_READER_QUEUE_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue_test.cc new file mode 100644 index 0000000000..ea6aa0a416 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue_test.cc @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/multi_reader_queue.h" + +#include "absl/types/optional.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +TEST(MultiReaderQueueTest, EmptyQueueEmptyForAllHeads) { + MultiReaderQueue<int> queue = MultiReaderQueue<int>(/*readers_count=*/10); + EXPECT_EQ(queue.size(), 0lu); + for (int i = 0; i < 10; ++i) { + EXPECT_TRUE(queue.IsEmpty(/*reader=*/i)); + EXPECT_EQ(queue.size(/*reader=*/i), 0lu); + EXPECT_FALSE(queue.PopFront(/*reader=*/i).has_value()); + EXPECT_FALSE(queue.Front(/*reader=*/i).has_value()); + } +} + +TEST(MultiReaderQueueTest, SizeIsEqualForAllHeadsAfterAddOnly) { + MultiReaderQueue<int> queue = MultiReaderQueue<int>(/*readers_count=*/10); + queue.PushBack(1); + queue.PushBack(2); + queue.PushBack(3); + EXPECT_EQ(queue.size(), 3lu); + for (int i = 0; i < 10; ++i) { + EXPECT_FALSE(queue.IsEmpty(/*reader=*/i)); + EXPECT_EQ(queue.size(/*reader=*/i), 3lu); + } +} + +TEST(MultiReaderQueueTest, SizeIsCorrectAfterRemoveFromOnlyOneHead) { + MultiReaderQueue<int> queue = MultiReaderQueue<int>(/*readers_count=*/10); + for (int i = 0; i < 5; ++i) { + queue.PushBack(i); + } + EXPECT_EQ(queue.size(), 5lu); + // Removing elements from queue #0 + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(queue.size(/*reader=*/0), static_cast<size_t>(5 - i)); + EXPECT_EQ(queue.PopFront(/*reader=*/0), absl::optional<int>(i)); + for (int j = 1; j < 10; ++j) { + EXPECT_EQ(queue.size(/*reader=*/j), 5lu); + } + } + EXPECT_EQ(queue.size(/*reader=*/0), 0lu); + EXPECT_TRUE(queue.IsEmpty(/*reader=*/0)); +} + +TEST(MultiReaderQueueTest, SingleHeadOneAddOneRemove) { + MultiReaderQueue<int> queue = MultiReaderQueue<int>(/*readers_count=*/1); + queue.PushBack(1); + EXPECT_EQ(queue.size(), 1lu); + EXPECT_TRUE(queue.Front(/*reader=*/0).has_value()); + EXPECT_EQ(queue.Front(/*reader=*/0).value(), 1); + absl::optional<int> value = queue.PopFront(/*reader=*/0); + EXPECT_TRUE(value.has_value()); + EXPECT_EQ(value.value(), 1); + EXPECT_EQ(queue.size(), 0lu); + EXPECT_TRUE(queue.IsEmpty(/*reader=*/0)); +} + +TEST(MultiReaderQueueTest, SingleHead) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/1); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + EXPECT_EQ(queue.size(), i + 1); + } + for (size_t i = 0; i < 10; ++i) { + EXPECT_EQ(queue.Front(/*reader=*/0), absl::optional<size_t>(i)); + EXPECT_EQ(queue.PopFront(/*reader=*/0), absl::optional<size_t>(i)); + EXPECT_EQ(queue.size(), 10 - i - 1); + } +} + +TEST(MultiReaderQueueTest, ThreeHeadsAddAllRemoveAllPerHead) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/3); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + EXPECT_EQ(queue.size(), i + 1); + } + for (size_t i = 0; i < 10; ++i) { + absl::optional<size_t> value = queue.PopFront(/*reader=*/0); + EXPECT_EQ(queue.size(), 10lu); + ASSERT_TRUE(value.has_value()); + EXPECT_EQ(value.value(), i); + } + for (size_t i = 0; i < 10; ++i) { + absl::optional<size_t> value = queue.PopFront(/*reader=*/1); + EXPECT_EQ(queue.size(), 10lu); + ASSERT_TRUE(value.has_value()); + EXPECT_EQ(value.value(), i); + } + for (size_t i = 0; i < 10; ++i) { + absl::optional<size_t> value = queue.PopFront(/*reader=*/2); + EXPECT_EQ(queue.size(), 10 - i - 1); + ASSERT_TRUE(value.has_value()); + EXPECT_EQ(value.value(), i); + } +} + +TEST(MultiReaderQueueTest, ThreeHeadsAddAllRemoveAll) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/3); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + EXPECT_EQ(queue.size(), i + 1); + } + for (size_t i = 0; i < 10; ++i) { + absl::optional<size_t> value1 = queue.PopFront(/*reader=*/0); + absl::optional<size_t> value2 = queue.PopFront(/*reader=*/1); + absl::optional<size_t> value3 = queue.PopFront(/*reader=*/2); + EXPECT_EQ(queue.size(), 10 - i - 1); + ASSERT_TRUE(value1.has_value()); + ASSERT_TRUE(value2.has_value()); + ASSERT_TRUE(value3.has_value()); + EXPECT_EQ(value1.value(), i); + EXPECT_EQ(value2.value(), i); + EXPECT_EQ(value3.value(), i); + } +} + +TEST(MultiReaderQueueTest, AddReaderSeeElementsOnlyFromReaderToCopy) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/2); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + } + for (size_t i = 0; i < 5; ++i) { + queue.PopFront(0); + } + + queue.AddReader(/*reader=*/2, /*reader_to_copy=*/0); + + EXPECT_EQ(queue.readers_count(), 3lu); + for (size_t i = 5; i < 10; ++i) { + absl::optional<size_t> value = queue.PopFront(/*reader=*/2); + EXPECT_EQ(queue.size(/*reader=*/2), 10 - i - 1); + ASSERT_TRUE(value.has_value()); + EXPECT_EQ(value.value(), i); + } +} + +TEST(MultiReaderQueueTest, AddReaderWithoutReaderToCopySeeFullQueue) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/2); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + } + for (size_t i = 0; i < 5; ++i) { + queue.PopFront(/*reader=*/0); + } + + queue.AddReader(/*reader=*/2); + + EXPECT_EQ(queue.readers_count(), 3lu); + for (size_t i = 0; i < 10; ++i) { + absl::optional<size_t> value = queue.PopFront(/*reader=*/2); + EXPECT_EQ(queue.size(/*reader=*/2), 10 - i - 1); + ASSERT_TRUE(value.has_value()); + EXPECT_EQ(value.value(), i); + } +} + +TEST(MultiReaderQueueTest, RemoveReaderWontChangeOthers) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/2); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + } + EXPECT_EQ(queue.size(/*reader=*/1), 10lu); + + queue.RemoveReader(0); + + EXPECT_EQ(queue.readers_count(), 1lu); + EXPECT_EQ(queue.size(/*reader=*/1), 10lu); +} + +TEST(MultiReaderQueueTest, RemoveLastReaderMakesQueueEmpty) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/1); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + } + EXPECT_EQ(queue.size(), 10lu); + + queue.RemoveReader(0); + + EXPECT_EQ(queue.size(), 0lu); + EXPECT_EQ(queue.readers_count(), 0lu); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.cc new file mode 100644 index 0000000000..3ccab620f8 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.cc @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/names_collection.h" + +#include <set> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" + +namespace webrtc { + +NamesCollection::NamesCollection(rtc::ArrayView<const std::string> names) { + names_ = std::vector<std::string>(names.begin(), names.end()); + for (size_t i = 0; i < names_.size(); ++i) { + index_.emplace(names_[i], i); + removed_.emplace_back(false); + } + size_ = names_.size(); +} + +bool NamesCollection::HasName(absl::string_view name) const { + auto it = index_.find(name); + if (it == index_.end()) { + return false; + } + return !removed_[it->second]; +} + +size_t NamesCollection::AddIfAbsent(absl::string_view name) { + auto it = index_.find(name); + if (it != index_.end()) { + // Name was registered in the collection before: we need to restore it. + size_t index = it->second; + if (removed_[index]) { + removed_[index] = false; + size_++; + } + return index; + } + size_t out = names_.size(); + size_t old_capacity = names_.capacity(); + names_.emplace_back(name); + removed_.emplace_back(false); + size_++; + size_t new_capacity = names_.capacity(); + + if (old_capacity == new_capacity) { + index_.emplace(names_[out], out); + } else { + // Reallocation happened in the vector, so we need to rebuild `index_` to + // fix absl::string_view internal references. + index_.clear(); + for (size_t i = 0; i < names_.size(); ++i) { + index_.emplace(names_[i], i); + } + } + return out; +} + +absl::optional<size_t> NamesCollection::RemoveIfPresent( + absl::string_view name) { + auto it = index_.find(name); + if (it == index_.end()) { + return absl::nullopt; + } + size_t index = it->second; + if (removed_[index]) { + return absl::nullopt; + } + removed_[index] = true; + size_--; + return index; +} + +std::set<size_t> NamesCollection::GetPresentIndexes() const { + std::set<size_t> out; + for (size_t i = 0; i < removed_.size(); ++i) { + if (!removed_[i]) { + out.insert(i); + } + } + return out; +} + +std::set<size_t> NamesCollection::GetAllIndexes() const { + std::set<size_t> out; + for (size_t i = 0; i < names_.size(); ++i) { + out.insert(i); + } + return out; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.h new file mode 100644 index 0000000000..f9a13a2a11 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_NAMES_COLLECTION_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_NAMES_COLLECTION_H_ + +#include <map> +#include <set> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/array_view.h" + +namespace webrtc { + +// Contains mapping between string names and unique size_t values (indexes). +// Once the name is added to the collection it is guaranteed: +// 1. Name will have the same index until collection will be destructed +// 2. Adding, removing and re-adding name won't change its index +// +// The name is considered in the collection if it was added and wasn't removed. +// Adding the name when it is in the collection won't change the collection, the +// same as removing the name when it is removed. +// +// Collection will return name's index and name for the index independently from +// was name removed or not. Once the name was added to the collection the index +// will be allocated for it. To check if name is in collection right now user +// has to explicitly call to `HasName` function. +class NamesCollection { + public: + NamesCollection() = default; + + explicit NamesCollection(rtc::ArrayView<const std::string> names); + + // Returns amount of currently presented names in the collection. + size_t size() const { return size_; } + + // Returns amount of all names known to this collection. + size_t GetKnownSize() const { return names_.size(); } + + // Returns index of the `name` which was known to the collection. Crashes + // if `name` was never registered in the collection. + size_t index(absl::string_view name) const { return index_.at(name); } + + // Returns name which was known to the collection for the specified `index`. + // Crashes if there was no any name registered in the collection for such + // `index`. + const std::string& name(size_t index) const { return names_.at(index); } + + // Returns if `name` is currently presented in this collection. + bool HasName(absl::string_view name) const; + + // Adds specified `name` to the collection if it isn't presented. + // Returns index which corresponds to specified `name`. + size_t AddIfAbsent(absl::string_view name); + + // Removes specified `name` from the collection if it is presented. + // + // After name was removed, collection size will be decreased, but `name` index + // will be preserved. Collection will return false for `HasName(name)`, but + // will continue to return previously known index for `index(name)` and return + // `name` for `name(index(name))`. + // + // Returns the index of the removed value or absl::nullopt if no such `name` + // registered in the collection. + absl::optional<size_t> RemoveIfPresent(absl::string_view name); + + // Returns a set of indexes for all currently present names in the + // collection. + std::set<size_t> GetPresentIndexes() const; + + // Returns a set of all indexes known to the collection including indexes for + // names that were removed. + std::set<size_t> GetAllIndexes() const; + + private: + std::vector<std::string> names_; + std::vector<bool> removed_; + std::map<absl::string_view, size_t> index_; + size_t size_ = 0; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_NAMES_COLLECTION_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection_test.cc new file mode 100644 index 0000000000..6c52f96975 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection_test.cc @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/names_collection.h" + +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +using ::testing::Eq; +using ::testing::Ne; + +TEST(NamesCollectionTest, NamesFromCtorHasUniqueIndexes) { + NamesCollection collection(std::vector<std::string>{"alice", "bob"}); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(2))); + EXPECT_TRUE(collection.HasName("alice")); + EXPECT_THAT(collection.name(collection.index("alice")), Eq("alice")); + + EXPECT_TRUE(collection.HasName("bob")); + EXPECT_THAT(collection.name(collection.index("bob")), Eq("bob")); + + EXPECT_THAT(collection.index("bob"), Ne(collection.index("alice"))); +} + +TEST(NamesCollectionTest, AddedNamesHasIndexes) { + NamesCollection collection(std::vector<std::string>{}); + collection.AddIfAbsent("alice"); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + EXPECT_TRUE(collection.HasName("alice")); + EXPECT_THAT(collection.name(collection.index("alice")), Eq("alice")); +} + +TEST(NamesCollectionTest, AddBobDoesNotChangeAliceIndex) { + NamesCollection collection(std::vector<std::string>{"alice"}); + + size_t alice_index = collection.index("alice"); + + collection.AddIfAbsent("bob"); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(2))); + EXPECT_THAT(collection.index("alice"), Eq(alice_index)); + EXPECT_THAT(collection.index("bob"), Ne(alice_index)); +} + +TEST(NamesCollectionTest, AddAliceSecondTimeDoesNotChangeIndex) { + NamesCollection collection(std::vector<std::string>{"alice"}); + + size_t alice_index = collection.index("alice"); + + EXPECT_THAT(collection.AddIfAbsent("alice"), Eq(alice_index)); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + EXPECT_THAT(collection.index("alice"), Eq(alice_index)); +} + +TEST(NamesCollectionTest, RemoveRemovesFromCollectionButNotIndex) { + NamesCollection collection(std::vector<std::string>{"alice", "bob"}); + + size_t bob_index = collection.index("bob"); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(2))); + + EXPECT_THAT(collection.RemoveIfPresent("bob"), + Eq(absl::optional<size_t>(bob_index))); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + EXPECT_FALSE(collection.HasName("bob")); + + EXPECT_THAT(collection.index("bob"), Eq(bob_index)); + EXPECT_THAT(collection.name(bob_index), Eq("bob")); +} + +TEST(NamesCollectionTest, RemoveOfAliceDoesNotChangeBobIndex) { + NamesCollection collection(std::vector<std::string>{"alice", "bob"}); + + size_t alice_index = collection.index("alice"); + size_t bob_index = collection.index("bob"); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(2))); + + EXPECT_THAT(collection.RemoveIfPresent("alice"), + Eq(absl::optional<size_t>(alice_index))); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + EXPECT_THAT(collection.index("bob"), Eq(bob_index)); + EXPECT_THAT(collection.name(bob_index), Eq("bob")); +} + +TEST(NamesCollectionTest, RemoveSecondTimeHasNoEffect) { + NamesCollection collection(std::vector<std::string>{"bob"}); + + size_t bob_index = collection.index("bob"); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + EXPECT_THAT(collection.RemoveIfPresent("bob"), + Eq(absl::optional<size_t>(bob_index))); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(0))); + EXPECT_THAT(collection.RemoveIfPresent("bob"), Eq(absl::nullopt)); +} + +TEST(NamesCollectionTest, RemoveOfNotExistingHasNoEffect) { + NamesCollection collection(std::vector<std::string>{"bob"}); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + EXPECT_THAT(collection.RemoveIfPresent("alice"), Eq(absl::nullopt)); + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); +} + +TEST(NamesCollectionTest, AddRemoveAddPreserveTheIndex) { + NamesCollection collection(std::vector<std::string>{}); + + size_t alice_index = collection.AddIfAbsent("alice"); + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + + EXPECT_THAT(collection.RemoveIfPresent("alice"), + Eq(absl::optional<size_t>(alice_index))); + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(0))); + + EXPECT_THAT(collection.AddIfAbsent("alice"), Eq(alice_index)); + EXPECT_THAT(collection.index("alice"), Eq(alice_index)); + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); +} + +TEST(NamesCollectionTest, GetKnownSizeReturnsForRemovedNames) { + NamesCollection collection(std::vector<std::string>{}); + + size_t alice_index = collection.AddIfAbsent("alice"); + EXPECT_THAT(collection.GetKnownSize(), Eq(static_cast<size_t>(1))); + + EXPECT_THAT(collection.RemoveIfPresent("alice"), + Eq(absl::optional<size_t>(alice_index))); + EXPECT_THAT(collection.GetKnownSize(), Eq(static_cast<size_t>(1))); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc new file mode 100644 index 0000000000..b958f4d027 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h" + +#include <cstdint> +#include <cstring> +#include <memory> +#include <utility> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" +#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +QualityAnalyzingVideoDecoder::QualityAnalyzingVideoDecoder( + absl::string_view peer_name, + std::unique_ptr<VideoDecoder> delegate, + EncodedImageDataExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer) + : peer_name_(peer_name), + implementation_name_("AnalyzingDecoder-" + + std::string(delegate->ImplementationName())), + delegate_(std::move(delegate)), + extractor_(extractor), + analyzer_(analyzer) { + analyzing_callback_ = std::make_unique<DecoderCallback>(this); +} +QualityAnalyzingVideoDecoder::~QualityAnalyzingVideoDecoder() = default; + +bool QualityAnalyzingVideoDecoder::Configure(const Settings& settings) { + { + MutexLock lock(&mutex_); + codec_name_ = std::string(CodecTypeToPayloadString(settings.codec_type())) + + "_" + delegate_->GetDecoderInfo().implementation_name; + } + return delegate_->Configure(settings); +} + +int32_t QualityAnalyzingVideoDecoder::Decode(const EncodedImage& input_image, + bool missing_frames, + int64_t render_time_ms) { + // Image extractor extracts id from provided EncodedImage and also returns + // the image with the original buffer. Buffer can be modified in place, so + // owner of original buffer will be responsible for deleting it, or extractor + // can create a new buffer. In such case extractor will be responsible for + // deleting it. + EncodedImageExtractionResult out = extractor_->ExtractData(input_image); + + if (out.discard) { + // To partly emulate behavior of Selective Forwarding Unit (SFU) in the + // test, on receiver side we will "discard" frames from irrelevant streams. + // When all encoded images were marked to discarded, black frame have to be + // returned. Because simulcast streams will be received by receiver as 3 + // different independent streams we don't want that irrelevant streams + // affect video quality metrics and also we don't want to use CPU time to + // decode them to prevent regressions on relevant streams. Also we can't + // just drop frame, because in such case, receiving part will be confused + // with all frames missing and will request a key frame, which will result + // into extra load on network and sender side. Because of it, discarded + // image will be always decoded as black frame and will be passed to + // callback directly without reaching decoder and video quality analyzer. + // + // For more details see QualityAnalyzingVideoEncoder. + return analyzing_callback_->IrrelevantSimulcastStreamDecoded( + out.id.value_or(VideoFrame::kNotSetId), input_image.Timestamp()); + } + + EncodedImage* origin_image; + { + MutexLock lock(&mutex_); + // Store id to be able to retrieve it in analyzing callback. + timestamp_to_frame_id_.insert({input_image.Timestamp(), out.id}); + // Store encoded image to prevent its destruction while it is used in + // decoder. + origin_image = &( + decoding_images_.insert({input_image.Timestamp(), std::move(out.image)}) + .first->second); + } + // We can safely dereference `origin_image`, because it can be removed from + // the map only after `delegate_` Decode method will be invoked. Image will + // be removed inside DecodedImageCallback, which can be done on separate + // thread. + analyzer_->OnFramePreDecode( + peer_name_, out.id.value_or(VideoFrame::kNotSetId), *origin_image); + int32_t result = + delegate_->Decode(*origin_image, missing_frames, render_time_ms); + if (result != WEBRTC_VIDEO_CODEC_OK) { + // If delegate decoder failed, then cleanup data for this image. + VideoQualityAnalyzerInterface::DecoderStats stats; + { + MutexLock lock(&mutex_); + timestamp_to_frame_id_.erase(input_image.Timestamp()); + decoding_images_.erase(input_image.Timestamp()); + stats.decoder_name = codec_name_; + } + analyzer_->OnDecoderError( + peer_name_, out.id.value_or(VideoFrame::kNotSetId), result, stats); + } + return result; +} + +int32_t QualityAnalyzingVideoDecoder::RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) { + analyzing_callback_->SetDelegateCallback(callback); + return delegate_->RegisterDecodeCompleteCallback(analyzing_callback_.get()); +} + +int32_t QualityAnalyzingVideoDecoder::Release() { + // Release decoder first. During release process it can still decode some + // frames, so we don't take a lock to prevent deadlock. + int32_t result = delegate_->Release(); + + MutexLock lock(&mutex_); + analyzing_callback_->SetDelegateCallback(nullptr); + timestamp_to_frame_id_.clear(); + decoding_images_.clear(); + return result; +} + +VideoDecoder::DecoderInfo QualityAnalyzingVideoDecoder::GetDecoderInfo() const { + DecoderInfo info = delegate_->GetDecoderInfo(); + info.implementation_name = implementation_name_; + return info; +} + +const char* QualityAnalyzingVideoDecoder::ImplementationName() const { + return implementation_name_.c_str(); +} + +QualityAnalyzingVideoDecoder::DecoderCallback::DecoderCallback( + QualityAnalyzingVideoDecoder* decoder) + : decoder_(decoder), delegate_callback_(nullptr) {} +QualityAnalyzingVideoDecoder::DecoderCallback::~DecoderCallback() = default; + +void QualityAnalyzingVideoDecoder::DecoderCallback::SetDelegateCallback( + DecodedImageCallback* delegate) { + MutexLock lock(&callback_mutex_); + delegate_callback_ = delegate; +} + +// We have to implement all next 3 methods because we don't know which one +// exactly is implemented in `delegate_callback_`, so we need to call the same +// method on `delegate_callback_`, as was called on `this` callback. +int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( + VideoFrame& decodedImage) { + decoder_->OnFrameDecoded(&decodedImage, /*decode_time_ms=*/absl::nullopt, + /*qp=*/absl::nullopt); + + MutexLock lock(&callback_mutex_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->Decoded(decodedImage); +} + +int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( + VideoFrame& decodedImage, + int64_t decode_time_ms) { + decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, /*qp=*/absl::nullopt); + + MutexLock lock(&callback_mutex_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->Decoded(decodedImage, decode_time_ms); +} + +void QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( + VideoFrame& decodedImage, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp) { + decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, qp); + + MutexLock lock(&callback_mutex_); + RTC_DCHECK(delegate_callback_); + delegate_callback_->Decoded(decodedImage, decode_time_ms, qp); +} + +int32_t +QualityAnalyzingVideoDecoder::DecoderCallback::IrrelevantSimulcastStreamDecoded( + uint16_t frame_id, + uint32_t timestamp_ms) { + webrtc::VideoFrame dummy_frame = + webrtc::VideoFrame::Builder() + .set_video_frame_buffer(GetDummyFrameBuffer()) + .set_timestamp_rtp(timestamp_ms) + .set_id(frame_id) + .build(); + MutexLock lock(&callback_mutex_); + RTC_DCHECK(delegate_callback_); + delegate_callback_->Decoded(dummy_frame, absl::nullopt, absl::nullopt); + return WEBRTC_VIDEO_CODEC_OK; +} + +rtc::scoped_refptr<webrtc::VideoFrameBuffer> +QualityAnalyzingVideoDecoder::DecoderCallback::GetDummyFrameBuffer() { + if (!dummy_frame_buffer_) { + dummy_frame_buffer_ = CreateDummyFrameBuffer(); + } + + return dummy_frame_buffer_; +} + +void QualityAnalyzingVideoDecoder::OnFrameDecoded( + VideoFrame* frame, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp) { + absl::optional<uint16_t> frame_id; + std::string codec_name; + { + MutexLock lock(&mutex_); + auto it = timestamp_to_frame_id_.find(frame->timestamp()); + if (it == timestamp_to_frame_id_.end()) { + // Ensure, that we have info about this frame. It can happen that for some + // reasons decoder response, that it failed to decode, when we were + // posting frame to it, but then call the callback for this frame. + RTC_LOG(LS_ERROR) << "QualityAnalyzingVideoDecoder::OnFrameDecoded: No " + "frame id for frame for frame->timestamp()=" + << frame->timestamp(); + return; + } + frame_id = it->second; + timestamp_to_frame_id_.erase(it); + decoding_images_.erase(frame->timestamp()); + codec_name = codec_name_; + } + // Set frame id to the value, that was extracted from corresponding encoded + // image. + frame->set_id(frame_id.value_or(VideoFrame::kNotSetId)); + VideoQualityAnalyzerInterface::DecoderStats stats; + stats.decoder_name = codec_name; + stats.decode_time_ms = decode_time_ms; + analyzer_->OnFrameDecoded(peer_name_, *frame, stats); +} + +QualityAnalyzingVideoDecoderFactory::QualityAnalyzingVideoDecoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoDecoderFactory> delegate, + EncodedImageDataExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer) + : peer_name_(peer_name), + delegate_(std::move(delegate)), + extractor_(extractor), + analyzer_(analyzer) {} +QualityAnalyzingVideoDecoderFactory::~QualityAnalyzingVideoDecoderFactory() = + default; + +std::vector<SdpVideoFormat> +QualityAnalyzingVideoDecoderFactory::GetSupportedFormats() const { + return delegate_->GetSupportedFormats(); +} + +std::unique_ptr<VideoDecoder> +QualityAnalyzingVideoDecoderFactory::CreateVideoDecoder( + const SdpVideoFormat& format) { + std::unique_ptr<VideoDecoder> decoder = delegate_->CreateVideoDecoder(format); + return std::make_unique<QualityAnalyzingVideoDecoder>( + peer_name_, std::move(decoder), extractor_, analyzer_); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h new file mode 100644 index 0000000000..a86f4196b0 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ + +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "rtc_base/synchronization/mutex.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// QualityAnalyzingVideoDecoder is used to wrap origin video decoder and inject +// VideoQualityAnalyzerInterface before and after decoder. +// +// QualityAnalyzingVideoDecoder propagates all calls to the origin decoder. +// It registers its own DecodedImageCallback in the origin decoder and will +// store user specified callback inside itself. +// +// When Decode(...) will be invoked, quality decoder first will extract frame id +// from passed EncodedImage with EncodedImageIdExtracor that was specified in +// constructor, then will call video quality analyzer, with correct +// EncodedImage and only then will pass image to origin decoder. +// +// When origin decoder decodes the image it will call quality decoder's special +// callback, where video analyzer will be called again and then decoded frame +// will be passed to origin callback, provided by user. +// +// Quality decoder registers its own callback in origin decoder, at the same +// time the user registers their callback in quality decoder. +class QualityAnalyzingVideoDecoder : public VideoDecoder { + public: + QualityAnalyzingVideoDecoder(absl::string_view peer_name, + std::unique_ptr<VideoDecoder> delegate, + EncodedImageDataExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoDecoder() override; + + // Methods of VideoDecoder interface. + bool Configure(const Settings& settings) override; + int32_t Decode(const EncodedImage& input_image, + bool missing_frames, + int64_t render_time_ms) override; + int32_t RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) override; + int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; + const char* ImplementationName() const override; + + private: + class DecoderCallback : public DecodedImageCallback { + public: + explicit DecoderCallback(QualityAnalyzingVideoDecoder* decoder); + ~DecoderCallback() override; + + void SetDelegateCallback(DecodedImageCallback* delegate); + + // Methods of DecodedImageCallback interface. + int32_t Decoded(VideoFrame& decodedImage) override; + int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) override; + void Decoded(VideoFrame& decodedImage, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp) override; + + int32_t IrrelevantSimulcastStreamDecoded(uint16_t frame_id, + uint32_t timestamp_ms); + + private: + rtc::scoped_refptr<webrtc::VideoFrameBuffer> GetDummyFrameBuffer(); + + QualityAnalyzingVideoDecoder* const decoder_; + + rtc::scoped_refptr<webrtc::VideoFrameBuffer> dummy_frame_buffer_; + + Mutex callback_mutex_; + DecodedImageCallback* delegate_callback_ RTC_GUARDED_BY(callback_mutex_); + }; + + void OnFrameDecoded(VideoFrame* frame, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp); + + const std::string peer_name_; + const std::string implementation_name_; + std::unique_ptr<VideoDecoder> delegate_; + EncodedImageDataExtractor* const extractor_; + VideoQualityAnalyzerInterface* const analyzer_; + std::unique_ptr<DecoderCallback> analyzing_callback_; + + // VideoDecoder interface assumes async delivery of decoded video frames. + // This lock is used to protect shared state, that have to be propagated + // from received EncodedImage to resulted VideoFrame. + Mutex mutex_; + + // Name of the video codec type used. Ex: VP8, VP9, H264 etc. + std::string codec_name_ RTC_GUARDED_BY(mutex_); + std::map<uint32_t, absl::optional<uint16_t>> timestamp_to_frame_id_ + RTC_GUARDED_BY(mutex_); + // Stores currently being decoded images by timestamp. Because + // EncodedImageDataExtractor can create new copy on EncodedImage we need to + // ensure, that this image won't be deleted during async decoding. To do it + // all images are putted into this map and removed from here inside callback. + std::map<uint32_t, EncodedImage> decoding_images_ RTC_GUARDED_BY(mutex_); +}; + +// Produces QualityAnalyzingVideoDecoder, which hold decoders, produced by +// specified factory as delegates. Forwards all other calls to specified +// factory. +class QualityAnalyzingVideoDecoderFactory : public VideoDecoderFactory { + public: + QualityAnalyzingVideoDecoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoDecoderFactory> delegate, + EncodedImageDataExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoDecoderFactory() override; + + // Methods of VideoDecoderFactory interface. + std::vector<SdpVideoFormat> GetSupportedFormats() const override; + std::unique_ptr<VideoDecoder> CreateVideoDecoder( + const SdpVideoFormat& format) override; + + private: + const std::string peer_name_; + std::unique_ptr<VideoDecoderFactory> delegate_; + EncodedImageDataExtractor* const extractor_; + VideoQualityAnalyzerInterface* const analyzer_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc new file mode 100644 index 0000000000..e814ba88b7 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc @@ -0,0 +1,403 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h" + +#include <cmath> +#include <memory> +#include <utility> + +#include "absl/strings/string_view.h" +#include "api/video/video_codec_type.h" +#include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/svc/scalability_mode_util.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using EmulatedSFUConfigMap = + ::webrtc::webrtc_pc_e2e::QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap; + +constexpr size_t kMaxFrameInPipelineCount = 1000; +constexpr double kNoMultiplier = 1.0; +constexpr double kEps = 1e-6; + +std::pair<uint32_t, uint32_t> GetMinMaxBitratesBps(const VideoCodec& codec, + size_t spatial_idx) { + uint32_t min_bitrate = codec.minBitrate; + uint32_t max_bitrate = codec.maxBitrate; + if (spatial_idx < codec.numberOfSimulcastStreams && + codec.codecType != VideoCodecType::kVideoCodecVP9) { + min_bitrate = + std::max(min_bitrate, codec.simulcastStream[spatial_idx].minBitrate); + max_bitrate = + std::min(max_bitrate, codec.simulcastStream[spatial_idx].maxBitrate); + } + if (codec.codecType == VideoCodecType::kVideoCodecVP9 && + spatial_idx < codec.VP9().numberOfSpatialLayers) { + min_bitrate = + std::max(min_bitrate, codec.spatialLayers[spatial_idx].minBitrate); + max_bitrate = + std::min(max_bitrate, codec.spatialLayers[spatial_idx].maxBitrate); + } + RTC_DCHECK_GT(max_bitrate, min_bitrate); + return {min_bitrate * 1000, max_bitrate * 1000}; +} + +} // namespace + +QualityAnalyzingVideoEncoder::QualityAnalyzingVideoEncoder( + absl::string_view peer_name, + std::unique_ptr<VideoEncoder> delegate, + double bitrate_multiplier, + EmulatedSFUConfigMap stream_to_sfu_config, + EncodedImageDataInjector* injector, + VideoQualityAnalyzerInterface* analyzer) + : peer_name_(peer_name), + delegate_(std::move(delegate)), + bitrate_multiplier_(bitrate_multiplier), + stream_to_sfu_config_(std::move(stream_to_sfu_config)), + injector_(injector), + analyzer_(analyzer), + mode_(SimulcastMode::kNormal), + delegate_callback_(nullptr) {} +QualityAnalyzingVideoEncoder::~QualityAnalyzingVideoEncoder() = default; + +void QualityAnalyzingVideoEncoder::SetFecControllerOverride( + FecControllerOverride* fec_controller_override) { + // Ignored. +} + +int32_t QualityAnalyzingVideoEncoder::InitEncode( + const VideoCodec* codec_settings, + const Settings& settings) { + MutexLock lock(&mutex_); + codec_settings_ = *codec_settings; + mode_ = SimulcastMode::kNormal; + absl::optional<InterLayerPredMode> inter_layer_pred_mode; + if (codec_settings->GetScalabilityMode().has_value()) { + inter_layer_pred_mode = ScalabilityModeToInterLayerPredMode( + *codec_settings->GetScalabilityMode()); + } else if (codec_settings->codecType == kVideoCodecVP9) { + if (codec_settings->VP9().numberOfSpatialLayers > 1) { + inter_layer_pred_mode = codec_settings->VP9().interLayerPred; + } + } + if (inter_layer_pred_mode.has_value()) { + switch (*inter_layer_pred_mode) { + case InterLayerPredMode::kOn: + mode_ = SimulcastMode::kSVC; + break; + case InterLayerPredMode::kOnKeyPic: + mode_ = SimulcastMode::kKSVC; + break; + case InterLayerPredMode::kOff: + mode_ = SimulcastMode::kSimulcast; + break; + default: + RTC_DCHECK_NOTREACHED() + << "Unknown InterLayerPredMode value " << *inter_layer_pred_mode; + break; + } + } + if (codec_settings->numberOfSimulcastStreams > 1) { + mode_ = SimulcastMode::kSimulcast; + } + return delegate_->InitEncode(codec_settings, settings); +} + +int32_t QualityAnalyzingVideoEncoder::RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) { + // We need to get a lock here because delegate_callback can be hypothetically + // accessed from different thread (encoder one) concurrently. + MutexLock lock(&mutex_); + delegate_callback_ = callback; + return delegate_->RegisterEncodeCompleteCallback(this); +} + +int32_t QualityAnalyzingVideoEncoder::Release() { + // Release encoder first. During release process it can still encode some + // frames, so we don't take a lock to prevent deadlock. + int32_t result = delegate_->Release(); + + MutexLock lock(&mutex_); + delegate_callback_ = nullptr; + return result; +} + +int32_t QualityAnalyzingVideoEncoder::Encode( + const VideoFrame& frame, + const std::vector<VideoFrameType>* frame_types) { + { + MutexLock lock(&mutex_); + // Store id to be able to retrieve it in analyzing callback. + timestamp_to_frame_id_list_.push_back({frame.timestamp(), frame.id()}); + // If this list is growing, it means that we are not receiving new encoded + // images from encoder. So it should be a bug in setup on in the encoder. + RTC_DCHECK_LT(timestamp_to_frame_id_list_.size(), kMaxFrameInPipelineCount); + } + analyzer_->OnFramePreEncode(peer_name_, frame); + int32_t result = delegate_->Encode(frame, frame_types); + if (result != WEBRTC_VIDEO_CODEC_OK) { + // If origin encoder failed, then cleanup data for this frame. + { + MutexLock lock(&mutex_); + // The timestamp-frame_id pair can be not the last one, so we need to + // find it first and then remove. We will search from the end, because + // usually it will be the last or close to the last one. + auto it = timestamp_to_frame_id_list_.end(); + while (it != timestamp_to_frame_id_list_.begin()) { + --it; + if (it->first == frame.timestamp()) { + timestamp_to_frame_id_list_.erase(it); + break; + } + } + } + analyzer_->OnEncoderError(peer_name_, frame, result); + } + return result; +} + +void QualityAnalyzingVideoEncoder::SetRates( + const VideoEncoder::RateControlParameters& parameters) { + RTC_DCHECK_GT(bitrate_multiplier_, 0.0); + if (fabs(bitrate_multiplier_ - kNoMultiplier) < kEps) { + { + MutexLock lock(&mutex_); + bitrate_allocation_ = parameters.bitrate; + } + return delegate_->SetRates(parameters); + } + + RateControlParameters adjusted_params = parameters; + { + MutexLock lock(&mutex_); + // Simulating encoder overshooting target bitrate, by configuring actual + // encoder too high. Take care not to adjust past limits of config, + // otherwise encoders may crash on DCHECK. + VideoBitrateAllocation multiplied_allocation; + for (size_t si = 0; si < kMaxSpatialLayers; ++si) { + const uint32_t spatial_layer_bitrate_bps = + parameters.bitrate.GetSpatialLayerSum(si); + if (spatial_layer_bitrate_bps == 0) { + continue; + } + + uint32_t min_bitrate_bps; + uint32_t max_bitrate_bps; + std::tie(min_bitrate_bps, max_bitrate_bps) = + GetMinMaxBitratesBps(codec_settings_, si); + double bitrate_multiplier = bitrate_multiplier_; + const uint32_t corrected_bitrate = rtc::checked_cast<uint32_t>( + bitrate_multiplier * spatial_layer_bitrate_bps); + if (corrected_bitrate < min_bitrate_bps) { + bitrate_multiplier = min_bitrate_bps / spatial_layer_bitrate_bps; + } else if (corrected_bitrate > max_bitrate_bps) { + bitrate_multiplier = max_bitrate_bps / spatial_layer_bitrate_bps; + } + + for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) { + if (parameters.bitrate.HasBitrate(si, ti)) { + multiplied_allocation.SetBitrate( + si, ti, + rtc::checked_cast<uint32_t>( + bitrate_multiplier * parameters.bitrate.GetBitrate(si, ti))); + } + } + } + + adjusted_params.bitrate = multiplied_allocation; + bitrate_allocation_ = adjusted_params.bitrate; + } + return delegate_->SetRates(adjusted_params); +} + +VideoEncoder::EncoderInfo QualityAnalyzingVideoEncoder::GetEncoderInfo() const { + return delegate_->GetEncoderInfo(); +} + +// It is assumed, that encoded callback will be always invoked with encoded +// images that correspond to the frames in the same sequence, that frames +// arrived. In other words, assume we have frames F1, F2 and F3 and they have +// corresponding encoded images I1, I2 and I3. In such case if we will call +// encode first with F1, then with F2 and then with F3, then encoder callback +// will be called first with all spatial layers for F1 (I1), then F2 (I2) and +// then F3 (I3). +// +// Basing on it we will use a list of timestamp-frame_id pairs like this: +// 1. If current encoded image timestamp is equals to timestamp in the front +// pair - pick frame id from that pair +// 2. If current encoded image timestamp isn't equals to timestamp in the front +// pair - remove the front pair and got to the step 1. +EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info) { + uint16_t frame_id; + bool discard = false; + uint32_t target_encode_bitrate = 0; + std::string codec_name; + { + MutexLock lock(&mutex_); + std::pair<uint32_t, uint16_t> timestamp_frame_id; + while (!timestamp_to_frame_id_list_.empty()) { + timestamp_frame_id = timestamp_to_frame_id_list_.front(); + if (timestamp_frame_id.first == encoded_image.Timestamp()) { + break; + } + timestamp_to_frame_id_list_.pop_front(); + } + + // After the loop the first element should point to current `encoded_image` + // frame id. We don't remove it from the list, because there may be + // multiple spatial layers for this frame, so encoder can produce more + // encoded images with this timestamp. The first element will be removed + // when the next frame would be encoded and EncodedImageCallback would be + // called with the next timestamp. + + if (timestamp_to_frame_id_list_.empty()) { + // Ensure, that we have info about this frame. It can happen that for some + // reasons encoder response, that he failed to decode, when we were + // posting frame to it, but then call the callback for this frame. + RTC_LOG(LS_ERROR) << "QualityAnalyzingVideoEncoder::OnEncodedImage: No " + "frame id for encoded_image.Timestamp()=" + << encoded_image.Timestamp(); + return EncodedImageCallback::Result( + EncodedImageCallback::Result::Error::OK); + } + frame_id = timestamp_frame_id.second; + + discard = ShouldDiscard(frame_id, encoded_image); + if (!discard) { + target_encode_bitrate = bitrate_allocation_.GetSpatialLayerSum( + encoded_image.SpatialIndex().value_or(0)); + } + codec_name = + std::string(CodecTypeToPayloadString(codec_settings_.codecType)) + "_" + + delegate_->GetEncoderInfo().implementation_name; + } + + VideoQualityAnalyzerInterface::EncoderStats stats; + stats.encoder_name = codec_name; + stats.target_encode_bitrate = target_encode_bitrate; + stats.qp = encoded_image.qp_; + analyzer_->OnFrameEncoded(peer_name_, frame_id, encoded_image, stats, + discard); + + // Image data injector injects frame id and discard flag into provided + // EncodedImage and returns the image with a) modified original buffer (in + // such case the current owner of the buffer will be responsible for deleting + // it) or b) a new buffer (in such case injector will be responsible for + // deleting it). + const EncodedImage& image = + injector_->InjectData(frame_id, discard, encoded_image); + { + MutexLock lock(&mutex_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->OnEncodedImage(image, codec_specific_info); + } +} + +void QualityAnalyzingVideoEncoder::OnDroppedFrame( + EncodedImageCallback::DropReason reason) { + MutexLock lock(&mutex_); + analyzer_->OnFrameDropped(peer_name_, reason); + RTC_DCHECK(delegate_callback_); + delegate_callback_->OnDroppedFrame(reason); +} + +bool QualityAnalyzingVideoEncoder::ShouldDiscard( + uint16_t frame_id, + const EncodedImage& encoded_image) { + std::string stream_label = analyzer_->GetStreamLabel(frame_id); + EmulatedSFUConfigMap::mapped_type emulated_sfu_config = + stream_to_sfu_config_[stream_label]; + + if (!emulated_sfu_config) + return false; + + int cur_spatial_index = encoded_image.SpatialIndex().value_or(0); + int cur_temporal_index = encoded_image.TemporalIndex().value_or(0); + + if (emulated_sfu_config->target_temporal_index && + cur_temporal_index > *emulated_sfu_config->target_temporal_index) + return true; + + if (emulated_sfu_config->target_layer_index) { + switch (mode_) { + case SimulcastMode::kSimulcast: + // In simulcast mode only encoded images with required spatial index are + // interested, so all others have to be discarded. + return cur_spatial_index != *emulated_sfu_config->target_layer_index; + case SimulcastMode::kSVC: + // In SVC mode encoded images with spatial indexes that are equal or + // less than required one are interesting, so all above have to be + // discarded. + return cur_spatial_index > *emulated_sfu_config->target_layer_index; + case SimulcastMode::kKSVC: + // In KSVC mode for key frame encoded images with spatial indexes that + // are equal or less than required one are interesting, so all above + // have to be discarded. For other frames only required spatial index + // is interesting, so all others except the ones depending on the + // keyframes can be discarded. There's no good test for that, so we keep + // all of temporal layer 0 for now. + if (encoded_image._frameType == VideoFrameType::kVideoFrameKey || + cur_temporal_index == 0) + return cur_spatial_index > *emulated_sfu_config->target_layer_index; + return cur_spatial_index != *emulated_sfu_config->target_layer_index; + case SimulcastMode::kNormal: + RTC_DCHECK_NOTREACHED() << "Analyzing encoder is in kNormal mode, but " + "target_layer_index is set"; + } + } + return false; +} + +QualityAnalyzingVideoEncoderFactory::QualityAnalyzingVideoEncoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoEncoderFactory> delegate, + double bitrate_multiplier, + EmulatedSFUConfigMap stream_to_sfu_config, + EncodedImageDataInjector* injector, + VideoQualityAnalyzerInterface* analyzer) + : peer_name_(peer_name), + delegate_(std::move(delegate)), + bitrate_multiplier_(bitrate_multiplier), + stream_to_sfu_config_(std::move(stream_to_sfu_config)), + injector_(injector), + analyzer_(analyzer) {} +QualityAnalyzingVideoEncoderFactory::~QualityAnalyzingVideoEncoderFactory() = + default; + +std::vector<SdpVideoFormat> +QualityAnalyzingVideoEncoderFactory::GetSupportedFormats() const { + return delegate_->GetSupportedFormats(); +} + +VideoEncoderFactory::CodecSupport +QualityAnalyzingVideoEncoderFactory::QueryCodecSupport( + const SdpVideoFormat& format, + absl::optional<std::string> scalability_mode) const { + return delegate_->QueryCodecSupport(format, scalability_mode); +} + +std::unique_ptr<VideoEncoder> +QualityAnalyzingVideoEncoderFactory::CreateVideoEncoder( + const SdpVideoFormat& format) { + return std::make_unique<QualityAnalyzingVideoEncoder>( + peer_name_, delegate_->CreateVideoEncoder(format), bitrate_multiplier_, + stream_to_sfu_config_, injector_, analyzer_); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h new file mode 100644 index 0000000000..4adeacc0cd --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ + +#include <list> +#include <memory> +#include <utility> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "rtc_base/synchronization/mutex.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// QualityAnalyzingVideoEncoder is used to wrap origin video encoder and inject +// VideoQualityAnalyzerInterface before and after encoder. +// +// QualityAnalyzingVideoEncoder propagates all calls to the origin encoder. +// It registers its own EncodedImageCallback in the origin encoder and will +// store user specified callback inside itself. +// +// When Encode(...) will be invoked, quality encoder first calls video quality +// analyzer with original frame, then encodes frame with original encoder. +// +// When origin encoder encodes the image it will call quality encoder's special +// callback, where video analyzer will be called again and then frame id will be +// injected into EncodedImage with passed EncodedImageDataInjector. Then new +// EncodedImage will be passed to origin callback, provided by user. +// +// Quality encoder registers its own callback in origin encoder, at the same +// time the user registers their callback in quality encoder. +class QualityAnalyzingVideoEncoder : public VideoEncoder, + public EncodedImageCallback { + public: + using EmulatedSFUConfigMap = + std::map<std::string, absl::optional<EmulatedSFUConfig>>; + + QualityAnalyzingVideoEncoder(absl::string_view peer_name, + std::unique_ptr<VideoEncoder> delegate, + double bitrate_multiplier, + EmulatedSFUConfigMap stream_to_sfu_config, + EncodedImageDataInjector* injector, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoEncoder() override; + + // Methods of VideoEncoder interface. + void SetFecControllerOverride( + FecControllerOverride* fec_controller_override) override; + int32_t InitEncode(const VideoCodec* codec_settings, + const Settings& settings) override; + int32_t RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) override; + int32_t Release() override; + int32_t Encode(const VideoFrame& frame, + const std::vector<VideoFrameType>* frame_types) override; + void SetRates(const VideoEncoder::RateControlParameters& parameters) override; + EncoderInfo GetEncoderInfo() const override; + + // Methods of EncodedImageCallback interface. + EncodedImageCallback::Result OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info) override; + void OnDroppedFrame(DropReason reason) override; + + private: + enum SimulcastMode { + // In this mode encoder assumes not more than 1 encoded image per video + // frame + kNormal, + + // Next modes are to test video conference behavior. For conference sender + // will send multiple spatial layers/simulcast streams for single video + // track and there is some Selective Forwarding Unit (SFU), that forwards + // only best one, that will pass through downlink to the receiver. + // + // Here this behavior will be partly emulated. Sender will send all spatial + // layers/simulcast streams and then some of them will be filtered out on + // the receiver side. During test setup user can specify which spatial + // layer/simulcast stream is required, what will simulated which spatial + // layer/simulcast stream will be chosen by SFU in the real world. Then + // sender will mark encoded images for all spatial layers above required or + // all simulcast streams except required as to be discarded and on receiver + // side they will be discarded in quality analyzing decoder and won't be + // passed into delegate decoder. + // + // If the sender for some reasons won't send specified spatial layer, then + // receiver still will fall back on lower spatial layers. But for simulcast + // streams if required one won't be sent, receiver will assume all frames + // in that period as dropped and will experience video freeze. + // + // Test based on this simulation will be used to evaluate video quality + // of concrete spatial layers/simulcast streams and also check distribution + // of bandwidth between spatial layers/simulcast streams by BWE. + + // In this mode encoder assumes that for each frame simulcast encoded + // images will be produced. So all simulcast streams except required will + // be marked as to be discarded in decoder and won't reach video quality + // analyzer. + kSimulcast, + // In this mode encoder assumes that for each frame encoded images for + // different spatial layers will be produced. So all spatial layers above + // required will be marked to be discarded in decoder and won't reach + // video quality analyzer. + kSVC, + // In this mode encoder assumes that for each frame encoded images for + // different spatial layers will be produced. Compared to kSVC mode + // spatial layers that are above required will be marked to be discarded + // only for key frames and for regular frames all except required spatial + // layer will be marked as to be discarded in decoder and won't reach video + // quality analyzer. + kKSVC + }; + + bool ShouldDiscard(uint16_t frame_id, const EncodedImage& encoded_image) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + const std::string peer_name_; + std::unique_ptr<VideoEncoder> delegate_; + const double bitrate_multiplier_; + // Contains mapping from stream label to optional spatial index. + // If we have stream label "Foo" and mapping contains + // 1. `absl::nullopt` means all streams are required + // 2. Concrete value means that particular simulcast/SVC stream have to be + // analyzed. + EmulatedSFUConfigMap stream_to_sfu_config_; + EncodedImageDataInjector* const injector_; + VideoQualityAnalyzerInterface* const analyzer_; + + // VideoEncoder interface assumes async delivery of encoded images. + // This lock is used to protect shared state, that have to be propagated + // from received VideoFrame to resulted EncodedImage. + Mutex mutex_; + + VideoCodec codec_settings_ RTC_GUARDED_BY(mutex_); + SimulcastMode mode_ RTC_GUARDED_BY(mutex_); + EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(mutex_); + std::list<std::pair<uint32_t, uint16_t>> timestamp_to_frame_id_list_ + RTC_GUARDED_BY(mutex_); + VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(mutex_); +}; + +// Produces QualityAnalyzingVideoEncoder, which hold decoders, produced by +// specified factory as delegates. Forwards all other calls to specified +// factory. +class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory { + public: + QualityAnalyzingVideoEncoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoEncoderFactory> delegate, + double bitrate_multiplier, + QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap stream_to_sfu_config, + EncodedImageDataInjector* injector, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoEncoderFactory() override; + + // Methods of VideoEncoderFactory interface. + std::vector<SdpVideoFormat> GetSupportedFormats() const override; + VideoEncoderFactory::CodecSupport QueryCodecSupport( + const SdpVideoFormat& format, + absl::optional<std::string> scalability_mode) const override; + std::unique_ptr<VideoEncoder> CreateVideoEncoder( + const SdpVideoFormat& format) override; + + private: + const std::string peer_name_; + std::unique_ptr<VideoEncoderFactory> delegate_; + const double bitrate_multiplier_; + QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap stream_to_sfu_config_; + EncodedImageDataInjector* const injector_; + VideoQualityAnalyzerInterface* const analyzer_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.cc new file mode 100644 index 0000000000..7a73b9f4f1 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.cc @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h" + +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_buffer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +constexpr char kIrrelatedSimulcastStreamFrameData[] = "Dummy!"; + +} // namespace + +rtc::scoped_refptr<webrtc::VideoFrameBuffer> CreateDummyFrameBuffer() { + // Use i420 buffer here as default one and supported by all codecs. + rtc::scoped_refptr<webrtc::I420Buffer> buffer = + webrtc::I420Buffer::Create(2, 2); + memcpy(buffer->MutableDataY(), kIrrelatedSimulcastStreamFrameData, 2); + memcpy(buffer->MutableDataY() + buffer->StrideY(), + kIrrelatedSimulcastStreamFrameData + 2, 2); + memcpy(buffer->MutableDataU(), kIrrelatedSimulcastStreamFrameData + 4, 1); + memcpy(buffer->MutableDataV(), kIrrelatedSimulcastStreamFrameData + 5, 1); + return buffer; +} + +bool IsDummyFrame(const webrtc::VideoFrame& video_frame) { + if (video_frame.width() != 2 || video_frame.height() != 2) { + return false; + } + rtc::scoped_refptr<webrtc::I420BufferInterface> buffer = + video_frame.video_frame_buffer()->ToI420(); + if (memcmp(buffer->DataY(), kIrrelatedSimulcastStreamFrameData, 2) != 0) { + return false; + } + if (memcmp(buffer->DataY() + buffer->StrideY(), + kIrrelatedSimulcastStreamFrameData + 2, 2) != 0) { + return false; + } + if (memcmp(buffer->DataU(), kIrrelatedSimulcastStreamFrameData + 4, 1) != 0) { + return false; + } + if (memcmp(buffer->DataV(), kIrrelatedSimulcastStreamFrameData + 5, 1) != 0) { + return false; + } + return true; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h new file mode 100644 index 0000000000..8ecfae7385 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_SIMULCAST_DUMMY_BUFFER_HELPER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_SIMULCAST_DUMMY_BUFFER_HELPER_H_ + +#include "api/video/video_frame.h" +#include "api/video/video_frame_buffer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Creates a special video frame buffer that should be used to create frames +// during Selective Forwarding Unit (SFU) emulation. Such frames are used when +// original was discarded and some frame is required to be passed upstream +// to make WebRTC pipeline happy and not request key frame on the received +// stream due to lack of incoming frames. +rtc::scoped_refptr<webrtc::VideoFrameBuffer> CreateDummyFrameBuffer(); + +// Tests if provided frame contains a buffer created by +// `CreateDummyFrameBuffer`. +bool IsDummyFrame(const webrtc::VideoFrame& video_frame); + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_SIMULCAST_DUMMY_BUFFER_HELPER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper_test.cc new file mode 100644 index 0000000000..db1030232d --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper_test.cc @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h" + +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_buffer.h" +#include "rtc_base/random.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +uint8_t RandByte(Random& random) { + return random.Rand(255); +} + +VideoFrame CreateRandom2x2VideoFrame(uint16_t id, Random& random) { + rtc::scoped_refptr<I420Buffer> buffer = I420Buffer::Create(2, 2); + + uint8_t data[6] = {RandByte(random), RandByte(random), RandByte(random), + RandByte(random), RandByte(random), RandByte(random)}; + + memcpy(buffer->MutableDataY(), data, 2); + memcpy(buffer->MutableDataY() + buffer->StrideY(), data + 2, 2); + memcpy(buffer->MutableDataU(), data + 4, 1); + memcpy(buffer->MutableDataV(), data + 5, 1); + + return VideoFrame::Builder() + .set_id(id) + .set_video_frame_buffer(buffer) + .set_timestamp_us(1) + .build(); +} + +TEST(CreateDummyFrameBufferTest, CreatedBufferIsDummy) { + VideoFrame dummy_frame = VideoFrame::Builder() + .set_video_frame_buffer(CreateDummyFrameBuffer()) + .build(); + + EXPECT_TRUE(IsDummyFrame(dummy_frame)); +} + +TEST(IsDummyFrameTest, NotEveryFrameIsDummy) { + Random random(/*seed=*/100); + VideoFrame frame = CreateRandom2x2VideoFrame(1, random); + EXPECT_FALSE(IsDummyFrame(frame)); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc new file mode 100644 index 0000000000..ccd2f03537 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h" + +#include <algorithm> +#include <cstddef> + +#include "absl/memory/memory.h" +#include "api/video/encoded_image.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +SingleProcessEncodedImageDataInjector::SingleProcessEncodedImageDataInjector() = + default; +SingleProcessEncodedImageDataInjector:: + ~SingleProcessEncodedImageDataInjector() = default; + +EncodedImage SingleProcessEncodedImageDataInjector::InjectData( + uint16_t id, + bool discard, + const EncodedImage& source) { + RTC_CHECK(source.size() >= ExtractionInfo::kUsedBufferSize); + + ExtractionInfo info; + info.discard = discard; + size_t insertion_pos = source.size() - ExtractionInfo::kUsedBufferSize; + memcpy(info.origin_data, &source.data()[insertion_pos], + ExtractionInfo::kUsedBufferSize); + { + MutexLock lock(&lock_); + // Will create new one if missed. + ExtractionInfoVector& ev = extraction_cache_[id]; + info.sub_id = ev.next_sub_id++; + ev.infos[info.sub_id] = info; + } + + auto buffer = EncodedImageBuffer::Create(source.data(), source.size()); + buffer->data()[insertion_pos] = id & 0x00ff; + buffer->data()[insertion_pos + 1] = (id & 0xff00) >> 8; + buffer->data()[insertion_pos + 2] = info.sub_id; + + EncodedImage out = source; + out.SetEncodedData(buffer); + return out; +} + +void SingleProcessEncodedImageDataInjector::AddParticipantInCall() { + MutexLock crit(&lock_); + expected_receivers_count_++; +} + +void SingleProcessEncodedImageDataInjector::RemoveParticipantInCall() { + MutexLock crit(&lock_); + expected_receivers_count_--; + // Now we need go over `extraction_cache_` and removed frames which have been + // received by `expected_receivers_count_`. + for (auto& [frame_id, extraction_infos] : extraction_cache_) { + for (auto it = extraction_infos.infos.begin(); + it != extraction_infos.infos.end();) { + // Frame is received if `received_count` equals to + // `expected_receivers_count_`. + if (it->second.received_count == expected_receivers_count_) { + it = extraction_infos.infos.erase(it); + } else { + ++it; + } + } + } +} + +EncodedImageExtractionResult SingleProcessEncodedImageDataInjector::ExtractData( + const EncodedImage& source) { + size_t size = source.size(); + auto buffer = EncodedImageBuffer::Create(source.data(), source.size()); + EncodedImage out = source; + out.SetEncodedData(buffer); + + std::vector<size_t> frame_sizes; + std::vector<size_t> frame_sl_index; + size_t max_spatial_index = out.SpatialIndex().value_or(0); + for (size_t i = 0; i <= max_spatial_index; ++i) { + auto frame_size = source.SpatialLayerFrameSize(i); + if (frame_size.value_or(0)) { + frame_sl_index.push_back(i); + frame_sizes.push_back(frame_size.value()); + } + } + if (frame_sizes.empty()) { + frame_sizes.push_back(size); + } + + size_t prev_frames_size = 0; + absl::optional<uint16_t> id = absl::nullopt; + bool discard = true; + std::vector<ExtractionInfo> extraction_infos; + for (size_t frame_size : frame_sizes) { + size_t insertion_pos = + prev_frames_size + frame_size - ExtractionInfo::kUsedBufferSize; + // Extract frame id from first 2 bytes starting from insertion pos. + uint16_t next_id = buffer->data()[insertion_pos] + + (buffer->data()[insertion_pos + 1] << 8); + // Extract frame sub id from second 3 byte starting from insertion pos. + uint8_t sub_id = buffer->data()[insertion_pos + 2]; + RTC_CHECK(!id || *id == next_id) + << "Different frames encoded into single encoded image: " << *id + << " vs " << next_id; + id = next_id; + ExtractionInfo info; + { + MutexLock lock(&lock_); + auto ext_vector_it = extraction_cache_.find(next_id); + RTC_CHECK(ext_vector_it != extraction_cache_.end()) + << "Unknown frame_id=" << next_id; + + auto info_it = ext_vector_it->second.infos.find(sub_id); + RTC_CHECK(info_it != ext_vector_it->second.infos.end()) + << "Unknown sub_id=" << sub_id << " for frame_id=" << next_id; + info_it->second.received_count++; + info = info_it->second; + if (info.received_count == expected_receivers_count_) { + ext_vector_it->second.infos.erase(info_it); + } + } + // We need to discard encoded image only if all concatenated encoded images + // have to be discarded. + discard = discard && info.discard; + + extraction_infos.push_back(info); + prev_frames_size += frame_size; + } + RTC_CHECK(id); + + if (discard) { + out.set_size(0); + for (size_t i = 0; i <= max_spatial_index; ++i) { + out.SetSpatialLayerFrameSize(i, 0); + } + return EncodedImageExtractionResult{*id, out, true}; + } + + // Make a pass from begin to end to restore origin payload and erase discarded + // encoded images. + size_t pos = 0; + for (size_t frame_index = 0; frame_index < frame_sizes.size(); + ++frame_index) { + RTC_CHECK(pos < size); + const size_t frame_size = frame_sizes[frame_index]; + const ExtractionInfo& info = extraction_infos[frame_index]; + if (info.discard) { + // If this encoded image is marked to be discarded - erase it's payload + // from the buffer. + memmove(&buffer->data()[pos], &buffer->data()[pos + frame_size], + size - pos - frame_size); + RTC_CHECK_LT(frame_index, frame_sl_index.size()) + << "codec doesn't support discard option or the image, that was " + "supposed to be discarded, is lost"; + out.SetSpatialLayerFrameSize(frame_sl_index[frame_index], 0); + size -= frame_size; + } else { + memcpy( + &buffer->data()[pos + frame_size - ExtractionInfo::kUsedBufferSize], + info.origin_data, ExtractionInfo::kUsedBufferSize); + pos += frame_size; + } + } + out.set_size(pos); + + return EncodedImageExtractionResult{*id, out, discard}; +} + +SingleProcessEncodedImageDataInjector::ExtractionInfoVector:: + ExtractionInfoVector() = default; +SingleProcessEncodedImageDataInjector::ExtractionInfoVector:: + ~ExtractionInfoVector() = default; + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h new file mode 100644 index 0000000000..1082440e2f --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_ + +#include <cstdint> +#include <map> +#include <memory> +#include <utility> +#include <vector> + +#include "api/video/encoded_image.h" +#include "rtc_base/synchronization/mutex.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Based on assumption that all call participants are in the same OS process +// and uses same QualityAnalyzingVideoContext to obtain +// EncodedImageDataInjector. +// +// To inject frame id and discard flag into EncodedImage injector uses last 3rd +// and 2nd bytes of EncodedImage payload. Then it uses last byte for frame +// sub id, that is required to distinguish different spatial layers. The origin +// data from these 3 bytes will be stored inside injector's internal storage and +// then will be restored during extraction phase. +// +// This injector won't add any extra overhead into EncodedImage payload and +// support frames with any size of payload. Also assumes that every EncodedImage +// payload size is greater or equals to 3 bytes +// +// This injector doesn't support video frames/encoded images without frame ID. +class SingleProcessEncodedImageDataInjector + : public EncodedImageDataPropagator { + public: + SingleProcessEncodedImageDataInjector(); + ~SingleProcessEncodedImageDataInjector() override; + + // Id and discard flag will be injected into EncodedImage buffer directly. + // This buffer won't be fully copied, so `source` image buffer will be also + // changed. + EncodedImage InjectData(uint16_t id, + bool discard, + const EncodedImage& source) override; + + void Start(int expected_receivers_count) override { + MutexLock crit(&lock_); + expected_receivers_count_ = expected_receivers_count; + } + void AddParticipantInCall() override; + void RemoveParticipantInCall() override; + EncodedImageExtractionResult ExtractData(const EncodedImage& source) override; + + private: + // Contains data required to extract frame id from EncodedImage and restore + // original buffer. + struct ExtractionInfo { + // Number of bytes from the beginning of the EncodedImage buffer that will + // be used to store frame id and sub id. + const static size_t kUsedBufferSize = 3; + // Frame sub id to distinguish encoded images for different spatial layers. + uint8_t sub_id; + // Flag to show is this encoded images should be discarded by analyzing + // decoder because of not required spatial layer/simulcast stream. + bool discard; + // Data from first 3 bytes of origin encoded image's payload. + uint8_t origin_data[ExtractionInfo::kUsedBufferSize]; + // Count of how many times this frame was received. + int received_count = 0; + }; + + struct ExtractionInfoVector { + ExtractionInfoVector(); + ~ExtractionInfoVector(); + + // Next sub id, that have to be used for this frame id. + uint8_t next_sub_id = 0; + std::map<uint8_t, ExtractionInfo> infos; + }; + + Mutex lock_; + int expected_receivers_count_ RTC_GUARDED_BY(lock_); + // Stores a mapping from frame id to extraction info for spatial layers + // for this frame id. There can be a lot of them, because if frame was + // dropped we can't clean it up, because we won't receive a signal on + // decoder side about that frame. In such case it will be replaced + // when sub id will overlap. + std::map<uint16_t, ExtractionInfoVector> extraction_cache_ + RTC_GUARDED_BY(lock_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc new file mode 100644 index 0000000000..f6fa40455a --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc @@ -0,0 +1,445 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h" + +#include <utility> + +#include "api/video/encoded_image.h" +#include "rtc_base/buffer.h" +#include "test/gtest.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +rtc::scoped_refptr<EncodedImageBuffer> +CreateEncodedImageBufferOfSizeNFilledWithValuesFromX(size_t n, uint8_t x) { + auto buffer = EncodedImageBuffer::Create(n); + for (size_t i = 0; i < n; ++i) { + buffer->data()[i] = static_cast<uint8_t>(x + i); + } + return buffer; +} + +EncodedImage CreateEncodedImageOfSizeNFilledWithValuesFromX(size_t n, + uint8_t x) { + EncodedImage image; + image.SetEncodedData( + CreateEncodedImageBufferOfSizeNFilledWithValuesFromX(n, x)); + return image; +} + +EncodedImage DeepCopyEncodedImage(const EncodedImage& source) { + EncodedImage copy = source; + copy.SetEncodedData(EncodedImageBuffer::Create(source.data(), source.size())); + return copy; +} + +TEST(SingleProcessEncodedImageDataInjectorTest, InjectExtractDiscardFalse) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + + EncodedImageExtractionResult out = + injector.ExtractData(injector.InjectData(512, false, source)); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, InjectExtractDiscardTrue) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + + EncodedImageExtractionResult out = + injector.ExtractData(injector.InjectData(512, true, source)); + EXPECT_EQ(out.id, 512); + EXPECT_TRUE(out.discard); + EXPECT_EQ(out.image.size(), 0ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); +} + +TEST(SingleProcessEncodedImageDataInjectorTest, + InjectWithUnsetSpatialLayerSizes) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + + EncodedImage intermediate = injector.InjectData(512, false, source); + intermediate.SetSpatialIndex(2); + + EncodedImageExtractionResult out = injector.ExtractData(intermediate); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } + EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2); + for (int i = 0; i < 3; ++i) { + EXPECT_EQ(out.image.SpatialLayerFrameSize(i).value_or(0), 0ul); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, + InjectWithZeroSpatialLayerSizes) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + + EncodedImage intermediate = injector.InjectData(512, false, source); + intermediate.SetSpatialIndex(2); + intermediate.SetSpatialLayerFrameSize(0, 0); + intermediate.SetSpatialLayerFrameSize(1, 0); + intermediate.SetSpatialLayerFrameSize(2, 0); + + EncodedImageExtractionResult out = injector.ExtractData(intermediate); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } + EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2); + for (int i = 0; i < 3; ++i) { + EXPECT_EQ(out.image.SpatialLayerFrameSize(i).value_or(0), 0ul); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, Inject3Extract3) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + // 1st frame + EncodedImage source1 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source1.SetTimestamp(123456710); + // 2nd frame 1st spatial layer + EncodedImage source2 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/11); + source2.SetTimestamp(123456720); + // 2nd frame 2nd spatial layer + EncodedImage source3 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/21); + source3.SetTimestamp(123456720); + + EncodedImage intermediate1 = injector.InjectData(510, false, source1); + EncodedImage intermediate2 = injector.InjectData(520, true, source2); + EncodedImage intermediate3 = injector.InjectData(520, false, source3); + + // Extract ids in different order. + EncodedImageExtractionResult out3 = injector.ExtractData(intermediate3); + EncodedImageExtractionResult out1 = injector.ExtractData(intermediate1); + EncodedImageExtractionResult out2 = injector.ExtractData(intermediate2); + + EXPECT_EQ(out1.id, 510); + EXPECT_FALSE(out1.discard); + EXPECT_EQ(out1.image.size(), 10ul); + EXPECT_EQ(out1.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out1.image.data()[i], i + 1); + } + EXPECT_EQ(out2.id, 520); + EXPECT_TRUE(out2.discard); + EXPECT_EQ(out2.image.size(), 0ul); + EXPECT_EQ(out2.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + EXPECT_EQ(out3.id, 520); + EXPECT_FALSE(out3.discard); + EXPECT_EQ(out3.image.size(), 10ul); + EXPECT_EQ(out3.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out3.image.data()[i], i + 21); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, InjectExtractFromConcatenated) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + EncodedImage source1 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source1.SetTimestamp(123456710); + EncodedImage source2 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/11); + source2.SetTimestamp(123456710); + EncodedImage source3 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/21); + source3.SetTimestamp(123456710); + + // Inject id into 3 images with same frame id. + EncodedImage intermediate1 = injector.InjectData(512, false, source1); + EncodedImage intermediate2 = injector.InjectData(512, true, source2); + EncodedImage intermediate3 = injector.InjectData(512, false, source3); + + // Concatenate them into single encoded image, like it can be done in jitter + // buffer. + size_t concatenated_length = + intermediate1.size() + intermediate2.size() + intermediate3.size(); + rtc::Buffer concatenated_buffer; + concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size()); + concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size()); + concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size()); + EncodedImage concatenated; + concatenated.SetEncodedData(EncodedImageBuffer::Create( + concatenated_buffer.data(), concatenated_length)); + concatenated.SetSpatialIndex(2); + concatenated.SetSpatialLayerFrameSize(0, intermediate1.size()); + concatenated.SetSpatialLayerFrameSize(1, intermediate2.size()); + concatenated.SetSpatialLayerFrameSize(2, intermediate3.size()); + + // Extract frame id from concatenated image + EncodedImageExtractionResult out = injector.ExtractData(concatenated); + + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 2 * 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + EXPECT_EQ(out.image.data()[i + 10], i + 21); + } + EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(1).value_or(0), 0ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(2).value_or(0), 10ul); +} + +TEST(SingleProcessEncodedImageDataInjector, + InjectExtractFromConcatenatedAllDiscarded) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + EncodedImage source1 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source1.SetTimestamp(123456710); + EncodedImage source2 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/11); + source2.SetTimestamp(123456710); + EncodedImage source3 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/21); + source3.SetTimestamp(123456710); + + // Inject id into 3 images with same frame id. + EncodedImage intermediate1 = injector.InjectData(512, true, source1); + EncodedImage intermediate2 = injector.InjectData(512, true, source2); + EncodedImage intermediate3 = injector.InjectData(512, true, source3); + + // Concatenate them into single encoded image, like it can be done in jitter + // buffer. + size_t concatenated_length = + intermediate1.size() + intermediate2.size() + intermediate3.size(); + rtc::Buffer concatenated_buffer; + concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size()); + concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size()); + concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size()); + EncodedImage concatenated; + concatenated.SetEncodedData(EncodedImageBuffer::Create( + concatenated_buffer.data(), concatenated_length)); + concatenated.SetSpatialIndex(2); + concatenated.SetSpatialLayerFrameSize(0, intermediate1.size()); + concatenated.SetSpatialLayerFrameSize(1, intermediate2.size()); + concatenated.SetSpatialLayerFrameSize(2, intermediate3.size()); + + // Extract frame id from concatenated image + EncodedImageExtractionResult out = injector.ExtractData(concatenated); + + EXPECT_EQ(out.id, 512); + EXPECT_TRUE(out.discard); + EXPECT_EQ(out.image.size(), 0ul); + EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2); + for (int i = 0; i < 3; ++i) { + EXPECT_EQ(out.image.SpatialLayerFrameSize(i).value_or(0), 0ul); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, InjectOnceExtractTwice) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/2); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + + EncodedImageExtractionResult out = injector.ExtractData( + injector.InjectData(/*id=*/512, /*discard=*/false, source)); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } + out = injector.ExtractData( + injector.InjectData(/*id=*/512, /*discard=*/false, source)); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, Add1stReceiverAfterStart) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/0); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + EncodedImage modified_image = injector.InjectData( + /*id=*/512, /*discard=*/false, source); + + injector.AddParticipantInCall(); + EncodedImageExtractionResult out = injector.ExtractData(modified_image); + + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, Add3rdReceiverAfterStart) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/2); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + EncodedImage modified_image = injector.InjectData( + /*id=*/512, /*discard=*/false, source); + injector.ExtractData(modified_image); + + injector.AddParticipantInCall(); + injector.ExtractData(modified_image); + EncodedImageExtractionResult out = injector.ExtractData(modified_image); + + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, + RemoveReceiverRemovesOnlyFullyReceivedFrames) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/3); + + EncodedImage source1 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source1.SetTimestamp(10); + EncodedImage source2 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source2.SetTimestamp(20); + + EncodedImage modified_image1 = injector.InjectData( + /*id=*/512, /*discard=*/false, source1); + EncodedImage modified_image2 = injector.InjectData( + /*id=*/513, /*discard=*/false, source2); + + // Out of 3 receivers 1st image received by 2 and 2nd image by 1 + injector.ExtractData(DeepCopyEncodedImage(modified_image1)); + injector.ExtractData(DeepCopyEncodedImage(modified_image1)); + injector.ExtractData(DeepCopyEncodedImage(modified_image2)); + + // When we removed one receiver, 2nd image should still be available for + // extraction. + injector.RemoveParticipantInCall(); + + EncodedImageExtractionResult out = + injector.ExtractData(DeepCopyEncodedImage(modified_image2)); + + EXPECT_EQ(out.id, 513); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +// Death tests. +// Disabled on Android because death tests misbehave on Android, see +// base/test/gtest_util.h. +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +TEST(SingleProcessEncodedImageDataInjectorTestDeathTest, + InjectOnceExtractMoreThenExpected) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/2); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + + EncodedImage modified = + injector.InjectData(/*id=*/512, /*discard=*/false, source); + + injector.ExtractData(DeepCopyEncodedImage(modified)); + injector.ExtractData(DeepCopyEncodedImage(modified)); + EXPECT_DEATH(injector.ExtractData(DeepCopyEncodedImage(modified)), + "Unknown sub_id=0 for frame_id=512"); +} + +TEST(SingleProcessEncodedImageDataInjectorTestDeathTest, + RemoveReceiverRemovesOnlyFullyReceivedFramesVerifyFrameIsRemoved) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/3); + + EncodedImage source1 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source1.SetTimestamp(10); + EncodedImage source2 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source2.SetTimestamp(20); + + EncodedImage modified_image1 = injector.InjectData( + /*id=*/512, /*discard=*/false, source1); + EncodedImage modified_image2 = injector.InjectData( + /*id=*/513, /*discard=*/false, source2); + + // Out of 3 receivers 1st image received by 2 and 2nd image by 1 + injector.ExtractData(DeepCopyEncodedImage(modified_image1)); + injector.ExtractData(DeepCopyEncodedImage(modified_image1)); + injector.ExtractData(DeepCopyEncodedImage(modified_image2)); + + // When we removed one receiver 1st image should be removed. + injector.RemoveParticipantInCall(); + + EXPECT_DEATH(injector.ExtractData(DeepCopyEncodedImage(modified_image1)), + "Unknown sub_id=0 for frame_id=512"); +} +#endif // RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.cc new file mode 100644 index 0000000000..4fec0a8f9e --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.cc @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/video_dumping.h" + +#include <stdio.h> + +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/test/video/video_frame_writer.h" +#include "api/video/video_frame.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/clock.h" +#include "test/testsupport/video_frame_writer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +class VideoFrameIdsWriter final : public test::VideoFrameWriter { + public: + explicit VideoFrameIdsWriter(absl::string_view file_name) + : file_name_(file_name) { + output_file_ = fopen(file_name_.c_str(), "wb"); + RTC_LOG(LS_INFO) << "Writing VideoFrame IDs into " << file_name_; + RTC_CHECK(output_file_ != nullptr) + << "Failed to open file to dump frame ids for writing: " << file_name_; + } + ~VideoFrameIdsWriter() override { Close(); } + + bool WriteFrame(const VideoFrame& frame) override { + RTC_CHECK(output_file_ != nullptr) << "Writer is already closed"; + int chars_written = fprintf(output_file_, "%d\n", frame.id()); + if (chars_written < 2) { + RTC_LOG(LS_ERROR) << "Failed to write frame id to the output file: " + << file_name_; + return false; + } + return true; + } + + void Close() override { + if (output_file_ != nullptr) { + RTC_LOG(LS_INFO) << "Closing file for VideoFrame IDs: " << file_name_; + fclose(output_file_); + output_file_ = nullptr; + } + } + + private: + const std::string file_name_; + FILE* output_file_; +}; + +// Broadcast received frame to multiple underlying frame writers. +class BroadcastingFrameWriter final : public test::VideoFrameWriter { + public: + explicit BroadcastingFrameWriter( + std::vector<std::unique_ptr<test::VideoFrameWriter>> delegates) + : delegates_(std::move(delegates)) {} + ~BroadcastingFrameWriter() override { Close(); } + + bool WriteFrame(const webrtc::VideoFrame& frame) override { + for (auto& delegate : delegates_) { + if (!delegate->WriteFrame(frame)) { + return false; + } + } + return true; + } + + void Close() override { + for (auto& delegate : delegates_) { + delegate->Close(); + } + } + + private: + std::vector<std::unique_ptr<test::VideoFrameWriter>> delegates_; +}; + +} // namespace + +VideoWriter::VideoWriter(test::VideoFrameWriter* video_writer, + int sampling_modulo) + : video_writer_(video_writer), sampling_modulo_(sampling_modulo) {} + +void VideoWriter::OnFrame(const VideoFrame& frame) { + if (frames_counter_++ % sampling_modulo_ != 0) { + return; + } + bool result = video_writer_->WriteFrame(frame); + RTC_CHECK(result) << "Failed to write frame"; +} + +std::unique_ptr<test::VideoFrameWriter> CreateVideoFrameWithIdsWriter( + std::unique_ptr<test::VideoFrameWriter> video_writer_delegate, + absl::string_view frame_ids_dump_file_name) { + std::vector<std::unique_ptr<test::VideoFrameWriter>> requested_writers; + requested_writers.push_back(std::move(video_writer_delegate)); + requested_writers.push_back( + std::make_unique<VideoFrameIdsWriter>(frame_ids_dump_file_name)); + return std::make_unique<BroadcastingFrameWriter>( + std::move(requested_writers)); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.h new file mode 100644 index 0000000000..cad4e1bdbf --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_DUMPING_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_DUMPING_H_ + +#include <memory> +#include <string> + +#include "absl/strings/string_view.h" +#include "api/test/video/video_frame_writer.h" +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "test/testsupport/video_frame_writer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// `VideoSinkInterface` to dump incoming video frames into specified video +// writer. +class VideoWriter final : public rtc::VideoSinkInterface<VideoFrame> { + public: + // Creates video writer. Caller keeps ownership of `video_writer` and is + // responsible for closing it after VideoWriter will be destroyed. + VideoWriter(test::VideoFrameWriter* video_writer, int sampling_modulo); + VideoWriter(const VideoWriter&) = delete; + VideoWriter& operator=(const VideoWriter&) = delete; + ~VideoWriter() override = default; + + void OnFrame(const VideoFrame& frame) override; + + private: + test::VideoFrameWriter* const video_writer_; + const int sampling_modulo_; + + int64_t frames_counter_ = 0; +}; + +// Creates a `VideoFrameWriter` to dump video frames together with their ids. +// It uses provided `video_writer_delegate` to write video itself. Frame ids +// will be logged into the specified file. +std::unique_ptr<test::VideoFrameWriter> CreateVideoFrameWithIdsWriter( + std::unique_ptr<test::VideoFrameWriter> video_writer_delegate, + absl::string_view frame_ids_dump_file_name); + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_DUMPING_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping_test.cc new file mode 100644 index 0000000000..5dd4021516 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping_test.cc @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/video_dumping.h" + +#include <stdio.h> + +#include <memory> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_buffer.h" +#include "rtc_base/random.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" +#include "test/testsupport/frame_reader.h" +#include "test/testsupport/video_frame_writer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::testing::ElementsAreArray; +using ::testing::Eq; +using ::testing::Test; + +uint8_t RandByte(Random& random) { + return random.Rand(255); +} + +VideoFrame CreateRandom2x2VideoFrame(uint16_t id, Random& random) { + rtc::scoped_refptr<I420Buffer> buffer = I420Buffer::Create(2, 2); + + uint8_t data[6] = {RandByte(random), RandByte(random), RandByte(random), + RandByte(random), RandByte(random), RandByte(random)}; + + memcpy(buffer->MutableDataY(), data, 2); + memcpy(buffer->MutableDataY() + buffer->StrideY(), data + 2, 2); + memcpy(buffer->MutableDataU(), data + 4, 1); + memcpy(buffer->MutableDataV(), data + 5, 1); + + return VideoFrame::Builder() + .set_id(id) + .set_video_frame_buffer(buffer) + .set_timestamp_us(1) + .build(); +} + +std::vector<uint8_t> AsVector(const uint8_t* data, size_t size) { + std::vector<uint8_t> out; + out.assign(data, data + size); + return out; +} + +void AssertFramesEqual(rtc::scoped_refptr<webrtc::I420BufferInterface> actual, + rtc::scoped_refptr<VideoFrameBuffer> expected) { + ASSERT_THAT(actual->width(), Eq(expected->width())); + ASSERT_THAT(actual->height(), Eq(expected->height())); + rtc::scoped_refptr<webrtc::I420BufferInterface> expected_i420 = + expected->ToI420(); + + int height = actual->height(); + + EXPECT_THAT(AsVector(actual->DataY(), actual->StrideY() * height), + ElementsAreArray(expected_i420->DataY(), + expected_i420->StrideY() * height)); + EXPECT_THAT(AsVector(actual->DataU(), actual->StrideU() * (height + 1) / 2), + ElementsAreArray(expected_i420->DataU(), + expected_i420->StrideU() * (height + 1) / 2)); + EXPECT_THAT(AsVector(actual->DataV(), actual->StrideV() * (height + 1) / 2), + ElementsAreArray(expected_i420->DataV(), + expected_i420->StrideV() * (height + 1) / 2)); +} + +void AssertFrameIdsAre(const std::string& filename, + std::vector<std::string> expected_ids) { + FILE* file = fopen(filename.c_str(), "r"); + ASSERT_TRUE(file != nullptr); + std::vector<std::string> actual_ids; + char buffer[8]; + while (fgets(buffer, sizeof buffer, file) != nullptr) { + std::string current_id(buffer); + ASSERT_GE(current_id.size(), 2lu); + // Trim "\n" at the end. + actual_ids.push_back(current_id.substr(0, current_id.size() - 1)); + } + EXPECT_THAT(actual_ids, ElementsAreArray(expected_ids)); +} + +class VideoDumpingTest : public Test { + protected: + ~VideoDumpingTest() override = default; + + void SetUp() override { + video_filename_ = webrtc::test::TempFilename(webrtc::test::OutputPath(), + "video_dumping_test"); + ids_filename_ = webrtc::test::TempFilename(webrtc::test::OutputPath(), + "video_dumping_test"); + } + + void TearDown() override { + remove(video_filename_.c_str()); + remove(ids_filename_.c_str()); + } + + std::string video_filename_; + std::string ids_filename_; +}; + +using CreateVideoFrameWithIdsWriterTest = VideoDumpingTest; + +TEST_F(CreateVideoFrameWithIdsWriterTest, VideoIsWritenWithFrameIds) { + Random random(/*seed=*/100); + VideoFrame frame1 = CreateRandom2x2VideoFrame(1, random); + VideoFrame frame2 = CreateRandom2x2VideoFrame(2, random); + + std::unique_ptr<test::VideoFrameWriter> writer = + CreateVideoFrameWithIdsWriter( + std::make_unique<test::Y4mVideoFrameWriterImpl>( + std::string(video_filename_), + /*width=*/2, /*height=*/2, /*fps=*/2), + ids_filename_); + + ASSERT_TRUE(writer->WriteFrame(frame1)); + ASSERT_TRUE(writer->WriteFrame(frame2)); + writer->Close(); + + auto frame_reader = test::CreateY4mFrameReader(video_filename_); + EXPECT_THAT(frame_reader->num_frames(), Eq(2)); + AssertFramesEqual(frame_reader->PullFrame(), frame1.video_frame_buffer()); + AssertFramesEqual(frame_reader->PullFrame(), frame2.video_frame_buffer()); + AssertFrameIdsAre(ids_filename_, {"1", "2"}); +} + +using VideoWriterTest = VideoDumpingTest; + +TEST_F(VideoWriterTest, AllFramesAreWrittenWithSamplingModulo1) { + Random random(/*seed=*/100); + VideoFrame frame1 = CreateRandom2x2VideoFrame(1, random); + VideoFrame frame2 = CreateRandom2x2VideoFrame(2, random); + + { + test::Y4mVideoFrameWriterImpl frame_writer(std::string(video_filename_), + /*width=*/2, /*height=*/2, + /*fps=*/2); + VideoWriter writer(&frame_writer, /*sampling_modulo=*/1); + + writer.OnFrame(frame1); + writer.OnFrame(frame2); + frame_writer.Close(); + } + + auto frame_reader = test::CreateY4mFrameReader(video_filename_); + EXPECT_THAT(frame_reader->num_frames(), Eq(2)); + AssertFramesEqual(frame_reader->PullFrame(), frame1.video_frame_buffer()); + AssertFramesEqual(frame_reader->PullFrame(), frame2.video_frame_buffer()); +} + +TEST_F(VideoWriterTest, OnlyEvery2ndFramesIsWrittenWithSamplingModulo2) { + Random random(/*seed=*/100); + VideoFrame frame1 = CreateRandom2x2VideoFrame(1, random); + VideoFrame frame2 = CreateRandom2x2VideoFrame(2, random); + VideoFrame frame3 = CreateRandom2x2VideoFrame(3, random); + + { + test::Y4mVideoFrameWriterImpl frame_writer(std::string(video_filename_), + /*width=*/2, /*height=*/2, + /*fps=*/2); + VideoWriter writer(&frame_writer, /*sampling_modulo=*/2); + + writer.OnFrame(frame1); + writer.OnFrame(frame2); + writer.OnFrame(frame3); + frame_writer.Close(); + } + + auto frame_reader = test::CreateY4mFrameReader(video_filename_); + EXPECT_THAT(frame_reader->num_frames(), Eq(2)); + AssertFramesEqual(frame_reader->PullFrame(), frame1.video_frame_buffer()); + AssertFramesEqual(frame_reader->PullFrame(), frame3.video_frame_buffer()); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.cc new file mode 100644 index 0000000000..5a74d60250 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.cc @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h" + +#include "absl/memory/memory.h" +#include "api/video/encoded_image.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +EncodedImage VideoFrameTrackingIdInjector::InjectData( + uint16_t id, + bool unused_discard, + const EncodedImage& source) { + RTC_CHECK(!unused_discard); + EncodedImage out = source; + out.SetVideoFrameTrackingId(id); + return out; +} + +EncodedImageExtractionResult VideoFrameTrackingIdInjector::ExtractData( + const EncodedImage& source) { + return EncodedImageExtractionResult{source.VideoFrameTrackingId(), source, + /*discard=*/false}; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h new file mode 100644 index 0000000000..ecc3cd3f51 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_FRAME_TRACKING_ID_INJECTOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_FRAME_TRACKING_ID_INJECTOR_H_ + +#include <cstdint> + +#include "api/video/encoded_image.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// This injector sets and retrieves the provided id in the EncodedImage +// video_frame_tracking_id field. This is only possible with the RTP header +// extension VideoFrameTrackingIdExtension that will propagate the input +// tracking id to the received EncodedImage. This RTP header extension is +// enabled with the field trial WebRTC-VideoFrameTrackingIdAdvertised +// (http://www.webrtc.org/experiments/rtp-hdrext/video-frame-tracking-id). +// +// Note that this injector doesn't allow to discard frames. +class VideoFrameTrackingIdInjector : public EncodedImageDataPropagator { + public: + EncodedImage InjectData(uint16_t id, + bool unused_discard, + const EncodedImage& source) override; + + EncodedImageExtractionResult ExtractData(const EncodedImage& source) override; + + void Start(int) override {} + void AddParticipantInCall() override {} + void RemoveParticipantInCall() override {} +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_FRAME_TRACKING_ID_INJECTOR_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector_unittest.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector_unittest.cc new file mode 100644 index 0000000000..c7d453c4bb --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector_unittest.cc @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h" + +#include "api/video/encoded_image.h" +#include "rtc_base/buffer.h" +#include "test/gtest.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +EncodedImage CreateEncodedImageOfSizeN(size_t n) { + EncodedImage image; + rtc::scoped_refptr<EncodedImageBuffer> buffer = EncodedImageBuffer::Create(n); + for (size_t i = 0; i < n; ++i) { + buffer->data()[i] = static_cast<uint8_t>(i); + } + image.SetEncodedData(buffer); + return image; +} + +TEST(VideoFrameTrackingIdInjectorTest, InjectExtractDiscardFalse) { + VideoFrameTrackingIdInjector injector; + EncodedImage source = CreateEncodedImageOfSizeN(10); + EncodedImageExtractionResult out = + injector.ExtractData(injector.InjectData(512, false, source)); + + ASSERT_TRUE(out.id.has_value()); + EXPECT_EQ(*out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(source.data()[i], out.image.data()[i]); + } +} + +#if GTEST_HAS_DEATH_TEST +TEST(VideoFrameTrackingIdInjectorTest, InjectExtractDiscardTrue) { + VideoFrameTrackingIdInjector injector; + EncodedImage source = CreateEncodedImageOfSizeN(10); + + EXPECT_DEATH(injector.InjectData(512, true, source), ""); +} +#endif // GTEST_HAS_DEATH_TEST + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc new file mode 100644 index 0000000000..87c11886cc --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc @@ -0,0 +1,264 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h" + +#include <stdio.h> + +#include <memory> +#include <utility> +#include <vector> + +#include "absl/memory/memory.h" +#include "absl/strings/string_view.h" +#include "api/array_view.h" +#include "api/test/pclf/media_configuration.h" +#include "api/video/i420_buffer.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/strings/string_builder.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/analyzing_video_sink.h" +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h" +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h" +#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h" +#include "test/pc/e2e/analyzer/video/video_dumping.h" +#include "test/testsupport/fixed_fps_video_frame_writer_adapter.h" +#include "test/video_renderer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using webrtc::webrtc_pc_e2e::VideoConfig; +using EmulatedSFUConfigMap = + ::webrtc::webrtc_pc_e2e::QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap; + +class AnalyzingFramePreprocessor + : public test::TestVideoCapturer::FramePreprocessor { + public: + AnalyzingFramePreprocessor( + absl::string_view peer_name, + absl::string_view stream_label, + VideoQualityAnalyzerInterface* analyzer, + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> sinks) + : peer_name_(peer_name), + stream_label_(stream_label), + analyzer_(analyzer), + sinks_(std::move(sinks)) {} + ~AnalyzingFramePreprocessor() override = default; + + VideoFrame Preprocess(const VideoFrame& source_frame) override { + // Copy VideoFrame to be able to set id on it. + VideoFrame frame = source_frame; + uint16_t frame_id = + analyzer_->OnFrameCaptured(peer_name_, stream_label_, frame); + frame.set_id(frame_id); + + for (auto& sink : sinks_) { + sink->OnFrame(frame); + } + return frame; + } + + private: + const std::string peer_name_; + const std::string stream_label_; + VideoQualityAnalyzerInterface* const analyzer_; + const std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> + sinks_; +}; + +} // namespace + +VideoQualityAnalyzerInjectionHelper::VideoQualityAnalyzerInjectionHelper( + Clock* clock, + std::unique_ptr<VideoQualityAnalyzerInterface> analyzer, + EncodedImageDataInjector* injector, + EncodedImageDataExtractor* extractor) + : clock_(clock), + analyzer_(std::move(analyzer)), + injector_(injector), + extractor_(extractor) { + RTC_DCHECK(clock_); + RTC_DCHECK(injector_); + RTC_DCHECK(extractor_); +} +VideoQualityAnalyzerInjectionHelper::~VideoQualityAnalyzerInjectionHelper() = + default; + +std::unique_ptr<VideoEncoderFactory> +VideoQualityAnalyzerInjectionHelper::WrapVideoEncoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoEncoderFactory> delegate, + double bitrate_multiplier, + EmulatedSFUConfigMap stream_to_sfu_config) const { + return std::make_unique<QualityAnalyzingVideoEncoderFactory>( + peer_name, std::move(delegate), bitrate_multiplier, + std::move(stream_to_sfu_config), injector_, analyzer_.get()); +} + +std::unique_ptr<VideoDecoderFactory> +VideoQualityAnalyzerInjectionHelper::WrapVideoDecoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoDecoderFactory> delegate) const { + return std::make_unique<QualityAnalyzingVideoDecoderFactory>( + peer_name, std::move(delegate), extractor_, analyzer_.get()); +} + +std::unique_ptr<test::TestVideoCapturer::FramePreprocessor> +VideoQualityAnalyzerInjectionHelper::CreateFramePreprocessor( + absl::string_view peer_name, + const VideoConfig& config) { + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> sinks; + if (config.input_dump_options.has_value()) { + std::unique_ptr<test::VideoFrameWriter> writer = + config.input_dump_options->CreateInputDumpVideoFrameWriter( + *config.stream_label, config.GetResolution()); + sinks.push_back(std::make_unique<VideoWriter>( + writer.get(), config.input_dump_options->sampling_modulo())); + video_writers_.push_back(std::move(writer)); + } + if (config.show_on_screen) { + sinks.push_back(absl::WrapUnique( + test::VideoRenderer::Create((*config.stream_label + "-capture").c_str(), + config.width, config.height))); + } + sinks_helper_.AddConfig(peer_name, config); + { + MutexLock lock(&mutex_); + known_video_configs_.insert({*config.stream_label, config}); + } + return std::make_unique<AnalyzingFramePreprocessor>( + peer_name, std::move(*config.stream_label), analyzer_.get(), + std::move(sinks)); +} + +std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> +VideoQualityAnalyzerInjectionHelper::CreateVideoSink( + absl::string_view peer_name) { + return std::make_unique<AnalyzingVideoSink2>(peer_name, this); +} + +std::unique_ptr<AnalyzingVideoSink> +VideoQualityAnalyzerInjectionHelper::CreateVideoSink( + absl::string_view peer_name, + const VideoSubscription& subscription, + bool report_infra_metrics) { + return std::make_unique<AnalyzingVideoSink>(peer_name, clock_, *analyzer_, + sinks_helper_, subscription, + report_infra_metrics); +} + +void VideoQualityAnalyzerInjectionHelper::Start( + std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count) { + analyzer_->Start(std::move(test_case_name), peer_names, max_threads_count); + extractor_->Start(peer_names.size()); + MutexLock lock(&mutex_); + peers_count_ = peer_names.size(); +} + +void VideoQualityAnalyzerInjectionHelper::RegisterParticipantInCall( + absl::string_view peer_name) { + analyzer_->RegisterParticipantInCall(peer_name); + extractor_->AddParticipantInCall(); + MutexLock lock(&mutex_); + peers_count_++; +} + +void VideoQualityAnalyzerInjectionHelper::UnregisterParticipantInCall( + absl::string_view peer_name) { + analyzer_->UnregisterParticipantInCall(peer_name); + extractor_->RemoveParticipantInCall(); + MutexLock lock(&mutex_); + peers_count_--; +} + +void VideoQualityAnalyzerInjectionHelper::OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) { + analyzer_->OnStatsReports(pc_label, report); +} + +void VideoQualityAnalyzerInjectionHelper::Stop() { + analyzer_->Stop(); + for (const auto& video_writer : video_writers_) { + video_writer->Close(); + } + video_writers_.clear(); + sinks_helper_.Clear(); +} + +void VideoQualityAnalyzerInjectionHelper::OnFrame(absl::string_view peer_name, + const VideoFrame& frame) { + if (IsDummyFrame(frame)) { + // This is dummy frame, so we don't need to process it further. + return; + } + // Copy entire video frame including video buffer to ensure that analyzer + // won't hold any WebRTC internal buffers. + VideoFrame frame_copy = frame; + frame_copy.set_video_frame_buffer( + I420Buffer::Copy(*frame.video_frame_buffer()->ToI420())); + analyzer_->OnFrameRendered(peer_name, frame_copy); + + if (frame.id() != VideoFrame::kNotSetId) { + std::string stream_label = analyzer_->GetStreamLabel(frame.id()); + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>* sinks = + PopulateSinks(ReceiverStream(peer_name, stream_label)); + if (sinks == nullptr) { + return; + } + for (auto& sink : *sinks) { + sink->OnFrame(frame); + } + } +} + +std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>* +VideoQualityAnalyzerInjectionHelper::PopulateSinks( + const ReceiverStream& receiver_stream) { + MutexLock lock(&mutex_); + auto sinks_it = sinks_.find(receiver_stream); + if (sinks_it != sinks_.end()) { + return &sinks_it->second; + } + auto it = known_video_configs_.find(receiver_stream.stream_label); + RTC_DCHECK(it != known_video_configs_.end()) + << "No video config for stream " << receiver_stream.stream_label; + const VideoConfig& config = it->second; + + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> sinks; + if (config.output_dump_options.has_value()) { + std::unique_ptr<test::VideoFrameWriter> writer = + config.output_dump_options->CreateOutputDumpVideoFrameWriter( + receiver_stream.stream_label, receiver_stream.peer_name, + config.GetResolution()); + if (config.output_dump_use_fixed_framerate) { + writer = std::make_unique<test::FixedFpsVideoFrameWriterAdapter>( + config.fps, clock_, std::move(writer)); + } + sinks.push_back(std::make_unique<VideoWriter>( + writer.get(), config.output_dump_options->sampling_modulo())); + video_writers_.push_back(std::move(writer)); + } + if (config.show_on_screen) { + sinks.push_back(absl::WrapUnique( + test::VideoRenderer::Create((*config.stream_label + "-render").c_str(), + config.width, config.height))); + } + sinks_.insert({receiver_stream, std::move(sinks)}); + return &(sinks_.find(receiver_stream)->second); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h new file mode 100644 index 0000000000..7421c8e4a7 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_ANALYZER_INJECTION_HELPER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_ANALYZER_INJECTION_HELPER_H_ + +#include <stdio.h> + +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/array_view.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/stats_observer_interface.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "rtc_base/synchronization/mutex.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/analyzing_video_sink.h" +#include "test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h" +#include "test/test_video_capturer.h" +#include "test/testsupport/video_frame_writer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Provides factory methods for components, that will be used to inject +// VideoQualityAnalyzerInterface into PeerConnection pipeline. +class VideoQualityAnalyzerInjectionHelper : public StatsObserverInterface { + public: + VideoQualityAnalyzerInjectionHelper( + Clock* clock, + std::unique_ptr<VideoQualityAnalyzerInterface> analyzer, + EncodedImageDataInjector* injector, + EncodedImageDataExtractor* extractor); + ~VideoQualityAnalyzerInjectionHelper() override; + + // Wraps video encoder factory to give video quality analyzer access to frames + // before encoding and encoded images after. + std::unique_ptr<VideoEncoderFactory> WrapVideoEncoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoEncoderFactory> delegate, + double bitrate_multiplier, + QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap stream_to_sfu_config) + const; + // Wraps video decoder factory to give video quality analyzer access to + // received encoded images and frames, that were decoded from them. + std::unique_ptr<VideoDecoderFactory> WrapVideoDecoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoDecoderFactory> delegate) const; + + // Creates VideoFrame preprocessor, that will allow video quality analyzer to + // get access to the captured frames. If provided config also specifies + // `input_dump_file_name`, video will be written into that file. + std::unique_ptr<test::TestVideoCapturer::FramePreprocessor> + CreateFramePreprocessor(absl::string_view peer_name, + const webrtc::webrtc_pc_e2e::VideoConfig& config); + // Creates sink, that will allow video quality analyzer to get access to + // the rendered frames. If corresponding video track has + // `output_dump_file_name` in its VideoConfig, which was used for + // CreateFramePreprocessor(...), then video also will be written + // into that file. + // TODO(titovartem): Remove method with `peer_name` only parameter. + std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> CreateVideoSink( + absl::string_view peer_name); + std::unique_ptr<AnalyzingVideoSink> CreateVideoSink( + absl::string_view peer_name, + const VideoSubscription& subscription, + bool report_infra_metrics); + + void Start(std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count = 1); + + // Registers new call participant to the underlying video quality analyzer. + // The method should be called before the participant is actually added. + void RegisterParticipantInCall(absl::string_view peer_name); + + // Will be called after test removed existing participant in the middle of the + // call. + void UnregisterParticipantInCall(absl::string_view peer_name); + + // Forwards `stats_reports` for Peer Connection `pc_label` to + // `analyzer_`. + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override; + + // Stops VideoQualityAnalyzerInterface to populate final data and metrics. + // Should be invoked after analyzed video tracks are disposed. + void Stop(); + + private: + // Deprecated, to be removed when old API isn't used anymore. + class AnalyzingVideoSink2 final : public rtc::VideoSinkInterface<VideoFrame> { + public: + explicit AnalyzingVideoSink2(absl::string_view peer_name, + VideoQualityAnalyzerInjectionHelper* helper) + : peer_name_(peer_name), helper_(helper) {} + ~AnalyzingVideoSink2() override = default; + + void OnFrame(const VideoFrame& frame) override { + helper_->OnFrame(peer_name_, frame); + } + + private: + const std::string peer_name_; + VideoQualityAnalyzerInjectionHelper* const helper_; + }; + + struct ReceiverStream { + ReceiverStream(absl::string_view peer_name, absl::string_view stream_label) + : peer_name(peer_name), stream_label(stream_label) {} + + std::string peer_name; + std::string stream_label; + + // Define operators required to use ReceiverStream as std::map key. + bool operator==(const ReceiverStream& o) const { + return peer_name == o.peer_name && stream_label == o.stream_label; + } + bool operator<(const ReceiverStream& o) const { + return (peer_name == o.peer_name) ? stream_label < o.stream_label + : peer_name < o.peer_name; + } + }; + + // Creates a deep copy of the frame and passes it to the video analyzer, while + // passing real frame to the sinks + void OnFrame(absl::string_view peer_name, const VideoFrame& frame); + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>* + PopulateSinks(const ReceiverStream& receiver_stream); + + Clock* const clock_; + std::unique_ptr<VideoQualityAnalyzerInterface> analyzer_; + EncodedImageDataInjector* injector_; + EncodedImageDataExtractor* extractor_; + + std::vector<std::unique_ptr<test::VideoFrameWriter>> video_writers_; + + AnalyzingVideoSinksHelper sinks_helper_; + Mutex mutex_; + int peers_count_ RTC_GUARDED_BY(mutex_); + // Map from stream label to the video config. + std::map<std::string, webrtc::webrtc_pc_e2e::VideoConfig> known_video_configs_ + RTC_GUARDED_BY(mutex_); + std::map<ReceiverStream, + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>> + sinks_ RTC_GUARDED_BY(mutex_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_ANALYZER_INJECTION_HELPER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc new file mode 100644 index 0000000000..8049af308e --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h" + +#include <map> +#include <string> + +#include "api/stats/rtc_stats.h" +#include "api/stats/rtcstats_objects.h" +#include "api/test/metrics/metric.h" +#include "api/units/data_rate.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "rtc_base/checks.h" +#include "test/pc/e2e/metric_metadata_keys.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Unit; +using ::webrtc::webrtc_pc_e2e::MetricMetadataKey; + +SamplesStatsCounter BytesPerSecondToKbps(const SamplesStatsCounter& counter) { + return counter * 0.008; +} + +} // namespace + +VideoQualityMetricsReporter::VideoQualityMetricsReporter( + Clock* const clock, + test::MetricsLogger* const metrics_logger) + : clock_(clock), metrics_logger_(metrics_logger) { + RTC_CHECK(metrics_logger_); +} + +void VideoQualityMetricsReporter::Start( + absl::string_view test_case_name, + const TrackIdStreamInfoMap* /*reporter_helper*/) { + test_case_name_ = std::string(test_case_name); + start_time_ = Now(); +} + +void VideoQualityMetricsReporter::OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) { + RTC_CHECK(start_time_) + << "Please invoke Start(...) method before calling OnStatsReports(...)"; + + auto transport_stats = report->GetStatsOfType<RTCTransportStats>(); + if (transport_stats.size() == 0u || + !transport_stats[0]->selected_candidate_pair_id.is_defined()) { + return; + } + RTC_DCHECK_EQ(transport_stats.size(), 1); + std::string selected_ice_id = + transport_stats[0]->selected_candidate_pair_id.ValueToString(); + // Use the selected ICE candidate pair ID to get the appropriate ICE stats. + const RTCIceCandidatePairStats ice_candidate_pair_stats = + report->Get(selected_ice_id)->cast_to<const RTCIceCandidatePairStats>(); + + auto outbound_rtp_stats = report->GetStatsOfType<RTCOutboundRTPStreamStats>(); + StatsSample sample; + for (auto& s : outbound_rtp_stats) { + if (!s->kind.is_defined()) { + continue; + } + if (!(*s->kind == RTCMediaStreamTrackKind::kVideo)) { + continue; + } + if (s->timestamp() > sample.sample_time) { + sample.sample_time = s->timestamp(); + } + sample.retransmitted_bytes_sent += + DataSize::Bytes(s->retransmitted_bytes_sent.ValueOrDefault(0ul)); + sample.bytes_sent += DataSize::Bytes(s->bytes_sent.ValueOrDefault(0ul)); + sample.header_bytes_sent += + DataSize::Bytes(s->header_bytes_sent.ValueOrDefault(0ul)); + } + + MutexLock lock(&video_bwe_stats_lock_); + VideoBweStats& video_bwe_stats = video_bwe_stats_[std::string(pc_label)]; + if (ice_candidate_pair_stats.available_outgoing_bitrate.is_defined()) { + video_bwe_stats.available_send_bandwidth.AddSample( + DataRate::BitsPerSec( + *ice_candidate_pair_stats.available_outgoing_bitrate) + .bytes_per_sec()); + } + + StatsSample prev_sample = last_stats_sample_[std::string(pc_label)]; + if (prev_sample.sample_time.IsZero()) { + prev_sample.sample_time = start_time_.value(); + } + last_stats_sample_[std::string(pc_label)] = sample; + + TimeDelta time_between_samples = sample.sample_time - prev_sample.sample_time; + if (time_between_samples.IsZero()) { + return; + } + + DataRate retransmission_bitrate = + (sample.retransmitted_bytes_sent - prev_sample.retransmitted_bytes_sent) / + time_between_samples; + video_bwe_stats.retransmission_bitrate.AddSample( + retransmission_bitrate.bytes_per_sec()); + DataRate transmission_bitrate = + (sample.bytes_sent + sample.header_bytes_sent - prev_sample.bytes_sent - + prev_sample.header_bytes_sent) / + time_between_samples; + video_bwe_stats.transmission_bitrate.AddSample( + transmission_bitrate.bytes_per_sec()); +} + +void VideoQualityMetricsReporter::StopAndReportResults() { + MutexLock video_bwemutex_(&video_bwe_stats_lock_); + for (const auto& item : video_bwe_stats_) { + ReportVideoBweResults(item.first, item.second); + } +} + +std::string VideoQualityMetricsReporter::GetTestCaseName( + const std::string& peer_name) const { + return test_case_name_ + "/" + peer_name; +} + +void VideoQualityMetricsReporter::ReportVideoBweResults( + const std::string& peer_name, + const VideoBweStats& video_bwe_stats) { + std::string test_case_name = GetTestCaseName(peer_name); + // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey. + std::map<std::string, std::string> metric_metadata{ + {MetricMetadataKey::kPeerMetadataKey, peer_name}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_case_name_}}; + + metrics_logger_->LogMetric( + "available_send_bandwidth", test_case_name, + BytesPerSecondToKbps(video_bwe_stats.available_send_bandwidth), + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter, + metric_metadata); + metrics_logger_->LogMetric( + "transmission_bitrate", test_case_name, + BytesPerSecondToKbps(video_bwe_stats.transmission_bitrate), + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter, + metric_metadata); + metrics_logger_->LogMetric( + "retransmission_bitrate", test_case_name, + BytesPerSecondToKbps(video_bwe_stats.retransmission_bitrate), + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter, + metric_metadata); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h new file mode 100644 index 0000000000..d3d976343b --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_METRICS_REPORTER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_METRICS_REPORTER_H_ + +#include <map> +#include <string> + +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metrics_logger.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/test/track_id_stream_info_map.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +struct VideoBweStats { + SamplesStatsCounter available_send_bandwidth; + SamplesStatsCounter transmission_bitrate; + SamplesStatsCounter retransmission_bitrate; +}; + +class VideoQualityMetricsReporter + : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { + public: + VideoQualityMetricsReporter(Clock* const clock, + test::MetricsLogger* const metrics_logger); + ~VideoQualityMetricsReporter() override = default; + + void Start(absl::string_view test_case_name, + const TrackIdStreamInfoMap* reporter_helper) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override; + void StopAndReportResults() override; + + private: + struct StatsSample { + DataSize bytes_sent = DataSize::Zero(); + DataSize header_bytes_sent = DataSize::Zero(); + DataSize retransmitted_bytes_sent = DataSize::Zero(); + + Timestamp sample_time = Timestamp::Zero(); + }; + + std::string GetTestCaseName(const std::string& peer_name) const; + void ReportVideoBweResults(const std::string& peer_name, + const VideoBweStats& video_bwe_stats); + Timestamp Now() const { return clock_->CurrentTime(); } + + Clock* const clock_; + test::MetricsLogger* const metrics_logger_; + + std::string test_case_name_; + absl::optional<Timestamp> start_time_; + + Mutex video_bwe_stats_lock_; + // Map between a peer connection label (provided by the framework) and + // its video BWE stats. + std::map<std::string, VideoBweStats> video_bwe_stats_ + RTC_GUARDED_BY(video_bwe_stats_lock_); + std::map<std::string, StatsSample> last_stats_sample_ + RTC_GUARDED_BY(video_bwe_stats_lock_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_METRICS_REPORTER_H_ |