diff options
Diffstat (limited to 'third_party/libwebrtc/test/pc')
99 files changed, 22642 insertions, 0 deletions
diff --git a/third_party/libwebrtc/test/pc/e2e/BUILD.gn b/third_party/libwebrtc/test/pc/e2e/BUILD.gn new file mode 100644 index 0000000000..7354aa8ba4 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/BUILD.gn @@ -0,0 +1,573 @@ +# Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../webrtc.gni") + +rtc_library("metric_metadata_keys") { + testonly = true + sources = [ "metric_metadata_keys.h" ] +} + +if (!build_with_chromium) { + group("e2e") { + testonly = true + + deps = [ ":metric_metadata_keys" ] + if (rtc_include_tests) { + deps += [ + ":peerconnection_quality_test", + ":test_peer", + ] + } + } + + if (rtc_include_tests) { + group("e2e_unittests") { + testonly = true + + deps = [ + ":peer_connection_e2e_smoke_test", + ":peer_connection_quality_test_metric_names_test", + ":peer_connection_quality_test_test", + ":stats_based_network_quality_metrics_reporter_test", + ":stats_poller_test", + ] + } + } + + if (rtc_include_tests) { + rtc_library("echo_emulation") { + testonly = true + sources = [ + "echo/echo_emulation.cc", + "echo/echo_emulation.h", + ] + deps = [ + "../../../api/test/pclf:media_configuration", + "../../../modules/audio_device:audio_device_impl", + "../../../rtc_base:swap_queue", + ] + } + + rtc_library("test_peer") { + testonly = true + sources = [ + "test_peer.cc", + "test_peer.h", + ] + deps = [ + ":stats_provider", + "../../../api:frame_generator_api", + "../../../api:function_view", + "../../../api:libjingle_peerconnection_api", + "../../../api:scoped_refptr", + "../../../api:sequence_checker", + "../../../api/task_queue:pending_task_safety_flag", + "../../../api/test/pclf:media_configuration", + "../../../api/test/pclf:media_quality_test_params", + "../../../api/test/pclf:peer_configurer", + "../../../modules/audio_processing:api", + "../../../pc:peerconnection_wrapper", + "../../../rtc_base:logging", + "../../../rtc_base:refcount", + "../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:variant", + ] + } + + rtc_library("test_peer_factory") { + testonly = true + sources = [ + "test_peer_factory.cc", + "test_peer_factory.h", + ] + deps = [ + ":echo_emulation", + ":test_peer", + "../..:copy_to_file_audio_capturer", + "../../../api:create_time_controller", + "../../../api:time_controller", + "../../../api/rtc_event_log:rtc_event_log_factory", + "../../../api/task_queue:default_task_queue_factory", + "../../../api/test/pclf:media_configuration", + "../../../api/test/pclf:media_quality_test_params", + "../../../api/test/pclf:peer_configurer", + "../../../api/transport:field_trial_based_config", + "../../../api/video_codecs:builtin_video_decoder_factory", + "../../../api/video_codecs:builtin_video_encoder_factory", + "../../../media:rtc_audio_video", + "../../../media:rtc_media_engine_defaults", + "../../../modules/audio_device:audio_device_impl", + "../../../modules/audio_processing/aec_dump", + "../../../p2p:rtc_p2p", + "../../../rtc_base:rtc_task_queue", + "../../../rtc_base:threading", + "analyzer/video:quality_analyzing_video_encoder", + "analyzer/video:video_quality_analyzer_injection_helper", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + ] + } + + rtc_library("media_helper") { + testonly = true + sources = [ + "media/media_helper.cc", + "media/media_helper.h", + "media/test_video_capturer_video_track_source.h", + ] + deps = [ + ":test_peer", + "../..:fileutils", + "../..:platform_video_capturer", + "../..:video_test_common", + "../../../api:create_frame_generator", + "../../../api:frame_generator_api", + "../../../api:media_stream_interface", + "../../../api/test/pclf:media_configuration", + "../../../api/test/pclf:peer_configurer", + "../../../api/video:video_frame", + "../../../pc:session_description", + "../../../pc:video_track_source", + "analyzer/video:video_quality_analyzer_injection_helper", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:variant" ] + } + + rtc_library("peer_params_preprocessor") { + visibility = [ "*" ] + testonly = true + sources = [ + "peer_params_preprocessor.cc", + "peer_params_preprocessor.h", + ] + deps = [ + "../..:fileutils", + "../../../api:peer_network_dependencies", + "../../../api/test/pclf:media_configuration", + "../../../api/test/pclf:media_quality_test_params", + "../../../api/test/pclf:peer_configurer", + "../../../modules/video_coding/svc:scalability_mode_util", + "../../../modules/video_coding/svc:scalability_structures", + "../../../rtc_base:macromagic", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + + rtc_library("test_activities_executor") { + testonly = true + sources = [ + "test_activities_executor.cc", + "test_activities_executor.h", + ] + deps = [ + "../../../api/task_queue", + "../../../api/units:time_delta", + "../../../api/units:timestamp", + "../../../rtc_base:checks", + "../../../rtc_base:criticalsection", + "../../../rtc_base:logging", + "../../../rtc_base:task_queue_for_test", + "../../../rtc_base/synchronization:mutex", + "../../../rtc_base/task_utils:repeating_task", + "../../../system_wrappers", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + + rtc_library("peerconnection_quality_test") { + testonly = true + + sources = [ + "peer_connection_quality_test.cc", + "peer_connection_quality_test.h", + ] + deps = [ + ":analyzer_helper", + ":cross_media_metrics_reporter", + ":default_audio_quality_analyzer", + ":media_helper", + ":metric_metadata_keys", + ":peer_params_preprocessor", + ":sdp_changer", + ":stats_poller", + ":test_activities_executor", + ":test_peer", + ":test_peer_factory", + "../..:field_trial", + "../..:fileutils", + "../..:perf_test", + "../../../api:audio_quality_analyzer_api", + "../../../api:libjingle_peerconnection_api", + "../../../api:media_stream_interface", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:rtc_event_log_output_file", + "../../../api:scoped_refptr", + "../../../api:time_controller", + "../../../api:video_quality_analyzer_api", + "../../../api/rtc_event_log", + "../../../api/task_queue", + "../../../api/test/metrics:metric", + "../../../api/test/metrics:metrics_logger", + "../../../api/test/pclf:media_configuration", + "../../../api/test/pclf:media_quality_test_params", + "../../../api/test/pclf:peer_configurer", + "../../../api/units:time_delta", + "../../../api/units:timestamp", + "../../../pc:pc_test_utils", + "../../../pc:sdp_utils", + "../../../rtc_base:gunit_helpers", + "../../../rtc_base:macromagic", + "../../../rtc_base:safe_conversions", + "../../../rtc_base:stringutils", + "../../../rtc_base:task_queue_for_test", + "../../../rtc_base:threading", + "../../../rtc_base/synchronization:mutex", + "../../../system_wrappers", + "../../../system_wrappers:field_trial", + "analyzer/video:default_video_quality_analyzer", + "analyzer/video:single_process_encoded_image_data_injector", + "analyzer/video:video_frame_tracking_id_injector", + "analyzer/video:video_quality_analyzer_injection_helper", + "analyzer/video:video_quality_metrics_reporter", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + + peer_connection_e2e_smoke_test_resources = [ + "../../../resources/pc_quality_smoke_test_alice_source.wav", + "../../../resources/pc_quality_smoke_test_bob_source.wav", + ] + if (is_ios) { + bundle_data("peer_connection_e2e_smoke_test_resources_bundle_data") { + testonly = true + sources = peer_connection_e2e_smoke_test_resources + outputs = [ "{{bundle_resources_dir}}/{{source_file_part}}" ] + } + } + + rtc_library("peer_connection_e2e_smoke_test") { + testonly = true + + sources = [ "peer_connection_e2e_smoke_test.cc" ] + deps = [ + ":default_audio_quality_analyzer", + ":network_quality_metrics_reporter", + ":stats_based_network_quality_metrics_reporter", + "../../../api:callfactory_api", + "../../../api:create_network_emulation_manager", + "../../../api:create_peer_connection_quality_test_frame_generator", + "../../../api:create_peerconnection_quality_test_fixture", + "../../../api:libjingle_peerconnection_api", + "../../../api:media_stream_interface", + "../../../api:network_emulation_manager_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:scoped_refptr", + "../../../api:simulated_network_api", + "../../../api/audio_codecs:builtin_audio_decoder_factory", + "../../../api/audio_codecs:builtin_audio_encoder_factory", + "../../../api/test/metrics:global_metrics_logger_and_exporter", + "../../../api/test/pclf:media_configuration", + "../../../api/test/pclf:media_quality_test_params", + "../../../api/test/pclf:peer_configurer", + "../../../api/video_codecs:builtin_video_decoder_factory", + "../../../api/video_codecs:builtin_video_encoder_factory", + "../../../call:simulated_network", + "../../../media:rtc_audio_video", + "../../../modules/audio_device:audio_device_impl", + "../../../p2p:rtc_p2p", + "../../../pc:pc_test_utils", + "../../../pc:peerconnection_wrapper", + "../../../rtc_base:gunit_helpers", + "../../../rtc_base:logging", + "../../../rtc_base:rtc_event", + "../../../system_wrappers:field_trial", + "../../../test:field_trial", + "../../../test:fileutils", + "../../../test:test_support", + "analyzer/video:default_video_quality_analyzer", + "analyzer/video:default_video_quality_analyzer_shared", + ] + data = peer_connection_e2e_smoke_test_resources + if (is_mac || is_ios) { + deps += [ "../../../modules/video_coding:objc_codec_factory_helper" ] + } + if (is_ios) { + deps += [ ":peer_connection_e2e_smoke_test_resources_bundle_data" ] + } + } + + rtc_library("peer_connection_quality_test_metric_names_test") { + testonly = true + sources = [ "peer_connection_quality_test_metric_names_test.cc" ] + deps = [ + ":metric_metadata_keys", + ":peerconnection_quality_test", + ":stats_based_network_quality_metrics_reporter", + "../..:test_support", + "../../../api:create_network_emulation_manager", + "../../../api:create_peer_connection_quality_test_frame_generator", + "../../../api:network_emulation_manager_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api/test/metrics:metrics_logger", + "../../../api/test/metrics:stdout_metrics_exporter", + "../../../api/test/pclf:media_configuration", + "../../../api/test/pclf:media_quality_test_params", + "../../../api/test/pclf:peer_configurer", + "../../../api/units:time_delta", + ] + } + + rtc_library("stats_based_network_quality_metrics_reporter_test") { + testonly = true + sources = [ "stats_based_network_quality_metrics_reporter_test.cc" ] + deps = [ + ":metric_metadata_keys", + ":peerconnection_quality_test", + ":stats_based_network_quality_metrics_reporter", + "../..:test_support", + "../../../api:array_view", + "../../../api:create_network_emulation_manager", + "../../../api:create_peer_connection_quality_test_frame_generator", + "../../../api:network_emulation_manager_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api/test/metrics:metrics_logger", + "../../../api/test/metrics:stdout_metrics_exporter", + "../../../api/test/pclf:media_configuration", + "../../../api/test/pclf:media_quality_test_params", + "../../../api/test/pclf:peer_configurer", + "../../../api/units:time_delta", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + + rtc_library("peer_connection_quality_test_test") { + testonly = true + sources = [ "peer_connection_quality_test_test.cc" ] + deps = [ + ":peerconnection_quality_test", + "../..:fileutils", + "../..:test_support", + "../..:video_test_support", + "../../../api:create_network_emulation_manager", + "../../../api:network_emulation_manager_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api/test/metrics:global_metrics_logger_and_exporter", + "../../../api/test/pclf:media_configuration", + "../../../api/test/pclf:media_quality_test_params", + "../../../api/test/pclf:peer_configurer", + "../../../api/units:time_delta", + "../../../rtc_base:timeutils", + ] + } + + rtc_library("stats_provider") { + testonly = true + sources = [ "stats_provider.h" ] + deps = [ "../../../api:rtc_stats_api" ] + } + + rtc_library("stats_poller") { + testonly = true + sources = [ + "stats_poller.cc", + "stats_poller.h", + ] + deps = [ + ":stats_provider", + ":test_peer", + "../../../api:libjingle_peerconnection_api", + "../../../api:rtc_stats_api", + "../../../api:stats_observer_interface", + "../../../rtc_base:logging", + "../../../rtc_base:macromagic", + "../../../rtc_base/synchronization:mutex", + ] + } + + rtc_library("stats_poller_test") { + testonly = true + sources = [ "stats_poller_test.cc" ] + deps = [ + ":stats_poller", + "../..:test_support", + "../../../api:rtc_stats_api", + ] + } + } + + rtc_library("analyzer_helper") { + sources = [ + "analyzer_helper.cc", + "analyzer_helper.h", + ] + deps = [ + "../../../api:sequence_checker", + "../../../api:track_id_stream_info_map", + "../../../rtc_base:macromagic", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + + rtc_library("default_audio_quality_analyzer") { + testonly = true + sources = [ + "analyzer/audio/default_audio_quality_analyzer.cc", + "analyzer/audio/default_audio_quality_analyzer.h", + ] + + deps = [ + ":metric_metadata_keys", + "../..:perf_test", + "../../../api:audio_quality_analyzer_api", + "../../../api:rtc_stats_api", + "../../../api:stats_observer_interface", + "../../../api:track_id_stream_info_map", + "../../../api/numerics", + "../../../api/test/metrics:metric", + "../../../api/test/metrics:metrics_logger", + "../../../api/units:time_delta", + "../../../api/units:timestamp", + "../../../rtc_base:checks", + "../../../rtc_base:criticalsection", + "../../../rtc_base:logging", + "../../../rtc_base:rtc_numerics", + "../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + + rtc_library("network_quality_metrics_reporter") { + testonly = true + sources = [ + "network_quality_metrics_reporter.cc", + "network_quality_metrics_reporter.h", + ] + deps = [ + "../..:perf_test", + "../../../api:network_emulation_manager_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:rtc_stats_api", + "../../../api:track_id_stream_info_map", + "../../../api/test/metrics:metric", + "../../../api/test/metrics:metrics_logger", + "../../../api/units:data_size", + "../../../rtc_base:checks", + "../../../rtc_base:criticalsection", + "../../../rtc_base:rtc_event", + "../../../rtc_base/synchronization:mutex", + "../../../system_wrappers:field_trial", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + + rtc_library("stats_based_network_quality_metrics_reporter") { + testonly = true + sources = [ + "stats_based_network_quality_metrics_reporter.cc", + "stats_based_network_quality_metrics_reporter.h", + ] + deps = [ + ":metric_metadata_keys", + "../..:perf_test", + "../../../api:array_view", + "../../../api:network_emulation_manager_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:rtc_stats_api", + "../../../api:scoped_refptr", + "../../../api:sequence_checker", + "../../../api/numerics", + "../../../api/test/metrics:metric", + "../../../api/test/metrics:metrics_logger", + "../../../api/test/network_emulation", + "../../../api/units:data_rate", + "../../../api/units:data_size", + "../../../api/units:timestamp", + "../../../rtc_base:checks", + "../../../rtc_base:ip_address", + "../../../rtc_base:rtc_event", + "../../../rtc_base:stringutils", + "../../../rtc_base/synchronization:mutex", + "../../../rtc_base/system:no_unique_address", + "../../../system_wrappers:field_trial", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + + rtc_library("cross_media_metrics_reporter") { + testonly = true + sources = [ + "cross_media_metrics_reporter.cc", + "cross_media_metrics_reporter.h", + ] + deps = [ + ":metric_metadata_keys", + "../..:perf_test", + "../../../api:network_emulation_manager_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:rtc_stats_api", + "../../../api:track_id_stream_info_map", + "../../../api/numerics", + "../../../api/test/metrics:metric", + "../../../api/test/metrics:metrics_logger", + "../../../api/units:timestamp", + "../../../rtc_base:checks", + "../../../rtc_base:criticalsection", + "../../../rtc_base:rtc_event", + "../../../rtc_base:rtc_numerics", + "../../../rtc_base/synchronization:mutex", + "../../../system_wrappers:field_trial", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + + rtc_library("sdp_changer") { + testonly = true + sources = [ + "sdp/sdp_changer.cc", + "sdp/sdp_changer.h", + ] + deps = [ + "../../../api:array_view", + "../../../api:libjingle_peerconnection_api", + "../../../api:rtp_parameters", + "../../../api/test/pclf:media_configuration", + "../../../media:media_constants", + "../../../media:rid_description", + "../../../media:rtc_media_base", + "../../../p2p:rtc_p2p", + "../../../pc:sdp_utils", + "../../../pc:session_description", + "../../../pc:simulcast_description", + "../../../rtc_base:stringutils", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings:strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + } +} diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc new file mode 100644 index 0000000000..98d0c533c2 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h" + +#include "api/stats/rtc_stats.h" +#include "api/stats/rtcstats_objects.h" +#include "api/test/metrics/metric.h" +#include "api/test/track_id_stream_info_map.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "test/pc/e2e/metric_metadata_keys.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Unit; + +DefaultAudioQualityAnalyzer::DefaultAudioQualityAnalyzer( + test::MetricsLogger* const metrics_logger) + : metrics_logger_(metrics_logger) { + RTC_CHECK(metrics_logger_); +} + +void DefaultAudioQualityAnalyzer::Start(std::string test_case_name, + TrackIdStreamInfoMap* analyzer_helper) { + test_case_name_ = std::move(test_case_name); + analyzer_helper_ = analyzer_helper; +} + +void DefaultAudioQualityAnalyzer::OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) { + auto stats = report->GetStatsOfType<RTCInboundRTPStreamStats>(); + + for (auto& stat : stats) { + if (!stat->kind.is_defined() || + !(*stat->kind == RTCMediaStreamTrackKind::kAudio)) { + continue; + } + + StatsSample sample; + sample.total_samples_received = + stat->total_samples_received.ValueOrDefault(0ul); + sample.concealed_samples = stat->concealed_samples.ValueOrDefault(0ul); + sample.removed_samples_for_acceleration = + stat->removed_samples_for_acceleration.ValueOrDefault(0ul); + sample.inserted_samples_for_deceleration = + stat->inserted_samples_for_deceleration.ValueOrDefault(0ul); + sample.silent_concealed_samples = + stat->silent_concealed_samples.ValueOrDefault(0ul); + sample.jitter_buffer_delay = + TimeDelta::Seconds(stat->jitter_buffer_delay.ValueOrDefault(0.)); + sample.jitter_buffer_target_delay = + TimeDelta::Seconds(stat->jitter_buffer_target_delay.ValueOrDefault(0.)); + sample.jitter_buffer_emitted_count = + stat->jitter_buffer_emitted_count.ValueOrDefault(0ul); + + TrackIdStreamInfoMap::StreamInfo stream_info = + analyzer_helper_->GetStreamInfoFromTrackId(*stat->track_identifier); + + MutexLock lock(&lock_); + stream_info_.emplace(stream_info.stream_label, stream_info); + StatsSample prev_sample = last_stats_sample_[stream_info.stream_label]; + RTC_CHECK_GE(sample.total_samples_received, + prev_sample.total_samples_received); + double total_samples_diff = static_cast<double>( + sample.total_samples_received - prev_sample.total_samples_received); + if (total_samples_diff == 0) { + return; + } + + AudioStreamStats& audio_stream_stats = + streams_stats_[stream_info.stream_label]; + audio_stream_stats.expand_rate.AddSample( + (sample.concealed_samples - prev_sample.concealed_samples) / + total_samples_diff); + audio_stream_stats.accelerate_rate.AddSample( + (sample.removed_samples_for_acceleration - + prev_sample.removed_samples_for_acceleration) / + total_samples_diff); + audio_stream_stats.preemptive_rate.AddSample( + (sample.inserted_samples_for_deceleration - + prev_sample.inserted_samples_for_deceleration) / + total_samples_diff); + + int64_t speech_concealed_samples = + sample.concealed_samples - sample.silent_concealed_samples; + int64_t prev_speech_concealed_samples = + prev_sample.concealed_samples - prev_sample.silent_concealed_samples; + audio_stream_stats.speech_expand_rate.AddSample( + (speech_concealed_samples - prev_speech_concealed_samples) / + total_samples_diff); + + int64_t jitter_buffer_emitted_count_diff = + sample.jitter_buffer_emitted_count - + prev_sample.jitter_buffer_emitted_count; + if (jitter_buffer_emitted_count_diff > 0) { + TimeDelta jitter_buffer_delay_diff = + sample.jitter_buffer_delay - prev_sample.jitter_buffer_delay; + TimeDelta jitter_buffer_target_delay_diff = + sample.jitter_buffer_target_delay - + prev_sample.jitter_buffer_target_delay; + audio_stream_stats.average_jitter_buffer_delay_ms.AddSample( + jitter_buffer_delay_diff.ms<double>() / + jitter_buffer_emitted_count_diff); + audio_stream_stats.preferred_buffer_size_ms.AddSample( + jitter_buffer_target_delay_diff.ms<double>() / + jitter_buffer_emitted_count_diff); + } + + last_stats_sample_[stream_info.stream_label] = sample; + } +} + +std::string DefaultAudioQualityAnalyzer::GetTestCaseName( + const std::string& stream_label) const { + return test_case_name_ + "/" + stream_label; +} + +void DefaultAudioQualityAnalyzer::Stop() { + MutexLock lock(&lock_); + for (auto& item : streams_stats_) { + const TrackIdStreamInfoMap::StreamInfo& stream_info = + stream_info_[item.first]; + // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey. + std::map<std::string, std::string> metric_metadata{ + {MetricMetadataKey::kAudioStreamMetadataKey, item.first}, + {MetricMetadataKey::kPeerMetadataKey, stream_info.receiver_peer}, + {MetricMetadataKey::kReceiverMetadataKey, stream_info.receiver_peer}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_case_name_}}; + + metrics_logger_->LogMetric("expand_rate", GetTestCaseName(item.first), + item.second.expand_rate, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter, + metric_metadata); + metrics_logger_->LogMetric("accelerate_rate", GetTestCaseName(item.first), + item.second.accelerate_rate, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter, + metric_metadata); + metrics_logger_->LogMetric("preemptive_rate", GetTestCaseName(item.first), + item.second.preemptive_rate, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter, + metric_metadata); + metrics_logger_->LogMetric( + "speech_expand_rate", GetTestCaseName(item.first), + item.second.speech_expand_rate, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "average_jitter_buffer_delay_ms", GetTestCaseName(item.first), + item.second.average_jitter_buffer_delay_ms, Unit::kMilliseconds, + ImprovementDirection::kNeitherIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "preferred_buffer_size_ms", GetTestCaseName(item.first), + item.second.preferred_buffer_size_ms, Unit::kMilliseconds, + ImprovementDirection::kNeitherIsBetter, metric_metadata); + } +} + +std::map<std::string, AudioStreamStats> +DefaultAudioQualityAnalyzer::GetAudioStreamsStats() const { + MutexLock lock(&lock_); + return streams_stats_; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h b/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h new file mode 100644 index 0000000000..9e427afed8 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_AUDIO_DEFAULT_AUDIO_QUALITY_ANALYZER_H_ +#define TEST_PC_E2E_ANALYZER_AUDIO_DEFAULT_AUDIO_QUALITY_ANALYZER_H_ + +#include <map> +#include <string> + +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/audio_quality_analyzer_interface.h" +#include "api/test/metrics/metrics_logger.h" +#include "api/test/track_id_stream_info_map.h" +#include "api/units/time_delta.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +struct AudioStreamStats { + SamplesStatsCounter expand_rate; + SamplesStatsCounter accelerate_rate; + SamplesStatsCounter preemptive_rate; + SamplesStatsCounter speech_expand_rate; + SamplesStatsCounter average_jitter_buffer_delay_ms; + SamplesStatsCounter preferred_buffer_size_ms; +}; + +class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface { + public: + explicit DefaultAudioQualityAnalyzer( + test::MetricsLogger* const metrics_logger); + + void Start(std::string test_case_name, + TrackIdStreamInfoMap* analyzer_helper) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override; + void Stop() override; + + // Returns audio quality stats per stream label. + std::map<std::string, AudioStreamStats> GetAudioStreamsStats() const; + + private: + struct StatsSample { + uint64_t total_samples_received = 0; + uint64_t concealed_samples = 0; + uint64_t removed_samples_for_acceleration = 0; + uint64_t inserted_samples_for_deceleration = 0; + uint64_t silent_concealed_samples = 0; + TimeDelta jitter_buffer_delay = TimeDelta::Zero(); + TimeDelta jitter_buffer_target_delay = TimeDelta::Zero(); + uint64_t jitter_buffer_emitted_count = 0; + }; + + std::string GetTestCaseName(const std::string& stream_label) const; + + test::MetricsLogger* const metrics_logger_; + + std::string test_case_name_; + TrackIdStreamInfoMap* analyzer_helper_; + + mutable Mutex lock_; + std::map<std::string, AudioStreamStats> streams_stats_ RTC_GUARDED_BY(lock_); + std::map<std::string, TrackIdStreamInfoMap::StreamInfo> stream_info_ + RTC_GUARDED_BY(lock_); + std::map<std::string, StatsSample> last_stats_sample_ RTC_GUARDED_BY(lock_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_AUDIO_DEFAULT_AUDIO_QUALITY_ANALYZER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/BUILD.gn b/third_party/libwebrtc/test/pc/e2e/analyzer/video/BUILD.gn new file mode 100644 index 0000000000..cbb4c078f3 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/BUILD.gn @@ -0,0 +1,573 @@ +# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../../../webrtc.gni") + +if (!build_with_chromium) { + group("video_analyzer") { + testonly = true + + deps = [ + ":analyzing_video_sinks_helper", + ":default_video_quality_analyzer_internal", + ":encoded_image_data_injector_api", + ":example_video_quality_analyzer", + ":multi_reader_queue", + ":quality_analyzing_video_decoder", + ":quality_analyzing_video_encoder", + ":simulcast_dummy_buffer_helper", + ":single_process_encoded_image_data_injector", + ":video_dumping", + ":video_frame_tracking_id_injector", + ":video_quality_metrics_reporter", + ] + if (rtc_include_tests) { + deps += [ + ":analyzing_video_sink", + ":video_quality_analyzer_injection_helper", + ] + } + } + + if (rtc_include_tests) { + group("video_analyzer_unittests") { + testonly = true + + deps = [ + ":analyzing_video_sink_test", + ":analyzing_video_sinks_helper_test", + ":default_video_quality_analyzer_frames_comparator_test", + ":default_video_quality_analyzer_metric_names_test", + ":default_video_quality_analyzer_stream_state_test", + ":default_video_quality_analyzer_test", + ":multi_reader_queue_test", + ":names_collection_test", + ":simulcast_dummy_buffer_helper_test", + ":single_process_encoded_image_data_injector_unittest", + ":video_dumping_test", + ":video_frame_tracking_id_injector_unittest", + ] + } + } +} + +rtc_library("video_dumping") { + testonly = true + sources = [ + "video_dumping.cc", + "video_dumping.h", + ] + deps = [ + "../../../..:video_test_support", + "../../../../../api/test/video:video_frame_writer", + "../../../../../api/video:video_frame", + "../../../../../rtc_base:logging", + "../../../../../system_wrappers", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +rtc_library("encoded_image_data_injector_api") { + testonly = true + sources = [ "encoded_image_data_injector.h" ] + + deps = [ "../../../../../api/video:encoded_image" ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("single_process_encoded_image_data_injector") { + testonly = true + sources = [ + "single_process_encoded_image_data_injector.cc", + "single_process_encoded_image_data_injector.h", + ] + + deps = [ + ":encoded_image_data_injector_api", + "../../../../../api/video:encoded_image", + "../../../../../rtc_base:checks", + "../../../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/memory" ] +} + +rtc_library("video_frame_tracking_id_injector") { + testonly = true + sources = [ + "video_frame_tracking_id_injector.cc", + "video_frame_tracking_id_injector.h", + ] + + deps = [ + ":encoded_image_data_injector_api", + "../../../../../api/video:encoded_image", + "../../../../../rtc_base:checks", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/memory" ] +} + +rtc_library("simulcast_dummy_buffer_helper") { + testonly = true + sources = [ + "simulcast_dummy_buffer_helper.cc", + "simulcast_dummy_buffer_helper.h", + ] + deps = [ "../../../../../api/video:video_frame" ] +} + +rtc_library("quality_analyzing_video_decoder") { + testonly = true + sources = [ + "quality_analyzing_video_decoder.cc", + "quality_analyzing_video_decoder.h", + ] + deps = [ + ":encoded_image_data_injector_api", + ":simulcast_dummy_buffer_helper", + "../../../../../api:video_quality_analyzer_api", + "../../../../../api/video:encoded_image", + "../../../../../api/video:video_frame", + "../../../../../api/video_codecs:video_codecs_api", + "../../../../../modules/video_coding:video_codec_interface", + "../../../../../rtc_base:logging", + "../../../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("quality_analyzing_video_encoder") { + testonly = true + sources = [ + "quality_analyzing_video_encoder.cc", + "quality_analyzing_video_encoder.h", + ] + deps = [ + ":encoded_image_data_injector_api", + "../../../../../api:video_quality_analyzer_api", + "../../../../../api/test/pclf:media_configuration", + "../../../../../api/video:video_frame", + "../../../../../api/video_codecs:video_codecs_api", + "../../../../../modules/video_coding:video_codec_interface", + "../../../../../modules/video_coding/svc:scalability_mode_util", + "../../../../../rtc_base:logging", + "../../../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +rtc_library("analyzing_video_sinks_helper") { + testonly = true + sources = [ + "analyzing_video_sinks_helper.cc", + "analyzing_video_sinks_helper.h", + ] + deps = [ + "../../../../../api/test/pclf:media_configuration", + "../../../../../api/test/video:video_frame_writer", + "../../../../../rtc_base:macromagic", + "../../../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("example_video_quality_analyzer") { + testonly = true + sources = [ + "example_video_quality_analyzer.cc", + "example_video_quality_analyzer.h", + ] + + deps = [ + "../../../../../api:array_view", + "../../../../../api:video_quality_analyzer_api", + "../../../../../api/video:encoded_image", + "../../../../../api/video:video_frame", + "../../../../../rtc_base:logging", + "../../../../../rtc_base/synchronization:mutex", + ] +} + +# This target contains implementation details of DefaultVideoQualityAnalyzer, +# so headers exported by it shouldn't be used in other places. +rtc_library("default_video_quality_analyzer_internal") { + visibility = [ + ":default_video_quality_analyzer", + ":default_video_quality_analyzer_frames_comparator_test", + ":default_video_quality_analyzer_stream_state_test", + ":names_collection_test", + ":video_analyzer", + ] + + testonly = true + sources = [ + "default_video_quality_analyzer_cpu_measurer.cc", + "default_video_quality_analyzer_cpu_measurer.h", + "default_video_quality_analyzer_frame_in_flight.cc", + "default_video_quality_analyzer_frame_in_flight.h", + "default_video_quality_analyzer_frames_comparator.cc", + "default_video_quality_analyzer_frames_comparator.h", + "default_video_quality_analyzer_internal_shared_objects.cc", + "default_video_quality_analyzer_internal_shared_objects.h", + "default_video_quality_analyzer_stream_state.cc", + "default_video_quality_analyzer_stream_state.h", + "names_collection.cc", + "names_collection.h", + ] + + deps = [ + ":default_video_quality_analyzer_shared", + ":multi_reader_queue", + "../..:metric_metadata_keys", + "../../../../../api:array_view", + "../../../../../api:scoped_refptr", + "../../../../../api/numerics", + "../../../../../api/units:data_size", + "../../../../../api/units:timestamp", + "../../../../../api/video:video_frame", + "../../../../../api/video:video_frame_type", + "../../../../../common_video", + "../../../../../rtc_base:checks", + "../../../../../rtc_base:platform_thread", + "../../../../../rtc_base:rtc_base_tests_utils", + "../../../../../rtc_base:rtc_event", + "../../../../../rtc_base:stringutils", + "../../../../../rtc_base:timeutils", + "../../../../../rtc_base/synchronization:mutex", + "../../../../../rtc_tools:video_quality_analysis", + "../../../../../system_wrappers", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings:strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("multi_reader_queue") { + testonly = true + sources = [ "multi_reader_queue.h" ] + deps = [ "../../../../../rtc_base:checks" ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("video_quality_metrics_reporter") { + testonly = true + sources = [ + "video_quality_metrics_reporter.cc", + "video_quality_metrics_reporter.h", + ] + deps = [ + "../..:metric_metadata_keys", + "../../../../../api:peer_connection_quality_test_fixture_api", + "../../../../../api:rtc_stats_api", + "../../../../../api:track_id_stream_info_map", + "../../../../../api/numerics", + "../../../../../api/test/metrics:metric", + "../../../../../api/test/metrics:metrics_logger", + "../../../../../api/units:data_rate", + "../../../../../api/units:data_size", + "../../../../../api/units:time_delta", + "../../../../../api/units:timestamp", + "../../../../../rtc_base:checks", + "../../../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +rtc_library("default_video_quality_analyzer") { + testonly = true + sources = [ + "default_video_quality_analyzer.cc", + "default_video_quality_analyzer.h", + ] + + deps = [ + ":default_video_quality_analyzer_internal", + ":default_video_quality_analyzer_shared", + "../..:metric_metadata_keys", + "../../../../../api:array_view", + "../../../../../api:video_quality_analyzer_api", + "../../../../../api/numerics", + "../../../../../api/test/metrics:metric", + "../../../../../api/test/metrics:metrics_logger", + "../../../../../api/units:data_size", + "../../../../../api/units:time_delta", + "../../../../../api/units:timestamp", + "../../../../../api/video:encoded_image", + "../../../../../api/video:video_frame", + "../../../../../rtc_base:checks", + "../../../../../rtc_base:logging", + "../../../../../rtc_base:macromagic", + "../../../../../rtc_base:stringutils", + "../../../../../rtc_base/synchronization:mutex", + "../../../../../system_wrappers", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("default_video_quality_analyzer_shared") { + testonly = true + sources = [ + "default_video_quality_analyzer_shared_objects.cc", + "default_video_quality_analyzer_shared_objects.h", + ] + + deps = [ + "../../../../../api/numerics", + "../../../../../api/units:timestamp", + "../../../../../rtc_base:checks", + "../../../../../rtc_base:stringutils", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("analyzing_video_sink") { + testonly = true + sources = [ + "analyzing_video_sink.cc", + "analyzing_video_sink.h", + ] + deps = [ + ":analyzing_video_sinks_helper", + ":simulcast_dummy_buffer_helper", + ":video_dumping", + "../../../..:fixed_fps_video_frame_writer_adapter", + "../../../..:test_renderer", + "../../../../../api:video_quality_analyzer_api", + "../../../../../api/numerics", + "../../../../../api/test/pclf:media_configuration", + "../../../../../api/test/video:video_frame_writer", + "../../../../../api/units:timestamp", + "../../../../../api/video:video_frame", + "../../../../../rtc_base:checks", + "../../../../../rtc_base:logging", + "../../../../../rtc_base:macromagic", + "../../../../../rtc_base/synchronization:mutex", + "../../../../../system_wrappers", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory:memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("video_quality_analyzer_injection_helper") { + testonly = true + sources = [ + "video_quality_analyzer_injection_helper.cc", + "video_quality_analyzer_injection_helper.h", + ] + deps = [ + ":analyzing_video_sink", + ":analyzing_video_sinks_helper", + ":encoded_image_data_injector_api", + ":quality_analyzing_video_decoder", + ":quality_analyzing_video_encoder", + ":simulcast_dummy_buffer_helper", + ":video_dumping", + "../../../..:fixed_fps_video_frame_writer_adapter", + "../../../..:test_renderer", + "../../../..:video_test_common", + "../../../..:video_test_support", + "../../../../../api:array_view", + "../../../../../api:stats_observer_interface", + "../../../../../api:video_quality_analyzer_api", + "../../../../../api/test/pclf:media_configuration", + "../../../../../api/video:video_frame", + "../../../../../api/video_codecs:video_codecs_api", + "../../../../../rtc_base:checks", + "../../../../../rtc_base:logging", + "../../../../../rtc_base:stringutils", + "../../../../../rtc_base/synchronization:mutex", + "../../../../../system_wrappers", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + ] +} + +if (rtc_include_tests) { + rtc_library("simulcast_dummy_buffer_helper_test") { + testonly = true + sources = [ "simulcast_dummy_buffer_helper_test.cc" ] + deps = [ + ":simulcast_dummy_buffer_helper", + "../../../..:test_support", + "../../../../../api/video:video_frame", + "../../../../../rtc_base:random", + ] + } + + rtc_library("analyzing_video_sink_test") { + testonly = true + sources = [ "analyzing_video_sink_test.cc" ] + deps = [ + ":analyzing_video_sink", + ":example_video_quality_analyzer", + "../../../..:fileutils", + "../../../..:test_support", + "../../../..:video_test_support", + "../../../../../api:create_frame_generator", + "../../../../../api:frame_generator_api", + "../../../../../api:scoped_refptr", + "../../../../../api/test/pclf:media_configuration", + "../../../../../api/units:time_delta", + "../../../../../api/units:timestamp", + "../../../../../api/video:video_frame", + "../../../../../common_video", + "../../../../../rtc_base:timeutils", + "../../../../../system_wrappers", + "../../../../time_controller", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + + rtc_library("analyzing_video_sinks_helper_test") { + testonly = true + sources = [ "analyzing_video_sinks_helper_test.cc" ] + deps = [ + ":analyzing_video_sinks_helper", + "../../../..:test_support", + "../../../../../api/test/pclf:media_configuration", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_library("default_video_quality_analyzer_frames_comparator_test") { + testonly = true + sources = [ "default_video_quality_analyzer_frames_comparator_test.cc" ] + deps = [ + ":default_video_quality_analyzer_internal", + ":default_video_quality_analyzer_shared", + "../../../..:test_support", + "../../../../../api:create_frame_generator", + "../../../../../api/units:timestamp", + "../../../../../rtc_base:stringutils", + "../../../../../system_wrappers", + ] + } + + rtc_library("names_collection_test") { + testonly = true + sources = [ "names_collection_test.cc" ] + deps = [ + ":default_video_quality_analyzer_internal", + "../../../..:test_support", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings:strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + + rtc_library("multi_reader_queue_test") { + testonly = true + sources = [ "multi_reader_queue_test.cc" ] + deps = [ + ":multi_reader_queue", + "../../../..:test_support", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_library("default_video_quality_analyzer_stream_state_test") { + testonly = true + sources = [ "default_video_quality_analyzer_stream_state_test.cc" ] + deps = [ + ":default_video_quality_analyzer_internal", + "../../../..:test_support", + "../../../../../api/units:timestamp", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_library("default_video_quality_analyzer_test") { + testonly = true + sources = [ "default_video_quality_analyzer_test.cc" ] + deps = [ + ":default_video_quality_analyzer", + ":default_video_quality_analyzer_shared", + "../../../..:test_support", + "../../../../../api:create_frame_generator", + "../../../../../api:rtp_packet_info", + "../../../../../api/test/metrics:global_metrics_logger_and_exporter", + "../../../../../api/video:encoded_image", + "../../../../../api/video:video_frame", + "../../../../../common_video", + "../../../../../rtc_base:stringutils", + "../../../../../rtc_tools:video_quality_analysis", + "../../../../../system_wrappers", + ] + } + + rtc_library("default_video_quality_analyzer_metric_names_test") { + testonly = true + sources = [ "default_video_quality_analyzer_metric_names_test.cc" ] + deps = [ + ":default_video_quality_analyzer", + "../../../..:test_support", + "../../../../../api:create_frame_generator", + "../../../../../api:rtp_packet_info", + "../../../../../api/test/metrics:metric", + "../../../../../api/test/metrics:metrics_logger", + "../../../../../api/test/metrics:stdout_metrics_exporter", + "../../../../../api/video:encoded_image", + "../../../../../api/video:video_frame", + "../../../../../common_video", + "../../../../../rtc_tools:video_quality_analysis", + "../../../../../system_wrappers", + ] + } + + rtc_library("video_dumping_test") { + testonly = true + sources = [ "video_dumping_test.cc" ] + deps = [ + ":video_dumping", + "../../../..:fileutils", + "../../../..:test_support", + "../../../..:video_test_support", + "../../../../../api:scoped_refptr", + "../../../../../api/video:video_frame", + "../../../../../rtc_base:random", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_library("single_process_encoded_image_data_injector_unittest") { + testonly = true + sources = [ "single_process_encoded_image_data_injector_unittest.cc" ] + deps = [ + ":single_process_encoded_image_data_injector", + "../../../..:test_support", + "../../../../../api/video:encoded_image", + "../../../../../rtc_base:buffer", + ] + } + + rtc_library("video_frame_tracking_id_injector_unittest") { + testonly = true + sources = [ "video_frame_tracking_id_injector_unittest.cc" ] + deps = [ + ":video_frame_tracking_id_injector", + "../../../..:test_support", + "../../../../../api/video:encoded_image", + "../../../../../rtc_base:buffer", + ] + } +} diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.cc new file mode 100644 index 0000000000..fb221e6797 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.cc @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/analyzing_video_sink.h" + +#include <memory> +#include <set> +#include <utility> + +#include "absl/memory/memory.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/video/video_frame_writer.h" +#include "api/units/timestamp.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/synchronization/mutex.h" +#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h" +#include "test/pc/e2e/analyzer/video/video_dumping.h" +#include "test/testsupport/fixed_fps_video_frame_writer_adapter.h" +#include "test/video_renderer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +AnalyzingVideoSink::AnalyzingVideoSink(absl::string_view peer_name, + Clock* clock, + VideoQualityAnalyzerInterface& analyzer, + AnalyzingVideoSinksHelper& sinks_helper, + const VideoSubscription& subscription, + bool report_infra_stats) + : peer_name_(peer_name), + report_infra_stats_(report_infra_stats), + clock_(clock), + analyzer_(&analyzer), + sinks_helper_(&sinks_helper), + subscription_(subscription) {} + +void AnalyzingVideoSink::UpdateSubscription( + const VideoSubscription& subscription) { + // For peers with changed resolutions we need to close current writers and + // open new ones. This is done by removing existing sinks, which will force + // creation of the new sinks when next frame will be received. + std::set<test::VideoFrameWriter*> writers_to_close; + { + MutexLock lock(&mutex_); + subscription_ = subscription; + for (auto it = stream_sinks_.cbegin(); it != stream_sinks_.cend();) { + absl::optional<VideoResolution> new_requested_resolution = + subscription_.GetResolutionForPeer(it->second.sender_peer_name); + if (!new_requested_resolution.has_value() || + (*new_requested_resolution != it->second.resolution)) { + RTC_LOG(LS_INFO) << peer_name_ << ": Subscribed resolution for stream " + << it->first << " from " << it->second.sender_peer_name + << " was updated from " + << it->second.resolution.ToString() << " to " + << new_requested_resolution->ToString() + << ". Repopulating all video sinks and recreating " + << "requested video writers"; + writers_to_close.insert(it->second.video_frame_writer); + it = stream_sinks_.erase(it); + } else { + ++it; + } + } + } + sinks_helper_->CloseAndRemoveVideoWriters(writers_to_close); +} + +void AnalyzingVideoSink::OnFrame(const VideoFrame& frame) { + if (IsDummyFrame(frame)) { + // This is dummy frame, so we don't need to process it further. + return; + } + + if (frame.id() == VideoFrame::kNotSetId) { + // If frame ID is unknown we can't get required render resolution, so pass + // to the analyzer in the actual resolution of the frame. + AnalyzeFrame(frame); + } else { + std::string stream_label = analyzer_->GetStreamLabel(frame.id()); + MutexLock lock(&mutex_); + Timestamp processing_started = clock_->CurrentTime(); + SinksDescriptor* sinks_descriptor = PopulateSinks(stream_label); + RTC_CHECK(sinks_descriptor != nullptr); + + VideoFrame scaled_frame = + ScaleVideoFrame(frame, sinks_descriptor->resolution); + AnalyzeFrame(scaled_frame); + for (auto& sink : sinks_descriptor->sinks) { + sink->OnFrame(scaled_frame); + } + Timestamp processing_finished = clock_->CurrentTime(); + + if (report_infra_stats_) { + stats_.analyzing_sink_processing_time_ms.AddSample( + (processing_finished - processing_started).ms<double>()); + } + } +} + +AnalyzingVideoSink::Stats AnalyzingVideoSink::stats() const { + MutexLock lock(&mutex_); + return stats_; +} + +VideoFrame AnalyzingVideoSink::ScaleVideoFrame( + const VideoFrame& frame, + const VideoResolution& required_resolution) { + Timestamp processing_started = clock_->CurrentTime(); + if (required_resolution.width() == static_cast<size_t>(frame.width()) && + required_resolution.height() == static_cast<size_t>(frame.height())) { + if (report_infra_stats_) { + stats_.scaling_tims_ms.AddSample( + (clock_->CurrentTime() - processing_started).ms<double>()); + } + return frame; + } + + // We allow some difference in the aspect ration because when decoder + // downscales video stream it may round up some dimensions to make them even, + // ex: 960x540 -> 480x270 -> 240x136 instead of 240x135. + RTC_CHECK_LE(std::abs(static_cast<double>(required_resolution.width()) / + required_resolution.height() - + static_cast<double>(frame.width()) / frame.height()), + 0.1) + << peer_name_ + << ": Received frame has too different aspect ratio compared to " + << "requested video resolution: required resolution=" + << required_resolution.ToString() + << "; actual resolution=" << frame.width() << "x" << frame.height(); + + rtc::scoped_refptr<I420Buffer> scaled_buffer(I420Buffer::Create( + required_resolution.width(), required_resolution.height())); + scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420()); + + VideoFrame scaled_frame = frame; + scaled_frame.set_video_frame_buffer(scaled_buffer); + if (report_infra_stats_) { + stats_.scaling_tims_ms.AddSample( + (clock_->CurrentTime() - processing_started).ms<double>()); + } + return scaled_frame; +} + +void AnalyzingVideoSink::AnalyzeFrame(const VideoFrame& frame) { + VideoFrame frame_copy = frame; + frame_copy.set_video_frame_buffer( + I420Buffer::Copy(*frame.video_frame_buffer()->ToI420())); + analyzer_->OnFrameRendered(peer_name_, frame_copy); +} + +AnalyzingVideoSink::SinksDescriptor* AnalyzingVideoSink::PopulateSinks( + absl::string_view stream_label) { + // Fast pass: sinks already exists. + auto sinks_it = stream_sinks_.find(std::string(stream_label)); + if (sinks_it != stream_sinks_.end()) { + return &sinks_it->second; + } + + // Slow pass: we need to create and save sinks + absl::optional<std::pair<std::string, VideoConfig>> peer_and_config = + sinks_helper_->GetPeerAndConfig(stream_label); + RTC_CHECK(peer_and_config.has_value()) + << "No video config for stream " << stream_label; + const std::string& sender_peer_name = peer_and_config->first; + const VideoConfig& config = peer_and_config->second; + + absl::optional<VideoResolution> resolution = + subscription_.GetResolutionForPeer(sender_peer_name); + if (!resolution.has_value()) { + RTC_LOG(LS_ERROR) << peer_name_ << " received stream " << stream_label + << " from " << sender_peer_name + << " for which they were not subscribed"; + resolution = config.GetResolution(); + } + if (!resolution->IsRegular()) { + RTC_LOG(LS_ERROR) << peer_name_ << " received stream " << stream_label + << " from " << sender_peer_name + << " for which resolution wasn't resolved"; + resolution = config.GetResolution(); + } + + RTC_CHECK(resolution.has_value()); + + SinksDescriptor sinks_descriptor(sender_peer_name, *resolution); + if (config.output_dump_options.has_value()) { + std::unique_ptr<test::VideoFrameWriter> writer = + config.output_dump_options->CreateOutputDumpVideoFrameWriter( + stream_label, peer_name_, *resolution); + if (config.output_dump_use_fixed_framerate) { + writer = std::make_unique<test::FixedFpsVideoFrameWriterAdapter>( + resolution->fps(), clock_, std::move(writer)); + } + sinks_descriptor.sinks.push_back(std::make_unique<VideoWriter>( + writer.get(), config.output_dump_options->sampling_modulo())); + sinks_descriptor.video_frame_writer = + sinks_helper_->AddVideoWriter(std::move(writer)); + } + if (config.show_on_screen) { + sinks_descriptor.sinks.push_back( + absl::WrapUnique(test::VideoRenderer::Create( + (*config.stream_label + "-render").c_str(), resolution->width(), + resolution->height()))); + } + return &stream_sinks_.emplace(stream_label, std::move(sinks_descriptor)) + .first->second; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.h new file mode 100644 index 0000000000..1834bbe469 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINK_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINK_H_ + +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/video/video_frame_writer.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// A sink to inject video quality analyzer as a sink into WebRTC. +class AnalyzingVideoSink : public rtc::VideoSinkInterface<VideoFrame> { + public: + struct Stats { + // Time required to scale video frame to the requested rendered resolution. + // Collected only for frames with ID set and iff `report_infra_stats` is + // true. + SamplesStatsCounter scaling_tims_ms; + // Time required to process single video frame. Collected only for frames + // with ID set and iff `report_infra_stats` is true. + SamplesStatsCounter analyzing_sink_processing_time_ms; + }; + + AnalyzingVideoSink(absl::string_view peer_name, + Clock* clock, + VideoQualityAnalyzerInterface& analyzer, + AnalyzingVideoSinksHelper& sinks_helper, + const VideoSubscription& subscription, + bool report_infra_stats); + + // Updates subscription used by this peer to render received video. + void UpdateSubscription(const VideoSubscription& subscription); + + void OnFrame(const VideoFrame& frame) override; + + Stats stats() const; + + private: + struct SinksDescriptor { + SinksDescriptor(absl::string_view sender_peer_name, + const VideoResolution& resolution) + : sender_peer_name(sender_peer_name), resolution(resolution) {} + + // Required to be able to resolve resolutions on new subscription and + // understand if we need to recreate `video_frame_writer` and `sinks`. + std::string sender_peer_name; + // Resolution which was used to create `video_frame_writer` and `sinks`. + VideoResolution resolution; + + // Is set if dumping of output video was requested; + test::VideoFrameWriter* video_frame_writer = nullptr; + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> sinks; + }; + + // Scales video frame to `required_resolution` if necessary. Crashes if video + // frame and `required_resolution` have different aspect ratio. + VideoFrame ScaleVideoFrame(const VideoFrame& frame, + const VideoResolution& required_resolution) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + // Creates full copy of the frame to free any frame owned internal buffers + // and passes created copy to analyzer. Uses `I420Buffer` to represent + // frame content. + void AnalyzeFrame(const VideoFrame& frame); + // Populates sink for specified stream and caches them in `stream_sinks_`. + SinksDescriptor* PopulateSinks(absl::string_view stream_label) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + const std::string peer_name_; + const bool report_infra_stats_; + Clock* const clock_; + VideoQualityAnalyzerInterface* const analyzer_; + AnalyzingVideoSinksHelper* const sinks_helper_; + + mutable Mutex mutex_; + VideoSubscription subscription_ RTC_GUARDED_BY(mutex_); + std::map<std::string, SinksDescriptor> stream_sinks_ RTC_GUARDED_BY(mutex_); + Stats stats_ RTC_GUARDED_BY(mutex_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINK_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink_test.cc new file mode 100644 index 0000000000..6cd89551ea --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink_test.cc @@ -0,0 +1,598 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/analyzing_video_sink.h" + +#include <stdio.h> + +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "api/test/create_frame_generator.h" +#include "api/test/frame_generator_interface.h" +#include "api/test/pclf/media_configuration.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/clock.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/pc/e2e/analyzer/video/example_video_quality_analyzer.h" +#include "test/testsupport/file_utils.h" +#include "test/testsupport/frame_reader.h" +#include "test/time_controller/simulated_time_controller.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::testing::ElementsAreArray; +using ::testing::Eq; +using ::testing::Ge; +using ::testing::Test; + +// Remove files and directories in a directory non-recursively. +void CleanDir(absl::string_view dir, size_t expected_output_files_count) { + absl::optional<std::vector<std::string>> dir_content = + test::ReadDirectory(dir); + if (expected_output_files_count == 0) { + ASSERT_TRUE(!dir_content.has_value() || dir_content->empty()) + << "Empty directory is expected"; + } else { + ASSERT_TRUE(dir_content.has_value()) << "Test directory is empty!"; + EXPECT_EQ(dir_content->size(), expected_output_files_count); + for (const auto& entry : *dir_content) { + if (test::DirExists(entry)) { + EXPECT_TRUE(test::RemoveDir(entry)) + << "Failed to remove sub directory: " << entry; + } else if (test::FileExists(entry)) { + EXPECT_TRUE(test::RemoveFile(entry)) + << "Failed to remove file: " << entry; + } else { + FAIL() << "Can't remove unknown file type: " << entry; + } + } + } + EXPECT_TRUE(test::RemoveDir(dir)) << "Failed to remove directory: " << dir; +} + +VideoFrame CreateFrame(test::FrameGeneratorInterface& frame_generator) { + test::FrameGeneratorInterface::VideoFrameData frame_data = + frame_generator.NextFrame(); + return VideoFrame::Builder() + .set_video_frame_buffer(frame_data.buffer) + .set_update_rect(frame_data.update_rect) + .build(); +} + +std::unique_ptr<test::FrameGeneratorInterface> CreateFrameGenerator( + size_t width, + size_t height) { + return test::CreateSquareFrameGenerator(width, height, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); +} + +void AssertFrameIdsAre(const std::string& filename, + std::vector<std::string> expected_ids) { + FILE* file = fopen(filename.c_str(), "r"); + ASSERT_TRUE(file != nullptr) << "Failed to open frame ids file: " << filename; + std::vector<std::string> actual_ids; + char buffer[8]; + while (fgets(buffer, sizeof buffer, file) != nullptr) { + std::string current_id(buffer); + EXPECT_GE(current_id.size(), 2lu) + << "Found invalid frame id: [" << current_id << "]"; + if (current_id.size() < 2) { + continue; + } + // Trim "\n" at the end. + actual_ids.push_back(current_id.substr(0, current_id.size() - 1)); + } + fclose(file); + EXPECT_THAT(actual_ids, ElementsAreArray(expected_ids)); +} + +class AnalyzingVideoSinkTest : public Test { + protected: + ~AnalyzingVideoSinkTest() override = default; + + void SetUp() override { + // Create an empty temporary directory for this test. + test_directory_ = test::JoinFilename( + test::OutputPath(), + "TestDir_AnalyzingVideoSinkTest_" + + std::string( + testing::UnitTest::GetInstance()->current_test_info()->name())); + test::CreateDir(test_directory_); + } + + void TearDown() override { + CleanDir(test_directory_, expected_output_files_count_); + } + + void ExpectOutputFilesCount(size_t count) { + expected_output_files_count_ = count; + } + + std::string test_directory_; + size_t expected_output_files_count_ = 0; +}; + +TEST_F(AnalyzingVideoSinkTest, VideoFramesAreDumpedCorrectly) { + VideoSubscription subscription; + subscription.SubscribeToPeer( + "alice", VideoResolution(/*width=*/640, /*height=*/360, /*fps=*/30)); + VideoConfig video_config("alice_video", /*width=*/1280, /*height=*/720, + /*fps=*/30); + video_config.output_dump_options = VideoDumpOptions(test_directory_); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/1280, /*height=*/720); + VideoFrame frame = CreateFrame(*frame_generator); + frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame)); + + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription, /*report_infra_stats=*/false); + sink.OnFrame(frame); + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(1))); + + auto frame_reader = test::CreateY4mFrameReader( + test::JoinFilename(test_directory_, "alice_video_bob_640x360_30.y4m")); + EXPECT_THAT(frame_reader->num_frames(), Eq(1)); + rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame(); + rtc::scoped_refptr<I420BufferInterface> expected_frame = + frame.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Actual should be downscaled version of expected. + EXPECT_GT(ssim, 0.98); + EXPECT_GT(psnr, 38); + + ExpectOutputFilesCount(1); +} + +TEST_F(AnalyzingVideoSinkTest, + FallbackOnConfigResolutionIfNoSubscriptionProvided) { + VideoSubscription subscription; + VideoConfig video_config("alice_video", /*width=*/320, /*height=*/240, + /*fps=*/30); + video_config.output_dump_options = VideoDumpOptions(test_directory_); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/320, /*height=*/240); + VideoFrame frame = CreateFrame(*frame_generator); + frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame)); + + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription, /*report_infra_stats=*/false); + sink.OnFrame(frame); + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(1))); + + auto frame_reader = test::CreateY4mFrameReader( + test::JoinFilename(test_directory_, "alice_video_bob_320x240_30.y4m")); + EXPECT_THAT(frame_reader->num_frames(), Eq(1)); + rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame(); + rtc::scoped_refptr<I420BufferInterface> expected_frame = + frame.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Frames should be equal. + EXPECT_DOUBLE_EQ(ssim, 1.00); + EXPECT_DOUBLE_EQ(psnr, 48); + + ExpectOutputFilesCount(1); +} + +TEST_F(AnalyzingVideoSinkTest, + FallbackOnConfigResolutionIfNoSubscriptionIsNotResolved) { + VideoSubscription subscription; + subscription.SubscribeToAllPeers( + VideoResolution(VideoResolution::Spec::kMaxFromSender)); + VideoConfig video_config("alice_video", /*width=*/320, /*height=*/240, + /*fps=*/30); + video_config.output_dump_options = VideoDumpOptions(test_directory_); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/320, /*height=*/240); + VideoFrame frame = CreateFrame(*frame_generator); + frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame)); + + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription, /*report_infra_stats=*/false); + sink.OnFrame(frame); + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(1))); + + auto frame_reader = test::CreateY4mFrameReader( + test::JoinFilename(test_directory_, "alice_video_bob_320x240_30.y4m")); + EXPECT_THAT(frame_reader->num_frames(), Eq(1)); + rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame(); + rtc::scoped_refptr<I420BufferInterface> expected_frame = + frame.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Frames should be equal. + EXPECT_DOUBLE_EQ(ssim, 1.00); + EXPECT_DOUBLE_EQ(psnr, 48); + + ExpectOutputFilesCount(1); +} + +TEST_F(AnalyzingVideoSinkTest, + VideoFramesAreDumpedCorrectlyWhenSubscriptionChanged) { + VideoSubscription subscription_before; + subscription_before.SubscribeToPeer( + "alice", VideoResolution(/*width=*/1280, /*height=*/720, /*fps=*/30)); + VideoSubscription subscription_after; + subscription_after.SubscribeToPeer( + "alice", VideoResolution(/*width=*/640, /*height=*/360, /*fps=*/30)); + VideoConfig video_config("alice_video", /*width=*/1280, /*height=*/720, + /*fps=*/30); + video_config.output_dump_options = VideoDumpOptions(test_directory_); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/1280, /*height=*/720); + VideoFrame frame_before = CreateFrame(*frame_generator); + frame_before.set_id( + analyzer.OnFrameCaptured("alice", "alice_video", frame_before)); + VideoFrame frame_after = CreateFrame(*frame_generator); + frame_after.set_id( + analyzer.OnFrameCaptured("alice", "alice_video", frame_after)); + + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription_before, /*report_infra_stats=*/false); + sink.OnFrame(frame_before); + + sink.UpdateSubscription(subscription_after); + sink.OnFrame(frame_after); + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(2))); + + { + auto frame_reader = test::CreateY4mFrameReader( + test::JoinFilename(test_directory_, "alice_video_bob_1280x720_30.y4m")); + EXPECT_THAT(frame_reader->num_frames(), Eq(1)); + rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame(); + rtc::scoped_refptr<I420BufferInterface> expected_frame = + frame_before.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Frames should be equal. + EXPECT_DOUBLE_EQ(ssim, 1.00); + EXPECT_DOUBLE_EQ(psnr, 48); + } + { + auto frame_reader = test::CreateY4mFrameReader( + test::JoinFilename(test_directory_, "alice_video_bob_640x360_30.y4m")); + EXPECT_THAT(frame_reader->num_frames(), Eq(1)); + rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame(); + rtc::scoped_refptr<I420BufferInterface> expected_frame = + frame_after.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Actual should be downscaled version of expected. + EXPECT_GT(ssim, 0.98); + EXPECT_GT(psnr, 38); + } + + ExpectOutputFilesCount(2); +} + +TEST_F(AnalyzingVideoSinkTest, + VideoFramesAreDumpedCorrectlyWhenSubscriptionChangedOnTheSameOne) { + VideoSubscription subscription_before; + subscription_before.SubscribeToPeer( + "alice", VideoResolution(/*width=*/640, /*height=*/360, /*fps=*/30)); + VideoSubscription subscription_after; + subscription_after.SubscribeToPeer( + "alice", VideoResolution(/*width=*/640, /*height=*/360, /*fps=*/30)); + VideoConfig video_config("alice_video", /*width=*/640, /*height=*/360, + /*fps=*/30); + video_config.output_dump_options = VideoDumpOptions(test_directory_); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/640, /*height=*/360); + VideoFrame frame_before = CreateFrame(*frame_generator); + frame_before.set_id( + analyzer.OnFrameCaptured("alice", "alice_video", frame_before)); + VideoFrame frame_after = CreateFrame(*frame_generator); + frame_after.set_id( + analyzer.OnFrameCaptured("alice", "alice_video", frame_after)); + + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription_before, /*report_infra_stats=*/false); + sink.OnFrame(frame_before); + + sink.UpdateSubscription(subscription_after); + sink.OnFrame(frame_after); + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(2))); + + { + auto frame_reader = test::CreateY4mFrameReader( + test::JoinFilename(test_directory_, "alice_video_bob_640x360_30.y4m")); + EXPECT_THAT(frame_reader->num_frames(), Eq(2)); + // Read the first frame. + rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame(); + rtc::scoped_refptr<I420BufferInterface> expected_frame = + frame_before.video_frame_buffer()->ToI420(); + // Frames should be equal. + EXPECT_DOUBLE_EQ(I420SSIM(*expected_frame, *actual_frame), 1.00); + EXPECT_DOUBLE_EQ(I420PSNR(*expected_frame, *actual_frame), 48); + // Read the second frame. + actual_frame = frame_reader->PullFrame(); + expected_frame = frame_after.video_frame_buffer()->ToI420(); + // Frames should be equal. + EXPECT_DOUBLE_EQ(I420SSIM(*expected_frame, *actual_frame), 1.00); + EXPECT_DOUBLE_EQ(I420PSNR(*expected_frame, *actual_frame), 48); + } + + ExpectOutputFilesCount(1); +} + +TEST_F(AnalyzingVideoSinkTest, SmallDiviationsInAspectRationAreAllowed) { + VideoSubscription subscription; + subscription.SubscribeToPeer( + "alice", VideoResolution(/*width=*/480, /*height=*/270, /*fps=*/30)); + VideoConfig video_config("alice_video", /*width=*/480, /*height=*/270, + /*fps=*/30); + video_config.output_dump_options = VideoDumpOptions(test_directory_); + + ExampleVideoQualityAnalyzer analyzer; + // Generator produces downscaled frames with a bit different aspect ration. + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/240, /*height=*/136); + VideoFrame frame = CreateFrame(*frame_generator); + frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame)); + + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription, /*report_infra_stats=*/false); + sink.OnFrame(frame); + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(1))); + + { + auto frame_reader = test::CreateY4mFrameReader( + test::JoinFilename(test_directory_, "alice_video_bob_480x270_30.y4m")); + EXPECT_THAT(frame_reader->num_frames(), Eq(1)); + // Read the first frame. + rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame(); + rtc::scoped_refptr<I420BufferInterface> expected_frame = + frame.video_frame_buffer()->ToI420(); + // Actual frame is upscaled version of the expected. But because rendered + // resolution is equal to the actual frame size we need to upscale expected + // during comparison and then they have to be the same. + EXPECT_DOUBLE_EQ(I420SSIM(*actual_frame, *expected_frame), 1); + EXPECT_DOUBLE_EQ(I420PSNR(*actual_frame, *expected_frame), 48); + } + + ExpectOutputFilesCount(1); +} + +TEST_F(AnalyzingVideoSinkTest, VideoFramesIdsAreDumpedWhenRequested) { + VideoSubscription subscription; + subscription.SubscribeToPeer( + "alice", VideoResolution(/*width=*/320, /*height=*/240, /*fps=*/30)); + VideoConfig video_config("alice_video", /*width=*/320, /*height=*/240, + /*fps=*/30); + video_config.output_dump_options = + VideoDumpOptions(test_directory_, /*export_frame_ids=*/true); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/320, /*height=*/240); + + std::vector<std::string> expected_frame_ids; + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription, /*report_infra_stats=*/false); + for (int i = 0; i < 10; ++i) { + VideoFrame frame = CreateFrame(*frame_generator); + frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame)); + expected_frame_ids.push_back(std::to_string(frame.id())); + sink.OnFrame(frame); + } + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(10))); + + AssertFrameIdsAre( + test::JoinFilename(test_directory_, + "alice_video_bob_320x240_30.frame_ids.txt"), + expected_frame_ids); + + ExpectOutputFilesCount(2); +} + +TEST_F(AnalyzingVideoSinkTest, + VideoFramesAndIdsAreDumpedWithFixedFpsWhenRequested) { + GlobalSimulatedTimeController simulated_time(Timestamp::Seconds(100000)); + + VideoSubscription subscription; + subscription.SubscribeToPeer( + "alice", VideoResolution(/*width=*/320, /*height=*/240, /*fps=*/10)); + VideoConfig video_config("alice_video", /*width=*/320, /*height=*/240, + /*fps=*/10); + video_config.output_dump_options = + VideoDumpOptions(test_directory_, /*export_frame_ids=*/true); + video_config.output_dump_use_fixed_framerate = true; + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/320, /*height=*/240); + + VideoFrame frame1 = CreateFrame(*frame_generator); + frame1.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame1)); + VideoFrame frame2 = CreateFrame(*frame_generator); + frame2.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame2)); + + { + // `helper` and `sink` has to be destroyed so all frames will be written + // to the disk. + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", simulated_time.GetClock(), analyzer, helper, + subscription, /*report_infra_stats=*/false); + sink.OnFrame(frame1); + // Advance almost 1 second, so the first frame has to be repeated 9 time + // more. + simulated_time.AdvanceTime(TimeDelta::Millis(990)); + sink.OnFrame(frame2); + simulated_time.AdvanceTime(TimeDelta::Millis(100)); + } + + EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(2))); + + auto frame_reader = test::CreateY4mFrameReader( + test::JoinFilename(test_directory_, "alice_video_bob_320x240_10.y4m")); + EXPECT_THAT(frame_reader->num_frames(), Eq(11)); + for (int i = 0; i < 10; ++i) { + rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame(); + rtc::scoped_refptr<I420BufferInterface> expected_frame = + frame1.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Frames should be equal. + EXPECT_DOUBLE_EQ(ssim, 1.00); + EXPECT_DOUBLE_EQ(psnr, 48); + } + rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame(); + rtc::scoped_refptr<I420BufferInterface> expected_frame = + frame2.video_frame_buffer()->ToI420(); + double psnr = I420PSNR(*expected_frame, *actual_frame); + double ssim = I420SSIM(*expected_frame, *actual_frame); + // Frames should be equal. + EXPECT_DOUBLE_EQ(ssim, 1.00); + EXPECT_DOUBLE_EQ(psnr, 48); + + AssertFrameIdsAre( + test::JoinFilename(test_directory_, + "alice_video_bob_320x240_10.frame_ids.txt"), + {std::to_string(frame1.id()), std::to_string(frame1.id()), + std::to_string(frame1.id()), std::to_string(frame1.id()), + std::to_string(frame1.id()), std::to_string(frame1.id()), + std::to_string(frame1.id()), std::to_string(frame1.id()), + std::to_string(frame1.id()), std::to_string(frame1.id()), + std::to_string(frame2.id())}); + + ExpectOutputFilesCount(2); +} + +TEST_F(AnalyzingVideoSinkTest, InfraMetricsCollectedWhenRequested) { + VideoSubscription subscription; + subscription.SubscribeToPeer( + "alice", VideoResolution(/*width=*/1280, /*height=*/720, /*fps=*/30)); + VideoConfig video_config("alice_video", /*width=*/640, /*height=*/360, + /*fps=*/30); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/640, /*height=*/360); + VideoFrame frame = CreateFrame(*frame_generator); + frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame)); + + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription, /*report_infra_stats=*/true); + sink.OnFrame(frame); + + AnalyzingVideoSink::Stats stats = sink.stats(); + EXPECT_THAT(stats.scaling_tims_ms.NumSamples(), Eq(1)); + EXPECT_THAT(stats.scaling_tims_ms.GetAverage(), Ge(0)); + EXPECT_THAT(stats.analyzing_sink_processing_time_ms.NumSamples(), Eq(1)); + EXPECT_THAT(stats.analyzing_sink_processing_time_ms.GetAverage(), + Ge(stats.scaling_tims_ms.GetAverage())); + + ExpectOutputFilesCount(0); +} + +TEST_F(AnalyzingVideoSinkTest, InfraMetricsNotCollectedWhenNotRequested) { + VideoSubscription subscription; + subscription.SubscribeToPeer( + "alice", VideoResolution(/*width=*/1280, /*height=*/720, /*fps=*/30)); + VideoConfig video_config("alice_video", /*width=*/640, /*height=*/360, + /*fps=*/30); + + ExampleVideoQualityAnalyzer analyzer; + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + CreateFrameGenerator(/*width=*/640, /*height=*/360); + VideoFrame frame = CreateFrame(*frame_generator); + frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame)); + + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", video_config); + AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper, + subscription, /*report_infra_stats=*/false); + sink.OnFrame(frame); + + AnalyzingVideoSink::Stats stats = sink.stats(); + EXPECT_THAT(stats.scaling_tims_ms.NumSamples(), Eq(0)); + EXPECT_THAT(stats.analyzing_sink_processing_time_ms.NumSamples(), Eq(0)); + + ExpectOutputFilesCount(0); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.cc new file mode 100644 index 0000000000..70dc4b00b5 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.cc @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h" + +#include <memory> +#include <set> +#include <string> +#include <utility> + +#include "absl/strings/string_view.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/video/video_frame_writer.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +void AnalyzingVideoSinksHelper::AddConfig(absl::string_view sender_peer_name, + VideoConfig config) { + MutexLock lock(&mutex_); + auto it = video_configs_.find(*config.stream_label); + if (it == video_configs_.end()) { + std::string stream_label = *config.stream_label; + video_configs_.emplace( + std::move(stream_label), + std::pair{std::string(sender_peer_name), std::move(config)}); + } else { + it->second = std::pair{std::string(sender_peer_name), std::move(config)}; + } +} + +absl::optional<std::pair<std::string, VideoConfig>> +AnalyzingVideoSinksHelper::GetPeerAndConfig(absl::string_view stream_label) { + MutexLock lock(&mutex_); + auto it = video_configs_.find(std::string(stream_label)); + if (it == video_configs_.end()) { + return absl::nullopt; + } + return it->second; +} + +void AnalyzingVideoSinksHelper::RemoveConfig(absl::string_view stream_label) { + MutexLock lock(&mutex_); + video_configs_.erase(std::string(stream_label)); +} + +test::VideoFrameWriter* AnalyzingVideoSinksHelper::AddVideoWriter( + std::unique_ptr<test::VideoFrameWriter> video_writer) { + MutexLock lock(&mutex_); + test::VideoFrameWriter* out = video_writer.get(); + video_writers_.push_back(std::move(video_writer)); + return out; +} + +void AnalyzingVideoSinksHelper::CloseAndRemoveVideoWriters( + std::set<test::VideoFrameWriter*> writers_to_close) { + MutexLock lock(&mutex_); + for (auto it = video_writers_.cbegin(); it != video_writers_.cend();) { + if (writers_to_close.find(it->get()) != writers_to_close.end()) { + (*it)->Close(); + it = video_writers_.erase(it); + } else { + ++it; + } + } +} + +void AnalyzingVideoSinksHelper::Clear() { + MutexLock lock(&mutex_); + video_configs_.clear(); + for (const auto& video_writer : video_writers_) { + video_writer->Close(); + } + video_writers_.clear(); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h new file mode 100644 index 0000000000..5f38c5a40e --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINKS_HELPER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINKS_HELPER_H_ + +#include <list> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <utility> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/video/video_frame_writer.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Registry of known video configs and video writers. +// This class is thread safe. +class AnalyzingVideoSinksHelper { + public: + // Adds config in the registry. If config with such stream label was + // registered before, the new value will override the old one. + void AddConfig(absl::string_view sender_peer_name, VideoConfig config); + absl::optional<std::pair<std::string, VideoConfig>> GetPeerAndConfig( + absl::string_view stream_label); + // Removes video config for specified stream label. If there are no know video + // config for such stream label - does nothing. + void RemoveConfig(absl::string_view stream_label); + + // Takes ownership of the provided video writer. All video writers owned by + // this class will be closed during `AnalyzingVideoSinksHelper` destruction + // and guaranteed to be alive either until explicitly removed by + // `CloseAndRemoveVideoWriters` or until `AnalyzingVideoSinksHelper` is + // destroyed. + // + // Returns pointer to the added writer. Ownership is maintained by + // `AnalyzingVideoSinksHelper`. + test::VideoFrameWriter* AddVideoWriter( + std::unique_ptr<test::VideoFrameWriter> video_writer); + // For each provided `writers_to_close`, if it is known, will close and + // destroy it, otherwise does nothing with it. + void CloseAndRemoveVideoWriters( + std::set<test::VideoFrameWriter*> writers_to_close); + + // Removes all added configs and close and removes all added writers. + void Clear(); + + private: + Mutex mutex_; + std::map<std::string, std::pair<std::string, VideoConfig>> video_configs_ + RTC_GUARDED_BY(mutex_); + std::list<std::unique_ptr<test::VideoFrameWriter>> video_writers_ + RTC_GUARDED_BY(mutex_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINKS_HELPER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper_test.cc new file mode 100644 index 0000000000..1a820a5229 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper_test.cc @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h" + +#include <memory> +#include <string> +#include <utility> + +#include "absl/types/optional.h" +#include "api/test/pclf/media_configuration.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::testing::Eq; + +// Asserts equality of the main fields of the video config. We don't compare +// the full config due to the lack of equality definition for a lot of subtypes. +void AssertConfigsAreEquals(const VideoConfig& actual, + const VideoConfig& expected) { + EXPECT_THAT(actual.stream_label, Eq(expected.stream_label)); + EXPECT_THAT(actual.width, Eq(expected.width)); + EXPECT_THAT(actual.height, Eq(expected.height)); + EXPECT_THAT(actual.fps, Eq(expected.fps)); +} + +TEST(AnalyzingVideoSinksHelperTest, ConfigsCanBeAdded) { + VideoConfig config("alice_video", /*width=*/1280, /*height=*/720, /*fps=*/30); + + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", config); + + absl::optional<std::pair<std::string, VideoConfig>> registred_config = + helper.GetPeerAndConfig("alice_video"); + ASSERT_TRUE(registred_config.has_value()); + EXPECT_THAT(registred_config->first, Eq("alice")); + AssertConfigsAreEquals(registred_config->second, config); +} + +TEST(AnalyzingVideoSinksHelperTest, AddingForExistingLabelWillOverwriteValue) { + VideoConfig config_before("alice_video", /*width=*/1280, /*height=*/720, + /*fps=*/30); + VideoConfig config_after("alice_video", /*width=*/640, /*height=*/360, + /*fps=*/15); + + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", config_before); + + absl::optional<std::pair<std::string, VideoConfig>> registred_config = + helper.GetPeerAndConfig("alice_video"); + ASSERT_TRUE(registred_config.has_value()); + EXPECT_THAT(registred_config->first, Eq("alice")); + AssertConfigsAreEquals(registred_config->second, config_before); + + helper.AddConfig("alice", config_after); + + registred_config = helper.GetPeerAndConfig("alice_video"); + ASSERT_TRUE(registred_config.has_value()); + EXPECT_THAT(registred_config->first, Eq("alice")); + AssertConfigsAreEquals(registred_config->second, config_after); +} + +TEST(AnalyzingVideoSinksHelperTest, ConfigsCanBeRemoved) { + VideoConfig config("alice_video", /*width=*/1280, /*height=*/720, /*fps=*/30); + + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", config); + + ASSERT_TRUE(helper.GetPeerAndConfig("alice_video").has_value()); + + helper.RemoveConfig("alice_video"); + ASSERT_FALSE(helper.GetPeerAndConfig("alice_video").has_value()); +} + +TEST(AnalyzingVideoSinksHelperTest, RemoveOfNonExistingConfigDontCrash) { + AnalyzingVideoSinksHelper helper; + helper.RemoveConfig("alice_video"); +} + +TEST(AnalyzingVideoSinksHelperTest, ClearRemovesAllConfigs) { + VideoConfig config1("alice_video", /*width=*/640, /*height=*/360, /*fps=*/30); + VideoConfig config2("bob_video", /*width=*/640, /*height=*/360, /*fps=*/30); + + AnalyzingVideoSinksHelper helper; + helper.AddConfig("alice", config1); + helper.AddConfig("bob", config2); + + ASSERT_TRUE(helper.GetPeerAndConfig("alice_video").has_value()); + ASSERT_TRUE(helper.GetPeerAndConfig("bob_video").has_value()); + + helper.Clear(); + ASSERT_FALSE(helper.GetPeerAndConfig("alice_video").has_value()); + ASSERT_FALSE(helper.GetPeerAndConfig("bob_video").has_value()); +} + +struct TestVideoFrameWriterFactory { + int closed_writers_count = 0; + int deleted_writers_count = 0; + + std::unique_ptr<test::VideoFrameWriter> CreateWriter() { + return std::make_unique<TestVideoFrameWriter>(this); + } + + private: + class TestVideoFrameWriter : public test::VideoFrameWriter { + public: + explicit TestVideoFrameWriter(TestVideoFrameWriterFactory* factory) + : factory_(factory) {} + ~TestVideoFrameWriter() override { factory_->deleted_writers_count++; } + + bool WriteFrame(const VideoFrame& frame) override { return true; } + + void Close() override { factory_->closed_writers_count++; } + + private: + TestVideoFrameWriterFactory* factory_; + }; +}; + +TEST(AnalyzingVideoSinksHelperTest, RemovingWritersCloseAndDestroyAllOfThem) { + TestVideoFrameWriterFactory factory; + + AnalyzingVideoSinksHelper helper; + test::VideoFrameWriter* writer1 = + helper.AddVideoWriter(factory.CreateWriter()); + test::VideoFrameWriter* writer2 = + helper.AddVideoWriter(factory.CreateWriter()); + + helper.CloseAndRemoveVideoWriters({writer1, writer2}); + + EXPECT_THAT(factory.closed_writers_count, Eq(2)); + EXPECT_THAT(factory.deleted_writers_count, Eq(2)); +} + +TEST(AnalyzingVideoSinksHelperTest, ClearCloseAndDestroyAllWriters) { + TestVideoFrameWriterFactory factory; + + AnalyzingVideoSinksHelper helper; + helper.AddVideoWriter(factory.CreateWriter()); + helper.AddVideoWriter(factory.CreateWriter()); + + helper.Clear(); + + EXPECT_THAT(factory.closed_writers_count, Eq(2)); + EXPECT_THAT(factory.deleted_writers_count, Eq(2)); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc new file mode 100644 index 0000000000..59144589fc --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -0,0 +1,1228 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" + +#include <algorithm> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <utility> +#include <vector> + +#include "api/array_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metric.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "api/video/video_frame.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/strings/string_builder.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h" +#include "test/pc/e2e/metric_metadata_keys.h" + +namespace webrtc { +namespace { + +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Unit; +using ::webrtc::webrtc_pc_e2e::MetricMetadataKey; + +constexpr int kBitsInByte = 8; +constexpr absl::string_view kSkipRenderedFrameReasonProcessed = "processed"; +constexpr absl::string_view kSkipRenderedFrameReasonRendered = "rendered"; +constexpr absl::string_view kSkipRenderedFrameReasonDropped = + "considered dropped"; + +void LogFrameCounters(const std::string& name, const FrameCounters& counters) { + RTC_LOG(LS_INFO) << "[" << name + << "] Captured : " << counters.captured; + RTC_LOG(LS_INFO) << "[" << name + << "] Pre encoded : " << counters.pre_encoded; + RTC_LOG(LS_INFO) << "[" << name + << "] Encoded : " << counters.encoded; + RTC_LOG(LS_INFO) << "[" << name + << "] Received : " << counters.received; + RTC_LOG(LS_INFO) << "[" << name + << "] Decoded : " << counters.decoded; + RTC_LOG(LS_INFO) << "[" << name + << "] Rendered : " << counters.rendered; + RTC_LOG(LS_INFO) << "[" << name + << "] Dropped : " << counters.dropped; + RTC_LOG(LS_INFO) << "[" << name + << "] Failed to decode : " << counters.failed_to_decode; +} + +void LogStreamInternalStats(const std::string& name, + const StreamStats& stats, + Timestamp start_time) { + for (const auto& entry : stats.dropped_by_phase) { + RTC_LOG(LS_INFO) << "[" << name << "] Dropped at " << ToString(entry.first) + << ": " << entry.second; + } + Timestamp first_encoded_frame_time = Timestamp::PlusInfinity(); + for (const StreamCodecInfo& encoder : stats.encoders) { + RTC_DCHECK(encoder.switched_on_at.IsFinite()); + RTC_DCHECK(encoder.switched_from_at.IsFinite()); + if (first_encoded_frame_time.IsInfinite()) { + first_encoded_frame_time = encoder.switched_on_at; + } + RTC_LOG(LS_INFO) + << "[" << name << "] Used encoder: \"" << encoder.codec_name + << "\" used from (frame_id=" << encoder.first_frame_id + << "; from_stream_start=" + << (encoder.switched_on_at - stats.stream_started_time).ms() + << "ms, from_call_start=" << (encoder.switched_on_at - start_time).ms() + << "ms) until (frame_id=" << encoder.last_frame_id + << "; from_stream_start=" + << (encoder.switched_from_at - stats.stream_started_time).ms() + << "ms, from_call_start=" + << (encoder.switched_from_at - start_time).ms() << "ms)"; + } + for (const StreamCodecInfo& decoder : stats.decoders) { + RTC_DCHECK(decoder.switched_on_at.IsFinite()); + RTC_DCHECK(decoder.switched_from_at.IsFinite()); + RTC_LOG(LS_INFO) + << "[" << name << "] Used decoder: \"" << decoder.codec_name + << "\" used from (frame_id=" << decoder.first_frame_id + << "; from_stream_start=" + << (decoder.switched_on_at - stats.stream_started_time).ms() + << "ms, from_call_start=" << (decoder.switched_on_at - start_time).ms() + << "ms) until (frame_id=" << decoder.last_frame_id + << "; from_stream_start=" + << (decoder.switched_from_at - stats.stream_started_time).ms() + << "ms, from_call_start=" + << (decoder.switched_from_at - start_time).ms() << "ms)"; + } +} + +template <typename T> +absl::optional<T> MaybeGetValue(const std::map<size_t, T>& map, size_t key) { + auto it = map.find(key); + if (it == map.end()) { + return absl::nullopt; + } + return it->second; +} + +SamplesStatsCounter::StatsSample StatsSample(double value, + Timestamp sampling_time) { + return SamplesStatsCounter::StatsSample{value, sampling_time}; +} + +} // namespace + +DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer( + webrtc::Clock* clock, + test::MetricsLogger* metrics_logger, + DefaultVideoQualityAnalyzerOptions options) + : options_(options), + clock_(clock), + metrics_logger_(metrics_logger), + frames_comparator_(clock, cpu_measurer_, options) { + RTC_CHECK(metrics_logger_); +} + +DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() { + Stop(); +} + +void DefaultVideoQualityAnalyzer::Start( + std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count) { + test_label_ = std::move(test_case_name); + frames_comparator_.Start(max_threads_count); + { + MutexLock lock(&mutex_); + peers_ = std::make_unique<NamesCollection>(peer_names); + RTC_CHECK(start_time_.IsMinusInfinity()); + + RTC_CHECK_EQ(state_, State::kNew) + << "DefaultVideoQualityAnalyzer is already started"; + state_ = State::kActive; + start_time_ = Now(); + } +} + +uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured( + absl::string_view peer_name, + const std::string& stream_label, + const webrtc::VideoFrame& frame) { + // `next_frame_id` is atomic, so we needn't lock here. + Timestamp captured_time = Now(); + Timestamp start_time = Timestamp::MinusInfinity(); + size_t peer_index = -1; + size_t peers_count = -1; + size_t stream_index; + uint16_t frame_id = VideoFrame::kNotSetId; + { + MutexLock lock(&mutex_); + frame_id = GetNextFrameId(); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + // Create a local copy of `start_time_`, peer's index and total peers count + // to access it without holding a `mutex_` during access to + // `frames_comparator_`. + start_time = start_time_; + peer_index = peers_->index(peer_name); + peers_count = peers_->size(); + stream_index = streams_.AddIfAbsent(stream_label); + } + // Ensure stats for this stream exists. + frames_comparator_.EnsureStatsForStream(stream_index, peer_index, peers_count, + captured_time, start_time); + { + MutexLock lock(&mutex_); + stream_to_sender_[stream_index] = peer_index; + frame_counters_.captured++; + for (size_t i : peers_->GetAllIndexes()) { + if (i != peer_index || options_.enable_receive_own_stream) { + InternalStatsKey key(stream_index, peer_index, i); + stream_frame_counters_[key].captured++; + } + } + + std::set<size_t> frame_receivers_indexes = peers_->GetPresentIndexes(); + if (!options_.enable_receive_own_stream) { + frame_receivers_indexes.erase(peer_index); + } + + auto state_it = stream_states_.find(stream_index); + if (state_it == stream_states_.end()) { + stream_states_.emplace( + stream_index, + StreamState(peer_index, frame_receivers_indexes, captured_time)); + } + StreamState* state = &stream_states_.at(stream_index); + state->PushBack(frame_id); + // Update frames in flight info. + auto it = captured_frames_in_flight_.find(frame_id); + if (it != captured_frames_in_flight_.end()) { + // If we overflow uint16_t and hit previous frame id and this frame is + // still in flight, it means that this stream wasn't rendered for long + // time and we need to process existing frame as dropped. + for (size_t i : peers_->GetPresentIndexes()) { + if (i == peer_index && !options_.enable_receive_own_stream) { + continue; + } + + uint16_t oldest_frame_id = state->PopFront(i); + RTC_DCHECK_EQ(frame_id, oldest_frame_id); + frame_counters_.dropped++; + InternalStatsKey key(stream_index, peer_index, i); + stream_frame_counters_.at(key).dropped++; + + analyzer_stats_.frames_in_flight_left_count.AddSample( + StatsSample(captured_frames_in_flight_.size(), Now())); + frames_comparator_.AddComparison( + InternalStatsKey(stream_index, peer_index, i), + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, FrameComparisonType::kDroppedFrame, + it->second.GetStatsForPeer(i)); + } + + captured_frames_in_flight_.erase(it); + } + captured_frames_in_flight_.emplace( + frame_id, FrameInFlight(stream_index, frame, captured_time, + std::move(frame_receivers_indexes))); + // Set frame id on local copy of the frame + captured_frames_in_flight_.at(frame_id).SetFrameId(frame_id); + + // Update history stream<->frame mapping + for (auto it = stream_to_frame_id_history_.begin(); + it != stream_to_frame_id_history_.end(); ++it) { + it->second.erase(frame_id); + } + stream_to_frame_id_history_[stream_index].insert(frame_id); + stream_to_frame_id_full_history_[stream_index].push_back(frame_id); + + // If state has too many frames that are in flight => remove the oldest + // queued frame in order to avoid to use too much memory. + if (state->GetAliveFramesCount() > + options_.max_frames_in_flight_per_stream_count) { + uint16_t frame_id_to_remove = state->MarkNextAliveFrameAsDead(); + auto it = captured_frames_in_flight_.find(frame_id_to_remove); + RTC_CHECK(it != captured_frames_in_flight_.end()) + << "Frame with ID " << frame_id_to_remove + << " is expected to be in flight, but hasn't been found in " + << "|captured_frames_in_flight_|"; + bool is_removed = it->second.RemoveFrame(); + RTC_DCHECK(is_removed) + << "Invalid stream state: alive frame is removed already"; + } + if (options_.report_infra_metrics) { + analyzer_stats_.on_frame_captured_processing_time_ms.AddSample( + (Now() - captured_time).ms<double>()); + } + } + return frame_id; +} + +void DefaultVideoQualityAnalyzer::OnFramePreEncode( + absl::string_view peer_name, + const webrtc::VideoFrame& frame) { + Timestamp processing_started = Now(); + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + auto it = captured_frames_in_flight_.find(frame.id()); + RTC_CHECK(it != captured_frames_in_flight_.end()) + << "Frame id=" << frame.id() << " not found"; + FrameInFlight& frame_in_flight = it->second; + frame_counters_.pre_encoded++; + size_t peer_index = peers_->index(peer_name); + for (size_t i : peers_->GetAllIndexes()) { + if (i != peer_index || options_.enable_receive_own_stream) { + InternalStatsKey key(frame_in_flight.stream(), peer_index, i); + stream_frame_counters_.at(key).pre_encoded++; + } + } + frame_in_flight.SetPreEncodeTime(Now()); + + if (options_.report_infra_metrics) { + analyzer_stats_.on_frame_pre_encode_processing_time_ms.AddSample( + (Now() - processing_started).ms<double>()); + } +} + +void DefaultVideoQualityAnalyzer::OnFrameEncoded( + absl::string_view peer_name, + uint16_t frame_id, + const webrtc::EncodedImage& encoded_image, + const EncoderStats& stats, + bool discarded) { + if (discarded) + return; + + Timestamp processing_started = Now(); + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + auto it = captured_frames_in_flight_.find(frame_id); + if (it == captured_frames_in_flight_.end()) { + RTC_LOG(LS_WARNING) + << "The encoding of video frame with id [" << frame_id << "] for peer [" + << peer_name << "] finished after all receivers rendered this frame or " + << "were removed. It can be OK for simulcast/SVC if higher quality " + << "stream is not required or the last receiver was unregistered " + << "between encoding of different layers, but it may indicate an ERROR " + << "for singlecast or if it happens often."; + return; + } + FrameInFlight& frame_in_flight = it->second; + // For SVC we can receive multiple encoded images for one frame, so to cover + // all cases we have to pick the last encode time. + if (!frame_in_flight.HasEncodedTime()) { + // Increase counters only when we meet this frame first time. + frame_counters_.encoded++; + size_t peer_index = peers_->index(peer_name); + for (size_t i : peers_->GetAllIndexes()) { + if (i != peer_index || options_.enable_receive_own_stream) { + InternalStatsKey key(frame_in_flight.stream(), peer_index, i); + stream_frame_counters_.at(key).encoded++; + } + } + } + Timestamp now = Now(); + StreamCodecInfo used_encoder; + used_encoder.codec_name = stats.encoder_name; + used_encoder.first_frame_id = frame_id; + used_encoder.last_frame_id = frame_id; + used_encoder.switched_on_at = now; + used_encoder.switched_from_at = now; + frame_in_flight.OnFrameEncoded( + now, encoded_image._frameType, DataSize::Bytes(encoded_image.size()), + stats.target_encode_bitrate, encoded_image.SpatialIndex().value_or(0), + stats.qp, used_encoder); + + if (options_.report_infra_metrics) { + analyzer_stats_.on_frame_encoded_processing_time_ms.AddSample( + (Now() - processing_started).ms<double>()); + } +} + +void DefaultVideoQualityAnalyzer::OnFrameDropped( + absl::string_view peer_name, + webrtc::EncodedImageCallback::DropReason reason) { + // Here we do nothing, because we will see this drop on renderer side. +} + +void DefaultVideoQualityAnalyzer::OnFramePreDecode( + absl::string_view peer_name, + uint16_t frame_id, + const webrtc::EncodedImage& input_image) { + Timestamp processing_started = Now(); + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + size_t peer_index = peers_->index(peer_name); + + if (frame_id == VideoFrame::kNotSetId) { + frame_counters_.received++; + unknown_sender_frame_counters_[std::string(peer_name)].received++; + return; + } + + auto it = captured_frames_in_flight_.find(frame_id); + if (it == captured_frames_in_flight_.end() || + it->second.HasReceivedTime(peer_index)) { + // It means this frame was predecoded before, so we can skip it. It may + // happen when we have multiple simulcast streams in one track and received + // the same picture from two different streams because SFU can't reliably + // correlate two simulcast streams and started relaying the second stream + // from the same frame it has relayed right before for the first stream. + return; + } + + frame_counters_.received++; + InternalStatsKey key(it->second.stream(), + stream_to_sender_.at(it->second.stream()), peer_index); + stream_frame_counters_.at(key).received++; + // Determine the time of the last received packet of this video frame. + RTC_DCHECK(!input_image.PacketInfos().empty()); + Timestamp last_receive_time = + std::max_element(input_image.PacketInfos().cbegin(), + input_image.PacketInfos().cend(), + [](const RtpPacketInfo& a, const RtpPacketInfo& b) { + return a.receive_time() < b.receive_time(); + }) + ->receive_time(); + it->second.OnFramePreDecode(peer_index, + /*received_time=*/last_receive_time, + /*decode_start_time=*/Now(), + input_image._frameType, + DataSize::Bytes(input_image.size())); + + if (options_.report_infra_metrics) { + analyzer_stats_.on_frame_pre_decode_processing_time_ms.AddSample( + (Now() - processing_started).ms<double>()); + } +} + +void DefaultVideoQualityAnalyzer::OnFrameDecoded( + absl::string_view peer_name, + const webrtc::VideoFrame& frame, + const DecoderStats& stats) { + Timestamp processing_started = Now(); + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + size_t peer_index = peers_->index(peer_name); + + if (frame.id() == VideoFrame::kNotSetId) { + frame_counters_.decoded++; + unknown_sender_frame_counters_[std::string(peer_name)].decoded++; + return; + } + + auto it = captured_frames_in_flight_.find(frame.id()); + if (it == captured_frames_in_flight_.end() || + it->second.HasDecodeEndTime(peer_index)) { + // It means this frame was decoded before, so we can skip it. It may happen + // when we have multiple simulcast streams in one track and received + // the same frame from two different streams because SFU can't reliably + // correlate two simulcast streams and started relaying the second stream + // from the same frame it has relayed right before for the first stream. + return; + } + frame_counters_.decoded++; + InternalStatsKey key(it->second.stream(), + stream_to_sender_.at(it->second.stream()), peer_index); + stream_frame_counters_.at(key).decoded++; + Timestamp now = Now(); + StreamCodecInfo used_decoder; + used_decoder.codec_name = stats.decoder_name; + used_decoder.first_frame_id = frame.id(); + used_decoder.last_frame_id = frame.id(); + used_decoder.switched_on_at = now; + used_decoder.switched_from_at = now; + it->second.OnFrameDecoded(peer_index, now, frame.width(), frame.height(), + used_decoder); + + if (options_.report_infra_metrics) { + analyzer_stats_.on_frame_decoded_processing_time_ms.AddSample( + (Now() - processing_started).ms<double>()); + } +} + +void DefaultVideoQualityAnalyzer::OnFrameRendered( + absl::string_view peer_name, + const webrtc::VideoFrame& frame) { + Timestamp processing_started = Now(); + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + size_t peer_index = peers_->index(peer_name); + + if (frame.id() == VideoFrame::kNotSetId) { + frame_counters_.rendered++; + unknown_sender_frame_counters_[std::string(peer_name)].rendered++; + return; + } + + auto frame_it = captured_frames_in_flight_.find(frame.id()); + if (frame_it == captured_frames_in_flight_.end() || + frame_it->second.HasRenderedTime(peer_index) || + frame_it->second.IsDropped(peer_index)) { + // It means this frame was rendered or dropped before, so we can skip it. + // It may happen when we have multiple simulcast streams in one track and + // received the same frame from two different streams because SFU can't + // reliably correlate two simulcast streams and started relaying the second + // stream from the same frame it has relayed right before for the first + // stream. + absl::string_view reason = kSkipRenderedFrameReasonProcessed; + if (frame_it != captured_frames_in_flight_.end()) { + if (frame_it->second.HasRenderedTime(peer_index)) { + reason = kSkipRenderedFrameReasonRendered; + } else if (frame_it->second.IsDropped(peer_index)) { + reason = kSkipRenderedFrameReasonDropped; + } + } + RTC_LOG(LS_WARNING) + << "Peer " << peer_name + << "; Received frame out of order: received frame with id " + << frame.id() << " which was " << reason << " before"; + return; + } + + // Find corresponding captured frame. + FrameInFlight* frame_in_flight = &frame_it->second; + absl::optional<VideoFrame> captured_frame = frame_in_flight->frame(); + + const size_t stream_index = frame_in_flight->stream(); + StreamState* state = &stream_states_.at(stream_index); + const InternalStatsKey stats_key(stream_index, state->sender(), peer_index); + + // Update frames counters. + frame_counters_.rendered++; + stream_frame_counters_.at(stats_key).rendered++; + + // Update current frame stats. + frame_in_flight->OnFrameRendered(peer_index, Now()); + + // After we received frame here we need to check if there are any dropped + // frames between this one and last one, that was rendered for this video + // stream. + int dropped_count = 0; + while (!state->IsEmpty(peer_index) && + state->Front(peer_index) != frame.id()) { + dropped_count++; + uint16_t dropped_frame_id = state->PopFront(peer_index); + // Frame with id `dropped_frame_id` was dropped. We need: + // 1. Update global and stream frame counters + // 2. Extract corresponding frame from `captured_frames_in_flight_` + // 3. Send extracted frame to comparison with dropped=true + // 4. Cleanup dropped frame + frame_counters_.dropped++; + stream_frame_counters_.at(stats_key).dropped++; + + auto dropped_frame_it = captured_frames_in_flight_.find(dropped_frame_id); + RTC_DCHECK(dropped_frame_it != captured_frames_in_flight_.end()); + dropped_frame_it->second.MarkDropped(peer_index); + + analyzer_stats_.frames_in_flight_left_count.AddSample( + StatsSample(captured_frames_in_flight_.size(), Now())); + frames_comparator_.AddComparison( + stats_key, /*captured=*/absl::nullopt, /*rendered=*/absl::nullopt, + FrameComparisonType::kDroppedFrame, + dropped_frame_it->second.GetStatsForPeer(peer_index)); + + if (dropped_frame_it->second.HaveAllPeersReceived()) { + captured_frames_in_flight_.erase(dropped_frame_it); + } + } + RTC_DCHECK(!state->IsEmpty(peer_index)); + state->PopFront(peer_index); + + if (state->last_rendered_frame_time(peer_index)) { + frame_in_flight->SetPrevFrameRenderedTime( + peer_index, state->last_rendered_frame_time(peer_index).value()); + } + state->SetLastRenderedFrameTime(peer_index, + frame_in_flight->rendered_time(peer_index)); + analyzer_stats_.frames_in_flight_left_count.AddSample( + StatsSample(captured_frames_in_flight_.size(), Now())); + frames_comparator_.AddComparison( + stats_key, dropped_count, captured_frame, /*rendered=*/frame, + FrameComparisonType::kRegular, + frame_in_flight->GetStatsForPeer(peer_index)); + + if (frame_it->second.HaveAllPeersReceived()) { + captured_frames_in_flight_.erase(frame_it); + } + + if (options_.report_infra_metrics) { + analyzer_stats_.on_frame_rendered_processing_time_ms.AddSample( + (Now() - processing_started).ms<double>()); + } +} + +void DefaultVideoQualityAnalyzer::OnEncoderError( + absl::string_view peer_name, + const webrtc::VideoFrame& frame, + int32_t error_code) { + RTC_LOG(LS_ERROR) << "Encoder error for frame.id=" << frame.id() + << ", code=" << error_code; +} + +void DefaultVideoQualityAnalyzer::OnDecoderError(absl::string_view peer_name, + uint16_t frame_id, + int32_t error_code, + const DecoderStats& stats) { + RTC_LOG(LS_ERROR) << "Decoder error for frame_id=" << frame_id + << ", code=" << error_code; + + Timestamp processing_started = Now(); + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + size_t peer_index = peers_->index(peer_name); + + if (frame_id == VideoFrame::kNotSetId) { + frame_counters_.failed_to_decode++; + unknown_sender_frame_counters_[std::string(peer_name)].failed_to_decode++; + return; + } + + auto it = captured_frames_in_flight_.find(frame_id); + if (it == captured_frames_in_flight_.end() || + it->second.HasDecodeEndTime(peer_index)) { + // It means this frame was decoded before, so we can skip it. It may happen + // when we have multiple simulcast streams in one track and received + // the same frame from two different streams because SFU can't reliably + // correlate two simulcast streams and started relaying the second stream + // from the same frame it has relayed right before for the first stream. + return; + } + frame_counters_.failed_to_decode++; + InternalStatsKey key(it->second.stream(), + stream_to_sender_.at(it->second.stream()), peer_index); + stream_frame_counters_.at(key).failed_to_decode++; + Timestamp now = Now(); + StreamCodecInfo used_decoder; + used_decoder.codec_name = stats.decoder_name; + used_decoder.first_frame_id = frame_id; + used_decoder.last_frame_id = frame_id; + used_decoder.switched_on_at = now; + used_decoder.switched_from_at = now; + it->second.OnDecoderError(peer_index, used_decoder); + + if (options_.report_infra_metrics) { + analyzer_stats_.on_decoder_error_processing_time_ms.AddSample( + (Now() - processing_started).ms<double>()); + } +} + +void DefaultVideoQualityAnalyzer::RegisterParticipantInCall( + absl::string_view peer_name) { + MutexLock lock(&mutex_); + RTC_CHECK(!peers_->HasName(peer_name)); + size_t new_peer_index = peers_->AddIfAbsent(peer_name); + + // Ensure stats for receiving (for frames from other peers to this one) + // streams exists. Since in flight frames will be sent to the new peer + // as well. Sending stats (from this peer to others) will be added by + // DefaultVideoQualityAnalyzer::OnFrameCaptured. + std::vector<std::pair<InternalStatsKey, Timestamp>> stream_started_time; + for (auto [stream_index, sender_peer_index] : stream_to_sender_) { + InternalStatsKey key(stream_index, sender_peer_index, new_peer_index); + + // To initiate `FrameCounters` for the stream we should pick frame + // counters with the same stream index and the same sender's peer index + // and any receiver's peer index and copy from its sender side + // counters. + FrameCounters counters; + for (size_t i : peers_->GetPresentIndexes()) { + InternalStatsKey prototype_key(stream_index, sender_peer_index, i); + auto it = stream_frame_counters_.find(prototype_key); + if (it != stream_frame_counters_.end()) { + counters.captured = it->second.captured; + counters.pre_encoded = it->second.pre_encoded; + counters.encoded = it->second.encoded; + break; + } + } + // It may happen if we had only one peer before this method was invoked, + // then `counters` will be empty. In such case empty `counters` are ok. + stream_frame_counters_.insert({key, std::move(counters)}); + + stream_started_time.push_back( + {key, stream_states_.at(stream_index).stream_started_time()}); + } + frames_comparator_.RegisterParticipantInCall(stream_started_time, + start_time_); + // Ensure, that frames states are handled correctly + // (e.g. dropped frames tracking). + for (auto& [stream_index, stream_state] : stream_states_) { + stream_state.AddPeer(new_peer_index); + } + // Register new peer for every frame in flight. + // It is guaranteed, that no garbage FrameInFlight objects will stay in + // memory because of adding new peer. Even if the new peer won't receive the + // frame, the frame will be removed by OnFrameRendered after next frame comes + // for the new peer. It is important because FrameInFlight is a large object. + for (auto& [frame_id, frame_in_flight] : captured_frames_in_flight_) { + frame_in_flight.AddExpectedReceiver(new_peer_index); + } +} + +void DefaultVideoQualityAnalyzer::UnregisterParticipantInCall( + absl::string_view peer_name) { + MutexLock lock(&mutex_); + RTC_CHECK(peers_->HasName(peer_name)); + absl::optional<size_t> peer_index = peers_->RemoveIfPresent(peer_name); + RTC_CHECK(peer_index.has_value()); + + for (auto& [stream_index, stream_state] : stream_states_) { + if (!options_.enable_receive_own_stream && + peer_index == stream_state.sender()) { + continue; + } + + AddExistingFramesInFlightForStreamToComparator(stream_index, stream_state, + *peer_index); + + stream_state.RemovePeer(*peer_index); + } + + // Remove peer from every frame in flight. If we removed that last expected + // receiver for the frame, then we should removed this frame if it was + // already encoded. If frame wasn't encoded, it still will be used by sender + // side pipeline, so we can't delete it yet. + for (auto it = captured_frames_in_flight_.begin(); + it != captured_frames_in_flight_.end();) { + FrameInFlight& frame_in_flight = it->second; + frame_in_flight.RemoveExpectedReceiver(*peer_index); + // If frame was fully sent and all receivers received it, then erase it. + // It may happen that when we remove FrameInFlight only some Simulcast/SVC + // layers were encoded and frame has encoded time, but more layers might be + // encoded after removal. In such case it's safe to still remove a frame, + // because OnFrameEncoded method will correctly handle the case when there + // is no FrameInFlight for the received encoded image. + if (frame_in_flight.HasEncodedTime() && + frame_in_flight.HaveAllPeersReceived()) { + it = captured_frames_in_flight_.erase(it); + } else { + it++; + } + } +} + +void DefaultVideoQualityAnalyzer::Stop() { + std::map<InternalStatsKey, Timestamp> last_rendered_frame_times; + { + MutexLock lock(&mutex_); + if (state_ == State::kStopped) { + return; + } + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + state_ = State::kStopped; + + // Add the amount of frames in flight to the analyzer stats before all left + // frames in flight will be sent to the `frames_compartor_`. + analyzer_stats_.frames_in_flight_left_count.AddSample( + StatsSample(captured_frames_in_flight_.size(), Now())); + + for (auto& state_entry : stream_states_) { + const size_t stream_index = state_entry.first; + StreamState& stream_state = state_entry.second; + + // Populate `last_rendered_frame_times` map for all peers that were met in + // call, not only for the currently presented ones. + for (size_t peer_index : peers_->GetAllIndexes()) { + if (peer_index == stream_state.sender() && + !options_.enable_receive_own_stream) { + continue; + } + + InternalStatsKey stats_key(stream_index, stream_state.sender(), + peer_index); + + // If there are no freezes in the call we have to report + // time_between_freezes_ms as call duration and in such case + // `stream_last_freeze_end_time` for this stream will be `start_time_`. + // If there is freeze, then we need add time from last rendered frame + // to last freeze end as time between freezes. + if (stream_state.last_rendered_frame_time(peer_index)) { + last_rendered_frame_times.emplace( + stats_key, + stream_state.last_rendered_frame_time(peer_index).value()); + } + } + + // Push left frame in flight for analysis for the peers that are still in + // the call. + for (size_t peer_index : peers_->GetPresentIndexes()) { + if (peer_index == stream_state.sender() && + !options_.enable_receive_own_stream) { + continue; + } + + AddExistingFramesInFlightForStreamToComparator( + stream_index, stream_state, peer_index); + } + } + } + frames_comparator_.Stop(last_rendered_frame_times); + + // Perform final Metrics update. On this place analyzer is stopped and no one + // holds any locks. + { + MutexLock lock(&mutex_); + FramesComparatorStats frames_comparator_stats = + frames_comparator_.frames_comparator_stats(); + analyzer_stats_.comparisons_queue_size = + std::move(frames_comparator_stats.comparisons_queue_size); + analyzer_stats_.comparisons_done = frames_comparator_stats.comparisons_done; + analyzer_stats_.cpu_overloaded_comparisons_done = + frames_comparator_stats.cpu_overloaded_comparisons_done; + analyzer_stats_.memory_overloaded_comparisons_done = + frames_comparator_stats.memory_overloaded_comparisons_done; + } + ReportResults(); +} + +std::string DefaultVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) { + MutexLock lock1(&mutex_); + auto it = captured_frames_in_flight_.find(frame_id); + if (it != captured_frames_in_flight_.end()) { + return streams_.name(it->second.stream()); + } + for (auto hist_it = stream_to_frame_id_history_.begin(); + hist_it != stream_to_frame_id_history_.end(); ++hist_it) { + auto hist_set_it = hist_it->second.find(frame_id); + if (hist_set_it != hist_it->second.end()) { + return streams_.name(hist_it->first); + } + } + RTC_CHECK(false) << "Unknown frame_id=" << frame_id; +} + +std::set<StatsKey> DefaultVideoQualityAnalyzer::GetKnownVideoStreams() const { + MutexLock lock(&mutex_); + std::set<StatsKey> out; + for (auto& item : frames_comparator_.stream_stats()) { + RTC_LOG(LS_INFO) << item.first.ToString() << " ==> " + << ToStatsKey(item.first).ToString(); + out.insert(ToStatsKey(item.first)); + } + return out; +} + +VideoStreamsInfo DefaultVideoQualityAnalyzer::GetKnownStreams() const { + MutexLock lock(&mutex_); + std::map<std::string, std::string> stream_to_sender; + std::map<std::string, std::set<std::string>> sender_to_streams; + std::map<std::string, std::set<std::string>> stream_to_receivers; + + for (auto& item : frames_comparator_.stream_stats()) { + const std::string& stream_label = streams_.name(item.first.stream); + const std::string& sender = peers_->name(item.first.sender); + const std::string& receiver = peers_->name(item.first.receiver); + RTC_LOG(LS_INFO) << item.first.ToString() << " ==> " + << "stream=" << stream_label << "; sender=" << sender + << "; receiver=" << receiver; + stream_to_sender.emplace(stream_label, sender); + auto streams_it = sender_to_streams.find(sender); + if (streams_it != sender_to_streams.end()) { + streams_it->second.emplace(stream_label); + } else { + sender_to_streams.emplace(sender, std::set<std::string>{stream_label}); + } + auto receivers_it = stream_to_receivers.find(stream_label); + if (receivers_it != stream_to_receivers.end()) { + receivers_it->second.emplace(receiver); + } else { + stream_to_receivers.emplace(stream_label, + std::set<std::string>{receiver}); + } + } + + return VideoStreamsInfo(std::move(stream_to_sender), + std::move(sender_to_streams), + std::move(stream_to_receivers)); +} + +FrameCounters DefaultVideoQualityAnalyzer::GetGlobalCounters() const { + MutexLock lock(&mutex_); + return frame_counters_; +} + +std::map<std::string, FrameCounters> +DefaultVideoQualityAnalyzer::GetUnknownSenderFrameCounters() const { + MutexLock lock(&mutex_); + return unknown_sender_frame_counters_; +} + +std::map<StatsKey, FrameCounters> +DefaultVideoQualityAnalyzer::GetPerStreamCounters() const { + MutexLock lock(&mutex_); + std::map<StatsKey, FrameCounters> out; + for (auto& item : stream_frame_counters_) { + out.emplace(ToStatsKey(item.first), item.second); + } + return out; +} + +std::map<StatsKey, StreamStats> DefaultVideoQualityAnalyzer::GetStats() const { + MutexLock lock1(&mutex_); + std::map<StatsKey, StreamStats> out; + for (auto& item : frames_comparator_.stream_stats()) { + out.emplace(ToStatsKey(item.first), item.second); + } + return out; +} + +AnalyzerStats DefaultVideoQualityAnalyzer::GetAnalyzerStats() const { + MutexLock lock(&mutex_); + return analyzer_stats_; +} + +uint16_t DefaultVideoQualityAnalyzer::GetNextFrameId() { + uint16_t frame_id = next_frame_id_++; + if (next_frame_id_ == VideoFrame::kNotSetId) { + next_frame_id_ = 1; + } + return frame_id; +} + +void DefaultVideoQualityAnalyzer:: + AddExistingFramesInFlightForStreamToComparator(size_t stream_index, + StreamState& stream_state, + size_t peer_index) { + InternalStatsKey stats_key(stream_index, stream_state.sender(), peer_index); + + // Add frames in flight for this stream into frames comparator. + // Frames in flight were not rendered, so they won't affect stream's + // last rendered frame time. + while (!stream_state.IsEmpty(peer_index)) { + uint16_t frame_id = stream_state.PopFront(peer_index); + auto it = captured_frames_in_flight_.find(frame_id); + RTC_DCHECK(it != captured_frames_in_flight_.end()); + FrameInFlight& frame = it->second; + + frames_comparator_.AddComparison(stats_key, /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, + frame.GetStatsForPeer(peer_index)); + } +} + +void DefaultVideoQualityAnalyzer::ReportResults() { + MutexLock lock(&mutex_); + for (auto& item : frames_comparator_.stream_stats()) { + ReportResults(item.first, item.second, + stream_frame_counters_.at(item.first)); + } + // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey. + metrics_logger_->LogSingleValueMetric( + "cpu_usage_%", test_label_, GetCpuUsagePercent(), Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter, + {{MetricMetadataKey::kExperimentalTestNameMetadataKey, test_label_}}); + LogFrameCounters("Global", frame_counters_); + if (!unknown_sender_frame_counters_.empty()) { + RTC_LOG(LS_INFO) << "Received frame counters with unknown frame id:"; + for (const auto& [peer_name, frame_counters] : + unknown_sender_frame_counters_) { + LogFrameCounters(peer_name, frame_counters); + } + } + RTC_LOG(LS_INFO) << "Received frame counters per stream:"; + for (const auto& [stats_key, stream_stats] : + frames_comparator_.stream_stats()) { + LogFrameCounters(ToStatsKey(stats_key).ToString(), + stream_frame_counters_.at(stats_key)); + LogStreamInternalStats(ToStatsKey(stats_key).ToString(), stream_stats, + start_time_); + } + if (!analyzer_stats_.comparisons_queue_size.IsEmpty()) { + RTC_LOG(LS_INFO) << "comparisons_queue_size min=" + << analyzer_stats_.comparisons_queue_size.GetMin() + << "; max=" + << analyzer_stats_.comparisons_queue_size.GetMax() + << "; 99%=" + << analyzer_stats_.comparisons_queue_size.GetPercentile( + 0.99); + } + RTC_LOG(LS_INFO) << "comparisons_done=" << analyzer_stats_.comparisons_done; + RTC_LOG(LS_INFO) << "cpu_overloaded_comparisons_done=" + << analyzer_stats_.cpu_overloaded_comparisons_done; + RTC_LOG(LS_INFO) << "memory_overloaded_comparisons_done=" + << analyzer_stats_.memory_overloaded_comparisons_done; + if (options_.report_infra_metrics) { + metrics_logger_->LogMetric("comparisons_queue_size", test_label_, + analyzer_stats_.comparisons_queue_size, + Unit::kCount, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric("frames_in_flight_left_count", test_label_, + analyzer_stats_.frames_in_flight_left_count, + Unit::kCount, + ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogSingleValueMetric( + "comparisons_done", test_label_, analyzer_stats_.comparisons_done, + Unit::kCount, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "cpu_overloaded_comparisons_done", test_label_, + analyzer_stats_.cpu_overloaded_comparisons_done, Unit::kCount, + ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "memory_overloaded_comparisons_done", test_label_, + analyzer_stats_.memory_overloaded_comparisons_done, Unit::kCount, + ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "test_duration", test_label_, (Now() - start_time_).ms(), + Unit::kMilliseconds, ImprovementDirection::kNeitherIsBetter); + + metrics_logger_->LogMetric( + "on_frame_captured_processing_time_ms", test_label_, + analyzer_stats_.on_frame_captured_processing_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric( + "on_frame_pre_encode_processing_time_ms", test_label_, + analyzer_stats_.on_frame_pre_encode_processing_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric( + "on_frame_encoded_processing_time_ms", test_label_, + analyzer_stats_.on_frame_encoded_processing_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric( + "on_frame_pre_decode_processing_time_ms", test_label_, + analyzer_stats_.on_frame_pre_decode_processing_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric( + "on_frame_decoded_processing_time_ms", test_label_, + analyzer_stats_.on_frame_decoded_processing_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric( + "on_frame_rendered_processing_time_ms", test_label_, + analyzer_stats_.on_frame_rendered_processing_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); + metrics_logger_->LogMetric( + "on_decoder_error_processing_time_ms", test_label_, + analyzer_stats_.on_decoder_error_processing_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter); + } +} + +void DefaultVideoQualityAnalyzer::ReportResults( + const InternalStatsKey& key, + const StreamStats& stats, + const FrameCounters& frame_counters) { + TimeDelta test_duration = Now() - start_time_; + std::string test_case_name = GetTestCaseName(ToMetricName(key)); + // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey. + std::map<std::string, std::string> metric_metadata{ + {MetricMetadataKey::kPeerMetadataKey, peers_->name(key.sender)}, + {MetricMetadataKey::kVideoStreamMetadataKey, streams_.name(key.stream)}, + {MetricMetadataKey::kSenderMetadataKey, peers_->name(key.sender)}, + {MetricMetadataKey::kReceiverMetadataKey, peers_->name(key.receiver)}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_label_}}; + + double sum_squared_interframe_delays_secs = 0; + Timestamp video_start_time = Timestamp::PlusInfinity(); + Timestamp video_end_time = Timestamp::MinusInfinity(); + for (const SamplesStatsCounter::StatsSample& sample : + stats.time_between_rendered_frames_ms.GetTimedSamples()) { + double interframe_delay_ms = sample.value; + const double interframe_delays_secs = interframe_delay_ms / 1000.0; + // Sum of squared inter frame intervals is used to calculate the harmonic + // frame rate metric. The metric aims to reflect overall experience related + // to smoothness of video playback and includes both freezes and pauses. + sum_squared_interframe_delays_secs += + interframe_delays_secs * interframe_delays_secs; + if (sample.time < video_start_time) { + video_start_time = sample.time; + } + if (sample.time > video_end_time) { + video_end_time = sample.time; + } + } + double harmonic_framerate_fps = 0; + TimeDelta video_duration = video_end_time - video_start_time; + if (sum_squared_interframe_delays_secs > 0.0 && video_duration.IsFinite()) { + harmonic_framerate_fps = + video_duration.seconds<double>() / sum_squared_interframe_delays_secs; + } + + metrics_logger_->LogMetric( + "psnr_dB", test_case_name, stats.psnr, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "ssim", test_case_name, stats.ssim, Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogMetric("transport_time", test_case_name, + stats.transport_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, + metric_metadata); + metrics_logger_->LogMetric( + "total_delay_incl_transport", test_case_name, + stats.total_delay_incl_transport_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "time_between_rendered_frames", test_case_name, + stats.time_between_rendered_frames_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "harmonic_framerate", test_case_name, harmonic_framerate_fps, + Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "encode_frame_rate", test_case_name, + stats.encode_frame_rate.IsEmpty() + ? 0 + : stats.encode_frame_rate.GetEventsPerSecond(), + Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "encode_time", test_case_name, stats.encode_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogMetric("time_between_freezes", test_case_name, + stats.time_between_freezes_ms, Unit::kMilliseconds, + ImprovementDirection::kBiggerIsBetter, + metric_metadata); + metrics_logger_->LogMetric("freeze_time_ms", test_case_name, + stats.freeze_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, + metric_metadata); + metrics_logger_->LogMetric( + "pixels_per_frame", test_case_name, stats.resolution_of_decoded_frame, + Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "min_psnr_dB", test_case_name, + stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), Unit::kUnitless, + ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "decode_time", test_case_name, stats.decode_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "receive_to_render_time", test_case_name, stats.receive_to_render_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter, + metric_metadata); + metrics_logger_->LogSingleValueMetric( + "dropped_frames", test_case_name, frame_counters.dropped, Unit::kCount, + ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "frames_in_flight", test_case_name, + frame_counters.captured - frame_counters.rendered - + frame_counters.dropped, + Unit::kCount, ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "rendered_frames", test_case_name, frame_counters.rendered, Unit::kCount, + ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "max_skipped", test_case_name, stats.skipped_between_rendered, + Unit::kCount, ImprovementDirection::kSmallerIsBetter, metric_metadata); + metrics_logger_->LogMetric( + "target_encode_bitrate", test_case_name, + stats.target_encode_bitrate / 1000, Unit::kKilobitsPerSecond, + ImprovementDirection::kNeitherIsBetter, metric_metadata); + for (const auto& [spatial_layer, qp] : stats.spatial_layers_qp) { + std::map<std::string, std::string> qp_metadata = metric_metadata; + qp_metadata[MetricMetadataKey::kSpatialLayerMetadataKey] = + std::to_string(spatial_layer); + metrics_logger_->LogMetric("qp_sl" + std::to_string(spatial_layer), + test_case_name, qp, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter, + std::move(qp_metadata)); + } + metrics_logger_->LogSingleValueMetric( + "actual_encode_bitrate", test_case_name, + static_cast<double>(stats.total_encoded_images_payload) / + test_duration.seconds<double>() * kBitsInByte / 1000, + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter, + metric_metadata); + + if (options_.report_detailed_frame_stats) { + metrics_logger_->LogSingleValueMetric( + "capture_frame_rate", test_case_name, + stats.capture_frame_rate.IsEmpty() + ? 0 + : stats.capture_frame_rate.GetEventsPerSecond(), + Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "num_encoded_frames", test_case_name, frame_counters.encoded, + Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "num_decoded_frames", test_case_name, frame_counters.decoded, + Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "num_send_key_frames", test_case_name, stats.num_send_key_frames, + Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "num_recv_key_frames", test_case_name, stats.num_recv_key_frames, + Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata); + + metrics_logger_->LogMetric("recv_key_frame_size_bytes", test_case_name, + stats.recv_key_frame_size_bytes, Unit::kCount, + ImprovementDirection::kBiggerIsBetter, + metric_metadata); + metrics_logger_->LogMetric("recv_delta_frame_size_bytes", test_case_name, + stats.recv_delta_frame_size_bytes, Unit::kCount, + ImprovementDirection::kBiggerIsBetter, + metric_metadata); + } +} + +std::string DefaultVideoQualityAnalyzer::GetTestCaseName( + const std::string& stream_label) const { + return test_label_ + "/" + stream_label; +} + +Timestamp DefaultVideoQualityAnalyzer::Now() { + return clock_->CurrentTime(); +} + +StatsKey DefaultVideoQualityAnalyzer::ToStatsKey( + const InternalStatsKey& key) const { + return StatsKey(streams_.name(key.stream), peers_->name(key.receiver)); +} + +std::string DefaultVideoQualityAnalyzer::ToMetricName( + const InternalStatsKey& key) const { + const std::string& stream_label = streams_.name(key.stream); + if (peers_->GetKnownSize() <= 2 && key.sender != key.receiver) { + // TODO(titovartem): remove this special case. + return stream_label; + } + rtc::StringBuilder out; + out << stream_label << "_" << peers_->name(key.sender) << "_" + << peers_->name(key.receiver); + return out.str(); +} + +double DefaultVideoQualityAnalyzer::GetCpuUsagePercent() { + return cpu_measurer_.GetCpuUsagePercent(); +} + +std::map<std::string, std::vector<uint16_t>> +DefaultVideoQualityAnalyzer::GetStreamFrames() const { + MutexLock lock(&mutex_); + std::map<std::string, std::vector<uint16_t>> out; + for (auto entry_it : stream_to_frame_id_full_history_) { + out.insert({streams_.name(entry_it.first), entry_it.second}); + } + return out; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h new file mode 100644 index 0000000000..b67e5a0147 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_H_ + +#include <atomic> +#include <cstdint> +#include <deque> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <vector> + +#include "api/array_view.h" +#include "api/test/metrics/metrics_logger.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h" +#include "test/pc/e2e/analyzer/video/names_collection.h" + +namespace webrtc { + +class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { + public: + DefaultVideoQualityAnalyzer(webrtc::Clock* clock, + test::MetricsLogger* metrics_logger, + DefaultVideoQualityAnalyzerOptions options = {}); + ~DefaultVideoQualityAnalyzer() override; + + void Start(std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count) override; + uint16_t OnFrameCaptured(absl::string_view peer_name, + const std::string& stream_label, + const VideoFrame& frame) override; + void OnFramePreEncode(absl::string_view peer_name, + const VideoFrame& frame) override; + void OnFrameEncoded(absl::string_view peer_name, + uint16_t frame_id, + const EncodedImage& encoded_image, + const EncoderStats& stats, + bool discarded) override; + void OnFrameDropped(absl::string_view peer_name, + EncodedImageCallback::DropReason reason) override; + void OnFramePreDecode(absl::string_view peer_name, + uint16_t frame_id, + const EncodedImage& input_image) override; + void OnFrameDecoded(absl::string_view peer_name, + const VideoFrame& frame, + const DecoderStats& stats) override; + void OnFrameRendered(absl::string_view peer_name, + const VideoFrame& frame) override; + void OnEncoderError(absl::string_view peer_name, + const VideoFrame& frame, + int32_t error_code) override; + void OnDecoderError(absl::string_view peer_name, + uint16_t frame_id, + int32_t error_code, + const DecoderStats& stats) override; + + void RegisterParticipantInCall(absl::string_view peer_name) override; + void UnregisterParticipantInCall(absl::string_view peer_name) override; + + void Stop() override; + std::string GetStreamLabel(uint16_t frame_id) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override {} + + // Returns set of stream labels, that were met during test call. + std::set<StatsKey> GetKnownVideoStreams() const; + VideoStreamsInfo GetKnownStreams() const; + FrameCounters GetGlobalCounters() const; + // Returns frame counter for frames received without frame id set. + std::map<std::string, FrameCounters> GetUnknownSenderFrameCounters() const; + // Returns frame counter per stream label. Valid stream labels can be obtained + // by calling GetKnownVideoStreams() + std::map<StatsKey, FrameCounters> GetPerStreamCounters() const; + // Returns video quality stats per stream label. Valid stream labels can be + // obtained by calling GetKnownVideoStreams() + std::map<StatsKey, StreamStats> GetStats() const; + AnalyzerStats GetAnalyzerStats() const; + double GetCpuUsagePercent(); + + // Returns mapping from the stream label to the history of frames that were + // met in this stream in the order as they were captured. + std::map<std::string, std::vector<uint16_t>> GetStreamFrames() const; + + private: + enum State { kNew, kActive, kStopped }; + + // Returns next frame id to use. Frame ID can't be `VideoFrame::kNotSetId`, + // because this value is reserved by `VideoFrame` as "ID not set". + uint16_t GetNextFrameId() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + void AddExistingFramesInFlightForStreamToComparator(size_t stream_index, + StreamState& stream_state, + size_t peer_index) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Report results for all metrics for all streams. + void ReportResults(); + void ReportResults(const InternalStatsKey& key, + const StreamStats& stats, + const FrameCounters& frame_counters) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + // Returns name of current test case for reporting. + std::string GetTestCaseName(const std::string& stream_label) const; + Timestamp Now(); + StatsKey ToStatsKey(const InternalStatsKey& key) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + // Returns string representation of stats key for metrics naming. Used for + // backward compatibility by metrics naming for 2 peers cases. + std::string ToMetricName(const InternalStatsKey& key) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + static const uint16_t kStartingFrameId = 1; + + const DefaultVideoQualityAnalyzerOptions options_; + webrtc::Clock* const clock_; + test::MetricsLogger* const metrics_logger_; + + std::string test_label_; + + mutable Mutex mutex_; + uint16_t next_frame_id_ RTC_GUARDED_BY(mutex_) = kStartingFrameId; + std::unique_ptr<NamesCollection> peers_ RTC_GUARDED_BY(mutex_); + State state_ RTC_GUARDED_BY(mutex_) = State::kNew; + Timestamp start_time_ RTC_GUARDED_BY(mutex_) = Timestamp::MinusInfinity(); + // Mapping from stream label to unique size_t value to use in stats and avoid + // extra string copying. + NamesCollection streams_ RTC_GUARDED_BY(mutex_); + // Frames that were captured by all streams and still aren't rendered on + // receivers or deemed dropped. Frame with id X can be removed from this map + // if: + // 1. The frame with id X was received in OnFrameRendered by all expected + // receivers. + // 2. The frame with id Y > X was received in OnFrameRendered by all expected + // receivers. + // 3. Next available frame id for newly captured frame is X + // 4. There too many frames in flight for current video stream and X is the + // oldest frame id in this stream. In such case only the frame content + // will be removed, but the map entry will be preserved. + std::map<uint16_t, FrameInFlight> captured_frames_in_flight_ + RTC_GUARDED_BY(mutex_); + // Global frames count for all video streams. + FrameCounters frame_counters_ RTC_GUARDED_BY(mutex_); + // Frame counters for received frames without video frame id set. + // Map from peer name to the frame counters. + std::map<std::string, FrameCounters> unknown_sender_frame_counters_ + RTC_GUARDED_BY(mutex_); + // Frame counters per each stream per each receiver. + std::map<InternalStatsKey, FrameCounters> stream_frame_counters_ + RTC_GUARDED_BY(mutex_); + // Map from stream index in `streams_` to its StreamState. + std::map<size_t, StreamState> stream_states_ RTC_GUARDED_BY(mutex_); + // Map from stream index in `streams_` to sender peer index in `peers_`. + std::map<size_t, size_t> stream_to_sender_ RTC_GUARDED_BY(mutex_); + + // Stores history mapping between stream index in `streams_` and frame ids. + // Updated when frame id overlap. It required to properly return stream label + // after 1st frame from simulcast streams was already rendered and last is + // still encoding. + std::map<size_t, std::set<uint16_t>> stream_to_frame_id_history_ + RTC_GUARDED_BY(mutex_); + // Map from stream index to the list of frames as they were met in the stream. + std::map<size_t, std::vector<uint16_t>> stream_to_frame_id_full_history_ + RTC_GUARDED_BY(mutex_); + AnalyzerStats analyzer_stats_ RTC_GUARDED_BY(mutex_); + + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer_; + DefaultVideoQualityAnalyzerFramesComparator frames_comparator_; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.cc new file mode 100644 index 0000000000..847c9f09a6 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.cc @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h" + +#include "rtc_base/cpu_time.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/system_time.h" + +namespace webrtc { + +void DefaultVideoQualityAnalyzerCpuMeasurer::StartMeasuringCpuProcessTime() { + MutexLock lock(&mutex_); + cpu_time_ -= rtc::GetProcessCpuTimeNanos(); + wallclock_time_ -= rtc::SystemTimeNanos(); +} + +void DefaultVideoQualityAnalyzerCpuMeasurer::StopMeasuringCpuProcessTime() { + MutexLock lock(&mutex_); + cpu_time_ += rtc::GetProcessCpuTimeNanos(); + wallclock_time_ += rtc::SystemTimeNanos(); +} + +void DefaultVideoQualityAnalyzerCpuMeasurer::StartExcludingCpuThreadTime() { + MutexLock lock(&mutex_); + cpu_time_ += rtc::GetThreadCpuTimeNanos(); +} + +void DefaultVideoQualityAnalyzerCpuMeasurer::StopExcludingCpuThreadTime() { + MutexLock lock(&mutex_); + cpu_time_ -= rtc::GetThreadCpuTimeNanos(); +} + +double DefaultVideoQualityAnalyzerCpuMeasurer::GetCpuUsagePercent() { + MutexLock lock(&mutex_); + return static_cast<double>(cpu_time_) / wallclock_time_ * 100.0; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h new file mode 100644 index 0000000000..dd9fa07af2 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_CPU_MEASURER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_CPU_MEASURER_H_ + +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { + +// This class is thread safe. +class DefaultVideoQualityAnalyzerCpuMeasurer { + public: + double GetCpuUsagePercent(); + + void StartMeasuringCpuProcessTime(); + void StopMeasuringCpuProcessTime(); + void StartExcludingCpuThreadTime(); + void StopExcludingCpuThreadTime(); + + private: + Mutex mutex_; + int64_t cpu_time_ RTC_GUARDED_BY(mutex_) = 0; + int64_t wallclock_time_ RTC_GUARDED_BY(mutex_) = 0; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_CPU_MEASURER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc new file mode 100644 index 0000000000..df34dadaf0 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h" + +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_type.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" + +namespace webrtc { +namespace { + +template <typename T> +absl::optional<T> MaybeGetValue(const std::map<size_t, T>& map, size_t key) { + auto it = map.find(key); + if (it == map.end()) { + return absl::nullopt; + } + return it->second; +} + +} // namespace + +FrameInFlight::FrameInFlight(size_t stream, + VideoFrame frame, + Timestamp captured_time, + std::set<size_t> expected_receivers) + : stream_(stream), + expected_receivers_(std::move(expected_receivers)), + frame_(std::move(frame)), + captured_time_(captured_time) {} + +bool FrameInFlight::RemoveFrame() { + if (!frame_) { + return false; + } + frame_ = absl::nullopt; + return true; +} + +void FrameInFlight::SetFrameId(uint16_t id) { + if (frame_) { + frame_->set_id(id); + } + frame_id_ = id; +} + +std::vector<size_t> FrameInFlight::GetPeersWhichDidntReceive() const { + std::vector<size_t> out; + for (size_t peer : expected_receivers_) { + auto it = receiver_stats_.find(peer); + if (it == receiver_stats_.end() || + (!it->second.dropped && it->second.rendered_time.IsInfinite())) { + out.push_back(peer); + } + } + return out; +} + +bool FrameInFlight::HaveAllPeersReceived() const { + for (size_t peer : expected_receivers_) { + auto it = receiver_stats_.find(peer); + if (it == receiver_stats_.end()) { + return false; + } + + if (!it->second.dropped && it->second.rendered_time.IsInfinite()) { + return false; + } + } + return true; +} + +void FrameInFlight::OnFrameEncoded(webrtc::Timestamp time, + VideoFrameType frame_type, + DataSize encoded_image_size, + uint32_t target_encode_bitrate, + int spatial_layer, + int qp, + StreamCodecInfo used_encoder) { + encoded_time_ = time; + frame_type_ = frame_type; + encoded_image_size_ = encoded_image_size; + target_encode_bitrate_ += target_encode_bitrate; + spatial_layers_qp_[spatial_layer].AddSample(SamplesStatsCounter::StatsSample{ + .value = static_cast<double>(qp), .time = time}); + // Update used encoder info. If simulcast/SVC is used, this method can + // be called multiple times, in such case we should preserve the value + // of `used_encoder_.switched_on_at` from the first invocation as the + // smallest one. + Timestamp encoder_switched_on_at = used_encoder_.has_value() + ? used_encoder_->switched_on_at + : Timestamp::PlusInfinity(); + RTC_DCHECK(used_encoder.switched_on_at.IsFinite()); + RTC_DCHECK(used_encoder.switched_from_at.IsFinite()); + used_encoder_ = used_encoder; + if (encoder_switched_on_at < used_encoder_->switched_on_at) { + used_encoder_->switched_on_at = encoder_switched_on_at; + } +} + +void FrameInFlight::OnFramePreDecode(size_t peer, + webrtc::Timestamp received_time, + webrtc::Timestamp decode_start_time, + VideoFrameType frame_type, + DataSize encoded_image_size) { + receiver_stats_[peer].received_time = received_time; + receiver_stats_[peer].decode_start_time = decode_start_time; + receiver_stats_[peer].frame_type = frame_type; + receiver_stats_[peer].encoded_image_size = encoded_image_size; +} + +bool FrameInFlight::HasReceivedTime(size_t peer) const { + auto it = receiver_stats_.find(peer); + if (it == receiver_stats_.end()) { + return false; + } + return it->second.received_time.IsFinite(); +} + +void FrameInFlight::OnFrameDecoded(size_t peer, + webrtc::Timestamp time, + int width, + int height, + const StreamCodecInfo& used_decoder) { + receiver_stats_[peer].decode_end_time = time; + receiver_stats_[peer].used_decoder = used_decoder; + receiver_stats_[peer].decoded_frame_width = width; + receiver_stats_[peer].decoded_frame_height = height; +} + +void FrameInFlight::OnDecoderError(size_t peer, + const StreamCodecInfo& used_decoder) { + receiver_stats_[peer].decoder_failed = true; + receiver_stats_[peer].used_decoder = used_decoder; +} + +bool FrameInFlight::HasDecodeEndTime(size_t peer) const { + auto it = receiver_stats_.find(peer); + if (it == receiver_stats_.end()) { + return false; + } + return it->second.decode_end_time.IsFinite(); +} + +void FrameInFlight::OnFrameRendered(size_t peer, webrtc::Timestamp time) { + receiver_stats_[peer].rendered_time = time; +} + +bool FrameInFlight::HasRenderedTime(size_t peer) const { + auto it = receiver_stats_.find(peer); + if (it == receiver_stats_.end()) { + return false; + } + return it->second.rendered_time.IsFinite(); +} + +bool FrameInFlight::IsDropped(size_t peer) const { + auto it = receiver_stats_.find(peer); + if (it == receiver_stats_.end()) { + return false; + } + return it->second.dropped; +} + +FrameStats FrameInFlight::GetStatsForPeer(size_t peer) const { + RTC_DCHECK_NE(frame_id_, VideoFrame::kNotSetId) + << "Frame id isn't initialized"; + FrameStats stats(frame_id_, captured_time_); + stats.pre_encode_time = pre_encode_time_; + stats.encoded_time = encoded_time_; + stats.target_encode_bitrate = target_encode_bitrate_; + stats.encoded_frame_type = frame_type_; + stats.encoded_image_size = encoded_image_size_; + stats.used_encoder = used_encoder_; + stats.spatial_layers_qp = spatial_layers_qp_; + + absl::optional<ReceiverFrameStats> receiver_stats = + MaybeGetValue<ReceiverFrameStats>(receiver_stats_, peer); + if (receiver_stats.has_value()) { + stats.received_time = receiver_stats->received_time; + stats.decode_start_time = receiver_stats->decode_start_time; + stats.decode_end_time = receiver_stats->decode_end_time; + stats.rendered_time = receiver_stats->rendered_time; + stats.prev_frame_rendered_time = receiver_stats->prev_frame_rendered_time; + stats.decoded_frame_width = receiver_stats->decoded_frame_width; + stats.decoded_frame_height = receiver_stats->decoded_frame_height; + stats.used_decoder = receiver_stats->used_decoder; + stats.pre_decoded_frame_type = receiver_stats->frame_type; + stats.pre_decoded_image_size = receiver_stats->encoded_image_size; + stats.decoder_failed = receiver_stats->decoder_failed; + } + return stats; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h new file mode 100644 index 0000000000..52a526d09b --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAME_IN_FLIGHT_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAME_IN_FLIGHT_H_ + +#include <map> +#include <set> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_type.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" + +namespace webrtc { + +struct ReceiverFrameStats { + // Time when last packet of a frame was received. + Timestamp received_time = Timestamp::MinusInfinity(); + Timestamp decode_start_time = Timestamp::MinusInfinity(); + Timestamp decode_end_time = Timestamp::MinusInfinity(); + Timestamp rendered_time = Timestamp::MinusInfinity(); + Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity(); + + // Type and encoded size of received frame. + VideoFrameType frame_type = VideoFrameType::kEmptyFrame; + DataSize encoded_image_size = DataSize::Bytes(0); + + absl::optional<int> decoded_frame_width = absl::nullopt; + absl::optional<int> decoded_frame_height = absl::nullopt; + + // Can be not set if frame was dropped in the network. + absl::optional<StreamCodecInfo> used_decoder = absl::nullopt; + + bool dropped = false; + bool decoder_failed = false; +}; + +// Represents a frame which was sent by sender and is currently on the way to +// multiple receivers. Some receivers may receive this frame and some don't. +// +// Contains all statistic associated with the frame and gathered in multiple +// points of the video pipeline. +// +// Internally may store the copy of the source frame which was sent. In such +// case this frame is "alive". +class FrameInFlight { + public: + FrameInFlight(size_t stream, + VideoFrame frame, + Timestamp captured_time, + std::set<size_t> expected_receivers); + + size_t stream() const { return stream_; } + // Returns internal copy of source `VideoFrame` or `absl::nullopt` if it was + // removed before. + const absl::optional<VideoFrame>& frame() const { return frame_; } + // Removes internal copy of the source `VideoFrame` to free up extra memory. + // Returns was frame removed or not. + bool RemoveFrame(); + void SetFrameId(uint16_t id); + + void AddExpectedReceiver(size_t peer) { expected_receivers_.insert(peer); } + + void RemoveExpectedReceiver(size_t peer) { expected_receivers_.erase(peer); } + + std::vector<size_t> GetPeersWhichDidntReceive() const; + + // Returns if all peers which were expected to receive this frame actually + // received it or not. + bool HaveAllPeersReceived() const; + + void SetPreEncodeTime(webrtc::Timestamp time) { pre_encode_time_ = time; } + + void OnFrameEncoded(webrtc::Timestamp time, + VideoFrameType frame_type, + DataSize encoded_image_size, + uint32_t target_encode_bitrate, + int spatial_layer, + int qp, + StreamCodecInfo used_encoder); + + bool HasEncodedTime() const { return encoded_time_.IsFinite(); } + + void OnFramePreDecode(size_t peer, + webrtc::Timestamp received_time, + webrtc::Timestamp decode_start_time, + VideoFrameType frame_type, + DataSize encoded_image_size); + + bool HasReceivedTime(size_t peer) const; + + void OnFrameDecoded(size_t peer, + webrtc::Timestamp time, + int width, + int height, + const StreamCodecInfo& used_decoder); + void OnDecoderError(size_t peer, const StreamCodecInfo& used_decoder); + + bool HasDecodeEndTime(size_t peer) const; + + void OnFrameRendered(size_t peer, webrtc::Timestamp time); + + bool HasRenderedTime(size_t peer) const; + + // Crash if rendered time is not set for specified `peer`. + webrtc::Timestamp rendered_time(size_t peer) const { + return receiver_stats_.at(peer).rendered_time; + } + + // Marks that frame was dropped and wasn't seen by particular `peer`. + void MarkDropped(size_t peer) { receiver_stats_[peer].dropped = true; } + bool IsDropped(size_t peer) const; + + void SetPrevFrameRenderedTime(size_t peer, webrtc::Timestamp time) { + receiver_stats_[peer].prev_frame_rendered_time = time; + } + + FrameStats GetStatsForPeer(size_t peer) const; + + private: + const size_t stream_; + // Set of peer's indexes who are expected to receive this frame. This is not + // the set of peer's indexes that received the frame. For example, if peer A + // was among expected receivers, it received frame and then left the call, A + // will be removed from this set, but the Stats for peer A still will be + // preserved in the FrameInFlight. + // + // This set is used to determine if this frame is expected to be received by + // any peer or can be safely deleted. It is responsibility of the user of this + // object to decide when it should be deleted. + std::set<size_t> expected_receivers_; + absl::optional<VideoFrame> frame_; + // Store frame id separately because `frame_` can be removed when we have too + // much memory consuption. + uint16_t frame_id_ = VideoFrame::kNotSetId; + + // Frame events timestamp. + Timestamp captured_time_; + Timestamp pre_encode_time_ = Timestamp::MinusInfinity(); + Timestamp encoded_time_ = Timestamp::MinusInfinity(); + // Type and encoded size of sent frame. + VideoFrameType frame_type_ = VideoFrameType::kEmptyFrame; + DataSize encoded_image_size_ = DataSize::Bytes(0); + uint32_t target_encode_bitrate_ = 0; + // Sender side qp values per spatial layer. In case when spatial layer is not + // set for `webrtc::EncodedImage`, 0 is used as default. + std::map<int, SamplesStatsCounter> spatial_layers_qp_; + // Can be not set if frame was dropped by encoder. + absl::optional<StreamCodecInfo> used_encoder_ = absl::nullopt; + // Map from the receiver peer's index to frame stats for that peer. + std::map<size_t, ReceiverFrameStats> receiver_stats_; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAME_IN_FLIGHT_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc new file mode 100644 index 0000000000..cbc0b7e8f3 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc @@ -0,0 +1,575 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" + +#include <algorithm> +#include <map> +#include <string> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/scoped_refptr.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame_type.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_base/checks.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_tools/frame_analyzer/video_geometry_aligner.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" +#include "test/pc/e2e/metric_metadata_keys.h" + +namespace webrtc { +namespace { + +using ::webrtc::webrtc_pc_e2e::SampleMetadataKey; + +constexpr TimeDelta kFreezeThreshold = TimeDelta::Millis(150); +constexpr int kMaxActiveComparisons = 10; + +SamplesStatsCounter::StatsSample StatsSample( + double value, + Timestamp sampling_time, + std::map<std::string, std::string> metadata) { + return SamplesStatsCounter::StatsSample{value, sampling_time, + std::move(metadata)}; +} + +SamplesStatsCounter::StatsSample StatsSample( + TimeDelta duration, + Timestamp sampling_time, + std::map<std::string, std::string> metadata) { + return SamplesStatsCounter::StatsSample{duration.ms<double>(), sampling_time, + std::move(metadata)}; +} + +FrameComparison ValidateFrameComparison(FrameComparison comparison) { + RTC_DCHECK(comparison.frame_stats.captured_time.IsFinite()) + << "Any comparison has to have finite captured_time"; + switch (comparison.type) { + case FrameComparisonType::kRegular: + // Regular comparison has to have all FrameStats filled in. + RTC_DCHECK(comparison.captured.has_value() || + comparison.overload_reason != OverloadReason::kNone) + << "Regular comparison has to have captured frame if it's not " + << "overloaded comparison"; + RTC_DCHECK(comparison.rendered.has_value() || + comparison.overload_reason != OverloadReason::kNone) + << "rendered frame has to be presented if it's not overloaded " + << "comparison"; + RTC_DCHECK(comparison.frame_stats.pre_encode_time.IsFinite()) + << "Regular comparison has to have finite pre_encode_time"; + RTC_DCHECK(comparison.frame_stats.encoded_time.IsFinite()) + << "Regular comparison has to have finite encoded_time"; + RTC_DCHECK(comparison.frame_stats.received_time.IsFinite()) + << "Regular comparison has to have finite received_time"; + RTC_DCHECK(comparison.frame_stats.decode_start_time.IsFinite()) + << "Regular comparison has to have finite decode_start_time"; + RTC_DCHECK(comparison.frame_stats.decode_end_time.IsFinite()) + << "Regular comparison has to have finite decode_end_time"; + RTC_DCHECK(comparison.frame_stats.rendered_time.IsFinite()) + << "Regular comparison has to have finite rendered_time"; + RTC_DCHECK(comparison.frame_stats.decoded_frame_width.has_value()) + << "Regular comparison has to have decoded_frame_width"; + RTC_DCHECK(comparison.frame_stats.decoded_frame_height.has_value()) + << "Regular comparison has to have decoded_frame_height"; + RTC_DCHECK(comparison.frame_stats.used_encoder.has_value()) + << "Regular comparison has to have used_encoder"; + RTC_DCHECK(comparison.frame_stats.used_decoder.has_value()) + << "Regular comparison has to have used_decoder"; + RTC_DCHECK(!comparison.frame_stats.decoder_failed) + << "Regular comparison can't have decoder failure"; + break; + case FrameComparisonType::kDroppedFrame: + // Frame can be dropped before encoder, by encoder, inside network or + // after decoder. + RTC_DCHECK(!comparison.captured.has_value()) + << "Dropped frame comparison can't have captured frame"; + RTC_DCHECK(!comparison.rendered.has_value()) + << "Dropped frame comparison can't have rendered frame"; + + if (comparison.frame_stats.encoded_time.IsFinite()) { + RTC_DCHECK(comparison.frame_stats.used_encoder.has_value()) + << "Dropped frame comparison has to have used_encoder when " + << "encoded_time is set"; + RTC_DCHECK(comparison.frame_stats.pre_encode_time.IsFinite()) + << "Dropped frame comparison has to have finite pre_encode_time " + << "when encoded_time is finite."; + } + + if (comparison.frame_stats.decode_end_time.IsFinite() || + comparison.frame_stats.decoder_failed) { + RTC_DCHECK(comparison.frame_stats.received_time.IsFinite()) + << "Dropped frame comparison has to have received_time when " + << "decode_end_time is set or decoder_failed is true"; + RTC_DCHECK(comparison.frame_stats.decode_start_time.IsFinite()) + << "Dropped frame comparison has to have decode_start_time when " + << "decode_end_time is set or decoder_failed is true"; + RTC_DCHECK(comparison.frame_stats.used_decoder.has_value()) + << "Dropped frame comparison has to have used_decoder when " + << "decode_end_time is set or decoder_failed is true"; + } else if (comparison.frame_stats.decode_end_time.IsFinite()) { + RTC_DCHECK(comparison.frame_stats.decoded_frame_width.has_value()) + << "Dropped frame comparison has to have decoded_frame_width when " + << "decode_end_time is set"; + RTC_DCHECK(comparison.frame_stats.decoded_frame_height.has_value()) + << "Dropped frame comparison has to have decoded_frame_height when " + << "decode_end_time is set"; + } + RTC_DCHECK(!comparison.frame_stats.rendered_time.IsFinite()) + << "Dropped frame comparison can't have rendered_time"; + break; + case FrameComparisonType::kFrameInFlight: + // Frame in flight comparison may miss almost any FrameStats, but if + // stats for stage X are set, then stats for stage X - 1 also has to be + // set. Also these frames were never rendered. + RTC_DCHECK(!comparison.captured.has_value()) + << "Frame in flight comparison can't have captured frame"; + RTC_DCHECK(!comparison.rendered.has_value()) + << "Frame in flight comparison can't have rendered frame"; + RTC_DCHECK(!comparison.frame_stats.rendered_time.IsFinite()) + << "Frame in flight comparison can't have rendered_time"; + + if (comparison.frame_stats.decode_end_time.IsFinite() || + comparison.frame_stats.decoder_failed) { + RTC_DCHECK(comparison.frame_stats.used_decoder.has_value()) + << "Frame in flight comparison has to have used_decoder when " + << "decode_end_time is set or decoder_failed is true."; + RTC_DCHECK(comparison.frame_stats.decode_start_time.IsFinite()) + << "Frame in flight comparison has to have finite " + << "decode_start_time when decode_end_time is finite or " + << "decoder_failed is true."; + } + if (comparison.frame_stats.decode_end_time.IsFinite()) { + RTC_DCHECK(comparison.frame_stats.decoded_frame_width.has_value()) + << "Frame in flight comparison has to have decoded_frame_width " + << "when decode_end_time is set."; + RTC_DCHECK(comparison.frame_stats.decoded_frame_height.has_value()) + << "Frame in flight comparison has to have decoded_frame_height " + << "when decode_end_time is set."; + } + if (comparison.frame_stats.decode_start_time.IsFinite()) { + RTC_DCHECK(comparison.frame_stats.received_time.IsFinite()) + << "Frame in flight comparison has to have finite received_time " + << "when decode_start_time is finite."; + } + if (comparison.frame_stats.received_time.IsFinite()) { + RTC_DCHECK(comparison.frame_stats.encoded_time.IsFinite()) + << "Frame in flight comparison has to have finite encoded_time " + << "when received_time is finite."; + } + if (comparison.frame_stats.encoded_time.IsFinite()) { + RTC_DCHECK(comparison.frame_stats.used_encoder.has_value()) + << "Frame in flight comparison has to have used_encoder when " + << "encoded_time is set"; + RTC_DCHECK(comparison.frame_stats.pre_encode_time.IsFinite()) + << "Frame in flight comparison has to have finite pre_encode_time " + << "when encoded_time is finite."; + } + break; + } + return comparison; +} + +} // namespace + +void DefaultVideoQualityAnalyzerFramesComparator::Start(int max_threads_count) { + for (int i = 0; i < max_threads_count; i++) { + thread_pool_.push_back(rtc::PlatformThread::SpawnJoinable( + [this] { ProcessComparisons(); }, + "DefaultVideoQualityAnalyzerFramesComparator-" + std::to_string(i))); + } + { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kNew) << "Frames comparator is already started"; + state_ = State::kActive; + } + cpu_measurer_.StartMeasuringCpuProcessTime(); +} + +void DefaultVideoQualityAnalyzerFramesComparator::Stop( + const std::map<InternalStatsKey, Timestamp>& last_rendered_frame_times) { + { + MutexLock lock(&mutex_); + if (state_ == State::kStopped) { + return; + } + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + state_ = State::kStopped; + } + cpu_measurer_.StopMeasuringCpuProcessTime(); + comparison_available_event_.Set(); + thread_pool_.clear(); + + { + MutexLock lock(&mutex_); + // Perform final Metrics update. On this place analyzer is stopped and no + // one holds any locks. + + // Time between freezes. + // Count time since the last freeze to the end of the call as time + // between freezes. + for (auto& entry : last_rendered_frame_times) { + const InternalStatsKey& stats_key = entry.first; + const Timestamp& last_rendered_frame_time = entry.second; + + // If there are no freezes in the call we have to report + // time_between_freezes_ms as call duration and in such case + // `last_rendered_frame_time` for this stream will be stream start time. + // If there is freeze, then we need add time from last rendered frame + // to last freeze end as time between freezes. + stream_stats_.at(stats_key).time_between_freezes_ms.AddSample(StatsSample( + last_rendered_frame_time - stream_last_freeze_end_time_.at(stats_key), + Now(), /*metadata=*/{})); + } + + // Freeze Time: + // If there were no freezes on a video stream, add only one sample with + // value 0 (0ms freezes time). + for (auto& [key, stream_stats] : stream_stats_) { + if (stream_stats.freeze_time_ms.IsEmpty()) { + stream_stats.freeze_time_ms.AddSample(0); + } + } + } +} + +void DefaultVideoQualityAnalyzerFramesComparator::EnsureStatsForStream( + size_t stream_index, + size_t sender_peer_index, + size_t peers_count, + Timestamp captured_time, + Timestamp start_time) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + + for (size_t i = 0; i < peers_count; ++i) { + if (i == sender_peer_index && !options_.enable_receive_own_stream) { + continue; + } + InternalStatsKey stats_key(stream_index, sender_peer_index, i); + if (stream_stats_.find(stats_key) == stream_stats_.end()) { + stream_stats_.insert({stats_key, StreamStats(captured_time)}); + // Assume that the first freeze was before first stream frame captured. + // This way time before the first freeze would be counted as time + // between freezes. + stream_last_freeze_end_time_.insert({stats_key, start_time}); + } else { + // When we see some `stream_label` for the first time we need to create + // stream stats object for it and set up some states, but we need to do + // it only once and for all receivers, so on the next frame on the same + // `stream_label` we can be sure, that it's already done and we needn't + // to scan though all peers again. + break; + } + } +} + +void DefaultVideoQualityAnalyzerFramesComparator::RegisterParticipantInCall( + rtc::ArrayView<std::pair<InternalStatsKey, Timestamp>> stream_started_time, + Timestamp start_time) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + + for (const std::pair<InternalStatsKey, Timestamp>& pair : + stream_started_time) { + stream_stats_.insert({pair.first, StreamStats(pair.second)}); + stream_last_freeze_end_time_.insert({pair.first, start_time}); + } +} + +void DefaultVideoQualityAnalyzerFramesComparator::AddComparison( + InternalStatsKey stats_key, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + AddComparisonInternal(std::move(stats_key), std::move(captured), + std::move(rendered), type, std::move(frame_stats)); +} + +void DefaultVideoQualityAnalyzerFramesComparator::AddComparison( + InternalStatsKey stats_key, + int skipped_between_rendered, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + stream_stats_.at(stats_key).skipped_between_rendered.AddSample( + StatsSample(skipped_between_rendered, Now(), + /*metadata=*/ + {{SampleMetadataKey::kFrameIdMetadataKey, + std::to_string(frame_stats.frame_id)}})); + AddComparisonInternal(std::move(stats_key), std::move(captured), + std::move(rendered), type, std::move(frame_stats)); +} + +void DefaultVideoQualityAnalyzerFramesComparator::AddComparisonInternal( + InternalStatsKey stats_key, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats) { + cpu_measurer_.StartExcludingCpuThreadTime(); + frames_comparator_stats_.comparisons_queue_size.AddSample( + StatsSample(comparisons_.size(), Now(), /*metadata=*/{})); + // If there too many computations waiting in the queue, we won't provide + // frames itself to make future computations lighter. + if (comparisons_.size() >= kMaxActiveComparisons) { + comparisons_.emplace_back(ValidateFrameComparison( + FrameComparison(std::move(stats_key), /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, type, + std::move(frame_stats), OverloadReason::kCpu))); + } else { + OverloadReason overload_reason = OverloadReason::kNone; + if (!captured && type == FrameComparisonType::kRegular) { + overload_reason = OverloadReason::kMemory; + } + comparisons_.emplace_back(ValidateFrameComparison(FrameComparison( + std::move(stats_key), std::move(captured), std::move(rendered), type, + std::move(frame_stats), overload_reason))); + } + comparison_available_event_.Set(); + cpu_measurer_.StopExcludingCpuThreadTime(); +} + +void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparisons() { + while (true) { + // Try to pick next comparison to perform from the queue. + absl::optional<FrameComparison> comparison = absl::nullopt; + bool more_new_comparisons_expected; + { + MutexLock lock(&mutex_); + if (!comparisons_.empty()) { + comparison = comparisons_.front(); + comparisons_.pop_front(); + if (!comparisons_.empty()) { + comparison_available_event_.Set(); + } + } + // If state is stopped => no new frame comparisons are expected. + more_new_comparisons_expected = state_ != State::kStopped; + } + if (!comparison) { + if (!more_new_comparisons_expected) { + comparison_available_event_.Set(); + return; + } + comparison_available_event_.Wait(TimeDelta::Seconds(1)); + continue; + } + + cpu_measurer_.StartExcludingCpuThreadTime(); + ProcessComparison(comparison.value()); + cpu_measurer_.StopExcludingCpuThreadTime(); + } +} + +void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison( + const FrameComparison& comparison) { + // Comparison is checked to be valid before adding, so we can use this + // assumptions during computations. + + // Perform expensive psnr and ssim calculations while not holding lock. + double psnr = -1.0; + double ssim = -1.0; + if ((options_.compute_psnr || options_.compute_ssim) && + comparison.captured.has_value() && comparison.rendered.has_value()) { + rtc::scoped_refptr<I420BufferInterface> reference_buffer = + comparison.captured->video_frame_buffer()->ToI420(); + rtc::scoped_refptr<I420BufferInterface> test_buffer = + comparison.rendered->video_frame_buffer()->ToI420(); + if (options_.adjust_cropping_before_comparing_frames) { + test_buffer = ScaleVideoFrameBuffer( + *test_buffer, reference_buffer->width(), reference_buffer->height()); + reference_buffer = test::AdjustCropping(reference_buffer, test_buffer); + } + if (options_.compute_psnr) { + psnr = options_.use_weighted_psnr + ? I420WeightedPSNR(*reference_buffer, *test_buffer) + : I420PSNR(*reference_buffer, *test_buffer); + } + if (options_.compute_ssim) { + ssim = I420SSIM(*reference_buffer, *test_buffer); + } + } + + const FrameStats& frame_stats = comparison.frame_stats; + + MutexLock lock(&mutex_); + auto stats_it = stream_stats_.find(comparison.stats_key); + RTC_CHECK(stats_it != stream_stats_.end()) << comparison.stats_key.ToString(); + StreamStats* stats = &stats_it->second; + + frames_comparator_stats_.comparisons_done++; + if (comparison.overload_reason == OverloadReason::kCpu) { + frames_comparator_stats_.cpu_overloaded_comparisons_done++; + } else if (comparison.overload_reason == OverloadReason::kMemory) { + frames_comparator_stats_.memory_overloaded_comparisons_done++; + } + + std::map<std::string, std::string> metadata; + metadata.emplace(SampleMetadataKey::kFrameIdMetadataKey, + std::to_string(frame_stats.frame_id)); + + if (psnr > 0) { + stats->psnr.AddSample( + StatsSample(psnr, frame_stats.rendered_time, metadata)); + } + if (ssim > 0) { + stats->ssim.AddSample( + StatsSample(ssim, frame_stats.received_time, metadata)); + } + stats->capture_frame_rate.AddEvent(frame_stats.captured_time); + + // Compute dropped phase for dropped frame + if (comparison.type == FrameComparisonType::kDroppedFrame) { + FrameDropPhase dropped_phase; + if (frame_stats.decode_end_time.IsFinite()) { + dropped_phase = FrameDropPhase::kAfterDecoder; + } else if (frame_stats.decode_start_time.IsFinite()) { + dropped_phase = FrameDropPhase::kByDecoder; + } else if (frame_stats.encoded_time.IsFinite()) { + dropped_phase = FrameDropPhase::kTransport; + } else if (frame_stats.pre_encode_time.IsFinite()) { + dropped_phase = FrameDropPhase::kByEncoder; + } else { + dropped_phase = FrameDropPhase::kBeforeEncoder; + } + stats->dropped_by_phase[dropped_phase]++; + } + + if (frame_stats.encoded_time.IsFinite()) { + stats->encode_time_ms.AddSample( + StatsSample(frame_stats.encoded_time - frame_stats.pre_encode_time, + frame_stats.encoded_time, metadata)); + stats->encode_frame_rate.AddEvent(frame_stats.encoded_time); + stats->total_encoded_images_payload += + frame_stats.encoded_image_size.bytes(); + stats->target_encode_bitrate.AddSample(StatsSample( + frame_stats.target_encode_bitrate, frame_stats.encoded_time, metadata)); + for (const auto& [spatial_layer, qp_values] : + frame_stats.spatial_layers_qp) { + for (SamplesStatsCounter::StatsSample qp : qp_values.GetTimedSamples()) { + qp.metadata = metadata; + stats->spatial_layers_qp[spatial_layer].AddSample(std::move(qp)); + } + } + + // Stats sliced on encoded frame type. + if (frame_stats.encoded_frame_type == VideoFrameType::kVideoFrameKey) { + ++stats->num_send_key_frames; + } + } + // Next stats can be calculated only if frame was received on remote side. + if (comparison.type != FrameComparisonType::kDroppedFrame || + comparison.frame_stats.decoder_failed) { + if (frame_stats.rendered_time.IsFinite()) { + stats->total_delay_incl_transport_ms.AddSample( + StatsSample(frame_stats.rendered_time - frame_stats.captured_time, + frame_stats.received_time, metadata)); + stats->receive_to_render_time_ms.AddSample( + StatsSample(frame_stats.rendered_time - frame_stats.received_time, + frame_stats.rendered_time, metadata)); + } + if (frame_stats.decode_start_time.IsFinite()) { + stats->transport_time_ms.AddSample( + StatsSample(frame_stats.decode_start_time - frame_stats.encoded_time, + frame_stats.decode_start_time, metadata)); + + // Stats sliced on decoded frame type. + if (frame_stats.pre_decoded_frame_type == + VideoFrameType::kVideoFrameKey) { + ++stats->num_recv_key_frames; + stats->recv_key_frame_size_bytes.AddSample( + StatsSample(frame_stats.pre_decoded_image_size.bytes(), + frame_stats.decode_start_time, metadata)); + } else if (frame_stats.pre_decoded_frame_type == + VideoFrameType::kVideoFrameDelta) { + stats->recv_delta_frame_size_bytes.AddSample( + StatsSample(frame_stats.pre_decoded_image_size.bytes(), + frame_stats.decode_start_time, metadata)); + } + } + if (frame_stats.decode_end_time.IsFinite()) { + stats->decode_time_ms.AddSample(StatsSample( + frame_stats.decode_end_time - frame_stats.decode_start_time, + frame_stats.decode_end_time, metadata)); + stats->resolution_of_decoded_frame.AddSample( + StatsSample(*comparison.frame_stats.decoded_frame_width * + *comparison.frame_stats.decoded_frame_height, + frame_stats.decode_end_time, metadata)); + } + + if (frame_stats.prev_frame_rendered_time.IsFinite() && + frame_stats.rendered_time.IsFinite()) { + TimeDelta time_between_rendered_frames = + frame_stats.rendered_time - frame_stats.prev_frame_rendered_time; + stats->time_between_rendered_frames_ms.AddSample(StatsSample( + time_between_rendered_frames, frame_stats.rendered_time, metadata)); + TimeDelta average_time_between_rendered_frames = TimeDelta::Millis( + stats->time_between_rendered_frames_ms.GetAverage()); + if (time_between_rendered_frames > + std::max(kFreezeThreshold + average_time_between_rendered_frames, + 3 * average_time_between_rendered_frames)) { + stats->freeze_time_ms.AddSample(StatsSample( + time_between_rendered_frames, frame_stats.rendered_time, metadata)); + auto freeze_end_it = + stream_last_freeze_end_time_.find(comparison.stats_key); + RTC_DCHECK(freeze_end_it != stream_last_freeze_end_time_.end()); + stats->time_between_freezes_ms.AddSample(StatsSample( + frame_stats.prev_frame_rendered_time - freeze_end_it->second, + frame_stats.rendered_time, metadata)); + freeze_end_it->second = frame_stats.rendered_time; + } + } + } + // Compute stream codec info. + if (frame_stats.used_encoder.has_value()) { + if (stats->encoders.empty() || stats->encoders.back().codec_name != + frame_stats.used_encoder->codec_name) { + stats->encoders.push_back(*frame_stats.used_encoder); + } + stats->encoders.back().last_frame_id = + frame_stats.used_encoder->last_frame_id; + stats->encoders.back().switched_from_at = + frame_stats.used_encoder->switched_from_at; + } + + if (frame_stats.used_decoder.has_value()) { + if (stats->decoders.empty() || stats->decoders.back().codec_name != + frame_stats.used_decoder->codec_name) { + stats->decoders.push_back(*frame_stats.used_decoder); + } + stats->decoders.back().last_frame_id = + frame_stats.used_decoder->last_frame_id; + stats->decoders.back().switched_from_at = + frame_stats.used_decoder->switched_from_at; + } +} + +Timestamp DefaultVideoQualityAnalyzerFramesComparator::Now() { + return clock_->CurrentTime(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h new file mode 100644 index 0000000000..006c3eb9bf --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_ + +#include <deque> +#include <map> +#include <utility> +#include <vector> + +#include "api/array_view.h" +#include "rtc_base/event.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/synchronization/mutex.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +namespace webrtc { + +struct FramesComparatorStats { + // Size of analyzer internal comparisons queue, measured when new element + // id added to the queue. + SamplesStatsCounter comparisons_queue_size; + // Number of performed comparisons of 2 video frames from captured and + // rendered streams. + int64_t comparisons_done = 0; + // Number of cpu overloaded comparisons. Comparison is cpu overloaded if it is + // queued when there are too many not processed comparisons in the queue. + // Overloaded comparison doesn't include metrics like SSIM and PSNR that + // require heavy computations. + int64_t cpu_overloaded_comparisons_done = 0; + // Number of memory overloaded comparisons. Comparison is memory overloaded if + // it is queued when its captured frame was already removed due to high memory + // usage for that video stream. + int64_t memory_overloaded_comparisons_done = 0; +}; + +// Performs comparisons of added frames and tracks frames related statistics. +// This class is thread safe. +class DefaultVideoQualityAnalyzerFramesComparator { + public: + // Creates frames comparator. + // Frames comparator doesn't use `options.enable_receive_own_stream` for any + // purposes, because it's unrelated to its functionality. + DefaultVideoQualityAnalyzerFramesComparator( + webrtc::Clock* clock, + DefaultVideoQualityAnalyzerCpuMeasurer& cpu_measurer, + DefaultVideoQualityAnalyzerOptions options = {}) + : options_(options), clock_(clock), cpu_measurer_(cpu_measurer) {} + ~DefaultVideoQualityAnalyzerFramesComparator() { Stop({}); } + + // Starts frames comparator. This method must be invoked before calling + // any other method on this object. + void Start(int max_threads_count); + // Stops frames comparator. This method will block until all added frame + // comparisons will be processed. After `Stop()` is invoked no more new + // comparisons can be added to this frames comparator. + // + // `last_rendered_frame_time` contains timestamps of last rendered frame for + // each (stream, sender, receiver) tuple to properly update time between + // freezes: it has include time from the last freeze until and of call. + void Stop( + const std::map<InternalStatsKey, Timestamp>& last_rendered_frame_times); + + // Ensures that stream `stream_index` has stats objects created for all + // potential receivers. This method must be called before adding any + // frames comparison for that stream. + void EnsureStatsForStream(size_t stream_index, + size_t sender_peer_index, + size_t peers_count, + Timestamp captured_time, + Timestamp start_time); + // Ensures that newly added participant will have stream stats objects created + // for all streams which they can receive. This method must be called before + // any frames comparison will be added for the newly added participant. + // + // `stream_started_time` - start time of each stream for which stats object + // has to be created. + // `start_time` - call start time. + void RegisterParticipantInCall( + rtc::ArrayView<std::pair<InternalStatsKey, Timestamp>> + stream_started_time, + Timestamp start_time); + + // `captured` - video frame captured by sender to use for PSNR/SSIM + // computation. If `type` is `FrameComparisonType::kRegular` and + // `captured` is `absl::nullopt` comparison is assumed to be overloaded + // due to memory constraints. + // `rendered` - video frame rendered by receiver to use for PSNR/SSIM + // computation. Required only if `type` is + // `FrameComparisonType::kRegular`, but can still be omitted if + // `captured` is `absl::nullopt`. + void AddComparison(InternalStatsKey stats_key, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats); + // `skipped_between_rendered` - amount of frames dropped on this stream before + // last received frame and current frame. + void AddComparison(InternalStatsKey stats_key, + int skipped_between_rendered, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats); + + std::map<InternalStatsKey, StreamStats> stream_stats() const { + MutexLock lock(&mutex_); + return stream_stats_; + } + FramesComparatorStats frames_comparator_stats() const { + MutexLock lock(&mutex_); + return frames_comparator_stats_; + } + + private: + enum State { kNew, kActive, kStopped }; + + void AddComparisonInternal(InternalStatsKey stats_key, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + void ProcessComparisons(); + void ProcessComparison(const FrameComparison& comparison); + Timestamp Now(); + + const DefaultVideoQualityAnalyzerOptions options_; + webrtc::Clock* const clock_; + DefaultVideoQualityAnalyzerCpuMeasurer& cpu_measurer_; + + mutable Mutex mutex_; + State state_ RTC_GUARDED_BY(mutex_) = State::kNew; + std::map<InternalStatsKey, StreamStats> stream_stats_ RTC_GUARDED_BY(mutex_); + std::map<InternalStatsKey, Timestamp> stream_last_freeze_end_time_ + RTC_GUARDED_BY(mutex_); + std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(mutex_); + FramesComparatorStats frames_comparator_stats_ RTC_GUARDED_BY(mutex_); + + std::vector<rtc::PlatformThread> thread_pool_; + rtc::Event comparison_available_event_; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc new file mode 100644 index 0000000000..8d3cd47ed6 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc @@ -0,0 +1,1648 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" + +#include <map> +#include <string> +#include <vector> + +#include "api/test/create_frame_generator.h" +#include "api/units/timestamp.h" +#include "rtc_base/strings/string_builder.h" +#include "system_wrappers/include/clock.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +namespace webrtc { +namespace { + +using ::testing::Contains; +using ::testing::DoubleEq; +using ::testing::Each; +using ::testing::Eq; +using ::testing::IsEmpty; +using ::testing::Pair; +using ::testing::SizeIs; + +using StatsSample = ::webrtc::SamplesStatsCounter::StatsSample; + +constexpr int kMaxFramesInFlightPerStream = 10; + +DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() { + DefaultVideoQualityAnalyzerOptions options; + options.compute_psnr = false; + options.compute_ssim = false; + options.adjust_cropping_before_comparing_frames = false; + options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream; + return options; +} + +VideoFrame CreateFrame(uint16_t frame_id, + int width, + int height, + Timestamp timestamp) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(width, height, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + test::FrameGeneratorInterface::VideoFrameData frame_data = + frame_generator->NextFrame(); + return VideoFrame::Builder() + .set_id(frame_id) + .set_video_frame_buffer(frame_data.buffer) + .set_update_rect(frame_data.update_rect) + .set_timestamp_us(timestamp.us()) + .build(); +} + +StreamCodecInfo Vp8CodecForOneFrame(uint16_t frame_id, Timestamp time) { + StreamCodecInfo info; + info.codec_name = "VP8"; + info.first_frame_id = frame_id; + info.last_frame_id = frame_id; + info.switched_on_at = time; + info.switched_from_at = time; + return info; +} + +FrameStats FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + uint16_t frame_id, + Timestamp captured_time) { + FrameStats frame_stats(frame_id, captured_time); + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + // Decode time is in microseconds. + frame_stats.decode_end_time = captured_time + TimeDelta::Micros(40010); + frame_stats.rendered_time = captured_time + TimeDelta::Millis(60); + frame_stats.used_encoder = Vp8CodecForOneFrame(1, frame_stats.encoded_time); + frame_stats.used_decoder = + Vp8CodecForOneFrame(1, frame_stats.decode_end_time); + frame_stats.decoded_frame_width = 10; + frame_stats.decoded_frame_height = 10; + return frame_stats; +} + +FrameStats ShiftStatsOn(const FrameStats& stats, TimeDelta delta) { + FrameStats frame_stats(stats.frame_id, stats.captured_time + delta); + frame_stats.pre_encode_time = stats.pre_encode_time + delta; + frame_stats.encoded_time = stats.encoded_time + delta; + frame_stats.received_time = stats.received_time + delta; + frame_stats.decode_start_time = stats.decode_start_time + delta; + frame_stats.decode_end_time = stats.decode_end_time + delta; + frame_stats.rendered_time = stats.rendered_time + delta; + + frame_stats.used_encoder = stats.used_encoder; + frame_stats.used_decoder = stats.used_decoder; + frame_stats.decoded_frame_width = stats.decoded_frame_width; + frame_stats.decoded_frame_height = stats.decoded_frame_height; + + return frame_stats; +} + +SamplesStatsCounter StatsCounter( + const std::vector<std::pair<double, Timestamp>>& samples) { + SamplesStatsCounter counter; + for (const std::pair<double, Timestamp>& sample : samples) { + counter.AddSample(SamplesStatsCounter::StatsSample{.value = sample.first, + .time = sample.second}); + } + return counter; +} + +double GetFirstOrDie(const SamplesStatsCounter& counter) { + EXPECT_FALSE(counter.IsEmpty()) << "Counter has to be not empty"; + return counter.GetSamples()[0]; +} + +void AssertFirstMetadataHasField(const SamplesStatsCounter& counter, + const std::string& field_name, + const std::string& field_value) { + EXPECT_FALSE(counter.IsEmpty()) << "Coutner has to be not empty"; + EXPECT_THAT(counter.GetTimedSamples()[0].metadata, + Contains(Pair(field_name, field_value))); +} + +std::string ToString(const SamplesStatsCounter& counter) { + rtc::StringBuilder out; + for (const StatsSample& s : counter.GetTimedSamples()) { + out << "{ time_ms=" << s.time.ms() << "; value=" << s.value << "}, "; + } + return out.str(); +} + +void ExpectEmpty(const SamplesStatsCounter& counter) { + EXPECT_TRUE(counter.IsEmpty()) + << "Expected empty SamplesStatsCounter, but got " << ToString(counter); +} + +void ExpectEmpty(const SamplesRateCounter& counter) { + EXPECT_TRUE(counter.IsEmpty()) + << "Expected empty SamplesRateCounter, but got " + << counter.GetEventsPerSecond(); +} + +void ExpectSizeAndAllElementsAre(const SamplesStatsCounter& counter, + int size, + double value) { + EXPECT_EQ(counter.NumSamples(), size); + EXPECT_THAT(counter.GetSamples(), Each(DoubleEq(value))); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + StatsPresentedAfterAddingOneComparison) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest()); + + Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + size_t peers_count = 2; + InternalStatsKey stats_key(stream, sender, receiver); + + FrameStats frame_stats = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + /*frame_id=*/1, stream_start_time); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, peers_count, + stream_start_time, stream_start_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kRegular, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + std::map<InternalStatsKey, StreamStats> stats = comparator.stream_stats(); + ExpectSizeAndAllElementsAre(stats.at(stats_key).transport_time_ms, /*size=*/1, + /*value=*/20.0); + ExpectSizeAndAllElementsAre(stats.at(stats_key).total_delay_incl_transport_ms, + /*size=*/1, /*value=*/60.0); + ExpectSizeAndAllElementsAre(stats.at(stats_key).encode_time_ms, /*size=*/1, + /*value=*/10.0); + ExpectSizeAndAllElementsAre(stats.at(stats_key).decode_time_ms, /*size=*/1, + /*value=*/0.01); + ExpectSizeAndAllElementsAre(stats.at(stats_key).receive_to_render_time_ms, + /*size=*/1, /*value=*/30.0); + ExpectSizeAndAllElementsAre(stats.at(stats_key).resolution_of_decoded_frame, + /*size=*/1, /*value=*/100.0); +} + +TEST( + DefaultVideoQualityAnalyzerFramesComparatorTest, + MultiFrameStatsPresentedWithMetadataAfterAddingTwoComparisonWith10msDelay) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest()); + + Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + size_t peers_count = 2; + InternalStatsKey stats_key(stream, sender, receiver); + + FrameStats frame_stats1 = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + /*frame_id=*/1, stream_start_time); + FrameStats frame_stats2 = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + /*frame_id=*/2, stream_start_time + TimeDelta::Millis(15)); + frame_stats2.prev_frame_rendered_time = frame_stats1.rendered_time; + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, peers_count, + stream_start_time, stream_start_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kRegular, frame_stats1); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kRegular, frame_stats2); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + std::map<InternalStatsKey, StreamStats> stats = comparator.stream_stats(); + ExpectSizeAndAllElementsAre( + stats.at(stats_key).time_between_rendered_frames_ms, /*size=*/1, + /*value=*/15.0); + AssertFirstMetadataHasField( + stats.at(stats_key).time_between_rendered_frames_ms, "frame_id", "2"); + EXPECT_DOUBLE_EQ(stats.at(stats_key).encode_frame_rate.GetEventsPerSecond(), + 2.0 / 15 * 1000) + << "There should be 2 events with interval of 15 ms"; +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + FrameInFlightStatsAreHandledCorrectly) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest()); + + Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + size_t peers_count = 2; + InternalStatsKey stats_key(stream, sender, receiver); + + // There are 7 different timings inside frame stats: captured, pre_encode, + // encoded, received, decode_start, decode_end, rendered. captured is always + // set and received is set together with decode_start. So we create 6 + // different frame stats with interval of 15 ms, where for each stat next + // timings will be set + // * 1st - captured + // * 2nd - captured, pre_encode + // * 3rd - captured, pre_encode, encoded + // * 4th - captured, pre_encode, encoded, received, decode_start + // * 5th - captured, pre_encode, encoded, received, decode_start, decode_end + // * 6th - all of them set + std::vector<FrameStats> stats; + // 1st stat + FrameStats frame_stats(/*frame_id=*/1, stream_start_time); + stats.push_back(frame_stats); + // 2nd stat + frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.frame_id = 2; + frame_stats.pre_encode_time = + frame_stats.captured_time + TimeDelta::Millis(10); + stats.push_back(frame_stats); + // 3rd stat + frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.frame_id = 3; + frame_stats.encoded_time = frame_stats.captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = Vp8CodecForOneFrame(1, frame_stats.encoded_time); + stats.push_back(frame_stats); + // 4th stat + frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.frame_id = 4; + frame_stats.received_time = frame_stats.captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = + frame_stats.captured_time + TimeDelta::Millis(40); + stats.push_back(frame_stats); + // 5th stat + frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.frame_id = 5; + frame_stats.decode_end_time = + frame_stats.captured_time + TimeDelta::Millis(50); + frame_stats.used_decoder = + Vp8CodecForOneFrame(1, frame_stats.decode_end_time); + frame_stats.decoded_frame_width = 10; + frame_stats.decoded_frame_height = 10; + stats.push_back(frame_stats); + // 6th stat + frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.frame_id = 6; + frame_stats.rendered_time = frame_stats.captured_time + TimeDelta::Millis(60); + stats.push_back(frame_stats); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, peers_count, + stream_start_time, stream_start_time); + for (size_t i = 0; i < stats.size() - 1; ++i) { + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, stats[i]); + } + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kRegular, + stats[stats.size() - 1]); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats result_stats = comparator.stream_stats().at(stats_key); + + EXPECT_DOUBLE_EQ(result_stats.transport_time_ms.GetAverage(), 20.0) + << ToString(result_stats.transport_time_ms); + EXPECT_EQ(result_stats.transport_time_ms.NumSamples(), 3); + + EXPECT_DOUBLE_EQ(result_stats.total_delay_incl_transport_ms.GetAverage(), + 60.0) + << ToString(result_stats.total_delay_incl_transport_ms); + EXPECT_EQ(result_stats.total_delay_incl_transport_ms.NumSamples(), 1); + + EXPECT_DOUBLE_EQ(result_stats.encode_time_ms.GetAverage(), 10) + << ToString(result_stats.encode_time_ms); + EXPECT_EQ(result_stats.encode_time_ms.NumSamples(), 4); + + EXPECT_DOUBLE_EQ(result_stats.decode_time_ms.GetAverage(), 10) + << ToString(result_stats.decode_time_ms); + EXPECT_EQ(result_stats.decode_time_ms.NumSamples(), 2); + + EXPECT_DOUBLE_EQ(result_stats.receive_to_render_time_ms.GetAverage(), 30) + << ToString(result_stats.receive_to_render_time_ms); + EXPECT_EQ(result_stats.receive_to_render_time_ms.NumSamples(), 1); + + EXPECT_DOUBLE_EQ(result_stats.resolution_of_decoded_frame.GetAverage(), 100) + << ToString(result_stats.resolution_of_decoded_frame); + EXPECT_EQ(result_stats.resolution_of_decoded_frame.NumSamples(), 2); + + EXPECT_DOUBLE_EQ(result_stats.encode_frame_rate.GetEventsPerSecond(), + 4.0 / 45 * 1000) + << "There should be 4 events with interval of 15 ms"; +} + +// Tests to validate that stats for each possible input frame are computed +// correctly. +// Frame in flight start +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + CapturedOnlyInFlightFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectEmpty(stats.encode_time_ms); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectEmpty(stats.target_encode_bitrate); + EXPECT_THAT(stats.spatial_layers_qp, IsEmpty()); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 0); + EXPECT_EQ(stats.num_send_key_frames, 0); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_THAT(stats.encoders, IsEmpty()); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + PreEncodedInFlightFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectEmpty(stats.encode_time_ms); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectEmpty(stats.target_encode_bitrate); + EXPECT_THAT(stats.spatial_layers_qp, IsEmpty()); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 0); + EXPECT_EQ(stats.num_send_key_frames, 0); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_THAT(stats.encoders, IsEmpty()); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + EncodedInFlightKeyFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 1); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + EncodedInFlightDeltaFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameDelta; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 0); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + PreDecodedInFlightKeyFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + // Frame pre decoded + frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.pre_decoded_image_size = DataSize::Bytes(500); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectSizeAndAllElementsAre(stats.transport_time_ms, /*size=*/1, + /*value=*/20.0); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1, + /*value=*/500.0); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 1); + EXPECT_EQ(stats.num_recv_key_frames, 1); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + DecodedInFlightKeyFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + // Frame pre decoded + frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.pre_decoded_image_size = DataSize::Bytes(500); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + // Frame decoded + frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50); + frame_stats.decoded_frame_width = 200; + frame_stats.decoded_frame_height = 100; + + frame_stats.used_decoder = + Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectSizeAndAllElementsAre(stats.transport_time_ms, /*size=*/1, + /*value=*/20.0); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectSizeAndAllElementsAre(stats.decode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + EXPECT_GE(GetFirstOrDie(stats.resolution_of_decoded_frame), 200 * 100.0); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1, + /*value=*/500.0); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 1); + EXPECT_EQ(stats.num_recv_key_frames, 1); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_EQ(stats.decoders, + std::vector<StreamCodecInfo>{*frame_stats.used_decoder}); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + DecoderFailureOnInFlightKeyFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + // Frame pre decoded + frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.pre_decoded_image_size = DataSize::Bytes(500); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + // Frame decoded + frame_stats.decoder_failed = true; + frame_stats.used_decoder = + Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectSizeAndAllElementsAre(stats.transport_time_ms, /*size=*/1, + /*value=*/20.0); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1, + /*value=*/500.0); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 1); + EXPECT_EQ(stats.num_recv_key_frames, 1); + // All frame in flight are not considered as dropped. + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_EQ(stats.decoders, + std::vector<StreamCodecInfo>{*frame_stats.used_decoder}); +} +// Frame in flight end + +// Dropped frame start +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + CapturedOnlyDroppedFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kDroppedFrame, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectEmpty(stats.encode_time_ms); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectEmpty(stats.target_encode_bitrate); + EXPECT_THAT(stats.spatial_layers_qp, IsEmpty()); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 0); + EXPECT_EQ(stats.num_send_key_frames, 0); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 1}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_THAT(stats.encoders, IsEmpty()); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + PreEncodedDroppedFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kDroppedFrame, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectEmpty(stats.encode_time_ms); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectEmpty(stats.target_encode_bitrate); + EXPECT_THAT(stats.spatial_layers_qp, IsEmpty()); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 0); + EXPECT_EQ(stats.num_send_key_frames, 0); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 1}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_THAT(stats.encoders, IsEmpty()); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + EncodedDroppedKeyFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kDroppedFrame, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 1); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 1}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + EncodedDroppedDeltaFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameDelta; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kDroppedFrame, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 0); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 1}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + PreDecodedDroppedKeyFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + // Frame pre decoded + frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.pre_decoded_image_size = DataSize::Bytes(500); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kDroppedFrame, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 1); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 1}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_THAT(stats.decoders, IsEmpty()); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + DecodedDroppedKeyFrameAccountedInStats) { + // We don't really drop frames after decoder, so it's a bit unclear what is + // correct way to account such frames in stats, so this test just fixes some + // current way. + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + // Frame pre decoded + frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.pre_decoded_image_size = DataSize::Bytes(500); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + // Frame decoded + frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50); + frame_stats.used_decoder = + Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time); + frame_stats.decoded_frame_width = 200; + frame_stats.decoded_frame_height = 100; + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kDroppedFrame, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectEmpty(stats.transport_time_ms); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectEmpty(stats.recv_key_frame_size_bytes); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 1); + EXPECT_EQ(stats.num_recv_key_frames, 0); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 1}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_EQ(stats.decoders, + std::vector<StreamCodecInfo>{*frame_stats.used_decoder}); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + DecoderFailedDroppedKeyFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + // Frame pre decoded + frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.pre_decoded_image_size = DataSize::Bytes(500); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + // Frame decoded + frame_stats.decoder_failed = true; + frame_stats.used_decoder = + Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kDroppedFrame, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + ExpectEmpty(stats.psnr); + ExpectEmpty(stats.ssim); + ExpectSizeAndAllElementsAre(stats.transport_time_ms, /*size=*/1, + /*value=*/20.0); + ExpectEmpty(stats.total_delay_incl_transport_ms); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + ExpectEmpty(stats.decode_time_ms); + ExpectEmpty(stats.receive_to_render_time_ms); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + ExpectEmpty(stats.resolution_of_decoded_frame); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1, + /*value=*/500.0); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 1); + EXPECT_EQ(stats.num_recv_key_frames, 1); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 1}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_EQ(stats.decoders, + std::vector<StreamCodecInfo>{*frame_stats.used_decoder}); +} +// Dropped frame end + +// Regular frame start +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + RenderedKeyFrameAccountedInStats) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + VideoFrame frame = + CreateFrame(frame_id, /*width=*/320, /*height=*/180, captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + // Frame pre decoded + frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.pre_decoded_image_size = DataSize::Bytes(500); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + // Frame decoded + frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50); + frame_stats.used_decoder = + Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time); + frame_stats.decoded_frame_width = 200; + frame_stats.decoded_frame_height = 100; + // Frame rendered + frame_stats.rendered_time = captured_time + TimeDelta::Millis(60); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/frame, + /*rendered=*/frame, FrameComparisonType::kRegular, + frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + EXPECT_EQ(stats.stream_started_time, captured_time); + EXPECT_GE(GetFirstOrDie(stats.psnr), 20); + EXPECT_GE(GetFirstOrDie(stats.ssim), 0.5); + ExpectSizeAndAllElementsAre(stats.transport_time_ms, /*size=*/1, + /*value=*/20.0); + EXPECT_GE(GetFirstOrDie(stats.total_delay_incl_transport_ms), 60.0); + ExpectEmpty(stats.time_between_rendered_frames_ms); + ExpectEmpty(stats.encode_frame_rate); + ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0); + EXPECT_GE(GetFirstOrDie(stats.decode_time_ms), 10.0); + EXPECT_GE(GetFirstOrDie(stats.receive_to_render_time_ms), 30.0); + ExpectEmpty(stats.skipped_between_rendered); + ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0); + ExpectEmpty(stats.time_between_freezes_ms); + EXPECT_GE(GetFirstOrDie(stats.resolution_of_decoded_frame), 200 * 100.0); + ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1, + /*value=*/2000.0); + EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1)); + ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2, + /*value=*/5.0); + ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1, + /*value=*/500.0); + ExpectEmpty(stats.recv_delta_frame_size_bytes); + EXPECT_EQ(stats.total_encoded_images_payload, 1000); + EXPECT_EQ(stats.num_send_key_frames, 1); + EXPECT_EQ(stats.num_recv_key_frames, 1); + EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{ + {FrameDropPhase::kBeforeEncoder, 0}, + {FrameDropPhase::kByEncoder, 0}, + {FrameDropPhase::kTransport, 0}, + {FrameDropPhase::kByDecoder, 0}, + {FrameDropPhase::kAfterDecoder, 0}})); + EXPECT_EQ(stats.encoders, + std::vector<StreamCodecInfo>{*frame_stats.used_encoder}); + EXPECT_EQ(stats.decoders, + std::vector<StreamCodecInfo>{*frame_stats.used_decoder}); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, AllStatsHaveMetadataSet) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, + DefaultVideoQualityAnalyzerOptions()); + + Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime(); + uint16_t frame_id = 1; + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + InternalStatsKey stats_key(stream, sender, receiver); + + // Frame captured + VideoFrame frame = + CreateFrame(frame_id, /*width=*/320, /*height=*/180, captured_time); + FrameStats frame_stats(/*frame_id=*/1, captured_time); + // Frame pre encoded + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + // Frame encoded + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = + Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time); + frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.encoded_image_size = DataSize::Bytes(1000); + frame_stats.target_encode_bitrate = 2000; + frame_stats.spatial_layers_qp = { + {0, StatsCounter( + /*samples=*/{{5, Timestamp::Seconds(1)}, + {5, Timestamp::Seconds(2)}})}}; + // Frame pre decoded + frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey; + frame_stats.pre_decoded_image_size = DataSize::Bytes(500); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + // Frame decoded + frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50); + frame_stats.used_decoder = + Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time); + // Frame rendered + frame_stats.rendered_time = captured_time + TimeDelta::Millis(60); + frame_stats.decoded_frame_width = 200; + frame_stats.decoded_frame_height = 100; + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2, + captured_time, captured_time); + comparator.AddComparison(stats_key, + /*captured=*/frame, + /*rendered=*/frame, FrameComparisonType::kRegular, + frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats stats = comparator.stream_stats().at(stats_key); + AssertFirstMetadataHasField(stats.psnr, "frame_id", "1"); + AssertFirstMetadataHasField(stats.ssim, "frame_id", "1"); + AssertFirstMetadataHasField(stats.transport_time_ms, "frame_id", "1"); + AssertFirstMetadataHasField(stats.total_delay_incl_transport_ms, "frame_id", + "1"); + AssertFirstMetadataHasField(stats.encode_time_ms, "frame_id", "1"); + AssertFirstMetadataHasField(stats.decode_time_ms, "frame_id", "1"); + AssertFirstMetadataHasField(stats.receive_to_render_time_ms, "frame_id", "1"); + AssertFirstMetadataHasField(stats.resolution_of_decoded_frame, "frame_id", + "1"); + AssertFirstMetadataHasField(stats.target_encode_bitrate, "frame_id", "1"); + AssertFirstMetadataHasField(stats.spatial_layers_qp[0], "frame_id", "1"); + AssertFirstMetadataHasField(stats.recv_key_frame_size_bytes, "frame_id", "1"); + + ExpectEmpty(stats.recv_delta_frame_size_bytes); +} +// Regular frame end + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + FreezeStatsPresentedWithMetadataAfterAddFrameWithSkippedAndDelay) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest()); + + Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + size_t peers_count = 2; + InternalStatsKey stats_key(stream, sender, receiver); + + comparator.Start(/*max_threads_count=*/1); + comparator.EnsureStatsForStream(stream, sender, peers_count, + stream_start_time, stream_start_time); + + // Add 5 frames which were rendered with 30 fps (~30ms between frames) + // Frame ids are in [1..5] and last frame is with 120ms offset from first. + Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity(); + for (int i = 0; i < 5; ++i) { + FrameStats frame_stats = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + /*frame_id=*/i + 1, stream_start_time + TimeDelta::Millis(30 * i)); + frame_stats.prev_frame_rendered_time = prev_frame_rendered_time; + prev_frame_rendered_time = frame_stats.rendered_time; + + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kRegular, frame_stats); + } + + // Next frame was rendered with 4 frames skipped and delay 300ms after last + // frame. + FrameStats freeze_frame_stats = + FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + /*frame_id=*/10, stream_start_time + TimeDelta::Millis(120 + 300)); + freeze_frame_stats.prev_frame_rendered_time = prev_frame_rendered_time; + + comparator.AddComparison(stats_key, + /*skipped_between_rendered=*/4, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kRegular, freeze_frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + StreamStats stats = comparator.stream_stats().at(stats_key); + ASSERT_THAT(GetFirstOrDie(stats.skipped_between_rendered), Eq(4)); + AssertFirstMetadataHasField(stats.skipped_between_rendered, "frame_id", "10"); + ASSERT_THAT(GetFirstOrDie(stats.freeze_time_ms), Eq(300)); + AssertFirstMetadataHasField(stats.freeze_time_ms, "frame_id", "10"); + // 180ms is time from the stream start to the rendered time of the last frame + // among first 5 frames which were received before freeze. + ASSERT_THAT(GetFirstOrDie(stats.time_between_freezes_ms), Eq(180)); + AssertFirstMetadataHasField(stats.time_between_freezes_ms, "frame_id", "10"); +} +// Stats validation tests end. + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.cc new file mode 100644 index 0000000000..16f49ef154 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.cc @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" + +#include "api/video/video_frame.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { + +std::string InternalStatsKey::ToString() const { + rtc::StringBuilder out; + out << "stream=" << stream << "_sender=" << sender + << "_receiver=" << receiver; + return out.str(); +} + +bool operator<(const InternalStatsKey& a, const InternalStatsKey& b) { + if (a.stream != b.stream) { + return a.stream < b.stream; + } + if (a.sender != b.sender) { + return a.sender < b.sender; + } + return a.receiver < b.receiver; +} + +bool operator==(const InternalStatsKey& a, const InternalStatsKey& b) { + return a.stream == b.stream && a.sender == b.sender && + a.receiver == b.receiver; +} + +FrameComparison::FrameComparison(InternalStatsKey stats_key, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats, + OverloadReason overload_reason) + : stats_key(std::move(stats_key)), + captured(std::move(captured)), + rendered(std::move(rendered)), + type(type), + frame_stats(std::move(frame_stats)), + overload_reason(overload_reason) {} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h new file mode 100644 index 0000000000..10f1314f46 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_INTERNAL_SHARED_OBJECTS_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_INTERNAL_SHARED_OBJECTS_H_ + +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_type.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +namespace webrtc { + +struct InternalStatsKey { + InternalStatsKey(size_t stream, size_t sender, size_t receiver) + : stream(stream), sender(sender), receiver(receiver) {} + + std::string ToString() const; + + size_t stream; + size_t sender; + size_t receiver; +}; + +// Required to use InternalStatsKey as std::map key. +bool operator<(const InternalStatsKey& a, const InternalStatsKey& b); +bool operator==(const InternalStatsKey& a, const InternalStatsKey& b); + +// Final stats computed for frame after it went through the whole video +// pipeline from capturing to rendering or dropping. +struct FrameStats { + FrameStats(uint16_t frame_id, Timestamp captured_time) + : frame_id(frame_id), captured_time(captured_time) {} + + uint16_t frame_id; + // Frame events timestamp. + Timestamp captured_time; + Timestamp pre_encode_time = Timestamp::MinusInfinity(); + Timestamp encoded_time = Timestamp::MinusInfinity(); + // Time when last packet of a frame was received. + Timestamp received_time = Timestamp::MinusInfinity(); + Timestamp decode_start_time = Timestamp::MinusInfinity(); + Timestamp decode_end_time = Timestamp::MinusInfinity(); + Timestamp rendered_time = Timestamp::MinusInfinity(); + Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity(); + + VideoFrameType encoded_frame_type = VideoFrameType::kEmptyFrame; + DataSize encoded_image_size = DataSize::Bytes(0); + VideoFrameType pre_decoded_frame_type = VideoFrameType::kEmptyFrame; + DataSize pre_decoded_image_size = DataSize::Bytes(0); + uint32_t target_encode_bitrate = 0; + // Sender side qp values per spatial layer. In case when spatial layer is not + // set for `webrtc::EncodedImage`, 0 is used as default. + std::map<int, SamplesStatsCounter> spatial_layers_qp; + + absl::optional<int> decoded_frame_width = absl::nullopt; + absl::optional<int> decoded_frame_height = absl::nullopt; + + // Can be not set if frame was dropped by encoder. + absl::optional<StreamCodecInfo> used_encoder = absl::nullopt; + // Can be not set if frame was dropped in the network. + absl::optional<StreamCodecInfo> used_decoder = absl::nullopt; + + bool decoder_failed = false; +}; + +// Describes why comparison was done in overloaded mode (without calculating +// PSNR and SSIM). +enum class OverloadReason { + kNone, + // Not enough CPU to process all incoming comparisons. + kCpu, + // Not enough memory to store captured frames for all comparisons. + kMemory +}; + +enum class FrameComparisonType { + // Comparison for captured and rendered frame. + kRegular, + // Comparison for captured frame that is known to be dropped somewhere in + // video pipeline. + kDroppedFrame, + // Comparison for captured frame that was still in the video pipeline when + // test was stopped. It's unknown is this frame dropped or would it be + // delivered if test continue. + kFrameInFlight +}; + +// Represents comparison between two VideoFrames. Contains video frames itself +// and stats. Can be one of two types: +// 1. Normal - in this case `captured` is presented and either `rendered` is +// presented and `dropped` is false, either `rendered` is omitted and +// `dropped` is true. +// 2. Overloaded - in this case both `captured` and `rendered` are omitted +// because there were too many comparisons in the queue. `dropped` can be +// true or false showing was frame dropped or not. +struct FrameComparison { + FrameComparison(InternalStatsKey stats_key, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats, + OverloadReason overload_reason); + + InternalStatsKey stats_key; + // Frames can be omitted if there too many computations waiting in the + // queue. + absl::optional<VideoFrame> captured; + absl::optional<VideoFrame> rendered; + FrameComparisonType type; + FrameStats frame_stats; + OverloadReason overload_reason; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_INTERNAL_SHARED_OBJECTS_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc new file mode 100644 index 0000000000..f5029ac956 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc @@ -0,0 +1,682 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <memory> +#include <string> +#include <vector> + +#include "api/rtp_packet_info.h" +#include "api/rtp_packet_infos.h" +#include "api/test/create_frame_generator.h" +#include "api/test/metrics/metric.h" +#include "api/test/metrics/metrics_logger.h" +#include "api/test/metrics/stdout_metrics_exporter.h" +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_tools/frame_analyzer/video_geometry_aligner.h" +#include "system_wrappers/include/sleep.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" + +namespace webrtc { +namespace { + +using ::testing::Contains; +using ::testing::SizeIs; +using ::testing::UnorderedElementsAre; + +using ::webrtc::test::DefaultMetricsLogger; +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Metric; +using ::webrtc::test::MetricsExporter; +using ::webrtc::test::StdoutMetricsExporter; +using ::webrtc::test::Unit; + +constexpr int kAnalyzerMaxThreadsCount = 1; +constexpr int kMaxFramesInFlightPerStream = 10; +constexpr int kFrameWidth = 320; +constexpr int kFrameHeight = 240; + +DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() { + DefaultVideoQualityAnalyzerOptions options; + options.compute_psnr = true; + options.compute_ssim = true; + options.adjust_cropping_before_comparing_frames = false; + options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream; + options.report_detailed_frame_stats = true; + return options; +} + +VideoFrame NextFrame(test::FrameGeneratorInterface* frame_generator, + int64_t timestamp_us) { + test::FrameGeneratorInterface::VideoFrameData frame_data = + frame_generator->NextFrame(); + return VideoFrame::Builder() + .set_video_frame_buffer(frame_data.buffer) + .set_update_rect(frame_data.update_rect) + .set_timestamp_us(timestamp_us) + .build(); +} + +EncodedImage FakeEncode(const VideoFrame& frame) { + EncodedImage image; + std::vector<RtpPacketInfo> packet_infos; + packet_infos.push_back(RtpPacketInfo( + /*ssrc=*/1, + /*csrcs=*/{}, + /*rtp_timestamp=*/frame.timestamp(), + /*receive_time=*/Timestamp::Micros(frame.timestamp_us() + 10000))); + image.SetPacketInfos(RtpPacketInfos(packet_infos)); + return image; +} + +VideoFrame DeepCopy(const VideoFrame& frame) { + VideoFrame copy = frame; + copy.set_video_frame_buffer( + I420Buffer::Copy(*frame.video_frame_buffer()->ToI420())); + return copy; +} + +void PassFramesThroughAnalyzer(DefaultVideoQualityAnalyzer& analyzer, + absl::string_view sender, + absl::string_view stream_label, + std::vector<absl::string_view> receivers, + int frames_count, + test::FrameGeneratorInterface& frame_generator, + int interframe_delay_ms = 0) { + for (int i = 0; i < frames_count; ++i) { + VideoFrame frame = NextFrame(&frame_generator, /*timestamp_us=*/1); + uint16_t frame_id = + analyzer.OnFrameCaptured(sender, std::string(stream_label), frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode(sender, frame); + analyzer.OnFrameEncoded(sender, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + for (absl::string_view receiver : receivers) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(receiver, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(receiver, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(receiver, received_frame); + } + if (i < frames_count - 1 && interframe_delay_ms > 0) { + SleepMs(interframe_delay_ms); + } + } +} + +// Metric fields to assert on +struct MetricValidationInfo { + std::string test_case; + std::string name; + Unit unit; + ImprovementDirection improvement_direction; +}; + +bool operator==(const MetricValidationInfo& a, const MetricValidationInfo& b) { + return a.name == b.name && a.test_case == b.test_case && a.unit == b.unit && + a.improvement_direction == b.improvement_direction; +} + +std::ostream& operator<<(std::ostream& os, const MetricValidationInfo& m) { + os << "{ test_case=" << m.test_case << "; name=" << m.name + << "; unit=" << test::ToString(m.unit) + << "; improvement_direction=" << test::ToString(m.improvement_direction) + << " }"; + return os; +} + +std::vector<MetricValidationInfo> ToValidationInfo( + const std::vector<Metric>& metrics) { + std::vector<MetricValidationInfo> out; + for (const Metric& m : metrics) { + out.push_back( + MetricValidationInfo{.test_case = m.test_case, + .name = m.name, + .unit = m.unit, + .improvement_direction = m.improvement_direction}); + } + return out; +} + +std::vector<std::string> ToTestCases(const std::vector<Metric>& metrics) { + std::vector<std::string> out; + for (const Metric& m : metrics) { + out.push_back(m.test_case); + } + return out; +} + +TEST(DefaultVideoQualityAnalyzerMetricNamesTest, MetricNamesForP2PAreCorrect) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultMetricsLogger metrics_logger(Clock::GetRealTimeClock()); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + &metrics_logger, options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"}, + /*frames_count=*/5, *frame_generator, + /*interframe_delay_ms=*/50); + analyzer.Stop(); + + std::vector<MetricValidationInfo> metrics = + ToValidationInfo(metrics_logger.GetCollectedMetrics()); + EXPECT_THAT( + metrics, + UnorderedElementsAre( + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "ssim", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "transport_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "total_delay_incl_transport", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "time_between_rendered_frames", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "harmonic_framerate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "encode_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "encode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "time_between_freezes", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "freeze_time_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "pixels_per_frame", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "min_psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "decode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "receive_to_render_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "dropped_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "frames_in_flight", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "rendered_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "max_skipped", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "target_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "qp_sl0", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "actual_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "capture_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "num_encoded_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "num_decoded_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "num_send_key_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "num_recv_key_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "recv_key_frame_size_bytes", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "recv_delta_frame_size_bytes", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{.test_case = "test_case", + .name = "cpu_usage_%", + .unit = Unit::kUnitless, + .improvement_direction = + ImprovementDirection::kSmallerIsBetter})); +} + +TEST(DefaultVideoQualityAnalyzerMetricNamesTest, + MetricNamesFor3PeersAreCorrect) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultMetricsLogger metrics_logger(Clock::GetRealTimeClock()); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + &metrics_logger, options); + analyzer.Start("test_case", + std::vector<std::string>{"alice", "bob", "charlie"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"bob", "charlie"}, + /*frames_count=*/5, *frame_generator, + /*interframe_delay_ms=*/50); + analyzer.Stop(); + + std::vector<MetricValidationInfo> metrics = + ToValidationInfo(metrics_logger.GetCollectedMetrics()); + EXPECT_THAT( + metrics, + UnorderedElementsAre( + // Bob + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "ssim", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "transport_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "total_delay_incl_transport", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "time_between_rendered_frames", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "harmonic_framerate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "encode_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "encode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "time_between_freezes", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "freeze_time_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "pixels_per_frame", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "min_psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "decode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "receive_to_render_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "dropped_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "frames_in_flight", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "rendered_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "max_skipped", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "target_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "qp_sl0", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "actual_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "capture_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "num_encoded_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "num_decoded_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "num_send_key_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "num_recv_key_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "recv_key_frame_size_bytes", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_bob", + .name = "recv_delta_frame_size_bytes", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + + // Charlie + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "ssim", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "transport_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "total_delay_incl_transport", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "time_between_rendered_frames", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "harmonic_framerate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "encode_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "encode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "time_between_freezes", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "freeze_time_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "pixels_per_frame", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "min_psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "decode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "receive_to_render_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "dropped_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "frames_in_flight", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "rendered_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "max_skipped", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "target_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "qp_sl0", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "actual_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "capture_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "num_encoded_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "num_decoded_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "num_send_key_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "num_recv_key_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "recv_key_frame_size_bytes", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{ + .test_case = "test_case/alice_video_alice_charlie", + .name = "recv_delta_frame_size_bytes", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter}, + MetricValidationInfo{.test_case = "test_case", + .name = "cpu_usage_%", + .unit = Unit::kUnitless, + .improvement_direction = + ImprovementDirection::kSmallerIsBetter})); +} + +TEST(DefaultVideoQualityAnalyzerMetricNamesTest, + TestCaseFor3PeerIsTheSameAfterAllPeersLeft) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultMetricsLogger metrics_logger(Clock::GetRealTimeClock()); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + &metrics_logger, options); + analyzer.Start("test_case", + std::vector<std::string>{"alice", "bob", "charlie"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"bob", "charlie"}, + /*frames_count=*/5, *frame_generator, + /*interframe_delay_ms=*/50); + analyzer.UnregisterParticipantInCall("alice"); + analyzer.UnregisterParticipantInCall("bob"); + analyzer.UnregisterParticipantInCall("charlie"); + analyzer.Stop(); + + std::vector<std::string> metrics = + ToTestCases(metrics_logger.GetCollectedMetrics()); + EXPECT_THAT(metrics, SizeIs(57)); + EXPECT_THAT(metrics, Contains("test_case/alice_video_alice_bob").Times(28)); + EXPECT_THAT(metrics, + Contains("test_case/alice_video_alice_charlie").Times(28)); + EXPECT_THAT(metrics, Contains("test_case").Times(1)); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc new file mode 100644 index 0000000000..79b9286e2d --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +#include <algorithm> +#include <iterator> +#include <ostream> +#include <string> + +#include "api/units/timestamp.h" +#include "rtc_base/checks.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { +namespace { + +constexpr int kMicrosPerSecond = 1000000; + +} // namespace + +std::string StreamCodecInfo::ToString() const { + rtc::StringBuilder out; + out << "{codec_name=" << codec_name << "; first_frame_id=" << first_frame_id + << "; last_frame_id=" << last_frame_id + << "; switched_on_at=" << webrtc::ToString(switched_on_at) + << "; switched_from_at=" << webrtc::ToString(switched_from_at) << " }"; + return out.str(); +} + +std::ostream& operator<<(std::ostream& os, const StreamCodecInfo& state) { + return os << state.ToString(); +} + +rtc::StringBuilder& operator<<(rtc::StringBuilder& sb, + const StreamCodecInfo& state) { + return sb << state.ToString(); +} + +bool operator==(const StreamCodecInfo& a, const StreamCodecInfo& b) { + return a.codec_name == b.codec_name && a.first_frame_id == b.first_frame_id && + a.last_frame_id == b.last_frame_id && + a.switched_on_at == b.switched_on_at && + a.switched_from_at == b.switched_from_at; +} + +std::string ToString(FrameDropPhase phase) { + switch (phase) { + case FrameDropPhase::kBeforeEncoder: + return "kBeforeEncoder"; + case FrameDropPhase::kByEncoder: + return "kByEncoder"; + case FrameDropPhase::kTransport: + return "kTransport"; + case FrameDropPhase::kByDecoder: + return "kByDecoder"; + case FrameDropPhase::kAfterDecoder: + return "kAfterDecoder"; + case FrameDropPhase::kLastValue: + return "kLastValue"; + } +} + +std::ostream& operator<<(std::ostream& os, FrameDropPhase phase) { + return os << ToString(phase); +} +rtc::StringBuilder& operator<<(rtc::StringBuilder& sb, FrameDropPhase phase) { + return sb << ToString(phase); +} + +void SamplesRateCounter::AddEvent(Timestamp event_time) { + if (event_first_time_.IsMinusInfinity()) { + event_first_time_ = event_time; + } + event_last_time_ = event_time; + events_count_++; +} + +double SamplesRateCounter::GetEventsPerSecond() const { + RTC_DCHECK(!IsEmpty()); + // Divide on us and multiply on kMicrosPerSecond to correctly process cases + // where there were too small amount of events, so difference is less then 1 + // sec. We can use us here, because Timestamp has us resolution. + return static_cast<double>(events_count_) / + (event_last_time_ - event_first_time_).us() * kMicrosPerSecond; +} + +StreamStats::StreamStats(Timestamp stream_started_time) + : stream_started_time(stream_started_time) { + for (int i = static_cast<int>(FrameDropPhase::kBeforeEncoder); + i < static_cast<int>(FrameDropPhase::kLastValue); ++i) { + dropped_by_phase.emplace(static_cast<FrameDropPhase>(i), 0); + } +} + +std::string StatsKey::ToString() const { + rtc::StringBuilder out; + out << stream_label << "_" << receiver; + return out.str(); +} + +bool operator<(const StatsKey& a, const StatsKey& b) { + if (a.stream_label != b.stream_label) { + return a.stream_label < b.stream_label; + } + return a.receiver < b.receiver; +} + +bool operator==(const StatsKey& a, const StatsKey& b) { + return a.stream_label == b.stream_label && a.receiver == b.receiver; +} + +VideoStreamsInfo::VideoStreamsInfo( + std::map<std::string, std::string> stream_to_sender, + std::map<std::string, std::set<std::string>> sender_to_streams, + std::map<std::string, std::set<std::string>> stream_to_receivers) + : stream_to_sender_(std::move(stream_to_sender)), + sender_to_streams_(std::move(sender_to_streams)), + stream_to_receivers_(std::move(stream_to_receivers)) {} + +std::set<StatsKey> VideoStreamsInfo::GetStatsKeys() const { + std::set<StatsKey> out; + for (const std::string& stream_label : GetStreams()) { + for (const std::string& receiver : GetReceivers(stream_label)) { + out.insert(StatsKey(stream_label, receiver)); + } + } + return out; +} + +std::set<std::string> VideoStreamsInfo::GetStreams() const { + std::set<std::string> out; + std::transform(stream_to_sender_.begin(), stream_to_sender_.end(), + std::inserter(out, out.end()), + [](auto map_entry) { return map_entry.first; }); + return out; +} + +std::set<std::string> VideoStreamsInfo::GetStreams( + absl::string_view sender_name) const { + auto it = sender_to_streams_.find(std::string(sender_name)); + if (it == sender_to_streams_.end()) { + return {}; + } + return it->second; +} + +absl::optional<std::string> VideoStreamsInfo::GetSender( + absl::string_view stream_label) const { + auto it = stream_to_sender_.find(std::string(stream_label)); + if (it == stream_to_sender_.end()) { + return absl::nullopt; + } + return it->second; +} + +std::set<std::string> VideoStreamsInfo::GetReceivers( + absl::string_view stream_label) const { + auto it = stream_to_receivers_.find(std::string(stream_label)); + if (it == stream_to_receivers_.end()) { + return {}; + } + return it->second; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h new file mode 100644 index 0000000000..175f777b68 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h @@ -0,0 +1,284 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_SHARED_OBJECTS_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_SHARED_OBJECTS_H_ + +#include <cstdint> +#include <map> +#include <memory> +#include <ostream> +#include <set> +#include <string> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/units/timestamp.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { + +// WebRTC will request a key frame after 3 seconds if no frames were received. +// We assume max frame rate ~60 fps, so 270 frames will cover max freeze without +// key frame request. +constexpr size_t kDefaultMaxFramesInFlightPerStream = 270; + +class SamplesRateCounter { + public: + void AddEvent(Timestamp event_time); + + bool IsEmpty() const { return event_first_time_ == event_last_time_; } + + double GetEventsPerSecond() const; + + private: + Timestamp event_first_time_ = Timestamp::MinusInfinity(); + Timestamp event_last_time_ = Timestamp::MinusInfinity(); + int64_t events_count_ = 0; +}; + +struct FrameCounters { + // Count of frames, that were passed into WebRTC pipeline by video stream + // source. + int64_t captured = 0; + // Count of frames that reached video encoder. + int64_t pre_encoded = 0; + // Count of encoded images that were produced by encoder for all requested + // spatial layers and simulcast streams. + int64_t encoded = 0; + // Count of encoded images received in decoder for all requested spatial + // layers and simulcast streams. + int64_t received = 0; + // Count of frames that were produced by decoder. + int64_t decoded = 0; + // Count of frames that went out from WebRTC pipeline to video sink. + int64_t rendered = 0; + // Count of frames that were dropped in any point between capturing and + // rendering. + int64_t dropped = 0; + // Count of frames for which decoder returned error when they were sent for + // decoding. + int64_t failed_to_decode = 0; +}; + +// Contains information about the codec that was used for encoding or decoding +// the stream. +struct StreamCodecInfo { + // Codec implementation name. + std::string codec_name; + // Id of the first frame for which this codec was used. + uint16_t first_frame_id; + // Id of the last frame for which this codec was used. + uint16_t last_frame_id; + // Timestamp when the first frame was handled by the encode/decoder. + Timestamp switched_on_at = Timestamp::PlusInfinity(); + // Timestamp when this codec was used last time. + Timestamp switched_from_at = Timestamp::PlusInfinity(); + + std::string ToString() const; +}; + +std::ostream& operator<<(std::ostream& os, const StreamCodecInfo& state); +rtc::StringBuilder& operator<<(rtc::StringBuilder& sb, + const StreamCodecInfo& state); +bool operator==(const StreamCodecInfo& a, const StreamCodecInfo& b); + +// Represents phases where video frame can be dropped and such drop will be +// detected by analyzer. +enum class FrameDropPhase : int { + kBeforeEncoder, + kByEncoder, + kTransport, + kByDecoder, + kAfterDecoder, + // kLastValue must be the last value in this enumeration. + kLastValue +}; + +std::string ToString(FrameDropPhase phase); +std::ostream& operator<<(std::ostream& os, FrameDropPhase phase); +rtc::StringBuilder& operator<<(rtc::StringBuilder& sb, FrameDropPhase phase); + +struct StreamStats { + explicit StreamStats(Timestamp stream_started_time); + + // The time when the first frame of this stream was captured. + Timestamp stream_started_time; + + // Spatial quality metrics. + SamplesStatsCounter psnr; + SamplesStatsCounter ssim; + + // Time from frame encoded (time point on exit from encoder) to the + // encoded image received in decoder (time point on entrance to decoder). + SamplesStatsCounter transport_time_ms; + // Time from frame was captured on device to time frame was displayed on + // device. + SamplesStatsCounter total_delay_incl_transport_ms; + // Time between frames out from renderer. + SamplesStatsCounter time_between_rendered_frames_ms; + SamplesRateCounter capture_frame_rate; + SamplesRateCounter encode_frame_rate; + SamplesStatsCounter encode_time_ms; + SamplesStatsCounter decode_time_ms; + // Time from last packet of frame is received until it's sent to the renderer. + SamplesStatsCounter receive_to_render_time_ms; + // Max frames skipped between two nearest. + SamplesStatsCounter skipped_between_rendered; + // In the next 2 metrics freeze is a pause that is longer, than maximum: + // 1. 150ms + // 2. 3 * average time between two sequential frames. + // Item 1 will cover high fps video and is a duration, that is noticeable by + // human eye. Item 2 will cover low fps video like screen sharing. + // Freeze duration. + SamplesStatsCounter freeze_time_ms; + // Mean time between one freeze end and next freeze start. + SamplesStatsCounter time_between_freezes_ms; + SamplesStatsCounter resolution_of_decoded_frame; + SamplesStatsCounter target_encode_bitrate; + // Sender side qp values per spatial layer. In case when spatial layer is not + // set for `webrtc::EncodedImage`, 0 is used as default. + std::map<int, SamplesStatsCounter> spatial_layers_qp; + + int64_t total_encoded_images_payload = 0; + // Counters on which phase how many frames were dropped. + std::map<FrameDropPhase, int64_t> dropped_by_phase; + + // Frame count metrics. + int64_t num_send_key_frames = 0; + int64_t num_recv_key_frames = 0; + + // Encoded frame size (in bytes) metrics. + SamplesStatsCounter recv_key_frame_size_bytes; + SamplesStatsCounter recv_delta_frame_size_bytes; + + // Vector of encoders used for this stream by sending client. + std::vector<StreamCodecInfo> encoders; + // Vectors of decoders used for this stream by receiving client. + std::vector<StreamCodecInfo> decoders; +}; + +struct AnalyzerStats { + // Size of analyzer internal comparisons queue, measured when new element + // id added to the queue. + SamplesStatsCounter comparisons_queue_size; + // Number of performed comparisons of 2 video frames from captured and + // rendered streams. + int64_t comparisons_done = 0; + // Number of cpu overloaded comparisons. Comparison is cpu overloaded if it is + // queued when there are too many not processed comparisons in the queue. + // Overloaded comparison doesn't include metrics like SSIM and PSNR that + // require heavy computations. + int64_t cpu_overloaded_comparisons_done = 0; + // Number of memory overloaded comparisons. Comparison is memory overloaded if + // it is queued when its captured frame was already removed due to high memory + // usage for that video stream. + int64_t memory_overloaded_comparisons_done = 0; + // Count of frames in flight in analyzer measured when new comparison is added + // and after analyzer was stopped. + SamplesStatsCounter frames_in_flight_left_count; + + // Next metrics are collected and reported iff + // `DefaultVideoQualityAnalyzerOptions::report_infra_metrics` is true. + SamplesStatsCounter on_frame_captured_processing_time_ms; + SamplesStatsCounter on_frame_pre_encode_processing_time_ms; + SamplesStatsCounter on_frame_encoded_processing_time_ms; + SamplesStatsCounter on_frame_pre_decode_processing_time_ms; + SamplesStatsCounter on_frame_decoded_processing_time_ms; + SamplesStatsCounter on_frame_rendered_processing_time_ms; + SamplesStatsCounter on_decoder_error_processing_time_ms; +}; + +struct StatsKey { + StatsKey(std::string stream_label, std::string receiver) + : stream_label(std::move(stream_label)), receiver(std::move(receiver)) {} + + std::string ToString() const; + + // Label of video stream to which stats belongs to. + std::string stream_label; + // Name of the peer on which stream was received. + std::string receiver; +}; + +// Required to use StatsKey as std::map key. +bool operator<(const StatsKey& a, const StatsKey& b); +bool operator==(const StatsKey& a, const StatsKey& b); + +// Contains all metadata related to the video streams that were seen by the +// video analyzer. +class VideoStreamsInfo { + public: + std::set<StatsKey> GetStatsKeys() const; + + // Returns all stream labels that are known to the video analyzer. + std::set<std::string> GetStreams() const; + + // Returns set of the stream for specified `sender_name`. If sender didn't + // send any streams or `sender_name` isn't known to the video analyzer + // empty set will be returned. + std::set<std::string> GetStreams(absl::string_view sender_name) const; + + // Returns sender name for specified `stream_label`. Returns `absl::nullopt` + // if provided `stream_label` isn't known to the video analyzer. + absl::optional<std::string> GetSender(absl::string_view stream_label) const; + + // Returns set of the receivers for specified `stream_label`. If stream wasn't + // received by any peer or `stream_label` isn't known to the video analyzer + // empty set will be returned. + std::set<std::string> GetReceivers(absl::string_view stream_label) const; + + protected: + friend class DefaultVideoQualityAnalyzer; + VideoStreamsInfo( + std::map<std::string, std::string> stream_to_sender, + std::map<std::string, std::set<std::string>> sender_to_streams, + std::map<std::string, std::set<std::string>> stream_to_receivers); + + private: + std::map<std::string, std::string> stream_to_sender_; + std::map<std::string, std::set<std::string>> sender_to_streams_; + std::map<std::string, std::set<std::string>> stream_to_receivers_; +}; + +struct DefaultVideoQualityAnalyzerOptions { + // Tells DefaultVideoQualityAnalyzer if heavy metrics have to be computed. + bool compute_psnr = true; + bool compute_ssim = true; + // If true, weights the luma plane more than the chroma planes in the PSNR. + bool use_weighted_psnr = false; + // Tells DefaultVideoQualityAnalyzer if detailed frame stats should be + // reported. + bool report_detailed_frame_stats = false; + // Tells DefaultVideoQualityAnalyzer if infra metrics related to the + // performance and stability of the analyzer itself should be reported. + bool report_infra_metrics = false; + // If true DefaultVideoQualityAnalyzer will try to adjust frames before + // computing PSNR and SSIM for them. In some cases picture may be shifted by + // a few pixels after the encode/decode step. Those difference is invisible + // for a human eye, but it affects the metrics. So the adjustment is used to + // get metrics that are closer to how human perceive the video. This feature + // significantly slows down the comparison, so turn it on only when it is + // needed. + bool adjust_cropping_before_comparing_frames = false; + // Amount of frames that are queued in the DefaultVideoQualityAnalyzer from + // the point they were captured to the point they were rendered on all + // receivers per stream. + size_t max_frames_in_flight_per_stream_count = + kDefaultMaxFramesInFlightPerStream; + // If true, the analyzer will expect peers to receive their own video streams. + bool enable_receive_own_stream = false; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_SHARED_OBJECTS_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.cc new file mode 100644 index 0000000000..d59ef12c63 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.cc @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h" + +#include <map> +#include <set> + +#include "absl/types/optional.h" +#include "api/units/timestamp.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +template <typename T> +absl::optional<T> MaybeGetValue(const std::map<size_t, T>& map, size_t key) { + auto it = map.find(key); + if (it == map.end()) { + return absl::nullopt; + } + return it->second; +} + +} // namespace + +StreamState::StreamState(size_t sender, + std::set<size_t> receivers, + Timestamp stream_started_time) + : sender_(sender), + stream_started_time_(stream_started_time), + receivers_(receivers), + frame_ids_(std::move(receivers)) { + frame_ids_.AddReader(kAliveFramesQueueIndex); + RTC_CHECK_NE(sender_, kAliveFramesQueueIndex); + for (size_t receiver : receivers_) { + RTC_CHECK_NE(receiver, kAliveFramesQueueIndex); + } +} + +uint16_t StreamState::PopFront(size_t peer) { + RTC_CHECK_NE(peer, kAliveFramesQueueIndex); + absl::optional<uint16_t> frame_id = frame_ids_.PopFront(peer); + RTC_DCHECK(frame_id.has_value()); + + // If alive's frame queue is longer than all others, than also pop frame from + // it, because that frame is received by all receivers. + size_t alive_size = frame_ids_.size(kAliveFramesQueueIndex); + size_t other_size = GetLongestReceiverQueue(); + // Pops frame from alive queue if alive's queue is the longest one. + if (alive_size > other_size) { + absl::optional<uint16_t> alive_frame_id = + frame_ids_.PopFront(kAliveFramesQueueIndex); + RTC_DCHECK(alive_frame_id.has_value()); + RTC_DCHECK_EQ(frame_id.value(), alive_frame_id.value()); + } + + return frame_id.value(); +} + +void StreamState::AddPeer(size_t peer) { + RTC_CHECK_NE(peer, kAliveFramesQueueIndex); + frame_ids_.AddReader(peer, kAliveFramesQueueIndex); + receivers_.insert(peer); +} + +void StreamState::RemovePeer(size_t peer) { + RTC_CHECK_NE(peer, kAliveFramesQueueIndex); + frame_ids_.RemoveReader(peer); + receivers_.erase(peer); + + // If we removed the last receiver for the alive frames, we need to pop them + // from the queue, because now they received by all receivers. + size_t alive_size = frame_ids_.size(kAliveFramesQueueIndex); + size_t other_size = GetLongestReceiverQueue(); + while (alive_size > other_size) { + frame_ids_.PopFront(kAliveFramesQueueIndex); + alive_size--; + } +} + +uint16_t StreamState::MarkNextAliveFrameAsDead() { + absl::optional<uint16_t> frame_id = + frame_ids_.PopFront(kAliveFramesQueueIndex); + RTC_DCHECK(frame_id.has_value()); + return frame_id.value(); +} + +void StreamState::SetLastRenderedFrameTime(size_t peer, Timestamp time) { + auto it = last_rendered_frame_time_.find(peer); + if (it == last_rendered_frame_time_.end()) { + last_rendered_frame_time_.insert({peer, time}); + } else { + it->second = time; + } +} + +absl::optional<Timestamp> StreamState::last_rendered_frame_time( + size_t peer) const { + return MaybeGetValue(last_rendered_frame_time_, peer); +} + +size_t StreamState::GetLongestReceiverQueue() const { + size_t max = 0; + for (size_t receiver : receivers_) { + size_t cur_size = frame_ids_.size(receiver); + if (cur_size > max) { + max = cur_size; + } + } + return max; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h new file mode 100644 index 0000000000..829a79c7bf --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_STREAM_STATE_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_STREAM_STATE_H_ + +#include <limits> +#include <map> +#include <set> + +#include "absl/types/optional.h" +#include "api/units/timestamp.h" +#include "test/pc/e2e/analyzer/video/multi_reader_queue.h" + +namespace webrtc { + +// Represents a current state of video stream inside +// DefaultVideoQualityAnalyzer. +// +// Maintains the sequence of frames for each video stream and keeps track about +// which frames were seen by each of the possible stream receiver. +// +// Keeps information about which frames are alive and which are dead. Frame is +// alive if it contains VideoFrame payload for corresponding FrameInFlight +// object inside DefaultVideoQualityAnalyzer, otherwise frame is considered +// dead. +// +// Supports peer indexes from 0 to max(size_t) - 1. +class StreamState { + public: + StreamState(size_t sender, + std::set<size_t> receivers, + Timestamp stream_started_time); + + size_t sender() const { return sender_; } + Timestamp stream_started_time() const { return stream_started_time_; } + + void PushBack(uint16_t frame_id) { frame_ids_.PushBack(frame_id); } + // Crash if state is empty. + uint16_t PopFront(size_t peer); + bool IsEmpty(size_t peer) const { return frame_ids_.IsEmpty(peer); } + // Crash if state is empty. + uint16_t Front(size_t peer) const { return frame_ids_.Front(peer).value(); } + + // Adds a new peer to the state. All currently alive frames will be expected + // to be received by the newly added peer. + void AddPeer(size_t peer); + + // Removes peer from the state. Frames that were expected to be received by + // this peer will be removed from it. On the other hand last rendered frame + // time for the removed peer will be preserved, because + // DefaultVideoQualityAnalyzer still may request it for stats processing. + void RemovePeer(size_t peer); + + size_t GetAliveFramesCount() const { + return frame_ids_.size(kAliveFramesQueueIndex); + } + uint16_t MarkNextAliveFrameAsDead(); + + void SetLastRenderedFrameTime(size_t peer, Timestamp time); + absl::optional<Timestamp> last_rendered_frame_time(size_t peer) const; + + private: + // Index of the `frame_ids_` queue which is used to track alive frames for + // this stream. + static constexpr size_t kAliveFramesQueueIndex = + std::numeric_limits<size_t>::max(); + + size_t GetLongestReceiverQueue() const; + + // Index of the owner. Owner's queue in `frame_ids_` will keep alive frames. + const size_t sender_; + const Timestamp stream_started_time_; + std::set<size_t> receivers_; + // To correctly determine dropped frames we have to know sequence of frames + // in each stream so we will keep a list of frame ids inside the stream. + // This list is represented by multi head queue of frame ids with separate + // head for each receiver. When the frame is rendered, we will pop ids from + // the corresponding head until id will match with rendered one. All ids + // before matched one can be considered as dropped: + // + // | frame_id1 |->| frame_id2 |->| frame_id3 |->| frame_id4 | + // + // If we received frame with id frame_id3, then we will pop frame_id1 and + // frame_id2 and consider those frames as dropped and then compare received + // frame with the one from `FrameInFlight` with id frame_id3. + MultiReaderQueue<uint16_t> frame_ids_; + std::map<size_t, Timestamp> last_rendered_frame_time_; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_STREAM_STATE_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state_test.cc new file mode 100644 index 0000000000..01a6aab28a --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state_test.cc @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h" + +#include <set> + +#include "api/units/timestamp.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +TEST(StreamStateTest, PopFrontAndFrontIndependentForEachPeer) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + state.PushBack(/*frame_id=*/2); + + EXPECT_EQ(state.Front(/*peer=*/1), 1); + EXPECT_EQ(state.PopFront(/*peer=*/1), 1); + EXPECT_EQ(state.Front(/*peer=*/1), 2); + EXPECT_EQ(state.PopFront(/*peer=*/1), 2); + EXPECT_EQ(state.Front(/*peer=*/2), 1); + EXPECT_EQ(state.PopFront(/*peer=*/2), 1); + EXPECT_EQ(state.Front(/*peer=*/2), 2); + EXPECT_EQ(state.PopFront(/*peer=*/2), 2); +} + +TEST(StreamStateTest, IsEmpty) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + + EXPECT_FALSE(state.IsEmpty(/*peer=*/1)); + + state.PopFront(/*peer=*/1); + + EXPECT_TRUE(state.IsEmpty(/*peer=*/1)); +} + +TEST(StreamStateTest, PopFrontForOnlyOnePeerDontChangeAliveFramesCount) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + state.PushBack(/*frame_id=*/2); + + EXPECT_EQ(state.GetAliveFramesCount(), 2lu); + + state.PopFront(/*peer=*/1); + state.PopFront(/*peer=*/1); + + EXPECT_EQ(state.GetAliveFramesCount(), 2lu); +} + +TEST(StreamStateTest, PopFrontForAllPeersReducesAliveFramesCount) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + state.PushBack(/*frame_id=*/2); + + EXPECT_EQ(state.GetAliveFramesCount(), 2lu); + + state.PopFront(/*peer=*/1); + state.PopFront(/*peer=*/2); + + EXPECT_EQ(state.GetAliveFramesCount(), 1lu); +} + +TEST(StreamStateTest, RemovePeerForLastExpectedReceiverUpdatesAliveFrames) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + state.PushBack(/*frame_id=*/2); + + state.PopFront(/*peer=*/1); + + EXPECT_EQ(state.GetAliveFramesCount(), 2lu); + + state.RemovePeer(/*peer=*/2); + + EXPECT_EQ(state.GetAliveFramesCount(), 1lu); +} + +TEST(StreamStateTest, MarkNextAliveFrameAsDeadDecreseAliveFramesCount) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + state.PushBack(/*frame_id=*/2); + + EXPECT_EQ(state.GetAliveFramesCount(), 2lu); + + state.MarkNextAliveFrameAsDead(); + + EXPECT_EQ(state.GetAliveFramesCount(), 1lu); +} + +TEST(StreamStateTest, MarkNextAliveFrameAsDeadDoesntAffectFrontFrameForPeer) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + state.PushBack(/*frame_id=*/2); + + EXPECT_EQ(state.Front(/*peer=*/1), 1); + + state.MarkNextAliveFrameAsDead(); + + EXPECT_EQ(state.Front(/*peer=*/1), 1); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc new file mode 100644 index 0000000000..fc970e1ea2 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc @@ -0,0 +1,2204 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" + +#include <algorithm> +#include <map> +#include <memory> +#include <vector> + +#include "api/rtp_packet_info.h" +#include "api/rtp_packet_infos.h" +#include "api/test/create_frame_generator.h" +#include "api/test/metrics/global_metrics_logger_and_exporter.h" +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_tools/frame_analyzer/video_geometry_aligner.h" +#include "system_wrappers/include/sleep.h" +#include "test/gtest.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +namespace webrtc { +namespace { + +using ::testing::TestWithParam; +using ::testing::ValuesIn; + +using StatsSample = ::webrtc::SamplesStatsCounter::StatsSample; + +constexpr int kAnalyzerMaxThreadsCount = 1; +constexpr int kMaxFramesInFlightPerStream = 10; +constexpr int kFrameWidth = 320; +constexpr int kFrameHeight = 240; +constexpr double kMaxSsim = 1; +constexpr char kStreamLabel[] = "video-stream"; +constexpr char kSenderPeerName[] = "alice"; +constexpr char kReceiverPeerName[] = "bob"; + +DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() { + DefaultVideoQualityAnalyzerOptions options; + options.compute_psnr = false; + options.compute_ssim = false; + options.adjust_cropping_before_comparing_frames = false; + options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream; + return options; +} + +VideoFrame NextFrame(test::FrameGeneratorInterface* frame_generator, + int64_t timestamp_us) { + test::FrameGeneratorInterface::VideoFrameData frame_data = + frame_generator->NextFrame(); + return VideoFrame::Builder() + .set_video_frame_buffer(frame_data.buffer) + .set_update_rect(frame_data.update_rect) + .set_timestamp_us(timestamp_us) + .build(); +} + +EncodedImage FakeEncode(const VideoFrame& frame) { + EncodedImage image; + std::vector<RtpPacketInfo> packet_infos; + packet_infos.push_back(RtpPacketInfo( + /*ssrc=*/1, + /*csrcs=*/{}, + /*rtp_timestamp=*/frame.timestamp(), + /*receive_time=*/Timestamp::Micros(frame.timestamp_us() + 10000))); + image.SetPacketInfos(RtpPacketInfos(packet_infos)); + return image; +} + +VideoFrame DeepCopy(const VideoFrame& frame) { + VideoFrame copy = frame; + copy.set_video_frame_buffer( + I420Buffer::Copy(*frame.video_frame_buffer()->ToI420())); + return copy; +} + +std::vector<StatsSample> GetSortedSamples(const SamplesStatsCounter& counter) { + rtc::ArrayView<const StatsSample> view = counter.GetTimedSamples(); + std::vector<StatsSample> out(view.begin(), view.end()); + std::sort(out.begin(), out.end(), + [](const StatsSample& a, const StatsSample& b) { + return a.time < b.time; + }); + return out; +} + +std::string ToString(const std::vector<StatsSample>& values) { + rtc::StringBuilder out; + for (const auto& v : values) { + out << "{ time_ms=" << v.time.ms() << "; value=" << v.value << "}, "; + } + return out.str(); +} + +void FakeCPULoad() { + std::vector<int> temp(1000000); + for (size_t i = 0; i < temp.size(); ++i) { + temp[i] = rand(); + } + std::sort(temp.begin(), temp.end()); + ASSERT_TRUE(std::is_sorted(temp.begin(), temp.end())); +} + +void PassFramesThroughAnalyzer(DefaultVideoQualityAnalyzer& analyzer, + absl::string_view sender, + absl::string_view stream_label, + std::vector<absl::string_view> receivers, + int frames_count, + test::FrameGeneratorInterface& frame_generator, + int interframe_delay_ms = 0) { + for (int i = 0; i < frames_count; ++i) { + VideoFrame frame = NextFrame(&frame_generator, /*timestamp_us=*/1); + uint16_t frame_id = + analyzer.OnFrameCaptured(sender, std::string(stream_label), frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode(sender, frame); + analyzer.OnFrameEncoded(sender, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + for (absl::string_view receiver : receivers) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(receiver, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(receiver, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(receiver, received_frame); + } + if (i < frames_count - 1 && interframe_delay_ms > 0) { + SleepMs(interframe_delay_ms); + } + } +} + +TEST(DefaultVideoQualityAnalyzerTest, + MemoryOverloadedAndThenAllFramesReceived) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + for (int i = 0; i < kMaxFramesInFlightPerStream * 2; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + for (const uint16_t& frame_id : frames_order) { + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, + kMaxFramesInFlightPerStream); + EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream * 2); + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream * 2); + EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream * 2); + EXPECT_EQ(frame_counters.dropped, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, + FillMaxMemoryReceiveAllMemoryOverloadedAndThenAllFramesReceived) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + // Feel analyzer's memory up to limit + for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + // Receive all frames. + for (const uint16_t& frame_id : frames_order) { + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + frames_order.clear(); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + + // Overload analyzer's memory up to limit + for (int i = 0; i < 2 * kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + // Receive all frames. + for (const uint16_t& frame_id : frames_order) { + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, + kMaxFramesInFlightPerStream); + EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream * 3); + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream * 3); + EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream * 3); + EXPECT_EQ(frame_counters.dropped, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, + MemoryOverloadedHalfDroppedAndThenHalfFramesReceived) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + for (int i = 0; i < kMaxFramesInFlightPerStream * 2; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + for (size_t i = kMaxFramesInFlightPerStream; i < frames_order.size(); ++i) { + uint16_t frame_id = frames_order.at(i); + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream * 2); + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream * 2); + EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream); + EXPECT_EQ(frame_counters.dropped, kMaxFramesInFlightPerStream); +} + +TEST(DefaultVideoQualityAnalyzerTest, NormalScenario) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + for (size_t i = 1; i < frames_order.size(); i += 2) { + uint16_t frame_id = frames_order.at(i); + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream); + + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream); + EXPECT_EQ(frame_counters.received, kMaxFramesInFlightPerStream / 2); + EXPECT_EQ(frame_counters.decoded, kMaxFramesInFlightPerStream / 2); + EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream / 2); + EXPECT_EQ(frame_counters.dropped, kMaxFramesInFlightPerStream / 2); +} + +TEST(DefaultVideoQualityAnalyzerTest, OneFrameReceivedTwice) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + VideoFrame captured_frame = NextFrame(frame_generator.get(), 0); + captured_frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, captured_frame)); + analyzer.OnFramePreEncode(kSenderPeerName, captured_frame); + analyzer.OnFrameEncoded(kSenderPeerName, captured_frame.id(), + FakeEncode(captured_frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + VideoFrame received_frame = DeepCopy(captured_frame); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + + received_frame = DeepCopy(captured_frame); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(stats.comparisons_done, 1); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 1); + EXPECT_EQ(frame_counters.received, 1); + EXPECT_EQ(frame_counters.decoded, 1); + EXPECT_EQ(frame_counters.rendered, 1); + EXPECT_EQ(frame_counters.dropped, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, NormalScenario2Receivers) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + constexpr char kAlice[] = "alice"; + constexpr char kBob[] = "bob"; + constexpr char kCharlie[] = "charlie"; + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie}, + kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id(analyzer.OnFrameCaptured(kAlice, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kAlice, frame); + SleepMs(20); + analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + SleepMs(50); + + for (size_t i = 1; i < frames_order.size(); i += 2) { + uint16_t frame_id = frames_order.at(i); + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kBob, received_frame.id(), + FakeEncode(received_frame)); + SleepMs(30); + analyzer.OnFrameDecoded(kBob, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + SleepMs(10); + analyzer.OnFrameRendered(kBob, received_frame); + } + + for (size_t i = 1; i < frames_order.size(); i += 2) { + uint16_t frame_id = frames_order.at(i); + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kCharlie, received_frame.id(), + FakeEncode(received_frame)); + SleepMs(40); + analyzer.OnFrameDecoded(kCharlie, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + SleepMs(5); + analyzer.OnFrameRendered(kCharlie, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(analyzer_stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(analyzer_stats.comparisons_done, kMaxFramesInFlightPerStream * 2); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream); + EXPECT_EQ(frame_counters.received, kMaxFramesInFlightPerStream); + EXPECT_EQ(frame_counters.decoded, kMaxFramesInFlightPerStream); + EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream); + EXPECT_EQ(frame_counters.dropped, kMaxFramesInFlightPerStream); + + VideoStreamsInfo streams_info = analyzer.GetKnownStreams(); + EXPECT_EQ(streams_info.GetStreams(), std::set<std::string>{kStreamLabel}); + EXPECT_EQ(streams_info.GetStreams(kAlice), + std::set<std::string>{kStreamLabel}); + EXPECT_EQ(streams_info.GetSender(kStreamLabel), kAlice); + EXPECT_EQ(streams_info.GetReceivers(kStreamLabel), + (std::set<std::string>{kBob, kCharlie})); + + EXPECT_EQ(streams_info.GetStatsKeys().size(), 2lu); + for (auto stream_key : streams_info.GetStatsKeys()) { + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(stream_key); + // On some devices the pipeline can be too slow, so we actually can't + // force real constraints here. Lets just check, that at least 1 + // frame passed whole pipeline. + EXPECT_GE(stream_conters.captured, 10); + EXPECT_GE(stream_conters.pre_encoded, 10); + EXPECT_GE(stream_conters.encoded, 10); + EXPECT_GE(stream_conters.received, 5); + EXPECT_GE(stream_conters.decoded, 5); + EXPECT_GE(stream_conters.rendered, 5); + EXPECT_GE(stream_conters.dropped, 5); + } + + std::map<StatsKey, StreamStats> stats = analyzer.GetStats(); + const StatsKey kAliceBobStats(kStreamLabel, kBob); + const StatsKey kAliceCharlieStats(kStreamLabel, kCharlie); + EXPECT_EQ(stats.size(), 2lu); + { + auto it = stats.find(kAliceBobStats); + EXPECT_FALSE(it == stats.end()); + ASSERT_FALSE(it->second.encode_time_ms.IsEmpty()); + EXPECT_GE(it->second.encode_time_ms.GetMin(), 20); + ASSERT_FALSE(it->second.decode_time_ms.IsEmpty()); + EXPECT_GE(it->second.decode_time_ms.GetMin(), 30); + ASSERT_FALSE(it->second.resolution_of_decoded_frame.IsEmpty()); + EXPECT_GE(it->second.resolution_of_decoded_frame.GetMin(), + kFrameWidth * kFrameHeight - 1); + EXPECT_LE(it->second.resolution_of_decoded_frame.GetMax(), + kFrameWidth * kFrameHeight + 1); + } + { + auto it = stats.find(kAliceCharlieStats); + EXPECT_FALSE(it == stats.end()); + ASSERT_FALSE(it->second.encode_time_ms.IsEmpty()); + EXPECT_GE(it->second.encode_time_ms.GetMin(), 20); + ASSERT_FALSE(it->second.decode_time_ms.IsEmpty()); + EXPECT_GE(it->second.decode_time_ms.GetMin(), 30); + ASSERT_FALSE(it->second.resolution_of_decoded_frame.IsEmpty()); + EXPECT_GE(it->second.resolution_of_decoded_frame.GetMin(), + kFrameWidth * kFrameHeight - 1); + EXPECT_LE(it->second.resolution_of_decoded_frame.GetMax(), + kFrameWidth * kFrameHeight + 1); + } +} + +// Test the case which can happen when SFU is switching from one layer to +// another, so the same frame can be received twice by the same peer. +TEST(DefaultVideoQualityAnalyzerTest, + OneFrameReceivedTwiceBySamePeerWith2Receivers) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + constexpr char kAlice[] = "alice"; + constexpr char kBob[] = "bob"; + constexpr char kCharlie[] = "charlie"; + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie}, + kAnalyzerMaxThreadsCount); + + VideoFrame captured_frame = NextFrame(frame_generator.get(), 0); + captured_frame.set_id( + analyzer.OnFrameCaptured(kAlice, kStreamLabel, captured_frame)); + analyzer.OnFramePreEncode(kAlice, captured_frame); + analyzer.OnFrameEncoded(kAlice, captured_frame.id(), + FakeEncode(captured_frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + VideoFrame received_frame = DeepCopy(captured_frame); + analyzer.OnFramePreDecode(kBob, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kBob, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kBob, received_frame); + + received_frame = DeepCopy(captured_frame); + analyzer.OnFramePreDecode(kBob, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kBob, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kBob, received_frame); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + // We have 2 comparisons here because 1 for the frame received by Bob and + // 1 for the frame in flight from Alice to Charlie. + EXPECT_EQ(stats.comparisons_done, 2); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 1); + EXPECT_EQ(frame_counters.received, 1); + EXPECT_EQ(frame_counters.decoded, 1); + EXPECT_EQ(frame_counters.rendered, 1); + EXPECT_EQ(frame_counters.dropped, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, HeavyQualityMetricsFromEqualFrames) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions analyzer_options; + analyzer_options.compute_psnr = true; + analyzer_options.compute_ssim = true; + analyzer_options.adjust_cropping_before_comparing_frames = false; + analyzer_options.max_frames_in_flight_per_stream_count = + kMaxFramesInFlightPerStream; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + analyzer_options); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. Heavy metrics + // computation is turned on, so giving some extra time to be sure that + // computatio have ended. + SleepMs(500); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream); + + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + std::map<StatsKey, StreamStats> stream_stats = analyzer.GetStats(); + const StatsKey kAliceBobStats(kStreamLabel, kReceiverPeerName); + EXPECT_EQ(stream_stats.size(), 1lu); + + auto it = stream_stats.find(kAliceBobStats); + EXPECT_GE(it->second.psnr.GetMin(), kPerfectPSNR); + EXPECT_GE(it->second.ssim.GetMin(), kMaxSsim); +} + +TEST(DefaultVideoQualityAnalyzerTest, + HeavyQualityMetricsFromShiftedFramesWithAdjustment) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions analyzer_options; + analyzer_options.compute_psnr = true; + analyzer_options.compute_ssim = true; + analyzer_options.adjust_cropping_before_comparing_frames = true; + analyzer_options.max_frames_in_flight_per_stream_count = + kMaxFramesInFlightPerStream; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + analyzer_options); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + + VideoFrame received_frame = frame; + // Shift frame by a few pixels. + test::CropRegion crop_region{0, 1, 3, 0}; + rtc::scoped_refptr<VideoFrameBuffer> cropped_buffer = + CropAndZoom(crop_region, received_frame.video_frame_buffer()->ToI420()); + received_frame.set_video_frame_buffer(cropped_buffer); + + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. Heavy metrics + // computation is turned on, so giving some extra time to be sure that + // computatio have ended. + SleepMs(500); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream); + + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + std::map<StatsKey, StreamStats> stream_stats = analyzer.GetStats(); + const StatsKey kAliceBobStats(kStreamLabel, kReceiverPeerName); + EXPECT_EQ(stream_stats.size(), 1lu); + + auto it = stream_stats.find(kAliceBobStats); + EXPECT_GE(it->second.psnr.GetMin(), kPerfectPSNR); + EXPECT_GE(it->second.ssim.GetMin(), kMaxSsim); +} + +TEST(DefaultVideoQualityAnalyzerTest, CpuUsage) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + // Windows CPU clock has low accuracy. We need to fake some additional load to + // be sure that the clock ticks (https://crbug.com/webrtc/12249). + FakeCPULoad(); + + for (size_t i = 1; i < frames_order.size(); i += 2) { + uint16_t frame_id = frames_order.at(i); + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + double cpu_usage = analyzer.GetCpuUsagePercent(); + ASSERT_GT(cpu_usage, 0); + + SleepMs(100); + analyzer.Stop(); + + EXPECT_EQ(analyzer.GetCpuUsagePercent(), cpu_usage); +} + +TEST(DefaultVideoQualityAnalyzerTest, RuntimeParticipantsAdding) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + constexpr char kAlice[] = "alice"; + constexpr char kBob[] = "bob"; + constexpr char kCharlie[] = "charlie"; + constexpr char kKatie[] = "katie"; + + constexpr int kFramesCount = 9; + constexpr int kOneThirdFrames = kFramesCount / 3; + constexpr int kTwoThirdFrames = 2 * kOneThirdFrames; + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", {}, kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + analyzer.RegisterParticipantInCall(kAlice); + analyzer.RegisterParticipantInCall(kBob); + + // Alice is sending frames. + for (int i = 0; i < kFramesCount; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id(analyzer.OnFrameCaptured(kAlice, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kAlice, frame); + analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + // Bob receives one third of the sent frames. + for (int i = 0; i < kOneThirdFrames; ++i) { + uint16_t frame_id = frames_order.at(i); + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kBob, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kBob, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kBob, received_frame); + } + + analyzer.RegisterParticipantInCall(kCharlie); + analyzer.RegisterParticipantInCall(kKatie); + + // New participants were dynamically added. Bob and Charlie receive second + // third of the sent frames. Katie drops the frames. + for (int i = kOneThirdFrames; i < kTwoThirdFrames; ++i) { + uint16_t frame_id = frames_order.at(i); + VideoFrame bob_received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kBob, bob_received_frame.id(), + FakeEncode(bob_received_frame)); + analyzer.OnFrameDecoded(kBob, bob_received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kBob, bob_received_frame); + + VideoFrame charlie_received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kCharlie, charlie_received_frame.id(), + FakeEncode(charlie_received_frame)); + analyzer.OnFrameDecoded(kCharlie, charlie_received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kCharlie, charlie_received_frame); + } + + // Bob, Charlie and Katie receive the rest of the sent frames. + for (int i = kTwoThirdFrames; i < kFramesCount; ++i) { + uint16_t frame_id = frames_order.at(i); + VideoFrame bob_received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kBob, bob_received_frame.id(), + FakeEncode(bob_received_frame)); + analyzer.OnFrameDecoded(kBob, bob_received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kBob, bob_received_frame); + + VideoFrame charlie_received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kCharlie, charlie_received_frame.id(), + FakeEncode(charlie_received_frame)); + analyzer.OnFrameDecoded(kCharlie, charlie_received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kCharlie, charlie_received_frame); + + VideoFrame katie_received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kKatie, katie_received_frame.id(), + FakeEncode(katie_received_frame)); + analyzer.OnFrameDecoded(kKatie, katie_received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kKatie, katie_received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(stats.comparisons_done, kFramesCount + 2 * kTwoThirdFrames); + + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, kFramesCount); + EXPECT_EQ(frame_counters.received, 2 * kFramesCount); + EXPECT_EQ(frame_counters.decoded, 2 * kFramesCount); + EXPECT_EQ(frame_counters.rendered, 2 * kFramesCount); + EXPECT_EQ(frame_counters.dropped, kOneThirdFrames); + + const StatsKey kAliceBobStats(kStreamLabel, kBob); + const StatsKey kAliceCharlieStats(kStreamLabel, kCharlie); + const StatsKey kAliceKatieStats(kStreamLabel, kKatie); + EXPECT_EQ(analyzer.GetKnownStreams().GetStatsKeys(), + (std::set<StatsKey>{kAliceBobStats, kAliceCharlieStats, + kAliceKatieStats})); + { + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(kAliceBobStats); + EXPECT_EQ(stream_conters.captured, kFramesCount); + EXPECT_EQ(stream_conters.pre_encoded, kFramesCount); + EXPECT_EQ(stream_conters.encoded, kFramesCount); + EXPECT_EQ(stream_conters.received, kFramesCount); + EXPECT_EQ(stream_conters.decoded, kFramesCount); + EXPECT_EQ(stream_conters.rendered, kFramesCount); + } + { + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(kAliceCharlieStats); + EXPECT_EQ(stream_conters.captured, kFramesCount); + EXPECT_EQ(stream_conters.pre_encoded, kFramesCount); + EXPECT_EQ(stream_conters.encoded, kFramesCount); + EXPECT_EQ(stream_conters.received, kTwoThirdFrames); + EXPECT_EQ(stream_conters.decoded, kTwoThirdFrames); + EXPECT_EQ(stream_conters.rendered, kTwoThirdFrames); + } + { + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(kAliceKatieStats); + EXPECT_EQ(stream_conters.captured, kFramesCount); + EXPECT_EQ(stream_conters.pre_encoded, kFramesCount); + EXPECT_EQ(stream_conters.encoded, kFramesCount); + EXPECT_EQ(stream_conters.received, kOneThirdFrames); + EXPECT_EQ(stream_conters.decoded, kOneThirdFrames); + EXPECT_EQ(stream_conters.rendered, kOneThirdFrames); + } +} + +TEST(DefaultVideoQualityAnalyzerTest, + SimulcastFrameWasFullyReceivedByAllPeersBeforeEncodeFinish) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + constexpr char kAlice[] = "alice"; + constexpr char kBob[] = "bob"; + constexpr char kCharlie[] = "charlie"; + analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), 1); + + frame.set_id(analyzer.OnFrameCaptured(kAlice, kStreamLabel, frame)); + analyzer.OnFramePreEncode(kAlice, frame); + // Encode 1st simulcast layer + analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + // Receive by Bob + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kBob, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kBob, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kBob, received_frame); + // Receive by Charlie + received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kCharlie, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kCharlie, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kCharlie, received_frame); + + // Encode 2nd simulcast layer + analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.comparisons_done, 2); + + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 1); + EXPECT_EQ(frame_counters.rendered, 2); +} + +TEST(DefaultVideoQualityAnalyzerTest, + FrameCanBeReceivedBySenderAfterItWasReceivedByReceiver) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::vector<VideoFrame> frames; + for (int i = 0; i < 3; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), 1); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames.push_back(frame); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + // Receive by 2nd peer. + for (VideoFrame& frame : frames) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Check that we still have that frame in flight. + AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats(); + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(analyzer_stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 3) + << "Expected that frame is still in flight, " + << "because it wasn't received by sender" + << ToString(frames_in_flight_sizes); + + // Receive by sender + for (VideoFrame& frame : frames) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kSenderPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kSenderPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kSenderPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + analyzer_stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(analyzer_stats.comparisons_done, 6); + + frames_in_flight_sizes = + GetSortedSamples(analyzer_stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 3); + EXPECT_EQ(frame_counters.rendered, 6); + + EXPECT_EQ(analyzer.GetStats().size(), 2lu); + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kReceiverPeerName)); + EXPECT_EQ(stream_conters.captured, 3); + EXPECT_EQ(stream_conters.pre_encoded, 3); + EXPECT_EQ(stream_conters.encoded, 3); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 3); + EXPECT_EQ(stream_conters.rendered, 3); + } + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kSenderPeerName)); + EXPECT_EQ(stream_conters.captured, 3); + EXPECT_EQ(stream_conters.pre_encoded, 3); + EXPECT_EQ(stream_conters.encoded, 3); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 3); + EXPECT_EQ(stream_conters.rendered, 3); + } +} + +TEST(DefaultVideoQualityAnalyzerTest, + FrameCanBeReceivedByReceiverAfterItWasReceivedBySender) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::vector<VideoFrame> frames; + for (int i = 0; i < 3; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), 1); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames.push_back(frame); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + + // Receive by sender + for (VideoFrame& frame : frames) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kSenderPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kSenderPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kSenderPeerName, received_frame); + } + + // Check that we still have that frame in flight. + AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats(); + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(analyzer_stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 3) + << "Expected that frame is still in flight, " + << "because it wasn't received by sender" + << ToString(frames_in_flight_sizes); + + // Receive by 2nd peer. + for (VideoFrame& frame : frames) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + analyzer_stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(analyzer_stats.comparisons_done, 6); + + frames_in_flight_sizes = + GetSortedSamples(analyzer_stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 3); + EXPECT_EQ(frame_counters.rendered, 6); + + EXPECT_EQ(analyzer.GetStats().size(), 2lu); + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kReceiverPeerName)); + EXPECT_EQ(stream_conters.captured, 3); + EXPECT_EQ(stream_conters.pre_encoded, 3); + EXPECT_EQ(stream_conters.encoded, 3); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 3); + EXPECT_EQ(stream_conters.rendered, 3); + } + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kSenderPeerName)); + EXPECT_EQ(stream_conters.captured, 3); + EXPECT_EQ(stream_conters.pre_encoded, 3); + EXPECT_EQ(stream_conters.encoded, 3); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 3); + EXPECT_EQ(stream_conters.rendered, 3); + } +} + +TEST(DefaultVideoQualityAnalyzerTest, CodecTrackedCorrectly) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + VideoQualityAnalyzerInterface::EncoderStats encoder_stats; + std::vector<std::string> codec_names = {"codec_1", "codec_2"}; + std::vector<VideoFrame> frames; + // Send 3 frame for each codec. + for (size_t i = 0; i < codec_names.size(); ++i) { + for (size_t j = 0; j < 3; ++j) { + VideoFrame frame = NextFrame(frame_generator.get(), 3 * i + j); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + encoder_stats.encoder_name = codec_names[i]; + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + encoder_stats, false); + frames.push_back(std::move(frame)); + } + } + + // Receive 3 frame for each codec. + VideoQualityAnalyzerInterface::DecoderStats decoder_stats; + for (size_t i = 0; i < codec_names.size(); ++i) { + for (size_t j = 0; j < 3; ++j) { + VideoFrame received_frame = DeepCopy(frames[3 * i + j]); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + decoder_stats.decoder_name = codec_names[i]; + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, decoder_stats); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + std::map<StatsKey, StreamStats> stats = analyzer.GetStats(); + ASSERT_EQ(stats.size(), 1lu); + const StreamStats& stream_stats = + stats.at(StatsKey(kStreamLabel, kReceiverPeerName)); + ASSERT_EQ(stream_stats.encoders.size(), 2lu); + EXPECT_EQ(stream_stats.encoders[0].codec_name, codec_names[0]); + EXPECT_EQ(stream_stats.encoders[0].first_frame_id, frames[0].id()); + EXPECT_EQ(stream_stats.encoders[0].last_frame_id, frames[2].id()); + EXPECT_EQ(stream_stats.encoders[1].codec_name, codec_names[1]); + EXPECT_EQ(stream_stats.encoders[1].first_frame_id, frames[3].id()); + EXPECT_EQ(stream_stats.encoders[1].last_frame_id, frames[5].id()); + + ASSERT_EQ(stream_stats.decoders.size(), 2lu); + EXPECT_EQ(stream_stats.decoders[0].codec_name, codec_names[0]); + EXPECT_EQ(stream_stats.decoders[0].first_frame_id, frames[0].id()); + EXPECT_EQ(stream_stats.decoders[0].last_frame_id, frames[2].id()); + EXPECT_EQ(stream_stats.decoders[1].codec_name, codec_names[1]); + EXPECT_EQ(stream_stats.decoders[1].first_frame_id, frames[3].id()); + EXPECT_EQ(stream_stats.decoders[1].last_frame_id, frames[5].id()); +} + +TEST(DefaultVideoQualityAnalyzerTest, + FramesInFlightAreCorrectlySentToTheComparatorAfterStop) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + // There are 7 different timings inside frame stats: captured, pre_encode, + // encoded, received, decode_start, decode_end, rendered. captured is always + // set and received is set together with decode_start. So we create 6 + // different frames, where for each frame next timings will be set + // * 1st - all of them set + // * 2nd - captured, pre_encode, encoded, received, decode_start, decode_end + // * 3rd - captured, pre_encode, encoded, received, decode_start + // * 4th - captured, pre_encode, encoded + // * 5th - captured, pre_encode + // * 6th - captured + std::vector<VideoFrame> frames; + // Sender side actions + for (int i = 0; i < 6; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), 1); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames.push_back(frame); + } + for (int i = 0; i < 5; ++i) { + analyzer.OnFramePreEncode(kSenderPeerName, frames[i]); + } + for (int i = 0; i < 4; ++i) { + analyzer.OnFrameEncoded( + kSenderPeerName, frames[i].id(), FakeEncode(frames[i]), + VideoQualityAnalyzerInterface::EncoderStats(), false); + } + + // Receiver side actions + for (int i = 0; i < 3; ++i) { + analyzer.OnFramePreDecode(kReceiverPeerName, frames[i].id(), + FakeEncode(frames[i])); + } + for (int i = 0; i < 2; ++i) { + analyzer.OnFrameDecoded(kReceiverPeerName, DeepCopy(frames[i]), + VideoQualityAnalyzerInterface::DecoderStats()); + } + for (int i = 0; i < 1; ++i) { + analyzer.OnFrameRendered(kReceiverPeerName, DeepCopy(frames[i])); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(analyzer_stats.comparisons_done, 6); + + // The last frames in flight size has to reflect the amount of frame in flight + // before all of them were sent to the comparison when Stop() was invoked. + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(analyzer_stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 5) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 6); + EXPECT_EQ(frame_counters.pre_encoded, 5); + EXPECT_EQ(frame_counters.encoded, 4); + EXPECT_EQ(frame_counters.received, 3); + EXPECT_EQ(frame_counters.decoded, 2); + EXPECT_EQ(frame_counters.rendered, 1); + + EXPECT_EQ(analyzer.GetStats().size(), 1lu); + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kReceiverPeerName)); + EXPECT_EQ(stream_conters.captured, 6); + EXPECT_EQ(stream_conters.pre_encoded, 5); + EXPECT_EQ(stream_conters.encoded, 4); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 2); + EXPECT_EQ(stream_conters.rendered, 1); + } +} + +TEST( + DefaultVideoQualityAnalyzerTest, + FramesInFlightAreCorrectlySentToTheComparatorAfterStopForSenderAndReceiver) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + // There are 7 different timings inside frame stats: captured, pre_encode, + // encoded, received, decode_start, decode_end, rendered. captured is always + // set and received is set together with decode_start. So we create 6 + // different frames, where for each frame next timings will be set + // * 1st - all of them set + // * 2nd - captured, pre_encode, encoded, received, decode_start, decode_end + // * 3rd - captured, pre_encode, encoded, received, decode_start + // * 4th - captured, pre_encode, encoded + // * 5th - captured, pre_encode + // * 6th - captured + std::vector<VideoFrame> frames; + // Sender side actions + for (int i = 0; i < 6; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), 1); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames.push_back(frame); + } + for (int i = 0; i < 5; ++i) { + analyzer.OnFramePreEncode(kSenderPeerName, frames[i]); + } + for (int i = 0; i < 4; ++i) { + analyzer.OnFrameEncoded( + kSenderPeerName, frames[i].id(), FakeEncode(frames[i]), + VideoQualityAnalyzerInterface::EncoderStats(), false); + } + + // Receiver side actions + for (int i = 0; i < 3; ++i) { + analyzer.OnFramePreDecode(kSenderPeerName, frames[i].id(), + FakeEncode(frames[i])); + analyzer.OnFramePreDecode(kReceiverPeerName, frames[i].id(), + FakeEncode(frames[i])); + } + for (int i = 0; i < 2; ++i) { + analyzer.OnFrameDecoded(kSenderPeerName, DeepCopy(frames[i]), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameDecoded(kReceiverPeerName, DeepCopy(frames[i]), + VideoQualityAnalyzerInterface::DecoderStats()); + } + for (int i = 0; i < 1; ++i) { + analyzer.OnFrameRendered(kSenderPeerName, DeepCopy(frames[i])); + analyzer.OnFrameRendered(kReceiverPeerName, DeepCopy(frames[i])); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(analyzer_stats.comparisons_done, 12); + + // The last frames in flight size has to reflect the amount of frame in flight + // before all of them were sent to the comparison when Stop() was invoked. + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(analyzer_stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 5) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 6); + EXPECT_EQ(frame_counters.pre_encoded, 5); + EXPECT_EQ(frame_counters.encoded, 4); + EXPECT_EQ(frame_counters.received, 6); + EXPECT_EQ(frame_counters.decoded, 4); + EXPECT_EQ(frame_counters.rendered, 2); + + EXPECT_EQ(analyzer.GetStats().size(), 2lu); + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kReceiverPeerName)); + EXPECT_EQ(stream_conters.captured, 6); + EXPECT_EQ(stream_conters.pre_encoded, 5); + EXPECT_EQ(stream_conters.encoded, 4); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 2); + EXPECT_EQ(stream_conters.rendered, 1); + } + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kSenderPeerName)); + EXPECT_EQ(stream_conters.captured, 6); + EXPECT_EQ(stream_conters.pre_encoded, 5); + EXPECT_EQ(stream_conters.encoded, 4); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 2); + EXPECT_EQ(stream_conters.rendered, 1); + } +} + +TEST(DefaultVideoQualityAnalyzerTest, GetStreamFrames) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + // The order in which peers captured frames and passed them to analyzer. + std::vector<std::string> frame_capturers_sequence{ + "alice", "alice", "bob", "bob", "bob", + "bob", "bob", "alice", "alice", "alice", + }; + + std::map<std::string, std::vector<uint16_t>> stream_to_frame_ids; + stream_to_frame_ids.emplace("alice_video", std::vector<uint16_t>{}); + stream_to_frame_ids.emplace("bob_video", std::vector<uint16_t>{}); + + std::vector<VideoFrame> frames; + for (const std::string& sender : frame_capturers_sequence) { + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = + analyzer.OnFrameCaptured(sender, sender + "_video", frame); + frame.set_id(frame_id); + stream_to_frame_ids.find(sender + "_video")->second.push_back(frame_id); + frames.push_back(frame); + analyzer.OnFramePreEncode(sender, frame); + analyzer.OnFrameEncoded(sender, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), + false); + } + // We don't need to receive frames for stats to be gathered correctly. + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + EXPECT_EQ(analyzer.GetStreamFrames(), stream_to_frame_ids); +} + +TEST(DefaultVideoQualityAnalyzerTest, ReceiverReceivedFramesWhenSenderRemoved) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + analyzer.UnregisterParticipantInCall("alice"); + + analyzer.OnFramePreDecode("bob", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("bob", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("bob", DeepCopy(frame)); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 1); + EXPECT_EQ(stream_conters.decoded, 1); + EXPECT_EQ(stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, + ReceiverReceivedFramesWhenSenderRemovedWithSelfview) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + analyzer.UnregisterParticipantInCall("alice"); + + analyzer.OnFramePreDecode("bob", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("bob", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("bob", DeepCopy(frame)); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 1); + EXPECT_EQ(stream_conters.decoded, 1); + EXPECT_EQ(stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, + SenderReceivedFramesWhenReceiverRemovedWithSelfview) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + analyzer.UnregisterParticipantInCall("bob"); + + analyzer.OnFramePreDecode("alice", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("alice", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("alice", DeepCopy(frame)); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "alice")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 1); + EXPECT_EQ(stream_conters.decoded, 1); + EXPECT_EQ(stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, + SenderAndReceiverReceivedFramesWhenReceiverRemovedWithSelfview) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + analyzer.OnFramePreDecode("bob", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("bob", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("bob", DeepCopy(frame)); + + analyzer.UnregisterParticipantInCall("bob"); + + analyzer.OnFramePreDecode("alice", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("alice", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("alice", DeepCopy(frame)); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters alice_alice_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "alice")); + EXPECT_EQ(alice_alice_stream_conters.captured, 1); + EXPECT_EQ(alice_alice_stream_conters.pre_encoded, 1); + EXPECT_EQ(alice_alice_stream_conters.encoded, 1); + EXPECT_EQ(alice_alice_stream_conters.received, 1); + EXPECT_EQ(alice_alice_stream_conters.decoded, 1); + EXPECT_EQ(alice_alice_stream_conters.rendered, 1); + + FrameCounters alice_bob_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(alice_bob_stream_conters.captured, 1); + EXPECT_EQ(alice_bob_stream_conters.pre_encoded, 1); + EXPECT_EQ(alice_bob_stream_conters.encoded, 1); + EXPECT_EQ(alice_bob_stream_conters.received, 1); + EXPECT_EQ(alice_bob_stream_conters.decoded, 1); + EXPECT_EQ(alice_bob_stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeCapturing2ndFrame) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"}, + /*frames_count=*/1, *frame_generator); + analyzer.UnregisterParticipantInCall("bob"); + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {}, + /*frames_count=*/1, *frame_generator); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 2); + EXPECT_EQ(global_stream_conters.pre_encoded, 2); + EXPECT_EQ(global_stream_conters.encoded, 2); + EXPECT_EQ(global_stream_conters.received, 1); + EXPECT_EQ(global_stream_conters.decoded, 1); + EXPECT_EQ(global_stream_conters.rendered, 1); + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 2); + EXPECT_EQ(stream_conters.pre_encoded, 2); + EXPECT_EQ(stream_conters.encoded, 2); + EXPECT_EQ(stream_conters.received, 1); + EXPECT_EQ(stream_conters.decoded, 1); + EXPECT_EQ(stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforePreEncoded) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.UnregisterParticipantInCall("bob"); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 1); + EXPECT_EQ(global_stream_conters.pre_encoded, 1); + EXPECT_EQ(global_stream_conters.encoded, 1); + EXPECT_EQ(global_stream_conters.received, 0); + EXPECT_EQ(global_stream_conters.decoded, 0); + EXPECT_EQ(global_stream_conters.rendered, 0); + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 0); + EXPECT_EQ(stream_conters.decoded, 0); + EXPECT_EQ(stream_conters.rendered, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeEncoded) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.UnregisterParticipantInCall("bob"); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 1); + EXPECT_EQ(global_stream_conters.pre_encoded, 1); + EXPECT_EQ(global_stream_conters.encoded, 1); + EXPECT_EQ(global_stream_conters.received, 0); + EXPECT_EQ(global_stream_conters.decoded, 0); + EXPECT_EQ(global_stream_conters.rendered, 0); + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 0); + EXPECT_EQ(stream_conters.decoded, 0); + EXPECT_EQ(stream_conters.rendered, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, + ReceiverRemovedBetweenSimulcastLayersEncoded) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + // 1st simulcast layer encoded + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + analyzer.UnregisterParticipantInCall("bob"); + // 2nd simulcast layer encoded + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 1); + EXPECT_EQ(global_stream_conters.pre_encoded, 1); + EXPECT_EQ(global_stream_conters.encoded, 1); + EXPECT_EQ(global_stream_conters.received, 0); + EXPECT_EQ(global_stream_conters.decoded, 0); + EXPECT_EQ(global_stream_conters.rendered, 0); + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 0); + EXPECT_EQ(stream_conters.decoded, 0); + EXPECT_EQ(stream_conters.rendered, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, UnregisterOneAndRegisterAnother) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", + std::vector<std::string>{"alice", "bob", "charlie"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"bob", "charlie"}, + /*frames_count=*/2, *frame_generator); + analyzer.UnregisterParticipantInCall("bob"); + analyzer.RegisterParticipantInCall("david"); + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"charlie", "david"}, + /*frames_count=*/4, *frame_generator); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 6); + EXPECT_EQ(global_stream_conters.pre_encoded, 6); + EXPECT_EQ(global_stream_conters.encoded, 6); + EXPECT_EQ(global_stream_conters.received, 12); + EXPECT_EQ(global_stream_conters.decoded, 12); + EXPECT_EQ(global_stream_conters.rendered, 12); + FrameCounters alice_bob_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(alice_bob_stream_conters.captured, 6); + EXPECT_EQ(alice_bob_stream_conters.pre_encoded, 6); + EXPECT_EQ(alice_bob_stream_conters.encoded, 6); + EXPECT_EQ(alice_bob_stream_conters.received, 2); + EXPECT_EQ(alice_bob_stream_conters.decoded, 2); + EXPECT_EQ(alice_bob_stream_conters.rendered, 2); + FrameCounters alice_charlie_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "charlie")); + EXPECT_EQ(alice_charlie_stream_conters.captured, 6); + EXPECT_EQ(alice_charlie_stream_conters.pre_encoded, 6); + EXPECT_EQ(alice_charlie_stream_conters.encoded, 6); + EXPECT_EQ(alice_charlie_stream_conters.received, 6); + EXPECT_EQ(alice_charlie_stream_conters.decoded, 6); + EXPECT_EQ(alice_charlie_stream_conters.rendered, 6); + FrameCounters alice_david_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "david")); + EXPECT_EQ(alice_david_stream_conters.captured, 6); + EXPECT_EQ(alice_david_stream_conters.pre_encoded, 6); + EXPECT_EQ(alice_david_stream_conters.encoded, 6); + EXPECT_EQ(alice_david_stream_conters.received, 4); + EXPECT_EQ(alice_david_stream_conters.decoded, 4); + EXPECT_EQ(alice_david_stream_conters.rendered, 4); +} + +TEST(DefaultVideoQualityAnalyzerTest, + UnregisterOneAndRegisterAnotherRegisterBack) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", + std::vector<std::string>{"alice", "bob", "charlie"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"bob", "charlie"}, + /*frames_count=*/2, *frame_generator); + analyzer.UnregisterParticipantInCall("bob"); + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"charlie"}, + /*frames_count=*/4, *frame_generator); + analyzer.RegisterParticipantInCall("bob"); + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"bob", "charlie"}, + /*frames_count=*/6, *frame_generator); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 12); + EXPECT_EQ(global_stream_conters.pre_encoded, 12); + EXPECT_EQ(global_stream_conters.encoded, 12); + EXPECT_EQ(global_stream_conters.received, 20); + EXPECT_EQ(global_stream_conters.decoded, 20); + EXPECT_EQ(global_stream_conters.rendered, 20); + FrameCounters alice_bob_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(alice_bob_stream_conters.captured, 12); + EXPECT_EQ(alice_bob_stream_conters.pre_encoded, 12); + EXPECT_EQ(alice_bob_stream_conters.encoded, 12); + EXPECT_EQ(alice_bob_stream_conters.received, 8); + EXPECT_EQ(alice_bob_stream_conters.decoded, 8); + EXPECT_EQ(alice_bob_stream_conters.rendered, 8); + FrameCounters alice_charlie_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "charlie")); + EXPECT_EQ(alice_charlie_stream_conters.captured, 12); + EXPECT_EQ(alice_charlie_stream_conters.pre_encoded, 12); + EXPECT_EQ(alice_charlie_stream_conters.encoded, 12); + EXPECT_EQ(alice_charlie_stream_conters.received, 12); + EXPECT_EQ(alice_charlie_stream_conters.decoded, 12); + EXPECT_EQ(alice_charlie_stream_conters.rendered, 12); +} + +TEST(DefaultVideoQualityAnalyzerTest, + FramesInFlightAreAccountedForUnregisterPeers) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + // Add one frame in flight which has encode time >= 10ms. + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + SleepMs(10); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + + analyzer.UnregisterParticipantInCall("bob"); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + StreamStats stats = analyzer.GetStats().at(StatsKey("alice_video", "bob")); + ASSERT_EQ(stats.encode_time_ms.NumSamples(), 1); + EXPECT_GE(stats.encode_time_ms.GetAverage(), 10); +} + +TEST(DefaultVideoQualityAnalyzerTest, InfraMetricsAreReportedWhenRequested) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.report_infra_metrics = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"}, + /*frames_count=*/1, *frame_generator); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.on_frame_captured_processing_time_ms.NumSamples(), 1); + EXPECT_EQ(stats.on_frame_pre_encode_processing_time_ms.NumSamples(), 1); + EXPECT_EQ(stats.on_frame_encoded_processing_time_ms.NumSamples(), 1); + EXPECT_EQ(stats.on_frame_pre_decode_processing_time_ms.NumSamples(), 1); + EXPECT_EQ(stats.on_frame_decoded_processing_time_ms.NumSamples(), 1); + EXPECT_EQ(stats.on_frame_rendered_processing_time_ms.NumSamples(), 1); + EXPECT_EQ(stats.on_decoder_error_processing_time_ms.NumSamples(), 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, InfraMetricsNotCollectedByDefault) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.report_infra_metrics = false; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"}, + /*frames_count=*/1, *frame_generator); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.on_frame_captured_processing_time_ms.NumSamples(), 0); + EXPECT_EQ(stats.on_frame_pre_encode_processing_time_ms.NumSamples(), 0); + EXPECT_EQ(stats.on_frame_encoded_processing_time_ms.NumSamples(), 0); + EXPECT_EQ(stats.on_frame_pre_decode_processing_time_ms.NumSamples(), 0); + EXPECT_EQ(stats.on_frame_decoded_processing_time_ms.NumSamples(), 0); + EXPECT_EQ(stats.on_frame_rendered_processing_time_ms.NumSamples(), 0); + EXPECT_EQ(stats.on_decoder_error_processing_time_ms.NumSamples(), 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, + FrameDroppedByDecoderIsAccountedCorrectly) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.report_infra_metrics = false; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame to_be_dropped_frame = + NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = + analyzer.OnFrameCaptured("alice", "alice_video", to_be_dropped_frame); + to_be_dropped_frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", to_be_dropped_frame); + analyzer.OnFrameEncoded("alice", to_be_dropped_frame.id(), + FakeEncode(to_be_dropped_frame), + VideoQualityAnalyzerInterface::EncoderStats(), false); + VideoFrame received_to_be_dropped_frame = DeepCopy(to_be_dropped_frame); + analyzer.OnFramePreDecode("bob", received_to_be_dropped_frame.id(), + FakeEncode(received_to_be_dropped_frame)); + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"}, + /*frames_count=*/1, *frame_generator); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + StreamStats stats = analyzer.GetStats().at(StatsKey("alice_video", "bob")); + ASSERT_EQ(stats.dropped_by_phase[FrameDropPhase::kByDecoder], 1); +} + +class DefaultVideoQualityAnalyzerTimeBetweenFreezesTest + : public TestWithParam<bool> {}; + +TEST_P(DefaultVideoQualityAnalyzerTimeBetweenFreezesTest, + TimeBetweenFreezesIsEqualToStreamDurationWhenThereAreNoFeeezes) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + test::GetGlobalMetricsLogger(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"}, + /*frames_count=*/5, *frame_generator, + /*interframe_delay_ms=*/50); + if (GetParam()) { + analyzer.UnregisterParticipantInCall("bob"); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(50); + analyzer.Stop(); + + StreamStats stats = analyzer.GetStats().at(StatsKey("alice_video", "bob")); + ASSERT_EQ(stats.time_between_freezes_ms.NumSamples(), 1); + EXPECT_GE(stats.time_between_freezes_ms.GetAverage(), 200); +} + +INSTANTIATE_TEST_SUITE_P(WithRegisteredAndUnregisteredPeerAtTheEndOfTheCall, + DefaultVideoQualityAnalyzerTimeBetweenFreezesTest, + ValuesIn({true, false})); + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h new file mode 100644 index 0000000000..384e901462 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_ + +#include <cstdint> +#include <utility> + +#include "absl/types/optional.h" +#include "api/video/encoded_image.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Injects frame id into EncodedImage on encoder side +class EncodedImageDataInjector { + public: + virtual ~EncodedImageDataInjector() = default; + + // Return encoded image with specified `id` and `discard` flag injected into + // its payload. `discard` flag mean does analyzing decoder should discard this + // encoded image because it belongs to unnecessary simulcast stream or spatial + // layer. + virtual EncodedImage InjectData(uint16_t id, + bool discard, + const EncodedImage& source) = 0; +}; + +struct EncodedImageExtractionResult { + absl::optional<uint16_t> id; + EncodedImage image; + // Is true if encoded image should be discarded. It is used to filter out + // unnecessary spatial layers and simulcast streams. + bool discard; +}; + +// Extracts frame id from EncodedImage on decoder side. +class EncodedImageDataExtractor { + public: + virtual ~EncodedImageDataExtractor() = default; + + // Invoked by framework before any image will come to the extractor. + // `expected_receivers_count` is the expected amount of receivers for each + // encoded image. + virtual void Start(int expected_receivers_count) = 0; + + // Invoked by framework when it is required to add one more receiver for + // frames. Will be invoked before that receiver will start receive data. + virtual void AddParticipantInCall() = 0; + + // Invoked by framework when it is required to remove receiver for frames. + // Will be invoked after that receiver will stop receiving data. + virtual void RemoveParticipantInCall() = 0; + + // Returns encoded image id, extracted from payload and also encoded image + // with its original payload. For concatenated spatial layers it should be the + // same id. + virtual EncodedImageExtractionResult ExtractData( + const EncodedImage& source) = 0; +}; + +class EncodedImageDataPropagator : public EncodedImageDataInjector, + public EncodedImageDataExtractor { + public: + ~EncodedImageDataPropagator() override = default; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc new file mode 100644 index 0000000000..da9c53beb9 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/example_video_quality_analyzer.h" + +#include "api/array_view.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +ExampleVideoQualityAnalyzer::ExampleVideoQualityAnalyzer() = default; +ExampleVideoQualityAnalyzer::~ExampleVideoQualityAnalyzer() = default; + +void ExampleVideoQualityAnalyzer::Start( + std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count) {} + +uint16_t ExampleVideoQualityAnalyzer::OnFrameCaptured( + absl::string_view peer_name, + const std::string& stream_label, + const webrtc::VideoFrame& frame) { + MutexLock lock(&lock_); + uint16_t frame_id = next_frame_id_++; + if (frame_id == VideoFrame::kNotSetId) { + frame_id = next_frame_id_++; + } + auto it = frames_in_flight_.find(frame_id); + if (it == frames_in_flight_.end()) { + frames_in_flight_.insert(frame_id); + frames_to_stream_label_.insert({frame_id, stream_label}); + } else { + RTC_LOG(LS_WARNING) << "Meet new frame with the same id: " << frame_id + << ". Assumes old one as dropped"; + // We needn't insert frame to frames_in_flight_, because it is already + // there. + ++frames_dropped_; + auto stream_it = frames_to_stream_label_.find(frame_id); + RTC_CHECK(stream_it != frames_to_stream_label_.end()); + stream_it->second = stream_label; + } + ++frames_captured_; + return frame_id; +} + +void ExampleVideoQualityAnalyzer::OnFramePreEncode( + absl::string_view peer_name, + const webrtc::VideoFrame& frame) { + MutexLock lock(&lock_); + ++frames_pre_encoded_; +} + +void ExampleVideoQualityAnalyzer::OnFrameEncoded( + absl::string_view peer_name, + uint16_t frame_id, + const webrtc::EncodedImage& encoded_image, + const EncoderStats& stats, + bool discarded) { + MutexLock lock(&lock_); + ++frames_encoded_; +} + +void ExampleVideoQualityAnalyzer::OnFrameDropped( + absl::string_view peer_name, + webrtc::EncodedImageCallback::DropReason reason) { + RTC_LOG(LS_INFO) << "Frame dropped by encoder"; + MutexLock lock(&lock_); + ++frames_dropped_; +} + +void ExampleVideoQualityAnalyzer::OnFramePreDecode( + absl::string_view peer_name, + uint16_t frame_id, + const webrtc::EncodedImage& encoded_image) { + MutexLock lock(&lock_); + ++frames_received_; +} + +void ExampleVideoQualityAnalyzer::OnFrameDecoded( + absl::string_view peer_name, + const webrtc::VideoFrame& frame, + const DecoderStats& stats) { + MutexLock lock(&lock_); + ++frames_decoded_; +} + +void ExampleVideoQualityAnalyzer::OnFrameRendered( + absl::string_view peer_name, + const webrtc::VideoFrame& frame) { + MutexLock lock(&lock_); + frames_in_flight_.erase(frame.id()); + ++frames_rendered_; +} + +void ExampleVideoQualityAnalyzer::OnEncoderError( + absl::string_view peer_name, + const webrtc::VideoFrame& frame, + int32_t error_code) { + RTC_LOG(LS_ERROR) << "Failed to encode frame " << frame.id() + << ". Code: " << error_code; +} + +void ExampleVideoQualityAnalyzer::OnDecoderError(absl::string_view peer_name, + uint16_t frame_id, + int32_t error_code, + const DecoderStats& stats) { + RTC_LOG(LS_ERROR) << "Failed to decode frame " << frame_id + << ". Code: " << error_code; +} + +void ExampleVideoQualityAnalyzer::Stop() { + MutexLock lock(&lock_); + RTC_LOG(LS_INFO) << "There are " << frames_in_flight_.size() + << " frames in flight, assuming all of them are dropped"; + frames_dropped_ += frames_in_flight_.size(); +} + +std::string ExampleVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) { + MutexLock lock(&lock_); + auto it = frames_to_stream_label_.find(frame_id); + RTC_DCHECK(it != frames_to_stream_label_.end()) + << "Unknown frame_id=" << frame_id; + return it->second; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_captured() const { + MutexLock lock(&lock_); + return frames_captured_; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_pre_encoded() const { + MutexLock lock(&lock_); + return frames_pre_encoded_; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_encoded() const { + MutexLock lock(&lock_); + return frames_encoded_; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_received() const { + MutexLock lock(&lock_); + return frames_received_; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_decoded() const { + MutexLock lock(&lock_); + return frames_decoded_; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_rendered() const { + MutexLock lock(&lock_); + return frames_rendered_; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_dropped() const { + MutexLock lock(&lock_); + return frames_dropped_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h new file mode 100644 index 0000000000..af4868a961 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_EXAMPLE_VIDEO_QUALITY_ANALYZER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_EXAMPLE_VIDEO_QUALITY_ANALYZER_H_ + +#include <atomic> +#include <map> +#include <set> +#include <string> + +#include "api/array_view.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { + +// This class is an example implementation of +// webrtc::VideoQualityAnalyzerInterface and calculates simple metrics +// just to demonstration purposes. Assumed to be used in the single process +// test cases, where both peers are in the same process. +class ExampleVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { + public: + ExampleVideoQualityAnalyzer(); + ~ExampleVideoQualityAnalyzer() override; + + void Start(std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count) override; + uint16_t OnFrameCaptured(absl::string_view peer_name, + const std::string& stream_label, + const VideoFrame& frame) override; + void OnFramePreEncode(absl::string_view peer_name, + const VideoFrame& frame) override; + void OnFrameEncoded(absl::string_view peer_name, + uint16_t frame_id, + const EncodedImage& encoded_image, + const EncoderStats& stats, + bool discarded) override; + void OnFrameDropped(absl::string_view peer_name, + EncodedImageCallback::DropReason reason) override; + void OnFramePreDecode(absl::string_view peer_name, + uint16_t frame_id, + const EncodedImage& encoded_image) override; + void OnFrameDecoded(absl::string_view peer_name, + const VideoFrame& frame, + const DecoderStats& stats) override; + void OnFrameRendered(absl::string_view peer_name, + const VideoFrame& frame) override; + void OnEncoderError(absl::string_view peer_name, + const VideoFrame& frame, + int32_t error_code) override; + void OnDecoderError(absl::string_view peer_name, + uint16_t frame_id, + int32_t error_code, + const DecoderStats& stats) override; + void Stop() override; + std::string GetStreamLabel(uint16_t frame_id) override; + + uint64_t frames_captured() const; + uint64_t frames_pre_encoded() const; + uint64_t frames_encoded() const; + uint64_t frames_received() const; + uint64_t frames_decoded() const; + uint64_t frames_rendered() const; + uint64_t frames_dropped() const; + + private: + // When peer A captured the frame it will come into analyzer's OnFrameCaptured + // and will be stored in frames_in_flight_. It will be removed from there + // when it will be received in peer B, so we need to guard it with lock. + // Also because analyzer will serve for all video streams it can be called + // from different threads inside one peer. + mutable Mutex lock_; + // Stores frame ids, that are currently going from one peer to another. We + // need to keep them to correctly determine dropped frames and also correctly + // process frame id overlap. + std::set<uint16_t> frames_in_flight_ RTC_GUARDED_BY(lock_); + std::map<uint16_t, std::string> frames_to_stream_label_ RTC_GUARDED_BY(lock_); + uint16_t next_frame_id_ RTC_GUARDED_BY(lock_) = 1; + uint64_t frames_captured_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_pre_encoded_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_encoded_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_received_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_decoded_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_rendered_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_dropped_ RTC_GUARDED_BY(lock_) = 0; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_EXAMPLE_VIDEO_QUALITY_ANALYZER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue.h new file mode 100644 index 0000000000..39d26b42bc --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue.h @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_MULTI_READER_QUEUE_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_MULTI_READER_QUEUE_H_ + +#include <deque> +#include <memory> +#include <set> +#include <unordered_map> + +#include "absl/types/optional.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +// Represents the queue which can be read by multiple readers. Each reader reads +// from its own queue head. When an element is added it will become visible for +// all readers. When an element will be removed by all the readers, the element +// will be removed from the queue. +template <typename T> +class MultiReaderQueue { + public: + // Creates queue with exactly `readers_count` readers named from 0 to + // `readers_count - 1`. + explicit MultiReaderQueue(size_t readers_count) { + for (size_t i = 0; i < readers_count; ++i) { + heads_[i] = 0; + } + } + // Creates queue with specified readers. + explicit MultiReaderQueue(std::set<size_t> readers) { + for (size_t reader : readers) { + heads_[reader] = 0; + } + } + + // Adds a new `reader`, initializing its reading position (the reader's head) + // equal to the one of `reader_to_copy`. + // Complexity O(MultiReaderQueue::size(reader_to_copy)). + void AddReader(size_t reader, size_t reader_to_copy) { + size_t pos = GetHeadPositionOrDie(reader_to_copy); + + auto it = heads_.find(reader); + RTC_CHECK(it == heads_.end()) + << "Reader " << reader << " is already in the queue"; + heads_[reader] = heads_[reader_to_copy]; + for (size_t i = pos; i < queue_.size(); ++i) { + in_queues_[i]++; + } + } + + // Adds a new `reader`, initializing its reading position equal to first + // element in the queue. + // Complexity O(MultiReaderQueue::size()). + void AddReader(size_t reader) { + auto it = heads_.find(reader); + RTC_CHECK(it == heads_.end()) + << "Reader " << reader << " is already in the queue"; + heads_[reader] = removed_elements_count_; + for (size_t i = 0; i < queue_.size(); ++i) { + in_queues_[i]++; + } + } + + // Removes specified `reader` from the queue. + // Complexity O(MultiReaderQueue::size(reader)). + void RemoveReader(size_t reader) { + size_t pos = GetHeadPositionOrDie(reader); + for (size_t i = pos; i < queue_.size(); ++i) { + in_queues_[i]--; + } + while (!in_queues_.empty() && in_queues_[0] == 0) { + PopFront(); + } + heads_.erase(reader); + } + + // Add value to the end of the queue. Complexity O(1). + void PushBack(T value) { + queue_.push_back(value); + in_queues_.push_back(heads_.size()); + } + + // Extract element from specified head. Complexity O(1). + absl::optional<T> PopFront(size_t reader) { + size_t pos = GetHeadPositionOrDie(reader); + if (pos >= queue_.size()) { + return absl::nullopt; + } + + T out = queue_[pos]; + + in_queues_[pos]--; + heads_[reader]++; + + if (in_queues_[pos] == 0) { + RTC_DCHECK_EQ(pos, 0); + PopFront(); + } + return out; + } + + // Returns element at specified head. Complexity O(1). + absl::optional<T> Front(size_t reader) const { + size_t pos = GetHeadPositionOrDie(reader); + if (pos >= queue_.size()) { + return absl::nullopt; + } + return queue_[pos]; + } + + // Returns true if for specified head there are no more elements in the queue + // or false otherwise. Complexity O(1). + bool IsEmpty(size_t reader) const { + size_t pos = GetHeadPositionOrDie(reader); + return pos >= queue_.size(); + } + + // Returns size of the longest queue between all readers. + // Complexity O(1). + size_t size() const { return queue_.size(); } + + // Returns size of the specified queue. Complexity O(1). + size_t size(size_t reader) const { + size_t pos = GetHeadPositionOrDie(reader); + return queue_.size() - pos; + } + + // Complexity O(1). + size_t readers_count() const { return heads_.size(); } + + private: + size_t GetHeadPositionOrDie(size_t reader) const { + auto it = heads_.find(reader); + RTC_CHECK(it != heads_.end()) << "No queue for reader " << reader; + return it->second - removed_elements_count_; + } + + void PopFront() { + RTC_DCHECK(!queue_.empty()); + RTC_DCHECK_EQ(in_queues_[0], 0); + queue_.pop_front(); + in_queues_.pop_front(); + removed_elements_count_++; + } + + // Number of the elements that were removed from the queue. It is used to + // subtract from each head to compute the right index inside `queue_`; + size_t removed_elements_count_ = 0; + std::deque<T> queue_; + // In how may queues the element at index `i` is. An element can be removed + // from the front if and only if it is in 0 queues. + std::deque<size_t> in_queues_; + // Map from the reader to the head position in the queue. + std::unordered_map<size_t, size_t> heads_; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_MULTI_READER_QUEUE_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue_test.cc new file mode 100644 index 0000000000..ea6aa0a416 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue_test.cc @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/multi_reader_queue.h" + +#include "absl/types/optional.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +TEST(MultiReaderQueueTest, EmptyQueueEmptyForAllHeads) { + MultiReaderQueue<int> queue = MultiReaderQueue<int>(/*readers_count=*/10); + EXPECT_EQ(queue.size(), 0lu); + for (int i = 0; i < 10; ++i) { + EXPECT_TRUE(queue.IsEmpty(/*reader=*/i)); + EXPECT_EQ(queue.size(/*reader=*/i), 0lu); + EXPECT_FALSE(queue.PopFront(/*reader=*/i).has_value()); + EXPECT_FALSE(queue.Front(/*reader=*/i).has_value()); + } +} + +TEST(MultiReaderQueueTest, SizeIsEqualForAllHeadsAfterAddOnly) { + MultiReaderQueue<int> queue = MultiReaderQueue<int>(/*readers_count=*/10); + queue.PushBack(1); + queue.PushBack(2); + queue.PushBack(3); + EXPECT_EQ(queue.size(), 3lu); + for (int i = 0; i < 10; ++i) { + EXPECT_FALSE(queue.IsEmpty(/*reader=*/i)); + EXPECT_EQ(queue.size(/*reader=*/i), 3lu); + } +} + +TEST(MultiReaderQueueTest, SizeIsCorrectAfterRemoveFromOnlyOneHead) { + MultiReaderQueue<int> queue = MultiReaderQueue<int>(/*readers_count=*/10); + for (int i = 0; i < 5; ++i) { + queue.PushBack(i); + } + EXPECT_EQ(queue.size(), 5lu); + // Removing elements from queue #0 + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(queue.size(/*reader=*/0), static_cast<size_t>(5 - i)); + EXPECT_EQ(queue.PopFront(/*reader=*/0), absl::optional<int>(i)); + for (int j = 1; j < 10; ++j) { + EXPECT_EQ(queue.size(/*reader=*/j), 5lu); + } + } + EXPECT_EQ(queue.size(/*reader=*/0), 0lu); + EXPECT_TRUE(queue.IsEmpty(/*reader=*/0)); +} + +TEST(MultiReaderQueueTest, SingleHeadOneAddOneRemove) { + MultiReaderQueue<int> queue = MultiReaderQueue<int>(/*readers_count=*/1); + queue.PushBack(1); + EXPECT_EQ(queue.size(), 1lu); + EXPECT_TRUE(queue.Front(/*reader=*/0).has_value()); + EXPECT_EQ(queue.Front(/*reader=*/0).value(), 1); + absl::optional<int> value = queue.PopFront(/*reader=*/0); + EXPECT_TRUE(value.has_value()); + EXPECT_EQ(value.value(), 1); + EXPECT_EQ(queue.size(), 0lu); + EXPECT_TRUE(queue.IsEmpty(/*reader=*/0)); +} + +TEST(MultiReaderQueueTest, SingleHead) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/1); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + EXPECT_EQ(queue.size(), i + 1); + } + for (size_t i = 0; i < 10; ++i) { + EXPECT_EQ(queue.Front(/*reader=*/0), absl::optional<size_t>(i)); + EXPECT_EQ(queue.PopFront(/*reader=*/0), absl::optional<size_t>(i)); + EXPECT_EQ(queue.size(), 10 - i - 1); + } +} + +TEST(MultiReaderQueueTest, ThreeHeadsAddAllRemoveAllPerHead) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/3); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + EXPECT_EQ(queue.size(), i + 1); + } + for (size_t i = 0; i < 10; ++i) { + absl::optional<size_t> value = queue.PopFront(/*reader=*/0); + EXPECT_EQ(queue.size(), 10lu); + ASSERT_TRUE(value.has_value()); + EXPECT_EQ(value.value(), i); + } + for (size_t i = 0; i < 10; ++i) { + absl::optional<size_t> value = queue.PopFront(/*reader=*/1); + EXPECT_EQ(queue.size(), 10lu); + ASSERT_TRUE(value.has_value()); + EXPECT_EQ(value.value(), i); + } + for (size_t i = 0; i < 10; ++i) { + absl::optional<size_t> value = queue.PopFront(/*reader=*/2); + EXPECT_EQ(queue.size(), 10 - i - 1); + ASSERT_TRUE(value.has_value()); + EXPECT_EQ(value.value(), i); + } +} + +TEST(MultiReaderQueueTest, ThreeHeadsAddAllRemoveAll) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/3); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + EXPECT_EQ(queue.size(), i + 1); + } + for (size_t i = 0; i < 10; ++i) { + absl::optional<size_t> value1 = queue.PopFront(/*reader=*/0); + absl::optional<size_t> value2 = queue.PopFront(/*reader=*/1); + absl::optional<size_t> value3 = queue.PopFront(/*reader=*/2); + EXPECT_EQ(queue.size(), 10 - i - 1); + ASSERT_TRUE(value1.has_value()); + ASSERT_TRUE(value2.has_value()); + ASSERT_TRUE(value3.has_value()); + EXPECT_EQ(value1.value(), i); + EXPECT_EQ(value2.value(), i); + EXPECT_EQ(value3.value(), i); + } +} + +TEST(MultiReaderQueueTest, AddReaderSeeElementsOnlyFromReaderToCopy) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/2); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + } + for (size_t i = 0; i < 5; ++i) { + queue.PopFront(0); + } + + queue.AddReader(/*reader=*/2, /*reader_to_copy=*/0); + + EXPECT_EQ(queue.readers_count(), 3lu); + for (size_t i = 5; i < 10; ++i) { + absl::optional<size_t> value = queue.PopFront(/*reader=*/2); + EXPECT_EQ(queue.size(/*reader=*/2), 10 - i - 1); + ASSERT_TRUE(value.has_value()); + EXPECT_EQ(value.value(), i); + } +} + +TEST(MultiReaderQueueTest, AddReaderWithoutReaderToCopySeeFullQueue) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/2); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + } + for (size_t i = 0; i < 5; ++i) { + queue.PopFront(/*reader=*/0); + } + + queue.AddReader(/*reader=*/2); + + EXPECT_EQ(queue.readers_count(), 3lu); + for (size_t i = 0; i < 10; ++i) { + absl::optional<size_t> value = queue.PopFront(/*reader=*/2); + EXPECT_EQ(queue.size(/*reader=*/2), 10 - i - 1); + ASSERT_TRUE(value.has_value()); + EXPECT_EQ(value.value(), i); + } +} + +TEST(MultiReaderQueueTest, RemoveReaderWontChangeOthers) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/2); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + } + EXPECT_EQ(queue.size(/*reader=*/1), 10lu); + + queue.RemoveReader(0); + + EXPECT_EQ(queue.readers_count(), 1lu); + EXPECT_EQ(queue.size(/*reader=*/1), 10lu); +} + +TEST(MultiReaderQueueTest, RemoveLastReaderMakesQueueEmpty) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/1); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + } + EXPECT_EQ(queue.size(), 10lu); + + queue.RemoveReader(0); + + EXPECT_EQ(queue.size(), 0lu); + EXPECT_EQ(queue.readers_count(), 0lu); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.cc new file mode 100644 index 0000000000..3ccab620f8 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.cc @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/names_collection.h" + +#include <set> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" + +namespace webrtc { + +NamesCollection::NamesCollection(rtc::ArrayView<const std::string> names) { + names_ = std::vector<std::string>(names.begin(), names.end()); + for (size_t i = 0; i < names_.size(); ++i) { + index_.emplace(names_[i], i); + removed_.emplace_back(false); + } + size_ = names_.size(); +} + +bool NamesCollection::HasName(absl::string_view name) const { + auto it = index_.find(name); + if (it == index_.end()) { + return false; + } + return !removed_[it->second]; +} + +size_t NamesCollection::AddIfAbsent(absl::string_view name) { + auto it = index_.find(name); + if (it != index_.end()) { + // Name was registered in the collection before: we need to restore it. + size_t index = it->second; + if (removed_[index]) { + removed_[index] = false; + size_++; + } + return index; + } + size_t out = names_.size(); + size_t old_capacity = names_.capacity(); + names_.emplace_back(name); + removed_.emplace_back(false); + size_++; + size_t new_capacity = names_.capacity(); + + if (old_capacity == new_capacity) { + index_.emplace(names_[out], out); + } else { + // Reallocation happened in the vector, so we need to rebuild `index_` to + // fix absl::string_view internal references. + index_.clear(); + for (size_t i = 0; i < names_.size(); ++i) { + index_.emplace(names_[i], i); + } + } + return out; +} + +absl::optional<size_t> NamesCollection::RemoveIfPresent( + absl::string_view name) { + auto it = index_.find(name); + if (it == index_.end()) { + return absl::nullopt; + } + size_t index = it->second; + if (removed_[index]) { + return absl::nullopt; + } + removed_[index] = true; + size_--; + return index; +} + +std::set<size_t> NamesCollection::GetPresentIndexes() const { + std::set<size_t> out; + for (size_t i = 0; i < removed_.size(); ++i) { + if (!removed_[i]) { + out.insert(i); + } + } + return out; +} + +std::set<size_t> NamesCollection::GetAllIndexes() const { + std::set<size_t> out; + for (size_t i = 0; i < names_.size(); ++i) { + out.insert(i); + } + return out; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.h new file mode 100644 index 0000000000..f9a13a2a11 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_NAMES_COLLECTION_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_NAMES_COLLECTION_H_ + +#include <map> +#include <set> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/array_view.h" + +namespace webrtc { + +// Contains mapping between string names and unique size_t values (indexes). +// Once the name is added to the collection it is guaranteed: +// 1. Name will have the same index until collection will be destructed +// 2. Adding, removing and re-adding name won't change its index +// +// The name is considered in the collection if it was added and wasn't removed. +// Adding the name when it is in the collection won't change the collection, the +// same as removing the name when it is removed. +// +// Collection will return name's index and name for the index independently from +// was name removed or not. Once the name was added to the collection the index +// will be allocated for it. To check if name is in collection right now user +// has to explicitly call to `HasName` function. +class NamesCollection { + public: + NamesCollection() = default; + + explicit NamesCollection(rtc::ArrayView<const std::string> names); + + // Returns amount of currently presented names in the collection. + size_t size() const { return size_; } + + // Returns amount of all names known to this collection. + size_t GetKnownSize() const { return names_.size(); } + + // Returns index of the `name` which was known to the collection. Crashes + // if `name` was never registered in the collection. + size_t index(absl::string_view name) const { return index_.at(name); } + + // Returns name which was known to the collection for the specified `index`. + // Crashes if there was no any name registered in the collection for such + // `index`. + const std::string& name(size_t index) const { return names_.at(index); } + + // Returns if `name` is currently presented in this collection. + bool HasName(absl::string_view name) const; + + // Adds specified `name` to the collection if it isn't presented. + // Returns index which corresponds to specified `name`. + size_t AddIfAbsent(absl::string_view name); + + // Removes specified `name` from the collection if it is presented. + // + // After name was removed, collection size will be decreased, but `name` index + // will be preserved. Collection will return false for `HasName(name)`, but + // will continue to return previously known index for `index(name)` and return + // `name` for `name(index(name))`. + // + // Returns the index of the removed value or absl::nullopt if no such `name` + // registered in the collection. + absl::optional<size_t> RemoveIfPresent(absl::string_view name); + + // Returns a set of indexes for all currently present names in the + // collection. + std::set<size_t> GetPresentIndexes() const; + + // Returns a set of all indexes known to the collection including indexes for + // names that were removed. + std::set<size_t> GetAllIndexes() const; + + private: + std::vector<std::string> names_; + std::vector<bool> removed_; + std::map<absl::string_view, size_t> index_; + size_t size_ = 0; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_NAMES_COLLECTION_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection_test.cc new file mode 100644 index 0000000000..6c52f96975 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection_test.cc @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/names_collection.h" + +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +using ::testing::Eq; +using ::testing::Ne; + +TEST(NamesCollectionTest, NamesFromCtorHasUniqueIndexes) { + NamesCollection collection(std::vector<std::string>{"alice", "bob"}); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(2))); + EXPECT_TRUE(collection.HasName("alice")); + EXPECT_THAT(collection.name(collection.index("alice")), Eq("alice")); + + EXPECT_TRUE(collection.HasName("bob")); + EXPECT_THAT(collection.name(collection.index("bob")), Eq("bob")); + + EXPECT_THAT(collection.index("bob"), Ne(collection.index("alice"))); +} + +TEST(NamesCollectionTest, AddedNamesHasIndexes) { + NamesCollection collection(std::vector<std::string>{}); + collection.AddIfAbsent("alice"); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + EXPECT_TRUE(collection.HasName("alice")); + EXPECT_THAT(collection.name(collection.index("alice")), Eq("alice")); +} + +TEST(NamesCollectionTest, AddBobDoesNotChangeAliceIndex) { + NamesCollection collection(std::vector<std::string>{"alice"}); + + size_t alice_index = collection.index("alice"); + + collection.AddIfAbsent("bob"); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(2))); + EXPECT_THAT(collection.index("alice"), Eq(alice_index)); + EXPECT_THAT(collection.index("bob"), Ne(alice_index)); +} + +TEST(NamesCollectionTest, AddAliceSecondTimeDoesNotChangeIndex) { + NamesCollection collection(std::vector<std::string>{"alice"}); + + size_t alice_index = collection.index("alice"); + + EXPECT_THAT(collection.AddIfAbsent("alice"), Eq(alice_index)); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + EXPECT_THAT(collection.index("alice"), Eq(alice_index)); +} + +TEST(NamesCollectionTest, RemoveRemovesFromCollectionButNotIndex) { + NamesCollection collection(std::vector<std::string>{"alice", "bob"}); + + size_t bob_index = collection.index("bob"); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(2))); + + EXPECT_THAT(collection.RemoveIfPresent("bob"), + Eq(absl::optional<size_t>(bob_index))); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + EXPECT_FALSE(collection.HasName("bob")); + + EXPECT_THAT(collection.index("bob"), Eq(bob_index)); + EXPECT_THAT(collection.name(bob_index), Eq("bob")); +} + +TEST(NamesCollectionTest, RemoveOfAliceDoesNotChangeBobIndex) { + NamesCollection collection(std::vector<std::string>{"alice", "bob"}); + + size_t alice_index = collection.index("alice"); + size_t bob_index = collection.index("bob"); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(2))); + + EXPECT_THAT(collection.RemoveIfPresent("alice"), + Eq(absl::optional<size_t>(alice_index))); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + EXPECT_THAT(collection.index("bob"), Eq(bob_index)); + EXPECT_THAT(collection.name(bob_index), Eq("bob")); +} + +TEST(NamesCollectionTest, RemoveSecondTimeHasNoEffect) { + NamesCollection collection(std::vector<std::string>{"bob"}); + + size_t bob_index = collection.index("bob"); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + EXPECT_THAT(collection.RemoveIfPresent("bob"), + Eq(absl::optional<size_t>(bob_index))); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(0))); + EXPECT_THAT(collection.RemoveIfPresent("bob"), Eq(absl::nullopt)); +} + +TEST(NamesCollectionTest, RemoveOfNotExistingHasNoEffect) { + NamesCollection collection(std::vector<std::string>{"bob"}); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + EXPECT_THAT(collection.RemoveIfPresent("alice"), Eq(absl::nullopt)); + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); +} + +TEST(NamesCollectionTest, AddRemoveAddPreserveTheIndex) { + NamesCollection collection(std::vector<std::string>{}); + + size_t alice_index = collection.AddIfAbsent("alice"); + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + + EXPECT_THAT(collection.RemoveIfPresent("alice"), + Eq(absl::optional<size_t>(alice_index))); + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(0))); + + EXPECT_THAT(collection.AddIfAbsent("alice"), Eq(alice_index)); + EXPECT_THAT(collection.index("alice"), Eq(alice_index)); + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); +} + +TEST(NamesCollectionTest, GetKnownSizeReturnsForRemovedNames) { + NamesCollection collection(std::vector<std::string>{}); + + size_t alice_index = collection.AddIfAbsent("alice"); + EXPECT_THAT(collection.GetKnownSize(), Eq(static_cast<size_t>(1))); + + EXPECT_THAT(collection.RemoveIfPresent("alice"), + Eq(absl::optional<size_t>(alice_index))); + EXPECT_THAT(collection.GetKnownSize(), Eq(static_cast<size_t>(1))); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc new file mode 100644 index 0000000000..b958f4d027 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h" + +#include <cstdint> +#include <cstring> +#include <memory> +#include <utility> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" +#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +QualityAnalyzingVideoDecoder::QualityAnalyzingVideoDecoder( + absl::string_view peer_name, + std::unique_ptr<VideoDecoder> delegate, + EncodedImageDataExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer) + : peer_name_(peer_name), + implementation_name_("AnalyzingDecoder-" + + std::string(delegate->ImplementationName())), + delegate_(std::move(delegate)), + extractor_(extractor), + analyzer_(analyzer) { + analyzing_callback_ = std::make_unique<DecoderCallback>(this); +} +QualityAnalyzingVideoDecoder::~QualityAnalyzingVideoDecoder() = default; + +bool QualityAnalyzingVideoDecoder::Configure(const Settings& settings) { + { + MutexLock lock(&mutex_); + codec_name_ = std::string(CodecTypeToPayloadString(settings.codec_type())) + + "_" + delegate_->GetDecoderInfo().implementation_name; + } + return delegate_->Configure(settings); +} + +int32_t QualityAnalyzingVideoDecoder::Decode(const EncodedImage& input_image, + bool missing_frames, + int64_t render_time_ms) { + // Image extractor extracts id from provided EncodedImage and also returns + // the image with the original buffer. Buffer can be modified in place, so + // owner of original buffer will be responsible for deleting it, or extractor + // can create a new buffer. In such case extractor will be responsible for + // deleting it. + EncodedImageExtractionResult out = extractor_->ExtractData(input_image); + + if (out.discard) { + // To partly emulate behavior of Selective Forwarding Unit (SFU) in the + // test, on receiver side we will "discard" frames from irrelevant streams. + // When all encoded images were marked to discarded, black frame have to be + // returned. Because simulcast streams will be received by receiver as 3 + // different independent streams we don't want that irrelevant streams + // affect video quality metrics and also we don't want to use CPU time to + // decode them to prevent regressions on relevant streams. Also we can't + // just drop frame, because in such case, receiving part will be confused + // with all frames missing and will request a key frame, which will result + // into extra load on network and sender side. Because of it, discarded + // image will be always decoded as black frame and will be passed to + // callback directly without reaching decoder and video quality analyzer. + // + // For more details see QualityAnalyzingVideoEncoder. + return analyzing_callback_->IrrelevantSimulcastStreamDecoded( + out.id.value_or(VideoFrame::kNotSetId), input_image.Timestamp()); + } + + EncodedImage* origin_image; + { + MutexLock lock(&mutex_); + // Store id to be able to retrieve it in analyzing callback. + timestamp_to_frame_id_.insert({input_image.Timestamp(), out.id}); + // Store encoded image to prevent its destruction while it is used in + // decoder. + origin_image = &( + decoding_images_.insert({input_image.Timestamp(), std::move(out.image)}) + .first->second); + } + // We can safely dereference `origin_image`, because it can be removed from + // the map only after `delegate_` Decode method will be invoked. Image will + // be removed inside DecodedImageCallback, which can be done on separate + // thread. + analyzer_->OnFramePreDecode( + peer_name_, out.id.value_or(VideoFrame::kNotSetId), *origin_image); + int32_t result = + delegate_->Decode(*origin_image, missing_frames, render_time_ms); + if (result != WEBRTC_VIDEO_CODEC_OK) { + // If delegate decoder failed, then cleanup data for this image. + VideoQualityAnalyzerInterface::DecoderStats stats; + { + MutexLock lock(&mutex_); + timestamp_to_frame_id_.erase(input_image.Timestamp()); + decoding_images_.erase(input_image.Timestamp()); + stats.decoder_name = codec_name_; + } + analyzer_->OnDecoderError( + peer_name_, out.id.value_or(VideoFrame::kNotSetId), result, stats); + } + return result; +} + +int32_t QualityAnalyzingVideoDecoder::RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) { + analyzing_callback_->SetDelegateCallback(callback); + return delegate_->RegisterDecodeCompleteCallback(analyzing_callback_.get()); +} + +int32_t QualityAnalyzingVideoDecoder::Release() { + // Release decoder first. During release process it can still decode some + // frames, so we don't take a lock to prevent deadlock. + int32_t result = delegate_->Release(); + + MutexLock lock(&mutex_); + analyzing_callback_->SetDelegateCallback(nullptr); + timestamp_to_frame_id_.clear(); + decoding_images_.clear(); + return result; +} + +VideoDecoder::DecoderInfo QualityAnalyzingVideoDecoder::GetDecoderInfo() const { + DecoderInfo info = delegate_->GetDecoderInfo(); + info.implementation_name = implementation_name_; + return info; +} + +const char* QualityAnalyzingVideoDecoder::ImplementationName() const { + return implementation_name_.c_str(); +} + +QualityAnalyzingVideoDecoder::DecoderCallback::DecoderCallback( + QualityAnalyzingVideoDecoder* decoder) + : decoder_(decoder), delegate_callback_(nullptr) {} +QualityAnalyzingVideoDecoder::DecoderCallback::~DecoderCallback() = default; + +void QualityAnalyzingVideoDecoder::DecoderCallback::SetDelegateCallback( + DecodedImageCallback* delegate) { + MutexLock lock(&callback_mutex_); + delegate_callback_ = delegate; +} + +// We have to implement all next 3 methods because we don't know which one +// exactly is implemented in `delegate_callback_`, so we need to call the same +// method on `delegate_callback_`, as was called on `this` callback. +int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( + VideoFrame& decodedImage) { + decoder_->OnFrameDecoded(&decodedImage, /*decode_time_ms=*/absl::nullopt, + /*qp=*/absl::nullopt); + + MutexLock lock(&callback_mutex_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->Decoded(decodedImage); +} + +int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( + VideoFrame& decodedImage, + int64_t decode_time_ms) { + decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, /*qp=*/absl::nullopt); + + MutexLock lock(&callback_mutex_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->Decoded(decodedImage, decode_time_ms); +} + +void QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( + VideoFrame& decodedImage, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp) { + decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, qp); + + MutexLock lock(&callback_mutex_); + RTC_DCHECK(delegate_callback_); + delegate_callback_->Decoded(decodedImage, decode_time_ms, qp); +} + +int32_t +QualityAnalyzingVideoDecoder::DecoderCallback::IrrelevantSimulcastStreamDecoded( + uint16_t frame_id, + uint32_t timestamp_ms) { + webrtc::VideoFrame dummy_frame = + webrtc::VideoFrame::Builder() + .set_video_frame_buffer(GetDummyFrameBuffer()) + .set_timestamp_rtp(timestamp_ms) + .set_id(frame_id) + .build(); + MutexLock lock(&callback_mutex_); + RTC_DCHECK(delegate_callback_); + delegate_callback_->Decoded(dummy_frame, absl::nullopt, absl::nullopt); + return WEBRTC_VIDEO_CODEC_OK; +} + +rtc::scoped_refptr<webrtc::VideoFrameBuffer> +QualityAnalyzingVideoDecoder::DecoderCallback::GetDummyFrameBuffer() { + if (!dummy_frame_buffer_) { + dummy_frame_buffer_ = CreateDummyFrameBuffer(); + } + + return dummy_frame_buffer_; +} + +void QualityAnalyzingVideoDecoder::OnFrameDecoded( + VideoFrame* frame, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp) { + absl::optional<uint16_t> frame_id; + std::string codec_name; + { + MutexLock lock(&mutex_); + auto it = timestamp_to_frame_id_.find(frame->timestamp()); + if (it == timestamp_to_frame_id_.end()) { + // Ensure, that we have info about this frame. It can happen that for some + // reasons decoder response, that it failed to decode, when we were + // posting frame to it, but then call the callback for this frame. + RTC_LOG(LS_ERROR) << "QualityAnalyzingVideoDecoder::OnFrameDecoded: No " + "frame id for frame for frame->timestamp()=" + << frame->timestamp(); + return; + } + frame_id = it->second; + timestamp_to_frame_id_.erase(it); + decoding_images_.erase(frame->timestamp()); + codec_name = codec_name_; + } + // Set frame id to the value, that was extracted from corresponding encoded + // image. + frame->set_id(frame_id.value_or(VideoFrame::kNotSetId)); + VideoQualityAnalyzerInterface::DecoderStats stats; + stats.decoder_name = codec_name; + stats.decode_time_ms = decode_time_ms; + analyzer_->OnFrameDecoded(peer_name_, *frame, stats); +} + +QualityAnalyzingVideoDecoderFactory::QualityAnalyzingVideoDecoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoDecoderFactory> delegate, + EncodedImageDataExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer) + : peer_name_(peer_name), + delegate_(std::move(delegate)), + extractor_(extractor), + analyzer_(analyzer) {} +QualityAnalyzingVideoDecoderFactory::~QualityAnalyzingVideoDecoderFactory() = + default; + +std::vector<SdpVideoFormat> +QualityAnalyzingVideoDecoderFactory::GetSupportedFormats() const { + return delegate_->GetSupportedFormats(); +} + +std::unique_ptr<VideoDecoder> +QualityAnalyzingVideoDecoderFactory::CreateVideoDecoder( + const SdpVideoFormat& format) { + std::unique_ptr<VideoDecoder> decoder = delegate_->CreateVideoDecoder(format); + return std::make_unique<QualityAnalyzingVideoDecoder>( + peer_name_, std::move(decoder), extractor_, analyzer_); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h new file mode 100644 index 0000000000..a86f4196b0 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ + +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "rtc_base/synchronization/mutex.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// QualityAnalyzingVideoDecoder is used to wrap origin video decoder and inject +// VideoQualityAnalyzerInterface before and after decoder. +// +// QualityAnalyzingVideoDecoder propagates all calls to the origin decoder. +// It registers its own DecodedImageCallback in the origin decoder and will +// store user specified callback inside itself. +// +// When Decode(...) will be invoked, quality decoder first will extract frame id +// from passed EncodedImage with EncodedImageIdExtracor that was specified in +// constructor, then will call video quality analyzer, with correct +// EncodedImage and only then will pass image to origin decoder. +// +// When origin decoder decodes the image it will call quality decoder's special +// callback, where video analyzer will be called again and then decoded frame +// will be passed to origin callback, provided by user. +// +// Quality decoder registers its own callback in origin decoder, at the same +// time the user registers their callback in quality decoder. +class QualityAnalyzingVideoDecoder : public VideoDecoder { + public: + QualityAnalyzingVideoDecoder(absl::string_view peer_name, + std::unique_ptr<VideoDecoder> delegate, + EncodedImageDataExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoDecoder() override; + + // Methods of VideoDecoder interface. + bool Configure(const Settings& settings) override; + int32_t Decode(const EncodedImage& input_image, + bool missing_frames, + int64_t render_time_ms) override; + int32_t RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) override; + int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; + const char* ImplementationName() const override; + + private: + class DecoderCallback : public DecodedImageCallback { + public: + explicit DecoderCallback(QualityAnalyzingVideoDecoder* decoder); + ~DecoderCallback() override; + + void SetDelegateCallback(DecodedImageCallback* delegate); + + // Methods of DecodedImageCallback interface. + int32_t Decoded(VideoFrame& decodedImage) override; + int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) override; + void Decoded(VideoFrame& decodedImage, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp) override; + + int32_t IrrelevantSimulcastStreamDecoded(uint16_t frame_id, + uint32_t timestamp_ms); + + private: + rtc::scoped_refptr<webrtc::VideoFrameBuffer> GetDummyFrameBuffer(); + + QualityAnalyzingVideoDecoder* const decoder_; + + rtc::scoped_refptr<webrtc::VideoFrameBuffer> dummy_frame_buffer_; + + Mutex callback_mutex_; + DecodedImageCallback* delegate_callback_ RTC_GUARDED_BY(callback_mutex_); + }; + + void OnFrameDecoded(VideoFrame* frame, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp); + + const std::string peer_name_; + const std::string implementation_name_; + std::unique_ptr<VideoDecoder> delegate_; + EncodedImageDataExtractor* const extractor_; + VideoQualityAnalyzerInterface* const analyzer_; + std::unique_ptr<DecoderCallback> analyzing_callback_; + + // VideoDecoder interface assumes async delivery of decoded video frames. + // This lock is used to protect shared state, that have to be propagated + // from received EncodedImage to resulted VideoFrame. + Mutex mutex_; + + // Name of the video codec type used. Ex: VP8, VP9, H264 etc. + std::string codec_name_ RTC_GUARDED_BY(mutex_); + std::map<uint32_t, absl::optional<uint16_t>> timestamp_to_frame_id_ + RTC_GUARDED_BY(mutex_); + // Stores currently being decoded images by timestamp. Because + // EncodedImageDataExtractor can create new copy on EncodedImage we need to + // ensure, that this image won't be deleted during async decoding. To do it + // all images are putted into this map and removed from here inside callback. + std::map<uint32_t, EncodedImage> decoding_images_ RTC_GUARDED_BY(mutex_); +}; + +// Produces QualityAnalyzingVideoDecoder, which hold decoders, produced by +// specified factory as delegates. Forwards all other calls to specified +// factory. +class QualityAnalyzingVideoDecoderFactory : public VideoDecoderFactory { + public: + QualityAnalyzingVideoDecoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoDecoderFactory> delegate, + EncodedImageDataExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoDecoderFactory() override; + + // Methods of VideoDecoderFactory interface. + std::vector<SdpVideoFormat> GetSupportedFormats() const override; + std::unique_ptr<VideoDecoder> CreateVideoDecoder( + const SdpVideoFormat& format) override; + + private: + const std::string peer_name_; + std::unique_ptr<VideoDecoderFactory> delegate_; + EncodedImageDataExtractor* const extractor_; + VideoQualityAnalyzerInterface* const analyzer_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc new file mode 100644 index 0000000000..e814ba88b7 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc @@ -0,0 +1,403 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h" + +#include <cmath> +#include <memory> +#include <utility> + +#include "absl/strings/string_view.h" +#include "api/video/video_codec_type.h" +#include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/svc/scalability_mode_util.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using EmulatedSFUConfigMap = + ::webrtc::webrtc_pc_e2e::QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap; + +constexpr size_t kMaxFrameInPipelineCount = 1000; +constexpr double kNoMultiplier = 1.0; +constexpr double kEps = 1e-6; + +std::pair<uint32_t, uint32_t> GetMinMaxBitratesBps(const VideoCodec& codec, + size_t spatial_idx) { + uint32_t min_bitrate = codec.minBitrate; + uint32_t max_bitrate = codec.maxBitrate; + if (spatial_idx < codec.numberOfSimulcastStreams && + codec.codecType != VideoCodecType::kVideoCodecVP9) { + min_bitrate = + std::max(min_bitrate, codec.simulcastStream[spatial_idx].minBitrate); + max_bitrate = + std::min(max_bitrate, codec.simulcastStream[spatial_idx].maxBitrate); + } + if (codec.codecType == VideoCodecType::kVideoCodecVP9 && + spatial_idx < codec.VP9().numberOfSpatialLayers) { + min_bitrate = + std::max(min_bitrate, codec.spatialLayers[spatial_idx].minBitrate); + max_bitrate = + std::min(max_bitrate, codec.spatialLayers[spatial_idx].maxBitrate); + } + RTC_DCHECK_GT(max_bitrate, min_bitrate); + return {min_bitrate * 1000, max_bitrate * 1000}; +} + +} // namespace + +QualityAnalyzingVideoEncoder::QualityAnalyzingVideoEncoder( + absl::string_view peer_name, + std::unique_ptr<VideoEncoder> delegate, + double bitrate_multiplier, + EmulatedSFUConfigMap stream_to_sfu_config, + EncodedImageDataInjector* injector, + VideoQualityAnalyzerInterface* analyzer) + : peer_name_(peer_name), + delegate_(std::move(delegate)), + bitrate_multiplier_(bitrate_multiplier), + stream_to_sfu_config_(std::move(stream_to_sfu_config)), + injector_(injector), + analyzer_(analyzer), + mode_(SimulcastMode::kNormal), + delegate_callback_(nullptr) {} +QualityAnalyzingVideoEncoder::~QualityAnalyzingVideoEncoder() = default; + +void QualityAnalyzingVideoEncoder::SetFecControllerOverride( + FecControllerOverride* fec_controller_override) { + // Ignored. +} + +int32_t QualityAnalyzingVideoEncoder::InitEncode( + const VideoCodec* codec_settings, + const Settings& settings) { + MutexLock lock(&mutex_); + codec_settings_ = *codec_settings; + mode_ = SimulcastMode::kNormal; + absl::optional<InterLayerPredMode> inter_layer_pred_mode; + if (codec_settings->GetScalabilityMode().has_value()) { + inter_layer_pred_mode = ScalabilityModeToInterLayerPredMode( + *codec_settings->GetScalabilityMode()); + } else if (codec_settings->codecType == kVideoCodecVP9) { + if (codec_settings->VP9().numberOfSpatialLayers > 1) { + inter_layer_pred_mode = codec_settings->VP9().interLayerPred; + } + } + if (inter_layer_pred_mode.has_value()) { + switch (*inter_layer_pred_mode) { + case InterLayerPredMode::kOn: + mode_ = SimulcastMode::kSVC; + break; + case InterLayerPredMode::kOnKeyPic: + mode_ = SimulcastMode::kKSVC; + break; + case InterLayerPredMode::kOff: + mode_ = SimulcastMode::kSimulcast; + break; + default: + RTC_DCHECK_NOTREACHED() + << "Unknown InterLayerPredMode value " << *inter_layer_pred_mode; + break; + } + } + if (codec_settings->numberOfSimulcastStreams > 1) { + mode_ = SimulcastMode::kSimulcast; + } + return delegate_->InitEncode(codec_settings, settings); +} + +int32_t QualityAnalyzingVideoEncoder::RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) { + // We need to get a lock here because delegate_callback can be hypothetically + // accessed from different thread (encoder one) concurrently. + MutexLock lock(&mutex_); + delegate_callback_ = callback; + return delegate_->RegisterEncodeCompleteCallback(this); +} + +int32_t QualityAnalyzingVideoEncoder::Release() { + // Release encoder first. During release process it can still encode some + // frames, so we don't take a lock to prevent deadlock. + int32_t result = delegate_->Release(); + + MutexLock lock(&mutex_); + delegate_callback_ = nullptr; + return result; +} + +int32_t QualityAnalyzingVideoEncoder::Encode( + const VideoFrame& frame, + const std::vector<VideoFrameType>* frame_types) { + { + MutexLock lock(&mutex_); + // Store id to be able to retrieve it in analyzing callback. + timestamp_to_frame_id_list_.push_back({frame.timestamp(), frame.id()}); + // If this list is growing, it means that we are not receiving new encoded + // images from encoder. So it should be a bug in setup on in the encoder. + RTC_DCHECK_LT(timestamp_to_frame_id_list_.size(), kMaxFrameInPipelineCount); + } + analyzer_->OnFramePreEncode(peer_name_, frame); + int32_t result = delegate_->Encode(frame, frame_types); + if (result != WEBRTC_VIDEO_CODEC_OK) { + // If origin encoder failed, then cleanup data for this frame. + { + MutexLock lock(&mutex_); + // The timestamp-frame_id pair can be not the last one, so we need to + // find it first and then remove. We will search from the end, because + // usually it will be the last or close to the last one. + auto it = timestamp_to_frame_id_list_.end(); + while (it != timestamp_to_frame_id_list_.begin()) { + --it; + if (it->first == frame.timestamp()) { + timestamp_to_frame_id_list_.erase(it); + break; + } + } + } + analyzer_->OnEncoderError(peer_name_, frame, result); + } + return result; +} + +void QualityAnalyzingVideoEncoder::SetRates( + const VideoEncoder::RateControlParameters& parameters) { + RTC_DCHECK_GT(bitrate_multiplier_, 0.0); + if (fabs(bitrate_multiplier_ - kNoMultiplier) < kEps) { + { + MutexLock lock(&mutex_); + bitrate_allocation_ = parameters.bitrate; + } + return delegate_->SetRates(parameters); + } + + RateControlParameters adjusted_params = parameters; + { + MutexLock lock(&mutex_); + // Simulating encoder overshooting target bitrate, by configuring actual + // encoder too high. Take care not to adjust past limits of config, + // otherwise encoders may crash on DCHECK. + VideoBitrateAllocation multiplied_allocation; + for (size_t si = 0; si < kMaxSpatialLayers; ++si) { + const uint32_t spatial_layer_bitrate_bps = + parameters.bitrate.GetSpatialLayerSum(si); + if (spatial_layer_bitrate_bps == 0) { + continue; + } + + uint32_t min_bitrate_bps; + uint32_t max_bitrate_bps; + std::tie(min_bitrate_bps, max_bitrate_bps) = + GetMinMaxBitratesBps(codec_settings_, si); + double bitrate_multiplier = bitrate_multiplier_; + const uint32_t corrected_bitrate = rtc::checked_cast<uint32_t>( + bitrate_multiplier * spatial_layer_bitrate_bps); + if (corrected_bitrate < min_bitrate_bps) { + bitrate_multiplier = min_bitrate_bps / spatial_layer_bitrate_bps; + } else if (corrected_bitrate > max_bitrate_bps) { + bitrate_multiplier = max_bitrate_bps / spatial_layer_bitrate_bps; + } + + for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) { + if (parameters.bitrate.HasBitrate(si, ti)) { + multiplied_allocation.SetBitrate( + si, ti, + rtc::checked_cast<uint32_t>( + bitrate_multiplier * parameters.bitrate.GetBitrate(si, ti))); + } + } + } + + adjusted_params.bitrate = multiplied_allocation; + bitrate_allocation_ = adjusted_params.bitrate; + } + return delegate_->SetRates(adjusted_params); +} + +VideoEncoder::EncoderInfo QualityAnalyzingVideoEncoder::GetEncoderInfo() const { + return delegate_->GetEncoderInfo(); +} + +// It is assumed, that encoded callback will be always invoked with encoded +// images that correspond to the frames in the same sequence, that frames +// arrived. In other words, assume we have frames F1, F2 and F3 and they have +// corresponding encoded images I1, I2 and I3. In such case if we will call +// encode first with F1, then with F2 and then with F3, then encoder callback +// will be called first with all spatial layers for F1 (I1), then F2 (I2) and +// then F3 (I3). +// +// Basing on it we will use a list of timestamp-frame_id pairs like this: +// 1. If current encoded image timestamp is equals to timestamp in the front +// pair - pick frame id from that pair +// 2. If current encoded image timestamp isn't equals to timestamp in the front +// pair - remove the front pair and got to the step 1. +EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info) { + uint16_t frame_id; + bool discard = false; + uint32_t target_encode_bitrate = 0; + std::string codec_name; + { + MutexLock lock(&mutex_); + std::pair<uint32_t, uint16_t> timestamp_frame_id; + while (!timestamp_to_frame_id_list_.empty()) { + timestamp_frame_id = timestamp_to_frame_id_list_.front(); + if (timestamp_frame_id.first == encoded_image.Timestamp()) { + break; + } + timestamp_to_frame_id_list_.pop_front(); + } + + // After the loop the first element should point to current `encoded_image` + // frame id. We don't remove it from the list, because there may be + // multiple spatial layers for this frame, so encoder can produce more + // encoded images with this timestamp. The first element will be removed + // when the next frame would be encoded and EncodedImageCallback would be + // called with the next timestamp. + + if (timestamp_to_frame_id_list_.empty()) { + // Ensure, that we have info about this frame. It can happen that for some + // reasons encoder response, that he failed to decode, when we were + // posting frame to it, but then call the callback for this frame. + RTC_LOG(LS_ERROR) << "QualityAnalyzingVideoEncoder::OnEncodedImage: No " + "frame id for encoded_image.Timestamp()=" + << encoded_image.Timestamp(); + return EncodedImageCallback::Result( + EncodedImageCallback::Result::Error::OK); + } + frame_id = timestamp_frame_id.second; + + discard = ShouldDiscard(frame_id, encoded_image); + if (!discard) { + target_encode_bitrate = bitrate_allocation_.GetSpatialLayerSum( + encoded_image.SpatialIndex().value_or(0)); + } + codec_name = + std::string(CodecTypeToPayloadString(codec_settings_.codecType)) + "_" + + delegate_->GetEncoderInfo().implementation_name; + } + + VideoQualityAnalyzerInterface::EncoderStats stats; + stats.encoder_name = codec_name; + stats.target_encode_bitrate = target_encode_bitrate; + stats.qp = encoded_image.qp_; + analyzer_->OnFrameEncoded(peer_name_, frame_id, encoded_image, stats, + discard); + + // Image data injector injects frame id and discard flag into provided + // EncodedImage and returns the image with a) modified original buffer (in + // such case the current owner of the buffer will be responsible for deleting + // it) or b) a new buffer (in such case injector will be responsible for + // deleting it). + const EncodedImage& image = + injector_->InjectData(frame_id, discard, encoded_image); + { + MutexLock lock(&mutex_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->OnEncodedImage(image, codec_specific_info); + } +} + +void QualityAnalyzingVideoEncoder::OnDroppedFrame( + EncodedImageCallback::DropReason reason) { + MutexLock lock(&mutex_); + analyzer_->OnFrameDropped(peer_name_, reason); + RTC_DCHECK(delegate_callback_); + delegate_callback_->OnDroppedFrame(reason); +} + +bool QualityAnalyzingVideoEncoder::ShouldDiscard( + uint16_t frame_id, + const EncodedImage& encoded_image) { + std::string stream_label = analyzer_->GetStreamLabel(frame_id); + EmulatedSFUConfigMap::mapped_type emulated_sfu_config = + stream_to_sfu_config_[stream_label]; + + if (!emulated_sfu_config) + return false; + + int cur_spatial_index = encoded_image.SpatialIndex().value_or(0); + int cur_temporal_index = encoded_image.TemporalIndex().value_or(0); + + if (emulated_sfu_config->target_temporal_index && + cur_temporal_index > *emulated_sfu_config->target_temporal_index) + return true; + + if (emulated_sfu_config->target_layer_index) { + switch (mode_) { + case SimulcastMode::kSimulcast: + // In simulcast mode only encoded images with required spatial index are + // interested, so all others have to be discarded. + return cur_spatial_index != *emulated_sfu_config->target_layer_index; + case SimulcastMode::kSVC: + // In SVC mode encoded images with spatial indexes that are equal or + // less than required one are interesting, so all above have to be + // discarded. + return cur_spatial_index > *emulated_sfu_config->target_layer_index; + case SimulcastMode::kKSVC: + // In KSVC mode for key frame encoded images with spatial indexes that + // are equal or less than required one are interesting, so all above + // have to be discarded. For other frames only required spatial index + // is interesting, so all others except the ones depending on the + // keyframes can be discarded. There's no good test for that, so we keep + // all of temporal layer 0 for now. + if (encoded_image._frameType == VideoFrameType::kVideoFrameKey || + cur_temporal_index == 0) + return cur_spatial_index > *emulated_sfu_config->target_layer_index; + return cur_spatial_index != *emulated_sfu_config->target_layer_index; + case SimulcastMode::kNormal: + RTC_DCHECK_NOTREACHED() << "Analyzing encoder is in kNormal mode, but " + "target_layer_index is set"; + } + } + return false; +} + +QualityAnalyzingVideoEncoderFactory::QualityAnalyzingVideoEncoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoEncoderFactory> delegate, + double bitrate_multiplier, + EmulatedSFUConfigMap stream_to_sfu_config, + EncodedImageDataInjector* injector, + VideoQualityAnalyzerInterface* analyzer) + : peer_name_(peer_name), + delegate_(std::move(delegate)), + bitrate_multiplier_(bitrate_multiplier), + stream_to_sfu_config_(std::move(stream_to_sfu_config)), + injector_(injector), + analyzer_(analyzer) {} +QualityAnalyzingVideoEncoderFactory::~QualityAnalyzingVideoEncoderFactory() = + default; + +std::vector<SdpVideoFormat> +QualityAnalyzingVideoEncoderFactory::GetSupportedFormats() const { + return delegate_->GetSupportedFormats(); +} + +VideoEncoderFactory::CodecSupport +QualityAnalyzingVideoEncoderFactory::QueryCodecSupport( + const SdpVideoFormat& format, + absl::optional<std::string> scalability_mode) const { + return delegate_->QueryCodecSupport(format, scalability_mode); +} + +std::unique_ptr<VideoEncoder> +QualityAnalyzingVideoEncoderFactory::CreateVideoEncoder( + const SdpVideoFormat& format) { + return std::make_unique<QualityAnalyzingVideoEncoder>( + peer_name_, delegate_->CreateVideoEncoder(format), bitrate_multiplier_, + stream_to_sfu_config_, injector_, analyzer_); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h new file mode 100644 index 0000000000..4adeacc0cd --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ + +#include <list> +#include <memory> +#include <utility> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "rtc_base/synchronization/mutex.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// QualityAnalyzingVideoEncoder is used to wrap origin video encoder and inject +// VideoQualityAnalyzerInterface before and after encoder. +// +// QualityAnalyzingVideoEncoder propagates all calls to the origin encoder. +// It registers its own EncodedImageCallback in the origin encoder and will +// store user specified callback inside itself. +// +// When Encode(...) will be invoked, quality encoder first calls video quality +// analyzer with original frame, then encodes frame with original encoder. +// +// When origin encoder encodes the image it will call quality encoder's special +// callback, where video analyzer will be called again and then frame id will be +// injected into EncodedImage with passed EncodedImageDataInjector. Then new +// EncodedImage will be passed to origin callback, provided by user. +// +// Quality encoder registers its own callback in origin encoder, at the same +// time the user registers their callback in quality encoder. +class QualityAnalyzingVideoEncoder : public VideoEncoder, + public EncodedImageCallback { + public: + using EmulatedSFUConfigMap = + std::map<std::string, absl::optional<EmulatedSFUConfig>>; + + QualityAnalyzingVideoEncoder(absl::string_view peer_name, + std::unique_ptr<VideoEncoder> delegate, + double bitrate_multiplier, + EmulatedSFUConfigMap stream_to_sfu_config, + EncodedImageDataInjector* injector, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoEncoder() override; + + // Methods of VideoEncoder interface. + void SetFecControllerOverride( + FecControllerOverride* fec_controller_override) override; + int32_t InitEncode(const VideoCodec* codec_settings, + const Settings& settings) override; + int32_t RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) override; + int32_t Release() override; + int32_t Encode(const VideoFrame& frame, + const std::vector<VideoFrameType>* frame_types) override; + void SetRates(const VideoEncoder::RateControlParameters& parameters) override; + EncoderInfo GetEncoderInfo() const override; + + // Methods of EncodedImageCallback interface. + EncodedImageCallback::Result OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info) override; + void OnDroppedFrame(DropReason reason) override; + + private: + enum SimulcastMode { + // In this mode encoder assumes not more than 1 encoded image per video + // frame + kNormal, + + // Next modes are to test video conference behavior. For conference sender + // will send multiple spatial layers/simulcast streams for single video + // track and there is some Selective Forwarding Unit (SFU), that forwards + // only best one, that will pass through downlink to the receiver. + // + // Here this behavior will be partly emulated. Sender will send all spatial + // layers/simulcast streams and then some of them will be filtered out on + // the receiver side. During test setup user can specify which spatial + // layer/simulcast stream is required, what will simulated which spatial + // layer/simulcast stream will be chosen by SFU in the real world. Then + // sender will mark encoded images for all spatial layers above required or + // all simulcast streams except required as to be discarded and on receiver + // side they will be discarded in quality analyzing decoder and won't be + // passed into delegate decoder. + // + // If the sender for some reasons won't send specified spatial layer, then + // receiver still will fall back on lower spatial layers. But for simulcast + // streams if required one won't be sent, receiver will assume all frames + // in that period as dropped and will experience video freeze. + // + // Test based on this simulation will be used to evaluate video quality + // of concrete spatial layers/simulcast streams and also check distribution + // of bandwidth between spatial layers/simulcast streams by BWE. + + // In this mode encoder assumes that for each frame simulcast encoded + // images will be produced. So all simulcast streams except required will + // be marked as to be discarded in decoder and won't reach video quality + // analyzer. + kSimulcast, + // In this mode encoder assumes that for each frame encoded images for + // different spatial layers will be produced. So all spatial layers above + // required will be marked to be discarded in decoder and won't reach + // video quality analyzer. + kSVC, + // In this mode encoder assumes that for each frame encoded images for + // different spatial layers will be produced. Compared to kSVC mode + // spatial layers that are above required will be marked to be discarded + // only for key frames and for regular frames all except required spatial + // layer will be marked as to be discarded in decoder and won't reach video + // quality analyzer. + kKSVC + }; + + bool ShouldDiscard(uint16_t frame_id, const EncodedImage& encoded_image) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + const std::string peer_name_; + std::unique_ptr<VideoEncoder> delegate_; + const double bitrate_multiplier_; + // Contains mapping from stream label to optional spatial index. + // If we have stream label "Foo" and mapping contains + // 1. `absl::nullopt` means all streams are required + // 2. Concrete value means that particular simulcast/SVC stream have to be + // analyzed. + EmulatedSFUConfigMap stream_to_sfu_config_; + EncodedImageDataInjector* const injector_; + VideoQualityAnalyzerInterface* const analyzer_; + + // VideoEncoder interface assumes async delivery of encoded images. + // This lock is used to protect shared state, that have to be propagated + // from received VideoFrame to resulted EncodedImage. + Mutex mutex_; + + VideoCodec codec_settings_ RTC_GUARDED_BY(mutex_); + SimulcastMode mode_ RTC_GUARDED_BY(mutex_); + EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(mutex_); + std::list<std::pair<uint32_t, uint16_t>> timestamp_to_frame_id_list_ + RTC_GUARDED_BY(mutex_); + VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(mutex_); +}; + +// Produces QualityAnalyzingVideoEncoder, which hold decoders, produced by +// specified factory as delegates. Forwards all other calls to specified +// factory. +class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory { + public: + QualityAnalyzingVideoEncoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoEncoderFactory> delegate, + double bitrate_multiplier, + QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap stream_to_sfu_config, + EncodedImageDataInjector* injector, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoEncoderFactory() override; + + // Methods of VideoEncoderFactory interface. + std::vector<SdpVideoFormat> GetSupportedFormats() const override; + VideoEncoderFactory::CodecSupport QueryCodecSupport( + const SdpVideoFormat& format, + absl::optional<std::string> scalability_mode) const override; + std::unique_ptr<VideoEncoder> CreateVideoEncoder( + const SdpVideoFormat& format) override; + + private: + const std::string peer_name_; + std::unique_ptr<VideoEncoderFactory> delegate_; + const double bitrate_multiplier_; + QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap stream_to_sfu_config_; + EncodedImageDataInjector* const injector_; + VideoQualityAnalyzerInterface* const analyzer_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.cc new file mode 100644 index 0000000000..7a73b9f4f1 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.cc @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h" + +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_buffer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +constexpr char kIrrelatedSimulcastStreamFrameData[] = "Dummy!"; + +} // namespace + +rtc::scoped_refptr<webrtc::VideoFrameBuffer> CreateDummyFrameBuffer() { + // Use i420 buffer here as default one and supported by all codecs. + rtc::scoped_refptr<webrtc::I420Buffer> buffer = + webrtc::I420Buffer::Create(2, 2); + memcpy(buffer->MutableDataY(), kIrrelatedSimulcastStreamFrameData, 2); + memcpy(buffer->MutableDataY() + buffer->StrideY(), + kIrrelatedSimulcastStreamFrameData + 2, 2); + memcpy(buffer->MutableDataU(), kIrrelatedSimulcastStreamFrameData + 4, 1); + memcpy(buffer->MutableDataV(), kIrrelatedSimulcastStreamFrameData + 5, 1); + return buffer; +} + +bool IsDummyFrame(const webrtc::VideoFrame& video_frame) { + if (video_frame.width() != 2 || video_frame.height() != 2) { + return false; + } + rtc::scoped_refptr<webrtc::I420BufferInterface> buffer = + video_frame.video_frame_buffer()->ToI420(); + if (memcmp(buffer->DataY(), kIrrelatedSimulcastStreamFrameData, 2) != 0) { + return false; + } + if (memcmp(buffer->DataY() + buffer->StrideY(), + kIrrelatedSimulcastStreamFrameData + 2, 2) != 0) { + return false; + } + if (memcmp(buffer->DataU(), kIrrelatedSimulcastStreamFrameData + 4, 1) != 0) { + return false; + } + if (memcmp(buffer->DataV(), kIrrelatedSimulcastStreamFrameData + 5, 1) != 0) { + return false; + } + return true; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h new file mode 100644 index 0000000000..8ecfae7385 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_SIMULCAST_DUMMY_BUFFER_HELPER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_SIMULCAST_DUMMY_BUFFER_HELPER_H_ + +#include "api/video/video_frame.h" +#include "api/video/video_frame_buffer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Creates a special video frame buffer that should be used to create frames +// during Selective Forwarding Unit (SFU) emulation. Such frames are used when +// original was discarded and some frame is required to be passed upstream +// to make WebRTC pipeline happy and not request key frame on the received +// stream due to lack of incoming frames. +rtc::scoped_refptr<webrtc::VideoFrameBuffer> CreateDummyFrameBuffer(); + +// Tests if provided frame contains a buffer created by +// `CreateDummyFrameBuffer`. +bool IsDummyFrame(const webrtc::VideoFrame& video_frame); + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_SIMULCAST_DUMMY_BUFFER_HELPER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper_test.cc new file mode 100644 index 0000000000..db1030232d --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper_test.cc @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h" + +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_buffer.h" +#include "rtc_base/random.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +uint8_t RandByte(Random& random) { + return random.Rand(255); +} + +VideoFrame CreateRandom2x2VideoFrame(uint16_t id, Random& random) { + rtc::scoped_refptr<I420Buffer> buffer = I420Buffer::Create(2, 2); + + uint8_t data[6] = {RandByte(random), RandByte(random), RandByte(random), + RandByte(random), RandByte(random), RandByte(random)}; + + memcpy(buffer->MutableDataY(), data, 2); + memcpy(buffer->MutableDataY() + buffer->StrideY(), data + 2, 2); + memcpy(buffer->MutableDataU(), data + 4, 1); + memcpy(buffer->MutableDataV(), data + 5, 1); + + return VideoFrame::Builder() + .set_id(id) + .set_video_frame_buffer(buffer) + .set_timestamp_us(1) + .build(); +} + +TEST(CreateDummyFrameBufferTest, CreatedBufferIsDummy) { + VideoFrame dummy_frame = VideoFrame::Builder() + .set_video_frame_buffer(CreateDummyFrameBuffer()) + .build(); + + EXPECT_TRUE(IsDummyFrame(dummy_frame)); +} + +TEST(IsDummyFrameTest, NotEveryFrameIsDummy) { + Random random(/*seed=*/100); + VideoFrame frame = CreateRandom2x2VideoFrame(1, random); + EXPECT_FALSE(IsDummyFrame(frame)); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc new file mode 100644 index 0000000000..ccd2f03537 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h" + +#include <algorithm> +#include <cstddef> + +#include "absl/memory/memory.h" +#include "api/video/encoded_image.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +SingleProcessEncodedImageDataInjector::SingleProcessEncodedImageDataInjector() = + default; +SingleProcessEncodedImageDataInjector:: + ~SingleProcessEncodedImageDataInjector() = default; + +EncodedImage SingleProcessEncodedImageDataInjector::InjectData( + uint16_t id, + bool discard, + const EncodedImage& source) { + RTC_CHECK(source.size() >= ExtractionInfo::kUsedBufferSize); + + ExtractionInfo info; + info.discard = discard; + size_t insertion_pos = source.size() - ExtractionInfo::kUsedBufferSize; + memcpy(info.origin_data, &source.data()[insertion_pos], + ExtractionInfo::kUsedBufferSize); + { + MutexLock lock(&lock_); + // Will create new one if missed. + ExtractionInfoVector& ev = extraction_cache_[id]; + info.sub_id = ev.next_sub_id++; + ev.infos[info.sub_id] = info; + } + + auto buffer = EncodedImageBuffer::Create(source.data(), source.size()); + buffer->data()[insertion_pos] = id & 0x00ff; + buffer->data()[insertion_pos + 1] = (id & 0xff00) >> 8; + buffer->data()[insertion_pos + 2] = info.sub_id; + + EncodedImage out = source; + out.SetEncodedData(buffer); + return out; +} + +void SingleProcessEncodedImageDataInjector::AddParticipantInCall() { + MutexLock crit(&lock_); + expected_receivers_count_++; +} + +void SingleProcessEncodedImageDataInjector::RemoveParticipantInCall() { + MutexLock crit(&lock_); + expected_receivers_count_--; + // Now we need go over `extraction_cache_` and removed frames which have been + // received by `expected_receivers_count_`. + for (auto& [frame_id, extraction_infos] : extraction_cache_) { + for (auto it = extraction_infos.infos.begin(); + it != extraction_infos.infos.end();) { + // Frame is received if `received_count` equals to + // `expected_receivers_count_`. + if (it->second.received_count == expected_receivers_count_) { + it = extraction_infos.infos.erase(it); + } else { + ++it; + } + } + } +} + +EncodedImageExtractionResult SingleProcessEncodedImageDataInjector::ExtractData( + const EncodedImage& source) { + size_t size = source.size(); + auto buffer = EncodedImageBuffer::Create(source.data(), source.size()); + EncodedImage out = source; + out.SetEncodedData(buffer); + + std::vector<size_t> frame_sizes; + std::vector<size_t> frame_sl_index; + size_t max_spatial_index = out.SpatialIndex().value_or(0); + for (size_t i = 0; i <= max_spatial_index; ++i) { + auto frame_size = source.SpatialLayerFrameSize(i); + if (frame_size.value_or(0)) { + frame_sl_index.push_back(i); + frame_sizes.push_back(frame_size.value()); + } + } + if (frame_sizes.empty()) { + frame_sizes.push_back(size); + } + + size_t prev_frames_size = 0; + absl::optional<uint16_t> id = absl::nullopt; + bool discard = true; + std::vector<ExtractionInfo> extraction_infos; + for (size_t frame_size : frame_sizes) { + size_t insertion_pos = + prev_frames_size + frame_size - ExtractionInfo::kUsedBufferSize; + // Extract frame id from first 2 bytes starting from insertion pos. + uint16_t next_id = buffer->data()[insertion_pos] + + (buffer->data()[insertion_pos + 1] << 8); + // Extract frame sub id from second 3 byte starting from insertion pos. + uint8_t sub_id = buffer->data()[insertion_pos + 2]; + RTC_CHECK(!id || *id == next_id) + << "Different frames encoded into single encoded image: " << *id + << " vs " << next_id; + id = next_id; + ExtractionInfo info; + { + MutexLock lock(&lock_); + auto ext_vector_it = extraction_cache_.find(next_id); + RTC_CHECK(ext_vector_it != extraction_cache_.end()) + << "Unknown frame_id=" << next_id; + + auto info_it = ext_vector_it->second.infos.find(sub_id); + RTC_CHECK(info_it != ext_vector_it->second.infos.end()) + << "Unknown sub_id=" << sub_id << " for frame_id=" << next_id; + info_it->second.received_count++; + info = info_it->second; + if (info.received_count == expected_receivers_count_) { + ext_vector_it->second.infos.erase(info_it); + } + } + // We need to discard encoded image only if all concatenated encoded images + // have to be discarded. + discard = discard && info.discard; + + extraction_infos.push_back(info); + prev_frames_size += frame_size; + } + RTC_CHECK(id); + + if (discard) { + out.set_size(0); + for (size_t i = 0; i <= max_spatial_index; ++i) { + out.SetSpatialLayerFrameSize(i, 0); + } + return EncodedImageExtractionResult{*id, out, true}; + } + + // Make a pass from begin to end to restore origin payload and erase discarded + // encoded images. + size_t pos = 0; + for (size_t frame_index = 0; frame_index < frame_sizes.size(); + ++frame_index) { + RTC_CHECK(pos < size); + const size_t frame_size = frame_sizes[frame_index]; + const ExtractionInfo& info = extraction_infos[frame_index]; + if (info.discard) { + // If this encoded image is marked to be discarded - erase it's payload + // from the buffer. + memmove(&buffer->data()[pos], &buffer->data()[pos + frame_size], + size - pos - frame_size); + RTC_CHECK_LT(frame_index, frame_sl_index.size()) + << "codec doesn't support discard option or the image, that was " + "supposed to be discarded, is lost"; + out.SetSpatialLayerFrameSize(frame_sl_index[frame_index], 0); + size -= frame_size; + } else { + memcpy( + &buffer->data()[pos + frame_size - ExtractionInfo::kUsedBufferSize], + info.origin_data, ExtractionInfo::kUsedBufferSize); + pos += frame_size; + } + } + out.set_size(pos); + + return EncodedImageExtractionResult{*id, out, discard}; +} + +SingleProcessEncodedImageDataInjector::ExtractionInfoVector:: + ExtractionInfoVector() = default; +SingleProcessEncodedImageDataInjector::ExtractionInfoVector:: + ~ExtractionInfoVector() = default; + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h new file mode 100644 index 0000000000..1082440e2f --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_ + +#include <cstdint> +#include <map> +#include <memory> +#include <utility> +#include <vector> + +#include "api/video/encoded_image.h" +#include "rtc_base/synchronization/mutex.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Based on assumption that all call participants are in the same OS process +// and uses same QualityAnalyzingVideoContext to obtain +// EncodedImageDataInjector. +// +// To inject frame id and discard flag into EncodedImage injector uses last 3rd +// and 2nd bytes of EncodedImage payload. Then it uses last byte for frame +// sub id, that is required to distinguish different spatial layers. The origin +// data from these 3 bytes will be stored inside injector's internal storage and +// then will be restored during extraction phase. +// +// This injector won't add any extra overhead into EncodedImage payload and +// support frames with any size of payload. Also assumes that every EncodedImage +// payload size is greater or equals to 3 bytes +// +// This injector doesn't support video frames/encoded images without frame ID. +class SingleProcessEncodedImageDataInjector + : public EncodedImageDataPropagator { + public: + SingleProcessEncodedImageDataInjector(); + ~SingleProcessEncodedImageDataInjector() override; + + // Id and discard flag will be injected into EncodedImage buffer directly. + // This buffer won't be fully copied, so `source` image buffer will be also + // changed. + EncodedImage InjectData(uint16_t id, + bool discard, + const EncodedImage& source) override; + + void Start(int expected_receivers_count) override { + MutexLock crit(&lock_); + expected_receivers_count_ = expected_receivers_count; + } + void AddParticipantInCall() override; + void RemoveParticipantInCall() override; + EncodedImageExtractionResult ExtractData(const EncodedImage& source) override; + + private: + // Contains data required to extract frame id from EncodedImage and restore + // original buffer. + struct ExtractionInfo { + // Number of bytes from the beginning of the EncodedImage buffer that will + // be used to store frame id and sub id. + const static size_t kUsedBufferSize = 3; + // Frame sub id to distinguish encoded images for different spatial layers. + uint8_t sub_id; + // Flag to show is this encoded images should be discarded by analyzing + // decoder because of not required spatial layer/simulcast stream. + bool discard; + // Data from first 3 bytes of origin encoded image's payload. + uint8_t origin_data[ExtractionInfo::kUsedBufferSize]; + // Count of how many times this frame was received. + int received_count = 0; + }; + + struct ExtractionInfoVector { + ExtractionInfoVector(); + ~ExtractionInfoVector(); + + // Next sub id, that have to be used for this frame id. + uint8_t next_sub_id = 0; + std::map<uint8_t, ExtractionInfo> infos; + }; + + Mutex lock_; + int expected_receivers_count_ RTC_GUARDED_BY(lock_); + // Stores a mapping from frame id to extraction info for spatial layers + // for this frame id. There can be a lot of them, because if frame was + // dropped we can't clean it up, because we won't receive a signal on + // decoder side about that frame. In such case it will be replaced + // when sub id will overlap. + std::map<uint16_t, ExtractionInfoVector> extraction_cache_ + RTC_GUARDED_BY(lock_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc new file mode 100644 index 0000000000..f6fa40455a --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc @@ -0,0 +1,445 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h" + +#include <utility> + +#include "api/video/encoded_image.h" +#include "rtc_base/buffer.h" +#include "test/gtest.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +rtc::scoped_refptr<EncodedImageBuffer> +CreateEncodedImageBufferOfSizeNFilledWithValuesFromX(size_t n, uint8_t x) { + auto buffer = EncodedImageBuffer::Create(n); + for (size_t i = 0; i < n; ++i) { + buffer->data()[i] = static_cast<uint8_t>(x + i); + } + return buffer; +} + +EncodedImage CreateEncodedImageOfSizeNFilledWithValuesFromX(size_t n, + uint8_t x) { + EncodedImage image; + image.SetEncodedData( + CreateEncodedImageBufferOfSizeNFilledWithValuesFromX(n, x)); + return image; +} + +EncodedImage DeepCopyEncodedImage(const EncodedImage& source) { + EncodedImage copy = source; + copy.SetEncodedData(EncodedImageBuffer::Create(source.data(), source.size())); + return copy; +} + +TEST(SingleProcessEncodedImageDataInjectorTest, InjectExtractDiscardFalse) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + + EncodedImageExtractionResult out = + injector.ExtractData(injector.InjectData(512, false, source)); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, InjectExtractDiscardTrue) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + + EncodedImageExtractionResult out = + injector.ExtractData(injector.InjectData(512, true, source)); + EXPECT_EQ(out.id, 512); + EXPECT_TRUE(out.discard); + EXPECT_EQ(out.image.size(), 0ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); +} + +TEST(SingleProcessEncodedImageDataInjectorTest, + InjectWithUnsetSpatialLayerSizes) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + + EncodedImage intermediate = injector.InjectData(512, false, source); + intermediate.SetSpatialIndex(2); + + EncodedImageExtractionResult out = injector.ExtractData(intermediate); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } + EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2); + for (int i = 0; i < 3; ++i) { + EXPECT_EQ(out.image.SpatialLayerFrameSize(i).value_or(0), 0ul); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, + InjectWithZeroSpatialLayerSizes) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + + EncodedImage intermediate = injector.InjectData(512, false, source); + intermediate.SetSpatialIndex(2); + intermediate.SetSpatialLayerFrameSize(0, 0); + intermediate.SetSpatialLayerFrameSize(1, 0); + intermediate.SetSpatialLayerFrameSize(2, 0); + + EncodedImageExtractionResult out = injector.ExtractData(intermediate); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } + EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2); + for (int i = 0; i < 3; ++i) { + EXPECT_EQ(out.image.SpatialLayerFrameSize(i).value_or(0), 0ul); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, Inject3Extract3) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + // 1st frame + EncodedImage source1 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source1.SetTimestamp(123456710); + // 2nd frame 1st spatial layer + EncodedImage source2 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/11); + source2.SetTimestamp(123456720); + // 2nd frame 2nd spatial layer + EncodedImage source3 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/21); + source3.SetTimestamp(123456720); + + EncodedImage intermediate1 = injector.InjectData(510, false, source1); + EncodedImage intermediate2 = injector.InjectData(520, true, source2); + EncodedImage intermediate3 = injector.InjectData(520, false, source3); + + // Extract ids in different order. + EncodedImageExtractionResult out3 = injector.ExtractData(intermediate3); + EncodedImageExtractionResult out1 = injector.ExtractData(intermediate1); + EncodedImageExtractionResult out2 = injector.ExtractData(intermediate2); + + EXPECT_EQ(out1.id, 510); + EXPECT_FALSE(out1.discard); + EXPECT_EQ(out1.image.size(), 10ul); + EXPECT_EQ(out1.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out1.image.data()[i], i + 1); + } + EXPECT_EQ(out2.id, 520); + EXPECT_TRUE(out2.discard); + EXPECT_EQ(out2.image.size(), 0ul); + EXPECT_EQ(out2.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + EXPECT_EQ(out3.id, 520); + EXPECT_FALSE(out3.discard); + EXPECT_EQ(out3.image.size(), 10ul); + EXPECT_EQ(out3.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out3.image.data()[i], i + 21); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, InjectExtractFromConcatenated) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + EncodedImage source1 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source1.SetTimestamp(123456710); + EncodedImage source2 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/11); + source2.SetTimestamp(123456710); + EncodedImage source3 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/21); + source3.SetTimestamp(123456710); + + // Inject id into 3 images with same frame id. + EncodedImage intermediate1 = injector.InjectData(512, false, source1); + EncodedImage intermediate2 = injector.InjectData(512, true, source2); + EncodedImage intermediate3 = injector.InjectData(512, false, source3); + + // Concatenate them into single encoded image, like it can be done in jitter + // buffer. + size_t concatenated_length = + intermediate1.size() + intermediate2.size() + intermediate3.size(); + rtc::Buffer concatenated_buffer; + concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size()); + concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size()); + concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size()); + EncodedImage concatenated; + concatenated.SetEncodedData(EncodedImageBuffer::Create( + concatenated_buffer.data(), concatenated_length)); + concatenated.SetSpatialIndex(2); + concatenated.SetSpatialLayerFrameSize(0, intermediate1.size()); + concatenated.SetSpatialLayerFrameSize(1, intermediate2.size()); + concatenated.SetSpatialLayerFrameSize(2, intermediate3.size()); + + // Extract frame id from concatenated image + EncodedImageExtractionResult out = injector.ExtractData(concatenated); + + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 2 * 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + EXPECT_EQ(out.image.data()[i + 10], i + 21); + } + EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(1).value_or(0), 0ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(2).value_or(0), 10ul); +} + +TEST(SingleProcessEncodedImageDataInjector, + InjectExtractFromConcatenatedAllDiscarded) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + EncodedImage source1 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source1.SetTimestamp(123456710); + EncodedImage source2 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/11); + source2.SetTimestamp(123456710); + EncodedImage source3 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/21); + source3.SetTimestamp(123456710); + + // Inject id into 3 images with same frame id. + EncodedImage intermediate1 = injector.InjectData(512, true, source1); + EncodedImage intermediate2 = injector.InjectData(512, true, source2); + EncodedImage intermediate3 = injector.InjectData(512, true, source3); + + // Concatenate them into single encoded image, like it can be done in jitter + // buffer. + size_t concatenated_length = + intermediate1.size() + intermediate2.size() + intermediate3.size(); + rtc::Buffer concatenated_buffer; + concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size()); + concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size()); + concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size()); + EncodedImage concatenated; + concatenated.SetEncodedData(EncodedImageBuffer::Create( + concatenated_buffer.data(), concatenated_length)); + concatenated.SetSpatialIndex(2); + concatenated.SetSpatialLayerFrameSize(0, intermediate1.size()); + concatenated.SetSpatialLayerFrameSize(1, intermediate2.size()); + concatenated.SetSpatialLayerFrameSize(2, intermediate3.size()); + + // Extract frame id from concatenated image + EncodedImageExtractionResult out = injector.ExtractData(concatenated); + + EXPECT_EQ(out.id, 512); + EXPECT_TRUE(out.discard); + EXPECT_EQ(out.image.size(), 0ul); + EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2); + for (int i = 0; i < 3; ++i) { + EXPECT_EQ(out.image.SpatialLayerFrameSize(i).value_or(0), 0ul); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, InjectOnceExtractTwice) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/2); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + + EncodedImageExtractionResult out = injector.ExtractData( + injector.InjectData(/*id=*/512, /*discard=*/false, source)); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } + out = injector.ExtractData( + injector.InjectData(/*id=*/512, /*discard=*/false, source)); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, Add1stReceiverAfterStart) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/0); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + EncodedImage modified_image = injector.InjectData( + /*id=*/512, /*discard=*/false, source); + + injector.AddParticipantInCall(); + EncodedImageExtractionResult out = injector.ExtractData(modified_image); + + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, Add3rdReceiverAfterStart) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/2); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + EncodedImage modified_image = injector.InjectData( + /*id=*/512, /*discard=*/false, source); + injector.ExtractData(modified_image); + + injector.AddParticipantInCall(); + injector.ExtractData(modified_image); + EncodedImageExtractionResult out = injector.ExtractData(modified_image); + + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, + RemoveReceiverRemovesOnlyFullyReceivedFrames) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/3); + + EncodedImage source1 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source1.SetTimestamp(10); + EncodedImage source2 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source2.SetTimestamp(20); + + EncodedImage modified_image1 = injector.InjectData( + /*id=*/512, /*discard=*/false, source1); + EncodedImage modified_image2 = injector.InjectData( + /*id=*/513, /*discard=*/false, source2); + + // Out of 3 receivers 1st image received by 2 and 2nd image by 1 + injector.ExtractData(DeepCopyEncodedImage(modified_image1)); + injector.ExtractData(DeepCopyEncodedImage(modified_image1)); + injector.ExtractData(DeepCopyEncodedImage(modified_image2)); + + // When we removed one receiver, 2nd image should still be available for + // extraction. + injector.RemoveParticipantInCall(); + + EncodedImageExtractionResult out = + injector.ExtractData(DeepCopyEncodedImage(modified_image2)); + + EXPECT_EQ(out.id, 513); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +// Death tests. +// Disabled on Android because death tests misbehave on Android, see +// base/test/gtest_util.h. +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +TEST(SingleProcessEncodedImageDataInjectorTestDeathTest, + InjectOnceExtractMoreThenExpected) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/2); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + + EncodedImage modified = + injector.InjectData(/*id=*/512, /*discard=*/false, source); + + injector.ExtractData(DeepCopyEncodedImage(modified)); + injector.ExtractData(DeepCopyEncodedImage(modified)); + EXPECT_DEATH(injector.ExtractData(DeepCopyEncodedImage(modified)), + "Unknown sub_id=0 for frame_id=512"); +} + +TEST(SingleProcessEncodedImageDataInjectorTestDeathTest, + RemoveReceiverRemovesOnlyFullyReceivedFramesVerifyFrameIsRemoved) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/3); + + EncodedImage source1 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source1.SetTimestamp(10); + EncodedImage source2 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source2.SetTimestamp(20); + + EncodedImage modified_image1 = injector.InjectData( + /*id=*/512, /*discard=*/false, source1); + EncodedImage modified_image2 = injector.InjectData( + /*id=*/513, /*discard=*/false, source2); + + // Out of 3 receivers 1st image received by 2 and 2nd image by 1 + injector.ExtractData(DeepCopyEncodedImage(modified_image1)); + injector.ExtractData(DeepCopyEncodedImage(modified_image1)); + injector.ExtractData(DeepCopyEncodedImage(modified_image2)); + + // When we removed one receiver 1st image should be removed. + injector.RemoveParticipantInCall(); + + EXPECT_DEATH(injector.ExtractData(DeepCopyEncodedImage(modified_image1)), + "Unknown sub_id=0 for frame_id=512"); +} +#endif // RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.cc new file mode 100644 index 0000000000..4fec0a8f9e --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.cc @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/video_dumping.h" + +#include <stdio.h> + +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/test/video/video_frame_writer.h" +#include "api/video/video_frame.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/clock.h" +#include "test/testsupport/video_frame_writer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +class VideoFrameIdsWriter final : public test::VideoFrameWriter { + public: + explicit VideoFrameIdsWriter(absl::string_view file_name) + : file_name_(file_name) { + output_file_ = fopen(file_name_.c_str(), "wb"); + RTC_LOG(LS_INFO) << "Writing VideoFrame IDs into " << file_name_; + RTC_CHECK(output_file_ != nullptr) + << "Failed to open file to dump frame ids for writing: " << file_name_; + } + ~VideoFrameIdsWriter() override { Close(); } + + bool WriteFrame(const VideoFrame& frame) override { + RTC_CHECK(output_file_ != nullptr) << "Writer is already closed"; + int chars_written = fprintf(output_file_, "%d\n", frame.id()); + if (chars_written < 2) { + RTC_LOG(LS_ERROR) << "Failed to write frame id to the output file: " + << file_name_; + return false; + } + return true; + } + + void Close() override { + if (output_file_ != nullptr) { + RTC_LOG(LS_INFO) << "Closing file for VideoFrame IDs: " << file_name_; + fclose(output_file_); + output_file_ = nullptr; + } + } + + private: + const std::string file_name_; + FILE* output_file_; +}; + +// Broadcast received frame to multiple underlying frame writers. +class BroadcastingFrameWriter final : public test::VideoFrameWriter { + public: + explicit BroadcastingFrameWriter( + std::vector<std::unique_ptr<test::VideoFrameWriter>> delegates) + : delegates_(std::move(delegates)) {} + ~BroadcastingFrameWriter() override { Close(); } + + bool WriteFrame(const webrtc::VideoFrame& frame) override { + for (auto& delegate : delegates_) { + if (!delegate->WriteFrame(frame)) { + return false; + } + } + return true; + } + + void Close() override { + for (auto& delegate : delegates_) { + delegate->Close(); + } + } + + private: + std::vector<std::unique_ptr<test::VideoFrameWriter>> delegates_; +}; + +} // namespace + +VideoWriter::VideoWriter(test::VideoFrameWriter* video_writer, + int sampling_modulo) + : video_writer_(video_writer), sampling_modulo_(sampling_modulo) {} + +void VideoWriter::OnFrame(const VideoFrame& frame) { + if (frames_counter_++ % sampling_modulo_ != 0) { + return; + } + bool result = video_writer_->WriteFrame(frame); + RTC_CHECK(result) << "Failed to write frame"; +} + +std::unique_ptr<test::VideoFrameWriter> CreateVideoFrameWithIdsWriter( + std::unique_ptr<test::VideoFrameWriter> video_writer_delegate, + absl::string_view frame_ids_dump_file_name) { + std::vector<std::unique_ptr<test::VideoFrameWriter>> requested_writers; + requested_writers.push_back(std::move(video_writer_delegate)); + requested_writers.push_back( + std::make_unique<VideoFrameIdsWriter>(frame_ids_dump_file_name)); + return std::make_unique<BroadcastingFrameWriter>( + std::move(requested_writers)); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.h new file mode 100644 index 0000000000..cad4e1bdbf --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_DUMPING_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_DUMPING_H_ + +#include <memory> +#include <string> + +#include "absl/strings/string_view.h" +#include "api/test/video/video_frame_writer.h" +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "test/testsupport/video_frame_writer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// `VideoSinkInterface` to dump incoming video frames into specified video +// writer. +class VideoWriter final : public rtc::VideoSinkInterface<VideoFrame> { + public: + // Creates video writer. Caller keeps ownership of `video_writer` and is + // responsible for closing it after VideoWriter will be destroyed. + VideoWriter(test::VideoFrameWriter* video_writer, int sampling_modulo); + VideoWriter(const VideoWriter&) = delete; + VideoWriter& operator=(const VideoWriter&) = delete; + ~VideoWriter() override = default; + + void OnFrame(const VideoFrame& frame) override; + + private: + test::VideoFrameWriter* const video_writer_; + const int sampling_modulo_; + + int64_t frames_counter_ = 0; +}; + +// Creates a `VideoFrameWriter` to dump video frames together with their ids. +// It uses provided `video_writer_delegate` to write video itself. Frame ids +// will be logged into the specified file. +std::unique_ptr<test::VideoFrameWriter> CreateVideoFrameWithIdsWriter( + std::unique_ptr<test::VideoFrameWriter> video_writer_delegate, + absl::string_view frame_ids_dump_file_name); + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_DUMPING_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping_test.cc new file mode 100644 index 0000000000..5dd4021516 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping_test.cc @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/video_dumping.h" + +#include <stdio.h> + +#include <memory> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_buffer.h" +#include "rtc_base/random.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" +#include "test/testsupport/frame_reader.h" +#include "test/testsupport/video_frame_writer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::testing::ElementsAreArray; +using ::testing::Eq; +using ::testing::Test; + +uint8_t RandByte(Random& random) { + return random.Rand(255); +} + +VideoFrame CreateRandom2x2VideoFrame(uint16_t id, Random& random) { + rtc::scoped_refptr<I420Buffer> buffer = I420Buffer::Create(2, 2); + + uint8_t data[6] = {RandByte(random), RandByte(random), RandByte(random), + RandByte(random), RandByte(random), RandByte(random)}; + + memcpy(buffer->MutableDataY(), data, 2); + memcpy(buffer->MutableDataY() + buffer->StrideY(), data + 2, 2); + memcpy(buffer->MutableDataU(), data + 4, 1); + memcpy(buffer->MutableDataV(), data + 5, 1); + + return VideoFrame::Builder() + .set_id(id) + .set_video_frame_buffer(buffer) + .set_timestamp_us(1) + .build(); +} + +std::vector<uint8_t> AsVector(const uint8_t* data, size_t size) { + std::vector<uint8_t> out; + out.assign(data, data + size); + return out; +} + +void AssertFramesEqual(rtc::scoped_refptr<webrtc::I420BufferInterface> actual, + rtc::scoped_refptr<VideoFrameBuffer> expected) { + ASSERT_THAT(actual->width(), Eq(expected->width())); + ASSERT_THAT(actual->height(), Eq(expected->height())); + rtc::scoped_refptr<webrtc::I420BufferInterface> expected_i420 = + expected->ToI420(); + + int height = actual->height(); + + EXPECT_THAT(AsVector(actual->DataY(), actual->StrideY() * height), + ElementsAreArray(expected_i420->DataY(), + expected_i420->StrideY() * height)); + EXPECT_THAT(AsVector(actual->DataU(), actual->StrideU() * (height + 1) / 2), + ElementsAreArray(expected_i420->DataU(), + expected_i420->StrideU() * (height + 1) / 2)); + EXPECT_THAT(AsVector(actual->DataV(), actual->StrideV() * (height + 1) / 2), + ElementsAreArray(expected_i420->DataV(), + expected_i420->StrideV() * (height + 1) / 2)); +} + +void AssertFrameIdsAre(const std::string& filename, + std::vector<std::string> expected_ids) { + FILE* file = fopen(filename.c_str(), "r"); + ASSERT_TRUE(file != nullptr); + std::vector<std::string> actual_ids; + char buffer[8]; + while (fgets(buffer, sizeof buffer, file) != nullptr) { + std::string current_id(buffer); + ASSERT_GE(current_id.size(), 2lu); + // Trim "\n" at the end. + actual_ids.push_back(current_id.substr(0, current_id.size() - 1)); + } + EXPECT_THAT(actual_ids, ElementsAreArray(expected_ids)); +} + +class VideoDumpingTest : public Test { + protected: + ~VideoDumpingTest() override = default; + + void SetUp() override { + video_filename_ = webrtc::test::TempFilename(webrtc::test::OutputPath(), + "video_dumping_test"); + ids_filename_ = webrtc::test::TempFilename(webrtc::test::OutputPath(), + "video_dumping_test"); + } + + void TearDown() override { + remove(video_filename_.c_str()); + remove(ids_filename_.c_str()); + } + + std::string video_filename_; + std::string ids_filename_; +}; + +using CreateVideoFrameWithIdsWriterTest = VideoDumpingTest; + +TEST_F(CreateVideoFrameWithIdsWriterTest, VideoIsWritenWithFrameIds) { + Random random(/*seed=*/100); + VideoFrame frame1 = CreateRandom2x2VideoFrame(1, random); + VideoFrame frame2 = CreateRandom2x2VideoFrame(2, random); + + std::unique_ptr<test::VideoFrameWriter> writer = + CreateVideoFrameWithIdsWriter( + std::make_unique<test::Y4mVideoFrameWriterImpl>( + std::string(video_filename_), + /*width=*/2, /*height=*/2, /*fps=*/2), + ids_filename_); + + ASSERT_TRUE(writer->WriteFrame(frame1)); + ASSERT_TRUE(writer->WriteFrame(frame2)); + writer->Close(); + + auto frame_reader = test::CreateY4mFrameReader(video_filename_); + EXPECT_THAT(frame_reader->num_frames(), Eq(2)); + AssertFramesEqual(frame_reader->PullFrame(), frame1.video_frame_buffer()); + AssertFramesEqual(frame_reader->PullFrame(), frame2.video_frame_buffer()); + AssertFrameIdsAre(ids_filename_, {"1", "2"}); +} + +using VideoWriterTest = VideoDumpingTest; + +TEST_F(VideoWriterTest, AllFramesAreWrittenWithSamplingModulo1) { + Random random(/*seed=*/100); + VideoFrame frame1 = CreateRandom2x2VideoFrame(1, random); + VideoFrame frame2 = CreateRandom2x2VideoFrame(2, random); + + { + test::Y4mVideoFrameWriterImpl frame_writer(std::string(video_filename_), + /*width=*/2, /*height=*/2, + /*fps=*/2); + VideoWriter writer(&frame_writer, /*sampling_modulo=*/1); + + writer.OnFrame(frame1); + writer.OnFrame(frame2); + frame_writer.Close(); + } + + auto frame_reader = test::CreateY4mFrameReader(video_filename_); + EXPECT_THAT(frame_reader->num_frames(), Eq(2)); + AssertFramesEqual(frame_reader->PullFrame(), frame1.video_frame_buffer()); + AssertFramesEqual(frame_reader->PullFrame(), frame2.video_frame_buffer()); +} + +TEST_F(VideoWriterTest, OnlyEvery2ndFramesIsWrittenWithSamplingModulo2) { + Random random(/*seed=*/100); + VideoFrame frame1 = CreateRandom2x2VideoFrame(1, random); + VideoFrame frame2 = CreateRandom2x2VideoFrame(2, random); + VideoFrame frame3 = CreateRandom2x2VideoFrame(3, random); + + { + test::Y4mVideoFrameWriterImpl frame_writer(std::string(video_filename_), + /*width=*/2, /*height=*/2, + /*fps=*/2); + VideoWriter writer(&frame_writer, /*sampling_modulo=*/2); + + writer.OnFrame(frame1); + writer.OnFrame(frame2); + writer.OnFrame(frame3); + frame_writer.Close(); + } + + auto frame_reader = test::CreateY4mFrameReader(video_filename_); + EXPECT_THAT(frame_reader->num_frames(), Eq(2)); + AssertFramesEqual(frame_reader->PullFrame(), frame1.video_frame_buffer()); + AssertFramesEqual(frame_reader->PullFrame(), frame3.video_frame_buffer()); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.cc new file mode 100644 index 0000000000..5a74d60250 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.cc @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h" + +#include "absl/memory/memory.h" +#include "api/video/encoded_image.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +EncodedImage VideoFrameTrackingIdInjector::InjectData( + uint16_t id, + bool unused_discard, + const EncodedImage& source) { + RTC_CHECK(!unused_discard); + EncodedImage out = source; + out.SetVideoFrameTrackingId(id); + return out; +} + +EncodedImageExtractionResult VideoFrameTrackingIdInjector::ExtractData( + const EncodedImage& source) { + return EncodedImageExtractionResult{source.VideoFrameTrackingId(), source, + /*discard=*/false}; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h new file mode 100644 index 0000000000..ecc3cd3f51 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_FRAME_TRACKING_ID_INJECTOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_FRAME_TRACKING_ID_INJECTOR_H_ + +#include <cstdint> + +#include "api/video/encoded_image.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// This injector sets and retrieves the provided id in the EncodedImage +// video_frame_tracking_id field. This is only possible with the RTP header +// extension VideoFrameTrackingIdExtension that will propagate the input +// tracking id to the received EncodedImage. This RTP header extension is +// enabled with the field trial WebRTC-VideoFrameTrackingIdAdvertised +// (http://www.webrtc.org/experiments/rtp-hdrext/video-frame-tracking-id). +// +// Note that this injector doesn't allow to discard frames. +class VideoFrameTrackingIdInjector : public EncodedImageDataPropagator { + public: + EncodedImage InjectData(uint16_t id, + bool unused_discard, + const EncodedImage& source) override; + + EncodedImageExtractionResult ExtractData(const EncodedImage& source) override; + + void Start(int) override {} + void AddParticipantInCall() override {} + void RemoveParticipantInCall() override {} +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_FRAME_TRACKING_ID_INJECTOR_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector_unittest.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector_unittest.cc new file mode 100644 index 0000000000..c7d453c4bb --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector_unittest.cc @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h" + +#include "api/video/encoded_image.h" +#include "rtc_base/buffer.h" +#include "test/gtest.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +EncodedImage CreateEncodedImageOfSizeN(size_t n) { + EncodedImage image; + rtc::scoped_refptr<EncodedImageBuffer> buffer = EncodedImageBuffer::Create(n); + for (size_t i = 0; i < n; ++i) { + buffer->data()[i] = static_cast<uint8_t>(i); + } + image.SetEncodedData(buffer); + return image; +} + +TEST(VideoFrameTrackingIdInjectorTest, InjectExtractDiscardFalse) { + VideoFrameTrackingIdInjector injector; + EncodedImage source = CreateEncodedImageOfSizeN(10); + EncodedImageExtractionResult out = + injector.ExtractData(injector.InjectData(512, false, source)); + + ASSERT_TRUE(out.id.has_value()); + EXPECT_EQ(*out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(source.data()[i], out.image.data()[i]); + } +} + +#if GTEST_HAS_DEATH_TEST +TEST(VideoFrameTrackingIdInjectorTest, InjectExtractDiscardTrue) { + VideoFrameTrackingIdInjector injector; + EncodedImage source = CreateEncodedImageOfSizeN(10); + + EXPECT_DEATH(injector.InjectData(512, true, source), ""); +} +#endif // GTEST_HAS_DEATH_TEST + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc new file mode 100644 index 0000000000..87c11886cc --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc @@ -0,0 +1,264 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h" + +#include <stdio.h> + +#include <memory> +#include <utility> +#include <vector> + +#include "absl/memory/memory.h" +#include "absl/strings/string_view.h" +#include "api/array_view.h" +#include "api/test/pclf/media_configuration.h" +#include "api/video/i420_buffer.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/strings/string_builder.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/analyzing_video_sink.h" +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h" +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h" +#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h" +#include "test/pc/e2e/analyzer/video/video_dumping.h" +#include "test/testsupport/fixed_fps_video_frame_writer_adapter.h" +#include "test/video_renderer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using webrtc::webrtc_pc_e2e::VideoConfig; +using EmulatedSFUConfigMap = + ::webrtc::webrtc_pc_e2e::QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap; + +class AnalyzingFramePreprocessor + : public test::TestVideoCapturer::FramePreprocessor { + public: + AnalyzingFramePreprocessor( + absl::string_view peer_name, + absl::string_view stream_label, + VideoQualityAnalyzerInterface* analyzer, + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> sinks) + : peer_name_(peer_name), + stream_label_(stream_label), + analyzer_(analyzer), + sinks_(std::move(sinks)) {} + ~AnalyzingFramePreprocessor() override = default; + + VideoFrame Preprocess(const VideoFrame& source_frame) override { + // Copy VideoFrame to be able to set id on it. + VideoFrame frame = source_frame; + uint16_t frame_id = + analyzer_->OnFrameCaptured(peer_name_, stream_label_, frame); + frame.set_id(frame_id); + + for (auto& sink : sinks_) { + sink->OnFrame(frame); + } + return frame; + } + + private: + const std::string peer_name_; + const std::string stream_label_; + VideoQualityAnalyzerInterface* const analyzer_; + const std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> + sinks_; +}; + +} // namespace + +VideoQualityAnalyzerInjectionHelper::VideoQualityAnalyzerInjectionHelper( + Clock* clock, + std::unique_ptr<VideoQualityAnalyzerInterface> analyzer, + EncodedImageDataInjector* injector, + EncodedImageDataExtractor* extractor) + : clock_(clock), + analyzer_(std::move(analyzer)), + injector_(injector), + extractor_(extractor) { + RTC_DCHECK(clock_); + RTC_DCHECK(injector_); + RTC_DCHECK(extractor_); +} +VideoQualityAnalyzerInjectionHelper::~VideoQualityAnalyzerInjectionHelper() = + default; + +std::unique_ptr<VideoEncoderFactory> +VideoQualityAnalyzerInjectionHelper::WrapVideoEncoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoEncoderFactory> delegate, + double bitrate_multiplier, + EmulatedSFUConfigMap stream_to_sfu_config) const { + return std::make_unique<QualityAnalyzingVideoEncoderFactory>( + peer_name, std::move(delegate), bitrate_multiplier, + std::move(stream_to_sfu_config), injector_, analyzer_.get()); +} + +std::unique_ptr<VideoDecoderFactory> +VideoQualityAnalyzerInjectionHelper::WrapVideoDecoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoDecoderFactory> delegate) const { + return std::make_unique<QualityAnalyzingVideoDecoderFactory>( + peer_name, std::move(delegate), extractor_, analyzer_.get()); +} + +std::unique_ptr<test::TestVideoCapturer::FramePreprocessor> +VideoQualityAnalyzerInjectionHelper::CreateFramePreprocessor( + absl::string_view peer_name, + const VideoConfig& config) { + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> sinks; + if (config.input_dump_options.has_value()) { + std::unique_ptr<test::VideoFrameWriter> writer = + config.input_dump_options->CreateInputDumpVideoFrameWriter( + *config.stream_label, config.GetResolution()); + sinks.push_back(std::make_unique<VideoWriter>( + writer.get(), config.input_dump_options->sampling_modulo())); + video_writers_.push_back(std::move(writer)); + } + if (config.show_on_screen) { + sinks.push_back(absl::WrapUnique( + test::VideoRenderer::Create((*config.stream_label + "-capture").c_str(), + config.width, config.height))); + } + sinks_helper_.AddConfig(peer_name, config); + { + MutexLock lock(&mutex_); + known_video_configs_.insert({*config.stream_label, config}); + } + return std::make_unique<AnalyzingFramePreprocessor>( + peer_name, std::move(*config.stream_label), analyzer_.get(), + std::move(sinks)); +} + +std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> +VideoQualityAnalyzerInjectionHelper::CreateVideoSink( + absl::string_view peer_name) { + return std::make_unique<AnalyzingVideoSink2>(peer_name, this); +} + +std::unique_ptr<AnalyzingVideoSink> +VideoQualityAnalyzerInjectionHelper::CreateVideoSink( + absl::string_view peer_name, + const VideoSubscription& subscription, + bool report_infra_metrics) { + return std::make_unique<AnalyzingVideoSink>(peer_name, clock_, *analyzer_, + sinks_helper_, subscription, + report_infra_metrics); +} + +void VideoQualityAnalyzerInjectionHelper::Start( + std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count) { + analyzer_->Start(std::move(test_case_name), peer_names, max_threads_count); + extractor_->Start(peer_names.size()); + MutexLock lock(&mutex_); + peers_count_ = peer_names.size(); +} + +void VideoQualityAnalyzerInjectionHelper::RegisterParticipantInCall( + absl::string_view peer_name) { + analyzer_->RegisterParticipantInCall(peer_name); + extractor_->AddParticipantInCall(); + MutexLock lock(&mutex_); + peers_count_++; +} + +void VideoQualityAnalyzerInjectionHelper::UnregisterParticipantInCall( + absl::string_view peer_name) { + analyzer_->UnregisterParticipantInCall(peer_name); + extractor_->RemoveParticipantInCall(); + MutexLock lock(&mutex_); + peers_count_--; +} + +void VideoQualityAnalyzerInjectionHelper::OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) { + analyzer_->OnStatsReports(pc_label, report); +} + +void VideoQualityAnalyzerInjectionHelper::Stop() { + analyzer_->Stop(); + for (const auto& video_writer : video_writers_) { + video_writer->Close(); + } + video_writers_.clear(); + sinks_helper_.Clear(); +} + +void VideoQualityAnalyzerInjectionHelper::OnFrame(absl::string_view peer_name, + const VideoFrame& frame) { + if (IsDummyFrame(frame)) { + // This is dummy frame, so we don't need to process it further. + return; + } + // Copy entire video frame including video buffer to ensure that analyzer + // won't hold any WebRTC internal buffers. + VideoFrame frame_copy = frame; + frame_copy.set_video_frame_buffer( + I420Buffer::Copy(*frame.video_frame_buffer()->ToI420())); + analyzer_->OnFrameRendered(peer_name, frame_copy); + + if (frame.id() != VideoFrame::kNotSetId) { + std::string stream_label = analyzer_->GetStreamLabel(frame.id()); + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>* sinks = + PopulateSinks(ReceiverStream(peer_name, stream_label)); + if (sinks == nullptr) { + return; + } + for (auto& sink : *sinks) { + sink->OnFrame(frame); + } + } +} + +std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>* +VideoQualityAnalyzerInjectionHelper::PopulateSinks( + const ReceiverStream& receiver_stream) { + MutexLock lock(&mutex_); + auto sinks_it = sinks_.find(receiver_stream); + if (sinks_it != sinks_.end()) { + return &sinks_it->second; + } + auto it = known_video_configs_.find(receiver_stream.stream_label); + RTC_DCHECK(it != known_video_configs_.end()) + << "No video config for stream " << receiver_stream.stream_label; + const VideoConfig& config = it->second; + + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> sinks; + if (config.output_dump_options.has_value()) { + std::unique_ptr<test::VideoFrameWriter> writer = + config.output_dump_options->CreateOutputDumpVideoFrameWriter( + receiver_stream.stream_label, receiver_stream.peer_name, + config.GetResolution()); + if (config.output_dump_use_fixed_framerate) { + writer = std::make_unique<test::FixedFpsVideoFrameWriterAdapter>( + config.fps, clock_, std::move(writer)); + } + sinks.push_back(std::make_unique<VideoWriter>( + writer.get(), config.output_dump_options->sampling_modulo())); + video_writers_.push_back(std::move(writer)); + } + if (config.show_on_screen) { + sinks.push_back(absl::WrapUnique( + test::VideoRenderer::Create((*config.stream_label + "-render").c_str(), + config.width, config.height))); + } + sinks_.insert({receiver_stream, std::move(sinks)}); + return &(sinks_.find(receiver_stream)->second); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h new file mode 100644 index 0000000000..7421c8e4a7 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_ANALYZER_INJECTION_HELPER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_ANALYZER_INJECTION_HELPER_H_ + +#include <stdio.h> + +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/array_view.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/stats_observer_interface.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "rtc_base/synchronization/mutex.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/analyzing_video_sink.h" +#include "test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h" +#include "test/test_video_capturer.h" +#include "test/testsupport/video_frame_writer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Provides factory methods for components, that will be used to inject +// VideoQualityAnalyzerInterface into PeerConnection pipeline. +class VideoQualityAnalyzerInjectionHelper : public StatsObserverInterface { + public: + VideoQualityAnalyzerInjectionHelper( + Clock* clock, + std::unique_ptr<VideoQualityAnalyzerInterface> analyzer, + EncodedImageDataInjector* injector, + EncodedImageDataExtractor* extractor); + ~VideoQualityAnalyzerInjectionHelper() override; + + // Wraps video encoder factory to give video quality analyzer access to frames + // before encoding and encoded images after. + std::unique_ptr<VideoEncoderFactory> WrapVideoEncoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoEncoderFactory> delegate, + double bitrate_multiplier, + QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap stream_to_sfu_config) + const; + // Wraps video decoder factory to give video quality analyzer access to + // received encoded images and frames, that were decoded from them. + std::unique_ptr<VideoDecoderFactory> WrapVideoDecoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoDecoderFactory> delegate) const; + + // Creates VideoFrame preprocessor, that will allow video quality analyzer to + // get access to the captured frames. If provided config also specifies + // `input_dump_file_name`, video will be written into that file. + std::unique_ptr<test::TestVideoCapturer::FramePreprocessor> + CreateFramePreprocessor(absl::string_view peer_name, + const webrtc::webrtc_pc_e2e::VideoConfig& config); + // Creates sink, that will allow video quality analyzer to get access to + // the rendered frames. If corresponding video track has + // `output_dump_file_name` in its VideoConfig, which was used for + // CreateFramePreprocessor(...), then video also will be written + // into that file. + // TODO(titovartem): Remove method with `peer_name` only parameter. + std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> CreateVideoSink( + absl::string_view peer_name); + std::unique_ptr<AnalyzingVideoSink> CreateVideoSink( + absl::string_view peer_name, + const VideoSubscription& subscription, + bool report_infra_metrics); + + void Start(std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count = 1); + + // Registers new call participant to the underlying video quality analyzer. + // The method should be called before the participant is actually added. + void RegisterParticipantInCall(absl::string_view peer_name); + + // Will be called after test removed existing participant in the middle of the + // call. + void UnregisterParticipantInCall(absl::string_view peer_name); + + // Forwards `stats_reports` for Peer Connection `pc_label` to + // `analyzer_`. + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override; + + // Stops VideoQualityAnalyzerInterface to populate final data and metrics. + // Should be invoked after analyzed video tracks are disposed. + void Stop(); + + private: + // Deprecated, to be removed when old API isn't used anymore. + class AnalyzingVideoSink2 final : public rtc::VideoSinkInterface<VideoFrame> { + public: + explicit AnalyzingVideoSink2(absl::string_view peer_name, + VideoQualityAnalyzerInjectionHelper* helper) + : peer_name_(peer_name), helper_(helper) {} + ~AnalyzingVideoSink2() override = default; + + void OnFrame(const VideoFrame& frame) override { + helper_->OnFrame(peer_name_, frame); + } + + private: + const std::string peer_name_; + VideoQualityAnalyzerInjectionHelper* const helper_; + }; + + struct ReceiverStream { + ReceiverStream(absl::string_view peer_name, absl::string_view stream_label) + : peer_name(peer_name), stream_label(stream_label) {} + + std::string peer_name; + std::string stream_label; + + // Define operators required to use ReceiverStream as std::map key. + bool operator==(const ReceiverStream& o) const { + return peer_name == o.peer_name && stream_label == o.stream_label; + } + bool operator<(const ReceiverStream& o) const { + return (peer_name == o.peer_name) ? stream_label < o.stream_label + : peer_name < o.peer_name; + } + }; + + // Creates a deep copy of the frame and passes it to the video analyzer, while + // passing real frame to the sinks + void OnFrame(absl::string_view peer_name, const VideoFrame& frame); + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>* + PopulateSinks(const ReceiverStream& receiver_stream); + + Clock* const clock_; + std::unique_ptr<VideoQualityAnalyzerInterface> analyzer_; + EncodedImageDataInjector* injector_; + EncodedImageDataExtractor* extractor_; + + std::vector<std::unique_ptr<test::VideoFrameWriter>> video_writers_; + + AnalyzingVideoSinksHelper sinks_helper_; + Mutex mutex_; + int peers_count_ RTC_GUARDED_BY(mutex_); + // Map from stream label to the video config. + std::map<std::string, webrtc::webrtc_pc_e2e::VideoConfig> known_video_configs_ + RTC_GUARDED_BY(mutex_); + std::map<ReceiverStream, + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>> + sinks_ RTC_GUARDED_BY(mutex_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_ANALYZER_INJECTION_HELPER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc new file mode 100644 index 0000000000..8049af308e --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h" + +#include <map> +#include <string> + +#include "api/stats/rtc_stats.h" +#include "api/stats/rtcstats_objects.h" +#include "api/test/metrics/metric.h" +#include "api/units/data_rate.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "rtc_base/checks.h" +#include "test/pc/e2e/metric_metadata_keys.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Unit; +using ::webrtc::webrtc_pc_e2e::MetricMetadataKey; + +SamplesStatsCounter BytesPerSecondToKbps(const SamplesStatsCounter& counter) { + return counter * 0.008; +} + +} // namespace + +VideoQualityMetricsReporter::VideoQualityMetricsReporter( + Clock* const clock, + test::MetricsLogger* const metrics_logger) + : clock_(clock), metrics_logger_(metrics_logger) { + RTC_CHECK(metrics_logger_); +} + +void VideoQualityMetricsReporter::Start( + absl::string_view test_case_name, + const TrackIdStreamInfoMap* /*reporter_helper*/) { + test_case_name_ = std::string(test_case_name); + start_time_ = Now(); +} + +void VideoQualityMetricsReporter::OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) { + RTC_CHECK(start_time_) + << "Please invoke Start(...) method before calling OnStatsReports(...)"; + + auto transport_stats = report->GetStatsOfType<RTCTransportStats>(); + if (transport_stats.size() == 0u || + !transport_stats[0]->selected_candidate_pair_id.is_defined()) { + return; + } + RTC_DCHECK_EQ(transport_stats.size(), 1); + std::string selected_ice_id = + transport_stats[0]->selected_candidate_pair_id.ValueToString(); + // Use the selected ICE candidate pair ID to get the appropriate ICE stats. + const RTCIceCandidatePairStats ice_candidate_pair_stats = + report->Get(selected_ice_id)->cast_to<const RTCIceCandidatePairStats>(); + + auto outbound_rtp_stats = report->GetStatsOfType<RTCOutboundRTPStreamStats>(); + StatsSample sample; + for (auto& s : outbound_rtp_stats) { + if (!s->kind.is_defined()) { + continue; + } + if (!(*s->kind == RTCMediaStreamTrackKind::kVideo)) { + continue; + } + if (s->timestamp() > sample.sample_time) { + sample.sample_time = s->timestamp(); + } + sample.retransmitted_bytes_sent += + DataSize::Bytes(s->retransmitted_bytes_sent.ValueOrDefault(0ul)); + sample.bytes_sent += DataSize::Bytes(s->bytes_sent.ValueOrDefault(0ul)); + sample.header_bytes_sent += + DataSize::Bytes(s->header_bytes_sent.ValueOrDefault(0ul)); + } + + MutexLock lock(&video_bwe_stats_lock_); + VideoBweStats& video_bwe_stats = video_bwe_stats_[std::string(pc_label)]; + if (ice_candidate_pair_stats.available_outgoing_bitrate.is_defined()) { + video_bwe_stats.available_send_bandwidth.AddSample( + DataRate::BitsPerSec( + *ice_candidate_pair_stats.available_outgoing_bitrate) + .bytes_per_sec()); + } + + StatsSample prev_sample = last_stats_sample_[std::string(pc_label)]; + if (prev_sample.sample_time.IsZero()) { + prev_sample.sample_time = start_time_.value(); + } + last_stats_sample_[std::string(pc_label)] = sample; + + TimeDelta time_between_samples = sample.sample_time - prev_sample.sample_time; + if (time_between_samples.IsZero()) { + return; + } + + DataRate retransmission_bitrate = + (sample.retransmitted_bytes_sent - prev_sample.retransmitted_bytes_sent) / + time_between_samples; + video_bwe_stats.retransmission_bitrate.AddSample( + retransmission_bitrate.bytes_per_sec()); + DataRate transmission_bitrate = + (sample.bytes_sent + sample.header_bytes_sent - prev_sample.bytes_sent - + prev_sample.header_bytes_sent) / + time_between_samples; + video_bwe_stats.transmission_bitrate.AddSample( + transmission_bitrate.bytes_per_sec()); +} + +void VideoQualityMetricsReporter::StopAndReportResults() { + MutexLock video_bwemutex_(&video_bwe_stats_lock_); + for (const auto& item : video_bwe_stats_) { + ReportVideoBweResults(item.first, item.second); + } +} + +std::string VideoQualityMetricsReporter::GetTestCaseName( + const std::string& peer_name) const { + return test_case_name_ + "/" + peer_name; +} + +void VideoQualityMetricsReporter::ReportVideoBweResults( + const std::string& peer_name, + const VideoBweStats& video_bwe_stats) { + std::string test_case_name = GetTestCaseName(peer_name); + // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey. + std::map<std::string, std::string> metric_metadata{ + {MetricMetadataKey::kPeerMetadataKey, peer_name}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_case_name_}}; + + metrics_logger_->LogMetric( + "available_send_bandwidth", test_case_name, + BytesPerSecondToKbps(video_bwe_stats.available_send_bandwidth), + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter, + metric_metadata); + metrics_logger_->LogMetric( + "transmission_bitrate", test_case_name, + BytesPerSecondToKbps(video_bwe_stats.transmission_bitrate), + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter, + metric_metadata); + metrics_logger_->LogMetric( + "retransmission_bitrate", test_case_name, + BytesPerSecondToKbps(video_bwe_stats.retransmission_bitrate), + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter, + metric_metadata); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h new file mode 100644 index 0000000000..d3d976343b --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_METRICS_REPORTER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_METRICS_REPORTER_H_ + +#include <map> +#include <string> + +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metrics_logger.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/test/track_id_stream_info_map.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +struct VideoBweStats { + SamplesStatsCounter available_send_bandwidth; + SamplesStatsCounter transmission_bitrate; + SamplesStatsCounter retransmission_bitrate; +}; + +class VideoQualityMetricsReporter + : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { + public: + VideoQualityMetricsReporter(Clock* const clock, + test::MetricsLogger* const metrics_logger); + ~VideoQualityMetricsReporter() override = default; + + void Start(absl::string_view test_case_name, + const TrackIdStreamInfoMap* reporter_helper) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override; + void StopAndReportResults() override; + + private: + struct StatsSample { + DataSize bytes_sent = DataSize::Zero(); + DataSize header_bytes_sent = DataSize::Zero(); + DataSize retransmitted_bytes_sent = DataSize::Zero(); + + Timestamp sample_time = Timestamp::Zero(); + }; + + std::string GetTestCaseName(const std::string& peer_name) const; + void ReportVideoBweResults(const std::string& peer_name, + const VideoBweStats& video_bwe_stats); + Timestamp Now() const { return clock_->CurrentTime(); } + + Clock* const clock_; + test::MetricsLogger* const metrics_logger_; + + std::string test_case_name_; + absl::optional<Timestamp> start_time_; + + Mutex video_bwe_stats_lock_; + // Map between a peer connection label (provided by the framework) and + // its video BWE stats. + std::map<std::string, VideoBweStats> video_bwe_stats_ + RTC_GUARDED_BY(video_bwe_stats_lock_); + std::map<std::string, StatsSample> last_stats_sample_ + RTC_GUARDED_BY(video_bwe_stats_lock_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_METRICS_REPORTER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer_helper.cc b/third_party/libwebrtc/test/pc/e2e/analyzer_helper.cc new file mode 100644 index 0000000000..76cd9a7c78 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer_helper.cc @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer_helper.h" + +#include <string> +#include <utility> + +namespace webrtc { +namespace webrtc_pc_e2e { + +AnalyzerHelper::AnalyzerHelper() { + signaling_sequence_checker_.Detach(); +} + +void AnalyzerHelper::AddTrackToStreamMapping( + absl::string_view track_id, + absl::string_view receiver_peer, + absl::string_view stream_label, + absl::optional<std::string> sync_group) { + RTC_DCHECK_RUN_ON(&signaling_sequence_checker_); + track_to_stream_map_.insert( + {std::string(track_id), + StreamInfo{.receiver_peer = std::string(receiver_peer), + .stream_label = std::string(stream_label), + .sync_group = sync_group.has_value() + ? *sync_group + : std::string(stream_label)}}); +} + +void AnalyzerHelper::AddTrackToStreamMapping(std::string track_id, + std::string stream_label) { + RTC_DCHECK_RUN_ON(&signaling_sequence_checker_); + track_to_stream_map_.insert( + {std::move(track_id), StreamInfo{stream_label, stream_label}}); +} + +void AnalyzerHelper::AddTrackToStreamMapping(std::string track_id, + std::string stream_label, + std::string sync_group) { + RTC_DCHECK_RUN_ON(&signaling_sequence_checker_); + track_to_stream_map_.insert( + {std::move(track_id), + StreamInfo{std::move(stream_label), std::move(sync_group)}}); +} + +AnalyzerHelper::StreamInfo AnalyzerHelper::GetStreamInfoFromTrackId( + absl::string_view track_id) const { + RTC_DCHECK_RUN_ON(&signaling_sequence_checker_); + auto track_to_stream_pair = track_to_stream_map_.find(std::string(track_id)); + RTC_CHECK(track_to_stream_pair != track_to_stream_map_.end()); + return track_to_stream_pair->second; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer_helper.h b/third_party/libwebrtc/test/pc/e2e/analyzer_helper.h new file mode 100644 index 0000000000..d0b47c4fb9 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer_helper.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_HELPER_H_ +#define TEST_PC_E2E_ANALYZER_HELPER_H_ + +#include <map> +#include <string> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/sequence_checker.h" +#include "api/test/track_id_stream_info_map.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// This class is a utility that provides bookkeeping capabilities that +// are useful to associate stats reports track_ids to the remote stream info. +// The framework will populate an instance of this class and it will pass +// it to the Start method of Media Quality Analyzers. +// An instance of AnalyzerHelper must only be accessed from a single +// thread and since stats collection happens on the signaling thread, +// AddTrackToStreamMapping, GetStreamLabelFromTrackId and +// GetSyncGroupLabelFromTrackId must be invoked from the signaling thread. Get +// methods should be invoked only after all data is added. Mixing Get methods +// with adding new data may lead to undefined behavior. +class AnalyzerHelper : public TrackIdStreamInfoMap { + public: + AnalyzerHelper(); + + void AddTrackToStreamMapping(absl::string_view track_id, + absl::string_view receiver_peer, + absl::string_view stream_label, + absl::optional<std::string> sync_group); + void AddTrackToStreamMapping(std::string track_id, std::string stream_label); + void AddTrackToStreamMapping(std::string track_id, + std::string stream_label, + std::string sync_group); + + StreamInfo GetStreamInfoFromTrackId( + absl::string_view track_id) const override; + + private: + SequenceChecker signaling_sequence_checker_; + std::map<std::string, StreamInfo> track_to_stream_map_ + RTC_GUARDED_BY(signaling_sequence_checker_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_HELPER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.cc b/third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.cc new file mode 100644 index 0000000000..0d4fe7478d --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.cc @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/cross_media_metrics_reporter.h" + +#include <utility> +#include <vector> + +#include "api/stats/rtc_stats.h" +#include "api/stats/rtcstats_objects.h" +#include "api/test/metrics/metric.h" +#include "api/units/timestamp.h" +#include "rtc_base/checks.h" +#include "rtc_base/event.h" +#include "system_wrappers/include/field_trial.h" +#include "test/pc/e2e/metric_metadata_keys.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Unit; + +CrossMediaMetricsReporter::CrossMediaMetricsReporter( + test::MetricsLogger* metrics_logger) + : metrics_logger_(metrics_logger) { + RTC_CHECK(metrics_logger_); +} + +void CrossMediaMetricsReporter::Start( + absl::string_view test_case_name, + const TrackIdStreamInfoMap* reporter_helper) { + test_case_name_ = std::string(test_case_name); + reporter_helper_ = reporter_helper; +} + +void CrossMediaMetricsReporter::OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) { + auto inbound_stats = report->GetStatsOfType<RTCInboundRTPStreamStats>(); + std::map<std::string, std::vector<const RTCInboundRTPStreamStats*>> + sync_group_stats; + for (const auto& stat : inbound_stats) { + if (stat->estimated_playout_timestamp.ValueOrDefault(0.) > 0 && + stat->track_identifier.is_defined()) { + sync_group_stats[reporter_helper_ + ->GetStreamInfoFromTrackId(*stat->track_identifier) + .sync_group] + .push_back(stat); + } + } + + MutexLock lock(&mutex_); + for (const auto& pair : sync_group_stats) { + // If there is less than two streams, it is not a sync group. + if (pair.second.size() < 2) { + continue; + } + auto sync_group = std::string(pair.first); + const RTCInboundRTPStreamStats* audio_stat = pair.second[0]; + const RTCInboundRTPStreamStats* video_stat = pair.second[1]; + + RTC_CHECK(pair.second.size() == 2 && audio_stat->kind.is_defined() && + video_stat->kind.is_defined() && + *audio_stat->kind != *video_stat->kind) + << "Sync group should consist of one audio and one video stream."; + + if (*audio_stat->kind == RTCMediaStreamTrackKind::kVideo) { + std::swap(audio_stat, video_stat); + } + // Stream labels of a sync group are same for all polls, so we need it add + // it only once. + if (stats_info_.find(sync_group) == stats_info_.end()) { + RTC_CHECK(audio_stat->track_identifier.is_defined()); + RTC_CHECK(video_stat->track_identifier.is_defined()); + stats_info_[sync_group].audio_stream_info = + reporter_helper_->GetStreamInfoFromTrackId( + *audio_stat->track_identifier); + stats_info_[sync_group].video_stream_info = + reporter_helper_->GetStreamInfoFromTrackId( + *video_stat->track_identifier); + } + + double audio_video_playout_diff = *audio_stat->estimated_playout_timestamp - + *video_stat->estimated_playout_timestamp; + if (audio_video_playout_diff > 0) { + stats_info_[sync_group].audio_ahead_ms.AddSample( + audio_video_playout_diff); + stats_info_[sync_group].video_ahead_ms.AddSample(0); + } else { + stats_info_[sync_group].audio_ahead_ms.AddSample(0); + stats_info_[sync_group].video_ahead_ms.AddSample( + std::abs(audio_video_playout_diff)); + } + } +} + +void CrossMediaMetricsReporter::StopAndReportResults() { + MutexLock lock(&mutex_); + for (const auto& pair : stats_info_) { + const std::string& sync_group = pair.first; + // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey. + std::map<std::string, std::string> audio_metric_metadata{ + {MetricMetadataKey::kPeerSyncGroupMetadataKey, sync_group}, + {MetricMetadataKey::kAudioStreamMetadataKey, + pair.second.audio_stream_info.stream_label}, + {MetricMetadataKey::kPeerMetadataKey, + pair.second.audio_stream_info.receiver_peer}, + {MetricMetadataKey::kReceiverMetadataKey, + pair.second.audio_stream_info.receiver_peer}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_case_name_}}; + metrics_logger_->LogMetric( + "audio_ahead_ms", + GetTestCaseName(pair.second.audio_stream_info.stream_label, sync_group), + pair.second.audio_ahead_ms, Unit::kMilliseconds, + webrtc::test::ImprovementDirection::kSmallerIsBetter, + std::move(audio_metric_metadata)); + + // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey. + std::map<std::string, std::string> video_metric_metadata{ + {MetricMetadataKey::kPeerSyncGroupMetadataKey, sync_group}, + {MetricMetadataKey::kAudioStreamMetadataKey, + pair.second.video_stream_info.stream_label}, + {MetricMetadataKey::kPeerMetadataKey, + pair.second.video_stream_info.receiver_peer}, + {MetricMetadataKey::kReceiverMetadataKey, + pair.second.video_stream_info.receiver_peer}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_case_name_}}; + metrics_logger_->LogMetric( + "video_ahead_ms", + GetTestCaseName(pair.second.video_stream_info.stream_label, sync_group), + pair.second.video_ahead_ms, Unit::kMilliseconds, + webrtc::test::ImprovementDirection::kSmallerIsBetter, + std::move(video_metric_metadata)); + } +} + +std::string CrossMediaMetricsReporter::GetTestCaseName( + const std::string& stream_label, + const std::string& sync_group) const { + return test_case_name_ + "/" + sync_group + "_" + stream_label; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.h b/third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.h new file mode 100644 index 0000000000..2d51ebb20f --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_CROSS_MEDIA_METRICS_REPORTER_H_ +#define TEST_PC_E2E_CROSS_MEDIA_METRICS_REPORTER_H_ + +#include <map> +#include <string> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metrics_logger.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/test/track_id_stream_info_map.h" +#include "api/units/timestamp.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class CrossMediaMetricsReporter + : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { + public: + explicit CrossMediaMetricsReporter(test::MetricsLogger* metrics_logger); + ~CrossMediaMetricsReporter() override = default; + + void Start(absl::string_view test_case_name, + const TrackIdStreamInfoMap* reporter_helper) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override; + void StopAndReportResults() override; + + private: + struct StatsInfo { + SamplesStatsCounter audio_ahead_ms; + SamplesStatsCounter video_ahead_ms; + + TrackIdStreamInfoMap::StreamInfo audio_stream_info; + TrackIdStreamInfoMap::StreamInfo video_stream_info; + std::string audio_stream_label; + std::string video_stream_label; + }; + + std::string GetTestCaseName(const std::string& stream_label, + const std::string& sync_group) const; + + test::MetricsLogger* const metrics_logger_; + + std::string test_case_name_; + const TrackIdStreamInfoMap* reporter_helper_; + + Mutex mutex_; + std::map<std::string, StatsInfo> stats_info_ RTC_GUARDED_BY(mutex_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_CROSS_MEDIA_METRICS_REPORTER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.cc b/third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.cc new file mode 100644 index 0000000000..8fdabeb16f --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.cc @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/echo/echo_emulation.h" + +#include <limits> +#include <utility> + +#include "api/test/pclf/media_configuration.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +constexpr int kSingleBufferDurationMs = 10; + +} // namespace + +EchoEmulatingCapturer::EchoEmulatingCapturer( + std::unique_ptr<TestAudioDeviceModule::Capturer> capturer, + EchoEmulationConfig config) + : delegate_(std::move(capturer)), + config_(config), + renderer_queue_(2 * config_.echo_delay.ms() / kSingleBufferDurationMs), + queue_input_(TestAudioDeviceModule::SamplesPerFrame( + delegate_->SamplingFrequency()) * + delegate_->NumChannels()), + queue_output_(TestAudioDeviceModule::SamplesPerFrame( + delegate_->SamplingFrequency()) * + delegate_->NumChannels()) { + renderer_thread_.Detach(); + capturer_thread_.Detach(); +} + +void EchoEmulatingCapturer::OnAudioRendered( + rtc::ArrayView<const int16_t> data) { + RTC_DCHECK_RUN_ON(&renderer_thread_); + if (!recording_started_) { + // Because rendering can start before capturing in the beginning we can have + // a set of empty audio data frames. So we will skip them and will start + // fill the queue only after 1st non-empty audio data frame will arrive. + bool is_empty = true; + for (auto d : data) { + if (d != 0) { + is_empty = false; + break; + } + } + if (is_empty) { + return; + } + recording_started_ = true; + } + queue_input_.assign(data.begin(), data.end()); + if (!renderer_queue_.Insert(&queue_input_)) { + RTC_LOG(LS_WARNING) << "Echo queue is full"; + } +} + +bool EchoEmulatingCapturer::Capture(rtc::BufferT<int16_t>* buffer) { + RTC_DCHECK_RUN_ON(&capturer_thread_); + bool result = delegate_->Capture(buffer); + // Now we have to reduce input signal to avoid saturation when mixing in the + // fake echo. + for (size_t i = 0; i < buffer->size(); ++i) { + (*buffer)[i] /= 2; + } + + // When we accumulated enough delay in the echo buffer we will pop from + // that buffer on each ::Capture(...) call. If the buffer become empty it + // will mean some bug, so we will crash during removing item from the queue. + if (!delay_accumulated_) { + delay_accumulated_ = + renderer_queue_.SizeAtLeast() >= + static_cast<size_t>(config_.echo_delay.ms() / kSingleBufferDurationMs); + } + + if (delay_accumulated_) { + RTC_CHECK(renderer_queue_.Remove(&queue_output_)); + for (size_t i = 0; i < buffer->size() && i < queue_output_.size(); ++i) { + int32_t res = (*buffer)[i] + queue_output_[i]; + if (res < std::numeric_limits<int16_t>::min()) { + res = std::numeric_limits<int16_t>::min(); + } + if (res > std::numeric_limits<int16_t>::max()) { + res = std::numeric_limits<int16_t>::max(); + } + (*buffer)[i] = static_cast<int16_t>(res); + } + } + + return result; +} + +EchoEmulatingRenderer::EchoEmulatingRenderer( + std::unique_ptr<TestAudioDeviceModule::Renderer> renderer, + EchoEmulatingCapturer* echo_emulating_capturer) + : delegate_(std::move(renderer)), + echo_emulating_capturer_(echo_emulating_capturer) { + RTC_DCHECK(echo_emulating_capturer_); +} + +bool EchoEmulatingRenderer::Render(rtc::ArrayView<const int16_t> data) { + if (data.size() > 0) { + echo_emulating_capturer_->OnAudioRendered(data); + } + return delegate_->Render(data); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.h b/third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.h new file mode 100644 index 0000000000..359a481e46 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ECHO_ECHO_EMULATION_H_ +#define TEST_PC_E2E_ECHO_ECHO_EMULATION_H_ + +#include <atomic> +#include <deque> +#include <memory> +#include <vector> + +#include "api/test/pclf/media_configuration.h" +#include "modules/audio_device/include/test_audio_device.h" +#include "rtc_base/swap_queue.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Reduces audio input strength from provided capturer twice and adds input +// provided into EchoEmulatingCapturer::OnAudioRendered(...). +class EchoEmulatingCapturer : public TestAudioDeviceModule::Capturer { + public: + EchoEmulatingCapturer( + std::unique_ptr<TestAudioDeviceModule::Capturer> capturer, + EchoEmulationConfig config); + + void OnAudioRendered(rtc::ArrayView<const int16_t> data); + + int SamplingFrequency() const override { + return delegate_->SamplingFrequency(); + } + int NumChannels() const override { return delegate_->NumChannels(); } + bool Capture(rtc::BufferT<int16_t>* buffer) override; + + private: + std::unique_ptr<TestAudioDeviceModule::Capturer> delegate_; + const EchoEmulationConfig config_; + + SwapQueue<std::vector<int16_t>> renderer_queue_; + + SequenceChecker renderer_thread_; + std::vector<int16_t> queue_input_ RTC_GUARDED_BY(renderer_thread_); + bool recording_started_ RTC_GUARDED_BY(renderer_thread_) = false; + + SequenceChecker capturer_thread_; + std::vector<int16_t> queue_output_ RTC_GUARDED_BY(capturer_thread_); + bool delay_accumulated_ RTC_GUARDED_BY(capturer_thread_) = false; +}; + +// Renders output into provided renderer and also copy output into provided +// EchoEmulationCapturer. +class EchoEmulatingRenderer : public TestAudioDeviceModule::Renderer { + public: + EchoEmulatingRenderer( + std::unique_ptr<TestAudioDeviceModule::Renderer> renderer, + EchoEmulatingCapturer* echo_emulating_capturer); + + int SamplingFrequency() const override { + return delegate_->SamplingFrequency(); + } + int NumChannels() const override { return delegate_->NumChannels(); } + bool Render(rtc::ArrayView<const int16_t> data) override; + + private: + std::unique_ptr<TestAudioDeviceModule::Renderer> delegate_; + EchoEmulatingCapturer* echo_emulating_capturer_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ECHO_ECHO_EMULATION_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/architecture.md b/third_party/libwebrtc/test/pc/e2e/g3doc/architecture.md new file mode 100644 index 0000000000..1b68c6db2c --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/g3doc/architecture.md @@ -0,0 +1,209 @@ +<!-- go/cmark --> +<!--* freshness: {owner: 'titovartem' reviewed: '2021-04-12'} *--> + +# PeerConnection level framework fixture architecture + +## Overview + +The main implementation of +[`webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture`][1] is +[`webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTest`][2]. Internally it owns +the next main pieces: + +* [`MediaHelper`][3] - responsible for adding audio and video tracks to the + peers. +* [`VideoQualityAnalyzerInjectionHelper`][4] and + [`SingleProcessEncodedImageDataInjector`][5] - used to inject video quality + analysis and properly match captured and rendered video frames. You can read + more about it in + [DefaultVideoQualityAnalyzer](default_video_quality_analyzer.md) section. +* [`AudioQualityAnalyzerInterface`][6] - used to measure audio quality metrics +* [`TestActivitiesExecutor`][7] - used to support [`ExecuteAt(...)`][8] and + [`ExecuteEvery(...)`][9] API of `PeerConnectionE2EQualityTestFixture` to run + any arbitrary action during test execution timely synchronized with a test + call. +* A vector of [`QualityMetricsReporter`][10] added by the + `PeerConnectionE2EQualityTestFixture` user. +* Two peers: Alice and Bob represented by instances of [`TestPeer`][11] + object. + +Also it keeps a reference to [`webrtc::TimeController`][12], which is used to +create all required threads, task queues, task queue factories and time related +objects. + +## TestPeer + +Call participants are represented by instances of `TestPeer` object. +[`TestPeerFactory`][13] is used to create them. `TestPeer` owns all instances +related to the `webrtc::PeerConnection`, including required listeners and +callbacks. Also it provides an API to do offer/answer exchange and ICE candidate +exchange. For this purposes internally it uses an instance of +[`webrtc::PeerConnectionWrapper`][14]. + +The `TestPeer` also owns the `PeerConnection` worker thread. The signaling +thread for all `PeerConnection`'s is owned by +`PeerConnectionE2EQualityTestFixture` and shared between all participants in the +call. The network thread is owned by the network layer (it maybe either emulated +network provided by [Network Emulation Framework][24] or network thread and +`rtc::NetworkManager` provided by user) and provided when peer is added to the +fixture via [`AddPeer(...)`][15] API. + +## GetStats API based metrics reporters + +`PeerConnectionE2EQualityTestFixture` gives the user ability to provide +different `QualityMetricsReporter`s which will listen for `PeerConnection` +[`GetStats`][16] API. Then such reporters will be able to report various metrics +that user wants to measure. + +`PeerConnectionE2EQualityTestFixture` itself also uses this mechanism to +measure: + +* Audio quality metrics +* Audio/Video sync metrics (with help of [`CrossMediaMetricsReporter`][17]) + +Also framework provides a [`StatsBasedNetworkQualityMetricsReporter`][18] to +measure network related WebRTC metrics and print debug raw emulated network +statistic. This reporter should be added by user via +[`AddQualityMetricsReporter(...)`][19] API if requried. + +Internally stats gathering is done by [`StatsPoller`][20]. Stats are requested +once per second for each `PeerConnection` and then resulted object is provided +into each stats listener. + +## Offer/Answer exchange + +`PeerConnectionE2EQualityTest` provides ability to test Simulcast and SVC for +video. These features aren't supported by P2P call and in general requires a +Selective Forwarding Unit (SFU). So special logic is applied to mimic SFU +behavior in P2P call. This logic is located inside [`SignalingInterceptor`][21], +[`QualityAnalyzingVideoEncoder`][22] and [`QualityAnalyzingVideoDecoder`][23] +and consist of SDP modification during offer/answer exchange and special +handling of video frames from unrelated Simulcast/SVC streams during decoding. + +### Simulcast + +In case of Simulcast we have a video track, which internally contains multiple +video streams, for example low resolution, medium resolution and high +resolution. WebRTC client doesn't support receiving an offer with multiple +streams in it, because usually SFU will keep only single stream for the client. +To bypass it framework will modify offer by converting a single track with three +video streams into three independent video tracks. Then sender will think that +it send simulcast, but receiver will think that it receives 3 independent +tracks. + +To achieve such behavior some extra tweaks are required: + +* MID RTP header extension from original offer have to be removed +* RID RTP header extension from original offer is replaced with MID RTP header + extension, so the ID that sender uses for RID on receiver will be parsed as + MID. +* Answer have to be modified in the opposite way. + +Described modifications are illustrated on the picture below. + + + +The exchange will look like this: + +1. Alice creates an offer +2. Alice sets offer as local description +3. Do described offer modification +4. Alice sends modified offer to Bob +5. Bob sets modified offer as remote description +6. Bob creates answer +7. Bob sets answer as local description +8. Do reverse modifications on answer +9. Bob sends modified answer to Alice +10. Alice sets modified answer as remote description + +Such mechanism put a constraint that RTX streams are not supported, because they +don't have RID RTP header extension in their packets. + +### SVC + +In case of SVC the framework will update the sender's offer before even setting +it as local description on the sender side. Then no changes to answer will be +required. + +`ssrc` is a 32 bit random value that is generated in RTP to denote a specific +source used to send media in an RTP connection. In original offer video track +section will look like this: + +``` +m=video 9 UDP/TLS/RTP/SAVPF 98 100 99 101 +... +a=ssrc-group:FID <primary ssrc> <retransmission ssrc> +a=ssrc:<primary ssrc> cname:... +.... +a=ssrc:<retransmission ssrc> cname:... +.... +``` + +To enable SVC for such video track framework will add extra `ssrc`s for each SVC +stream that is required like this: + +``` +a=ssrc-group:FID <Low resolution primary ssrc> <Low resolution retransmission ssrc> +a=ssrc:<Low resolution primary ssrc> cname:... +.... +a=ssrc:<Low resolution retransmission ssrc> cname:.... +... +a=ssrc-group:FID <Medium resolution primary ssrc> <Medium resolution retransmission ssrc> +a=ssrc:<Medium resolution primary ssrc> cname:... +.... +a=ssrc:<Medium resolution retransmission ssrc> cname:.... +... +a=ssrc-group:FID <High resolution primary ssrc> <High resolution retransmission ssrc> +a=ssrc:<High resolution primary ssrc> cname:... +.... +a=ssrc:<High resolution retransmission ssrc> cname:.... +... +``` + +The next line will also be added to the video track section of the offer: + +``` +a=ssrc-group:SIM <Low resolution primary ssrc> <Medium resolution primary ssrc> <High resolution primary ssrc> +``` + +It will tell PeerConnection that this track should be configured as SVC. It +utilize WebRTC Plan B offer structure to achieve SVC behavior, also it modifies +offer before setting it as local description which violates WebRTC standard. +Also it adds limitations that on lossy networks only top resolution streams can +be analyzed, because WebRTC won't try to restore low resolution streams in case +of loss, because it still receives higher stream. + +### Handling in encoder/decoder + +In the encoder, the framework for each encoded video frame will propagate +information requried for the fake SFU to know if it belongs to an interesting +simulcast stream/spatial layer of if it should be "discarded". + +On the decoder side frames that should be "discarded" by fake SFU will be auto +decoded into single pixel images and only the interesting simulcast +stream/spatial layer will go into real decoder and then will be analyzed. + +[1]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=55;drc=484acf27231d931dbc99aedce85bc27e06486b96 +[2]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/peer_connection_quality_test.h;l=44;drc=6cc893ad778a0965e2b7a8e614f3c98aa81bee5b +[3]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/media/media_helper.h;l=27;drc=d46db9f1523ae45909b4a6fdc90a140443068bc6 +[4]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h;l=38;drc=79020414fd5c71f9ec1f25445ea5f1c8001e1a49 +[5]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h;l=40;drc=79020414fd5c71f9ec1f25445ea5f1c8001e1a49 +[6]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/audio_quality_analyzer_interface.h;l=23;drc=20f45823e37fd7272aa841831c029c21f29742c2 +[7]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/test_activities_executor.h;l=28;drc=6cc893ad778a0965e2b7a8e614f3c98aa81bee5b +[8]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=439;drc=484acf27231d931dbc99aedce85bc27e06486b96 +[9]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=445;drc=484acf27231d931dbc99aedce85bc27e06486b96 +[10]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=413;drc=9438fb3fff97c803d1ead34c0e4f223db168526f +[11]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/test_activities_executor.h;l=28;drc=6cc893ad778a0965e2b7a8e614f3c98aa81bee5b +[12]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/test_activities_executor.h;l=28;drc=6cc893ad778a0965e2b7a8e614f3c98aa81bee5b +[13]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/test_peer_factory.h;l=46;drc=0ef4a2488a466a24ab97b31fdddde55440d451f9 +[14]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/pc/peer_connection_wrapper.h;l=47;drc=5ab79e62f691875a237ea28ca3975ea1f0ed62ec +[15]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=459;drc=484acf27231d931dbc99aedce85bc27e06486b96 +[16]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/peer_connection_interface.h;l=886;drc=9438fb3fff97c803d1ead34c0e4f223db168526f +[17]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/cross_media_metrics_reporter.h;l=29;drc=9d777620236ec76754cfce19f6e82dd18e52d22c +[18]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/cross_media_metrics_reporter.h;l=29;drc=9d777620236ec76754cfce19f6e82dd18e52d22c +[19]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=450;drc=484acf27231d931dbc99aedce85bc27e06486b96 +[20]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/stats_poller.h;l=52;drc=9b526180c9e9722d3fc7f8689da6ec094fc7fc0a +[21]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/sdp/sdp_changer.h;l=79;drc=ee558dcca89fd8b105114ededf9e74d948da85e8 +[22]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h;l=54;drc=79020414fd5c71f9ec1f25445ea5f1c8001e1a49 +[23]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h;l=50;drc=79020414fd5c71f9ec1f25445ea5f1c8001e1a49 +[24]: /test/network/g3doc/index.md diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/default_video_quality_analyzer.md b/third_party/libwebrtc/test/pc/e2e/g3doc/default_video_quality_analyzer.md new file mode 100644 index 0000000000..67596777f2 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/g3doc/default_video_quality_analyzer.md @@ -0,0 +1,197 @@ +<!-- go/cmark --> +<!--* freshness: {owner: 'titovartem' reviewed: '2021-02-21'} *--> + +# DefaultVideoQualityAnalyzer + +## Audience + +This document is for users of +[`webrtc::webrtc_pc_e2e::DefaultVideoQualityAnalyzer`][1]. + +## Overview + +`DefaultVideoQualityAnalyzer` implements +[`webrtc::VideoQualityAnalyzerInterface`][2] and is a main +implementation of video quality analyzer for WebRTC. To operate correctly it +requires to receive video frame on each step: + +1. On frame captured - analyzer will generate a unique ID for the frame, that + caller should attach to the it. +2. Immediately before frame enter the encoder. +3. Immediately after the frame was encoded. +4. After the frame was received and immediately before it entered the decoder. +5. Immediately after the frame was decoded. +6. When the frame was rendered. + + + +The analyzer updates its internal metrics per frame when it was rendered and +reports all of them after it was stopped through +[WebRTC perf results reporting system][10]. + +To properly inject `DefaultVideoQualityAnalyzer` into pipeline the following helpers can be used: + +### VideoQualityAnalyzerInjectionHelper + +[`webrtc::webrtc_pc_e2e::VideoQualityAnalyzerInjectionHelper`][3] provides +factory methods for components, that will be used to inject +`VideoQualityAnalyzerInterface` into the `PeerConnection` pipeline: + +* Wrappers for [`webrtc::VideoEncoderFactory`][4] and + [`webrtc::VideoDecodeFactory`][5] which will properly pass + [`webrtc::VideoFrame`][6] and [`webrtc::EncodedImage`][7] into analyzer + before and after real video encode and decoder. +* [`webrtc::test::TestVideoCapturer::FramePreprocessor`][8] which is used to + pass generated frames into analyzer on capturing and then set the returned + frame ID. It also configures dumping of captured frames if requried. +* [`rtc::VideoSinkInterface<VideoFrame>`][9] which is used to pass frames to + the analyzer before they will be rendered to compute per frame metrics. It + also configures dumping of rendered video if requried. + +Besides factories `VideoQualityAnalyzerInjectionHelper` has method to +orchestrate `VideoQualityAnalyzerInterface` workflow: + +* `Start` - to start video analyzer, so it will be able to receive and analyze + video frames. +* `RegisterParticipantInCall` - to add new participants after analyzer was + started. +* `Stop` - to stop analyzer, compute all metrics for frames that were recevied + before and report them. + +Also `VideoQualityAnalyzerInjectionHelper` implements +[`webrtc::webrtc_pc_e2e::StatsObserverInterface`][11] to propagate WebRTC stats +to `VideoQualityAnalyzerInterface`. + +### EncodedImageDataInjector and EncodedImageDataExtractor + +[`webrtc::webrtc_pc_e2e::EncodedImageDataInjector`][14] and +[`webrtc::webrtc_pc_e2e::EncodedImageDataInjector`][15] are used to inject and +extract data into `webrtc::EncodedImage` to propagate frame ID and other +required information through the network. + +By default [`webrtc::webrtc_pc_e2e::SingleProcessEncodedImageDataInjector`][16] +is used. It assumes `webrtc::EncodedImage` payload as black box which is +remaining unchanged from encoder to decoder and stores the information required +for its work in the last 3 bytes of the payload, replacing the original data +during injection and restoring it back during extraction. Also +`SingleProcessEncodedImageDataInjector` requires that sender and receiver were +inside single process. + + + +## Exported metrics + +Exported metrics are reported to WebRTC perf results reporting system. + +### General + +* *`cpu_usage`* - CPU usage excluding video analyzer + +### Video + +* *`psnr`* - peak signal-to-noise ratio: + [wikipedia](https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio) +* *`ssim`* - structural similarity: + [wikipedia](https://en.wikipedia.org/wiki/Structural_similarity). +* *`min_psnr`* - minimum value of psnr across all frames of video stream. +* *`encode_time`* - time to encode a single frame. +* *`decode_time`* - time to decode a single frame. +* *`transport_time`* - time from frame encoded to frame received for decoding. +* *`receive_to_render_time`* - time from frame received for decoding to frame + rendered. +* *`total_delay_incl_transport`* - time from frame was captured on device to + time when frame was displayed on device. +* *`encode_frame_rate`* - frame rate after encoder. +* *`harmonic_framerate`* - video duration divided on squared sum of interframe + delays. Reflects render frame rate penalized by freezes. +* *`time_between_rendered_frames`* - time between frames out to renderer. +* *`dropped_frames`* - amount of frames that were sent, but weren't rendered + and are known not to be “on the way” from sender to receiver. + +Freeze is a pause when no new frames from decoder arrived for 150ms + avg time +between frames or 3 * avg time between frames. + +* *`time_between_freezes`* - mean time from previous freeze end to new freeze + start. +* *`freeze_time_ms`* - total freeze time in ms. +* *`max_skipped`* - frames skipped between two nearest rendered. +* *`pixels_per_frame`* - amount of pixels on frame (width * height). +* *`target_encode_bitrate`* - target encode bitrate provided by BWE to + encoder. +* *`actual_encode_bitrate -`* - actual encode bitrate produced by encoder. +* *`available_send_bandwidth -`* - available send bandwidth estimated by BWE. +* *`transmission_bitrate`* - bitrate of media in the emulated network, not + counting retransmissions FEC, and RTCP messages +* *`retransmission_bitrate`* - bitrate of retransmission streams only. + +### Framework stability + +* *`frames_in_flight`* - amount of frames that were captured but wasn't seen + on receiver. + +## Debug metrics + +Debug metrics are not reported to WebRTC perf results reporting system, but are +available through `DefaultVideoQualityAnalyzer` API. + +### [FrameCounters][12] + +Frame counters consist of next counters: + +* *`captured`* - count of frames, that were passed into WebRTC pipeline by + video stream source +* *`pre_encoded`* - count of frames that reached video encoder. +* *`encoded`* - count of encoded images that were produced by encoder for all + requested spatial layers and simulcast streams. +* *`received`* - count of encoded images received in decoder for all requested + spatial layers and simulcast streams. +* *`decoded`* - count of frames that were produced by decoder. +* *`rendered`* - count of frames that went out from WebRTC pipeline to video + sink. +* *`dropped`* - count of frames that were dropped in any point between + capturing and rendering. + +`DefaultVideoQualityAnalyzer` exports these frame counters: + +* *`GlobalCounters`* - frame counters for frames met on each stage of analysis + for all media streams. +* *`PerStreamCounters`* - frame counters for frames met on each stage of + analysis separated per individual video track (single media section in the + SDP offer). + +### [AnalyzerStats][13] + +Contains metrics about internal state of video analyzer during its work + +* *`comparisons_queue_size`* - size of analyzer internal queue used to perform + captured and rendered frames comparisons measured when new element is added + to the queue. +* *`comparisons_done`* - number of performed comparisons of 2 video frames + from captured and rendered streams. +* *`cpu_overloaded_comparisons_done`* - number of cpu overloaded comparisons. + Comparison is cpu overloaded if it is queued when there are too many not + processed comparisons in the queue. Overloaded comparison doesn't include + metrics like SSIM and PSNR that require heavy computations. +* *`memory_overloaded_comparisons_done`* - number of memory overloaded + comparisons. Comparison is memory overloaded if it is queued when its + captured frame was already removed due to high memory usage for that video + stream. +* *`frames_in_flight_left_count`* - count of frames in flight in analyzer + measured when new comparison is added and after analyzer was stopped. + +[1]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h;l=188;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[2]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/test/video_quality_analyzer_interface.h;l=56;drc=d7808f1c464a07c8f1e2f97ec7ee92fda998d590 +[3]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h;l=39;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[4]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/video_codecs/video_encoder_factory.h;l=27;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[5]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/video_codecs/video_decoder_factory.h;l=27;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[6]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/video/video_frame.h;l=30;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[7]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/video/encoded_image.h;l=71;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[8]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/test_video_capturer.h;l=28;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[9]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/video/video_sink_interface.h;l=19;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[10]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/testsupport/perf_test.h;drc=0710b401b1e5b500b8e84946fb657656ba1b58b7 +[11]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/test/stats_observer_interface.h;l=21;drc=9b526180c9e9722d3fc7f8689da6ec094fc7fc0a +[12]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h;l=57;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[13]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h;l=113;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[14]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h;l=23;drc=c57089a97a3df454f4356d882cc8df173e8b3ead +[15]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h;l=46;drc=c57089a97a3df454f4356d882cc8df173e8b3ead +[16]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h;l=40;drc=c57089a97a3df454f4356d882cc8df173e8b3ead diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/in_test_psnr_plot.png b/third_party/libwebrtc/test/pc/e2e/g3doc/in_test_psnr_plot.png Binary files differnew file mode 100644 index 0000000000..3f36725727 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/g3doc/in_test_psnr_plot.png diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/index.md b/third_party/libwebrtc/test/pc/e2e/g3doc/index.md new file mode 100644 index 0000000000..678262bb2b --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/g3doc/index.md @@ -0,0 +1,224 @@ +<!-- go/cmark --> +<!--* freshness: {owner: 'titovartem' reviewed: '2021-04-12'} *--> + +# PeerConnection Level Framework + +## API + +* [Fixture][1] +* [Fixture factory function][2] + +## Documentation + +The PeerConnection level framework is designed for end-to-end media quality +testing through the PeerConnection level public API. The framework uses the +*Unified plan* API to generate offers/answers during the signaling phase. The +framework also wraps the video encoder/decoder and inject it into +*`webrtc::PeerConnection`* to measure video quality, performing 1:1 frames +matching between captured and rendered frames without any extra requirements to +input video. For audio quality evaluation the standard `GetStats()` API from +PeerConnection is used. + +The framework API is located in the namespace *`webrtc::webrtc_pc_e2e`*. + +### Supported features + +* Single or bidirectional media in the call +* RTC Event log dump per peer +* AEC dump per peer +* Compatible with *`webrtc::TimeController`* for both real and simulated time +* Media + * AV sync +* Video + * Any amount of video tracks both from caller and callee sides + * Input video from + * Video generator + * Specified file + * Any instance of *`webrtc::test::FrameGeneratorInterface`* + * Dumping of captured/rendered video into file + * Screen sharing + * Vp8 simulcast from caller side + * Vp9 SVC from caller side + * Choosing of video codec (name and parameters), having multiple codecs + negotiated to support codec-switching testing. + * FEC (ULP or Flex) + * Forced codec overshooting (for encoder overshoot emulation on some + mobile devices, when hardware encoder can overshoot target bitrate) +* Audio + * Up to 1 audio track both from caller and callee sides + * Generated audio + * Audio from specified file + * Dumping of captured/rendered audio into file + * Parameterizing of `cricket::AudioOptions` + * Echo emulation +* Injection of various WebRTC components into underlying + *`webrtc::PeerConnection`* or *`webrtc::PeerConnectionFactory`*. You can see + the full list [here][11] +* Scheduling of events, that can happen during the test, for example: + * Changes in network configuration + * User statistics measurements + * Custom defined actions +* User defined statistics reporting via + *`webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::QualityMetricsReporter`* + interface + +## Exported metrics + +### General + +* *`<peer_name>_connected`* - peer successfully established connection to + remote side +* *`cpu_usage`* - CPU usage excluding video analyzer +* *`audio_ahead_ms`* - Used to estimate how much audio and video is out of + sync when the two tracks were from the same source. Stats are polled + periodically during a call. The metric represents how much earlier was audio + played out on average over the call. If, during a stats poll, video is + ahead, then audio_ahead_ms will be equal to 0 for this poll. +* *`video_ahead_ms`* - Used to estimate how much audio and video is out of + sync when the two tracks were from the same source. Stats are polled + periodically during a call. The metric represents how much earlier was video + played out on average over the call. If, during a stats poll, audio is + ahead, then video_ahead_ms will be equal to 0 for this poll. + +### Video + +See documentation for +[*`DefaultVideoQualityAnalyzer`*](default_video_quality_analyzer.md#exported-metrics) + +### Audio + +* *`accelerate_rate`* - when playout is sped up, this counter is increased by + the difference between the number of samples received and the number of + samples played out. If speedup is achieved by removing samples, this will be + the count of samples removed. Rate is calculated as difference between + nearby samples divided on sample interval. +* *`expand_rate`* - the total number of samples that are concealed samples + over time. A concealed sample is a sample that was replaced with synthesized + samples generated locally before being played out. Examples of samples that + have to be concealed are samples from lost packets or samples from packets + that arrive too late to be played out +* *`speech_expand_rate`* - the total number of samples that are concealed + samples minus the total number of concealed samples inserted that are + "silent" over time. Playing out silent samples results in silence or comfort + noise. +* *`preemptive_rate`* - when playout is slowed down, this counter is increased + by the difference between the number of samples received and the number of + samples played out. If playout is slowed down by inserting samples, this + will be the number of inserted samples. Rate is calculated as difference + between nearby samples divided on sample interval. +* *`average_jitter_buffer_delay_ms`* - average size of NetEQ jitter buffer. +* *`preferred_buffer_size_ms`* - preferred size of NetEQ jitter buffer. +* *`visqol_mos`* - proxy for audio quality itself. +* *`asdm_samples`* - measure of how much acceleration/deceleration was in the + signal. +* *`word_error_rate`* - measure of how intelligible the audio was (percent of + words that could not be recognized in output audio). + +### Network + +* *`bytes_sent`* - represents the total number of payload bytes sent on this + PeerConnection, i.e., not including headers or padding +* *`packets_sent`* - represents the total number of packets sent over this + PeerConnection’s transports. +* *`average_send_rate`* - average send rate calculated on bytes_sent divided + by test duration. +* *`payload_bytes_sent`* - total number of bytes sent for all SSRC plus total + number of RTP header and padding bytes sent for all SSRC. This does not + include the size of transport layer headers such as IP or UDP. +* *`sent_packets_loss`* - packets_sent minus corresponding packets_received. +* *`bytes_received`* - represents the total number of bytes received on this + PeerConnection, i.e., not including headers or padding. +* *`packets_received`* - represents the total number of packets received on + this PeerConnection’s transports. +* *`average_receive_rate`* - average receive rate calculated on bytes_received + divided by test duration. +* *`payload_bytes_received`* - total number of bytes received for all SSRC + plus total number of RTP header and padding bytes received for all SSRC. + This does not include the size of transport layer headers such as IP or UDP. + +### Framework stability + +* *`frames_in_flight`* - amount of frames that were captured but wasn't seen + on receiver in the way that also all frames after also weren't seen on + receiver. +* *`bytes_discarded_no_receiver`* - total number of bytes that were received + on network interfaces related to the peer, but destination port was closed. +* *`packets_discarded_no_receiver`* - total number of packets that were + received on network interfaces related to the peer, but destination port was + closed. + +## Examples + +Examples can be found in + +* [peer_connection_e2e_smoke_test.cc][3] +* [pc_full_stack_tests.cc][4] + +## Stats plotting + +### Description + +Stats plotting provides ability to plot statistic collected during the test. +Right now it is used in PeerConnection level framework and give ability to see +how video quality metrics changed during test execution. + +### Usage + +To make any metrics plottable you need: + +1. Collect metric data with [SamplesStatsCounter][5] which internally will + store all intermediate points and timestamps when these points were added. +2. Then you need to report collected data with + [`webrtc::test::PrintResult(...)`][6]. By using these method you will also + specify name of the plottable metric. + +After these steps it will be possible to export your metric for plotting. There +are several options how you can do this: + +1. Use [`webrtc::TestMain::Create()`][7] as `main` function implementation, for + example use [`test/test_main.cc`][8] as `main` function for your test. + + In such case your binary will have flag `--plot`, where you can provide a + list of metrics, that you want to plot or specify `all` to plot all + available metrics. + + If `--plot` is specified, the binary will output metrics data into `stdout`. + Then you need to pipe this `stdout` into python plotter script + [`rtc_tools/metrics_plotter.py`][9], which will plot data. + + Examples: + + ```shell + $ ./out/Default/test_support_unittests \ + --gtest_filter=PeerConnectionE2EQualityTestSmokeTest.Svc \ + --nologs \ + --plot=all \ + | python rtc_tools/metrics_plotter.py + ``` + + ```shell + $ ./out/Default/test_support_unittests \ + --gtest_filter=PeerConnectionE2EQualityTestSmokeTest.Svc \ + --nologs \ + --plot=psnr,ssim \ + | python rtc_tools/metrics_plotter.py + ``` + + Example chart:  + +2. Use API from [`test/testsupport/perf_test.h`][10] directly by invoking + `webrtc::test::PrintPlottableResults(const std::vector<std::string>& + desired_graphs)` to print plottable metrics to stdout. Then as in previous + option you need to pipe result into plotter script. + +[1]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;drc=cbe6e8a2589a925d4c91a2ac2c69201f03de9c39 +[2]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/create_peerconnection_quality_test_fixture.h;drc=cbe6e8a2589a925d4c91a2ac2c69201f03de9c39 +[3]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/peer_connection_e2e_smoke_test.cc;drc=cbe6e8a2589a925d4c91a2ac2c69201f03de9c39 +[4]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/video/pc_full_stack_tests.cc;drc=cbe6e8a2589a925d4c91a2ac2c69201f03de9c39 +[5]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/numerics/samples_stats_counter.h;drc=cbe6e8a2589a925d4c91a2ac2c69201f03de9c39 +[6]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/testsupport/perf_test.h;l=86;drc=0710b401b1e5b500b8e84946fb657656ba1b58b7 +[7]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/test_main_lib.h;l=23;drc=bcb42f1e4be136c390986a40d9d5cb3ad0de260b +[8]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/test_main.cc;drc=bcb42f1e4be136c390986a40d9d5cb3ad0de260b +[9]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/rtc_tools/metrics_plotter.py;drc=8cc6695652307929edfc877cd64b75cd9ec2d615 +[10]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/testsupport/perf_test.h;l=105;drc=0710b401b1e5b500b8e84946fb657656ba1b58b7 +[11]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=272;drc=484acf27231d931dbc99aedce85bc27e06486b96 diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/single_process_encoded_image_data_injector.png b/third_party/libwebrtc/test/pc/e2e/g3doc/single_process_encoded_image_data_injector.png Binary files differnew file mode 100644 index 0000000000..73480bafbe --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/g3doc/single_process_encoded_image_data_injector.png diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/video_quality_analyzer_pipeline.png b/third_party/libwebrtc/test/pc/e2e/g3doc/video_quality_analyzer_pipeline.png Binary files differnew file mode 100644 index 0000000000..6cddb91110 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/g3doc/video_quality_analyzer_pipeline.png diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/vp8_simulcast_offer_modification.png b/third_party/libwebrtc/test/pc/e2e/g3doc/vp8_simulcast_offer_modification.png Binary files differnew file mode 100644 index 0000000000..c7eaa04c0e --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/g3doc/vp8_simulcast_offer_modification.png diff --git a/third_party/libwebrtc/test/pc/e2e/media/media_helper.cc b/third_party/libwebrtc/test/pc/e2e/media/media_helper.cc new file mode 100644 index 0000000000..e945bd4dae --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/media/media_helper.cc @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/media/media_helper.h" + +#include <string> +#include <utility> + +#include "absl/types/variant.h" +#include "api/media_stream_interface.h" +#include "api/test/create_frame_generator.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/pclf/peer_configurer.h" +#include "test/frame_generator_capturer.h" +#include "test/platform_video_capturer.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +void MediaHelper::MaybeAddAudio(TestPeer* peer) { + if (!peer->params().audio_config) { + return; + } + const AudioConfig& audio_config = peer->params().audio_config.value(); + rtc::scoped_refptr<webrtc::AudioSourceInterface> source = + peer->pc_factory()->CreateAudioSource(audio_config.audio_options); + rtc::scoped_refptr<AudioTrackInterface> track = + peer->pc_factory()->CreateAudioTrack(*audio_config.stream_label, + source.get()); + std::string sync_group = audio_config.sync_group + ? audio_config.sync_group.value() + : audio_config.stream_label.value() + "-sync"; + peer->AddTrack(track, {sync_group, *audio_config.stream_label}); +} + +std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>> +MediaHelper::MaybeAddVideo(TestPeer* peer) { + // Params here valid because of pre-run validation. + const Params& params = peer->params(); + const ConfigurableParams& configurable_params = peer->configurable_params(); + std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>> out; + for (size_t i = 0; i < configurable_params.video_configs.size(); ++i) { + const VideoConfig& video_config = configurable_params.video_configs[i]; + // Setup input video source into peer connection. + std::unique_ptr<test::TestVideoCapturer> capturer = CreateVideoCapturer( + video_config, peer->ReleaseVideoSource(i), + video_quality_analyzer_injection_helper_->CreateFramePreprocessor( + params.name.value(), video_config)); + bool is_screencast = + video_config.content_hint == VideoTrackInterface::ContentHint::kText || + video_config.content_hint == + VideoTrackInterface::ContentHint::kDetailed; + rtc::scoped_refptr<TestVideoCapturerVideoTrackSource> source = + rtc::make_ref_counted<TestVideoCapturerVideoTrackSource>( + std::move(capturer), is_screencast); + out.push_back(source); + RTC_LOG(LS_INFO) << "Adding video with video_config.stream_label=" + << video_config.stream_label.value(); + rtc::scoped_refptr<VideoTrackInterface> track = + peer->pc_factory()->CreateVideoTrack(video_config.stream_label.value(), + source.get()); + if (video_config.content_hint.has_value()) { + track->set_content_hint(video_config.content_hint.value()); + } + std::string sync_group = video_config.sync_group + ? video_config.sync_group.value() + : video_config.stream_label.value() + "-sync"; + RTCErrorOr<rtc::scoped_refptr<RtpSenderInterface>> sender = + peer->AddTrack(track, {sync_group, *video_config.stream_label}); + RTC_CHECK(sender.ok()); + if (video_config.temporal_layers_count || + video_config.degradation_preference) { + RtpParameters rtp_parameters = sender.value()->GetParameters(); + if (video_config.temporal_layers_count) { + for (auto& encoding_parameters : rtp_parameters.encodings) { + encoding_parameters.num_temporal_layers = + video_config.temporal_layers_count; + } + } + if (video_config.degradation_preference) { + rtp_parameters.degradation_preference = + video_config.degradation_preference; + } + RTCError res = sender.value()->SetParameters(rtp_parameters); + RTC_CHECK(res.ok()) << "Failed to set RTP parameters"; + } + } + return out; +} + +std::unique_ptr<test::TestVideoCapturer> MediaHelper::CreateVideoCapturer( + const VideoConfig& video_config, + PeerConfigurer::VideoSource source, + std::unique_ptr<test::TestVideoCapturer::FramePreprocessor> + frame_preprocessor) { + CapturingDeviceIndex* capturing_device_index = + absl::get_if<CapturingDeviceIndex>(&source); + if (capturing_device_index != nullptr) { + std::unique_ptr<test::TestVideoCapturer> capturer = + test::CreateVideoCapturer(video_config.width, video_config.height, + video_config.fps, + static_cast<size_t>(*capturing_device_index)); + RTC_CHECK(capturer) + << "Failed to obtain input stream from capturing device #" + << *capturing_device_index; + capturer->SetFramePreprocessor(std::move(frame_preprocessor)); + return capturer; + } + + auto capturer = std::make_unique<test::FrameGeneratorCapturer>( + clock_, + absl::get<std::unique_ptr<test::FrameGeneratorInterface>>( + std::move(source)), + video_config.fps, *task_queue_factory_); + capturer->SetFramePreprocessor(std::move(frame_preprocessor)); + capturer->Init(); + return capturer; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/media/media_helper.h b/third_party/libwebrtc/test/pc/e2e/media/media_helper.h new file mode 100644 index 0000000000..2d163d009e --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/media/media_helper.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_MEDIA_MEDIA_HELPER_H_ +#define TEST_PC_E2E_MEDIA_MEDIA_HELPER_H_ + +#include <memory> +#include <vector> + +#include "api/test/frame_generator_interface.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/pclf/peer_configurer.h" +#include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h" +#include "test/pc/e2e/media/test_video_capturer_video_track_source.h" +#include "test/pc/e2e/test_peer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class MediaHelper { + public: + MediaHelper(VideoQualityAnalyzerInjectionHelper* + video_quality_analyzer_injection_helper, + TaskQueueFactory* task_queue_factory, + Clock* clock) + : clock_(clock), + task_queue_factory_(task_queue_factory), + video_quality_analyzer_injection_helper_( + video_quality_analyzer_injection_helper) {} + + void MaybeAddAudio(TestPeer* peer); + + std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>> + MaybeAddVideo(TestPeer* peer); + + private: + std::unique_ptr<test::TestVideoCapturer> CreateVideoCapturer( + const VideoConfig& video_config, + PeerConfigurer::VideoSource source, + std::unique_ptr<test::TestVideoCapturer::FramePreprocessor> + frame_preprocessor); + + Clock* const clock_; + TaskQueueFactory* const task_queue_factory_; + VideoQualityAnalyzerInjectionHelper* video_quality_analyzer_injection_helper_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_MEDIA_MEDIA_HELPER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/media/test_video_capturer_video_track_source.h b/third_party/libwebrtc/test/pc/e2e/media/test_video_capturer_video_track_source.h new file mode 100644 index 0000000000..c883a2e8e9 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/media/test_video_capturer_video_track_source.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_MEDIA_TEST_VIDEO_CAPTURER_VIDEO_TRACK_SOURCE_H_ +#define TEST_PC_E2E_MEDIA_TEST_VIDEO_CAPTURER_VIDEO_TRACK_SOURCE_H_ + +#include <memory> +#include <utility> + +#include "api/video/video_frame.h" +#include "api/video/video_source_interface.h" +#include "pc/video_track_source.h" +#include "test/test_video_capturer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class TestVideoCapturerVideoTrackSource : public VideoTrackSource { + public: + TestVideoCapturerVideoTrackSource( + std::unique_ptr<test::TestVideoCapturer> video_capturer, + bool is_screencast) + : VideoTrackSource(/*remote=*/false), + video_capturer_(std::move(video_capturer)), + is_screencast_(is_screencast) {} + + ~TestVideoCapturerVideoTrackSource() = default; + + void Start() { SetState(kLive); } + + void Stop() { SetState(kMuted); } + + bool is_screencast() const override { return is_screencast_; } + + protected: + rtc::VideoSourceInterface<VideoFrame>* source() override { + return video_capturer_.get(); + } + + private: + std::unique_ptr<test::TestVideoCapturer> video_capturer_; + const bool is_screencast_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_MEDIA_TEST_VIDEO_CAPTURER_VIDEO_TRACK_SOURCE_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/metric_metadata_keys.h b/third_party/libwebrtc/test/pc/e2e/metric_metadata_keys.h new file mode 100644 index 0000000000..fbcd3b90fe --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/metric_metadata_keys.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef TEST_PC_E2E_METRIC_METADATA_KEYS_H_ +#define TEST_PC_E2E_METRIC_METADATA_KEYS_H_ + +#include <string> + +namespace webrtc { +namespace webrtc_pc_e2e { + +// All metadata fields are present only if applicable for particular metric. +class MetricMetadataKey { + public: + // Represents on peer with whom the metric is associated. + static constexpr char kPeerMetadataKey[] = "peer"; + // Represents sender of the media stream. + static constexpr char kSenderMetadataKey[] = "sender"; + // Represents receiver of the media stream. + static constexpr char kReceiverMetadataKey[] = "receiver"; + // Represents name of the audio stream. + static constexpr char kAudioStreamMetadataKey[] = "audio_stream"; + // Represents name of the video stream. + static constexpr char kVideoStreamMetadataKey[] = "video_stream"; + // Represents name of the sync group to which stream belongs. + static constexpr char kPeerSyncGroupMetadataKey[] = "peer_sync_group"; + // Represents the test name (without any peer and stream data appended to it + // as it currently happens with the webrtc.test_metrics.Metric.test_case + // field). This metadata is temporary and it will be removed once this + // information is moved to webrtc.test_metrics.Metric.test_case. + // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey. + static constexpr char kExperimentalTestNameMetadataKey[] = + "experimental_test_name"; + // Represents index of a video spatial layer to which metric belongs. + static constexpr char kSpatialLayerMetadataKey[] = "spatial_layer"; + + private: + MetricMetadataKey() = default; +}; + +// All metadata fields are presented only if applicable for particular metric. +class SampleMetadataKey { + public: + // Represents a frame ID with which data point is associated. + static constexpr char kFrameIdMetadataKey[] = "frame_id"; + + private: + SampleMetadataKey() = default; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_METRIC_METADATA_KEYS_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.cc b/third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.cc new file mode 100644 index 0000000000..0bb28f0847 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.cc @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/network_quality_metrics_reporter.h" + +#include <utility> + +#include "api/stats/rtc_stats.h" +#include "api/stats/rtcstats_objects.h" +#include "api/test/metrics/metric.h" +#include "rtc_base/checks.h" +#include "rtc_base/event.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Unit; + +constexpr TimeDelta kStatsWaitTimeout = TimeDelta::Seconds(1); + +// Field trial which controls whether to report standard-compliant bytes +// sent/received per stream. If enabled, padding and headers are not included +// in bytes sent or received. +constexpr char kUseStandardBytesStats[] = "WebRTC-UseStandardBytesStats"; + +} // namespace + +NetworkQualityMetricsReporter::NetworkQualityMetricsReporter( + EmulatedNetworkManagerInterface* alice_network, + EmulatedNetworkManagerInterface* bob_network, + test::MetricsLogger* metrics_logger) + : alice_network_(alice_network), + bob_network_(bob_network), + metrics_logger_(metrics_logger) { + RTC_CHECK(metrics_logger_); +} + +void NetworkQualityMetricsReporter::Start( + absl::string_view test_case_name, + const TrackIdStreamInfoMap* /*reporter_helper*/) { + test_case_name_ = std::string(test_case_name); + // Check that network stats are clean before test execution. + EmulatedNetworkStats alice_stats = PopulateStats(alice_network_); + RTC_CHECK_EQ(alice_stats.overall_outgoing_stats.packets_sent, 0); + RTC_CHECK_EQ(alice_stats.overall_incoming_stats.packets_received, 0); + EmulatedNetworkStats bob_stats = PopulateStats(bob_network_); + RTC_CHECK_EQ(bob_stats.overall_outgoing_stats.packets_sent, 0); + RTC_CHECK_EQ(bob_stats.overall_incoming_stats.packets_received, 0); +} + +void NetworkQualityMetricsReporter::OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) { + DataSize payload_received = DataSize::Zero(); + DataSize payload_sent = DataSize::Zero(); + + auto inbound_stats = report->GetStatsOfType<RTCInboundRTPStreamStats>(); + for (const auto& stat : inbound_stats) { + payload_received += + DataSize::Bytes(stat->bytes_received.ValueOrDefault(0ul) + + stat->header_bytes_received.ValueOrDefault(0ul)); + } + + auto outbound_stats = report->GetStatsOfType<RTCOutboundRTPStreamStats>(); + for (const auto& stat : outbound_stats) { + payload_sent += + DataSize::Bytes(stat->bytes_sent.ValueOrDefault(0ul) + + stat->header_bytes_sent.ValueOrDefault(0ul)); + } + + MutexLock lock(&lock_); + PCStats& stats = pc_stats_[std::string(pc_label)]; + stats.payload_received = payload_received; + stats.payload_sent = payload_sent; +} + +void NetworkQualityMetricsReporter::StopAndReportResults() { + EmulatedNetworkStats alice_stats = PopulateStats(alice_network_); + EmulatedNetworkStats bob_stats = PopulateStats(bob_network_); + int64_t alice_packets_loss = + alice_stats.overall_outgoing_stats.packets_sent - + bob_stats.overall_incoming_stats.packets_received; + int64_t bob_packets_loss = + bob_stats.overall_outgoing_stats.packets_sent - + alice_stats.overall_incoming_stats.packets_received; + ReportStats("alice", alice_stats, alice_packets_loss); + ReportStats("bob", bob_stats, bob_packets_loss); + + if (!webrtc::field_trial::IsEnabled(kUseStandardBytesStats)) { + RTC_LOG(LS_ERROR) + << "Non-standard GetStats; \"payload\" counts include RTP headers"; + } + + MutexLock lock(&lock_); + for (const auto& pair : pc_stats_) { + ReportPCStats(pair.first, pair.second); + } +} + +EmulatedNetworkStats NetworkQualityMetricsReporter::PopulateStats( + EmulatedNetworkManagerInterface* network) { + rtc::Event wait; + EmulatedNetworkStats stats; + network->GetStats([&](EmulatedNetworkStats s) { + stats = std::move(s); + wait.Set(); + }); + bool stats_received = wait.Wait(kStatsWaitTimeout); + RTC_CHECK(stats_received); + return stats; +} + +void NetworkQualityMetricsReporter::ReportStats( + const std::string& network_label, + const EmulatedNetworkStats& stats, + int64_t packet_loss) { + metrics_logger_->LogSingleValueMetric( + "bytes_sent", GetTestCaseName(network_label), + stats.overall_outgoing_stats.bytes_sent.bytes(), Unit::kBytes, + ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "packets_sent", GetTestCaseName(network_label), + stats.overall_outgoing_stats.packets_sent, Unit::kUnitless, + ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "average_send_rate", GetTestCaseName(network_label), + stats.overall_outgoing_stats.packets_sent >= 2 + ? stats.overall_outgoing_stats.AverageSendRate().kbps<double>() + : 0, + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "bytes_discarded_no_receiver", GetTestCaseName(network_label), + stats.overall_incoming_stats.bytes_discarded_no_receiver.bytes(), + Unit::kBytes, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "packets_discarded_no_receiver", GetTestCaseName(network_label), + stats.overall_incoming_stats.packets_discarded_no_receiver, + Unit::kUnitless, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "bytes_received", GetTestCaseName(network_label), + stats.overall_incoming_stats.bytes_received.bytes(), Unit::kBytes, + ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "packets_received", GetTestCaseName(network_label), + stats.overall_incoming_stats.packets_received, Unit::kUnitless, + ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "average_receive_rate", GetTestCaseName(network_label), + stats.overall_incoming_stats.packets_received >= 2 + ? stats.overall_incoming_stats.AverageReceiveRate().kbps<double>() + : 0, + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "sent_packets_loss", GetTestCaseName(network_label), packet_loss, + Unit::kUnitless, ImprovementDirection::kNeitherIsBetter); +} + +void NetworkQualityMetricsReporter::ReportPCStats(const std::string& pc_label, + const PCStats& stats) { + metrics_logger_->LogSingleValueMetric( + "payload_bytes_received", pc_label, stats.payload_received.bytes(), + Unit::kBytes, ImprovementDirection::kNeitherIsBetter); + metrics_logger_->LogSingleValueMetric( + "payload_bytes_sent", pc_label, stats.payload_sent.bytes(), Unit::kBytes, + ImprovementDirection::kNeitherIsBetter); +} + +std::string NetworkQualityMetricsReporter::GetTestCaseName( + const std::string& network_label) const { + return test_case_name_ + "/" + network_label; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.h b/third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.h new file mode 100644 index 0000000000..ed894bcf54 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_ +#define TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_ + +#include <memory> +#include <string> + +#include "absl/strings/string_view.h" +#include "api/test/metrics/metrics_logger.h" +#include "api/test/network_emulation_manager.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/test/track_id_stream_info_map.h" +#include "api/units/data_size.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class NetworkQualityMetricsReporter + : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { + public: + NetworkQualityMetricsReporter(EmulatedNetworkManagerInterface* alice_network, + EmulatedNetworkManagerInterface* bob_network, + test::MetricsLogger* metrics_logger); + ~NetworkQualityMetricsReporter() override = default; + + // Network stats must be empty when this method will be invoked. + void Start(absl::string_view test_case_name, + const TrackIdStreamInfoMap* reporter_helper) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override; + void StopAndReportResults() override; + + private: + struct PCStats { + // TODO(nisse): Separate audio and video counters. Depends on standard stat + // counters, enabled by field trial "WebRTC-UseStandardBytesStats". + DataSize payload_received = DataSize::Zero(); + DataSize payload_sent = DataSize::Zero(); + }; + + static EmulatedNetworkStats PopulateStats( + EmulatedNetworkManagerInterface* network); + void ReportStats(const std::string& network_label, + const EmulatedNetworkStats& stats, + int64_t packet_loss); + void ReportPCStats(const std::string& pc_label, const PCStats& stats); + std::string GetTestCaseName(const std::string& network_label) const; + + std::string test_case_name_; + + EmulatedNetworkManagerInterface* const alice_network_; + EmulatedNetworkManagerInterface* const bob_network_; + test::MetricsLogger* const metrics_logger_; + Mutex lock_; + std::map<std::string, PCStats> pc_stats_ RTC_GUARDED_BY(lock_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/peer_connection_e2e_smoke_test.cc b/third_party/libwebrtc/test/pc/e2e/peer_connection_e2e_smoke_test.cc new file mode 100644 index 0000000000..0e7993e5be --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/peer_connection_e2e_smoke_test.cc @@ -0,0 +1,536 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <cstdint> +#include <memory> +#include <string> + +#include "api/media_stream_interface.h" +#include "api/test/create_network_emulation_manager.h" +#include "api/test/create_peer_connection_quality_test_frame_generator.h" +#include "api/test/create_peerconnection_quality_test_fixture.h" +#include "api/test/metrics/global_metrics_logger_and_exporter.h" +#include "api/test/network_emulation_manager.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/pclf/media_quality_test_params.h" +#include "api/test/pclf/peer_configurer.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "call/simulated_network.h" +#include "system_wrappers/include/field_trial.h" +#include "test/field_trial.h" +#include "test/gtest.h" +#include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" +#include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h" +#include "test/testsupport/file_utils.h" + +#if defined(WEBRTC_MAC) || defined(WEBRTC_IOS) +#include "modules/video_coding/codecs/test/objc_codec_factory_helper.h" // nogncheck +#endif + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test { + public: + void SetUp() override { + network_emulation_ = CreateNetworkEmulationManager(); + auto video_quality_analyzer = std::make_unique<DefaultVideoQualityAnalyzer>( + network_emulation_->time_controller()->GetClock(), + test::GetGlobalMetricsLogger()); + video_quality_analyzer_ = video_quality_analyzer.get(); + fixture_ = CreatePeerConnectionE2EQualityTestFixture( + testing::UnitTest::GetInstance()->current_test_info()->name(), + *network_emulation_->time_controller(), + /*audio_quality_analyzer=*/nullptr, std::move(video_quality_analyzer)); + } + + std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> + CreateNetwork() { + EmulatedNetworkNode* alice_node = network_emulation_->CreateEmulatedNode( + std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig())); + EmulatedNetworkNode* bob_node = network_emulation_->CreateEmulatedNode( + std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig())); + + EmulatedEndpoint* alice_endpoint = + network_emulation_->CreateEndpoint(EmulatedEndpointConfig()); + EmulatedEndpoint* bob_endpoint = + network_emulation_->CreateEndpoint(EmulatedEndpointConfig()); + + network_emulation_->CreateRoute(alice_endpoint, {alice_node}, bob_endpoint); + network_emulation_->CreateRoute(bob_endpoint, {bob_node}, alice_endpoint); + + EmulatedNetworkManagerInterface* alice_network = + network_emulation_->CreateEmulatedNetworkManagerInterface( + {alice_endpoint}); + EmulatedNetworkManagerInterface* bob_network = + network_emulation_->CreateEmulatedNetworkManagerInterface( + {bob_endpoint}); + + return std::make_pair(alice_network, bob_network); + } + + void AddPeer(EmulatedNetworkManagerInterface* network, + rtc::FunctionView<void(PeerConfigurer*)> update_configurer) { + auto configurer = + std::make_unique<PeerConfigurer>(network->network_dependencies()); + update_configurer(configurer.get()); + fixture_->AddPeer(std::move(configurer)); + } + + void RunAndCheckEachVideoStreamReceivedFrames(const RunParams& run_params) { + fixture_->Run(run_params); + + EXPECT_GE(fixture_->GetRealTestDuration(), run_params.run_duration); + VideoStreamsInfo known_streams = video_quality_analyzer_->GetKnownStreams(); + for (const StatsKey& stream_key : known_streams.GetStatsKeys()) { + FrameCounters stream_conters = + video_quality_analyzer_->GetPerStreamCounters().at(stream_key); + // On some devices the pipeline can be too slow, so we actually can't + // force real constraints here. Lets just check, that at least 1 + // frame passed whole pipeline. + int64_t expected_min_fps = run_params.run_duration.seconds() * 15; + EXPECT_GE(stream_conters.captured, expected_min_fps) + << stream_key.ToString(); + EXPECT_GE(stream_conters.pre_encoded, 1) << stream_key.ToString(); + EXPECT_GE(stream_conters.encoded, 1) << stream_key.ToString(); + EXPECT_GE(stream_conters.received, 1) << stream_key.ToString(); + EXPECT_GE(stream_conters.decoded, 1) << stream_key.ToString(); + EXPECT_GE(stream_conters.rendered, 1) << stream_key.ToString(); + } + } + + NetworkEmulationManager* network_emulation() { + return network_emulation_.get(); + } + + PeerConnectionE2EQualityTestFixture* fixture() { return fixture_.get(); } + + private: + std::unique_ptr<NetworkEmulationManager> network_emulation_; + DefaultVideoQualityAnalyzer* video_quality_analyzer_; + std::unique_ptr<PeerConnectionE2EQualityTestFixture> fixture_; +}; + +// IOS debug builds can be quite slow, disabling to avoid issues with timeouts. +#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) +#define MAYBE_Smoke DISABLED_Smoke +#else +#define MAYBE_Smoke Smoke +#endif +TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Smoke) { + std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + VideoConfig video(160, 120, 15); + video.stream_label = "alice-video"; + video.sync_group = "alice-media"; + alice->AddVideoConfig(std::move(video)); + + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + audio.sampling_frequency_in_hz = 48000; + audio.sync_group = "alice-media"; + alice->SetAudioConfig(std::move(audio)); + alice->SetVideoCodecs( + {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}); + + alice->SetUseFlexFEC(true); + alice->SetUseUlpFEC(true); + alice->SetVideoEncoderBitrateMultiplier(1.1); + }); + AddPeer(network_links.second, [](PeerConfigurer* charlie) { + charlie->SetName("charlie"); + VideoConfig video(160, 120, 15); + video.stream_label = "charlie-video"; + video.temporal_layers_count = 2; + charlie->AddVideoConfig(std::move(video)); + + AudioConfig audio; + audio.stream_label = "charlie-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_bob_source", "wav"); + charlie->SetAudioConfig(std::move(audio)); + charlie->SetVideoCodecs( + {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}); + + charlie->SetUseFlexFEC(true); + charlie->SetUseUlpFEC(true); + charlie->SetVideoEncoderBitrateMultiplier(1.1); + }); + fixture()->AddQualityMetricsReporter( + std::make_unique<StatsBasedNetworkQualityMetricsReporter>( + std::map<std::string, std::vector<EmulatedEndpoint*>>( + {{"alice", network_links.first->endpoints()}, + {"charlie", network_links.second->endpoints()}}), + network_emulation(), test::GetGlobalMetricsLogger())); + RunParams run_params(TimeDelta::Seconds(2)); + run_params.enable_flex_fec_support = true; + RunAndCheckEachVideoStreamReceivedFrames(run_params); +} + +// IOS debug builds can be quite slow, disabling to avoid issues with timeouts. +#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) +#define MAYBE_Smoke DISABLED_Smoke +#else +#define MAYBE_SendAndReceivePacketsOnOneThread \ + SmokeSendAndReceivePacketsOnOneThread +#endif +// Only use the network thread for sending and receiving packets. +// The one and only network thread is used as a worker thread in all +// PeerConnections. Pacing when sending packets is done on the worker thread. +// See bugs.webrtc.org/14502. +TEST_F(PeerConnectionE2EQualityTestSmokeTest, + MAYBE_SendAndReceivePacketsOnOneThread) { + test::ScopedFieldTrials trials( + std::string(field_trial::GetFieldTrialString()) + + "WebRTC-SendPacketsOnWorkerThread/Enabled/"); + + std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + // Peerconnection use the network thread as the worker thread. + alice->SetUseNetworkThreadAsWorkerThread(); + VideoConfig video(160, 120, 15); + video.stream_label = "alice-video"; + video.sync_group = "alice-media"; + alice->AddVideoConfig(std::move(video)); + + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + audio.sampling_frequency_in_hz = 48000; + audio.sync_group = "alice-media"; + alice->SetAudioConfig(std::move(audio)); + alice->SetVideoCodecs( + {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}); + }); + AddPeer(network_links.second, [](PeerConfigurer* charlie) { + // Peerconnection use the network thread as the worker thread. + charlie->SetUseNetworkThreadAsWorkerThread(); + charlie->SetName("charlie"); + VideoConfig video(160, 120, 15); + video.stream_label = "charlie-video"; + video.temporal_layers_count = 2; + charlie->AddVideoConfig(std::move(video)); + + AudioConfig audio; + audio.stream_label = "charlie-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_bob_source", "wav"); + charlie->SetAudioConfig(std::move(audio)); + charlie->SetVideoCodecs( + {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}); + charlie->SetVideoEncoderBitrateMultiplier(1.1); + }); + fixture()->AddQualityMetricsReporter( + std::make_unique<StatsBasedNetworkQualityMetricsReporter>( + std::map<std::string, std::vector<EmulatedEndpoint*>>( + {{"alice", network_links.first->endpoints()}, + {"charlie", network_links.second->endpoints()}}), + network_emulation(), test::GetGlobalMetricsLogger())); + RunParams run_params(TimeDelta::Seconds(2)); + RunAndCheckEachVideoStreamReceivedFrames(run_params); +} + +#if defined(WEBRTC_MAC) || defined(WEBRTC_IOS) +TEST_F(PeerConnectionE2EQualityTestSmokeTest, SmokeH264) { + std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> + network_links = CreateNetwork(); + + AddPeer(network_links.first, [](PeerConfigurer* alice) { + VideoConfig video(160, 120, 15); + video.stream_label = "alice-video"; + video.sync_group = "alice-media"; + alice->AddVideoConfig(std::move(video)); + + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + audio.sampling_frequency_in_hz = 48000; + audio.sync_group = "alice-media"; + alice->SetAudioConfig(std::move(audio)); + alice->SetVideoCodecs({VideoCodecConfig(cricket::kH264CodecName)}); + alice->SetVideoEncoderFactory(webrtc::test::CreateObjCEncoderFactory()); + alice->SetVideoDecoderFactory(webrtc::test::CreateObjCDecoderFactory()); + }); + AddPeer(network_links.second, [](PeerConfigurer* charlie) { + charlie->SetName("charlie"); + VideoConfig video(160, 120, 15); + video.stream_label = "charlie-video"; + video.temporal_layers_count = 2; + charlie->AddVideoConfig(std::move(video)); + + AudioConfig audio; + audio.stream_label = "charlie-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_bob_source", "wav"); + charlie->SetAudioConfig(std::move(audio)); + charlie->SetVideoCodecs({VideoCodecConfig(cricket::kH264CodecName)}); + charlie->SetVideoEncoderFactory(webrtc::test::CreateObjCEncoderFactory()); + charlie->SetVideoDecoderFactory(webrtc::test::CreateObjCDecoderFactory()); + }); + + fixture()->AddQualityMetricsReporter( + std::make_unique<StatsBasedNetworkQualityMetricsReporter>( + std::map<std::string, std::vector<EmulatedEndpoint*>>( + {{"alice", network_links.first->endpoints()}, + {"charlie", network_links.second->endpoints()}}), + network_emulation(), test::GetGlobalMetricsLogger())); + RunParams run_params(TimeDelta::Seconds(2)); + run_params.enable_flex_fec_support = true; + RunAndCheckEachVideoStreamReceivedFrames(run_params); +} +#endif + +// IOS debug builds can be quite slow, disabling to avoid issues with timeouts. +#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) +#define MAYBE_ChangeNetworkConditions DISABLED_ChangeNetworkConditions +#else +#define MAYBE_ChangeNetworkConditions ChangeNetworkConditions +#endif +TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_ChangeNetworkConditions) { + NetworkEmulationManager::SimulatedNetworkNode alice_node = + network_emulation() + ->NodeBuilder() + .config(BuiltInNetworkBehaviorConfig()) + .Build(); + NetworkEmulationManager::SimulatedNetworkNode bob_node = + network_emulation() + ->NodeBuilder() + .config(BuiltInNetworkBehaviorConfig()) + .Build(); + + EmulatedEndpoint* alice_endpoint = + network_emulation()->CreateEndpoint(EmulatedEndpointConfig()); + EmulatedEndpoint* bob_endpoint = + network_emulation()->CreateEndpoint(EmulatedEndpointConfig()); + + network_emulation()->CreateRoute(alice_endpoint, {alice_node.node}, + bob_endpoint); + network_emulation()->CreateRoute(bob_endpoint, {bob_node.node}, + alice_endpoint); + + EmulatedNetworkManagerInterface* alice_network = + network_emulation()->CreateEmulatedNetworkManagerInterface( + {alice_endpoint}); + EmulatedNetworkManagerInterface* bob_network = + network_emulation()->CreateEmulatedNetworkManagerInterface( + {bob_endpoint}); + + AddPeer(alice_network, [](PeerConfigurer* alice) { + VideoConfig video(160, 120, 15); + video.stream_label = "alice-video"; + video.sync_group = "alice-media"; + alice->AddVideoConfig(std::move(video)); + alice->SetVideoCodecs( + {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}); + + alice->SetUseFlexFEC(true); + alice->SetUseUlpFEC(true); + alice->SetVideoEncoderBitrateMultiplier(1.1); + }); + AddPeer(bob_network, [](PeerConfigurer* bob) { + bob->SetVideoCodecs( + {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}); + + bob->SetUseFlexFEC(true); + bob->SetUseUlpFEC(true); + bob->SetVideoEncoderBitrateMultiplier(1.1); + }); + fixture()->AddQualityMetricsReporter( + std::make_unique<StatsBasedNetworkQualityMetricsReporter>( + std::map<std::string, std::vector<EmulatedEndpoint*>>( + {{"alice", alice_network->endpoints()}, + {"bob", bob_network->endpoints()}}), + network_emulation(), test::GetGlobalMetricsLogger())); + + fixture()->ExecuteAt(TimeDelta::Seconds(1), [alice_node](TimeDelta) { + BuiltInNetworkBehaviorConfig config; + config.loss_percent = 5; + alice_node.simulation->SetConfig(config); + }); + + RunParams run_params(TimeDelta::Seconds(2)); + run_params.enable_flex_fec_support = true; + RunAndCheckEachVideoStreamReceivedFrames(run_params); +} + +// IOS debug builds can be quite slow, disabling to avoid issues with timeouts. +#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) +#define MAYBE_Screenshare DISABLED_Screenshare +#else +#define MAYBE_Screenshare Screenshare +#endif +TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Screenshare) { + std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + VideoConfig screenshare(320, 180, 30); + screenshare.stream_label = "alice-screenshare"; + screenshare.content_hint = VideoTrackInterface::ContentHint::kText; + ScreenShareConfig screen_share_config = + ScreenShareConfig(TimeDelta::Seconds(2)); + screen_share_config.scrolling_params = + ScrollingParams{.duration = TimeDelta::Millis(1800)}; + auto screen_share_frame_generator = + CreateScreenShareFrameGenerator(screenshare, screen_share_config); + alice->AddVideoConfig(std::move(screenshare), + std::move(screen_share_frame_generator)); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) {}); + RunAndCheckEachVideoStreamReceivedFrames(RunParams(TimeDelta::Seconds(2))); +} + +// IOS debug builds can be quite slow, disabling to avoid issues with timeouts. +#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) +#define MAYBE_Echo DISABLED_Echo +#else +#define MAYBE_Echo Echo +#endif +TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Echo) { + std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + audio.sampling_frequency_in_hz = 48000; + alice->SetAudioConfig(std::move(audio)); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) { + AudioConfig audio; + audio.stream_label = "bob-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_bob_source", "wav"); + bob->SetAudioConfig(std::move(audio)); + }); + RunParams run_params(TimeDelta::Seconds(2)); + run_params.echo_emulation_config = EchoEmulationConfig(); + RunAndCheckEachVideoStreamReceivedFrames(run_params); +} + +// IOS debug builds can be quite slow, disabling to avoid issues with timeouts. +#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) +#define MAYBE_Simulcast DISABLED_Simulcast +#else +#define MAYBE_Simulcast Simulcast +#endif +TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Simulcast) { + std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + VideoConfig simulcast(1280, 720, 15); + simulcast.stream_label = "alice-simulcast"; + simulcast.simulcast_config = VideoSimulcastConfig(2); + simulcast.emulated_sfu_config = EmulatedSFUConfig(0); + alice->AddVideoConfig(std::move(simulcast)); + + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + alice->SetAudioConfig(std::move(audio)); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) {}); + RunParams run_params(TimeDelta::Seconds(2)); + RunAndCheckEachVideoStreamReceivedFrames(run_params); +} + +// IOS debug builds can be quite slow, disabling to avoid issues with timeouts. +#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) +#define MAYBE_Svc DISABLED_Svc +#else +#define MAYBE_Svc Svc +#endif +TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Svc) { + std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + VideoConfig simulcast("alice-svc", 1280, 720, 15); + // Because we have network with packets loss we can analyze only the + // highest spatial layer in SVC mode. + simulcast.simulcast_config = VideoSimulcastConfig(2); + simulcast.emulated_sfu_config = EmulatedSFUConfig(1); + alice->AddVideoConfig(std::move(simulcast)); + + AudioConfig audio("alice-audio"); + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + alice->SetAudioConfig(std::move(audio)); + alice->SetVideoCodecs({VideoCodecConfig(cricket::kVp9CodecName)}); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) { + bob->SetVideoCodecs({VideoCodecConfig(cricket::kVp9CodecName)}); + }); + RunParams run_params(TimeDelta::Seconds(2)); + RunAndCheckEachVideoStreamReceivedFrames(run_params); +} + +// IOS debug builds can be quite slow, disabling to avoid issues with timeouts. +#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) +#define MAYBE_HighBitrate DISABLED_HighBitrate +#else +#define MAYBE_HighBitrate HighBitrate +#endif +TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_HighBitrate) { + std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + BitrateSettings bitrate_settings; + bitrate_settings.start_bitrate_bps = 3'000'000; + bitrate_settings.max_bitrate_bps = 3'000'000; + alice->SetBitrateSettings(bitrate_settings); + VideoConfig video(800, 600, 15); + video.stream_label = "alice-video"; + RtpEncodingParameters encoding_parameters; + encoding_parameters.min_bitrate_bps = 500'000; + encoding_parameters.max_bitrate_bps = 3'000'000; + video.encoding_params.push_back(std::move(encoding_parameters)); + alice->AddVideoConfig(std::move(video)); + + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + audio.sampling_frequency_in_hz = 48000; + alice->SetAudioConfig(std::move(audio)); + alice->SetVideoCodecs( + {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) { + bob->SetVideoCodecs( + {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}); + }); + RunParams run_params(TimeDelta::Seconds(2)); + RunAndCheckEachVideoStreamReceivedFrames(run_params); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.cc b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.cc new file mode 100644 index 0000000000..83613118f9 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.cc @@ -0,0 +1,763 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/peer_connection_quality_test.h" + +#include <algorithm> +#include <memory> +#include <set> +#include <utility> + +#include "absl/strings/string_view.h" +#include "api/jsep.h" +#include "api/media_stream_interface.h" +#include "api/peer_connection_interface.h" +#include "api/rtc_event_log/rtc_event_log.h" +#include "api/rtc_event_log_output_file.h" +#include "api/scoped_refptr.h" +#include "api/test/metrics/metric.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/pclf/peer_configurer.h" +#include "api/test/time_controller.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "pc/sdp_utils.h" +#include "pc/test/mock_peer_connection_observers.h" +#include "rtc_base/gunit.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/task_queue_for_test.h" +#include "system_wrappers/include/cpu_info.h" +#include "system_wrappers/include/field_trial.h" +#include "test/field_trial.h" +#include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" +#include "test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h" +#include "test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h" +#include "test/pc/e2e/cross_media_metrics_reporter.h" +#include "test/pc/e2e/metric_metadata_keys.h" +#include "test/pc/e2e/peer_params_preprocessor.h" +#include "test/pc/e2e/stats_poller.h" +#include "test/pc/e2e/test_peer_factory.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Unit; + +constexpr TimeDelta kDefaultTimeout = TimeDelta::Seconds(10); +constexpr char kSignalThreadName[] = "signaling_thread"; +// 1 signaling, 2 network, 2 worker and 2 extra for codecs etc. +constexpr int kPeerConnectionUsedThreads = 7; +// Framework has extra thread for network layer and extra thread for peer +// connection stats polling. +constexpr int kFrameworkUsedThreads = 2; +constexpr int kMaxVideoAnalyzerThreads = 8; + +constexpr TimeDelta kStatsUpdateInterval = TimeDelta::Seconds(1); + +constexpr TimeDelta kAliveMessageLogInterval = TimeDelta::Seconds(30); + +constexpr TimeDelta kQuickTestModeRunDuration = TimeDelta::Millis(100); + +// Field trials to enable Flex FEC advertising and receiving. +constexpr char kFlexFecEnabledFieldTrials[] = + "WebRTC-FlexFEC-03-Advertised/Enabled/WebRTC-FlexFEC-03/Enabled/"; +constexpr char kUseStandardsBytesStats[] = + "WebRTC-UseStandardBytesStats/Enabled/"; + +class FixturePeerConnectionObserver : public MockPeerConnectionObserver { + public: + // `on_track_callback` will be called when any new track will be added to peer + // connection. + // `on_connected_callback` will be called when peer connection will come to + // either connected or completed state. Client should notice that in the case + // of reconnect this callback can be called again, so it should be tolerant + // to such behavior. + FixturePeerConnectionObserver( + std::function<void(rtc::scoped_refptr<RtpTransceiverInterface>)> + on_track_callback, + std::function<void()> on_connected_callback) + : on_track_callback_(std::move(on_track_callback)), + on_connected_callback_(std::move(on_connected_callback)) {} + + void OnTrack( + rtc::scoped_refptr<RtpTransceiverInterface> transceiver) override { + MockPeerConnectionObserver::OnTrack(transceiver); + on_track_callback_(transceiver); + } + + void OnIceConnectionChange( + PeerConnectionInterface::IceConnectionState new_state) override { + MockPeerConnectionObserver::OnIceConnectionChange(new_state); + if (ice_connected_) { + on_connected_callback_(); + } + } + + private: + std::function<void(rtc::scoped_refptr<RtpTransceiverInterface>)> + on_track_callback_; + std::function<void()> on_connected_callback_; +}; + +void ValidateP2PSimulcastParams( + const std::vector<std::unique_ptr<PeerConfigurer>>& peers) { + for (size_t i = 0; i < peers.size(); ++i) { + Params* params = peers[i]->params(); + ConfigurableParams* configurable_params = peers[i]->configurable_params(); + for (const VideoConfig& video_config : configurable_params->video_configs) { + if (video_config.simulcast_config) { + // When we simulate SFU we support only one video codec. + RTC_CHECK_EQ(params->video_codecs.size(), 1) + << "Only 1 video codec is supported when simulcast is enabled in " + << "at least 1 video config"; + } + } + } +} + +} // namespace + +PeerConnectionE2EQualityTest::PeerConnectionE2EQualityTest( + std::string test_case_name, + TimeController& time_controller, + std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer, + std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer) + : PeerConnectionE2EQualityTest(std::move(test_case_name), + time_controller, + std::move(audio_quality_analyzer), + std::move(video_quality_analyzer), + /*metrics_logger_=*/nullptr) {} + +PeerConnectionE2EQualityTest::PeerConnectionE2EQualityTest( + std::string test_case_name, + TimeController& time_controller, + std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer, + std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer, + test::MetricsLogger* metrics_logger) + : time_controller_(time_controller), + task_queue_factory_(time_controller_.CreateTaskQueueFactory()), + test_case_name_(std::move(test_case_name)), + executor_(std::make_unique<TestActivitiesExecutor>( + time_controller_.GetClock())), + metrics_logger_(metrics_logger) { + // Create default video quality analyzer. We will always create an analyzer, + // even if there are no video streams, because it will be installed into video + // encoder/decoder factories. + if (video_quality_analyzer == nullptr) { + video_quality_analyzer = std::make_unique<DefaultVideoQualityAnalyzer>( + time_controller_.GetClock(), metrics_logger_); + } + if (field_trial::IsEnabled("WebRTC-VideoFrameTrackingIdAdvertised")) { + encoded_image_data_propagator_ = + std::make_unique<VideoFrameTrackingIdInjector>(); + } else { + encoded_image_data_propagator_ = + std::make_unique<SingleProcessEncodedImageDataInjector>(); + } + video_quality_analyzer_injection_helper_ = + std::make_unique<VideoQualityAnalyzerInjectionHelper>( + time_controller_.GetClock(), std::move(video_quality_analyzer), + encoded_image_data_propagator_.get(), + encoded_image_data_propagator_.get()); + + if (audio_quality_analyzer == nullptr) { + audio_quality_analyzer = + std::make_unique<DefaultAudioQualityAnalyzer>(metrics_logger_); + } + audio_quality_analyzer_.swap(audio_quality_analyzer); +} + +void PeerConnectionE2EQualityTest::ExecuteAt( + TimeDelta target_time_since_start, + std::function<void(TimeDelta)> func) { + executor_->ScheduleActivity(target_time_since_start, absl::nullopt, func); +} + +void PeerConnectionE2EQualityTest::ExecuteEvery( + TimeDelta initial_delay_since_start, + TimeDelta interval, + std::function<void(TimeDelta)> func) { + executor_->ScheduleActivity(initial_delay_since_start, interval, func); +} + +void PeerConnectionE2EQualityTest::AddQualityMetricsReporter( + std::unique_ptr<QualityMetricsReporter> quality_metrics_reporter) { + quality_metrics_reporters_.push_back(std::move(quality_metrics_reporter)); +} + +PeerConnectionE2EQualityTest::PeerHandle* PeerConnectionE2EQualityTest::AddPeer( + std::unique_ptr<PeerConfigurer> configurer) { + peer_configurations_.push_back(std::move(configurer)); + peer_handles_.push_back(PeerHandleImpl()); + return &peer_handles_.back(); +} + +void PeerConnectionE2EQualityTest::Run(RunParams run_params) { + webrtc::webrtc_pc_e2e::PeerParamsPreprocessor params_preprocessor; + for (auto& peer_configuration : peer_configurations_) { + params_preprocessor.SetDefaultValuesForMissingParams(*peer_configuration); + params_preprocessor.ValidateParams(*peer_configuration); + } + ValidateP2PSimulcastParams(peer_configurations_); + RTC_CHECK_EQ(peer_configurations_.size(), 2) + << "Only peer to peer calls are allowed, please add 2 peers"; + + std::unique_ptr<PeerConfigurer> alice_configurer = + std::move(peer_configurations_[0]); + std::unique_ptr<PeerConfigurer> bob_configurer = + std::move(peer_configurations_[1]); + peer_configurations_.clear(); + + for (size_t i = 0; + i < bob_configurer->configurable_params()->video_configs.size(); ++i) { + // We support simulcast only from caller. + RTC_CHECK(!bob_configurer->configurable_params() + ->video_configs[i] + .simulcast_config) + << "Only simulcast stream from first peer is supported"; + } + + test::ScopedFieldTrials field_trials(GetFieldTrials(run_params)); + + // Print test summary + RTC_LOG(LS_INFO) + << "Media quality test: " << *alice_configurer->params()->name + << " will make a call to " << *bob_configurer->params()->name + << " with media video=" + << !alice_configurer->configurable_params()->video_configs.empty() + << "; audio=" << alice_configurer->params()->audio_config.has_value() + << ". " << *bob_configurer->params()->name + << " will respond with media video=" + << !bob_configurer->configurable_params()->video_configs.empty() + << "; audio=" << bob_configurer->params()->audio_config.has_value(); + + const std::unique_ptr<rtc::Thread> signaling_thread = + time_controller_.CreateThread(kSignalThreadName); + media_helper_ = std::make_unique<MediaHelper>( + video_quality_analyzer_injection_helper_.get(), task_queue_factory_.get(), + time_controller_.GetClock()); + + // Create a `task_queue_`. + task_queue_ = std::make_unique<webrtc::TaskQueueForTest>( + time_controller_.GetTaskQueueFactory()->CreateTaskQueue( + "pc_e2e_quality_test", webrtc::TaskQueueFactory::Priority::NORMAL)); + + // Create call participants: Alice and Bob. + // Audio streams are intercepted in AudioDeviceModule, so if it is required to + // catch output of Alice's stream, Alice's output_dump_file_name should be + // passed to Bob's TestPeer setup as audio output file name. + absl::optional<RemotePeerAudioConfig> alice_remote_audio_config = + RemotePeerAudioConfig::Create(bob_configurer->params()->audio_config); + absl::optional<RemotePeerAudioConfig> bob_remote_audio_config = + RemotePeerAudioConfig::Create(alice_configurer->params()->audio_config); + // Copy Alice and Bob video configs, subscriptions and names to correctly pass + // them into lambdas. + VideoSubscription alice_subscription = + alice_configurer->configurable_params()->video_subscription; + std::vector<VideoConfig> alice_video_configs = + alice_configurer->configurable_params()->video_configs; + std::string alice_name = alice_configurer->params()->name.value(); + VideoSubscription bob_subscription = + alice_configurer->configurable_params()->video_subscription; + std::vector<VideoConfig> bob_video_configs = + bob_configurer->configurable_params()->video_configs; + std::string bob_name = bob_configurer->params()->name.value(); + + TestPeerFactory test_peer_factory( + signaling_thread.get(), time_controller_, + video_quality_analyzer_injection_helper_.get(), task_queue_.get()); + alice_ = test_peer_factory.CreateTestPeer( + std::move(alice_configurer), + std::make_unique<FixturePeerConnectionObserver>( + [this, alice_name, alice_subscription, bob_video_configs]( + rtc::scoped_refptr<RtpTransceiverInterface> transceiver) { + OnTrackCallback(alice_name, alice_subscription, transceiver, + bob_video_configs); + }, + [this]() { StartVideo(alice_video_sources_); }), + alice_remote_audio_config, run_params.echo_emulation_config); + bob_ = test_peer_factory.CreateTestPeer( + std::move(bob_configurer), + std::make_unique<FixturePeerConnectionObserver>( + [this, bob_name, bob_subscription, alice_video_configs]( + rtc::scoped_refptr<RtpTransceiverInterface> transceiver) { + OnTrackCallback(bob_name, bob_subscription, transceiver, + alice_video_configs); + }, + [this]() { StartVideo(bob_video_sources_); }), + bob_remote_audio_config, run_params.echo_emulation_config); + + int num_cores = CpuInfo::DetectNumberOfCores(); + RTC_DCHECK_GE(num_cores, 1); + + int video_analyzer_threads = + num_cores - kPeerConnectionUsedThreads - kFrameworkUsedThreads; + if (video_analyzer_threads <= 0) { + video_analyzer_threads = 1; + } + video_analyzer_threads = + std::min(video_analyzer_threads, kMaxVideoAnalyzerThreads); + RTC_LOG(LS_INFO) << "video_analyzer_threads=" << video_analyzer_threads; + quality_metrics_reporters_.push_back( + std::make_unique<VideoQualityMetricsReporter>(time_controller_.GetClock(), + metrics_logger_)); + quality_metrics_reporters_.push_back( + std::make_unique<CrossMediaMetricsReporter>(metrics_logger_)); + + video_quality_analyzer_injection_helper_->Start( + test_case_name_, + std::vector<std::string>{alice_->params().name.value(), + bob_->params().name.value()}, + video_analyzer_threads); + audio_quality_analyzer_->Start(test_case_name_, &analyzer_helper_); + for (auto& reporter : quality_metrics_reporters_) { + reporter->Start(test_case_name_, &analyzer_helper_); + } + + // Start RTCEventLog recording if requested. + if (alice_->params().rtc_event_log_path) { + auto alice_rtc_event_log = std::make_unique<webrtc::RtcEventLogOutputFile>( + alice_->params().rtc_event_log_path.value()); + alice_->pc()->StartRtcEventLog(std::move(alice_rtc_event_log), + webrtc::RtcEventLog::kImmediateOutput); + } + if (bob_->params().rtc_event_log_path) { + auto bob_rtc_event_log = std::make_unique<webrtc::RtcEventLogOutputFile>( + bob_->params().rtc_event_log_path.value()); + bob_->pc()->StartRtcEventLog(std::move(bob_rtc_event_log), + webrtc::RtcEventLog::kImmediateOutput); + } + + // Setup alive logging. It is done to prevent test infra to think that test is + // dead. + RepeatingTaskHandle::DelayedStart(task_queue_->Get(), + kAliveMessageLogInterval, []() { + std::printf("Test is still running...\n"); + return kAliveMessageLogInterval; + }); + + RTC_LOG(LS_INFO) << "Configuration is done. Now " << *alice_->params().name + << " is calling to " << *bob_->params().name << "..."; + + // Setup stats poller. + std::vector<StatsObserverInterface*> observers = { + audio_quality_analyzer_.get(), + video_quality_analyzer_injection_helper_.get()}; + for (auto& reporter : quality_metrics_reporters_) { + observers.push_back(reporter.get()); + } + StatsPoller stats_poller(observers, + std::map<std::string, StatsProvider*>{ + {*alice_->params().name, alice_.get()}, + {*bob_->params().name, bob_.get()}}); + executor_->ScheduleActivity(TimeDelta::Zero(), kStatsUpdateInterval, + [&stats_poller](TimeDelta) { + stats_poller.PollStatsAndNotifyObservers(); + }); + + // Setup call. + SendTask(signaling_thread.get(), + [this, &run_params] { SetupCallOnSignalingThread(run_params); }); + std::unique_ptr<SignalingInterceptor> signaling_interceptor = + CreateSignalingInterceptor(run_params); + // Connect peers. + SendTask(signaling_thread.get(), [this, &signaling_interceptor] { + ExchangeOfferAnswer(signaling_interceptor.get()); + }); + WaitUntilIceCandidatesGathered(signaling_thread.get()); + + SendTask(signaling_thread.get(), [this, &signaling_interceptor] { + ExchangeIceCandidates(signaling_interceptor.get()); + }); + WaitUntilPeersAreConnected(signaling_thread.get()); + + executor_->Start(task_queue_.get()); + Timestamp start_time = Now(); + + bool is_quick_test_enabled = field_trial::IsEnabled("WebRTC-QuickPerfTest"); + if (is_quick_test_enabled) { + time_controller_.AdvanceTime(kQuickTestModeRunDuration); + } else { + time_controller_.AdvanceTime(run_params.run_duration); + } + + RTC_LOG(LS_INFO) << "Test is done, initiating disconnect sequence."; + + // Stop all client started tasks to prevent their access to any call related + // objects after these objects will be destroyed during call tear down. + executor_->Stop(); + // There is no guarantee, that last stats collection will happen at the end + // of the call, so we force it after executor, which is among others is doing + // stats collection, was stopped. + task_queue_->SendTask([&stats_poller]() { + // Get final end-of-call stats. + stats_poller.PollStatsAndNotifyObservers(); + }); + // We need to detach AEC dumping from peers, because dump uses `task_queue_` + // inside. + alice_->DetachAecDump(); + bob_->DetachAecDump(); + // Tear down the call. + SendTask(signaling_thread.get(), [this] { TearDownCallOnSignalingThread(); }); + + Timestamp end_time = Now(); + RTC_LOG(LS_INFO) << "All peers are disconnected."; + { + MutexLock lock(&lock_); + real_test_duration_ = end_time - start_time; + } + + ReportGeneralTestResults(); + audio_quality_analyzer_->Stop(); + video_quality_analyzer_injection_helper_->Stop(); + for (auto& reporter : quality_metrics_reporters_) { + reporter->StopAndReportResults(); + } + + // Reset `task_queue_` after test to cleanup. + task_queue_.reset(); + + alice_ = nullptr; + bob_ = nullptr; + // Ensuring that TestVideoCapturerVideoTrackSource are destroyed on the right + // thread. + RTC_CHECK(alice_video_sources_.empty()); + RTC_CHECK(bob_video_sources_.empty()); +} + +std::string PeerConnectionE2EQualityTest::GetFieldTrials( + const RunParams& run_params) { + std::vector<absl::string_view> default_field_trials = { + kUseStandardsBytesStats}; + if (run_params.enable_flex_fec_support) { + default_field_trials.push_back(kFlexFecEnabledFieldTrials); + } + rtc::StringBuilder sb; + sb << field_trial::GetFieldTrialString(); + for (const absl::string_view& field_trial : default_field_trials) { + sb << field_trial; + } + return sb.Release(); +} + +void PeerConnectionE2EQualityTest::OnTrackCallback( + absl::string_view peer_name, + VideoSubscription peer_subscription, + rtc::scoped_refptr<RtpTransceiverInterface> transceiver, + std::vector<VideoConfig> remote_video_configs) { + const rtc::scoped_refptr<MediaStreamTrackInterface>& track = + transceiver->receiver()->track(); + RTC_CHECK_EQ(transceiver->receiver()->stream_ids().size(), 2) + << "Expected 2 stream ids: 1st - sync group, 2nd - unique stream label"; + std::string sync_group = transceiver->receiver()->stream_ids()[0]; + std::string stream_label = transceiver->receiver()->stream_ids()[1]; + analyzer_helper_.AddTrackToStreamMapping(track->id(), peer_name, stream_label, + sync_group); + if (track->kind() != MediaStreamTrackInterface::kVideoKind) { + return; + } + + // It is safe to cast here, because it is checked above that + // track->kind() is kVideoKind. + auto* video_track = static_cast<VideoTrackInterface*>(track.get()); + std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> video_sink = + video_quality_analyzer_injection_helper_->CreateVideoSink( + peer_name, peer_subscription, /*report_infra_stats=*/false); + video_track->AddOrUpdateSink(video_sink.get(), rtc::VideoSinkWants()); + output_video_sinks_.push_back(std::move(video_sink)); +} + +void PeerConnectionE2EQualityTest::SetupCallOnSignalingThread( + const RunParams& run_params) { + // We need receive-only transceivers for Bob's media stream, so there will + // be media section in SDP for that streams in Alice's offer, because it is + // forbidden to add new media sections in answer in Unified Plan. + RtpTransceiverInit receive_only_transceiver_init; + receive_only_transceiver_init.direction = RtpTransceiverDirection::kRecvOnly; + int alice_transceivers_counter = 0; + if (bob_->params().audio_config) { + // Setup receive audio transceiver if Bob has audio to send. If we'll need + // multiple audio streams, then we need transceiver for each Bob's audio + // stream. + RTCErrorOr<rtc::scoped_refptr<RtpTransceiverInterface>> result = + alice_->AddTransceiver(cricket::MediaType::MEDIA_TYPE_AUDIO, + receive_only_transceiver_init); + RTC_CHECK(result.ok()); + alice_transceivers_counter++; + } + + size_t alice_video_transceivers_non_simulcast_counter = 0; + for (auto& video_config : alice_->configurable_params().video_configs) { + RtpTransceiverInit transceiver_params; + if (video_config.simulcast_config) { + transceiver_params.direction = RtpTransceiverDirection::kSendOnly; + // Because simulcast enabled `alice_->params().video_codecs` has only 1 + // element. + if (alice_->params().video_codecs[0].name == cricket::kVp8CodecName) { + // For Vp8 simulcast we need to add as many RtpEncodingParameters to the + // track as many simulcast streams requested. If they specified in + // `video_config.simulcast_config` it should be copied from there. + for (int i = 0; + i < video_config.simulcast_config->simulcast_streams_count; ++i) { + RtpEncodingParameters enc_params; + if (!video_config.encoding_params.empty()) { + enc_params = video_config.encoding_params[i]; + } + // We need to be sure, that all rids will be unique with all mids. + enc_params.rid = std::to_string(alice_transceivers_counter) + "000" + + std::to_string(i); + transceiver_params.send_encodings.push_back(enc_params); + } + } + } else { + transceiver_params.direction = RtpTransceiverDirection::kSendRecv; + RtpEncodingParameters enc_params; + if (video_config.encoding_params.size() == 1) { + enc_params = video_config.encoding_params[0]; + } + transceiver_params.send_encodings.push_back(enc_params); + + alice_video_transceivers_non_simulcast_counter++; + } + RTCErrorOr<rtc::scoped_refptr<RtpTransceiverInterface>> result = + alice_->AddTransceiver(cricket::MediaType::MEDIA_TYPE_VIDEO, + transceiver_params); + RTC_CHECK(result.ok()); + + alice_transceivers_counter++; + } + + // Add receive only transceivers in case Bob has more video_configs than + // Alice. + for (size_t i = alice_video_transceivers_non_simulcast_counter; + i < bob_->configurable_params().video_configs.size(); ++i) { + RTCErrorOr<rtc::scoped_refptr<RtpTransceiverInterface>> result = + alice_->AddTransceiver(cricket::MediaType::MEDIA_TYPE_VIDEO, + receive_only_transceiver_init); + RTC_CHECK(result.ok()); + alice_transceivers_counter++; + } + + // Then add media for Alice and Bob + media_helper_->MaybeAddAudio(alice_.get()); + alice_video_sources_ = media_helper_->MaybeAddVideo(alice_.get()); + media_helper_->MaybeAddAudio(bob_.get()); + bob_video_sources_ = media_helper_->MaybeAddVideo(bob_.get()); + + SetPeerCodecPreferences(alice_.get()); + SetPeerCodecPreferences(bob_.get()); +} + +void PeerConnectionE2EQualityTest::TearDownCallOnSignalingThread() { + TearDownCall(); +} + +void PeerConnectionE2EQualityTest::SetPeerCodecPreferences(TestPeer* peer) { + std::vector<RtpCodecCapability> with_rtx_video_capabilities = + FilterVideoCodecCapabilities( + peer->params().video_codecs, true, peer->params().use_ulp_fec, + peer->params().use_flex_fec, + peer->pc_factory() + ->GetRtpSenderCapabilities(cricket::MediaType::MEDIA_TYPE_VIDEO) + .codecs); + std::vector<RtpCodecCapability> without_rtx_video_capabilities = + FilterVideoCodecCapabilities( + peer->params().video_codecs, false, peer->params().use_ulp_fec, + peer->params().use_flex_fec, + peer->pc_factory() + ->GetRtpSenderCapabilities(cricket::MediaType::MEDIA_TYPE_VIDEO) + .codecs); + + // Set codecs for transceivers + for (auto transceiver : peer->pc()->GetTransceivers()) { + if (transceiver->media_type() == cricket::MediaType::MEDIA_TYPE_VIDEO) { + if (transceiver->sender()->init_send_encodings().size() > 1) { + // If transceiver's sender has more then 1 send encodings, it means it + // has multiple simulcast streams, so we need disable RTX on it. + RTCError result = + transceiver->SetCodecPreferences(without_rtx_video_capabilities); + RTC_CHECK(result.ok()); + } else { + RTCError result = + transceiver->SetCodecPreferences(with_rtx_video_capabilities); + RTC_CHECK(result.ok()); + } + } + } +} + +std::unique_ptr<SignalingInterceptor> +PeerConnectionE2EQualityTest::CreateSignalingInterceptor( + const RunParams& run_params) { + std::map<std::string, int> stream_label_to_simulcast_streams_count; + // We add only Alice here, because simulcast/svc is supported only from the + // first peer. + for (auto& video_config : alice_->configurable_params().video_configs) { + if (video_config.simulcast_config) { + stream_label_to_simulcast_streams_count.insert( + {*video_config.stream_label, + video_config.simulcast_config->simulcast_streams_count}); + } + } + PatchingParams patching_params(run_params.use_conference_mode, + stream_label_to_simulcast_streams_count); + return std::make_unique<SignalingInterceptor>(patching_params); +} + +void PeerConnectionE2EQualityTest::WaitUntilIceCandidatesGathered( + rtc::Thread* signaling_thread) { + ASSERT_TRUE(time_controller_.Wait( + [&]() { + bool result; + SendTask(signaling_thread, [&]() { + result = alice_->IsIceGatheringDone() && bob_->IsIceGatheringDone(); + }); + return result; + }, + 2 * kDefaultTimeout)); +} + +void PeerConnectionE2EQualityTest::WaitUntilPeersAreConnected( + rtc::Thread* signaling_thread) { + // This means that ICE and DTLS are connected. + alice_connected_ = time_controller_.Wait( + [&]() { + bool result; + SendTask(signaling_thread, [&] { result = alice_->IsIceConnected(); }); + return result; + }, + kDefaultTimeout); + bob_connected_ = time_controller_.Wait( + [&]() { + bool result; + SendTask(signaling_thread, [&] { result = bob_->IsIceConnected(); }); + return result; + }, + kDefaultTimeout); +} + +void PeerConnectionE2EQualityTest::ExchangeOfferAnswer( + SignalingInterceptor* signaling_interceptor) { + std::string log_output; + + auto offer = alice_->CreateOffer(); + RTC_CHECK(offer); + offer->ToString(&log_output); + RTC_LOG(LS_INFO) << "Original offer: " << log_output; + LocalAndRemoteSdp patch_result = signaling_interceptor->PatchOffer( + std::move(offer), alice_->params().video_codecs[0]); + patch_result.local_sdp->ToString(&log_output); + RTC_LOG(LS_INFO) << "Offer to set as local description: " << log_output; + patch_result.remote_sdp->ToString(&log_output); + RTC_LOG(LS_INFO) << "Offer to set as remote description: " << log_output; + + bool set_local_offer = + alice_->SetLocalDescription(std::move(patch_result.local_sdp)); + RTC_CHECK(set_local_offer); + bool set_remote_offer = + bob_->SetRemoteDescription(std::move(patch_result.remote_sdp)); + RTC_CHECK(set_remote_offer); + auto answer = bob_->CreateAnswer(); + RTC_CHECK(answer); + answer->ToString(&log_output); + RTC_LOG(LS_INFO) << "Original answer: " << log_output; + patch_result = signaling_interceptor->PatchAnswer( + std::move(answer), bob_->params().video_codecs[0]); + patch_result.local_sdp->ToString(&log_output); + RTC_LOG(LS_INFO) << "Answer to set as local description: " << log_output; + patch_result.remote_sdp->ToString(&log_output); + RTC_LOG(LS_INFO) << "Answer to set as remote description: " << log_output; + + bool set_local_answer = + bob_->SetLocalDescription(std::move(patch_result.local_sdp)); + RTC_CHECK(set_local_answer); + bool set_remote_answer = + alice_->SetRemoteDescription(std::move(patch_result.remote_sdp)); + RTC_CHECK(set_remote_answer); +} + +void PeerConnectionE2EQualityTest::ExchangeIceCandidates( + SignalingInterceptor* signaling_interceptor) { + // Connect an ICE candidate pairs. + std::vector<std::unique_ptr<IceCandidateInterface>> alice_candidates = + signaling_interceptor->PatchOffererIceCandidates( + alice_->observer()->GetAllCandidates()); + for (auto& candidate : alice_candidates) { + std::string candidate_str; + RTC_CHECK(candidate->ToString(&candidate_str)); + RTC_LOG(LS_INFO) << *alice_->params().name + << " ICE candidate(mid= " << candidate->sdp_mid() + << "): " << candidate_str; + } + ASSERT_TRUE(bob_->AddIceCandidates(std::move(alice_candidates))); + std::vector<std::unique_ptr<IceCandidateInterface>> bob_candidates = + signaling_interceptor->PatchAnswererIceCandidates( + bob_->observer()->GetAllCandidates()); + for (auto& candidate : bob_candidates) { + std::string candidate_str; + RTC_CHECK(candidate->ToString(&candidate_str)); + RTC_LOG(LS_INFO) << *bob_->params().name + << " ICE candidate(mid= " << candidate->sdp_mid() + << "): " << candidate_str; + } + ASSERT_TRUE(alice_->AddIceCandidates(std::move(bob_candidates))); +} + +void PeerConnectionE2EQualityTest::StartVideo( + const std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>>& + sources) { + for (auto& source : sources) { + if (source->state() != MediaSourceInterface::SourceState::kLive) { + source->Start(); + } + } +} + +void PeerConnectionE2EQualityTest::TearDownCall() { + for (const auto& video_source : alice_video_sources_) { + video_source->Stop(); + } + for (const auto& video_source : bob_video_sources_) { + video_source->Stop(); + } + + alice_video_sources_.clear(); + bob_video_sources_.clear(); + + alice_->Close(); + bob_->Close(); + + media_helper_ = nullptr; +} + +void PeerConnectionE2EQualityTest::ReportGeneralTestResults() { + // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey. + metrics_logger_->LogSingleValueMetric( + *alice_->params().name + "_connected", test_case_name_, alice_connected_, + Unit::kUnitless, ImprovementDirection::kBiggerIsBetter, + {{MetricMetadataKey::kPeerMetadataKey, *alice_->params().name}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_case_name_}}); + // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey. + metrics_logger_->LogSingleValueMetric( + *bob_->params().name + "_connected", test_case_name_, bob_connected_, + Unit::kUnitless, ImprovementDirection::kBiggerIsBetter, + {{MetricMetadataKey::kPeerMetadataKey, *bob_->params().name}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_case_name_}}); +} + +Timestamp PeerConnectionE2EQualityTest::Now() const { + return time_controller_.GetClock()->CurrentTime(); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.h b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.h new file mode 100644 index 0000000000..6cbf232874 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.h @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef TEST_PC_E2E_PEER_CONNECTION_QUALITY_TEST_H_ +#define TEST_PC_E2E_PEER_CONNECTION_QUALITY_TEST_H_ + +#include <memory> +#include <queue> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/task_queue/task_queue_factory.h" +#include "api/test/audio_quality_analyzer_interface.h" +#include "api/test/metrics/metrics_logger.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/pclf/media_quality_test_params.h" +#include "api/test/pclf/peer_configurer.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/test/time_controller.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/task_queue_for_test.h" +#include "rtc_base/thread.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h" +#include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h" +#include "test/pc/e2e/analyzer_helper.h" +#include "test/pc/e2e/media/media_helper.h" +#include "test/pc/e2e/sdp/sdp_changer.h" +#include "test/pc/e2e/test_activities_executor.h" +#include "test/pc/e2e/test_peer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class PeerConnectionE2EQualityTest + : public PeerConnectionE2EQualityTestFixture { + public: + using QualityMetricsReporter = + PeerConnectionE2EQualityTestFixture::QualityMetricsReporter; + + PeerConnectionE2EQualityTest( + std::string test_case_name, + TimeController& time_controller, + std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer, + std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer); + PeerConnectionE2EQualityTest( + std::string test_case_name, + TimeController& time_controller, + std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer, + std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer, + test::MetricsLogger* metrics_logger); + + ~PeerConnectionE2EQualityTest() override = default; + + void ExecuteAt(TimeDelta target_time_since_start, + std::function<void(TimeDelta)> func) override; + void ExecuteEvery(TimeDelta initial_delay_since_start, + TimeDelta interval, + std::function<void(TimeDelta)> func) override; + + void AddQualityMetricsReporter(std::unique_ptr<QualityMetricsReporter> + quality_metrics_reporter) override; + + PeerHandle* AddPeer(std::unique_ptr<PeerConfigurer> configurer) override; + void Run(RunParams run_params) override; + + TimeDelta GetRealTestDuration() const override { + MutexLock lock(&lock_); + RTC_CHECK_NE(real_test_duration_, TimeDelta::Zero()); + return real_test_duration_; + } + + private: + class PeerHandleImpl : public PeerHandle { + public: + ~PeerHandleImpl() override = default; + }; + + // For some functionality some field trials have to be enabled, they will be + // enabled in Run(). + std::string GetFieldTrials(const RunParams& run_params); + void OnTrackCallback(absl::string_view peer_name, + VideoSubscription peer_subscription, + rtc::scoped_refptr<RtpTransceiverInterface> transceiver, + std::vector<VideoConfig> remote_video_configs); + // Have to be run on the signaling thread. + void SetupCallOnSignalingThread(const RunParams& run_params); + void TearDownCallOnSignalingThread(); + void SetPeerCodecPreferences(TestPeer* peer); + std::unique_ptr<SignalingInterceptor> CreateSignalingInterceptor( + const RunParams& run_params); + void WaitUntilIceCandidatesGathered(rtc::Thread* signaling_thread); + void WaitUntilPeersAreConnected(rtc::Thread* signaling_thread); + void ExchangeOfferAnswer(SignalingInterceptor* signaling_interceptor); + void ExchangeIceCandidates(SignalingInterceptor* signaling_interceptor); + void StartVideo( + const std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>>& + sources); + void TearDownCall(); + void ReportGeneralTestResults(); + Timestamp Now() const; + + TimeController& time_controller_; + const std::unique_ptr<TaskQueueFactory> task_queue_factory_; + std::string test_case_name_; + std::unique_ptr<VideoQualityAnalyzerInjectionHelper> + video_quality_analyzer_injection_helper_; + std::unique_ptr<MediaHelper> media_helper_; + std::unique_ptr<EncodedImageDataPropagator> encoded_image_data_propagator_; + std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer_; + std::unique_ptr<TestActivitiesExecutor> executor_; + test::MetricsLogger* const metrics_logger_; + + std::vector<std::unique_ptr<PeerConfigurer>> peer_configurations_; + std::vector<PeerHandleImpl> peer_handles_; + + std::unique_ptr<TestPeer> alice_; + std::unique_ptr<TestPeer> bob_; + std::vector<std::unique_ptr<QualityMetricsReporter>> + quality_metrics_reporters_; + + std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>> + alice_video_sources_; + std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>> + bob_video_sources_; + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> + output_video_sinks_; + AnalyzerHelper analyzer_helper_; + + mutable Mutex lock_; + TimeDelta real_test_duration_ RTC_GUARDED_BY(lock_) = TimeDelta::Zero(); + + // Task queue, that is used for running activities during test call. + // This task queue will be created before call set up and will be destroyed + // immediately before call tear down. + std::unique_ptr<TaskQueueForTest> task_queue_; + + bool alice_connected_ = false; + bool bob_connected_ = false; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_PEER_CONNECTION_QUALITY_TEST_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc new file mode 100644 index 0000000000..8a47e108e0 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc @@ -0,0 +1,1102 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <map> +#include <memory> +#include <string> + +#include "api/test/create_network_emulation_manager.h" +#include "api/test/create_peer_connection_quality_test_frame_generator.h" +#include "api/test/metrics/metrics_logger.h" +#include "api/test/metrics/stdout_metrics_exporter.h" +#include "api/test/network_emulation_manager.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/pclf/media_quality_test_params.h" +#include "api/test/pclf/peer_configurer.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/units/time_delta.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/pc/e2e/metric_metadata_keys.h" +#include "test/pc/e2e/peer_connection_quality_test.h" +#include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::testing::UnorderedElementsAre; + +using ::webrtc::test::DefaultMetricsLogger; +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Metric; +using ::webrtc::test::MetricsExporter; +using ::webrtc::test::StdoutMetricsExporter; +using ::webrtc::test::Unit; +using ::webrtc::webrtc_pc_e2e::PeerConfigurer; + +// Adds a peer with some audio and video (the client should not care about +// details about audio and video configs). +void AddDefaultAudioVideoPeer( + absl::string_view peer_name, + absl::string_view audio_stream_label, + absl::string_view video_stream_label, + const PeerNetworkDependencies& network_dependencies, + PeerConnectionE2EQualityTestFixture& fixture) { + AudioConfig audio{std::string(audio_stream_label)}; + audio.sync_group = std::string(peer_name); + VideoConfig video(std::string(video_stream_label), 320, 180, 15); + video.sync_group = std::string(peer_name); + auto peer = std::make_unique<PeerConfigurer>(network_dependencies); + peer->SetName(peer_name); + peer->SetAudioConfig(std::move(audio)); + peer->AddVideoConfig(std::move(video)); + peer->SetVideoCodecs({VideoCodecConfig(cricket::kVp8CodecName)}); + fixture.AddPeer(std::move(peer)); +} + +// Metric fields to assert on +struct MetricValidationInfo { + std::string test_case; + std::string name; + Unit unit; + ImprovementDirection improvement_direction; + std::map<std::string, std::string> metadata; +}; + +bool operator==(const MetricValidationInfo& a, const MetricValidationInfo& b) { + return a.name == b.name && a.test_case == b.test_case && a.unit == b.unit && + a.improvement_direction == b.improvement_direction && + a.metadata == b.metadata; +} + +std::ostream& operator<<(std::ostream& os, const MetricValidationInfo& m) { + os << "{ test_case=" << m.test_case << "; name=" << m.name + << "; unit=" << test::ToString(m.unit) + << "; improvement_direction=" << test::ToString(m.improvement_direction) + << "; metadata={ "; + for (const auto& [key, value] : m.metadata) { + os << "{ key=" << key << "; value=" << value << " }"; + } + os << " }}"; + return os; +} + +std::vector<MetricValidationInfo> ToValidationInfo( + const std::vector<Metric>& metrics) { + std::vector<MetricValidationInfo> out; + for (const Metric& m : metrics) { + out.push_back( + MetricValidationInfo{.test_case = m.test_case, + .name = m.name, + .unit = m.unit, + .improvement_direction = m.improvement_direction, + .metadata = m.metric_metadata}); + } + return out; +} + +TEST(PeerConnectionE2EQualityTestMetricNamesTest, + ExportedMetricsHasCorrectNamesAndAnnotation) { + std::unique_ptr<NetworkEmulationManager> network_emulation = + CreateNetworkEmulationManager(TimeMode::kSimulated); + DefaultMetricsLogger metrics_logger( + network_emulation->time_controller()->GetClock()); + PeerConnectionE2EQualityTest fixture( + "test_case", *network_emulation->time_controller(), + /*audio_quality_analyzer=*/nullptr, /*video_quality_analyzer=*/nullptr, + &metrics_logger); + + EmulatedEndpoint* alice_endpoint = + network_emulation->CreateEndpoint(EmulatedEndpointConfig()); + EmulatedEndpoint* bob_endpoint = + network_emulation->CreateEndpoint(EmulatedEndpointConfig()); + + network_emulation->CreateRoute( + alice_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()}, + bob_endpoint); + network_emulation->CreateRoute( + bob_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()}, + alice_endpoint); + + EmulatedNetworkManagerInterface* alice_network = + network_emulation->CreateEmulatedNetworkManagerInterface( + {alice_endpoint}); + EmulatedNetworkManagerInterface* bob_network = + network_emulation->CreateEmulatedNetworkManagerInterface({bob_endpoint}); + + AddDefaultAudioVideoPeer("alice", "alice_audio", "alice_video", + alice_network->network_dependencies(), fixture); + AddDefaultAudioVideoPeer("bob", "bob_audio", "bob_video", + bob_network->network_dependencies(), fixture); + fixture.AddQualityMetricsReporter( + std::make_unique<StatsBasedNetworkQualityMetricsReporter>( + std::map<std::string, std::vector<EmulatedEndpoint*>>( + {{"alice", alice_network->endpoints()}, + {"bob", bob_network->endpoints()}}), + network_emulation.get(), &metrics_logger)); + + // Run for at least 7 seconds, so AV-sync metrics will be collected. + fixture.Run(RunParams(TimeDelta::Seconds(7))); + + std::vector<MetricValidationInfo> metrics = + ToValidationInfo(metrics_logger.GetCollectedMetrics()); + EXPECT_THAT( + metrics, + UnorderedElementsAre( + // Metrics from PeerConnectionE2EQualityTest + MetricValidationInfo{ + .test_case = "test_case", + .name = "alice_connected", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case", + .name = "bob_connected", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + + // Metrics from DefaultAudioQualityAnalyzer + MetricValidationInfo{ + .test_case = "test_case/alice_audio", + .name = "expand_rate", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey, + "alice_audio"}, + {MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_audio", + .name = "accelerate_rate", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey, + "alice_audio"}, + {MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_audio", + .name = "preemptive_rate", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey, + "alice_audio"}, + {MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_audio", + .name = "speech_expand_rate", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey, + "alice_audio"}, + {MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_audio", + .name = "average_jitter_buffer_delay_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey, + "alice_audio"}, + {MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_audio", + .name = "preferred_buffer_size_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey, + "alice_audio"}, + {MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_audio", + .name = "expand_rate", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey, + "bob_audio"}, + {MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_audio", + .name = "accelerate_rate", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey, + "bob_audio"}, + {MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_audio", + .name = "preemptive_rate", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey, + "bob_audio"}, + {MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_audio", + .name = "speech_expand_rate", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey, + "bob_audio"}, + {MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_audio", + .name = "average_jitter_buffer_delay_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey, + "bob_audio"}, + {MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_audio", + .name = "preferred_buffer_size_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey, + "bob_audio"}, + {MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + + // Metrics from DefaultVideoQualityAnalyzer + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "ssim", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "transport_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "total_delay_incl_transport", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "time_between_rendered_frames", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "harmonic_framerate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "encode_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "encode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "time_between_freezes", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "freeze_time_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "pixels_per_frame", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "min_psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "decode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "receive_to_render_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "dropped_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "frames_in_flight", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "rendered_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "max_skipped", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "target_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "qp_sl0", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kSpatialLayerMetadataKey, "0"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_video", + .name = "actual_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "alice_video"}, + {MetricMetadataKey::kSenderMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "ssim", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "transport_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "total_delay_incl_transport", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "time_between_rendered_frames", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "harmonic_framerate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "encode_frame_rate", + .unit = Unit::kHertz, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "encode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "time_between_freezes", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "freeze_time_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "pixels_per_frame", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "min_psnr_dB", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "decode_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "receive_to_render_time", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "dropped_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "frames_in_flight", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "rendered_frames", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kBiggerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "max_skipped", + .unit = Unit::kCount, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "target_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "actual_encode_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_video", + .name = "qp_sl0", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kVideoStreamMetadataKey, + "bob_video"}, + {MetricMetadataKey::kSenderMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kSpatialLayerMetadataKey, "0"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case", + .name = "cpu_usage_%", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = {{MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + + // Metrics from StatsBasedNetworkQualityMetricsReporter + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "bytes_discarded_no_receiver", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "packets_discarded_no_receiver", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "payload_bytes_received", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "payload_bytes_sent", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "bytes_sent", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "packets_sent", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "average_send_rate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "bytes_received", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "packets_received", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "average_receive_rate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "sent_packets_loss", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "bytes_discarded_no_receiver", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "packets_discarded_no_receiver", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "payload_bytes_received", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "payload_bytes_sent", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "bytes_sent", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "packets_sent", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "average_send_rate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "bytes_received", + .unit = Unit::kBytes, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "packets_received", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "average_receive_rate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "sent_packets_loss", + .unit = Unit::kUnitless, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + + // Metrics from VideoQualityMetricsReporter + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "available_send_bandwidth", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "transmission_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice", + .name = "retransmission_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "available_send_bandwidth", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "transmission_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob", + .name = "retransmission_bitrate", + .unit = Unit::kKilobitsPerSecond, + .improvement_direction = ImprovementDirection::kNeitherIsBetter, + .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + + // Metrics from CrossMediaMetricsReporter + MetricValidationInfo{ + .test_case = "test_case/alice_alice_audio", + .name = "audio_ahead_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = + {{MetricMetadataKey::kAudioStreamMetadataKey, "alice_audio"}, + {MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kPeerSyncGroupMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/alice_alice_video", + .name = "video_ahead_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = + {{MetricMetadataKey::kAudioStreamMetadataKey, "alice_video"}, + {MetricMetadataKey::kPeerMetadataKey, "bob"}, + {MetricMetadataKey::kPeerSyncGroupMetadataKey, "alice"}, + {MetricMetadataKey::kReceiverMetadataKey, "bob"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_bob_audio", + .name = "audio_ahead_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = + {{MetricMetadataKey::kAudioStreamMetadataKey, "bob_audio"}, + {MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kPeerSyncGroupMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}}, + MetricValidationInfo{ + .test_case = "test_case/bob_bob_video", + .name = "video_ahead_ms", + .unit = Unit::kMilliseconds, + .improvement_direction = ImprovementDirection::kSmallerIsBetter, + .metadata = { + {MetricMetadataKey::kAudioStreamMetadataKey, "bob_video"}, + {MetricMetadataKey::kPeerMetadataKey, "alice"}, + {MetricMetadataKey::kPeerSyncGroupMetadataKey, "bob"}, + {MetricMetadataKey::kReceiverMetadataKey, "alice"}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, + "test_case"}}})); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_test.cc b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_test.cc new file mode 100644 index 0000000000..066fe7d8ee --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_test.cc @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/peer_connection_quality_test.h" + +#include <map> +#include <memory> +#include <string> +#include <utility> + +#include "api/test/create_network_emulation_manager.h" +#include "api/test/metrics/global_metrics_logger_and_exporter.h" +#include "api/test/network_emulation_manager.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/pclf/media_quality_test_params.h" +#include "api/test/pclf/peer_configurer.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/units/time_delta.h" +#include "rtc_base/time_utils.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" +#include "test/testsupport/frame_reader.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::testing::Eq; +using ::testing::Test; + +using ::webrtc::webrtc_pc_e2e::PeerConfigurer; + +// Remove files and directories in a directory non-recursively. +void CleanDir(absl::string_view dir, size_t expected_output_files_count) { + absl::optional<std::vector<std::string>> dir_content = + test::ReadDirectory(dir); + if (expected_output_files_count == 0) { + ASSERT_FALSE(dir_content.has_value()) << "Empty directory is expected"; + } else { + ASSERT_TRUE(dir_content.has_value()) << "Test directory is empty!"; + EXPECT_EQ(dir_content->size(), expected_output_files_count); + for (const auto& entry : *dir_content) { + if (test::DirExists(entry)) { + EXPECT_TRUE(test::RemoveDir(entry)) + << "Failed to remove sub directory: " << entry; + } else if (test::FileExists(entry)) { + EXPECT_TRUE(test::RemoveFile(entry)) + << "Failed to remove file: " << entry; + } else { + FAIL() << "Can't remove unknown file type: " << entry; + } + } + } + EXPECT_TRUE(test::RemoveDir(dir)) << "Failed to remove directory: " << dir; +} + +class PeerConnectionE2EQualityTestTest : public Test { + protected: + ~PeerConnectionE2EQualityTestTest() override = default; + + void SetUp() override { + // Create an empty temporary directory for this test. + test_directory_ = test::JoinFilename( + test::OutputPath(), + "TestDir_PeerConnectionE2EQualityTestTest_" + + std::string( + testing::UnitTest::GetInstance()->current_test_info()->name())); + test::CreateDir(test_directory_); + } + + void TearDown() override { + CleanDir(test_directory_, expected_output_files_count_); + } + + void ExpectOutputFilesCount(size_t count) { + expected_output_files_count_ = count; + } + + std::string test_directory_; + size_t expected_output_files_count_ = 0; +}; + +TEST_F(PeerConnectionE2EQualityTestTest, OutputVideoIsDumpedWhenRequested) { + std::unique_ptr<NetworkEmulationManager> network_emulation = + CreateNetworkEmulationManager(TimeMode::kSimulated); + PeerConnectionE2EQualityTest fixture( + "test_case", *network_emulation->time_controller(), + /*audio_quality_analyzer=*/nullptr, /*video_quality_analyzer=*/nullptr, + test::GetGlobalMetricsLogger()); + + EmulatedEndpoint* alice_endpoint = + network_emulation->CreateEndpoint(EmulatedEndpointConfig()); + EmulatedEndpoint* bob_endpoint = + network_emulation->CreateEndpoint(EmulatedEndpointConfig()); + + network_emulation->CreateRoute( + alice_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()}, + bob_endpoint); + network_emulation->CreateRoute( + bob_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()}, + alice_endpoint); + + EmulatedNetworkManagerInterface* alice_network = + network_emulation->CreateEmulatedNetworkManagerInterface( + {alice_endpoint}); + EmulatedNetworkManagerInterface* bob_network = + network_emulation->CreateEmulatedNetworkManagerInterface({bob_endpoint}); + + VideoConfig alice_video("alice_video", 320, 180, 15); + alice_video.output_dump_options = VideoDumpOptions(test_directory_); + PeerConfigurer alice(alice_network->network_dependencies()); + alice.SetName("alice"); + alice.AddVideoConfig(std::move(alice_video)); + fixture.AddPeer(std::make_unique<PeerConfigurer>(std::move(alice))); + + PeerConfigurer bob(bob_network->network_dependencies()); + bob.SetName("bob"); + fixture.AddPeer(std::make_unique<PeerConfigurer>(std::move(bob))); + + fixture.Run(RunParams(TimeDelta::Seconds(2))); + + auto frame_reader = test::CreateY4mFrameReader( + test::JoinFilename(test_directory_, "alice_video_bob_320x180_15.y4m")); + EXPECT_THAT(frame_reader->num_frames(), Eq(31)); // 2 seconds 15 fps + 1 + + ExpectOutputFilesCount(1); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/peer_params_preprocessor.cc b/third_party/libwebrtc/test/pc/e2e/peer_params_preprocessor.cc new file mode 100644 index 0000000000..05372125d2 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/peer_params_preprocessor.cc @@ -0,0 +1,217 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/peer_params_preprocessor.h" + +#include <set> +#include <string> + +#include "absl/strings/string_view.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/pclf/media_quality_test_params.h" +#include "api/test/pclf/peer_configurer.h" +#include "api/test/peer_network_dependencies.h" +#include "modules/video_coding/svc/create_scalability_structure.h" +#include "modules/video_coding/svc/scalability_mode_util.h" +#include "rtc_base/arraysize.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +// List of default names of generic participants according to +// https://en.wikipedia.org/wiki/Alice_and_Bob +constexpr absl::string_view kDefaultNames[] = {"alice", "bob", "charlie", + "david", "erin", "frank"}; + +} // namespace + +class PeerParamsPreprocessor::DefaultNamesProvider { + public: + // Caller have to ensure that default names array will outlive names provider + // instance. + explicit DefaultNamesProvider( + absl::string_view prefix, + rtc::ArrayView<const absl::string_view> default_names = {}) + : prefix_(prefix), default_names_(default_names) {} + + void MaybeSetName(absl::optional<std::string>& name) { + if (name.has_value()) { + known_names_.insert(name.value()); + } else { + name = GenerateName(); + } + } + + private: + std::string GenerateName() { + std::string name; + do { + name = GenerateNameInternal(); + } while (!known_names_.insert(name).second); + return name; + } + + std::string GenerateNameInternal() { + if (counter_ < default_names_.size()) { + return std::string(default_names_[counter_++]); + } + return prefix_ + std::to_string(counter_++); + } + + const std::string prefix_; + const rtc::ArrayView<const absl::string_view> default_names_; + + std::set<std::string> known_names_; + size_t counter_ = 0; +}; + +PeerParamsPreprocessor::PeerParamsPreprocessor() + : peer_names_provider_( + std::make_unique<DefaultNamesProvider>("peer_", kDefaultNames)) {} +PeerParamsPreprocessor::~PeerParamsPreprocessor() = default; + +void PeerParamsPreprocessor::SetDefaultValuesForMissingParams( + PeerConfigurer& peer) { + Params* params = peer.params(); + ConfigurableParams* configurable_params = peer.configurable_params(); + peer_names_provider_->MaybeSetName(params->name); + DefaultNamesProvider video_stream_names_provider(*params->name + + "_auto_video_stream_label_"); + for (VideoConfig& config : configurable_params->video_configs) { + video_stream_names_provider.MaybeSetName(config.stream_label); + } + if (params->audio_config) { + DefaultNamesProvider audio_stream_names_provider( + *params->name + "_auto_audio_stream_label_"); + audio_stream_names_provider.MaybeSetName( + params->audio_config->stream_label); + } + + if (params->video_codecs.empty()) { + params->video_codecs.push_back(VideoCodecConfig(cricket::kVp8CodecName)); + } +} + +void PeerParamsPreprocessor::ValidateParams(const PeerConfigurer& peer) { + const Params& p = peer.params(); + RTC_CHECK_GT(p.video_encoder_bitrate_multiplier, 0.0); + // Each peer should at least support 1 video codec. + RTC_CHECK_GE(p.video_codecs.size(), 1); + + { + RTC_CHECK(p.name); + bool inserted = peer_names_.insert(p.name.value()).second; + RTC_CHECK(inserted) << "Duplicate name=" << p.name.value(); + } + + // Validate that all video stream labels are unique and sync groups are + // valid. + for (const VideoConfig& video_config : + peer.configurable_params().video_configs) { + RTC_CHECK(video_config.stream_label); + bool inserted = + video_labels_.insert(video_config.stream_label.value()).second; + RTC_CHECK(inserted) << "Duplicate video_config.stream_label=" + << video_config.stream_label.value(); + + // TODO(bugs.webrtc.org/4762): remove this check after synchronization of + // more than two streams is supported. + if (video_config.sync_group.has_value()) { + bool sync_group_inserted = + video_sync_groups_.insert(video_config.sync_group.value()).second; + RTC_CHECK(sync_group_inserted) + << "Sync group shouldn't consist of more than two streams (one " + "video and one audio). Duplicate video_config.sync_group=" + << video_config.sync_group.value(); + } + + if (video_config.simulcast_config) { + if (!video_config.encoding_params.empty()) { + RTC_CHECK_EQ(video_config.simulcast_config->simulcast_streams_count, + video_config.encoding_params.size()) + << "|encoding_params| have to be specified for each simulcast " + << "stream in |video_config|."; + } + } else { + RTC_CHECK_LE(video_config.encoding_params.size(), 1) + << "|encoding_params| has multiple values but simulcast is not " + "enabled."; + } + + if (video_config.emulated_sfu_config) { + if (video_config.simulcast_config && + video_config.emulated_sfu_config->target_layer_index) { + RTC_CHECK_LT(*video_config.emulated_sfu_config->target_layer_index, + video_config.simulcast_config->simulcast_streams_count); + } + if (!video_config.encoding_params.empty()) { + bool is_svc = false; + for (const auto& encoding_param : video_config.encoding_params) { + if (!encoding_param.scalability_mode) + continue; + + absl::optional<ScalabilityMode> scalability_mode = + ScalabilityModeFromString(*encoding_param.scalability_mode); + RTC_CHECK(scalability_mode) << "Unknown scalability_mode requested"; + + absl::optional<ScalableVideoController::StreamLayersConfig> + stream_layers_config = + ScalabilityStructureConfig(*scalability_mode); + is_svc |= stream_layers_config->num_spatial_layers > 1; + RTC_CHECK(stream_layers_config->num_spatial_layers == 1 || + video_config.encoding_params.size() == 1) + << "Can't enable SVC modes with multiple spatial layers (" + << stream_layers_config->num_spatial_layers + << " layers) or simulcast (" + << video_config.encoding_params.size() << " layers)"; + if (video_config.emulated_sfu_config->target_layer_index) { + RTC_CHECK_LT(*video_config.emulated_sfu_config->target_layer_index, + stream_layers_config->num_spatial_layers); + } + } + if (!is_svc && video_config.emulated_sfu_config->target_layer_index) { + RTC_CHECK_LT(*video_config.emulated_sfu_config->target_layer_index, + video_config.encoding_params.size()); + } + } + } + } + if (p.audio_config) { + bool inserted = + audio_labels_.insert(p.audio_config->stream_label.value()).second; + RTC_CHECK(inserted) << "Duplicate audio_config.stream_label=" + << p.audio_config->stream_label.value(); + // TODO(bugs.webrtc.org/4762): remove this check after synchronization of + // more than two streams is supported. + if (p.audio_config->sync_group.has_value()) { + bool sync_group_inserted = + audio_sync_groups_.insert(p.audio_config->sync_group.value()).second; + RTC_CHECK(sync_group_inserted) + << "Sync group shouldn't consist of more than two streams (one " + "video and one audio). Duplicate audio_config.sync_group=" + << p.audio_config->sync_group.value(); + } + // Check that if mode input file name specified only if mode is kFile. + if (p.audio_config.value().mode == AudioConfig::Mode::kGenerated) { + RTC_CHECK(!p.audio_config.value().input_file_name); + } + if (p.audio_config.value().mode == AudioConfig::Mode::kFile) { + RTC_CHECK(p.audio_config.value().input_file_name); + RTC_CHECK( + test::FileExists(p.audio_config.value().input_file_name.value())) + << p.audio_config.value().input_file_name.value() << " doesn't exist"; + } + } +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/peer_params_preprocessor.h b/third_party/libwebrtc/test/pc/e2e/peer_params_preprocessor.h new file mode 100644 index 0000000000..c222811546 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/peer_params_preprocessor.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef TEST_PC_E2E_PEER_PARAMS_PREPROCESSOR_H_ +#define TEST_PC_E2E_PEER_PARAMS_PREPROCESSOR_H_ + +#include <memory> +#include <set> +#include <string> + +#include "api/test/pclf/peer_configurer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class PeerParamsPreprocessor { + public: + PeerParamsPreprocessor(); + ~PeerParamsPreprocessor(); + + // Set missing params to default values if it is required: + // * Generate video stream labels if some of them are missing + // * Generate audio stream labels if some of them are missing + // * Set video source generation mode if it is not specified + // * Video codecs under test + void SetDefaultValuesForMissingParams(PeerConfigurer& peer); + + // Validate peer's parameters, also ensure uniqueness of all video stream + // labels. + void ValidateParams(const PeerConfigurer& peer); + + private: + class DefaultNamesProvider; + std::unique_ptr<DefaultNamesProvider> peer_names_provider_; + + std::set<std::string> peer_names_; + std::set<std::string> video_labels_; + std::set<std::string> audio_labels_; + std::set<std::string> video_sync_groups_; + std::set<std::string> audio_sync_groups_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_PEER_PARAMS_PREPROCESSOR_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.cc b/third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.cc new file mode 100644 index 0000000000..af55f29175 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.cc @@ -0,0 +1,601 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/sdp/sdp_changer.h" + +#include <utility> + +#include "absl/memory/memory.h" +#include "api/jsep_session_description.h" +#include "api/test/pclf/media_configuration.h" +#include "media/base/media_constants.h" +#include "p2p/base/p2p_constants.h" +#include "pc/sdp_utils.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +std::string CodecRequiredParamsToString( + const std::map<std::string, std::string>& codec_required_params) { + rtc::StringBuilder out; + for (const auto& entry : codec_required_params) { + out << entry.first << "=" << entry.second << ";"; + } + return out.str(); +} + +std::string SupportedCodecsToString( + rtc::ArrayView<const RtpCodecCapability> supported_codecs) { + rtc::StringBuilder out; + for (const auto& codec : supported_codecs) { + out << codec.name; + if (!codec.parameters.empty()) { + out << "("; + for (const auto& param : codec.parameters) { + out << param.first << "=" << param.second << ";"; + } + out << ")"; + } + out << "; "; + } + return out.str(); +} + +} // namespace + +std::vector<RtpCodecCapability> FilterVideoCodecCapabilities( + rtc::ArrayView<const VideoCodecConfig> video_codecs, + bool use_rtx, + bool use_ulpfec, + bool use_flexfec, + rtc::ArrayView<const RtpCodecCapability> supported_codecs) { + std::vector<RtpCodecCapability> output_codecs; + // Find requested codecs among supported and add them to output in the order + // they were requested. + for (auto& codec_request : video_codecs) { + size_t size_before = output_codecs.size(); + for (auto& codec : supported_codecs) { + if (codec.name != codec_request.name) { + continue; + } + bool parameters_matched = true; + for (const auto& item : codec_request.required_params) { + auto it = codec.parameters.find(item.first); + if (it == codec.parameters.end()) { + parameters_matched = false; + break; + } + if (item.second != it->second) { + parameters_matched = false; + break; + } + } + if (parameters_matched) { + output_codecs.push_back(codec); + } + } + RTC_CHECK_GT(output_codecs.size(), size_before) + << "Codec with name=" << codec_request.name << " and params {" + << CodecRequiredParamsToString(codec_request.required_params) + << "} is unsupported for this peer connection. Supported codecs are: " + << SupportedCodecsToString(supported_codecs); + } + + // Add required FEC and RTX codecs to output. + for (auto& codec : supported_codecs) { + if (codec.name == cricket::kRtxCodecName && use_rtx) { + output_codecs.push_back(codec); + } else if (codec.name == cricket::kFlexfecCodecName && use_flexfec) { + output_codecs.push_back(codec); + } else if ((codec.name == cricket::kRedCodecName || + codec.name == cricket::kUlpfecCodecName) && + use_ulpfec) { + // Red and ulpfec should be enabled or disabled together. + output_codecs.push_back(codec); + } + } + return output_codecs; +} + +// If offer has no simulcast video sections - do nothing. +// +// If offer has simulcast video sections - for each section creates +// SimulcastSectionInfo and put it into `context_`. +void SignalingInterceptor::FillSimulcastContext( + SessionDescriptionInterface* offer) { + for (auto& content : offer->description()->contents()) { + cricket::MediaContentDescription* media_desc = content.media_description(); + if (media_desc->type() != cricket::MediaType::MEDIA_TYPE_VIDEO) { + continue; + } + if (media_desc->HasSimulcast()) { + // We support only single stream simulcast sections with rids. + RTC_CHECK_EQ(media_desc->mutable_streams().size(), 1); + RTC_CHECK(media_desc->mutable_streams()[0].has_rids()); + + // Create SimulcastSectionInfo for this video section. + SimulcastSectionInfo info(content.mid(), content.type, + media_desc->mutable_streams()[0].rids()); + + // Set new rids basing on created SimulcastSectionInfo. + std::vector<cricket::RidDescription> rids; + cricket::SimulcastDescription simulcast_description; + for (std::string& rid : info.rids) { + rids.emplace_back(rid, cricket::RidDirection::kSend); + simulcast_description.send_layers().AddLayer( + cricket::SimulcastLayer(rid, false)); + } + media_desc->mutable_streams()[0].set_rids(rids); + media_desc->set_simulcast_description(simulcast_description); + + info.simulcast_description = media_desc->simulcast_description(); + for (const auto& extension : media_desc->rtp_header_extensions()) { + if (extension.uri == RtpExtension::kMidUri) { + info.mid_extension = extension; + } else if (extension.uri == RtpExtension::kRidUri) { + info.rid_extension = extension; + } else if (extension.uri == RtpExtension::kRepairedRidUri) { + info.rrid_extension = extension; + } + } + RTC_CHECK_NE(info.rid_extension.id, 0); + RTC_CHECK_NE(info.mid_extension.id, 0); + bool transport_description_found = false; + for (auto& transport_info : offer->description()->transport_infos()) { + if (transport_info.content_name == info.mid) { + info.transport_description = transport_info.description; + transport_description_found = true; + break; + } + } + RTC_CHECK(transport_description_found); + + context_.AddSimulcastInfo(info); + } + } +} + +LocalAndRemoteSdp SignalingInterceptor::PatchOffer( + std::unique_ptr<SessionDescriptionInterface> offer, + const VideoCodecConfig& first_codec) { + for (auto& content : offer->description()->contents()) { + context_.mids_order.push_back(content.mid()); + cricket::MediaContentDescription* media_desc = content.media_description(); + if (media_desc->type() != cricket::MediaType::MEDIA_TYPE_VIDEO) { + continue; + } + if (content.media_description()->streams().empty()) { + // It means that this media section describes receive only media section + // in SDP. + RTC_CHECK_EQ(content.media_description()->direction(), + RtpTransceiverDirection::kRecvOnly); + continue; + } + media_desc->set_conference_mode(params_.use_conference_mode); + } + + if (!params_.stream_label_to_simulcast_streams_count.empty()) { + // Because simulcast enabled `params_.video_codecs` has only 1 element. + if (first_codec.name == cricket::kVp8CodecName) { + return PatchVp8Offer(std::move(offer)); + } + + if (first_codec.name == cricket::kVp9CodecName) { + return PatchVp9Offer(std::move(offer)); + } + } + + auto offer_for_remote = CloneSessionDescription(offer.get()); + return LocalAndRemoteSdp(std::move(offer), std::move(offer_for_remote)); +} + +LocalAndRemoteSdp SignalingInterceptor::PatchVp8Offer( + std::unique_ptr<SessionDescriptionInterface> offer) { + FillSimulcastContext(offer.get()); + if (!context_.HasSimulcast()) { + auto offer_for_remote = CloneSessionDescription(offer.get()); + return LocalAndRemoteSdp(std::move(offer), std::move(offer_for_remote)); + } + + // Clone original offer description. We mustn't access original offer after + // this point. + std::unique_ptr<cricket::SessionDescription> desc = + offer->description()->Clone(); + + for (auto& info : context_.simulcast_infos) { + // For each simulcast section we have to perform: + // 1. Swap MID and RID header extensions + // 2. Remove RIDs from streams and remove SimulcastDescription + // 3. For each RID duplicate media section + cricket::ContentInfo* simulcast_content = desc->GetContentByName(info.mid); + + // Now we need to prepare common prototype for "m=video" sections, in which + // single simulcast section will be converted. Do it before removing content + // because otherwise description will be deleted. + std::unique_ptr<cricket::MediaContentDescription> prototype_media_desc = + simulcast_content->media_description()->Clone(); + + // Remove simulcast video section from offer. + RTC_CHECK(desc->RemoveContentByName(simulcast_content->mid())); + // Clear `simulcast_content`, because now it is pointing to removed object. + simulcast_content = nullptr; + + // Swap mid and rid extensions, so remote peer will understand rid as mid. + // Also remove rid extension. + std::vector<webrtc::RtpExtension> extensions = + prototype_media_desc->rtp_header_extensions(); + for (auto ext_it = extensions.begin(); ext_it != extensions.end();) { + if (ext_it->uri == RtpExtension::kRidUri) { + // We don't need rid extension for remote peer. + ext_it = extensions.erase(ext_it); + continue; + } + if (ext_it->uri == RtpExtension::kRepairedRidUri) { + // We don't support RTX in simulcast. + ext_it = extensions.erase(ext_it); + continue; + } + if (ext_it->uri == RtpExtension::kMidUri) { + ext_it->id = info.rid_extension.id; + } + ++ext_it; + } + + prototype_media_desc->ClearRtpHeaderExtensions(); + prototype_media_desc->set_rtp_header_extensions(extensions); + + // We support only single stream inside video section with simulcast + RTC_CHECK_EQ(prototype_media_desc->mutable_streams().size(), 1); + // This stream must have rids. + RTC_CHECK(prototype_media_desc->mutable_streams()[0].has_rids()); + + // Remove rids and simulcast description from media description. + prototype_media_desc->mutable_streams()[0].set_rids({}); + prototype_media_desc->set_simulcast_description( + cricket::SimulcastDescription()); + + // For each rid add separate video section. + for (std::string& rid : info.rids) { + desc->AddContent(rid, info.media_protocol_type, + prototype_media_desc->Clone()); + } + } + + // Now we need to add bundle line to have all media bundled together. + cricket::ContentGroup bundle_group(cricket::GROUP_TYPE_BUNDLE); + for (auto& content : desc->contents()) { + bundle_group.AddContentName(content.mid()); + } + if (desc->HasGroup(cricket::GROUP_TYPE_BUNDLE)) { + desc->RemoveGroupByName(cricket::GROUP_TYPE_BUNDLE); + } + desc->AddGroup(bundle_group); + + // Update transport_infos to add TransportInfo for each new media section. + std::vector<cricket::TransportInfo> transport_infos = desc->transport_infos(); + transport_infos.erase(std::remove_if( + transport_infos.begin(), transport_infos.end(), + [this](const cricket::TransportInfo& ti) { + // Remove transport infos that correspond to simulcast video sections. + return context_.simulcast_infos_by_mid.find(ti.content_name) != + context_.simulcast_infos_by_mid.end(); + })); + for (auto& info : context_.simulcast_infos) { + for (auto& rid : info.rids) { + transport_infos.emplace_back(rid, info.transport_description); + } + } + desc->set_transport_infos(transport_infos); + + // Create patched offer. + auto patched_offer = + std::make_unique<JsepSessionDescription>(SdpType::kOffer); + patched_offer->Initialize(std::move(desc), offer->session_id(), + offer->session_version()); + return LocalAndRemoteSdp(std::move(offer), std::move(patched_offer)); +} + +LocalAndRemoteSdp SignalingInterceptor::PatchVp9Offer( + std::unique_ptr<SessionDescriptionInterface> offer) { + rtc::UniqueRandomIdGenerator ssrcs_generator; + for (auto& content : offer->description()->contents()) { + for (auto& stream : content.media_description()->streams()) { + for (auto& ssrc : stream.ssrcs) { + ssrcs_generator.AddKnownId(ssrc); + } + } + } + + for (auto& content : offer->description()->contents()) { + if (content.media_description()->type() != + cricket::MediaType::MEDIA_TYPE_VIDEO) { + // We are interested in only video tracks + continue; + } + if (content.media_description()->direction() == + RtpTransceiverDirection::kRecvOnly) { + // If direction is receive only, then there is no media in this track from + // sender side, so we needn't to do anything with this track. + continue; + } + RTC_CHECK_EQ(content.media_description()->streams().size(), 1); + cricket::StreamParams& stream = + content.media_description()->mutable_streams()[0]; + RTC_CHECK_EQ(stream.stream_ids().size(), 2) + << "Expected 2 stream ids in video stream: 1st - sync_group, 2nd - " + "unique label"; + std::string stream_label = stream.stream_ids()[1]; + + auto it = + params_.stream_label_to_simulcast_streams_count.find(stream_label); + if (it == params_.stream_label_to_simulcast_streams_count.end()) { + continue; + } + int svc_layers_count = it->second; + + RTC_CHECK(stream.has_ssrc_groups()) << "Only SVC with RTX is supported"; + RTC_CHECK_EQ(stream.ssrc_groups.size(), 1) + << "Too many ssrc groups in the track"; + std::vector<uint32_t> primary_ssrcs; + stream.GetPrimarySsrcs(&primary_ssrcs); + RTC_CHECK(primary_ssrcs.size() == 1); + for (int i = 1; i < svc_layers_count; ++i) { + uint32_t ssrc = ssrcs_generator.GenerateId(); + primary_ssrcs.push_back(ssrc); + stream.add_ssrc(ssrc); + stream.AddFidSsrc(ssrc, ssrcs_generator.GenerateId()); + } + stream.ssrc_groups.push_back( + cricket::SsrcGroup(cricket::kSimSsrcGroupSemantics, primary_ssrcs)); + } + auto offer_for_remote = CloneSessionDescription(offer.get()); + return LocalAndRemoteSdp(std::move(offer), std::move(offer_for_remote)); +} + +LocalAndRemoteSdp SignalingInterceptor::PatchAnswer( + std::unique_ptr<SessionDescriptionInterface> answer, + const VideoCodecConfig& first_codec) { + for (auto& content : answer->description()->contents()) { + cricket::MediaContentDescription* media_desc = content.media_description(); + if (media_desc->type() != cricket::MediaType::MEDIA_TYPE_VIDEO) { + continue; + } + if (content.media_description()->direction() != + RtpTransceiverDirection::kRecvOnly) { + continue; + } + media_desc->set_conference_mode(params_.use_conference_mode); + } + + if (!params_.stream_label_to_simulcast_streams_count.empty()) { + // Because simulcast enabled `params_.video_codecs` has only 1 element. + if (first_codec.name == cricket::kVp8CodecName) { + return PatchVp8Answer(std::move(answer)); + } + + if (first_codec.name == cricket::kVp9CodecName) { + return PatchVp9Answer(std::move(answer)); + } + } + + auto answer_for_remote = CloneSessionDescription(answer.get()); + return LocalAndRemoteSdp(std::move(answer), std::move(answer_for_remote)); +} + +LocalAndRemoteSdp SignalingInterceptor::PatchVp8Answer( + std::unique_ptr<SessionDescriptionInterface> answer) { + if (!context_.HasSimulcast()) { + auto answer_for_remote = CloneSessionDescription(answer.get()); + return LocalAndRemoteSdp(std::move(answer), std::move(answer_for_remote)); + } + + std::unique_ptr<cricket::SessionDescription> desc = + answer->description()->Clone(); + + for (auto& info : context_.simulcast_infos) { + cricket::ContentInfo* simulcast_content = + desc->GetContentByName(info.rids[0]); + RTC_CHECK(simulcast_content); + + // Get media description, which will be converted to simulcast answer. + std::unique_ptr<cricket::MediaContentDescription> media_desc = + simulcast_content->media_description()->Clone(); + // Set `simulcast_content` to nullptr, because then it will be removed, so + // it will point to deleted object. + simulcast_content = nullptr; + + // Remove separate media sections for simulcast streams. + for (auto& rid : info.rids) { + RTC_CHECK(desc->RemoveContentByName(rid)); + } + + // Patch `media_desc` to make it simulcast answer description. + // Restore mid/rid rtp header extensions + std::vector<webrtc::RtpExtension> extensions = + media_desc->rtp_header_extensions(); + // First remove existing rid/mid header extensions. + extensions.erase(std::remove_if(extensions.begin(), extensions.end(), + [](const webrtc::RtpExtension& e) { + return e.uri == RtpExtension::kMidUri || + e.uri == RtpExtension::kRidUri || + e.uri == + RtpExtension::kRepairedRidUri; + })); + + // Then add right ones. + extensions.push_back(info.mid_extension); + extensions.push_back(info.rid_extension); + // extensions.push_back(info.rrid_extension); + media_desc->ClearRtpHeaderExtensions(); + media_desc->set_rtp_header_extensions(extensions); + + // Add StreamParams with rids for receive. + RTC_CHECK_EQ(media_desc->mutable_streams().size(), 0); + std::vector<cricket::RidDescription> rids; + for (auto& rid : info.rids) { + rids.emplace_back(rid, cricket::RidDirection::kReceive); + } + cricket::StreamParams stream_params; + stream_params.set_rids(rids); + media_desc->mutable_streams().push_back(stream_params); + + // Restore SimulcastDescription. It should correspond to one from offer, + // but it have to have receive layers instead of send. So we need to put + // send layers from offer to receive layers in answer. + cricket::SimulcastDescription simulcast_description; + for (const auto& layer : info.simulcast_description.send_layers()) { + simulcast_description.receive_layers().AddLayerWithAlternatives(layer); + } + media_desc->set_simulcast_description(simulcast_description); + + // Add simulcast media section. + desc->AddContent(info.mid, info.media_protocol_type, std::move(media_desc)); + } + + desc = RestoreMediaSectionsOrder(std::move(desc)); + + // Now we need to add bundle line to have all media bundled together. + cricket::ContentGroup bundle_group(cricket::GROUP_TYPE_BUNDLE); + for (auto& content : desc->contents()) { + bundle_group.AddContentName(content.mid()); + } + if (desc->HasGroup(cricket::GROUP_TYPE_BUNDLE)) { + desc->RemoveGroupByName(cricket::GROUP_TYPE_BUNDLE); + } + desc->AddGroup(bundle_group); + + // Fix transport_infos: it have to have single info for simulcast section. + std::vector<cricket::TransportInfo> transport_infos = desc->transport_infos(); + std::map<std::string, cricket::TransportDescription> + mid_to_transport_description; + for (auto info_it = transport_infos.begin(); + info_it != transport_infos.end();) { + auto it = context_.simulcast_infos_by_rid.find(info_it->content_name); + if (it != context_.simulcast_infos_by_rid.end()) { + // This transport info correspond to some extra added media section. + mid_to_transport_description.insert( + {it->second->mid, info_it->description}); + info_it = transport_infos.erase(info_it); + } else { + ++info_it; + } + } + for (auto& info : context_.simulcast_infos) { + transport_infos.emplace_back(info.mid, + mid_to_transport_description.at(info.mid)); + } + desc->set_transport_infos(transport_infos); + + auto patched_answer = + std::make_unique<JsepSessionDescription>(SdpType::kAnswer); + patched_answer->Initialize(std::move(desc), answer->session_id(), + answer->session_version()); + return LocalAndRemoteSdp(std::move(answer), std::move(patched_answer)); +} + +std::unique_ptr<cricket::SessionDescription> +SignalingInterceptor::RestoreMediaSectionsOrder( + std::unique_ptr<cricket::SessionDescription> source) { + std::unique_ptr<cricket::SessionDescription> out = source->Clone(); + for (auto& mid : context_.mids_order) { + RTC_CHECK(out->RemoveContentByName(mid)); + } + RTC_CHECK_EQ(out->contents().size(), 0); + for (auto& mid : context_.mids_order) { + cricket::ContentInfo* content = source->GetContentByName(mid); + RTC_CHECK(content); + out->AddContent(mid, content->type, content->media_description()->Clone()); + } + return out; +} + +LocalAndRemoteSdp SignalingInterceptor::PatchVp9Answer( + std::unique_ptr<SessionDescriptionInterface> answer) { + auto answer_for_remote = CloneSessionDescription(answer.get()); + return LocalAndRemoteSdp(std::move(answer), std::move(answer_for_remote)); +} + +std::vector<std::unique_ptr<IceCandidateInterface>> +SignalingInterceptor::PatchOffererIceCandidates( + rtc::ArrayView<const IceCandidateInterface* const> candidates) { + std::vector<std::unique_ptr<IceCandidateInterface>> out; + for (auto* candidate : candidates) { + auto simulcast_info_it = + context_.simulcast_infos_by_mid.find(candidate->sdp_mid()); + if (simulcast_info_it != context_.simulcast_infos_by_mid.end()) { + // This is candidate for simulcast section, so it should be transformed + // into candidates for replicated sections. The sdpMLineIndex is set to + // -1 and ignored if the rid is present. + for (const std::string& rid : simulcast_info_it->second->rids) { + out.push_back(CreateIceCandidate(rid, -1, candidate->candidate())); + } + } else { + out.push_back(CreateIceCandidate(candidate->sdp_mid(), + candidate->sdp_mline_index(), + candidate->candidate())); + } + } + RTC_CHECK_GT(out.size(), 0); + return out; +} + +std::vector<std::unique_ptr<IceCandidateInterface>> +SignalingInterceptor::PatchAnswererIceCandidates( + rtc::ArrayView<const IceCandidateInterface* const> candidates) { + std::vector<std::unique_ptr<IceCandidateInterface>> out; + for (auto* candidate : candidates) { + auto simulcast_info_it = + context_.simulcast_infos_by_rid.find(candidate->sdp_mid()); + if (simulcast_info_it != context_.simulcast_infos_by_rid.end()) { + // This is candidate for replicated section, created from single simulcast + // section, so it should be transformed into candidates for simulcast + // section. + out.push_back(CreateIceCandidate(simulcast_info_it->second->mid, 0, + candidate->candidate())); + } else if (!context_.simulcast_infos_by_rid.empty()) { + // When using simulcast and bundle, put everything on the first m-line. + out.push_back(CreateIceCandidate("", 0, candidate->candidate())); + } else { + out.push_back(CreateIceCandidate(candidate->sdp_mid(), + candidate->sdp_mline_index(), + candidate->candidate())); + } + } + RTC_CHECK_GT(out.size(), 0); + return out; +} + +SignalingInterceptor::SimulcastSectionInfo::SimulcastSectionInfo( + const std::string& mid, + cricket::MediaProtocolType media_protocol_type, + const std::vector<cricket::RidDescription>& rids_desc) + : mid(mid), media_protocol_type(media_protocol_type) { + for (auto& rid : rids_desc) { + rids.push_back(rid.rid); + } +} + +void SignalingInterceptor::SignalingContext::AddSimulcastInfo( + const SimulcastSectionInfo& info) { + simulcast_infos.push_back(info); + bool inserted = + simulcast_infos_by_mid.insert({info.mid, &simulcast_infos.back()}).second; + RTC_CHECK(inserted); + for (auto& rid : info.rids) { + inserted = + simulcast_infos_by_rid.insert({rid, &simulcast_infos.back()}).second; + RTC_CHECK(inserted); + } +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.h b/third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.h new file mode 100644 index 0000000000..6f68d03f52 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.h @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_SDP_SDP_CHANGER_H_ +#define TEST_PC_E2E_SDP_SDP_CHANGER_H_ + +#include <map> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/jsep.h" +#include "api/rtp_parameters.h" +#include "api/test/pclf/media_configuration.h" +#include "media/base/rid_description.h" +#include "pc/session_description.h" +#include "pc/simulcast_description.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Creates list of capabilities, which can be set on RtpTransceiverInterface via +// RtpTransceiverInterface::SetCodecPreferences(...) to negotiate use of codecs +// from list of `supported_codecs` which will match `video_codecs`. If flags +// `ulpfec` or `flexfec` set to true corresponding FEC codec will be added. +// FEC and RTX codecs will be added after required codecs. +// +// All codecs will be added only if they exists in the list of +// `supported_codecs`. If multiple codecs from this list will match +// `video_codecs`, then all of them will be added to the output +// vector and they will be added in the same order, as they were in +// `supported_codecs`. +std::vector<RtpCodecCapability> FilterVideoCodecCapabilities( + rtc::ArrayView<const VideoCodecConfig> video_codecs, + bool use_rtx, + bool use_ulpfec, + bool use_flexfec, + rtc::ArrayView<const RtpCodecCapability> supported_codecs); + +struct LocalAndRemoteSdp { + LocalAndRemoteSdp(std::unique_ptr<SessionDescriptionInterface> local_sdp, + std::unique_ptr<SessionDescriptionInterface> remote_sdp) + : local_sdp(std::move(local_sdp)), remote_sdp(std::move(remote_sdp)) {} + + // Sdp, that should be as local description on the peer, that created it. + std::unique_ptr<SessionDescriptionInterface> local_sdp; + // Sdp, that should be set as remote description on the peer opposite to the + // one, who created it. + std::unique_ptr<SessionDescriptionInterface> remote_sdp; +}; + +struct PatchingParams { + PatchingParams( + bool use_conference_mode, + std::map<std::string, int> stream_label_to_simulcast_streams_count) + : use_conference_mode(use_conference_mode), + stream_label_to_simulcast_streams_count( + stream_label_to_simulcast_streams_count) {} + + bool use_conference_mode; + std::map<std::string, int> stream_label_to_simulcast_streams_count; +}; + +class SignalingInterceptor { + public: + explicit SignalingInterceptor(PatchingParams params) : params_(params) {} + + LocalAndRemoteSdp PatchOffer( + std::unique_ptr<SessionDescriptionInterface> offer, + const VideoCodecConfig& first_codec); + LocalAndRemoteSdp PatchAnswer( + std::unique_ptr<SessionDescriptionInterface> answer, + const VideoCodecConfig& first_codec); + + std::vector<std::unique_ptr<IceCandidateInterface>> PatchOffererIceCandidates( + rtc::ArrayView<const IceCandidateInterface* const> candidates); + std::vector<std::unique_ptr<IceCandidateInterface>> + PatchAnswererIceCandidates( + rtc::ArrayView<const IceCandidateInterface* const> candidates); + + private: + // Contains information about simulcast section, that is required to perform + // modified offer/answer and ice candidates exchange. + struct SimulcastSectionInfo { + SimulcastSectionInfo(const std::string& mid, + cricket::MediaProtocolType media_protocol_type, + const std::vector<cricket::RidDescription>& rids_desc); + + const std::string mid; + const cricket::MediaProtocolType media_protocol_type; + std::vector<std::string> rids; + cricket::SimulcastDescription simulcast_description; + webrtc::RtpExtension mid_extension; + webrtc::RtpExtension rid_extension; + webrtc::RtpExtension rrid_extension; + cricket::TransportDescription transport_description; + }; + + struct SignalingContext { + SignalingContext() = default; + // SignalingContext is not copyable and movable. + SignalingContext(SignalingContext&) = delete; + SignalingContext& operator=(SignalingContext&) = delete; + SignalingContext(SignalingContext&&) = delete; + SignalingContext& operator=(SignalingContext&&) = delete; + + void AddSimulcastInfo(const SimulcastSectionInfo& info); + bool HasSimulcast() const { return !simulcast_infos.empty(); } + + std::vector<SimulcastSectionInfo> simulcast_infos; + std::map<std::string, SimulcastSectionInfo*> simulcast_infos_by_mid; + std::map<std::string, SimulcastSectionInfo*> simulcast_infos_by_rid; + + std::vector<std::string> mids_order; + }; + + LocalAndRemoteSdp PatchVp8Offer( + std::unique_ptr<SessionDescriptionInterface> offer); + LocalAndRemoteSdp PatchVp9Offer( + std::unique_ptr<SessionDescriptionInterface> offer); + LocalAndRemoteSdp PatchVp8Answer( + std::unique_ptr<SessionDescriptionInterface> answer); + LocalAndRemoteSdp PatchVp9Answer( + std::unique_ptr<SessionDescriptionInterface> answer); + + void FillSimulcastContext(SessionDescriptionInterface* offer); + std::unique_ptr<cricket::SessionDescription> RestoreMediaSectionsOrder( + std::unique_ptr<cricket::SessionDescription> source); + + PatchingParams params_; + SignalingContext context_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_SDP_SDP_CHANGER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.cc b/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.cc new file mode 100644 index 0000000000..65dca5b518 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.cc @@ -0,0 +1,592 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h" + +#include <cstdint> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <type_traits> +#include <utility> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/array_view.h" +#include "api/scoped_refptr.h" +#include "api/sequence_checker.h" +#include "api/stats/rtc_stats.h" +#include "api/stats/rtcstats_objects.h" +#include "api/test/metrics/metric.h" +#include "api/test/network_emulation/network_emulation_interfaces.h" +#include "api/test/network_emulation_manager.h" +#include "api/units/data_rate.h" +#include "api/units/timestamp.h" +#include "rtc_base/checks.h" +#include "rtc_base/event.h" +#include "rtc_base/ip_address.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/system/no_unique_address.h" +#include "system_wrappers/include/field_trial.h" +#include "test/pc/e2e/metric_metadata_keys.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Unit; + +using NetworkLayerStats = + StatsBasedNetworkQualityMetricsReporter::NetworkLayerStats; + +constexpr TimeDelta kStatsWaitTimeout = TimeDelta::Seconds(1); + +// Field trial which controls whether to report standard-compliant bytes +// sent/received per stream. If enabled, padding and headers are not included +// in bytes sent or received. +constexpr char kUseStandardBytesStats[] = "WebRTC-UseStandardBytesStats"; + +EmulatedNetworkStats PopulateStats(std::vector<EmulatedEndpoint*> endpoints, + NetworkEmulationManager* network_emulation) { + rtc::Event stats_loaded; + EmulatedNetworkStats stats; + network_emulation->GetStats(endpoints, [&](EmulatedNetworkStats s) { + stats = std::move(s); + stats_loaded.Set(); + }); + bool stats_received = stats_loaded.Wait(kStatsWaitTimeout); + RTC_CHECK(stats_received); + return stats; +} + +std::map<rtc::IPAddress, std::string> PopulateIpToPeer( + const std::map<std::string, std::vector<EmulatedEndpoint*>>& + peer_endpoints) { + std::map<rtc::IPAddress, std::string> out; + for (const auto& entry : peer_endpoints) { + for (const EmulatedEndpoint* const endpoint : entry.second) { + RTC_CHECK(out.find(endpoint->GetPeerLocalAddress()) == out.end()) + << "Two peers can't share the same endpoint"; + out.emplace(endpoint->GetPeerLocalAddress(), entry.first); + } + } + return out; +} + +// Accumulates emulated network stats being executed on the network thread. +// When all stats are collected stores it in thread safe variable. +class EmulatedNetworkStatsAccumulator { + public: + // `expected_stats_count` - the number of calls to + // AddEndpointStats/AddUplinkStats/AddDownlinkStats the accumulator is going + // to wait. If called more than expected, the program will crash. + explicit EmulatedNetworkStatsAccumulator(size_t expected_stats_count) + : not_collected_stats_count_(expected_stats_count) { + RTC_DCHECK_GE(not_collected_stats_count_, 0); + if (not_collected_stats_count_ == 0) { + all_stats_collected_.Set(); + } + sequence_checker_.Detach(); + } + + // Has to be executed on network thread. + void AddEndpointStats(std::string peer_name, EmulatedNetworkStats stats) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + n_stats_[peer_name].endpoints_stats = std::move(stats); + DecrementNotCollectedStatsCount(); + } + + // Has to be executed on network thread. + void AddUplinkStats(std::string peer_name, EmulatedNetworkNodeStats stats) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + n_stats_[peer_name].uplink_stats = std::move(stats); + DecrementNotCollectedStatsCount(); + } + + // Has to be executed on network thread. + void AddDownlinkStats(std::string peer_name, EmulatedNetworkNodeStats stats) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + n_stats_[peer_name].downlink_stats = std::move(stats); + DecrementNotCollectedStatsCount(); + } + + // Can be executed on any thread. + // Returns true if count down was completed and false if timeout elapsed + // before. + bool Wait(TimeDelta timeout) { return all_stats_collected_.Wait(timeout); } + + // Can be called once. Returns all collected stats by moving underlying + // object. + std::map<std::string, NetworkLayerStats> ReleaseStats() { + RTC_DCHECK(!stats_released_); + stats_released_ = true; + MutexLock lock(&mutex_); + return std::move(stats_); + } + + private: + void DecrementNotCollectedStatsCount() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_CHECK_GT(not_collected_stats_count_, 0) + << "All stats are already collected"; + not_collected_stats_count_--; + if (not_collected_stats_count_ == 0) { + MutexLock lock(&mutex_); + stats_ = std::move(n_stats_); + all_stats_collected_.Set(); + } + } + + RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; + size_t not_collected_stats_count_ RTC_GUARDED_BY(sequence_checker_); + // Collected on the network thread. Moved into `stats_` after all stats are + // collected. + std::map<std::string, NetworkLayerStats> n_stats_ + RTC_GUARDED_BY(sequence_checker_); + + rtc::Event all_stats_collected_; + Mutex mutex_; + std::map<std::string, NetworkLayerStats> stats_ RTC_GUARDED_BY(mutex_); + bool stats_released_ = false; +}; + +} // namespace + +StatsBasedNetworkQualityMetricsReporter:: + StatsBasedNetworkQualityMetricsReporter( + std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints, + NetworkEmulationManager* network_emulation, + test::MetricsLogger* metrics_logger) + : collector_(std::move(peer_endpoints), network_emulation), + clock_(network_emulation->time_controller()->GetClock()), + metrics_logger_(metrics_logger) { + RTC_CHECK(metrics_logger_); +} + +StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector:: + NetworkLayerStatsCollector( + std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints, + NetworkEmulationManager* network_emulation) + : peer_endpoints_(std::move(peer_endpoints)), + ip_to_peer_(PopulateIpToPeer(peer_endpoints_)), + network_emulation_(network_emulation) {} + +void StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector:: + Start() { + MutexLock lock(&mutex_); + // Check that network stats are clean before test execution. + for (const auto& entry : peer_endpoints_) { + EmulatedNetworkStats stats = + PopulateStats(entry.second, network_emulation_); + RTC_CHECK_EQ(stats.overall_outgoing_stats.packets_sent, 0); + RTC_CHECK_EQ(stats.overall_incoming_stats.packets_received, 0); + } +} + +void StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector:: + AddPeer(absl::string_view peer_name, + std::vector<EmulatedEndpoint*> endpoints, + std::vector<EmulatedNetworkNode*> uplink, + std::vector<EmulatedNetworkNode*> downlink) { + MutexLock lock(&mutex_); + // When new peer is added not in the constructor, don't check if it has empty + // stats, because their endpoint could be used for traffic before. + peer_endpoints_.emplace(peer_name, std::move(endpoints)); + peer_uplinks_.emplace(peer_name, std::move(uplink)); + peer_downlinks_.emplace(peer_name, std::move(downlink)); + for (const EmulatedEndpoint* const endpoint : endpoints) { + RTC_CHECK(ip_to_peer_.find(endpoint->GetPeerLocalAddress()) == + ip_to_peer_.end()) + << "Two peers can't share the same endpoint"; + ip_to_peer_.emplace(endpoint->GetPeerLocalAddress(), peer_name); + } +} + +std::map<std::string, NetworkLayerStats> +StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector:: + GetStats() { + MutexLock lock(&mutex_); + EmulatedNetworkStatsAccumulator stats_accumulator( + peer_endpoints_.size() + peer_uplinks_.size() + peer_downlinks_.size()); + for (const auto& entry : peer_endpoints_) { + network_emulation_->GetStats( + entry.second, [&stats_accumulator, + peer = entry.first](EmulatedNetworkStats s) mutable { + stats_accumulator.AddEndpointStats(std::move(peer), std::move(s)); + }); + } + for (const auto& entry : peer_uplinks_) { + network_emulation_->GetStats( + entry.second, [&stats_accumulator, + peer = entry.first](EmulatedNetworkNodeStats s) mutable { + stats_accumulator.AddUplinkStats(std::move(peer), std::move(s)); + }); + } + for (const auto& entry : peer_downlinks_) { + network_emulation_->GetStats( + entry.second, [&stats_accumulator, + peer = entry.first](EmulatedNetworkNodeStats s) mutable { + stats_accumulator.AddDownlinkStats(std::move(peer), std::move(s)); + }); + } + bool stats_collected = stats_accumulator.Wait(kStatsWaitTimeout); + RTC_CHECK(stats_collected); + std::map<std::string, NetworkLayerStats> peer_to_stats = + stats_accumulator.ReleaseStats(); + std::map<std::string, std::vector<std::string>> sender_to_receivers; + for (const auto& entry : peer_endpoints_) { + const std::string& peer_name = entry.first; + const NetworkLayerStats& stats = peer_to_stats[peer_name]; + for (const auto& income_stats_entry : + stats.endpoints_stats.incoming_stats_per_source) { + const rtc::IPAddress& source_ip = income_stats_entry.first; + auto it = ip_to_peer_.find(source_ip); + if (it == ip_to_peer_.end()) { + // Source IP is unknown for this collector, so will be skipped. + continue; + } + sender_to_receivers[it->second].push_back(peer_name); + } + } + for (auto& entry : peer_to_stats) { + const std::vector<std::string>& receivers = + sender_to_receivers[entry.first]; + entry.second.receivers = + std::set<std::string>(receivers.begin(), receivers.end()); + } + return peer_to_stats; +} + +void StatsBasedNetworkQualityMetricsReporter::AddPeer( + absl::string_view peer_name, + std::vector<EmulatedEndpoint*> endpoints) { + collector_.AddPeer(peer_name, std::move(endpoints), /*uplink=*/{}, + /*downlink=*/{}); +} + +void StatsBasedNetworkQualityMetricsReporter::AddPeer( + absl::string_view peer_name, + std::vector<EmulatedEndpoint*> endpoints, + std::vector<EmulatedNetworkNode*> uplink, + std::vector<EmulatedNetworkNode*> downlink) { + collector_.AddPeer(peer_name, std::move(endpoints), std::move(uplink), + std::move(downlink)); +} + +void StatsBasedNetworkQualityMetricsReporter::Start( + absl::string_view test_case_name, + const TrackIdStreamInfoMap* reporter_helper) { + test_case_name_ = std::string(test_case_name); + collector_.Start(); + start_time_ = clock_->CurrentTime(); +} + +void StatsBasedNetworkQualityMetricsReporter::OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) { + PCStats cur_stats; + + auto inbound_stats = report->GetStatsOfType<RTCInboundRTPStreamStats>(); + for (const auto& stat : inbound_stats) { + cur_stats.payload_received += + DataSize::Bytes(stat->bytes_received.ValueOrDefault(0ul) + + stat->header_bytes_received.ValueOrDefault(0ul)); + } + + auto outbound_stats = report->GetStatsOfType<RTCOutboundRTPStreamStats>(); + for (const auto& stat : outbound_stats) { + cur_stats.payload_sent += + DataSize::Bytes(stat->bytes_sent.ValueOrDefault(0ul) + + stat->header_bytes_sent.ValueOrDefault(0ul)); + } + + auto candidate_pairs_stats = report->GetStatsOfType<RTCTransportStats>(); + for (const auto& stat : candidate_pairs_stats) { + cur_stats.total_received += + DataSize::Bytes(stat->bytes_received.ValueOrDefault(0ul)); + cur_stats.total_sent += + DataSize::Bytes(stat->bytes_sent.ValueOrDefault(0ul)); + cur_stats.packets_received += stat->packets_received.ValueOrDefault(0ul); + cur_stats.packets_sent += stat->packets_sent.ValueOrDefault(0ul); + } + + MutexLock lock(&mutex_); + pc_stats_[std::string(pc_label)] = cur_stats; +} + +void StatsBasedNetworkQualityMetricsReporter::StopAndReportResults() { + Timestamp end_time = clock_->CurrentTime(); + + if (!webrtc::field_trial::IsEnabled(kUseStandardBytesStats)) { + RTC_LOG(LS_ERROR) + << "Non-standard GetStats; \"payload\" counts include RTP headers"; + } + + std::map<std::string, NetworkLayerStats> stats = collector_.GetStats(); + for (const auto& entry : stats) { + LogNetworkLayerStats(entry.first, entry.second); + } + MutexLock lock(&mutex_); + for (const auto& pair : pc_stats_) { + auto it = stats.find(pair.first); + RTC_CHECK(it != stats.end()) + << "Peer name used for PeerConnection stats collection and peer name " + "used for endpoints naming doesn't match. No endpoints found for " + "peer " + << pair.first; + const NetworkLayerStats& network_layer_stats = it->second; + int64_t total_packets_received = 0; + bool found = false; + for (const auto& dest_peer : network_layer_stats.receivers) { + auto pc_stats_it = pc_stats_.find(dest_peer); + if (pc_stats_it == pc_stats_.end()) { + continue; + } + found = true; + total_packets_received += pc_stats_it->second.packets_received; + } + int64_t packet_loss = -1; + if (found) { + packet_loss = pair.second.packets_sent - total_packets_received; + } + ReportStats(pair.first, pair.second, network_layer_stats, packet_loss, + end_time); + } +} + +void StatsBasedNetworkQualityMetricsReporter::ReportStats( + const std::string& pc_label, + const PCStats& pc_stats, + const NetworkLayerStats& network_layer_stats, + int64_t packet_loss, + const Timestamp& end_time) { + // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey. + std::map<std::string, std::string> metric_metadata{ + {MetricMetadataKey::kPeerMetadataKey, pc_label}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_case_name_}}; + metrics_logger_->LogSingleValueMetric( + "bytes_discarded_no_receiver", GetTestCaseName(pc_label), + network_layer_stats.endpoints_stats.overall_incoming_stats + .bytes_discarded_no_receiver.bytes(), + Unit::kBytes, ImprovementDirection::kNeitherIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "packets_discarded_no_receiver", GetTestCaseName(pc_label), + network_layer_stats.endpoints_stats.overall_incoming_stats + .packets_discarded_no_receiver, + Unit::kUnitless, ImprovementDirection::kNeitherIsBetter, metric_metadata); + + metrics_logger_->LogSingleValueMetric( + "payload_bytes_received", GetTestCaseName(pc_label), + pc_stats.payload_received.bytes(), Unit::kBytes, + ImprovementDirection::kNeitherIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "payload_bytes_sent", GetTestCaseName(pc_label), + pc_stats.payload_sent.bytes(), Unit::kBytes, + ImprovementDirection::kNeitherIsBetter, metric_metadata); + + metrics_logger_->LogSingleValueMetric( + "bytes_sent", GetTestCaseName(pc_label), pc_stats.total_sent.bytes(), + Unit::kBytes, ImprovementDirection::kNeitherIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "packets_sent", GetTestCaseName(pc_label), pc_stats.packets_sent, + Unit::kUnitless, ImprovementDirection::kNeitherIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "average_send_rate", GetTestCaseName(pc_label), + (pc_stats.total_sent / (end_time - start_time_)).kbps<double>(), + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter, + metric_metadata); + metrics_logger_->LogSingleValueMetric( + "bytes_received", GetTestCaseName(pc_label), + pc_stats.total_received.bytes(), Unit::kBytes, + ImprovementDirection::kNeitherIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "packets_received", GetTestCaseName(pc_label), pc_stats.packets_received, + Unit::kUnitless, ImprovementDirection::kNeitherIsBetter, metric_metadata); + metrics_logger_->LogSingleValueMetric( + "average_receive_rate", GetTestCaseName(pc_label), + (pc_stats.total_received / (end_time - start_time_)).kbps<double>(), + Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter, + metric_metadata); + metrics_logger_->LogSingleValueMetric( + "sent_packets_loss", GetTestCaseName(pc_label), packet_loss, + Unit::kUnitless, ImprovementDirection::kNeitherIsBetter, metric_metadata); +} + +std::string StatsBasedNetworkQualityMetricsReporter::GetTestCaseName( + absl::string_view network_label) const { + rtc::StringBuilder builder; + builder << test_case_name_ << "/" << network_label.data(); + return builder.str(); +} + +void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats( + const std::string& peer_name, + const NetworkLayerStats& stats) const { + DataRate average_send_rate = + stats.endpoints_stats.overall_outgoing_stats.packets_sent >= 2 + ? stats.endpoints_stats.overall_outgoing_stats.AverageSendRate() + : DataRate::Zero(); + DataRate average_receive_rate = + stats.endpoints_stats.overall_incoming_stats.packets_received >= 2 + ? stats.endpoints_stats.overall_incoming_stats.AverageReceiveRate() + : DataRate::Zero(); + // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey. + std::map<std::string, std::string> metric_metadata{ + {MetricMetadataKey::kPeerMetadataKey, peer_name}, + {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_case_name_}}; + rtc::StringBuilder log; + log << "Raw network layer statistic for [" << peer_name << "]:\n" + << "Local IPs:\n"; + for (size_t i = 0; i < stats.endpoints_stats.local_addresses.size(); ++i) { + log << " " << stats.endpoints_stats.local_addresses[i].ToString() << "\n"; + } + if (!stats.endpoints_stats.overall_outgoing_stats.sent_packets_size + .IsEmpty()) { + metrics_logger_->LogMetric( + "sent_packets_size", GetTestCaseName(peer_name), + stats.endpoints_stats.overall_outgoing_stats.sent_packets_size, + Unit::kBytes, ImprovementDirection::kNeitherIsBetter, metric_metadata); + } + if (!stats.endpoints_stats.overall_incoming_stats.received_packets_size + .IsEmpty()) { + metrics_logger_->LogMetric( + "received_packets_size", GetTestCaseName(peer_name), + stats.endpoints_stats.overall_incoming_stats.received_packets_size, + Unit::kBytes, ImprovementDirection::kNeitherIsBetter, metric_metadata); + } + if (!stats.endpoints_stats.overall_incoming_stats + .packets_discarded_no_receiver_size.IsEmpty()) { + metrics_logger_->LogMetric( + "packets_discarded_no_receiver_size", GetTestCaseName(peer_name), + stats.endpoints_stats.overall_incoming_stats + .packets_discarded_no_receiver_size, + Unit::kBytes, ImprovementDirection::kNeitherIsBetter, metric_metadata); + } + if (!stats.endpoints_stats.sent_packets_queue_wait_time_us.IsEmpty()) { + metrics_logger_->LogMetric( + "sent_packets_queue_wait_time_us", GetTestCaseName(peer_name), + stats.endpoints_stats.sent_packets_queue_wait_time_us, Unit::kUnitless, + ImprovementDirection::kNeitherIsBetter, metric_metadata); + } + + log << "Send statistic:\n" + << " packets: " + << stats.endpoints_stats.overall_outgoing_stats.packets_sent << " bytes: " + << stats.endpoints_stats.overall_outgoing_stats.bytes_sent.bytes() + << " avg_rate (bytes/sec): " << average_send_rate.bytes_per_sec() + << " avg_rate (bps): " << average_send_rate.bps() << "\n" + << "Send statistic per destination:\n"; + + for (const auto& entry : + stats.endpoints_stats.outgoing_stats_per_destination) { + DataRate source_average_send_rate = entry.second.packets_sent >= 2 + ? entry.second.AverageSendRate() + : DataRate::Zero(); + log << "(" << entry.first.ToString() << "):\n" + << " packets: " << entry.second.packets_sent + << " bytes: " << entry.second.bytes_sent.bytes() + << " avg_rate (bytes/sec): " << source_average_send_rate.bytes_per_sec() + << " avg_rate (bps): " << source_average_send_rate.bps() << "\n"; + if (!entry.second.sent_packets_size.IsEmpty()) { + metrics_logger_->LogMetric( + "sent_packets_size", + GetTestCaseName(peer_name + "/" + entry.first.ToString()), + entry.second.sent_packets_size, Unit::kBytes, + ImprovementDirection::kNeitherIsBetter, metric_metadata); + } + } + + if (!stats.uplink_stats.packet_transport_time.IsEmpty()) { + log << "[Debug stats] packet_transport_time=(" + << stats.uplink_stats.packet_transport_time.GetAverage() << ", " + << stats.uplink_stats.packet_transport_time.GetStandardDeviation() + << ")\n"; + metrics_logger_->LogMetric( + "uplink_packet_transport_time", GetTestCaseName(peer_name), + stats.uplink_stats.packet_transport_time, Unit::kMilliseconds, + ImprovementDirection::kNeitherIsBetter, metric_metadata); + } + if (!stats.uplink_stats.size_to_packet_transport_time.IsEmpty()) { + log << "[Debug stats] size_to_packet_transport_time=(" + << stats.uplink_stats.size_to_packet_transport_time.GetAverage() << ", " + << stats.uplink_stats.size_to_packet_transport_time + .GetStandardDeviation() + << ")\n"; + metrics_logger_->LogMetric( + "uplink_size_to_packet_transport_time", GetTestCaseName(peer_name), + stats.uplink_stats.size_to_packet_transport_time, Unit::kUnitless, + ImprovementDirection::kNeitherIsBetter, metric_metadata); + } + + log << "Receive statistic:\n" + << " packets: " + << stats.endpoints_stats.overall_incoming_stats.packets_received + << " bytes: " + << stats.endpoints_stats.overall_incoming_stats.bytes_received.bytes() + << " avg_rate (bytes/sec): " << average_receive_rate.bytes_per_sec() + << " avg_rate (bps): " << average_receive_rate.bps() << "\n" + << "Receive statistic per source:\n"; + + for (const auto& entry : stats.endpoints_stats.incoming_stats_per_source) { + DataRate source_average_receive_rate = + entry.second.packets_received >= 2 ? entry.second.AverageReceiveRate() + : DataRate::Zero(); + log << "(" << entry.first.ToString() << "):\n" + << " packets: " << entry.second.packets_received + << " bytes: " << entry.second.bytes_received.bytes() + << " avg_rate (bytes/sec): " + << source_average_receive_rate.bytes_per_sec() + << " avg_rate (bps): " << source_average_receive_rate.bps() << "\n"; + if (!entry.second.received_packets_size.IsEmpty()) { + metrics_logger_->LogMetric( + "received_packets_size", + GetTestCaseName(peer_name + "/" + entry.first.ToString()), + entry.second.received_packets_size, Unit::kBytes, + ImprovementDirection::kNeitherIsBetter, metric_metadata); + } + if (!entry.second.packets_discarded_no_receiver_size.IsEmpty()) { + metrics_logger_->LogMetric( + "packets_discarded_no_receiver_size", + GetTestCaseName(peer_name + "/" + entry.first.ToString()), + entry.second.packets_discarded_no_receiver_size, Unit::kBytes, + ImprovementDirection::kNeitherIsBetter, metric_metadata); + } + } + if (!stats.downlink_stats.packet_transport_time.IsEmpty()) { + log << "[Debug stats] packet_transport_time=(" + << stats.downlink_stats.packet_transport_time.GetAverage() << ", " + << stats.downlink_stats.packet_transport_time.GetStandardDeviation() + << ")\n"; + metrics_logger_->LogMetric( + "downlink_packet_transport_time", GetTestCaseName(peer_name), + stats.downlink_stats.packet_transport_time, Unit::kMilliseconds, + ImprovementDirection::kNeitherIsBetter, metric_metadata); + } + if (!stats.downlink_stats.size_to_packet_transport_time.IsEmpty()) { + log << "[Debug stats] size_to_packet_transport_time=(" + << stats.downlink_stats.size_to_packet_transport_time.GetAverage() + << ", " + << stats.downlink_stats.size_to_packet_transport_time + .GetStandardDeviation() + << ")\n"; + metrics_logger_->LogMetric( + "downlink_size_to_packet_transport_time", GetTestCaseName(peer_name), + stats.downlink_stats.size_to_packet_transport_time, Unit::kUnitless, + ImprovementDirection::kNeitherIsBetter, metric_metadata); + } + + RTC_LOG(LS_INFO) << log.str(); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.h b/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.h new file mode 100644 index 0000000000..60daf40c8c --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.h @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_STATS_BASED_NETWORK_QUALITY_METRICS_REPORTER_H_ +#define TEST_PC_E2E_STATS_BASED_NETWORK_QUALITY_METRICS_REPORTER_H_ + +#include <cstdint> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <utility> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metrics_logger.h" +#include "api/test/network_emulation/network_emulation_interfaces.h" +#include "api/test/network_emulation_manager.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "rtc_base/ip_address.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// TODO(titovartem): make this class testable and add tests. +class StatsBasedNetworkQualityMetricsReporter + : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { + public: + // Emulated network layer stats for single peer. + struct NetworkLayerStats { + EmulatedNetworkStats endpoints_stats; + EmulatedNetworkNodeStats uplink_stats; + EmulatedNetworkNodeStats downlink_stats; + std::set<std::string> receivers; + }; + + // `networks` map peer name to network to report network layer stability stats + // and to log network layer metrics. + StatsBasedNetworkQualityMetricsReporter( + std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints, + NetworkEmulationManager* network_emulation, + test::MetricsLogger* metrics_logger); + ~StatsBasedNetworkQualityMetricsReporter() override = default; + + void AddPeer(absl::string_view peer_name, + std::vector<EmulatedEndpoint*> endpoints); + void AddPeer(absl::string_view peer_name, + std::vector<EmulatedEndpoint*> endpoints, + std::vector<EmulatedNetworkNode*> uplink, + std::vector<EmulatedNetworkNode*> downlink); + + // Network stats must be empty when this method will be invoked. + void Start(absl::string_view test_case_name, + const TrackIdStreamInfoMap* reporter_helper) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override; + void StopAndReportResults() override; + + private: + struct PCStats { + // TODO(bugs.webrtc.org/10525): Separate audio and video counters. Depends + // on standard stat counters, enabled by field trial + // "WebRTC-UseStandardBytesStats". + DataSize payload_received = DataSize::Zero(); + DataSize payload_sent = DataSize::Zero(); + + // Total bytes/packets sent/received in all RTCTransport's. + DataSize total_received = DataSize::Zero(); + DataSize total_sent = DataSize::Zero(); + int64_t packets_received = 0; + int64_t packets_sent = 0; + }; + + class NetworkLayerStatsCollector { + public: + NetworkLayerStatsCollector( + std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints, + NetworkEmulationManager* network_emulation); + + void Start(); + + void AddPeer(absl::string_view peer_name, + std::vector<EmulatedEndpoint*> endpoints, + std::vector<EmulatedNetworkNode*> uplink, + std::vector<EmulatedNetworkNode*> downlink); + + std::map<std::string, NetworkLayerStats> GetStats(); + + private: + Mutex mutex_; + std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints_ + RTC_GUARDED_BY(mutex_); + std::map<std::string, std::vector<EmulatedNetworkNode*>> peer_uplinks_ + RTC_GUARDED_BY(mutex_); + std::map<std::string, std::vector<EmulatedNetworkNode*>> peer_downlinks_ + RTC_GUARDED_BY(mutex_); + std::map<rtc::IPAddress, std::string> ip_to_peer_ RTC_GUARDED_BY(mutex_); + NetworkEmulationManager* const network_emulation_; + }; + + void ReportStats(const std::string& pc_label, + const PCStats& pc_stats, + const NetworkLayerStats& network_layer_stats, + int64_t packet_loss, + const Timestamp& end_time); + std::string GetTestCaseName(absl::string_view network_label) const; + void LogNetworkLayerStats(const std::string& peer_name, + const NetworkLayerStats& stats) const; + + NetworkLayerStatsCollector collector_; + Clock* const clock_; + test::MetricsLogger* const metrics_logger_; + + std::string test_case_name_; + Timestamp start_time_ = Timestamp::MinusInfinity(); + + Mutex mutex_; + std::map<std::string, PCStats> pc_stats_ RTC_GUARDED_BY(mutex_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_STATS_BASED_NETWORK_QUALITY_METRICS_REPORTER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter_test.cc b/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter_test.cc new file mode 100644 index 0000000000..be55149482 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter_test.cc @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h" + +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/test/create_network_emulation_manager.h" +#include "api/test/create_peer_connection_quality_test_frame_generator.h" +#include "api/test/metrics/metrics_logger.h" +#include "api/test/metrics/stdout_metrics_exporter.h" +#include "api/test/network_emulation_manager.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/pclf/media_quality_test_params.h" +#include "api/test/pclf/peer_configurer.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/units/time_delta.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/pc/e2e/metric_metadata_keys.h" +#include "test/pc/e2e/peer_connection_quality_test.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::testing::UnorderedElementsAre; + +using ::webrtc::test::DefaultMetricsLogger; +using ::webrtc::test::ImprovementDirection; +using ::webrtc::test::Metric; +using ::webrtc::test::Unit; +using ::webrtc::webrtc_pc_e2e::PeerConfigurer; + +// Adds a peer with some audio and video (the client should not care about +// details about audio and video configs). +void AddDefaultAudioVideoPeer( + absl::string_view peer_name, + absl::string_view audio_stream_label, + absl::string_view video_stream_label, + const PeerNetworkDependencies& network_dependencies, + PeerConnectionE2EQualityTestFixture& fixture) { + AudioConfig audio{std::string(audio_stream_label)}; + audio.sync_group = std::string(peer_name); + VideoConfig video(std::string(video_stream_label), 320, 180, 15); + video.sync_group = std::string(peer_name); + auto peer = std::make_unique<PeerConfigurer>(network_dependencies); + peer->SetName(peer_name); + peer->SetAudioConfig(std::move(audio)); + peer->AddVideoConfig(std::move(video)); + peer->SetVideoCodecs({VideoCodecConfig(cricket::kVp8CodecName)}); + fixture.AddPeer(std::move(peer)); +} + +absl::optional<Metric> FindMeetricByName(absl::string_view name, + rtc::ArrayView<const Metric> metrics) { + for (const Metric& metric : metrics) { + if (metric.name == name) { + return metric; + } + } + return absl::nullopt; +} + +TEST(StatsBasedNetworkQualityMetricsReporterTest, DebugStatsAreCollected) { + std::unique_ptr<NetworkEmulationManager> network_emulation = + CreateNetworkEmulationManager(TimeMode::kSimulated, + EmulatedNetworkStatsGatheringMode::kDebug); + DefaultMetricsLogger metrics_logger( + network_emulation->time_controller()->GetClock()); + PeerConnectionE2EQualityTest fixture( + "test_case", *network_emulation->time_controller(), + /*audio_quality_analyzer=*/nullptr, /*video_quality_analyzer=*/nullptr, + &metrics_logger); + + EmulatedEndpoint* alice_endpoint = + network_emulation->CreateEndpoint(EmulatedEndpointConfig()); + EmulatedEndpoint* bob_endpoint = + network_emulation->CreateEndpoint(EmulatedEndpointConfig()); + + EmulatedNetworkNode* alice_link = network_emulation->CreateEmulatedNode( + BuiltInNetworkBehaviorConfig{.link_capacity_kbps = 500}); + network_emulation->CreateRoute(alice_endpoint, {alice_link}, bob_endpoint); + EmulatedNetworkNode* bob_link = network_emulation->CreateEmulatedNode( + BuiltInNetworkBehaviorConfig{.link_capacity_kbps = 500}); + network_emulation->CreateRoute(bob_endpoint, {bob_link}, alice_endpoint); + + EmulatedNetworkManagerInterface* alice_network = + network_emulation->CreateEmulatedNetworkManagerInterface( + {alice_endpoint}); + EmulatedNetworkManagerInterface* bob_network = + network_emulation->CreateEmulatedNetworkManagerInterface({bob_endpoint}); + + AddDefaultAudioVideoPeer("alice", "alice_audio", "alice_video", + alice_network->network_dependencies(), fixture); + AddDefaultAudioVideoPeer("bob", "bob_audio", "bob_video", + bob_network->network_dependencies(), fixture); + + auto network_stats_reporter = + std::make_unique<StatsBasedNetworkQualityMetricsReporter>( + /*peer_endpoints=*/std::map<std::string, + std::vector<EmulatedEndpoint*>>{}, + network_emulation.get(), &metrics_logger); + network_stats_reporter->AddPeer("alice", alice_network->endpoints(), + /*uplink=*/{alice_link}, + /*downlink=*/{bob_link}); + network_stats_reporter->AddPeer("bob", bob_network->endpoints(), + /*uplink=*/{bob_link}, + /*downlink=*/{alice_link}); + fixture.AddQualityMetricsReporter(std::move(network_stats_reporter)); + + fixture.Run(RunParams(TimeDelta::Seconds(4))); + + std::vector<Metric> metrics = metrics_logger.GetCollectedMetrics(); + absl::optional<Metric> uplink_packet_transport_time = + FindMeetricByName("uplink_packet_transport_time", metrics); + ASSERT_TRUE(uplink_packet_transport_time.has_value()); + ASSERT_FALSE(uplink_packet_transport_time->time_series.samples.empty()); + absl::optional<Metric> uplink_size_to_packet_transport_time = + FindMeetricByName("uplink_size_to_packet_transport_time", metrics); + ASSERT_TRUE(uplink_size_to_packet_transport_time.has_value()); + ASSERT_FALSE( + uplink_size_to_packet_transport_time->time_series.samples.empty()); + absl::optional<Metric> downlink_packet_transport_time = + FindMeetricByName("downlink_packet_transport_time", metrics); + ASSERT_TRUE(downlink_packet_transport_time.has_value()); + ASSERT_FALSE(downlink_packet_transport_time->time_series.samples.empty()); + absl::optional<Metric> downlink_size_to_packet_transport_time = + FindMeetricByName("downlink_size_to_packet_transport_time", metrics); + ASSERT_TRUE(downlink_size_to_packet_transport_time.has_value()); + ASSERT_FALSE( + downlink_size_to_packet_transport_time->time_series.samples.empty()); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/stats_poller.cc b/third_party/libwebrtc/test/pc/e2e/stats_poller.cc new file mode 100644 index 0000000000..c04805fb20 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/stats_poller.cc @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/stats_poller.h" + +#include <utility> + +#include "rtc_base/logging.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +void InternalStatsObserver::PollStats() { + peer_->GetStats(this); +} + +void InternalStatsObserver::OnStatsDelivered( + const rtc::scoped_refptr<const RTCStatsReport>& report) { + for (auto* observer : observers_) { + observer->OnStatsReports(pc_label_, report); + } +} + +StatsPoller::StatsPoller(std::vector<StatsObserverInterface*> observers, + std::map<std::string, StatsProvider*> peers) + : observers_(std::move(observers)) { + webrtc::MutexLock lock(&mutex_); + for (auto& peer : peers) { + pollers_.push_back(rtc::make_ref_counted<InternalStatsObserver>( + peer.first, peer.second, observers_)); + } +} + +StatsPoller::StatsPoller(std::vector<StatsObserverInterface*> observers, + std::map<std::string, TestPeer*> peers) + : observers_(std::move(observers)) { + webrtc::MutexLock lock(&mutex_); + for (auto& peer : peers) { + pollers_.push_back(rtc::make_ref_counted<InternalStatsObserver>( + peer.first, peer.second, observers_)); + } +} + +void StatsPoller::PollStatsAndNotifyObservers() { + webrtc::MutexLock lock(&mutex_); + for (auto& poller : pollers_) { + poller->PollStats(); + } +} + +void StatsPoller::RegisterParticipantInCall(absl::string_view peer_name, + StatsProvider* peer) { + webrtc::MutexLock lock(&mutex_); + pollers_.push_back(rtc::make_ref_counted<InternalStatsObserver>( + peer_name, peer, observers_)); +} + +bool StatsPoller::UnregisterParticipantInCall(absl::string_view peer_name) { + webrtc::MutexLock lock(&mutex_); + for (auto it = pollers_.begin(); it != pollers_.end(); ++it) { + if ((*it)->pc_label() == peer_name) { + pollers_.erase(it); + return true; + } + } + return false; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/stats_poller.h b/third_party/libwebrtc/test/pc/e2e/stats_poller.h new file mode 100644 index 0000000000..3576f1bf05 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/stats_poller.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_STATS_POLLER_H_ +#define TEST_PC_E2E_STATS_POLLER_H_ + +#include <map> +#include <string> +#include <utility> +#include <vector> + +#include "api/peer_connection_interface.h" +#include "api/stats/rtc_stats_collector_callback.h" +#include "api/test/stats_observer_interface.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" +#include "test/pc/e2e/stats_provider.h" +#include "test/pc/e2e/test_peer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Helper class that will notify all the webrtc::test::StatsObserverInterface +// objects subscribed. +class InternalStatsObserver : public RTCStatsCollectorCallback { + public: + InternalStatsObserver(absl::string_view pc_label, + StatsProvider* peer, + std::vector<StatsObserverInterface*> observers) + : pc_label_(pc_label), peer_(peer), observers_(std::move(observers)) {} + + std::string pc_label() const { return pc_label_; } + + void PollStats(); + + void OnStatsDelivered( + const rtc::scoped_refptr<const RTCStatsReport>& report) override; + + private: + std::string pc_label_; + StatsProvider* peer_; + std::vector<StatsObserverInterface*> observers_; +}; + +// Helper class to invoke GetStats on a PeerConnection by passing a +// webrtc::StatsObserver that will notify all the +// webrtc::test::StatsObserverInterface subscribed. +class StatsPoller { + public: + StatsPoller(std::vector<StatsObserverInterface*> observers, + std::map<std::string, StatsProvider*> peers_to_observe); + StatsPoller(std::vector<StatsObserverInterface*> observers, + std::map<std::string, TestPeer*> peers_to_observe); + + void PollStatsAndNotifyObservers(); + + void RegisterParticipantInCall(absl::string_view peer_name, + StatsProvider* peer); + // Unregister participant from stats poller. Returns true if participant was + // removed and false if participant wasn't found. + bool UnregisterParticipantInCall(absl::string_view peer_name); + + private: + const std::vector<StatsObserverInterface*> observers_; + webrtc::Mutex mutex_; + std::vector<rtc::scoped_refptr<InternalStatsObserver>> pollers_ + RTC_GUARDED_BY(mutex_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_STATS_POLLER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/stats_poller_test.cc b/third_party/libwebrtc/test/pc/e2e/stats_poller_test.cc new file mode 100644 index 0000000000..02a323127b --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/stats_poller_test.cc @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/stats_poller.h" + +#include "api/stats/rtc_stats_collector_callback.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::testing::Eq; + +class TestStatsProvider : public StatsProvider { + public: + ~TestStatsProvider() override = default; + + void GetStats(RTCStatsCollectorCallback* callback) override { + stats_collections_count_++; + } + + int stats_collections_count() const { return stats_collections_count_; } + + private: + int stats_collections_count_ = 0; +}; + +class MockStatsObserver : public StatsObserverInterface { + public: + ~MockStatsObserver() override = default; + + MOCK_METHOD(void, + OnStatsReports, + (absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report)); +}; + +TEST(StatsPollerTest, UnregisterParticipantAddedInCtor) { + TestStatsProvider alice; + TestStatsProvider bob; + + MockStatsObserver stats_observer; + + StatsPoller poller(/*observers=*/{&stats_observer}, + /*peers_to_observe=*/{{"alice", &alice}, {"bob", &bob}}); + poller.PollStatsAndNotifyObservers(); + + EXPECT_THAT(alice.stats_collections_count(), Eq(1)); + EXPECT_THAT(bob.stats_collections_count(), Eq(1)); + + poller.UnregisterParticipantInCall("bob"); + poller.PollStatsAndNotifyObservers(); + + EXPECT_THAT(alice.stats_collections_count(), Eq(2)); + EXPECT_THAT(bob.stats_collections_count(), Eq(1)); +} + +TEST(StatsPollerTest, UnregisterParticipantRegisteredInCall) { + TestStatsProvider alice; + TestStatsProvider bob; + + MockStatsObserver stats_observer; + + StatsPoller poller(/*observers=*/{&stats_observer}, + /*peers_to_observe=*/{{"alice", &alice}}); + poller.RegisterParticipantInCall("bob", &bob); + poller.PollStatsAndNotifyObservers(); + + EXPECT_THAT(alice.stats_collections_count(), Eq(1)); + EXPECT_THAT(bob.stats_collections_count(), Eq(1)); + + poller.UnregisterParticipantInCall("bob"); + poller.PollStatsAndNotifyObservers(); + + EXPECT_THAT(alice.stats_collections_count(), Eq(2)); + EXPECT_THAT(bob.stats_collections_count(), Eq(1)); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/stats_provider.h b/third_party/libwebrtc/test/pc/e2e/stats_provider.h new file mode 100644 index 0000000000..eef62d779c --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/stats_provider.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_STATS_PROVIDER_H_ +#define TEST_PC_E2E_STATS_PROVIDER_H_ + +#include "api/stats/rtc_stats_collector_callback.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class StatsProvider { + public: + virtual ~StatsProvider() = default; + + virtual void GetStats(RTCStatsCollectorCallback* callback) = 0; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_STATS_PROVIDER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/test_activities_executor.cc b/third_party/libwebrtc/test/pc/e2e/test_activities_executor.cc new file mode 100644 index 0000000000..7bcf7dd6c3 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/test_activities_executor.cc @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/test_activities_executor.h" + +#include <memory> +#include <utility> + +#include "absl/memory/memory.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/task_queue_for_test.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +void TestActivitiesExecutor::Start(TaskQueueBase* task_queue) { + RTC_DCHECK(task_queue); + task_queue_ = task_queue; + MutexLock lock(&lock_); + start_time_ = Now(); + while (!scheduled_activities_.empty()) { + PostActivity(std::move(scheduled_activities_.front())); + scheduled_activities_.pop(); + } +} + +void TestActivitiesExecutor::Stop() { + if (task_queue_ == nullptr) { + // Already stopped or not started. + return; + } + SendTask(task_queue_, [this]() { + MutexLock lock(&lock_); + for (auto& handle : repeating_task_handles_) { + handle.Stop(); + } + }); + task_queue_ = nullptr; +} + +void TestActivitiesExecutor::ScheduleActivity( + TimeDelta initial_delay_since_start, + absl::optional<TimeDelta> interval, + std::function<void(TimeDelta)> func) { + RTC_CHECK(initial_delay_since_start.IsFinite() && + initial_delay_since_start >= TimeDelta::Zero()); + RTC_CHECK(!interval || + (interval->IsFinite() && *interval > TimeDelta::Zero())); + MutexLock lock(&lock_); + ScheduledActivity activity(initial_delay_since_start, interval, func); + if (start_time_.IsInfinite()) { + scheduled_activities_.push(std::move(activity)); + } else { + PostActivity(std::move(activity)); + } +} + +void TestActivitiesExecutor::PostActivity(ScheduledActivity activity) { + // Because start_time_ will never change at this point copy it to local + // variable to capture in in lambda without requirement to hold a lock. + Timestamp start_time = start_time_; + + TimeDelta remaining_delay = + activity.initial_delay_since_start == TimeDelta::Zero() + ? TimeDelta::Zero() + : activity.initial_delay_since_start - (Now() - start_time); + if (remaining_delay < TimeDelta::Zero()) { + RTC_LOG(LS_WARNING) << "Executing late task immediately, late by=" + << ToString(remaining_delay.Abs()); + remaining_delay = TimeDelta::Zero(); + } + + if (activity.interval) { + if (remaining_delay == TimeDelta::Zero()) { + repeating_task_handles_.push_back(RepeatingTaskHandle::Start( + task_queue_, [activity, start_time, this]() { + activity.func(Now() - start_time); + return *activity.interval; + })); + return; + } + repeating_task_handles_.push_back(RepeatingTaskHandle::DelayedStart( + task_queue_, remaining_delay, [activity, start_time, this]() { + activity.func(Now() - start_time); + return *activity.interval; + })); + return; + } + + if (remaining_delay == TimeDelta::Zero()) { + task_queue_->PostTask( + [activity, start_time, this]() { activity.func(Now() - start_time); }); + return; + } + + task_queue_->PostDelayedTask( + [activity, start_time, this]() { activity.func(Now() - start_time); }, + remaining_delay); +} + +Timestamp TestActivitiesExecutor::Now() const { + return clock_->CurrentTime(); +} + +TestActivitiesExecutor::ScheduledActivity::ScheduledActivity( + TimeDelta initial_delay_since_start, + absl::optional<TimeDelta> interval, + std::function<void(TimeDelta)> func) + : initial_delay_since_start(initial_delay_since_start), + interval(interval), + func(std::move(func)) {} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/test_activities_executor.h b/third_party/libwebrtc/test/pc/e2e/test_activities_executor.h new file mode 100644 index 0000000000..2469ac7f36 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/test_activities_executor.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_TEST_ACTIVITIES_EXECUTOR_H_ +#define TEST_PC_E2E_TEST_ACTIVITIES_EXECUTOR_H_ + +#include <queue> +#include <vector> + +#include "absl/types/optional.h" +#include "api/task_queue/task_queue_base.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/task_queue_for_test.h" +#include "rtc_base/task_utils/repeating_task.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class TestActivitiesExecutor { + public: + explicit TestActivitiesExecutor(Clock* clock) : clock_(clock) {} + ~TestActivitiesExecutor() { Stop(); } + + // Starts scheduled activities according to their schedule. All activities + // that will be scheduled after Start(...) was invoked will be executed + // immediately according to their schedule. + void Start(TaskQueueForTest* task_queue) { Start(task_queue->Get()); } + void Start(TaskQueueBase* task_queue); + void Stop(); + + // Schedule activity to be executed. If test isn't started yet, then activity + // will be executed according to its schedule after Start() will be invoked. + // If test is started, then it will be executed immediately according to its + // schedule. + void ScheduleActivity(TimeDelta initial_delay_since_start, + absl::optional<TimeDelta> interval, + std::function<void(TimeDelta)> func); + + private: + struct ScheduledActivity { + ScheduledActivity(TimeDelta initial_delay_since_start, + absl::optional<TimeDelta> interval, + std::function<void(TimeDelta)> func); + + TimeDelta initial_delay_since_start; + absl::optional<TimeDelta> interval; + std::function<void(TimeDelta)> func; + }; + + void PostActivity(ScheduledActivity activity) + RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_); + Timestamp Now() const; + + Clock* const clock_; + + TaskQueueBase* task_queue_; + + Mutex lock_; + // Time when test was started. Minus infinity means that it wasn't started + // yet. + Timestamp start_time_ RTC_GUARDED_BY(lock_) = Timestamp::MinusInfinity(); + // Queue of activities that were added before test was started. + // Activities from this queue will be posted on the `task_queue_` after test + // will be set up and then this queue will be unused. + std::queue<ScheduledActivity> scheduled_activities_ RTC_GUARDED_BY(lock_); + // List of task handles for activities, that are posted on `task_queue_` as + // repeated during the call. + std::vector<RepeatingTaskHandle> repeating_task_handles_ + RTC_GUARDED_BY(lock_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_TEST_ACTIVITIES_EXECUTOR_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/test_peer.cc b/third_party/libwebrtc/test/pc/e2e/test_peer.cc new file mode 100644 index 0000000000..b3a9e1c164 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/test_peer.cc @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/test_peer.h" + +#include <string> +#include <utility> + +#include "absl/memory/memory.h" +#include "absl/strings/string_view.h" +#include "api/scoped_refptr.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/pclf/peer_configurer.h" +#include "modules/audio_processing/include/audio_processing.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +class SetRemoteDescriptionCallback + : public webrtc::SetRemoteDescriptionObserverInterface { + public: + void OnSetRemoteDescriptionComplete(webrtc::RTCError error) override { + is_called_ = true; + error_ = error; + } + + bool is_called() const { return is_called_; } + + webrtc::RTCError error() const { return error_; } + + private: + bool is_called_ = false; + webrtc::RTCError error_; +}; + +} // namespace + +ConfigurableParams TestPeer::configurable_params() const { + MutexLock lock(&mutex_); + return configurable_params_; +} + +void TestPeer::AddVideoConfig(VideoConfig config) { + MutexLock lock(&mutex_); + configurable_params_.video_configs.push_back(std::move(config)); +} + +void TestPeer::RemoveVideoConfig(absl::string_view stream_label) { + MutexLock lock(&mutex_); + bool config_removed = false; + for (auto it = configurable_params_.video_configs.begin(); + it != configurable_params_.video_configs.end(); ++it) { + if (*it->stream_label == stream_label) { + configurable_params_.video_configs.erase(it); + config_removed = true; + break; + } + } + RTC_CHECK(config_removed) << *params_.name << ": No video config with label [" + << stream_label << "] was found"; +} + +void TestPeer::SetVideoSubscription(VideoSubscription subscription) { + MutexLock lock(&mutex_); + configurable_params_.video_subscription = std::move(subscription); +} + +void TestPeer::GetStats(RTCStatsCollectorCallback* callback) { + pc()->signaling_thread()->PostTask( + SafeTask(signaling_thread_task_safety_, + [this, callback]() { pc()->GetStats(callback); })); +} + +bool TestPeer::SetRemoteDescription( + std::unique_ptr<SessionDescriptionInterface> desc, + std::string* error_out) { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + + auto observer = rtc::make_ref_counted<SetRemoteDescriptionCallback>(); + // We're assuming (and asserting) that the PeerConnection implementation of + // SetRemoteDescription is synchronous when called on the signaling thread. + pc()->SetRemoteDescription(std::move(desc), observer); + RTC_CHECK(observer->is_called()); + if (!observer->error().ok()) { + RTC_LOG(LS_ERROR) << *params_.name << ": Failed to set remote description: " + << observer->error().message(); + if (error_out) { + *error_out = observer->error().message(); + } + } + return observer->error().ok(); +} + +bool TestPeer::AddIceCandidates( + std::vector<std::unique_ptr<IceCandidateInterface>> candidates) { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + bool success = true; + for (auto& candidate : candidates) { + if (!pc()->AddIceCandidate(candidate.get())) { + std::string candidate_str; + bool res = candidate->ToString(&candidate_str); + RTC_CHECK(res); + RTC_LOG(LS_ERROR) << "Failed to add ICE candidate, candidate_str=" + << candidate_str; + success = false; + } else { + remote_ice_candidates_.push_back(std::move(candidate)); + } + } + return success; +} + +void TestPeer::Close() { + signaling_thread_task_safety_->SetNotAlive(); + wrapper_->pc()->Close(); + remote_ice_candidates_.clear(); + audio_processing_ = nullptr; + video_sources_.clear(); + wrapper_ = nullptr; + worker_thread_ = nullptr; +} + +TestPeer::TestPeer( + rtc::scoped_refptr<PeerConnectionFactoryInterface> pc_factory, + rtc::scoped_refptr<PeerConnectionInterface> pc, + std::unique_ptr<MockPeerConnectionObserver> observer, + Params params, + ConfigurableParams configurable_params, + std::vector<PeerConfigurer::VideoSource> video_sources, + rtc::scoped_refptr<AudioProcessing> audio_processing, + std::unique_ptr<rtc::Thread> worker_thread) + : params_(std::move(params)), + configurable_params_(std::move(configurable_params)), + worker_thread_(std::move(worker_thread)), + wrapper_(std::make_unique<PeerConnectionWrapper>(std::move(pc_factory), + std::move(pc), + std::move(observer))), + video_sources_(std::move(video_sources)), + audio_processing_(audio_processing) { + signaling_thread_task_safety_ = PendingTaskSafetyFlag::CreateDetached(); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/test_peer.h b/third_party/libwebrtc/test/pc/e2e/test_peer.h new file mode 100644 index 0000000000..1088871817 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/test_peer.h @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_TEST_PEER_H_ +#define TEST_PC_E2E_TEST_PEER_H_ + +#include <memory> +#include <vector> + +#include "absl/memory/memory.h" +#include "absl/strings/string_view.h" +#include "api/function_view.h" +#include "api/scoped_refptr.h" +#include "api/sequence_checker.h" +#include "api/set_remote_description_observer_interface.h" +#include "api/task_queue/pending_task_safety_flag.h" +#include "api/test/frame_generator_interface.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/pclf/media_quality_test_params.h" +#include "api/test/pclf/peer_configurer.h" +#include "pc/peer_connection_wrapper.h" +#include "rtc_base/logging.h" +#include "rtc_base/synchronization/mutex.h" +#include "test/pc/e2e/stats_provider.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Describes a single participant in the call. +class TestPeer final : public StatsProvider { + public: + ~TestPeer() override = default; + + const Params& params() const { return params_; } + + ConfigurableParams configurable_params() const; + void AddVideoConfig(VideoConfig config); + // Removes video config with specified name. Crashes if the config with + // specified name isn't found. + void RemoveVideoConfig(absl::string_view stream_label); + void SetVideoSubscription(VideoSubscription subscription); + + void GetStats(RTCStatsCollectorCallback* callback) override; + + PeerConfigurer::VideoSource ReleaseVideoSource(size_t i) { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return std::move(video_sources_[i]); + } + + PeerConnectionFactoryInterface* pc_factory() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->pc_factory(); + } + PeerConnectionInterface* pc() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->pc(); + } + MockPeerConnectionObserver* observer() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->observer(); + } + + // Tell underlying `PeerConnection` to create an Offer. + // `observer` will be invoked on the signaling thread when offer is created. + void CreateOffer( + rtc::scoped_refptr<CreateSessionDescriptionObserver> observer) { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + pc()->CreateOffer(observer.get(), params_.rtc_offer_answer_options); + } + std::unique_ptr<SessionDescriptionInterface> CreateOffer() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->CreateOffer(params_.rtc_offer_answer_options); + } + + std::unique_ptr<SessionDescriptionInterface> CreateAnswer() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->CreateAnswer(); + } + + bool SetLocalDescription(std::unique_ptr<SessionDescriptionInterface> desc, + std::string* error_out = nullptr) { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->SetLocalDescription(std::move(desc), error_out); + } + + // `error_out` will be set only if returned value is false. + bool SetRemoteDescription(std::unique_ptr<SessionDescriptionInterface> desc, + std::string* error_out = nullptr); + + rtc::scoped_refptr<RtpTransceiverInterface> AddTransceiver( + cricket::MediaType media_type, + const RtpTransceiverInit& init) { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->AddTransceiver(media_type, init); + } + + rtc::scoped_refptr<RtpSenderInterface> AddTrack( + rtc::scoped_refptr<MediaStreamTrackInterface> track, + const std::vector<std::string>& stream_ids = {}) { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->AddTrack(track, stream_ids); + } + + rtc::scoped_refptr<DataChannelInterface> CreateDataChannel( + const std::string& label) { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->CreateDataChannel(label); + } + + PeerConnectionInterface::SignalingState signaling_state() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->signaling_state(); + } + + bool IsIceGatheringDone() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->IsIceGatheringDone(); + } + + bool IsIceConnected() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->IsIceConnected(); + } + + rtc::scoped_refptr<const RTCStatsReport> GetStats() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->GetStats(); + } + + void DetachAecDump() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + if (audio_processing_) { + audio_processing_->DetachAecDump(); + } + } + + // Adds provided `candidates` to the owned peer connection. + bool AddIceCandidates( + std::vector<std::unique_ptr<IceCandidateInterface>> candidates); + + // Closes underlying peer connection and destroys all related objects freeing + // up related resources. + void Close(); + + protected: + friend class TestPeerFactory; + TestPeer(rtc::scoped_refptr<PeerConnectionFactoryInterface> pc_factory, + rtc::scoped_refptr<PeerConnectionInterface> pc, + std::unique_ptr<MockPeerConnectionObserver> observer, + Params params, + ConfigurableParams configurable_params, + std::vector<PeerConfigurer::VideoSource> video_sources, + rtc::scoped_refptr<AudioProcessing> audio_processing, + std::unique_ptr<rtc::Thread> worker_thread); + + private: + const Params params_; + + mutable Mutex mutex_; + ConfigurableParams configurable_params_ RTC_GUARDED_BY(mutex_); + + // Safety flag to protect all tasks posted on the signaling thread to not be + // executed after `wrapper_` object is destructed. + rtc::scoped_refptr<PendingTaskSafetyFlag> signaling_thread_task_safety_ = + nullptr; + + // Keeps ownership of worker thread. It has to be destroyed after `wrapper_`. + // `worker_thread_`can be null if the Peer use only one thread as both the + // worker thread and network thread. + std::unique_ptr<rtc::Thread> worker_thread_; + std::unique_ptr<PeerConnectionWrapper> wrapper_; + std::vector<PeerConfigurer::VideoSource> video_sources_; + rtc::scoped_refptr<AudioProcessing> audio_processing_; + + std::vector<std::unique_ptr<IceCandidateInterface>> remote_ice_candidates_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_TEST_PEER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/test_peer_factory.cc b/third_party/libwebrtc/test/pc/e2e/test_peer_factory.cc new file mode 100644 index 0000000000..7fc12f2c11 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/test_peer_factory.cc @@ -0,0 +1,374 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/test_peer_factory.h" + +#include <utility> + +#include "absl/memory/memory.h" +#include "absl/strings/string_view.h" +#include "api/task_queue/default_task_queue_factory.h" +#include "api/test/create_time_controller.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/pclf/peer_configurer.h" +#include "api/test/time_controller.h" +#include "api/transport/field_trial_based_config.h" +#include "api/video_codecs/builtin_video_decoder_factory.h" +#include "api/video_codecs/builtin_video_encoder_factory.h" +#include "media/engine/webrtc_media_engine.h" +#include "media/engine/webrtc_media_engine_defaults.h" +#include "modules/audio_processing/aec_dump/aec_dump_factory.h" +#include "p2p/client/basic_port_allocator.h" +#include "rtc_base/thread.h" +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h" +#include "test/pc/e2e/echo/echo_emulation.h" +#include "test/testsupport/copy_to_file_audio_capturer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using EmulatedSFUConfigMap = + ::webrtc::webrtc_pc_e2e::QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap; + +constexpr int16_t kGeneratedAudioMaxAmplitude = 32000; +constexpr int kDefaultSamplingFrequencyInHz = 48000; + +// Sets mandatory entities in injectable components like `pcf_dependencies` +// and `pc_dependencies` if they are omitted. Also setup required +// dependencies, that won't be specially provided by factory and will be just +// transferred to peer connection creation code. +void SetMandatoryEntities(InjectableComponents* components, + TimeController& time_controller) { + RTC_DCHECK(components->pcf_dependencies); + RTC_DCHECK(components->pc_dependencies); + + // Setup required peer connection factory dependencies. + if (components->pcf_dependencies->task_queue_factory == nullptr) { + components->pcf_dependencies->task_queue_factory = + time_controller.CreateTaskQueueFactory(); + } + if (components->pcf_dependencies->call_factory == nullptr) { + components->pcf_dependencies->call_factory = + CreateTimeControllerBasedCallFactory(&time_controller); + } + if (components->pcf_dependencies->event_log_factory == nullptr) { + components->pcf_dependencies->event_log_factory = + std::make_unique<RtcEventLogFactory>( + components->pcf_dependencies->task_queue_factory.get()); + } + if (!components->pcf_dependencies->trials) { + components->pcf_dependencies->trials = + std::make_unique<FieldTrialBasedConfig>(); + } +} + +// Returns mapping from stream label to optional spatial index. +// If we have stream label "Foo" and mapping contains +// 1. `absl::nullopt` means all simulcast/SVC streams are required +// 2. Concrete value means that particular simulcast/SVC stream have to be +// analyzed. +EmulatedSFUConfigMap CalculateRequiredSpatialIndexPerStream( + const std::vector<VideoConfig>& video_configs) { + EmulatedSFUConfigMap result; + for (auto& video_config : video_configs) { + // Stream label should be set by fixture implementation here. + RTC_DCHECK(video_config.stream_label); + bool res = result + .insert({*video_config.stream_label, + video_config.emulated_sfu_config}) + .second; + RTC_DCHECK(res) << "Duplicate video_config.stream_label=" + << *video_config.stream_label; + } + return result; +} + +std::unique_ptr<TestAudioDeviceModule::Renderer> CreateAudioRenderer( + const absl::optional<RemotePeerAudioConfig>& config) { + if (!config) { + // Return default renderer because we always require some renderer. + return TestAudioDeviceModule::CreateDiscardRenderer( + kDefaultSamplingFrequencyInHz); + } + if (config->output_file_name) { + return TestAudioDeviceModule::CreateBoundedWavFileWriter( + config->output_file_name.value(), config->sampling_frequency_in_hz); + } + return TestAudioDeviceModule::CreateDiscardRenderer( + config->sampling_frequency_in_hz); +} + +std::unique_ptr<TestAudioDeviceModule::Capturer> CreateAudioCapturer( + const absl::optional<AudioConfig>& audio_config) { + if (!audio_config) { + // If we have no audio config we still need to provide some audio device. + // In such case use generated capturer. Despite of we provided audio here, + // in test media setup audio stream won't be added into peer connection. + return TestAudioDeviceModule::CreatePulsedNoiseCapturer( + kGeneratedAudioMaxAmplitude, kDefaultSamplingFrequencyInHz); + } + + switch (audio_config->mode) { + case AudioConfig::Mode::kGenerated: + return TestAudioDeviceModule::CreatePulsedNoiseCapturer( + kGeneratedAudioMaxAmplitude, audio_config->sampling_frequency_in_hz); + case AudioConfig::Mode::kFile: + RTC_DCHECK(audio_config->input_file_name); + return TestAudioDeviceModule::CreateWavFileReader( + audio_config->input_file_name.value(), /*repeat=*/true); + } +} + +rtc::scoped_refptr<AudioDeviceModule> CreateAudioDeviceModule( + absl::optional<AudioConfig> audio_config, + absl::optional<RemotePeerAudioConfig> remote_audio_config, + absl::optional<EchoEmulationConfig> echo_emulation_config, + TaskQueueFactory* task_queue_factory) { + std::unique_ptr<TestAudioDeviceModule::Renderer> renderer = + CreateAudioRenderer(remote_audio_config); + std::unique_ptr<TestAudioDeviceModule::Capturer> capturer = + CreateAudioCapturer(audio_config); + RTC_DCHECK(renderer); + RTC_DCHECK(capturer); + + // Setup echo emulation if required. + if (echo_emulation_config) { + capturer = std::make_unique<EchoEmulatingCapturer>(std::move(capturer), + *echo_emulation_config); + renderer = std::make_unique<EchoEmulatingRenderer>( + std::move(renderer), + static_cast<EchoEmulatingCapturer*>(capturer.get())); + } + + // Setup input stream dumping if required. + if (audio_config && audio_config->input_dump_file_name) { + capturer = std::make_unique<test::CopyToFileAudioCapturer>( + std::move(capturer), audio_config->input_dump_file_name.value()); + } + + return TestAudioDeviceModule::Create(task_queue_factory, std::move(capturer), + std::move(renderer), /*speed=*/1.f); +} + +std::unique_ptr<cricket::MediaEngineInterface> CreateMediaEngine( + PeerConnectionFactoryComponents* pcf_dependencies, + rtc::scoped_refptr<AudioDeviceModule> audio_device_module) { + cricket::MediaEngineDependencies media_deps; + media_deps.task_queue_factory = pcf_dependencies->task_queue_factory.get(); + media_deps.adm = audio_device_module; + media_deps.audio_processing = pcf_dependencies->audio_processing; + media_deps.audio_mixer = pcf_dependencies->audio_mixer; + media_deps.video_encoder_factory = + std::move(pcf_dependencies->video_encoder_factory); + media_deps.video_decoder_factory = + std::move(pcf_dependencies->video_decoder_factory); + webrtc::SetMediaEngineDefaults(&media_deps); + RTC_DCHECK(pcf_dependencies->trials); + media_deps.trials = pcf_dependencies->trials.get(); + + return cricket::CreateMediaEngine(std::move(media_deps)); +} + +void WrapVideoEncoderFactory( + absl::string_view peer_name, + double bitrate_multiplier, + EmulatedSFUConfigMap stream_to_sfu_config, + PeerConnectionFactoryComponents* pcf_dependencies, + VideoQualityAnalyzerInjectionHelper* video_analyzer_helper) { + std::unique_ptr<VideoEncoderFactory> video_encoder_factory; + if (pcf_dependencies->video_encoder_factory != nullptr) { + video_encoder_factory = std::move(pcf_dependencies->video_encoder_factory); + } else { + video_encoder_factory = CreateBuiltinVideoEncoderFactory(); + } + pcf_dependencies->video_encoder_factory = + video_analyzer_helper->WrapVideoEncoderFactory( + peer_name, std::move(video_encoder_factory), bitrate_multiplier, + std::move(stream_to_sfu_config)); +} + +void WrapVideoDecoderFactory( + absl::string_view peer_name, + PeerConnectionFactoryComponents* pcf_dependencies, + VideoQualityAnalyzerInjectionHelper* video_analyzer_helper) { + std::unique_ptr<VideoDecoderFactory> video_decoder_factory; + if (pcf_dependencies->video_decoder_factory != nullptr) { + video_decoder_factory = std::move(pcf_dependencies->video_decoder_factory); + } else { + video_decoder_factory = CreateBuiltinVideoDecoderFactory(); + } + pcf_dependencies->video_decoder_factory = + video_analyzer_helper->WrapVideoDecoderFactory( + peer_name, std::move(video_decoder_factory)); +} + +// Creates PeerConnectionFactoryDependencies objects, providing entities +// from InjectableComponents::PeerConnectionFactoryComponents. +PeerConnectionFactoryDependencies CreatePCFDependencies( + std::unique_ptr<PeerConnectionFactoryComponents> pcf_dependencies, + std::unique_ptr<cricket::MediaEngineInterface> media_engine, + rtc::Thread* signaling_thread, + rtc::Thread* worker_thread, + rtc::Thread* network_thread) { + PeerConnectionFactoryDependencies pcf_deps; + pcf_deps.signaling_thread = signaling_thread; + pcf_deps.worker_thread = worker_thread; + pcf_deps.network_thread = network_thread; + pcf_deps.media_engine = std::move(media_engine); + + pcf_deps.call_factory = std::move(pcf_dependencies->call_factory); + pcf_deps.event_log_factory = std::move(pcf_dependencies->event_log_factory); + pcf_deps.task_queue_factory = std::move(pcf_dependencies->task_queue_factory); + + if (pcf_dependencies->fec_controller_factory != nullptr) { + pcf_deps.fec_controller_factory = + std::move(pcf_dependencies->fec_controller_factory); + } + if (pcf_dependencies->network_controller_factory != nullptr) { + pcf_deps.network_controller_factory = + std::move(pcf_dependencies->network_controller_factory); + } + if (pcf_dependencies->neteq_factory != nullptr) { + pcf_deps.neteq_factory = std::move(pcf_dependencies->neteq_factory); + } + if (pcf_dependencies->trials != nullptr) { + pcf_deps.trials = std::move(pcf_dependencies->trials); + } + + return pcf_deps; +} + +// Creates PeerConnectionDependencies objects, providing entities +// from InjectableComponents::PeerConnectionComponents. +PeerConnectionDependencies CreatePCDependencies( + MockPeerConnectionObserver* observer, + uint32_t port_allocator_extra_flags, + std::unique_ptr<PeerConnectionComponents> pc_dependencies) { + PeerConnectionDependencies pc_deps(observer); + + auto port_allocator = std::make_unique<cricket::BasicPortAllocator>( + pc_dependencies->network_manager, pc_dependencies->packet_socket_factory); + + // This test does not support TCP + int flags = port_allocator_extra_flags | cricket::PORTALLOCATOR_DISABLE_TCP; + port_allocator->set_flags(port_allocator->flags() | flags); + + pc_deps.allocator = std::move(port_allocator); + + if (pc_dependencies->async_resolver_factory != nullptr) { + pc_deps.async_resolver_factory = + std::move(pc_dependencies->async_resolver_factory); + } + if (pc_dependencies->cert_generator != nullptr) { + pc_deps.cert_generator = std::move(pc_dependencies->cert_generator); + } + if (pc_dependencies->tls_cert_verifier != nullptr) { + pc_deps.tls_cert_verifier = std::move(pc_dependencies->tls_cert_verifier); + } + if (pc_dependencies->ice_transport_factory != nullptr) { + pc_deps.ice_transport_factory = + std::move(pc_dependencies->ice_transport_factory); + } + return pc_deps; +} + +} // namespace + +absl::optional<RemotePeerAudioConfig> RemotePeerAudioConfig::Create( + absl::optional<AudioConfig> config) { + if (!config) { + return absl::nullopt; + } + return RemotePeerAudioConfig(config.value()); +} + +std::unique_ptr<TestPeer> TestPeerFactory::CreateTestPeer( + std::unique_ptr<PeerConfigurer> configurer, + std::unique_ptr<MockPeerConnectionObserver> observer, + absl::optional<RemotePeerAudioConfig> remote_audio_config, + absl::optional<EchoEmulationConfig> echo_emulation_config) { + std::unique_ptr<InjectableComponents> components = + configurer->ReleaseComponents(); + std::unique_ptr<Params> params = configurer->ReleaseParams(); + std::unique_ptr<ConfigurableParams> configurable_params = + configurer->ReleaseConfigurableParams(); + std::vector<PeerConfigurer::VideoSource> video_sources = + configurer->ReleaseVideoSources(); + RTC_DCHECK(components); + RTC_DCHECK(params); + RTC_DCHECK(configurable_params); + RTC_DCHECK_EQ(configurable_params->video_configs.size(), + video_sources.size()); + SetMandatoryEntities(components.get(), time_controller_); + params->rtc_configuration.sdp_semantics = SdpSemantics::kUnifiedPlan; + + // Create peer connection factory. + if (components->pcf_dependencies->audio_processing == nullptr) { + components->pcf_dependencies->audio_processing = + webrtc::AudioProcessingBuilder().Create(); + } + if (params->aec_dump_path) { + components->pcf_dependencies->audio_processing->CreateAndAttachAecDump( + *params->aec_dump_path, -1, task_queue_); + } + rtc::scoped_refptr<AudioDeviceModule> audio_device_module = + CreateAudioDeviceModule( + params->audio_config, remote_audio_config, echo_emulation_config, + components->pcf_dependencies->task_queue_factory.get()); + WrapVideoEncoderFactory( + params->name.value(), params->video_encoder_bitrate_multiplier, + CalculateRequiredSpatialIndexPerStream( + configurable_params->video_configs), + components->pcf_dependencies.get(), video_analyzer_helper_); + WrapVideoDecoderFactory(params->name.value(), + components->pcf_dependencies.get(), + video_analyzer_helper_); + std::unique_ptr<cricket::MediaEngineInterface> media_engine = + CreateMediaEngine(components->pcf_dependencies.get(), + audio_device_module); + + std::unique_ptr<rtc::Thread> owned_worker_thread = + components->worker_thread != nullptr + ? nullptr + : time_controller_.CreateThread("worker_thread"); + if (components->worker_thread == nullptr) { + components->worker_thread = owned_worker_thread.get(); + } + + // Store `webrtc::AudioProcessing` into local variable before move of + // `components->pcf_dependencies` + rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing = + components->pcf_dependencies->audio_processing; + PeerConnectionFactoryDependencies pcf_deps = CreatePCFDependencies( + std::move(components->pcf_dependencies), std::move(media_engine), + signaling_thread_, components->worker_thread, components->network_thread); + rtc::scoped_refptr<PeerConnectionFactoryInterface> peer_connection_factory = + CreateModularPeerConnectionFactory(std::move(pcf_deps)); + + // Create peer connection. + PeerConnectionDependencies pc_deps = + CreatePCDependencies(observer.get(), params->port_allocator_extra_flags, + std::move(components->pc_dependencies)); + rtc::scoped_refptr<PeerConnectionInterface> peer_connection = + peer_connection_factory + ->CreatePeerConnectionOrError(params->rtc_configuration, + std::move(pc_deps)) + .MoveValue(); + peer_connection->SetBitrate(params->bitrate_settings); + + return absl::WrapUnique( + new TestPeer(peer_connection_factory, peer_connection, + std::move(observer), std::move(*params), + std::move(*configurable_params), std::move(video_sources), + audio_processing, std::move(owned_worker_thread))); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/test_peer_factory.h b/third_party/libwebrtc/test/pc/e2e/test_peer_factory.h new file mode 100644 index 0000000000..f2698e2a15 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/test_peer_factory.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_TEST_PEER_FACTORY_H_ +#define TEST_PC_E2E_TEST_PEER_FACTORY_H_ + +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/rtc_event_log/rtc_event_log_factory.h" +#include "api/test/pclf/media_configuration.h" +#include "api/test/pclf/media_quality_test_params.h" +#include "api/test/pclf/peer_configurer.h" +#include "api/test/time_controller.h" +#include "modules/audio_device/include/test_audio_device.h" +#include "rtc_base/task_queue.h" +#include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h" +#include "test/pc/e2e/test_peer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +struct RemotePeerAudioConfig { + explicit RemotePeerAudioConfig(AudioConfig config) + : sampling_frequency_in_hz(config.sampling_frequency_in_hz), + output_file_name(config.output_dump_file_name) {} + + static absl::optional<RemotePeerAudioConfig> Create( + absl::optional<AudioConfig> config); + + int sampling_frequency_in_hz; + absl::optional<std::string> output_file_name; +}; + +class TestPeerFactory { + public: + // Creates a test peer factory. + // `signaling_thread` will be used as a signaling thread for all peers created + // by this factory. + // `time_controller` will be used to create required threads, task queue + // factories and call factory. + // `video_analyzer_helper` will be used to setup video quality analysis for + // created peers. + // `task_queue` will be used for AEC dump if it is requested. + TestPeerFactory(rtc::Thread* signaling_thread, + TimeController& time_controller, + VideoQualityAnalyzerInjectionHelper* video_analyzer_helper, + rtc::TaskQueue* task_queue) + : signaling_thread_(signaling_thread), + time_controller_(time_controller), + video_analyzer_helper_(video_analyzer_helper), + task_queue_(task_queue) {} + + // Setups all components, that should be provided to WebRTC + // PeerConnectionFactory and PeerConnection creation methods, + // also will setup dependencies, that are required for media analyzers + // injection. + std::unique_ptr<TestPeer> CreateTestPeer( + std::unique_ptr<PeerConfigurer> configurer, + std::unique_ptr<MockPeerConnectionObserver> observer, + absl::optional<RemotePeerAudioConfig> remote_audio_config, + absl::optional<EchoEmulationConfig> echo_emulation_config); + + private: + rtc::Thread* signaling_thread_; + TimeController& time_controller_; + VideoQualityAnalyzerInjectionHelper* video_analyzer_helper_; + rtc::TaskQueue* task_queue_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_TEST_PEER_FACTORY_H_ diff --git a/third_party/libwebrtc/test/pc/sctp/BUILD.gn b/third_party/libwebrtc/test/pc/sctp/BUILD.gn new file mode 100644 index 0000000000..f088a5b20c --- /dev/null +++ b/third_party/libwebrtc/test/pc/sctp/BUILD.gn @@ -0,0 +1,18 @@ +# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../webrtc.gni") + +rtc_source_set("fake_sctp_transport") { + visibility = [ "*" ] + sources = [ "fake_sctp_transport.h" ] + deps = [ + "../../../api/transport:sctp_transport_factory_interface", + "../../../media:rtc_data_sctp_transport_internal", + ] +} diff --git a/third_party/libwebrtc/test/pc/sctp/fake_sctp_transport.h b/third_party/libwebrtc/test/pc/sctp/fake_sctp_transport.h new file mode 100644 index 0000000000..a1bb0e219c --- /dev/null +++ b/third_party/libwebrtc/test/pc/sctp/fake_sctp_transport.h @@ -0,0 +1,79 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_SCTP_FAKE_SCTP_TRANSPORT_H_ +#define TEST_PC_SCTP_FAKE_SCTP_TRANSPORT_H_ + +#include <memory> + +#include "api/transport/sctp_transport_factory_interface.h" +#include "media/sctp/sctp_transport_internal.h" + +// Used for tests in this file to verify that PeerConnection responds to signals +// from the SctpTransport correctly, and calls Start with the correct +// local/remote ports. +class FakeSctpTransport : public cricket::SctpTransportInternal { + public: + void SetOnConnectedCallback(std::function<void()> callback) override {} + void SetDataChannelSink(webrtc::DataChannelSink* sink) override {} + void SetDtlsTransport(rtc::PacketTransportInternal* transport) override {} + bool Start(int local_port, int remote_port, int max_message_size) override { + local_port_.emplace(local_port); + remote_port_.emplace(remote_port); + max_message_size_ = max_message_size; + return true; + } + bool OpenStream(int sid) override { return true; } + bool ResetStream(int sid) override { return true; } + bool SendData(int sid, + const webrtc::SendDataParams& params, + const rtc::CopyOnWriteBuffer& payload, + cricket::SendDataResult* result = nullptr) override { + return true; + } + bool ReadyToSendData() override { return true; } + void set_debug_name_for_testing(const char* debug_name) override {} + + int max_message_size() const { return max_message_size_; } + absl::optional<int> max_outbound_streams() const { return absl::nullopt; } + absl::optional<int> max_inbound_streams() const { return absl::nullopt; } + int local_port() const { + RTC_DCHECK(local_port_); + return *local_port_; + } + int remote_port() const { + RTC_DCHECK(remote_port_); + return *remote_port_; + } + + private: + absl::optional<int> local_port_; + absl::optional<int> remote_port_; + int max_message_size_; +}; + +class FakeSctpTransportFactory : public webrtc::SctpTransportFactoryInterface { + public: + std::unique_ptr<cricket::SctpTransportInternal> CreateSctpTransport( + rtc::PacketTransportInternal*) override { + last_fake_sctp_transport_ = new FakeSctpTransport(); + return std::unique_ptr<cricket::SctpTransportInternal>( + last_fake_sctp_transport_); + } + + FakeSctpTransport* last_fake_sctp_transport() { + return last_fake_sctp_transport_; + } + + private: + FakeSctpTransport* last_fake_sctp_transport_ = nullptr; +}; + +#endif // TEST_PC_SCTP_FAKE_SCTP_TRANSPORT_H_ |
