diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 09:22:09 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 09:22:09 +0000 |
commit | 43a97878ce14b72f0981164f87f2e35e14151312 (patch) | |
tree | 620249daf56c0258faa40cbdcf9cfba06de2a846 /third_party/libwebrtc/test/pc/e2e | |
parent | Initial commit. (diff) | |
download | firefox-43a97878ce14b72f0981164f87f2e35e14151312.tar.xz firefox-43a97878ce14b72f0981164f87f2e35e14151312.zip |
Adding upstream version 110.0.1.upstream/110.0.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/libwebrtc/test/pc/e2e')
83 files changed, 16670 insertions, 0 deletions
diff --git a/third_party/libwebrtc/test/pc/e2e/BUILD.gn b/third_party/libwebrtc/test/pc/e2e/BUILD.gn new file mode 100644 index 0000000000..b0f1f0f86f --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/BUILD.gn @@ -0,0 +1,923 @@ +# Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../webrtc.gni") + +if (!build_with_chromium) { + group("e2e") { + testonly = true + + deps = [ + ":encoded_image_data_injector_api", + ":example_video_quality_analyzer", + ":quality_analyzing_video_decoder", + ":quality_analyzing_video_encoder", + ":single_process_encoded_image_data_injector", + ":video_frame_tracking_id_injector", + ] + if (rtc_include_tests) { + deps += [ + ":peerconnection_quality_test", + ":test_peer", + ":video_quality_analyzer_injection_helper", + ] + } + } + + if (rtc_include_tests) { + group("e2e_unittests") { + testonly = true + + deps = [ + ":default_video_quality_analyzer_frames_comparator_test", + ":default_video_quality_analyzer_stream_state_test", + ":default_video_quality_analyzer_test", + ":multi_reader_queue_test", + ":names_collection_test", + ":peer_connection_e2e_smoke_test", + ":single_process_encoded_image_data_injector_unittest", + ":stats_poller_test", + ":video_frame_tracking_id_injector_unittest", + ] + } + } + + rtc_library("peer_connection_quality_test_params") { + visibility = [ "*" ] + testonly = true + sources = [ "peer_connection_quality_test_params.h" ] + + deps = [ + "../../../api:callfactory_api", + "../../../api:fec_controller_api", + "../../../api:field_trials_view", + "../../../api:libjingle_peerconnection_api", + "../../../api:packet_socket_factory", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api/audio:audio_mixer_api", + "../../../api/rtc_event_log", + "../../../api/task_queue", + "../../../api/transport:network_control", + "../../../api/video_codecs:video_codecs_api", + "../../../modules/audio_processing:api", + "../../../p2p:rtc_p2p", + "../../../rtc_base", + "../../../rtc_base:threading", + ] + } + + rtc_library("encoded_image_data_injector_api") { + visibility = [ "*" ] + testonly = true + sources = [ "analyzer/video/encoded_image_data_injector.h" ] + + deps = [ "../../../api/video:encoded_image" ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_library("single_process_encoded_image_data_injector") { + visibility = [ "*" ] + testonly = true + sources = [ + "analyzer/video/single_process_encoded_image_data_injector.cc", + "analyzer/video/single_process_encoded_image_data_injector.h", + ] + + deps = [ + ":encoded_image_data_injector_api", + "../../../api/video:encoded_image", + "../../../rtc_base:checks", + "../../../rtc_base:criticalsection", + "../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/memory" ] + } + + rtc_library("video_frame_tracking_id_injector") { + visibility = [ "*" ] + testonly = true + sources = [ + "analyzer/video/video_frame_tracking_id_injector.cc", + "analyzer/video/video_frame_tracking_id_injector.h", + ] + + deps = [ + ":encoded_image_data_injector_api", + "../../../api/video:encoded_image", + "../../../rtc_base:checks", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/memory" ] + } + + rtc_library("simulcast_dummy_buffer_helper") { + visibility = [ "*" ] + testonly = true + sources = [ + "analyzer/video/simulcast_dummy_buffer_helper.cc", + "analyzer/video/simulcast_dummy_buffer_helper.h", + ] + deps = [ "../../../api/video:video_frame" ] + } + + rtc_library("quality_analyzing_video_decoder") { + visibility = [ "*" ] + testonly = true + sources = [ + "analyzer/video/quality_analyzing_video_decoder.cc", + "analyzer/video/quality_analyzing_video_decoder.h", + ] + deps = [ + ":encoded_image_data_injector_api", + ":simulcast_dummy_buffer_helper", + "../../../api:video_quality_analyzer_api", + "../../../api/video:encoded_image", + "../../../api/video:video_frame", + "../../../api/video:video_rtp_headers", + "../../../api/video_codecs:video_codecs_api", + "../../../modules/video_coding:video_codec_interface", + "../../../rtc_base:criticalsection", + "../../../rtc_base:logging", + "../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + + rtc_library("quality_analyzing_video_encoder") { + visibility = [ "*" ] + testonly = true + sources = [ + "analyzer/video/quality_analyzing_video_encoder.cc", + "analyzer/video/quality_analyzing_video_encoder.h", + ] + deps = [ + ":encoded_image_data_injector_api", + "../../../api:video_quality_analyzer_api", + "../../../api/video:encoded_image", + "../../../api/video:video_frame", + "../../../api/video:video_rtp_headers", + "../../../api/video_codecs:video_codecs_api", + "../../../modules/video_coding:video_codec_interface", + "../../../rtc_base:criticalsection", + "../../../rtc_base:logging", + "../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + + if (rtc_include_tests) { + rtc_library("video_quality_analyzer_injection_helper") { + visibility = [ "*" ] + testonly = true + sources = [ + "analyzer/video/video_quality_analyzer_injection_helper.cc", + "analyzer/video/video_quality_analyzer_injection_helper.h", + ] + deps = [ + ":encoded_image_data_injector_api", + ":quality_analyzing_video_decoder", + ":quality_analyzing_video_encoder", + ":simulcast_dummy_buffer_helper", + "../..:fixed_fps_video_frame_writer_adapter", + "../..:test_renderer", + "../../../api:array_view", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:stats_observer_interface", + "../../../api:video_quality_analyzer_api", + "../../../api/video:video_frame", + "../../../api/video:video_rtp_headers", + "../../../api/video_codecs:video_codecs_api", + "../../../rtc_base:criticalsection", + "../../../rtc_base:stringutils", + "../../../rtc_base/synchronization:mutex", + "../../../system_wrappers", + "../../../test:video_test_common", + "../../../test:video_test_support", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + ] + } + + rtc_library("echo_emulation") { + visibility = [ "*" ] + testonly = true + sources = [ + "echo/echo_emulation.cc", + "echo/echo_emulation.h", + ] + deps = [ + "../../../api:peer_connection_quality_test_fixture_api", + "../../../modules/audio_device:audio_device_impl", + "../../../rtc_base:swap_queue", + ] + } + + rtc_library("test_peer") { + visibility = [ "*" ] + testonly = true + sources = [ + "test_peer.cc", + "test_peer.h", + ] + deps = [ + ":peer_configurer", + ":peer_connection_quality_test_params", + ":stats_provider", + "../../../api:frame_generator_api", + "../../../api:function_view", + "../../../api:libjingle_peerconnection_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:scoped_refptr", + "../../../api:sequence_checker", + "../../../api/task_queue:pending_task_safety_flag", + "../../../modules/audio_processing:api", + "../../../pc:peerconnection_wrapper", + "../../../rtc_base:logging", + "../../../rtc_base:refcount", + "../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:variant", + ] + } + + rtc_library("test_peer_factory") { + visibility = [ "*" ] + testonly = true + sources = [ + "test_peer_factory.cc", + "test_peer_factory.h", + ] + deps = [ + ":echo_emulation", + ":peer_configurer", + ":peer_connection_quality_test_params", + ":quality_analyzing_video_encoder", + ":test_peer", + ":video_quality_analyzer_injection_helper", + "../..:copy_to_file_audio_capturer", + "../../../api:create_time_controller", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:time_controller", + "../../../api/rtc_event_log:rtc_event_log_factory", + "../../../api/task_queue:default_task_queue_factory", + "../../../api/transport:field_trial_based_config", + "../../../api/video_codecs:builtin_video_decoder_factory", + "../../../api/video_codecs:builtin_video_encoder_factory", + "../../../media:rtc_audio_video", + "../../../media:rtc_media_engine_defaults", + "../../../modules/audio_device:audio_device_impl", + "../../../modules/audio_processing/aec_dump", + "../../../p2p:rtc_p2p", + "../../../rtc_base:rtc_task_queue", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + ] + } + + rtc_library("media_helper") { + visibility = [ "*" ] + testonly = true + sources = [ + "media/media_helper.cc", + "media/media_helper.h", + "media/test_video_capturer_video_track_source.h", + ] + deps = [ + ":peer_configurer", + ":test_peer", + ":video_quality_analyzer_injection_helper", + "../..:fileutils", + "../..:platform_video_capturer", + "../..:video_test_common", + "../../../api:create_frame_generator", + "../../../api:frame_generator_api", + "../../../api:media_stream_interface", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api/video:video_frame", + "../../../pc:session_description", + "../../../pc:video_track_source", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:variant" ] + } + + rtc_library("peer_configurer") { + visibility = [ "*" ] + testonly = true + sources = [ + "peer_configurer.cc", + "peer_configurer.h", + ] + deps = [ + ":peer_connection_quality_test_params", + "../..:fileutils", + "../../../api:callfactory_api", + "../../../api:create_peer_connection_quality_test_frame_generator", + "../../../api:fec_controller_api", + "../../../api:packet_socket_factory", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api/audio:audio_mixer_api", + "../../../api/rtc_event_log", + "../../../api/task_queue", + "../../../api/transport:network_control", + "../../../api/video_codecs:video_codecs_api", + "../../../modules/audio_processing:api", + "../../../rtc_base", + "../../../rtc_base:macromagic", + "../../../rtc_base:threading", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + + rtc_library("test_activities_executor") { + visibility = [ "*" ] + testonly = true + sources = [ + "test_activities_executor.cc", + "test_activities_executor.h", + ] + deps = [ + "../../../api/task_queue", + "../../../api/units:time_delta", + "../../../api/units:timestamp", + "../../../rtc_base:checks", + "../../../rtc_base:criticalsection", + "../../../rtc_base:logging", + "../../../rtc_base:task_queue_for_test", + "../../../rtc_base/synchronization:mutex", + "../../../rtc_base/task_utils:repeating_task", + "../../../system_wrappers", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + + rtc_library("peerconnection_quality_test") { + visibility = [ "*" ] + testonly = true + + sources = [ + "peer_connection_quality_test.cc", + "peer_connection_quality_test.h", + ] + deps = [ + ":analyzer_helper", + ":cross_media_metrics_reporter", + ":default_audio_quality_analyzer", + ":default_video_quality_analyzer", + ":media_helper", + ":peer_configurer", + ":peer_connection_quality_test_params", + ":sdp_changer", + ":single_process_encoded_image_data_injector", + ":stats_poller", + ":test_activities_executor", + ":test_peer", + ":test_peer_factory", + ":video_frame_tracking_id_injector", + ":video_quality_analyzer_injection_helper", + ":video_quality_metrics_reporter", + "../..:field_trial", + "../..:fileutils", + "../..:perf_test", + "../../../api:audio_quality_analyzer_api", + "../../../api:libjingle_peerconnection_api", + "../../../api:media_stream_interface", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:rtc_event_log_output_file", + "../../../api:scoped_refptr", + "../../../api:time_controller", + "../../../api:video_quality_analyzer_api", + "../../../api/rtc_event_log", + "../../../api/task_queue", + "../../../api/units:time_delta", + "../../../api/units:timestamp", + "../../../pc:pc_test_utils", + "../../../pc:sdp_utils", + "../../../rtc_base", + "../../../rtc_base:gunit_helpers", + "../../../rtc_base:macromagic", + "../../../rtc_base:safe_conversions", + "../../../rtc_base:stringutils", + "../../../rtc_base:task_queue_for_test", + "../../../rtc_base:threading", + "../../../rtc_base/synchronization:mutex", + "../../../system_wrappers", + "../../../system_wrappers:field_trial", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + + rtc_library("single_process_encoded_image_data_injector_unittest") { + testonly = true + sources = [ + "analyzer/video/single_process_encoded_image_data_injector_unittest.cc", + ] + deps = [ + ":single_process_encoded_image_data_injector", + "../../../api/video:encoded_image", + "../../../rtc_base:buffer", + "../../../test:test_support", + ] + } + + rtc_library("video_frame_tracking_id_injector_unittest") { + testonly = true + sources = + [ "analyzer/video/video_frame_tracking_id_injector_unittest.cc" ] + deps = [ + ":video_frame_tracking_id_injector", + "../../../api/video:encoded_image", + "../../../rtc_base:buffer", + "../../../test:test_support", + ] + } + + peer_connection_e2e_smoke_test_resources = [ + "../../../resources/pc_quality_smoke_test_alice_source.wav", + "../../../resources/pc_quality_smoke_test_bob_source.wav", + ] + if (is_ios) { + bundle_data("peer_connection_e2e_smoke_test_resources_bundle_data") { + testonly = true + sources = peer_connection_e2e_smoke_test_resources + outputs = [ "{{bundle_resources_dir}}/{{source_file_part}}" ] + } + } + + rtc_library("peer_connection_e2e_smoke_test") { + testonly = true + + sources = [ "peer_connection_e2e_smoke_test.cc" ] + deps = [ + ":default_audio_quality_analyzer", + ":default_video_quality_analyzer", + ":default_video_quality_analyzer_shared", + ":network_quality_metrics_reporter", + ":stats_based_network_quality_metrics_reporter", + "../../../api:callfactory_api", + "../../../api:create_network_emulation_manager", + "../../../api:create_peer_connection_quality_test_frame_generator", + "../../../api:create_peerconnection_quality_test_fixture", + "../../../api:libjingle_peerconnection_api", + "../../../api:media_stream_interface", + "../../../api:network_emulation_manager_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:scoped_refptr", + "../../../api:simulated_network_api", + "../../../api/audio_codecs:builtin_audio_decoder_factory", + "../../../api/audio_codecs:builtin_audio_encoder_factory", + "../../../api/video_codecs:builtin_video_decoder_factory", + "../../../api/video_codecs:builtin_video_encoder_factory", + "../../../call:simulated_network", + "../../../media:rtc_audio_video", + "../../../modules/audio_device:audio_device_impl", + "../../../p2p:rtc_p2p", + "../../../pc:pc_test_utils", + "../../../pc:peerconnection_wrapper", + "../../../rtc_base", + "../../../rtc_base:gunit_helpers", + "../../../rtc_base:logging", + "../../../rtc_base:rtc_event", + "../../../system_wrappers:field_trial", + "../../../test:field_trial", + "../../../test:fileutils", + "../../../test:test_support", + ] + data = peer_connection_e2e_smoke_test_resources + if (is_mac || is_ios) { + deps += [ "../../../modules/video_coding:objc_codec_factory_helper" ] + } + if (is_ios) { + deps += [ ":peer_connection_e2e_smoke_test_resources_bundle_data" ] + } + } + + rtc_library("stats_provider") { + visibility = [ "*" ] + testonly = true + sources = [ "stats_provider.h" ] + deps = [ "../../../api:rtc_stats_api" ] + } + + rtc_library("stats_poller") { + visibility = [ "*" ] + testonly = true + sources = [ + "stats_poller.cc", + "stats_poller.h", + ] + deps = [ + ":stats_provider", + ":test_peer", + "../../../api:libjingle_peerconnection_api", + "../../../api:rtc_stats_api", + "../../../api:stats_observer_interface", + "../../../rtc_base:logging", + "../../../rtc_base:macromagic", + "../../../rtc_base/synchronization:mutex", + ] + } + + rtc_library("stats_poller_test") { + testonly = true + sources = [ "stats_poller_test.cc" ] + deps = [ + ":stats_poller", + "../..:test_support", + "../../../api:rtc_stats_api", + ] + } + + rtc_library("default_video_quality_analyzer_test") { + testonly = true + sources = [ "analyzer/video/default_video_quality_analyzer_test.cc" ] + deps = [ + ":default_video_quality_analyzer", + ":default_video_quality_analyzer_shared", + "../..:test_support", + "../../../api:create_frame_generator", + "../../../api:rtp_packet_info", + "../../../api/video:encoded_image", + "../../../api/video:video_frame", + "../../../common_video", + "../../../modules/rtp_rtcp:rtp_rtcp_format", + "../../../rtc_base:stringutils", + "../../../rtc_tools:video_quality_analysis", + "../../../system_wrappers", + ] + } + + rtc_library("default_video_quality_analyzer_frames_comparator_test") { + testonly = true + sources = [ "analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc" ] + deps = [ + ":default_video_quality_analyzer_internal", + ":default_video_quality_analyzer_shared", + "../..:test_support", + "../../../api:create_frame_generator", + "../../../api/units:timestamp", + "../../../rtc_base:stringutils", + "../../../system_wrappers", + ] + } + + rtc_library("names_collection_test") { + testonly = true + sources = [ "analyzer/video/names_collection_test.cc" ] + deps = [ + ":default_video_quality_analyzer_internal", + "../..:test_support", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings:strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + + rtc_library("multi_reader_queue_test") { + testonly = true + sources = [ "analyzer/video/multi_reader_queue_test.cc" ] + deps = [ + ":multi_reader_queue", + "../../../test:test_support", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_library("default_video_quality_analyzer_stream_state_test") { + testonly = true + sources = [ + "analyzer/video/default_video_quality_analyzer_stream_state_test.cc", + ] + deps = [ + ":default_video_quality_analyzer_internal", + "../../../api/units:timestamp", + "../../../test:test_support", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + } + + rtc_library("analyzer_helper") { + visibility = [ "*" ] + sources = [ + "analyzer_helper.cc", + "analyzer_helper.h", + ] + deps = [ + "../../../api:sequence_checker", + "../../../api:track_id_stream_info_map", + "../../../rtc_base:macromagic", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + + rtc_library("default_audio_quality_analyzer") { + visibility = [ "*" ] + testonly = true + sources = [ + "analyzer/audio/default_audio_quality_analyzer.cc", + "analyzer/audio/default_audio_quality_analyzer.h", + ] + + deps = [ + "../..:perf_test", + "../../../api:audio_quality_analyzer_api", + "../../../api:rtc_stats_api", + "../../../api:stats_observer_interface", + "../../../api:track_id_stream_info_map", + "../../../api/numerics", + "../../../api/units:time_delta", + "../../../api/units:timestamp", + "../../../rtc_base:criticalsection", + "../../../rtc_base:logging", + "../../../rtc_base:rtc_numerics", + "../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + + rtc_library("example_video_quality_analyzer") { + visibility = [ "*" ] + testonly = true + sources = [ + "analyzer/video/example_video_quality_analyzer.cc", + "analyzer/video/example_video_quality_analyzer.h", + ] + + deps = [ + "../../../api:array_view", + "../../../api:video_quality_analyzer_api", + "../../../api/video:encoded_image", + "../../../api/video:video_frame", + "../../../api/video:video_rtp_headers", + "../../../rtc_base:criticalsection", + "../../../rtc_base:logging", + "../../../rtc_base/synchronization:mutex", + ] + } + + rtc_library("video_quality_metrics_reporter") { + visibility = [ "*" ] + + testonly = true + sources = [ + "analyzer/video/video_quality_metrics_reporter.cc", + "analyzer/video/video_quality_metrics_reporter.h", + ] + deps = [ + "../..:perf_test", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:rtc_stats_api", + "../../../api:track_id_stream_info_map", + "../../../api/numerics", + "../../../api/units:data_rate", + "../../../api/units:data_size", + "../../../api/units:time_delta", + "../../../api/units:timestamp", + "../../../rtc_base:criticalsection", + "../../../rtc_base:rtc_numerics", + "../../../rtc_base/synchronization:mutex", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + + rtc_library("default_video_quality_analyzer") { + visibility = [ "*" ] + + testonly = true + sources = [ + "analyzer/video/default_video_quality_analyzer.cc", + "analyzer/video/default_video_quality_analyzer.h", + ] + + deps = [ + ":default_video_quality_analyzer_internal", + ":default_video_quality_analyzer_shared", + "../..:perf_test", + "../../../api:array_view", + "../../../api:video_quality_analyzer_api", + "../../../api/numerics", + "../../../api/units:data_size", + "../../../api/units:time_delta", + "../../../api/units:timestamp", + "../../../api/video:encoded_image", + "../../../api/video:video_frame", + "../../../api/video:video_frame_type", + "../../../api/video:video_rtp_headers", + "../../../common_video", + "../../../rtc_base:checks", + "../../../rtc_base:criticalsection", + "../../../rtc_base:logging", + "../../../rtc_base:macromagic", + "../../../rtc_base:platform_thread", + "../../../rtc_base:rtc_event", + "../../../rtc_base:rtc_numerics", + "../../../rtc_base:stringutils", + "../../../rtc_base:timeutils", + "../../../rtc_base/synchronization:mutex", + "../../../rtc_tools:video_quality_analysis", + "../../../system_wrappers", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + # This target contains implementation details of DefaultVideoQualityAnalyzer, + # so headers exported by it shouldn't be used in other places. + rtc_library("default_video_quality_analyzer_internal") { + visibility = [ + ":default_video_quality_analyzer", + ":default_video_quality_analyzer_frames_comparator_test", + ":default_video_quality_analyzer_stream_state_test", + ":names_collection_test", + ] + + testonly = true + sources = [ + "analyzer/video/default_video_quality_analyzer_cpu_measurer.cc", + "analyzer/video/default_video_quality_analyzer_cpu_measurer.h", + "analyzer/video/default_video_quality_analyzer_frame_in_flight.cc", + "analyzer/video/default_video_quality_analyzer_frame_in_flight.h", + "analyzer/video/default_video_quality_analyzer_frames_comparator.cc", + "analyzer/video/default_video_quality_analyzer_frames_comparator.h", + "analyzer/video/default_video_quality_analyzer_internal_shared_objects.cc", + "analyzer/video/default_video_quality_analyzer_internal_shared_objects.h", + "analyzer/video/default_video_quality_analyzer_stream_state.cc", + "analyzer/video/default_video_quality_analyzer_stream_state.h", + "analyzer/video/names_collection.cc", + "analyzer/video/names_collection.h", + ] + + deps = [ + ":default_video_quality_analyzer_shared", + ":multi_reader_queue", + "../../../api:array_view", + "../../../api:scoped_refptr", + "../../../api/numerics:numerics", + "../../../api/units:data_size", + "../../../api/units:timestamp", + "../../../api/video:video_frame", + "../../../api/video:video_frame_type", + "../../../common_video", + "../../../rtc_base:checks", + "../../../rtc_base:platform_thread", + "../../../rtc_base:rtc_base_tests_utils", + "../../../rtc_base:rtc_event", + "../../../rtc_base:stringutils", + "../../../rtc_base:timeutils", + "../../../rtc_base/synchronization:mutex", + "../../../rtc_tools:video_quality_analysis", + "../../../system_wrappers:system_wrappers", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings:strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + + rtc_library("default_video_quality_analyzer_shared") { + visibility = [ "*" ] + + testonly = true + sources = [ + "analyzer/video/default_video_quality_analyzer_shared_objects.cc", + "analyzer/video/default_video_quality_analyzer_shared_objects.h", + ] + + deps = [ + "../../../api/numerics:numerics", + "../../../api/units:timestamp", + "../../../rtc_base:checks", + "../../../rtc_base:stringutils", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_library("network_quality_metrics_reporter") { + visibility = [ "*" ] + testonly = true + sources = [ + "network_quality_metrics_reporter.cc", + "network_quality_metrics_reporter.h", + ] + deps = [ + "../..:perf_test", + "../../../api:network_emulation_manager_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:rtc_stats_api", + "../../../api:track_id_stream_info_map", + "../../../api/units:data_size", + "../../../rtc_base:criticalsection", + "../../../rtc_base:rtc_event", + "../../../rtc_base/synchronization:mutex", + "../../../system_wrappers:field_trial", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + + rtc_library("stats_based_network_quality_metrics_reporter") { + visibility = [ "*" ] + testonly = true + sources = [ + "stats_based_network_quality_metrics_reporter.cc", + "stats_based_network_quality_metrics_reporter.h", + ] + deps = [ + "../..:perf_test", + "../../../api:array_view", + "../../../api:network_emulation_manager_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:rtc_stats_api", + "../../../api:scoped_refptr", + "../../../api/numerics", + "../../../api/test/network_emulation", + "../../../api/units:data_rate", + "../../../api/units:data_size", + "../../../api/units:timestamp", + "../../../rtc_base", + "../../../rtc_base:ip_address", + "../../../rtc_base:rtc_event", + "../../../rtc_base:stringutils", + "../../../rtc_base/synchronization:mutex", + "../../../system_wrappers:field_trial", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + } + + rtc_library("cross_media_metrics_reporter") { + visibility = [ "*" ] + testonly = true + sources = [ + "cross_media_metrics_reporter.cc", + "cross_media_metrics_reporter.h", + ] + deps = [ + "../..:perf_test", + "../../../api:network_emulation_manager_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:rtc_stats_api", + "../../../api:track_id_stream_info_map", + "../../../api/numerics", + "../../../api/units:timestamp", + "../../../rtc_base:criticalsection", + "../../../rtc_base:rtc_event", + "../../../rtc_base:rtc_numerics", + "../../../rtc_base/synchronization:mutex", + "../../../system_wrappers:field_trial", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + + rtc_library("sdp_changer") { + visibility = [ "*" ] + testonly = true + sources = [ + "sdp/sdp_changer.cc", + "sdp/sdp_changer.h", + ] + deps = [ + "../../../api:array_view", + "../../../api:libjingle_peerconnection_api", + "../../../api:peer_connection_quality_test_fixture_api", + "../../../api:rtp_parameters", + "../../../media:rtc_media_base", + "../../../p2p:rtc_p2p", + "../../../pc:sdp_utils", + "../../../pc:session_description", + "../../../pc:simulcast_description", + "../../../rtc_base:stringutils", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings:strings", + "//third_party/abseil-cpp/absl/types:optional", + ] + } + + rtc_library("multi_reader_queue") { + visibility = [ "*" ] + testonly = true + sources = [ "analyzer/video/multi_reader_queue.h" ] + deps = [ "../../../rtc_base:checks" ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } +} diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc new file mode 100644 index 0000000000..0577bccb62 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h" + +#include "api/stats/rtc_stats.h" +#include "api/stats/rtcstats_objects.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +void DefaultAudioQualityAnalyzer::Start(std::string test_case_name, + TrackIdStreamInfoMap* analyzer_helper) { + test_case_name_ = std::move(test_case_name); + analyzer_helper_ = analyzer_helper; +} + +void DefaultAudioQualityAnalyzer::OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) { + auto stats = report->GetStatsOfType<RTCInboundRTPStreamStats>(); + + for (auto& stat : stats) { + if (!stat->kind.is_defined() || + !(*stat->kind == RTCMediaStreamTrackKind::kAudio)) { + continue; + } + + StatsSample sample; + sample.total_samples_received = + stat->total_samples_received.ValueOrDefault(0ul); + sample.concealed_samples = stat->concealed_samples.ValueOrDefault(0ul); + sample.removed_samples_for_acceleration = + stat->removed_samples_for_acceleration.ValueOrDefault(0ul); + sample.inserted_samples_for_deceleration = + stat->inserted_samples_for_deceleration.ValueOrDefault(0ul); + sample.silent_concealed_samples = + stat->silent_concealed_samples.ValueOrDefault(0ul); + sample.jitter_buffer_delay = + TimeDelta::Seconds(stat->jitter_buffer_delay.ValueOrDefault(0.)); + sample.jitter_buffer_target_delay = + TimeDelta::Seconds(stat->jitter_buffer_target_delay.ValueOrDefault(0.)); + sample.jitter_buffer_emitted_count = + stat->jitter_buffer_emitted_count.ValueOrDefault(0ul); + + const std::string stream_label = std::string( + analyzer_helper_->GetStreamLabelFromTrackId(*stat->track_identifier)); + + MutexLock lock(&lock_); + StatsSample prev_sample = last_stats_sample_[stream_label]; + RTC_CHECK_GE(sample.total_samples_received, + prev_sample.total_samples_received); + double total_samples_diff = static_cast<double>( + sample.total_samples_received - prev_sample.total_samples_received); + if (total_samples_diff == 0) { + return; + } + + AudioStreamStats& audio_stream_stats = streams_stats_[stream_label]; + audio_stream_stats.expand_rate.AddSample( + (sample.concealed_samples - prev_sample.concealed_samples) / + total_samples_diff); + audio_stream_stats.accelerate_rate.AddSample( + (sample.removed_samples_for_acceleration - + prev_sample.removed_samples_for_acceleration) / + total_samples_diff); + audio_stream_stats.preemptive_rate.AddSample( + (sample.inserted_samples_for_deceleration - + prev_sample.inserted_samples_for_deceleration) / + total_samples_diff); + + int64_t speech_concealed_samples = + sample.concealed_samples - sample.silent_concealed_samples; + int64_t prev_speech_concealed_samples = + prev_sample.concealed_samples - prev_sample.silent_concealed_samples; + audio_stream_stats.speech_expand_rate.AddSample( + (speech_concealed_samples - prev_speech_concealed_samples) / + total_samples_diff); + + int64_t jitter_buffer_emitted_count_diff = + sample.jitter_buffer_emitted_count - + prev_sample.jitter_buffer_emitted_count; + if (jitter_buffer_emitted_count_diff > 0) { + TimeDelta jitter_buffer_delay_diff = + sample.jitter_buffer_delay - prev_sample.jitter_buffer_delay; + TimeDelta jitter_buffer_target_delay_diff = + sample.jitter_buffer_target_delay - + prev_sample.jitter_buffer_target_delay; + audio_stream_stats.average_jitter_buffer_delay_ms.AddSample( + jitter_buffer_delay_diff.ms<double>() / + jitter_buffer_emitted_count_diff); + audio_stream_stats.preferred_buffer_size_ms.AddSample( + jitter_buffer_target_delay_diff.ms<double>() / + jitter_buffer_emitted_count_diff); + } + + last_stats_sample_[stream_label] = sample; + } +} + +std::string DefaultAudioQualityAnalyzer::GetTestCaseName( + const std::string& stream_label) const { + return test_case_name_ + "/" + stream_label; +} + +void DefaultAudioQualityAnalyzer::Stop() { + using ::webrtc::test::ImproveDirection; + MutexLock lock(&lock_); + for (auto& item : streams_stats_) { + ReportResult("expand_rate", item.first, item.second.expand_rate, "unitless", + ImproveDirection::kSmallerIsBetter); + ReportResult("accelerate_rate", item.first, item.second.accelerate_rate, + "unitless", ImproveDirection::kSmallerIsBetter); + ReportResult("preemptive_rate", item.first, item.second.preemptive_rate, + "unitless", ImproveDirection::kSmallerIsBetter); + ReportResult("speech_expand_rate", item.first, + item.second.speech_expand_rate, "unitless", + ImproveDirection::kSmallerIsBetter); + ReportResult("average_jitter_buffer_delay_ms", item.first, + item.second.average_jitter_buffer_delay_ms, "ms", + ImproveDirection::kNone); + ReportResult("preferred_buffer_size_ms", item.first, + item.second.preferred_buffer_size_ms, "ms", + ImproveDirection::kNone); + } +} + +std::map<std::string, AudioStreamStats> +DefaultAudioQualityAnalyzer::GetAudioStreamsStats() const { + MutexLock lock(&lock_); + return streams_stats_; +} + +void DefaultAudioQualityAnalyzer::ReportResult( + const std::string& metric_name, + const std::string& stream_label, + const SamplesStatsCounter& counter, + const std::string& unit, + webrtc::test::ImproveDirection improve_direction) const { + test::PrintResultMeanAndError( + metric_name, /*modifier=*/"", GetTestCaseName(stream_label), + counter.IsEmpty() ? 0 : counter.GetAverage(), + counter.IsEmpty() ? 0 : counter.GetStandardDeviation(), unit, + /*important=*/false, improve_direction); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h b/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h new file mode 100644 index 0000000000..4ad0dd3da2 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_AUDIO_DEFAULT_AUDIO_QUALITY_ANALYZER_H_ +#define TEST_PC_E2E_ANALYZER_AUDIO_DEFAULT_AUDIO_QUALITY_ANALYZER_H_ + +#include <map> +#include <string> + +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/audio_quality_analyzer_interface.h" +#include "api/test/track_id_stream_info_map.h" +#include "api/units/time_delta.h" +#include "rtc_base/synchronization/mutex.h" +#include "test/testsupport/perf_test.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +struct AudioStreamStats { + SamplesStatsCounter expand_rate; + SamplesStatsCounter accelerate_rate; + SamplesStatsCounter preemptive_rate; + SamplesStatsCounter speech_expand_rate; + SamplesStatsCounter average_jitter_buffer_delay_ms; + SamplesStatsCounter preferred_buffer_size_ms; +}; + +class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface { + public: + void Start(std::string test_case_name, + TrackIdStreamInfoMap* analyzer_helper) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override; + void Stop() override; + + // Returns audio quality stats per stream label. + std::map<std::string, AudioStreamStats> GetAudioStreamsStats() const; + + private: + struct StatsSample { + uint64_t total_samples_received = 0; + uint64_t concealed_samples = 0; + uint64_t removed_samples_for_acceleration = 0; + uint64_t inserted_samples_for_deceleration = 0; + uint64_t silent_concealed_samples = 0; + TimeDelta jitter_buffer_delay = TimeDelta::Zero(); + TimeDelta jitter_buffer_target_delay = TimeDelta::Zero(); + uint64_t jitter_buffer_emitted_count = 0; + }; + + std::string GetTestCaseName(const std::string& stream_label) const; + void ReportResult(const std::string& metric_name, + const std::string& stream_label, + const SamplesStatsCounter& counter, + const std::string& unit, + webrtc::test::ImproveDirection improve_direction) const; + + std::string test_case_name_; + TrackIdStreamInfoMap* analyzer_helper_; + + mutable Mutex lock_; + std::map<std::string, AudioStreamStats> streams_stats_ RTC_GUARDED_BY(lock_); + std::map<std::string, StatsSample> last_stats_sample_ RTC_GUARDED_BY(lock_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_AUDIO_DEFAULT_AUDIO_QUALITY_ANALYZER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc new file mode 100644 index 0000000000..e31843c8eb --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc @@ -0,0 +1,1036 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" + +#include <algorithm> +#include <memory> +#include <set> +#include <utility> +#include <vector> + +#include "api/array_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/units/time_delta.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_base/logging.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/time_utils.h" +#include "rtc_tools/frame_analyzer/video_geometry_aligner.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h" + +namespace webrtc { +namespace { + +constexpr int kBitsInByte = 8; +constexpr absl::string_view kSkipRenderedFrameReasonProcessed = "processed"; +constexpr absl::string_view kSkipRenderedFrameReasonRendered = "rendered"; +constexpr absl::string_view kSkipRenderedFrameReasonDropped = + "considered dropped"; + +void LogFrameCounters(const std::string& name, const FrameCounters& counters) { + RTC_LOG(LS_INFO) << "[" << name << "] Captured : " << counters.captured; + RTC_LOG(LS_INFO) << "[" << name << "] Pre encoded : " << counters.pre_encoded; + RTC_LOG(LS_INFO) << "[" << name << "] Encoded : " << counters.encoded; + RTC_LOG(LS_INFO) << "[" << name << "] Received : " << counters.received; + RTC_LOG(LS_INFO) << "[" << name << "] Decoded : " << counters.decoded; + RTC_LOG(LS_INFO) << "[" << name << "] Rendered : " << counters.rendered; + RTC_LOG(LS_INFO) << "[" << name << "] Dropped : " << counters.dropped; +} + +absl::string_view ToString(FrameDropPhase phase) { + switch (phase) { + case FrameDropPhase::kBeforeEncoder: + return "kBeforeEncoder"; + case FrameDropPhase::kByEncoder: + return "kByEncoder"; + case FrameDropPhase::kTransport: + return "kTransport"; + case FrameDropPhase::kAfterDecoder: + return "kAfterDecoder"; + case FrameDropPhase::kLastValue: + RTC_CHECK(false) << "FrameDropPhase::kLastValue mustn't be used"; + } +} + +void LogStreamInternalStats(const std::string& name, + const StreamStats& stats, + Timestamp start_time) { + for (const auto& entry : stats.dropped_by_phase) { + RTC_LOG(LS_INFO) << "[" << name << "] Dropped at " << ToString(entry.first) + << ": " << entry.second; + } + Timestamp first_encoded_frame_time = Timestamp::PlusInfinity(); + for (const StreamCodecInfo& encoder : stats.encoders) { + RTC_DCHECK(encoder.switched_on_at.IsFinite()); + RTC_DCHECK(encoder.switched_from_at.IsFinite()); + if (first_encoded_frame_time.IsInfinite()) { + first_encoded_frame_time = encoder.switched_on_at; + } + RTC_LOG(LS_INFO) + << "[" << name << "] Used encoder: \"" << encoder.codec_name + << "\" used from (frame_id=" << encoder.first_frame_id + << "; from_stream_start=" + << (encoder.switched_on_at - stats.stream_started_time).ms() + << "ms, from_call_start=" << (encoder.switched_on_at - start_time).ms() + << "ms) until (frame_id=" << encoder.last_frame_id + << "; from_stream_start=" + << (encoder.switched_from_at - stats.stream_started_time).ms() + << "ms, from_call_start=" + << (encoder.switched_from_at - start_time).ms() << "ms)"; + } + for (const StreamCodecInfo& decoder : stats.decoders) { + RTC_DCHECK(decoder.switched_on_at.IsFinite()); + RTC_DCHECK(decoder.switched_from_at.IsFinite()); + RTC_LOG(LS_INFO) + << "[" << name << "] Used decoder: \"" << decoder.codec_name + << "\" used from (frame_id=" << decoder.first_frame_id + << "; from_stream_start=" + << (decoder.switched_on_at - stats.stream_started_time).ms() + << "ms, from_call_start=" << (decoder.switched_on_at - start_time).ms() + << "ms) until (frame_id=" << decoder.last_frame_id + << "; from_stream_start=" + << (decoder.switched_from_at - stats.stream_started_time).ms() + << "ms, from_call_start=" + << (decoder.switched_from_at - start_time).ms() << "ms)"; + } +} + +template <typename T> +absl::optional<T> MaybeGetValue(const std::map<size_t, T>& map, size_t key) { + auto it = map.find(key); + if (it == map.end()) { + return absl::nullopt; + } + return it->second; +} + +SamplesStatsCounter::StatsSample StatsSample(double value, + Timestamp sampling_time) { + return SamplesStatsCounter::StatsSample{value, sampling_time}; +} + +} // namespace + +DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer( + webrtc::Clock* clock, + DefaultVideoQualityAnalyzerOptions options) + : options_(options), + clock_(clock), + frames_comparator_(clock, cpu_measurer_, options) {} +DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() { + Stop(); +} + +void DefaultVideoQualityAnalyzer::Start( + std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count) { + test_label_ = std::move(test_case_name); + frames_comparator_.Start(max_threads_count); + { + MutexLock lock(&mutex_); + peers_ = std::make_unique<NamesCollection>(peer_names); + RTC_CHECK(start_time_.IsMinusInfinity()); + + RTC_CHECK_EQ(state_, State::kNew) + << "DefaultVideoQualityAnalyzer is already started"; + state_ = State::kActive; + start_time_ = Now(); + } +} + +uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured( + absl::string_view peer_name, + const std::string& stream_label, + const webrtc::VideoFrame& frame) { + // `next_frame_id` is atomic, so we needn't lock here. + Timestamp captured_time = Now(); + Timestamp start_time = Timestamp::MinusInfinity(); + size_t peer_index = -1; + size_t peers_count = -1; + size_t stream_index; + uint16_t frame_id = VideoFrame::kNotSetId; + { + MutexLock lock(&mutex_); + frame_id = GetNextFrameId(); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + // Create a local copy of `start_time_`, peer's index and total peers count + // to access it without holding a `mutex_` during access to + // `frames_comparator_`. + start_time = start_time_; + peer_index = peers_->index(peer_name); + peers_count = peers_->size(); + stream_index = streams_.AddIfAbsent(stream_label); + } + // Ensure stats for this stream exists. + frames_comparator_.EnsureStatsForStream(stream_index, peer_index, peers_count, + captured_time, start_time); + { + MutexLock lock(&mutex_); + stream_to_sender_[stream_index] = peer_index; + frame_counters_.captured++; + for (size_t i : peers_->GetAllIndexes()) { + if (i != peer_index || options_.enable_receive_own_stream) { + InternalStatsKey key(stream_index, peer_index, i); + stream_frame_counters_[key].captured++; + } + } + + std::set<size_t> frame_receivers_indexes = peers_->GetPresentIndexes(); + if (!options_.enable_receive_own_stream) { + frame_receivers_indexes.erase(peer_index); + } + + auto state_it = stream_states_.find(stream_index); + if (state_it == stream_states_.end()) { + stream_states_.emplace( + stream_index, + StreamState(peer_index, frame_receivers_indexes, captured_time)); + } + StreamState* state = &stream_states_.at(stream_index); + state->PushBack(frame_id); + // Update frames in flight info. + auto it = captured_frames_in_flight_.find(frame_id); + if (it != captured_frames_in_flight_.end()) { + // If we overflow uint16_t and hit previous frame id and this frame is + // still in flight, it means that this stream wasn't rendered for long + // time and we need to process existing frame as dropped. + for (size_t i : peers_->GetPresentIndexes()) { + if (i == peer_index && !options_.enable_receive_own_stream) { + continue; + } + + uint16_t oldest_frame_id = state->PopFront(i); + RTC_DCHECK_EQ(frame_id, oldest_frame_id); + frame_counters_.dropped++; + InternalStatsKey key(stream_index, peer_index, i); + stream_frame_counters_.at(key).dropped++; + + analyzer_stats_.frames_in_flight_left_count.AddSample( + StatsSample(captured_frames_in_flight_.size(), Now())); + frames_comparator_.AddComparison( + InternalStatsKey(stream_index, peer_index, i), + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, FrameComparisonType::kDroppedFrame, + it->second.GetStatsForPeer(i)); + } + + captured_frames_in_flight_.erase(it); + } + captured_frames_in_flight_.emplace( + frame_id, FrameInFlight(stream_index, frame, captured_time, + std::move(frame_receivers_indexes))); + // Set frame id on local copy of the frame + captured_frames_in_flight_.at(frame_id).SetFrameId(frame_id); + + // Update history stream<->frame mapping + for (auto it = stream_to_frame_id_history_.begin(); + it != stream_to_frame_id_history_.end(); ++it) { + it->second.erase(frame_id); + } + stream_to_frame_id_history_[stream_index].insert(frame_id); + stream_to_frame_id_full_history_[stream_index].push_back(frame_id); + + // If state has too many frames that are in flight => remove the oldest + // queued frame in order to avoid to use too much memory. + if (state->GetAliveFramesCount() > + options_.max_frames_in_flight_per_stream_count) { + uint16_t frame_id_to_remove = state->MarkNextAliveFrameAsDead(); + auto it = captured_frames_in_flight_.find(frame_id_to_remove); + RTC_CHECK(it != captured_frames_in_flight_.end()) + << "Frame with ID " << frame_id_to_remove + << " is expected to be in flight, but hasn't been found in " + << "|captured_frames_in_flight_|"; + bool is_removed = it->second.RemoveFrame(); + RTC_DCHECK(is_removed) + << "Invalid stream state: alive frame is removed already"; + } + } + return frame_id; +} + +void DefaultVideoQualityAnalyzer::OnFramePreEncode( + absl::string_view peer_name, + const webrtc::VideoFrame& frame) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + auto it = captured_frames_in_flight_.find(frame.id()); + RTC_CHECK(it != captured_frames_in_flight_.end()) + << "Frame id=" << frame.id() << " not found"; + FrameInFlight& frame_in_flight = it->second; + frame_counters_.pre_encoded++; + size_t peer_index = peers_->index(peer_name); + for (size_t i : peers_->GetAllIndexes()) { + if (i != peer_index || options_.enable_receive_own_stream) { + InternalStatsKey key(frame_in_flight.stream(), peer_index, i); + stream_frame_counters_.at(key).pre_encoded++; + } + } + frame_in_flight.SetPreEncodeTime(Now()); +} + +void DefaultVideoQualityAnalyzer::OnFrameEncoded( + absl::string_view peer_name, + uint16_t frame_id, + const webrtc::EncodedImage& encoded_image, + const EncoderStats& stats) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + auto it = captured_frames_in_flight_.find(frame_id); + if (it == captured_frames_in_flight_.end()) { + RTC_LOG(LS_WARNING) + << "The encoding of video frame with id [" << frame_id << "] for peer [" + << peer_name << "] finished after all receivers rendered this frame or " + << "were removed. It can be OK for simulcast/SVC if higher quality " + << "stream is not required or the last receiver was unregistered " + << "between encoding of different layers, but it may indicate an ERROR " + << "for singlecast or if it happens often."; + return; + } + FrameInFlight& frame_in_flight = it->second; + // For SVC we can receive multiple encoded images for one frame, so to cover + // all cases we have to pick the last encode time. + if (!frame_in_flight.HasEncodedTime()) { + // Increase counters only when we meet this frame first time. + frame_counters_.encoded++; + size_t peer_index = peers_->index(peer_name); + for (size_t i : peers_->GetAllIndexes()) { + if (i != peer_index || options_.enable_receive_own_stream) { + InternalStatsKey key(frame_in_flight.stream(), peer_index, i); + stream_frame_counters_.at(key).encoded++; + } + } + } + Timestamp now = Now(); + StreamCodecInfo used_encoder; + used_encoder.codec_name = stats.encoder_name; + used_encoder.first_frame_id = frame_id; + used_encoder.last_frame_id = frame_id; + used_encoder.switched_on_at = now; + used_encoder.switched_from_at = now; + frame_in_flight.OnFrameEncoded(now, encoded_image._frameType, + DataSize::Bytes(encoded_image.size()), + stats.target_encode_bitrate, used_encoder); +} + +void DefaultVideoQualityAnalyzer::OnFrameDropped( + absl::string_view peer_name, + webrtc::EncodedImageCallback::DropReason reason) { + // Here we do nothing, because we will see this drop on renderer side. +} + +void DefaultVideoQualityAnalyzer::OnFramePreDecode( + absl::string_view peer_name, + uint16_t frame_id, + const webrtc::EncodedImage& input_image) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + size_t peer_index = peers_->index(peer_name); + + if (frame_id == VideoFrame::kNotSetId) { + frame_counters_.received++; + unknown_sender_frame_counters_[std::string(peer_name)].received++; + return; + } + + auto it = captured_frames_in_flight_.find(frame_id); + if (it == captured_frames_in_flight_.end() || + it->second.HasReceivedTime(peer_index)) { + // It means this frame was predecoded before, so we can skip it. It may + // happen when we have multiple simulcast streams in one track and received + // the same picture from two different streams because SFU can't reliably + // correlate two simulcast streams and started relaying the second stream + // from the same frame it has relayed right before for the first stream. + return; + } + + frame_counters_.received++; + InternalStatsKey key(it->second.stream(), + stream_to_sender_.at(it->second.stream()), peer_index); + stream_frame_counters_.at(key).received++; + // Determine the time of the last received packet of this video frame. + RTC_DCHECK(!input_image.PacketInfos().empty()); + Timestamp last_receive_time = + std::max_element(input_image.PacketInfos().cbegin(), + input_image.PacketInfos().cend(), + [](const RtpPacketInfo& a, const RtpPacketInfo& b) { + return a.receive_time() < b.receive_time(); + }) + ->receive_time(); + it->second.OnFramePreDecode(peer_index, + /*received_time=*/last_receive_time, + /*decode_start_time=*/Now(), + input_image._frameType, + DataSize::Bytes(input_image.size())); +} + +void DefaultVideoQualityAnalyzer::OnFrameDecoded( + absl::string_view peer_name, + const webrtc::VideoFrame& frame, + const DecoderStats& stats) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + size_t peer_index = peers_->index(peer_name); + + if (frame.id() == VideoFrame::kNotSetId) { + frame_counters_.decoded++; + unknown_sender_frame_counters_[std::string(peer_name)].decoded++; + return; + } + + auto it = captured_frames_in_flight_.find(frame.id()); + if (it == captured_frames_in_flight_.end() || + it->second.HasDecodeEndTime(peer_index)) { + // It means this frame was decoded before, so we can skip it. It may happen + // when we have multiple simulcast streams in one track and received + // the same picture from two different streams because SFU can't reliably + // correlate two simulcast streams and started relaying the second stream + // from the same frame it has relayed right before for the first stream. + return; + } + frame_counters_.decoded++; + InternalStatsKey key(it->second.stream(), + stream_to_sender_.at(it->second.stream()), peer_index); + stream_frame_counters_.at(key).decoded++; + Timestamp now = Now(); + StreamCodecInfo used_decoder; + used_decoder.codec_name = stats.decoder_name; + used_decoder.first_frame_id = frame.id(); + used_decoder.last_frame_id = frame.id(); + used_decoder.switched_on_at = now; + used_decoder.switched_from_at = now; + it->second.OnFrameDecoded(peer_index, now, used_decoder); +} + +void DefaultVideoQualityAnalyzer::OnFrameRendered( + absl::string_view peer_name, + const webrtc::VideoFrame& frame) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + size_t peer_index = peers_->index(peer_name); + + if (frame.id() == VideoFrame::kNotSetId) { + frame_counters_.rendered++; + unknown_sender_frame_counters_[std::string(peer_name)].rendered++; + return; + } + + auto frame_it = captured_frames_in_flight_.find(frame.id()); + if (frame_it == captured_frames_in_flight_.end() || + frame_it->second.HasRenderedTime(peer_index) || + frame_it->second.IsDropped(peer_index)) { + // It means this frame was rendered or dropped before, so we can skip it. + // It may happen when we have multiple simulcast streams in one track and + // received the same frame from two different streams because SFU can't + // reliably correlate two simulcast streams and started relaying the second + // stream from the same frame it has relayed right before for the first + // stream. + absl::string_view reason = kSkipRenderedFrameReasonProcessed; + if (frame_it != captured_frames_in_flight_.end()) { + if (frame_it->second.HasRenderedTime(peer_index)) { + reason = kSkipRenderedFrameReasonRendered; + } else if (frame_it->second.IsDropped(peer_index)) { + reason = kSkipRenderedFrameReasonDropped; + } + } + RTC_LOG(LS_WARNING) + << "Peer " << peer_name + << "; Received frame out of order: received frame with id " + << frame.id() << " which was " << reason << " before"; + return; + } + + // Find corresponding captured frame. + FrameInFlight* frame_in_flight = &frame_it->second; + absl::optional<VideoFrame> captured_frame = frame_in_flight->frame(); + + const size_t stream_index = frame_in_flight->stream(); + StreamState* state = &stream_states_.at(stream_index); + const InternalStatsKey stats_key(stream_index, state->sender(), peer_index); + + // Update frames counters. + frame_counters_.rendered++; + stream_frame_counters_.at(stats_key).rendered++; + + // Update current frame stats. + frame_in_flight->OnFrameRendered(peer_index, Now(), frame.width(), + frame.height()); + + // After we received frame here we need to check if there are any dropped + // frames between this one and last one, that was rendered for this video + // stream. + int dropped_count = 0; + while (!state->IsEmpty(peer_index) && + state->Front(peer_index) != frame.id()) { + dropped_count++; + uint16_t dropped_frame_id = state->PopFront(peer_index); + // Frame with id `dropped_frame_id` was dropped. We need: + // 1. Update global and stream frame counters + // 2. Extract corresponding frame from `captured_frames_in_flight_` + // 3. Send extracted frame to comparison with dropped=true + // 4. Cleanup dropped frame + frame_counters_.dropped++; + stream_frame_counters_.at(stats_key).dropped++; + + auto dropped_frame_it = captured_frames_in_flight_.find(dropped_frame_id); + RTC_DCHECK(dropped_frame_it != captured_frames_in_flight_.end()); + dropped_frame_it->second.MarkDropped(peer_index); + + analyzer_stats_.frames_in_flight_left_count.AddSample( + StatsSample(captured_frames_in_flight_.size(), Now())); + frames_comparator_.AddComparison( + stats_key, /*captured=*/absl::nullopt, /*rendered=*/absl::nullopt, + FrameComparisonType::kDroppedFrame, + dropped_frame_it->second.GetStatsForPeer(peer_index)); + + if (dropped_frame_it->second.HaveAllPeersReceived()) { + captured_frames_in_flight_.erase(dropped_frame_it); + } + } + RTC_DCHECK(!state->IsEmpty(peer_index)); + state->PopFront(peer_index); + + if (state->last_rendered_frame_time(peer_index)) { + frame_in_flight->SetPrevFrameRenderedTime( + peer_index, state->last_rendered_frame_time(peer_index).value()); + } + state->SetLastRenderedFrameTime(peer_index, + frame_in_flight->rendered_time(peer_index)); + analyzer_stats_.frames_in_flight_left_count.AddSample( + StatsSample(captured_frames_in_flight_.size(), Now())); + frames_comparator_.AddComparison( + stats_key, dropped_count, captured_frame, /*rendered=*/frame, + FrameComparisonType::kRegular, + frame_in_flight->GetStatsForPeer(peer_index)); + + if (frame_it->second.HaveAllPeersReceived()) { + captured_frames_in_flight_.erase(frame_it); + } +} + +void DefaultVideoQualityAnalyzer::OnEncoderError( + absl::string_view peer_name, + const webrtc::VideoFrame& frame, + int32_t error_code) { + RTC_LOG(LS_ERROR) << "Encoder error for frame.id=" << frame.id() + << ", code=" << error_code; +} + +void DefaultVideoQualityAnalyzer::OnDecoderError(absl::string_view peer_name, + uint16_t frame_id, + int32_t error_code) { + RTC_LOG(LS_ERROR) << "Decoder error for frame_id=" << frame_id + << ", code=" << error_code; +} + +void DefaultVideoQualityAnalyzer::RegisterParticipantInCall( + absl::string_view peer_name) { + MutexLock lock(&mutex_); + RTC_CHECK(!peers_->HasName(peer_name)); + size_t new_peer_index = peers_->AddIfAbsent(peer_name); + + // Ensure stats for receiving (for frames from other peers to this one) + // streams exists. Since in flight frames will be sent to the new peer + // as well. Sending stats (from this peer to others) will be added by + // DefaultVideoQualityAnalyzer::OnFrameCaptured. + std::vector<std::pair<InternalStatsKey, Timestamp>> stream_started_time; + for (auto [stream_index, sender_peer_index] : stream_to_sender_) { + InternalStatsKey key(stream_index, sender_peer_index, new_peer_index); + + // To initiate `FrameCounters` for the stream we should pick frame + // counters with the same stream index and the same sender's peer index + // and any receiver's peer index and copy from its sender side + // counters. + FrameCounters counters; + for (size_t i : peers_->GetPresentIndexes()) { + InternalStatsKey prototype_key(stream_index, sender_peer_index, i); + auto it = stream_frame_counters_.find(prototype_key); + if (it != stream_frame_counters_.end()) { + counters.captured = it->second.captured; + counters.pre_encoded = it->second.pre_encoded; + counters.encoded = it->second.encoded; + break; + } + } + // It may happen if we had only one peer before this method was invoked, + // then `counters` will be empty. In such case empty `counters` are ok. + stream_frame_counters_.insert({key, std::move(counters)}); + + stream_started_time.push_back( + {key, stream_states_.at(stream_index).stream_started_time()}); + } + frames_comparator_.RegisterParticipantInCall(stream_started_time, + start_time_); + // Ensure, that frames states are handled correctly + // (e.g. dropped frames tracking). + for (auto& [stream_index, stream_state] : stream_states_) { + stream_state.AddPeer(new_peer_index); + } + // Register new peer for every frame in flight. + // It is guaranteed, that no garbage FrameInFlight objects will stay in + // memory because of adding new peer. Even if the new peer won't receive the + // frame, the frame will be removed by OnFrameRendered after next frame comes + // for the new peer. It is important because FrameInFlight is a large object. + for (auto& [frame_id, frame_in_flight] : captured_frames_in_flight_) { + frame_in_flight.AddExpectedReceiver(new_peer_index); + } +} + +void DefaultVideoQualityAnalyzer::UnregisterParticipantInCall( + absl::string_view peer_name) { + MutexLock lock(&mutex_); + RTC_CHECK(peers_->HasName(peer_name)); + absl::optional<size_t> peer_index = peers_->RemoveIfPresent(peer_name); + RTC_CHECK(peer_index.has_value()); + + for (auto& [stream_index, stream_state] : stream_states_) { + if (!options_.enable_receive_own_stream && + peer_index == stream_state.sender()) { + continue; + } + stream_state.RemovePeer(*peer_index); + } + + // Remove peer from every frame in flight. If we removed that last expected + // receiver for the frame, then we should removed this frame if it was + // already encoded. If frame wasn't encoded, it still will be used by sender + // side pipeline, so we can't delete it yet. + for (auto it = captured_frames_in_flight_.begin(); + it != captured_frames_in_flight_.end();) { + FrameInFlight& frame_in_flight = it->second; + frame_in_flight.RemoveExpectedReceiver(*peer_index); + // If frame was fully sent and all receivers received it, then erase it. + // It may happen that when we remove FrameInFlight only some Simulcast/SVC + // layers were encoded and frame has encoded time, but more layers might be + // encoded after removal. In such case it's safe to still remove a frame, + // because OnFrameEncoded method will correctly handle the case when there + // is no FrameInFlight for the received encoded image. + if (frame_in_flight.HasEncodedTime() && + frame_in_flight.HaveAllPeersReceived()) { + it = captured_frames_in_flight_.erase(it); + } else { + it++; + } + } +} + +void DefaultVideoQualityAnalyzer::Stop() { + std::map<InternalStatsKey, Timestamp> last_rendered_frame_times; + { + MutexLock lock(&mutex_); + if (state_ == State::kStopped) { + return; + } + RTC_CHECK_EQ(state_, State::kActive) + << "DefaultVideoQualityAnalyzer has to be started before use"; + + state_ = State::kStopped; + + // Add the amount of frames in flight to the analyzer stats before all left + // frames in flight will be sent to the `frames_compartor_`. + analyzer_stats_.frames_in_flight_left_count.AddSample( + StatsSample(captured_frames_in_flight_.size(), Now())); + + for (auto& state_entry : stream_states_) { + const size_t stream_index = state_entry.first; + StreamState& stream_state = state_entry.second; + for (size_t peer_index : peers_->GetPresentIndexes()) { + if (peer_index == stream_state.sender() && + !options_.enable_receive_own_stream) { + continue; + } + + InternalStatsKey stats_key(stream_index, stream_state.sender(), + peer_index); + + // If there are no freezes in the call we have to report + // time_between_freezes_ms as call duration and in such case + // `stream_last_freeze_end_time` for this stream will be `start_time_`. + // If there is freeze, then we need add time from last rendered frame + // to last freeze end as time between freezes. + if (stream_state.last_rendered_frame_time(peer_index)) { + last_rendered_frame_times.emplace( + stats_key, + stream_state.last_rendered_frame_time(peer_index).value()); + } + + // Add frames in flight for this stream into frames comparator. + // Frames in flight were not rendered, so they won't affect stream's + // last rendered frame time. + while (!stream_state.IsEmpty(peer_index)) { + uint16_t frame_id = stream_state.PopFront(peer_index); + auto it = captured_frames_in_flight_.find(frame_id); + RTC_DCHECK(it != captured_frames_in_flight_.end()); + FrameInFlight& frame = it->second; + + frames_comparator_.AddComparison( + stats_key, /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, FrameComparisonType::kFrameInFlight, + frame.GetStatsForPeer(peer_index)); + + if (frame.HaveAllPeersReceived()) { + captured_frames_in_flight_.erase(it); + } + } + } + } + } + frames_comparator_.Stop(last_rendered_frame_times); + + // Perform final Metrics update. On this place analyzer is stopped and no one + // holds any locks. + { + MutexLock lock(&mutex_); + FramesComparatorStats frames_comparator_stats = + frames_comparator_.frames_comparator_stats(); + analyzer_stats_.comparisons_queue_size = + frames_comparator_stats.comparisons_queue_size; + analyzer_stats_.comparisons_done = frames_comparator_stats.comparisons_done; + analyzer_stats_.cpu_overloaded_comparisons_done = + frames_comparator_stats.cpu_overloaded_comparisons_done; + analyzer_stats_.memory_overloaded_comparisons_done = + frames_comparator_stats.memory_overloaded_comparisons_done; + } + ReportResults(); +} + +std::string DefaultVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) { + MutexLock lock1(&mutex_); + auto it = captured_frames_in_flight_.find(frame_id); + if (it != captured_frames_in_flight_.end()) { + return streams_.name(it->second.stream()); + } + for (auto hist_it = stream_to_frame_id_history_.begin(); + hist_it != stream_to_frame_id_history_.end(); ++hist_it) { + auto hist_set_it = hist_it->second.find(frame_id); + if (hist_set_it != hist_it->second.end()) { + return streams_.name(hist_it->first); + } + } + RTC_CHECK(false) << "Unknown frame_id=" << frame_id; +} + +std::set<StatsKey> DefaultVideoQualityAnalyzer::GetKnownVideoStreams() const { + MutexLock lock(&mutex_); + std::set<StatsKey> out; + for (auto& item : frames_comparator_.stream_stats()) { + RTC_LOG(LS_INFO) << item.first.ToString() << " ==> " + << ToStatsKey(item.first).ToString(); + out.insert(ToStatsKey(item.first)); + } + return out; +} + +VideoStreamsInfo DefaultVideoQualityAnalyzer::GetKnownStreams() const { + MutexLock lock(&mutex_); + std::map<std::string, std::string> stream_to_sender; + std::map<std::string, std::set<std::string>> sender_to_streams; + std::map<std::string, std::set<std::string>> stream_to_receivers; + + for (auto& item : frames_comparator_.stream_stats()) { + const std::string& stream_label = streams_.name(item.first.stream); + const std::string& sender = peers_->name(item.first.sender); + const std::string& receiver = peers_->name(item.first.receiver); + RTC_LOG(LS_INFO) << item.first.ToString() << " ==> " + << "stream=" << stream_label << "; sender=" << sender + << "; receiver=" << receiver; + stream_to_sender.emplace(stream_label, sender); + auto streams_it = sender_to_streams.find(sender); + if (streams_it != sender_to_streams.end()) { + streams_it->second.emplace(stream_label); + } else { + sender_to_streams.emplace(sender, std::set<std::string>{stream_label}); + } + auto receivers_it = stream_to_receivers.find(stream_label); + if (receivers_it != stream_to_receivers.end()) { + receivers_it->second.emplace(receiver); + } else { + stream_to_receivers.emplace(stream_label, + std::set<std::string>{receiver}); + } + } + + return VideoStreamsInfo(std::move(stream_to_sender), + std::move(sender_to_streams), + std::move(stream_to_receivers)); +} + +FrameCounters DefaultVideoQualityAnalyzer::GetGlobalCounters() const { + MutexLock lock(&mutex_); + return frame_counters_; +} + +std::map<std::string, FrameCounters> +DefaultVideoQualityAnalyzer::GetUnknownSenderFrameCounters() const { + MutexLock lock(&mutex_); + return unknown_sender_frame_counters_; +} + +std::map<StatsKey, FrameCounters> +DefaultVideoQualityAnalyzer::GetPerStreamCounters() const { + MutexLock lock(&mutex_); + std::map<StatsKey, FrameCounters> out; + for (auto& item : stream_frame_counters_) { + out.emplace(ToStatsKey(item.first), item.second); + } + return out; +} + +std::map<StatsKey, StreamStats> DefaultVideoQualityAnalyzer::GetStats() const { + MutexLock lock1(&mutex_); + std::map<StatsKey, StreamStats> out; + for (auto& item : frames_comparator_.stream_stats()) { + out.emplace(ToStatsKey(item.first), item.second); + } + return out; +} + +AnalyzerStats DefaultVideoQualityAnalyzer::GetAnalyzerStats() const { + MutexLock lock(&mutex_); + return analyzer_stats_; +} + +uint16_t DefaultVideoQualityAnalyzer::GetNextFrameId() { + uint16_t frame_id = next_frame_id_++; + if (next_frame_id_ == VideoFrame::kNotSetId) { + next_frame_id_ = 1; + } + return frame_id; +} + +void DefaultVideoQualityAnalyzer::ReportResults() { + using ::webrtc::test::ImproveDirection; + + MutexLock lock(&mutex_); + for (auto& item : frames_comparator_.stream_stats()) { + ReportResults(GetTestCaseName(ToMetricName(item.first)), item.second, + stream_frame_counters_.at(item.first)); + } + test::PrintResult("cpu_usage", "", test_label_.c_str(), GetCpuUsagePercent(), + "%", false, ImproveDirection::kSmallerIsBetter); + LogFrameCounters("Global", frame_counters_); + if (!unknown_sender_frame_counters_.empty()) { + RTC_LOG(LS_INFO) << "Received frame counters with unknown frame id:"; + for (const auto& [peer_name, frame_counters] : + unknown_sender_frame_counters_) { + LogFrameCounters(peer_name, frame_counters); + } + } + RTC_LOG(LS_INFO) << "Received frame counters per stream:"; + for (const auto& [stats_key, stream_stats] : + frames_comparator_.stream_stats()) { + LogFrameCounters(ToStatsKey(stats_key).ToString(), + stream_frame_counters_.at(stats_key)); + LogStreamInternalStats(ToStatsKey(stats_key).ToString(), stream_stats, + start_time_); + } + if (!analyzer_stats_.comparisons_queue_size.IsEmpty()) { + RTC_LOG(LS_INFO) << "comparisons_queue_size min=" + << analyzer_stats_.comparisons_queue_size.GetMin() + << "; max=" + << analyzer_stats_.comparisons_queue_size.GetMax() + << "; 99%=" + << analyzer_stats_.comparisons_queue_size.GetPercentile( + 0.99); + } + RTC_LOG(LS_INFO) << "comparisons_done=" << analyzer_stats_.comparisons_done; + RTC_LOG(LS_INFO) << "cpu_overloaded_comparisons_done=" + << analyzer_stats_.cpu_overloaded_comparisons_done; + RTC_LOG(LS_INFO) << "memory_overloaded_comparisons_done=" + << analyzer_stats_.memory_overloaded_comparisons_done; +} + +void DefaultVideoQualityAnalyzer::ReportResults( + const std::string& test_case_name, + const StreamStats& stats, + const FrameCounters& frame_counters) { + using ::webrtc::test::ImproveDirection; + TimeDelta test_duration = Now() - start_time_; + + double sum_squared_interframe_delays_secs = 0; + Timestamp video_start_time = Timestamp::PlusInfinity(); + Timestamp video_end_time = Timestamp::MinusInfinity(); + for (const SamplesStatsCounter::StatsSample& sample : + stats.time_between_rendered_frames_ms.GetTimedSamples()) { + double interframe_delay_ms = sample.value; + const double interframe_delays_secs = interframe_delay_ms / 1000.0; + // Sum of squared inter frame intervals is used to calculate the harmonic + // frame rate metric. The metric aims to reflect overall experience related + // to smoothness of video playback and includes both freezes and pauses. + sum_squared_interframe_delays_secs += + interframe_delays_secs * interframe_delays_secs; + if (sample.time < video_start_time) { + video_start_time = sample.time; + } + if (sample.time > video_end_time) { + video_end_time = sample.time; + } + } + double harmonic_framerate_fps = 0; + TimeDelta video_duration = video_end_time - video_start_time; + if (sum_squared_interframe_delays_secs > 0.0 && video_duration.IsFinite()) { + harmonic_framerate_fps = + video_duration.seconds<double>() / sum_squared_interframe_delays_secs; + } + + ReportResult("psnr", test_case_name, stats.psnr, "dB", + ImproveDirection::kBiggerIsBetter); + ReportResult("ssim", test_case_name, stats.ssim, "unitless", + ImproveDirection::kBiggerIsBetter); + ReportResult("transport_time", test_case_name, stats.transport_time_ms, "ms", + ImproveDirection::kSmallerIsBetter); + ReportResult("total_delay_incl_transport", test_case_name, + stats.total_delay_incl_transport_ms, "ms", + ImproveDirection::kSmallerIsBetter); + ReportResult("time_between_rendered_frames", test_case_name, + stats.time_between_rendered_frames_ms, "ms", + ImproveDirection::kSmallerIsBetter); + test::PrintResult("harmonic_framerate", "", test_case_name, + harmonic_framerate_fps, "Hz", /*important=*/false, + ImproveDirection::kBiggerIsBetter); + test::PrintResult("encode_frame_rate", "", test_case_name, + stats.encode_frame_rate.IsEmpty() + ? 0 + : stats.encode_frame_rate.GetEventsPerSecond(), + "Hz", /*important=*/false, + ImproveDirection::kBiggerIsBetter); + ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms", + ImproveDirection::kSmallerIsBetter); + ReportResult("time_between_freezes", test_case_name, + stats.time_between_freezes_ms, "ms", + ImproveDirection::kBiggerIsBetter); + ReportResult("freeze_time_ms", test_case_name, stats.freeze_time_ms, "ms", + ImproveDirection::kSmallerIsBetter); + ReportResult("pixels_per_frame", test_case_name, + stats.resolution_of_rendered_frame, "count", + ImproveDirection::kBiggerIsBetter); + test::PrintResult("min_psnr", "", test_case_name, + stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), "dB", + /*important=*/false, ImproveDirection::kBiggerIsBetter); + ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms", + ImproveDirection::kSmallerIsBetter); + ReportResult("receive_to_render_time", test_case_name, + stats.receive_to_render_time_ms, "ms", + ImproveDirection::kSmallerIsBetter); + test::PrintResult("dropped_frames", "", test_case_name, + frame_counters.dropped, "count", + /*important=*/false, ImproveDirection::kSmallerIsBetter); + test::PrintResult("frames_in_flight", "", test_case_name, + frame_counters.captured - frame_counters.rendered - + frame_counters.dropped, + "count", + /*important=*/false, ImproveDirection::kSmallerIsBetter); + test::PrintResult("rendered_frames", "", test_case_name, + frame_counters.rendered, "count", /*important=*/false, + ImproveDirection::kBiggerIsBetter); + ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered, + "count", ImproveDirection::kSmallerIsBetter); + ReportResult("target_encode_bitrate", test_case_name, + stats.target_encode_bitrate / kBitsInByte, "bytesPerSecond", + ImproveDirection::kNone); + test::PrintResult("actual_encode_bitrate", "", test_case_name, + static_cast<double>(stats.total_encoded_images_payload) / + test_duration.seconds<double>(), + "bytesPerSecond", /*important=*/false, + ImproveDirection::kNone); + + if (options_.report_detailed_frame_stats) { + test::PrintResult("num_encoded_frames", "", test_case_name, + frame_counters.encoded, "count", + /*important=*/false, ImproveDirection::kBiggerIsBetter); + test::PrintResult("num_decoded_frames", "", test_case_name, + frame_counters.decoded, "count", + /*important=*/false, ImproveDirection::kBiggerIsBetter); + test::PrintResult("num_send_key_frames", "", test_case_name, + stats.num_send_key_frames, "count", + /*important=*/false, ImproveDirection::kBiggerIsBetter); + test::PrintResult("num_recv_key_frames", "", test_case_name, + stats.num_recv_key_frames, "count", + /*important=*/false, ImproveDirection::kBiggerIsBetter); + + ReportResult("recv_key_frame_size_bytes", test_case_name, + stats.recv_key_frame_size_bytes, "count", + ImproveDirection::kBiggerIsBetter); + ReportResult("recv_delta_frame_size_bytes", test_case_name, + stats.recv_delta_frame_size_bytes, "count", + ImproveDirection::kBiggerIsBetter); + } +} + +void DefaultVideoQualityAnalyzer::ReportResult( + const std::string& metric_name, + const std::string& test_case_name, + const SamplesStatsCounter& counter, + const std::string& unit, + webrtc::test::ImproveDirection improve_direction) { + test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit, + /*important=*/false, improve_direction); +} + +std::string DefaultVideoQualityAnalyzer::GetTestCaseName( + const std::string& stream_label) const { + return test_label_ + "/" + stream_label; +} + +Timestamp DefaultVideoQualityAnalyzer::Now() { + return clock_->CurrentTime(); +} + +StatsKey DefaultVideoQualityAnalyzer::ToStatsKey( + const InternalStatsKey& key) const { + return StatsKey(streams_.name(key.stream), peers_->name(key.receiver)); +} + +std::string DefaultVideoQualityAnalyzer::ToMetricName( + const InternalStatsKey& key) const { + const std::string& stream_label = streams_.name(key.stream); + if (peers_->size() <= 2 && key.sender != key.receiver) { + return stream_label; + } + rtc::StringBuilder out; + out << stream_label << "_" << peers_->name(key.sender) << "_" + << peers_->name(key.receiver); + return out.str(); +} + +double DefaultVideoQualityAnalyzer::GetCpuUsagePercent() { + return cpu_measurer_.GetCpuUsagePercent(); +} + +std::map<std::string, std::vector<uint16_t>> +DefaultVideoQualityAnalyzer::GetStreamFrames() const { + MutexLock lock(&mutex_); + std::map<std::string, std::vector<uint16_t>> out; + for (auto entry_it : stream_to_frame_id_full_history_) { + out.insert({streams_.name(entry_it.first), entry_it.second}); + } + return out; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h new file mode 100644 index 0000000000..5aec7c1e26 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_H_ + +#include <atomic> +#include <cstdint> +#include <deque> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <vector> + +#include "api/array_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_type.h" +#include "rtc_base/event.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h" +#include "test/pc/e2e/analyzer/video/names_collection.h" +#include "test/testsupport/perf_test.h" + +namespace webrtc { + +class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { + public: + explicit DefaultVideoQualityAnalyzer( + webrtc::Clock* clock, + DefaultVideoQualityAnalyzerOptions options = {}); + ~DefaultVideoQualityAnalyzer() override; + + void Start(std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count) override; + uint16_t OnFrameCaptured(absl::string_view peer_name, + const std::string& stream_label, + const VideoFrame& frame) override; + void OnFramePreEncode(absl::string_view peer_name, + const VideoFrame& frame) override; + void OnFrameEncoded(absl::string_view peer_name, + uint16_t frame_id, + const EncodedImage& encoded_image, + const EncoderStats& stats) override; + void OnFrameDropped(absl::string_view peer_name, + EncodedImageCallback::DropReason reason) override; + void OnFramePreDecode(absl::string_view peer_name, + uint16_t frame_id, + const EncodedImage& input_image) override; + void OnFrameDecoded(absl::string_view peer_name, + const VideoFrame& frame, + const DecoderStats& stats) override; + void OnFrameRendered(absl::string_view peer_name, + const VideoFrame& frame) override; + void OnEncoderError(absl::string_view peer_name, + const VideoFrame& frame, + int32_t error_code) override; + void OnDecoderError(absl::string_view peer_name, + uint16_t frame_id, + int32_t error_code) override; + + void RegisterParticipantInCall(absl::string_view peer_name) override; + void UnregisterParticipantInCall(absl::string_view peer_name) override; + + void Stop() override; + std::string GetStreamLabel(uint16_t frame_id) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override {} + + // Returns set of stream labels, that were met during test call. + std::set<StatsKey> GetKnownVideoStreams() const; + VideoStreamsInfo GetKnownStreams() const; + FrameCounters GetGlobalCounters() const; + // Returns frame counter for frames received without frame id set. + std::map<std::string, FrameCounters> GetUnknownSenderFrameCounters() const; + // Returns frame counter per stream label. Valid stream labels can be obtained + // by calling GetKnownVideoStreams() + std::map<StatsKey, FrameCounters> GetPerStreamCounters() const; + // Returns video quality stats per stream label. Valid stream labels can be + // obtained by calling GetKnownVideoStreams() + std::map<StatsKey, StreamStats> GetStats() const; + AnalyzerStats GetAnalyzerStats() const; + double GetCpuUsagePercent(); + + // Returns mapping from the stream label to the history of frames that were + // met in this stream in the order as they were captured. + std::map<std::string, std::vector<uint16_t>> GetStreamFrames() const; + + private: + enum State { kNew, kActive, kStopped }; + + // Returns next frame id to use. Frame ID can't be `VideoFrame::kNotSetId`, + // because this value is reserved by `VideoFrame` as "ID not set". + uint16_t GetNextFrameId() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Report results for all metrics for all streams. + void ReportResults(); + void ReportResults(const std::string& test_case_name, + const StreamStats& stats, + const FrameCounters& frame_counters) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + // Report result for single metric for specified stream. + static void ReportResult(const std::string& metric_name, + const std::string& test_case_name, + const SamplesStatsCounter& counter, + const std::string& unit, + webrtc::test::ImproveDirection improve_direction = + webrtc::test::ImproveDirection::kNone); + // Returns name of current test case for reporting. + std::string GetTestCaseName(const std::string& stream_label) const; + Timestamp Now(); + StatsKey ToStatsKey(const InternalStatsKey& key) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + // Returns string representation of stats key for metrics naming. Used for + // backward compatibility by metrics naming for 2 peers cases. + std::string ToMetricName(const InternalStatsKey& key) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + static const uint16_t kStartingFrameId = 1; + + const DefaultVideoQualityAnalyzerOptions options_; + webrtc::Clock* const clock_; + + std::string test_label_; + + mutable Mutex mutex_; + uint16_t next_frame_id_ RTC_GUARDED_BY(mutex_) = kStartingFrameId; + std::unique_ptr<NamesCollection> peers_ RTC_GUARDED_BY(mutex_); + State state_ RTC_GUARDED_BY(mutex_) = State::kNew; + Timestamp start_time_ RTC_GUARDED_BY(mutex_) = Timestamp::MinusInfinity(); + // Mapping from stream label to unique size_t value to use in stats and avoid + // extra string copying. + NamesCollection streams_ RTC_GUARDED_BY(mutex_); + // Frames that were captured by all streams and still aren't rendered on + // receivers or deemed dropped. Frame with id X can be removed from this map + // if: + // 1. The frame with id X was received in OnFrameRendered by all expected + // receivers. + // 2. The frame with id Y > X was received in OnFrameRendered by all expected + // receivers. + // 3. Next available frame id for newly captured frame is X + // 4. There too many frames in flight for current video stream and X is the + // oldest frame id in this stream. In such case only the frame content + // will be removed, but the map entry will be preserved. + std::map<uint16_t, FrameInFlight> captured_frames_in_flight_ + RTC_GUARDED_BY(mutex_); + // Global frames count for all video streams. + FrameCounters frame_counters_ RTC_GUARDED_BY(mutex_); + // Frame counters for received frames without video frame id set. + // Map from peer name to the frame counters. + std::map<std::string, FrameCounters> unknown_sender_frame_counters_ + RTC_GUARDED_BY(mutex_); + // Frame counters per each stream per each receiver. + std::map<InternalStatsKey, FrameCounters> stream_frame_counters_ + RTC_GUARDED_BY(mutex_); + // Map from stream index in `streams_` to its StreamState. + std::map<size_t, StreamState> stream_states_ RTC_GUARDED_BY(mutex_); + // Map from stream index in `streams_` to sender peer index in `peers_`. + std::map<size_t, size_t> stream_to_sender_ RTC_GUARDED_BY(mutex_); + + // Stores history mapping between stream index in `streams_` and frame ids. + // Updated when frame id overlap. It required to properly return stream label + // after 1st frame from simulcast streams was already rendered and last is + // still encoding. + std::map<size_t, std::set<uint16_t>> stream_to_frame_id_history_ + RTC_GUARDED_BY(mutex_); + // Map from stream index to the list of frames as they were met in the stream. + std::map<size_t, std::vector<uint16_t>> stream_to_frame_id_full_history_ + RTC_GUARDED_BY(mutex_); + AnalyzerStats analyzer_stats_ RTC_GUARDED_BY(mutex_); + + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer_; + DefaultVideoQualityAnalyzerFramesComparator frames_comparator_; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.cc new file mode 100644 index 0000000000..847c9f09a6 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.cc @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h" + +#include "rtc_base/cpu_time.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/system_time.h" + +namespace webrtc { + +void DefaultVideoQualityAnalyzerCpuMeasurer::StartMeasuringCpuProcessTime() { + MutexLock lock(&mutex_); + cpu_time_ -= rtc::GetProcessCpuTimeNanos(); + wallclock_time_ -= rtc::SystemTimeNanos(); +} + +void DefaultVideoQualityAnalyzerCpuMeasurer::StopMeasuringCpuProcessTime() { + MutexLock lock(&mutex_); + cpu_time_ += rtc::GetProcessCpuTimeNanos(); + wallclock_time_ += rtc::SystemTimeNanos(); +} + +void DefaultVideoQualityAnalyzerCpuMeasurer::StartExcludingCpuThreadTime() { + MutexLock lock(&mutex_); + cpu_time_ += rtc::GetThreadCpuTimeNanos(); +} + +void DefaultVideoQualityAnalyzerCpuMeasurer::StopExcludingCpuThreadTime() { + MutexLock lock(&mutex_); + cpu_time_ -= rtc::GetThreadCpuTimeNanos(); +} + +double DefaultVideoQualityAnalyzerCpuMeasurer::GetCpuUsagePercent() { + MutexLock lock(&mutex_); + return static_cast<double>(cpu_time_) / wallclock_time_ * 100.0; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h new file mode 100644 index 0000000000..dd9fa07af2 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_CPU_MEASURER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_CPU_MEASURER_H_ + +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { + +// This class is thread safe. +class DefaultVideoQualityAnalyzerCpuMeasurer { + public: + double GetCpuUsagePercent(); + + void StartMeasuringCpuProcessTime(); + void StopMeasuringCpuProcessTime(); + void StartExcludingCpuThreadTime(); + void StopExcludingCpuThreadTime(); + + private: + Mutex mutex_; + int64_t cpu_time_ RTC_GUARDED_BY(mutex_) = 0; + int64_t wallclock_time_ RTC_GUARDED_BY(mutex_) = 0; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_CPU_MEASURER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc new file mode 100644 index 0000000000..13e77b4586 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h" + +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "api/video/video_frame_type.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" + +namespace webrtc { +namespace { + +template <typename T> +absl::optional<T> MaybeGetValue(const std::map<size_t, T>& map, size_t key) { + auto it = map.find(key); + if (it == map.end()) { + return absl::nullopt; + } + return it->second; +} + +} // namespace + +FrameInFlight::FrameInFlight(size_t stream, + VideoFrame frame, + Timestamp captured_time, + std::set<size_t> expected_receivers) + : stream_(stream), + expected_receivers_(std::move(expected_receivers)), + frame_(std::move(frame)), + captured_time_(captured_time) {} + +bool FrameInFlight::RemoveFrame() { + if (!frame_) { + return false; + } + frame_ = absl::nullopt; + return true; +} + +void FrameInFlight::SetFrameId(uint16_t id) { + if (frame_) { + frame_->set_id(id); + } +} + +std::vector<size_t> FrameInFlight::GetPeersWhichDidntReceive() const { + std::vector<size_t> out; + for (size_t peer : expected_receivers_) { + auto it = receiver_stats_.find(peer); + if (it == receiver_stats_.end() || + (!it->second.dropped && it->second.rendered_time.IsInfinite())) { + out.push_back(peer); + } + } + return out; +} + +bool FrameInFlight::HaveAllPeersReceived() const { + for (size_t peer : expected_receivers_) { + auto it = receiver_stats_.find(peer); + if (it == receiver_stats_.end()) { + return false; + } + + if (!it->second.dropped && it->second.rendered_time.IsInfinite()) { + return false; + } + } + return true; +} + +void FrameInFlight::OnFrameEncoded(webrtc::Timestamp time, + VideoFrameType frame_type, + DataSize encoded_image_size, + uint32_t target_encode_bitrate, + StreamCodecInfo used_encoder) { + encoded_time_ = time; + frame_type_ = frame_type; + encoded_image_size_ = encoded_image_size; + target_encode_bitrate_ += target_encode_bitrate; + // Update used encoder info. If simulcast/SVC is used, this method can + // be called multiple times, in such case we should preserve the value + // of `used_encoder_.switched_on_at` from the first invocation as the + // smallest one. + Timestamp encoder_switched_on_at = used_encoder_.has_value() + ? used_encoder_->switched_on_at + : Timestamp::PlusInfinity(); + RTC_DCHECK(used_encoder.switched_on_at.IsFinite()); + RTC_DCHECK(used_encoder.switched_from_at.IsFinite()); + used_encoder_ = used_encoder; + if (encoder_switched_on_at < used_encoder_->switched_on_at) { + used_encoder_->switched_on_at = encoder_switched_on_at; + } +} + +void FrameInFlight::OnFramePreDecode(size_t peer, + webrtc::Timestamp received_time, + webrtc::Timestamp decode_start_time, + VideoFrameType frame_type, + DataSize encoded_image_size) { + receiver_stats_[peer].received_time = received_time; + receiver_stats_[peer].decode_start_time = decode_start_time; + receiver_stats_[peer].frame_type = frame_type; + receiver_stats_[peer].encoded_image_size = encoded_image_size; +} + +bool FrameInFlight::HasReceivedTime(size_t peer) const { + auto it = receiver_stats_.find(peer); + if (it == receiver_stats_.end()) { + return false; + } + return it->second.received_time.IsFinite(); +} + +void FrameInFlight::OnFrameDecoded(size_t peer, + webrtc::Timestamp time, + StreamCodecInfo used_decoder) { + receiver_stats_[peer].decode_end_time = time; + receiver_stats_[peer].used_decoder = used_decoder; +} + +bool FrameInFlight::HasDecodeEndTime(size_t peer) const { + auto it = receiver_stats_.find(peer); + if (it == receiver_stats_.end()) { + return false; + } + return it->second.decode_end_time.IsFinite(); +} + +void FrameInFlight::OnFrameRendered(size_t peer, + webrtc::Timestamp time, + int width, + int height) { + receiver_stats_[peer].rendered_time = time; + receiver_stats_[peer].rendered_frame_width = width; + receiver_stats_[peer].rendered_frame_height = height; +} + +bool FrameInFlight::HasRenderedTime(size_t peer) const { + auto it = receiver_stats_.find(peer); + if (it == receiver_stats_.end()) { + return false; + } + return it->second.rendered_time.IsFinite(); +} + +bool FrameInFlight::IsDropped(size_t peer) const { + auto it = receiver_stats_.find(peer); + if (it == receiver_stats_.end()) { + return false; + } + return it->second.dropped; +} + +FrameStats FrameInFlight::GetStatsForPeer(size_t peer) const { + FrameStats stats(captured_time_); + stats.pre_encode_time = pre_encode_time_; + stats.encoded_time = encoded_time_; + stats.target_encode_bitrate = target_encode_bitrate_; + stats.encoded_frame_type = frame_type_; + stats.encoded_image_size = encoded_image_size_; + stats.used_encoder = used_encoder_; + + absl::optional<ReceiverFrameStats> receiver_stats = + MaybeGetValue<ReceiverFrameStats>(receiver_stats_, peer); + if (receiver_stats.has_value()) { + stats.received_time = receiver_stats->received_time; + stats.decode_start_time = receiver_stats->decode_start_time; + stats.decode_end_time = receiver_stats->decode_end_time; + stats.rendered_time = receiver_stats->rendered_time; + stats.prev_frame_rendered_time = receiver_stats->prev_frame_rendered_time; + stats.rendered_frame_width = receiver_stats->rendered_frame_width; + stats.rendered_frame_height = receiver_stats->rendered_frame_height; + stats.used_decoder = receiver_stats->used_decoder; + stats.pre_decoded_frame_type = receiver_stats->frame_type; + stats.pre_decoded_image_size = receiver_stats->encoded_image_size; + } + return stats; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h new file mode 100644 index 0000000000..a1ce8faf5b --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAME_IN_FLIGHT_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAME_IN_FLIGHT_H_ + +#include <map> +#include <set> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_type.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" + +namespace webrtc { + +struct ReceiverFrameStats { + // Time when last packet of a frame was received. + Timestamp received_time = Timestamp::MinusInfinity(); + Timestamp decode_start_time = Timestamp::MinusInfinity(); + Timestamp decode_end_time = Timestamp::MinusInfinity(); + Timestamp rendered_time = Timestamp::MinusInfinity(); + Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity(); + + // Type and encoded size of received frame. + VideoFrameType frame_type = VideoFrameType::kEmptyFrame; + DataSize encoded_image_size = DataSize::Bytes(0); + + absl::optional<int> rendered_frame_width = absl::nullopt; + absl::optional<int> rendered_frame_height = absl::nullopt; + + // Can be not set if frame was dropped in the network. + absl::optional<StreamCodecInfo> used_decoder = absl::nullopt; + + bool dropped = false; +}; + +// Represents a frame which was sent by sender and is currently on the way to +// multiple receivers. Some receivers may receive this frame and some don't. +// +// Contains all statistic associated with the frame and gathered in multiple +// points of the video pipeline. +// +// Internally may store the copy of the source frame which was sent. In such +// case this frame is "alive". +class FrameInFlight { + public: + FrameInFlight(size_t stream, + VideoFrame frame, + Timestamp captured_time, + std::set<size_t> expected_receivers); + + size_t stream() const { return stream_; } + // Returns internal copy of source `VideoFrame` or `absl::nullopt` if it was + // removed before. + const absl::optional<VideoFrame>& frame() const { return frame_; } + // Removes internal copy of the source `VideoFrame` to free up extra memory. + // Returns was frame removed or not. + bool RemoveFrame(); + void SetFrameId(uint16_t id); + + void AddExpectedReceiver(size_t peer) { expected_receivers_.insert(peer); } + + void RemoveExpectedReceiver(size_t peer) { expected_receivers_.erase(peer); } + + std::vector<size_t> GetPeersWhichDidntReceive() const; + + // Returns if all peers which were expected to receive this frame actually + // received it or not. + bool HaveAllPeersReceived() const; + + void SetPreEncodeTime(webrtc::Timestamp time) { pre_encode_time_ = time; } + + void OnFrameEncoded(webrtc::Timestamp time, + VideoFrameType frame_type, + DataSize encoded_image_size, + uint32_t target_encode_bitrate, + StreamCodecInfo used_encoder); + + bool HasEncodedTime() const { return encoded_time_.IsFinite(); } + + void OnFramePreDecode(size_t peer, + webrtc::Timestamp received_time, + webrtc::Timestamp decode_start_time, + VideoFrameType frame_type, + DataSize encoded_image_size); + + bool HasReceivedTime(size_t peer) const; + + void OnFrameDecoded(size_t peer, + webrtc::Timestamp time, + StreamCodecInfo used_decoder); + + bool HasDecodeEndTime(size_t peer) const; + + void OnFrameRendered(size_t peer, + webrtc::Timestamp time, + int width, + int height); + + bool HasRenderedTime(size_t peer) const; + + // Crash if rendered time is not set for specified `peer`. + webrtc::Timestamp rendered_time(size_t peer) const { + return receiver_stats_.at(peer).rendered_time; + } + + // Marks that frame was dropped and wasn't seen by particular `peer`. + void MarkDropped(size_t peer) { receiver_stats_[peer].dropped = true; } + bool IsDropped(size_t peer) const; + + void SetPrevFrameRenderedTime(size_t peer, webrtc::Timestamp time) { + receiver_stats_[peer].prev_frame_rendered_time = time; + } + + FrameStats GetStatsForPeer(size_t peer) const; + + private: + const size_t stream_; + // Set of peer's indexes who are expected to receive this frame. This is not + // the set of peer's indexes that received the frame. For example, if peer A + // was among expected receivers, it received frame and then left the call, A + // will be removed from this set, but the Stats for peer A still will be + // preserved in the FrameInFlight. + // + // This set is used to determine if this frame is expected to be received by + // any peer or can be safely deleted. It is responsibility of the user of this + // object to decide when it should be deleted. + std::set<size_t> expected_receivers_; + absl::optional<VideoFrame> frame_; + + // Frame events timestamp. + Timestamp captured_time_; + Timestamp pre_encode_time_ = Timestamp::MinusInfinity(); + Timestamp encoded_time_ = Timestamp::MinusInfinity(); + // Type and encoded size of sent frame. + VideoFrameType frame_type_ = VideoFrameType::kEmptyFrame; + DataSize encoded_image_size_ = DataSize::Bytes(0); + uint32_t target_encode_bitrate_ = 0; + // Can be not set if frame was dropped by encoder. + absl::optional<StreamCodecInfo> used_encoder_ = absl::nullopt; + // Map from the receiver peer's index to frame stats for that peer. + std::map<size_t, ReceiverFrameStats> receiver_stats_; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAME_IN_FLIGHT_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc new file mode 100644 index 0000000000..e216c70c6f --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc @@ -0,0 +1,533 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" + +#include <algorithm> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/scoped_refptr.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame_type.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_base/checks.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_tools/frame_analyzer/video_geometry_aligner.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +namespace webrtc { +namespace { + +constexpr TimeDelta kFreezeThreshold = TimeDelta::Millis(150); +constexpr int kMaxActiveComparisons = 10; + +SamplesStatsCounter::StatsSample StatsSample(double value, + Timestamp sampling_time) { + return SamplesStatsCounter::StatsSample{value, sampling_time}; +} + +SamplesStatsCounter::StatsSample StatsSample(TimeDelta duration, + Timestamp sampling_time) { + return SamplesStatsCounter::StatsSample{duration.ms<double>(), sampling_time}; +} + +FrameComparison ValidateFrameComparison(FrameComparison comparison) { + RTC_DCHECK(comparison.frame_stats.captured_time.IsFinite()) + << "Any comparison has to have finite captured_time"; + switch (comparison.type) { + case FrameComparisonType::kRegular: + // Regular comparison has to have all FrameStats filled in. + RTC_DCHECK(comparison.captured.has_value() || + comparison.overload_reason != OverloadReason::kNone) + << "Regular comparison has to have captured frame if it's not " + << "overloaded comparison"; + RTC_DCHECK(comparison.rendered.has_value() || + comparison.overload_reason != OverloadReason::kNone) + << "rendered frame has to be presented if it's not overloaded " + << "comparison"; + RTC_DCHECK(comparison.frame_stats.pre_encode_time.IsFinite()) + << "Regular comparison has to have finite pre_encode_time"; + RTC_DCHECK(comparison.frame_stats.encoded_time.IsFinite()) + << "Regular comparison has to have finite encoded_time"; + RTC_DCHECK(comparison.frame_stats.received_time.IsFinite()) + << "Regular comparison has to have finite received_time"; + RTC_DCHECK(comparison.frame_stats.decode_start_time.IsFinite()) + << "Regular comparison has to have finite decode_start_time"; + RTC_DCHECK(comparison.frame_stats.decode_end_time.IsFinite()) + << "Regular comparison has to have finite decode_end_time"; + RTC_DCHECK(comparison.frame_stats.rendered_time.IsFinite()) + << "Regular comparison has to have finite rendered_time"; + RTC_DCHECK(comparison.frame_stats.rendered_frame_width.has_value()) + << "Regular comparison has to have rendered_frame_width"; + RTC_DCHECK(comparison.frame_stats.rendered_frame_height.has_value()) + << "Regular comparison has to have rendered_frame_height"; + RTC_DCHECK(comparison.frame_stats.used_encoder.has_value()) + << "Regular comparison has to have used_encoder"; + RTC_DCHECK(comparison.frame_stats.used_decoder.has_value()) + << "Regular comparison has to have used_decoder"; + break; + case FrameComparisonType::kDroppedFrame: + // Frame can be dropped before encoder, by encoder, inside network or + // after decoder. + RTC_DCHECK(!comparison.captured.has_value()) + << "Dropped frame comparison can't have captured frame"; + RTC_DCHECK(!comparison.rendered.has_value()) + << "Dropped frame comparison can't have rendered frame"; + + if (comparison.frame_stats.encoded_time.IsFinite()) { + RTC_DCHECK(comparison.frame_stats.used_encoder.has_value()) + << "Dropped frame comparison has to have used_encoder when " + << "encoded_time is set"; + RTC_DCHECK(comparison.frame_stats.pre_encode_time.IsFinite()) + << "Dropped frame comparison has to have finite pre_encode_time " + << "when encoded_time is finite."; + } + + if (comparison.frame_stats.decode_end_time.IsFinite()) { + RTC_DCHECK(comparison.frame_stats.received_time.IsFinite()) + << "Dropped frame comparison has to have received_time when " + << "decode_end_time is set"; + RTC_DCHECK(comparison.frame_stats.decode_start_time.IsFinite()) + << "Dropped frame comparison has to have decode_start_time when " + << "decode_end_time is set"; + RTC_DCHECK(comparison.frame_stats.used_decoder.has_value()) + << "Dropped frame comparison has to have used_decoder when " + << "decode_end_time is set"; + } else { + RTC_DCHECK(!comparison.frame_stats.received_time.IsFinite()) + << "Dropped frame comparison can't have received_time when " + << "decode_end_time is not set"; + RTC_DCHECK(!comparison.frame_stats.decode_start_time.IsFinite()) + << "Dropped frame comparison can't have decode_start_time when " + << "decode_end_time is not set"; + RTC_DCHECK(!comparison.frame_stats.used_decoder.has_value()) + << "Dropped frame comparison can't have used_decoder when " + << "decode_end_time is not set"; + } + RTC_DCHECK(!comparison.frame_stats.rendered_time.IsFinite()) + << "Dropped frame comparison can't have rendered_time"; + RTC_DCHECK(!comparison.frame_stats.rendered_frame_width.has_value()) + << "Dropped frame comparison can't have rendered_frame_width"; + RTC_DCHECK(!comparison.frame_stats.rendered_frame_height.has_value()) + << "Dropped frame comparison can't have rendered_frame_height"; + break; + case FrameComparisonType::kFrameInFlight: + // Frame in flight comparison may miss almost any FrameStats, but if + // stats for stage X are set, then stats for stage X - 1 also has to be + // set. Also these frames were never rendered. + RTC_DCHECK(!comparison.captured.has_value()) + << "Frame in flight comparison can't have captured frame"; + RTC_DCHECK(!comparison.rendered.has_value()) + << "Frame in flight comparison can't have rendered frame"; + RTC_DCHECK(!comparison.frame_stats.rendered_time.IsFinite()) + << "Frame in flight comparison can't have rendered_time"; + RTC_DCHECK(!comparison.frame_stats.rendered_frame_width.has_value()) + << "Frame in flight comparison can't have rendered_frame_width"; + RTC_DCHECK(!comparison.frame_stats.rendered_frame_height.has_value()) + << "Frame in flight comparison can't have rendered_frame_height"; + + if (comparison.frame_stats.decode_end_time.IsFinite()) { + RTC_DCHECK(comparison.frame_stats.used_decoder.has_value()) + << "Frame in flight comparison has to have used_decoder when " + << "decode_end_time is set"; + RTC_DCHECK(comparison.frame_stats.decode_start_time.IsFinite()) + << "Frame in flight comparison has to have finite " + << "decode_start_time when decode_end_time is finite."; + } + if (comparison.frame_stats.decode_start_time.IsFinite()) { + RTC_DCHECK(comparison.frame_stats.received_time.IsFinite()) + << "Frame in flight comparison has to have finite received_time " + << "when decode_start_time is finite."; + } + if (comparison.frame_stats.received_time.IsFinite()) { + RTC_DCHECK(comparison.frame_stats.encoded_time.IsFinite()) + << "Frame in flight comparison has to have finite encoded_time " + << "when received_time is finite."; + } + if (comparison.frame_stats.encoded_time.IsFinite()) { + RTC_DCHECK(comparison.frame_stats.used_encoder.has_value()) + << "Frame in flight comparison has to have used_encoder when " + << "encoded_time is set"; + RTC_DCHECK(comparison.frame_stats.pre_encode_time.IsFinite()) + << "Frame in flight comparison has to have finite pre_encode_time " + << "when encoded_time is finite."; + } + break; + } + return comparison; +} + +} // namespace + +void DefaultVideoQualityAnalyzerFramesComparator::Start(int max_threads_count) { + for (int i = 0; i < max_threads_count; i++) { + thread_pool_.push_back(rtc::PlatformThread::SpawnJoinable( + [this] { ProcessComparisons(); }, + "DefaultVideoQualityAnalyzerFramesComparator-" + std::to_string(i))); + } + { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kNew) << "Frames comparator is already started"; + state_ = State::kActive; + } + cpu_measurer_.StartMeasuringCpuProcessTime(); +} + +void DefaultVideoQualityAnalyzerFramesComparator::Stop( + const std::map<InternalStatsKey, Timestamp>& last_rendered_frame_times) { + { + MutexLock lock(&mutex_); + if (state_ == State::kStopped) { + return; + } + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + state_ = State::kStopped; + } + cpu_measurer_.StopMeasuringCpuProcessTime(); + comparison_available_event_.Set(); + thread_pool_.clear(); + + { + MutexLock lock(&mutex_); + // Perform final Metrics update. On this place analyzer is stopped and no + // one holds any locks. + + // Time between freezes. + // Count time since the last freeze to the end of the call as time + // between freezes. + for (auto& entry : last_rendered_frame_times) { + const InternalStatsKey& stats_key = entry.first; + const Timestamp& last_rendered_frame_time = entry.second; + + // If there are no freezes in the call we have to report + // time_between_freezes_ms as call duration and in such case + // `last_rendered_frame_time` for this stream will be stream start time. + // If there is freeze, then we need add time from last rendered frame + // to last freeze end as time between freezes. + stream_stats_.at(stats_key).time_between_freezes_ms.AddSample(StatsSample( + last_rendered_frame_time - stream_last_freeze_end_time_.at(stats_key), + Now())); + } + } +} + +void DefaultVideoQualityAnalyzerFramesComparator::EnsureStatsForStream( + size_t stream_index, + size_t sender_peer_index, + size_t peers_count, + Timestamp captured_time, + Timestamp start_time) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + + for (size_t i = 0; i < peers_count; ++i) { + if (i == sender_peer_index && !options_.enable_receive_own_stream) { + continue; + } + InternalStatsKey stats_key(stream_index, sender_peer_index, i); + if (stream_stats_.find(stats_key) == stream_stats_.end()) { + stream_stats_.insert({stats_key, StreamStats(captured_time)}); + // Assume that the first freeze was before first stream frame captured. + // This way time before the first freeze would be counted as time + // between freezes. + stream_last_freeze_end_time_.insert({stats_key, start_time}); + } else { + // When we see some `stream_label` for the first time we need to create + // stream stats object for it and set up some states, but we need to do + // it only once and for all receivers, so on the next frame on the same + // `stream_label` we can be sure, that it's already done and we needn't + // to scan though all peers again. + break; + } + } +} + +void DefaultVideoQualityAnalyzerFramesComparator::RegisterParticipantInCall( + rtc::ArrayView<std::pair<InternalStatsKey, Timestamp>> stream_started_time, + Timestamp start_time) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + + for (const std::pair<InternalStatsKey, Timestamp>& pair : + stream_started_time) { + stream_stats_.insert({pair.first, StreamStats(pair.second)}); + stream_last_freeze_end_time_.insert({pair.first, start_time}); + } +} + +void DefaultVideoQualityAnalyzerFramesComparator::AddComparison( + InternalStatsKey stats_key, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + AddComparisonInternal(std::move(stats_key), std::move(captured), + std::move(rendered), type, std::move(frame_stats)); +} + +void DefaultVideoQualityAnalyzerFramesComparator::AddComparison( + InternalStatsKey stats_key, + int skipped_between_rendered, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats) { + MutexLock lock(&mutex_); + RTC_CHECK_EQ(state_, State::kActive) + << "Frames comparator has to be started before it will be used"; + stream_stats_.at(stats_key).skipped_between_rendered.AddSample( + StatsSample(skipped_between_rendered, Now())); + AddComparisonInternal(std::move(stats_key), std::move(captured), + std::move(rendered), type, std::move(frame_stats)); +} + +void DefaultVideoQualityAnalyzerFramesComparator::AddComparisonInternal( + InternalStatsKey stats_key, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats) { + cpu_measurer_.StartExcludingCpuThreadTime(); + frames_comparator_stats_.comparisons_queue_size.AddSample( + StatsSample(comparisons_.size(), Now())); + // If there too many computations waiting in the queue, we won't provide + // frames itself to make future computations lighter. + if (comparisons_.size() >= kMaxActiveComparisons) { + comparisons_.emplace_back(ValidateFrameComparison( + FrameComparison(std::move(stats_key), /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, type, + std::move(frame_stats), OverloadReason::kCpu))); + } else { + OverloadReason overload_reason = OverloadReason::kNone; + if (!captured && type == FrameComparisonType::kRegular) { + overload_reason = OverloadReason::kMemory; + } + comparisons_.emplace_back(ValidateFrameComparison(FrameComparison( + std::move(stats_key), std::move(captured), std::move(rendered), type, + std::move(frame_stats), overload_reason))); + } + comparison_available_event_.Set(); + cpu_measurer_.StopExcludingCpuThreadTime(); +} + +void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparisons() { + while (true) { + // Try to pick next comparison to perform from the queue. + absl::optional<FrameComparison> comparison = absl::nullopt; + bool more_new_comparisons_expected; + { + MutexLock lock(&mutex_); + if (!comparisons_.empty()) { + comparison = comparisons_.front(); + comparisons_.pop_front(); + if (!comparisons_.empty()) { + comparison_available_event_.Set(); + } + } + // If state is stopped => no new frame comparisons are expected. + more_new_comparisons_expected = state_ != State::kStopped; + } + if (!comparison) { + if (!more_new_comparisons_expected) { + comparison_available_event_.Set(); + return; + } + comparison_available_event_.Wait(1000); + continue; + } + + cpu_measurer_.StartExcludingCpuThreadTime(); + ProcessComparison(comparison.value()); + cpu_measurer_.StopExcludingCpuThreadTime(); + } +} + +void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison( + const FrameComparison& comparison) { + // Comparison is checked to be valid before adding, so we can use this + // assumptions during computations. + + // Perform expensive psnr and ssim calculations while not holding lock. + double psnr = -1.0; + double ssim = -1.0; + if ((options_.compute_psnr || options_.compute_ssim) && + comparison.captured.has_value() && comparison.rendered.has_value()) { + rtc::scoped_refptr<I420BufferInterface> reference_buffer = + comparison.captured->video_frame_buffer()->ToI420(); + rtc::scoped_refptr<I420BufferInterface> test_buffer = + comparison.rendered->video_frame_buffer()->ToI420(); + if (options_.adjust_cropping_before_comparing_frames) { + test_buffer = ScaleVideoFrameBuffer( + *test_buffer, reference_buffer->width(), reference_buffer->height()); + reference_buffer = test::AdjustCropping(reference_buffer, test_buffer); + } + if (options_.compute_psnr) { + psnr = options_.use_weighted_psnr + ? I420WeightedPSNR(*reference_buffer, *test_buffer) + : I420PSNR(*reference_buffer, *test_buffer); + } + if (options_.compute_ssim) { + ssim = I420SSIM(*reference_buffer, *test_buffer); + } + } + + const FrameStats& frame_stats = comparison.frame_stats; + + MutexLock lock(&mutex_); + auto stats_it = stream_stats_.find(comparison.stats_key); + RTC_CHECK(stats_it != stream_stats_.end()) << comparison.stats_key.ToString(); + StreamStats* stats = &stats_it->second; + + frames_comparator_stats_.comparisons_done++; + if (comparison.overload_reason == OverloadReason::kCpu) { + frames_comparator_stats_.cpu_overloaded_comparisons_done++; + } else if (comparison.overload_reason == OverloadReason::kMemory) { + frames_comparator_stats_.memory_overloaded_comparisons_done++; + } + + if (psnr > 0) { + stats->psnr.AddSample(StatsSample(psnr, frame_stats.rendered_time)); + } + if (ssim > 0) { + stats->ssim.AddSample(StatsSample(ssim, frame_stats.received_time)); + } + + // Compute dropped phase for dropped frame + if (comparison.type == FrameComparisonType::kDroppedFrame) { + FrameDropPhase dropped_phase; + if (frame_stats.decode_end_time.IsFinite()) { + dropped_phase = FrameDropPhase::kAfterDecoder; + } else if (frame_stats.encoded_time.IsFinite()) { + dropped_phase = FrameDropPhase::kTransport; + } else if (frame_stats.pre_encode_time.IsFinite()) { + dropped_phase = FrameDropPhase::kByEncoder; + } else { + dropped_phase = FrameDropPhase::kBeforeEncoder; + } + stats->dropped_by_phase[dropped_phase]++; + } + + if (frame_stats.encoded_time.IsFinite()) { + stats->encode_time_ms.AddSample( + StatsSample(frame_stats.encoded_time - frame_stats.pre_encode_time, + frame_stats.encoded_time)); + stats->encode_frame_rate.AddEvent(frame_stats.encoded_time); + stats->total_encoded_images_payload += + frame_stats.encoded_image_size.bytes(); + stats->target_encode_bitrate.AddSample(StatsSample( + frame_stats.target_encode_bitrate, frame_stats.encoded_time)); + + // Stats sliced on encoded frame type. + if (frame_stats.encoded_frame_type == VideoFrameType::kVideoFrameKey) { + ++stats->num_send_key_frames; + } + } + // Next stats can be calculated only if frame was received on remote side. + if (comparison.type != FrameComparisonType::kDroppedFrame) { + if (frame_stats.rendered_time.IsFinite()) { + stats->resolution_of_rendered_frame.AddSample( + StatsSample(*comparison.frame_stats.rendered_frame_width * + *comparison.frame_stats.rendered_frame_height, + frame_stats.rendered_time)); + stats->total_delay_incl_transport_ms.AddSample( + StatsSample(frame_stats.rendered_time - frame_stats.captured_time, + frame_stats.received_time)); + stats->receive_to_render_time_ms.AddSample( + StatsSample(frame_stats.rendered_time - frame_stats.received_time, + frame_stats.rendered_time)); + } + if (frame_stats.decode_start_time.IsFinite()) { + stats->transport_time_ms.AddSample( + StatsSample(frame_stats.decode_start_time - frame_stats.encoded_time, + frame_stats.decode_start_time)); + + // Stats sliced on decoded frame type. + if (frame_stats.pre_decoded_frame_type == + VideoFrameType::kVideoFrameKey) { + ++stats->num_recv_key_frames; + stats->recv_key_frame_size_bytes.AddSample( + StatsSample(frame_stats.pre_decoded_image_size.bytes(), + frame_stats.decode_start_time)); + } else if (frame_stats.pre_decoded_frame_type == + VideoFrameType::kVideoFrameDelta) { + stats->recv_delta_frame_size_bytes.AddSample( + StatsSample(frame_stats.pre_decoded_image_size.bytes(), + frame_stats.decode_start_time)); + } + } + if (frame_stats.decode_end_time.IsFinite()) { + stats->decode_time_ms.AddSample(StatsSample( + frame_stats.decode_end_time - frame_stats.decode_start_time, + frame_stats.decode_end_time)); + } + + if (frame_stats.prev_frame_rendered_time.IsFinite() && + frame_stats.rendered_time.IsFinite()) { + TimeDelta time_between_rendered_frames = + frame_stats.rendered_time - frame_stats.prev_frame_rendered_time; + stats->time_between_rendered_frames_ms.AddSample( + StatsSample(time_between_rendered_frames, frame_stats.rendered_time)); + TimeDelta average_time_between_rendered_frames = TimeDelta::Millis( + stats->time_between_rendered_frames_ms.GetAverage()); + if (time_between_rendered_frames > + std::max(kFreezeThreshold + average_time_between_rendered_frames, + 3 * average_time_between_rendered_frames)) { + stats->freeze_time_ms.AddSample(StatsSample( + time_between_rendered_frames, frame_stats.rendered_time)); + auto freeze_end_it = + stream_last_freeze_end_time_.find(comparison.stats_key); + RTC_DCHECK(freeze_end_it != stream_last_freeze_end_time_.end()); + stats->time_between_freezes_ms.AddSample(StatsSample( + frame_stats.prev_frame_rendered_time - freeze_end_it->second, + frame_stats.rendered_time)); + freeze_end_it->second = frame_stats.rendered_time; + } + } + } + // Compute stream codec info. + if (frame_stats.used_encoder.has_value()) { + if (stats->encoders.empty() || stats->encoders.back().codec_name != + frame_stats.used_encoder->codec_name) { + stats->encoders.push_back(*frame_stats.used_encoder); + } + stats->encoders.back().last_frame_id = + frame_stats.used_encoder->last_frame_id; + stats->encoders.back().switched_from_at = + frame_stats.used_encoder->switched_from_at; + } + + if (frame_stats.used_decoder.has_value()) { + if (stats->decoders.empty() || stats->decoders.back().codec_name != + frame_stats.used_decoder->codec_name) { + stats->decoders.push_back(*frame_stats.used_decoder); + } + stats->decoders.back().last_frame_id = + frame_stats.used_decoder->last_frame_id; + stats->decoders.back().switched_from_at = + frame_stats.used_decoder->switched_from_at; + } +} + +Timestamp DefaultVideoQualityAnalyzerFramesComparator::Now() { + return clock_->CurrentTime(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h new file mode 100644 index 0000000000..006c3eb9bf --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_ + +#include <deque> +#include <map> +#include <utility> +#include <vector> + +#include "api/array_view.h" +#include "rtc_base/event.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/synchronization/mutex.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +namespace webrtc { + +struct FramesComparatorStats { + // Size of analyzer internal comparisons queue, measured when new element + // id added to the queue. + SamplesStatsCounter comparisons_queue_size; + // Number of performed comparisons of 2 video frames from captured and + // rendered streams. + int64_t comparisons_done = 0; + // Number of cpu overloaded comparisons. Comparison is cpu overloaded if it is + // queued when there are too many not processed comparisons in the queue. + // Overloaded comparison doesn't include metrics like SSIM and PSNR that + // require heavy computations. + int64_t cpu_overloaded_comparisons_done = 0; + // Number of memory overloaded comparisons. Comparison is memory overloaded if + // it is queued when its captured frame was already removed due to high memory + // usage for that video stream. + int64_t memory_overloaded_comparisons_done = 0; +}; + +// Performs comparisons of added frames and tracks frames related statistics. +// This class is thread safe. +class DefaultVideoQualityAnalyzerFramesComparator { + public: + // Creates frames comparator. + // Frames comparator doesn't use `options.enable_receive_own_stream` for any + // purposes, because it's unrelated to its functionality. + DefaultVideoQualityAnalyzerFramesComparator( + webrtc::Clock* clock, + DefaultVideoQualityAnalyzerCpuMeasurer& cpu_measurer, + DefaultVideoQualityAnalyzerOptions options = {}) + : options_(options), clock_(clock), cpu_measurer_(cpu_measurer) {} + ~DefaultVideoQualityAnalyzerFramesComparator() { Stop({}); } + + // Starts frames comparator. This method must be invoked before calling + // any other method on this object. + void Start(int max_threads_count); + // Stops frames comparator. This method will block until all added frame + // comparisons will be processed. After `Stop()` is invoked no more new + // comparisons can be added to this frames comparator. + // + // `last_rendered_frame_time` contains timestamps of last rendered frame for + // each (stream, sender, receiver) tuple to properly update time between + // freezes: it has include time from the last freeze until and of call. + void Stop( + const std::map<InternalStatsKey, Timestamp>& last_rendered_frame_times); + + // Ensures that stream `stream_index` has stats objects created for all + // potential receivers. This method must be called before adding any + // frames comparison for that stream. + void EnsureStatsForStream(size_t stream_index, + size_t sender_peer_index, + size_t peers_count, + Timestamp captured_time, + Timestamp start_time); + // Ensures that newly added participant will have stream stats objects created + // for all streams which they can receive. This method must be called before + // any frames comparison will be added for the newly added participant. + // + // `stream_started_time` - start time of each stream for which stats object + // has to be created. + // `start_time` - call start time. + void RegisterParticipantInCall( + rtc::ArrayView<std::pair<InternalStatsKey, Timestamp>> + stream_started_time, + Timestamp start_time); + + // `captured` - video frame captured by sender to use for PSNR/SSIM + // computation. If `type` is `FrameComparisonType::kRegular` and + // `captured` is `absl::nullopt` comparison is assumed to be overloaded + // due to memory constraints. + // `rendered` - video frame rendered by receiver to use for PSNR/SSIM + // computation. Required only if `type` is + // `FrameComparisonType::kRegular`, but can still be omitted if + // `captured` is `absl::nullopt`. + void AddComparison(InternalStatsKey stats_key, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats); + // `skipped_between_rendered` - amount of frames dropped on this stream before + // last received frame and current frame. + void AddComparison(InternalStatsKey stats_key, + int skipped_between_rendered, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats); + + std::map<InternalStatsKey, StreamStats> stream_stats() const { + MutexLock lock(&mutex_); + return stream_stats_; + } + FramesComparatorStats frames_comparator_stats() const { + MutexLock lock(&mutex_); + return frames_comparator_stats_; + } + + private: + enum State { kNew, kActive, kStopped }; + + void AddComparisonInternal(InternalStatsKey stats_key, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + void ProcessComparisons(); + void ProcessComparison(const FrameComparison& comparison); + Timestamp Now(); + + const DefaultVideoQualityAnalyzerOptions options_; + webrtc::Clock* const clock_; + DefaultVideoQualityAnalyzerCpuMeasurer& cpu_measurer_; + + mutable Mutex mutex_; + State state_ RTC_GUARDED_BY(mutex_) = State::kNew; + std::map<InternalStatsKey, StreamStats> stream_stats_ RTC_GUARDED_BY(mutex_); + std::map<InternalStatsKey, Timestamp> stream_last_freeze_end_time_ + RTC_GUARDED_BY(mutex_); + std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(mutex_); + FramesComparatorStats frames_comparator_stats_ RTC_GUARDED_BY(mutex_); + + std::vector<rtc::PlatformThread> thread_pool_; + rtc::Event comparison_available_event_; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc new file mode 100644 index 0000000000..b9b822072f --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h" + +#include <map> +#include <vector> + +#include "api/units/timestamp.h" +#include "rtc_base/strings/string_builder.h" +#include "system_wrappers/include/clock.h" +#include "test/gtest.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +namespace webrtc { +namespace { + +using StatsSample = ::webrtc::SamplesStatsCounter::StatsSample; + +constexpr int kMaxFramesInFlightPerStream = 10; + +DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() { + DefaultVideoQualityAnalyzerOptions options; + options.compute_psnr = false; + options.compute_ssim = false; + options.adjust_cropping_before_comparing_frames = false; + options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream; + return options; +} + +StreamCodecInfo Vp8CodecForOneFrame(uint16_t frame_id, Timestamp time) { + StreamCodecInfo info; + info.codec_name = "VP8"; + info.first_frame_id = frame_id; + info.last_frame_id = frame_id; + info.switched_on_at = time; + info.switched_from_at = time; + return info; +} + +FrameStats FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + Timestamp captured_time) { + FrameStats frame_stats(captured_time); + frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10); + frame_stats.encoded_time = captured_time + TimeDelta::Millis(20); + frame_stats.received_time = captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40); + // Decode time is in microseconds. + frame_stats.decode_end_time = captured_time + TimeDelta::Micros(40010); + frame_stats.rendered_time = captured_time + TimeDelta::Millis(60); + frame_stats.used_encoder = Vp8CodecForOneFrame(1, frame_stats.encoded_time); + frame_stats.used_decoder = + Vp8CodecForOneFrame(1, frame_stats.decode_end_time); + frame_stats.rendered_frame_width = 10; + frame_stats.rendered_frame_height = 10; + return frame_stats; +} + +FrameStats ShiftStatsOn(const FrameStats& stats, TimeDelta delta) { + FrameStats frame_stats(stats.captured_time + delta); + frame_stats.pre_encode_time = stats.pre_encode_time + delta; + frame_stats.encoded_time = stats.encoded_time + delta; + frame_stats.received_time = stats.received_time + delta; + frame_stats.decode_start_time = stats.decode_start_time + delta; + frame_stats.decode_end_time = stats.decode_end_time + delta; + frame_stats.rendered_time = stats.rendered_time + delta; + + frame_stats.used_encoder = stats.used_encoder; + frame_stats.used_decoder = stats.used_decoder; + frame_stats.rendered_frame_width = stats.rendered_frame_width; + frame_stats.rendered_frame_height = stats.rendered_frame_height; + + return frame_stats; +} + +double GetFirstOrDie(const SamplesStatsCounter& counter) { + EXPECT_TRUE(!counter.IsEmpty()) << "Counter has to be not empty"; + return counter.GetSamples()[0]; +} + +std::string ToString(const SamplesStatsCounter& counter) { + rtc::StringBuilder out; + for (const StatsSample& s : counter.GetTimedSamples()) { + out << "{ time_ms=" << s.time.ms() << "; value=" << s.value << "}, "; + } + return out.str(); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + StatsPresentedAfterAddingOneComparison) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest()); + + Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + size_t peers_count = 2; + InternalStatsKey stats_key(stream, sender, receiver); + + FrameStats frame_stats = + FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(stream_start_time); + + comparator.Start(1); + comparator.EnsureStatsForStream(stream, sender, peers_count, + stream_start_time, stream_start_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kRegular, frame_stats); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + std::map<InternalStatsKey, StreamStats> stats = comparator.stream_stats(); + EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).transport_time_ms), 20.0); + EXPECT_DOUBLE_EQ( + GetFirstOrDie(stats.at(stats_key).total_delay_incl_transport_ms), 60.0); + EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).encode_time_ms), 10.0); + EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).decode_time_ms), 0.01); + EXPECT_DOUBLE_EQ(GetFirstOrDie(stats.at(stats_key).receive_to_render_time_ms), + 30.0); + EXPECT_DOUBLE_EQ( + GetFirstOrDie(stats.at(stats_key).resolution_of_rendered_frame), 100.0); +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + MultiFrameStatsPresentedAfterAddingTwoComparisonWith10msDelay) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest()); + + Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + size_t peers_count = 2; + InternalStatsKey stats_key(stream, sender, receiver); + + FrameStats frame_stats1 = + FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(stream_start_time); + FrameStats frame_stats2 = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame( + stream_start_time + TimeDelta::Millis(15)); + frame_stats2.prev_frame_rendered_time = frame_stats1.rendered_time; + + comparator.Start(1); + comparator.EnsureStatsForStream(stream, sender, peers_count, + stream_start_time, stream_start_time); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kRegular, frame_stats1); + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kRegular, frame_stats2); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + std::map<InternalStatsKey, StreamStats> stats = comparator.stream_stats(); + EXPECT_DOUBLE_EQ( + GetFirstOrDie(stats.at(stats_key).time_between_rendered_frames_ms), 15.0); + EXPECT_DOUBLE_EQ(stats.at(stats_key).encode_frame_rate.GetEventsPerSecond(), + 2.0 / 15 * 1000) + << "There should be 2 events with interval of 15 ms"; + ; +} + +TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, + FrameInFlightStatsAreHandledCorrectly) { + DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer; + DefaultVideoQualityAnalyzerFramesComparator comparator( + Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest()); + + Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime(); + size_t stream = 0; + size_t sender = 0; + size_t receiver = 1; + size_t peers_count = 2; + InternalStatsKey stats_key(stream, sender, receiver); + + // There are 7 different timings inside frame stats: captured, pre_encode, + // encoded, received, decode_start, decode_end, rendered. captured is always + // set and received is set together with decode_start. So we create 6 + // different frame stats with interval of 15 ms, where for each stat next + // timings will be set + // * 1st - captured + // * 2nd - captured, pre_encode + // * 3rd - captured, pre_encode, encoded + // * 4th - captured, pre_encode, encoded, received, decode_start + // * 5th - captured, pre_encode, encoded, received, decode_start, decode_end + // * 6th - all of them set + std::vector<FrameStats> stats; + // 1st stat + FrameStats frame_stats(stream_start_time); + stats.push_back(frame_stats); + // 2nd stat + frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.pre_encode_time = + frame_stats.captured_time + TimeDelta::Millis(10); + stats.push_back(frame_stats); + // 3rd stat + frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.encoded_time = frame_stats.captured_time + TimeDelta::Millis(20); + frame_stats.used_encoder = Vp8CodecForOneFrame(1, frame_stats.encoded_time); + stats.push_back(frame_stats); + // 4th stat + frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.received_time = frame_stats.captured_time + TimeDelta::Millis(30); + frame_stats.decode_start_time = + frame_stats.captured_time + TimeDelta::Millis(40); + stats.push_back(frame_stats); + // 5th stat + frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.decode_end_time = + frame_stats.captured_time + TimeDelta::Millis(50); + frame_stats.used_decoder = + Vp8CodecForOneFrame(1, frame_stats.decode_end_time); + stats.push_back(frame_stats); + // 6th stat + frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15)); + frame_stats.rendered_time = frame_stats.captured_time + TimeDelta::Millis(60); + frame_stats.rendered_frame_width = 10; + frame_stats.rendered_frame_height = 10; + stats.push_back(frame_stats); + + comparator.Start(1); + comparator.EnsureStatsForStream(stream, sender, peers_count, + stream_start_time, stream_start_time); + for (size_t i = 0; i < stats.size() - 1; ++i) { + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kFrameInFlight, stats[i]); + } + comparator.AddComparison(stats_key, + /*captured=*/absl::nullopt, + /*rendered=*/absl::nullopt, + FrameComparisonType::kRegular, + stats[stats.size() - 1]); + comparator.Stop(/*last_rendered_frame_times=*/{}); + + EXPECT_EQ(comparator.stream_stats().size(), 1lu); + StreamStats result_stats = comparator.stream_stats().at(stats_key); + + EXPECT_DOUBLE_EQ(result_stats.transport_time_ms.GetAverage(), 20.0) + << ToString(result_stats.transport_time_ms); + EXPECT_EQ(result_stats.transport_time_ms.NumSamples(), 3); + + EXPECT_DOUBLE_EQ(result_stats.total_delay_incl_transport_ms.GetAverage(), + 60.0) + << ToString(result_stats.total_delay_incl_transport_ms); + EXPECT_EQ(result_stats.total_delay_incl_transport_ms.NumSamples(), 1); + + EXPECT_DOUBLE_EQ(result_stats.encode_time_ms.GetAverage(), 10) + << ToString(result_stats.encode_time_ms); + EXPECT_EQ(result_stats.encode_time_ms.NumSamples(), 4); + + EXPECT_DOUBLE_EQ(result_stats.decode_time_ms.GetAverage(), 10) + << ToString(result_stats.decode_time_ms); + EXPECT_EQ(result_stats.decode_time_ms.NumSamples(), 2); + + EXPECT_DOUBLE_EQ(result_stats.receive_to_render_time_ms.GetAverage(), 30) + << ToString(result_stats.receive_to_render_time_ms); + EXPECT_EQ(result_stats.receive_to_render_time_ms.NumSamples(), 1); + + EXPECT_DOUBLE_EQ(result_stats.resolution_of_rendered_frame.GetAverage(), 100) + << ToString(result_stats.resolution_of_rendered_frame); + EXPECT_EQ(result_stats.resolution_of_rendered_frame.NumSamples(), 1); + + EXPECT_DOUBLE_EQ(result_stats.encode_frame_rate.GetEventsPerSecond(), + 4.0 / 45 * 1000) + << "There should be 4 events with interval of 15 ms"; +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.cc new file mode 100644 index 0000000000..16f49ef154 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.cc @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" + +#include "api/video/video_frame.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { + +std::string InternalStatsKey::ToString() const { + rtc::StringBuilder out; + out << "stream=" << stream << "_sender=" << sender + << "_receiver=" << receiver; + return out.str(); +} + +bool operator<(const InternalStatsKey& a, const InternalStatsKey& b) { + if (a.stream != b.stream) { + return a.stream < b.stream; + } + if (a.sender != b.sender) { + return a.sender < b.sender; + } + return a.receiver < b.receiver; +} + +bool operator==(const InternalStatsKey& a, const InternalStatsKey& b) { + return a.stream == b.stream && a.sender == b.sender && + a.receiver == b.receiver; +} + +FrameComparison::FrameComparison(InternalStatsKey stats_key, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats, + OverloadReason overload_reason) + : stats_key(std::move(stats_key)), + captured(std::move(captured)), + rendered(std::move(rendered)), + type(type), + frame_stats(std::move(frame_stats)), + overload_reason(overload_reason) {} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h new file mode 100644 index 0000000000..04f653c02b --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_INTERNAL_SHARED_OBJECTS_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_INTERNAL_SHARED_OBJECTS_H_ + +#include <memory> +#include <string> +#include <utility> + +#include "absl/types/optional.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_type.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +namespace webrtc { + +struct InternalStatsKey { + InternalStatsKey(size_t stream, size_t sender, size_t receiver) + : stream(stream), sender(sender), receiver(receiver) {} + + std::string ToString() const; + + size_t stream; + size_t sender; + size_t receiver; +}; + +// Required to use InternalStatsKey as std::map key. +bool operator<(const InternalStatsKey& a, const InternalStatsKey& b); +bool operator==(const InternalStatsKey& a, const InternalStatsKey& b); + +// Final stats computed for frame after it went through the whole video +// pipeline from capturing to rendering or dropping. +struct FrameStats { + explicit FrameStats(Timestamp captured_time) : captured_time(captured_time) {} + + // Frame events timestamp. + Timestamp captured_time; + Timestamp pre_encode_time = Timestamp::MinusInfinity(); + Timestamp encoded_time = Timestamp::MinusInfinity(); + // Time when last packet of a frame was received. + Timestamp received_time = Timestamp::MinusInfinity(); + Timestamp decode_start_time = Timestamp::MinusInfinity(); + Timestamp decode_end_time = Timestamp::MinusInfinity(); + Timestamp rendered_time = Timestamp::MinusInfinity(); + Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity(); + + VideoFrameType encoded_frame_type = VideoFrameType::kEmptyFrame; + DataSize encoded_image_size = DataSize::Bytes(0); + VideoFrameType pre_decoded_frame_type = VideoFrameType::kEmptyFrame; + DataSize pre_decoded_image_size = DataSize::Bytes(0); + uint32_t target_encode_bitrate = 0; + + absl::optional<int> rendered_frame_width = absl::nullopt; + absl::optional<int> rendered_frame_height = absl::nullopt; + + // Can be not set if frame was dropped by encoder. + absl::optional<StreamCodecInfo> used_encoder = absl::nullopt; + // Can be not set if frame was dropped in the network. + absl::optional<StreamCodecInfo> used_decoder = absl::nullopt; +}; + +// Describes why comparison was done in overloaded mode (without calculating +// PSNR and SSIM). +enum class OverloadReason { + kNone, + // Not enough CPU to process all incoming comparisons. + kCpu, + // Not enough memory to store captured frames for all comparisons. + kMemory +}; + +enum class FrameComparisonType { + // Comparison for captured and rendered frame. + kRegular, + // Comparison for captured frame that is known to be dropped somewhere in + // video pipeline. + kDroppedFrame, + // Comparison for captured frame that was still in the video pipeline when + // test was stopped. It's unknown is this frame dropped or would it be + // delivered if test continue. + kFrameInFlight +}; + +// Represents comparison between two VideoFrames. Contains video frames itself +// and stats. Can be one of two types: +// 1. Normal - in this case `captured` is presented and either `rendered` is +// presented and `dropped` is false, either `rendered` is omitted and +// `dropped` is true. +// 2. Overloaded - in this case both `captured` and `rendered` are omitted +// because there were too many comparisons in the queue. `dropped` can be +// true or false showing was frame dropped or not. +struct FrameComparison { + FrameComparison(InternalStatsKey stats_key, + absl::optional<VideoFrame> captured, + absl::optional<VideoFrame> rendered, + FrameComparisonType type, + FrameStats frame_stats, + OverloadReason overload_reason); + + InternalStatsKey stats_key; + // Frames can be omitted if there too many computations waiting in the + // queue. + absl::optional<VideoFrame> captured; + absl::optional<VideoFrame> rendered; + FrameComparisonType type; + FrameStats frame_stats; + OverloadReason overload_reason; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_INTERNAL_SHARED_OBJECTS_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc new file mode 100644 index 0000000000..cc17429072 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +#include <algorithm> +#include <iterator> + +#include "api/units/timestamp.h" +#include "rtc_base/checks.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { +namespace { + +constexpr int kMicrosPerSecond = 1000000; + +} // namespace + +void SamplesRateCounter::AddEvent(Timestamp event_time) { + if (event_first_time_.IsMinusInfinity()) { + event_first_time_ = event_time; + } + event_last_time_ = event_time; + events_count_++; +} + +double SamplesRateCounter::GetEventsPerSecond() const { + RTC_DCHECK(!IsEmpty()); + // Divide on us and multiply on kMicrosPerSecond to correctly process cases + // where there were too small amount of events, so difference is less then 1 + // sec. We can use us here, because Timestamp has us resolution. + return static_cast<double>(events_count_) / + (event_last_time_ - event_first_time_).us() * kMicrosPerSecond; +} + +StreamStats::StreamStats(Timestamp stream_started_time) + : stream_started_time(stream_started_time) { + for (int i = static_cast<int>(FrameDropPhase::kBeforeEncoder); + i < static_cast<int>(FrameDropPhase::kLastValue); ++i) { + dropped_by_phase.emplace(static_cast<FrameDropPhase>(i), 0); + } +} + +std::string StatsKey::ToString() const { + rtc::StringBuilder out; + out << stream_label << "_" << receiver; + return out.str(); +} + +bool operator<(const StatsKey& a, const StatsKey& b) { + if (a.stream_label != b.stream_label) { + return a.stream_label < b.stream_label; + } + return a.receiver < b.receiver; +} + +bool operator==(const StatsKey& a, const StatsKey& b) { + return a.stream_label == b.stream_label && a.receiver == b.receiver; +} + +VideoStreamsInfo::VideoStreamsInfo( + std::map<std::string, std::string> stream_to_sender, + std::map<std::string, std::set<std::string>> sender_to_streams, + std::map<std::string, std::set<std::string>> stream_to_receivers) + : stream_to_sender_(std::move(stream_to_sender)), + sender_to_streams_(std::move(sender_to_streams)), + stream_to_receivers_(std::move(stream_to_receivers)) {} + +std::set<StatsKey> VideoStreamsInfo::GetStatsKeys() const { + std::set<StatsKey> out; + for (const std::string& stream_label : GetStreams()) { + for (const std::string& receiver : GetReceivers(stream_label)) { + out.insert(StatsKey(stream_label, receiver)); + } + } + return out; +} + +std::set<std::string> VideoStreamsInfo::GetStreams() const { + std::set<std::string> out; + std::transform(stream_to_sender_.begin(), stream_to_sender_.end(), + std::inserter(out, out.end()), + [](auto map_entry) { return map_entry.first; }); + return out; +} + +std::set<std::string> VideoStreamsInfo::GetStreams( + absl::string_view sender_name) const { + auto it = sender_to_streams_.find(std::string(sender_name)); + if (it == sender_to_streams_.end()) { + return {}; + } + return it->second; +} + +absl::optional<std::string> VideoStreamsInfo::GetSender( + absl::string_view stream_label) const { + auto it = stream_to_sender_.find(std::string(stream_label)); + if (it == stream_to_sender_.end()) { + return absl::nullopt; + } + return it->second; +} + +std::set<std::string> VideoStreamsInfo::GetReceivers( + absl::string_view stream_label) const { + auto it = stream_to_receivers_.find(std::string(stream_label)); + if (it == stream_to_receivers_.end()) { + return {}; + } + return it->second; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h new file mode 100644 index 0000000000..8f0afd36f7 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_SHARED_OBJECTS_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_SHARED_OBJECTS_H_ + +#include <map> +#include <memory> +#include <set> +#include <string> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/units/timestamp.h" + +namespace webrtc { + +// WebRTC will request a key frame after 3 seconds if no frames were received. +// We assume max frame rate ~60 fps, so 270 frames will cover max freeze without +// key frame request. +constexpr size_t kDefaultMaxFramesInFlightPerStream = 270; + +class SamplesRateCounter { + public: + void AddEvent(Timestamp event_time); + + bool IsEmpty() const { return event_first_time_ == event_last_time_; } + + double GetEventsPerSecond() const; + + private: + Timestamp event_first_time_ = Timestamp::MinusInfinity(); + Timestamp event_last_time_ = Timestamp::MinusInfinity(); + int64_t events_count_ = 0; +}; + +struct FrameCounters { + // Count of frames, that were passed into WebRTC pipeline by video stream + // source. + int64_t captured = 0; + // Count of frames that reached video encoder. + int64_t pre_encoded = 0; + // Count of encoded images that were produced by encoder for all requested + // spatial layers and simulcast streams. + int64_t encoded = 0; + // Count of encoded images received in decoder for all requested spatial + // layers and simulcast streams. + int64_t received = 0; + // Count of frames that were produced by decoder. + int64_t decoded = 0; + // Count of frames that went out from WebRTC pipeline to video sink. + int64_t rendered = 0; + // Count of frames that were dropped in any point between capturing and + // rendering. + int64_t dropped = 0; +}; + +// Contains information about the codec that was used for encoding or decoding +// the stream. +struct StreamCodecInfo { + // Codec implementation name. + std::string codec_name; + // Id of the first frame for which this codec was used. + uint16_t first_frame_id; + // Id of the last frame for which this codec was used. + uint16_t last_frame_id; + // Timestamp when the first frame was handled by the encode/decoder. + Timestamp switched_on_at = Timestamp::PlusInfinity(); + // Timestamp when this codec was used last time. + Timestamp switched_from_at = Timestamp::PlusInfinity(); +}; + +// Represents phases where video frame can be dropped and such drop will be +// detected by analyzer. +enum class FrameDropPhase : int { + kBeforeEncoder, + kByEncoder, + kTransport, + kAfterDecoder, + // kLastValue must be the last value in this enumeration. + kLastValue +}; + +struct StreamStats { + explicit StreamStats(Timestamp stream_started_time); + + // The time when the first frame of this stream was captured. + Timestamp stream_started_time; + + // Spatial quality metrics. + SamplesStatsCounter psnr; + SamplesStatsCounter ssim; + + // Time from frame encoded (time point on exit from encoder) to the + // encoded image received in decoder (time point on entrance to decoder). + SamplesStatsCounter transport_time_ms; + // Time from frame was captured on device to time frame was displayed on + // device. + SamplesStatsCounter total_delay_incl_transport_ms; + // Time between frames out from renderer. + SamplesStatsCounter time_between_rendered_frames_ms; + SamplesRateCounter encode_frame_rate; + SamplesStatsCounter encode_time_ms; + SamplesStatsCounter decode_time_ms; + // Time from last packet of frame is received until it's sent to the renderer. + SamplesStatsCounter receive_to_render_time_ms; + // Max frames skipped between two nearest. + SamplesStatsCounter skipped_between_rendered; + // In the next 2 metrics freeze is a pause that is longer, than maximum: + // 1. 150ms + // 2. 3 * average time between two sequential frames. + // Item 1 will cover high fps video and is a duration, that is noticeable by + // human eye. Item 2 will cover low fps video like screen sharing. + // Freeze duration. + SamplesStatsCounter freeze_time_ms; + // Mean time between one freeze end and next freeze start. + SamplesStatsCounter time_between_freezes_ms; + SamplesStatsCounter resolution_of_rendered_frame; + SamplesStatsCounter target_encode_bitrate; + + int64_t total_encoded_images_payload = 0; + // Counters on which phase how many frames were dropped. + std::map<FrameDropPhase, int64_t> dropped_by_phase; + + // Frame count metrics. + int64_t num_send_key_frames = 0; + int64_t num_recv_key_frames = 0; + + // Encoded frame size (in bytes) metrics. + SamplesStatsCounter recv_key_frame_size_bytes; + SamplesStatsCounter recv_delta_frame_size_bytes; + + // Vector of encoders used for this stream by sending client. + std::vector<StreamCodecInfo> encoders; + // Vectors of decoders used for this stream by receiving client. + std::vector<StreamCodecInfo> decoders; +}; + +struct AnalyzerStats { + // Size of analyzer internal comparisons queue, measured when new element + // id added to the queue. + SamplesStatsCounter comparisons_queue_size; + // Number of performed comparisons of 2 video frames from captured and + // rendered streams. + int64_t comparisons_done = 0; + // Number of cpu overloaded comparisons. Comparison is cpu overloaded if it is + // queued when there are too many not processed comparisons in the queue. + // Overloaded comparison doesn't include metrics like SSIM and PSNR that + // require heavy computations. + int64_t cpu_overloaded_comparisons_done = 0; + // Number of memory overloaded comparisons. Comparison is memory overloaded if + // it is queued when its captured frame was already removed due to high memory + // usage for that video stream. + int64_t memory_overloaded_comparisons_done = 0; + // Count of frames in flight in analyzer measured when new comparison is added + // and after analyzer was stopped. + SamplesStatsCounter frames_in_flight_left_count; +}; + +struct StatsKey { + StatsKey(std::string stream_label, std::string receiver) + : stream_label(std::move(stream_label)), receiver(std::move(receiver)) {} + + std::string ToString() const; + + // Label of video stream to which stats belongs to. + std::string stream_label; + // Name of the peer on which stream was received. + std::string receiver; +}; + +// Required to use StatsKey as std::map key. +bool operator<(const StatsKey& a, const StatsKey& b); +bool operator==(const StatsKey& a, const StatsKey& b); + +// Contains all metadata related to the video streams that were seen by the +// video analyzer. +class VideoStreamsInfo { + public: + std::set<StatsKey> GetStatsKeys() const; + + // Returns all stream labels that are known to the video analyzer. + std::set<std::string> GetStreams() const; + + // Returns set of the stream for specified `sender_name`. If sender didn't + // send any streams or `sender_name` isn't known to the video analyzer + // empty set will be returned. + std::set<std::string> GetStreams(absl::string_view sender_name) const; + + // Returns sender name for specified `stream_label`. Returns `absl::nullopt` + // if provided `stream_label` isn't known to the video analyzer. + absl::optional<std::string> GetSender(absl::string_view stream_label) const; + + // Returns set of the receivers for specified `stream_label`. If stream wasn't + // received by any peer or `stream_label` isn't known to the video analyzer + // empty set will be returned. + std::set<std::string> GetReceivers(absl::string_view stream_label) const; + + protected: + friend class DefaultVideoQualityAnalyzer; + VideoStreamsInfo( + std::map<std::string, std::string> stream_to_sender, + std::map<std::string, std::set<std::string>> sender_to_streams, + std::map<std::string, std::set<std::string>> stream_to_receivers); + + private: + std::map<std::string, std::string> stream_to_sender_; + std::map<std::string, std::set<std::string>> sender_to_streams_; + std::map<std::string, std::set<std::string>> stream_to_receivers_; +}; + +struct DefaultVideoQualityAnalyzerOptions { + // Tells DefaultVideoQualityAnalyzer if heavy metrics have to be computed. + bool compute_psnr = true; + bool compute_ssim = true; + // If true, weights the luma plane more than the chroma planes in the PSNR. + bool use_weighted_psnr = false; + // Tells DefaultVideoQualityAnalyzer if detailed frame stats should be + // reported. + bool report_detailed_frame_stats = false; + // If true DefaultVideoQualityAnalyzer will try to adjust frames before + // computing PSNR and SSIM for them. In some cases picture may be shifted by + // a few pixels after the encode/decode step. Those difference is invisible + // for a human eye, but it affects the metrics. So the adjustment is used to + // get metrics that are closer to how human perceive the video. This feature + // significantly slows down the comparison, so turn it on only when it is + // needed. + bool adjust_cropping_before_comparing_frames = false; + // Amount of frames that are queued in the DefaultVideoQualityAnalyzer from + // the point they were captured to the point they were rendered on all + // receivers per stream. + size_t max_frames_in_flight_per_stream_count = + kDefaultMaxFramesInFlightPerStream; + // If true, the analyzer will expect peers to receive their own video streams. + bool enable_receive_own_stream = false; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_SHARED_OBJECTS_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.cc new file mode 100644 index 0000000000..d59ef12c63 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.cc @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h" + +#include <map> +#include <set> + +#include "absl/types/optional.h" +#include "api/units/timestamp.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +template <typename T> +absl::optional<T> MaybeGetValue(const std::map<size_t, T>& map, size_t key) { + auto it = map.find(key); + if (it == map.end()) { + return absl::nullopt; + } + return it->second; +} + +} // namespace + +StreamState::StreamState(size_t sender, + std::set<size_t> receivers, + Timestamp stream_started_time) + : sender_(sender), + stream_started_time_(stream_started_time), + receivers_(receivers), + frame_ids_(std::move(receivers)) { + frame_ids_.AddReader(kAliveFramesQueueIndex); + RTC_CHECK_NE(sender_, kAliveFramesQueueIndex); + for (size_t receiver : receivers_) { + RTC_CHECK_NE(receiver, kAliveFramesQueueIndex); + } +} + +uint16_t StreamState::PopFront(size_t peer) { + RTC_CHECK_NE(peer, kAliveFramesQueueIndex); + absl::optional<uint16_t> frame_id = frame_ids_.PopFront(peer); + RTC_DCHECK(frame_id.has_value()); + + // If alive's frame queue is longer than all others, than also pop frame from + // it, because that frame is received by all receivers. + size_t alive_size = frame_ids_.size(kAliveFramesQueueIndex); + size_t other_size = GetLongestReceiverQueue(); + // Pops frame from alive queue if alive's queue is the longest one. + if (alive_size > other_size) { + absl::optional<uint16_t> alive_frame_id = + frame_ids_.PopFront(kAliveFramesQueueIndex); + RTC_DCHECK(alive_frame_id.has_value()); + RTC_DCHECK_EQ(frame_id.value(), alive_frame_id.value()); + } + + return frame_id.value(); +} + +void StreamState::AddPeer(size_t peer) { + RTC_CHECK_NE(peer, kAliveFramesQueueIndex); + frame_ids_.AddReader(peer, kAliveFramesQueueIndex); + receivers_.insert(peer); +} + +void StreamState::RemovePeer(size_t peer) { + RTC_CHECK_NE(peer, kAliveFramesQueueIndex); + frame_ids_.RemoveReader(peer); + receivers_.erase(peer); + + // If we removed the last receiver for the alive frames, we need to pop them + // from the queue, because now they received by all receivers. + size_t alive_size = frame_ids_.size(kAliveFramesQueueIndex); + size_t other_size = GetLongestReceiverQueue(); + while (alive_size > other_size) { + frame_ids_.PopFront(kAliveFramesQueueIndex); + alive_size--; + } +} + +uint16_t StreamState::MarkNextAliveFrameAsDead() { + absl::optional<uint16_t> frame_id = + frame_ids_.PopFront(kAliveFramesQueueIndex); + RTC_DCHECK(frame_id.has_value()); + return frame_id.value(); +} + +void StreamState::SetLastRenderedFrameTime(size_t peer, Timestamp time) { + auto it = last_rendered_frame_time_.find(peer); + if (it == last_rendered_frame_time_.end()) { + last_rendered_frame_time_.insert({peer, time}); + } else { + it->second = time; + } +} + +absl::optional<Timestamp> StreamState::last_rendered_frame_time( + size_t peer) const { + return MaybeGetValue(last_rendered_frame_time_, peer); +} + +size_t StreamState::GetLongestReceiverQueue() const { + size_t max = 0; + for (size_t receiver : receivers_) { + size_t cur_size = frame_ids_.size(receiver); + if (cur_size > max) { + max = cur_size; + } + } + return max; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h new file mode 100644 index 0000000000..829a79c7bf --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_STREAM_STATE_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_STREAM_STATE_H_ + +#include <limits> +#include <map> +#include <set> + +#include "absl/types/optional.h" +#include "api/units/timestamp.h" +#include "test/pc/e2e/analyzer/video/multi_reader_queue.h" + +namespace webrtc { + +// Represents a current state of video stream inside +// DefaultVideoQualityAnalyzer. +// +// Maintains the sequence of frames for each video stream and keeps track about +// which frames were seen by each of the possible stream receiver. +// +// Keeps information about which frames are alive and which are dead. Frame is +// alive if it contains VideoFrame payload for corresponding FrameInFlight +// object inside DefaultVideoQualityAnalyzer, otherwise frame is considered +// dead. +// +// Supports peer indexes from 0 to max(size_t) - 1. +class StreamState { + public: + StreamState(size_t sender, + std::set<size_t> receivers, + Timestamp stream_started_time); + + size_t sender() const { return sender_; } + Timestamp stream_started_time() const { return stream_started_time_; } + + void PushBack(uint16_t frame_id) { frame_ids_.PushBack(frame_id); } + // Crash if state is empty. + uint16_t PopFront(size_t peer); + bool IsEmpty(size_t peer) const { return frame_ids_.IsEmpty(peer); } + // Crash if state is empty. + uint16_t Front(size_t peer) const { return frame_ids_.Front(peer).value(); } + + // Adds a new peer to the state. All currently alive frames will be expected + // to be received by the newly added peer. + void AddPeer(size_t peer); + + // Removes peer from the state. Frames that were expected to be received by + // this peer will be removed from it. On the other hand last rendered frame + // time for the removed peer will be preserved, because + // DefaultVideoQualityAnalyzer still may request it for stats processing. + void RemovePeer(size_t peer); + + size_t GetAliveFramesCount() const { + return frame_ids_.size(kAliveFramesQueueIndex); + } + uint16_t MarkNextAliveFrameAsDead(); + + void SetLastRenderedFrameTime(size_t peer, Timestamp time); + absl::optional<Timestamp> last_rendered_frame_time(size_t peer) const; + + private: + // Index of the `frame_ids_` queue which is used to track alive frames for + // this stream. + static constexpr size_t kAliveFramesQueueIndex = + std::numeric_limits<size_t>::max(); + + size_t GetLongestReceiverQueue() const; + + // Index of the owner. Owner's queue in `frame_ids_` will keep alive frames. + const size_t sender_; + const Timestamp stream_started_time_; + std::set<size_t> receivers_; + // To correctly determine dropped frames we have to know sequence of frames + // in each stream so we will keep a list of frame ids inside the stream. + // This list is represented by multi head queue of frame ids with separate + // head for each receiver. When the frame is rendered, we will pop ids from + // the corresponding head until id will match with rendered one. All ids + // before matched one can be considered as dropped: + // + // | frame_id1 |->| frame_id2 |->| frame_id3 |->| frame_id4 | + // + // If we received frame with id frame_id3, then we will pop frame_id1 and + // frame_id2 and consider those frames as dropped and then compare received + // frame with the one from `FrameInFlight` with id frame_id3. + MultiReaderQueue<uint16_t> frame_ids_; + std::map<size_t, Timestamp> last_rendered_frame_time_; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_STREAM_STATE_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state_test.cc new file mode 100644 index 0000000000..01a6aab28a --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state_test.cc @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h" + +#include <set> + +#include "api/units/timestamp.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +TEST(StreamStateTest, PopFrontAndFrontIndependentForEachPeer) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + state.PushBack(/*frame_id=*/2); + + EXPECT_EQ(state.Front(/*peer=*/1), 1); + EXPECT_EQ(state.PopFront(/*peer=*/1), 1); + EXPECT_EQ(state.Front(/*peer=*/1), 2); + EXPECT_EQ(state.PopFront(/*peer=*/1), 2); + EXPECT_EQ(state.Front(/*peer=*/2), 1); + EXPECT_EQ(state.PopFront(/*peer=*/2), 1); + EXPECT_EQ(state.Front(/*peer=*/2), 2); + EXPECT_EQ(state.PopFront(/*peer=*/2), 2); +} + +TEST(StreamStateTest, IsEmpty) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + + EXPECT_FALSE(state.IsEmpty(/*peer=*/1)); + + state.PopFront(/*peer=*/1); + + EXPECT_TRUE(state.IsEmpty(/*peer=*/1)); +} + +TEST(StreamStateTest, PopFrontForOnlyOnePeerDontChangeAliveFramesCount) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + state.PushBack(/*frame_id=*/2); + + EXPECT_EQ(state.GetAliveFramesCount(), 2lu); + + state.PopFront(/*peer=*/1); + state.PopFront(/*peer=*/1); + + EXPECT_EQ(state.GetAliveFramesCount(), 2lu); +} + +TEST(StreamStateTest, PopFrontForAllPeersReducesAliveFramesCount) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + state.PushBack(/*frame_id=*/2); + + EXPECT_EQ(state.GetAliveFramesCount(), 2lu); + + state.PopFront(/*peer=*/1); + state.PopFront(/*peer=*/2); + + EXPECT_EQ(state.GetAliveFramesCount(), 1lu); +} + +TEST(StreamStateTest, RemovePeerForLastExpectedReceiverUpdatesAliveFrames) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + state.PushBack(/*frame_id=*/2); + + state.PopFront(/*peer=*/1); + + EXPECT_EQ(state.GetAliveFramesCount(), 2lu); + + state.RemovePeer(/*peer=*/2); + + EXPECT_EQ(state.GetAliveFramesCount(), 1lu); +} + +TEST(StreamStateTest, MarkNextAliveFrameAsDeadDecreseAliveFramesCount) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + state.PushBack(/*frame_id=*/2); + + EXPECT_EQ(state.GetAliveFramesCount(), 2lu); + + state.MarkNextAliveFrameAsDead(); + + EXPECT_EQ(state.GetAliveFramesCount(), 1lu); +} + +TEST(StreamStateTest, MarkNextAliveFrameAsDeadDoesntAffectFrontFrameForPeer) { + StreamState state(/*sender=*/0, + /*receivers=*/std::set<size_t>{1, 2}, + Timestamp::Seconds(1)); + state.PushBack(/*frame_id=*/1); + state.PushBack(/*frame_id=*/2); + + EXPECT_EQ(state.Front(/*peer=*/1), 1); + + state.MarkNextAliveFrameAsDead(); + + EXPECT_EQ(state.Front(/*peer=*/1), 1); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc new file mode 100644 index 0000000000..40662fd31d --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc @@ -0,0 +1,1980 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" + +#include <algorithm> +#include <map> +#include <memory> +#include <vector> + +#include "api/rtp_packet_info.h" +#include "api/rtp_packet_infos.h" +#include "api/test/create_frame_generator.h" +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_tools/frame_analyzer/video_geometry_aligner.h" +#include "system_wrappers/include/sleep.h" +#include "test/gtest.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" + +namespace webrtc { +namespace { + +using StatsSample = ::webrtc::SamplesStatsCounter::StatsSample; + +constexpr int kAnalyzerMaxThreadsCount = 1; +constexpr int kMaxFramesInFlightPerStream = 10; +constexpr int kFrameWidth = 320; +constexpr int kFrameHeight = 240; +constexpr double kMaxSsim = 1; +constexpr char kStreamLabel[] = "video-stream"; +constexpr char kSenderPeerName[] = "alice"; +constexpr char kReceiverPeerName[] = "bob"; + +DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() { + DefaultVideoQualityAnalyzerOptions options; + options.compute_psnr = false; + options.compute_ssim = false; + options.adjust_cropping_before_comparing_frames = false; + options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream; + return options; +} + +VideoFrame NextFrame(test::FrameGeneratorInterface* frame_generator, + int64_t timestamp_us) { + test::FrameGeneratorInterface::VideoFrameData frame_data = + frame_generator->NextFrame(); + return VideoFrame::Builder() + .set_video_frame_buffer(frame_data.buffer) + .set_update_rect(frame_data.update_rect) + .set_timestamp_us(timestamp_us) + .build(); +} + +EncodedImage FakeEncode(const VideoFrame& frame) { + EncodedImage image; + std::vector<RtpPacketInfo> packet_infos; + packet_infos.push_back(RtpPacketInfo( + /*ssrc=*/1, + /*csrcs=*/{}, + /*rtp_timestamp=*/frame.timestamp(), + /*audio_level=*/absl::nullopt, + /*absolute_capture_time=*/absl::nullopt, + /*receive_time=*/Timestamp::Micros(frame.timestamp_us() + 10000))); + image.SetPacketInfos(RtpPacketInfos(packet_infos)); + return image; +} + +VideoFrame DeepCopy(const VideoFrame& frame) { + VideoFrame copy = frame; + copy.set_video_frame_buffer( + I420Buffer::Copy(*frame.video_frame_buffer()->ToI420())); + return copy; +} + +std::vector<StatsSample> GetSortedSamples(const SamplesStatsCounter& counter) { + rtc::ArrayView<const StatsSample> view = counter.GetTimedSamples(); + std::vector<StatsSample> out(view.begin(), view.end()); + std::sort(out.begin(), out.end(), + [](const StatsSample& a, const StatsSample& b) { + return a.time < b.time; + }); + return out; +} + +std::string ToString(const std::vector<StatsSample>& values) { + rtc::StringBuilder out; + for (const auto& v : values) { + out << "{ time_ms=" << v.time.ms() << "; value=" << v.value << "}, "; + } + return out.str(); +} + +void FakeCPULoad() { + std::vector<int> temp(1000000); + for (size_t i = 0; i < temp.size(); ++i) { + temp[i] = rand(); + } + std::sort(temp.begin(), temp.end()); + ASSERT_TRUE(std::is_sorted(temp.begin(), temp.end())); +} + +void PassFramesThroughAnalyzer(DefaultVideoQualityAnalyzer& analyzer, + absl::string_view sender, + absl::string_view stream_label, + std::vector<absl::string_view> receivers, + int frames_count, + test::FrameGeneratorInterface& frame_generator) { + for (int i = 0; i < frames_count; ++i) { + VideoFrame frame = NextFrame(&frame_generator, /*timestamp_us=*/1); + uint16_t frame_id = + analyzer.OnFrameCaptured(sender, std::string(stream_label), frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode(sender, frame); + analyzer.OnFrameEncoded(sender, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + for (absl::string_view receiver : receivers) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(receiver, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(receiver, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(receiver, received_frame); + } + } +} + +TEST(DefaultVideoQualityAnalyzerTest, + MemoryOverloadedAndThenAllFramesReceived) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + for (int i = 0; i < kMaxFramesInFlightPerStream * 2; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + } + + for (const uint16_t& frame_id : frames_order) { + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, + kMaxFramesInFlightPerStream); + EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream * 2); + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream * 2); + EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream * 2); + EXPECT_EQ(frame_counters.dropped, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, + FillMaxMemoryReceiveAllMemoryOverloadedAndThenAllFramesReceived) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + // Feel analyzer's memory up to limit + for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + } + + // Receive all frames. + for (const uint16_t& frame_id : frames_order) { + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + frames_order.clear(); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + + // Overload analyzer's memory up to limit + for (int i = 0; i < 2 * kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + } + + // Receive all frames. + for (const uint16_t& frame_id : frames_order) { + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, + kMaxFramesInFlightPerStream); + EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream * 3); + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream * 3); + EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream * 3); + EXPECT_EQ(frame_counters.dropped, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, + MemoryOverloadedHalfDroppedAndThenHalfFramesReceived) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + for (int i = 0; i < kMaxFramesInFlightPerStream * 2; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + } + + for (size_t i = kMaxFramesInFlightPerStream; i < frames_order.size(); ++i) { + uint16_t frame_id = frames_order.at(i); + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream * 2); + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream * 2); + EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream); + EXPECT_EQ(frame_counters.dropped, kMaxFramesInFlightPerStream); +} + +TEST(DefaultVideoQualityAnalyzerTest, NormalScenario) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + } + + for (size_t i = 1; i < frames_order.size(); i += 2) { + uint16_t frame_id = frames_order.at(i); + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream); + + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream); + EXPECT_EQ(frame_counters.received, kMaxFramesInFlightPerStream / 2); + EXPECT_EQ(frame_counters.decoded, kMaxFramesInFlightPerStream / 2); + EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream / 2); + EXPECT_EQ(frame_counters.dropped, kMaxFramesInFlightPerStream / 2); +} + +TEST(DefaultVideoQualityAnalyzerTest, OneFrameReceivedTwice) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + VideoFrame captured_frame = NextFrame(frame_generator.get(), 0); + captured_frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, captured_frame)); + analyzer.OnFramePreEncode(kSenderPeerName, captured_frame); + analyzer.OnFrameEncoded(kSenderPeerName, captured_frame.id(), + FakeEncode(captured_frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + VideoFrame received_frame = DeepCopy(captured_frame); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + + received_frame = DeepCopy(captured_frame); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(stats.comparisons_done, 1); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 1); + EXPECT_EQ(frame_counters.received, 1); + EXPECT_EQ(frame_counters.decoded, 1); + EXPECT_EQ(frame_counters.rendered, 1); + EXPECT_EQ(frame_counters.dropped, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, NormalScenario2Receivers) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + constexpr char kAlice[] = "alice"; + constexpr char kBob[] = "bob"; + constexpr char kCharlie[] = "charlie"; + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie}, + kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id(analyzer.OnFrameCaptured(kAlice, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kAlice, frame); + SleepMs(20); + analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + } + + SleepMs(50); + + for (size_t i = 1; i < frames_order.size(); i += 2) { + uint16_t frame_id = frames_order.at(i); + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kBob, received_frame.id(), + FakeEncode(received_frame)); + SleepMs(30); + analyzer.OnFrameDecoded(kBob, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + SleepMs(10); + analyzer.OnFrameRendered(kBob, received_frame); + } + + for (size_t i = 1; i < frames_order.size(); i += 2) { + uint16_t frame_id = frames_order.at(i); + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kCharlie, received_frame.id(), + FakeEncode(received_frame)); + SleepMs(40); + analyzer.OnFrameDecoded(kCharlie, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + SleepMs(5); + analyzer.OnFrameRendered(kCharlie, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(analyzer_stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(analyzer_stats.comparisons_done, kMaxFramesInFlightPerStream * 2); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream); + EXPECT_EQ(frame_counters.received, kMaxFramesInFlightPerStream); + EXPECT_EQ(frame_counters.decoded, kMaxFramesInFlightPerStream); + EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream); + EXPECT_EQ(frame_counters.dropped, kMaxFramesInFlightPerStream); + + VideoStreamsInfo streams_info = analyzer.GetKnownStreams(); + EXPECT_EQ(streams_info.GetStreams(), std::set<std::string>{kStreamLabel}); + EXPECT_EQ(streams_info.GetStreams(kAlice), + std::set<std::string>{kStreamLabel}); + EXPECT_EQ(streams_info.GetSender(kStreamLabel), kAlice); + EXPECT_EQ(streams_info.GetReceivers(kStreamLabel), + (std::set<std::string>{kBob, kCharlie})); + + EXPECT_EQ(streams_info.GetStatsKeys().size(), 2lu); + for (auto stream_key : streams_info.GetStatsKeys()) { + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(stream_key); + // On some devices the pipeline can be too slow, so we actually can't + // force real constraints here. Lets just check, that at least 1 + // frame passed whole pipeline. + EXPECT_GE(stream_conters.captured, 10); + EXPECT_GE(stream_conters.pre_encoded, 10); + EXPECT_GE(stream_conters.encoded, 10); + EXPECT_GE(stream_conters.received, 5); + EXPECT_GE(stream_conters.decoded, 5); + EXPECT_GE(stream_conters.rendered, 5); + EXPECT_GE(stream_conters.dropped, 5); + } + + std::map<StatsKey, StreamStats> stats = analyzer.GetStats(); + const StatsKey kAliceBobStats(kStreamLabel, kBob); + const StatsKey kAliceCharlieStats(kStreamLabel, kCharlie); + EXPECT_EQ(stats.size(), 2lu); + { + auto it = stats.find(kAliceBobStats); + EXPECT_FALSE(it == stats.end()); + ASSERT_FALSE(it->second.encode_time_ms.IsEmpty()); + EXPECT_GE(it->second.encode_time_ms.GetMin(), 20); + ASSERT_FALSE(it->second.decode_time_ms.IsEmpty()); + EXPECT_GE(it->second.decode_time_ms.GetMin(), 30); + ASSERT_FALSE(it->second.resolution_of_rendered_frame.IsEmpty()); + EXPECT_GE(it->second.resolution_of_rendered_frame.GetMin(), + kFrameWidth * kFrameHeight - 1); + EXPECT_LE(it->second.resolution_of_rendered_frame.GetMax(), + kFrameWidth * kFrameHeight + 1); + } + { + auto it = stats.find(kAliceCharlieStats); + EXPECT_FALSE(it == stats.end()); + ASSERT_FALSE(it->second.encode_time_ms.IsEmpty()); + EXPECT_GE(it->second.encode_time_ms.GetMin(), 20); + ASSERT_FALSE(it->second.decode_time_ms.IsEmpty()); + EXPECT_GE(it->second.decode_time_ms.GetMin(), 30); + ASSERT_FALSE(it->second.resolution_of_rendered_frame.IsEmpty()); + EXPECT_GE(it->second.resolution_of_rendered_frame.GetMin(), + kFrameWidth * kFrameHeight - 1); + EXPECT_LE(it->second.resolution_of_rendered_frame.GetMax(), + kFrameWidth * kFrameHeight + 1); + } +} + +// Test the case which can happen when SFU is switching from one layer to +// another, so the same frame can be received twice by the same peer. +TEST(DefaultVideoQualityAnalyzerTest, + OneFrameReceivedTwiceBySamePeerWith2Receivers) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + constexpr char kAlice[] = "alice"; + constexpr char kBob[] = "bob"; + constexpr char kCharlie[] = "charlie"; + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie}, + kAnalyzerMaxThreadsCount); + + VideoFrame captured_frame = NextFrame(frame_generator.get(), 0); + captured_frame.set_id( + analyzer.OnFrameCaptured(kAlice, kStreamLabel, captured_frame)); + analyzer.OnFramePreEncode(kAlice, captured_frame); + analyzer.OnFrameEncoded(kAlice, captured_frame.id(), + FakeEncode(captured_frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + VideoFrame received_frame = DeepCopy(captured_frame); + analyzer.OnFramePreDecode(kBob, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kBob, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kBob, received_frame); + + received_frame = DeepCopy(captured_frame); + analyzer.OnFramePreDecode(kBob, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kBob, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kBob, received_frame); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + // We have 2 comparisons here because 1 for the frame received by Bob and + // 1 for the frame in flight from Alice to Charlie. + EXPECT_EQ(stats.comparisons_done, 2); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 1); + EXPECT_EQ(frame_counters.received, 1); + EXPECT_EQ(frame_counters.decoded, 1); + EXPECT_EQ(frame_counters.rendered, 1); + EXPECT_EQ(frame_counters.dropped, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, HeavyQualityMetricsFromEqualFrames) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions analyzer_options; + analyzer_options.compute_psnr = true; + analyzer_options.compute_ssim = true; + analyzer_options.adjust_cropping_before_comparing_frames = false; + analyzer_options.max_frames_in_flight_per_stream_count = + kMaxFramesInFlightPerStream; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + analyzer_options); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. Heavy metrics + // computation is turned on, so giving some extra time to be sure that + // computatio have ended. + SleepMs(500); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream); + + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + std::map<StatsKey, StreamStats> stream_stats = analyzer.GetStats(); + const StatsKey kAliceBobStats(kStreamLabel, kReceiverPeerName); + EXPECT_EQ(stream_stats.size(), 1lu); + + auto it = stream_stats.find(kAliceBobStats); + EXPECT_GE(it->second.psnr.GetMin(), kPerfectPSNR); + EXPECT_GE(it->second.ssim.GetMin(), kMaxSsim); +} + +TEST(DefaultVideoQualityAnalyzerTest, + HeavyQualityMetricsFromShiftedFramesWithAdjustment) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions analyzer_options; + analyzer_options.compute_psnr = true; + analyzer_options.compute_ssim = true; + analyzer_options.adjust_cropping_before_comparing_frames = true; + analyzer_options.max_frames_in_flight_per_stream_count = + kMaxFramesInFlightPerStream; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + analyzer_options); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + VideoFrame received_frame = frame; + // Shift frame by a few pixels. + test::CropRegion crop_region{0, 1, 3, 0}; + rtc::scoped_refptr<VideoFrameBuffer> cropped_buffer = + CropAndZoom(crop_region, received_frame.video_frame_buffer()->ToI420()); + received_frame.set_video_frame_buffer(cropped_buffer); + + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. Heavy metrics + // computation is turned on, so giving some extra time to be sure that + // computatio have ended. + SleepMs(500); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream); + + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + std::map<StatsKey, StreamStats> stream_stats = analyzer.GetStats(); + const StatsKey kAliceBobStats(kStreamLabel, kReceiverPeerName); + EXPECT_EQ(stream_stats.size(), 1lu); + + auto it = stream_stats.find(kAliceBobStats); + EXPECT_GE(it->second.psnr.GetMin(), kPerfectPSNR); + EXPECT_GE(it->second.ssim.GetMin(), kMaxSsim); +} + +TEST(DefaultVideoQualityAnalyzerTest, CpuUsage) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + } + + // Windows CPU clock has low accuracy. We need to fake some additional load to + // be sure that the clock ticks (https://crbug.com/webrtc/12249). + FakeCPULoad(); + + for (size_t i = 1; i < frames_order.size(); i += 2) { + uint16_t frame_id = frames_order.at(i); + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + double cpu_usage = analyzer.GetCpuUsagePercent(); + ASSERT_GT(cpu_usage, 0); + + SleepMs(100); + analyzer.Stop(); + + EXPECT_EQ(analyzer.GetCpuUsagePercent(), cpu_usage); +} + +TEST(DefaultVideoQualityAnalyzerTest, RuntimeParticipantsAdding) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + constexpr char kAlice[] = "alice"; + constexpr char kBob[] = "bob"; + constexpr char kCharlie[] = "charlie"; + constexpr char kKatie[] = "katie"; + + constexpr int kFramesCount = 9; + constexpr int kOneThirdFrames = kFramesCount / 3; + constexpr int kTwoThirdFrames = 2 * kOneThirdFrames; + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", {}, kAnalyzerMaxThreadsCount); + + std::map<uint16_t, VideoFrame> captured_frames; + std::vector<uint16_t> frames_order; + analyzer.RegisterParticipantInCall(kAlice); + analyzer.RegisterParticipantInCall(kBob); + + // Alice is sending frames. + for (int i = 0; i < kFramesCount; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), i); + frame.set_id(analyzer.OnFrameCaptured(kAlice, kStreamLabel, frame)); + frames_order.push_back(frame.id()); + captured_frames.insert({frame.id(), frame}); + analyzer.OnFramePreEncode(kAlice, frame); + analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + } + + // Bob receives one third of the sent frames. + for (int i = 0; i < kOneThirdFrames; ++i) { + uint16_t frame_id = frames_order.at(i); + VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kBob, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kBob, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kBob, received_frame); + } + + analyzer.RegisterParticipantInCall(kCharlie); + analyzer.RegisterParticipantInCall(kKatie); + + // New participants were dynamically added. Bob and Charlie receive second + // third of the sent frames. Katie drops the frames. + for (int i = kOneThirdFrames; i < kTwoThirdFrames; ++i) { + uint16_t frame_id = frames_order.at(i); + VideoFrame bob_received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kBob, bob_received_frame.id(), + FakeEncode(bob_received_frame)); + analyzer.OnFrameDecoded(kBob, bob_received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kBob, bob_received_frame); + + VideoFrame charlie_received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kCharlie, charlie_received_frame.id(), + FakeEncode(charlie_received_frame)); + analyzer.OnFrameDecoded(kCharlie, charlie_received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kCharlie, charlie_received_frame); + } + + // Bob, Charlie and Katie receive the rest of the sent frames. + for (int i = kTwoThirdFrames; i < kFramesCount; ++i) { + uint16_t frame_id = frames_order.at(i); + VideoFrame bob_received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kBob, bob_received_frame.id(), + FakeEncode(bob_received_frame)); + analyzer.OnFrameDecoded(kBob, bob_received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kBob, bob_received_frame); + + VideoFrame charlie_received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kCharlie, charlie_received_frame.id(), + FakeEncode(charlie_received_frame)); + analyzer.OnFrameDecoded(kCharlie, charlie_received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kCharlie, charlie_received_frame); + + VideoFrame katie_received_frame = DeepCopy(captured_frames.at(frame_id)); + analyzer.OnFramePreDecode(kKatie, katie_received_frame.id(), + FakeEncode(katie_received_frame)); + analyzer.OnFrameDecoded(kKatie, katie_received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kKatie, katie_received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0); + EXPECT_EQ(stats.comparisons_done, kFramesCount + 2 * kTwoThirdFrames); + + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, kFramesCount); + EXPECT_EQ(frame_counters.received, 2 * kFramesCount); + EXPECT_EQ(frame_counters.decoded, 2 * kFramesCount); + EXPECT_EQ(frame_counters.rendered, 2 * kFramesCount); + EXPECT_EQ(frame_counters.dropped, kOneThirdFrames); + + const StatsKey kAliceBobStats(kStreamLabel, kBob); + const StatsKey kAliceCharlieStats(kStreamLabel, kCharlie); + const StatsKey kAliceKatieStats(kStreamLabel, kKatie); + EXPECT_EQ(analyzer.GetKnownStreams().GetStatsKeys(), + (std::set<StatsKey>{kAliceBobStats, kAliceCharlieStats, + kAliceKatieStats})); + { + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(kAliceBobStats); + EXPECT_EQ(stream_conters.captured, kFramesCount); + EXPECT_EQ(stream_conters.pre_encoded, kFramesCount); + EXPECT_EQ(stream_conters.encoded, kFramesCount); + EXPECT_EQ(stream_conters.received, kFramesCount); + EXPECT_EQ(stream_conters.decoded, kFramesCount); + EXPECT_EQ(stream_conters.rendered, kFramesCount); + } + { + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(kAliceCharlieStats); + EXPECT_EQ(stream_conters.captured, kFramesCount); + EXPECT_EQ(stream_conters.pre_encoded, kFramesCount); + EXPECT_EQ(stream_conters.encoded, kFramesCount); + EXPECT_EQ(stream_conters.received, kTwoThirdFrames); + EXPECT_EQ(stream_conters.decoded, kTwoThirdFrames); + EXPECT_EQ(stream_conters.rendered, kTwoThirdFrames); + } + { + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(kAliceKatieStats); + EXPECT_EQ(stream_conters.captured, kFramesCount); + EXPECT_EQ(stream_conters.pre_encoded, kFramesCount); + EXPECT_EQ(stream_conters.encoded, kFramesCount); + EXPECT_EQ(stream_conters.received, kOneThirdFrames); + EXPECT_EQ(stream_conters.decoded, kOneThirdFrames); + EXPECT_EQ(stream_conters.rendered, kOneThirdFrames); + } +} + +TEST(DefaultVideoQualityAnalyzerTest, + SimulcastFrameWasFullyReceivedByAllPeersBeforeEncodeFinish) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + AnalyzerOptionsForTest()); + constexpr char kAlice[] = "alice"; + constexpr char kBob[] = "bob"; + constexpr char kCharlie[] = "charlie"; + analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), 1); + + frame.set_id(analyzer.OnFrameCaptured(kAlice, kStreamLabel, frame)); + analyzer.OnFramePreEncode(kAlice, frame); + // Encode 1st simulcast layer + analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + // Receive by Bob + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kBob, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kBob, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kBob, received_frame); + // Receive by Charlie + received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kCharlie, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kCharlie, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kCharlie, received_frame); + + // Encode 2nd simulcast layer + analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(stats.comparisons_done, 2); + + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 1); + EXPECT_EQ(frame_counters.rendered, 2); +} + +TEST(DefaultVideoQualityAnalyzerTest, + FrameCanBeReceivedBySenderAfterItWasReceivedByReceiver) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::vector<VideoFrame> frames; + for (int i = 0; i < 3; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), 1); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames.push_back(frame); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + } + + // Receive by 2nd peer. + for (VideoFrame& frame : frames) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Check that we still have that frame in flight. + AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats(); + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(analyzer_stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 3) + << "Expected that frame is still in flight, " + << "because it wasn't received by sender" + << ToString(frames_in_flight_sizes); + + // Receive by sender + for (VideoFrame& frame : frames) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kSenderPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kSenderPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kSenderPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + analyzer_stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(analyzer_stats.comparisons_done, 6); + + frames_in_flight_sizes = + GetSortedSamples(analyzer_stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 3); + EXPECT_EQ(frame_counters.rendered, 6); + + EXPECT_EQ(analyzer.GetStats().size(), 2lu); + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kReceiverPeerName)); + EXPECT_EQ(stream_conters.captured, 3); + EXPECT_EQ(stream_conters.pre_encoded, 3); + EXPECT_EQ(stream_conters.encoded, 3); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 3); + EXPECT_EQ(stream_conters.rendered, 3); + } + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kSenderPeerName)); + EXPECT_EQ(stream_conters.captured, 3); + EXPECT_EQ(stream_conters.pre_encoded, 3); + EXPECT_EQ(stream_conters.encoded, 3); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 3); + EXPECT_EQ(stream_conters.rendered, 3); + } +} + +TEST(DefaultVideoQualityAnalyzerTest, + FrameCanBeReceivedByReceiverAfterItWasReceivedBySender) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + std::vector<VideoFrame> frames; + for (int i = 0; i < 3; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), 1); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames.push_back(frame); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + } + + // Receive by sender + for (VideoFrame& frame : frames) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kSenderPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kSenderPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kSenderPeerName, received_frame); + } + + // Check that we still have that frame in flight. + AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats(); + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(analyzer_stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 3) + << "Expected that frame is still in flight, " + << "because it wasn't received by sender" + << ToString(frames_in_flight_sizes); + + // Receive by 2nd peer. + for (VideoFrame& frame : frames) { + VideoFrame received_frame = DeepCopy(frame); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + analyzer_stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(analyzer_stats.comparisons_done, 6); + + frames_in_flight_sizes = + GetSortedSamples(analyzer_stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 0) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 3); + EXPECT_EQ(frame_counters.rendered, 6); + + EXPECT_EQ(analyzer.GetStats().size(), 2lu); + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kReceiverPeerName)); + EXPECT_EQ(stream_conters.captured, 3); + EXPECT_EQ(stream_conters.pre_encoded, 3); + EXPECT_EQ(stream_conters.encoded, 3); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 3); + EXPECT_EQ(stream_conters.rendered, 3); + } + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kSenderPeerName)); + EXPECT_EQ(stream_conters.captured, 3); + EXPECT_EQ(stream_conters.pre_encoded, 3); + EXPECT_EQ(stream_conters.encoded, 3); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 3); + EXPECT_EQ(stream_conters.rendered, 3); + } +} + +TEST(DefaultVideoQualityAnalyzerTest, CodecTrackedCorrectly) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), + AnalyzerOptionsForTest()); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + VideoQualityAnalyzerInterface::EncoderStats encoder_stats; + std::vector<std::string> codec_names = {"codec_1", "codec_2"}; + std::vector<VideoFrame> frames; + // Send 3 frame for each codec. + for (size_t i = 0; i < codec_names.size(); ++i) { + for (size_t j = 0; j < 3; ++j) { + VideoFrame frame = NextFrame(frame_generator.get(), 3 * i + j); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + analyzer.OnFramePreEncode(kSenderPeerName, frame); + encoder_stats.encoder_name = codec_names[i]; + analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame), + encoder_stats); + frames.push_back(std::move(frame)); + } + } + + // Receive 3 frame for each codec. + VideoQualityAnalyzerInterface::DecoderStats decoder_stats; + for (size_t i = 0; i < codec_names.size(); ++i) { + for (size_t j = 0; j < 3; ++j) { + VideoFrame received_frame = DeepCopy(frames[3 * i + j]); + analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(), + FakeEncode(received_frame)); + decoder_stats.decoder_name = codec_names[i]; + analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, decoder_stats); + analyzer.OnFrameRendered(kReceiverPeerName, received_frame); + } + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + std::map<StatsKey, StreamStats> stats = analyzer.GetStats(); + ASSERT_EQ(stats.size(), 1lu); + const StreamStats& stream_stats = + stats.at(StatsKey(kStreamLabel, kReceiverPeerName)); + ASSERT_EQ(stream_stats.encoders.size(), 2lu); + EXPECT_EQ(stream_stats.encoders[0].codec_name, codec_names[0]); + EXPECT_EQ(stream_stats.encoders[0].first_frame_id, frames[0].id()); + EXPECT_EQ(stream_stats.encoders[0].last_frame_id, frames[2].id()); + EXPECT_EQ(stream_stats.encoders[1].codec_name, codec_names[1]); + EXPECT_EQ(stream_stats.encoders[1].first_frame_id, frames[3].id()); + EXPECT_EQ(stream_stats.encoders[1].last_frame_id, frames[5].id()); + + ASSERT_EQ(stream_stats.decoders.size(), 2lu); + EXPECT_EQ(stream_stats.decoders[0].codec_name, codec_names[0]); + EXPECT_EQ(stream_stats.decoders[0].first_frame_id, frames[0].id()); + EXPECT_EQ(stream_stats.decoders[0].last_frame_id, frames[2].id()); + EXPECT_EQ(stream_stats.decoders[1].codec_name, codec_names[1]); + EXPECT_EQ(stream_stats.decoders[1].first_frame_id, frames[3].id()); + EXPECT_EQ(stream_stats.decoders[1].last_frame_id, frames[5].id()); +} + +TEST(DefaultVideoQualityAnalyzerTest, + FramesInFlightAreCorrectlySentToTheComparatorAfterStop) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + // There are 7 different timings inside frame stats: captured, pre_encode, + // encoded, received, decode_start, decode_end, rendered. captured is always + // set and received is set together with decode_start. So we create 6 + // different frames, where for each frame next timings will be set + // * 1st - all of them set + // * 2nd - captured, pre_encode, encoded, received, decode_start, decode_end + // * 3rd - captured, pre_encode, encoded, received, decode_start + // * 4th - captured, pre_encode, encoded + // * 5th - captured, pre_encode + // * 6th - captured + std::vector<VideoFrame> frames; + // Sender side actions + for (int i = 0; i < 6; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), 1); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames.push_back(frame); + } + for (int i = 0; i < 5; ++i) { + analyzer.OnFramePreEncode(kSenderPeerName, frames[i]); + } + for (int i = 0; i < 4; ++i) { + analyzer.OnFrameEncoded(kSenderPeerName, frames[i].id(), + FakeEncode(frames[i]), + VideoQualityAnalyzerInterface::EncoderStats()); + } + + // Receiver side actions + for (int i = 0; i < 3; ++i) { + analyzer.OnFramePreDecode(kReceiverPeerName, frames[i].id(), + FakeEncode(frames[i])); + } + for (int i = 0; i < 2; ++i) { + analyzer.OnFrameDecoded(kReceiverPeerName, DeepCopy(frames[i]), + VideoQualityAnalyzerInterface::DecoderStats()); + } + for (int i = 0; i < 1; ++i) { + analyzer.OnFrameRendered(kReceiverPeerName, DeepCopy(frames[i])); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(analyzer_stats.comparisons_done, 6); + + // The last frames in flight size has to reflect the amount of frame in flight + // before all of them were sent to the comparison when Stop() was invoked. + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(analyzer_stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 5) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 6); + EXPECT_EQ(frame_counters.pre_encoded, 5); + EXPECT_EQ(frame_counters.encoded, 4); + EXPECT_EQ(frame_counters.received, 3); + EXPECT_EQ(frame_counters.decoded, 2); + EXPECT_EQ(frame_counters.rendered, 1); + + EXPECT_EQ(analyzer.GetStats().size(), 1lu); + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kReceiverPeerName)); + EXPECT_EQ(stream_conters.captured, 6); + EXPECT_EQ(stream_conters.pre_encoded, 5); + EXPECT_EQ(stream_conters.encoded, 4); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 2); + EXPECT_EQ(stream_conters.rendered, 1); + } +} + +TEST( + DefaultVideoQualityAnalyzerTest, + FramesInFlightAreCorrectlySentToTheComparatorAfterStopForSenderAndReceiver) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", + std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, + kAnalyzerMaxThreadsCount); + + // There are 7 different timings inside frame stats: captured, pre_encode, + // encoded, received, decode_start, decode_end, rendered. captured is always + // set and received is set together with decode_start. So we create 6 + // different frames, where for each frame next timings will be set + // * 1st - all of them set + // * 2nd - captured, pre_encode, encoded, received, decode_start, decode_end + // * 3rd - captured, pre_encode, encoded, received, decode_start + // * 4th - captured, pre_encode, encoded + // * 5th - captured, pre_encode + // * 6th - captured + std::vector<VideoFrame> frames; + // Sender side actions + for (int i = 0; i < 6; ++i) { + VideoFrame frame = NextFrame(frame_generator.get(), 1); + frame.set_id( + analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame)); + frames.push_back(frame); + } + for (int i = 0; i < 5; ++i) { + analyzer.OnFramePreEncode(kSenderPeerName, frames[i]); + } + for (int i = 0; i < 4; ++i) { + analyzer.OnFrameEncoded(kSenderPeerName, frames[i].id(), + FakeEncode(frames[i]), + VideoQualityAnalyzerInterface::EncoderStats()); + } + + // Receiver side actions + for (int i = 0; i < 3; ++i) { + analyzer.OnFramePreDecode(kSenderPeerName, frames[i].id(), + FakeEncode(frames[i])); + analyzer.OnFramePreDecode(kReceiverPeerName, frames[i].id(), + FakeEncode(frames[i])); + } + for (int i = 0; i < 2; ++i) { + analyzer.OnFrameDecoded(kSenderPeerName, DeepCopy(frames[i]), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameDecoded(kReceiverPeerName, DeepCopy(frames[i]), + VideoQualityAnalyzerInterface::DecoderStats()); + } + for (int i = 0; i < 1; ++i) { + analyzer.OnFrameRendered(kSenderPeerName, DeepCopy(frames[i])); + analyzer.OnFrameRendered(kReceiverPeerName, DeepCopy(frames[i])); + } + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats(); + EXPECT_EQ(analyzer_stats.comparisons_done, 12); + + // The last frames in flight size has to reflect the amount of frame in flight + // before all of them were sent to the comparison when Stop() was invoked. + std::vector<StatsSample> frames_in_flight_sizes = + GetSortedSamples(analyzer_stats.frames_in_flight_left_count); + EXPECT_EQ(frames_in_flight_sizes.back().value, 5) + << ToString(frames_in_flight_sizes); + + FrameCounters frame_counters = analyzer.GetGlobalCounters(); + EXPECT_EQ(frame_counters.captured, 6); + EXPECT_EQ(frame_counters.pre_encoded, 5); + EXPECT_EQ(frame_counters.encoded, 4); + EXPECT_EQ(frame_counters.received, 6); + EXPECT_EQ(frame_counters.decoded, 4); + EXPECT_EQ(frame_counters.rendered, 2); + + EXPECT_EQ(analyzer.GetStats().size(), 2lu); + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kReceiverPeerName)); + EXPECT_EQ(stream_conters.captured, 6); + EXPECT_EQ(stream_conters.pre_encoded, 5); + EXPECT_EQ(stream_conters.encoded, 4); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 2); + EXPECT_EQ(stream_conters.rendered, 1); + } + { + FrameCounters stream_conters = analyzer.GetPerStreamCounters().at( + StatsKey(kStreamLabel, kSenderPeerName)); + EXPECT_EQ(stream_conters.captured, 6); + EXPECT_EQ(stream_conters.pre_encoded, 5); + EXPECT_EQ(stream_conters.encoded, 4); + EXPECT_EQ(stream_conters.received, 3); + EXPECT_EQ(stream_conters.decoded, 2); + EXPECT_EQ(stream_conters.rendered, 1); + } +} + +TEST(DefaultVideoQualityAnalyzerTest, GetStreamFrames) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + // The order in which peers captured frames and passed them to analyzer. + std::vector<std::string> frame_capturers_sequence{ + "alice", "alice", "bob", "bob", "bob", + "bob", "bob", "alice", "alice", "alice", + }; + + std::map<std::string, std::vector<uint16_t>> stream_to_frame_ids; + stream_to_frame_ids.emplace("alice_video", std::vector<uint16_t>{}); + stream_to_frame_ids.emplace("bob_video", std::vector<uint16_t>{}); + + std::vector<VideoFrame> frames; + for (const std::string& sender : frame_capturers_sequence) { + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = + analyzer.OnFrameCaptured(sender, sender + "_video", frame); + frame.set_id(frame_id); + stream_to_frame_ids.find(sender + "_video")->second.push_back(frame_id); + frames.push_back(frame); + analyzer.OnFramePreEncode(sender, frame); + analyzer.OnFrameEncoded(sender, frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + } + // We don't need to receive frames for stats to be gathered correctly. + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + EXPECT_EQ(analyzer.GetStreamFrames(), stream_to_frame_ids); +} + +TEST(DefaultVideoQualityAnalyzerTest, ReceiverReceivedFramesWhenSenderRemoved) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + analyzer.UnregisterParticipantInCall("alice"); + + analyzer.OnFramePreDecode("bob", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("bob", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("bob", DeepCopy(frame)); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 1); + EXPECT_EQ(stream_conters.decoded, 1); + EXPECT_EQ(stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, + ReceiverReceivedFramesWhenSenderRemovedWithSelfview) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + analyzer.UnregisterParticipantInCall("alice"); + + analyzer.OnFramePreDecode("bob", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("bob", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("bob", DeepCopy(frame)); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 1); + EXPECT_EQ(stream_conters.decoded, 1); + EXPECT_EQ(stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, + SenderReceivedFramesWhenReceiverRemovedWithSelfview) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + analyzer.UnregisterParticipantInCall("bob"); + + analyzer.OnFramePreDecode("alice", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("alice", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("alice", DeepCopy(frame)); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "alice")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 1); + EXPECT_EQ(stream_conters.decoded, 1); + EXPECT_EQ(stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, + SenderAndReceiverReceivedFramesWhenReceiverRemovedWithSelfview) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + options.enable_receive_own_stream = true; + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + analyzer.OnFramePreDecode("bob", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("bob", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("bob", DeepCopy(frame)); + + analyzer.UnregisterParticipantInCall("bob"); + + analyzer.OnFramePreDecode("alice", frame.id(), FakeEncode(frame)); + analyzer.OnFrameDecoded("alice", DeepCopy(frame), + VideoQualityAnalyzerInterface::DecoderStats()); + analyzer.OnFrameRendered("alice", DeepCopy(frame)); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters alice_alice_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "alice")); + EXPECT_EQ(alice_alice_stream_conters.captured, 1); + EXPECT_EQ(alice_alice_stream_conters.pre_encoded, 1); + EXPECT_EQ(alice_alice_stream_conters.encoded, 1); + EXPECT_EQ(alice_alice_stream_conters.received, 1); + EXPECT_EQ(alice_alice_stream_conters.decoded, 1); + EXPECT_EQ(alice_alice_stream_conters.rendered, 1); + + FrameCounters alice_bob_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(alice_bob_stream_conters.captured, 1); + EXPECT_EQ(alice_bob_stream_conters.pre_encoded, 1); + EXPECT_EQ(alice_bob_stream_conters.encoded, 1); + EXPECT_EQ(alice_bob_stream_conters.received, 1); + EXPECT_EQ(alice_bob_stream_conters.decoded, 1); + EXPECT_EQ(alice_bob_stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeCapturing2ndFrame) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"}, + /*frames_count=*/1, *frame_generator); + analyzer.UnregisterParticipantInCall("bob"); + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {}, + /*frames_count=*/1, *frame_generator); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 2); + EXPECT_EQ(global_stream_conters.pre_encoded, 2); + EXPECT_EQ(global_stream_conters.encoded, 2); + EXPECT_EQ(global_stream_conters.received, 1); + EXPECT_EQ(global_stream_conters.decoded, 1); + EXPECT_EQ(global_stream_conters.rendered, 1); + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 2); + EXPECT_EQ(stream_conters.pre_encoded, 2); + EXPECT_EQ(stream_conters.encoded, 2); + EXPECT_EQ(stream_conters.received, 1); + EXPECT_EQ(stream_conters.decoded, 1); + EXPECT_EQ(stream_conters.rendered, 1); +} + +TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforePreEncoded) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.UnregisterParticipantInCall("bob"); + analyzer.OnFramePreEncode("alice", frame); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 1); + EXPECT_EQ(global_stream_conters.pre_encoded, 1); + EXPECT_EQ(global_stream_conters.encoded, 1); + EXPECT_EQ(global_stream_conters.received, 0); + EXPECT_EQ(global_stream_conters.decoded, 0); + EXPECT_EQ(global_stream_conters.rendered, 0); + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 0); + EXPECT_EQ(stream_conters.decoded, 0); + EXPECT_EQ(stream_conters.rendered, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeEncoded) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + analyzer.UnregisterParticipantInCall("bob"); + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 1); + EXPECT_EQ(global_stream_conters.pre_encoded, 1); + EXPECT_EQ(global_stream_conters.encoded, 1); + EXPECT_EQ(global_stream_conters.received, 0); + EXPECT_EQ(global_stream_conters.decoded, 0); + EXPECT_EQ(global_stream_conters.rendered, 0); + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 0); + EXPECT_EQ(stream_conters.decoded, 0); + EXPECT_EQ(stream_conters.rendered, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, + ReceiverRemovedBetweenSimulcastLayersEncoded) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, + kAnalyzerMaxThreadsCount); + + VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1); + uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame); + frame.set_id(frame_id); + analyzer.OnFramePreEncode("alice", frame); + // 1st simulcast layer encoded + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + analyzer.UnregisterParticipantInCall("bob"); + // 2nd simulcast layer encoded + analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame), + VideoQualityAnalyzerInterface::EncoderStats()); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 1); + EXPECT_EQ(global_stream_conters.pre_encoded, 1); + EXPECT_EQ(global_stream_conters.encoded, 1); + EXPECT_EQ(global_stream_conters.received, 0); + EXPECT_EQ(global_stream_conters.decoded, 0); + EXPECT_EQ(global_stream_conters.rendered, 0); + FrameCounters stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(stream_conters.captured, 1); + EXPECT_EQ(stream_conters.pre_encoded, 1); + EXPECT_EQ(stream_conters.encoded, 1); + EXPECT_EQ(stream_conters.received, 0); + EXPECT_EQ(stream_conters.decoded, 0); + EXPECT_EQ(stream_conters.rendered, 0); +} + +TEST(DefaultVideoQualityAnalyzerTest, UnregisterOneAndRegisterAnother) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", + std::vector<std::string>{"alice", "bob", "charlie"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"bob", "charlie"}, + /*frames_count=*/2, *frame_generator); + analyzer.UnregisterParticipantInCall("bob"); + analyzer.RegisterParticipantInCall("david"); + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"charlie", "david"}, + /*frames_count=*/4, *frame_generator); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 6); + EXPECT_EQ(global_stream_conters.pre_encoded, 6); + EXPECT_EQ(global_stream_conters.encoded, 6); + EXPECT_EQ(global_stream_conters.received, 12); + EXPECT_EQ(global_stream_conters.decoded, 12); + EXPECT_EQ(global_stream_conters.rendered, 12); + FrameCounters alice_bob_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(alice_bob_stream_conters.captured, 6); + EXPECT_EQ(alice_bob_stream_conters.pre_encoded, 6); + EXPECT_EQ(alice_bob_stream_conters.encoded, 6); + EXPECT_EQ(alice_bob_stream_conters.received, 2); + EXPECT_EQ(alice_bob_stream_conters.decoded, 2); + EXPECT_EQ(alice_bob_stream_conters.rendered, 2); + FrameCounters alice_charlie_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "charlie")); + EXPECT_EQ(alice_charlie_stream_conters.captured, 6); + EXPECT_EQ(alice_charlie_stream_conters.pre_encoded, 6); + EXPECT_EQ(alice_charlie_stream_conters.encoded, 6); + EXPECT_EQ(alice_charlie_stream_conters.received, 6); + EXPECT_EQ(alice_charlie_stream_conters.decoded, 6); + EXPECT_EQ(alice_charlie_stream_conters.rendered, 6); + FrameCounters alice_david_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "david")); + EXPECT_EQ(alice_david_stream_conters.captured, 6); + EXPECT_EQ(alice_david_stream_conters.pre_encoded, 6); + EXPECT_EQ(alice_david_stream_conters.encoded, 6); + EXPECT_EQ(alice_david_stream_conters.received, 4); + EXPECT_EQ(alice_david_stream_conters.decoded, 4); + EXPECT_EQ(alice_david_stream_conters.rendered, 4); +} + +TEST(DefaultVideoQualityAnalyzerTest, + UnregisterOneAndRegisterAnotherRegisterBack) { + std::unique_ptr<test::FrameGeneratorInterface> frame_generator = + test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight, + /*type=*/absl::nullopt, + /*num_squares=*/absl::nullopt); + + DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); + DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); + analyzer.Start("test_case", + std::vector<std::string>{"alice", "bob", "charlie"}, + kAnalyzerMaxThreadsCount); + + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"bob", "charlie"}, + /*frames_count=*/2, *frame_generator); + analyzer.UnregisterParticipantInCall("bob"); + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"charlie"}, + /*frames_count=*/4, *frame_generator); + analyzer.RegisterParticipantInCall("bob"); + PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", + {"bob", "charlie"}, + /*frames_count=*/6, *frame_generator); + + // Give analyzer some time to process frames on async thread. The computations + // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it + // means we have an issue! + SleepMs(100); + analyzer.Stop(); + + FrameCounters global_stream_conters = analyzer.GetGlobalCounters(); + EXPECT_EQ(global_stream_conters.captured, 12); + EXPECT_EQ(global_stream_conters.pre_encoded, 12); + EXPECT_EQ(global_stream_conters.encoded, 12); + EXPECT_EQ(global_stream_conters.received, 20); + EXPECT_EQ(global_stream_conters.decoded, 20); + EXPECT_EQ(global_stream_conters.rendered, 20); + FrameCounters alice_bob_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob")); + EXPECT_EQ(alice_bob_stream_conters.captured, 12); + EXPECT_EQ(alice_bob_stream_conters.pre_encoded, 12); + EXPECT_EQ(alice_bob_stream_conters.encoded, 12); + EXPECT_EQ(alice_bob_stream_conters.received, 8); + EXPECT_EQ(alice_bob_stream_conters.decoded, 8); + EXPECT_EQ(alice_bob_stream_conters.rendered, 8); + FrameCounters alice_charlie_stream_conters = + analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "charlie")); + EXPECT_EQ(alice_charlie_stream_conters.captured, 12); + EXPECT_EQ(alice_charlie_stream_conters.pre_encoded, 12); + EXPECT_EQ(alice_charlie_stream_conters.encoded, 12); + EXPECT_EQ(alice_charlie_stream_conters.received, 12); + EXPECT_EQ(alice_charlie_stream_conters.decoded, 12); + EXPECT_EQ(alice_charlie_stream_conters.rendered, 12); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h new file mode 100644 index 0000000000..384e901462 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_ + +#include <cstdint> +#include <utility> + +#include "absl/types/optional.h" +#include "api/video/encoded_image.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Injects frame id into EncodedImage on encoder side +class EncodedImageDataInjector { + public: + virtual ~EncodedImageDataInjector() = default; + + // Return encoded image with specified `id` and `discard` flag injected into + // its payload. `discard` flag mean does analyzing decoder should discard this + // encoded image because it belongs to unnecessary simulcast stream or spatial + // layer. + virtual EncodedImage InjectData(uint16_t id, + bool discard, + const EncodedImage& source) = 0; +}; + +struct EncodedImageExtractionResult { + absl::optional<uint16_t> id; + EncodedImage image; + // Is true if encoded image should be discarded. It is used to filter out + // unnecessary spatial layers and simulcast streams. + bool discard; +}; + +// Extracts frame id from EncodedImage on decoder side. +class EncodedImageDataExtractor { + public: + virtual ~EncodedImageDataExtractor() = default; + + // Invoked by framework before any image will come to the extractor. + // `expected_receivers_count` is the expected amount of receivers for each + // encoded image. + virtual void Start(int expected_receivers_count) = 0; + + // Invoked by framework when it is required to add one more receiver for + // frames. Will be invoked before that receiver will start receive data. + virtual void AddParticipantInCall() = 0; + + // Invoked by framework when it is required to remove receiver for frames. + // Will be invoked after that receiver will stop receiving data. + virtual void RemoveParticipantInCall() = 0; + + // Returns encoded image id, extracted from payload and also encoded image + // with its original payload. For concatenated spatial layers it should be the + // same id. + virtual EncodedImageExtractionResult ExtractData( + const EncodedImage& source) = 0; +}; + +class EncodedImageDataPropagator : public EncodedImageDataInjector, + public EncodedImageDataExtractor { + public: + ~EncodedImageDataPropagator() override = default; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc new file mode 100644 index 0000000000..0b9e2cd5a2 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/example_video_quality_analyzer.h" + +#include "api/array_view.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +ExampleVideoQualityAnalyzer::ExampleVideoQualityAnalyzer() = default; +ExampleVideoQualityAnalyzer::~ExampleVideoQualityAnalyzer() = default; + +void ExampleVideoQualityAnalyzer::Start( + std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count) {} + +uint16_t ExampleVideoQualityAnalyzer::OnFrameCaptured( + absl::string_view peer_name, + const std::string& stream_label, + const webrtc::VideoFrame& frame) { + MutexLock lock(&lock_); + uint16_t frame_id = next_frame_id_++; + auto it = frames_in_flight_.find(frame_id); + if (it == frames_in_flight_.end()) { + frames_in_flight_.insert(frame_id); + frames_to_stream_label_.insert({frame_id, stream_label}); + } else { + RTC_LOG(LS_WARNING) << "Meet new frame with the same id: " << frame_id + << ". Assumes old one as dropped"; + // We needn't insert frame to frames_in_flight_, because it is already + // there. + ++frames_dropped_; + auto stream_it = frames_to_stream_label_.find(frame_id); + RTC_CHECK(stream_it != frames_to_stream_label_.end()); + stream_it->second = stream_label; + } + ++frames_captured_; + return frame_id; +} + +void ExampleVideoQualityAnalyzer::OnFramePreEncode( + absl::string_view peer_name, + const webrtc::VideoFrame& frame) { + MutexLock lock(&lock_); + ++frames_pre_encoded_; +} + +void ExampleVideoQualityAnalyzer::OnFrameEncoded( + absl::string_view peer_name, + uint16_t frame_id, + const webrtc::EncodedImage& encoded_image, + const EncoderStats& stats) { + MutexLock lock(&lock_); + ++frames_encoded_; +} + +void ExampleVideoQualityAnalyzer::OnFrameDropped( + absl::string_view peer_name, + webrtc::EncodedImageCallback::DropReason reason) { + RTC_LOG(LS_INFO) << "Frame dropped by encoder"; + MutexLock lock(&lock_); + ++frames_dropped_; +} + +void ExampleVideoQualityAnalyzer::OnFramePreDecode( + absl::string_view peer_name, + uint16_t frame_id, + const webrtc::EncodedImage& encoded_image) { + MutexLock lock(&lock_); + ++frames_received_; +} + +void ExampleVideoQualityAnalyzer::OnFrameDecoded( + absl::string_view peer_name, + const webrtc::VideoFrame& frame, + const DecoderStats& stats) { + MutexLock lock(&lock_); + ++frames_decoded_; +} + +void ExampleVideoQualityAnalyzer::OnFrameRendered( + absl::string_view peer_name, + const webrtc::VideoFrame& frame) { + MutexLock lock(&lock_); + frames_in_flight_.erase(frame.id()); + ++frames_rendered_; +} + +void ExampleVideoQualityAnalyzer::OnEncoderError( + absl::string_view peer_name, + const webrtc::VideoFrame& frame, + int32_t error_code) { + RTC_LOG(LS_ERROR) << "Failed to encode frame " << frame.id() + << ". Code: " << error_code; +} + +void ExampleVideoQualityAnalyzer::OnDecoderError(absl::string_view peer_name, + uint16_t frame_id, + int32_t error_code) { + RTC_LOG(LS_ERROR) << "Failed to decode frame " << frame_id + << ". Code: " << error_code; +} + +void ExampleVideoQualityAnalyzer::Stop() { + MutexLock lock(&lock_); + RTC_LOG(LS_INFO) << "There are " << frames_in_flight_.size() + << " frames in flight, assuming all of them are dropped"; + frames_dropped_ += frames_in_flight_.size(); +} + +std::string ExampleVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) { + MutexLock lock(&lock_); + auto it = frames_to_stream_label_.find(frame_id); + RTC_DCHECK(it != frames_to_stream_label_.end()) + << "Unknown frame_id=" << frame_id; + return it->second; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_captured() const { + MutexLock lock(&lock_); + return frames_captured_; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_pre_encoded() const { + MutexLock lock(&lock_); + return frames_pre_encoded_; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_encoded() const { + MutexLock lock(&lock_); + return frames_encoded_; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_received() const { + MutexLock lock(&lock_); + return frames_received_; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_decoded() const { + MutexLock lock(&lock_); + return frames_decoded_; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_rendered() const { + MutexLock lock(&lock_); + return frames_rendered_; +} + +uint64_t ExampleVideoQualityAnalyzer::frames_dropped() const { + MutexLock lock(&lock_); + return frames_dropped_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h new file mode 100644 index 0000000000..f41e29e8ab --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_EXAMPLE_VIDEO_QUALITY_ANALYZER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_EXAMPLE_VIDEO_QUALITY_ANALYZER_H_ + +#include <atomic> +#include <map> +#include <set> +#include <string> + +#include "api/array_view.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { + +// This class is an example implementation of +// webrtc::VideoQualityAnalyzerInterface and calculates simple metrics +// just to demonstration purposes. Assumed to be used in the single process +// test cases, where both peers are in the same process. +class ExampleVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { + public: + ExampleVideoQualityAnalyzer(); + ~ExampleVideoQualityAnalyzer() override; + + void Start(std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count) override; + uint16_t OnFrameCaptured(absl::string_view peer_name, + const std::string& stream_label, + const VideoFrame& frame) override; + void OnFramePreEncode(absl::string_view peer_name, + const VideoFrame& frame) override; + void OnFrameEncoded(absl::string_view peer_name, + uint16_t frame_id, + const EncodedImage& encoded_image, + const EncoderStats& stats) override; + void OnFrameDropped(absl::string_view peer_name, + EncodedImageCallback::DropReason reason) override; + void OnFramePreDecode(absl::string_view peer_name, + uint16_t frame_id, + const EncodedImage& encoded_image) override; + void OnFrameDecoded(absl::string_view peer_name, + const VideoFrame& frame, + const DecoderStats& stats) override; + void OnFrameRendered(absl::string_view peer_name, + const VideoFrame& frame) override; + void OnEncoderError(absl::string_view peer_name, + const VideoFrame& frame, + int32_t error_code) override; + void OnDecoderError(absl::string_view peer_name, + uint16_t frame_id, + int32_t error_code) override; + void Stop() override; + std::string GetStreamLabel(uint16_t frame_id) override; + + uint64_t frames_captured() const; + uint64_t frames_pre_encoded() const; + uint64_t frames_encoded() const; + uint64_t frames_received() const; + uint64_t frames_decoded() const; + uint64_t frames_rendered() const; + uint64_t frames_dropped() const; + + private: + // When peer A captured the frame it will come into analyzer's OnFrameCaptured + // and will be stored in frames_in_flight_. It will be removed from there + // when it will be received in peer B, so we need to guard it with lock. + // Also because analyzer will serve for all video streams it can be called + // from different threads inside one peer. + mutable Mutex lock_; + // Stores frame ids, that are currently going from one peer to another. We + // need to keep them to correctly determine dropped frames and also correctly + // process frame id overlap. + std::set<uint16_t> frames_in_flight_ RTC_GUARDED_BY(lock_); + std::map<uint16_t, std::string> frames_to_stream_label_ RTC_GUARDED_BY(lock_); + uint16_t next_frame_id_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_captured_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_pre_encoded_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_encoded_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_received_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_decoded_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_rendered_ RTC_GUARDED_BY(lock_) = 0; + uint64_t frames_dropped_ RTC_GUARDED_BY(lock_) = 0; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_EXAMPLE_VIDEO_QUALITY_ANALYZER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue.h new file mode 100644 index 0000000000..39d26b42bc --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue.h @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_MULTI_READER_QUEUE_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_MULTI_READER_QUEUE_H_ + +#include <deque> +#include <memory> +#include <set> +#include <unordered_map> + +#include "absl/types/optional.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +// Represents the queue which can be read by multiple readers. Each reader reads +// from its own queue head. When an element is added it will become visible for +// all readers. When an element will be removed by all the readers, the element +// will be removed from the queue. +template <typename T> +class MultiReaderQueue { + public: + // Creates queue with exactly `readers_count` readers named from 0 to + // `readers_count - 1`. + explicit MultiReaderQueue(size_t readers_count) { + for (size_t i = 0; i < readers_count; ++i) { + heads_[i] = 0; + } + } + // Creates queue with specified readers. + explicit MultiReaderQueue(std::set<size_t> readers) { + for (size_t reader : readers) { + heads_[reader] = 0; + } + } + + // Adds a new `reader`, initializing its reading position (the reader's head) + // equal to the one of `reader_to_copy`. + // Complexity O(MultiReaderQueue::size(reader_to_copy)). + void AddReader(size_t reader, size_t reader_to_copy) { + size_t pos = GetHeadPositionOrDie(reader_to_copy); + + auto it = heads_.find(reader); + RTC_CHECK(it == heads_.end()) + << "Reader " << reader << " is already in the queue"; + heads_[reader] = heads_[reader_to_copy]; + for (size_t i = pos; i < queue_.size(); ++i) { + in_queues_[i]++; + } + } + + // Adds a new `reader`, initializing its reading position equal to first + // element in the queue. + // Complexity O(MultiReaderQueue::size()). + void AddReader(size_t reader) { + auto it = heads_.find(reader); + RTC_CHECK(it == heads_.end()) + << "Reader " << reader << " is already in the queue"; + heads_[reader] = removed_elements_count_; + for (size_t i = 0; i < queue_.size(); ++i) { + in_queues_[i]++; + } + } + + // Removes specified `reader` from the queue. + // Complexity O(MultiReaderQueue::size(reader)). + void RemoveReader(size_t reader) { + size_t pos = GetHeadPositionOrDie(reader); + for (size_t i = pos; i < queue_.size(); ++i) { + in_queues_[i]--; + } + while (!in_queues_.empty() && in_queues_[0] == 0) { + PopFront(); + } + heads_.erase(reader); + } + + // Add value to the end of the queue. Complexity O(1). + void PushBack(T value) { + queue_.push_back(value); + in_queues_.push_back(heads_.size()); + } + + // Extract element from specified head. Complexity O(1). + absl::optional<T> PopFront(size_t reader) { + size_t pos = GetHeadPositionOrDie(reader); + if (pos >= queue_.size()) { + return absl::nullopt; + } + + T out = queue_[pos]; + + in_queues_[pos]--; + heads_[reader]++; + + if (in_queues_[pos] == 0) { + RTC_DCHECK_EQ(pos, 0); + PopFront(); + } + return out; + } + + // Returns element at specified head. Complexity O(1). + absl::optional<T> Front(size_t reader) const { + size_t pos = GetHeadPositionOrDie(reader); + if (pos >= queue_.size()) { + return absl::nullopt; + } + return queue_[pos]; + } + + // Returns true if for specified head there are no more elements in the queue + // or false otherwise. Complexity O(1). + bool IsEmpty(size_t reader) const { + size_t pos = GetHeadPositionOrDie(reader); + return pos >= queue_.size(); + } + + // Returns size of the longest queue between all readers. + // Complexity O(1). + size_t size() const { return queue_.size(); } + + // Returns size of the specified queue. Complexity O(1). + size_t size(size_t reader) const { + size_t pos = GetHeadPositionOrDie(reader); + return queue_.size() - pos; + } + + // Complexity O(1). + size_t readers_count() const { return heads_.size(); } + + private: + size_t GetHeadPositionOrDie(size_t reader) const { + auto it = heads_.find(reader); + RTC_CHECK(it != heads_.end()) << "No queue for reader " << reader; + return it->second - removed_elements_count_; + } + + void PopFront() { + RTC_DCHECK(!queue_.empty()); + RTC_DCHECK_EQ(in_queues_[0], 0); + queue_.pop_front(); + in_queues_.pop_front(); + removed_elements_count_++; + } + + // Number of the elements that were removed from the queue. It is used to + // subtract from each head to compute the right index inside `queue_`; + size_t removed_elements_count_ = 0; + std::deque<T> queue_; + // In how may queues the element at index `i` is. An element can be removed + // from the front if and only if it is in 0 queues. + std::deque<size_t> in_queues_; + // Map from the reader to the head position in the queue. + std::unordered_map<size_t, size_t> heads_; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_MULTI_READER_QUEUE_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue_test.cc new file mode 100644 index 0000000000..ea6aa0a416 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue_test.cc @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/multi_reader_queue.h" + +#include "absl/types/optional.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +TEST(MultiReaderQueueTest, EmptyQueueEmptyForAllHeads) { + MultiReaderQueue<int> queue = MultiReaderQueue<int>(/*readers_count=*/10); + EXPECT_EQ(queue.size(), 0lu); + for (int i = 0; i < 10; ++i) { + EXPECT_TRUE(queue.IsEmpty(/*reader=*/i)); + EXPECT_EQ(queue.size(/*reader=*/i), 0lu); + EXPECT_FALSE(queue.PopFront(/*reader=*/i).has_value()); + EXPECT_FALSE(queue.Front(/*reader=*/i).has_value()); + } +} + +TEST(MultiReaderQueueTest, SizeIsEqualForAllHeadsAfterAddOnly) { + MultiReaderQueue<int> queue = MultiReaderQueue<int>(/*readers_count=*/10); + queue.PushBack(1); + queue.PushBack(2); + queue.PushBack(3); + EXPECT_EQ(queue.size(), 3lu); + for (int i = 0; i < 10; ++i) { + EXPECT_FALSE(queue.IsEmpty(/*reader=*/i)); + EXPECT_EQ(queue.size(/*reader=*/i), 3lu); + } +} + +TEST(MultiReaderQueueTest, SizeIsCorrectAfterRemoveFromOnlyOneHead) { + MultiReaderQueue<int> queue = MultiReaderQueue<int>(/*readers_count=*/10); + for (int i = 0; i < 5; ++i) { + queue.PushBack(i); + } + EXPECT_EQ(queue.size(), 5lu); + // Removing elements from queue #0 + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(queue.size(/*reader=*/0), static_cast<size_t>(5 - i)); + EXPECT_EQ(queue.PopFront(/*reader=*/0), absl::optional<int>(i)); + for (int j = 1; j < 10; ++j) { + EXPECT_EQ(queue.size(/*reader=*/j), 5lu); + } + } + EXPECT_EQ(queue.size(/*reader=*/0), 0lu); + EXPECT_TRUE(queue.IsEmpty(/*reader=*/0)); +} + +TEST(MultiReaderQueueTest, SingleHeadOneAddOneRemove) { + MultiReaderQueue<int> queue = MultiReaderQueue<int>(/*readers_count=*/1); + queue.PushBack(1); + EXPECT_EQ(queue.size(), 1lu); + EXPECT_TRUE(queue.Front(/*reader=*/0).has_value()); + EXPECT_EQ(queue.Front(/*reader=*/0).value(), 1); + absl::optional<int> value = queue.PopFront(/*reader=*/0); + EXPECT_TRUE(value.has_value()); + EXPECT_EQ(value.value(), 1); + EXPECT_EQ(queue.size(), 0lu); + EXPECT_TRUE(queue.IsEmpty(/*reader=*/0)); +} + +TEST(MultiReaderQueueTest, SingleHead) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/1); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + EXPECT_EQ(queue.size(), i + 1); + } + for (size_t i = 0; i < 10; ++i) { + EXPECT_EQ(queue.Front(/*reader=*/0), absl::optional<size_t>(i)); + EXPECT_EQ(queue.PopFront(/*reader=*/0), absl::optional<size_t>(i)); + EXPECT_EQ(queue.size(), 10 - i - 1); + } +} + +TEST(MultiReaderQueueTest, ThreeHeadsAddAllRemoveAllPerHead) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/3); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + EXPECT_EQ(queue.size(), i + 1); + } + for (size_t i = 0; i < 10; ++i) { + absl::optional<size_t> value = queue.PopFront(/*reader=*/0); + EXPECT_EQ(queue.size(), 10lu); + ASSERT_TRUE(value.has_value()); + EXPECT_EQ(value.value(), i); + } + for (size_t i = 0; i < 10; ++i) { + absl::optional<size_t> value = queue.PopFront(/*reader=*/1); + EXPECT_EQ(queue.size(), 10lu); + ASSERT_TRUE(value.has_value()); + EXPECT_EQ(value.value(), i); + } + for (size_t i = 0; i < 10; ++i) { + absl::optional<size_t> value = queue.PopFront(/*reader=*/2); + EXPECT_EQ(queue.size(), 10 - i - 1); + ASSERT_TRUE(value.has_value()); + EXPECT_EQ(value.value(), i); + } +} + +TEST(MultiReaderQueueTest, ThreeHeadsAddAllRemoveAll) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/3); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + EXPECT_EQ(queue.size(), i + 1); + } + for (size_t i = 0; i < 10; ++i) { + absl::optional<size_t> value1 = queue.PopFront(/*reader=*/0); + absl::optional<size_t> value2 = queue.PopFront(/*reader=*/1); + absl::optional<size_t> value3 = queue.PopFront(/*reader=*/2); + EXPECT_EQ(queue.size(), 10 - i - 1); + ASSERT_TRUE(value1.has_value()); + ASSERT_TRUE(value2.has_value()); + ASSERT_TRUE(value3.has_value()); + EXPECT_EQ(value1.value(), i); + EXPECT_EQ(value2.value(), i); + EXPECT_EQ(value3.value(), i); + } +} + +TEST(MultiReaderQueueTest, AddReaderSeeElementsOnlyFromReaderToCopy) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/2); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + } + for (size_t i = 0; i < 5; ++i) { + queue.PopFront(0); + } + + queue.AddReader(/*reader=*/2, /*reader_to_copy=*/0); + + EXPECT_EQ(queue.readers_count(), 3lu); + for (size_t i = 5; i < 10; ++i) { + absl::optional<size_t> value = queue.PopFront(/*reader=*/2); + EXPECT_EQ(queue.size(/*reader=*/2), 10 - i - 1); + ASSERT_TRUE(value.has_value()); + EXPECT_EQ(value.value(), i); + } +} + +TEST(MultiReaderQueueTest, AddReaderWithoutReaderToCopySeeFullQueue) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/2); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + } + for (size_t i = 0; i < 5; ++i) { + queue.PopFront(/*reader=*/0); + } + + queue.AddReader(/*reader=*/2); + + EXPECT_EQ(queue.readers_count(), 3lu); + for (size_t i = 0; i < 10; ++i) { + absl::optional<size_t> value = queue.PopFront(/*reader=*/2); + EXPECT_EQ(queue.size(/*reader=*/2), 10 - i - 1); + ASSERT_TRUE(value.has_value()); + EXPECT_EQ(value.value(), i); + } +} + +TEST(MultiReaderQueueTest, RemoveReaderWontChangeOthers) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/2); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + } + EXPECT_EQ(queue.size(/*reader=*/1), 10lu); + + queue.RemoveReader(0); + + EXPECT_EQ(queue.readers_count(), 1lu); + EXPECT_EQ(queue.size(/*reader=*/1), 10lu); +} + +TEST(MultiReaderQueueTest, RemoveLastReaderMakesQueueEmpty) { + MultiReaderQueue<size_t> queue = + MultiReaderQueue<size_t>(/*readers_count=*/1); + for (size_t i = 0; i < 10; ++i) { + queue.PushBack(i); + } + EXPECT_EQ(queue.size(), 10lu); + + queue.RemoveReader(0); + + EXPECT_EQ(queue.size(), 0lu); + EXPECT_EQ(queue.readers_count(), 0lu); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.cc new file mode 100644 index 0000000000..3ccab620f8 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.cc @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/names_collection.h" + +#include <set> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" + +namespace webrtc { + +NamesCollection::NamesCollection(rtc::ArrayView<const std::string> names) { + names_ = std::vector<std::string>(names.begin(), names.end()); + for (size_t i = 0; i < names_.size(); ++i) { + index_.emplace(names_[i], i); + removed_.emplace_back(false); + } + size_ = names_.size(); +} + +bool NamesCollection::HasName(absl::string_view name) const { + auto it = index_.find(name); + if (it == index_.end()) { + return false; + } + return !removed_[it->second]; +} + +size_t NamesCollection::AddIfAbsent(absl::string_view name) { + auto it = index_.find(name); + if (it != index_.end()) { + // Name was registered in the collection before: we need to restore it. + size_t index = it->second; + if (removed_[index]) { + removed_[index] = false; + size_++; + } + return index; + } + size_t out = names_.size(); + size_t old_capacity = names_.capacity(); + names_.emplace_back(name); + removed_.emplace_back(false); + size_++; + size_t new_capacity = names_.capacity(); + + if (old_capacity == new_capacity) { + index_.emplace(names_[out], out); + } else { + // Reallocation happened in the vector, so we need to rebuild `index_` to + // fix absl::string_view internal references. + index_.clear(); + for (size_t i = 0; i < names_.size(); ++i) { + index_.emplace(names_[i], i); + } + } + return out; +} + +absl::optional<size_t> NamesCollection::RemoveIfPresent( + absl::string_view name) { + auto it = index_.find(name); + if (it == index_.end()) { + return absl::nullopt; + } + size_t index = it->second; + if (removed_[index]) { + return absl::nullopt; + } + removed_[index] = true; + size_--; + return index; +} + +std::set<size_t> NamesCollection::GetPresentIndexes() const { + std::set<size_t> out; + for (size_t i = 0; i < removed_.size(); ++i) { + if (!removed_[i]) { + out.insert(i); + } + } + return out; +} + +std::set<size_t> NamesCollection::GetAllIndexes() const { + std::set<size_t> out; + for (size_t i = 0; i < names_.size(); ++i) { + out.insert(i); + } + return out; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.h new file mode 100644 index 0000000000..e29bad1ad4 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_NAMES_COLLECTION_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_NAMES_COLLECTION_H_ + +#include <map> +#include <set> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/array_view.h" + +namespace webrtc { + +// Contains mapping between string names and unique size_t values (indexes). +// Once the name is added to the collection it is guaranteed: +// 1. Name will have the same index until collection will be destructed +// 2. Adding, removing and re-adding name won't change its index +// +// The name is considered in the collection if it was added and wasn't removed. +// Adding the name when it is in the collection won't change the collection, the +// same as removing the name when it is removed. +// +// Collection will return name's index and name for the index independently from +// was name removed or not. Once the name was added to the collection the index +// will be allocated for it. To check if name is in collection right now user +// has to explicitly call to `HasName` function. +class NamesCollection { + public: + NamesCollection() = default; + + explicit NamesCollection(rtc::ArrayView<const std::string> names); + + // Returns amount of currently presented names in the collection. + size_t size() const { return size_; } + + // Returns index of the `name` which was known to the collection. Crashes + // if `name` was never registered in the collection. + size_t index(absl::string_view name) const { return index_.at(name); } + + // Returns name which was known to the collection for the specified `index`. + // Crashes if there was no any name registered in the collection for such + // `index`. + const std::string& name(size_t index) const { return names_.at(index); } + + // Returns if `name` is currently presented in this collection. + bool HasName(absl::string_view name) const; + + // Adds specified `name` to the collection if it isn't presented. + // Returns index which corresponds to specified `name`. + size_t AddIfAbsent(absl::string_view name); + + // Removes specified `name` from the collection if it is presented. + // + // After name was removed, collection size will be decreased, but `name` index + // will be preserved. Collection will return false for `HasName(name)`, but + // will continue to return previously known index for `index(name)` and return + // `name` for `name(index(name))`. + // + // Returns the index of the removed value or absl::nullopt if no such `name` + // registered in the collection. + absl::optional<size_t> RemoveIfPresent(absl::string_view name); + + // Returns a set of indexes for all currently present names in the + // collection. + std::set<size_t> GetPresentIndexes() const; + + // Returns a set of all indexes known to the collection including indexes for + // names that were removed. + std::set<size_t> GetAllIndexes() const; + + private: + std::vector<std::string> names_; + std::vector<bool> removed_; + std::map<absl::string_view, size_t> index_; + size_t size_ = 0; +}; + +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_NAMES_COLLECTION_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection_test.cc new file mode 100644 index 0000000000..65932ea26e --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection_test.cc @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/names_collection.h" + +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +using ::testing::Eq; +using ::testing::Ne; + +TEST(NamesCollectionTest, NamesFromCtorHasUniqueIndexes) { + NamesCollection collection(std::vector<std::string>{"alice", "bob"}); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(2))); + EXPECT_TRUE(collection.HasName("alice")); + EXPECT_THAT(collection.name(collection.index("alice")), Eq("alice")); + + EXPECT_TRUE(collection.HasName("bob")); + EXPECT_THAT(collection.name(collection.index("bob")), Eq("bob")); + + EXPECT_THAT(collection.index("bob"), Ne(collection.index("alice"))); +} + +TEST(NamesCollectionTest, AddedNamesHasIndexes) { + NamesCollection collection(std::vector<std::string>{}); + collection.AddIfAbsent("alice"); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + EXPECT_TRUE(collection.HasName("alice")); + EXPECT_THAT(collection.name(collection.index("alice")), Eq("alice")); +} + +TEST(NamesCollectionTest, AddBobDoesNotChangeAliceIndex) { + NamesCollection collection(std::vector<std::string>{"alice"}); + + size_t alice_index = collection.index("alice"); + + collection.AddIfAbsent("bob"); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(2))); + EXPECT_THAT(collection.index("alice"), Eq(alice_index)); + EXPECT_THAT(collection.index("bob"), Ne(alice_index)); +} + +TEST(NamesCollectionTest, AddAliceSecondTimeDoesNotChangeIndex) { + NamesCollection collection(std::vector<std::string>{"alice"}); + + size_t alice_index = collection.index("alice"); + + EXPECT_THAT(collection.AddIfAbsent("alice"), Eq(alice_index)); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + EXPECT_THAT(collection.index("alice"), Eq(alice_index)); +} + +TEST(NamesCollectionTest, RemoveRemovesFromCollectionButNotIndex) { + NamesCollection collection(std::vector<std::string>{"alice", "bob"}); + + size_t bob_index = collection.index("bob"); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(2))); + + EXPECT_THAT(collection.RemoveIfPresent("bob"), + Eq(absl::optional<size_t>(bob_index))); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + EXPECT_FALSE(collection.HasName("bob")); + + EXPECT_THAT(collection.index("bob"), Eq(bob_index)); + EXPECT_THAT(collection.name(bob_index), Eq("bob")); +} + +TEST(NamesCollectionTest, RemoveOfAliceDoesNotChangeBobIndex) { + NamesCollection collection(std::vector<std::string>{"alice", "bob"}); + + size_t alice_index = collection.index("alice"); + size_t bob_index = collection.index("bob"); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(2))); + + EXPECT_THAT(collection.RemoveIfPresent("alice"), + Eq(absl::optional<size_t>(alice_index))); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + EXPECT_THAT(collection.index("bob"), Eq(bob_index)); + EXPECT_THAT(collection.name(bob_index), Eq("bob")); +} + +TEST(NamesCollectionTest, RemoveSecondTimeHasNoEffect) { + NamesCollection collection(std::vector<std::string>{"bob"}); + + size_t bob_index = collection.index("bob"); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + EXPECT_THAT(collection.RemoveIfPresent("bob"), + Eq(absl::optional<size_t>(bob_index))); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(0))); + EXPECT_THAT(collection.RemoveIfPresent("bob"), Eq(absl::nullopt)); +} + +TEST(NamesCollectionTest, RemoveOfNotExistingHasNoEffect) { + NamesCollection collection(std::vector<std::string>{"bob"}); + + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + EXPECT_THAT(collection.RemoveIfPresent("alice"), Eq(absl::nullopt)); + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); +} + +TEST(NamesCollectionTest, AddRemoveAddPreserveTheIndex) { + NamesCollection collection(std::vector<std::string>{}); + + size_t alice_index = collection.AddIfAbsent("alice"); + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); + + EXPECT_THAT(collection.RemoveIfPresent("alice"), + Eq(absl::optional<size_t>(alice_index))); + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(0))); + + EXPECT_THAT(collection.AddIfAbsent("alice"), Eq(alice_index)); + EXPECT_THAT(collection.index("alice"), Eq(alice_index)); + EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1))); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc new file mode 100644 index 0000000000..41b8aec8a1 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc @@ -0,0 +1,270 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h" + +#include <cstdint> +#include <cstring> +#include <memory> +#include <utility> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" +#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +QualityAnalyzingVideoDecoder::QualityAnalyzingVideoDecoder( + absl::string_view peer_name, + std::unique_ptr<VideoDecoder> delegate, + EncodedImageDataExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer) + : peer_name_(peer_name), + implementation_name_("AnalyzingDecoder-" + + std::string(delegate->ImplementationName())), + delegate_(std::move(delegate)), + extractor_(extractor), + analyzer_(analyzer) { + analyzing_callback_ = std::make_unique<DecoderCallback>(this); +} +QualityAnalyzingVideoDecoder::~QualityAnalyzingVideoDecoder() = default; + +bool QualityAnalyzingVideoDecoder::Configure(const Settings& settings) { + { + MutexLock lock(&mutex_); + codec_name_ = std::string(CodecTypeToPayloadString(settings.codec_type())) + + "_" + delegate_->GetDecoderInfo().implementation_name; + } + return delegate_->Configure(settings); +} + +int32_t QualityAnalyzingVideoDecoder::Decode(const EncodedImage& input_image, + bool missing_frames, + int64_t render_time_ms) { + // Image extractor extracts id from provided EncodedImage and also returns + // the image with the original buffer. Buffer can be modified in place, so + // owner of original buffer will be responsible for deleting it, or extractor + // can create a new buffer. In such case extractor will be responsible for + // deleting it. + EncodedImageExtractionResult out = extractor_->ExtractData(input_image); + + if (out.discard) { + // To partly emulate behavior of Selective Forwarding Unit (SFU) in the + // test, on receiver side we will "discard" frames from irrelevant streams. + // When all encoded images were marked to discarded, black frame have to be + // returned. Because simulcast streams will be received by receiver as 3 + // different independent streams we don't want that irrelevant streams + // affect video quality metrics and also we don't want to use CPU time to + // decode them to prevent regressions on relevant streams. Also we can't + // just drop frame, because in such case, receiving part will be confused + // with all frames missing and will request a key frame, which will result + // into extra load on network and sender side. Because of it, discarded + // image will be always decoded as black frame and will be passed to + // callback directly without reaching decoder and video quality analyzer. + // + // For more details see QualityAnalyzingVideoEncoder. + return analyzing_callback_->IrrelevantSimulcastStreamDecoded( + out.id.value_or(VideoFrame::kNotSetId), input_image.Timestamp()); + } + + EncodedImage* origin_image; + { + MutexLock lock(&mutex_); + // Store id to be able to retrieve it in analyzing callback. + timestamp_to_frame_id_.insert({input_image.Timestamp(), out.id}); + // Store encoded image to prevent its destruction while it is used in + // decoder. + origin_image = &( + decoding_images_.insert({input_image.Timestamp(), std::move(out.image)}) + .first->second); + } + // We can safely dereference `origin_image`, because it can be removed from + // the map only after `delegate_` Decode method will be invoked. Image will + // be removed inside DecodedImageCallback, which can be done on separate + // thread. + analyzer_->OnFramePreDecode( + peer_name_, out.id.value_or(VideoFrame::kNotSetId), *origin_image); + int32_t result = + delegate_->Decode(*origin_image, missing_frames, render_time_ms); + if (result != WEBRTC_VIDEO_CODEC_OK) { + // If delegate decoder failed, then cleanup data for this image. + { + MutexLock lock(&mutex_); + timestamp_to_frame_id_.erase(input_image.Timestamp()); + decoding_images_.erase(input_image.Timestamp()); + } + analyzer_->OnDecoderError(peer_name_, + out.id.value_or(VideoFrame::kNotSetId), result); + } + return result; +} + +int32_t QualityAnalyzingVideoDecoder::RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) { + analyzing_callback_->SetDelegateCallback(callback); + return delegate_->RegisterDecodeCompleteCallback(analyzing_callback_.get()); +} + +int32_t QualityAnalyzingVideoDecoder::Release() { + // Release decoder first. During release process it can still decode some + // frames, so we don't take a lock to prevent deadlock. + int32_t result = delegate_->Release(); + + MutexLock lock(&mutex_); + analyzing_callback_->SetDelegateCallback(nullptr); + timestamp_to_frame_id_.clear(); + decoding_images_.clear(); + return result; +} + +VideoDecoder::DecoderInfo QualityAnalyzingVideoDecoder::GetDecoderInfo() const { + DecoderInfo info = delegate_->GetDecoderInfo(); + info.implementation_name = implementation_name_; + return info; +} + +const char* QualityAnalyzingVideoDecoder::ImplementationName() const { + return implementation_name_.c_str(); +} + +QualityAnalyzingVideoDecoder::DecoderCallback::DecoderCallback( + QualityAnalyzingVideoDecoder* decoder) + : decoder_(decoder), delegate_callback_(nullptr) {} +QualityAnalyzingVideoDecoder::DecoderCallback::~DecoderCallback() = default; + +void QualityAnalyzingVideoDecoder::DecoderCallback::SetDelegateCallback( + DecodedImageCallback* delegate) { + MutexLock lock(&callback_mutex_); + delegate_callback_ = delegate; +} + +// We have to implement all next 3 methods because we don't know which one +// exactly is implemented in `delegate_callback_`, so we need to call the same +// method on `delegate_callback_`, as was called on `this` callback. +int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( + VideoFrame& decodedImage) { + decoder_->OnFrameDecoded(&decodedImage, /*decode_time_ms=*/absl::nullopt, + /*qp=*/absl::nullopt); + + MutexLock lock(&callback_mutex_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->Decoded(decodedImage); +} + +int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( + VideoFrame& decodedImage, + int64_t decode_time_ms) { + decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, /*qp=*/absl::nullopt); + + MutexLock lock(&callback_mutex_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->Decoded(decodedImage, decode_time_ms); +} + +void QualityAnalyzingVideoDecoder::DecoderCallback::Decoded( + VideoFrame& decodedImage, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp) { + decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, qp); + + MutexLock lock(&callback_mutex_); + RTC_DCHECK(delegate_callback_); + delegate_callback_->Decoded(decodedImage, decode_time_ms, qp); +} + +int32_t +QualityAnalyzingVideoDecoder::DecoderCallback::IrrelevantSimulcastStreamDecoded( + uint16_t frame_id, + uint32_t timestamp_ms) { + webrtc::VideoFrame dummy_frame = + webrtc::VideoFrame::Builder() + .set_video_frame_buffer(GetDummyFrameBuffer()) + .set_timestamp_rtp(timestamp_ms) + .set_id(frame_id) + .build(); + MutexLock lock(&callback_mutex_); + RTC_DCHECK(delegate_callback_); + delegate_callback_->Decoded(dummy_frame, absl::nullopt, absl::nullopt); + return WEBRTC_VIDEO_CODEC_OK; +} + +rtc::scoped_refptr<webrtc::VideoFrameBuffer> +QualityAnalyzingVideoDecoder::DecoderCallback::GetDummyFrameBuffer() { + if (!dummy_frame_buffer_) { + dummy_frame_buffer_ = CreateDummyFrameBuffer(); + } + + return dummy_frame_buffer_; +} + +void QualityAnalyzingVideoDecoder::OnFrameDecoded( + VideoFrame* frame, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp) { + absl::optional<uint16_t> frame_id; + std::string codec_name; + { + MutexLock lock(&mutex_); + auto it = timestamp_to_frame_id_.find(frame->timestamp()); + if (it == timestamp_to_frame_id_.end()) { + // Ensure, that we have info about this frame. It can happen that for some + // reasons decoder response, that it failed to decode, when we were + // posting frame to it, but then call the callback for this frame. + RTC_LOG(LS_ERROR) << "QualityAnalyzingVideoDecoder::OnFrameDecoded: No " + "frame id for frame for frame->timestamp()=" + << frame->timestamp(); + return; + } + frame_id = it->second; + timestamp_to_frame_id_.erase(it); + decoding_images_.erase(frame->timestamp()); + codec_name = codec_name_; + } + // Set frame id to the value, that was extracted from corresponding encoded + // image. + frame->set_id(frame_id.value_or(VideoFrame::kNotSetId)); + VideoQualityAnalyzerInterface::DecoderStats stats; + stats.decoder_name = codec_name; + stats.decode_time_ms = decode_time_ms; + analyzer_->OnFrameDecoded(peer_name_, *frame, stats); +} + +QualityAnalyzingVideoDecoderFactory::QualityAnalyzingVideoDecoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoDecoderFactory> delegate, + EncodedImageDataExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer) + : peer_name_(peer_name), + delegate_(std::move(delegate)), + extractor_(extractor), + analyzer_(analyzer) {} +QualityAnalyzingVideoDecoderFactory::~QualityAnalyzingVideoDecoderFactory() = + default; + +std::vector<SdpVideoFormat> +QualityAnalyzingVideoDecoderFactory::GetSupportedFormats() const { + return delegate_->GetSupportedFormats(); +} + +std::unique_ptr<VideoDecoder> +QualityAnalyzingVideoDecoderFactory::CreateVideoDecoder( + const SdpVideoFormat& format) { + std::unique_ptr<VideoDecoder> decoder = delegate_->CreateVideoDecoder(format); + return std::make_unique<QualityAnalyzingVideoDecoder>( + peer_name_, std::move(decoder), extractor_, analyzer_); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h new file mode 100644 index 0000000000..a86f4196b0 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ + +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "rtc_base/synchronization/mutex.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// QualityAnalyzingVideoDecoder is used to wrap origin video decoder and inject +// VideoQualityAnalyzerInterface before and after decoder. +// +// QualityAnalyzingVideoDecoder propagates all calls to the origin decoder. +// It registers its own DecodedImageCallback in the origin decoder and will +// store user specified callback inside itself. +// +// When Decode(...) will be invoked, quality decoder first will extract frame id +// from passed EncodedImage with EncodedImageIdExtracor that was specified in +// constructor, then will call video quality analyzer, with correct +// EncodedImage and only then will pass image to origin decoder. +// +// When origin decoder decodes the image it will call quality decoder's special +// callback, where video analyzer will be called again and then decoded frame +// will be passed to origin callback, provided by user. +// +// Quality decoder registers its own callback in origin decoder, at the same +// time the user registers their callback in quality decoder. +class QualityAnalyzingVideoDecoder : public VideoDecoder { + public: + QualityAnalyzingVideoDecoder(absl::string_view peer_name, + std::unique_ptr<VideoDecoder> delegate, + EncodedImageDataExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoDecoder() override; + + // Methods of VideoDecoder interface. + bool Configure(const Settings& settings) override; + int32_t Decode(const EncodedImage& input_image, + bool missing_frames, + int64_t render_time_ms) override; + int32_t RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) override; + int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; + const char* ImplementationName() const override; + + private: + class DecoderCallback : public DecodedImageCallback { + public: + explicit DecoderCallback(QualityAnalyzingVideoDecoder* decoder); + ~DecoderCallback() override; + + void SetDelegateCallback(DecodedImageCallback* delegate); + + // Methods of DecodedImageCallback interface. + int32_t Decoded(VideoFrame& decodedImage) override; + int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) override; + void Decoded(VideoFrame& decodedImage, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp) override; + + int32_t IrrelevantSimulcastStreamDecoded(uint16_t frame_id, + uint32_t timestamp_ms); + + private: + rtc::scoped_refptr<webrtc::VideoFrameBuffer> GetDummyFrameBuffer(); + + QualityAnalyzingVideoDecoder* const decoder_; + + rtc::scoped_refptr<webrtc::VideoFrameBuffer> dummy_frame_buffer_; + + Mutex callback_mutex_; + DecodedImageCallback* delegate_callback_ RTC_GUARDED_BY(callback_mutex_); + }; + + void OnFrameDecoded(VideoFrame* frame, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp); + + const std::string peer_name_; + const std::string implementation_name_; + std::unique_ptr<VideoDecoder> delegate_; + EncodedImageDataExtractor* const extractor_; + VideoQualityAnalyzerInterface* const analyzer_; + std::unique_ptr<DecoderCallback> analyzing_callback_; + + // VideoDecoder interface assumes async delivery of decoded video frames. + // This lock is used to protect shared state, that have to be propagated + // from received EncodedImage to resulted VideoFrame. + Mutex mutex_; + + // Name of the video codec type used. Ex: VP8, VP9, H264 etc. + std::string codec_name_ RTC_GUARDED_BY(mutex_); + std::map<uint32_t, absl::optional<uint16_t>> timestamp_to_frame_id_ + RTC_GUARDED_BY(mutex_); + // Stores currently being decoded images by timestamp. Because + // EncodedImageDataExtractor can create new copy on EncodedImage we need to + // ensure, that this image won't be deleted during async decoding. To do it + // all images are putted into this map and removed from here inside callback. + std::map<uint32_t, EncodedImage> decoding_images_ RTC_GUARDED_BY(mutex_); +}; + +// Produces QualityAnalyzingVideoDecoder, which hold decoders, produced by +// specified factory as delegates. Forwards all other calls to specified +// factory. +class QualityAnalyzingVideoDecoderFactory : public VideoDecoderFactory { + public: + QualityAnalyzingVideoDecoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoDecoderFactory> delegate, + EncodedImageDataExtractor* extractor, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoDecoderFactory() override; + + // Methods of VideoDecoderFactory interface. + std::vector<SdpVideoFormat> GetSupportedFormats() const override; + std::unique_ptr<VideoDecoder> CreateVideoDecoder( + const SdpVideoFormat& format) override; + + private: + const std::string peer_name_; + std::unique_ptr<VideoDecoderFactory> delegate_; + EncodedImageDataExtractor* const extractor_; + VideoQualityAnalyzerInterface* const analyzer_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc new file mode 100644 index 0000000000..d3fbab08a0 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc @@ -0,0 +1,384 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h" + +#include <cmath> +#include <memory> +#include <utility> + +#include "absl/strings/string_view.h" +#include "api/video/video_codec_type.h" +#include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +constexpr size_t kMaxFrameInPipelineCount = 1000; +constexpr double kNoMultiplier = 1.0; +constexpr double kEps = 1e-6; + +std::pair<uint32_t, uint32_t> GetMinMaxBitratesBps(const VideoCodec& codec, + size_t spatial_idx) { + uint32_t min_bitrate = codec.minBitrate; + uint32_t max_bitrate = codec.maxBitrate; + if (spatial_idx < codec.numberOfSimulcastStreams && + codec.codecType != VideoCodecType::kVideoCodecVP9) { + min_bitrate = + std::max(min_bitrate, codec.simulcastStream[spatial_idx].minBitrate); + max_bitrate = + std::min(max_bitrate, codec.simulcastStream[spatial_idx].maxBitrate); + } + if (codec.codecType == VideoCodecType::kVideoCodecVP9 && + spatial_idx < codec.VP9().numberOfSpatialLayers) { + min_bitrate = + std::max(min_bitrate, codec.spatialLayers[spatial_idx].minBitrate); + max_bitrate = + std::min(max_bitrate, codec.spatialLayers[spatial_idx].maxBitrate); + } + RTC_DCHECK_GT(max_bitrate, min_bitrate); + return {min_bitrate * 1000, max_bitrate * 1000}; +} + +} // namespace + +QualityAnalyzingVideoEncoder::QualityAnalyzingVideoEncoder( + absl::string_view peer_name, + std::unique_ptr<VideoEncoder> delegate, + double bitrate_multiplier, + std::map<std::string, absl::optional<int>> stream_required_spatial_index, + EncodedImageDataInjector* injector, + VideoQualityAnalyzerInterface* analyzer) + : peer_name_(peer_name), + delegate_(std::move(delegate)), + bitrate_multiplier_(bitrate_multiplier), + stream_required_spatial_index_(std::move(stream_required_spatial_index)), + injector_(injector), + analyzer_(analyzer), + mode_(SimulcastMode::kNormal), + delegate_callback_(nullptr) {} +QualityAnalyzingVideoEncoder::~QualityAnalyzingVideoEncoder() = default; + +void QualityAnalyzingVideoEncoder::SetFecControllerOverride( + FecControllerOverride* fec_controller_override) { + // Ignored. +} + +int32_t QualityAnalyzingVideoEncoder::InitEncode( + const VideoCodec* codec_settings, + const Settings& settings) { + MutexLock lock(&mutex_); + codec_settings_ = *codec_settings; + mode_ = SimulcastMode::kNormal; + if (codec_settings->codecType == kVideoCodecVP9) { + if (codec_settings->VP9().numberOfSpatialLayers > 1) { + switch (codec_settings->VP9().interLayerPred) { + case InterLayerPredMode::kOn: + mode_ = SimulcastMode::kSVC; + break; + case InterLayerPredMode::kOnKeyPic: + mode_ = SimulcastMode::kKSVC; + break; + case InterLayerPredMode::kOff: + mode_ = SimulcastMode::kSimulcast; + break; + default: + RTC_DCHECK_NOTREACHED() + << "Unknown codec_settings->VP9().interLayerPred"; + break; + } + } + } + if (codec_settings->numberOfSimulcastStreams > 1) { + mode_ = SimulcastMode::kSimulcast; + } + return delegate_->InitEncode(codec_settings, settings); +} + +int32_t QualityAnalyzingVideoEncoder::RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) { + // We need to get a lock here because delegate_callback can be hypothetically + // accessed from different thread (encoder one) concurrently. + MutexLock lock(&mutex_); + delegate_callback_ = callback; + return delegate_->RegisterEncodeCompleteCallback(this); +} + +int32_t QualityAnalyzingVideoEncoder::Release() { + // Release encoder first. During release process it can still encode some + // frames, so we don't take a lock to prevent deadlock. + int32_t result = delegate_->Release(); + + MutexLock lock(&mutex_); + delegate_callback_ = nullptr; + return result; +} + +int32_t QualityAnalyzingVideoEncoder::Encode( + const VideoFrame& frame, + const std::vector<VideoFrameType>* frame_types) { + { + MutexLock lock(&mutex_); + // Store id to be able to retrieve it in analyzing callback. + timestamp_to_frame_id_list_.push_back({frame.timestamp(), frame.id()}); + // If this list is growing, it means that we are not receiving new encoded + // images from encoder. So it should be a bug in setup on in the encoder. + RTC_DCHECK_LT(timestamp_to_frame_id_list_.size(), kMaxFrameInPipelineCount); + } + analyzer_->OnFramePreEncode(peer_name_, frame); + int32_t result = delegate_->Encode(frame, frame_types); + if (result != WEBRTC_VIDEO_CODEC_OK) { + // If origin encoder failed, then cleanup data for this frame. + { + MutexLock lock(&mutex_); + // The timestamp-frame_id pair can be not the last one, so we need to + // find it first and then remove. We will search from the end, because + // usually it will be the last or close to the last one. + auto it = timestamp_to_frame_id_list_.end(); + while (it != timestamp_to_frame_id_list_.begin()) { + --it; + if (it->first == frame.timestamp()) { + timestamp_to_frame_id_list_.erase(it); + break; + } + } + } + analyzer_->OnEncoderError(peer_name_, frame, result); + } + return result; +} + +void QualityAnalyzingVideoEncoder::SetRates( + const VideoEncoder::RateControlParameters& parameters) { + RTC_DCHECK_GT(bitrate_multiplier_, 0.0); + if (fabs(bitrate_multiplier_ - kNoMultiplier) < kEps) { + { + MutexLock lock(&mutex_); + bitrate_allocation_ = parameters.bitrate; + } + return delegate_->SetRates(parameters); + } + + RateControlParameters adjusted_params = parameters; + { + MutexLock lock(&mutex_); + // Simulating encoder overshooting target bitrate, by configuring actual + // encoder too high. Take care not to adjust past limits of config, + // otherwise encoders may crash on DCHECK. + VideoBitrateAllocation multiplied_allocation; + for (size_t si = 0; si < kMaxSpatialLayers; ++si) { + const uint32_t spatial_layer_bitrate_bps = + parameters.bitrate.GetSpatialLayerSum(si); + if (spatial_layer_bitrate_bps == 0) { + continue; + } + + uint32_t min_bitrate_bps; + uint32_t max_bitrate_bps; + std::tie(min_bitrate_bps, max_bitrate_bps) = + GetMinMaxBitratesBps(codec_settings_, si); + double bitrate_multiplier = bitrate_multiplier_; + const uint32_t corrected_bitrate = rtc::checked_cast<uint32_t>( + bitrate_multiplier * spatial_layer_bitrate_bps); + if (corrected_bitrate < min_bitrate_bps) { + bitrate_multiplier = min_bitrate_bps / spatial_layer_bitrate_bps; + } else if (corrected_bitrate > max_bitrate_bps) { + bitrate_multiplier = max_bitrate_bps / spatial_layer_bitrate_bps; + } + + for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) { + if (parameters.bitrate.HasBitrate(si, ti)) { + multiplied_allocation.SetBitrate( + si, ti, + rtc::checked_cast<uint32_t>( + bitrate_multiplier * parameters.bitrate.GetBitrate(si, ti))); + } + } + } + + adjusted_params.bitrate = multiplied_allocation; + bitrate_allocation_ = adjusted_params.bitrate; + } + return delegate_->SetRates(adjusted_params); +} + +VideoEncoder::EncoderInfo QualityAnalyzingVideoEncoder::GetEncoderInfo() const { + return delegate_->GetEncoderInfo(); +} + +// It is assumed, that encoded callback will be always invoked with encoded +// images that correspond to the frames in the same sequence, that frames +// arrived. In other words, assume we have frames F1, F2 and F3 and they have +// corresponding encoded images I1, I2 and I3. In such case if we will call +// encode first with F1, then with F2 and then with F3, then encoder callback +// will be called first with all spatial layers for F1 (I1), then F2 (I2) and +// then F3 (I3). +// +// Basing on it we will use a list of timestamp-frame_id pairs like this: +// 1. If current encoded image timestamp is equals to timestamp in the front +// pair - pick frame id from that pair +// 2. If current encoded image timestamp isn't equals to timestamp in the front +// pair - remove the front pair and got to the step 1. +EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info) { + uint16_t frame_id; + bool discard = false; + uint32_t target_encode_bitrate = 0; + std::string codec_name; + { + MutexLock lock(&mutex_); + std::pair<uint32_t, uint16_t> timestamp_frame_id; + while (!timestamp_to_frame_id_list_.empty()) { + timestamp_frame_id = timestamp_to_frame_id_list_.front(); + if (timestamp_frame_id.first == encoded_image.Timestamp()) { + break; + } + timestamp_to_frame_id_list_.pop_front(); + } + + // After the loop the first element should point to current `encoded_image` + // frame id. We don't remove it from the list, because there may be + // multiple spatial layers for this frame, so encoder can produce more + // encoded images with this timestamp. The first element will be removed + // when the next frame would be encoded and EncodedImageCallback would be + // called with the next timestamp. + + if (timestamp_to_frame_id_list_.empty()) { + // Ensure, that we have info about this frame. It can happen that for some + // reasons encoder response, that he failed to decode, when we were + // posting frame to it, but then call the callback for this frame. + RTC_LOG(LS_ERROR) << "QualityAnalyzingVideoEncoder::OnEncodedImage: No " + "frame id for encoded_image.Timestamp()=" + << encoded_image.Timestamp(); + return EncodedImageCallback::Result( + EncodedImageCallback::Result::Error::OK); + } + frame_id = timestamp_frame_id.second; + + discard = ShouldDiscard(frame_id, encoded_image); + if (!discard) { + target_encode_bitrate = bitrate_allocation_.GetSpatialLayerSum( + encoded_image.SpatialIndex().value_or(0)); + } + codec_name = + std::string(CodecTypeToPayloadString(codec_settings_.codecType)) + "_" + + delegate_->GetEncoderInfo().implementation_name; + } + + if (!discard) { + // Analyzer should see only encoded images, that weren't discarded. But all + // not discarded layers have to be passed. + VideoQualityAnalyzerInterface::EncoderStats stats; + stats.encoder_name = codec_name; + stats.target_encode_bitrate = target_encode_bitrate; + analyzer_->OnFrameEncoded(peer_name_, frame_id, encoded_image, stats); + } + + // Image data injector injects frame id and discard flag into provided + // EncodedImage and returns the image with a) modified original buffer (in + // such case the current owner of the buffer will be responsible for deleting + // it) or b) a new buffer (in such case injector will be responsible for + // deleting it). + const EncodedImage& image = + injector_->InjectData(frame_id, discard, encoded_image); + { + MutexLock lock(&mutex_); + RTC_DCHECK(delegate_callback_); + return delegate_callback_->OnEncodedImage(image, codec_specific_info); + } +} + +void QualityAnalyzingVideoEncoder::OnDroppedFrame( + EncodedImageCallback::DropReason reason) { + MutexLock lock(&mutex_); + analyzer_->OnFrameDropped(peer_name_, reason); + RTC_DCHECK(delegate_callback_); + delegate_callback_->OnDroppedFrame(reason); +} + +bool QualityAnalyzingVideoEncoder::ShouldDiscard( + uint16_t frame_id, + const EncodedImage& encoded_image) { + std::string stream_label = analyzer_->GetStreamLabel(frame_id); + absl::optional<int> required_spatial_index = + stream_required_spatial_index_[stream_label]; + if (required_spatial_index) { + if (*required_spatial_index == kAnalyzeAnySpatialStream) { + return false; + } + absl::optional<int> cur_spatial_index = encoded_image.SpatialIndex(); + if (!cur_spatial_index) { + cur_spatial_index = 0; + } + RTC_CHECK(mode_ != SimulcastMode::kNormal) + << "Analyzing encoder is in kNormal " + "mode, but spatial layer/simulcast " + "stream met."; + if (mode_ == SimulcastMode::kSimulcast) { + // In simulcast mode only encoded images with required spatial index are + // interested, so all others have to be discarded. + return *cur_spatial_index != *required_spatial_index; + } else if (mode_ == SimulcastMode::kSVC) { + // In SVC mode encoded images with spatial indexes that are equal or + // less than required one are interesting, so all above have to be + // discarded. + return *cur_spatial_index > *required_spatial_index; + } else if (mode_ == SimulcastMode::kKSVC) { + // In KSVC mode for key frame encoded images with spatial indexes that + // are equal or less than required one are interesting, so all above + // have to be discarded. For other frames only required spatial index + // is interesting, so all others have to be discarded. + if (encoded_image._frameType == VideoFrameType::kVideoFrameKey) { + return *cur_spatial_index > *required_spatial_index; + } else { + return *cur_spatial_index != *required_spatial_index; + } + } else { + RTC_DCHECK_NOTREACHED() << "Unsupported encoder mode"; + } + } + return false; +} + +QualityAnalyzingVideoEncoderFactory::QualityAnalyzingVideoEncoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoEncoderFactory> delegate, + double bitrate_multiplier, + std::map<std::string, absl::optional<int>> stream_required_spatial_index, + EncodedImageDataInjector* injector, + VideoQualityAnalyzerInterface* analyzer) + : peer_name_(peer_name), + delegate_(std::move(delegate)), + bitrate_multiplier_(bitrate_multiplier), + stream_required_spatial_index_(std::move(stream_required_spatial_index)), + injector_(injector), + analyzer_(analyzer) {} +QualityAnalyzingVideoEncoderFactory::~QualityAnalyzingVideoEncoderFactory() = + default; + +std::vector<SdpVideoFormat> +QualityAnalyzingVideoEncoderFactory::GetSupportedFormats() const { + return delegate_->GetSupportedFormats(); +} + +std::unique_ptr<VideoEncoder> +QualityAnalyzingVideoEncoderFactory::CreateVideoEncoder( + const SdpVideoFormat& format) { + return std::make_unique<QualityAnalyzingVideoEncoder>( + peer_name_, delegate_->CreateVideoEncoder(format), bitrate_multiplier_, + stream_required_spatial_index_, injector_, analyzer_); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h new file mode 100644 index 0000000000..135c85c133 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ + +#include <list> +#include <memory> +#include <utility> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "rtc_base/synchronization/mutex.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Tells QualityAnalyzingVideoEncoder that it shouldn't mark any spatial stream +// as to be discarded. In such case the top stream will be passed to +// VideoQualityAnalyzerInterface as a reference. +constexpr int kAnalyzeAnySpatialStream = -1; + +// QualityAnalyzingVideoEncoder is used to wrap origin video encoder and inject +// VideoQualityAnalyzerInterface before and after encoder. +// +// QualityAnalyzingVideoEncoder propagates all calls to the origin encoder. +// It registers its own EncodedImageCallback in the origin encoder and will +// store user specified callback inside itself. +// +// When Encode(...) will be invoked, quality encoder first calls video quality +// analyzer with original frame, then encodes frame with original encoder. +// +// When origin encoder encodes the image it will call quality encoder's special +// callback, where video analyzer will be called again and then frame id will be +// injected into EncodedImage with passed EncodedImageDataInjector. Then new +// EncodedImage will be passed to origin callback, provided by user. +// +// Quality encoder registers its own callback in origin encoder, at the same +// time the user registers their callback in quality encoder. +class QualityAnalyzingVideoEncoder : public VideoEncoder, + public EncodedImageCallback { + public: + QualityAnalyzingVideoEncoder( + absl::string_view peer_name, + std::unique_ptr<VideoEncoder> delegate, + double bitrate_multiplier, + std::map<std::string, absl::optional<int>> stream_required_spatial_index, + EncodedImageDataInjector* injector, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoEncoder() override; + + // Methods of VideoEncoder interface. + void SetFecControllerOverride( + FecControllerOverride* fec_controller_override) override; + int32_t InitEncode(const VideoCodec* codec_settings, + const Settings& settings) override; + int32_t RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) override; + int32_t Release() override; + int32_t Encode(const VideoFrame& frame, + const std::vector<VideoFrameType>* frame_types) override; + void SetRates(const VideoEncoder::RateControlParameters& parameters) override; + EncoderInfo GetEncoderInfo() const override; + + // Methods of EncodedImageCallback interface. + EncodedImageCallback::Result OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info) override; + void OnDroppedFrame(DropReason reason) override; + + private: + enum SimulcastMode { + // In this mode encoder assumes not more than 1 encoded image per video + // frame + kNormal, + + // Next modes are to test video conference behavior. For conference sender + // will send multiple spatial layers/simulcast streams for single video + // track and there is some Selective Forwarding Unit (SFU), that forwards + // only best one, that will pass through downlink to the receiver. + // + // Here this behavior will be partly emulated. Sender will send all spatial + // layers/simulcast streams and then some of them will be filtered out on + // the receiver side. During test setup user can specify which spatial + // layer/simulcast stream is required, what will simulated which spatial + // layer/simulcast stream will be chosen by SFU in the real world. Then + // sender will mark encoded images for all spatial layers above required or + // all simulcast streams except required as to be discarded and on receiver + // side they will be discarded in quality analyzing decoder and won't be + // passed into delegate decoder. + // + // If the sender for some reasons won't send specified spatial layer, then + // receiver still will fall back on lower spatial layers. But for simulcast + // streams if required one won't be sent, receiver will assume all frames + // in that period as dropped and will experience video freeze. + // + // Test based on this simulation will be used to evaluate video quality + // of concrete spatial layers/simulcast streams and also check distribution + // of bandwidth between spatial layers/simulcast streams by BWE. + + // In this mode encoder assumes that for each frame simulcast encoded + // images will be produced. So all simulcast streams except required will + // be marked as to be discarded in decoder and won't reach video quality + // analyzer. + kSimulcast, + // In this mode encoder assumes that for each frame encoded images for + // different spatial layers will be produced. So all spatial layers above + // required will be marked to be discarded in decoder and won't reach + // video quality analyzer. + kSVC, + // In this mode encoder assumes that for each frame encoded images for + // different spatial layers will be produced. Compared to kSVC mode + // spatial layers that are above required will be marked to be discarded + // only for key frames and for regular frames all except required spatial + // layer will be marked as to be discarded in decoder and won't reach video + // quality analyzer. + kKSVC + }; + + bool ShouldDiscard(uint16_t frame_id, const EncodedImage& encoded_image) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + const std::string peer_name_; + std::unique_ptr<VideoEncoder> delegate_; + const double bitrate_multiplier_; + // Contains mapping from stream label to optional spatial index. + // If we have stream label "Foo" and mapping contains + // 1. `absl::nullopt` means "Foo" isn't simulcast/SVC stream + // 2. `kAnalyzeAnySpatialStream` means all simulcast/SVC streams are required + // 3. Concrete value means that particular simulcast/SVC stream have to be + // analyzed. + std::map<std::string, absl::optional<int>> stream_required_spatial_index_; + EncodedImageDataInjector* const injector_; + VideoQualityAnalyzerInterface* const analyzer_; + + // VideoEncoder interface assumes async delivery of encoded images. + // This lock is used to protect shared state, that have to be propagated + // from received VideoFrame to resulted EncodedImage. + Mutex mutex_; + + VideoCodec codec_settings_ RTC_GUARDED_BY(mutex_); + SimulcastMode mode_ RTC_GUARDED_BY(mutex_); + EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(mutex_); + std::list<std::pair<uint32_t, uint16_t>> timestamp_to_frame_id_list_ + RTC_GUARDED_BY(mutex_); + VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(mutex_); +}; + +// Produces QualityAnalyzingVideoEncoder, which hold decoders, produced by +// specified factory as delegates. Forwards all other calls to specified +// factory. +class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory { + public: + QualityAnalyzingVideoEncoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoEncoderFactory> delegate, + double bitrate_multiplier, + std::map<std::string, absl::optional<int>> stream_required_spatial_index, + EncodedImageDataInjector* injector, + VideoQualityAnalyzerInterface* analyzer); + ~QualityAnalyzingVideoEncoderFactory() override; + + // Methods of VideoEncoderFactory interface. + std::vector<SdpVideoFormat> GetSupportedFormats() const override; + std::unique_ptr<VideoEncoder> CreateVideoEncoder( + const SdpVideoFormat& format) override; + + private: + const std::string peer_name_; + std::unique_ptr<VideoEncoderFactory> delegate_; + const double bitrate_multiplier_; + std::map<std::string, absl::optional<int>> stream_required_spatial_index_; + EncodedImageDataInjector* const injector_; + VideoQualityAnalyzerInterface* const analyzer_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.cc new file mode 100644 index 0000000000..1825bfbe20 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.cc @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h" + +#include <cstring> + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +constexpr char kIrrelatedSimulcastStreamFrameData[] = "Dummy!"; + +} // namespace + +rtc::scoped_refptr<webrtc::VideoFrameBuffer> CreateDummyFrameBuffer() { + // Use i420 buffer here as default one and supported by all codecs. + rtc::scoped_refptr<webrtc::I420Buffer> buffer = + webrtc::I420Buffer::Create(2, 2); + memcpy(buffer->MutableDataY(), kIrrelatedSimulcastStreamFrameData, 2); + memcpy(buffer->MutableDataY() + buffer->StrideY(), + kIrrelatedSimulcastStreamFrameData + 2, 2); + memcpy(buffer->MutableDataU(), kIrrelatedSimulcastStreamFrameData + 4, 1); + memcpy(buffer->MutableDataV(), kIrrelatedSimulcastStreamFrameData + 5, 1); + return buffer; +} + +bool IsDummyFrameBuffer( + rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer) { + if (video_frame_buffer->width() != 2 || video_frame_buffer->height() != 2) { + return false; + } + rtc::scoped_refptr<webrtc::I420BufferInterface> buffer = + video_frame_buffer->ToI420(); + if (memcmp(buffer->DataY(), kIrrelatedSimulcastStreamFrameData, 2) != 0) { + return false; + } + if (memcmp(buffer->DataY() + buffer->StrideY(), + kIrrelatedSimulcastStreamFrameData + 2, 2) != 0) { + return false; + } + if (memcmp(buffer->DataU(), kIrrelatedSimulcastStreamFrameData + 4, 1) != 0) { + return false; + } + if (memcmp(buffer->DataV(), kIrrelatedSimulcastStreamFrameData + 5, 1) != 0) { + return false; + } + return true; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h new file mode 100644 index 0000000000..84c5abefbf --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_SIMULCAST_DUMMY_BUFFER_HELPER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_SIMULCAST_DUMMY_BUFFER_HELPER_H_ + +#include "api/video/i420_buffer.h" +#include "api/video/video_frame_buffer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +rtc::scoped_refptr<webrtc::VideoFrameBuffer> CreateDummyFrameBuffer(); + +bool IsDummyFrameBuffer( + rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer); + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_SIMULCAST_DUMMY_BUFFER_HELPER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc new file mode 100644 index 0000000000..ccd2f03537 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h" + +#include <algorithm> +#include <cstddef> + +#include "absl/memory/memory.h" +#include "api/video/encoded_image.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +SingleProcessEncodedImageDataInjector::SingleProcessEncodedImageDataInjector() = + default; +SingleProcessEncodedImageDataInjector:: + ~SingleProcessEncodedImageDataInjector() = default; + +EncodedImage SingleProcessEncodedImageDataInjector::InjectData( + uint16_t id, + bool discard, + const EncodedImage& source) { + RTC_CHECK(source.size() >= ExtractionInfo::kUsedBufferSize); + + ExtractionInfo info; + info.discard = discard; + size_t insertion_pos = source.size() - ExtractionInfo::kUsedBufferSize; + memcpy(info.origin_data, &source.data()[insertion_pos], + ExtractionInfo::kUsedBufferSize); + { + MutexLock lock(&lock_); + // Will create new one if missed. + ExtractionInfoVector& ev = extraction_cache_[id]; + info.sub_id = ev.next_sub_id++; + ev.infos[info.sub_id] = info; + } + + auto buffer = EncodedImageBuffer::Create(source.data(), source.size()); + buffer->data()[insertion_pos] = id & 0x00ff; + buffer->data()[insertion_pos + 1] = (id & 0xff00) >> 8; + buffer->data()[insertion_pos + 2] = info.sub_id; + + EncodedImage out = source; + out.SetEncodedData(buffer); + return out; +} + +void SingleProcessEncodedImageDataInjector::AddParticipantInCall() { + MutexLock crit(&lock_); + expected_receivers_count_++; +} + +void SingleProcessEncodedImageDataInjector::RemoveParticipantInCall() { + MutexLock crit(&lock_); + expected_receivers_count_--; + // Now we need go over `extraction_cache_` and removed frames which have been + // received by `expected_receivers_count_`. + for (auto& [frame_id, extraction_infos] : extraction_cache_) { + for (auto it = extraction_infos.infos.begin(); + it != extraction_infos.infos.end();) { + // Frame is received if `received_count` equals to + // `expected_receivers_count_`. + if (it->second.received_count == expected_receivers_count_) { + it = extraction_infos.infos.erase(it); + } else { + ++it; + } + } + } +} + +EncodedImageExtractionResult SingleProcessEncodedImageDataInjector::ExtractData( + const EncodedImage& source) { + size_t size = source.size(); + auto buffer = EncodedImageBuffer::Create(source.data(), source.size()); + EncodedImage out = source; + out.SetEncodedData(buffer); + + std::vector<size_t> frame_sizes; + std::vector<size_t> frame_sl_index; + size_t max_spatial_index = out.SpatialIndex().value_or(0); + for (size_t i = 0; i <= max_spatial_index; ++i) { + auto frame_size = source.SpatialLayerFrameSize(i); + if (frame_size.value_or(0)) { + frame_sl_index.push_back(i); + frame_sizes.push_back(frame_size.value()); + } + } + if (frame_sizes.empty()) { + frame_sizes.push_back(size); + } + + size_t prev_frames_size = 0; + absl::optional<uint16_t> id = absl::nullopt; + bool discard = true; + std::vector<ExtractionInfo> extraction_infos; + for (size_t frame_size : frame_sizes) { + size_t insertion_pos = + prev_frames_size + frame_size - ExtractionInfo::kUsedBufferSize; + // Extract frame id from first 2 bytes starting from insertion pos. + uint16_t next_id = buffer->data()[insertion_pos] + + (buffer->data()[insertion_pos + 1] << 8); + // Extract frame sub id from second 3 byte starting from insertion pos. + uint8_t sub_id = buffer->data()[insertion_pos + 2]; + RTC_CHECK(!id || *id == next_id) + << "Different frames encoded into single encoded image: " << *id + << " vs " << next_id; + id = next_id; + ExtractionInfo info; + { + MutexLock lock(&lock_); + auto ext_vector_it = extraction_cache_.find(next_id); + RTC_CHECK(ext_vector_it != extraction_cache_.end()) + << "Unknown frame_id=" << next_id; + + auto info_it = ext_vector_it->second.infos.find(sub_id); + RTC_CHECK(info_it != ext_vector_it->second.infos.end()) + << "Unknown sub_id=" << sub_id << " for frame_id=" << next_id; + info_it->second.received_count++; + info = info_it->second; + if (info.received_count == expected_receivers_count_) { + ext_vector_it->second.infos.erase(info_it); + } + } + // We need to discard encoded image only if all concatenated encoded images + // have to be discarded. + discard = discard && info.discard; + + extraction_infos.push_back(info); + prev_frames_size += frame_size; + } + RTC_CHECK(id); + + if (discard) { + out.set_size(0); + for (size_t i = 0; i <= max_spatial_index; ++i) { + out.SetSpatialLayerFrameSize(i, 0); + } + return EncodedImageExtractionResult{*id, out, true}; + } + + // Make a pass from begin to end to restore origin payload and erase discarded + // encoded images. + size_t pos = 0; + for (size_t frame_index = 0; frame_index < frame_sizes.size(); + ++frame_index) { + RTC_CHECK(pos < size); + const size_t frame_size = frame_sizes[frame_index]; + const ExtractionInfo& info = extraction_infos[frame_index]; + if (info.discard) { + // If this encoded image is marked to be discarded - erase it's payload + // from the buffer. + memmove(&buffer->data()[pos], &buffer->data()[pos + frame_size], + size - pos - frame_size); + RTC_CHECK_LT(frame_index, frame_sl_index.size()) + << "codec doesn't support discard option or the image, that was " + "supposed to be discarded, is lost"; + out.SetSpatialLayerFrameSize(frame_sl_index[frame_index], 0); + size -= frame_size; + } else { + memcpy( + &buffer->data()[pos + frame_size - ExtractionInfo::kUsedBufferSize], + info.origin_data, ExtractionInfo::kUsedBufferSize); + pos += frame_size; + } + } + out.set_size(pos); + + return EncodedImageExtractionResult{*id, out, discard}; +} + +SingleProcessEncodedImageDataInjector::ExtractionInfoVector:: + ExtractionInfoVector() = default; +SingleProcessEncodedImageDataInjector::ExtractionInfoVector:: + ~ExtractionInfoVector() = default; + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h new file mode 100644 index 0000000000..1082440e2f --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_ + +#include <cstdint> +#include <map> +#include <memory> +#include <utility> +#include <vector> + +#include "api/video/encoded_image.h" +#include "rtc_base/synchronization/mutex.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Based on assumption that all call participants are in the same OS process +// and uses same QualityAnalyzingVideoContext to obtain +// EncodedImageDataInjector. +// +// To inject frame id and discard flag into EncodedImage injector uses last 3rd +// and 2nd bytes of EncodedImage payload. Then it uses last byte for frame +// sub id, that is required to distinguish different spatial layers. The origin +// data from these 3 bytes will be stored inside injector's internal storage and +// then will be restored during extraction phase. +// +// This injector won't add any extra overhead into EncodedImage payload and +// support frames with any size of payload. Also assumes that every EncodedImage +// payload size is greater or equals to 3 bytes +// +// This injector doesn't support video frames/encoded images without frame ID. +class SingleProcessEncodedImageDataInjector + : public EncodedImageDataPropagator { + public: + SingleProcessEncodedImageDataInjector(); + ~SingleProcessEncodedImageDataInjector() override; + + // Id and discard flag will be injected into EncodedImage buffer directly. + // This buffer won't be fully copied, so `source` image buffer will be also + // changed. + EncodedImage InjectData(uint16_t id, + bool discard, + const EncodedImage& source) override; + + void Start(int expected_receivers_count) override { + MutexLock crit(&lock_); + expected_receivers_count_ = expected_receivers_count; + } + void AddParticipantInCall() override; + void RemoveParticipantInCall() override; + EncodedImageExtractionResult ExtractData(const EncodedImage& source) override; + + private: + // Contains data required to extract frame id from EncodedImage and restore + // original buffer. + struct ExtractionInfo { + // Number of bytes from the beginning of the EncodedImage buffer that will + // be used to store frame id and sub id. + const static size_t kUsedBufferSize = 3; + // Frame sub id to distinguish encoded images for different spatial layers. + uint8_t sub_id; + // Flag to show is this encoded images should be discarded by analyzing + // decoder because of not required spatial layer/simulcast stream. + bool discard; + // Data from first 3 bytes of origin encoded image's payload. + uint8_t origin_data[ExtractionInfo::kUsedBufferSize]; + // Count of how many times this frame was received. + int received_count = 0; + }; + + struct ExtractionInfoVector { + ExtractionInfoVector(); + ~ExtractionInfoVector(); + + // Next sub id, that have to be used for this frame id. + uint8_t next_sub_id = 0; + std::map<uint8_t, ExtractionInfo> infos; + }; + + Mutex lock_; + int expected_receivers_count_ RTC_GUARDED_BY(lock_); + // Stores a mapping from frame id to extraction info for spatial layers + // for this frame id. There can be a lot of them, because if frame was + // dropped we can't clean it up, because we won't receive a signal on + // decoder side about that frame. In such case it will be replaced + // when sub id will overlap. + std::map<uint16_t, ExtractionInfoVector> extraction_cache_ + RTC_GUARDED_BY(lock_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc new file mode 100644 index 0000000000..f6fa40455a --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc @@ -0,0 +1,445 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h" + +#include <utility> + +#include "api/video/encoded_image.h" +#include "rtc_base/buffer.h" +#include "test/gtest.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +rtc::scoped_refptr<EncodedImageBuffer> +CreateEncodedImageBufferOfSizeNFilledWithValuesFromX(size_t n, uint8_t x) { + auto buffer = EncodedImageBuffer::Create(n); + for (size_t i = 0; i < n; ++i) { + buffer->data()[i] = static_cast<uint8_t>(x + i); + } + return buffer; +} + +EncodedImage CreateEncodedImageOfSizeNFilledWithValuesFromX(size_t n, + uint8_t x) { + EncodedImage image; + image.SetEncodedData( + CreateEncodedImageBufferOfSizeNFilledWithValuesFromX(n, x)); + return image; +} + +EncodedImage DeepCopyEncodedImage(const EncodedImage& source) { + EncodedImage copy = source; + copy.SetEncodedData(EncodedImageBuffer::Create(source.data(), source.size())); + return copy; +} + +TEST(SingleProcessEncodedImageDataInjectorTest, InjectExtractDiscardFalse) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + + EncodedImageExtractionResult out = + injector.ExtractData(injector.InjectData(512, false, source)); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, InjectExtractDiscardTrue) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + + EncodedImageExtractionResult out = + injector.ExtractData(injector.InjectData(512, true, source)); + EXPECT_EQ(out.id, 512); + EXPECT_TRUE(out.discard); + EXPECT_EQ(out.image.size(), 0ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); +} + +TEST(SingleProcessEncodedImageDataInjectorTest, + InjectWithUnsetSpatialLayerSizes) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + + EncodedImage intermediate = injector.InjectData(512, false, source); + intermediate.SetSpatialIndex(2); + + EncodedImageExtractionResult out = injector.ExtractData(intermediate); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } + EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2); + for (int i = 0; i < 3; ++i) { + EXPECT_EQ(out.image.SpatialLayerFrameSize(i).value_or(0), 0ul); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, + InjectWithZeroSpatialLayerSizes) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + + EncodedImage intermediate = injector.InjectData(512, false, source); + intermediate.SetSpatialIndex(2); + intermediate.SetSpatialLayerFrameSize(0, 0); + intermediate.SetSpatialLayerFrameSize(1, 0); + intermediate.SetSpatialLayerFrameSize(2, 0); + + EncodedImageExtractionResult out = injector.ExtractData(intermediate); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } + EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2); + for (int i = 0; i < 3; ++i) { + EXPECT_EQ(out.image.SpatialLayerFrameSize(i).value_or(0), 0ul); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, Inject3Extract3) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + // 1st frame + EncodedImage source1 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source1.SetTimestamp(123456710); + // 2nd frame 1st spatial layer + EncodedImage source2 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/11); + source2.SetTimestamp(123456720); + // 2nd frame 2nd spatial layer + EncodedImage source3 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/21); + source3.SetTimestamp(123456720); + + EncodedImage intermediate1 = injector.InjectData(510, false, source1); + EncodedImage intermediate2 = injector.InjectData(520, true, source2); + EncodedImage intermediate3 = injector.InjectData(520, false, source3); + + // Extract ids in different order. + EncodedImageExtractionResult out3 = injector.ExtractData(intermediate3); + EncodedImageExtractionResult out1 = injector.ExtractData(intermediate1); + EncodedImageExtractionResult out2 = injector.ExtractData(intermediate2); + + EXPECT_EQ(out1.id, 510); + EXPECT_FALSE(out1.discard); + EXPECT_EQ(out1.image.size(), 10ul); + EXPECT_EQ(out1.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out1.image.data()[i], i + 1); + } + EXPECT_EQ(out2.id, 520); + EXPECT_TRUE(out2.discard); + EXPECT_EQ(out2.image.size(), 0ul); + EXPECT_EQ(out2.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + EXPECT_EQ(out3.id, 520); + EXPECT_FALSE(out3.discard); + EXPECT_EQ(out3.image.size(), 10ul); + EXPECT_EQ(out3.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out3.image.data()[i], i + 21); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, InjectExtractFromConcatenated) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + EncodedImage source1 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source1.SetTimestamp(123456710); + EncodedImage source2 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/11); + source2.SetTimestamp(123456710); + EncodedImage source3 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/21); + source3.SetTimestamp(123456710); + + // Inject id into 3 images with same frame id. + EncodedImage intermediate1 = injector.InjectData(512, false, source1); + EncodedImage intermediate2 = injector.InjectData(512, true, source2); + EncodedImage intermediate3 = injector.InjectData(512, false, source3); + + // Concatenate them into single encoded image, like it can be done in jitter + // buffer. + size_t concatenated_length = + intermediate1.size() + intermediate2.size() + intermediate3.size(); + rtc::Buffer concatenated_buffer; + concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size()); + concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size()); + concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size()); + EncodedImage concatenated; + concatenated.SetEncodedData(EncodedImageBuffer::Create( + concatenated_buffer.data(), concatenated_length)); + concatenated.SetSpatialIndex(2); + concatenated.SetSpatialLayerFrameSize(0, intermediate1.size()); + concatenated.SetSpatialLayerFrameSize(1, intermediate2.size()); + concatenated.SetSpatialLayerFrameSize(2, intermediate3.size()); + + // Extract frame id from concatenated image + EncodedImageExtractionResult out = injector.ExtractData(concatenated); + + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 2 * 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + EXPECT_EQ(out.image.data()[i + 10], i + 21); + } + EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(1).value_or(0), 0ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(2).value_or(0), 10ul); +} + +TEST(SingleProcessEncodedImageDataInjector, + InjectExtractFromConcatenatedAllDiscarded) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/1); + + EncodedImage source1 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source1.SetTimestamp(123456710); + EncodedImage source2 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/11); + source2.SetTimestamp(123456710); + EncodedImage source3 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/21); + source3.SetTimestamp(123456710); + + // Inject id into 3 images with same frame id. + EncodedImage intermediate1 = injector.InjectData(512, true, source1); + EncodedImage intermediate2 = injector.InjectData(512, true, source2); + EncodedImage intermediate3 = injector.InjectData(512, true, source3); + + // Concatenate them into single encoded image, like it can be done in jitter + // buffer. + size_t concatenated_length = + intermediate1.size() + intermediate2.size() + intermediate3.size(); + rtc::Buffer concatenated_buffer; + concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size()); + concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size()); + concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size()); + EncodedImage concatenated; + concatenated.SetEncodedData(EncodedImageBuffer::Create( + concatenated_buffer.data(), concatenated_length)); + concatenated.SetSpatialIndex(2); + concatenated.SetSpatialLayerFrameSize(0, intermediate1.size()); + concatenated.SetSpatialLayerFrameSize(1, intermediate2.size()); + concatenated.SetSpatialLayerFrameSize(2, intermediate3.size()); + + // Extract frame id from concatenated image + EncodedImageExtractionResult out = injector.ExtractData(concatenated); + + EXPECT_EQ(out.id, 512); + EXPECT_TRUE(out.discard); + EXPECT_EQ(out.image.size(), 0ul); + EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2); + for (int i = 0; i < 3; ++i) { + EXPECT_EQ(out.image.SpatialLayerFrameSize(i).value_or(0), 0ul); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, InjectOnceExtractTwice) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/2); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + + EncodedImageExtractionResult out = injector.ExtractData( + injector.InjectData(/*id=*/512, /*discard=*/false, source)); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } + out = injector.ExtractData( + injector.InjectData(/*id=*/512, /*discard=*/false, source)); + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, Add1stReceiverAfterStart) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/0); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + EncodedImage modified_image = injector.InjectData( + /*id=*/512, /*discard=*/false, source); + + injector.AddParticipantInCall(); + EncodedImageExtractionResult out = injector.ExtractData(modified_image); + + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, Add3rdReceiverAfterStart) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/2); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + EncodedImage modified_image = injector.InjectData( + /*id=*/512, /*discard=*/false, source); + injector.ExtractData(modified_image); + + injector.AddParticipantInCall(); + injector.ExtractData(modified_image); + EncodedImageExtractionResult out = injector.ExtractData(modified_image); + + EXPECT_EQ(out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +TEST(SingleProcessEncodedImageDataInjectorTest, + RemoveReceiverRemovesOnlyFullyReceivedFrames) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/3); + + EncodedImage source1 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source1.SetTimestamp(10); + EncodedImage source2 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source2.SetTimestamp(20); + + EncodedImage modified_image1 = injector.InjectData( + /*id=*/512, /*discard=*/false, source1); + EncodedImage modified_image2 = injector.InjectData( + /*id=*/513, /*discard=*/false, source2); + + // Out of 3 receivers 1st image received by 2 and 2nd image by 1 + injector.ExtractData(DeepCopyEncodedImage(modified_image1)); + injector.ExtractData(DeepCopyEncodedImage(modified_image1)); + injector.ExtractData(DeepCopyEncodedImage(modified_image2)); + + // When we removed one receiver, 2nd image should still be available for + // extraction. + injector.RemoveParticipantInCall(); + + EncodedImageExtractionResult out = + injector.ExtractData(DeepCopyEncodedImage(modified_image2)); + + EXPECT_EQ(out.id, 513); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(out.image.data()[i], i + 1); + } +} + +// Death tests. +// Disabled on Android because death tests misbehave on Android, see +// base/test/gtest_util.h. +#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) +TEST(SingleProcessEncodedImageDataInjectorTestDeathTest, + InjectOnceExtractMoreThenExpected) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/2); + + EncodedImage source = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source.SetTimestamp(123456789); + + EncodedImage modified = + injector.InjectData(/*id=*/512, /*discard=*/false, source); + + injector.ExtractData(DeepCopyEncodedImage(modified)); + injector.ExtractData(DeepCopyEncodedImage(modified)); + EXPECT_DEATH(injector.ExtractData(DeepCopyEncodedImage(modified)), + "Unknown sub_id=0 for frame_id=512"); +} + +TEST(SingleProcessEncodedImageDataInjectorTestDeathTest, + RemoveReceiverRemovesOnlyFullyReceivedFramesVerifyFrameIsRemoved) { + SingleProcessEncodedImageDataInjector injector; + injector.Start(/*expected_receivers_count=*/3); + + EncodedImage source1 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source1.SetTimestamp(10); + EncodedImage source2 = + CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1); + source2.SetTimestamp(20); + + EncodedImage modified_image1 = injector.InjectData( + /*id=*/512, /*discard=*/false, source1); + EncodedImage modified_image2 = injector.InjectData( + /*id=*/513, /*discard=*/false, source2); + + // Out of 3 receivers 1st image received by 2 and 2nd image by 1 + injector.ExtractData(DeepCopyEncodedImage(modified_image1)); + injector.ExtractData(DeepCopyEncodedImage(modified_image1)); + injector.ExtractData(DeepCopyEncodedImage(modified_image2)); + + // When we removed one receiver 1st image should be removed. + injector.RemoveParticipantInCall(); + + EXPECT_DEATH(injector.ExtractData(DeepCopyEncodedImage(modified_image1)), + "Unknown sub_id=0 for frame_id=512"); +} +#endif // RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.cc new file mode 100644 index 0000000000..5a74d60250 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.cc @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h" + +#include "absl/memory/memory.h" +#include "api/video/encoded_image.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +EncodedImage VideoFrameTrackingIdInjector::InjectData( + uint16_t id, + bool unused_discard, + const EncodedImage& source) { + RTC_CHECK(!unused_discard); + EncodedImage out = source; + out.SetVideoFrameTrackingId(id); + return out; +} + +EncodedImageExtractionResult VideoFrameTrackingIdInjector::ExtractData( + const EncodedImage& source) { + return EncodedImageExtractionResult{source.VideoFrameTrackingId(), source, + /*discard=*/false}; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h new file mode 100644 index 0000000000..ecc3cd3f51 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_FRAME_TRACKING_ID_INJECTOR_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_FRAME_TRACKING_ID_INJECTOR_H_ + +#include <cstdint> + +#include "api/video/encoded_image.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// This injector sets and retrieves the provided id in the EncodedImage +// video_frame_tracking_id field. This is only possible with the RTP header +// extension VideoFrameTrackingIdExtension that will propagate the input +// tracking id to the received EncodedImage. This RTP header extension is +// enabled with the field trial WebRTC-VideoFrameTrackingIdAdvertised +// (http://www.webrtc.org/experiments/rtp-hdrext/video-frame-tracking-id). +// +// Note that this injector doesn't allow to discard frames. +class VideoFrameTrackingIdInjector : public EncodedImageDataPropagator { + public: + EncodedImage InjectData(uint16_t id, + bool unused_discard, + const EncodedImage& source) override; + + EncodedImageExtractionResult ExtractData(const EncodedImage& source) override; + + void Start(int) override {} + void AddParticipantInCall() override {} + void RemoveParticipantInCall() override {} +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_FRAME_TRACKING_ID_INJECTOR_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector_unittest.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector_unittest.cc new file mode 100644 index 0000000000..c7d453c4bb --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector_unittest.cc @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h" + +#include "api/video/encoded_image.h" +#include "rtc_base/buffer.h" +#include "test/gtest.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +EncodedImage CreateEncodedImageOfSizeN(size_t n) { + EncodedImage image; + rtc::scoped_refptr<EncodedImageBuffer> buffer = EncodedImageBuffer::Create(n); + for (size_t i = 0; i < n; ++i) { + buffer->data()[i] = static_cast<uint8_t>(i); + } + image.SetEncodedData(buffer); + return image; +} + +TEST(VideoFrameTrackingIdInjectorTest, InjectExtractDiscardFalse) { + VideoFrameTrackingIdInjector injector; + EncodedImage source = CreateEncodedImageOfSizeN(10); + EncodedImageExtractionResult out = + injector.ExtractData(injector.InjectData(512, false, source)); + + ASSERT_TRUE(out.id.has_value()); + EXPECT_EQ(*out.id, 512); + EXPECT_FALSE(out.discard); + EXPECT_EQ(out.image.size(), 10ul); + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(source.data()[i], out.image.data()[i]); + } +} + +#if GTEST_HAS_DEATH_TEST +TEST(VideoFrameTrackingIdInjectorTest, InjectExtractDiscardTrue) { + VideoFrameTrackingIdInjector injector; + EncodedImage source = CreateEncodedImageOfSizeN(10); + + EXPECT_DEATH(injector.InjectData(512, true, source), ""); +} +#endif // GTEST_HAS_DEATH_TEST + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc new file mode 100644 index 0000000000..661208e888 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc @@ -0,0 +1,366 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h" + +#include <stdio.h> + +#include <utility> +#include <vector> + +#include "absl/memory/memory.h" +#include "absl/strings/string_view.h" +#include "api/array_view.h" +#include "rtc_base/strings/string_builder.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h" +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h" +#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h" +#include "test/testsupport/fixed_fps_video_frame_writer_adapter.h" +#include "test/video_renderer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +namespace { + +class VideoWriter final : public rtc::VideoSinkInterface<VideoFrame> { + public: + VideoWriter(test::VideoFrameWriter* video_writer, int sampling_modulo) + : video_writer_(video_writer), sampling_modulo_(sampling_modulo) {} + ~VideoWriter() override = default; + + void OnFrame(const VideoFrame& frame) override { + if (frames_counter_++ % sampling_modulo_ != 0) { + return; + } + bool result = video_writer_->WriteFrame(frame); + RTC_CHECK(result) << "Failed to write frame"; + } + + private: + test::VideoFrameWriter* const video_writer_; + const int sampling_modulo_; + + int64_t frames_counter_ = 0; +}; + +class AnalyzingFramePreprocessor + : public test::TestVideoCapturer::FramePreprocessor { + public: + AnalyzingFramePreprocessor( + absl::string_view peer_name, + absl::string_view stream_label, + VideoQualityAnalyzerInterface* analyzer, + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> sinks) + : peer_name_(peer_name), + stream_label_(stream_label), + analyzer_(analyzer), + sinks_(std::move(sinks)) {} + ~AnalyzingFramePreprocessor() override = default; + + VideoFrame Preprocess(const VideoFrame& source_frame) override { + // Copy VideoFrame to be able to set id on it. + VideoFrame frame = source_frame; + uint16_t frame_id = + analyzer_->OnFrameCaptured(peer_name_, stream_label_, frame); + frame.set_id(frame_id); + + for (auto& sink : sinks_) { + sink->OnFrame(frame); + } + return frame; + } + + private: + const std::string peer_name_; + const std::string stream_label_; + VideoQualityAnalyzerInterface* const analyzer_; + const std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> + sinks_; +}; + +} // namespace + +VideoQualityAnalyzerInjectionHelper::VideoFrameIdsWriter::VideoFrameIdsWriter( + absl::string_view file_name) + : file_name_(file_name) { + output_file_ = fopen(file_name_.c_str(), "wb"); + RTC_CHECK(output_file_ != nullptr) + << "Failed to open file to dump frame ids for writing: " << file_name_; +} + +VideoQualityAnalyzerInjectionHelper::VideoFrameIdsWriter:: + ~VideoFrameIdsWriter() { + fclose(output_file_); +} + +void VideoQualityAnalyzerInjectionHelper::VideoFrameIdsWriter::WriteFrameId( + uint16_t frame_id) { + int chars_written = fprintf(output_file_, "%d\n", frame_id); + RTC_CHECK_GE(chars_written, 2) + << "Failed to write frame id to the output file: " << file_name_; +} + +VideoQualityAnalyzerInjectionHelper::VideoWriter2::VideoWriter2( + test::VideoFrameWriter* video_writer, + VideoFrameIdsWriter* frame_ids_writer, + int sampling_modulo) + : video_writer_(video_writer), + frame_ids_writer_(frame_ids_writer), + sampling_modulo_(sampling_modulo) {} + +void VideoQualityAnalyzerInjectionHelper::VideoWriter2::OnFrame( + const VideoFrame& frame) { + if (frames_counter_++ % sampling_modulo_ != 0) { + return; + } + bool result = video_writer_->WriteFrame(frame); + RTC_CHECK(result) << "Failed to write frame"; + if (frame_ids_writer_) { + frame_ids_writer_->WriteFrameId(frame.id()); + } +} + +VideoQualityAnalyzerInjectionHelper::VideoQualityAnalyzerInjectionHelper( + Clock* clock, + std::unique_ptr<VideoQualityAnalyzerInterface> analyzer, + EncodedImageDataInjector* injector, + EncodedImageDataExtractor* extractor) + : clock_(clock), + analyzer_(std::move(analyzer)), + injector_(injector), + extractor_(extractor) { + RTC_DCHECK(clock_); + RTC_DCHECK(injector_); + RTC_DCHECK(extractor_); +} +VideoQualityAnalyzerInjectionHelper::~VideoQualityAnalyzerInjectionHelper() = + default; + +std::unique_ptr<VideoEncoderFactory> +VideoQualityAnalyzerInjectionHelper::WrapVideoEncoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoEncoderFactory> delegate, + double bitrate_multiplier, + std::map<std::string, absl::optional<int>> stream_required_spatial_index) + const { + return std::make_unique<QualityAnalyzingVideoEncoderFactory>( + peer_name, std::move(delegate), bitrate_multiplier, + std::move(stream_required_spatial_index), injector_, analyzer_.get()); +} + +std::unique_ptr<VideoDecoderFactory> +VideoQualityAnalyzerInjectionHelper::WrapVideoDecoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoDecoderFactory> delegate) const { + return std::make_unique<QualityAnalyzingVideoDecoderFactory>( + peer_name, std::move(delegate), extractor_, analyzer_.get()); +} + +std::unique_ptr<test::TestVideoCapturer::FramePreprocessor> +VideoQualityAnalyzerInjectionHelper::CreateFramePreprocessor( + absl::string_view peer_name, + const VideoConfig& config) { + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> sinks; + test::VideoFrameWriter* writer = nullptr; + if (config.input_dump_options.has_value()) { + // Using new API for video dumping. + writer = MaybeCreateVideoWriter( + config.input_dump_options->GetInputDumpFileName(*config.stream_label), + config); + RTC_CHECK(writer); + VideoFrameIdsWriter* frame_ids_writer = nullptr; + if (config.input_dump_options->export_frame_ids()) { + frame_ids_writers_.push_back(std::make_unique<VideoFrameIdsWriter>( + *config.input_dump_options->GetInputFrameIdsDumpFileName( + *config.stream_label))); + frame_ids_writer = frame_ids_writers_.back().get(); + } + sinks.push_back(std::make_unique<VideoWriter2>( + writer, frame_ids_writer, + config.input_dump_options->sampling_modulo())); + } else { + // Using old API. To be removed. + writer = MaybeCreateVideoWriter(config.input_dump_file_name, config); + if (writer) { + sinks.push_back(std::make_unique<VideoWriter>( + writer, config.input_dump_sampling_modulo)); + } + } + if (config.show_on_screen) { + sinks.push_back(absl::WrapUnique( + test::VideoRenderer::Create((*config.stream_label + "-capture").c_str(), + config.width, config.height))); + } + { + MutexLock lock(&mutex_); + known_video_configs_.insert({*config.stream_label, config}); + } + return std::make_unique<AnalyzingFramePreprocessor>( + peer_name, std::move(*config.stream_label), analyzer_.get(), + std::move(sinks)); +} + +std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> +VideoQualityAnalyzerInjectionHelper::CreateVideoSink( + absl::string_view peer_name) { + return std::make_unique<AnalyzingVideoSink>(peer_name, this); +} + +void VideoQualityAnalyzerInjectionHelper::Start( + std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count) { + analyzer_->Start(std::move(test_case_name), peer_names, max_threads_count); + extractor_->Start(peer_names.size()); + MutexLock lock(&mutex_); + peers_count_ = peer_names.size(); +} + +void VideoQualityAnalyzerInjectionHelper::RegisterParticipantInCall( + absl::string_view peer_name) { + analyzer_->RegisterParticipantInCall(peer_name); + extractor_->AddParticipantInCall(); + MutexLock lock(&mutex_); + peers_count_++; +} + +void VideoQualityAnalyzerInjectionHelper::UnregisterParticipantInCall( + absl::string_view peer_name) { + analyzer_->UnregisterParticipantInCall(peer_name); + extractor_->RemoveParticipantInCall(); + MutexLock lock(&mutex_); + peers_count_--; +} + +void VideoQualityAnalyzerInjectionHelper::OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) { + analyzer_->OnStatsReports(pc_label, report); +} + +void VideoQualityAnalyzerInjectionHelper::Stop() { + analyzer_->Stop(); + for (const auto& video_writer : video_writers_) { + video_writer->Close(); + } + video_writers_.clear(); + frame_ids_writers_.clear(); +} + +test::VideoFrameWriter* +VideoQualityAnalyzerInjectionHelper::MaybeCreateVideoWriter( + absl::optional<std::string> file_name, + const PeerConnectionE2EQualityTestFixture::VideoConfig& config) { + if (!file_name.has_value()) { + return nullptr; + } + // TODO(titovartem) create only one file writer for simulcast video track. + // For now this code will be invoked for each simulcast stream separately, but + // only one file will be used. + std::unique_ptr<test::VideoFrameWriter> video_writer = + std::make_unique<test::Y4mVideoFrameWriterImpl>( + *file_name, config.width, config.height, config.fps); + if (config.output_dump_use_fixed_framerate) { + video_writer = std::make_unique<test::FixedFpsVideoFrameWriterAdapter>( + config.fps, clock_, std::move(video_writer)); + } + test::VideoFrameWriter* out = video_writer.get(); + video_writers_.push_back(std::move(video_writer)); + return out; +} + +void VideoQualityAnalyzerInjectionHelper::OnFrame(absl::string_view peer_name, + const VideoFrame& frame) { + rtc::scoped_refptr<I420BufferInterface> i420_buffer = + frame.video_frame_buffer()->ToI420(); + if (IsDummyFrameBuffer(i420_buffer)) { + // This is dummy frame, so we don't need to process it further. + return; + } + // Copy entire video frame including video buffer to ensure that analyzer + // won't hold any WebRTC internal buffers. + VideoFrame frame_copy = frame; + frame_copy.set_video_frame_buffer(I420Buffer::Copy(*i420_buffer)); + analyzer_->OnFrameRendered(peer_name, frame_copy); + + if (frame.id() != VideoFrame::kNotSetId) { + std::string stream_label = analyzer_->GetStreamLabel(frame.id()); + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>* sinks = + PopulateSinks(ReceiverStream(peer_name, stream_label)); + if (sinks == nullptr) { + return; + } + for (auto& sink : *sinks) { + sink->OnFrame(frame); + } + } +} + +std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>* +VideoQualityAnalyzerInjectionHelper::PopulateSinks( + const ReceiverStream& receiver_stream) { + MutexLock lock(&mutex_); + auto sinks_it = sinks_.find(receiver_stream); + if (sinks_it != sinks_.end()) { + return &sinks_it->second; + } + auto it = known_video_configs_.find(receiver_stream.stream_label); + RTC_DCHECK(it != known_video_configs_.end()) + << "No video config for stream " << receiver_stream.stream_label; + const VideoConfig& config = it->second; + + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> sinks; + test::VideoFrameWriter* writer = nullptr; + if (config.output_dump_options.has_value()) { + // Using new API with output directory. + writer = MaybeCreateVideoWriter( + config.output_dump_options->GetOutputDumpFileName( + receiver_stream.stream_label, receiver_stream.peer_name), + config); + RTC_CHECK(writer); + VideoFrameIdsWriter* frame_ids_writer = nullptr; + if (config.output_dump_options->export_frame_ids()) { + frame_ids_writers_.push_back(std::make_unique<VideoFrameIdsWriter>( + *config.output_dump_options->GetOutputFrameIdsDumpFileName( + receiver_stream.stream_label, receiver_stream.peer_name))); + frame_ids_writer = frame_ids_writers_.back().get(); + } + sinks.push_back(std::make_unique<VideoWriter2>( + writer, frame_ids_writer, + config.output_dump_options->sampling_modulo())); + } else { + // Using old API. To be removed. + absl::optional<std::string> output_dump_file_name = + config.output_dump_file_name; + if (output_dump_file_name.has_value() && peers_count_ > 2) { + // TODO(titovartem): make this default behavior for any amount of peers. + rtc::StringBuilder builder(*output_dump_file_name); + builder << "." << receiver_stream.peer_name; + output_dump_file_name = builder.str(); + } + writer = MaybeCreateVideoWriter(output_dump_file_name, config); + if (writer) { + sinks.push_back(std::make_unique<VideoWriter>( + writer, config.output_dump_sampling_modulo)); + } + } + if (config.show_on_screen) { + sinks.push_back(absl::WrapUnique( + test::VideoRenderer::Create((*config.stream_label + "-render").c_str(), + config.width, config.height))); + } + sinks_.insert({receiver_stream, std::move(sinks)}); + return &(sinks_.find(receiver_stream)->second); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h new file mode 100644 index 0000000000..76b8b446bc --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_ANALYZER_INJECTION_HELPER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_ANALYZER_INJECTION_HELPER_H_ + +#include <stdio.h> + +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/array_view.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/test/stats_observer_interface.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "rtc_base/synchronization/mutex.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" +#include "test/test_video_capturer.h" +#include "test/testsupport/video_frame_writer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Provides factory methods for components, that will be used to inject +// VideoQualityAnalyzerInterface into PeerConnection pipeline. +class VideoQualityAnalyzerInjectionHelper : public StatsObserverInterface { + public: + using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig; + + VideoQualityAnalyzerInjectionHelper( + Clock* clock, + std::unique_ptr<VideoQualityAnalyzerInterface> analyzer, + EncodedImageDataInjector* injector, + EncodedImageDataExtractor* extractor); + ~VideoQualityAnalyzerInjectionHelper() override; + + // Wraps video encoder factory to give video quality analyzer access to frames + // before encoding and encoded images after. + std::unique_ptr<VideoEncoderFactory> WrapVideoEncoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoEncoderFactory> delegate, + double bitrate_multiplier, + std::map<std::string, absl::optional<int>> stream_required_spatial_index) + const; + // Wraps video decoder factory to give video quality analyzer access to + // received encoded images and frames, that were decoded from them. + std::unique_ptr<VideoDecoderFactory> WrapVideoDecoderFactory( + absl::string_view peer_name, + std::unique_ptr<VideoDecoderFactory> delegate) const; + + // Creates VideoFrame preprocessor, that will allow video quality analyzer to + // get access to the captured frames. If provided config also specifies + // `input_dump_file_name`, video will be written into that file. + std::unique_ptr<test::TestVideoCapturer::FramePreprocessor> + CreateFramePreprocessor(absl::string_view peer_name, + const VideoConfig& config); + // Creates sink, that will allow video quality analyzer to get access to + // the rendered frames. If corresponding video track has + // `output_dump_file_name` in its VideoConfig, which was used for + // CreateFramePreprocessor(...), then video also will be written + // into that file. + std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> CreateVideoSink( + absl::string_view peer_name); + + void Start(std::string test_case_name, + rtc::ArrayView<const std::string> peer_names, + int max_threads_count = 1); + + // Registers new call participant to the underlying video quality analyzer. + // The method should be called before the participant is actually added. + void RegisterParticipantInCall(absl::string_view peer_name); + + // Will be called after test removed existing participant in the middle of the + // call. + void UnregisterParticipantInCall(absl::string_view peer_name); + + // Forwards `stats_reports` for Peer Connection `pc_label` to + // `analyzer_`. + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override; + + // Stops VideoQualityAnalyzerInterface to populate final data and metrics. + // Should be invoked after analyzed video tracks are disposed. + void Stop(); + + private: + class AnalyzingVideoSink final : public rtc::VideoSinkInterface<VideoFrame> { + public: + explicit AnalyzingVideoSink(absl::string_view peer_name, + VideoQualityAnalyzerInjectionHelper* helper) + : peer_name_(peer_name), helper_(helper) {} + ~AnalyzingVideoSink() override = default; + + void OnFrame(const VideoFrame& frame) override { + helper_->OnFrame(peer_name_, frame); + } + + private: + const std::string peer_name_; + VideoQualityAnalyzerInjectionHelper* const helper_; + }; + + struct ReceiverStream { + ReceiverStream(absl::string_view peer_name, absl::string_view stream_label) + : peer_name(peer_name), stream_label(stream_label) {} + + std::string peer_name; + std::string stream_label; + + // Define operators required to use ReceiverStream as std::map key. + bool operator==(const ReceiverStream& o) const { + return peer_name == o.peer_name && stream_label == o.stream_label; + } + bool operator<(const ReceiverStream& o) const { + return (peer_name == o.peer_name) ? stream_label < o.stream_label + : peer_name < o.peer_name; + } + }; + + class VideoFrameIdsWriter { + public: + explicit VideoFrameIdsWriter(absl::string_view file_name); + ~VideoFrameIdsWriter(); + + void WriteFrameId(uint16_t frame_id); + + private: + const std::string file_name_; + FILE* output_file_; + }; + + class VideoWriter2 final : public rtc::VideoSinkInterface<VideoFrame> { + public: + VideoWriter2(test::VideoFrameWriter* video_writer, + VideoFrameIdsWriter* frame_ids_writer, + int sampling_modulo); + ~VideoWriter2() override = default; + + void OnFrame(const VideoFrame& frame) override; + + private: + test::VideoFrameWriter* const video_writer_; + VideoFrameIdsWriter* const frame_ids_writer_; + const int sampling_modulo_; + + int64_t frames_counter_ = 0; + }; + + test::VideoFrameWriter* MaybeCreateVideoWriter( + absl::optional<std::string> file_name, + const PeerConnectionE2EQualityTestFixture::VideoConfig& config); + // Creates a deep copy of the frame and passes it to the video analyzer, while + // passing real frame to the sinks + void OnFrame(absl::string_view peer_name, const VideoFrame& frame); + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>* + PopulateSinks(const ReceiverStream& receiver_stream); + + Clock* const clock_; + std::unique_ptr<VideoQualityAnalyzerInterface> analyzer_; + EncodedImageDataInjector* injector_; + EncodedImageDataExtractor* extractor_; + + std::vector<std::unique_ptr<test::VideoFrameWriter>> video_writers_; + std::vector<std::unique_ptr<VideoFrameIdsWriter>> frame_ids_writers_; + + Mutex mutex_; + int peers_count_ RTC_GUARDED_BY(mutex_); + // Map from stream label to the video config. + std::map<std::string, VideoConfig> known_video_configs_ + RTC_GUARDED_BY(mutex_); + std::map<ReceiverStream, + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>> + sinks_ RTC_GUARDED_BY(mutex_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_ANALYZER_INJECTION_HELPER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc new file mode 100644 index 0000000000..7a27ae0680 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h" + +#include "api/stats/rtc_stats.h" +#include "api/stats/rtcstats_objects.h" +#include "api/units/data_rate.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +void VideoQualityMetricsReporter::Start( + absl::string_view test_case_name, + const TrackIdStreamInfoMap* /*reporter_helper*/) { + test_case_name_ = std::string(test_case_name); + start_time_ = Now(); +} + +void VideoQualityMetricsReporter::OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) { + RTC_CHECK(start_time_) + << "Please invoke Start(...) method before calling OnStatsReports(...)"; + + auto transport_stats = report->GetStatsOfType<RTCTransportStats>(); + if (transport_stats.size() == 0u || + !transport_stats[0]->selected_candidate_pair_id.is_defined()) { + return; + } + RTC_DCHECK_EQ(transport_stats.size(), 1); + std::string selected_ice_id = + transport_stats[0]->selected_candidate_pair_id.ValueToString(); + // Use the selected ICE candidate pair ID to get the appropriate ICE stats. + const RTCIceCandidatePairStats ice_candidate_pair_stats = + report->Get(selected_ice_id)->cast_to<const RTCIceCandidatePairStats>(); + + auto outbound_rtp_stats = report->GetStatsOfType<RTCOutboundRTPStreamStats>(); + StatsSample sample; + for (auto& s : outbound_rtp_stats) { + if (!s->kind.is_defined()) { + continue; + } + if (!(*s->kind == RTCMediaStreamTrackKind::kVideo)) { + continue; + } + if (s->timestamp_us() > sample.sample_time.us()) { + sample.sample_time = Timestamp::Micros(s->timestamp_us()); + } + sample.retransmitted_bytes_sent += + DataSize::Bytes(s->retransmitted_bytes_sent.ValueOrDefault(0ul)); + sample.bytes_sent += DataSize::Bytes(s->bytes_sent.ValueOrDefault(0ul)); + sample.header_bytes_sent += + DataSize::Bytes(s->header_bytes_sent.ValueOrDefault(0ul)); + } + + MutexLock lock(&video_bwe_stats_lock_); + VideoBweStats& video_bwe_stats = video_bwe_stats_[std::string(pc_label)]; + if (ice_candidate_pair_stats.available_outgoing_bitrate.is_defined()) { + video_bwe_stats.available_send_bandwidth.AddSample( + DataRate::BitsPerSec( + *ice_candidate_pair_stats.available_outgoing_bitrate) + .bytes_per_sec()); + } + + StatsSample prev_sample = last_stats_sample_[std::string(pc_label)]; + if (prev_sample.sample_time.IsZero()) { + prev_sample.sample_time = start_time_.value(); + } + last_stats_sample_[std::string(pc_label)] = sample; + + TimeDelta time_between_samples = sample.sample_time - prev_sample.sample_time; + if (time_between_samples.IsZero()) { + return; + } + + DataRate retransmission_bitrate = + (sample.retransmitted_bytes_sent - prev_sample.retransmitted_bytes_sent) / + time_between_samples; + video_bwe_stats.retransmission_bitrate.AddSample( + retransmission_bitrate.bytes_per_sec()); + DataRate transmission_bitrate = + (sample.bytes_sent + sample.header_bytes_sent - prev_sample.bytes_sent - + prev_sample.header_bytes_sent) / + time_between_samples; + video_bwe_stats.transmission_bitrate.AddSample( + transmission_bitrate.bytes_per_sec()); +} + +void VideoQualityMetricsReporter::StopAndReportResults() { + MutexLock video_bwemutex_(&video_bwe_stats_lock_); + for (const auto& item : video_bwe_stats_) { + ReportVideoBweResults(GetTestCaseName(item.first), item.second); + } +} + +std::string VideoQualityMetricsReporter::GetTestCaseName( + const std::string& stream_label) const { + return test_case_name_ + "/" + stream_label; +} + +void VideoQualityMetricsReporter::ReportVideoBweResults( + const std::string& test_case_name, + const VideoBweStats& video_bwe_stats) { + ReportResult("available_send_bandwidth", test_case_name, + video_bwe_stats.available_send_bandwidth, "bytesPerSecond"); + ReportResult("transmission_bitrate", test_case_name, + video_bwe_stats.transmission_bitrate, "bytesPerSecond"); + ReportResult("retransmission_bitrate", test_case_name, + video_bwe_stats.retransmission_bitrate, "bytesPerSecond"); +} + +void VideoQualityMetricsReporter::ReportResult( + const std::string& metric_name, + const std::string& test_case_name, + const SamplesStatsCounter& counter, + const std::string& unit, + webrtc::test::ImproveDirection improve_direction) { + test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit, + /*important=*/false, improve_direction); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h new file mode 100644 index 0000000000..ff195a450e --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_METRICS_REPORTER_H_ +#define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_METRICS_REPORTER_H_ + +#include <map> +#include <string> + +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/test/track_id_stream_info_map.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "rtc_base/synchronization/mutex.h" +#include "test/testsupport/perf_test.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +struct VideoBweStats { + SamplesStatsCounter available_send_bandwidth; + SamplesStatsCounter transmission_bitrate; + SamplesStatsCounter retransmission_bitrate; +}; + +class VideoQualityMetricsReporter + : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { + public: + VideoQualityMetricsReporter(Clock* const clock) : clock_(clock) {} + ~VideoQualityMetricsReporter() override = default; + + void Start(absl::string_view test_case_name, + const TrackIdStreamInfoMap* reporter_helper) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override; + void StopAndReportResults() override; + + private: + struct StatsSample { + DataSize bytes_sent = DataSize::Zero(); + DataSize header_bytes_sent = DataSize::Zero(); + DataSize retransmitted_bytes_sent = DataSize::Zero(); + + Timestamp sample_time = Timestamp::Zero(); + }; + + std::string GetTestCaseName(const std::string& stream_label) const; + static void ReportVideoBweResults(const std::string& test_case_name, + const VideoBweStats& video_bwe_stats); + // Report result for single metric for specified stream. + static void ReportResult(const std::string& metric_name, + const std::string& test_case_name, + const SamplesStatsCounter& counter, + const std::string& unit, + webrtc::test::ImproveDirection improve_direction = + webrtc::test::ImproveDirection::kNone); + Timestamp Now() const { return clock_->CurrentTime(); } + + Clock* const clock_; + + std::string test_case_name_; + absl::optional<Timestamp> start_time_; + + Mutex video_bwe_stats_lock_; + // Map between a peer connection label (provided by the framework) and + // its video BWE stats. + std::map<std::string, VideoBweStats> video_bwe_stats_ + RTC_GUARDED_BY(video_bwe_stats_lock_); + std::map<std::string, StatsSample> last_stats_sample_ + RTC_GUARDED_BY(video_bwe_stats_lock_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_METRICS_REPORTER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer_helper.cc b/third_party/libwebrtc/test/pc/e2e/analyzer_helper.cc new file mode 100644 index 0000000000..852f0a3435 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer_helper.cc @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/analyzer_helper.h" + +#include <utility> + +namespace webrtc { +namespace webrtc_pc_e2e { + +AnalyzerHelper::AnalyzerHelper() { + signaling_sequence_checker_.Detach(); +} + +void AnalyzerHelper::AddTrackToStreamMapping(std::string track_id, + std::string stream_label) { + RTC_DCHECK_RUN_ON(&signaling_sequence_checker_); + track_to_stream_map_.insert( + {std::move(track_id), StreamInfo{stream_label, stream_label}}); +} + +void AnalyzerHelper::AddTrackToStreamMapping(std::string track_id, + std::string stream_label, + std::string sync_group) { + RTC_DCHECK_RUN_ON(&signaling_sequence_checker_); + track_to_stream_map_.insert( + {std::move(track_id), + StreamInfo{std::move(stream_label), std::move(sync_group)}}); +} + +const AnalyzerHelper::StreamInfo& AnalyzerHelper::GetStreamInfoFromTrackId( + absl::string_view track_id) const { + RTC_DCHECK_RUN_ON(&signaling_sequence_checker_); + auto track_to_stream_pair = track_to_stream_map_.find(std::string(track_id)); + RTC_CHECK(track_to_stream_pair != track_to_stream_map_.end()); + return track_to_stream_pair->second; +} + +absl::string_view AnalyzerHelper::GetStreamLabelFromTrackId( + absl::string_view track_id) const { + return GetStreamInfoFromTrackId(track_id).stream_label; +} + +absl::string_view AnalyzerHelper::GetSyncGroupLabelFromTrackId( + absl::string_view track_id) const { + return GetStreamInfoFromTrackId(track_id).sync_group; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer_helper.h b/third_party/libwebrtc/test/pc/e2e/analyzer_helper.h new file mode 100644 index 0000000000..9cebd7015e --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/analyzer_helper.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ANALYZER_HELPER_H_ +#define TEST_PC_E2E_ANALYZER_HELPER_H_ + +#include <map> +#include <string> + +#include "absl/strings/string_view.h" +#include "api/sequence_checker.h" +#include "api/test/track_id_stream_info_map.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// This class is a utility that provides bookkeeping capabilities that +// are useful to associate stats reports track_ids to the remote stream info. +// The framework will populate an instance of this class and it will pass +// it to the Start method of Media Quality Analyzers. +// An instance of AnalyzerHelper must only be accessed from a single +// thread and since stats collection happens on the signaling thread, +// AddTrackToStreamMapping, GetStreamLabelFromTrackId and +// GetSyncGroupLabelFromTrackId must be invoked from the signaling thread. Get +// methods should be invoked only after all data is added. Mixing Get methods +// with adding new data may lead to undefined behaviour. +class AnalyzerHelper : public TrackIdStreamInfoMap { + public: + AnalyzerHelper(); + + void AddTrackToStreamMapping(std::string track_id, std::string stream_label); + void AddTrackToStreamMapping(std::string track_id, + std::string stream_label, + std::string sync_group); + + absl::string_view GetStreamLabelFromTrackId( + absl::string_view track_id) const override; + + absl::string_view GetSyncGroupLabelFromTrackId( + absl::string_view track_id) const override; + + private: + struct StreamInfo { + std::string stream_label; + std::string sync_group; + }; + + const StreamInfo& GetStreamInfoFromTrackId(absl::string_view track_id) const; + + SequenceChecker signaling_sequence_checker_; + std::map<std::string, StreamInfo> track_to_stream_map_ + RTC_GUARDED_BY(signaling_sequence_checker_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ANALYZER_HELPER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.cc b/third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.cc new file mode 100644 index 0000000000..96f661fd4f --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.cc @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/cross_media_metrics_reporter.h" + +#include <utility> +#include <vector> + +#include "api/stats/rtc_stats.h" +#include "api/stats/rtcstats_objects.h" +#include "api/units/timestamp.h" +#include "rtc_base/event.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +void CrossMediaMetricsReporter::Start( + absl::string_view test_case_name, + const TrackIdStreamInfoMap* reporter_helper) { + test_case_name_ = std::string(test_case_name); + reporter_helper_ = reporter_helper; +} + +void CrossMediaMetricsReporter::OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) { + auto inbound_stats = report->GetStatsOfType<RTCInboundRTPStreamStats>(); + std::map<absl::string_view, std::vector<const RTCInboundRTPStreamStats*>> + sync_group_stats; + for (const auto& stat : inbound_stats) { + auto media_source_stat = + report->GetAs<RTCMediaStreamTrackStats>(*stat->track_id); + if (stat->estimated_playout_timestamp.ValueOrDefault(0.) > 0 && + media_source_stat->track_identifier.is_defined()) { + sync_group_stats[reporter_helper_->GetSyncGroupLabelFromTrackId( + *media_source_stat->track_identifier)] + .push_back(stat); + } + } + + MutexLock lock(&mutex_); + for (const auto& pair : sync_group_stats) { + // If there is less than two streams, it is not a sync group. + if (pair.second.size() < 2) { + continue; + } + auto sync_group = std::string(pair.first); + const RTCInboundRTPStreamStats* audio_stat = pair.second[0]; + const RTCInboundRTPStreamStats* video_stat = pair.second[1]; + + RTC_CHECK(pair.second.size() == 2 && audio_stat->kind.is_defined() && + video_stat->kind.is_defined() && + *audio_stat->kind != *video_stat->kind) + << "Sync group should consist of one audio and one video stream."; + + if (*audio_stat->kind == RTCMediaStreamTrackKind::kVideo) { + std::swap(audio_stat, video_stat); + } + // Stream labels of a sync group are same for all polls, so we need it add + // it only once. + if (stats_info_.find(sync_group) == stats_info_.end()) { + auto audio_source_stat = + report->GetAs<RTCMediaStreamTrackStats>(*audio_stat->track_id); + auto video_source_stat = + report->GetAs<RTCMediaStreamTrackStats>(*video_stat->track_id); + // *_source_stat->track_identifier is always defined here because we + // checked it while grouping stats. + stats_info_[sync_group].audio_stream_label = + std::string(reporter_helper_->GetStreamLabelFromTrackId( + *audio_source_stat->track_identifier)); + stats_info_[sync_group].video_stream_label = + std::string(reporter_helper_->GetStreamLabelFromTrackId( + *video_source_stat->track_identifier)); + } + + double audio_video_playout_diff = *audio_stat->estimated_playout_timestamp - + *video_stat->estimated_playout_timestamp; + if (audio_video_playout_diff > 0) { + stats_info_[sync_group].audio_ahead_ms.AddSample( + audio_video_playout_diff); + stats_info_[sync_group].video_ahead_ms.AddSample(0); + } else { + stats_info_[sync_group].audio_ahead_ms.AddSample(0); + stats_info_[sync_group].video_ahead_ms.AddSample( + std::abs(audio_video_playout_diff)); + } + } +} + +void CrossMediaMetricsReporter::StopAndReportResults() { + MutexLock lock(&mutex_); + for (const auto& pair : stats_info_) { + const std::string& sync_group = pair.first; + ReportResult("audio_ahead_ms", + GetTestCaseName(pair.second.audio_stream_label, sync_group), + pair.second.audio_ahead_ms, "ms", + webrtc::test::ImproveDirection::kSmallerIsBetter); + ReportResult("video_ahead_ms", + GetTestCaseName(pair.second.video_stream_label, sync_group), + pair.second.video_ahead_ms, "ms", + webrtc::test::ImproveDirection::kSmallerIsBetter); + } +} + +void CrossMediaMetricsReporter::ReportResult( + const std::string& metric_name, + const std::string& test_case_name, + const SamplesStatsCounter& counter, + const std::string& unit, + webrtc::test::ImproveDirection improve_direction) { + test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit, + /*important=*/false, improve_direction); +} + +std::string CrossMediaMetricsReporter::GetTestCaseName( + const std::string& stream_label, + const std::string& sync_group) const { + return test_case_name_ + "/" + sync_group + "_" + stream_label; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.h b/third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.h new file mode 100644 index 0000000000..6ddc994d1f --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_CROSS_MEDIA_METRICS_REPORTER_H_ +#define TEST_PC_E2E_CROSS_MEDIA_METRICS_REPORTER_H_ + +#include <map> +#include <string> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/test/track_id_stream_info_map.h" +#include "api/units/timestamp.h" +#include "rtc_base/synchronization/mutex.h" +#include "test/testsupport/perf_test.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class CrossMediaMetricsReporter + : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { + public: + CrossMediaMetricsReporter() = default; + ~CrossMediaMetricsReporter() override = default; + + void Start(absl::string_view test_case_name, + const TrackIdStreamInfoMap* reporter_helper) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override; + void StopAndReportResults() override; + + private: + struct StatsInfo { + SamplesStatsCounter audio_ahead_ms; + SamplesStatsCounter video_ahead_ms; + + std::string audio_stream_label; + std::string video_stream_label; + }; + + static void ReportResult(const std::string& metric_name, + const std::string& test_case_name, + const SamplesStatsCounter& counter, + const std::string& unit, + webrtc::test::ImproveDirection improve_direction = + webrtc::test::ImproveDirection::kNone); + std::string GetTestCaseName(const std::string& stream_label, + const std::string& sync_group) const; + + std::string test_case_name_; + const TrackIdStreamInfoMap* reporter_helper_; + + Mutex mutex_; + std::map<std::string, StatsInfo> stats_info_ RTC_GUARDED_BY(mutex_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_CROSS_MEDIA_METRICS_REPORTER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.cc b/third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.cc new file mode 100644 index 0000000000..f2b4be9e0d --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.cc @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/echo/echo_emulation.h" + +#include <limits> +#include <utility> + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +constexpr int kSingleBufferDurationMs = 10; + +} // namespace + +EchoEmulatingCapturer::EchoEmulatingCapturer( + std::unique_ptr<TestAudioDeviceModule::Capturer> capturer, + PeerConnectionE2EQualityTestFixture::EchoEmulationConfig config) + : delegate_(std::move(capturer)), + config_(config), + renderer_queue_(2 * config_.echo_delay.ms() / kSingleBufferDurationMs), + queue_input_(TestAudioDeviceModule::SamplesPerFrame( + delegate_->SamplingFrequency()) * + delegate_->NumChannels()), + queue_output_(TestAudioDeviceModule::SamplesPerFrame( + delegate_->SamplingFrequency()) * + delegate_->NumChannels()) { + renderer_thread_.Detach(); + capturer_thread_.Detach(); +} + +void EchoEmulatingCapturer::OnAudioRendered( + rtc::ArrayView<const int16_t> data) { + RTC_DCHECK_RUN_ON(&renderer_thread_); + if (!recording_started_) { + // Because rendering can start before capturing in the beginning we can have + // a set of empty audio data frames. So we will skip them and will start + // fill the queue only after 1st non-empty audio data frame will arrive. + bool is_empty = true; + for (auto d : data) { + if (d != 0) { + is_empty = false; + break; + } + } + if (is_empty) { + return; + } + recording_started_ = true; + } + queue_input_.assign(data.begin(), data.end()); + if (!renderer_queue_.Insert(&queue_input_)) { + RTC_LOG(LS_WARNING) << "Echo queue is full"; + } +} + +bool EchoEmulatingCapturer::Capture(rtc::BufferT<int16_t>* buffer) { + RTC_DCHECK_RUN_ON(&capturer_thread_); + bool result = delegate_->Capture(buffer); + // Now we have to reduce input signal to avoid saturation when mixing in the + // fake echo. + for (size_t i = 0; i < buffer->size(); ++i) { + (*buffer)[i] /= 2; + } + + // When we accumulated enough delay in the echo buffer we will pop from + // that buffer on each ::Capture(...) call. If the buffer become empty it + // will mean some bug, so we will crash during removing item from the queue. + if (!delay_accumulated_) { + delay_accumulated_ = + renderer_queue_.SizeAtLeast() >= + static_cast<size_t>(config_.echo_delay.ms() / kSingleBufferDurationMs); + } + + if (delay_accumulated_) { + RTC_CHECK(renderer_queue_.Remove(&queue_output_)); + for (size_t i = 0; i < buffer->size() && i < queue_output_.size(); ++i) { + int32_t res = (*buffer)[i] + queue_output_[i]; + if (res < std::numeric_limits<int16_t>::min()) { + res = std::numeric_limits<int16_t>::min(); + } + if (res > std::numeric_limits<int16_t>::max()) { + res = std::numeric_limits<int16_t>::max(); + } + (*buffer)[i] = static_cast<int16_t>(res); + } + } + + return result; +} + +EchoEmulatingRenderer::EchoEmulatingRenderer( + std::unique_ptr<TestAudioDeviceModule::Renderer> renderer, + EchoEmulatingCapturer* echo_emulating_capturer) + : delegate_(std::move(renderer)), + echo_emulating_capturer_(echo_emulating_capturer) { + RTC_DCHECK(echo_emulating_capturer_); +} + +bool EchoEmulatingRenderer::Render(rtc::ArrayView<const int16_t> data) { + if (data.size() > 0) { + echo_emulating_capturer_->OnAudioRendered(data); + } + return delegate_->Render(data); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.h b/third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.h new file mode 100644 index 0000000000..d1d41f63a8 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_ECHO_ECHO_EMULATION_H_ +#define TEST_PC_E2E_ECHO_ECHO_EMULATION_H_ + +#include <atomic> +#include <deque> +#include <memory> +#include <vector> + +#include "api/test/peerconnection_quality_test_fixture.h" +#include "modules/audio_device/include/test_audio_device.h" +#include "rtc_base/swap_queue.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Reduces audio input strength from provided capturer twice and adds input +// provided into EchoEmulatingCapturer::OnAudioRendered(...). +class EchoEmulatingCapturer : public TestAudioDeviceModule::Capturer { + public: + EchoEmulatingCapturer( + std::unique_ptr<TestAudioDeviceModule::Capturer> capturer, + PeerConnectionE2EQualityTestFixture::EchoEmulationConfig config); + + void OnAudioRendered(rtc::ArrayView<const int16_t> data); + + int SamplingFrequency() const override { + return delegate_->SamplingFrequency(); + } + int NumChannels() const override { return delegate_->NumChannels(); } + bool Capture(rtc::BufferT<int16_t>* buffer) override; + + private: + std::unique_ptr<TestAudioDeviceModule::Capturer> delegate_; + const PeerConnectionE2EQualityTestFixture::EchoEmulationConfig config_; + + SwapQueue<std::vector<int16_t>> renderer_queue_; + + SequenceChecker renderer_thread_; + std::vector<int16_t> queue_input_ RTC_GUARDED_BY(renderer_thread_); + bool recording_started_ RTC_GUARDED_BY(renderer_thread_) = false; + + SequenceChecker capturer_thread_; + std::vector<int16_t> queue_output_ RTC_GUARDED_BY(capturer_thread_); + bool delay_accumulated_ RTC_GUARDED_BY(capturer_thread_) = false; +}; + +// Renders output into provided renderer and also copy output into provided +// EchoEmulationCapturer. +class EchoEmulatingRenderer : public TestAudioDeviceModule::Renderer { + public: + EchoEmulatingRenderer( + std::unique_ptr<TestAudioDeviceModule::Renderer> renderer, + EchoEmulatingCapturer* echo_emulating_capturer); + + int SamplingFrequency() const override { + return delegate_->SamplingFrequency(); + } + int NumChannels() const override { return delegate_->NumChannels(); } + bool Render(rtc::ArrayView<const int16_t> data) override; + + private: + std::unique_ptr<TestAudioDeviceModule::Renderer> delegate_; + EchoEmulatingCapturer* echo_emulating_capturer_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_ECHO_ECHO_EMULATION_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/architecture.md b/third_party/libwebrtc/test/pc/e2e/g3doc/architecture.md new file mode 100644 index 0000000000..aacdf7e5d5 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/g3doc/architecture.md @@ -0,0 +1,208 @@ +<?% config.freshness.reviewed = '2021-04-12' %?> + +# PeerConnection level framework fixture architecture + +## Overview + +The main implementation of +[`webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture`][1] is +[`webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTest`][2]. Internally it owns +the next main pieces: + +* [`MediaHelper`][3] - responsible for adding audio and video tracks to the + peers. +* [`VideoQualityAnalyzerInjectionHelper`][4] and + [`SingleProcessEncodedImageDataInjector`][5] - used to inject video quality + analysis and properly match captured and rendered video frames. You can read + more about it in + [DefaultVideoQualityAnalyzer](default_video_quality_analyzer.md) section. +* [`AudioQualityAnalyzerInterface`][6] - used to measure audio quality metrics +* [`TestActivitiesExecutor`][7] - used to support [`ExecuteAt(...)`][8] and + [`ExecuteEvery(...)`][9] API of `PeerConnectionE2EQualityTestFixture` to run + any arbitrary action during test execution timely synchronized with a test + call. +* A vector of [`QualityMetricsReporter`][10] added by the + `PeerConnectionE2EQualityTestFixture` user. +* Two peers: Alice and Bob represented by instances of [`TestPeer`][11] + object. + +Also it keeps a reference to [`webrtc::TimeController`][12], which is used to +create all required threads, task queues, task queue factories and time related +objects. + +## TestPeer + +Call participants are represented by instances of `TestPeer` object. +[`TestPeerFactory`][13] is used to create them. `TestPeer` owns all instances +related to the `webrtc::PeerConnection`, including required listeners and +callbacks. Also it provides an API to do offer/answer exchange and ICE candidate +exchange. For this purposes internally it uses an instance of +[`webrtc::PeerConnectionWrapper`][14]. + +The `TestPeer` also owns the `PeerConnection` worker thread. The signaling +thread for all `PeerConnection`'s is owned by +`PeerConnectionE2EQualityTestFixture` and shared between all participants in the +call. The network thread is owned by the network layer (it maybe either emulated +network provided by [Network Emulation Framework][24] or network thread and +`rtc::NetworkManager` provided by user) and provided when peer is added to the +fixture via [`AddPeer(...)`][15] API. + +## GetStats API based metrics reporters + +`PeerConnectionE2EQualityTestFixture` gives the user ability to provide +different `QualityMetricsReporter`s which will listen for `PeerConnection` +[`GetStats`][16] API. Then such reporters will be able to report various metrics +that user wants to measure. + +`PeerConnectionE2EQualityTestFixture` itself also uses this mechanism to +measure: + +* Audio quality metrics +* Audio/Video sync metrics (with help of [`CrossMediaMetricsReporter`][17]) + +Also framework provides a [`StatsBasedNetworkQualityMetricsReporter`][18] to +measure network related WebRTC metrics and print debug raw emulated network +statistic. This reporter should be added by user via +[`AddQualityMetricsReporter(...)`][19] API if requried. + +Internally stats gathering is done by [`StatsPoller`][20]. Stats are requested +once per second for each `PeerConnection` and then resulted object is provided +into each stats listener. + +## Offer/Answer exchange + +`PeerConnectionE2EQualityTest` provides ability to test Simulcast and SVC for +video. These features aren't supported by P2P call and in general requires a +Selective Forwarding Unit (SFU). So special logic is applied to mimic SFU +behavior in P2P call. This logic is located inside [`SignalingInterceptor`][21], +[`QualityAnalyzingVideoEncoder`][22] and [`QualityAnalyzingVideoDecoder`][23] +and consist of SDP modification during offer/answer exchange and special +handling of video frames from unrelated Simulcast/SVC streams during decoding. + +### Simulcast + +In case of Simulcast we have a video track, which internally contains multiple +video streams, for example low resolution, medium resolution and high +resolution. WebRTC client doesn't support receiving an offer with multiple +streams in it, because usually SFU will keep only single stream for the client. +To bypass it framework will modify offer by converting a single track with three +video streams into three independent video tracks. Then sender will think that +it send simulcast, but receiver will think that it receives 3 independent +tracks. + +To achieve such behavior some extra tweaks are required: + +* MID RTP header extension from original offer have to be removed +* RID RTP header extension from original offer is replaced with MID RTP header + extension, so the ID that sender uses for RID on receiver will be parsed as + MID. +* Answer have to be modified in the opposite way. + +Described modifications are illustrated on the picture below. + +![VP8 Simulcast offer modification](vp8_simulcast_offer_modification.png "VP8 Simulcast offer modification") + +The exchange will look like this: + +1. Alice creates an offer +2. Alice sets offer as local description +3. Do described offer modification +4. Alice sends modified offer to Bob +5. Bob sets modified offer as remote description +6. Bob creates answer +7. Bob sets answer as local description +8. Do reverse modifications on answer +9. Bob sends modified answer to Alice +10. Alice sets modified answer as remote description + +Such mechanism put a constraint that RTX streams are not supported, because they +don't have RID RTP header extension in their packets. + +### SVC + +In case of SVC the framework will update the sender's offer before even setting +it as local description on the sender side. Then no changes to answer will be +required. + +`ssrc` is a 32 bit random value that is generated in RTP to denote a specific +source used to send media in an RTP connection. In original offer video track +section will look like this: + +``` +m=video 9 UDP/TLS/RTP/SAVPF 98 100 99 101 +... +a=ssrc-group:FID <primary ssrc> <retransmission ssrc> +a=ssrc:<primary ssrc> cname:... +.... +a=ssrc:<retransmission ssrc> cname:... +.... +``` + +To enable SVC for such video track framework will add extra `ssrc`s for each SVC +stream that is required like this: + +``` +a=ssrc-group:FID <Low resolution primary ssrc> <Low resolution retransmission ssrc> +a=ssrc:<Low resolution primary ssrc> cname:... +.... +a=ssrc:<Low resolution retransmission ssrc> cname:.... +... +a=ssrc-group:FID <Medium resolution primary ssrc> <Medium resolution retransmission ssrc> +a=ssrc:<Medium resolution primary ssrc> cname:... +.... +a=ssrc:<Medium resolution retransmission ssrc> cname:.... +... +a=ssrc-group:FID <High resolution primary ssrc> <High resolution retransmission ssrc> +a=ssrc:<High resolution primary ssrc> cname:... +.... +a=ssrc:<High resolution retransmission ssrc> cname:.... +... +``` + +The next line will also be added to the video track section of the offer: + +``` +a=ssrc-group:SIM <Low resolution primary ssrc> <Medium resolution primary ssrc> <High resolution primary ssrc> +``` + +It will tell PeerConnection that this track should be configured as SVC. It +utilize WebRTC Plan B offer structure to achieve SVC behavior, also it modifies +offer before setting it as local description which violates WebRTC standard. +Also it adds limitations that on lossy networks only top resolution streams can +be analyzed, because WebRTC won't try to restore low resolution streams in case +of loss, because it still receives higher stream. + +### Handling in encoder/decoder + +In the encoder, the framework for each encoded video frame will propagate +information requried for the fake SFU to know if it belongs to an interesting +simulcast stream/spatial layer of if it should be "discarded". + +On the decoder side frames that should be "discarded" by fake SFU will be auto +decoded into single pixel images and only the interesting simulcast +stream/spatial layer will go into real decoder and then will be analyzed. + +[1]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=55;drc=484acf27231d931dbc99aedce85bc27e06486b96 +[2]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/peer_connection_quality_test.h;l=44;drc=6cc893ad778a0965e2b7a8e614f3c98aa81bee5b +[3]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/media/media_helper.h;l=27;drc=d46db9f1523ae45909b4a6fdc90a140443068bc6 +[4]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h;l=38;drc=79020414fd5c71f9ec1f25445ea5f1c8001e1a49 +[5]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h;l=40;drc=79020414fd5c71f9ec1f25445ea5f1c8001e1a49 +[6]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/audio_quality_analyzer_interface.h;l=23;drc=20f45823e37fd7272aa841831c029c21f29742c2 +[7]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/test_activities_executor.h;l=28;drc=6cc893ad778a0965e2b7a8e614f3c98aa81bee5b +[8]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=439;drc=484acf27231d931dbc99aedce85bc27e06486b96 +[9]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=445;drc=484acf27231d931dbc99aedce85bc27e06486b96 +[10]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=413;drc=9438fb3fff97c803d1ead34c0e4f223db168526f +[11]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/test_activities_executor.h;l=28;drc=6cc893ad778a0965e2b7a8e614f3c98aa81bee5b +[12]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/test_activities_executor.h;l=28;drc=6cc893ad778a0965e2b7a8e614f3c98aa81bee5b +[13]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/test_peer_factory.h;l=46;drc=0ef4a2488a466a24ab97b31fdddde55440d451f9 +[14]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/pc/peer_connection_wrapper.h;l=47;drc=5ab79e62f691875a237ea28ca3975ea1f0ed62ec +[15]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=459;drc=484acf27231d931dbc99aedce85bc27e06486b96 +[16]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/peer_connection_interface.h;l=886;drc=9438fb3fff97c803d1ead34c0e4f223db168526f +[17]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/cross_media_metrics_reporter.h;l=29;drc=9d777620236ec76754cfce19f6e82dd18e52d22c +[18]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/cross_media_metrics_reporter.h;l=29;drc=9d777620236ec76754cfce19f6e82dd18e52d22c +[19]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=450;drc=484acf27231d931dbc99aedce85bc27e06486b96 +[20]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/stats_poller.h;l=52;drc=9b526180c9e9722d3fc7f8689da6ec094fc7fc0a +[21]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/sdp/sdp_changer.h;l=79;drc=ee558dcca89fd8b105114ededf9e74d948da85e8 +[22]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h;l=54;drc=79020414fd5c71f9ec1f25445ea5f1c8001e1a49 +[23]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h;l=50;drc=79020414fd5c71f9ec1f25445ea5f1c8001e1a49 +[24]: /test/network/g3doc/index.md diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/default_video_quality_analyzer.md b/third_party/libwebrtc/test/pc/e2e/g3doc/default_video_quality_analyzer.md new file mode 100644 index 0000000000..ed182d833d --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/g3doc/default_video_quality_analyzer.md @@ -0,0 +1,196 @@ +<?% config.freshness.reviewed = '2021-02-21' %?> + +# DefaultVideoQualityAnalyzer + +## Audience + +This document is for users of +[`webrtc::webrtc_pc_e2e::DefaultVideoQualityAnalyzer`][1]. + +## Overview + +`DefaultVideoQualityAnalyzer` implements +[`webrtc::VideoQualityAnalyzerInterface`][2] and is a main +implementation of video quality analyzer for WebRTC. To operate correctly it +requires to receive video frame on each step: + +1. On frame captured - analyzer will generate a unique ID for the frame, that + caller should attach to the it. +2. Immediately before frame enter the encoder. +3. Immediately after the frame was encoded. +4. After the frame was received and immediately before it entered the decoder. +5. Immediately after the frame was decoded. +6. When the frame was rendered. + +![VideoQualityAnalyzerInterface pipeline](video_quality_analyzer_pipeline.png "VideoQualityAnalyzerInterface pipeline") + +The analyzer updates its internal metrics per frame when it was rendered and +reports all of them after it was stopped through +[WebRTC perf results reporting system][10]. + +To properly inject `DefaultVideoQualityAnalyzer` into pipeline the following helpers can be used: + +### VideoQualityAnalyzerInjectionHelper + +[`webrtc::webrtc_pc_e2e::VideoQualityAnalyzerInjectionHelper`][3] provides +factory methods for components, that will be used to inject +`VideoQualityAnalyzerInterface` into the `PeerConnection` pipeline: + +* Wrappers for [`webrtc::VideoEncoderFactory`][4] and + [`webrtc::VideoDecodeFactory`][5] which will properly pass + [`webrtc::VideoFrame`][6] and [`webrtc::EncodedImage`][7] into analyzer + before and after real video encode and decoder. +* [`webrtc::test::TestVideoCapturer::FramePreprocessor`][8] which is used to + pass generated frames into analyzer on capturing and then set the returned + frame ID. It also configures dumping of captured frames if requried. +* [`rtc::VideoSinkInterface<VideoFrame>`][9] which is used to pass frames to + the analyzer before they will be rendered to compute per frame metrics. It + also configures dumping of rendered video if requried. + +Besides factories `VideoQualityAnalyzerInjectionHelper` has method to +orchestrate `VideoQualityAnalyzerInterface` workflow: + +* `Start` - to start video analyzer, so it will be able to receive and analyze + video frames. +* `RegisterParticipantInCall` - to add new participants after analyzer was + started. +* `Stop` - to stop analyzer, compute all metrics for frames that were recevied + before and report them. + +Also `VideoQualityAnalyzerInjectionHelper` implements +[`webrtc::webrtc_pc_e2e::StatsObserverInterface`][11] to propagate WebRTC stats +to `VideoQualityAnalyzerInterface`. + +### EncodedImageDataInjector and EncodedImageDataExtractor + +[`webrtc::webrtc_pc_e2e::EncodedImageDataInjector`][14] and +[`webrtc::webrtc_pc_e2e::EncodedImageDataInjector`][15] are used to inject and +extract data into `webrtc::EncodedImage` to propagate frame ID and other +required information through the network. + +By default [`webrtc::webrtc_pc_e2e::SingleProcessEncodedImageDataInjector`][16] +is used. It assumes `webrtc::EncodedImage` payload as black box which is +remaining unchanged from encoder to decoder and stores the information required +for its work in the last 3 bytes of the payload, replacing the original data +during injection and restoring it back during extraction. Also +`SingleProcessEncodedImageDataInjector` requires that sender and receiver were +inside single process. + +![SingleProcessEncodedImageDataInjector](single_process_encoded_image_data_injector.png "SingleProcessEncodedImageDataInjector") + +## Exported metrics + +Exported metrics are reported to WebRTC perf results reporting system. + +### General + +* *`cpu_usage`* - CPU usage excluding video analyzer + +### Video + +* *`psnr`* - peak signal-to-noise ratio: + [wikipedia](https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio) +* *`ssim`* - structural similarity: + [wikipedia](https://en.wikipedia.org/wiki/Structural_similarity). +* *`min_psnr`* - minimum value of psnr across all frames of video stream. +* *`encode_time`* - time to encode a single frame. +* *`decode_time`* - time to decode a single frame. +* *`transport_time`* - time from frame encoded to frame received for decoding. +* *`receive_to_render_time`* - time from frame received for decoding to frame + rendered. +* *`total_delay_incl_transport`* - time from frame was captured on device to + time when frame was displayed on device. +* *`encode_frame_rate`* - frame rate after encoder. +* *`harmonic_framerate`* - video duration divided on squared sum of interframe + delays. Reflects render frame rate penalized by freezes. +* *`time_between_rendered_frames`* - time between frames out to renderer. +* *`dropped_frames`* - amount of frames that were sent, but weren't rendered + and are known not to be “on the way” from sender to receiver. + +Freeze is a pause when no new frames from decoder arrived for 150ms + avg time +between frames or 3 * avg time between frames. + +* *`time_between_freezes`* - mean time from previous freeze end to new freeze + start. +* *`freeze_time_ms`* - total freeze time in ms. +* *`max_skipped`* - frames skipped between two nearest rendered. +* *`pixels_per_frame`* - amount of pixels on frame (width * height). +* *`target_encode_bitrate`* - target encode bitrate provided by BWE to + encoder. +* *`actual_encode_bitrate -`* - actual encode bitrate produced by encoder. +* *`available_send_bandwidth -`* - available send bandwidth estimated by BWE. +* *`transmission_bitrate`* - bitrate of media in the emulated network, not + counting retransmissions FEC, and RTCP messages +* *`retransmission_bitrate`* - bitrate of retransmission streams only. + +### Framework stability + +* *`frames_in_flight`* - amount of frames that were captured but wasn't seen + on receiver. + +## Debug metrics + +Debug metrics are not reported to WebRTC perf results reporting system, but are +available through `DefaultVideoQualityAnalyzer` API. + +### [FrameCounters][12] + +Frame counters consist of next counters: + +* *`captured`* - count of frames, that were passed into WebRTC pipeline by + video stream source +* *`pre_encoded`* - count of frames that reached video encoder. +* *`encoded`* - count of encoded images that were produced by encoder for all + requested spatial layers and simulcast streams. +* *`received`* - count of encoded images received in decoder for all requested + spatial layers and simulcast streams. +* *`decoded`* - count of frames that were produced by decoder. +* *`rendered`* - count of frames that went out from WebRTC pipeline to video + sink. +* *`dropped`* - count of frames that were dropped in any point between + capturing and rendering. + +`DefaultVideoQualityAnalyzer` exports these frame counters: + +* *`GlobalCounters`* - frame counters for frames met on each stage of analysis + for all media streams. +* *`PerStreamCounters`* - frame counters for frames met on each stage of + analysis separated per individual video track (single media section in the + SDP offer). + +### [AnalyzerStats][13] + +Contains metrics about internal state of video analyzer during its work + +* *`comparisons_queue_size`* - size of analyzer internal queue used to perform + captured and rendered frames comparisons measured when new element is added + to the queue. +* *`comparisons_done`* - number of performed comparisons of 2 video frames + from captured and rendered streams. +* *`cpu_overloaded_comparisons_done`* - number of cpu overloaded comparisons. + Comparison is cpu overloaded if it is queued when there are too many not + processed comparisons in the queue. Overloaded comparison doesn't include + metrics like SSIM and PSNR that require heavy computations. +* *`memory_overloaded_comparisons_done`* - number of memory overloaded + comparisons. Comparison is memory overloaded if it is queued when its + captured frame was already removed due to high memory usage for that video + stream. +* *`frames_in_flight_left_count`* - count of frames in flight in analyzer + measured when new comparison is added and after analyzer was stopped. + +[1]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h;l=188;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[2]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/test/video_quality_analyzer_interface.h;l=56;drc=d7808f1c464a07c8f1e2f97ec7ee92fda998d590 +[3]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h;l=39;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[4]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/video_codecs/video_encoder_factory.h;l=27;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[5]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/video_codecs/video_decoder_factory.h;l=27;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[6]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/video/video_frame.h;l=30;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[7]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/video/encoded_image.h;l=71;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[8]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/test_video_capturer.h;l=28;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[9]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/video/video_sink_interface.h;l=19;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[10]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/testsupport/perf_test.h;drc=0710b401b1e5b500b8e84946fb657656ba1b58b7 +[11]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/test/stats_observer_interface.h;l=21;drc=9b526180c9e9722d3fc7f8689da6ec094fc7fc0a +[12]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h;l=57;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[13]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h;l=113;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2 +[14]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h;l=23;drc=c57089a97a3df454f4356d882cc8df173e8b3ead +[15]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h;l=46;drc=c57089a97a3df454f4356d882cc8df173e8b3ead +[16]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h;l=40;drc=c57089a97a3df454f4356d882cc8df173e8b3ead diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/g3doc.lua b/third_party/libwebrtc/test/pc/e2e/g3doc/g3doc.lua new file mode 100644 index 0000000000..981393c826 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/g3doc/g3doc.lua @@ -0,0 +1,5 @@ +config = super() + +config.freshness.owner = 'titovartem' + +return config diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/in_test_psnr_plot.png b/third_party/libwebrtc/test/pc/e2e/g3doc/in_test_psnr_plot.png Binary files differnew file mode 100644 index 0000000000..3f36725727 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/g3doc/in_test_psnr_plot.png diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/index.md b/third_party/libwebrtc/test/pc/e2e/g3doc/index.md new file mode 100644 index 0000000000..5a3c9a7e41 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/g3doc/index.md @@ -0,0 +1,223 @@ +<?% config.freshness.reviewed = '2021-04-12' %?> + +# PeerConnection Level Framework + +## API + +* [Fixture][1] +* [Fixture factory function][2] + +## Documentation + +The PeerConnection level framework is designed for end-to-end media quality +testing through the PeerConnection level public API. The framework uses the +*Unified plan* API to generate offers/answers during the signaling phase. The +framework also wraps the video encoder/decoder and inject it into +*`webrtc::PeerConnection`* to measure video quality, performing 1:1 frames +matching between captured and rendered frames without any extra requirements to +input video. For audio quality evaluation the standard `GetStats()` API from +PeerConnection is used. + +The framework API is located in the namespace *`webrtc::webrtc_pc_e2e`*. + +### Supported features + +* Single or bidirectional media in the call +* RTC Event log dump per peer +* AEC dump per peer +* Compatible with *`webrtc::TimeController`* for both real and simulated time +* Media + * AV sync +* Video + * Any amount of video tracks both from caller and callee sides + * Input video from + * Video generator + * Specified file + * Any instance of *`webrtc::test::FrameGeneratorInterface`* + * Dumping of captured/rendered video into file + * Screen sharing + * Vp8 simulcast from caller side + * Vp9 SVC from caller side + * Choosing of video codec (name and parameters), having multiple codecs + negotiated to support codec-switching testing. + * FEC (ULP or Flex) + * Forced codec overshooting (for encoder overshoot emulation on some + mobile devices, when hardware encoder can overshoot target bitrate) +* Audio + * Up to 1 audio track both from caller and callee sides + * Generated audio + * Audio from specified file + * Dumping of captured/rendered audio into file + * Parameterizing of `cricket::AudioOptions` + * Echo emulation +* Injection of various WebRTC components into underlying + *`webrtc::PeerConnection`* or *`webrtc::PeerConnectionFactory`*. You can see + the full list [here][11] +* Scheduling of events, that can happen during the test, for example: + * Changes in network configuration + * User statistics measurements + * Custom defined actions +* User defined statistics reporting via + *`webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::QualityMetricsReporter`* + interface + +## Exported metrics + +### General + +* *`<peer_name>_connected`* - peer successfully established connection to + remote side +* *`cpu_usage`* - CPU usage excluding video analyzer +* *`audio_ahead_ms`* - Used to estimate how much audio and video is out of + sync when the two tracks were from the same source. Stats are polled + periodically during a call. The metric represents how much earlier was audio + played out on average over the call. If, during a stats poll, video is + ahead, then audio_ahead_ms will be equal to 0 for this poll. +* *`video_ahead_ms`* - Used to estimate how much audio and video is out of + sync when the two tracks were from the same source. Stats are polled + periodically during a call. The metric represents how much earlier was video + played out on average over the call. If, during a stats poll, audio is + ahead, then video_ahead_ms will be equal to 0 for this poll. + +### Video + +See documentation for +[*`DefaultVideoQualityAnalyzer`*](default_video_quality_analyzer.md#exported-metrics) + +### Audio + +* *`accelerate_rate`* - when playout is sped up, this counter is increased by + the difference between the number of samples received and the number of + samples played out. If speedup is achieved by removing samples, this will be + the count of samples removed. Rate is calculated as difference between + nearby samples divided on sample interval. +* *`expand_rate`* - the total number of samples that are concealed samples + over time. A concealed sample is a sample that was replaced with synthesized + samples generated locally before being played out. Examples of samples that + have to be concealed are samples from lost packets or samples from packets + that arrive too late to be played out +* *`speech_expand_rate`* - the total number of samples that are concealed + samples minus the total number of concealed samples inserted that are + "silent" over time. Playing out silent samples results in silence or comfort + noise. +* *`preemptive_rate`* - when playout is slowed down, this counter is increased + by the difference between the number of samples received and the number of + samples played out. If playout is slowed down by inserting samples, this + will be the number of inserted samples. Rate is calculated as difference + between nearby samples divided on sample interval. +* *`average_jitter_buffer_delay_ms`* - average size of NetEQ jitter buffer. +* *`preferred_buffer_size_ms`* - preferred size of NetEQ jitter buffer. +* *`visqol_mos`* - proxy for audio quality itself. +* *`asdm_samples`* - measure of how much acceleration/deceleration was in the + signal. +* *`word_error_rate`* - measure of how intelligible the audio was (percent of + words that could not be recognized in output audio). + +### Network + +* *`bytes_sent`* - represents the total number of payload bytes sent on this + PeerConnection, i.e., not including headers or padding +* *`packets_sent`* - represents the total number of packets sent over this + PeerConnection’s transports. +* *`average_send_rate`* - average send rate calculated on bytes_sent divided + by test duration. +* *`payload_bytes_sent`* - total number of bytes sent for all SSRC plus total + number of RTP header and padding bytes sent for all SSRC. This does not + include the size of transport layer headers such as IP or UDP. +* *`sent_packets_loss`* - packets_sent minus corresponding packets_received. +* *`bytes_received`* - represents the total number of bytes received on this + PeerConnection, i.e., not including headers or padding. +* *`packets_received`* - represents the total number of packets received on + this PeerConnection’s transports. +* *`average_receive_rate`* - average receive rate calculated on bytes_received + divided by test duration. +* *`payload_bytes_received`* - total number of bytes received for all SSRC + plus total number of RTP header and padding bytes received for all SSRC. + This does not include the size of transport layer headers such as IP or UDP. + +### Framework stability + +* *`frames_in_flight`* - amount of frames that were captured but wasn't seen + on receiver in the way that also all frames after also weren't seen on + receiver. +* *`bytes_discarded_no_receiver`* - total number of bytes that were received + on network interfaces related to the peer, but destination port was closed. +* *`packets_discarded_no_receiver`* - total number of packets that were + received on network interfaces related to the peer, but destination port was + closed. + +## Examples + +Examples can be found in + +* [peer_connection_e2e_smoke_test.cc][3] +* [pc_full_stack_tests.cc][4] + +## Stats plotting + +### Description + +Stats plotting provides ability to plot statistic collected during the test. +Right now it is used in PeerConnection level framework and give ability to see +how video quality metrics changed during test execution. + +### Usage + +To make any metrics plottable you need: + +1. Collect metric data with [SamplesStatsCounter][5] which internally will + store all intermediate points and timestamps when these points were added. +2. Then you need to report collected data with + [`webrtc::test::PrintResult(...)`][6]. By using these method you will also + specify name of the plottable metric. + +After these steps it will be possible to export your metric for plotting. There +are several options how you can do this: + +1. Use [`webrtc::TestMain::Create()`][7] as `main` function implementation, for + example use [`test/test_main.cc`][8] as `main` function for your test. + + In such case your binary will have flag `--plot`, where you can provide a + list of metrics, that you want to plot or specify `all` to plot all + available metrics. + + If `--plot` is specified, the binary will output metrics data into `stdout`. + Then you need to pipe this `stdout` into python plotter script + [`rtc_tools/metrics_plotter.py`][9], which will plot data. + + Examples: + + ```shell + $ ./out/Default/test_support_unittests \ + --gtest_filter=PeerConnectionE2EQualityTestSmokeTest.Svc \ + --nologs \ + --plot=all \ + | python rtc_tools/metrics_plotter.py + ``` + + ```shell + $ ./out/Default/test_support_unittests \ + --gtest_filter=PeerConnectionE2EQualityTestSmokeTest.Svc \ + --nologs \ + --plot=psnr,ssim \ + | python rtc_tools/metrics_plotter.py + ``` + + Example chart: ![PSNR changes during the test](in_test_psnr_plot.png) + +2. Use API from [`test/testsupport/perf_test.h`][10] directly by invoking + `webrtc::test::PrintPlottableResults(const std::vector<std::string>& + desired_graphs)` to print plottable metrics to stdout. Then as in previous + option you need to pipe result into plotter script. + +[1]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;drc=cbe6e8a2589a925d4c91a2ac2c69201f03de9c39 +[2]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/create_peerconnection_quality_test_fixture.h;drc=cbe6e8a2589a925d4c91a2ac2c69201f03de9c39 +[3]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/peer_connection_e2e_smoke_test.cc;drc=cbe6e8a2589a925d4c91a2ac2c69201f03de9c39 +[4]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/video/pc_full_stack_tests.cc;drc=cbe6e8a2589a925d4c91a2ac2c69201f03de9c39 +[5]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/numerics/samples_stats_counter.h;drc=cbe6e8a2589a925d4c91a2ac2c69201f03de9c39 +[6]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/testsupport/perf_test.h;l=86;drc=0710b401b1e5b500b8e84946fb657656ba1b58b7 +[7]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/test_main_lib.h;l=23;drc=bcb42f1e4be136c390986a40d9d5cb3ad0de260b +[8]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/test_main.cc;drc=bcb42f1e4be136c390986a40d9d5cb3ad0de260b +[9]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/rtc_tools/metrics_plotter.py;drc=8cc6695652307929edfc877cd64b75cd9ec2d615 +[10]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/testsupport/perf_test.h;l=105;drc=0710b401b1e5b500b8e84946fb657656ba1b58b7 +[11]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=272;drc=484acf27231d931dbc99aedce85bc27e06486b96 diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/single_process_encoded_image_data_injector.png b/third_party/libwebrtc/test/pc/e2e/g3doc/single_process_encoded_image_data_injector.png Binary files differnew file mode 100644 index 0000000000..73480bafbe --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/g3doc/single_process_encoded_image_data_injector.png diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/video_quality_analyzer_pipeline.png b/third_party/libwebrtc/test/pc/e2e/g3doc/video_quality_analyzer_pipeline.png Binary files differnew file mode 100644 index 0000000000..6cddb91110 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/g3doc/video_quality_analyzer_pipeline.png diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/vp8_simulcast_offer_modification.png b/third_party/libwebrtc/test/pc/e2e/g3doc/vp8_simulcast_offer_modification.png Binary files differnew file mode 100644 index 0000000000..c7eaa04c0e --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/g3doc/vp8_simulcast_offer_modification.png diff --git a/third_party/libwebrtc/test/pc/e2e/media/media_helper.cc b/third_party/libwebrtc/test/pc/e2e/media/media_helper.cc new file mode 100644 index 0000000000..22fcc51ffe --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/media/media_helper.cc @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/media/media_helper.h" + +#include <string> +#include <utility> + +#include "absl/types/variant.h" +#include "api/media_stream_interface.h" +#include "api/test/create_frame_generator.h" +#include "test/frame_generator_capturer.h" +#include "test/platform_video_capturer.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using VideoConfig = + ::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::VideoConfig; +using AudioConfig = + ::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::AudioConfig; +using CapturingDeviceIndex = ::webrtc::webrtc_pc_e2e:: + PeerConnectionE2EQualityTestFixture::CapturingDeviceIndex; + +} // namespace + +void MediaHelper::MaybeAddAudio(TestPeer* peer) { + if (!peer->params().audio_config) { + return; + } + const AudioConfig& audio_config = peer->params().audio_config.value(); + rtc::scoped_refptr<webrtc::AudioSourceInterface> source = + peer->pc_factory()->CreateAudioSource(audio_config.audio_options); + rtc::scoped_refptr<AudioTrackInterface> track = + peer->pc_factory()->CreateAudioTrack(*audio_config.stream_label, + source.get()); + std::string sync_group = audio_config.sync_group + ? audio_config.sync_group.value() + : audio_config.stream_label.value(); + peer->AddTrack(track, {sync_group, *audio_config.stream_label}); +} + +std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>> +MediaHelper::MaybeAddVideo(TestPeer* peer) { + // Params here valid because of pre-run validation. + const Params& params = peer->params(); + const ConfigurableParams& configurable_params = peer->configurable_params(); + std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>> out; + for (size_t i = 0; i < configurable_params.video_configs.size(); ++i) { + const VideoConfig& video_config = configurable_params.video_configs[i]; + // Setup input video source into peer connection. + std::unique_ptr<test::TestVideoCapturer> capturer = CreateVideoCapturer( + video_config, peer->ReleaseVideoSource(i), + video_quality_analyzer_injection_helper_->CreateFramePreprocessor( + params.name.value(), video_config)); + bool is_screencast = + video_config.content_hint == VideoTrackInterface::ContentHint::kText || + video_config.content_hint == + VideoTrackInterface::ContentHint::kDetailed; + rtc::scoped_refptr<TestVideoCapturerVideoTrackSource> source = + rtc::make_ref_counted<TestVideoCapturerVideoTrackSource>( + std::move(capturer), is_screencast); + out.push_back(source); + RTC_LOG(LS_INFO) << "Adding video with video_config.stream_label=" + << video_config.stream_label.value(); + rtc::scoped_refptr<VideoTrackInterface> track = + peer->pc_factory()->CreateVideoTrack(video_config.stream_label.value(), + source.get()); + if (video_config.content_hint.has_value()) { + track->set_content_hint(video_config.content_hint.value()); + } + std::string sync_group = video_config.sync_group + ? video_config.sync_group.value() + : video_config.stream_label.value(); + RTCErrorOr<rtc::scoped_refptr<RtpSenderInterface>> sender = + peer->AddTrack(track, {sync_group, *video_config.stream_label}); + RTC_CHECK(sender.ok()); + if (video_config.temporal_layers_count) { + RtpParameters rtp_parameters = sender.value()->GetParameters(); + for (auto& encoding_parameters : rtp_parameters.encodings) { + encoding_parameters.num_temporal_layers = + video_config.temporal_layers_count; + } + RTCError res = sender.value()->SetParameters(rtp_parameters); + RTC_CHECK(res.ok()) << "Failed to set RTP parameters"; + } + } + return out; +} + +std::unique_ptr<test::TestVideoCapturer> MediaHelper::CreateVideoCapturer( + const VideoConfig& video_config, + PeerConfigurerImpl::VideoSource source, + std::unique_ptr<test::TestVideoCapturer::FramePreprocessor> + frame_preprocessor) { + CapturingDeviceIndex* capturing_device_index = + absl::get_if<CapturingDeviceIndex>(&source); + if (capturing_device_index != nullptr) { + std::unique_ptr<test::TestVideoCapturer> capturer = + test::CreateVideoCapturer(video_config.width, video_config.height, + video_config.fps, + static_cast<size_t>(*capturing_device_index)); + RTC_CHECK(capturer) + << "Failed to obtain input stream from capturing device #" + << *capturing_device_index; + capturer->SetFramePreprocessor(std::move(frame_preprocessor)); + return capturer; + } + + auto capturer = std::make_unique<test::FrameGeneratorCapturer>( + clock_, + absl::get<std::unique_ptr<test::FrameGeneratorInterface>>( + std::move(source)), + video_config.fps, *task_queue_factory_); + capturer->SetFramePreprocessor(std::move(frame_preprocessor)); + capturer->Init(); + return capturer; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/media/media_helper.h b/third_party/libwebrtc/test/pc/e2e/media/media_helper.h new file mode 100644 index 0000000000..4e977e3002 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/media/media_helper.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_MEDIA_MEDIA_HELPER_H_ +#define TEST_PC_E2E_MEDIA_MEDIA_HELPER_H_ + +#include <memory> +#include <vector> + +#include "api/test/frame_generator_interface.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h" +#include "test/pc/e2e/media/test_video_capturer_video_track_source.h" +#include "test/pc/e2e/peer_configurer.h" +#include "test/pc/e2e/test_peer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class MediaHelper { + public: + MediaHelper(VideoQualityAnalyzerInjectionHelper* + video_quality_analyzer_injection_helper, + TaskQueueFactory* task_queue_factory, + Clock* clock) + : clock_(clock), + task_queue_factory_(task_queue_factory), + video_quality_analyzer_injection_helper_( + video_quality_analyzer_injection_helper) {} + + void MaybeAddAudio(TestPeer* peer); + + std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>> + MaybeAddVideo(TestPeer* peer); + + private: + std::unique_ptr<test::TestVideoCapturer> CreateVideoCapturer( + const PeerConnectionE2EQualityTestFixture::VideoConfig& video_config, + PeerConfigurerImpl::VideoSource source, + std::unique_ptr<test::TestVideoCapturer::FramePreprocessor> + frame_preprocessor); + + Clock* const clock_; + TaskQueueFactory* const task_queue_factory_; + VideoQualityAnalyzerInjectionHelper* video_quality_analyzer_injection_helper_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_MEDIA_MEDIA_HELPER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/media/test_video_capturer_video_track_source.h b/third_party/libwebrtc/test/pc/e2e/media/test_video_capturer_video_track_source.h new file mode 100644 index 0000000000..c883a2e8e9 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/media/test_video_capturer_video_track_source.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_MEDIA_TEST_VIDEO_CAPTURER_VIDEO_TRACK_SOURCE_H_ +#define TEST_PC_E2E_MEDIA_TEST_VIDEO_CAPTURER_VIDEO_TRACK_SOURCE_H_ + +#include <memory> +#include <utility> + +#include "api/video/video_frame.h" +#include "api/video/video_source_interface.h" +#include "pc/video_track_source.h" +#include "test/test_video_capturer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class TestVideoCapturerVideoTrackSource : public VideoTrackSource { + public: + TestVideoCapturerVideoTrackSource( + std::unique_ptr<test::TestVideoCapturer> video_capturer, + bool is_screencast) + : VideoTrackSource(/*remote=*/false), + video_capturer_(std::move(video_capturer)), + is_screencast_(is_screencast) {} + + ~TestVideoCapturerVideoTrackSource() = default; + + void Start() { SetState(kLive); } + + void Stop() { SetState(kMuted); } + + bool is_screencast() const override { return is_screencast_; } + + protected: + rtc::VideoSourceInterface<VideoFrame>* source() override { + return video_capturer_.get(); + } + + private: + std::unique_ptr<test::TestVideoCapturer> video_capturer_; + const bool is_screencast_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_MEDIA_TEST_VIDEO_CAPTURER_VIDEO_TRACK_SOURCE_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.cc b/third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.cc new file mode 100644 index 0000000000..513bdc0a5f --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.cc @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/network_quality_metrics_reporter.h" + +#include <utility> + +#include "api/stats/rtc_stats.h" +#include "api/stats/rtcstats_objects.h" +#include "rtc_base/event.h" +#include "system_wrappers/include/field_trial.h" +#include "test/testsupport/perf_test.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +constexpr int kStatsWaitTimeoutMs = 1000; + +// Field trial which controls whether to report standard-compliant bytes +// sent/received per stream. If enabled, padding and headers are not included +// in bytes sent or received. +constexpr char kUseStandardBytesStats[] = "WebRTC-UseStandardBytesStats"; +} + +void NetworkQualityMetricsReporter::Start( + absl::string_view test_case_name, + const TrackIdStreamInfoMap* /*reporter_helper*/) { + test_case_name_ = std::string(test_case_name); + // Check that network stats are clean before test execution. + std::unique_ptr<EmulatedNetworkStats> alice_stats = + PopulateStats(alice_network_); + RTC_CHECK_EQ(alice_stats->PacketsSent(), 0); + RTC_CHECK_EQ(alice_stats->PacketsReceived(), 0); + std::unique_ptr<EmulatedNetworkStats> bob_stats = PopulateStats(bob_network_); + RTC_CHECK_EQ(bob_stats->PacketsSent(), 0); + RTC_CHECK_EQ(bob_stats->PacketsReceived(), 0); +} + +void NetworkQualityMetricsReporter::OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) { + DataSize payload_received = DataSize::Zero(); + DataSize payload_sent = DataSize::Zero(); + + auto inbound_stats = report->GetStatsOfType<RTCInboundRTPStreamStats>(); + for (const auto& stat : inbound_stats) { + payload_received += + DataSize::Bytes(stat->bytes_received.ValueOrDefault(0ul) + + stat->header_bytes_received.ValueOrDefault(0ul)); + } + + auto outbound_stats = report->GetStatsOfType<RTCOutboundRTPStreamStats>(); + for (const auto& stat : outbound_stats) { + payload_sent += + DataSize::Bytes(stat->bytes_sent.ValueOrDefault(0ul) + + stat->header_bytes_sent.ValueOrDefault(0ul)); + } + + MutexLock lock(&lock_); + PCStats& stats = pc_stats_[std::string(pc_label)]; + stats.payload_received = payload_received; + stats.payload_sent = payload_sent; +} + +void NetworkQualityMetricsReporter::StopAndReportResults() { + std::unique_ptr<EmulatedNetworkStats> alice_stats = + PopulateStats(alice_network_); + std::unique_ptr<EmulatedNetworkStats> bob_stats = PopulateStats(bob_network_); + int64_t alice_packets_loss = + alice_stats->PacketsSent() - bob_stats->PacketsReceived(); + int64_t bob_packets_loss = + bob_stats->PacketsSent() - alice_stats->PacketsReceived(); + ReportStats("alice", std::move(alice_stats), alice_packets_loss); + ReportStats("bob", std::move(bob_stats), bob_packets_loss); + + if (!webrtc::field_trial::IsEnabled(kUseStandardBytesStats)) { + RTC_LOG(LS_ERROR) + << "Non-standard GetStats; \"payload\" counts include RTP headers"; + } + + MutexLock lock(&lock_); + for (const auto& pair : pc_stats_) { + ReportPCStats(pair.first, pair.second); + } +} + +std::unique_ptr<EmulatedNetworkStats> +NetworkQualityMetricsReporter::PopulateStats( + EmulatedNetworkManagerInterface* network) { + rtc::Event wait; + std::unique_ptr<EmulatedNetworkStats> stats; + network->GetStats([&](std::unique_ptr<EmulatedNetworkStats> s) { + stats = std::move(s); + wait.Set(); + }); + bool stats_received = wait.Wait(kStatsWaitTimeoutMs); + RTC_CHECK(stats_received); + return stats; +} + +void NetworkQualityMetricsReporter::ReportStats( + const std::string& network_label, + std::unique_ptr<EmulatedNetworkStats> stats, + int64_t packet_loss) { + ReportResult("bytes_sent", network_label, stats->BytesSent().bytes(), + "sizeInBytes"); + ReportResult("packets_sent", network_label, stats->PacketsSent(), "unitless"); + ReportResult( + "average_send_rate", network_label, + stats->PacketsSent() >= 2 ? stats->AverageSendRate().bytes_per_sec() : 0, + "bytesPerSecond"); + ReportResult("bytes_discarded_no_receiver", network_label, + stats->BytesDropped().bytes(), "sizeInBytes"); + ReportResult("packets_discarded_no_receiver", network_label, + stats->PacketsDropped(), "unitless"); + ReportResult("bytes_received", network_label, stats->BytesReceived().bytes(), + "sizeInBytes"); + ReportResult("packets_received", network_label, stats->PacketsReceived(), + "unitless"); + ReportResult("average_receive_rate", network_label, + stats->PacketsReceived() >= 2 + ? stats->AverageReceiveRate().bytes_per_sec() + : 0, + "bytesPerSecond"); + ReportResult("sent_packets_loss", network_label, packet_loss, "unitless"); +} + +void NetworkQualityMetricsReporter::ReportPCStats(const std::string& pc_label, + const PCStats& stats) { + ReportResult("payload_bytes_received", pc_label, + stats.payload_received.bytes(), "sizeInBytes"); + ReportResult("payload_bytes_sent", pc_label, stats.payload_sent.bytes(), + "sizeInBytes"); +} + +void NetworkQualityMetricsReporter::ReportResult( + const std::string& metric_name, + const std::string& network_label, + const double value, + const std::string& unit) const { + test::PrintResult(metric_name, /*modifier=*/"", + GetTestCaseName(network_label), value, unit, + /*important=*/false); +} + +std::string NetworkQualityMetricsReporter::GetTestCaseName( + const std::string& network_label) const { + return test_case_name_ + "/" + network_label; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.h b/third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.h new file mode 100644 index 0000000000..5cedce1e7f --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_ +#define TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_ + +#include <memory> +#include <string> + +#include "absl/strings/string_view.h" +#include "api/test/network_emulation_manager.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/test/track_id_stream_info_map.h" +#include "api/units/data_size.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class NetworkQualityMetricsReporter + : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { + public: + NetworkQualityMetricsReporter(EmulatedNetworkManagerInterface* alice_network, + EmulatedNetworkManagerInterface* bob_network) + : alice_network_(alice_network), bob_network_(bob_network) {} + ~NetworkQualityMetricsReporter() override = default; + + // Network stats must be empty when this method will be invoked. + void Start(absl::string_view test_case_name, + const TrackIdStreamInfoMap* reporter_helper) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override; + void StopAndReportResults() override; + + private: + struct PCStats { + // TODO(nisse): Separate audio and video counters. Depends on standard stat + // counters, enabled by field trial "WebRTC-UseStandardBytesStats". + DataSize payload_received = DataSize::Zero(); + DataSize payload_sent = DataSize::Zero(); + }; + + static std::unique_ptr<EmulatedNetworkStats> PopulateStats( + EmulatedNetworkManagerInterface* network); + void ReportStats(const std::string& network_label, + std::unique_ptr<EmulatedNetworkStats> stats, + int64_t packet_loss); + void ReportPCStats(const std::string& pc_label, const PCStats& stats); + void ReportResult(const std::string& metric_name, + const std::string& network_label, + double value, + const std::string& unit) const; + std::string GetTestCaseName(const std::string& network_label) const; + + std::string test_case_name_; + + EmulatedNetworkManagerInterface* alice_network_; + EmulatedNetworkManagerInterface* bob_network_; + Mutex lock_; + std::map<std::string, PCStats> pc_stats_ RTC_GUARDED_BY(lock_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/peer_configurer.cc b/third_party/libwebrtc/test/pc/e2e/peer_configurer.cc new file mode 100644 index 0000000000..998f789cc1 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/peer_configurer.cc @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/peer_configurer.h" + +#include <set> + +#include "absl/strings/string_view.h" +#include "rtc_base/arraysize.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using AudioConfig = PeerConnectionE2EQualityTestFixture::AudioConfig; +using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig; +using RunParams = PeerConnectionE2EQualityTestFixture::RunParams; +using VideoCodecConfig = PeerConnectionE2EQualityTestFixture::VideoCodecConfig; + +// List of default names of generic participants according to +// https://en.wikipedia.org/wiki/Alice_and_Bob +constexpr absl::string_view kDefaultNames[] = {"alice", "bob", "charlie", + "david", "erin", "frank"}; + +} // namespace + +DefaultNamesProvider::DefaultNamesProvider( + absl::string_view prefix, + rtc::ArrayView<const absl::string_view> default_names) + : prefix_(prefix), default_names_(default_names) {} + +void DefaultNamesProvider::MaybeSetName(absl::optional<std::string>& name) { + if (name.has_value()) { + known_names_.insert(name.value()); + } else { + name = GenerateName(); + } +} + +std::string DefaultNamesProvider::GenerateName() { + std::string name; + do { + name = GenerateNameInternal(); + } while (!known_names_.insert(name).second); + return name; +} + +std::string DefaultNamesProvider::GenerateNameInternal() { + if (counter_ < default_names_.size()) { + return std::string(default_names_[counter_++]); + } + return prefix_ + std::to_string(counter_++); +} + +PeerParamsPreprocessor::PeerParamsPreprocessor() + : peer_names_provider_("peer_", kDefaultNames) {} + +void PeerParamsPreprocessor::SetDefaultValuesForMissingParams( + PeerConfigurerImpl& peer) { + Params* params = peer.params(); + ConfigurableParams* configurable_params = peer.configurable_params(); + peer_names_provider_.MaybeSetName(params->name); + DefaultNamesProvider video_stream_names_provider(*params->name + + "_auto_video_stream_label_"); + for (VideoConfig& config : configurable_params->video_configs) { + video_stream_names_provider.MaybeSetName(config.stream_label); + } + if (params->audio_config) { + DefaultNamesProvider audio_stream_names_provider( + *params->name + "_auto_audio_stream_label_"); + audio_stream_names_provider.MaybeSetName( + params->audio_config->stream_label); + } + + if (params->video_codecs.empty()) { + params->video_codecs.push_back( + PeerConnectionE2EQualityTestFixture::VideoCodecConfig( + cricket::kVp8CodecName)); + } +} + +void PeerParamsPreprocessor::ValidateParams(const PeerConfigurerImpl& peer) { + const Params& p = peer.params(); + RTC_CHECK_GT(p.video_encoder_bitrate_multiplier, 0.0); + // Each peer should at least support 1 video codec. + RTC_CHECK_GE(p.video_codecs.size(), 1); + + { + RTC_CHECK(p.name); + bool inserted = peer_names_.insert(p.name.value()).second; + RTC_CHECK(inserted) << "Duplicate name=" << p.name.value(); + } + + // Validate that all video stream labels are unique and sync groups are + // valid. + for (const VideoConfig& video_config : + peer.configurable_params().video_configs) { + RTC_CHECK(video_config.stream_label); + bool inserted = + video_labels_.insert(video_config.stream_label.value()).second; + RTC_CHECK(inserted) << "Duplicate video_config.stream_label=" + << video_config.stream_label.value(); + + if (video_config.input_dump_file_name.has_value()) { + RTC_CHECK_GT(video_config.input_dump_sampling_modulo, 0) + << "video_config.input_dump_sampling_modulo must be greater than 0"; + } + if (video_config.output_dump_file_name.has_value()) { + RTC_CHECK_GT(video_config.output_dump_sampling_modulo, 0) + << "video_config.input_dump_sampling_modulo must be greater than 0"; + } + + // TODO(bugs.webrtc.org/4762): remove this check after synchronization of + // more than two streams is supported. + if (video_config.sync_group.has_value()) { + bool sync_group_inserted = + video_sync_groups_.insert(video_config.sync_group.value()).second; + RTC_CHECK(sync_group_inserted) + << "Sync group shouldn't consist of more than two streams (one " + "video and one audio). Duplicate video_config.sync_group=" + << video_config.sync_group.value(); + } + + if (video_config.simulcast_config) { + if (video_config.simulcast_config->target_spatial_index) { + RTC_CHECK_GE(*video_config.simulcast_config->target_spatial_index, 0); + RTC_CHECK_LT(*video_config.simulcast_config->target_spatial_index, + video_config.simulcast_config->simulcast_streams_count); + } + if (!video_config.encoding_params.empty()) { + RTC_CHECK_EQ(video_config.simulcast_config->simulcast_streams_count, + video_config.encoding_params.size()) + << "|encoding_params| have to be specified for each simulcast " + << "stream in |video_config|."; + } + } else { + RTC_CHECK_LE(video_config.encoding_params.size(), 1) + << "|encoding_params| has multiple values but simulcast is not " + "enabled."; + } + } + if (p.audio_config) { + bool inserted = + audio_labels_.insert(p.audio_config->stream_label.value()).second; + RTC_CHECK(inserted) << "Duplicate audio_config.stream_label=" + << p.audio_config->stream_label.value(); + // TODO(bugs.webrtc.org/4762): remove this check after synchronization of + // more than two streams is supported. + if (p.audio_config->sync_group.has_value()) { + bool sync_group_inserted = + audio_sync_groups_.insert(p.audio_config->sync_group.value()).second; + RTC_CHECK(sync_group_inserted) + << "Sync group shouldn't consist of more than two streams (one " + "video and one audio). Duplicate audio_config.sync_group=" + << p.audio_config->sync_group.value(); + } + // Check that if mode input file name specified only if mode is kFile. + if (p.audio_config.value().mode == AudioConfig::Mode::kGenerated) { + RTC_CHECK(!p.audio_config.value().input_file_name); + } + if (p.audio_config.value().mode == AudioConfig::Mode::kFile) { + RTC_CHECK(p.audio_config.value().input_file_name); + RTC_CHECK( + test::FileExists(p.audio_config.value().input_file_name.value())) + << p.audio_config.value().input_file_name.value() << " doesn't exist"; + } + } +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/peer_configurer.h b/third_party/libwebrtc/test/pc/e2e/peer_configurer.h new file mode 100644 index 0000000000..c8239c08d4 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/peer_configurer.h @@ -0,0 +1,332 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef TEST_PC_E2E_PEER_CONFIGURER_H_ +#define TEST_PC_E2E_PEER_CONFIGURER_H_ + +#include <memory> +#include <string> +#include <utility> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/async_resolver_factory.h" +#include "api/audio/audio_mixer.h" +#include "api/call/call_factory_interface.h" +#include "api/fec_controller.h" +#include "api/rtc_event_log/rtc_event_log_factory_interface.h" +#include "api/task_queue/task_queue_factory.h" +#include "api/test/create_peer_connection_quality_test_frame_generator.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/transport/network_control.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "rtc_base/network.h" +#include "rtc_base/rtc_certificate_generator.h" +#include "rtc_base/ssl_certificate.h" +#include "rtc_base/thread.h" +#include "test/pc/e2e/peer_connection_quality_test_params.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class PeerConfigurerImpl final + : public PeerConnectionE2EQualityTestFixture::PeerConfigurer { + public: + using VideoSource = + absl::variant<std::unique_ptr<test::FrameGeneratorInterface>, + PeerConnectionE2EQualityTestFixture::CapturingDeviceIndex>; + + PeerConfigurerImpl(rtc::Thread* network_thread, + rtc::NetworkManager* network_manager, + rtc::PacketSocketFactory* packet_socket_factory) + : components_( + std::make_unique<InjectableComponents>(network_thread, + network_manager, + packet_socket_factory)), + params_(std::make_unique<Params>()), + configurable_params_(std::make_unique<ConfigurableParams>()) {} + + PeerConfigurer* SetName(absl::string_view name) override { + params_->name = std::string(name); + return this; + } + + // Implementation of PeerConnectionE2EQualityTestFixture::PeerConfigurer. + PeerConfigurer* SetTaskQueueFactory( + std::unique_ptr<TaskQueueFactory> task_queue_factory) override { + components_->pcf_dependencies->task_queue_factory = + std::move(task_queue_factory); + return this; + } + PeerConfigurer* SetCallFactory( + std::unique_ptr<CallFactoryInterface> call_factory) override { + components_->pcf_dependencies->call_factory = std::move(call_factory); + return this; + } + PeerConfigurer* SetEventLogFactory( + std::unique_ptr<RtcEventLogFactoryInterface> event_log_factory) override { + components_->pcf_dependencies->event_log_factory = + std::move(event_log_factory); + return this; + } + PeerConfigurer* SetFecControllerFactory( + std::unique_ptr<FecControllerFactoryInterface> fec_controller_factory) + override { + components_->pcf_dependencies->fec_controller_factory = + std::move(fec_controller_factory); + return this; + } + PeerConfigurer* SetNetworkControllerFactory( + std::unique_ptr<NetworkControllerFactoryInterface> + network_controller_factory) override { + components_->pcf_dependencies->network_controller_factory = + std::move(network_controller_factory); + return this; + } + PeerConfigurer* SetVideoEncoderFactory( + std::unique_ptr<VideoEncoderFactory> video_encoder_factory) override { + components_->pcf_dependencies->video_encoder_factory = + std::move(video_encoder_factory); + return this; + } + PeerConfigurer* SetVideoDecoderFactory( + std::unique_ptr<VideoDecoderFactory> video_decoder_factory) override { + components_->pcf_dependencies->video_decoder_factory = + std::move(video_decoder_factory); + return this; + } + + PeerConfigurer* SetAsyncResolverFactory( + std::unique_ptr<webrtc::AsyncResolverFactory> async_resolver_factory) + override { + components_->pc_dependencies->async_resolver_factory = + std::move(async_resolver_factory); + return this; + } + PeerConfigurer* SetRTCCertificateGenerator( + std::unique_ptr<rtc::RTCCertificateGeneratorInterface> cert_generator) + override { + components_->pc_dependencies->cert_generator = std::move(cert_generator); + return this; + } + PeerConfigurer* SetSSLCertificateVerifier( + std::unique_ptr<rtc::SSLCertificateVerifier> tls_cert_verifier) override { + components_->pc_dependencies->tls_cert_verifier = + std::move(tls_cert_verifier); + return this; + } + + PeerConfigurer* AddVideoConfig( + PeerConnectionE2EQualityTestFixture::VideoConfig config) override { + video_sources_.push_back( + CreateSquareFrameGenerator(config, /*type=*/absl::nullopt)); + configurable_params_->video_configs.push_back(std::move(config)); + return this; + } + PeerConfigurer* AddVideoConfig( + PeerConnectionE2EQualityTestFixture::VideoConfig config, + std::unique_ptr<test::FrameGeneratorInterface> generator) override { + configurable_params_->video_configs.push_back(std::move(config)); + video_sources_.push_back(std::move(generator)); + return this; + } + PeerConfigurer* AddVideoConfig( + PeerConnectionE2EQualityTestFixture::VideoConfig config, + PeerConnectionE2EQualityTestFixture::CapturingDeviceIndex index) + override { + configurable_params_->video_configs.push_back(std::move(config)); + video_sources_.push_back(index); + return this; + } + PeerConfigurer* SetVideoSubscription( + PeerConnectionE2EQualityTestFixture::VideoSubscription subscription) + override { + configurable_params_->video_subscription = std::move(subscription); + return this; + } + PeerConfigurer* SetAudioConfig( + PeerConnectionE2EQualityTestFixture::AudioConfig config) override { + params_->audio_config = std::move(config); + return this; + } + PeerConfigurer* SetUseUlpFEC(bool value) override { + params_->use_ulp_fec = value; + return this; + } + PeerConfigurer* SetUseFlexFEC(bool value) override { + params_->use_flex_fec = value; + return this; + } + PeerConfigurer* SetVideoEncoderBitrateMultiplier(double multiplier) override { + params_->video_encoder_bitrate_multiplier = multiplier; + return this; + } + PeerConfigurer* SetNetEqFactory( + std::unique_ptr<NetEqFactory> neteq_factory) override { + components_->pcf_dependencies->neteq_factory = std::move(neteq_factory); + return this; + } + PeerConfigurer* SetAudioProcessing( + rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing) override { + components_->pcf_dependencies->audio_processing = audio_processing; + return this; + } + PeerConfigurer* SetAudioMixer( + rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer) override { + components_->pcf_dependencies->audio_mixer = audio_mixer; + return this; + } + PeerConfigurer* SetRtcEventLogPath(std::string path) override { + params_->rtc_event_log_path = std::move(path); + return this; + } + PeerConfigurer* SetAecDumpPath(std::string path) override { + params_->aec_dump_path = std::move(path); + return this; + } + PeerConfigurer* SetRTCConfiguration( + PeerConnectionInterface::RTCConfiguration configuration) override { + params_->rtc_configuration = std::move(configuration); + return this; + } + PeerConfigurer* SetRTCOfferAnswerOptions( + PeerConnectionInterface::RTCOfferAnswerOptions options) override { + params_->rtc_offer_answer_options = std::move(options); + return this; + } + PeerConfigurer* SetBitrateSettings( + BitrateSettings bitrate_settings) override { + params_->bitrate_settings = bitrate_settings; + return this; + } + PeerConfigurer* SetVideoCodecs( + std::vector<PeerConnectionE2EQualityTestFixture::VideoCodecConfig> + video_codecs) override { + params_->video_codecs = std::move(video_codecs); + return this; + } + + PeerConfigurer* SetIceTransportFactory( + std::unique_ptr<IceTransportFactory> factory) override { + components_->pc_dependencies->ice_transport_factory = std::move(factory); + return this; + } + + PeerConfigurer* SetPortAllocatorExtraFlags(uint32_t extra_flags) override { + params_->port_allocator_extra_flags = extra_flags; + return this; + } + // Implementation of PeerConnectionE2EQualityTestFixture::PeerConfigurer end. + + InjectableComponents* components() { return components_.get(); } + Params* params() { return params_.get(); } + ConfigurableParams* configurable_params() { + return configurable_params_.get(); + } + const Params& params() const { return *params_; } + const ConfigurableParams& configurable_params() const { + return *configurable_params_; + } + std::vector<VideoSource>* video_sources() { return &video_sources_; } + + // Returns InjectableComponents and transfer ownership to the caller. + // Can be called once. + std::unique_ptr<InjectableComponents> ReleaseComponents() { + RTC_CHECK(components_); + auto components = std::move(components_); + components_ = nullptr; + return components; + } + + // Returns Params and transfer ownership to the caller. + // Can be called once. + std::unique_ptr<Params> ReleaseParams() { + RTC_CHECK(params_); + auto params = std::move(params_); + params_ = nullptr; + return params; + } + + // Returns ConfigurableParams and transfer ownership to the caller. + // Can be called once. + std::unique_ptr<ConfigurableParams> ReleaseConfigurableParams() { + RTC_CHECK(configurable_params_); + auto configurable_params = std::move(configurable_params_); + configurable_params_ = nullptr; + return configurable_params; + } + + // Returns video sources and transfer frame generators ownership to the + // caller. Can be called once. + std::vector<VideoSource> ReleaseVideoSources() { + auto video_sources = std::move(video_sources_); + video_sources_.clear(); + return video_sources; + } + + private: + std::unique_ptr<InjectableComponents> components_; + std::unique_ptr<Params> params_; + std::unique_ptr<ConfigurableParams> configurable_params_; + std::vector<VideoSource> video_sources_; +}; + +class DefaultNamesProvider { + public: + // Caller have to ensure that default names array will outlive names provider + // instance. + explicit DefaultNamesProvider( + absl::string_view prefix, + rtc::ArrayView<const absl::string_view> default_names = {}); + + void MaybeSetName(absl::optional<std::string>& name); + + private: + std::string GenerateName(); + + std::string GenerateNameInternal(); + + const std::string prefix_; + const rtc::ArrayView<const absl::string_view> default_names_; + + std::set<std::string> known_names_; + size_t counter_ = 0; +}; + +class PeerParamsPreprocessor { + public: + PeerParamsPreprocessor(); + + // Set missing params to default values if it is required: + // * Generate video stream labels if some of them are missing + // * Generate audio stream labels if some of them are missing + // * Set video source generation mode if it is not specified + // * Video codecs under test + void SetDefaultValuesForMissingParams(PeerConfigurerImpl& peer); + + // Validate peer's parameters, also ensure uniqueness of all video stream + // labels. + void ValidateParams(const PeerConfigurerImpl& peer); + + private: + DefaultNamesProvider peer_names_provider_; + + std::set<std::string> peer_names_; + std::set<std::string> video_labels_; + std::set<std::string> audio_labels_; + std::set<std::string> video_sync_groups_; + std::set<std::string> audio_sync_groups_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_PEER_CONFIGURER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/peer_connection_e2e_smoke_test.cc b/third_party/libwebrtc/test/pc/e2e/peer_connection_e2e_smoke_test.cc new file mode 100644 index 0000000000..4ee6590419 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/peer_connection_e2e_smoke_test.cc @@ -0,0 +1,472 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <cstdint> +#include <memory> + +#include "api/media_stream_interface.h" +#include "api/test/create_network_emulation_manager.h" +#include "api/test/create_peer_connection_quality_test_frame_generator.h" +#include "api/test/create_peerconnection_quality_test_fixture.h" +#include "api/test/network_emulation_manager.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "call/simulated_network.h" +#include "system_wrappers/include/field_trial.h" +#include "test/field_trial.h" +#include "test/gtest.h" +#include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" +#include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h" +#include "test/testsupport/file_utils.h" + +#if defined(WEBRTC_MAC) || defined(WEBRTC_IOS) +#include "modules/video_coding/codecs/test/objc_codec_factory_helper.h" // nogncheck +#endif + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test { + public: + using PeerConfigurer = PeerConnectionE2EQualityTestFixture::PeerConfigurer; + using RunParams = PeerConnectionE2EQualityTestFixture::RunParams; + using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig; + using VideoCodecConfig = + PeerConnectionE2EQualityTestFixture::VideoCodecConfig; + using AudioConfig = PeerConnectionE2EQualityTestFixture::AudioConfig; + using ScreenShareConfig = + PeerConnectionE2EQualityTestFixture::ScreenShareConfig; + using ScrollingParams = PeerConnectionE2EQualityTestFixture::ScrollingParams; + using VideoSimulcastConfig = + PeerConnectionE2EQualityTestFixture::VideoSimulcastConfig; + using EchoEmulationConfig = + PeerConnectionE2EQualityTestFixture::EchoEmulationConfig; + + void SetUp() override { + network_emulation_ = CreateNetworkEmulationManager(); + auto video_quality_analyzer = std::make_unique<DefaultVideoQualityAnalyzer>( + network_emulation_->time_controller()->GetClock()); + video_quality_analyzer_ = video_quality_analyzer.get(); + fixture_ = CreatePeerConnectionE2EQualityTestFixture( + testing::UnitTest::GetInstance()->current_test_info()->name(), + *network_emulation_->time_controller(), + /*audio_quality_analyzer=*/nullptr, std::move(video_quality_analyzer)); + } + + std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> + CreateNetwork() { + EmulatedNetworkNode* alice_node = network_emulation_->CreateEmulatedNode( + std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig())); + EmulatedNetworkNode* bob_node = network_emulation_->CreateEmulatedNode( + std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig())); + + EmulatedEndpoint* alice_endpoint = + network_emulation_->CreateEndpoint(EmulatedEndpointConfig()); + EmulatedEndpoint* bob_endpoint = + network_emulation_->CreateEndpoint(EmulatedEndpointConfig()); + + network_emulation_->CreateRoute(alice_endpoint, {alice_node}, bob_endpoint); + network_emulation_->CreateRoute(bob_endpoint, {bob_node}, alice_endpoint); + + EmulatedNetworkManagerInterface* alice_network = + network_emulation_->CreateEmulatedNetworkManagerInterface( + {alice_endpoint}); + EmulatedNetworkManagerInterface* bob_network = + network_emulation_->CreateEmulatedNetworkManagerInterface( + {bob_endpoint}); + + return std::make_pair(alice_network, bob_network); + } + + void AddPeer(EmulatedNetworkManagerInterface* network, + rtc::FunctionView<void(PeerConfigurer*)> configurer) { + fixture_->AddPeer(network->network_dependencies(), configurer); + } + + void RunAndCheckEachVideoStreamReceivedFrames(const RunParams& run_params) { + fixture_->Run(run_params); + + EXPECT_GE(fixture_->GetRealTestDuration(), run_params.run_duration); + VideoStreamsInfo known_streams = video_quality_analyzer_->GetKnownStreams(); + for (const StatsKey& stream_key : known_streams.GetStatsKeys()) { + FrameCounters stream_conters = + video_quality_analyzer_->GetPerStreamCounters().at(stream_key); + // On some devices the pipeline can be too slow, so we actually can't + // force real constraints here. Lets just check, that at least 1 + // frame passed whole pipeline. + int64_t expected_min_fps = run_params.run_duration.seconds() * 15; + EXPECT_GE(stream_conters.captured, expected_min_fps) + << stream_key.ToString(); + EXPECT_GE(stream_conters.pre_encoded, 1) << stream_key.ToString(); + EXPECT_GE(stream_conters.encoded, 1) << stream_key.ToString(); + EXPECT_GE(stream_conters.received, 1) << stream_key.ToString(); + EXPECT_GE(stream_conters.decoded, 1) << stream_key.ToString(); + EXPECT_GE(stream_conters.rendered, 1) << stream_key.ToString(); + } + } + + NetworkEmulationManager* network_emulation() { + return network_emulation_.get(); + } + + PeerConnectionE2EQualityTestFixture* fixture() { return fixture_.get(); } + + private: + std::unique_ptr<NetworkEmulationManager> network_emulation_; + DefaultVideoQualityAnalyzer* video_quality_analyzer_; + std::unique_ptr<PeerConnectionE2EQualityTestFixture> fixture_; +}; + +// IOS debug builds can be quite slow, disabling to avoid issues with timeouts. +#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) +#define MAYBE_Smoke DISABLED_Smoke +#else +#define MAYBE_Smoke Smoke +#endif +TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Smoke) { + std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + VideoConfig video(160, 120, 15); + video.stream_label = "alice-video"; + video.sync_group = "alice-media"; + alice->AddVideoConfig(std::move(video)); + + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + audio.sampling_frequency_in_hz = 48000; + audio.sync_group = "alice-media"; + alice->SetAudioConfig(std::move(audio)); + alice->SetVideoCodecs( + {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}); + + alice->SetUseFlexFEC(true); + alice->SetUseUlpFEC(true); + alice->SetVideoEncoderBitrateMultiplier(1.1); + }); + AddPeer(network_links.second, [](PeerConfigurer* charlie) { + charlie->SetName("charlie"); + VideoConfig video(160, 120, 15); + video.stream_label = "charlie-video"; + video.temporal_layers_count = 2; + charlie->AddVideoConfig(std::move(video)); + + AudioConfig audio; + audio.stream_label = "charlie-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_bob_source", "wav"); + charlie->SetAudioConfig(std::move(audio)); + charlie->SetVideoCodecs( + {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}); + + charlie->SetUseFlexFEC(true); + charlie->SetUseUlpFEC(true); + charlie->SetVideoEncoderBitrateMultiplier(1.1); + }); + fixture()->AddQualityMetricsReporter( + std::make_unique<StatsBasedNetworkQualityMetricsReporter>( + std::map<std::string, std::vector<EmulatedEndpoint*>>( + {{"alice", network_links.first->endpoints()}, + {"charlie", network_links.second->endpoints()}}), + network_emulation())); + RunParams run_params(TimeDelta::Seconds(2)); + run_params.enable_flex_fec_support = true; + RunAndCheckEachVideoStreamReceivedFrames(run_params); +} + +#if defined(WEBRTC_MAC) || defined(WEBRTC_IOS) +TEST_F(PeerConnectionE2EQualityTestSmokeTest, SmokeH264) { + std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> + network_links = CreateNetwork(); + + AddPeer(network_links.first, [](PeerConfigurer* alice) { + VideoConfig video(160, 120, 15); + video.stream_label = "alice-video"; + video.sync_group = "alice-media"; + alice->AddVideoConfig(std::move(video)); + + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + audio.sampling_frequency_in_hz = 48000; + audio.sync_group = "alice-media"; + alice->SetAudioConfig(std::move(audio)); + alice->SetVideoCodecs({VideoCodecConfig(cricket::kH264CodecName)}); + alice->SetVideoEncoderFactory(webrtc::test::CreateObjCEncoderFactory()); + alice->SetVideoDecoderFactory(webrtc::test::CreateObjCDecoderFactory()); + }); + AddPeer(network_links.second, [](PeerConfigurer* charlie) { + charlie->SetName("charlie"); + VideoConfig video(160, 120, 15); + video.stream_label = "charlie-video"; + video.temporal_layers_count = 2; + charlie->AddVideoConfig(std::move(video)); + + AudioConfig audio; + audio.stream_label = "charlie-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_bob_source", "wav"); + charlie->SetAudioConfig(std::move(audio)); + charlie->SetVideoCodecs({VideoCodecConfig(cricket::kH264CodecName)}); + charlie->SetVideoEncoderFactory(webrtc::test::CreateObjCEncoderFactory()); + charlie->SetVideoDecoderFactory(webrtc::test::CreateObjCDecoderFactory()); + }); + + fixture()->AddQualityMetricsReporter( + std::make_unique<StatsBasedNetworkQualityMetricsReporter>( + std::map<std::string, std::vector<EmulatedEndpoint*>>( + {{"alice", network_links.first->endpoints()}, + {"charlie", network_links.second->endpoints()}}), + network_emulation())); + RunParams run_params(TimeDelta::Seconds(2)); + run_params.enable_flex_fec_support = true; + RunAndCheckEachVideoStreamReceivedFrames(run_params); +} +#endif + +// IOS debug builds can be quite slow, disabling to avoid issues with timeouts. +#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) +#define MAYBE_ChangeNetworkConditions DISABLED_ChangeNetworkConditions +#else +#define MAYBE_ChangeNetworkConditions ChangeNetworkConditions +#endif +TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_ChangeNetworkConditions) { + NetworkEmulationManager::SimulatedNetworkNode alice_node = + network_emulation() + ->NodeBuilder() + .config(BuiltInNetworkBehaviorConfig()) + .Build(); + NetworkEmulationManager::SimulatedNetworkNode bob_node = + network_emulation() + ->NodeBuilder() + .config(BuiltInNetworkBehaviorConfig()) + .Build(); + + EmulatedEndpoint* alice_endpoint = + network_emulation()->CreateEndpoint(EmulatedEndpointConfig()); + EmulatedEndpoint* bob_endpoint = + network_emulation()->CreateEndpoint(EmulatedEndpointConfig()); + + network_emulation()->CreateRoute(alice_endpoint, {alice_node.node}, + bob_endpoint); + network_emulation()->CreateRoute(bob_endpoint, {bob_node.node}, + alice_endpoint); + + EmulatedNetworkManagerInterface* alice_network = + network_emulation()->CreateEmulatedNetworkManagerInterface( + {alice_endpoint}); + EmulatedNetworkManagerInterface* bob_network = + network_emulation()->CreateEmulatedNetworkManagerInterface( + {bob_endpoint}); + + AddPeer(alice_network, [](PeerConfigurer* alice) { + VideoConfig video(160, 120, 15); + video.stream_label = "alice-video"; + video.sync_group = "alice-media"; + alice->AddVideoConfig(std::move(video)); + alice->SetVideoCodecs( + {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}); + + alice->SetUseFlexFEC(true); + alice->SetUseUlpFEC(true); + alice->SetVideoEncoderBitrateMultiplier(1.1); + }); + AddPeer(bob_network, [](PeerConfigurer* bob) { + bob->SetVideoCodecs( + {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}); + + bob->SetUseFlexFEC(true); + bob->SetUseUlpFEC(true); + bob->SetVideoEncoderBitrateMultiplier(1.1); + }); + fixture()->AddQualityMetricsReporter( + std::make_unique<StatsBasedNetworkQualityMetricsReporter>( + std::map<std::string, std::vector<EmulatedEndpoint*>>( + {{"alice", alice_network->endpoints()}, + {"bob", bob_network->endpoints()}}), + network_emulation())); + + fixture()->ExecuteAt(TimeDelta::Seconds(1), [alice_node](TimeDelta) { + BuiltInNetworkBehaviorConfig config; + config.loss_percent = 5; + alice_node.simulation->SetConfig(config); + }); + + RunParams run_params(TimeDelta::Seconds(2)); + run_params.enable_flex_fec_support = true; + RunAndCheckEachVideoStreamReceivedFrames(run_params); +} + +// IOS debug builds can be quite slow, disabling to avoid issues with timeouts. +#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) +#define MAYBE_Screenshare DISABLED_Screenshare +#else +#define MAYBE_Screenshare Screenshare +#endif +TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Screenshare) { + std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + VideoConfig screenshare(320, 180, 30); + screenshare.stream_label = "alice-screenshare"; + screenshare.content_hint = VideoTrackInterface::ContentHint::kText; + ScreenShareConfig screen_share_config = + ScreenShareConfig(TimeDelta::Seconds(2)); + screen_share_config.scrolling_params = ScrollingParams( + TimeDelta::Millis(1800), kDefaultSlidesWidth, kDefaultSlidesHeight); + auto screen_share_frame_generator = + CreateScreenShareFrameGenerator(screenshare, screen_share_config); + alice->AddVideoConfig(std::move(screenshare), + std::move(screen_share_frame_generator)); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) {}); + RunAndCheckEachVideoStreamReceivedFrames(RunParams(TimeDelta::Seconds(2))); +} + +// IOS debug builds can be quite slow, disabling to avoid issues with timeouts. +#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) +#define MAYBE_Echo DISABLED_Echo +#else +#define MAYBE_Echo Echo +#endif +TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Echo) { + std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + audio.sampling_frequency_in_hz = 48000; + alice->SetAudioConfig(std::move(audio)); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) { + AudioConfig audio; + audio.stream_label = "bob-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_bob_source", "wav"); + bob->SetAudioConfig(std::move(audio)); + }); + RunParams run_params(TimeDelta::Seconds(2)); + run_params.echo_emulation_config = EchoEmulationConfig(); + RunAndCheckEachVideoStreamReceivedFrames(run_params); +} + +// IOS debug builds can be quite slow, disabling to avoid issues with timeouts. +#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) +#define MAYBE_Simulcast DISABLED_Simulcast +#else +#define MAYBE_Simulcast Simulcast +#endif +TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Simulcast) { + std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + VideoConfig simulcast(1280, 720, 15); + simulcast.stream_label = "alice-simulcast"; + simulcast.simulcast_config = VideoSimulcastConfig(2, 0); + alice->AddVideoConfig(std::move(simulcast)); + + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + alice->SetAudioConfig(std::move(audio)); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) {}); + RunParams run_params(TimeDelta::Seconds(2)); + RunAndCheckEachVideoStreamReceivedFrames(run_params); +} + +// IOS debug builds can be quite slow, disabling to avoid issues with timeouts. +#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) +#define MAYBE_Svc DISABLED_Svc +#else +#define MAYBE_Svc Svc +#endif +TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Svc) { + std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + VideoConfig simulcast("alice-svc", 1280, 720, 15); + // Because we have network with packets loss we can analyze only the + // highest spatial layer in SVC mode. + simulcast.simulcast_config = VideoSimulcastConfig(2, 1); + alice->AddVideoConfig(std::move(simulcast)); + + AudioConfig audio("alice-audio"); + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + alice->SetAudioConfig(std::move(audio)); + alice->SetVideoCodecs({VideoCodecConfig(cricket::kVp9CodecName)}); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) { + bob->SetVideoCodecs({VideoCodecConfig(cricket::kVp9CodecName)}); + }); + RunParams run_params(TimeDelta::Seconds(2)); + RunAndCheckEachVideoStreamReceivedFrames(run_params); +} + +// IOS debug builds can be quite slow, disabling to avoid issues with timeouts. +#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG) +#define MAYBE_HighBitrate DISABLED_HighBitrate +#else +#define MAYBE_HighBitrate HighBitrate +#endif +TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_HighBitrate) { + std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*> + network_links = CreateNetwork(); + AddPeer(network_links.first, [](PeerConfigurer* alice) { + BitrateSettings bitrate_settings; + bitrate_settings.start_bitrate_bps = 3'000'000; + bitrate_settings.max_bitrate_bps = 3'000'000; + alice->SetBitrateSettings(bitrate_settings); + VideoConfig video(800, 600, 15); + video.stream_label = "alice-video"; + RtpEncodingParameters encoding_parameters; + encoding_parameters.min_bitrate_bps = 500'000; + encoding_parameters.max_bitrate_bps = 3'000'000; + video.encoding_params.push_back(std::move(encoding_parameters)); + alice->AddVideoConfig(std::move(video)); + + AudioConfig audio; + audio.stream_label = "alice-audio"; + audio.mode = AudioConfig::Mode::kFile; + audio.input_file_name = + test::ResourcePath("pc_quality_smoke_test_alice_source", "wav"); + audio.sampling_frequency_in_hz = 48000; + alice->SetAudioConfig(std::move(audio)); + alice->SetVideoCodecs( + {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}); + }); + AddPeer(network_links.second, [](PeerConfigurer* bob) { + bob->SetVideoCodecs( + {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})}); + }); + RunParams run_params(TimeDelta::Seconds(2)); + RunAndCheckEachVideoStreamReceivedFrames(run_params); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.cc b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.cc new file mode 100644 index 0000000000..89d5da0d42 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.cc @@ -0,0 +1,737 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/peer_connection_quality_test.h" + +#include <algorithm> +#include <memory> +#include <set> +#include <utility> + +#include "absl/strings/string_view.h" +#include "api/jsep.h" +#include "api/media_stream_interface.h" +#include "api/peer_connection_interface.h" +#include "api/rtc_event_log/rtc_event_log.h" +#include "api/rtc_event_log_output_file.h" +#include "api/scoped_refptr.h" +#include "api/test/time_controller.h" +#include "api/test/video_quality_analyzer_interface.h" +#include "pc/sdp_utils.h" +#include "pc/test/mock_peer_connection_observers.h" +#include "rtc_base/gunit.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/task_queue_for_test.h" +#include "system_wrappers/include/cpu_info.h" +#include "system_wrappers/include/field_trial.h" +#include "test/field_trial.h" +#include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h" +#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h" +#include "test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h" +#include "test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h" +#include "test/pc/e2e/cross_media_metrics_reporter.h" +#include "test/pc/e2e/stats_poller.h" +#include "test/pc/e2e/test_peer_factory.h" +#include "test/testsupport/file_utils.h" +#include "test/testsupport/perf_test.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig; +using VideoCodecConfig = PeerConnectionE2EQualityTestFixture::VideoCodecConfig; + +constexpr TimeDelta kDefaultTimeout = TimeDelta::Seconds(10); +constexpr char kSignalThreadName[] = "signaling_thread"; +// 1 signaling, 2 network, 2 worker and 2 extra for codecs etc. +constexpr int kPeerConnectionUsedThreads = 7; +// Framework has extra thread for network layer and extra thread for peer +// connection stats polling. +constexpr int kFrameworkUsedThreads = 2; +constexpr int kMaxVideoAnalyzerThreads = 8; + +constexpr TimeDelta kStatsUpdateInterval = TimeDelta::Seconds(1); + +constexpr TimeDelta kAliveMessageLogInterval = TimeDelta::Seconds(30); + +constexpr TimeDelta kQuickTestModeRunDuration = TimeDelta::Millis(100); + +// Field trials to enable Flex FEC advertising and receiving. +constexpr char kFlexFecEnabledFieldTrials[] = + "WebRTC-FlexFEC-03-Advertised/Enabled/WebRTC-FlexFEC-03/Enabled/"; +constexpr char kUseStandardsBytesStats[] = + "WebRTC-UseStandardBytesStats/Enabled/"; + +class FixturePeerConnectionObserver : public MockPeerConnectionObserver { + public: + // `on_track_callback` will be called when any new track will be added to peer + // connection. + // `on_connected_callback` will be called when peer connection will come to + // either connected or completed state. Client should notice that in the case + // of reconnect this callback can be called again, so it should be tolerant + // to such behavior. + FixturePeerConnectionObserver( + std::function<void(rtc::scoped_refptr<RtpTransceiverInterface>)> + on_track_callback, + std::function<void()> on_connected_callback) + : on_track_callback_(std::move(on_track_callback)), + on_connected_callback_(std::move(on_connected_callback)) {} + + void OnTrack( + rtc::scoped_refptr<RtpTransceiverInterface> transceiver) override { + MockPeerConnectionObserver::OnTrack(transceiver); + on_track_callback_(transceiver); + } + + void OnIceConnectionChange( + PeerConnectionInterface::IceConnectionState new_state) override { + MockPeerConnectionObserver::OnIceConnectionChange(new_state); + if (ice_connected_) { + on_connected_callback_(); + } + } + + private: + std::function<void(rtc::scoped_refptr<RtpTransceiverInterface>)> + on_track_callback_; + std::function<void()> on_connected_callback_; +}; + +void ValidateP2PSimulcastParams( + const std::vector<std::unique_ptr<PeerConfigurerImpl>>& peers) { + for (size_t i = 0; i < peers.size(); ++i) { + Params* params = peers[i]->params(); + ConfigurableParams* configurable_params = peers[i]->configurable_params(); + for (const VideoConfig& video_config : configurable_params->video_configs) { + if (video_config.simulcast_config) { + // When we simulate SFU we support only one video codec. + RTC_CHECK_EQ(params->video_codecs.size(), 1) + << "Only 1 video codec is supported when simulcast is enabled in " + << "at least 1 video config"; + } + } + } +} + +} // namespace + +PeerConnectionE2EQualityTest::PeerConnectionE2EQualityTest( + std::string test_case_name, + TimeController& time_controller, + std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer, + std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer) + : time_controller_(time_controller), + task_queue_factory_(time_controller_.CreateTaskQueueFactory()), + test_case_name_(std::move(test_case_name)), + executor_(std::make_unique<TestActivitiesExecutor>( + time_controller_.GetClock())) { + // Create default video quality analyzer. We will always create an analyzer, + // even if there are no video streams, because it will be installed into video + // encoder/decoder factories. + if (video_quality_analyzer == nullptr) { + video_quality_analyzer = std::make_unique<DefaultVideoQualityAnalyzer>( + time_controller_.GetClock()); + } + if (field_trial::IsEnabled("WebRTC-VideoFrameTrackingIdAdvertised")) { + encoded_image_data_propagator_ = + std::make_unique<VideoFrameTrackingIdInjector>(); + } else { + encoded_image_data_propagator_ = + std::make_unique<SingleProcessEncodedImageDataInjector>(); + } + video_quality_analyzer_injection_helper_ = + std::make_unique<VideoQualityAnalyzerInjectionHelper>( + time_controller_.GetClock(), std::move(video_quality_analyzer), + encoded_image_data_propagator_.get(), + encoded_image_data_propagator_.get()); + + if (audio_quality_analyzer == nullptr) { + audio_quality_analyzer = std::make_unique<DefaultAudioQualityAnalyzer>(); + } + audio_quality_analyzer_.swap(audio_quality_analyzer); +} + +void PeerConnectionE2EQualityTest::ExecuteAt( + TimeDelta target_time_since_start, + std::function<void(TimeDelta)> func) { + executor_->ScheduleActivity(target_time_since_start, absl::nullopt, func); +} + +void PeerConnectionE2EQualityTest::ExecuteEvery( + TimeDelta initial_delay_since_start, + TimeDelta interval, + std::function<void(TimeDelta)> func) { + executor_->ScheduleActivity(initial_delay_since_start, interval, func); +} + +void PeerConnectionE2EQualityTest::AddQualityMetricsReporter( + std::unique_ptr<QualityMetricsReporter> quality_metrics_reporter) { + quality_metrics_reporters_.push_back(std::move(quality_metrics_reporter)); +} + +PeerConnectionE2EQualityTest::PeerHandle* PeerConnectionE2EQualityTest::AddPeer( + const PeerNetworkDependencies& network_dependencies, + rtc::FunctionView<void(PeerConfigurer*)> configurer) { + peer_configurations_.push_back(std::make_unique<PeerConfigurerImpl>( + network_dependencies.network_thread, network_dependencies.network_manager, + network_dependencies.packet_socket_factory)); + configurer(peer_configurations_.back().get()); + peer_handles_.push_back(PeerHandleImpl()); + return &peer_handles_.back(); +} + +void PeerConnectionE2EQualityTest::Run(RunParams run_params) { + webrtc::webrtc_pc_e2e::PeerParamsPreprocessor params_preprocessor; + for (auto& peer_configuration : peer_configurations_) { + params_preprocessor.SetDefaultValuesForMissingParams(*peer_configuration); + params_preprocessor.ValidateParams(*peer_configuration); + } + ValidateP2PSimulcastParams(peer_configurations_); + RTC_CHECK_EQ(peer_configurations_.size(), 2) + << "Only peer to peer calls are allowed, please add 2 peers"; + + std::unique_ptr<PeerConfigurerImpl> alice_configurer = + std::move(peer_configurations_[0]); + std::unique_ptr<PeerConfigurerImpl> bob_configurer = + std::move(peer_configurations_[1]); + peer_configurations_.clear(); + + for (size_t i = 0; + i < bob_configurer->configurable_params()->video_configs.size(); ++i) { + // We support simulcast only from caller. + RTC_CHECK(!bob_configurer->configurable_params() + ->video_configs[i] + .simulcast_config) + << "Only simulcast stream from first peer is supported"; + } + + test::ScopedFieldTrials field_trials(GetFieldTrials(run_params)); + + // Print test summary + RTC_LOG(LS_INFO) + << "Media quality test: " << *alice_configurer->params()->name + << " will make a call to " << *bob_configurer->params()->name + << " with media video=" + << !alice_configurer->configurable_params()->video_configs.empty() + << "; audio=" << alice_configurer->params()->audio_config.has_value() + << ". " << *bob_configurer->params()->name + << " will respond with media video=" + << !bob_configurer->configurable_params()->video_configs.empty() + << "; audio=" << bob_configurer->params()->audio_config.has_value(); + + const std::unique_ptr<rtc::Thread> signaling_thread = + time_controller_.CreateThread(kSignalThreadName); + media_helper_ = std::make_unique<MediaHelper>( + video_quality_analyzer_injection_helper_.get(), task_queue_factory_.get(), + time_controller_.GetClock()); + + // Create a `task_queue_`. + task_queue_ = std::make_unique<webrtc::TaskQueueForTest>( + time_controller_.GetTaskQueueFactory()->CreateTaskQueue( + "pc_e2e_quality_test", webrtc::TaskQueueFactory::Priority::NORMAL)); + + // Create call participants: Alice and Bob. + // Audio streams are intercepted in AudioDeviceModule, so if it is required to + // catch output of Alice's stream, Alice's output_dump_file_name should be + // passed to Bob's TestPeer setup as audio output file name. + absl::optional<RemotePeerAudioConfig> alice_remote_audio_config = + RemotePeerAudioConfig::Create(bob_configurer->params()->audio_config); + absl::optional<RemotePeerAudioConfig> bob_remote_audio_config = + RemotePeerAudioConfig::Create(alice_configurer->params()->audio_config); + // Copy Alice and Bob video configs and names to correctly pass them into + // lambdas. + std::vector<VideoConfig> alice_video_configs = + alice_configurer->configurable_params()->video_configs; + std::string alice_name = alice_configurer->params()->name.value(); + std::vector<VideoConfig> bob_video_configs = + bob_configurer->configurable_params()->video_configs; + std::string bob_name = bob_configurer->params()->name.value(); + + TestPeerFactory test_peer_factory( + signaling_thread.get(), time_controller_, + video_quality_analyzer_injection_helper_.get(), task_queue_.get()); + alice_ = test_peer_factory.CreateTestPeer( + std::move(alice_configurer), + std::make_unique<FixturePeerConnectionObserver>( + [this, bob_video_configs, alice_name]( + rtc::scoped_refptr<RtpTransceiverInterface> transceiver) { + OnTrackCallback(alice_name, transceiver, bob_video_configs); + }, + [this]() { StartVideo(alice_video_sources_); }), + alice_remote_audio_config, run_params.echo_emulation_config); + bob_ = test_peer_factory.CreateTestPeer( + std::move(bob_configurer), + std::make_unique<FixturePeerConnectionObserver>( + [this, alice_video_configs, + bob_name](rtc::scoped_refptr<RtpTransceiverInterface> transceiver) { + OnTrackCallback(bob_name, transceiver, alice_video_configs); + }, + [this]() { StartVideo(bob_video_sources_); }), + bob_remote_audio_config, run_params.echo_emulation_config); + + int num_cores = CpuInfo::DetectNumberOfCores(); + RTC_DCHECK_GE(num_cores, 1); + + int video_analyzer_threads = + num_cores - kPeerConnectionUsedThreads - kFrameworkUsedThreads; + if (video_analyzer_threads <= 0) { + video_analyzer_threads = 1; + } + video_analyzer_threads = + std::min(video_analyzer_threads, kMaxVideoAnalyzerThreads); + RTC_LOG(LS_INFO) << "video_analyzer_threads=" << video_analyzer_threads; + quality_metrics_reporters_.push_back( + std::make_unique<VideoQualityMetricsReporter>( + time_controller_.GetClock())); + quality_metrics_reporters_.push_back( + std::make_unique<CrossMediaMetricsReporter>()); + + video_quality_analyzer_injection_helper_->Start( + test_case_name_, + std::vector<std::string>{alice_->params().name.value(), + bob_->params().name.value()}, + video_analyzer_threads); + audio_quality_analyzer_->Start(test_case_name_, &analyzer_helper_); + for (auto& reporter : quality_metrics_reporters_) { + reporter->Start(test_case_name_, &analyzer_helper_); + } + + // Start RTCEventLog recording if requested. + if (alice_->params().rtc_event_log_path) { + auto alice_rtc_event_log = std::make_unique<webrtc::RtcEventLogOutputFile>( + alice_->params().rtc_event_log_path.value()); + alice_->pc()->StartRtcEventLog(std::move(alice_rtc_event_log), + webrtc::RtcEventLog::kImmediateOutput); + } + if (bob_->params().rtc_event_log_path) { + auto bob_rtc_event_log = std::make_unique<webrtc::RtcEventLogOutputFile>( + bob_->params().rtc_event_log_path.value()); + bob_->pc()->StartRtcEventLog(std::move(bob_rtc_event_log), + webrtc::RtcEventLog::kImmediateOutput); + } + + // Setup alive logging. It is done to prevent test infra to think that test is + // dead. + RepeatingTaskHandle::DelayedStart(task_queue_->Get(), + kAliveMessageLogInterval, []() { + std::printf("Test is still running...\n"); + return kAliveMessageLogInterval; + }); + + RTC_LOG(LS_INFO) << "Configuration is done. Now " << *alice_->params().name + << " is calling to " << *bob_->params().name << "..."; + + // Setup stats poller. + std::vector<StatsObserverInterface*> observers = { + audio_quality_analyzer_.get(), + video_quality_analyzer_injection_helper_.get()}; + for (auto& reporter : quality_metrics_reporters_) { + observers.push_back(reporter.get()); + } + StatsPoller stats_poller(observers, + std::map<std::string, StatsProvider*>{ + {*alice_->params().name, alice_.get()}, + {*bob_->params().name, bob_.get()}}); + executor_->ScheduleActivity(TimeDelta::Zero(), kStatsUpdateInterval, + [&stats_poller](TimeDelta) { + stats_poller.PollStatsAndNotifyObservers(); + }); + + // Setup call. + SendTask(signaling_thread.get(), + [this, &run_params] { SetupCallOnSignalingThread(run_params); }); + std::unique_ptr<SignalingInterceptor> signaling_interceptor = + CreateSignalingInterceptor(run_params); + // Connect peers. + SendTask(signaling_thread.get(), [this, &signaling_interceptor] { + ExchangeOfferAnswer(signaling_interceptor.get()); + }); + WaitUntilIceCandidatesGathered(signaling_thread.get()); + + SendTask(signaling_thread.get(), [this, &signaling_interceptor] { + ExchangeIceCandidates(signaling_interceptor.get()); + }); + WaitUntilPeersAreConnected(signaling_thread.get()); + + executor_->Start(task_queue_.get()); + Timestamp start_time = Now(); + + bool is_quick_test_enabled = field_trial::IsEnabled("WebRTC-QuickPerfTest"); + if (is_quick_test_enabled) { + time_controller_.AdvanceTime(kQuickTestModeRunDuration); + } else { + time_controller_.AdvanceTime(run_params.run_duration); + } + + RTC_LOG(LS_INFO) << "Test is done, initiating disconnect sequence."; + + // Stop all client started tasks to prevent their access to any call related + // objects after these objects will be destroyed during call tear down. + executor_->Stop(); + // There is no guarantee, that last stats collection will happen at the end + // of the call, so we force it after executor, which is among others is doing + // stats collection, was stopped. + task_queue_->SendTask([&stats_poller]() { + // Get final end-of-call stats. + stats_poller.PollStatsAndNotifyObservers(); + }); + // We need to detach AEC dumping from peers, because dump uses `task_queue_` + // inside. + alice_->DetachAecDump(); + bob_->DetachAecDump(); + // Tear down the call. + SendTask(signaling_thread.get(), [this] { TearDownCallOnSignalingThread(); }); + + Timestamp end_time = Now(); + RTC_LOG(LS_INFO) << "All peers are disconnected."; + { + MutexLock lock(&lock_); + real_test_duration_ = end_time - start_time; + } + + ReportGeneralTestResults(); + audio_quality_analyzer_->Stop(); + video_quality_analyzer_injection_helper_->Stop(); + for (auto& reporter : quality_metrics_reporters_) { + reporter->StopAndReportResults(); + } + + // Reset `task_queue_` after test to cleanup. + task_queue_.reset(); + + alice_ = nullptr; + bob_ = nullptr; + // Ensuring that TestVideoCapturerVideoTrackSource are destroyed on the right + // thread. + RTC_CHECK(alice_video_sources_.empty()); + RTC_CHECK(bob_video_sources_.empty()); +} + +std::string PeerConnectionE2EQualityTest::GetFieldTrials( + const RunParams& run_params) { + std::vector<absl::string_view> default_field_trials = { + kUseStandardsBytesStats}; + if (run_params.enable_flex_fec_support) { + default_field_trials.push_back(kFlexFecEnabledFieldTrials); + } + rtc::StringBuilder sb; + sb << field_trial::GetFieldTrialString(); + for (const absl::string_view& field_trial : default_field_trials) { + sb << field_trial; + } + return sb.Release(); +} + +void PeerConnectionE2EQualityTest::OnTrackCallback( + absl::string_view peer_name, + rtc::scoped_refptr<RtpTransceiverInterface> transceiver, + std::vector<VideoConfig> remote_video_configs) { + const rtc::scoped_refptr<MediaStreamTrackInterface>& track = + transceiver->receiver()->track(); + RTC_CHECK_EQ(transceiver->receiver()->stream_ids().size(), 2) + << "Expected 2 stream ids: 1st - sync group, 2nd - unique stream label"; + std::string sync_group = transceiver->receiver()->stream_ids()[0]; + std::string stream_label = transceiver->receiver()->stream_ids()[1]; + analyzer_helper_.AddTrackToStreamMapping(track->id(), stream_label, + sync_group); + if (track->kind() != MediaStreamTrackInterface::kVideoKind) { + return; + } + + // It is safe to cast here, because it is checked above that + // track->kind() is kVideoKind. + auto* video_track = static_cast<VideoTrackInterface*>(track.get()); + std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> video_sink = + video_quality_analyzer_injection_helper_->CreateVideoSink(peer_name); + video_track->AddOrUpdateSink(video_sink.get(), rtc::VideoSinkWants()); + output_video_sinks_.push_back(std::move(video_sink)); +} + +void PeerConnectionE2EQualityTest::SetupCallOnSignalingThread( + const RunParams& run_params) { + // We need receive-only transceivers for Bob's media stream, so there will + // be media section in SDP for that streams in Alice's offer, because it is + // forbidden to add new media sections in answer in Unified Plan. + RtpTransceiverInit receive_only_transceiver_init; + receive_only_transceiver_init.direction = RtpTransceiverDirection::kRecvOnly; + int alice_transceivers_counter = 0; + if (bob_->params().audio_config) { + // Setup receive audio transceiver if Bob has audio to send. If we'll need + // multiple audio streams, then we need transceiver for each Bob's audio + // stream. + RTCErrorOr<rtc::scoped_refptr<RtpTransceiverInterface>> result = + alice_->AddTransceiver(cricket::MediaType::MEDIA_TYPE_AUDIO, + receive_only_transceiver_init); + RTC_CHECK(result.ok()); + alice_transceivers_counter++; + } + + size_t alice_video_transceivers_non_simulcast_counter = 0; + for (auto& video_config : alice_->configurable_params().video_configs) { + RtpTransceiverInit transceiver_params; + if (video_config.simulcast_config) { + transceiver_params.direction = RtpTransceiverDirection::kSendOnly; + // Because simulcast enabled `alice_->params().video_codecs` has only 1 + // element. + if (alice_->params().video_codecs[0].name == cricket::kVp8CodecName) { + // For Vp8 simulcast we need to add as many RtpEncodingParameters to the + // track as many simulcast streams requested. If they specified in + // `video_config.simulcast_config` it should be copied from there. + for (int i = 0; + i < video_config.simulcast_config->simulcast_streams_count; ++i) { + RtpEncodingParameters enc_params; + if (!video_config.encoding_params.empty()) { + enc_params = video_config.encoding_params[i]; + } + // We need to be sure, that all rids will be unique with all mids. + enc_params.rid = std::to_string(alice_transceivers_counter) + "000" + + std::to_string(i); + transceiver_params.send_encodings.push_back(enc_params); + } + } + } else { + transceiver_params.direction = RtpTransceiverDirection::kSendRecv; + RtpEncodingParameters enc_params; + if (video_config.encoding_params.size() == 1) { + enc_params = video_config.encoding_params[0]; + } + transceiver_params.send_encodings.push_back(enc_params); + + alice_video_transceivers_non_simulcast_counter++; + } + RTCErrorOr<rtc::scoped_refptr<RtpTransceiverInterface>> result = + alice_->AddTransceiver(cricket::MediaType::MEDIA_TYPE_VIDEO, + transceiver_params); + RTC_CHECK(result.ok()); + + alice_transceivers_counter++; + } + + // Add receive only transceivers in case Bob has more video_configs than + // Alice. + for (size_t i = alice_video_transceivers_non_simulcast_counter; + i < bob_->configurable_params().video_configs.size(); ++i) { + RTCErrorOr<rtc::scoped_refptr<RtpTransceiverInterface>> result = + alice_->AddTransceiver(cricket::MediaType::MEDIA_TYPE_VIDEO, + receive_only_transceiver_init); + RTC_CHECK(result.ok()); + alice_transceivers_counter++; + } + + // Then add media for Alice and Bob + media_helper_->MaybeAddAudio(alice_.get()); + alice_video_sources_ = media_helper_->MaybeAddVideo(alice_.get()); + media_helper_->MaybeAddAudio(bob_.get()); + bob_video_sources_ = media_helper_->MaybeAddVideo(bob_.get()); + + SetPeerCodecPreferences(alice_.get()); + SetPeerCodecPreferences(bob_.get()); +} + +void PeerConnectionE2EQualityTest::TearDownCallOnSignalingThread() { + TearDownCall(); +} + +void PeerConnectionE2EQualityTest::SetPeerCodecPreferences(TestPeer* peer) { + std::vector<RtpCodecCapability> with_rtx_video_capabilities = + FilterVideoCodecCapabilities( + peer->params().video_codecs, true, peer->params().use_ulp_fec, + peer->params().use_flex_fec, + peer->pc_factory() + ->GetRtpSenderCapabilities(cricket::MediaType::MEDIA_TYPE_VIDEO) + .codecs); + std::vector<RtpCodecCapability> without_rtx_video_capabilities = + FilterVideoCodecCapabilities( + peer->params().video_codecs, false, peer->params().use_ulp_fec, + peer->params().use_flex_fec, + peer->pc_factory() + ->GetRtpSenderCapabilities(cricket::MediaType::MEDIA_TYPE_VIDEO) + .codecs); + + // Set codecs for transceivers + for (auto transceiver : peer->pc()->GetTransceivers()) { + if (transceiver->media_type() == cricket::MediaType::MEDIA_TYPE_VIDEO) { + if (transceiver->sender()->init_send_encodings().size() > 1) { + // If transceiver's sender has more then 1 send encodings, it means it + // has multiple simulcast streams, so we need disable RTX on it. + RTCError result = + transceiver->SetCodecPreferences(without_rtx_video_capabilities); + RTC_CHECK(result.ok()); + } else { + RTCError result = + transceiver->SetCodecPreferences(with_rtx_video_capabilities); + RTC_CHECK(result.ok()); + } + } + } +} + +std::unique_ptr<SignalingInterceptor> +PeerConnectionE2EQualityTest::CreateSignalingInterceptor( + const RunParams& run_params) { + std::map<std::string, int> stream_label_to_simulcast_streams_count; + // We add only Alice here, because simulcast/svc is supported only from the + // first peer. + for (auto& video_config : alice_->configurable_params().video_configs) { + if (video_config.simulcast_config) { + stream_label_to_simulcast_streams_count.insert( + {*video_config.stream_label, + video_config.simulcast_config->simulcast_streams_count}); + } + } + PatchingParams patching_params(run_params.use_conference_mode, + stream_label_to_simulcast_streams_count); + return std::make_unique<SignalingInterceptor>(patching_params); +} + +void PeerConnectionE2EQualityTest::WaitUntilIceCandidatesGathered( + rtc::Thread* signaling_thread) { + ASSERT_TRUE(time_controller_.Wait( + [&]() { + bool result; + SendTask(signaling_thread, [&]() { + result = alice_->IsIceGatheringDone() && bob_->IsIceGatheringDone(); + }); + return result; + }, + 2 * kDefaultTimeout)); +} + +void PeerConnectionE2EQualityTest::WaitUntilPeersAreConnected( + rtc::Thread* signaling_thread) { + // This means that ICE and DTLS are connected. + alice_connected_ = time_controller_.Wait( + [&]() { + bool result; + SendTask(signaling_thread, [&] { result = alice_->IsIceConnected(); }); + return result; + }, + kDefaultTimeout); + bob_connected_ = time_controller_.Wait( + [&]() { + bool result; + SendTask(signaling_thread, [&] { result = bob_->IsIceConnected(); }); + return result; + }, + kDefaultTimeout); +} + +void PeerConnectionE2EQualityTest::ExchangeOfferAnswer( + SignalingInterceptor* signaling_interceptor) { + std::string log_output; + + auto offer = alice_->CreateOffer(); + RTC_CHECK(offer); + offer->ToString(&log_output); + RTC_LOG(LS_INFO) << "Original offer: " << log_output; + LocalAndRemoteSdp patch_result = signaling_interceptor->PatchOffer( + std::move(offer), alice_->params().video_codecs[0]); + patch_result.local_sdp->ToString(&log_output); + RTC_LOG(LS_INFO) << "Offer to set as local description: " << log_output; + patch_result.remote_sdp->ToString(&log_output); + RTC_LOG(LS_INFO) << "Offer to set as remote description: " << log_output; + + bool set_local_offer = + alice_->SetLocalDescription(std::move(patch_result.local_sdp)); + RTC_CHECK(set_local_offer); + bool set_remote_offer = + bob_->SetRemoteDescription(std::move(patch_result.remote_sdp)); + RTC_CHECK(set_remote_offer); + auto answer = bob_->CreateAnswer(); + RTC_CHECK(answer); + answer->ToString(&log_output); + RTC_LOG(LS_INFO) << "Original answer: " << log_output; + patch_result = signaling_interceptor->PatchAnswer( + std::move(answer), bob_->params().video_codecs[0]); + patch_result.local_sdp->ToString(&log_output); + RTC_LOG(LS_INFO) << "Answer to set as local description: " << log_output; + patch_result.remote_sdp->ToString(&log_output); + RTC_LOG(LS_INFO) << "Answer to set as remote description: " << log_output; + + bool set_local_answer = + bob_->SetLocalDescription(std::move(patch_result.local_sdp)); + RTC_CHECK(set_local_answer); + bool set_remote_answer = + alice_->SetRemoteDescription(std::move(patch_result.remote_sdp)); + RTC_CHECK(set_remote_answer); +} + +void PeerConnectionE2EQualityTest::ExchangeIceCandidates( + SignalingInterceptor* signaling_interceptor) { + // Connect an ICE candidate pairs. + std::vector<std::unique_ptr<IceCandidateInterface>> alice_candidates = + signaling_interceptor->PatchOffererIceCandidates( + alice_->observer()->GetAllCandidates()); + for (auto& candidate : alice_candidates) { + std::string candidate_str; + RTC_CHECK(candidate->ToString(&candidate_str)); + RTC_LOG(LS_INFO) << *alice_->params().name + << " ICE candidate(mid= " << candidate->sdp_mid() + << "): " << candidate_str; + } + ASSERT_TRUE(bob_->AddIceCandidates(std::move(alice_candidates))); + std::vector<std::unique_ptr<IceCandidateInterface>> bob_candidates = + signaling_interceptor->PatchAnswererIceCandidates( + bob_->observer()->GetAllCandidates()); + for (auto& candidate : bob_candidates) { + std::string candidate_str; + RTC_CHECK(candidate->ToString(&candidate_str)); + RTC_LOG(LS_INFO) << *bob_->params().name + << " ICE candidate(mid= " << candidate->sdp_mid() + << "): " << candidate_str; + } + ASSERT_TRUE(alice_->AddIceCandidates(std::move(bob_candidates))); +} + +void PeerConnectionE2EQualityTest::StartVideo( + const std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>>& + sources) { + for (auto& source : sources) { + if (source->state() != MediaSourceInterface::SourceState::kLive) { + source->Start(); + } + } +} + +void PeerConnectionE2EQualityTest::TearDownCall() { + for (const auto& video_source : alice_video_sources_) { + video_source->Stop(); + } + for (const auto& video_source : bob_video_sources_) { + video_source->Stop(); + } + + alice_video_sources_.clear(); + bob_video_sources_.clear(); + + alice_->Close(); + bob_->Close(); + + media_helper_ = nullptr; +} + +void PeerConnectionE2EQualityTest::ReportGeneralTestResults() { + test::PrintResult(*alice_->params().name + "_connected", "", test_case_name_, + alice_connected_, "unitless", + /*important=*/false, + test::ImproveDirection::kBiggerIsBetter); + test::PrintResult(*bob_->params().name + "_connected", "", test_case_name_, + bob_connected_, "unitless", + /*important=*/false, + test::ImproveDirection::kBiggerIsBetter); +} + +Timestamp PeerConnectionE2EQualityTest::Now() const { + return time_controller_.GetClock()->CurrentTime(); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.h b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.h new file mode 100644 index 0000000000..7738a83329 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.h @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef TEST_PC_E2E_PEER_CONNECTION_QUALITY_TEST_H_ +#define TEST_PC_E2E_PEER_CONNECTION_QUALITY_TEST_H_ + +#include <memory> +#include <queue> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/task_queue/task_queue_factory.h" +#include "api/test/audio_quality_analyzer_interface.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/test/time_controller.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/task_queue_for_test.h" +#include "rtc_base/thread.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/clock.h" +#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h" +#include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h" +#include "test/pc/e2e/analyzer_helper.h" +#include "test/pc/e2e/media/media_helper.h" +#include "test/pc/e2e/peer_configurer.h" +#include "test/pc/e2e/peer_connection_quality_test_params.h" +#include "test/pc/e2e/sdp/sdp_changer.h" +#include "test/pc/e2e/test_activities_executor.h" +#include "test/pc/e2e/test_peer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class PeerConnectionE2EQualityTest + : public PeerConnectionE2EQualityTestFixture { + public: + using RunParams = PeerConnectionE2EQualityTestFixture::RunParams; + using VideoConfig = PeerConnectionE2EQualityTestFixture::VideoConfig; + using VideoSimulcastConfig = + PeerConnectionE2EQualityTestFixture::VideoSimulcastConfig; + using PeerConfigurer = PeerConnectionE2EQualityTestFixture::PeerConfigurer; + using QualityMetricsReporter = + PeerConnectionE2EQualityTestFixture::QualityMetricsReporter; + + PeerConnectionE2EQualityTest( + std::string test_case_name, + TimeController& time_controller, + std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer, + std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer); + + ~PeerConnectionE2EQualityTest() override = default; + + void ExecuteAt(TimeDelta target_time_since_start, + std::function<void(TimeDelta)> func) override; + void ExecuteEvery(TimeDelta initial_delay_since_start, + TimeDelta interval, + std::function<void(TimeDelta)> func) override; + + void AddQualityMetricsReporter(std::unique_ptr<QualityMetricsReporter> + quality_metrics_reporter) override; + + PeerHandle* AddPeer( + const PeerNetworkDependencies& network_dependencies, + rtc::FunctionView<void(PeerConfigurer*)> configurer) override; + void Run(RunParams run_params) override; + + TimeDelta GetRealTestDuration() const override { + MutexLock lock(&lock_); + RTC_CHECK_NE(real_test_duration_, TimeDelta::Zero()); + return real_test_duration_; + } + + private: + class PeerHandleImpl : public PeerHandle { + public: + ~PeerHandleImpl() override = default; + }; + + // For some functionality some field trials have to be enabled, they will be + // enabled in Run(). + std::string GetFieldTrials(const RunParams& run_params); + void OnTrackCallback(absl::string_view peer_name, + rtc::scoped_refptr<RtpTransceiverInterface> transceiver, + std::vector<VideoConfig> remote_video_configs); + // Have to be run on the signaling thread. + void SetupCallOnSignalingThread(const RunParams& run_params); + void TearDownCallOnSignalingThread(); + void SetPeerCodecPreferences(TestPeer* peer); + std::unique_ptr<SignalingInterceptor> CreateSignalingInterceptor( + const RunParams& run_params); + void WaitUntilIceCandidatesGathered(rtc::Thread* signaling_thread); + void WaitUntilPeersAreConnected(rtc::Thread* signaling_thread); + void ExchangeOfferAnswer(SignalingInterceptor* signaling_interceptor); + void ExchangeIceCandidates(SignalingInterceptor* signaling_interceptor); + void StartVideo( + const std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>>& + sources); + void TearDownCall(); + void ReportGeneralTestResults(); + Timestamp Now() const; + + TimeController& time_controller_; + const std::unique_ptr<TaskQueueFactory> task_queue_factory_; + std::string test_case_name_; + std::unique_ptr<VideoQualityAnalyzerInjectionHelper> + video_quality_analyzer_injection_helper_; + std::unique_ptr<MediaHelper> media_helper_; + std::unique_ptr<EncodedImageDataPropagator> encoded_image_data_propagator_; + std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer_; + std::unique_ptr<TestActivitiesExecutor> executor_; + + std::vector<std::unique_ptr<PeerConfigurerImpl>> peer_configurations_; + std::vector<PeerHandleImpl> peer_handles_; + + std::unique_ptr<TestPeer> alice_; + std::unique_ptr<TestPeer> bob_; + std::vector<std::unique_ptr<QualityMetricsReporter>> + quality_metrics_reporters_; + + std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>> + alice_video_sources_; + std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>> + bob_video_sources_; + std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> + output_video_sinks_; + AnalyzerHelper analyzer_helper_; + + mutable Mutex lock_; + TimeDelta real_test_duration_ RTC_GUARDED_BY(lock_) = TimeDelta::Zero(); + + // Task queue, that is used for running activities during test call. + // This task queue will be created before call set up and will be destroyed + // immediately before call tear down. + std::unique_ptr<TaskQueueForTest> task_queue_; + + bool alice_connected_ = false; + bool bob_connected_ = false; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_PEER_CONNECTION_QUALITY_TEST_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_params.h b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_params.h new file mode 100644 index 0000000000..645ed7a290 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_params.h @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef TEST_PC_E2E_PEER_CONNECTION_QUALITY_TEST_PARAMS_H_ +#define TEST_PC_E2E_PEER_CONNECTION_QUALITY_TEST_PARAMS_H_ + +#include <memory> +#include <string> +#include <vector> + +#include "api/async_resolver_factory.h" +#include "api/audio/audio_mixer.h" +#include "api/call/call_factory_interface.h" +#include "api/fec_controller.h" +#include "api/field_trials_view.h" +#include "api/rtc_event_log/rtc_event_log_factory_interface.h" +#include "api/task_queue/task_queue_factory.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/transport/network_control.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/audio_processing/include/audio_processing.h" +#include "p2p/base/port_allocator.h" +#include "rtc_base/network.h" +#include "rtc_base/rtc_certificate_generator.h" +#include "rtc_base/ssl_certificate.h" +#include "rtc_base/thread.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Contains most part from PeerConnectionFactoryDependencies. Also all fields +// are optional and defaults will be provided by fixture implementation if +// any will be omitted. +// +// Separate class was introduced to clarify which components can be +// overridden. For example worker and signaling threads will be provided by +// fixture implementation. The same is applicable to the media engine. So user +// can override only some parts of media engine like video encoder/decoder +// factories. +struct PeerConnectionFactoryComponents { + std::unique_ptr<TaskQueueFactory> task_queue_factory; + std::unique_ptr<CallFactoryInterface> call_factory; + std::unique_ptr<RtcEventLogFactoryInterface> event_log_factory; + std::unique_ptr<FecControllerFactoryInterface> fec_controller_factory; + std::unique_ptr<NetworkControllerFactoryInterface> network_controller_factory; + std::unique_ptr<NetEqFactory> neteq_factory; + + // Will be passed to MediaEngineInterface, that will be used in + // PeerConnectionFactory. + std::unique_ptr<VideoEncoderFactory> video_encoder_factory; + std::unique_ptr<VideoDecoderFactory> video_decoder_factory; + + std::unique_ptr<FieldTrialsView> trials; + + rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing; + rtc::scoped_refptr<webrtc::AudioMixer> audio_mixer; +}; + +// Contains most parts from PeerConnectionDependencies. Also all fields are +// optional and defaults will be provided by fixture implementation if any +// will be omitted. +// +// Separate class was introduced to clarify which components can be +// overridden. For example observer, which is required to +// PeerConnectionDependencies, will be provided by fixture implementation, +// so client can't inject its own. Also only network manager can be overridden +// inside port allocator. +struct PeerConnectionComponents { + PeerConnectionComponents(rtc::NetworkManager* network_manager, + rtc::PacketSocketFactory* packet_socket_factory) + : network_manager(network_manager), + packet_socket_factory(packet_socket_factory) { + RTC_CHECK(network_manager); + } + + rtc::NetworkManager* const network_manager; + rtc::PacketSocketFactory* const packet_socket_factory; + std::unique_ptr<webrtc::AsyncResolverFactory> async_resolver_factory; + std::unique_ptr<rtc::RTCCertificateGeneratorInterface> cert_generator; + std::unique_ptr<rtc::SSLCertificateVerifier> tls_cert_verifier; + std::unique_ptr<IceTransportFactory> ice_transport_factory; +}; + +// Contains all components, that can be overridden in peer connection. Also +// has a network thread, that will be used to communicate with another peers. +struct InjectableComponents { + InjectableComponents(rtc::Thread* network_thread, + rtc::NetworkManager* network_manager, + rtc::PacketSocketFactory* packet_socket_factory) + : network_thread(network_thread), + pcf_dependencies(std::make_unique<PeerConnectionFactoryComponents>()), + pc_dependencies( + std::make_unique<PeerConnectionComponents>(network_manager, + packet_socket_factory)) { + RTC_CHECK(network_thread); + } + + rtc::Thread* const network_thread; + + std::unique_ptr<PeerConnectionFactoryComponents> pcf_dependencies; + std::unique_ptr<PeerConnectionComponents> pc_dependencies; +}; + +// Contains information about call media streams (up to 1 audio stream and +// unlimited amount of video streams) and rtc configuration, that will be used +// to set up peer connection. +struct Params { + // Peer name. If empty - default one will be set by the fixture. + absl::optional<std::string> name; + // If `audio_config` is set audio stream will be configured + absl::optional<PeerConnectionE2EQualityTestFixture::AudioConfig> audio_config; + // Flags to set on `cricket::PortAllocator`. These flags will be added + // to the default ones that are presented on the port allocator. + uint32_t port_allocator_extra_flags = cricket::kDefaultPortAllocatorFlags; + // If `rtc_event_log_path` is set, an RTCEventLog will be saved in that + // location and it will be available for further analysis. + absl::optional<std::string> rtc_event_log_path; + // If `aec_dump_path` is set, an AEC dump will be saved in that location and + // it will be available for further analysis. + absl::optional<std::string> aec_dump_path; + + bool use_ulp_fec = false; + bool use_flex_fec = false; + // Specifies how much video encoder target bitrate should be different than + // target bitrate, provided by WebRTC stack. Must be greater then 0. Can be + // used to emulate overshooting of video encoders. This multiplier will + // be applied for all video encoder on both sides for all layers. Bitrate + // estimated by WebRTC stack will be multiplied by this multiplier and then + // provided into VideoEncoder::SetRates(...). + double video_encoder_bitrate_multiplier = 1.0; + + PeerConnectionInterface::RTCConfiguration rtc_configuration; + PeerConnectionInterface::RTCOfferAnswerOptions rtc_offer_answer_options; + BitrateSettings bitrate_settings; + std::vector<PeerConnectionE2EQualityTestFixture::VideoCodecConfig> + video_codecs; +}; + +// Contains parameters that maybe changed by test writer during the test call. +struct ConfigurableParams { + // If `video_configs` is empty - no video should be added to the test call. + std::vector<PeerConnectionE2EQualityTestFixture::VideoConfig> video_configs; + + PeerConnectionE2EQualityTestFixture::VideoSubscription video_subscription = + PeerConnectionE2EQualityTestFixture::VideoSubscription() + .SubscribeToAllPeers(); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_PEER_CONNECTION_QUALITY_TEST_PARAMS_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.cc b/third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.cc new file mode 100644 index 0000000000..b3ee3b78bb --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.cc @@ -0,0 +1,602 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/sdp/sdp_changer.h" + +#include <utility> + +#include "absl/memory/memory.h" +#include "api/jsep_session_description.h" +#include "media/base/media_constants.h" +#include "p2p/base/p2p_constants.h" +#include "pc/sdp_utils.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using VideoCodecConfig = PeerConnectionE2EQualityTestFixture::VideoCodecConfig; + +std::string CodecRequiredParamsToString( + const std::map<std::string, std::string>& codec_required_params) { + rtc::StringBuilder out; + for (const auto& entry : codec_required_params) { + out << entry.first << "=" << entry.second << ";"; + } + return out.str(); +} + +std::string SupportedCodecsToString( + rtc::ArrayView<const RtpCodecCapability> supported_codecs) { + rtc::StringBuilder out; + for (const auto& codec : supported_codecs) { + out << codec.name; + if (!codec.parameters.empty()) { + out << "("; + for (const auto& param : codec.parameters) { + out << param.first << "=" << param.second << ";"; + } + out << ")"; + } + out << "; "; + } + return out.str(); +} + +} // namespace + +std::vector<RtpCodecCapability> FilterVideoCodecCapabilities( + rtc::ArrayView<const VideoCodecConfig> video_codecs, + bool use_rtx, + bool use_ulpfec, + bool use_flexfec, + rtc::ArrayView<const RtpCodecCapability> supported_codecs) { + std::vector<RtpCodecCapability> output_codecs; + // Find requested codecs among supported and add them to output in the order + // they were requested. + for (auto& codec_request : video_codecs) { + size_t size_before = output_codecs.size(); + for (auto& codec : supported_codecs) { + if (codec.name != codec_request.name) { + continue; + } + bool parameters_matched = true; + for (const auto& item : codec_request.required_params) { + auto it = codec.parameters.find(item.first); + if (it == codec.parameters.end()) { + parameters_matched = false; + break; + } + if (item.second != it->second) { + parameters_matched = false; + break; + } + } + if (parameters_matched) { + output_codecs.push_back(codec); + } + } + RTC_CHECK_GT(output_codecs.size(), size_before) + << "Codec with name=" << codec_request.name << " and params {" + << CodecRequiredParamsToString(codec_request.required_params) + << "} is unsupported for this peer connection. Supported codecs are: " + << SupportedCodecsToString(supported_codecs); + } + + // Add required FEC and RTX codecs to output. + for (auto& codec : supported_codecs) { + if (codec.name == cricket::kRtxCodecName && use_rtx) { + output_codecs.push_back(codec); + } else if (codec.name == cricket::kFlexfecCodecName && use_flexfec) { + output_codecs.push_back(codec); + } else if ((codec.name == cricket::kRedCodecName || + codec.name == cricket::kUlpfecCodecName) && + use_ulpfec) { + // Red and ulpfec should be enabled or disabled together. + output_codecs.push_back(codec); + } + } + return output_codecs; +} + +// If offer has no simulcast video sections - do nothing. +// +// If offer has simulcast video sections - for each section creates +// SimulcastSectionInfo and put it into `context_`. +void SignalingInterceptor::FillSimulcastContext( + SessionDescriptionInterface* offer) { + for (auto& content : offer->description()->contents()) { + cricket::MediaContentDescription* media_desc = content.media_description(); + if (media_desc->type() != cricket::MediaType::MEDIA_TYPE_VIDEO) { + continue; + } + if (media_desc->HasSimulcast()) { + // We support only single stream simulcast sections with rids. + RTC_CHECK_EQ(media_desc->mutable_streams().size(), 1); + RTC_CHECK(media_desc->mutable_streams()[0].has_rids()); + + // Create SimulcastSectionInfo for this video section. + SimulcastSectionInfo info(content.mid(), content.type, + media_desc->mutable_streams()[0].rids()); + + // Set new rids basing on created SimulcastSectionInfo. + std::vector<cricket::RidDescription> rids; + cricket::SimulcastDescription simulcast_description; + for (std::string& rid : info.rids) { + rids.emplace_back(rid, cricket::RidDirection::kSend); + simulcast_description.send_layers().AddLayer( + cricket::SimulcastLayer(rid, false)); + } + media_desc->mutable_streams()[0].set_rids(rids); + media_desc->set_simulcast_description(simulcast_description); + + info.simulcast_description = media_desc->simulcast_description(); + for (const auto& extension : media_desc->rtp_header_extensions()) { + if (extension.uri == RtpExtension::kMidUri) { + info.mid_extension = extension; + } else if (extension.uri == RtpExtension::kRidUri) { + info.rid_extension = extension; + } else if (extension.uri == RtpExtension::kRepairedRidUri) { + info.rrid_extension = extension; + } + } + RTC_CHECK_NE(info.rid_extension.id, 0); + RTC_CHECK_NE(info.mid_extension.id, 0); + bool transport_description_found = false; + for (auto& transport_info : offer->description()->transport_infos()) { + if (transport_info.content_name == info.mid) { + info.transport_description = transport_info.description; + transport_description_found = true; + break; + } + } + RTC_CHECK(transport_description_found); + + context_.AddSimulcastInfo(info); + } + } +} + +LocalAndRemoteSdp SignalingInterceptor::PatchOffer( + std::unique_ptr<SessionDescriptionInterface> offer, + const PeerConnectionE2EQualityTestFixture::VideoCodecConfig& first_codec) { + for (auto& content : offer->description()->contents()) { + context_.mids_order.push_back(content.mid()); + cricket::MediaContentDescription* media_desc = content.media_description(); + if (media_desc->type() != cricket::MediaType::MEDIA_TYPE_VIDEO) { + continue; + } + if (content.media_description()->streams().empty()) { + // It means that this media section describes receive only media section + // in SDP. + RTC_CHECK_EQ(content.media_description()->direction(), + RtpTransceiverDirection::kRecvOnly); + continue; + } + media_desc->set_conference_mode(params_.use_conference_mode); + } + + if (!params_.stream_label_to_simulcast_streams_count.empty()) { + // Because simulcast enabled `params_.video_codecs` has only 1 element. + if (first_codec.name == cricket::kVp8CodecName) { + return PatchVp8Offer(std::move(offer)); + } + + if (first_codec.name == cricket::kVp9CodecName) { + return PatchVp9Offer(std::move(offer)); + } + } + + auto offer_for_remote = CloneSessionDescription(offer.get()); + return LocalAndRemoteSdp(std::move(offer), std::move(offer_for_remote)); +} + +LocalAndRemoteSdp SignalingInterceptor::PatchVp8Offer( + std::unique_ptr<SessionDescriptionInterface> offer) { + FillSimulcastContext(offer.get()); + if (!context_.HasSimulcast()) { + auto offer_for_remote = CloneSessionDescription(offer.get()); + return LocalAndRemoteSdp(std::move(offer), std::move(offer_for_remote)); + } + + // Clone original offer description. We mustn't access original offer after + // this point. + std::unique_ptr<cricket::SessionDescription> desc = + offer->description()->Clone(); + + for (auto& info : context_.simulcast_infos) { + // For each simulcast section we have to perform: + // 1. Swap MID and RID header extensions + // 2. Remove RIDs from streams and remove SimulcastDescription + // 3. For each RID duplicate media section + cricket::ContentInfo* simulcast_content = desc->GetContentByName(info.mid); + + // Now we need to prepare common prototype for "m=video" sections, in which + // single simulcast section will be converted. Do it before removing content + // because otherwise description will be deleted. + std::unique_ptr<cricket::MediaContentDescription> prototype_media_desc = + simulcast_content->media_description()->Clone(); + + // Remove simulcast video section from offer. + RTC_CHECK(desc->RemoveContentByName(simulcast_content->mid())); + // Clear `simulcast_content`, because now it is pointing to removed object. + simulcast_content = nullptr; + + // Swap mid and rid extensions, so remote peer will understand rid as mid. + // Also remove rid extension. + std::vector<webrtc::RtpExtension> extensions = + prototype_media_desc->rtp_header_extensions(); + for (auto ext_it = extensions.begin(); ext_it != extensions.end();) { + if (ext_it->uri == RtpExtension::kRidUri) { + // We don't need rid extension for remote peer. + ext_it = extensions.erase(ext_it); + continue; + } + if (ext_it->uri == RtpExtension::kRepairedRidUri) { + // We don't support RTX in simulcast. + ext_it = extensions.erase(ext_it); + continue; + } + if (ext_it->uri == RtpExtension::kMidUri) { + ext_it->id = info.rid_extension.id; + } + ++ext_it; + } + + prototype_media_desc->ClearRtpHeaderExtensions(); + prototype_media_desc->set_rtp_header_extensions(extensions); + + // We support only single stream inside video section with simulcast + RTC_CHECK_EQ(prototype_media_desc->mutable_streams().size(), 1); + // This stream must have rids. + RTC_CHECK(prototype_media_desc->mutable_streams()[0].has_rids()); + + // Remove rids and simulcast description from media description. + prototype_media_desc->mutable_streams()[0].set_rids({}); + prototype_media_desc->set_simulcast_description( + cricket::SimulcastDescription()); + + // For each rid add separate video section. + for (std::string& rid : info.rids) { + desc->AddContent(rid, info.media_protocol_type, + prototype_media_desc->Clone()); + } + } + + // Now we need to add bundle line to have all media bundled together. + cricket::ContentGroup bundle_group(cricket::GROUP_TYPE_BUNDLE); + for (auto& content : desc->contents()) { + bundle_group.AddContentName(content.mid()); + } + if (desc->HasGroup(cricket::GROUP_TYPE_BUNDLE)) { + desc->RemoveGroupByName(cricket::GROUP_TYPE_BUNDLE); + } + desc->AddGroup(bundle_group); + + // Update transport_infos to add TransportInfo for each new media section. + std::vector<cricket::TransportInfo> transport_infos = desc->transport_infos(); + transport_infos.erase(std::remove_if( + transport_infos.begin(), transport_infos.end(), + [this](const cricket::TransportInfo& ti) { + // Remove transport infos that correspond to simulcast video sections. + return context_.simulcast_infos_by_mid.find(ti.content_name) != + context_.simulcast_infos_by_mid.end(); + })); + for (auto& info : context_.simulcast_infos) { + for (auto& rid : info.rids) { + transport_infos.emplace_back(rid, info.transport_description); + } + } + desc->set_transport_infos(transport_infos); + + // Create patched offer. + auto patched_offer = + std::make_unique<JsepSessionDescription>(SdpType::kOffer); + patched_offer->Initialize(std::move(desc), offer->session_id(), + offer->session_version()); + return LocalAndRemoteSdp(std::move(offer), std::move(patched_offer)); +} + +LocalAndRemoteSdp SignalingInterceptor::PatchVp9Offer( + std::unique_ptr<SessionDescriptionInterface> offer) { + rtc::UniqueRandomIdGenerator ssrcs_generator; + for (auto& content : offer->description()->contents()) { + for (auto& stream : content.media_description()->streams()) { + for (auto& ssrc : stream.ssrcs) { + ssrcs_generator.AddKnownId(ssrc); + } + } + } + + for (auto& content : offer->description()->contents()) { + if (content.media_description()->type() != + cricket::MediaType::MEDIA_TYPE_VIDEO) { + // We are interested in only video tracks + continue; + } + if (content.media_description()->direction() == + RtpTransceiverDirection::kRecvOnly) { + // If direction is receive only, then there is no media in this track from + // sender side, so we needn't to do anything with this track. + continue; + } + RTC_CHECK_EQ(content.media_description()->streams().size(), 1); + cricket::StreamParams& stream = + content.media_description()->mutable_streams()[0]; + RTC_CHECK_EQ(stream.stream_ids().size(), 2) + << "Expected 2 stream ids in video stream: 1st - sync_group, 2nd - " + "unique label"; + std::string stream_label = stream.stream_ids()[1]; + + auto it = + params_.stream_label_to_simulcast_streams_count.find(stream_label); + if (it == params_.stream_label_to_simulcast_streams_count.end()) { + continue; + } + int svc_layers_count = it->second; + + RTC_CHECK(stream.has_ssrc_groups()) << "Only SVC with RTX is supported"; + RTC_CHECK_EQ(stream.ssrc_groups.size(), 1) + << "Too many ssrc groups in the track"; + std::vector<uint32_t> primary_ssrcs; + stream.GetPrimarySsrcs(&primary_ssrcs); + RTC_CHECK(primary_ssrcs.size() == 1); + for (int i = 1; i < svc_layers_count; ++i) { + uint32_t ssrc = ssrcs_generator.GenerateId(); + primary_ssrcs.push_back(ssrc); + stream.add_ssrc(ssrc); + stream.AddFidSsrc(ssrc, ssrcs_generator.GenerateId()); + } + stream.ssrc_groups.push_back( + cricket::SsrcGroup(cricket::kSimSsrcGroupSemantics, primary_ssrcs)); + } + auto offer_for_remote = CloneSessionDescription(offer.get()); + return LocalAndRemoteSdp(std::move(offer), std::move(offer_for_remote)); +} + +LocalAndRemoteSdp SignalingInterceptor::PatchAnswer( + std::unique_ptr<SessionDescriptionInterface> answer, + const PeerConnectionE2EQualityTestFixture::VideoCodecConfig& first_codec) { + for (auto& content : answer->description()->contents()) { + cricket::MediaContentDescription* media_desc = content.media_description(); + if (media_desc->type() != cricket::MediaType::MEDIA_TYPE_VIDEO) { + continue; + } + if (content.media_description()->direction() != + RtpTransceiverDirection::kRecvOnly) { + continue; + } + media_desc->set_conference_mode(params_.use_conference_mode); + } + + if (!params_.stream_label_to_simulcast_streams_count.empty()) { + // Because simulcast enabled `params_.video_codecs` has only 1 element. + if (first_codec.name == cricket::kVp8CodecName) { + return PatchVp8Answer(std::move(answer)); + } + + if (first_codec.name == cricket::kVp9CodecName) { + return PatchVp9Answer(std::move(answer)); + } + } + + auto answer_for_remote = CloneSessionDescription(answer.get()); + return LocalAndRemoteSdp(std::move(answer), std::move(answer_for_remote)); +} + +LocalAndRemoteSdp SignalingInterceptor::PatchVp8Answer( + std::unique_ptr<SessionDescriptionInterface> answer) { + if (!context_.HasSimulcast()) { + auto answer_for_remote = CloneSessionDescription(answer.get()); + return LocalAndRemoteSdp(std::move(answer), std::move(answer_for_remote)); + } + + std::unique_ptr<cricket::SessionDescription> desc = + answer->description()->Clone(); + + for (auto& info : context_.simulcast_infos) { + cricket::ContentInfo* simulcast_content = + desc->GetContentByName(info.rids[0]); + RTC_CHECK(simulcast_content); + + // Get media description, which will be converted to simulcast answer. + std::unique_ptr<cricket::MediaContentDescription> media_desc = + simulcast_content->media_description()->Clone(); + // Set `simulcast_content` to nullptr, because then it will be removed, so + // it will point to deleted object. + simulcast_content = nullptr; + + // Remove separate media sections for simulcast streams. + for (auto& rid : info.rids) { + RTC_CHECK(desc->RemoveContentByName(rid)); + } + + // Patch `media_desc` to make it simulcast answer description. + // Restore mid/rid rtp header extensions + std::vector<webrtc::RtpExtension> extensions = + media_desc->rtp_header_extensions(); + // First remove existing rid/mid header extensions. + extensions.erase(std::remove_if(extensions.begin(), extensions.end(), + [](const webrtc::RtpExtension& e) { + return e.uri == RtpExtension::kMidUri || + e.uri == RtpExtension::kRidUri || + e.uri == + RtpExtension::kRepairedRidUri; + })); + + // Then add right ones. + extensions.push_back(info.mid_extension); + extensions.push_back(info.rid_extension); + // extensions.push_back(info.rrid_extension); + media_desc->ClearRtpHeaderExtensions(); + media_desc->set_rtp_header_extensions(extensions); + + // Add StreamParams with rids for receive. + RTC_CHECK_EQ(media_desc->mutable_streams().size(), 0); + std::vector<cricket::RidDescription> rids; + for (auto& rid : info.rids) { + rids.emplace_back(rid, cricket::RidDirection::kReceive); + } + cricket::StreamParams stream_params; + stream_params.set_rids(rids); + media_desc->mutable_streams().push_back(stream_params); + + // Restore SimulcastDescription. It should correspond to one from offer, + // but it have to have receive layers instead of send. So we need to put + // send layers from offer to receive layers in answer. + cricket::SimulcastDescription simulcast_description; + for (const auto& layer : info.simulcast_description.send_layers()) { + simulcast_description.receive_layers().AddLayerWithAlternatives(layer); + } + media_desc->set_simulcast_description(simulcast_description); + + // Add simulcast media section. + desc->AddContent(info.mid, info.media_protocol_type, std::move(media_desc)); + } + + desc = RestoreMediaSectionsOrder(std::move(desc)); + + // Now we need to add bundle line to have all media bundled together. + cricket::ContentGroup bundle_group(cricket::GROUP_TYPE_BUNDLE); + for (auto& content : desc->contents()) { + bundle_group.AddContentName(content.mid()); + } + if (desc->HasGroup(cricket::GROUP_TYPE_BUNDLE)) { + desc->RemoveGroupByName(cricket::GROUP_TYPE_BUNDLE); + } + desc->AddGroup(bundle_group); + + // Fix transport_infos: it have to have single info for simulcast section. + std::vector<cricket::TransportInfo> transport_infos = desc->transport_infos(); + std::map<std::string, cricket::TransportDescription> + mid_to_transport_description; + for (auto info_it = transport_infos.begin(); + info_it != transport_infos.end();) { + auto it = context_.simulcast_infos_by_rid.find(info_it->content_name); + if (it != context_.simulcast_infos_by_rid.end()) { + // This transport info correspond to some extra added media section. + mid_to_transport_description.insert( + {it->second->mid, info_it->description}); + info_it = transport_infos.erase(info_it); + } else { + ++info_it; + } + } + for (auto& info : context_.simulcast_infos) { + transport_infos.emplace_back(info.mid, + mid_to_transport_description.at(info.mid)); + } + desc->set_transport_infos(transport_infos); + + auto patched_answer = + std::make_unique<JsepSessionDescription>(SdpType::kAnswer); + patched_answer->Initialize(std::move(desc), answer->session_id(), + answer->session_version()); + return LocalAndRemoteSdp(std::move(answer), std::move(patched_answer)); +} + +std::unique_ptr<cricket::SessionDescription> +SignalingInterceptor::RestoreMediaSectionsOrder( + std::unique_ptr<cricket::SessionDescription> source) { + std::unique_ptr<cricket::SessionDescription> out = source->Clone(); + for (auto& mid : context_.mids_order) { + RTC_CHECK(out->RemoveContentByName(mid)); + } + RTC_CHECK_EQ(out->contents().size(), 0); + for (auto& mid : context_.mids_order) { + cricket::ContentInfo* content = source->GetContentByName(mid); + RTC_CHECK(content); + out->AddContent(mid, content->type, content->media_description()->Clone()); + } + return out; +} + +LocalAndRemoteSdp SignalingInterceptor::PatchVp9Answer( + std::unique_ptr<SessionDescriptionInterface> answer) { + auto answer_for_remote = CloneSessionDescription(answer.get()); + return LocalAndRemoteSdp(std::move(answer), std::move(answer_for_remote)); +} + +std::vector<std::unique_ptr<IceCandidateInterface>> +SignalingInterceptor::PatchOffererIceCandidates( + rtc::ArrayView<const IceCandidateInterface* const> candidates) { + std::vector<std::unique_ptr<IceCandidateInterface>> out; + for (auto* candidate : candidates) { + auto simulcast_info_it = + context_.simulcast_infos_by_mid.find(candidate->sdp_mid()); + if (simulcast_info_it != context_.simulcast_infos_by_mid.end()) { + // This is candidate for simulcast section, so it should be transformed + // into candidates for replicated sections. The sdpMLineIndex is set to + // -1 and ignored if the rid is present. + for (const std::string& rid : simulcast_info_it->second->rids) { + out.push_back(CreateIceCandidate(rid, -1, candidate->candidate())); + } + } else { + out.push_back(CreateIceCandidate(candidate->sdp_mid(), + candidate->sdp_mline_index(), + candidate->candidate())); + } + } + RTC_CHECK_GT(out.size(), 0); + return out; +} + +std::vector<std::unique_ptr<IceCandidateInterface>> +SignalingInterceptor::PatchAnswererIceCandidates( + rtc::ArrayView<const IceCandidateInterface* const> candidates) { + std::vector<std::unique_ptr<IceCandidateInterface>> out; + for (auto* candidate : candidates) { + auto simulcast_info_it = + context_.simulcast_infos_by_rid.find(candidate->sdp_mid()); + if (simulcast_info_it != context_.simulcast_infos_by_rid.end()) { + // This is candidate for replicated section, created from single simulcast + // section, so it should be transformed into candidates for simulcast + // section. + out.push_back(CreateIceCandidate(simulcast_info_it->second->mid, 0, + candidate->candidate())); + } else if (!context_.simulcast_infos_by_rid.empty()) { + // When using simulcast and bundle, put everything on the first m-line. + out.push_back(CreateIceCandidate("", 0, candidate->candidate())); + } else { + out.push_back(CreateIceCandidate(candidate->sdp_mid(), + candidate->sdp_mline_index(), + candidate->candidate())); + } + } + RTC_CHECK_GT(out.size(), 0); + return out; +} + +SignalingInterceptor::SimulcastSectionInfo::SimulcastSectionInfo( + const std::string& mid, + cricket::MediaProtocolType media_protocol_type, + const std::vector<cricket::RidDescription>& rids_desc) + : mid(mid), media_protocol_type(media_protocol_type) { + for (auto& rid : rids_desc) { + rids.push_back(rid.rid); + } +} + +void SignalingInterceptor::SignalingContext::AddSimulcastInfo( + const SimulcastSectionInfo& info) { + simulcast_infos.push_back(info); + bool inserted = + simulcast_infos_by_mid.insert({info.mid, &simulcast_infos.back()}).second; + RTC_CHECK(inserted); + for (auto& rid : info.rids) { + inserted = + simulcast_infos_by_rid.insert({rid, &simulcast_infos.back()}).second; + RTC_CHECK(inserted); + } +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.h b/third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.h new file mode 100644 index 0000000000..115ed5ba2c --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.h @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_SDP_SDP_CHANGER_H_ +#define TEST_PC_E2E_SDP_SDP_CHANGER_H_ + +#include <map> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/jsep.h" +#include "api/rtp_parameters.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "media/base/rid_description.h" +#include "pc/session_description.h" +#include "pc/simulcast_description.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Creates list of capabilities, which can be set on RtpTransceiverInterface via +// RtpTransceiverInterface::SetCodecPreferences(...) to negotiate use of codecs +// from list of `supported_codecs` which will match `video_codecs`. If flags +// `ulpfec` or `flexfec` set to true corresponding FEC codec will be added. +// FEC and RTX codecs will be added after required codecs. +// +// All codecs will be added only if they exists in the list of +// `supported_codecs`. If multiple codecs from this list will match +// `video_codecs`, then all of them will be added to the output +// vector and they will be added in the same order, as they were in +// `supported_codecs`. +std::vector<RtpCodecCapability> FilterVideoCodecCapabilities( + rtc::ArrayView<const PeerConnectionE2EQualityTestFixture::VideoCodecConfig> + video_codecs, + bool use_rtx, + bool use_ulpfec, + bool use_flexfec, + rtc::ArrayView<const RtpCodecCapability> supported_codecs); + +struct LocalAndRemoteSdp { + LocalAndRemoteSdp(std::unique_ptr<SessionDescriptionInterface> local_sdp, + std::unique_ptr<SessionDescriptionInterface> remote_sdp) + : local_sdp(std::move(local_sdp)), remote_sdp(std::move(remote_sdp)) {} + + // Sdp, that should be as local description on the peer, that created it. + std::unique_ptr<SessionDescriptionInterface> local_sdp; + // Sdp, that should be set as remote description on the peer opposite to the + // one, who created it. + std::unique_ptr<SessionDescriptionInterface> remote_sdp; +}; + +struct PatchingParams { + PatchingParams( + bool use_conference_mode, + std::map<std::string, int> stream_label_to_simulcast_streams_count) + : use_conference_mode(use_conference_mode), + stream_label_to_simulcast_streams_count( + stream_label_to_simulcast_streams_count) {} + + bool use_conference_mode; + std::map<std::string, int> stream_label_to_simulcast_streams_count; +}; + +class SignalingInterceptor { + public: + explicit SignalingInterceptor(PatchingParams params) : params_(params) {} + + LocalAndRemoteSdp PatchOffer( + std::unique_ptr<SessionDescriptionInterface> offer, + const PeerConnectionE2EQualityTestFixture::VideoCodecConfig& first_codec); + LocalAndRemoteSdp PatchAnswer( + std::unique_ptr<SessionDescriptionInterface> answer, + const PeerConnectionE2EQualityTestFixture::VideoCodecConfig& first_codec); + + std::vector<std::unique_ptr<IceCandidateInterface>> PatchOffererIceCandidates( + rtc::ArrayView<const IceCandidateInterface* const> candidates); + std::vector<std::unique_ptr<IceCandidateInterface>> + PatchAnswererIceCandidates( + rtc::ArrayView<const IceCandidateInterface* const> candidates); + + private: + // Contains information about simulcast section, that is required to perform + // modified offer/answer and ice candidates exchange. + struct SimulcastSectionInfo { + SimulcastSectionInfo(const std::string& mid, + cricket::MediaProtocolType media_protocol_type, + const std::vector<cricket::RidDescription>& rids_desc); + + const std::string mid; + const cricket::MediaProtocolType media_protocol_type; + std::vector<std::string> rids; + cricket::SimulcastDescription simulcast_description; + webrtc::RtpExtension mid_extension; + webrtc::RtpExtension rid_extension; + webrtc::RtpExtension rrid_extension; + cricket::TransportDescription transport_description; + }; + + struct SignalingContext { + SignalingContext() = default; + // SignalingContext is not copyable and movable. + SignalingContext(SignalingContext&) = delete; + SignalingContext& operator=(SignalingContext&) = delete; + SignalingContext(SignalingContext&&) = delete; + SignalingContext& operator=(SignalingContext&&) = delete; + + void AddSimulcastInfo(const SimulcastSectionInfo& info); + bool HasSimulcast() const { return !simulcast_infos.empty(); } + + std::vector<SimulcastSectionInfo> simulcast_infos; + std::map<std::string, SimulcastSectionInfo*> simulcast_infos_by_mid; + std::map<std::string, SimulcastSectionInfo*> simulcast_infos_by_rid; + + std::vector<std::string> mids_order; + }; + + LocalAndRemoteSdp PatchVp8Offer( + std::unique_ptr<SessionDescriptionInterface> offer); + LocalAndRemoteSdp PatchVp9Offer( + std::unique_ptr<SessionDescriptionInterface> offer); + LocalAndRemoteSdp PatchVp8Answer( + std::unique_ptr<SessionDescriptionInterface> answer); + LocalAndRemoteSdp PatchVp9Answer( + std::unique_ptr<SessionDescriptionInterface> answer); + + void FillSimulcastContext(SessionDescriptionInterface* offer); + std::unique_ptr<cricket::SessionDescription> RestoreMediaSectionsOrder( + std::unique_ptr<cricket::SessionDescription> source); + + PatchingParams params_; + SignalingContext context_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_SDP_SDP_CHANGER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.cc b/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.cc new file mode 100644 index 0000000000..94a46fbae5 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.cc @@ -0,0 +1,385 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h" + +#include <cstdint> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <type_traits> +#include <utility> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/array_view.h" +#include "api/scoped_refptr.h" +#include "api/stats/rtc_stats.h" +#include "api/stats/rtcstats_objects.h" +#include "api/test/network_emulation/network_emulation_interfaces.h" +#include "api/test/network_emulation_manager.h" +#include "api/units/data_rate.h" +#include "api/units/timestamp.h" +#include "rtc_base/event.h" +#include "rtc_base/ip_address.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/synchronization/mutex.h" +#include "system_wrappers/include/field_trial.h" +#include "test/testsupport/perf_test.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +constexpr int kStatsWaitTimeoutMs = 1000; + +// Field trial which controls whether to report standard-compliant bytes +// sent/received per stream. If enabled, padding and headers are not included +// in bytes sent or received. +constexpr char kUseStandardBytesStats[] = "WebRTC-UseStandardBytesStats"; + +std::unique_ptr<EmulatedNetworkStats> PopulateStats( + std::vector<EmulatedEndpoint*> endpoints, + NetworkEmulationManager* network_emulation) { + rtc::Event stats_loaded; + std::unique_ptr<EmulatedNetworkStats> stats; + network_emulation->GetStats(endpoints, + [&](std::unique_ptr<EmulatedNetworkStats> s) { + stats = std::move(s); + stats_loaded.Set(); + }); + bool stats_received = stats_loaded.Wait(kStatsWaitTimeoutMs); + RTC_CHECK(stats_received); + return stats; +} + +std::map<rtc::IPAddress, std::string> PopulateIpToPeer( + const std::map<std::string, std::vector<EmulatedEndpoint*>>& + peer_endpoints) { + std::map<rtc::IPAddress, std::string> out; + for (const auto& entry : peer_endpoints) { + for (const EmulatedEndpoint* const endpoint : entry.second) { + RTC_CHECK(out.find(endpoint->GetPeerLocalAddress()) == out.end()) + << "Two peers can't share the same endpoint"; + out.emplace(endpoint->GetPeerLocalAddress(), entry.first); + } + } + return out; +} + +} // namespace + +StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector:: + NetworkLayerStatsCollector( + std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints, + NetworkEmulationManager* network_emulation) + : peer_endpoints_(std::move(peer_endpoints)), + ip_to_peer_(PopulateIpToPeer(peer_endpoints_)), + network_emulation_(network_emulation) {} + +void StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector:: + Start() { + MutexLock lock(&mutex_); + // Check that network stats are clean before test execution. + for (const auto& entry : peer_endpoints_) { + std::unique_ptr<EmulatedNetworkStats> stats = + PopulateStats(entry.second, network_emulation_); + RTC_CHECK_EQ(stats->PacketsSent(), 0); + RTC_CHECK_EQ(stats->PacketsReceived(), 0); + } +} + +void StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector:: + AddPeer(absl::string_view peer_name, + std::vector<EmulatedEndpoint*> endpoints) { + MutexLock lock(&mutex_); + // When new peer is added not in the constructor, don't check if it has empty + // stats, because their endpoint could be used for traffic before. + peer_endpoints_.emplace(peer_name, std::move(endpoints)); + for (const EmulatedEndpoint* const endpoint : endpoints) { + RTC_CHECK(ip_to_peer_.find(endpoint->GetPeerLocalAddress()) == + ip_to_peer_.end()) + << "Two peers can't share the same endpoint"; + ip_to_peer_.emplace(endpoint->GetPeerLocalAddress(), peer_name); + } +} + +std::map<std::string, + StatsBasedNetworkQualityMetricsReporter::NetworkLayerStats> +StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector:: + GetStats() { + MutexLock lock(&mutex_); + std::map<std::string, NetworkLayerStats> peer_to_stats; + std::map<std::string, std::vector<std::string>> sender_to_receivers; + for (const auto& entry : peer_endpoints_) { + NetworkLayerStats stats; + stats.stats = PopulateStats(entry.second, network_emulation_); + const std::string& peer_name = entry.first; + for (const auto& income_stats_entry : + stats.stats->IncomingStatsPerSource()) { + const rtc::IPAddress& source_ip = income_stats_entry.first; + auto it = ip_to_peer_.find(source_ip); + if (it == ip_to_peer_.end()) { + // Source IP is unknown for this collector, so will be skipped. + continue; + } + sender_to_receivers[it->second].push_back(peer_name); + } + peer_to_stats.emplace(peer_name, std::move(stats)); + } + for (auto& entry : peer_to_stats) { + const std::vector<std::string>& receivers = + sender_to_receivers[entry.first]; + entry.second.receivers = + std::set<std::string>(receivers.begin(), receivers.end()); + } + return peer_to_stats; +} + +void StatsBasedNetworkQualityMetricsReporter::AddPeer( + absl::string_view peer_name, + std::vector<EmulatedEndpoint*> endpoints) { + collector_.AddPeer(peer_name, std::move(endpoints)); +} + +void StatsBasedNetworkQualityMetricsReporter::Start( + absl::string_view test_case_name, + const TrackIdStreamInfoMap* reporter_helper) { + test_case_name_ = std::string(test_case_name); + collector_.Start(); + start_time_ = clock_->CurrentTime(); +} + +void StatsBasedNetworkQualityMetricsReporter::OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) { + PCStats cur_stats; + + auto inbound_stats = report->GetStatsOfType<RTCInboundRTPStreamStats>(); + for (const auto& stat : inbound_stats) { + cur_stats.payload_received += + DataSize::Bytes(stat->bytes_received.ValueOrDefault(0ul) + + stat->header_bytes_received.ValueOrDefault(0ul)); + } + + auto outbound_stats = report->GetStatsOfType<RTCOutboundRTPStreamStats>(); + for (const auto& stat : outbound_stats) { + cur_stats.payload_sent += + DataSize::Bytes(stat->bytes_sent.ValueOrDefault(0ul) + + stat->header_bytes_sent.ValueOrDefault(0ul)); + } + + auto candidate_pairs_stats = report->GetStatsOfType<RTCTransportStats>(); + for (const auto& stat : candidate_pairs_stats) { + cur_stats.total_received += + DataSize::Bytes(stat->bytes_received.ValueOrDefault(0ul)); + cur_stats.total_sent += + DataSize::Bytes(stat->bytes_sent.ValueOrDefault(0ul)); + cur_stats.packets_received += stat->packets_received.ValueOrDefault(0ul); + cur_stats.packets_sent += stat->packets_sent.ValueOrDefault(0ul); + } + + MutexLock lock(&mutex_); + pc_stats_[std::string(pc_label)] = cur_stats; +} + +void StatsBasedNetworkQualityMetricsReporter::StopAndReportResults() { + Timestamp end_time = clock_->CurrentTime(); + + if (!webrtc::field_trial::IsEnabled(kUseStandardBytesStats)) { + RTC_LOG(LS_ERROR) + << "Non-standard GetStats; \"payload\" counts include RTP headers"; + } + + std::map<std::string, NetworkLayerStats> stats = collector_.GetStats(); + for (const auto& entry : stats) { + LogNetworkLayerStats(entry.first, entry.second); + } + MutexLock lock(&mutex_); + for (const auto& pair : pc_stats_) { + auto it = stats.find(pair.first); + RTC_CHECK(it != stats.end()) + << "Peer name used for PeerConnection stats collection and peer name " + "used for endpoints naming doesn't match. No endpoints found for " + "peer " + << pair.first; + const NetworkLayerStats& network_layer_stats = it->second; + int64_t total_packets_received = 0; + bool found = false; + for (const auto& dest_peer : network_layer_stats.receivers) { + auto pc_stats_it = pc_stats_.find(dest_peer); + if (pc_stats_it == pc_stats_.end()) { + continue; + } + found = true; + total_packets_received += pc_stats_it->second.packets_received; + } + int64_t packet_loss = -1; + if (found) { + packet_loss = pair.second.packets_sent - total_packets_received; + } + ReportStats(pair.first, pair.second, network_layer_stats, packet_loss, + end_time); + } +} + +void StatsBasedNetworkQualityMetricsReporter::ReportStats( + const std::string& pc_label, + const PCStats& pc_stats, + const NetworkLayerStats& network_layer_stats, + int64_t packet_loss, + const Timestamp& end_time) { + ReportResult("bytes_discarded_no_receiver", pc_label, + network_layer_stats.stats->BytesDropped().bytes(), + "sizeInBytes"); + ReportResult("packets_discarded_no_receiver", pc_label, + network_layer_stats.stats->PacketsDropped(), "unitless"); + + ReportResult("payload_bytes_received", pc_label, + pc_stats.payload_received.bytes(), "sizeInBytes"); + ReportResult("payload_bytes_sent", pc_label, pc_stats.payload_sent.bytes(), + "sizeInBytes"); + + ReportResult("bytes_sent", pc_label, pc_stats.total_sent.bytes(), + "sizeInBytes"); + ReportResult("packets_sent", pc_label, pc_stats.packets_sent, "unitless"); + ReportResult("average_send_rate", pc_label, + (pc_stats.total_sent / (end_time - start_time_)).bytes_per_sec(), + "bytesPerSecond"); + ReportResult("bytes_received", pc_label, pc_stats.total_received.bytes(), + "sizeInBytes"); + ReportResult("packets_received", pc_label, pc_stats.packets_received, + "unitless"); + ReportResult( + "average_receive_rate", pc_label, + (pc_stats.total_received / (end_time - start_time_)).bytes_per_sec(), + "bytesPerSecond"); + ReportResult("sent_packets_loss", pc_label, packet_loss, "unitless"); +} + +void StatsBasedNetworkQualityMetricsReporter::ReportResult( + const std::string& metric_name, + const std::string& network_label, + const double value, + const std::string& unit) const { + test::PrintResult(metric_name, /*modifier=*/"", + GetTestCaseName(network_label), value, unit, + /*important=*/false); +} + +void StatsBasedNetworkQualityMetricsReporter::ReportResult( + const std::string& metric_name, + const std::string& network_label, + const SamplesStatsCounter& value, + const std::string& unit) const { + test::PrintResult(metric_name, /*modifier=*/"", + GetTestCaseName(network_label), value, unit, + /*important=*/false); +} + +std::string StatsBasedNetworkQualityMetricsReporter::GetTestCaseName( + absl::string_view network_label) const { + rtc::StringBuilder builder; + builder << test_case_name_ << "/" << network_label.data(); + return builder.str(); +} + +void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats( + const std::string& peer_name, + const NetworkLayerStats& stats) const { + DataRate average_send_rate = stats.stats->PacketsSent() >= 2 + ? stats.stats->AverageSendRate() + : DataRate::Zero(); + DataRate average_receive_rate = stats.stats->PacketsReceived() >= 2 + ? stats.stats->AverageReceiveRate() + : DataRate::Zero(); + rtc::StringBuilder log; + log << "Raw network layer statistic for [" << peer_name << "]:\n" + << "Local IPs:\n"; + std::vector<rtc::IPAddress> local_ips = stats.stats->LocalAddresses(); + for (size_t i = 0; i < local_ips.size(); ++i) { + log << " " << local_ips[i].ToString() << "\n"; + } + if (!stats.stats->SentPacketsSizeCounter().IsEmpty()) { + ReportResult("sent_packets_size", peer_name, + stats.stats->SentPacketsSizeCounter(), "sizeInBytes"); + } + if (!stats.stats->ReceivedPacketsSizeCounter().IsEmpty()) { + ReportResult("received_packets_size", peer_name, + stats.stats->ReceivedPacketsSizeCounter(), "sizeInBytes"); + } + if (!stats.stats->DroppedPacketsSizeCounter().IsEmpty()) { + ReportResult("dropped_packets_size", peer_name, + stats.stats->DroppedPacketsSizeCounter(), "sizeInBytes"); + } + if (!stats.stats->SentPacketsQueueWaitTimeUs().IsEmpty()) { + ReportResult("sent_packets_queue_wait_time_us", peer_name, + stats.stats->SentPacketsQueueWaitTimeUs(), "unitless"); + } + + log << "Send statistic:\n" + << " packets: " << stats.stats->PacketsSent() + << " bytes: " << stats.stats->BytesSent().bytes() + << " avg_rate (bytes/sec): " << average_send_rate.bytes_per_sec() + << " avg_rate (bps): " << average_send_rate.bps() << "\n" + << "Send statistic per destination:\n"; + + for (const auto& entry : stats.stats->OutgoingStatsPerDestination()) { + DataRate source_average_send_rate = entry.second->PacketsSent() >= 2 + ? entry.second->AverageSendRate() + : DataRate::Zero(); + log << "(" << entry.first.ToString() << "):\n" + << " packets: " << entry.second->PacketsSent() + << " bytes: " << entry.second->BytesSent().bytes() + << " avg_rate (bytes/sec): " << source_average_send_rate.bytes_per_sec() + << " avg_rate (bps): " << source_average_send_rate.bps() << "\n"; + if (!entry.second->SentPacketsSizeCounter().IsEmpty()) { + ReportResult("sent_packets_size", + peer_name + "/" + entry.first.ToString(), + stats.stats->SentPacketsSizeCounter(), "sizeInBytes"); + } + } + + log << "Receive statistic:\n" + << " packets: " << stats.stats->PacketsReceived() + << " bytes: " << stats.stats->BytesReceived().bytes() + << " avg_rate (bytes/sec): " << average_receive_rate.bytes_per_sec() + << " avg_rate (bps): " << average_receive_rate.bps() << "\n" + << "Receive statistic per source:\n"; + + for (const auto& entry : stats.stats->IncomingStatsPerSource()) { + DataRate source_average_receive_rate = + entry.second->PacketsReceived() >= 2 + ? entry.second->AverageReceiveRate() + : DataRate::Zero(); + log << "(" << entry.first.ToString() << "):\n" + << " packets: " << entry.second->PacketsReceived() + << " bytes: " << entry.second->BytesReceived().bytes() + << " avg_rate (bytes/sec): " + << source_average_receive_rate.bytes_per_sec() + << " avg_rate (bps): " << source_average_receive_rate.bps() << "\n"; + if (!entry.second->ReceivedPacketsSizeCounter().IsEmpty()) { + ReportResult("received_packets_size", + peer_name + "/" + entry.first.ToString(), + stats.stats->ReceivedPacketsSizeCounter(), "sizeInBytes"); + } + if (!entry.second->DroppedPacketsSizeCounter().IsEmpty()) { + ReportResult("dropped_packets_size", + peer_name + "/" + entry.first.ToString(), + stats.stats->DroppedPacketsSizeCounter(), "sizeInBytes"); + } + } + + RTC_LOG(LS_INFO) << log.str(); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.h b/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.h new file mode 100644 index 0000000000..a940d81df8 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.h @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_STATS_BASED_NETWORK_QUALITY_METRICS_REPORTER_H_ +#define TEST_PC_E2E_STATS_BASED_NETWORK_QUALITY_METRICS_REPORTER_H_ + +#include <cstdint> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <utility> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/network_emulation/network_emulation_interfaces.h" +#include "api/test/network_emulation_manager.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" +#include "rtc_base/ip_address.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// TODO(titovartem): make this class testable and add tests. +class StatsBasedNetworkQualityMetricsReporter + : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { + public: + // `networks` map peer name to network to report network layer stability stats + // and to log network layer metrics. + StatsBasedNetworkQualityMetricsReporter( + std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints, + NetworkEmulationManager* network_emulation) + : collector_(std::move(peer_endpoints), network_emulation), + clock_(network_emulation->time_controller()->GetClock()) {} + ~StatsBasedNetworkQualityMetricsReporter() override = default; + + void AddPeer(absl::string_view peer_name, + std::vector<EmulatedEndpoint*> endpoints); + + // Network stats must be empty when this method will be invoked. + void Start(absl::string_view test_case_name, + const TrackIdStreamInfoMap* reporter_helper) override; + void OnStatsReports( + absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report) override; + void StopAndReportResults() override; + + private: + struct PCStats { + // TODO(bugs.webrtc.org/10525): Separate audio and video counters. Depends + // on standard stat counters, enabled by field trial + // "WebRTC-UseStandardBytesStats". + DataSize payload_received = DataSize::Zero(); + DataSize payload_sent = DataSize::Zero(); + + // Total bytes/packets sent/received in all RTCTransport's. + DataSize total_received = DataSize::Zero(); + DataSize total_sent = DataSize::Zero(); + int64_t packets_received = 0; + int64_t packets_sent = 0; + }; + + struct NetworkLayerStats { + std::unique_ptr<EmulatedNetworkStats> stats; + std::set<std::string> receivers; + }; + + class NetworkLayerStatsCollector { + public: + NetworkLayerStatsCollector( + std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints, + NetworkEmulationManager* network_emulation); + + void Start(); + + void AddPeer(absl::string_view peer_name, + std::vector<EmulatedEndpoint*> endpoints); + + std::map<std::string, NetworkLayerStats> GetStats(); + + private: + Mutex mutex_; + std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints_ + RTC_GUARDED_BY(mutex_); + std::map<rtc::IPAddress, std::string> ip_to_peer_ RTC_GUARDED_BY(mutex_); + NetworkEmulationManager* const network_emulation_; + }; + + void ReportStats(const std::string& pc_label, + const PCStats& pc_stats, + const NetworkLayerStats& network_layer_stats, + int64_t packet_loss, + const Timestamp& end_time); + void ReportResult(const std::string& metric_name, + const std::string& network_label, + double value, + const std::string& unit) const; + void ReportResult(const std::string& metric_name, + const std::string& network_label, + const SamplesStatsCounter& value, + const std::string& unit) const; + std::string GetTestCaseName(absl::string_view network_label) const; + void LogNetworkLayerStats(const std::string& peer_name, + const NetworkLayerStats& stats) const; + + NetworkLayerStatsCollector collector_; + Clock* const clock_; + + std::string test_case_name_; + Timestamp start_time_ = Timestamp::MinusInfinity(); + + Mutex mutex_; + std::map<std::string, PCStats> pc_stats_ RTC_GUARDED_BY(mutex_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_STATS_BASED_NETWORK_QUALITY_METRICS_REPORTER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/stats_poller.cc b/third_party/libwebrtc/test/pc/e2e/stats_poller.cc new file mode 100644 index 0000000000..c04805fb20 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/stats_poller.cc @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/stats_poller.h" + +#include <utility> + +#include "rtc_base/logging.h" +#include "rtc_base/synchronization/mutex.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +void InternalStatsObserver::PollStats() { + peer_->GetStats(this); +} + +void InternalStatsObserver::OnStatsDelivered( + const rtc::scoped_refptr<const RTCStatsReport>& report) { + for (auto* observer : observers_) { + observer->OnStatsReports(pc_label_, report); + } +} + +StatsPoller::StatsPoller(std::vector<StatsObserverInterface*> observers, + std::map<std::string, StatsProvider*> peers) + : observers_(std::move(observers)) { + webrtc::MutexLock lock(&mutex_); + for (auto& peer : peers) { + pollers_.push_back(rtc::make_ref_counted<InternalStatsObserver>( + peer.first, peer.second, observers_)); + } +} + +StatsPoller::StatsPoller(std::vector<StatsObserverInterface*> observers, + std::map<std::string, TestPeer*> peers) + : observers_(std::move(observers)) { + webrtc::MutexLock lock(&mutex_); + for (auto& peer : peers) { + pollers_.push_back(rtc::make_ref_counted<InternalStatsObserver>( + peer.first, peer.second, observers_)); + } +} + +void StatsPoller::PollStatsAndNotifyObservers() { + webrtc::MutexLock lock(&mutex_); + for (auto& poller : pollers_) { + poller->PollStats(); + } +} + +void StatsPoller::RegisterParticipantInCall(absl::string_view peer_name, + StatsProvider* peer) { + webrtc::MutexLock lock(&mutex_); + pollers_.push_back(rtc::make_ref_counted<InternalStatsObserver>( + peer_name, peer, observers_)); +} + +bool StatsPoller::UnregisterParticipantInCall(absl::string_view peer_name) { + webrtc::MutexLock lock(&mutex_); + for (auto it = pollers_.begin(); it != pollers_.end(); ++it) { + if ((*it)->pc_label() == peer_name) { + pollers_.erase(it); + return true; + } + } + return false; +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/stats_poller.h b/third_party/libwebrtc/test/pc/e2e/stats_poller.h new file mode 100644 index 0000000000..3576f1bf05 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/stats_poller.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_STATS_POLLER_H_ +#define TEST_PC_E2E_STATS_POLLER_H_ + +#include <map> +#include <string> +#include <utility> +#include <vector> + +#include "api/peer_connection_interface.h" +#include "api/stats/rtc_stats_collector_callback.h" +#include "api/test/stats_observer_interface.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" +#include "test/pc/e2e/stats_provider.h" +#include "test/pc/e2e/test_peer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Helper class that will notify all the webrtc::test::StatsObserverInterface +// objects subscribed. +class InternalStatsObserver : public RTCStatsCollectorCallback { + public: + InternalStatsObserver(absl::string_view pc_label, + StatsProvider* peer, + std::vector<StatsObserverInterface*> observers) + : pc_label_(pc_label), peer_(peer), observers_(std::move(observers)) {} + + std::string pc_label() const { return pc_label_; } + + void PollStats(); + + void OnStatsDelivered( + const rtc::scoped_refptr<const RTCStatsReport>& report) override; + + private: + std::string pc_label_; + StatsProvider* peer_; + std::vector<StatsObserverInterface*> observers_; +}; + +// Helper class to invoke GetStats on a PeerConnection by passing a +// webrtc::StatsObserver that will notify all the +// webrtc::test::StatsObserverInterface subscribed. +class StatsPoller { + public: + StatsPoller(std::vector<StatsObserverInterface*> observers, + std::map<std::string, StatsProvider*> peers_to_observe); + StatsPoller(std::vector<StatsObserverInterface*> observers, + std::map<std::string, TestPeer*> peers_to_observe); + + void PollStatsAndNotifyObservers(); + + void RegisterParticipantInCall(absl::string_view peer_name, + StatsProvider* peer); + // Unregister participant from stats poller. Returns true if participant was + // removed and false if participant wasn't found. + bool UnregisterParticipantInCall(absl::string_view peer_name); + + private: + const std::vector<StatsObserverInterface*> observers_; + webrtc::Mutex mutex_; + std::vector<rtc::scoped_refptr<InternalStatsObserver>> pollers_ + RTC_GUARDED_BY(mutex_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_STATS_POLLER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/stats_poller_test.cc b/third_party/libwebrtc/test/pc/e2e/stats_poller_test.cc new file mode 100644 index 0000000000..02a323127b --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/stats_poller_test.cc @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/stats_poller.h" + +#include "api/stats/rtc_stats_collector_callback.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using ::testing::Eq; + +class TestStatsProvider : public StatsProvider { + public: + ~TestStatsProvider() override = default; + + void GetStats(RTCStatsCollectorCallback* callback) override { + stats_collections_count_++; + } + + int stats_collections_count() const { return stats_collections_count_; } + + private: + int stats_collections_count_ = 0; +}; + +class MockStatsObserver : public StatsObserverInterface { + public: + ~MockStatsObserver() override = default; + + MOCK_METHOD(void, + OnStatsReports, + (absl::string_view pc_label, + const rtc::scoped_refptr<const RTCStatsReport>& report)); +}; + +TEST(StatsPollerTest, UnregisterParticipantAddedInCtor) { + TestStatsProvider alice; + TestStatsProvider bob; + + MockStatsObserver stats_observer; + + StatsPoller poller(/*observers=*/{&stats_observer}, + /*peers_to_observe=*/{{"alice", &alice}, {"bob", &bob}}); + poller.PollStatsAndNotifyObservers(); + + EXPECT_THAT(alice.stats_collections_count(), Eq(1)); + EXPECT_THAT(bob.stats_collections_count(), Eq(1)); + + poller.UnregisterParticipantInCall("bob"); + poller.PollStatsAndNotifyObservers(); + + EXPECT_THAT(alice.stats_collections_count(), Eq(2)); + EXPECT_THAT(bob.stats_collections_count(), Eq(1)); +} + +TEST(StatsPollerTest, UnregisterParticipantRegisteredInCall) { + TestStatsProvider alice; + TestStatsProvider bob; + + MockStatsObserver stats_observer; + + StatsPoller poller(/*observers=*/{&stats_observer}, + /*peers_to_observe=*/{{"alice", &alice}}); + poller.RegisterParticipantInCall("bob", &bob); + poller.PollStatsAndNotifyObservers(); + + EXPECT_THAT(alice.stats_collections_count(), Eq(1)); + EXPECT_THAT(bob.stats_collections_count(), Eq(1)); + + poller.UnregisterParticipantInCall("bob"); + poller.PollStatsAndNotifyObservers(); + + EXPECT_THAT(alice.stats_collections_count(), Eq(2)); + EXPECT_THAT(bob.stats_collections_count(), Eq(1)); +} + +} // namespace +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/stats_provider.h b/third_party/libwebrtc/test/pc/e2e/stats_provider.h new file mode 100644 index 0000000000..eef62d779c --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/stats_provider.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_STATS_PROVIDER_H_ +#define TEST_PC_E2E_STATS_PROVIDER_H_ + +#include "api/stats/rtc_stats_collector_callback.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class StatsProvider { + public: + virtual ~StatsProvider() = default; + + virtual void GetStats(RTCStatsCollectorCallback* callback) = 0; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_STATS_PROVIDER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/test_activities_executor.cc b/third_party/libwebrtc/test/pc/e2e/test_activities_executor.cc new file mode 100644 index 0000000000..7bcf7dd6c3 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/test_activities_executor.cc @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/pc/e2e/test_activities_executor.h" + +#include <memory> +#include <utility> + +#include "absl/memory/memory.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/task_queue_for_test.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +void TestActivitiesExecutor::Start(TaskQueueBase* task_queue) { + RTC_DCHECK(task_queue); + task_queue_ = task_queue; + MutexLock lock(&lock_); + start_time_ = Now(); + while (!scheduled_activities_.empty()) { + PostActivity(std::move(scheduled_activities_.front())); + scheduled_activities_.pop(); + } +} + +void TestActivitiesExecutor::Stop() { + if (task_queue_ == nullptr) { + // Already stopped or not started. + return; + } + SendTask(task_queue_, [this]() { + MutexLock lock(&lock_); + for (auto& handle : repeating_task_handles_) { + handle.Stop(); + } + }); + task_queue_ = nullptr; +} + +void TestActivitiesExecutor::ScheduleActivity( + TimeDelta initial_delay_since_start, + absl::optional<TimeDelta> interval, + std::function<void(TimeDelta)> func) { + RTC_CHECK(initial_delay_since_start.IsFinite() && + initial_delay_since_start >= TimeDelta::Zero()); + RTC_CHECK(!interval || + (interval->IsFinite() && *interval > TimeDelta::Zero())); + MutexLock lock(&lock_); + ScheduledActivity activity(initial_delay_since_start, interval, func); + if (start_time_.IsInfinite()) { + scheduled_activities_.push(std::move(activity)); + } else { + PostActivity(std::move(activity)); + } +} + +void TestActivitiesExecutor::PostActivity(ScheduledActivity activity) { + // Because start_time_ will never change at this point copy it to local + // variable to capture in in lambda without requirement to hold a lock. + Timestamp start_time = start_time_; + + TimeDelta remaining_delay = + activity.initial_delay_since_start == TimeDelta::Zero() + ? TimeDelta::Zero() + : activity.initial_delay_since_start - (Now() - start_time); + if (remaining_delay < TimeDelta::Zero()) { + RTC_LOG(LS_WARNING) << "Executing late task immediately, late by=" + << ToString(remaining_delay.Abs()); + remaining_delay = TimeDelta::Zero(); + } + + if (activity.interval) { + if (remaining_delay == TimeDelta::Zero()) { + repeating_task_handles_.push_back(RepeatingTaskHandle::Start( + task_queue_, [activity, start_time, this]() { + activity.func(Now() - start_time); + return *activity.interval; + })); + return; + } + repeating_task_handles_.push_back(RepeatingTaskHandle::DelayedStart( + task_queue_, remaining_delay, [activity, start_time, this]() { + activity.func(Now() - start_time); + return *activity.interval; + })); + return; + } + + if (remaining_delay == TimeDelta::Zero()) { + task_queue_->PostTask( + [activity, start_time, this]() { activity.func(Now() - start_time); }); + return; + } + + task_queue_->PostDelayedTask( + [activity, start_time, this]() { activity.func(Now() - start_time); }, + remaining_delay); +} + +Timestamp TestActivitiesExecutor::Now() const { + return clock_->CurrentTime(); +} + +TestActivitiesExecutor::ScheduledActivity::ScheduledActivity( + TimeDelta initial_delay_since_start, + absl::optional<TimeDelta> interval, + std::function<void(TimeDelta)> func) + : initial_delay_since_start(initial_delay_since_start), + interval(interval), + func(std::move(func)) {} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/test_activities_executor.h b/third_party/libwebrtc/test/pc/e2e/test_activities_executor.h new file mode 100644 index 0000000000..2469ac7f36 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/test_activities_executor.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_TEST_ACTIVITIES_EXECUTOR_H_ +#define TEST_PC_E2E_TEST_ACTIVITIES_EXECUTOR_H_ + +#include <queue> +#include <vector> + +#include "absl/types/optional.h" +#include "api/task_queue/task_queue_base.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/task_queue_for_test.h" +#include "rtc_base/task_utils/repeating_task.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +class TestActivitiesExecutor { + public: + explicit TestActivitiesExecutor(Clock* clock) : clock_(clock) {} + ~TestActivitiesExecutor() { Stop(); } + + // Starts scheduled activities according to their schedule. All activities + // that will be scheduled after Start(...) was invoked will be executed + // immediately according to their schedule. + void Start(TaskQueueForTest* task_queue) { Start(task_queue->Get()); } + void Start(TaskQueueBase* task_queue); + void Stop(); + + // Schedule activity to be executed. If test isn't started yet, then activity + // will be executed according to its schedule after Start() will be invoked. + // If test is started, then it will be executed immediately according to its + // schedule. + void ScheduleActivity(TimeDelta initial_delay_since_start, + absl::optional<TimeDelta> interval, + std::function<void(TimeDelta)> func); + + private: + struct ScheduledActivity { + ScheduledActivity(TimeDelta initial_delay_since_start, + absl::optional<TimeDelta> interval, + std::function<void(TimeDelta)> func); + + TimeDelta initial_delay_since_start; + absl::optional<TimeDelta> interval; + std::function<void(TimeDelta)> func; + }; + + void PostActivity(ScheduledActivity activity) + RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_); + Timestamp Now() const; + + Clock* const clock_; + + TaskQueueBase* task_queue_; + + Mutex lock_; + // Time when test was started. Minus infinity means that it wasn't started + // yet. + Timestamp start_time_ RTC_GUARDED_BY(lock_) = Timestamp::MinusInfinity(); + // Queue of activities that were added before test was started. + // Activities from this queue will be posted on the `task_queue_` after test + // will be set up and then this queue will be unused. + std::queue<ScheduledActivity> scheduled_activities_ RTC_GUARDED_BY(lock_); + // List of task handles for activities, that are posted on `task_queue_` as + // repeated during the call. + std::vector<RepeatingTaskHandle> repeating_task_handles_ + RTC_GUARDED_BY(lock_); +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_TEST_ACTIVITIES_EXECUTOR_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/test_peer.cc b/third_party/libwebrtc/test/pc/e2e/test_peer.cc new file mode 100644 index 0000000000..d978f10665 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/test_peer.cc @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/test_peer.h" + +#include <string> +#include <utility> + +#include "absl/memory/memory.h" +#include "absl/strings/string_view.h" +#include "api/scoped_refptr.h" +#include "modules/audio_processing/include/audio_processing.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using VideoSubscription = ::webrtc::webrtc_pc_e2e:: + PeerConnectionE2EQualityTestFixture::VideoSubscription; +using VideoConfig = + ::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::VideoConfig; + +class SetRemoteDescriptionCallback + : public webrtc::SetRemoteDescriptionObserverInterface { + public: + void OnSetRemoteDescriptionComplete(webrtc::RTCError error) override { + is_called_ = true; + error_ = error; + } + + bool is_called() const { return is_called_; } + + webrtc::RTCError error() const { return error_; } + + private: + bool is_called_ = false; + webrtc::RTCError error_; +}; + +} // namespace + +ConfigurableParams TestPeer::configurable_params() const { + MutexLock lock(&mutex_); + return configurable_params_; +} + +void TestPeer::AddVideoConfig(VideoConfig config) { + MutexLock lock(&mutex_); + configurable_params_.video_configs.push_back(std::move(config)); +} + +void TestPeer::RemoveVideoConfig(absl::string_view stream_label) { + MutexLock lock(&mutex_); + bool config_removed = false; + for (auto it = configurable_params_.video_configs.begin(); + it != configurable_params_.video_configs.end(); ++it) { + if (*it->stream_label == stream_label) { + configurable_params_.video_configs.erase(it); + config_removed = true; + break; + } + } + RTC_CHECK(config_removed) << *params_.name << ": No video config with label [" + << stream_label << "] was found"; +} + +void TestPeer::SetVideoSubscription(VideoSubscription subscription) { + MutexLock lock(&mutex_); + configurable_params_.video_subscription = std::move(subscription); +} + +void TestPeer::GetStats(RTCStatsCollectorCallback* callback) { + pc()->signaling_thread()->PostTask( + SafeTask(signaling_thread_task_safety_, + [this, callback]() { pc()->GetStats(callback); })); +} + +bool TestPeer::SetRemoteDescription( + std::unique_ptr<SessionDescriptionInterface> desc, + std::string* error_out) { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + + auto observer = rtc::make_ref_counted<SetRemoteDescriptionCallback>(); + // We're assuming (and asserting) that the PeerConnection implementation of + // SetRemoteDescription is synchronous when called on the signaling thread. + pc()->SetRemoteDescription(std::move(desc), observer); + RTC_CHECK(observer->is_called()); + if (!observer->error().ok()) { + RTC_LOG(LS_ERROR) << *params_.name << ": Failed to set remote description: " + << observer->error().message(); + if (error_out) { + *error_out = observer->error().message(); + } + } + return observer->error().ok(); +} + +bool TestPeer::AddIceCandidates( + std::vector<std::unique_ptr<IceCandidateInterface>> candidates) { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + bool success = true; + for (auto& candidate : candidates) { + if (!pc()->AddIceCandidate(candidate.get())) { + std::string candidate_str; + bool res = candidate->ToString(&candidate_str); + RTC_CHECK(res); + RTC_LOG(LS_ERROR) << "Failed to add ICE candidate, candidate_str=" + << candidate_str; + success = false; + } else { + remote_ice_candidates_.push_back(std::move(candidate)); + } + } + return success; +} + +void TestPeer::Close() { + signaling_thread_task_safety_->SetNotAlive(); + wrapper_->pc()->Close(); + remote_ice_candidates_.clear(); + audio_processing_ = nullptr; + video_sources_.clear(); + wrapper_ = nullptr; + worker_thread_ = nullptr; +} + +TestPeer::TestPeer( + rtc::scoped_refptr<PeerConnectionFactoryInterface> pc_factory, + rtc::scoped_refptr<PeerConnectionInterface> pc, + std::unique_ptr<MockPeerConnectionObserver> observer, + Params params, + ConfigurableParams configurable_params, + std::vector<PeerConfigurerImpl::VideoSource> video_sources, + rtc::scoped_refptr<AudioProcessing> audio_processing, + std::unique_ptr<rtc::Thread> worker_thread) + : params_(std::move(params)), + configurable_params_(std::move(configurable_params)), + worker_thread_(std::move(worker_thread)), + wrapper_(std::make_unique<PeerConnectionWrapper>(std::move(pc_factory), + std::move(pc), + std::move(observer))), + video_sources_(std::move(video_sources)), + audio_processing_(audio_processing) { + signaling_thread_task_safety_ = PendingTaskSafetyFlag::CreateDetached(); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/test_peer.h b/third_party/libwebrtc/test/pc/e2e/test_peer.h new file mode 100644 index 0000000000..741e2e73f1 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/test_peer.h @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_TEST_PEER_H_ +#define TEST_PC_E2E_TEST_PEER_H_ + +#include <memory> +#include <vector> + +#include "absl/memory/memory.h" +#include "absl/strings/string_view.h" +#include "api/function_view.h" +#include "api/scoped_refptr.h" +#include "api/sequence_checker.h" +#include "api/set_remote_description_observer_interface.h" +#include "api/task_queue/pending_task_safety_flag.h" +#include "api/test/frame_generator_interface.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "pc/peer_connection_wrapper.h" +#include "rtc_base/logging.h" +#include "rtc_base/synchronization/mutex.h" +#include "test/pc/e2e/peer_configurer.h" +#include "test/pc/e2e/peer_connection_quality_test_params.h" +#include "test/pc/e2e/stats_provider.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +// Describes a single participant in the call. +class TestPeer final : public StatsProvider { + public: + ~TestPeer() override = default; + + const Params& params() const { return params_; } + + ConfigurableParams configurable_params() const; + void AddVideoConfig(PeerConnectionE2EQualityTestFixture::VideoConfig config); + // Removes video config with specified name. Crashes if the config with + // specified name isn't found. + void RemoveVideoConfig(absl::string_view stream_label); + void SetVideoSubscription( + PeerConnectionE2EQualityTestFixture::VideoSubscription subscription); + + void GetStats(RTCStatsCollectorCallback* callback) override; + + PeerConfigurerImpl::VideoSource ReleaseVideoSource(size_t i) { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return std::move(video_sources_[i]); + } + + PeerConnectionFactoryInterface* pc_factory() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->pc_factory(); + } + PeerConnectionInterface* pc() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->pc(); + } + MockPeerConnectionObserver* observer() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->observer(); + } + + // Tell underlying `PeerConnection` to create an Offer. + // `observer` will be invoked on the signaling thread when offer is created. + void CreateOffer( + rtc::scoped_refptr<CreateSessionDescriptionObserver> observer) { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + pc()->CreateOffer(observer.get(), params_.rtc_offer_answer_options); + } + std::unique_ptr<SessionDescriptionInterface> CreateOffer() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->CreateOffer(params_.rtc_offer_answer_options); + } + + std::unique_ptr<SessionDescriptionInterface> CreateAnswer() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->CreateAnswer(); + } + + bool SetLocalDescription(std::unique_ptr<SessionDescriptionInterface> desc, + std::string* error_out = nullptr) { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->SetLocalDescription(std::move(desc), error_out); + } + + // `error_out` will be set only if returned value is false. + bool SetRemoteDescription(std::unique_ptr<SessionDescriptionInterface> desc, + std::string* error_out = nullptr); + + rtc::scoped_refptr<RtpTransceiverInterface> AddTransceiver( + cricket::MediaType media_type, + const RtpTransceiverInit& init) { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->AddTransceiver(media_type, init); + } + + rtc::scoped_refptr<RtpSenderInterface> AddTrack( + rtc::scoped_refptr<MediaStreamTrackInterface> track, + const std::vector<std::string>& stream_ids = {}) { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->AddTrack(track, stream_ids); + } + + rtc::scoped_refptr<DataChannelInterface> CreateDataChannel( + const std::string& label) { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->CreateDataChannel(label); + } + + PeerConnectionInterface::SignalingState signaling_state() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->signaling_state(); + } + + bool IsIceGatheringDone() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->IsIceGatheringDone(); + } + + bool IsIceConnected() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->IsIceConnected(); + } + + rtc::scoped_refptr<const RTCStatsReport> GetStats() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + return wrapper_->GetStats(); + } + + void DetachAecDump() { + RTC_CHECK(wrapper_) << "TestPeer is already closed"; + if (audio_processing_) { + audio_processing_->DetachAecDump(); + } + } + + // Adds provided `candidates` to the owned peer connection. + bool AddIceCandidates( + std::vector<std::unique_ptr<IceCandidateInterface>> candidates); + + // Closes underlying peer connection and destroys all related objects freeing + // up related resources. + void Close(); + + protected: + friend class TestPeerFactory; + TestPeer(rtc::scoped_refptr<PeerConnectionFactoryInterface> pc_factory, + rtc::scoped_refptr<PeerConnectionInterface> pc, + std::unique_ptr<MockPeerConnectionObserver> observer, + Params params, + ConfigurableParams configurable_params, + std::vector<PeerConfigurerImpl::VideoSource> video_sources, + rtc::scoped_refptr<AudioProcessing> audio_processing, + std::unique_ptr<rtc::Thread> worker_thread); + + private: + const Params params_; + + mutable Mutex mutex_; + ConfigurableParams configurable_params_ RTC_GUARDED_BY(mutex_); + + // Safety flag to protect all tasks posted on the signaling thread to not be + // executed after `wrapper_` object is destructed. + rtc::scoped_refptr<PendingTaskSafetyFlag> signaling_thread_task_safety_ = + nullptr; + + // Keeps ownership of worker thread. It has to be destroyed after `wrapper_`. + std::unique_ptr<rtc::Thread> worker_thread_; + std::unique_ptr<PeerConnectionWrapper> wrapper_; + std::vector<PeerConfigurerImpl::VideoSource> video_sources_; + rtc::scoped_refptr<AudioProcessing> audio_processing_; + + std::vector<std::unique_ptr<IceCandidateInterface>> remote_ice_candidates_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_TEST_PEER_H_ diff --git a/third_party/libwebrtc/test/pc/e2e/test_peer_factory.cc b/third_party/libwebrtc/test/pc/e2e/test_peer_factory.cc new file mode 100644 index 0000000000..01388d90ea --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/test_peer_factory.cc @@ -0,0 +1,376 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "test/pc/e2e/test_peer_factory.h" + +#include <utility> + +#include "absl/memory/memory.h" +#include "absl/strings/string_view.h" +#include "api/task_queue/default_task_queue_factory.h" +#include "api/test/create_time_controller.h" +#include "api/test/time_controller.h" +#include "api/transport/field_trial_based_config.h" +#include "api/video_codecs/builtin_video_decoder_factory.h" +#include "api/video_codecs/builtin_video_encoder_factory.h" +#include "media/engine/webrtc_media_engine.h" +#include "media/engine/webrtc_media_engine_defaults.h" +#include "modules/audio_processing/aec_dump/aec_dump_factory.h" +#include "p2p/client/basic_port_allocator.h" +#include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h" +#include "test/pc/e2e/echo/echo_emulation.h" +#include "test/pc/e2e/peer_configurer.h" +#include "test/testsupport/copy_to_file_audio_capturer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { +namespace { + +using AudioConfig = + ::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::AudioConfig; +using VideoConfig = + ::webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::VideoConfig; +using EchoEmulationConfig = ::webrtc::webrtc_pc_e2e:: + PeerConnectionE2EQualityTestFixture::EchoEmulationConfig; + +constexpr int16_t kGeneratedAudioMaxAmplitude = 32000; +constexpr int kDefaultSamplingFrequencyInHz = 48000; + +// Sets mandatory entities in injectable components like `pcf_dependencies` +// and `pc_dependencies` if they are omitted. Also setup required +// dependencies, that won't be specially provided by factory and will be just +// transferred to peer connection creation code. +void SetMandatoryEntities(InjectableComponents* components, + TimeController& time_controller) { + RTC_DCHECK(components->pcf_dependencies); + RTC_DCHECK(components->pc_dependencies); + + // Setup required peer connection factory dependencies. + if (components->pcf_dependencies->task_queue_factory == nullptr) { + components->pcf_dependencies->task_queue_factory = + time_controller.CreateTaskQueueFactory(); + } + if (components->pcf_dependencies->call_factory == nullptr) { + components->pcf_dependencies->call_factory = + CreateTimeControllerBasedCallFactory(&time_controller); + } + if (components->pcf_dependencies->event_log_factory == nullptr) { + components->pcf_dependencies->event_log_factory = + std::make_unique<RtcEventLogFactory>( + components->pcf_dependencies->task_queue_factory.get()); + } + if (!components->pcf_dependencies->trials) { + components->pcf_dependencies->trials = + std::make_unique<FieldTrialBasedConfig>(); + } +} + +// Returns mapping from stream label to optional spatial index. +// If we have stream label "Foo" and mapping contains +// 1. `absl::nullopt` means "Foo" isn't simulcast/SVC stream +// 2. `kAnalyzeAnySpatialStream` means all simulcast/SVC streams are required +// 3. Concrete value means that particular simulcast/SVC stream have to be +// analyzed. +std::map<std::string, absl::optional<int>> +CalculateRequiredSpatialIndexPerStream( + const std::vector<VideoConfig>& video_configs) { + std::map<std::string, absl::optional<int>> out; + for (auto& video_config : video_configs) { + // Stream label should be set by fixture implementation here. + RTC_DCHECK(video_config.stream_label); + absl::optional<int> spatial_index; + if (video_config.simulcast_config) { + spatial_index = video_config.simulcast_config->target_spatial_index; + if (!spatial_index) { + spatial_index = kAnalyzeAnySpatialStream; + } + } + bool res = out.insert({*video_config.stream_label, spatial_index}).second; + RTC_DCHECK(res) << "Duplicate video_config.stream_label=" + << *video_config.stream_label; + } + return out; +} + +std::unique_ptr<TestAudioDeviceModule::Renderer> CreateAudioRenderer( + const absl::optional<RemotePeerAudioConfig>& config) { + if (!config) { + // Return default renderer because we always require some renderer. + return TestAudioDeviceModule::CreateDiscardRenderer( + kDefaultSamplingFrequencyInHz); + } + if (config->output_file_name) { + return TestAudioDeviceModule::CreateBoundedWavFileWriter( + config->output_file_name.value(), config->sampling_frequency_in_hz); + } + return TestAudioDeviceModule::CreateDiscardRenderer( + config->sampling_frequency_in_hz); +} + +std::unique_ptr<TestAudioDeviceModule::Capturer> CreateAudioCapturer( + const absl::optional<AudioConfig>& audio_config) { + if (!audio_config) { + // If we have no audio config we still need to provide some audio device. + // In such case use generated capturer. Despite of we provided audio here, + // in test media setup audio stream won't be added into peer connection. + return TestAudioDeviceModule::CreatePulsedNoiseCapturer( + kGeneratedAudioMaxAmplitude, kDefaultSamplingFrequencyInHz); + } + + switch (audio_config->mode) { + case AudioConfig::Mode::kGenerated: + return TestAudioDeviceModule::CreatePulsedNoiseCapturer( + kGeneratedAudioMaxAmplitude, audio_config->sampling_frequency_in_hz); + case AudioConfig::Mode::kFile: + RTC_DCHECK(audio_config->input_file_name); + return TestAudioDeviceModule::CreateWavFileReader( + audio_config->input_file_name.value(), /*repeat=*/true); + } +} + +rtc::scoped_refptr<AudioDeviceModule> CreateAudioDeviceModule( + absl::optional<AudioConfig> audio_config, + absl::optional<RemotePeerAudioConfig> remote_audio_config, + absl::optional<EchoEmulationConfig> echo_emulation_config, + TaskQueueFactory* task_queue_factory) { + std::unique_ptr<TestAudioDeviceModule::Renderer> renderer = + CreateAudioRenderer(remote_audio_config); + std::unique_ptr<TestAudioDeviceModule::Capturer> capturer = + CreateAudioCapturer(audio_config); + RTC_DCHECK(renderer); + RTC_DCHECK(capturer); + + // Setup echo emulation if required. + if (echo_emulation_config) { + capturer = std::make_unique<EchoEmulatingCapturer>(std::move(capturer), + *echo_emulation_config); + renderer = std::make_unique<EchoEmulatingRenderer>( + std::move(renderer), + static_cast<EchoEmulatingCapturer*>(capturer.get())); + } + + // Setup input stream dumping if required. + if (audio_config && audio_config->input_dump_file_name) { + capturer = std::make_unique<test::CopyToFileAudioCapturer>( + std::move(capturer), audio_config->input_dump_file_name.value()); + } + + return TestAudioDeviceModule::Create(task_queue_factory, std::move(capturer), + std::move(renderer), /*speed=*/1.f); +} + +std::unique_ptr<cricket::MediaEngineInterface> CreateMediaEngine( + PeerConnectionFactoryComponents* pcf_dependencies, + rtc::scoped_refptr<AudioDeviceModule> audio_device_module) { + cricket::MediaEngineDependencies media_deps; + media_deps.task_queue_factory = pcf_dependencies->task_queue_factory.get(); + media_deps.adm = audio_device_module; + media_deps.audio_processing = pcf_dependencies->audio_processing; + media_deps.audio_mixer = pcf_dependencies->audio_mixer; + media_deps.video_encoder_factory = + std::move(pcf_dependencies->video_encoder_factory); + media_deps.video_decoder_factory = + std::move(pcf_dependencies->video_decoder_factory); + webrtc::SetMediaEngineDefaults(&media_deps); + RTC_DCHECK(pcf_dependencies->trials); + media_deps.trials = pcf_dependencies->trials.get(); + + return cricket::CreateMediaEngine(std::move(media_deps)); +} + +void WrapVideoEncoderFactory( + absl::string_view peer_name, + double bitrate_multiplier, + std::map<std::string, absl::optional<int>> stream_required_spatial_index, + PeerConnectionFactoryComponents* pcf_dependencies, + VideoQualityAnalyzerInjectionHelper* video_analyzer_helper) { + std::unique_ptr<VideoEncoderFactory> video_encoder_factory; + if (pcf_dependencies->video_encoder_factory != nullptr) { + video_encoder_factory = std::move(pcf_dependencies->video_encoder_factory); + } else { + video_encoder_factory = CreateBuiltinVideoEncoderFactory(); + } + pcf_dependencies->video_encoder_factory = + video_analyzer_helper->WrapVideoEncoderFactory( + peer_name, std::move(video_encoder_factory), bitrate_multiplier, + std::move(stream_required_spatial_index)); +} + +void WrapVideoDecoderFactory( + absl::string_view peer_name, + PeerConnectionFactoryComponents* pcf_dependencies, + VideoQualityAnalyzerInjectionHelper* video_analyzer_helper) { + std::unique_ptr<VideoDecoderFactory> video_decoder_factory; + if (pcf_dependencies->video_decoder_factory != nullptr) { + video_decoder_factory = std::move(pcf_dependencies->video_decoder_factory); + } else { + video_decoder_factory = CreateBuiltinVideoDecoderFactory(); + } + pcf_dependencies->video_decoder_factory = + video_analyzer_helper->WrapVideoDecoderFactory( + peer_name, std::move(video_decoder_factory)); +} + +// Creates PeerConnectionFactoryDependencies objects, providing entities +// from InjectableComponents::PeerConnectionFactoryComponents. +PeerConnectionFactoryDependencies CreatePCFDependencies( + std::unique_ptr<PeerConnectionFactoryComponents> pcf_dependencies, + std::unique_ptr<cricket::MediaEngineInterface> media_engine, + rtc::Thread* signaling_thread, + rtc::Thread* worker_thread, + rtc::Thread* network_thread) { + PeerConnectionFactoryDependencies pcf_deps; + pcf_deps.signaling_thread = signaling_thread; + pcf_deps.worker_thread = worker_thread; + pcf_deps.network_thread = network_thread; + pcf_deps.media_engine = std::move(media_engine); + + pcf_deps.call_factory = std::move(pcf_dependencies->call_factory); + pcf_deps.event_log_factory = std::move(pcf_dependencies->event_log_factory); + pcf_deps.task_queue_factory = std::move(pcf_dependencies->task_queue_factory); + + if (pcf_dependencies->fec_controller_factory != nullptr) { + pcf_deps.fec_controller_factory = + std::move(pcf_dependencies->fec_controller_factory); + } + if (pcf_dependencies->network_controller_factory != nullptr) { + pcf_deps.network_controller_factory = + std::move(pcf_dependencies->network_controller_factory); + } + if (pcf_dependencies->neteq_factory != nullptr) { + pcf_deps.neteq_factory = std::move(pcf_dependencies->neteq_factory); + } + if (pcf_dependencies->trials != nullptr) { + pcf_deps.trials = std::move(pcf_dependencies->trials); + } + + return pcf_deps; +} + +// Creates PeerConnectionDependencies objects, providing entities +// from InjectableComponents::PeerConnectionComponents. +PeerConnectionDependencies CreatePCDependencies( + MockPeerConnectionObserver* observer, + uint32_t port_allocator_extra_flags, + std::unique_ptr<PeerConnectionComponents> pc_dependencies) { + PeerConnectionDependencies pc_deps(observer); + + auto port_allocator = std::make_unique<cricket::BasicPortAllocator>( + pc_dependencies->network_manager, pc_dependencies->packet_socket_factory); + + // This test does not support TCP + int flags = port_allocator_extra_flags | cricket::PORTALLOCATOR_DISABLE_TCP; + port_allocator->set_flags(port_allocator->flags() | flags); + + pc_deps.allocator = std::move(port_allocator); + + if (pc_dependencies->async_resolver_factory != nullptr) { + pc_deps.async_resolver_factory = + std::move(pc_dependencies->async_resolver_factory); + } + if (pc_dependencies->cert_generator != nullptr) { + pc_deps.cert_generator = std::move(pc_dependencies->cert_generator); + } + if (pc_dependencies->tls_cert_verifier != nullptr) { + pc_deps.tls_cert_verifier = std::move(pc_dependencies->tls_cert_verifier); + } + if (pc_dependencies->ice_transport_factory != nullptr) { + pc_deps.ice_transport_factory = + std::move(pc_dependencies->ice_transport_factory); + } + return pc_deps; +} + +} // namespace + +absl::optional<RemotePeerAudioConfig> RemotePeerAudioConfig::Create( + absl::optional<AudioConfig> config) { + if (!config) { + return absl::nullopt; + } + return RemotePeerAudioConfig(config.value()); +} + +std::unique_ptr<TestPeer> TestPeerFactory::CreateTestPeer( + std::unique_ptr<PeerConfigurerImpl> configurer, + std::unique_ptr<MockPeerConnectionObserver> observer, + absl::optional<RemotePeerAudioConfig> remote_audio_config, + absl::optional<PeerConnectionE2EQualityTestFixture::EchoEmulationConfig> + echo_emulation_config) { + std::unique_ptr<InjectableComponents> components = + configurer->ReleaseComponents(); + std::unique_ptr<Params> params = configurer->ReleaseParams(); + std::unique_ptr<ConfigurableParams> configurable_params = + configurer->ReleaseConfigurableParams(); + std::vector<PeerConfigurerImpl::VideoSource> video_sources = + configurer->ReleaseVideoSources(); + RTC_DCHECK(components); + RTC_DCHECK(params); + RTC_DCHECK(configurable_params); + RTC_DCHECK_EQ(configurable_params->video_configs.size(), + video_sources.size()); + SetMandatoryEntities(components.get(), time_controller_); + params->rtc_configuration.sdp_semantics = SdpSemantics::kUnifiedPlan; + + // Create peer connection factory. + if (components->pcf_dependencies->audio_processing == nullptr) { + components->pcf_dependencies->audio_processing = + webrtc::AudioProcessingBuilder().Create(); + } + if (params->aec_dump_path) { + components->pcf_dependencies->audio_processing->CreateAndAttachAecDump( + *params->aec_dump_path, -1, task_queue_); + } + rtc::scoped_refptr<AudioDeviceModule> audio_device_module = + CreateAudioDeviceModule( + params->audio_config, remote_audio_config, echo_emulation_config, + components->pcf_dependencies->task_queue_factory.get()); + WrapVideoEncoderFactory( + params->name.value(), params->video_encoder_bitrate_multiplier, + CalculateRequiredSpatialIndexPerStream( + configurable_params->video_configs), + components->pcf_dependencies.get(), video_analyzer_helper_); + WrapVideoDecoderFactory(params->name.value(), + components->pcf_dependencies.get(), + video_analyzer_helper_); + std::unique_ptr<cricket::MediaEngineInterface> media_engine = + CreateMediaEngine(components->pcf_dependencies.get(), + audio_device_module); + + std::unique_ptr<rtc::Thread> worker_thread = + time_controller_.CreateThread("worker_thread"); + // Store `webrtc::AudioProcessing` into local variable before move of + // `components->pcf_dependencies` + rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing = + components->pcf_dependencies->audio_processing; + PeerConnectionFactoryDependencies pcf_deps = CreatePCFDependencies( + std::move(components->pcf_dependencies), std::move(media_engine), + signaling_thread_, worker_thread.get(), components->network_thread); + rtc::scoped_refptr<PeerConnectionFactoryInterface> peer_connection_factory = + CreateModularPeerConnectionFactory(std::move(pcf_deps)); + + // Create peer connection. + PeerConnectionDependencies pc_deps = + CreatePCDependencies(observer.get(), params->port_allocator_extra_flags, + std::move(components->pc_dependencies)); + rtc::scoped_refptr<PeerConnectionInterface> peer_connection = + peer_connection_factory + ->CreatePeerConnectionOrError(params->rtc_configuration, + std::move(pc_deps)) + .MoveValue(); + peer_connection->SetBitrate(params->bitrate_settings); + + return absl::WrapUnique(new TestPeer( + peer_connection_factory, peer_connection, std::move(observer), + std::move(*params), std::move(*configurable_params), + std::move(video_sources), audio_processing, std::move(worker_thread))); +} + +} // namespace webrtc_pc_e2e +} // namespace webrtc diff --git a/third_party/libwebrtc/test/pc/e2e/test_peer_factory.h b/third_party/libwebrtc/test/pc/e2e/test_peer_factory.h new file mode 100644 index 0000000000..8d78e2f8d9 --- /dev/null +++ b/third_party/libwebrtc/test/pc/e2e/test_peer_factory.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_PC_E2E_TEST_PEER_FACTORY_H_ +#define TEST_PC_E2E_TEST_PEER_FACTORY_H_ + +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "absl/strings/string_view.h" +#include "api/rtc_event_log/rtc_event_log_factory.h" +#include "api/test/peerconnection_quality_test_fixture.h" +#include "api/test/time_controller.h" +#include "modules/audio_device/include/test_audio_device.h" +#include "rtc_base/task_queue.h" +#include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h" +#include "test/pc/e2e/peer_configurer.h" +#include "test/pc/e2e/peer_connection_quality_test_params.h" +#include "test/pc/e2e/test_peer.h" + +namespace webrtc { +namespace webrtc_pc_e2e { + +struct RemotePeerAudioConfig { + explicit RemotePeerAudioConfig( + PeerConnectionE2EQualityTestFixture::AudioConfig config) + : sampling_frequency_in_hz(config.sampling_frequency_in_hz), + output_file_name(config.output_dump_file_name) {} + + static absl::optional<RemotePeerAudioConfig> Create( + absl::optional<PeerConnectionE2EQualityTestFixture::AudioConfig> config); + + int sampling_frequency_in_hz; + absl::optional<std::string> output_file_name; +}; + +class TestPeerFactory { + public: + // Creates a test peer factory. + // `signaling_thread` will be used as a signaling thread for all peers created + // by this factory. + // `time_controller` will be used to create required threads, task queue + // factories and call factory. + // `video_analyzer_helper` will be used to setup video quality analysis for + // created peers. + // `task_queue` will be used for AEC dump if it is requested. + TestPeerFactory(rtc::Thread* signaling_thread, + TimeController& time_controller, + VideoQualityAnalyzerInjectionHelper* video_analyzer_helper, + rtc::TaskQueue* task_queue) + : signaling_thread_(signaling_thread), + time_controller_(time_controller), + video_analyzer_helper_(video_analyzer_helper), + task_queue_(task_queue) {} + + // Setups all components, that should be provided to WebRTC + // PeerConnectionFactory and PeerConnection creation methods, + // also will setup dependencies, that are required for media analyzers + // injection. + std::unique_ptr<TestPeer> CreateTestPeer( + std::unique_ptr<PeerConfigurerImpl> configurer, + std::unique_ptr<MockPeerConnectionObserver> observer, + absl::optional<RemotePeerAudioConfig> remote_audio_config, + absl::optional<PeerConnectionE2EQualityTestFixture::EchoEmulationConfig> + echo_emulation_config); + + private: + rtc::Thread* signaling_thread_; + TimeController& time_controller_; + VideoQualityAnalyzerInjectionHelper* video_analyzer_helper_; + rtc::TaskQueue* task_queue_; +}; + +} // namespace webrtc_pc_e2e +} // namespace webrtc + +#endif // TEST_PC_E2E_TEST_PEER_FACTORY_H_ |