summaryrefslogtreecommitdiffstats
path: root/third_party/libwebrtc/test/pc
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-21 11:44:51 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-21 11:44:51 +0000
commit9e3c08db40b8916968b9f30096c7be3f00ce9647 (patch)
treea68f146d7fa01f0134297619fbe7e33db084e0aa /third_party/libwebrtc/test/pc
parentInitial commit. (diff)
downloadthunderbird-9e3c08db40b8916968b9f30096c7be3f00ce9647.tar.xz
thunderbird-9e3c08db40b8916968b9f30096c7be3f00ce9647.zip
Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/libwebrtc/test/pc')
-rw-r--r--third_party/libwebrtc/test/pc/e2e/BUILD.gn573
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc175
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h81
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/BUILD.gn573
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.cc220
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.h106
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink_test.cc598
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.cc85
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h73
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper_test.cc160
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc1228
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h197
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.cc45
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h36
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc209
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h169
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc575
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h157
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc1648
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.cc52
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h132
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc682
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc172
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h284
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.cc121
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h100
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state_test.cc126
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc2204
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h79
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc168
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h101
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue.h168
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue_test.cc206
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.cc101
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.h94
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection_test.cc152
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc272
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h153
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc403
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h194
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.cc60
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h34
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper_test.cc61
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc187
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h104
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc445
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.cc118
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.h56
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping_test.cc196
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.cc37
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h46
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector_unittest.cc57
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc264
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h170
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc162
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h81
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer_helper.cc63
-rw-r--r--third_party/libwebrtc/test/pc/e2e/analyzer_helper.h61
-rw-r--r--third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.cc151
-rw-r--r--third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.h68
-rw-r--r--third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.cc117
-rw-r--r--third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.h79
-rw-r--r--third_party/libwebrtc/test/pc/e2e/g3doc/architecture.md209
-rw-r--r--third_party/libwebrtc/test/pc/e2e/g3doc/default_video_quality_analyzer.md197
-rw-r--r--third_party/libwebrtc/test/pc/e2e/g3doc/in_test_psnr_plot.pngbin0 -> 39236 bytes
-rw-r--r--third_party/libwebrtc/test/pc/e2e/g3doc/index.md224
-rw-r--r--third_party/libwebrtc/test/pc/e2e/g3doc/single_process_encoded_image_data_injector.pngbin0 -> 78481 bytes
-rw-r--r--third_party/libwebrtc/test/pc/e2e/g3doc/video_quality_analyzer_pipeline.pngbin0 -> 35899 bytes
-rw-r--r--third_party/libwebrtc/test/pc/e2e/g3doc/vp8_simulcast_offer_modification.pngbin0 -> 79641 bytes
-rw-r--r--third_party/libwebrtc/test/pc/e2e/media/media_helper.cc128
-rw-r--r--third_party/libwebrtc/test/pc/e2e/media/media_helper.h58
-rw-r--r--third_party/libwebrtc/test/pc/e2e/media/test_video_capturer_video_track_source.h55
-rw-r--r--third_party/libwebrtc/test/pc/e2e/metric_metadata_keys.h60
-rw-r--r--third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.cc183
-rw-r--r--third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.h72
-rw-r--r--third_party/libwebrtc/test/pc/e2e/peer_connection_e2e_smoke_test.cc536
-rw-r--r--third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.cc763
-rw-r--r--third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.h155
-rw-r--r--third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc1102
-rw-r--r--third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_test.cc139
-rw-r--r--third_party/libwebrtc/test/pc/e2e/peer_params_preprocessor.cc217
-rw-r--r--third_party/libwebrtc/test/pc/e2e/peer_params_preprocessor.h52
-rw-r--r--third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.cc601
-rw-r--r--third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.h146
-rw-r--r--third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.cc592
-rw-r--r--third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.h136
-rw-r--r--third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter_test.cc150
-rw-r--r--third_party/libwebrtc/test/pc/e2e/stats_poller.cc78
-rw-r--r--third_party/libwebrtc/test/pc/e2e/stats_poller.h80
-rw-r--r--third_party/libwebrtc/test/pc/e2e/stats_poller_test.cc90
-rw-r--r--third_party/libwebrtc/test/pc/e2e/stats_provider.h29
-rw-r--r--third_party/libwebrtc/test/pc/e2e/test_activities_executor.cc122
-rw-r--r--third_party/libwebrtc/test/pc/e2e/test_activities_executor.h85
-rw-r--r--third_party/libwebrtc/test/pc/e2e/test_peer.cc151
-rw-r--r--third_party/libwebrtc/test/pc/e2e/test_peer.h188
-rw-r--r--third_party/libwebrtc/test/pc/e2e/test_peer_factory.cc374
-rw-r--r--third_party/libwebrtc/test/pc/e2e/test_peer_factory.h84
-rw-r--r--third_party/libwebrtc/test/pc/sctp/BUILD.gn18
-rw-r--r--third_party/libwebrtc/test/pc/sctp/fake_sctp_transport.h79
99 files changed, 22642 insertions, 0 deletions
diff --git a/third_party/libwebrtc/test/pc/e2e/BUILD.gn b/third_party/libwebrtc/test/pc/e2e/BUILD.gn
new file mode 100644
index 0000000000..7354aa8ba4
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/BUILD.gn
@@ -0,0 +1,573 @@
+# Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import("../../../webrtc.gni")
+
+rtc_library("metric_metadata_keys") {
+ testonly = true
+ sources = [ "metric_metadata_keys.h" ]
+}
+
+if (!build_with_chromium) {
+ group("e2e") {
+ testonly = true
+
+ deps = [ ":metric_metadata_keys" ]
+ if (rtc_include_tests) {
+ deps += [
+ ":peerconnection_quality_test",
+ ":test_peer",
+ ]
+ }
+ }
+
+ if (rtc_include_tests) {
+ group("e2e_unittests") {
+ testonly = true
+
+ deps = [
+ ":peer_connection_e2e_smoke_test",
+ ":peer_connection_quality_test_metric_names_test",
+ ":peer_connection_quality_test_test",
+ ":stats_based_network_quality_metrics_reporter_test",
+ ":stats_poller_test",
+ ]
+ }
+ }
+
+ if (rtc_include_tests) {
+ rtc_library("echo_emulation") {
+ testonly = true
+ sources = [
+ "echo/echo_emulation.cc",
+ "echo/echo_emulation.h",
+ ]
+ deps = [
+ "../../../api/test/pclf:media_configuration",
+ "../../../modules/audio_device:audio_device_impl",
+ "../../../rtc_base:swap_queue",
+ ]
+ }
+
+ rtc_library("test_peer") {
+ testonly = true
+ sources = [
+ "test_peer.cc",
+ "test_peer.h",
+ ]
+ deps = [
+ ":stats_provider",
+ "../../../api:frame_generator_api",
+ "../../../api:function_view",
+ "../../../api:libjingle_peerconnection_api",
+ "../../../api:scoped_refptr",
+ "../../../api:sequence_checker",
+ "../../../api/task_queue:pending_task_safety_flag",
+ "../../../api/test/pclf:media_configuration",
+ "../../../api/test/pclf:media_quality_test_params",
+ "../../../api/test/pclf:peer_configurer",
+ "../../../modules/audio_processing:api",
+ "../../../pc:peerconnection_wrapper",
+ "../../../rtc_base:logging",
+ "../../../rtc_base:refcount",
+ "../../../rtc_base/synchronization:mutex",
+ ]
+ absl_deps = [
+ "//third_party/abseil-cpp/absl/memory",
+ "//third_party/abseil-cpp/absl/strings",
+ "//third_party/abseil-cpp/absl/types:variant",
+ ]
+ }
+
+ rtc_library("test_peer_factory") {
+ testonly = true
+ sources = [
+ "test_peer_factory.cc",
+ "test_peer_factory.h",
+ ]
+ deps = [
+ ":echo_emulation",
+ ":test_peer",
+ "../..:copy_to_file_audio_capturer",
+ "../../../api:create_time_controller",
+ "../../../api:time_controller",
+ "../../../api/rtc_event_log:rtc_event_log_factory",
+ "../../../api/task_queue:default_task_queue_factory",
+ "../../../api/test/pclf:media_configuration",
+ "../../../api/test/pclf:media_quality_test_params",
+ "../../../api/test/pclf:peer_configurer",
+ "../../../api/transport:field_trial_based_config",
+ "../../../api/video_codecs:builtin_video_decoder_factory",
+ "../../../api/video_codecs:builtin_video_encoder_factory",
+ "../../../media:rtc_audio_video",
+ "../../../media:rtc_media_engine_defaults",
+ "../../../modules/audio_device:audio_device_impl",
+ "../../../modules/audio_processing/aec_dump",
+ "../../../p2p:rtc_p2p",
+ "../../../rtc_base:rtc_task_queue",
+ "../../../rtc_base:threading",
+ "analyzer/video:quality_analyzing_video_encoder",
+ "analyzer/video:video_quality_analyzer_injection_helper",
+ ]
+ absl_deps = [
+ "//third_party/abseil-cpp/absl/memory",
+ "//third_party/abseil-cpp/absl/strings",
+ ]
+ }
+
+ rtc_library("media_helper") {
+ testonly = true
+ sources = [
+ "media/media_helper.cc",
+ "media/media_helper.h",
+ "media/test_video_capturer_video_track_source.h",
+ ]
+ deps = [
+ ":test_peer",
+ "../..:fileutils",
+ "../..:platform_video_capturer",
+ "../..:video_test_common",
+ "../../../api:create_frame_generator",
+ "../../../api:frame_generator_api",
+ "../../../api:media_stream_interface",
+ "../../../api/test/pclf:media_configuration",
+ "../../../api/test/pclf:peer_configurer",
+ "../../../api/video:video_frame",
+ "../../../pc:session_description",
+ "../../../pc:video_track_source",
+ "analyzer/video:video_quality_analyzer_injection_helper",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:variant" ]
+ }
+
+ rtc_library("peer_params_preprocessor") {
+ visibility = [ "*" ]
+ testonly = true
+ sources = [
+ "peer_params_preprocessor.cc",
+ "peer_params_preprocessor.h",
+ ]
+ deps = [
+ "../..:fileutils",
+ "../../../api:peer_network_dependencies",
+ "../../../api/test/pclf:media_configuration",
+ "../../../api/test/pclf:media_quality_test_params",
+ "../../../api/test/pclf:peer_configurer",
+ "../../../modules/video_coding/svc:scalability_mode_util",
+ "../../../modules/video_coding/svc:scalability_structures",
+ "../../../rtc_base:macromagic",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
+ }
+
+ rtc_library("test_activities_executor") {
+ testonly = true
+ sources = [
+ "test_activities_executor.cc",
+ "test_activities_executor.h",
+ ]
+ deps = [
+ "../../../api/task_queue",
+ "../../../api/units:time_delta",
+ "../../../api/units:timestamp",
+ "../../../rtc_base:checks",
+ "../../../rtc_base:criticalsection",
+ "../../../rtc_base:logging",
+ "../../../rtc_base:task_queue_for_test",
+ "../../../rtc_base/synchronization:mutex",
+ "../../../rtc_base/task_utils:repeating_task",
+ "../../../system_wrappers",
+ ]
+ absl_deps = [
+ "//third_party/abseil-cpp/absl/memory",
+ "//third_party/abseil-cpp/absl/types:optional",
+ ]
+ }
+
+ rtc_library("peerconnection_quality_test") {
+ testonly = true
+
+ sources = [
+ "peer_connection_quality_test.cc",
+ "peer_connection_quality_test.h",
+ ]
+ deps = [
+ ":analyzer_helper",
+ ":cross_media_metrics_reporter",
+ ":default_audio_quality_analyzer",
+ ":media_helper",
+ ":metric_metadata_keys",
+ ":peer_params_preprocessor",
+ ":sdp_changer",
+ ":stats_poller",
+ ":test_activities_executor",
+ ":test_peer",
+ ":test_peer_factory",
+ "../..:field_trial",
+ "../..:fileutils",
+ "../..:perf_test",
+ "../../../api:audio_quality_analyzer_api",
+ "../../../api:libjingle_peerconnection_api",
+ "../../../api:media_stream_interface",
+ "../../../api:peer_connection_quality_test_fixture_api",
+ "../../../api:rtc_event_log_output_file",
+ "../../../api:scoped_refptr",
+ "../../../api:time_controller",
+ "../../../api:video_quality_analyzer_api",
+ "../../../api/rtc_event_log",
+ "../../../api/task_queue",
+ "../../../api/test/metrics:metric",
+ "../../../api/test/metrics:metrics_logger",
+ "../../../api/test/pclf:media_configuration",
+ "../../../api/test/pclf:media_quality_test_params",
+ "../../../api/test/pclf:peer_configurer",
+ "../../../api/units:time_delta",
+ "../../../api/units:timestamp",
+ "../../../pc:pc_test_utils",
+ "../../../pc:sdp_utils",
+ "../../../rtc_base:gunit_helpers",
+ "../../../rtc_base:macromagic",
+ "../../../rtc_base:safe_conversions",
+ "../../../rtc_base:stringutils",
+ "../../../rtc_base:task_queue_for_test",
+ "../../../rtc_base:threading",
+ "../../../rtc_base/synchronization:mutex",
+ "../../../system_wrappers",
+ "../../../system_wrappers:field_trial",
+ "analyzer/video:default_video_quality_analyzer",
+ "analyzer/video:single_process_encoded_image_data_injector",
+ "analyzer/video:video_frame_tracking_id_injector",
+ "analyzer/video:video_quality_analyzer_injection_helper",
+ "analyzer/video:video_quality_metrics_reporter",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
+ }
+
+ peer_connection_e2e_smoke_test_resources = [
+ "../../../resources/pc_quality_smoke_test_alice_source.wav",
+ "../../../resources/pc_quality_smoke_test_bob_source.wav",
+ ]
+ if (is_ios) {
+ bundle_data("peer_connection_e2e_smoke_test_resources_bundle_data") {
+ testonly = true
+ sources = peer_connection_e2e_smoke_test_resources
+ outputs = [ "{{bundle_resources_dir}}/{{source_file_part}}" ]
+ }
+ }
+
+ rtc_library("peer_connection_e2e_smoke_test") {
+ testonly = true
+
+ sources = [ "peer_connection_e2e_smoke_test.cc" ]
+ deps = [
+ ":default_audio_quality_analyzer",
+ ":network_quality_metrics_reporter",
+ ":stats_based_network_quality_metrics_reporter",
+ "../../../api:callfactory_api",
+ "../../../api:create_network_emulation_manager",
+ "../../../api:create_peer_connection_quality_test_frame_generator",
+ "../../../api:create_peerconnection_quality_test_fixture",
+ "../../../api:libjingle_peerconnection_api",
+ "../../../api:media_stream_interface",
+ "../../../api:network_emulation_manager_api",
+ "../../../api:peer_connection_quality_test_fixture_api",
+ "../../../api:scoped_refptr",
+ "../../../api:simulated_network_api",
+ "../../../api/audio_codecs:builtin_audio_decoder_factory",
+ "../../../api/audio_codecs:builtin_audio_encoder_factory",
+ "../../../api/test/metrics:global_metrics_logger_and_exporter",
+ "../../../api/test/pclf:media_configuration",
+ "../../../api/test/pclf:media_quality_test_params",
+ "../../../api/test/pclf:peer_configurer",
+ "../../../api/video_codecs:builtin_video_decoder_factory",
+ "../../../api/video_codecs:builtin_video_encoder_factory",
+ "../../../call:simulated_network",
+ "../../../media:rtc_audio_video",
+ "../../../modules/audio_device:audio_device_impl",
+ "../../../p2p:rtc_p2p",
+ "../../../pc:pc_test_utils",
+ "../../../pc:peerconnection_wrapper",
+ "../../../rtc_base:gunit_helpers",
+ "../../../rtc_base:logging",
+ "../../../rtc_base:rtc_event",
+ "../../../system_wrappers:field_trial",
+ "../../../test:field_trial",
+ "../../../test:fileutils",
+ "../../../test:test_support",
+ "analyzer/video:default_video_quality_analyzer",
+ "analyzer/video:default_video_quality_analyzer_shared",
+ ]
+ data = peer_connection_e2e_smoke_test_resources
+ if (is_mac || is_ios) {
+ deps += [ "../../../modules/video_coding:objc_codec_factory_helper" ]
+ }
+ if (is_ios) {
+ deps += [ ":peer_connection_e2e_smoke_test_resources_bundle_data" ]
+ }
+ }
+
+ rtc_library("peer_connection_quality_test_metric_names_test") {
+ testonly = true
+ sources = [ "peer_connection_quality_test_metric_names_test.cc" ]
+ deps = [
+ ":metric_metadata_keys",
+ ":peerconnection_quality_test",
+ ":stats_based_network_quality_metrics_reporter",
+ "../..:test_support",
+ "../../../api:create_network_emulation_manager",
+ "../../../api:create_peer_connection_quality_test_frame_generator",
+ "../../../api:network_emulation_manager_api",
+ "../../../api:peer_connection_quality_test_fixture_api",
+ "../../../api/test/metrics:metrics_logger",
+ "../../../api/test/metrics:stdout_metrics_exporter",
+ "../../../api/test/pclf:media_configuration",
+ "../../../api/test/pclf:media_quality_test_params",
+ "../../../api/test/pclf:peer_configurer",
+ "../../../api/units:time_delta",
+ ]
+ }
+
+ rtc_library("stats_based_network_quality_metrics_reporter_test") {
+ testonly = true
+ sources = [ "stats_based_network_quality_metrics_reporter_test.cc" ]
+ deps = [
+ ":metric_metadata_keys",
+ ":peerconnection_quality_test",
+ ":stats_based_network_quality_metrics_reporter",
+ "../..:test_support",
+ "../../../api:array_view",
+ "../../../api:create_network_emulation_manager",
+ "../../../api:create_peer_connection_quality_test_frame_generator",
+ "../../../api:network_emulation_manager_api",
+ "../../../api:peer_connection_quality_test_fixture_api",
+ "../../../api/test/metrics:metrics_logger",
+ "../../../api/test/metrics:stdout_metrics_exporter",
+ "../../../api/test/pclf:media_configuration",
+ "../../../api/test/pclf:media_quality_test_params",
+ "../../../api/test/pclf:peer_configurer",
+ "../../../api/units:time_delta",
+ ]
+ absl_deps = [
+ "//third_party/abseil-cpp/absl/strings",
+ "//third_party/abseil-cpp/absl/types:optional",
+ ]
+ }
+
+ rtc_library("peer_connection_quality_test_test") {
+ testonly = true
+ sources = [ "peer_connection_quality_test_test.cc" ]
+ deps = [
+ ":peerconnection_quality_test",
+ "../..:fileutils",
+ "../..:test_support",
+ "../..:video_test_support",
+ "../../../api:create_network_emulation_manager",
+ "../../../api:network_emulation_manager_api",
+ "../../../api:peer_connection_quality_test_fixture_api",
+ "../../../api/test/metrics:global_metrics_logger_and_exporter",
+ "../../../api/test/pclf:media_configuration",
+ "../../../api/test/pclf:media_quality_test_params",
+ "../../../api/test/pclf:peer_configurer",
+ "../../../api/units:time_delta",
+ "../../../rtc_base:timeutils",
+ ]
+ }
+
+ rtc_library("stats_provider") {
+ testonly = true
+ sources = [ "stats_provider.h" ]
+ deps = [ "../../../api:rtc_stats_api" ]
+ }
+
+ rtc_library("stats_poller") {
+ testonly = true
+ sources = [
+ "stats_poller.cc",
+ "stats_poller.h",
+ ]
+ deps = [
+ ":stats_provider",
+ ":test_peer",
+ "../../../api:libjingle_peerconnection_api",
+ "../../../api:rtc_stats_api",
+ "../../../api:stats_observer_interface",
+ "../../../rtc_base:logging",
+ "../../../rtc_base:macromagic",
+ "../../../rtc_base/synchronization:mutex",
+ ]
+ }
+
+ rtc_library("stats_poller_test") {
+ testonly = true
+ sources = [ "stats_poller_test.cc" ]
+ deps = [
+ ":stats_poller",
+ "../..:test_support",
+ "../../../api:rtc_stats_api",
+ ]
+ }
+ }
+
+ rtc_library("analyzer_helper") {
+ sources = [
+ "analyzer_helper.cc",
+ "analyzer_helper.h",
+ ]
+ deps = [
+ "../../../api:sequence_checker",
+ "../../../api:track_id_stream_info_map",
+ "../../../rtc_base:macromagic",
+ ]
+ absl_deps = [
+ "//third_party/abseil-cpp/absl/strings",
+ "//third_party/abseil-cpp/absl/types:optional",
+ ]
+ }
+
+ rtc_library("default_audio_quality_analyzer") {
+ testonly = true
+ sources = [
+ "analyzer/audio/default_audio_quality_analyzer.cc",
+ "analyzer/audio/default_audio_quality_analyzer.h",
+ ]
+
+ deps = [
+ ":metric_metadata_keys",
+ "../..:perf_test",
+ "../../../api:audio_quality_analyzer_api",
+ "../../../api:rtc_stats_api",
+ "../../../api:stats_observer_interface",
+ "../../../api:track_id_stream_info_map",
+ "../../../api/numerics",
+ "../../../api/test/metrics:metric",
+ "../../../api/test/metrics:metrics_logger",
+ "../../../api/units:time_delta",
+ "../../../api/units:timestamp",
+ "../../../rtc_base:checks",
+ "../../../rtc_base:criticalsection",
+ "../../../rtc_base:logging",
+ "../../../rtc_base:rtc_numerics",
+ "../../../rtc_base/synchronization:mutex",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
+ }
+
+ rtc_library("network_quality_metrics_reporter") {
+ testonly = true
+ sources = [
+ "network_quality_metrics_reporter.cc",
+ "network_quality_metrics_reporter.h",
+ ]
+ deps = [
+ "../..:perf_test",
+ "../../../api:network_emulation_manager_api",
+ "../../../api:peer_connection_quality_test_fixture_api",
+ "../../../api:rtc_stats_api",
+ "../../../api:track_id_stream_info_map",
+ "../../../api/test/metrics:metric",
+ "../../../api/test/metrics:metrics_logger",
+ "../../../api/units:data_size",
+ "../../../rtc_base:checks",
+ "../../../rtc_base:criticalsection",
+ "../../../rtc_base:rtc_event",
+ "../../../rtc_base/synchronization:mutex",
+ "../../../system_wrappers:field_trial",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
+ }
+
+ rtc_library("stats_based_network_quality_metrics_reporter") {
+ testonly = true
+ sources = [
+ "stats_based_network_quality_metrics_reporter.cc",
+ "stats_based_network_quality_metrics_reporter.h",
+ ]
+ deps = [
+ ":metric_metadata_keys",
+ "../..:perf_test",
+ "../../../api:array_view",
+ "../../../api:network_emulation_manager_api",
+ "../../../api:peer_connection_quality_test_fixture_api",
+ "../../../api:rtc_stats_api",
+ "../../../api:scoped_refptr",
+ "../../../api:sequence_checker",
+ "../../../api/numerics",
+ "../../../api/test/metrics:metric",
+ "../../../api/test/metrics:metrics_logger",
+ "../../../api/test/network_emulation",
+ "../../../api/units:data_rate",
+ "../../../api/units:data_size",
+ "../../../api/units:timestamp",
+ "../../../rtc_base:checks",
+ "../../../rtc_base:ip_address",
+ "../../../rtc_base:rtc_event",
+ "../../../rtc_base:stringutils",
+ "../../../rtc_base/synchronization:mutex",
+ "../../../rtc_base/system:no_unique_address",
+ "../../../system_wrappers:field_trial",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
+ }
+
+ rtc_library("cross_media_metrics_reporter") {
+ testonly = true
+ sources = [
+ "cross_media_metrics_reporter.cc",
+ "cross_media_metrics_reporter.h",
+ ]
+ deps = [
+ ":metric_metadata_keys",
+ "../..:perf_test",
+ "../../../api:network_emulation_manager_api",
+ "../../../api:peer_connection_quality_test_fixture_api",
+ "../../../api:rtc_stats_api",
+ "../../../api:track_id_stream_info_map",
+ "../../../api/numerics",
+ "../../../api/test/metrics:metric",
+ "../../../api/test/metrics:metrics_logger",
+ "../../../api/units:timestamp",
+ "../../../rtc_base:checks",
+ "../../../rtc_base:criticalsection",
+ "../../../rtc_base:rtc_event",
+ "../../../rtc_base:rtc_numerics",
+ "../../../rtc_base/synchronization:mutex",
+ "../../../system_wrappers:field_trial",
+ ]
+ absl_deps = [
+ "//third_party/abseil-cpp/absl/strings",
+ "//third_party/abseil-cpp/absl/types:optional",
+ ]
+ }
+
+ rtc_library("sdp_changer") {
+ testonly = true
+ sources = [
+ "sdp/sdp_changer.cc",
+ "sdp/sdp_changer.h",
+ ]
+ deps = [
+ "../../../api:array_view",
+ "../../../api:libjingle_peerconnection_api",
+ "../../../api:rtp_parameters",
+ "../../../api/test/pclf:media_configuration",
+ "../../../media:media_constants",
+ "../../../media:rid_description",
+ "../../../media:rtc_media_base",
+ "../../../p2p:rtc_p2p",
+ "../../../pc:sdp_utils",
+ "../../../pc:session_description",
+ "../../../pc:simulcast_description",
+ "../../../rtc_base:stringutils",
+ ]
+ absl_deps = [
+ "//third_party/abseil-cpp/absl/memory",
+ "//third_party/abseil-cpp/absl/strings:strings",
+ "//third_party/abseil-cpp/absl/types:optional",
+ ]
+ }
+}
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc
new file mode 100644
index 0000000000..98d0c533c2
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc
@@ -0,0 +1,175 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h"
+
+#include "api/stats/rtc_stats.h"
+#include "api/stats/rtcstats_objects.h"
+#include "api/test/metrics/metric.h"
+#include "api/test/track_id_stream_info_map.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+#include "test/pc/e2e/metric_metadata_keys.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+using ::webrtc::test::ImprovementDirection;
+using ::webrtc::test::Unit;
+
+DefaultAudioQualityAnalyzer::DefaultAudioQualityAnalyzer(
+ test::MetricsLogger* const metrics_logger)
+ : metrics_logger_(metrics_logger) {
+ RTC_CHECK(metrics_logger_);
+}
+
+void DefaultAudioQualityAnalyzer::Start(std::string test_case_name,
+ TrackIdStreamInfoMap* analyzer_helper) {
+ test_case_name_ = std::move(test_case_name);
+ analyzer_helper_ = analyzer_helper;
+}
+
+void DefaultAudioQualityAnalyzer::OnStatsReports(
+ absl::string_view pc_label,
+ const rtc::scoped_refptr<const RTCStatsReport>& report) {
+ auto stats = report->GetStatsOfType<RTCInboundRTPStreamStats>();
+
+ for (auto& stat : stats) {
+ if (!stat->kind.is_defined() ||
+ !(*stat->kind == RTCMediaStreamTrackKind::kAudio)) {
+ continue;
+ }
+
+ StatsSample sample;
+ sample.total_samples_received =
+ stat->total_samples_received.ValueOrDefault(0ul);
+ sample.concealed_samples = stat->concealed_samples.ValueOrDefault(0ul);
+ sample.removed_samples_for_acceleration =
+ stat->removed_samples_for_acceleration.ValueOrDefault(0ul);
+ sample.inserted_samples_for_deceleration =
+ stat->inserted_samples_for_deceleration.ValueOrDefault(0ul);
+ sample.silent_concealed_samples =
+ stat->silent_concealed_samples.ValueOrDefault(0ul);
+ sample.jitter_buffer_delay =
+ TimeDelta::Seconds(stat->jitter_buffer_delay.ValueOrDefault(0.));
+ sample.jitter_buffer_target_delay =
+ TimeDelta::Seconds(stat->jitter_buffer_target_delay.ValueOrDefault(0.));
+ sample.jitter_buffer_emitted_count =
+ stat->jitter_buffer_emitted_count.ValueOrDefault(0ul);
+
+ TrackIdStreamInfoMap::StreamInfo stream_info =
+ analyzer_helper_->GetStreamInfoFromTrackId(*stat->track_identifier);
+
+ MutexLock lock(&lock_);
+ stream_info_.emplace(stream_info.stream_label, stream_info);
+ StatsSample prev_sample = last_stats_sample_[stream_info.stream_label];
+ RTC_CHECK_GE(sample.total_samples_received,
+ prev_sample.total_samples_received);
+ double total_samples_diff = static_cast<double>(
+ sample.total_samples_received - prev_sample.total_samples_received);
+ if (total_samples_diff == 0) {
+ return;
+ }
+
+ AudioStreamStats& audio_stream_stats =
+ streams_stats_[stream_info.stream_label];
+ audio_stream_stats.expand_rate.AddSample(
+ (sample.concealed_samples - prev_sample.concealed_samples) /
+ total_samples_diff);
+ audio_stream_stats.accelerate_rate.AddSample(
+ (sample.removed_samples_for_acceleration -
+ prev_sample.removed_samples_for_acceleration) /
+ total_samples_diff);
+ audio_stream_stats.preemptive_rate.AddSample(
+ (sample.inserted_samples_for_deceleration -
+ prev_sample.inserted_samples_for_deceleration) /
+ total_samples_diff);
+
+ int64_t speech_concealed_samples =
+ sample.concealed_samples - sample.silent_concealed_samples;
+ int64_t prev_speech_concealed_samples =
+ prev_sample.concealed_samples - prev_sample.silent_concealed_samples;
+ audio_stream_stats.speech_expand_rate.AddSample(
+ (speech_concealed_samples - prev_speech_concealed_samples) /
+ total_samples_diff);
+
+ int64_t jitter_buffer_emitted_count_diff =
+ sample.jitter_buffer_emitted_count -
+ prev_sample.jitter_buffer_emitted_count;
+ if (jitter_buffer_emitted_count_diff > 0) {
+ TimeDelta jitter_buffer_delay_diff =
+ sample.jitter_buffer_delay - prev_sample.jitter_buffer_delay;
+ TimeDelta jitter_buffer_target_delay_diff =
+ sample.jitter_buffer_target_delay -
+ prev_sample.jitter_buffer_target_delay;
+ audio_stream_stats.average_jitter_buffer_delay_ms.AddSample(
+ jitter_buffer_delay_diff.ms<double>() /
+ jitter_buffer_emitted_count_diff);
+ audio_stream_stats.preferred_buffer_size_ms.AddSample(
+ jitter_buffer_target_delay_diff.ms<double>() /
+ jitter_buffer_emitted_count_diff);
+ }
+
+ last_stats_sample_[stream_info.stream_label] = sample;
+ }
+}
+
+std::string DefaultAudioQualityAnalyzer::GetTestCaseName(
+ const std::string& stream_label) const {
+ return test_case_name_ + "/" + stream_label;
+}
+
+void DefaultAudioQualityAnalyzer::Stop() {
+ MutexLock lock(&lock_);
+ for (auto& item : streams_stats_) {
+ const TrackIdStreamInfoMap::StreamInfo& stream_info =
+ stream_info_[item.first];
+ // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey.
+ std::map<std::string, std::string> metric_metadata{
+ {MetricMetadataKey::kAudioStreamMetadataKey, item.first},
+ {MetricMetadataKey::kPeerMetadataKey, stream_info.receiver_peer},
+ {MetricMetadataKey::kReceiverMetadataKey, stream_info.receiver_peer},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_case_name_}};
+
+ metrics_logger_->LogMetric("expand_rate", GetTestCaseName(item.first),
+ item.second.expand_rate, Unit::kUnitless,
+ ImprovementDirection::kSmallerIsBetter,
+ metric_metadata);
+ metrics_logger_->LogMetric("accelerate_rate", GetTestCaseName(item.first),
+ item.second.accelerate_rate, Unit::kUnitless,
+ ImprovementDirection::kSmallerIsBetter,
+ metric_metadata);
+ metrics_logger_->LogMetric("preemptive_rate", GetTestCaseName(item.first),
+ item.second.preemptive_rate, Unit::kUnitless,
+ ImprovementDirection::kSmallerIsBetter,
+ metric_metadata);
+ metrics_logger_->LogMetric(
+ "speech_expand_rate", GetTestCaseName(item.first),
+ item.second.speech_expand_rate, Unit::kUnitless,
+ ImprovementDirection::kSmallerIsBetter, metric_metadata);
+ metrics_logger_->LogMetric(
+ "average_jitter_buffer_delay_ms", GetTestCaseName(item.first),
+ item.second.average_jitter_buffer_delay_ms, Unit::kMilliseconds,
+ ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ metrics_logger_->LogMetric(
+ "preferred_buffer_size_ms", GetTestCaseName(item.first),
+ item.second.preferred_buffer_size_ms, Unit::kMilliseconds,
+ ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ }
+}
+
+std::map<std::string, AudioStreamStats>
+DefaultAudioQualityAnalyzer::GetAudioStreamsStats() const {
+ MutexLock lock(&lock_);
+ return streams_stats_;
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h b/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h
new file mode 100644
index 0000000000..9e427afed8
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_AUDIO_DEFAULT_AUDIO_QUALITY_ANALYZER_H_
+#define TEST_PC_E2E_ANALYZER_AUDIO_DEFAULT_AUDIO_QUALITY_ANALYZER_H_
+
+#include <map>
+#include <string>
+
+#include "absl/strings/string_view.h"
+#include "api/numerics/samples_stats_counter.h"
+#include "api/test/audio_quality_analyzer_interface.h"
+#include "api/test/metrics/metrics_logger.h"
+#include "api/test/track_id_stream_info_map.h"
+#include "api/units/time_delta.h"
+#include "rtc_base/synchronization/mutex.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+struct AudioStreamStats {
+ SamplesStatsCounter expand_rate;
+ SamplesStatsCounter accelerate_rate;
+ SamplesStatsCounter preemptive_rate;
+ SamplesStatsCounter speech_expand_rate;
+ SamplesStatsCounter average_jitter_buffer_delay_ms;
+ SamplesStatsCounter preferred_buffer_size_ms;
+};
+
+class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface {
+ public:
+ explicit DefaultAudioQualityAnalyzer(
+ test::MetricsLogger* const metrics_logger);
+
+ void Start(std::string test_case_name,
+ TrackIdStreamInfoMap* analyzer_helper) override;
+ void OnStatsReports(
+ absl::string_view pc_label,
+ const rtc::scoped_refptr<const RTCStatsReport>& report) override;
+ void Stop() override;
+
+ // Returns audio quality stats per stream label.
+ std::map<std::string, AudioStreamStats> GetAudioStreamsStats() const;
+
+ private:
+ struct StatsSample {
+ uint64_t total_samples_received = 0;
+ uint64_t concealed_samples = 0;
+ uint64_t removed_samples_for_acceleration = 0;
+ uint64_t inserted_samples_for_deceleration = 0;
+ uint64_t silent_concealed_samples = 0;
+ TimeDelta jitter_buffer_delay = TimeDelta::Zero();
+ TimeDelta jitter_buffer_target_delay = TimeDelta::Zero();
+ uint64_t jitter_buffer_emitted_count = 0;
+ };
+
+ std::string GetTestCaseName(const std::string& stream_label) const;
+
+ test::MetricsLogger* const metrics_logger_;
+
+ std::string test_case_name_;
+ TrackIdStreamInfoMap* analyzer_helper_;
+
+ mutable Mutex lock_;
+ std::map<std::string, AudioStreamStats> streams_stats_ RTC_GUARDED_BY(lock_);
+ std::map<std::string, TrackIdStreamInfoMap::StreamInfo> stream_info_
+ RTC_GUARDED_BY(lock_);
+ std::map<std::string, StatsSample> last_stats_sample_ RTC_GUARDED_BY(lock_);
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_AUDIO_DEFAULT_AUDIO_QUALITY_ANALYZER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/BUILD.gn b/third_party/libwebrtc/test/pc/e2e/analyzer/video/BUILD.gn
new file mode 100644
index 0000000000..cbb4c078f3
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/BUILD.gn
@@ -0,0 +1,573 @@
+# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import("../../../../../webrtc.gni")
+
+if (!build_with_chromium) {
+ group("video_analyzer") {
+ testonly = true
+
+ deps = [
+ ":analyzing_video_sinks_helper",
+ ":default_video_quality_analyzer_internal",
+ ":encoded_image_data_injector_api",
+ ":example_video_quality_analyzer",
+ ":multi_reader_queue",
+ ":quality_analyzing_video_decoder",
+ ":quality_analyzing_video_encoder",
+ ":simulcast_dummy_buffer_helper",
+ ":single_process_encoded_image_data_injector",
+ ":video_dumping",
+ ":video_frame_tracking_id_injector",
+ ":video_quality_metrics_reporter",
+ ]
+ if (rtc_include_tests) {
+ deps += [
+ ":analyzing_video_sink",
+ ":video_quality_analyzer_injection_helper",
+ ]
+ }
+ }
+
+ if (rtc_include_tests) {
+ group("video_analyzer_unittests") {
+ testonly = true
+
+ deps = [
+ ":analyzing_video_sink_test",
+ ":analyzing_video_sinks_helper_test",
+ ":default_video_quality_analyzer_frames_comparator_test",
+ ":default_video_quality_analyzer_metric_names_test",
+ ":default_video_quality_analyzer_stream_state_test",
+ ":default_video_quality_analyzer_test",
+ ":multi_reader_queue_test",
+ ":names_collection_test",
+ ":simulcast_dummy_buffer_helper_test",
+ ":single_process_encoded_image_data_injector_unittest",
+ ":video_dumping_test",
+ ":video_frame_tracking_id_injector_unittest",
+ ]
+ }
+ }
+}
+
+rtc_library("video_dumping") {
+ testonly = true
+ sources = [
+ "video_dumping.cc",
+ "video_dumping.h",
+ ]
+ deps = [
+ "../../../..:video_test_support",
+ "../../../../../api/test/video:video_frame_writer",
+ "../../../../../api/video:video_frame",
+ "../../../../../rtc_base:logging",
+ "../../../../../system_wrappers",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
+}
+
+rtc_library("encoded_image_data_injector_api") {
+ testonly = true
+ sources = [ "encoded_image_data_injector.h" ]
+
+ deps = [ "../../../../../api/video:encoded_image" ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+}
+
+rtc_library("single_process_encoded_image_data_injector") {
+ testonly = true
+ sources = [
+ "single_process_encoded_image_data_injector.cc",
+ "single_process_encoded_image_data_injector.h",
+ ]
+
+ deps = [
+ ":encoded_image_data_injector_api",
+ "../../../../../api/video:encoded_image",
+ "../../../../../rtc_base:checks",
+ "../../../../../rtc_base/synchronization:mutex",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/memory" ]
+}
+
+rtc_library("video_frame_tracking_id_injector") {
+ testonly = true
+ sources = [
+ "video_frame_tracking_id_injector.cc",
+ "video_frame_tracking_id_injector.h",
+ ]
+
+ deps = [
+ ":encoded_image_data_injector_api",
+ "../../../../../api/video:encoded_image",
+ "../../../../../rtc_base:checks",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/memory" ]
+}
+
+rtc_library("simulcast_dummy_buffer_helper") {
+ testonly = true
+ sources = [
+ "simulcast_dummy_buffer_helper.cc",
+ "simulcast_dummy_buffer_helper.h",
+ ]
+ deps = [ "../../../../../api/video:video_frame" ]
+}
+
+rtc_library("quality_analyzing_video_decoder") {
+ testonly = true
+ sources = [
+ "quality_analyzing_video_decoder.cc",
+ "quality_analyzing_video_decoder.h",
+ ]
+ deps = [
+ ":encoded_image_data_injector_api",
+ ":simulcast_dummy_buffer_helper",
+ "../../../../../api:video_quality_analyzer_api",
+ "../../../../../api/video:encoded_image",
+ "../../../../../api/video:video_frame",
+ "../../../../../api/video_codecs:video_codecs_api",
+ "../../../../../modules/video_coding:video_codec_interface",
+ "../../../../../rtc_base:logging",
+ "../../../../../rtc_base/synchronization:mutex",
+ ]
+ absl_deps = [
+ "//third_party/abseil-cpp/absl/strings",
+ "//third_party/abseil-cpp/absl/types:optional",
+ ]
+}
+
+rtc_library("quality_analyzing_video_encoder") {
+ testonly = true
+ sources = [
+ "quality_analyzing_video_encoder.cc",
+ "quality_analyzing_video_encoder.h",
+ ]
+ deps = [
+ ":encoded_image_data_injector_api",
+ "../../../../../api:video_quality_analyzer_api",
+ "../../../../../api/test/pclf:media_configuration",
+ "../../../../../api/video:video_frame",
+ "../../../../../api/video_codecs:video_codecs_api",
+ "../../../../../modules/video_coding:video_codec_interface",
+ "../../../../../modules/video_coding/svc:scalability_mode_util",
+ "../../../../../rtc_base:logging",
+ "../../../../../rtc_base/synchronization:mutex",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
+}
+
+rtc_library("analyzing_video_sinks_helper") {
+ testonly = true
+ sources = [
+ "analyzing_video_sinks_helper.cc",
+ "analyzing_video_sinks_helper.h",
+ ]
+ deps = [
+ "../../../../../api/test/pclf:media_configuration",
+ "../../../../../api/test/video:video_frame_writer",
+ "../../../../../rtc_base:macromagic",
+ "../../../../../rtc_base/synchronization:mutex",
+ ]
+ absl_deps = [
+ "//third_party/abseil-cpp/absl/strings",
+ "//third_party/abseil-cpp/absl/types:optional",
+ ]
+}
+
+rtc_library("example_video_quality_analyzer") {
+ testonly = true
+ sources = [
+ "example_video_quality_analyzer.cc",
+ "example_video_quality_analyzer.h",
+ ]
+
+ deps = [
+ "../../../../../api:array_view",
+ "../../../../../api:video_quality_analyzer_api",
+ "../../../../../api/video:encoded_image",
+ "../../../../../api/video:video_frame",
+ "../../../../../rtc_base:logging",
+ "../../../../../rtc_base/synchronization:mutex",
+ ]
+}
+
+# This target contains implementation details of DefaultVideoQualityAnalyzer,
+# so headers exported by it shouldn't be used in other places.
+rtc_library("default_video_quality_analyzer_internal") {
+ visibility = [
+ ":default_video_quality_analyzer",
+ ":default_video_quality_analyzer_frames_comparator_test",
+ ":default_video_quality_analyzer_stream_state_test",
+ ":names_collection_test",
+ ":video_analyzer",
+ ]
+
+ testonly = true
+ sources = [
+ "default_video_quality_analyzer_cpu_measurer.cc",
+ "default_video_quality_analyzer_cpu_measurer.h",
+ "default_video_quality_analyzer_frame_in_flight.cc",
+ "default_video_quality_analyzer_frame_in_flight.h",
+ "default_video_quality_analyzer_frames_comparator.cc",
+ "default_video_quality_analyzer_frames_comparator.h",
+ "default_video_quality_analyzer_internal_shared_objects.cc",
+ "default_video_quality_analyzer_internal_shared_objects.h",
+ "default_video_quality_analyzer_stream_state.cc",
+ "default_video_quality_analyzer_stream_state.h",
+ "names_collection.cc",
+ "names_collection.h",
+ ]
+
+ deps = [
+ ":default_video_quality_analyzer_shared",
+ ":multi_reader_queue",
+ "../..:metric_metadata_keys",
+ "../../../../../api:array_view",
+ "../../../../../api:scoped_refptr",
+ "../../../../../api/numerics",
+ "../../../../../api/units:data_size",
+ "../../../../../api/units:timestamp",
+ "../../../../../api/video:video_frame",
+ "../../../../../api/video:video_frame_type",
+ "../../../../../common_video",
+ "../../../../../rtc_base:checks",
+ "../../../../../rtc_base:platform_thread",
+ "../../../../../rtc_base:rtc_base_tests_utils",
+ "../../../../../rtc_base:rtc_event",
+ "../../../../../rtc_base:stringutils",
+ "../../../../../rtc_base:timeutils",
+ "../../../../../rtc_base/synchronization:mutex",
+ "../../../../../rtc_tools:video_quality_analysis",
+ "../../../../../system_wrappers",
+ ]
+ absl_deps = [
+ "//third_party/abseil-cpp/absl/strings:strings",
+ "//third_party/abseil-cpp/absl/types:optional",
+ ]
+}
+
+rtc_library("multi_reader_queue") {
+ testonly = true
+ sources = [ "multi_reader_queue.h" ]
+ deps = [ "../../../../../rtc_base:checks" ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+}
+
+rtc_library("video_quality_metrics_reporter") {
+ testonly = true
+ sources = [
+ "video_quality_metrics_reporter.cc",
+ "video_quality_metrics_reporter.h",
+ ]
+ deps = [
+ "../..:metric_metadata_keys",
+ "../../../../../api:peer_connection_quality_test_fixture_api",
+ "../../../../../api:rtc_stats_api",
+ "../../../../../api:track_id_stream_info_map",
+ "../../../../../api/numerics",
+ "../../../../../api/test/metrics:metric",
+ "../../../../../api/test/metrics:metrics_logger",
+ "../../../../../api/units:data_rate",
+ "../../../../../api/units:data_size",
+ "../../../../../api/units:time_delta",
+ "../../../../../api/units:timestamp",
+ "../../../../../rtc_base:checks",
+ "../../../../../rtc_base/synchronization:mutex",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
+}
+
+rtc_library("default_video_quality_analyzer") {
+ testonly = true
+ sources = [
+ "default_video_quality_analyzer.cc",
+ "default_video_quality_analyzer.h",
+ ]
+
+ deps = [
+ ":default_video_quality_analyzer_internal",
+ ":default_video_quality_analyzer_shared",
+ "../..:metric_metadata_keys",
+ "../../../../../api:array_view",
+ "../../../../../api:video_quality_analyzer_api",
+ "../../../../../api/numerics",
+ "../../../../../api/test/metrics:metric",
+ "../../../../../api/test/metrics:metrics_logger",
+ "../../../../../api/units:data_size",
+ "../../../../../api/units:time_delta",
+ "../../../../../api/units:timestamp",
+ "../../../../../api/video:encoded_image",
+ "../../../../../api/video:video_frame",
+ "../../../../../rtc_base:checks",
+ "../../../../../rtc_base:logging",
+ "../../../../../rtc_base:macromagic",
+ "../../../../../rtc_base:stringutils",
+ "../../../../../rtc_base/synchronization:mutex",
+ "../../../../../system_wrappers",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+}
+
+rtc_library("default_video_quality_analyzer_shared") {
+ testonly = true
+ sources = [
+ "default_video_quality_analyzer_shared_objects.cc",
+ "default_video_quality_analyzer_shared_objects.h",
+ ]
+
+ deps = [
+ "../../../../../api/numerics",
+ "../../../../../api/units:timestamp",
+ "../../../../../rtc_base:checks",
+ "../../../../../rtc_base:stringutils",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+}
+
+rtc_library("analyzing_video_sink") {
+ testonly = true
+ sources = [
+ "analyzing_video_sink.cc",
+ "analyzing_video_sink.h",
+ ]
+ deps = [
+ ":analyzing_video_sinks_helper",
+ ":simulcast_dummy_buffer_helper",
+ ":video_dumping",
+ "../../../..:fixed_fps_video_frame_writer_adapter",
+ "../../../..:test_renderer",
+ "../../../../../api:video_quality_analyzer_api",
+ "../../../../../api/numerics",
+ "../../../../../api/test/pclf:media_configuration",
+ "../../../../../api/test/video:video_frame_writer",
+ "../../../../../api/units:timestamp",
+ "../../../../../api/video:video_frame",
+ "../../../../../rtc_base:checks",
+ "../../../../../rtc_base:logging",
+ "../../../../../rtc_base:macromagic",
+ "../../../../../rtc_base/synchronization:mutex",
+ "../../../../../system_wrappers",
+ ]
+ absl_deps = [
+ "//third_party/abseil-cpp/absl/memory:memory",
+ "//third_party/abseil-cpp/absl/strings",
+ "//third_party/abseil-cpp/absl/types:optional",
+ ]
+}
+
+rtc_library("video_quality_analyzer_injection_helper") {
+ testonly = true
+ sources = [
+ "video_quality_analyzer_injection_helper.cc",
+ "video_quality_analyzer_injection_helper.h",
+ ]
+ deps = [
+ ":analyzing_video_sink",
+ ":analyzing_video_sinks_helper",
+ ":encoded_image_data_injector_api",
+ ":quality_analyzing_video_decoder",
+ ":quality_analyzing_video_encoder",
+ ":simulcast_dummy_buffer_helper",
+ ":video_dumping",
+ "../../../..:fixed_fps_video_frame_writer_adapter",
+ "../../../..:test_renderer",
+ "../../../..:video_test_common",
+ "../../../..:video_test_support",
+ "../../../../../api:array_view",
+ "../../../../../api:stats_observer_interface",
+ "../../../../../api:video_quality_analyzer_api",
+ "../../../../../api/test/pclf:media_configuration",
+ "../../../../../api/video:video_frame",
+ "../../../../../api/video_codecs:video_codecs_api",
+ "../../../../../rtc_base:checks",
+ "../../../../../rtc_base:logging",
+ "../../../../../rtc_base:stringutils",
+ "../../../../../rtc_base/synchronization:mutex",
+ "../../../../../system_wrappers",
+ ]
+ absl_deps = [
+ "//third_party/abseil-cpp/absl/memory",
+ "//third_party/abseil-cpp/absl/strings",
+ ]
+}
+
+if (rtc_include_tests) {
+ rtc_library("simulcast_dummy_buffer_helper_test") {
+ testonly = true
+ sources = [ "simulcast_dummy_buffer_helper_test.cc" ]
+ deps = [
+ ":simulcast_dummy_buffer_helper",
+ "../../../..:test_support",
+ "../../../../../api/video:video_frame",
+ "../../../../../rtc_base:random",
+ ]
+ }
+
+ rtc_library("analyzing_video_sink_test") {
+ testonly = true
+ sources = [ "analyzing_video_sink_test.cc" ]
+ deps = [
+ ":analyzing_video_sink",
+ ":example_video_quality_analyzer",
+ "../../../..:fileutils",
+ "../../../..:test_support",
+ "../../../..:video_test_support",
+ "../../../../../api:create_frame_generator",
+ "../../../../../api:frame_generator_api",
+ "../../../../../api:scoped_refptr",
+ "../../../../../api/test/pclf:media_configuration",
+ "../../../../../api/units:time_delta",
+ "../../../../../api/units:timestamp",
+ "../../../../../api/video:video_frame",
+ "../../../../../common_video",
+ "../../../../../rtc_base:timeutils",
+ "../../../../../system_wrappers",
+ "../../../../time_controller",
+ ]
+ absl_deps = [
+ "//third_party/abseil-cpp/absl/strings",
+ "//third_party/abseil-cpp/absl/types:optional",
+ ]
+ }
+
+ rtc_library("analyzing_video_sinks_helper_test") {
+ testonly = true
+ sources = [ "analyzing_video_sinks_helper_test.cc" ]
+ deps = [
+ ":analyzing_video_sinks_helper",
+ "../../../..:test_support",
+ "../../../../../api/test/pclf:media_configuration",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+ }
+
+ rtc_library("default_video_quality_analyzer_frames_comparator_test") {
+ testonly = true
+ sources = [ "default_video_quality_analyzer_frames_comparator_test.cc" ]
+ deps = [
+ ":default_video_quality_analyzer_internal",
+ ":default_video_quality_analyzer_shared",
+ "../../../..:test_support",
+ "../../../../../api:create_frame_generator",
+ "../../../../../api/units:timestamp",
+ "../../../../../rtc_base:stringutils",
+ "../../../../../system_wrappers",
+ ]
+ }
+
+ rtc_library("names_collection_test") {
+ testonly = true
+ sources = [ "names_collection_test.cc" ]
+ deps = [
+ ":default_video_quality_analyzer_internal",
+ "../../../..:test_support",
+ ]
+ absl_deps = [
+ "//third_party/abseil-cpp/absl/strings:strings",
+ "//third_party/abseil-cpp/absl/types:optional",
+ ]
+ }
+
+ rtc_library("multi_reader_queue_test") {
+ testonly = true
+ sources = [ "multi_reader_queue_test.cc" ]
+ deps = [
+ ":multi_reader_queue",
+ "../../../..:test_support",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+ }
+
+ rtc_library("default_video_quality_analyzer_stream_state_test") {
+ testonly = true
+ sources = [ "default_video_quality_analyzer_stream_state_test.cc" ]
+ deps = [
+ ":default_video_quality_analyzer_internal",
+ "../../../..:test_support",
+ "../../../../../api/units:timestamp",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+ }
+
+ rtc_library("default_video_quality_analyzer_test") {
+ testonly = true
+ sources = [ "default_video_quality_analyzer_test.cc" ]
+ deps = [
+ ":default_video_quality_analyzer",
+ ":default_video_quality_analyzer_shared",
+ "../../../..:test_support",
+ "../../../../../api:create_frame_generator",
+ "../../../../../api:rtp_packet_info",
+ "../../../../../api/test/metrics:global_metrics_logger_and_exporter",
+ "../../../../../api/video:encoded_image",
+ "../../../../../api/video:video_frame",
+ "../../../../../common_video",
+ "../../../../../rtc_base:stringutils",
+ "../../../../../rtc_tools:video_quality_analysis",
+ "../../../../../system_wrappers",
+ ]
+ }
+
+ rtc_library("default_video_quality_analyzer_metric_names_test") {
+ testonly = true
+ sources = [ "default_video_quality_analyzer_metric_names_test.cc" ]
+ deps = [
+ ":default_video_quality_analyzer",
+ "../../../..:test_support",
+ "../../../../../api:create_frame_generator",
+ "../../../../../api:rtp_packet_info",
+ "../../../../../api/test/metrics:metric",
+ "../../../../../api/test/metrics:metrics_logger",
+ "../../../../../api/test/metrics:stdout_metrics_exporter",
+ "../../../../../api/video:encoded_image",
+ "../../../../../api/video:video_frame",
+ "../../../../../common_video",
+ "../../../../../rtc_tools:video_quality_analysis",
+ "../../../../../system_wrappers",
+ ]
+ }
+
+ rtc_library("video_dumping_test") {
+ testonly = true
+ sources = [ "video_dumping_test.cc" ]
+ deps = [
+ ":video_dumping",
+ "../../../..:fileutils",
+ "../../../..:test_support",
+ "../../../..:video_test_support",
+ "../../../../../api:scoped_refptr",
+ "../../../../../api/video:video_frame",
+ "../../../../../rtc_base:random",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+ }
+
+ rtc_library("single_process_encoded_image_data_injector_unittest") {
+ testonly = true
+ sources = [ "single_process_encoded_image_data_injector_unittest.cc" ]
+ deps = [
+ ":single_process_encoded_image_data_injector",
+ "../../../..:test_support",
+ "../../../../../api/video:encoded_image",
+ "../../../../../rtc_base:buffer",
+ ]
+ }
+
+ rtc_library("video_frame_tracking_id_injector_unittest") {
+ testonly = true
+ sources = [ "video_frame_tracking_id_injector_unittest.cc" ]
+ deps = [
+ ":video_frame_tracking_id_injector",
+ "../../../..:test_support",
+ "../../../../../api/video:encoded_image",
+ "../../../../../rtc_base:buffer",
+ ]
+ }
+}
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.cc
new file mode 100644
index 0000000000..fb221e6797
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.cc
@@ -0,0 +1,220 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/analyzer/video/analyzing_video_sink.h"
+
+#include <memory>
+#include <set>
+#include <utility>
+
+#include "absl/memory/memory.h"
+#include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/video/video_frame_writer.h"
+#include "api/units/timestamp.h"
+#include "api/video/i420_buffer.h"
+#include "api/video/video_frame.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h"
+#include "test/pc/e2e/analyzer/video/video_dumping.h"
+#include "test/testsupport/fixed_fps_video_frame_writer_adapter.h"
+#include "test/video_renderer.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+AnalyzingVideoSink::AnalyzingVideoSink(absl::string_view peer_name,
+ Clock* clock,
+ VideoQualityAnalyzerInterface& analyzer,
+ AnalyzingVideoSinksHelper& sinks_helper,
+ const VideoSubscription& subscription,
+ bool report_infra_stats)
+ : peer_name_(peer_name),
+ report_infra_stats_(report_infra_stats),
+ clock_(clock),
+ analyzer_(&analyzer),
+ sinks_helper_(&sinks_helper),
+ subscription_(subscription) {}
+
+void AnalyzingVideoSink::UpdateSubscription(
+ const VideoSubscription& subscription) {
+ // For peers with changed resolutions we need to close current writers and
+ // open new ones. This is done by removing existing sinks, which will force
+ // creation of the new sinks when next frame will be received.
+ std::set<test::VideoFrameWriter*> writers_to_close;
+ {
+ MutexLock lock(&mutex_);
+ subscription_ = subscription;
+ for (auto it = stream_sinks_.cbegin(); it != stream_sinks_.cend();) {
+ absl::optional<VideoResolution> new_requested_resolution =
+ subscription_.GetResolutionForPeer(it->second.sender_peer_name);
+ if (!new_requested_resolution.has_value() ||
+ (*new_requested_resolution != it->second.resolution)) {
+ RTC_LOG(LS_INFO) << peer_name_ << ": Subscribed resolution for stream "
+ << it->first << " from " << it->second.sender_peer_name
+ << " was updated from "
+ << it->second.resolution.ToString() << " to "
+ << new_requested_resolution->ToString()
+ << ". Repopulating all video sinks and recreating "
+ << "requested video writers";
+ writers_to_close.insert(it->second.video_frame_writer);
+ it = stream_sinks_.erase(it);
+ } else {
+ ++it;
+ }
+ }
+ }
+ sinks_helper_->CloseAndRemoveVideoWriters(writers_to_close);
+}
+
+void AnalyzingVideoSink::OnFrame(const VideoFrame& frame) {
+ if (IsDummyFrame(frame)) {
+ // This is dummy frame, so we don't need to process it further.
+ return;
+ }
+
+ if (frame.id() == VideoFrame::kNotSetId) {
+ // If frame ID is unknown we can't get required render resolution, so pass
+ // to the analyzer in the actual resolution of the frame.
+ AnalyzeFrame(frame);
+ } else {
+ std::string stream_label = analyzer_->GetStreamLabel(frame.id());
+ MutexLock lock(&mutex_);
+ Timestamp processing_started = clock_->CurrentTime();
+ SinksDescriptor* sinks_descriptor = PopulateSinks(stream_label);
+ RTC_CHECK(sinks_descriptor != nullptr);
+
+ VideoFrame scaled_frame =
+ ScaleVideoFrame(frame, sinks_descriptor->resolution);
+ AnalyzeFrame(scaled_frame);
+ for (auto& sink : sinks_descriptor->sinks) {
+ sink->OnFrame(scaled_frame);
+ }
+ Timestamp processing_finished = clock_->CurrentTime();
+
+ if (report_infra_stats_) {
+ stats_.analyzing_sink_processing_time_ms.AddSample(
+ (processing_finished - processing_started).ms<double>());
+ }
+ }
+}
+
+AnalyzingVideoSink::Stats AnalyzingVideoSink::stats() const {
+ MutexLock lock(&mutex_);
+ return stats_;
+}
+
+VideoFrame AnalyzingVideoSink::ScaleVideoFrame(
+ const VideoFrame& frame,
+ const VideoResolution& required_resolution) {
+ Timestamp processing_started = clock_->CurrentTime();
+ if (required_resolution.width() == static_cast<size_t>(frame.width()) &&
+ required_resolution.height() == static_cast<size_t>(frame.height())) {
+ if (report_infra_stats_) {
+ stats_.scaling_tims_ms.AddSample(
+ (clock_->CurrentTime() - processing_started).ms<double>());
+ }
+ return frame;
+ }
+
+ // We allow some difference in the aspect ration because when decoder
+ // downscales video stream it may round up some dimensions to make them even,
+ // ex: 960x540 -> 480x270 -> 240x136 instead of 240x135.
+ RTC_CHECK_LE(std::abs(static_cast<double>(required_resolution.width()) /
+ required_resolution.height() -
+ static_cast<double>(frame.width()) / frame.height()),
+ 0.1)
+ << peer_name_
+ << ": Received frame has too different aspect ratio compared to "
+ << "requested video resolution: required resolution="
+ << required_resolution.ToString()
+ << "; actual resolution=" << frame.width() << "x" << frame.height();
+
+ rtc::scoped_refptr<I420Buffer> scaled_buffer(I420Buffer::Create(
+ required_resolution.width(), required_resolution.height()));
+ scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420());
+
+ VideoFrame scaled_frame = frame;
+ scaled_frame.set_video_frame_buffer(scaled_buffer);
+ if (report_infra_stats_) {
+ stats_.scaling_tims_ms.AddSample(
+ (clock_->CurrentTime() - processing_started).ms<double>());
+ }
+ return scaled_frame;
+}
+
+void AnalyzingVideoSink::AnalyzeFrame(const VideoFrame& frame) {
+ VideoFrame frame_copy = frame;
+ frame_copy.set_video_frame_buffer(
+ I420Buffer::Copy(*frame.video_frame_buffer()->ToI420()));
+ analyzer_->OnFrameRendered(peer_name_, frame_copy);
+}
+
+AnalyzingVideoSink::SinksDescriptor* AnalyzingVideoSink::PopulateSinks(
+ absl::string_view stream_label) {
+ // Fast pass: sinks already exists.
+ auto sinks_it = stream_sinks_.find(std::string(stream_label));
+ if (sinks_it != stream_sinks_.end()) {
+ return &sinks_it->second;
+ }
+
+ // Slow pass: we need to create and save sinks
+ absl::optional<std::pair<std::string, VideoConfig>> peer_and_config =
+ sinks_helper_->GetPeerAndConfig(stream_label);
+ RTC_CHECK(peer_and_config.has_value())
+ << "No video config for stream " << stream_label;
+ const std::string& sender_peer_name = peer_and_config->first;
+ const VideoConfig& config = peer_and_config->second;
+
+ absl::optional<VideoResolution> resolution =
+ subscription_.GetResolutionForPeer(sender_peer_name);
+ if (!resolution.has_value()) {
+ RTC_LOG(LS_ERROR) << peer_name_ << " received stream " << stream_label
+ << " from " << sender_peer_name
+ << " for which they were not subscribed";
+ resolution = config.GetResolution();
+ }
+ if (!resolution->IsRegular()) {
+ RTC_LOG(LS_ERROR) << peer_name_ << " received stream " << stream_label
+ << " from " << sender_peer_name
+ << " for which resolution wasn't resolved";
+ resolution = config.GetResolution();
+ }
+
+ RTC_CHECK(resolution.has_value());
+
+ SinksDescriptor sinks_descriptor(sender_peer_name, *resolution);
+ if (config.output_dump_options.has_value()) {
+ std::unique_ptr<test::VideoFrameWriter> writer =
+ config.output_dump_options->CreateOutputDumpVideoFrameWriter(
+ stream_label, peer_name_, *resolution);
+ if (config.output_dump_use_fixed_framerate) {
+ writer = std::make_unique<test::FixedFpsVideoFrameWriterAdapter>(
+ resolution->fps(), clock_, std::move(writer));
+ }
+ sinks_descriptor.sinks.push_back(std::make_unique<VideoWriter>(
+ writer.get(), config.output_dump_options->sampling_modulo()));
+ sinks_descriptor.video_frame_writer =
+ sinks_helper_->AddVideoWriter(std::move(writer));
+ }
+ if (config.show_on_screen) {
+ sinks_descriptor.sinks.push_back(
+ absl::WrapUnique(test::VideoRenderer::Create(
+ (*config.stream_label + "-render").c_str(), resolution->width(),
+ resolution->height())));
+ }
+ return &stream_sinks_.emplace(stream_label, std::move(sinks_descriptor))
+ .first->second;
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.h
new file mode 100644
index 0000000000..1834bbe469
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink.h
@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINK_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINK_H_
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/strings/string_view.h"
+#include "api/numerics/samples_stats_counter.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/video/video_frame_writer.h"
+#include "api/test/video_quality_analyzer_interface.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_sink_interface.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "rtc_base/thread_annotations.h"
+#include "system_wrappers/include/clock.h"
+#include "test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+// A sink to inject video quality analyzer as a sink into WebRTC.
+class AnalyzingVideoSink : public rtc::VideoSinkInterface<VideoFrame> {
+ public:
+ struct Stats {
+ // Time required to scale video frame to the requested rendered resolution.
+ // Collected only for frames with ID set and iff `report_infra_stats` is
+ // true.
+ SamplesStatsCounter scaling_tims_ms;
+ // Time required to process single video frame. Collected only for frames
+ // with ID set and iff `report_infra_stats` is true.
+ SamplesStatsCounter analyzing_sink_processing_time_ms;
+ };
+
+ AnalyzingVideoSink(absl::string_view peer_name,
+ Clock* clock,
+ VideoQualityAnalyzerInterface& analyzer,
+ AnalyzingVideoSinksHelper& sinks_helper,
+ const VideoSubscription& subscription,
+ bool report_infra_stats);
+
+ // Updates subscription used by this peer to render received video.
+ void UpdateSubscription(const VideoSubscription& subscription);
+
+ void OnFrame(const VideoFrame& frame) override;
+
+ Stats stats() const;
+
+ private:
+ struct SinksDescriptor {
+ SinksDescriptor(absl::string_view sender_peer_name,
+ const VideoResolution& resolution)
+ : sender_peer_name(sender_peer_name), resolution(resolution) {}
+
+ // Required to be able to resolve resolutions on new subscription and
+ // understand if we need to recreate `video_frame_writer` and `sinks`.
+ std::string sender_peer_name;
+ // Resolution which was used to create `video_frame_writer` and `sinks`.
+ VideoResolution resolution;
+
+ // Is set if dumping of output video was requested;
+ test::VideoFrameWriter* video_frame_writer = nullptr;
+ std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> sinks;
+ };
+
+ // Scales video frame to `required_resolution` if necessary. Crashes if video
+ // frame and `required_resolution` have different aspect ratio.
+ VideoFrame ScaleVideoFrame(const VideoFrame& frame,
+ const VideoResolution& required_resolution)
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+ // Creates full copy of the frame to free any frame owned internal buffers
+ // and passes created copy to analyzer. Uses `I420Buffer` to represent
+ // frame content.
+ void AnalyzeFrame(const VideoFrame& frame);
+ // Populates sink for specified stream and caches them in `stream_sinks_`.
+ SinksDescriptor* PopulateSinks(absl::string_view stream_label)
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+
+ const std::string peer_name_;
+ const bool report_infra_stats_;
+ Clock* const clock_;
+ VideoQualityAnalyzerInterface* const analyzer_;
+ AnalyzingVideoSinksHelper* const sinks_helper_;
+
+ mutable Mutex mutex_;
+ VideoSubscription subscription_ RTC_GUARDED_BY(mutex_);
+ std::map<std::string, SinksDescriptor> stream_sinks_ RTC_GUARDED_BY(mutex_);
+ Stats stats_ RTC_GUARDED_BY(mutex_);
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINK_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink_test.cc
new file mode 100644
index 0000000000..6cd89551ea
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sink_test.cc
@@ -0,0 +1,598 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/analyzer/video/analyzing_video_sink.h"
+
+#include <stdio.h>
+
+#include <string>
+#include <vector>
+
+#include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
+#include "api/scoped_refptr.h"
+#include "api/test/create_frame_generator.h"
+#include "api/test/frame_generator_interface.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/units/time_delta.h"
+#include "api/units/timestamp.h"
+#include "api/video/i420_buffer.h"
+#include "api/video/video_frame.h"
+#include "common_video/libyuv/include/webrtc_libyuv.h"
+#include "rtc_base/time_utils.h"
+#include "system_wrappers/include/clock.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/pc/e2e/analyzer/video/example_video_quality_analyzer.h"
+#include "test/testsupport/file_utils.h"
+#include "test/testsupport/frame_reader.h"
+#include "test/time_controller/simulated_time_controller.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+using ::testing::ElementsAreArray;
+using ::testing::Eq;
+using ::testing::Ge;
+using ::testing::Test;
+
+// Remove files and directories in a directory non-recursively.
+void CleanDir(absl::string_view dir, size_t expected_output_files_count) {
+ absl::optional<std::vector<std::string>> dir_content =
+ test::ReadDirectory(dir);
+ if (expected_output_files_count == 0) {
+ ASSERT_TRUE(!dir_content.has_value() || dir_content->empty())
+ << "Empty directory is expected";
+ } else {
+ ASSERT_TRUE(dir_content.has_value()) << "Test directory is empty!";
+ EXPECT_EQ(dir_content->size(), expected_output_files_count);
+ for (const auto& entry : *dir_content) {
+ if (test::DirExists(entry)) {
+ EXPECT_TRUE(test::RemoveDir(entry))
+ << "Failed to remove sub directory: " << entry;
+ } else if (test::FileExists(entry)) {
+ EXPECT_TRUE(test::RemoveFile(entry))
+ << "Failed to remove file: " << entry;
+ } else {
+ FAIL() << "Can't remove unknown file type: " << entry;
+ }
+ }
+ }
+ EXPECT_TRUE(test::RemoveDir(dir)) << "Failed to remove directory: " << dir;
+}
+
+VideoFrame CreateFrame(test::FrameGeneratorInterface& frame_generator) {
+ test::FrameGeneratorInterface::VideoFrameData frame_data =
+ frame_generator.NextFrame();
+ return VideoFrame::Builder()
+ .set_video_frame_buffer(frame_data.buffer)
+ .set_update_rect(frame_data.update_rect)
+ .build();
+}
+
+std::unique_ptr<test::FrameGeneratorInterface> CreateFrameGenerator(
+ size_t width,
+ size_t height) {
+ return test::CreateSquareFrameGenerator(width, height,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+}
+
+void AssertFrameIdsAre(const std::string& filename,
+ std::vector<std::string> expected_ids) {
+ FILE* file = fopen(filename.c_str(), "r");
+ ASSERT_TRUE(file != nullptr) << "Failed to open frame ids file: " << filename;
+ std::vector<std::string> actual_ids;
+ char buffer[8];
+ while (fgets(buffer, sizeof buffer, file) != nullptr) {
+ std::string current_id(buffer);
+ EXPECT_GE(current_id.size(), 2lu)
+ << "Found invalid frame id: [" << current_id << "]";
+ if (current_id.size() < 2) {
+ continue;
+ }
+ // Trim "\n" at the end.
+ actual_ids.push_back(current_id.substr(0, current_id.size() - 1));
+ }
+ fclose(file);
+ EXPECT_THAT(actual_ids, ElementsAreArray(expected_ids));
+}
+
+class AnalyzingVideoSinkTest : public Test {
+ protected:
+ ~AnalyzingVideoSinkTest() override = default;
+
+ void SetUp() override {
+ // Create an empty temporary directory for this test.
+ test_directory_ = test::JoinFilename(
+ test::OutputPath(),
+ "TestDir_AnalyzingVideoSinkTest_" +
+ std::string(
+ testing::UnitTest::GetInstance()->current_test_info()->name()));
+ test::CreateDir(test_directory_);
+ }
+
+ void TearDown() override {
+ CleanDir(test_directory_, expected_output_files_count_);
+ }
+
+ void ExpectOutputFilesCount(size_t count) {
+ expected_output_files_count_ = count;
+ }
+
+ std::string test_directory_;
+ size_t expected_output_files_count_ = 0;
+};
+
+TEST_F(AnalyzingVideoSinkTest, VideoFramesAreDumpedCorrectly) {
+ VideoSubscription subscription;
+ subscription.SubscribeToPeer(
+ "alice", VideoResolution(/*width=*/640, /*height=*/360, /*fps=*/30));
+ VideoConfig video_config("alice_video", /*width=*/1280, /*height=*/720,
+ /*fps=*/30);
+ video_config.output_dump_options = VideoDumpOptions(test_directory_);
+
+ ExampleVideoQualityAnalyzer analyzer;
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ CreateFrameGenerator(/*width=*/1280, /*height=*/720);
+ VideoFrame frame = CreateFrame(*frame_generator);
+ frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame));
+
+ {
+ // `helper` and `sink` has to be destroyed so all frames will be written
+ // to the disk.
+ AnalyzingVideoSinksHelper helper;
+ helper.AddConfig("alice", video_config);
+ AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper,
+ subscription, /*report_infra_stats=*/false);
+ sink.OnFrame(frame);
+ }
+
+ EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(1)));
+
+ auto frame_reader = test::CreateY4mFrameReader(
+ test::JoinFilename(test_directory_, "alice_video_bob_640x360_30.y4m"));
+ EXPECT_THAT(frame_reader->num_frames(), Eq(1));
+ rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame();
+ rtc::scoped_refptr<I420BufferInterface> expected_frame =
+ frame.video_frame_buffer()->ToI420();
+ double psnr = I420PSNR(*expected_frame, *actual_frame);
+ double ssim = I420SSIM(*expected_frame, *actual_frame);
+ // Actual should be downscaled version of expected.
+ EXPECT_GT(ssim, 0.98);
+ EXPECT_GT(psnr, 38);
+
+ ExpectOutputFilesCount(1);
+}
+
+TEST_F(AnalyzingVideoSinkTest,
+ FallbackOnConfigResolutionIfNoSubscriptionProvided) {
+ VideoSubscription subscription;
+ VideoConfig video_config("alice_video", /*width=*/320, /*height=*/240,
+ /*fps=*/30);
+ video_config.output_dump_options = VideoDumpOptions(test_directory_);
+
+ ExampleVideoQualityAnalyzer analyzer;
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ CreateFrameGenerator(/*width=*/320, /*height=*/240);
+ VideoFrame frame = CreateFrame(*frame_generator);
+ frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame));
+
+ {
+ // `helper` and `sink` has to be destroyed so all frames will be written
+ // to the disk.
+ AnalyzingVideoSinksHelper helper;
+ helper.AddConfig("alice", video_config);
+ AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper,
+ subscription, /*report_infra_stats=*/false);
+ sink.OnFrame(frame);
+ }
+
+ EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(1)));
+
+ auto frame_reader = test::CreateY4mFrameReader(
+ test::JoinFilename(test_directory_, "alice_video_bob_320x240_30.y4m"));
+ EXPECT_THAT(frame_reader->num_frames(), Eq(1));
+ rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame();
+ rtc::scoped_refptr<I420BufferInterface> expected_frame =
+ frame.video_frame_buffer()->ToI420();
+ double psnr = I420PSNR(*expected_frame, *actual_frame);
+ double ssim = I420SSIM(*expected_frame, *actual_frame);
+ // Frames should be equal.
+ EXPECT_DOUBLE_EQ(ssim, 1.00);
+ EXPECT_DOUBLE_EQ(psnr, 48);
+
+ ExpectOutputFilesCount(1);
+}
+
+TEST_F(AnalyzingVideoSinkTest,
+ FallbackOnConfigResolutionIfNoSubscriptionIsNotResolved) {
+ VideoSubscription subscription;
+ subscription.SubscribeToAllPeers(
+ VideoResolution(VideoResolution::Spec::kMaxFromSender));
+ VideoConfig video_config("alice_video", /*width=*/320, /*height=*/240,
+ /*fps=*/30);
+ video_config.output_dump_options = VideoDumpOptions(test_directory_);
+
+ ExampleVideoQualityAnalyzer analyzer;
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ CreateFrameGenerator(/*width=*/320, /*height=*/240);
+ VideoFrame frame = CreateFrame(*frame_generator);
+ frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame));
+
+ {
+ // `helper` and `sink` has to be destroyed so all frames will be written
+ // to the disk.
+ AnalyzingVideoSinksHelper helper;
+ helper.AddConfig("alice", video_config);
+ AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper,
+ subscription, /*report_infra_stats=*/false);
+ sink.OnFrame(frame);
+ }
+
+ EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(1)));
+
+ auto frame_reader = test::CreateY4mFrameReader(
+ test::JoinFilename(test_directory_, "alice_video_bob_320x240_30.y4m"));
+ EXPECT_THAT(frame_reader->num_frames(), Eq(1));
+ rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame();
+ rtc::scoped_refptr<I420BufferInterface> expected_frame =
+ frame.video_frame_buffer()->ToI420();
+ double psnr = I420PSNR(*expected_frame, *actual_frame);
+ double ssim = I420SSIM(*expected_frame, *actual_frame);
+ // Frames should be equal.
+ EXPECT_DOUBLE_EQ(ssim, 1.00);
+ EXPECT_DOUBLE_EQ(psnr, 48);
+
+ ExpectOutputFilesCount(1);
+}
+
+TEST_F(AnalyzingVideoSinkTest,
+ VideoFramesAreDumpedCorrectlyWhenSubscriptionChanged) {
+ VideoSubscription subscription_before;
+ subscription_before.SubscribeToPeer(
+ "alice", VideoResolution(/*width=*/1280, /*height=*/720, /*fps=*/30));
+ VideoSubscription subscription_after;
+ subscription_after.SubscribeToPeer(
+ "alice", VideoResolution(/*width=*/640, /*height=*/360, /*fps=*/30));
+ VideoConfig video_config("alice_video", /*width=*/1280, /*height=*/720,
+ /*fps=*/30);
+ video_config.output_dump_options = VideoDumpOptions(test_directory_);
+
+ ExampleVideoQualityAnalyzer analyzer;
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ CreateFrameGenerator(/*width=*/1280, /*height=*/720);
+ VideoFrame frame_before = CreateFrame(*frame_generator);
+ frame_before.set_id(
+ analyzer.OnFrameCaptured("alice", "alice_video", frame_before));
+ VideoFrame frame_after = CreateFrame(*frame_generator);
+ frame_after.set_id(
+ analyzer.OnFrameCaptured("alice", "alice_video", frame_after));
+
+ {
+ // `helper` and `sink` has to be destroyed so all frames will be written
+ // to the disk.
+ AnalyzingVideoSinksHelper helper;
+ helper.AddConfig("alice", video_config);
+ AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper,
+ subscription_before, /*report_infra_stats=*/false);
+ sink.OnFrame(frame_before);
+
+ sink.UpdateSubscription(subscription_after);
+ sink.OnFrame(frame_after);
+ }
+
+ EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(2)));
+
+ {
+ auto frame_reader = test::CreateY4mFrameReader(
+ test::JoinFilename(test_directory_, "alice_video_bob_1280x720_30.y4m"));
+ EXPECT_THAT(frame_reader->num_frames(), Eq(1));
+ rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame();
+ rtc::scoped_refptr<I420BufferInterface> expected_frame =
+ frame_before.video_frame_buffer()->ToI420();
+ double psnr = I420PSNR(*expected_frame, *actual_frame);
+ double ssim = I420SSIM(*expected_frame, *actual_frame);
+ // Frames should be equal.
+ EXPECT_DOUBLE_EQ(ssim, 1.00);
+ EXPECT_DOUBLE_EQ(psnr, 48);
+ }
+ {
+ auto frame_reader = test::CreateY4mFrameReader(
+ test::JoinFilename(test_directory_, "alice_video_bob_640x360_30.y4m"));
+ EXPECT_THAT(frame_reader->num_frames(), Eq(1));
+ rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame();
+ rtc::scoped_refptr<I420BufferInterface> expected_frame =
+ frame_after.video_frame_buffer()->ToI420();
+ double psnr = I420PSNR(*expected_frame, *actual_frame);
+ double ssim = I420SSIM(*expected_frame, *actual_frame);
+ // Actual should be downscaled version of expected.
+ EXPECT_GT(ssim, 0.98);
+ EXPECT_GT(psnr, 38);
+ }
+
+ ExpectOutputFilesCount(2);
+}
+
+TEST_F(AnalyzingVideoSinkTest,
+ VideoFramesAreDumpedCorrectlyWhenSubscriptionChangedOnTheSameOne) {
+ VideoSubscription subscription_before;
+ subscription_before.SubscribeToPeer(
+ "alice", VideoResolution(/*width=*/640, /*height=*/360, /*fps=*/30));
+ VideoSubscription subscription_after;
+ subscription_after.SubscribeToPeer(
+ "alice", VideoResolution(/*width=*/640, /*height=*/360, /*fps=*/30));
+ VideoConfig video_config("alice_video", /*width=*/640, /*height=*/360,
+ /*fps=*/30);
+ video_config.output_dump_options = VideoDumpOptions(test_directory_);
+
+ ExampleVideoQualityAnalyzer analyzer;
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ CreateFrameGenerator(/*width=*/640, /*height=*/360);
+ VideoFrame frame_before = CreateFrame(*frame_generator);
+ frame_before.set_id(
+ analyzer.OnFrameCaptured("alice", "alice_video", frame_before));
+ VideoFrame frame_after = CreateFrame(*frame_generator);
+ frame_after.set_id(
+ analyzer.OnFrameCaptured("alice", "alice_video", frame_after));
+
+ {
+ // `helper` and `sink` has to be destroyed so all frames will be written
+ // to the disk.
+ AnalyzingVideoSinksHelper helper;
+ helper.AddConfig("alice", video_config);
+ AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper,
+ subscription_before, /*report_infra_stats=*/false);
+ sink.OnFrame(frame_before);
+
+ sink.UpdateSubscription(subscription_after);
+ sink.OnFrame(frame_after);
+ }
+
+ EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(2)));
+
+ {
+ auto frame_reader = test::CreateY4mFrameReader(
+ test::JoinFilename(test_directory_, "alice_video_bob_640x360_30.y4m"));
+ EXPECT_THAT(frame_reader->num_frames(), Eq(2));
+ // Read the first frame.
+ rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame();
+ rtc::scoped_refptr<I420BufferInterface> expected_frame =
+ frame_before.video_frame_buffer()->ToI420();
+ // Frames should be equal.
+ EXPECT_DOUBLE_EQ(I420SSIM(*expected_frame, *actual_frame), 1.00);
+ EXPECT_DOUBLE_EQ(I420PSNR(*expected_frame, *actual_frame), 48);
+ // Read the second frame.
+ actual_frame = frame_reader->PullFrame();
+ expected_frame = frame_after.video_frame_buffer()->ToI420();
+ // Frames should be equal.
+ EXPECT_DOUBLE_EQ(I420SSIM(*expected_frame, *actual_frame), 1.00);
+ EXPECT_DOUBLE_EQ(I420PSNR(*expected_frame, *actual_frame), 48);
+ }
+
+ ExpectOutputFilesCount(1);
+}
+
+TEST_F(AnalyzingVideoSinkTest, SmallDiviationsInAspectRationAreAllowed) {
+ VideoSubscription subscription;
+ subscription.SubscribeToPeer(
+ "alice", VideoResolution(/*width=*/480, /*height=*/270, /*fps=*/30));
+ VideoConfig video_config("alice_video", /*width=*/480, /*height=*/270,
+ /*fps=*/30);
+ video_config.output_dump_options = VideoDumpOptions(test_directory_);
+
+ ExampleVideoQualityAnalyzer analyzer;
+ // Generator produces downscaled frames with a bit different aspect ration.
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ CreateFrameGenerator(/*width=*/240, /*height=*/136);
+ VideoFrame frame = CreateFrame(*frame_generator);
+ frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame));
+
+ {
+ // `helper` and `sink` has to be destroyed so all frames will be written
+ // to the disk.
+ AnalyzingVideoSinksHelper helper;
+ helper.AddConfig("alice", video_config);
+ AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper,
+ subscription, /*report_infra_stats=*/false);
+ sink.OnFrame(frame);
+ }
+
+ EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(1)));
+
+ {
+ auto frame_reader = test::CreateY4mFrameReader(
+ test::JoinFilename(test_directory_, "alice_video_bob_480x270_30.y4m"));
+ EXPECT_THAT(frame_reader->num_frames(), Eq(1));
+ // Read the first frame.
+ rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame();
+ rtc::scoped_refptr<I420BufferInterface> expected_frame =
+ frame.video_frame_buffer()->ToI420();
+ // Actual frame is upscaled version of the expected. But because rendered
+ // resolution is equal to the actual frame size we need to upscale expected
+ // during comparison and then they have to be the same.
+ EXPECT_DOUBLE_EQ(I420SSIM(*actual_frame, *expected_frame), 1);
+ EXPECT_DOUBLE_EQ(I420PSNR(*actual_frame, *expected_frame), 48);
+ }
+
+ ExpectOutputFilesCount(1);
+}
+
+TEST_F(AnalyzingVideoSinkTest, VideoFramesIdsAreDumpedWhenRequested) {
+ VideoSubscription subscription;
+ subscription.SubscribeToPeer(
+ "alice", VideoResolution(/*width=*/320, /*height=*/240, /*fps=*/30));
+ VideoConfig video_config("alice_video", /*width=*/320, /*height=*/240,
+ /*fps=*/30);
+ video_config.output_dump_options =
+ VideoDumpOptions(test_directory_, /*export_frame_ids=*/true);
+
+ ExampleVideoQualityAnalyzer analyzer;
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ CreateFrameGenerator(/*width=*/320, /*height=*/240);
+
+ std::vector<std::string> expected_frame_ids;
+ {
+ // `helper` and `sink` has to be destroyed so all frames will be written
+ // to the disk.
+ AnalyzingVideoSinksHelper helper;
+ helper.AddConfig("alice", video_config);
+ AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper,
+ subscription, /*report_infra_stats=*/false);
+ for (int i = 0; i < 10; ++i) {
+ VideoFrame frame = CreateFrame(*frame_generator);
+ frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame));
+ expected_frame_ids.push_back(std::to_string(frame.id()));
+ sink.OnFrame(frame);
+ }
+ }
+
+ EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(10)));
+
+ AssertFrameIdsAre(
+ test::JoinFilename(test_directory_,
+ "alice_video_bob_320x240_30.frame_ids.txt"),
+ expected_frame_ids);
+
+ ExpectOutputFilesCount(2);
+}
+
+TEST_F(AnalyzingVideoSinkTest,
+ VideoFramesAndIdsAreDumpedWithFixedFpsWhenRequested) {
+ GlobalSimulatedTimeController simulated_time(Timestamp::Seconds(100000));
+
+ VideoSubscription subscription;
+ subscription.SubscribeToPeer(
+ "alice", VideoResolution(/*width=*/320, /*height=*/240, /*fps=*/10));
+ VideoConfig video_config("alice_video", /*width=*/320, /*height=*/240,
+ /*fps=*/10);
+ video_config.output_dump_options =
+ VideoDumpOptions(test_directory_, /*export_frame_ids=*/true);
+ video_config.output_dump_use_fixed_framerate = true;
+
+ ExampleVideoQualityAnalyzer analyzer;
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ CreateFrameGenerator(/*width=*/320, /*height=*/240);
+
+ VideoFrame frame1 = CreateFrame(*frame_generator);
+ frame1.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame1));
+ VideoFrame frame2 = CreateFrame(*frame_generator);
+ frame2.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame2));
+
+ {
+ // `helper` and `sink` has to be destroyed so all frames will be written
+ // to the disk.
+ AnalyzingVideoSinksHelper helper;
+ helper.AddConfig("alice", video_config);
+ AnalyzingVideoSink sink("bob", simulated_time.GetClock(), analyzer, helper,
+ subscription, /*report_infra_stats=*/false);
+ sink.OnFrame(frame1);
+ // Advance almost 1 second, so the first frame has to be repeated 9 time
+ // more.
+ simulated_time.AdvanceTime(TimeDelta::Millis(990));
+ sink.OnFrame(frame2);
+ simulated_time.AdvanceTime(TimeDelta::Millis(100));
+ }
+
+ EXPECT_THAT(analyzer.frames_rendered(), Eq(static_cast<uint64_t>(2)));
+
+ auto frame_reader = test::CreateY4mFrameReader(
+ test::JoinFilename(test_directory_, "alice_video_bob_320x240_10.y4m"));
+ EXPECT_THAT(frame_reader->num_frames(), Eq(11));
+ for (int i = 0; i < 10; ++i) {
+ rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame();
+ rtc::scoped_refptr<I420BufferInterface> expected_frame =
+ frame1.video_frame_buffer()->ToI420();
+ double psnr = I420PSNR(*expected_frame, *actual_frame);
+ double ssim = I420SSIM(*expected_frame, *actual_frame);
+ // Frames should be equal.
+ EXPECT_DOUBLE_EQ(ssim, 1.00);
+ EXPECT_DOUBLE_EQ(psnr, 48);
+ }
+ rtc::scoped_refptr<I420Buffer> actual_frame = frame_reader->PullFrame();
+ rtc::scoped_refptr<I420BufferInterface> expected_frame =
+ frame2.video_frame_buffer()->ToI420();
+ double psnr = I420PSNR(*expected_frame, *actual_frame);
+ double ssim = I420SSIM(*expected_frame, *actual_frame);
+ // Frames should be equal.
+ EXPECT_DOUBLE_EQ(ssim, 1.00);
+ EXPECT_DOUBLE_EQ(psnr, 48);
+
+ AssertFrameIdsAre(
+ test::JoinFilename(test_directory_,
+ "alice_video_bob_320x240_10.frame_ids.txt"),
+ {std::to_string(frame1.id()), std::to_string(frame1.id()),
+ std::to_string(frame1.id()), std::to_string(frame1.id()),
+ std::to_string(frame1.id()), std::to_string(frame1.id()),
+ std::to_string(frame1.id()), std::to_string(frame1.id()),
+ std::to_string(frame1.id()), std::to_string(frame1.id()),
+ std::to_string(frame2.id())});
+
+ ExpectOutputFilesCount(2);
+}
+
+TEST_F(AnalyzingVideoSinkTest, InfraMetricsCollectedWhenRequested) {
+ VideoSubscription subscription;
+ subscription.SubscribeToPeer(
+ "alice", VideoResolution(/*width=*/1280, /*height=*/720, /*fps=*/30));
+ VideoConfig video_config("alice_video", /*width=*/640, /*height=*/360,
+ /*fps=*/30);
+
+ ExampleVideoQualityAnalyzer analyzer;
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ CreateFrameGenerator(/*width=*/640, /*height=*/360);
+ VideoFrame frame = CreateFrame(*frame_generator);
+ frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame));
+
+ AnalyzingVideoSinksHelper helper;
+ helper.AddConfig("alice", video_config);
+ AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper,
+ subscription, /*report_infra_stats=*/true);
+ sink.OnFrame(frame);
+
+ AnalyzingVideoSink::Stats stats = sink.stats();
+ EXPECT_THAT(stats.scaling_tims_ms.NumSamples(), Eq(1));
+ EXPECT_THAT(stats.scaling_tims_ms.GetAverage(), Ge(0));
+ EXPECT_THAT(stats.analyzing_sink_processing_time_ms.NumSamples(), Eq(1));
+ EXPECT_THAT(stats.analyzing_sink_processing_time_ms.GetAverage(),
+ Ge(stats.scaling_tims_ms.GetAverage()));
+
+ ExpectOutputFilesCount(0);
+}
+
+TEST_F(AnalyzingVideoSinkTest, InfraMetricsNotCollectedWhenNotRequested) {
+ VideoSubscription subscription;
+ subscription.SubscribeToPeer(
+ "alice", VideoResolution(/*width=*/1280, /*height=*/720, /*fps=*/30));
+ VideoConfig video_config("alice_video", /*width=*/640, /*height=*/360,
+ /*fps=*/30);
+
+ ExampleVideoQualityAnalyzer analyzer;
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ CreateFrameGenerator(/*width=*/640, /*height=*/360);
+ VideoFrame frame = CreateFrame(*frame_generator);
+ frame.set_id(analyzer.OnFrameCaptured("alice", "alice_video", frame));
+
+ AnalyzingVideoSinksHelper helper;
+ helper.AddConfig("alice", video_config);
+ AnalyzingVideoSink sink("bob", Clock::GetRealTimeClock(), analyzer, helper,
+ subscription, /*report_infra_stats=*/false);
+ sink.OnFrame(frame);
+
+ AnalyzingVideoSink::Stats stats = sink.stats();
+ EXPECT_THAT(stats.scaling_tims_ms.NumSamples(), Eq(0));
+ EXPECT_THAT(stats.analyzing_sink_processing_time_ms.NumSamples(), Eq(0));
+
+ ExpectOutputFilesCount(0);
+}
+
+} // namespace
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.cc
new file mode 100644
index 0000000000..70dc4b00b5
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.cc
@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h"
+
+#include <memory>
+#include <set>
+#include <string>
+#include <utility>
+
+#include "absl/strings/string_view.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/video/video_frame_writer.h"
+#include "rtc_base/synchronization/mutex.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+void AnalyzingVideoSinksHelper::AddConfig(absl::string_view sender_peer_name,
+ VideoConfig config) {
+ MutexLock lock(&mutex_);
+ auto it = video_configs_.find(*config.stream_label);
+ if (it == video_configs_.end()) {
+ std::string stream_label = *config.stream_label;
+ video_configs_.emplace(
+ std::move(stream_label),
+ std::pair{std::string(sender_peer_name), std::move(config)});
+ } else {
+ it->second = std::pair{std::string(sender_peer_name), std::move(config)};
+ }
+}
+
+absl::optional<std::pair<std::string, VideoConfig>>
+AnalyzingVideoSinksHelper::GetPeerAndConfig(absl::string_view stream_label) {
+ MutexLock lock(&mutex_);
+ auto it = video_configs_.find(std::string(stream_label));
+ if (it == video_configs_.end()) {
+ return absl::nullopt;
+ }
+ return it->second;
+}
+
+void AnalyzingVideoSinksHelper::RemoveConfig(absl::string_view stream_label) {
+ MutexLock lock(&mutex_);
+ video_configs_.erase(std::string(stream_label));
+}
+
+test::VideoFrameWriter* AnalyzingVideoSinksHelper::AddVideoWriter(
+ std::unique_ptr<test::VideoFrameWriter> video_writer) {
+ MutexLock lock(&mutex_);
+ test::VideoFrameWriter* out = video_writer.get();
+ video_writers_.push_back(std::move(video_writer));
+ return out;
+}
+
+void AnalyzingVideoSinksHelper::CloseAndRemoveVideoWriters(
+ std::set<test::VideoFrameWriter*> writers_to_close) {
+ MutexLock lock(&mutex_);
+ for (auto it = video_writers_.cbegin(); it != video_writers_.cend();) {
+ if (writers_to_close.find(it->get()) != writers_to_close.end()) {
+ (*it)->Close();
+ it = video_writers_.erase(it);
+ } else {
+ ++it;
+ }
+ }
+}
+
+void AnalyzingVideoSinksHelper::Clear() {
+ MutexLock lock(&mutex_);
+ video_configs_.clear();
+ for (const auto& video_writer : video_writers_) {
+ video_writer->Close();
+ }
+ video_writers_.clear();
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h
new file mode 100644
index 0000000000..5f38c5a40e
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINKS_HELPER_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINKS_HELPER_H_
+
+#include <list>
+#include <map>
+#include <memory>
+#include <set>
+#include <string>
+#include <utility>
+
+#include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/video/video_frame_writer.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "rtc_base/thread_annotations.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+// Registry of known video configs and video writers.
+// This class is thread safe.
+class AnalyzingVideoSinksHelper {
+ public:
+ // Adds config in the registry. If config with such stream label was
+ // registered before, the new value will override the old one.
+ void AddConfig(absl::string_view sender_peer_name, VideoConfig config);
+ absl::optional<std::pair<std::string, VideoConfig>> GetPeerAndConfig(
+ absl::string_view stream_label);
+ // Removes video config for specified stream label. If there are no know video
+ // config for such stream label - does nothing.
+ void RemoveConfig(absl::string_view stream_label);
+
+ // Takes ownership of the provided video writer. All video writers owned by
+ // this class will be closed during `AnalyzingVideoSinksHelper` destruction
+ // and guaranteed to be alive either until explicitly removed by
+ // `CloseAndRemoveVideoWriters` or until `AnalyzingVideoSinksHelper` is
+ // destroyed.
+ //
+ // Returns pointer to the added writer. Ownership is maintained by
+ // `AnalyzingVideoSinksHelper`.
+ test::VideoFrameWriter* AddVideoWriter(
+ std::unique_ptr<test::VideoFrameWriter> video_writer);
+ // For each provided `writers_to_close`, if it is known, will close and
+ // destroy it, otherwise does nothing with it.
+ void CloseAndRemoveVideoWriters(
+ std::set<test::VideoFrameWriter*> writers_to_close);
+
+ // Removes all added configs and close and removes all added writers.
+ void Clear();
+
+ private:
+ Mutex mutex_;
+ std::map<std::string, std::pair<std::string, VideoConfig>> video_configs_
+ RTC_GUARDED_BY(mutex_);
+ std::list<std::unique_ptr<test::VideoFrameWriter>> video_writers_
+ RTC_GUARDED_BY(mutex_);
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_ANALYZING_VIDEO_SINKS_HELPER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper_test.cc
new file mode 100644
index 0000000000..1a820a5229
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/analyzing_video_sinks_helper_test.cc
@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h"
+
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "absl/types/optional.h"
+#include "api/test/pclf/media_configuration.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+using ::testing::Eq;
+
+// Asserts equality of the main fields of the video config. We don't compare
+// the full config due to the lack of equality definition for a lot of subtypes.
+void AssertConfigsAreEquals(const VideoConfig& actual,
+ const VideoConfig& expected) {
+ EXPECT_THAT(actual.stream_label, Eq(expected.stream_label));
+ EXPECT_THAT(actual.width, Eq(expected.width));
+ EXPECT_THAT(actual.height, Eq(expected.height));
+ EXPECT_THAT(actual.fps, Eq(expected.fps));
+}
+
+TEST(AnalyzingVideoSinksHelperTest, ConfigsCanBeAdded) {
+ VideoConfig config("alice_video", /*width=*/1280, /*height=*/720, /*fps=*/30);
+
+ AnalyzingVideoSinksHelper helper;
+ helper.AddConfig("alice", config);
+
+ absl::optional<std::pair<std::string, VideoConfig>> registred_config =
+ helper.GetPeerAndConfig("alice_video");
+ ASSERT_TRUE(registred_config.has_value());
+ EXPECT_THAT(registred_config->first, Eq("alice"));
+ AssertConfigsAreEquals(registred_config->second, config);
+}
+
+TEST(AnalyzingVideoSinksHelperTest, AddingForExistingLabelWillOverwriteValue) {
+ VideoConfig config_before("alice_video", /*width=*/1280, /*height=*/720,
+ /*fps=*/30);
+ VideoConfig config_after("alice_video", /*width=*/640, /*height=*/360,
+ /*fps=*/15);
+
+ AnalyzingVideoSinksHelper helper;
+ helper.AddConfig("alice", config_before);
+
+ absl::optional<std::pair<std::string, VideoConfig>> registred_config =
+ helper.GetPeerAndConfig("alice_video");
+ ASSERT_TRUE(registred_config.has_value());
+ EXPECT_THAT(registred_config->first, Eq("alice"));
+ AssertConfigsAreEquals(registred_config->second, config_before);
+
+ helper.AddConfig("alice", config_after);
+
+ registred_config = helper.GetPeerAndConfig("alice_video");
+ ASSERT_TRUE(registred_config.has_value());
+ EXPECT_THAT(registred_config->first, Eq("alice"));
+ AssertConfigsAreEquals(registred_config->second, config_after);
+}
+
+TEST(AnalyzingVideoSinksHelperTest, ConfigsCanBeRemoved) {
+ VideoConfig config("alice_video", /*width=*/1280, /*height=*/720, /*fps=*/30);
+
+ AnalyzingVideoSinksHelper helper;
+ helper.AddConfig("alice", config);
+
+ ASSERT_TRUE(helper.GetPeerAndConfig("alice_video").has_value());
+
+ helper.RemoveConfig("alice_video");
+ ASSERT_FALSE(helper.GetPeerAndConfig("alice_video").has_value());
+}
+
+TEST(AnalyzingVideoSinksHelperTest, RemoveOfNonExistingConfigDontCrash) {
+ AnalyzingVideoSinksHelper helper;
+ helper.RemoveConfig("alice_video");
+}
+
+TEST(AnalyzingVideoSinksHelperTest, ClearRemovesAllConfigs) {
+ VideoConfig config1("alice_video", /*width=*/640, /*height=*/360, /*fps=*/30);
+ VideoConfig config2("bob_video", /*width=*/640, /*height=*/360, /*fps=*/30);
+
+ AnalyzingVideoSinksHelper helper;
+ helper.AddConfig("alice", config1);
+ helper.AddConfig("bob", config2);
+
+ ASSERT_TRUE(helper.GetPeerAndConfig("alice_video").has_value());
+ ASSERT_TRUE(helper.GetPeerAndConfig("bob_video").has_value());
+
+ helper.Clear();
+ ASSERT_FALSE(helper.GetPeerAndConfig("alice_video").has_value());
+ ASSERT_FALSE(helper.GetPeerAndConfig("bob_video").has_value());
+}
+
+struct TestVideoFrameWriterFactory {
+ int closed_writers_count = 0;
+ int deleted_writers_count = 0;
+
+ std::unique_ptr<test::VideoFrameWriter> CreateWriter() {
+ return std::make_unique<TestVideoFrameWriter>(this);
+ }
+
+ private:
+ class TestVideoFrameWriter : public test::VideoFrameWriter {
+ public:
+ explicit TestVideoFrameWriter(TestVideoFrameWriterFactory* factory)
+ : factory_(factory) {}
+ ~TestVideoFrameWriter() override { factory_->deleted_writers_count++; }
+
+ bool WriteFrame(const VideoFrame& frame) override { return true; }
+
+ void Close() override { factory_->closed_writers_count++; }
+
+ private:
+ TestVideoFrameWriterFactory* factory_;
+ };
+};
+
+TEST(AnalyzingVideoSinksHelperTest, RemovingWritersCloseAndDestroyAllOfThem) {
+ TestVideoFrameWriterFactory factory;
+
+ AnalyzingVideoSinksHelper helper;
+ test::VideoFrameWriter* writer1 =
+ helper.AddVideoWriter(factory.CreateWriter());
+ test::VideoFrameWriter* writer2 =
+ helper.AddVideoWriter(factory.CreateWriter());
+
+ helper.CloseAndRemoveVideoWriters({writer1, writer2});
+
+ EXPECT_THAT(factory.closed_writers_count, Eq(2));
+ EXPECT_THAT(factory.deleted_writers_count, Eq(2));
+}
+
+TEST(AnalyzingVideoSinksHelperTest, ClearCloseAndDestroyAllWriters) {
+ TestVideoFrameWriterFactory factory;
+
+ AnalyzingVideoSinksHelper helper;
+ helper.AddVideoWriter(factory.CreateWriter());
+ helper.AddVideoWriter(factory.CreateWriter());
+
+ helper.Clear();
+
+ EXPECT_THAT(factory.closed_writers_count, Eq(2));
+ EXPECT_THAT(factory.deleted_writers_count, Eq(2));
+}
+
+} // namespace
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
new file mode 100644
index 0000000000..59144589fc
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
@@ -0,0 +1,1228 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
+
+#include <algorithm>
+#include <map>
+#include <memory>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "api/array_view.h"
+#include "api/numerics/samples_stats_counter.h"
+#include "api/test/metrics/metric.h"
+#include "api/units/time_delta.h"
+#include "api/units/timestamp.h"
+#include "api/video/video_frame.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/strings/string_builder.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h"
+#include "test/pc/e2e/metric_metadata_keys.h"
+
+namespace webrtc {
+namespace {
+
+using ::webrtc::test::ImprovementDirection;
+using ::webrtc::test::Unit;
+using ::webrtc::webrtc_pc_e2e::MetricMetadataKey;
+
+constexpr int kBitsInByte = 8;
+constexpr absl::string_view kSkipRenderedFrameReasonProcessed = "processed";
+constexpr absl::string_view kSkipRenderedFrameReasonRendered = "rendered";
+constexpr absl::string_view kSkipRenderedFrameReasonDropped =
+ "considered dropped";
+
+void LogFrameCounters(const std::string& name, const FrameCounters& counters) {
+ RTC_LOG(LS_INFO) << "[" << name
+ << "] Captured : " << counters.captured;
+ RTC_LOG(LS_INFO) << "[" << name
+ << "] Pre encoded : " << counters.pre_encoded;
+ RTC_LOG(LS_INFO) << "[" << name
+ << "] Encoded : " << counters.encoded;
+ RTC_LOG(LS_INFO) << "[" << name
+ << "] Received : " << counters.received;
+ RTC_LOG(LS_INFO) << "[" << name
+ << "] Decoded : " << counters.decoded;
+ RTC_LOG(LS_INFO) << "[" << name
+ << "] Rendered : " << counters.rendered;
+ RTC_LOG(LS_INFO) << "[" << name
+ << "] Dropped : " << counters.dropped;
+ RTC_LOG(LS_INFO) << "[" << name
+ << "] Failed to decode : " << counters.failed_to_decode;
+}
+
+void LogStreamInternalStats(const std::string& name,
+ const StreamStats& stats,
+ Timestamp start_time) {
+ for (const auto& entry : stats.dropped_by_phase) {
+ RTC_LOG(LS_INFO) << "[" << name << "] Dropped at " << ToString(entry.first)
+ << ": " << entry.second;
+ }
+ Timestamp first_encoded_frame_time = Timestamp::PlusInfinity();
+ for (const StreamCodecInfo& encoder : stats.encoders) {
+ RTC_DCHECK(encoder.switched_on_at.IsFinite());
+ RTC_DCHECK(encoder.switched_from_at.IsFinite());
+ if (first_encoded_frame_time.IsInfinite()) {
+ first_encoded_frame_time = encoder.switched_on_at;
+ }
+ RTC_LOG(LS_INFO)
+ << "[" << name << "] Used encoder: \"" << encoder.codec_name
+ << "\" used from (frame_id=" << encoder.first_frame_id
+ << "; from_stream_start="
+ << (encoder.switched_on_at - stats.stream_started_time).ms()
+ << "ms, from_call_start=" << (encoder.switched_on_at - start_time).ms()
+ << "ms) until (frame_id=" << encoder.last_frame_id
+ << "; from_stream_start="
+ << (encoder.switched_from_at - stats.stream_started_time).ms()
+ << "ms, from_call_start="
+ << (encoder.switched_from_at - start_time).ms() << "ms)";
+ }
+ for (const StreamCodecInfo& decoder : stats.decoders) {
+ RTC_DCHECK(decoder.switched_on_at.IsFinite());
+ RTC_DCHECK(decoder.switched_from_at.IsFinite());
+ RTC_LOG(LS_INFO)
+ << "[" << name << "] Used decoder: \"" << decoder.codec_name
+ << "\" used from (frame_id=" << decoder.first_frame_id
+ << "; from_stream_start="
+ << (decoder.switched_on_at - stats.stream_started_time).ms()
+ << "ms, from_call_start=" << (decoder.switched_on_at - start_time).ms()
+ << "ms) until (frame_id=" << decoder.last_frame_id
+ << "; from_stream_start="
+ << (decoder.switched_from_at - stats.stream_started_time).ms()
+ << "ms, from_call_start="
+ << (decoder.switched_from_at - start_time).ms() << "ms)";
+ }
+}
+
+template <typename T>
+absl::optional<T> MaybeGetValue(const std::map<size_t, T>& map, size_t key) {
+ auto it = map.find(key);
+ if (it == map.end()) {
+ return absl::nullopt;
+ }
+ return it->second;
+}
+
+SamplesStatsCounter::StatsSample StatsSample(double value,
+ Timestamp sampling_time) {
+ return SamplesStatsCounter::StatsSample{value, sampling_time};
+}
+
+} // namespace
+
+DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(
+ webrtc::Clock* clock,
+ test::MetricsLogger* metrics_logger,
+ DefaultVideoQualityAnalyzerOptions options)
+ : options_(options),
+ clock_(clock),
+ metrics_logger_(metrics_logger),
+ frames_comparator_(clock, cpu_measurer_, options) {
+ RTC_CHECK(metrics_logger_);
+}
+
+DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() {
+ Stop();
+}
+
+void DefaultVideoQualityAnalyzer::Start(
+ std::string test_case_name,
+ rtc::ArrayView<const std::string> peer_names,
+ int max_threads_count) {
+ test_label_ = std::move(test_case_name);
+ frames_comparator_.Start(max_threads_count);
+ {
+ MutexLock lock(&mutex_);
+ peers_ = std::make_unique<NamesCollection>(peer_names);
+ RTC_CHECK(start_time_.IsMinusInfinity());
+
+ RTC_CHECK_EQ(state_, State::kNew)
+ << "DefaultVideoQualityAnalyzer is already started";
+ state_ = State::kActive;
+ start_time_ = Now();
+ }
+}
+
+uint16_t DefaultVideoQualityAnalyzer::OnFrameCaptured(
+ absl::string_view peer_name,
+ const std::string& stream_label,
+ const webrtc::VideoFrame& frame) {
+ // `next_frame_id` is atomic, so we needn't lock here.
+ Timestamp captured_time = Now();
+ Timestamp start_time = Timestamp::MinusInfinity();
+ size_t peer_index = -1;
+ size_t peers_count = -1;
+ size_t stream_index;
+ uint16_t frame_id = VideoFrame::kNotSetId;
+ {
+ MutexLock lock(&mutex_);
+ frame_id = GetNextFrameId();
+ RTC_CHECK_EQ(state_, State::kActive)
+ << "DefaultVideoQualityAnalyzer has to be started before use";
+ // Create a local copy of `start_time_`, peer's index and total peers count
+ // to access it without holding a `mutex_` during access to
+ // `frames_comparator_`.
+ start_time = start_time_;
+ peer_index = peers_->index(peer_name);
+ peers_count = peers_->size();
+ stream_index = streams_.AddIfAbsent(stream_label);
+ }
+ // Ensure stats for this stream exists.
+ frames_comparator_.EnsureStatsForStream(stream_index, peer_index, peers_count,
+ captured_time, start_time);
+ {
+ MutexLock lock(&mutex_);
+ stream_to_sender_[stream_index] = peer_index;
+ frame_counters_.captured++;
+ for (size_t i : peers_->GetAllIndexes()) {
+ if (i != peer_index || options_.enable_receive_own_stream) {
+ InternalStatsKey key(stream_index, peer_index, i);
+ stream_frame_counters_[key].captured++;
+ }
+ }
+
+ std::set<size_t> frame_receivers_indexes = peers_->GetPresentIndexes();
+ if (!options_.enable_receive_own_stream) {
+ frame_receivers_indexes.erase(peer_index);
+ }
+
+ auto state_it = stream_states_.find(stream_index);
+ if (state_it == stream_states_.end()) {
+ stream_states_.emplace(
+ stream_index,
+ StreamState(peer_index, frame_receivers_indexes, captured_time));
+ }
+ StreamState* state = &stream_states_.at(stream_index);
+ state->PushBack(frame_id);
+ // Update frames in flight info.
+ auto it = captured_frames_in_flight_.find(frame_id);
+ if (it != captured_frames_in_flight_.end()) {
+ // If we overflow uint16_t and hit previous frame id and this frame is
+ // still in flight, it means that this stream wasn't rendered for long
+ // time and we need to process existing frame as dropped.
+ for (size_t i : peers_->GetPresentIndexes()) {
+ if (i == peer_index && !options_.enable_receive_own_stream) {
+ continue;
+ }
+
+ uint16_t oldest_frame_id = state->PopFront(i);
+ RTC_DCHECK_EQ(frame_id, oldest_frame_id);
+ frame_counters_.dropped++;
+ InternalStatsKey key(stream_index, peer_index, i);
+ stream_frame_counters_.at(key).dropped++;
+
+ analyzer_stats_.frames_in_flight_left_count.AddSample(
+ StatsSample(captured_frames_in_flight_.size(), Now()));
+ frames_comparator_.AddComparison(
+ InternalStatsKey(stream_index, peer_index, i),
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt, FrameComparisonType::kDroppedFrame,
+ it->second.GetStatsForPeer(i));
+ }
+
+ captured_frames_in_flight_.erase(it);
+ }
+ captured_frames_in_flight_.emplace(
+ frame_id, FrameInFlight(stream_index, frame, captured_time,
+ std::move(frame_receivers_indexes)));
+ // Set frame id on local copy of the frame
+ captured_frames_in_flight_.at(frame_id).SetFrameId(frame_id);
+
+ // Update history stream<->frame mapping
+ for (auto it = stream_to_frame_id_history_.begin();
+ it != stream_to_frame_id_history_.end(); ++it) {
+ it->second.erase(frame_id);
+ }
+ stream_to_frame_id_history_[stream_index].insert(frame_id);
+ stream_to_frame_id_full_history_[stream_index].push_back(frame_id);
+
+ // If state has too many frames that are in flight => remove the oldest
+ // queued frame in order to avoid to use too much memory.
+ if (state->GetAliveFramesCount() >
+ options_.max_frames_in_flight_per_stream_count) {
+ uint16_t frame_id_to_remove = state->MarkNextAliveFrameAsDead();
+ auto it = captured_frames_in_flight_.find(frame_id_to_remove);
+ RTC_CHECK(it != captured_frames_in_flight_.end())
+ << "Frame with ID " << frame_id_to_remove
+ << " is expected to be in flight, but hasn't been found in "
+ << "|captured_frames_in_flight_|";
+ bool is_removed = it->second.RemoveFrame();
+ RTC_DCHECK(is_removed)
+ << "Invalid stream state: alive frame is removed already";
+ }
+ if (options_.report_infra_metrics) {
+ analyzer_stats_.on_frame_captured_processing_time_ms.AddSample(
+ (Now() - captured_time).ms<double>());
+ }
+ }
+ return frame_id;
+}
+
+void DefaultVideoQualityAnalyzer::OnFramePreEncode(
+ absl::string_view peer_name,
+ const webrtc::VideoFrame& frame) {
+ Timestamp processing_started = Now();
+ MutexLock lock(&mutex_);
+ RTC_CHECK_EQ(state_, State::kActive)
+ << "DefaultVideoQualityAnalyzer has to be started before use";
+
+ auto it = captured_frames_in_flight_.find(frame.id());
+ RTC_CHECK(it != captured_frames_in_flight_.end())
+ << "Frame id=" << frame.id() << " not found";
+ FrameInFlight& frame_in_flight = it->second;
+ frame_counters_.pre_encoded++;
+ size_t peer_index = peers_->index(peer_name);
+ for (size_t i : peers_->GetAllIndexes()) {
+ if (i != peer_index || options_.enable_receive_own_stream) {
+ InternalStatsKey key(frame_in_flight.stream(), peer_index, i);
+ stream_frame_counters_.at(key).pre_encoded++;
+ }
+ }
+ frame_in_flight.SetPreEncodeTime(Now());
+
+ if (options_.report_infra_metrics) {
+ analyzer_stats_.on_frame_pre_encode_processing_time_ms.AddSample(
+ (Now() - processing_started).ms<double>());
+ }
+}
+
+void DefaultVideoQualityAnalyzer::OnFrameEncoded(
+ absl::string_view peer_name,
+ uint16_t frame_id,
+ const webrtc::EncodedImage& encoded_image,
+ const EncoderStats& stats,
+ bool discarded) {
+ if (discarded)
+ return;
+
+ Timestamp processing_started = Now();
+ MutexLock lock(&mutex_);
+ RTC_CHECK_EQ(state_, State::kActive)
+ << "DefaultVideoQualityAnalyzer has to be started before use";
+
+ auto it = captured_frames_in_flight_.find(frame_id);
+ if (it == captured_frames_in_flight_.end()) {
+ RTC_LOG(LS_WARNING)
+ << "The encoding of video frame with id [" << frame_id << "] for peer ["
+ << peer_name << "] finished after all receivers rendered this frame or "
+ << "were removed. It can be OK for simulcast/SVC if higher quality "
+ << "stream is not required or the last receiver was unregistered "
+ << "between encoding of different layers, but it may indicate an ERROR "
+ << "for singlecast or if it happens often.";
+ return;
+ }
+ FrameInFlight& frame_in_flight = it->second;
+ // For SVC we can receive multiple encoded images for one frame, so to cover
+ // all cases we have to pick the last encode time.
+ if (!frame_in_flight.HasEncodedTime()) {
+ // Increase counters only when we meet this frame first time.
+ frame_counters_.encoded++;
+ size_t peer_index = peers_->index(peer_name);
+ for (size_t i : peers_->GetAllIndexes()) {
+ if (i != peer_index || options_.enable_receive_own_stream) {
+ InternalStatsKey key(frame_in_flight.stream(), peer_index, i);
+ stream_frame_counters_.at(key).encoded++;
+ }
+ }
+ }
+ Timestamp now = Now();
+ StreamCodecInfo used_encoder;
+ used_encoder.codec_name = stats.encoder_name;
+ used_encoder.first_frame_id = frame_id;
+ used_encoder.last_frame_id = frame_id;
+ used_encoder.switched_on_at = now;
+ used_encoder.switched_from_at = now;
+ frame_in_flight.OnFrameEncoded(
+ now, encoded_image._frameType, DataSize::Bytes(encoded_image.size()),
+ stats.target_encode_bitrate, encoded_image.SpatialIndex().value_or(0),
+ stats.qp, used_encoder);
+
+ if (options_.report_infra_metrics) {
+ analyzer_stats_.on_frame_encoded_processing_time_ms.AddSample(
+ (Now() - processing_started).ms<double>());
+ }
+}
+
+void DefaultVideoQualityAnalyzer::OnFrameDropped(
+ absl::string_view peer_name,
+ webrtc::EncodedImageCallback::DropReason reason) {
+ // Here we do nothing, because we will see this drop on renderer side.
+}
+
+void DefaultVideoQualityAnalyzer::OnFramePreDecode(
+ absl::string_view peer_name,
+ uint16_t frame_id,
+ const webrtc::EncodedImage& input_image) {
+ Timestamp processing_started = Now();
+ MutexLock lock(&mutex_);
+ RTC_CHECK_EQ(state_, State::kActive)
+ << "DefaultVideoQualityAnalyzer has to be started before use";
+
+ size_t peer_index = peers_->index(peer_name);
+
+ if (frame_id == VideoFrame::kNotSetId) {
+ frame_counters_.received++;
+ unknown_sender_frame_counters_[std::string(peer_name)].received++;
+ return;
+ }
+
+ auto it = captured_frames_in_flight_.find(frame_id);
+ if (it == captured_frames_in_flight_.end() ||
+ it->second.HasReceivedTime(peer_index)) {
+ // It means this frame was predecoded before, so we can skip it. It may
+ // happen when we have multiple simulcast streams in one track and received
+ // the same picture from two different streams because SFU can't reliably
+ // correlate two simulcast streams and started relaying the second stream
+ // from the same frame it has relayed right before for the first stream.
+ return;
+ }
+
+ frame_counters_.received++;
+ InternalStatsKey key(it->second.stream(),
+ stream_to_sender_.at(it->second.stream()), peer_index);
+ stream_frame_counters_.at(key).received++;
+ // Determine the time of the last received packet of this video frame.
+ RTC_DCHECK(!input_image.PacketInfos().empty());
+ Timestamp last_receive_time =
+ std::max_element(input_image.PacketInfos().cbegin(),
+ input_image.PacketInfos().cend(),
+ [](const RtpPacketInfo& a, const RtpPacketInfo& b) {
+ return a.receive_time() < b.receive_time();
+ })
+ ->receive_time();
+ it->second.OnFramePreDecode(peer_index,
+ /*received_time=*/last_receive_time,
+ /*decode_start_time=*/Now(),
+ input_image._frameType,
+ DataSize::Bytes(input_image.size()));
+
+ if (options_.report_infra_metrics) {
+ analyzer_stats_.on_frame_pre_decode_processing_time_ms.AddSample(
+ (Now() - processing_started).ms<double>());
+ }
+}
+
+void DefaultVideoQualityAnalyzer::OnFrameDecoded(
+ absl::string_view peer_name,
+ const webrtc::VideoFrame& frame,
+ const DecoderStats& stats) {
+ Timestamp processing_started = Now();
+ MutexLock lock(&mutex_);
+ RTC_CHECK_EQ(state_, State::kActive)
+ << "DefaultVideoQualityAnalyzer has to be started before use";
+
+ size_t peer_index = peers_->index(peer_name);
+
+ if (frame.id() == VideoFrame::kNotSetId) {
+ frame_counters_.decoded++;
+ unknown_sender_frame_counters_[std::string(peer_name)].decoded++;
+ return;
+ }
+
+ auto it = captured_frames_in_flight_.find(frame.id());
+ if (it == captured_frames_in_flight_.end() ||
+ it->second.HasDecodeEndTime(peer_index)) {
+ // It means this frame was decoded before, so we can skip it. It may happen
+ // when we have multiple simulcast streams in one track and received
+ // the same frame from two different streams because SFU can't reliably
+ // correlate two simulcast streams and started relaying the second stream
+ // from the same frame it has relayed right before for the first stream.
+ return;
+ }
+ frame_counters_.decoded++;
+ InternalStatsKey key(it->second.stream(),
+ stream_to_sender_.at(it->second.stream()), peer_index);
+ stream_frame_counters_.at(key).decoded++;
+ Timestamp now = Now();
+ StreamCodecInfo used_decoder;
+ used_decoder.codec_name = stats.decoder_name;
+ used_decoder.first_frame_id = frame.id();
+ used_decoder.last_frame_id = frame.id();
+ used_decoder.switched_on_at = now;
+ used_decoder.switched_from_at = now;
+ it->second.OnFrameDecoded(peer_index, now, frame.width(), frame.height(),
+ used_decoder);
+
+ if (options_.report_infra_metrics) {
+ analyzer_stats_.on_frame_decoded_processing_time_ms.AddSample(
+ (Now() - processing_started).ms<double>());
+ }
+}
+
+void DefaultVideoQualityAnalyzer::OnFrameRendered(
+ absl::string_view peer_name,
+ const webrtc::VideoFrame& frame) {
+ Timestamp processing_started = Now();
+ MutexLock lock(&mutex_);
+ RTC_CHECK_EQ(state_, State::kActive)
+ << "DefaultVideoQualityAnalyzer has to be started before use";
+
+ size_t peer_index = peers_->index(peer_name);
+
+ if (frame.id() == VideoFrame::kNotSetId) {
+ frame_counters_.rendered++;
+ unknown_sender_frame_counters_[std::string(peer_name)].rendered++;
+ return;
+ }
+
+ auto frame_it = captured_frames_in_flight_.find(frame.id());
+ if (frame_it == captured_frames_in_flight_.end() ||
+ frame_it->second.HasRenderedTime(peer_index) ||
+ frame_it->second.IsDropped(peer_index)) {
+ // It means this frame was rendered or dropped before, so we can skip it.
+ // It may happen when we have multiple simulcast streams in one track and
+ // received the same frame from two different streams because SFU can't
+ // reliably correlate two simulcast streams and started relaying the second
+ // stream from the same frame it has relayed right before for the first
+ // stream.
+ absl::string_view reason = kSkipRenderedFrameReasonProcessed;
+ if (frame_it != captured_frames_in_flight_.end()) {
+ if (frame_it->second.HasRenderedTime(peer_index)) {
+ reason = kSkipRenderedFrameReasonRendered;
+ } else if (frame_it->second.IsDropped(peer_index)) {
+ reason = kSkipRenderedFrameReasonDropped;
+ }
+ }
+ RTC_LOG(LS_WARNING)
+ << "Peer " << peer_name
+ << "; Received frame out of order: received frame with id "
+ << frame.id() << " which was " << reason << " before";
+ return;
+ }
+
+ // Find corresponding captured frame.
+ FrameInFlight* frame_in_flight = &frame_it->second;
+ absl::optional<VideoFrame> captured_frame = frame_in_flight->frame();
+
+ const size_t stream_index = frame_in_flight->stream();
+ StreamState* state = &stream_states_.at(stream_index);
+ const InternalStatsKey stats_key(stream_index, state->sender(), peer_index);
+
+ // Update frames counters.
+ frame_counters_.rendered++;
+ stream_frame_counters_.at(stats_key).rendered++;
+
+ // Update current frame stats.
+ frame_in_flight->OnFrameRendered(peer_index, Now());
+
+ // After we received frame here we need to check if there are any dropped
+ // frames between this one and last one, that was rendered for this video
+ // stream.
+ int dropped_count = 0;
+ while (!state->IsEmpty(peer_index) &&
+ state->Front(peer_index) != frame.id()) {
+ dropped_count++;
+ uint16_t dropped_frame_id = state->PopFront(peer_index);
+ // Frame with id `dropped_frame_id` was dropped. We need:
+ // 1. Update global and stream frame counters
+ // 2. Extract corresponding frame from `captured_frames_in_flight_`
+ // 3. Send extracted frame to comparison with dropped=true
+ // 4. Cleanup dropped frame
+ frame_counters_.dropped++;
+ stream_frame_counters_.at(stats_key).dropped++;
+
+ auto dropped_frame_it = captured_frames_in_flight_.find(dropped_frame_id);
+ RTC_DCHECK(dropped_frame_it != captured_frames_in_flight_.end());
+ dropped_frame_it->second.MarkDropped(peer_index);
+
+ analyzer_stats_.frames_in_flight_left_count.AddSample(
+ StatsSample(captured_frames_in_flight_.size(), Now()));
+ frames_comparator_.AddComparison(
+ stats_key, /*captured=*/absl::nullopt, /*rendered=*/absl::nullopt,
+ FrameComparisonType::kDroppedFrame,
+ dropped_frame_it->second.GetStatsForPeer(peer_index));
+
+ if (dropped_frame_it->second.HaveAllPeersReceived()) {
+ captured_frames_in_flight_.erase(dropped_frame_it);
+ }
+ }
+ RTC_DCHECK(!state->IsEmpty(peer_index));
+ state->PopFront(peer_index);
+
+ if (state->last_rendered_frame_time(peer_index)) {
+ frame_in_flight->SetPrevFrameRenderedTime(
+ peer_index, state->last_rendered_frame_time(peer_index).value());
+ }
+ state->SetLastRenderedFrameTime(peer_index,
+ frame_in_flight->rendered_time(peer_index));
+ analyzer_stats_.frames_in_flight_left_count.AddSample(
+ StatsSample(captured_frames_in_flight_.size(), Now()));
+ frames_comparator_.AddComparison(
+ stats_key, dropped_count, captured_frame, /*rendered=*/frame,
+ FrameComparisonType::kRegular,
+ frame_in_flight->GetStatsForPeer(peer_index));
+
+ if (frame_it->second.HaveAllPeersReceived()) {
+ captured_frames_in_flight_.erase(frame_it);
+ }
+
+ if (options_.report_infra_metrics) {
+ analyzer_stats_.on_frame_rendered_processing_time_ms.AddSample(
+ (Now() - processing_started).ms<double>());
+ }
+}
+
+void DefaultVideoQualityAnalyzer::OnEncoderError(
+ absl::string_view peer_name,
+ const webrtc::VideoFrame& frame,
+ int32_t error_code) {
+ RTC_LOG(LS_ERROR) << "Encoder error for frame.id=" << frame.id()
+ << ", code=" << error_code;
+}
+
+void DefaultVideoQualityAnalyzer::OnDecoderError(absl::string_view peer_name,
+ uint16_t frame_id,
+ int32_t error_code,
+ const DecoderStats& stats) {
+ RTC_LOG(LS_ERROR) << "Decoder error for frame_id=" << frame_id
+ << ", code=" << error_code;
+
+ Timestamp processing_started = Now();
+ MutexLock lock(&mutex_);
+ RTC_CHECK_EQ(state_, State::kActive)
+ << "DefaultVideoQualityAnalyzer has to be started before use";
+
+ size_t peer_index = peers_->index(peer_name);
+
+ if (frame_id == VideoFrame::kNotSetId) {
+ frame_counters_.failed_to_decode++;
+ unknown_sender_frame_counters_[std::string(peer_name)].failed_to_decode++;
+ return;
+ }
+
+ auto it = captured_frames_in_flight_.find(frame_id);
+ if (it == captured_frames_in_flight_.end() ||
+ it->second.HasDecodeEndTime(peer_index)) {
+ // It means this frame was decoded before, so we can skip it. It may happen
+ // when we have multiple simulcast streams in one track and received
+ // the same frame from two different streams because SFU can't reliably
+ // correlate two simulcast streams and started relaying the second stream
+ // from the same frame it has relayed right before for the first stream.
+ return;
+ }
+ frame_counters_.failed_to_decode++;
+ InternalStatsKey key(it->second.stream(),
+ stream_to_sender_.at(it->second.stream()), peer_index);
+ stream_frame_counters_.at(key).failed_to_decode++;
+ Timestamp now = Now();
+ StreamCodecInfo used_decoder;
+ used_decoder.codec_name = stats.decoder_name;
+ used_decoder.first_frame_id = frame_id;
+ used_decoder.last_frame_id = frame_id;
+ used_decoder.switched_on_at = now;
+ used_decoder.switched_from_at = now;
+ it->second.OnDecoderError(peer_index, used_decoder);
+
+ if (options_.report_infra_metrics) {
+ analyzer_stats_.on_decoder_error_processing_time_ms.AddSample(
+ (Now() - processing_started).ms<double>());
+ }
+}
+
+void DefaultVideoQualityAnalyzer::RegisterParticipantInCall(
+ absl::string_view peer_name) {
+ MutexLock lock(&mutex_);
+ RTC_CHECK(!peers_->HasName(peer_name));
+ size_t new_peer_index = peers_->AddIfAbsent(peer_name);
+
+ // Ensure stats for receiving (for frames from other peers to this one)
+ // streams exists. Since in flight frames will be sent to the new peer
+ // as well. Sending stats (from this peer to others) will be added by
+ // DefaultVideoQualityAnalyzer::OnFrameCaptured.
+ std::vector<std::pair<InternalStatsKey, Timestamp>> stream_started_time;
+ for (auto [stream_index, sender_peer_index] : stream_to_sender_) {
+ InternalStatsKey key(stream_index, sender_peer_index, new_peer_index);
+
+ // To initiate `FrameCounters` for the stream we should pick frame
+ // counters with the same stream index and the same sender's peer index
+ // and any receiver's peer index and copy from its sender side
+ // counters.
+ FrameCounters counters;
+ for (size_t i : peers_->GetPresentIndexes()) {
+ InternalStatsKey prototype_key(stream_index, sender_peer_index, i);
+ auto it = stream_frame_counters_.find(prototype_key);
+ if (it != stream_frame_counters_.end()) {
+ counters.captured = it->second.captured;
+ counters.pre_encoded = it->second.pre_encoded;
+ counters.encoded = it->second.encoded;
+ break;
+ }
+ }
+ // It may happen if we had only one peer before this method was invoked,
+ // then `counters` will be empty. In such case empty `counters` are ok.
+ stream_frame_counters_.insert({key, std::move(counters)});
+
+ stream_started_time.push_back(
+ {key, stream_states_.at(stream_index).stream_started_time()});
+ }
+ frames_comparator_.RegisterParticipantInCall(stream_started_time,
+ start_time_);
+ // Ensure, that frames states are handled correctly
+ // (e.g. dropped frames tracking).
+ for (auto& [stream_index, stream_state] : stream_states_) {
+ stream_state.AddPeer(new_peer_index);
+ }
+ // Register new peer for every frame in flight.
+ // It is guaranteed, that no garbage FrameInFlight objects will stay in
+ // memory because of adding new peer. Even if the new peer won't receive the
+ // frame, the frame will be removed by OnFrameRendered after next frame comes
+ // for the new peer. It is important because FrameInFlight is a large object.
+ for (auto& [frame_id, frame_in_flight] : captured_frames_in_flight_) {
+ frame_in_flight.AddExpectedReceiver(new_peer_index);
+ }
+}
+
+void DefaultVideoQualityAnalyzer::UnregisterParticipantInCall(
+ absl::string_view peer_name) {
+ MutexLock lock(&mutex_);
+ RTC_CHECK(peers_->HasName(peer_name));
+ absl::optional<size_t> peer_index = peers_->RemoveIfPresent(peer_name);
+ RTC_CHECK(peer_index.has_value());
+
+ for (auto& [stream_index, stream_state] : stream_states_) {
+ if (!options_.enable_receive_own_stream &&
+ peer_index == stream_state.sender()) {
+ continue;
+ }
+
+ AddExistingFramesInFlightForStreamToComparator(stream_index, stream_state,
+ *peer_index);
+
+ stream_state.RemovePeer(*peer_index);
+ }
+
+ // Remove peer from every frame in flight. If we removed that last expected
+ // receiver for the frame, then we should removed this frame if it was
+ // already encoded. If frame wasn't encoded, it still will be used by sender
+ // side pipeline, so we can't delete it yet.
+ for (auto it = captured_frames_in_flight_.begin();
+ it != captured_frames_in_flight_.end();) {
+ FrameInFlight& frame_in_flight = it->second;
+ frame_in_flight.RemoveExpectedReceiver(*peer_index);
+ // If frame was fully sent and all receivers received it, then erase it.
+ // It may happen that when we remove FrameInFlight only some Simulcast/SVC
+ // layers were encoded and frame has encoded time, but more layers might be
+ // encoded after removal. In such case it's safe to still remove a frame,
+ // because OnFrameEncoded method will correctly handle the case when there
+ // is no FrameInFlight for the received encoded image.
+ if (frame_in_flight.HasEncodedTime() &&
+ frame_in_flight.HaveAllPeersReceived()) {
+ it = captured_frames_in_flight_.erase(it);
+ } else {
+ it++;
+ }
+ }
+}
+
+void DefaultVideoQualityAnalyzer::Stop() {
+ std::map<InternalStatsKey, Timestamp> last_rendered_frame_times;
+ {
+ MutexLock lock(&mutex_);
+ if (state_ == State::kStopped) {
+ return;
+ }
+ RTC_CHECK_EQ(state_, State::kActive)
+ << "DefaultVideoQualityAnalyzer has to be started before use";
+
+ state_ = State::kStopped;
+
+ // Add the amount of frames in flight to the analyzer stats before all left
+ // frames in flight will be sent to the `frames_compartor_`.
+ analyzer_stats_.frames_in_flight_left_count.AddSample(
+ StatsSample(captured_frames_in_flight_.size(), Now()));
+
+ for (auto& state_entry : stream_states_) {
+ const size_t stream_index = state_entry.first;
+ StreamState& stream_state = state_entry.second;
+
+ // Populate `last_rendered_frame_times` map for all peers that were met in
+ // call, not only for the currently presented ones.
+ for (size_t peer_index : peers_->GetAllIndexes()) {
+ if (peer_index == stream_state.sender() &&
+ !options_.enable_receive_own_stream) {
+ continue;
+ }
+
+ InternalStatsKey stats_key(stream_index, stream_state.sender(),
+ peer_index);
+
+ // If there are no freezes in the call we have to report
+ // time_between_freezes_ms as call duration and in such case
+ // `stream_last_freeze_end_time` for this stream will be `start_time_`.
+ // If there is freeze, then we need add time from last rendered frame
+ // to last freeze end as time between freezes.
+ if (stream_state.last_rendered_frame_time(peer_index)) {
+ last_rendered_frame_times.emplace(
+ stats_key,
+ stream_state.last_rendered_frame_time(peer_index).value());
+ }
+ }
+
+ // Push left frame in flight for analysis for the peers that are still in
+ // the call.
+ for (size_t peer_index : peers_->GetPresentIndexes()) {
+ if (peer_index == stream_state.sender() &&
+ !options_.enable_receive_own_stream) {
+ continue;
+ }
+
+ AddExistingFramesInFlightForStreamToComparator(
+ stream_index, stream_state, peer_index);
+ }
+ }
+ }
+ frames_comparator_.Stop(last_rendered_frame_times);
+
+ // Perform final Metrics update. On this place analyzer is stopped and no one
+ // holds any locks.
+ {
+ MutexLock lock(&mutex_);
+ FramesComparatorStats frames_comparator_stats =
+ frames_comparator_.frames_comparator_stats();
+ analyzer_stats_.comparisons_queue_size =
+ std::move(frames_comparator_stats.comparisons_queue_size);
+ analyzer_stats_.comparisons_done = frames_comparator_stats.comparisons_done;
+ analyzer_stats_.cpu_overloaded_comparisons_done =
+ frames_comparator_stats.cpu_overloaded_comparisons_done;
+ analyzer_stats_.memory_overloaded_comparisons_done =
+ frames_comparator_stats.memory_overloaded_comparisons_done;
+ }
+ ReportResults();
+}
+
+std::string DefaultVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) {
+ MutexLock lock1(&mutex_);
+ auto it = captured_frames_in_flight_.find(frame_id);
+ if (it != captured_frames_in_flight_.end()) {
+ return streams_.name(it->second.stream());
+ }
+ for (auto hist_it = stream_to_frame_id_history_.begin();
+ hist_it != stream_to_frame_id_history_.end(); ++hist_it) {
+ auto hist_set_it = hist_it->second.find(frame_id);
+ if (hist_set_it != hist_it->second.end()) {
+ return streams_.name(hist_it->first);
+ }
+ }
+ RTC_CHECK(false) << "Unknown frame_id=" << frame_id;
+}
+
+std::set<StatsKey> DefaultVideoQualityAnalyzer::GetKnownVideoStreams() const {
+ MutexLock lock(&mutex_);
+ std::set<StatsKey> out;
+ for (auto& item : frames_comparator_.stream_stats()) {
+ RTC_LOG(LS_INFO) << item.first.ToString() << " ==> "
+ << ToStatsKey(item.first).ToString();
+ out.insert(ToStatsKey(item.first));
+ }
+ return out;
+}
+
+VideoStreamsInfo DefaultVideoQualityAnalyzer::GetKnownStreams() const {
+ MutexLock lock(&mutex_);
+ std::map<std::string, std::string> stream_to_sender;
+ std::map<std::string, std::set<std::string>> sender_to_streams;
+ std::map<std::string, std::set<std::string>> stream_to_receivers;
+
+ for (auto& item : frames_comparator_.stream_stats()) {
+ const std::string& stream_label = streams_.name(item.first.stream);
+ const std::string& sender = peers_->name(item.first.sender);
+ const std::string& receiver = peers_->name(item.first.receiver);
+ RTC_LOG(LS_INFO) << item.first.ToString() << " ==> "
+ << "stream=" << stream_label << "; sender=" << sender
+ << "; receiver=" << receiver;
+ stream_to_sender.emplace(stream_label, sender);
+ auto streams_it = sender_to_streams.find(sender);
+ if (streams_it != sender_to_streams.end()) {
+ streams_it->second.emplace(stream_label);
+ } else {
+ sender_to_streams.emplace(sender, std::set<std::string>{stream_label});
+ }
+ auto receivers_it = stream_to_receivers.find(stream_label);
+ if (receivers_it != stream_to_receivers.end()) {
+ receivers_it->second.emplace(receiver);
+ } else {
+ stream_to_receivers.emplace(stream_label,
+ std::set<std::string>{receiver});
+ }
+ }
+
+ return VideoStreamsInfo(std::move(stream_to_sender),
+ std::move(sender_to_streams),
+ std::move(stream_to_receivers));
+}
+
+FrameCounters DefaultVideoQualityAnalyzer::GetGlobalCounters() const {
+ MutexLock lock(&mutex_);
+ return frame_counters_;
+}
+
+std::map<std::string, FrameCounters>
+DefaultVideoQualityAnalyzer::GetUnknownSenderFrameCounters() const {
+ MutexLock lock(&mutex_);
+ return unknown_sender_frame_counters_;
+}
+
+std::map<StatsKey, FrameCounters>
+DefaultVideoQualityAnalyzer::GetPerStreamCounters() const {
+ MutexLock lock(&mutex_);
+ std::map<StatsKey, FrameCounters> out;
+ for (auto& item : stream_frame_counters_) {
+ out.emplace(ToStatsKey(item.first), item.second);
+ }
+ return out;
+}
+
+std::map<StatsKey, StreamStats> DefaultVideoQualityAnalyzer::GetStats() const {
+ MutexLock lock1(&mutex_);
+ std::map<StatsKey, StreamStats> out;
+ for (auto& item : frames_comparator_.stream_stats()) {
+ out.emplace(ToStatsKey(item.first), item.second);
+ }
+ return out;
+}
+
+AnalyzerStats DefaultVideoQualityAnalyzer::GetAnalyzerStats() const {
+ MutexLock lock(&mutex_);
+ return analyzer_stats_;
+}
+
+uint16_t DefaultVideoQualityAnalyzer::GetNextFrameId() {
+ uint16_t frame_id = next_frame_id_++;
+ if (next_frame_id_ == VideoFrame::kNotSetId) {
+ next_frame_id_ = 1;
+ }
+ return frame_id;
+}
+
+void DefaultVideoQualityAnalyzer::
+ AddExistingFramesInFlightForStreamToComparator(size_t stream_index,
+ StreamState& stream_state,
+ size_t peer_index) {
+ InternalStatsKey stats_key(stream_index, stream_state.sender(), peer_index);
+
+ // Add frames in flight for this stream into frames comparator.
+ // Frames in flight were not rendered, so they won't affect stream's
+ // last rendered frame time.
+ while (!stream_state.IsEmpty(peer_index)) {
+ uint16_t frame_id = stream_state.PopFront(peer_index);
+ auto it = captured_frames_in_flight_.find(frame_id);
+ RTC_DCHECK(it != captured_frames_in_flight_.end());
+ FrameInFlight& frame = it->second;
+
+ frames_comparator_.AddComparison(stats_key, /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kFrameInFlight,
+ frame.GetStatsForPeer(peer_index));
+ }
+}
+
+void DefaultVideoQualityAnalyzer::ReportResults() {
+ MutexLock lock(&mutex_);
+ for (auto& item : frames_comparator_.stream_stats()) {
+ ReportResults(item.first, item.second,
+ stream_frame_counters_.at(item.first));
+ }
+ // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey.
+ metrics_logger_->LogSingleValueMetric(
+ "cpu_usage_%", test_label_, GetCpuUsagePercent(), Unit::kUnitless,
+ ImprovementDirection::kSmallerIsBetter,
+ {{MetricMetadataKey::kExperimentalTestNameMetadataKey, test_label_}});
+ LogFrameCounters("Global", frame_counters_);
+ if (!unknown_sender_frame_counters_.empty()) {
+ RTC_LOG(LS_INFO) << "Received frame counters with unknown frame id:";
+ for (const auto& [peer_name, frame_counters] :
+ unknown_sender_frame_counters_) {
+ LogFrameCounters(peer_name, frame_counters);
+ }
+ }
+ RTC_LOG(LS_INFO) << "Received frame counters per stream:";
+ for (const auto& [stats_key, stream_stats] :
+ frames_comparator_.stream_stats()) {
+ LogFrameCounters(ToStatsKey(stats_key).ToString(),
+ stream_frame_counters_.at(stats_key));
+ LogStreamInternalStats(ToStatsKey(stats_key).ToString(), stream_stats,
+ start_time_);
+ }
+ if (!analyzer_stats_.comparisons_queue_size.IsEmpty()) {
+ RTC_LOG(LS_INFO) << "comparisons_queue_size min="
+ << analyzer_stats_.comparisons_queue_size.GetMin()
+ << "; max="
+ << analyzer_stats_.comparisons_queue_size.GetMax()
+ << "; 99%="
+ << analyzer_stats_.comparisons_queue_size.GetPercentile(
+ 0.99);
+ }
+ RTC_LOG(LS_INFO) << "comparisons_done=" << analyzer_stats_.comparisons_done;
+ RTC_LOG(LS_INFO) << "cpu_overloaded_comparisons_done="
+ << analyzer_stats_.cpu_overloaded_comparisons_done;
+ RTC_LOG(LS_INFO) << "memory_overloaded_comparisons_done="
+ << analyzer_stats_.memory_overloaded_comparisons_done;
+ if (options_.report_infra_metrics) {
+ metrics_logger_->LogMetric("comparisons_queue_size", test_label_,
+ analyzer_stats_.comparisons_queue_size,
+ Unit::kCount,
+ ImprovementDirection::kSmallerIsBetter);
+ metrics_logger_->LogMetric("frames_in_flight_left_count", test_label_,
+ analyzer_stats_.frames_in_flight_left_count,
+ Unit::kCount,
+ ImprovementDirection::kSmallerIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "comparisons_done", test_label_, analyzer_stats_.comparisons_done,
+ Unit::kCount, ImprovementDirection::kNeitherIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "cpu_overloaded_comparisons_done", test_label_,
+ analyzer_stats_.cpu_overloaded_comparisons_done, Unit::kCount,
+ ImprovementDirection::kNeitherIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "memory_overloaded_comparisons_done", test_label_,
+ analyzer_stats_.memory_overloaded_comparisons_done, Unit::kCount,
+ ImprovementDirection::kNeitherIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "test_duration", test_label_, (Now() - start_time_).ms(),
+ Unit::kMilliseconds, ImprovementDirection::kNeitherIsBetter);
+
+ metrics_logger_->LogMetric(
+ "on_frame_captured_processing_time_ms", test_label_,
+ analyzer_stats_.on_frame_captured_processing_time_ms,
+ Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter);
+ metrics_logger_->LogMetric(
+ "on_frame_pre_encode_processing_time_ms", test_label_,
+ analyzer_stats_.on_frame_pre_encode_processing_time_ms,
+ Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter);
+ metrics_logger_->LogMetric(
+ "on_frame_encoded_processing_time_ms", test_label_,
+ analyzer_stats_.on_frame_encoded_processing_time_ms,
+ Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter);
+ metrics_logger_->LogMetric(
+ "on_frame_pre_decode_processing_time_ms", test_label_,
+ analyzer_stats_.on_frame_pre_decode_processing_time_ms,
+ Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter);
+ metrics_logger_->LogMetric(
+ "on_frame_decoded_processing_time_ms", test_label_,
+ analyzer_stats_.on_frame_decoded_processing_time_ms,
+ Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter);
+ metrics_logger_->LogMetric(
+ "on_frame_rendered_processing_time_ms", test_label_,
+ analyzer_stats_.on_frame_rendered_processing_time_ms,
+ Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter);
+ metrics_logger_->LogMetric(
+ "on_decoder_error_processing_time_ms", test_label_,
+ analyzer_stats_.on_decoder_error_processing_time_ms,
+ Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter);
+ }
+}
+
+void DefaultVideoQualityAnalyzer::ReportResults(
+ const InternalStatsKey& key,
+ const StreamStats& stats,
+ const FrameCounters& frame_counters) {
+ TimeDelta test_duration = Now() - start_time_;
+ std::string test_case_name = GetTestCaseName(ToMetricName(key));
+ // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey.
+ std::map<std::string, std::string> metric_metadata{
+ {MetricMetadataKey::kPeerMetadataKey, peers_->name(key.sender)},
+ {MetricMetadataKey::kVideoStreamMetadataKey, streams_.name(key.stream)},
+ {MetricMetadataKey::kSenderMetadataKey, peers_->name(key.sender)},
+ {MetricMetadataKey::kReceiverMetadataKey, peers_->name(key.receiver)},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_label_}};
+
+ double sum_squared_interframe_delays_secs = 0;
+ Timestamp video_start_time = Timestamp::PlusInfinity();
+ Timestamp video_end_time = Timestamp::MinusInfinity();
+ for (const SamplesStatsCounter::StatsSample& sample :
+ stats.time_between_rendered_frames_ms.GetTimedSamples()) {
+ double interframe_delay_ms = sample.value;
+ const double interframe_delays_secs = interframe_delay_ms / 1000.0;
+ // Sum of squared inter frame intervals is used to calculate the harmonic
+ // frame rate metric. The metric aims to reflect overall experience related
+ // to smoothness of video playback and includes both freezes and pauses.
+ sum_squared_interframe_delays_secs +=
+ interframe_delays_secs * interframe_delays_secs;
+ if (sample.time < video_start_time) {
+ video_start_time = sample.time;
+ }
+ if (sample.time > video_end_time) {
+ video_end_time = sample.time;
+ }
+ }
+ double harmonic_framerate_fps = 0;
+ TimeDelta video_duration = video_end_time - video_start_time;
+ if (sum_squared_interframe_delays_secs > 0.0 && video_duration.IsFinite()) {
+ harmonic_framerate_fps =
+ video_duration.seconds<double>() / sum_squared_interframe_delays_secs;
+ }
+
+ metrics_logger_->LogMetric(
+ "psnr_dB", test_case_name, stats.psnr, Unit::kUnitless,
+ ImprovementDirection::kBiggerIsBetter, metric_metadata);
+ metrics_logger_->LogMetric(
+ "ssim", test_case_name, stats.ssim, Unit::kUnitless,
+ ImprovementDirection::kBiggerIsBetter, metric_metadata);
+ metrics_logger_->LogMetric("transport_time", test_case_name,
+ stats.transport_time_ms, Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter,
+ metric_metadata);
+ metrics_logger_->LogMetric(
+ "total_delay_incl_transport", test_case_name,
+ stats.total_delay_incl_transport_ms, Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter, metric_metadata);
+ metrics_logger_->LogMetric(
+ "time_between_rendered_frames", test_case_name,
+ stats.time_between_rendered_frames_ms, Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter, metric_metadata);
+ metrics_logger_->LogSingleValueMetric(
+ "harmonic_framerate", test_case_name, harmonic_framerate_fps,
+ Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata);
+ metrics_logger_->LogSingleValueMetric(
+ "encode_frame_rate", test_case_name,
+ stats.encode_frame_rate.IsEmpty()
+ ? 0
+ : stats.encode_frame_rate.GetEventsPerSecond(),
+ Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata);
+ metrics_logger_->LogMetric(
+ "encode_time", test_case_name, stats.encode_time_ms, Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter, metric_metadata);
+ metrics_logger_->LogMetric("time_between_freezes", test_case_name,
+ stats.time_between_freezes_ms, Unit::kMilliseconds,
+ ImprovementDirection::kBiggerIsBetter,
+ metric_metadata);
+ metrics_logger_->LogMetric("freeze_time_ms", test_case_name,
+ stats.freeze_time_ms, Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter,
+ metric_metadata);
+ metrics_logger_->LogMetric(
+ "pixels_per_frame", test_case_name, stats.resolution_of_decoded_frame,
+ Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata);
+ metrics_logger_->LogSingleValueMetric(
+ "min_psnr_dB", test_case_name,
+ stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), Unit::kUnitless,
+ ImprovementDirection::kBiggerIsBetter, metric_metadata);
+ metrics_logger_->LogMetric(
+ "decode_time", test_case_name, stats.decode_time_ms, Unit::kMilliseconds,
+ ImprovementDirection::kSmallerIsBetter, metric_metadata);
+ metrics_logger_->LogMetric(
+ "receive_to_render_time", test_case_name, stats.receive_to_render_time_ms,
+ Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter,
+ metric_metadata);
+ metrics_logger_->LogSingleValueMetric(
+ "dropped_frames", test_case_name, frame_counters.dropped, Unit::kCount,
+ ImprovementDirection::kSmallerIsBetter, metric_metadata);
+ metrics_logger_->LogSingleValueMetric(
+ "frames_in_flight", test_case_name,
+ frame_counters.captured - frame_counters.rendered -
+ frame_counters.dropped,
+ Unit::kCount, ImprovementDirection::kSmallerIsBetter, metric_metadata);
+ metrics_logger_->LogSingleValueMetric(
+ "rendered_frames", test_case_name, frame_counters.rendered, Unit::kCount,
+ ImprovementDirection::kBiggerIsBetter, metric_metadata);
+ metrics_logger_->LogMetric(
+ "max_skipped", test_case_name, stats.skipped_between_rendered,
+ Unit::kCount, ImprovementDirection::kSmallerIsBetter, metric_metadata);
+ metrics_logger_->LogMetric(
+ "target_encode_bitrate", test_case_name,
+ stats.target_encode_bitrate / 1000, Unit::kKilobitsPerSecond,
+ ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ for (const auto& [spatial_layer, qp] : stats.spatial_layers_qp) {
+ std::map<std::string, std::string> qp_metadata = metric_metadata;
+ qp_metadata[MetricMetadataKey::kSpatialLayerMetadataKey] =
+ std::to_string(spatial_layer);
+ metrics_logger_->LogMetric("qp_sl" + std::to_string(spatial_layer),
+ test_case_name, qp, Unit::kUnitless,
+ ImprovementDirection::kSmallerIsBetter,
+ std::move(qp_metadata));
+ }
+ metrics_logger_->LogSingleValueMetric(
+ "actual_encode_bitrate", test_case_name,
+ static_cast<double>(stats.total_encoded_images_payload) /
+ test_duration.seconds<double>() * kBitsInByte / 1000,
+ Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter,
+ metric_metadata);
+
+ if (options_.report_detailed_frame_stats) {
+ metrics_logger_->LogSingleValueMetric(
+ "capture_frame_rate", test_case_name,
+ stats.capture_frame_rate.IsEmpty()
+ ? 0
+ : stats.capture_frame_rate.GetEventsPerSecond(),
+ Unit::kHertz, ImprovementDirection::kBiggerIsBetter, metric_metadata);
+ metrics_logger_->LogSingleValueMetric(
+ "num_encoded_frames", test_case_name, frame_counters.encoded,
+ Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata);
+ metrics_logger_->LogSingleValueMetric(
+ "num_decoded_frames", test_case_name, frame_counters.decoded,
+ Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata);
+ metrics_logger_->LogSingleValueMetric(
+ "num_send_key_frames", test_case_name, stats.num_send_key_frames,
+ Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata);
+ metrics_logger_->LogSingleValueMetric(
+ "num_recv_key_frames", test_case_name, stats.num_recv_key_frames,
+ Unit::kCount, ImprovementDirection::kBiggerIsBetter, metric_metadata);
+
+ metrics_logger_->LogMetric("recv_key_frame_size_bytes", test_case_name,
+ stats.recv_key_frame_size_bytes, Unit::kCount,
+ ImprovementDirection::kBiggerIsBetter,
+ metric_metadata);
+ metrics_logger_->LogMetric("recv_delta_frame_size_bytes", test_case_name,
+ stats.recv_delta_frame_size_bytes, Unit::kCount,
+ ImprovementDirection::kBiggerIsBetter,
+ metric_metadata);
+ }
+}
+
+std::string DefaultVideoQualityAnalyzer::GetTestCaseName(
+ const std::string& stream_label) const {
+ return test_label_ + "/" + stream_label;
+}
+
+Timestamp DefaultVideoQualityAnalyzer::Now() {
+ return clock_->CurrentTime();
+}
+
+StatsKey DefaultVideoQualityAnalyzer::ToStatsKey(
+ const InternalStatsKey& key) const {
+ return StatsKey(streams_.name(key.stream), peers_->name(key.receiver));
+}
+
+std::string DefaultVideoQualityAnalyzer::ToMetricName(
+ const InternalStatsKey& key) const {
+ const std::string& stream_label = streams_.name(key.stream);
+ if (peers_->GetKnownSize() <= 2 && key.sender != key.receiver) {
+ // TODO(titovartem): remove this special case.
+ return stream_label;
+ }
+ rtc::StringBuilder out;
+ out << stream_label << "_" << peers_->name(key.sender) << "_"
+ << peers_->name(key.receiver);
+ return out.str();
+}
+
+double DefaultVideoQualityAnalyzer::GetCpuUsagePercent() {
+ return cpu_measurer_.GetCpuUsagePercent();
+}
+
+std::map<std::string, std::vector<uint16_t>>
+DefaultVideoQualityAnalyzer::GetStreamFrames() const {
+ MutexLock lock(&mutex_);
+ std::map<std::string, std::vector<uint16_t>> out;
+ for (auto entry_it : stream_to_frame_id_full_history_) {
+ out.insert({streams_.name(entry_it.first), entry_it.second});
+ }
+ return out;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
new file mode 100644
index 0000000000..b67e5a0147
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
@@ -0,0 +1,197 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_H_
+
+#include <atomic>
+#include <cstdint>
+#include <deque>
+#include <map>
+#include <memory>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "api/array_view.h"
+#include "api/test/metrics/metrics_logger.h"
+#include "api/test/video_quality_analyzer_interface.h"
+#include "api/units/data_size.h"
+#include "api/units/timestamp.h"
+#include "api/video/encoded_image.h"
+#include "api/video/video_frame.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "rtc_base/thread_annotations.h"
+#include "system_wrappers/include/clock.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h"
+#include "test/pc/e2e/analyzer/video/names_collection.h"
+
+namespace webrtc {
+
+class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
+ public:
+ DefaultVideoQualityAnalyzer(webrtc::Clock* clock,
+ test::MetricsLogger* metrics_logger,
+ DefaultVideoQualityAnalyzerOptions options = {});
+ ~DefaultVideoQualityAnalyzer() override;
+
+ void Start(std::string test_case_name,
+ rtc::ArrayView<const std::string> peer_names,
+ int max_threads_count) override;
+ uint16_t OnFrameCaptured(absl::string_view peer_name,
+ const std::string& stream_label,
+ const VideoFrame& frame) override;
+ void OnFramePreEncode(absl::string_view peer_name,
+ const VideoFrame& frame) override;
+ void OnFrameEncoded(absl::string_view peer_name,
+ uint16_t frame_id,
+ const EncodedImage& encoded_image,
+ const EncoderStats& stats,
+ bool discarded) override;
+ void OnFrameDropped(absl::string_view peer_name,
+ EncodedImageCallback::DropReason reason) override;
+ void OnFramePreDecode(absl::string_view peer_name,
+ uint16_t frame_id,
+ const EncodedImage& input_image) override;
+ void OnFrameDecoded(absl::string_view peer_name,
+ const VideoFrame& frame,
+ const DecoderStats& stats) override;
+ void OnFrameRendered(absl::string_view peer_name,
+ const VideoFrame& frame) override;
+ void OnEncoderError(absl::string_view peer_name,
+ const VideoFrame& frame,
+ int32_t error_code) override;
+ void OnDecoderError(absl::string_view peer_name,
+ uint16_t frame_id,
+ int32_t error_code,
+ const DecoderStats& stats) override;
+
+ void RegisterParticipantInCall(absl::string_view peer_name) override;
+ void UnregisterParticipantInCall(absl::string_view peer_name) override;
+
+ void Stop() override;
+ std::string GetStreamLabel(uint16_t frame_id) override;
+ void OnStatsReports(
+ absl::string_view pc_label,
+ const rtc::scoped_refptr<const RTCStatsReport>& report) override {}
+
+ // Returns set of stream labels, that were met during test call.
+ std::set<StatsKey> GetKnownVideoStreams() const;
+ VideoStreamsInfo GetKnownStreams() const;
+ FrameCounters GetGlobalCounters() const;
+ // Returns frame counter for frames received without frame id set.
+ std::map<std::string, FrameCounters> GetUnknownSenderFrameCounters() const;
+ // Returns frame counter per stream label. Valid stream labels can be obtained
+ // by calling GetKnownVideoStreams()
+ std::map<StatsKey, FrameCounters> GetPerStreamCounters() const;
+ // Returns video quality stats per stream label. Valid stream labels can be
+ // obtained by calling GetKnownVideoStreams()
+ std::map<StatsKey, StreamStats> GetStats() const;
+ AnalyzerStats GetAnalyzerStats() const;
+ double GetCpuUsagePercent();
+
+ // Returns mapping from the stream label to the history of frames that were
+ // met in this stream in the order as they were captured.
+ std::map<std::string, std::vector<uint16_t>> GetStreamFrames() const;
+
+ private:
+ enum State { kNew, kActive, kStopped };
+
+ // Returns next frame id to use. Frame ID can't be `VideoFrame::kNotSetId`,
+ // because this value is reserved by `VideoFrame` as "ID not set".
+ uint16_t GetNextFrameId() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+
+ void AddExistingFramesInFlightForStreamToComparator(size_t stream_index,
+ StreamState& stream_state,
+ size_t peer_index)
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+
+ // Report results for all metrics for all streams.
+ void ReportResults();
+ void ReportResults(const InternalStatsKey& key,
+ const StreamStats& stats,
+ const FrameCounters& frame_counters)
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+ // Returns name of current test case for reporting.
+ std::string GetTestCaseName(const std::string& stream_label) const;
+ Timestamp Now();
+ StatsKey ToStatsKey(const InternalStatsKey& key) const
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+ // Returns string representation of stats key for metrics naming. Used for
+ // backward compatibility by metrics naming for 2 peers cases.
+ std::string ToMetricName(const InternalStatsKey& key) const
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+
+ static const uint16_t kStartingFrameId = 1;
+
+ const DefaultVideoQualityAnalyzerOptions options_;
+ webrtc::Clock* const clock_;
+ test::MetricsLogger* const metrics_logger_;
+
+ std::string test_label_;
+
+ mutable Mutex mutex_;
+ uint16_t next_frame_id_ RTC_GUARDED_BY(mutex_) = kStartingFrameId;
+ std::unique_ptr<NamesCollection> peers_ RTC_GUARDED_BY(mutex_);
+ State state_ RTC_GUARDED_BY(mutex_) = State::kNew;
+ Timestamp start_time_ RTC_GUARDED_BY(mutex_) = Timestamp::MinusInfinity();
+ // Mapping from stream label to unique size_t value to use in stats and avoid
+ // extra string copying.
+ NamesCollection streams_ RTC_GUARDED_BY(mutex_);
+ // Frames that were captured by all streams and still aren't rendered on
+ // receivers or deemed dropped. Frame with id X can be removed from this map
+ // if:
+ // 1. The frame with id X was received in OnFrameRendered by all expected
+ // receivers.
+ // 2. The frame with id Y > X was received in OnFrameRendered by all expected
+ // receivers.
+ // 3. Next available frame id for newly captured frame is X
+ // 4. There too many frames in flight for current video stream and X is the
+ // oldest frame id in this stream. In such case only the frame content
+ // will be removed, but the map entry will be preserved.
+ std::map<uint16_t, FrameInFlight> captured_frames_in_flight_
+ RTC_GUARDED_BY(mutex_);
+ // Global frames count for all video streams.
+ FrameCounters frame_counters_ RTC_GUARDED_BY(mutex_);
+ // Frame counters for received frames without video frame id set.
+ // Map from peer name to the frame counters.
+ std::map<std::string, FrameCounters> unknown_sender_frame_counters_
+ RTC_GUARDED_BY(mutex_);
+ // Frame counters per each stream per each receiver.
+ std::map<InternalStatsKey, FrameCounters> stream_frame_counters_
+ RTC_GUARDED_BY(mutex_);
+ // Map from stream index in `streams_` to its StreamState.
+ std::map<size_t, StreamState> stream_states_ RTC_GUARDED_BY(mutex_);
+ // Map from stream index in `streams_` to sender peer index in `peers_`.
+ std::map<size_t, size_t> stream_to_sender_ RTC_GUARDED_BY(mutex_);
+
+ // Stores history mapping between stream index in `streams_` and frame ids.
+ // Updated when frame id overlap. It required to properly return stream label
+ // after 1st frame from simulcast streams was already rendered and last is
+ // still encoding.
+ std::map<size_t, std::set<uint16_t>> stream_to_frame_id_history_
+ RTC_GUARDED_BY(mutex_);
+ // Map from stream index to the list of frames as they were met in the stream.
+ std::map<size_t, std::vector<uint16_t>> stream_to_frame_id_full_history_
+ RTC_GUARDED_BY(mutex_);
+ AnalyzerStats analyzer_stats_ RTC_GUARDED_BY(mutex_);
+
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer_;
+ DefaultVideoQualityAnalyzerFramesComparator frames_comparator_;
+};
+
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.cc
new file mode 100644
index 0000000000..847c9f09a6
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.cc
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h"
+
+#include "rtc_base/cpu_time.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "rtc_base/system_time.h"
+
+namespace webrtc {
+
+void DefaultVideoQualityAnalyzerCpuMeasurer::StartMeasuringCpuProcessTime() {
+ MutexLock lock(&mutex_);
+ cpu_time_ -= rtc::GetProcessCpuTimeNanos();
+ wallclock_time_ -= rtc::SystemTimeNanos();
+}
+
+void DefaultVideoQualityAnalyzerCpuMeasurer::StopMeasuringCpuProcessTime() {
+ MutexLock lock(&mutex_);
+ cpu_time_ += rtc::GetProcessCpuTimeNanos();
+ wallclock_time_ += rtc::SystemTimeNanos();
+}
+
+void DefaultVideoQualityAnalyzerCpuMeasurer::StartExcludingCpuThreadTime() {
+ MutexLock lock(&mutex_);
+ cpu_time_ += rtc::GetThreadCpuTimeNanos();
+}
+
+void DefaultVideoQualityAnalyzerCpuMeasurer::StopExcludingCpuThreadTime() {
+ MutexLock lock(&mutex_);
+ cpu_time_ -= rtc::GetThreadCpuTimeNanos();
+}
+
+double DefaultVideoQualityAnalyzerCpuMeasurer::GetCpuUsagePercent() {
+ MutexLock lock(&mutex_);
+ return static_cast<double>(cpu_time_) / wallclock_time_ * 100.0;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h
new file mode 100644
index 0000000000..dd9fa07af2
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_CPU_MEASURER_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_CPU_MEASURER_H_
+
+#include "rtc_base/synchronization/mutex.h"
+
+namespace webrtc {
+
+// This class is thread safe.
+class DefaultVideoQualityAnalyzerCpuMeasurer {
+ public:
+ double GetCpuUsagePercent();
+
+ void StartMeasuringCpuProcessTime();
+ void StopMeasuringCpuProcessTime();
+ void StartExcludingCpuThreadTime();
+ void StopExcludingCpuThreadTime();
+
+ private:
+ Mutex mutex_;
+ int64_t cpu_time_ RTC_GUARDED_BY(mutex_) = 0;
+ int64_t wallclock_time_ RTC_GUARDED_BY(mutex_) = 0;
+};
+
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_CPU_MEASURER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc
new file mode 100644
index 0000000000..df34dadaf0
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.cc
@@ -0,0 +1,209 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h"
+
+#include <utility>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/units/data_size.h"
+#include "api/units/timestamp.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_frame_type.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
+
+namespace webrtc {
+namespace {
+
+template <typename T>
+absl::optional<T> MaybeGetValue(const std::map<size_t, T>& map, size_t key) {
+ auto it = map.find(key);
+ if (it == map.end()) {
+ return absl::nullopt;
+ }
+ return it->second;
+}
+
+} // namespace
+
+FrameInFlight::FrameInFlight(size_t stream,
+ VideoFrame frame,
+ Timestamp captured_time,
+ std::set<size_t> expected_receivers)
+ : stream_(stream),
+ expected_receivers_(std::move(expected_receivers)),
+ frame_(std::move(frame)),
+ captured_time_(captured_time) {}
+
+bool FrameInFlight::RemoveFrame() {
+ if (!frame_) {
+ return false;
+ }
+ frame_ = absl::nullopt;
+ return true;
+}
+
+void FrameInFlight::SetFrameId(uint16_t id) {
+ if (frame_) {
+ frame_->set_id(id);
+ }
+ frame_id_ = id;
+}
+
+std::vector<size_t> FrameInFlight::GetPeersWhichDidntReceive() const {
+ std::vector<size_t> out;
+ for (size_t peer : expected_receivers_) {
+ auto it = receiver_stats_.find(peer);
+ if (it == receiver_stats_.end() ||
+ (!it->second.dropped && it->second.rendered_time.IsInfinite())) {
+ out.push_back(peer);
+ }
+ }
+ return out;
+}
+
+bool FrameInFlight::HaveAllPeersReceived() const {
+ for (size_t peer : expected_receivers_) {
+ auto it = receiver_stats_.find(peer);
+ if (it == receiver_stats_.end()) {
+ return false;
+ }
+
+ if (!it->second.dropped && it->second.rendered_time.IsInfinite()) {
+ return false;
+ }
+ }
+ return true;
+}
+
+void FrameInFlight::OnFrameEncoded(webrtc::Timestamp time,
+ VideoFrameType frame_type,
+ DataSize encoded_image_size,
+ uint32_t target_encode_bitrate,
+ int spatial_layer,
+ int qp,
+ StreamCodecInfo used_encoder) {
+ encoded_time_ = time;
+ frame_type_ = frame_type;
+ encoded_image_size_ = encoded_image_size;
+ target_encode_bitrate_ += target_encode_bitrate;
+ spatial_layers_qp_[spatial_layer].AddSample(SamplesStatsCounter::StatsSample{
+ .value = static_cast<double>(qp), .time = time});
+ // Update used encoder info. If simulcast/SVC is used, this method can
+ // be called multiple times, in such case we should preserve the value
+ // of `used_encoder_.switched_on_at` from the first invocation as the
+ // smallest one.
+ Timestamp encoder_switched_on_at = used_encoder_.has_value()
+ ? used_encoder_->switched_on_at
+ : Timestamp::PlusInfinity();
+ RTC_DCHECK(used_encoder.switched_on_at.IsFinite());
+ RTC_DCHECK(used_encoder.switched_from_at.IsFinite());
+ used_encoder_ = used_encoder;
+ if (encoder_switched_on_at < used_encoder_->switched_on_at) {
+ used_encoder_->switched_on_at = encoder_switched_on_at;
+ }
+}
+
+void FrameInFlight::OnFramePreDecode(size_t peer,
+ webrtc::Timestamp received_time,
+ webrtc::Timestamp decode_start_time,
+ VideoFrameType frame_type,
+ DataSize encoded_image_size) {
+ receiver_stats_[peer].received_time = received_time;
+ receiver_stats_[peer].decode_start_time = decode_start_time;
+ receiver_stats_[peer].frame_type = frame_type;
+ receiver_stats_[peer].encoded_image_size = encoded_image_size;
+}
+
+bool FrameInFlight::HasReceivedTime(size_t peer) const {
+ auto it = receiver_stats_.find(peer);
+ if (it == receiver_stats_.end()) {
+ return false;
+ }
+ return it->second.received_time.IsFinite();
+}
+
+void FrameInFlight::OnFrameDecoded(size_t peer,
+ webrtc::Timestamp time,
+ int width,
+ int height,
+ const StreamCodecInfo& used_decoder) {
+ receiver_stats_[peer].decode_end_time = time;
+ receiver_stats_[peer].used_decoder = used_decoder;
+ receiver_stats_[peer].decoded_frame_width = width;
+ receiver_stats_[peer].decoded_frame_height = height;
+}
+
+void FrameInFlight::OnDecoderError(size_t peer,
+ const StreamCodecInfo& used_decoder) {
+ receiver_stats_[peer].decoder_failed = true;
+ receiver_stats_[peer].used_decoder = used_decoder;
+}
+
+bool FrameInFlight::HasDecodeEndTime(size_t peer) const {
+ auto it = receiver_stats_.find(peer);
+ if (it == receiver_stats_.end()) {
+ return false;
+ }
+ return it->second.decode_end_time.IsFinite();
+}
+
+void FrameInFlight::OnFrameRendered(size_t peer, webrtc::Timestamp time) {
+ receiver_stats_[peer].rendered_time = time;
+}
+
+bool FrameInFlight::HasRenderedTime(size_t peer) const {
+ auto it = receiver_stats_.find(peer);
+ if (it == receiver_stats_.end()) {
+ return false;
+ }
+ return it->second.rendered_time.IsFinite();
+}
+
+bool FrameInFlight::IsDropped(size_t peer) const {
+ auto it = receiver_stats_.find(peer);
+ if (it == receiver_stats_.end()) {
+ return false;
+ }
+ return it->second.dropped;
+}
+
+FrameStats FrameInFlight::GetStatsForPeer(size_t peer) const {
+ RTC_DCHECK_NE(frame_id_, VideoFrame::kNotSetId)
+ << "Frame id isn't initialized";
+ FrameStats stats(frame_id_, captured_time_);
+ stats.pre_encode_time = pre_encode_time_;
+ stats.encoded_time = encoded_time_;
+ stats.target_encode_bitrate = target_encode_bitrate_;
+ stats.encoded_frame_type = frame_type_;
+ stats.encoded_image_size = encoded_image_size_;
+ stats.used_encoder = used_encoder_;
+ stats.spatial_layers_qp = spatial_layers_qp_;
+
+ absl::optional<ReceiverFrameStats> receiver_stats =
+ MaybeGetValue<ReceiverFrameStats>(receiver_stats_, peer);
+ if (receiver_stats.has_value()) {
+ stats.received_time = receiver_stats->received_time;
+ stats.decode_start_time = receiver_stats->decode_start_time;
+ stats.decode_end_time = receiver_stats->decode_end_time;
+ stats.rendered_time = receiver_stats->rendered_time;
+ stats.prev_frame_rendered_time = receiver_stats->prev_frame_rendered_time;
+ stats.decoded_frame_width = receiver_stats->decoded_frame_width;
+ stats.decoded_frame_height = receiver_stats->decoded_frame_height;
+ stats.used_decoder = receiver_stats->used_decoder;
+ stats.pre_decoded_frame_type = receiver_stats->frame_type;
+ stats.pre_decoded_image_size = receiver_stats->encoded_image_size;
+ stats.decoder_failed = receiver_stats->decoder_failed;
+ }
+ return stats;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h
new file mode 100644
index 0000000000..52a526d09b
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frame_in_flight.h
@@ -0,0 +1,169 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAME_IN_FLIGHT_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAME_IN_FLIGHT_H_
+
+#include <map>
+#include <set>
+#include <utility>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/numerics/samples_stats_counter.h"
+#include "api/units/data_size.h"
+#include "api/units/timestamp.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_frame_type.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
+
+namespace webrtc {
+
+struct ReceiverFrameStats {
+ // Time when last packet of a frame was received.
+ Timestamp received_time = Timestamp::MinusInfinity();
+ Timestamp decode_start_time = Timestamp::MinusInfinity();
+ Timestamp decode_end_time = Timestamp::MinusInfinity();
+ Timestamp rendered_time = Timestamp::MinusInfinity();
+ Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity();
+
+ // Type and encoded size of received frame.
+ VideoFrameType frame_type = VideoFrameType::kEmptyFrame;
+ DataSize encoded_image_size = DataSize::Bytes(0);
+
+ absl::optional<int> decoded_frame_width = absl::nullopt;
+ absl::optional<int> decoded_frame_height = absl::nullopt;
+
+ // Can be not set if frame was dropped in the network.
+ absl::optional<StreamCodecInfo> used_decoder = absl::nullopt;
+
+ bool dropped = false;
+ bool decoder_failed = false;
+};
+
+// Represents a frame which was sent by sender and is currently on the way to
+// multiple receivers. Some receivers may receive this frame and some don't.
+//
+// Contains all statistic associated with the frame and gathered in multiple
+// points of the video pipeline.
+//
+// Internally may store the copy of the source frame which was sent. In such
+// case this frame is "alive".
+class FrameInFlight {
+ public:
+ FrameInFlight(size_t stream,
+ VideoFrame frame,
+ Timestamp captured_time,
+ std::set<size_t> expected_receivers);
+
+ size_t stream() const { return stream_; }
+ // Returns internal copy of source `VideoFrame` or `absl::nullopt` if it was
+ // removed before.
+ const absl::optional<VideoFrame>& frame() const { return frame_; }
+ // Removes internal copy of the source `VideoFrame` to free up extra memory.
+ // Returns was frame removed or not.
+ bool RemoveFrame();
+ void SetFrameId(uint16_t id);
+
+ void AddExpectedReceiver(size_t peer) { expected_receivers_.insert(peer); }
+
+ void RemoveExpectedReceiver(size_t peer) { expected_receivers_.erase(peer); }
+
+ std::vector<size_t> GetPeersWhichDidntReceive() const;
+
+ // Returns if all peers which were expected to receive this frame actually
+ // received it or not.
+ bool HaveAllPeersReceived() const;
+
+ void SetPreEncodeTime(webrtc::Timestamp time) { pre_encode_time_ = time; }
+
+ void OnFrameEncoded(webrtc::Timestamp time,
+ VideoFrameType frame_type,
+ DataSize encoded_image_size,
+ uint32_t target_encode_bitrate,
+ int spatial_layer,
+ int qp,
+ StreamCodecInfo used_encoder);
+
+ bool HasEncodedTime() const { return encoded_time_.IsFinite(); }
+
+ void OnFramePreDecode(size_t peer,
+ webrtc::Timestamp received_time,
+ webrtc::Timestamp decode_start_time,
+ VideoFrameType frame_type,
+ DataSize encoded_image_size);
+
+ bool HasReceivedTime(size_t peer) const;
+
+ void OnFrameDecoded(size_t peer,
+ webrtc::Timestamp time,
+ int width,
+ int height,
+ const StreamCodecInfo& used_decoder);
+ void OnDecoderError(size_t peer, const StreamCodecInfo& used_decoder);
+
+ bool HasDecodeEndTime(size_t peer) const;
+
+ void OnFrameRendered(size_t peer, webrtc::Timestamp time);
+
+ bool HasRenderedTime(size_t peer) const;
+
+ // Crash if rendered time is not set for specified `peer`.
+ webrtc::Timestamp rendered_time(size_t peer) const {
+ return receiver_stats_.at(peer).rendered_time;
+ }
+
+ // Marks that frame was dropped and wasn't seen by particular `peer`.
+ void MarkDropped(size_t peer) { receiver_stats_[peer].dropped = true; }
+ bool IsDropped(size_t peer) const;
+
+ void SetPrevFrameRenderedTime(size_t peer, webrtc::Timestamp time) {
+ receiver_stats_[peer].prev_frame_rendered_time = time;
+ }
+
+ FrameStats GetStatsForPeer(size_t peer) const;
+
+ private:
+ const size_t stream_;
+ // Set of peer's indexes who are expected to receive this frame. This is not
+ // the set of peer's indexes that received the frame. For example, if peer A
+ // was among expected receivers, it received frame and then left the call, A
+ // will be removed from this set, but the Stats for peer A still will be
+ // preserved in the FrameInFlight.
+ //
+ // This set is used to determine if this frame is expected to be received by
+ // any peer or can be safely deleted. It is responsibility of the user of this
+ // object to decide when it should be deleted.
+ std::set<size_t> expected_receivers_;
+ absl::optional<VideoFrame> frame_;
+ // Store frame id separately because `frame_` can be removed when we have too
+ // much memory consuption.
+ uint16_t frame_id_ = VideoFrame::kNotSetId;
+
+ // Frame events timestamp.
+ Timestamp captured_time_;
+ Timestamp pre_encode_time_ = Timestamp::MinusInfinity();
+ Timestamp encoded_time_ = Timestamp::MinusInfinity();
+ // Type and encoded size of sent frame.
+ VideoFrameType frame_type_ = VideoFrameType::kEmptyFrame;
+ DataSize encoded_image_size_ = DataSize::Bytes(0);
+ uint32_t target_encode_bitrate_ = 0;
+ // Sender side qp values per spatial layer. In case when spatial layer is not
+ // set for `webrtc::EncodedImage`, 0 is used as default.
+ std::map<int, SamplesStatsCounter> spatial_layers_qp_;
+ // Can be not set if frame was dropped by encoder.
+ absl::optional<StreamCodecInfo> used_encoder_ = absl::nullopt;
+ // Map from the receiver peer's index to frame stats for that peer.
+ std::map<size_t, ReceiverFrameStats> receiver_stats_;
+};
+
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAME_IN_FLIGHT_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc
new file mode 100644
index 0000000000..cbc0b7e8f3
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.cc
@@ -0,0 +1,575 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h"
+
+#include <algorithm>
+#include <map>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/array_view.h"
+#include "api/scoped_refptr.h"
+#include "api/video/i420_buffer.h"
+#include "api/video/video_frame_type.h"
+#include "common_video/libyuv/include/webrtc_libyuv.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/platform_thread.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "rtc_tools/frame_analyzer/video_geometry_aligner.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
+#include "test/pc/e2e/metric_metadata_keys.h"
+
+namespace webrtc {
+namespace {
+
+using ::webrtc::webrtc_pc_e2e::SampleMetadataKey;
+
+constexpr TimeDelta kFreezeThreshold = TimeDelta::Millis(150);
+constexpr int kMaxActiveComparisons = 10;
+
+SamplesStatsCounter::StatsSample StatsSample(
+ double value,
+ Timestamp sampling_time,
+ std::map<std::string, std::string> metadata) {
+ return SamplesStatsCounter::StatsSample{value, sampling_time,
+ std::move(metadata)};
+}
+
+SamplesStatsCounter::StatsSample StatsSample(
+ TimeDelta duration,
+ Timestamp sampling_time,
+ std::map<std::string, std::string> metadata) {
+ return SamplesStatsCounter::StatsSample{duration.ms<double>(), sampling_time,
+ std::move(metadata)};
+}
+
+FrameComparison ValidateFrameComparison(FrameComparison comparison) {
+ RTC_DCHECK(comparison.frame_stats.captured_time.IsFinite())
+ << "Any comparison has to have finite captured_time";
+ switch (comparison.type) {
+ case FrameComparisonType::kRegular:
+ // Regular comparison has to have all FrameStats filled in.
+ RTC_DCHECK(comparison.captured.has_value() ||
+ comparison.overload_reason != OverloadReason::kNone)
+ << "Regular comparison has to have captured frame if it's not "
+ << "overloaded comparison";
+ RTC_DCHECK(comparison.rendered.has_value() ||
+ comparison.overload_reason != OverloadReason::kNone)
+ << "rendered frame has to be presented if it's not overloaded "
+ << "comparison";
+ RTC_DCHECK(comparison.frame_stats.pre_encode_time.IsFinite())
+ << "Regular comparison has to have finite pre_encode_time";
+ RTC_DCHECK(comparison.frame_stats.encoded_time.IsFinite())
+ << "Regular comparison has to have finite encoded_time";
+ RTC_DCHECK(comparison.frame_stats.received_time.IsFinite())
+ << "Regular comparison has to have finite received_time";
+ RTC_DCHECK(comparison.frame_stats.decode_start_time.IsFinite())
+ << "Regular comparison has to have finite decode_start_time";
+ RTC_DCHECK(comparison.frame_stats.decode_end_time.IsFinite())
+ << "Regular comparison has to have finite decode_end_time";
+ RTC_DCHECK(comparison.frame_stats.rendered_time.IsFinite())
+ << "Regular comparison has to have finite rendered_time";
+ RTC_DCHECK(comparison.frame_stats.decoded_frame_width.has_value())
+ << "Regular comparison has to have decoded_frame_width";
+ RTC_DCHECK(comparison.frame_stats.decoded_frame_height.has_value())
+ << "Regular comparison has to have decoded_frame_height";
+ RTC_DCHECK(comparison.frame_stats.used_encoder.has_value())
+ << "Regular comparison has to have used_encoder";
+ RTC_DCHECK(comparison.frame_stats.used_decoder.has_value())
+ << "Regular comparison has to have used_decoder";
+ RTC_DCHECK(!comparison.frame_stats.decoder_failed)
+ << "Regular comparison can't have decoder failure";
+ break;
+ case FrameComparisonType::kDroppedFrame:
+ // Frame can be dropped before encoder, by encoder, inside network or
+ // after decoder.
+ RTC_DCHECK(!comparison.captured.has_value())
+ << "Dropped frame comparison can't have captured frame";
+ RTC_DCHECK(!comparison.rendered.has_value())
+ << "Dropped frame comparison can't have rendered frame";
+
+ if (comparison.frame_stats.encoded_time.IsFinite()) {
+ RTC_DCHECK(comparison.frame_stats.used_encoder.has_value())
+ << "Dropped frame comparison has to have used_encoder when "
+ << "encoded_time is set";
+ RTC_DCHECK(comparison.frame_stats.pre_encode_time.IsFinite())
+ << "Dropped frame comparison has to have finite pre_encode_time "
+ << "when encoded_time is finite.";
+ }
+
+ if (comparison.frame_stats.decode_end_time.IsFinite() ||
+ comparison.frame_stats.decoder_failed) {
+ RTC_DCHECK(comparison.frame_stats.received_time.IsFinite())
+ << "Dropped frame comparison has to have received_time when "
+ << "decode_end_time is set or decoder_failed is true";
+ RTC_DCHECK(comparison.frame_stats.decode_start_time.IsFinite())
+ << "Dropped frame comparison has to have decode_start_time when "
+ << "decode_end_time is set or decoder_failed is true";
+ RTC_DCHECK(comparison.frame_stats.used_decoder.has_value())
+ << "Dropped frame comparison has to have used_decoder when "
+ << "decode_end_time is set or decoder_failed is true";
+ } else if (comparison.frame_stats.decode_end_time.IsFinite()) {
+ RTC_DCHECK(comparison.frame_stats.decoded_frame_width.has_value())
+ << "Dropped frame comparison has to have decoded_frame_width when "
+ << "decode_end_time is set";
+ RTC_DCHECK(comparison.frame_stats.decoded_frame_height.has_value())
+ << "Dropped frame comparison has to have decoded_frame_height when "
+ << "decode_end_time is set";
+ }
+ RTC_DCHECK(!comparison.frame_stats.rendered_time.IsFinite())
+ << "Dropped frame comparison can't have rendered_time";
+ break;
+ case FrameComparisonType::kFrameInFlight:
+ // Frame in flight comparison may miss almost any FrameStats, but if
+ // stats for stage X are set, then stats for stage X - 1 also has to be
+ // set. Also these frames were never rendered.
+ RTC_DCHECK(!comparison.captured.has_value())
+ << "Frame in flight comparison can't have captured frame";
+ RTC_DCHECK(!comparison.rendered.has_value())
+ << "Frame in flight comparison can't have rendered frame";
+ RTC_DCHECK(!comparison.frame_stats.rendered_time.IsFinite())
+ << "Frame in flight comparison can't have rendered_time";
+
+ if (comparison.frame_stats.decode_end_time.IsFinite() ||
+ comparison.frame_stats.decoder_failed) {
+ RTC_DCHECK(comparison.frame_stats.used_decoder.has_value())
+ << "Frame in flight comparison has to have used_decoder when "
+ << "decode_end_time is set or decoder_failed is true.";
+ RTC_DCHECK(comparison.frame_stats.decode_start_time.IsFinite())
+ << "Frame in flight comparison has to have finite "
+ << "decode_start_time when decode_end_time is finite or "
+ << "decoder_failed is true.";
+ }
+ if (comparison.frame_stats.decode_end_time.IsFinite()) {
+ RTC_DCHECK(comparison.frame_stats.decoded_frame_width.has_value())
+ << "Frame in flight comparison has to have decoded_frame_width "
+ << "when decode_end_time is set.";
+ RTC_DCHECK(comparison.frame_stats.decoded_frame_height.has_value())
+ << "Frame in flight comparison has to have decoded_frame_height "
+ << "when decode_end_time is set.";
+ }
+ if (comparison.frame_stats.decode_start_time.IsFinite()) {
+ RTC_DCHECK(comparison.frame_stats.received_time.IsFinite())
+ << "Frame in flight comparison has to have finite received_time "
+ << "when decode_start_time is finite.";
+ }
+ if (comparison.frame_stats.received_time.IsFinite()) {
+ RTC_DCHECK(comparison.frame_stats.encoded_time.IsFinite())
+ << "Frame in flight comparison has to have finite encoded_time "
+ << "when received_time is finite.";
+ }
+ if (comparison.frame_stats.encoded_time.IsFinite()) {
+ RTC_DCHECK(comparison.frame_stats.used_encoder.has_value())
+ << "Frame in flight comparison has to have used_encoder when "
+ << "encoded_time is set";
+ RTC_DCHECK(comparison.frame_stats.pre_encode_time.IsFinite())
+ << "Frame in flight comparison has to have finite pre_encode_time "
+ << "when encoded_time is finite.";
+ }
+ break;
+ }
+ return comparison;
+}
+
+} // namespace
+
+void DefaultVideoQualityAnalyzerFramesComparator::Start(int max_threads_count) {
+ for (int i = 0; i < max_threads_count; i++) {
+ thread_pool_.push_back(rtc::PlatformThread::SpawnJoinable(
+ [this] { ProcessComparisons(); },
+ "DefaultVideoQualityAnalyzerFramesComparator-" + std::to_string(i)));
+ }
+ {
+ MutexLock lock(&mutex_);
+ RTC_CHECK_EQ(state_, State::kNew) << "Frames comparator is already started";
+ state_ = State::kActive;
+ }
+ cpu_measurer_.StartMeasuringCpuProcessTime();
+}
+
+void DefaultVideoQualityAnalyzerFramesComparator::Stop(
+ const std::map<InternalStatsKey, Timestamp>& last_rendered_frame_times) {
+ {
+ MutexLock lock(&mutex_);
+ if (state_ == State::kStopped) {
+ return;
+ }
+ RTC_CHECK_EQ(state_, State::kActive)
+ << "Frames comparator has to be started before it will be used";
+ state_ = State::kStopped;
+ }
+ cpu_measurer_.StopMeasuringCpuProcessTime();
+ comparison_available_event_.Set();
+ thread_pool_.clear();
+
+ {
+ MutexLock lock(&mutex_);
+ // Perform final Metrics update. On this place analyzer is stopped and no
+ // one holds any locks.
+
+ // Time between freezes.
+ // Count time since the last freeze to the end of the call as time
+ // between freezes.
+ for (auto& entry : last_rendered_frame_times) {
+ const InternalStatsKey& stats_key = entry.first;
+ const Timestamp& last_rendered_frame_time = entry.second;
+
+ // If there are no freezes in the call we have to report
+ // time_between_freezes_ms as call duration and in such case
+ // `last_rendered_frame_time` for this stream will be stream start time.
+ // If there is freeze, then we need add time from last rendered frame
+ // to last freeze end as time between freezes.
+ stream_stats_.at(stats_key).time_between_freezes_ms.AddSample(StatsSample(
+ last_rendered_frame_time - stream_last_freeze_end_time_.at(stats_key),
+ Now(), /*metadata=*/{}));
+ }
+
+ // Freeze Time:
+ // If there were no freezes on a video stream, add only one sample with
+ // value 0 (0ms freezes time).
+ for (auto& [key, stream_stats] : stream_stats_) {
+ if (stream_stats.freeze_time_ms.IsEmpty()) {
+ stream_stats.freeze_time_ms.AddSample(0);
+ }
+ }
+ }
+}
+
+void DefaultVideoQualityAnalyzerFramesComparator::EnsureStatsForStream(
+ size_t stream_index,
+ size_t sender_peer_index,
+ size_t peers_count,
+ Timestamp captured_time,
+ Timestamp start_time) {
+ MutexLock lock(&mutex_);
+ RTC_CHECK_EQ(state_, State::kActive)
+ << "Frames comparator has to be started before it will be used";
+
+ for (size_t i = 0; i < peers_count; ++i) {
+ if (i == sender_peer_index && !options_.enable_receive_own_stream) {
+ continue;
+ }
+ InternalStatsKey stats_key(stream_index, sender_peer_index, i);
+ if (stream_stats_.find(stats_key) == stream_stats_.end()) {
+ stream_stats_.insert({stats_key, StreamStats(captured_time)});
+ // Assume that the first freeze was before first stream frame captured.
+ // This way time before the first freeze would be counted as time
+ // between freezes.
+ stream_last_freeze_end_time_.insert({stats_key, start_time});
+ } else {
+ // When we see some `stream_label` for the first time we need to create
+ // stream stats object for it and set up some states, but we need to do
+ // it only once and for all receivers, so on the next frame on the same
+ // `stream_label` we can be sure, that it's already done and we needn't
+ // to scan though all peers again.
+ break;
+ }
+ }
+}
+
+void DefaultVideoQualityAnalyzerFramesComparator::RegisterParticipantInCall(
+ rtc::ArrayView<std::pair<InternalStatsKey, Timestamp>> stream_started_time,
+ Timestamp start_time) {
+ MutexLock lock(&mutex_);
+ RTC_CHECK_EQ(state_, State::kActive)
+ << "Frames comparator has to be started before it will be used";
+
+ for (const std::pair<InternalStatsKey, Timestamp>& pair :
+ stream_started_time) {
+ stream_stats_.insert({pair.first, StreamStats(pair.second)});
+ stream_last_freeze_end_time_.insert({pair.first, start_time});
+ }
+}
+
+void DefaultVideoQualityAnalyzerFramesComparator::AddComparison(
+ InternalStatsKey stats_key,
+ absl::optional<VideoFrame> captured,
+ absl::optional<VideoFrame> rendered,
+ FrameComparisonType type,
+ FrameStats frame_stats) {
+ MutexLock lock(&mutex_);
+ RTC_CHECK_EQ(state_, State::kActive)
+ << "Frames comparator has to be started before it will be used";
+ AddComparisonInternal(std::move(stats_key), std::move(captured),
+ std::move(rendered), type, std::move(frame_stats));
+}
+
+void DefaultVideoQualityAnalyzerFramesComparator::AddComparison(
+ InternalStatsKey stats_key,
+ int skipped_between_rendered,
+ absl::optional<VideoFrame> captured,
+ absl::optional<VideoFrame> rendered,
+ FrameComparisonType type,
+ FrameStats frame_stats) {
+ MutexLock lock(&mutex_);
+ RTC_CHECK_EQ(state_, State::kActive)
+ << "Frames comparator has to be started before it will be used";
+ stream_stats_.at(stats_key).skipped_between_rendered.AddSample(
+ StatsSample(skipped_between_rendered, Now(),
+ /*metadata=*/
+ {{SampleMetadataKey::kFrameIdMetadataKey,
+ std::to_string(frame_stats.frame_id)}}));
+ AddComparisonInternal(std::move(stats_key), std::move(captured),
+ std::move(rendered), type, std::move(frame_stats));
+}
+
+void DefaultVideoQualityAnalyzerFramesComparator::AddComparisonInternal(
+ InternalStatsKey stats_key,
+ absl::optional<VideoFrame> captured,
+ absl::optional<VideoFrame> rendered,
+ FrameComparisonType type,
+ FrameStats frame_stats) {
+ cpu_measurer_.StartExcludingCpuThreadTime();
+ frames_comparator_stats_.comparisons_queue_size.AddSample(
+ StatsSample(comparisons_.size(), Now(), /*metadata=*/{}));
+ // If there too many computations waiting in the queue, we won't provide
+ // frames itself to make future computations lighter.
+ if (comparisons_.size() >= kMaxActiveComparisons) {
+ comparisons_.emplace_back(ValidateFrameComparison(
+ FrameComparison(std::move(stats_key), /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt, type,
+ std::move(frame_stats), OverloadReason::kCpu)));
+ } else {
+ OverloadReason overload_reason = OverloadReason::kNone;
+ if (!captured && type == FrameComparisonType::kRegular) {
+ overload_reason = OverloadReason::kMemory;
+ }
+ comparisons_.emplace_back(ValidateFrameComparison(FrameComparison(
+ std::move(stats_key), std::move(captured), std::move(rendered), type,
+ std::move(frame_stats), overload_reason)));
+ }
+ comparison_available_event_.Set();
+ cpu_measurer_.StopExcludingCpuThreadTime();
+}
+
+void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparisons() {
+ while (true) {
+ // Try to pick next comparison to perform from the queue.
+ absl::optional<FrameComparison> comparison = absl::nullopt;
+ bool more_new_comparisons_expected;
+ {
+ MutexLock lock(&mutex_);
+ if (!comparisons_.empty()) {
+ comparison = comparisons_.front();
+ comparisons_.pop_front();
+ if (!comparisons_.empty()) {
+ comparison_available_event_.Set();
+ }
+ }
+ // If state is stopped => no new frame comparisons are expected.
+ more_new_comparisons_expected = state_ != State::kStopped;
+ }
+ if (!comparison) {
+ if (!more_new_comparisons_expected) {
+ comparison_available_event_.Set();
+ return;
+ }
+ comparison_available_event_.Wait(TimeDelta::Seconds(1));
+ continue;
+ }
+
+ cpu_measurer_.StartExcludingCpuThreadTime();
+ ProcessComparison(comparison.value());
+ cpu_measurer_.StopExcludingCpuThreadTime();
+ }
+}
+
+void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison(
+ const FrameComparison& comparison) {
+ // Comparison is checked to be valid before adding, so we can use this
+ // assumptions during computations.
+
+ // Perform expensive psnr and ssim calculations while not holding lock.
+ double psnr = -1.0;
+ double ssim = -1.0;
+ if ((options_.compute_psnr || options_.compute_ssim) &&
+ comparison.captured.has_value() && comparison.rendered.has_value()) {
+ rtc::scoped_refptr<I420BufferInterface> reference_buffer =
+ comparison.captured->video_frame_buffer()->ToI420();
+ rtc::scoped_refptr<I420BufferInterface> test_buffer =
+ comparison.rendered->video_frame_buffer()->ToI420();
+ if (options_.adjust_cropping_before_comparing_frames) {
+ test_buffer = ScaleVideoFrameBuffer(
+ *test_buffer, reference_buffer->width(), reference_buffer->height());
+ reference_buffer = test::AdjustCropping(reference_buffer, test_buffer);
+ }
+ if (options_.compute_psnr) {
+ psnr = options_.use_weighted_psnr
+ ? I420WeightedPSNR(*reference_buffer, *test_buffer)
+ : I420PSNR(*reference_buffer, *test_buffer);
+ }
+ if (options_.compute_ssim) {
+ ssim = I420SSIM(*reference_buffer, *test_buffer);
+ }
+ }
+
+ const FrameStats& frame_stats = comparison.frame_stats;
+
+ MutexLock lock(&mutex_);
+ auto stats_it = stream_stats_.find(comparison.stats_key);
+ RTC_CHECK(stats_it != stream_stats_.end()) << comparison.stats_key.ToString();
+ StreamStats* stats = &stats_it->second;
+
+ frames_comparator_stats_.comparisons_done++;
+ if (comparison.overload_reason == OverloadReason::kCpu) {
+ frames_comparator_stats_.cpu_overloaded_comparisons_done++;
+ } else if (comparison.overload_reason == OverloadReason::kMemory) {
+ frames_comparator_stats_.memory_overloaded_comparisons_done++;
+ }
+
+ std::map<std::string, std::string> metadata;
+ metadata.emplace(SampleMetadataKey::kFrameIdMetadataKey,
+ std::to_string(frame_stats.frame_id));
+
+ if (psnr > 0) {
+ stats->psnr.AddSample(
+ StatsSample(psnr, frame_stats.rendered_time, metadata));
+ }
+ if (ssim > 0) {
+ stats->ssim.AddSample(
+ StatsSample(ssim, frame_stats.received_time, metadata));
+ }
+ stats->capture_frame_rate.AddEvent(frame_stats.captured_time);
+
+ // Compute dropped phase for dropped frame
+ if (comparison.type == FrameComparisonType::kDroppedFrame) {
+ FrameDropPhase dropped_phase;
+ if (frame_stats.decode_end_time.IsFinite()) {
+ dropped_phase = FrameDropPhase::kAfterDecoder;
+ } else if (frame_stats.decode_start_time.IsFinite()) {
+ dropped_phase = FrameDropPhase::kByDecoder;
+ } else if (frame_stats.encoded_time.IsFinite()) {
+ dropped_phase = FrameDropPhase::kTransport;
+ } else if (frame_stats.pre_encode_time.IsFinite()) {
+ dropped_phase = FrameDropPhase::kByEncoder;
+ } else {
+ dropped_phase = FrameDropPhase::kBeforeEncoder;
+ }
+ stats->dropped_by_phase[dropped_phase]++;
+ }
+
+ if (frame_stats.encoded_time.IsFinite()) {
+ stats->encode_time_ms.AddSample(
+ StatsSample(frame_stats.encoded_time - frame_stats.pre_encode_time,
+ frame_stats.encoded_time, metadata));
+ stats->encode_frame_rate.AddEvent(frame_stats.encoded_time);
+ stats->total_encoded_images_payload +=
+ frame_stats.encoded_image_size.bytes();
+ stats->target_encode_bitrate.AddSample(StatsSample(
+ frame_stats.target_encode_bitrate, frame_stats.encoded_time, metadata));
+ for (const auto& [spatial_layer, qp_values] :
+ frame_stats.spatial_layers_qp) {
+ for (SamplesStatsCounter::StatsSample qp : qp_values.GetTimedSamples()) {
+ qp.metadata = metadata;
+ stats->spatial_layers_qp[spatial_layer].AddSample(std::move(qp));
+ }
+ }
+
+ // Stats sliced on encoded frame type.
+ if (frame_stats.encoded_frame_type == VideoFrameType::kVideoFrameKey) {
+ ++stats->num_send_key_frames;
+ }
+ }
+ // Next stats can be calculated only if frame was received on remote side.
+ if (comparison.type != FrameComparisonType::kDroppedFrame ||
+ comparison.frame_stats.decoder_failed) {
+ if (frame_stats.rendered_time.IsFinite()) {
+ stats->total_delay_incl_transport_ms.AddSample(
+ StatsSample(frame_stats.rendered_time - frame_stats.captured_time,
+ frame_stats.received_time, metadata));
+ stats->receive_to_render_time_ms.AddSample(
+ StatsSample(frame_stats.rendered_time - frame_stats.received_time,
+ frame_stats.rendered_time, metadata));
+ }
+ if (frame_stats.decode_start_time.IsFinite()) {
+ stats->transport_time_ms.AddSample(
+ StatsSample(frame_stats.decode_start_time - frame_stats.encoded_time,
+ frame_stats.decode_start_time, metadata));
+
+ // Stats sliced on decoded frame type.
+ if (frame_stats.pre_decoded_frame_type ==
+ VideoFrameType::kVideoFrameKey) {
+ ++stats->num_recv_key_frames;
+ stats->recv_key_frame_size_bytes.AddSample(
+ StatsSample(frame_stats.pre_decoded_image_size.bytes(),
+ frame_stats.decode_start_time, metadata));
+ } else if (frame_stats.pre_decoded_frame_type ==
+ VideoFrameType::kVideoFrameDelta) {
+ stats->recv_delta_frame_size_bytes.AddSample(
+ StatsSample(frame_stats.pre_decoded_image_size.bytes(),
+ frame_stats.decode_start_time, metadata));
+ }
+ }
+ if (frame_stats.decode_end_time.IsFinite()) {
+ stats->decode_time_ms.AddSample(StatsSample(
+ frame_stats.decode_end_time - frame_stats.decode_start_time,
+ frame_stats.decode_end_time, metadata));
+ stats->resolution_of_decoded_frame.AddSample(
+ StatsSample(*comparison.frame_stats.decoded_frame_width *
+ *comparison.frame_stats.decoded_frame_height,
+ frame_stats.decode_end_time, metadata));
+ }
+
+ if (frame_stats.prev_frame_rendered_time.IsFinite() &&
+ frame_stats.rendered_time.IsFinite()) {
+ TimeDelta time_between_rendered_frames =
+ frame_stats.rendered_time - frame_stats.prev_frame_rendered_time;
+ stats->time_between_rendered_frames_ms.AddSample(StatsSample(
+ time_between_rendered_frames, frame_stats.rendered_time, metadata));
+ TimeDelta average_time_between_rendered_frames = TimeDelta::Millis(
+ stats->time_between_rendered_frames_ms.GetAverage());
+ if (time_between_rendered_frames >
+ std::max(kFreezeThreshold + average_time_between_rendered_frames,
+ 3 * average_time_between_rendered_frames)) {
+ stats->freeze_time_ms.AddSample(StatsSample(
+ time_between_rendered_frames, frame_stats.rendered_time, metadata));
+ auto freeze_end_it =
+ stream_last_freeze_end_time_.find(comparison.stats_key);
+ RTC_DCHECK(freeze_end_it != stream_last_freeze_end_time_.end());
+ stats->time_between_freezes_ms.AddSample(StatsSample(
+ frame_stats.prev_frame_rendered_time - freeze_end_it->second,
+ frame_stats.rendered_time, metadata));
+ freeze_end_it->second = frame_stats.rendered_time;
+ }
+ }
+ }
+ // Compute stream codec info.
+ if (frame_stats.used_encoder.has_value()) {
+ if (stats->encoders.empty() || stats->encoders.back().codec_name !=
+ frame_stats.used_encoder->codec_name) {
+ stats->encoders.push_back(*frame_stats.used_encoder);
+ }
+ stats->encoders.back().last_frame_id =
+ frame_stats.used_encoder->last_frame_id;
+ stats->encoders.back().switched_from_at =
+ frame_stats.used_encoder->switched_from_at;
+ }
+
+ if (frame_stats.used_decoder.has_value()) {
+ if (stats->decoders.empty() || stats->decoders.back().codec_name !=
+ frame_stats.used_decoder->codec_name) {
+ stats->decoders.push_back(*frame_stats.used_decoder);
+ }
+ stats->decoders.back().last_frame_id =
+ frame_stats.used_decoder->last_frame_id;
+ stats->decoders.back().switched_from_at =
+ frame_stats.used_decoder->switched_from_at;
+ }
+}
+
+Timestamp DefaultVideoQualityAnalyzerFramesComparator::Now() {
+ return clock_->CurrentTime();
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h
new file mode 100644
index 0000000000..006c3eb9bf
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h
@@ -0,0 +1,157 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_
+
+#include <deque>
+#include <map>
+#include <utility>
+#include <vector>
+
+#include "api/array_view.h"
+#include "rtc_base/event.h"
+#include "rtc_base/platform_thread.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "system_wrappers/include/clock.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
+
+namespace webrtc {
+
+struct FramesComparatorStats {
+ // Size of analyzer internal comparisons queue, measured when new element
+ // id added to the queue.
+ SamplesStatsCounter comparisons_queue_size;
+ // Number of performed comparisons of 2 video frames from captured and
+ // rendered streams.
+ int64_t comparisons_done = 0;
+ // Number of cpu overloaded comparisons. Comparison is cpu overloaded if it is
+ // queued when there are too many not processed comparisons in the queue.
+ // Overloaded comparison doesn't include metrics like SSIM and PSNR that
+ // require heavy computations.
+ int64_t cpu_overloaded_comparisons_done = 0;
+ // Number of memory overloaded comparisons. Comparison is memory overloaded if
+ // it is queued when its captured frame was already removed due to high memory
+ // usage for that video stream.
+ int64_t memory_overloaded_comparisons_done = 0;
+};
+
+// Performs comparisons of added frames and tracks frames related statistics.
+// This class is thread safe.
+class DefaultVideoQualityAnalyzerFramesComparator {
+ public:
+ // Creates frames comparator.
+ // Frames comparator doesn't use `options.enable_receive_own_stream` for any
+ // purposes, because it's unrelated to its functionality.
+ DefaultVideoQualityAnalyzerFramesComparator(
+ webrtc::Clock* clock,
+ DefaultVideoQualityAnalyzerCpuMeasurer& cpu_measurer,
+ DefaultVideoQualityAnalyzerOptions options = {})
+ : options_(options), clock_(clock), cpu_measurer_(cpu_measurer) {}
+ ~DefaultVideoQualityAnalyzerFramesComparator() { Stop({}); }
+
+ // Starts frames comparator. This method must be invoked before calling
+ // any other method on this object.
+ void Start(int max_threads_count);
+ // Stops frames comparator. This method will block until all added frame
+ // comparisons will be processed. After `Stop()` is invoked no more new
+ // comparisons can be added to this frames comparator.
+ //
+ // `last_rendered_frame_time` contains timestamps of last rendered frame for
+ // each (stream, sender, receiver) tuple to properly update time between
+ // freezes: it has include time from the last freeze until and of call.
+ void Stop(
+ const std::map<InternalStatsKey, Timestamp>& last_rendered_frame_times);
+
+ // Ensures that stream `stream_index` has stats objects created for all
+ // potential receivers. This method must be called before adding any
+ // frames comparison for that stream.
+ void EnsureStatsForStream(size_t stream_index,
+ size_t sender_peer_index,
+ size_t peers_count,
+ Timestamp captured_time,
+ Timestamp start_time);
+ // Ensures that newly added participant will have stream stats objects created
+ // for all streams which they can receive. This method must be called before
+ // any frames comparison will be added for the newly added participant.
+ //
+ // `stream_started_time` - start time of each stream for which stats object
+ // has to be created.
+ // `start_time` - call start time.
+ void RegisterParticipantInCall(
+ rtc::ArrayView<std::pair<InternalStatsKey, Timestamp>>
+ stream_started_time,
+ Timestamp start_time);
+
+ // `captured` - video frame captured by sender to use for PSNR/SSIM
+ // computation. If `type` is `FrameComparisonType::kRegular` and
+ // `captured` is `absl::nullopt` comparison is assumed to be overloaded
+ // due to memory constraints.
+ // `rendered` - video frame rendered by receiver to use for PSNR/SSIM
+ // computation. Required only if `type` is
+ // `FrameComparisonType::kRegular`, but can still be omitted if
+ // `captured` is `absl::nullopt`.
+ void AddComparison(InternalStatsKey stats_key,
+ absl::optional<VideoFrame> captured,
+ absl::optional<VideoFrame> rendered,
+ FrameComparisonType type,
+ FrameStats frame_stats);
+ // `skipped_between_rendered` - amount of frames dropped on this stream before
+ // last received frame and current frame.
+ void AddComparison(InternalStatsKey stats_key,
+ int skipped_between_rendered,
+ absl::optional<VideoFrame> captured,
+ absl::optional<VideoFrame> rendered,
+ FrameComparisonType type,
+ FrameStats frame_stats);
+
+ std::map<InternalStatsKey, StreamStats> stream_stats() const {
+ MutexLock lock(&mutex_);
+ return stream_stats_;
+ }
+ FramesComparatorStats frames_comparator_stats() const {
+ MutexLock lock(&mutex_);
+ return frames_comparator_stats_;
+ }
+
+ private:
+ enum State { kNew, kActive, kStopped };
+
+ void AddComparisonInternal(InternalStatsKey stats_key,
+ absl::optional<VideoFrame> captured,
+ absl::optional<VideoFrame> rendered,
+ FrameComparisonType type,
+ FrameStats frame_stats)
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+ void ProcessComparisons();
+ void ProcessComparison(const FrameComparison& comparison);
+ Timestamp Now();
+
+ const DefaultVideoQualityAnalyzerOptions options_;
+ webrtc::Clock* const clock_;
+ DefaultVideoQualityAnalyzerCpuMeasurer& cpu_measurer_;
+
+ mutable Mutex mutex_;
+ State state_ RTC_GUARDED_BY(mutex_) = State::kNew;
+ std::map<InternalStatsKey, StreamStats> stream_stats_ RTC_GUARDED_BY(mutex_);
+ std::map<InternalStatsKey, Timestamp> stream_last_freeze_end_time_
+ RTC_GUARDED_BY(mutex_);
+ std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(mutex_);
+ FramesComparatorStats frames_comparator_stats_ RTC_GUARDED_BY(mutex_);
+
+ std::vector<rtc::PlatformThread> thread_pool_;
+ rtc::Event comparison_available_event_;
+};
+
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc
new file mode 100644
index 0000000000..8d3cd47ed6
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator_test.cc
@@ -0,0 +1,1648 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h"
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "api/test/create_frame_generator.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/strings/string_builder.h"
+#include "system_wrappers/include/clock.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
+
+namespace webrtc {
+namespace {
+
+using ::testing::Contains;
+using ::testing::DoubleEq;
+using ::testing::Each;
+using ::testing::Eq;
+using ::testing::IsEmpty;
+using ::testing::Pair;
+using ::testing::SizeIs;
+
+using StatsSample = ::webrtc::SamplesStatsCounter::StatsSample;
+
+constexpr int kMaxFramesInFlightPerStream = 10;
+
+DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() {
+ DefaultVideoQualityAnalyzerOptions options;
+ options.compute_psnr = false;
+ options.compute_ssim = false;
+ options.adjust_cropping_before_comparing_frames = false;
+ options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream;
+ return options;
+}
+
+VideoFrame CreateFrame(uint16_t frame_id,
+ int width,
+ int height,
+ Timestamp timestamp) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(width, height,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+ test::FrameGeneratorInterface::VideoFrameData frame_data =
+ frame_generator->NextFrame();
+ return VideoFrame::Builder()
+ .set_id(frame_id)
+ .set_video_frame_buffer(frame_data.buffer)
+ .set_update_rect(frame_data.update_rect)
+ .set_timestamp_us(timestamp.us())
+ .build();
+}
+
+StreamCodecInfo Vp8CodecForOneFrame(uint16_t frame_id, Timestamp time) {
+ StreamCodecInfo info;
+ info.codec_name = "VP8";
+ info.first_frame_id = frame_id;
+ info.last_frame_id = frame_id;
+ info.switched_on_at = time;
+ info.switched_from_at = time;
+ return info;
+}
+
+FrameStats FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(
+ uint16_t frame_id,
+ Timestamp captured_time) {
+ FrameStats frame_stats(frame_id, captured_time);
+ frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
+ frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
+ frame_stats.received_time = captured_time + TimeDelta::Millis(30);
+ frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
+ // Decode time is in microseconds.
+ frame_stats.decode_end_time = captured_time + TimeDelta::Micros(40010);
+ frame_stats.rendered_time = captured_time + TimeDelta::Millis(60);
+ frame_stats.used_encoder = Vp8CodecForOneFrame(1, frame_stats.encoded_time);
+ frame_stats.used_decoder =
+ Vp8CodecForOneFrame(1, frame_stats.decode_end_time);
+ frame_stats.decoded_frame_width = 10;
+ frame_stats.decoded_frame_height = 10;
+ return frame_stats;
+}
+
+FrameStats ShiftStatsOn(const FrameStats& stats, TimeDelta delta) {
+ FrameStats frame_stats(stats.frame_id, stats.captured_time + delta);
+ frame_stats.pre_encode_time = stats.pre_encode_time + delta;
+ frame_stats.encoded_time = stats.encoded_time + delta;
+ frame_stats.received_time = stats.received_time + delta;
+ frame_stats.decode_start_time = stats.decode_start_time + delta;
+ frame_stats.decode_end_time = stats.decode_end_time + delta;
+ frame_stats.rendered_time = stats.rendered_time + delta;
+
+ frame_stats.used_encoder = stats.used_encoder;
+ frame_stats.used_decoder = stats.used_decoder;
+ frame_stats.decoded_frame_width = stats.decoded_frame_width;
+ frame_stats.decoded_frame_height = stats.decoded_frame_height;
+
+ return frame_stats;
+}
+
+SamplesStatsCounter StatsCounter(
+ const std::vector<std::pair<double, Timestamp>>& samples) {
+ SamplesStatsCounter counter;
+ for (const std::pair<double, Timestamp>& sample : samples) {
+ counter.AddSample(SamplesStatsCounter::StatsSample{.value = sample.first,
+ .time = sample.second});
+ }
+ return counter;
+}
+
+double GetFirstOrDie(const SamplesStatsCounter& counter) {
+ EXPECT_FALSE(counter.IsEmpty()) << "Counter has to be not empty";
+ return counter.GetSamples()[0];
+}
+
+void AssertFirstMetadataHasField(const SamplesStatsCounter& counter,
+ const std::string& field_name,
+ const std::string& field_value) {
+ EXPECT_FALSE(counter.IsEmpty()) << "Coutner has to be not empty";
+ EXPECT_THAT(counter.GetTimedSamples()[0].metadata,
+ Contains(Pair(field_name, field_value)));
+}
+
+std::string ToString(const SamplesStatsCounter& counter) {
+ rtc::StringBuilder out;
+ for (const StatsSample& s : counter.GetTimedSamples()) {
+ out << "{ time_ms=" << s.time.ms() << "; value=" << s.value << "}, ";
+ }
+ return out.str();
+}
+
+void ExpectEmpty(const SamplesStatsCounter& counter) {
+ EXPECT_TRUE(counter.IsEmpty())
+ << "Expected empty SamplesStatsCounter, but got " << ToString(counter);
+}
+
+void ExpectEmpty(const SamplesRateCounter& counter) {
+ EXPECT_TRUE(counter.IsEmpty())
+ << "Expected empty SamplesRateCounter, but got "
+ << counter.GetEventsPerSecond();
+}
+
+void ExpectSizeAndAllElementsAre(const SamplesStatsCounter& counter,
+ int size,
+ double value) {
+ EXPECT_EQ(counter.NumSamples(), size);
+ EXPECT_THAT(counter.GetSamples(), Each(DoubleEq(value)));
+}
+
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ StatsPresentedAfterAddingOneComparison) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest());
+
+ Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime();
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ size_t peers_count = 2;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ FrameStats frame_stats = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(
+ /*frame_id=*/1, stream_start_time);
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, peers_count,
+ stream_start_time, stream_start_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kRegular, frame_stats);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ std::map<InternalStatsKey, StreamStats> stats = comparator.stream_stats();
+ ExpectSizeAndAllElementsAre(stats.at(stats_key).transport_time_ms, /*size=*/1,
+ /*value=*/20.0);
+ ExpectSizeAndAllElementsAre(stats.at(stats_key).total_delay_incl_transport_ms,
+ /*size=*/1, /*value=*/60.0);
+ ExpectSizeAndAllElementsAre(stats.at(stats_key).encode_time_ms, /*size=*/1,
+ /*value=*/10.0);
+ ExpectSizeAndAllElementsAre(stats.at(stats_key).decode_time_ms, /*size=*/1,
+ /*value=*/0.01);
+ ExpectSizeAndAllElementsAre(stats.at(stats_key).receive_to_render_time_ms,
+ /*size=*/1, /*value=*/30.0);
+ ExpectSizeAndAllElementsAre(stats.at(stats_key).resolution_of_decoded_frame,
+ /*size=*/1, /*value=*/100.0);
+}
+
+TEST(
+ DefaultVideoQualityAnalyzerFramesComparatorTest,
+ MultiFrameStatsPresentedWithMetadataAfterAddingTwoComparisonWith10msDelay) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest());
+
+ Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime();
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ size_t peers_count = 2;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ FrameStats frame_stats1 = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(
+ /*frame_id=*/1, stream_start_time);
+ FrameStats frame_stats2 = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(
+ /*frame_id=*/2, stream_start_time + TimeDelta::Millis(15));
+ frame_stats2.prev_frame_rendered_time = frame_stats1.rendered_time;
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, peers_count,
+ stream_start_time, stream_start_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kRegular, frame_stats1);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kRegular, frame_stats2);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ std::map<InternalStatsKey, StreamStats> stats = comparator.stream_stats();
+ ExpectSizeAndAllElementsAre(
+ stats.at(stats_key).time_between_rendered_frames_ms, /*size=*/1,
+ /*value=*/15.0);
+ AssertFirstMetadataHasField(
+ stats.at(stats_key).time_between_rendered_frames_ms, "frame_id", "2");
+ EXPECT_DOUBLE_EQ(stats.at(stats_key).encode_frame_rate.GetEventsPerSecond(),
+ 2.0 / 15 * 1000)
+ << "There should be 2 events with interval of 15 ms";
+}
+
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ FrameInFlightStatsAreHandledCorrectly) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest());
+
+ Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime();
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ size_t peers_count = 2;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ // There are 7 different timings inside frame stats: captured, pre_encode,
+ // encoded, received, decode_start, decode_end, rendered. captured is always
+ // set and received is set together with decode_start. So we create 6
+ // different frame stats with interval of 15 ms, where for each stat next
+ // timings will be set
+ // * 1st - captured
+ // * 2nd - captured, pre_encode
+ // * 3rd - captured, pre_encode, encoded
+ // * 4th - captured, pre_encode, encoded, received, decode_start
+ // * 5th - captured, pre_encode, encoded, received, decode_start, decode_end
+ // * 6th - all of them set
+ std::vector<FrameStats> stats;
+ // 1st stat
+ FrameStats frame_stats(/*frame_id=*/1, stream_start_time);
+ stats.push_back(frame_stats);
+ // 2nd stat
+ frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15));
+ frame_stats.frame_id = 2;
+ frame_stats.pre_encode_time =
+ frame_stats.captured_time + TimeDelta::Millis(10);
+ stats.push_back(frame_stats);
+ // 3rd stat
+ frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15));
+ frame_stats.frame_id = 3;
+ frame_stats.encoded_time = frame_stats.captured_time + TimeDelta::Millis(20);
+ frame_stats.used_encoder = Vp8CodecForOneFrame(1, frame_stats.encoded_time);
+ stats.push_back(frame_stats);
+ // 4th stat
+ frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15));
+ frame_stats.frame_id = 4;
+ frame_stats.received_time = frame_stats.captured_time + TimeDelta::Millis(30);
+ frame_stats.decode_start_time =
+ frame_stats.captured_time + TimeDelta::Millis(40);
+ stats.push_back(frame_stats);
+ // 5th stat
+ frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15));
+ frame_stats.frame_id = 5;
+ frame_stats.decode_end_time =
+ frame_stats.captured_time + TimeDelta::Millis(50);
+ frame_stats.used_decoder =
+ Vp8CodecForOneFrame(1, frame_stats.decode_end_time);
+ frame_stats.decoded_frame_width = 10;
+ frame_stats.decoded_frame_height = 10;
+ stats.push_back(frame_stats);
+ // 6th stat
+ frame_stats = ShiftStatsOn(frame_stats, TimeDelta::Millis(15));
+ frame_stats.frame_id = 6;
+ frame_stats.rendered_time = frame_stats.captured_time + TimeDelta::Millis(60);
+ stats.push_back(frame_stats);
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, peers_count,
+ stream_start_time, stream_start_time);
+ for (size_t i = 0; i < stats.size() - 1; ++i) {
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kFrameInFlight, stats[i]);
+ }
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kRegular,
+ stats[stats.size() - 1]);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ EXPECT_EQ(comparator.stream_stats().size(), 1lu);
+ StreamStats result_stats = comparator.stream_stats().at(stats_key);
+
+ EXPECT_DOUBLE_EQ(result_stats.transport_time_ms.GetAverage(), 20.0)
+ << ToString(result_stats.transport_time_ms);
+ EXPECT_EQ(result_stats.transport_time_ms.NumSamples(), 3);
+
+ EXPECT_DOUBLE_EQ(result_stats.total_delay_incl_transport_ms.GetAverage(),
+ 60.0)
+ << ToString(result_stats.total_delay_incl_transport_ms);
+ EXPECT_EQ(result_stats.total_delay_incl_transport_ms.NumSamples(), 1);
+
+ EXPECT_DOUBLE_EQ(result_stats.encode_time_ms.GetAverage(), 10)
+ << ToString(result_stats.encode_time_ms);
+ EXPECT_EQ(result_stats.encode_time_ms.NumSamples(), 4);
+
+ EXPECT_DOUBLE_EQ(result_stats.decode_time_ms.GetAverage(), 10)
+ << ToString(result_stats.decode_time_ms);
+ EXPECT_EQ(result_stats.decode_time_ms.NumSamples(), 2);
+
+ EXPECT_DOUBLE_EQ(result_stats.receive_to_render_time_ms.GetAverage(), 30)
+ << ToString(result_stats.receive_to_render_time_ms);
+ EXPECT_EQ(result_stats.receive_to_render_time_ms.NumSamples(), 1);
+
+ EXPECT_DOUBLE_EQ(result_stats.resolution_of_decoded_frame.GetAverage(), 100)
+ << ToString(result_stats.resolution_of_decoded_frame);
+ EXPECT_EQ(result_stats.resolution_of_decoded_frame.NumSamples(), 2);
+
+ EXPECT_DOUBLE_EQ(result_stats.encode_frame_rate.GetEventsPerSecond(),
+ 4.0 / 45 * 1000)
+ << "There should be 4 events with interval of 15 ms";
+}
+
+// Tests to validate that stats for each possible input frame are computed
+// correctly.
+// Frame in flight start
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ CapturedOnlyInFlightFrameAccountedInStats) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer,
+ DefaultVideoQualityAnalyzerOptions());
+
+ Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ // Frame captured
+ FrameStats frame_stats(/*frame_id=*/1, captured_time);
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
+ captured_time, captured_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kFrameInFlight, frame_stats);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ EXPECT_EQ(comparator.stream_stats().size(), 1lu);
+ StreamStats stats = comparator.stream_stats().at(stats_key);
+ EXPECT_EQ(stats.stream_started_time, captured_time);
+ ExpectEmpty(stats.psnr);
+ ExpectEmpty(stats.ssim);
+ ExpectEmpty(stats.transport_time_ms);
+ ExpectEmpty(stats.total_delay_incl_transport_ms);
+ ExpectEmpty(stats.time_between_rendered_frames_ms);
+ ExpectEmpty(stats.encode_frame_rate);
+ ExpectEmpty(stats.encode_time_ms);
+ ExpectEmpty(stats.decode_time_ms);
+ ExpectEmpty(stats.receive_to_render_time_ms);
+ ExpectEmpty(stats.skipped_between_rendered);
+ ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0);
+ ExpectEmpty(stats.time_between_freezes_ms);
+ ExpectEmpty(stats.resolution_of_decoded_frame);
+ ExpectEmpty(stats.target_encode_bitrate);
+ EXPECT_THAT(stats.spatial_layers_qp, IsEmpty());
+ ExpectEmpty(stats.recv_key_frame_size_bytes);
+ ExpectEmpty(stats.recv_delta_frame_size_bytes);
+ EXPECT_EQ(stats.total_encoded_images_payload, 0);
+ EXPECT_EQ(stats.num_send_key_frames, 0);
+ EXPECT_EQ(stats.num_recv_key_frames, 0);
+ EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
+ {FrameDropPhase::kBeforeEncoder, 0},
+ {FrameDropPhase::kByEncoder, 0},
+ {FrameDropPhase::kTransport, 0},
+ {FrameDropPhase::kByDecoder, 0},
+ {FrameDropPhase::kAfterDecoder, 0}}));
+ EXPECT_THAT(stats.encoders, IsEmpty());
+ EXPECT_THAT(stats.decoders, IsEmpty());
+}
+
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ PreEncodedInFlightFrameAccountedInStats) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer,
+ DefaultVideoQualityAnalyzerOptions());
+
+ Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ // Frame captured
+ FrameStats frame_stats(/*frame_id=*/1, captured_time);
+ // Frame pre encoded
+ frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
+ captured_time, captured_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kFrameInFlight, frame_stats);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ EXPECT_EQ(comparator.stream_stats().size(), 1lu);
+ StreamStats stats = comparator.stream_stats().at(stats_key);
+ EXPECT_EQ(stats.stream_started_time, captured_time);
+ ExpectEmpty(stats.psnr);
+ ExpectEmpty(stats.ssim);
+ ExpectEmpty(stats.transport_time_ms);
+ ExpectEmpty(stats.total_delay_incl_transport_ms);
+ ExpectEmpty(stats.time_between_rendered_frames_ms);
+ ExpectEmpty(stats.encode_frame_rate);
+ ExpectEmpty(stats.encode_time_ms);
+ ExpectEmpty(stats.decode_time_ms);
+ ExpectEmpty(stats.receive_to_render_time_ms);
+ ExpectEmpty(stats.skipped_between_rendered);
+ ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0);
+ ExpectEmpty(stats.time_between_freezes_ms);
+ ExpectEmpty(stats.resolution_of_decoded_frame);
+ ExpectEmpty(stats.target_encode_bitrate);
+ EXPECT_THAT(stats.spatial_layers_qp, IsEmpty());
+ ExpectEmpty(stats.recv_key_frame_size_bytes);
+ ExpectEmpty(stats.recv_delta_frame_size_bytes);
+ EXPECT_EQ(stats.total_encoded_images_payload, 0);
+ EXPECT_EQ(stats.num_send_key_frames, 0);
+ EXPECT_EQ(stats.num_recv_key_frames, 0);
+ EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
+ {FrameDropPhase::kBeforeEncoder, 0},
+ {FrameDropPhase::kByEncoder, 0},
+ {FrameDropPhase::kTransport, 0},
+ {FrameDropPhase::kByDecoder, 0},
+ {FrameDropPhase::kAfterDecoder, 0}}));
+ EXPECT_THAT(stats.encoders, IsEmpty());
+ EXPECT_THAT(stats.decoders, IsEmpty());
+}
+
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ EncodedInFlightKeyFrameAccountedInStats) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer,
+ DefaultVideoQualityAnalyzerOptions());
+
+ Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
+ uint16_t frame_id = 1;
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ // Frame captured
+ FrameStats frame_stats(/*frame_id=*/1, captured_time);
+ // Frame pre encoded
+ frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
+ // Frame encoded
+ frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
+ frame_stats.used_encoder =
+ Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
+ frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
+ frame_stats.encoded_image_size = DataSize::Bytes(1000);
+ frame_stats.target_encode_bitrate = 2000;
+ frame_stats.spatial_layers_qp = {
+ {0, StatsCounter(
+ /*samples=*/{{5, Timestamp::Seconds(1)},
+ {5, Timestamp::Seconds(2)}})}};
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
+ captured_time, captured_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kFrameInFlight, frame_stats);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ EXPECT_EQ(comparator.stream_stats().size(), 1lu);
+ StreamStats stats = comparator.stream_stats().at(stats_key);
+ EXPECT_EQ(stats.stream_started_time, captured_time);
+ ExpectEmpty(stats.psnr);
+ ExpectEmpty(stats.ssim);
+ ExpectEmpty(stats.transport_time_ms);
+ ExpectEmpty(stats.total_delay_incl_transport_ms);
+ ExpectEmpty(stats.time_between_rendered_frames_ms);
+ ExpectEmpty(stats.encode_frame_rate);
+ ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0);
+ ExpectEmpty(stats.decode_time_ms);
+ ExpectEmpty(stats.receive_to_render_time_ms);
+ ExpectEmpty(stats.skipped_between_rendered);
+ ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0);
+ ExpectEmpty(stats.time_between_freezes_ms);
+ ExpectEmpty(stats.resolution_of_decoded_frame);
+ ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1,
+ /*value=*/2000.0);
+ EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1));
+ ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2,
+ /*value=*/5.0);
+ ExpectEmpty(stats.recv_key_frame_size_bytes);
+ ExpectEmpty(stats.recv_delta_frame_size_bytes);
+ EXPECT_EQ(stats.total_encoded_images_payload, 1000);
+ EXPECT_EQ(stats.num_send_key_frames, 1);
+ EXPECT_EQ(stats.num_recv_key_frames, 0);
+ EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
+ {FrameDropPhase::kBeforeEncoder, 0},
+ {FrameDropPhase::kByEncoder, 0},
+ {FrameDropPhase::kTransport, 0},
+ {FrameDropPhase::kByDecoder, 0},
+ {FrameDropPhase::kAfterDecoder, 0}}));
+ EXPECT_EQ(stats.encoders,
+ std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
+ EXPECT_THAT(stats.decoders, IsEmpty());
+}
+
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ EncodedInFlightDeltaFrameAccountedInStats) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer,
+ DefaultVideoQualityAnalyzerOptions());
+
+ Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
+ uint16_t frame_id = 1;
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ // Frame captured
+ FrameStats frame_stats(/*frame_id=*/1, captured_time);
+ // Frame pre encoded
+ frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
+ // Frame encoded
+ frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
+ frame_stats.used_encoder =
+ Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
+ frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameDelta;
+ frame_stats.encoded_image_size = DataSize::Bytes(1000);
+ frame_stats.target_encode_bitrate = 2000;
+ frame_stats.spatial_layers_qp = {
+ {0, StatsCounter(
+ /*samples=*/{{5, Timestamp::Seconds(1)},
+ {5, Timestamp::Seconds(2)}})}};
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
+ captured_time, captured_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kFrameInFlight, frame_stats);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ EXPECT_EQ(comparator.stream_stats().size(), 1lu);
+ StreamStats stats = comparator.stream_stats().at(stats_key);
+ EXPECT_EQ(stats.stream_started_time, captured_time);
+ ExpectEmpty(stats.psnr);
+ ExpectEmpty(stats.ssim);
+ ExpectEmpty(stats.transport_time_ms);
+ ExpectEmpty(stats.total_delay_incl_transport_ms);
+ ExpectEmpty(stats.time_between_rendered_frames_ms);
+ ExpectEmpty(stats.encode_frame_rate);
+ ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0);
+ ExpectEmpty(stats.decode_time_ms);
+ ExpectEmpty(stats.receive_to_render_time_ms);
+ ExpectEmpty(stats.skipped_between_rendered);
+ ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0);
+ ExpectEmpty(stats.time_between_freezes_ms);
+ ExpectEmpty(stats.resolution_of_decoded_frame);
+ ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1,
+ /*value=*/2000.0);
+ EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1));
+ ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2,
+ /*value=*/5.0);
+ ExpectEmpty(stats.recv_key_frame_size_bytes);
+ ExpectEmpty(stats.recv_delta_frame_size_bytes);
+ EXPECT_EQ(stats.total_encoded_images_payload, 1000);
+ EXPECT_EQ(stats.num_send_key_frames, 0);
+ EXPECT_EQ(stats.num_recv_key_frames, 0);
+ EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
+ {FrameDropPhase::kBeforeEncoder, 0},
+ {FrameDropPhase::kByEncoder, 0},
+ {FrameDropPhase::kTransport, 0},
+ {FrameDropPhase::kByDecoder, 0},
+ {FrameDropPhase::kAfterDecoder, 0}}));
+ EXPECT_EQ(stats.encoders,
+ std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
+ EXPECT_THAT(stats.decoders, IsEmpty());
+}
+
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ PreDecodedInFlightKeyFrameAccountedInStats) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer,
+ DefaultVideoQualityAnalyzerOptions());
+
+ Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
+ uint16_t frame_id = 1;
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ // Frame captured
+ FrameStats frame_stats(/*frame_id=*/1, captured_time);
+ // Frame pre encoded
+ frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
+ // Frame encoded
+ frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
+ frame_stats.used_encoder =
+ Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
+ frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
+ frame_stats.encoded_image_size = DataSize::Bytes(1000);
+ frame_stats.target_encode_bitrate = 2000;
+ frame_stats.spatial_layers_qp = {
+ {0, StatsCounter(
+ /*samples=*/{{5, Timestamp::Seconds(1)},
+ {5, Timestamp::Seconds(2)}})}};
+ // Frame pre decoded
+ frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey;
+ frame_stats.pre_decoded_image_size = DataSize::Bytes(500);
+ frame_stats.received_time = captured_time + TimeDelta::Millis(30);
+ frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
+ captured_time, captured_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kFrameInFlight, frame_stats);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ EXPECT_EQ(comparator.stream_stats().size(), 1lu);
+ StreamStats stats = comparator.stream_stats().at(stats_key);
+ EXPECT_EQ(stats.stream_started_time, captured_time);
+ ExpectEmpty(stats.psnr);
+ ExpectEmpty(stats.ssim);
+ ExpectSizeAndAllElementsAre(stats.transport_time_ms, /*size=*/1,
+ /*value=*/20.0);
+ ExpectEmpty(stats.total_delay_incl_transport_ms);
+ ExpectEmpty(stats.time_between_rendered_frames_ms);
+ ExpectEmpty(stats.encode_frame_rate);
+ ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0);
+ ExpectEmpty(stats.decode_time_ms);
+ ExpectEmpty(stats.receive_to_render_time_ms);
+ ExpectEmpty(stats.skipped_between_rendered);
+ ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0);
+ ExpectEmpty(stats.time_between_freezes_ms);
+ ExpectEmpty(stats.resolution_of_decoded_frame);
+ ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1,
+ /*value=*/2000.0);
+ EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1));
+ ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2,
+ /*value=*/5.0);
+ ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1,
+ /*value=*/500.0);
+ ExpectEmpty(stats.recv_delta_frame_size_bytes);
+ EXPECT_EQ(stats.total_encoded_images_payload, 1000);
+ EXPECT_EQ(stats.num_send_key_frames, 1);
+ EXPECT_EQ(stats.num_recv_key_frames, 1);
+ EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
+ {FrameDropPhase::kBeforeEncoder, 0},
+ {FrameDropPhase::kByEncoder, 0},
+ {FrameDropPhase::kTransport, 0},
+ {FrameDropPhase::kByDecoder, 0},
+ {FrameDropPhase::kAfterDecoder, 0}}));
+ EXPECT_EQ(stats.encoders,
+ std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
+ EXPECT_THAT(stats.decoders, IsEmpty());
+}
+
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ DecodedInFlightKeyFrameAccountedInStats) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer,
+ DefaultVideoQualityAnalyzerOptions());
+
+ Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
+ uint16_t frame_id = 1;
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ // Frame captured
+ FrameStats frame_stats(/*frame_id=*/1, captured_time);
+ // Frame pre encoded
+ frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
+ // Frame encoded
+ frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
+ frame_stats.used_encoder =
+ Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
+ frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
+ frame_stats.encoded_image_size = DataSize::Bytes(1000);
+ frame_stats.target_encode_bitrate = 2000;
+ frame_stats.spatial_layers_qp = {
+ {0, StatsCounter(
+ /*samples=*/{{5, Timestamp::Seconds(1)},
+ {5, Timestamp::Seconds(2)}})}};
+ // Frame pre decoded
+ frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey;
+ frame_stats.pre_decoded_image_size = DataSize::Bytes(500);
+ frame_stats.received_time = captured_time + TimeDelta::Millis(30);
+ frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
+ // Frame decoded
+ frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50);
+ frame_stats.decoded_frame_width = 200;
+ frame_stats.decoded_frame_height = 100;
+
+ frame_stats.used_decoder =
+ Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time);
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
+ captured_time, captured_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kFrameInFlight, frame_stats);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ EXPECT_EQ(comparator.stream_stats().size(), 1lu);
+ StreamStats stats = comparator.stream_stats().at(stats_key);
+ EXPECT_EQ(stats.stream_started_time, captured_time);
+ ExpectEmpty(stats.psnr);
+ ExpectEmpty(stats.ssim);
+ ExpectSizeAndAllElementsAre(stats.transport_time_ms, /*size=*/1,
+ /*value=*/20.0);
+ ExpectEmpty(stats.total_delay_incl_transport_ms);
+ ExpectEmpty(stats.time_between_rendered_frames_ms);
+ ExpectEmpty(stats.encode_frame_rate);
+ ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0);
+ ExpectSizeAndAllElementsAre(stats.decode_time_ms, /*size=*/1, /*value=*/10.0);
+ ExpectEmpty(stats.receive_to_render_time_ms);
+ ExpectEmpty(stats.skipped_between_rendered);
+ ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0);
+ ExpectEmpty(stats.time_between_freezes_ms);
+ EXPECT_GE(GetFirstOrDie(stats.resolution_of_decoded_frame), 200 * 100.0);
+ ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1,
+ /*value=*/2000.0);
+ EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1));
+ ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2,
+ /*value=*/5.0);
+ ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1,
+ /*value=*/500.0);
+ ExpectEmpty(stats.recv_delta_frame_size_bytes);
+ EXPECT_EQ(stats.total_encoded_images_payload, 1000);
+ EXPECT_EQ(stats.num_send_key_frames, 1);
+ EXPECT_EQ(stats.num_recv_key_frames, 1);
+ EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
+ {FrameDropPhase::kBeforeEncoder, 0},
+ {FrameDropPhase::kByEncoder, 0},
+ {FrameDropPhase::kTransport, 0},
+ {FrameDropPhase::kByDecoder, 0},
+ {FrameDropPhase::kAfterDecoder, 0}}));
+ EXPECT_EQ(stats.encoders,
+ std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
+ EXPECT_EQ(stats.decoders,
+ std::vector<StreamCodecInfo>{*frame_stats.used_decoder});
+}
+
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ DecoderFailureOnInFlightKeyFrameAccountedInStats) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer,
+ DefaultVideoQualityAnalyzerOptions());
+
+ Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
+ uint16_t frame_id = 1;
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ // Frame captured
+ FrameStats frame_stats(/*frame_id=*/1, captured_time);
+ // Frame pre encoded
+ frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
+ // Frame encoded
+ frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
+ frame_stats.used_encoder =
+ Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
+ frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
+ frame_stats.encoded_image_size = DataSize::Bytes(1000);
+ frame_stats.target_encode_bitrate = 2000;
+ frame_stats.spatial_layers_qp = {
+ {0, StatsCounter(
+ /*samples=*/{{5, Timestamp::Seconds(1)},
+ {5, Timestamp::Seconds(2)}})}};
+ // Frame pre decoded
+ frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey;
+ frame_stats.pre_decoded_image_size = DataSize::Bytes(500);
+ frame_stats.received_time = captured_time + TimeDelta::Millis(30);
+ frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
+ // Frame decoded
+ frame_stats.decoder_failed = true;
+ frame_stats.used_decoder =
+ Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time);
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
+ captured_time, captured_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kFrameInFlight, frame_stats);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ EXPECT_EQ(comparator.stream_stats().size(), 1lu);
+ StreamStats stats = comparator.stream_stats().at(stats_key);
+ EXPECT_EQ(stats.stream_started_time, captured_time);
+ ExpectEmpty(stats.psnr);
+ ExpectEmpty(stats.ssim);
+ ExpectSizeAndAllElementsAre(stats.transport_time_ms, /*size=*/1,
+ /*value=*/20.0);
+ ExpectEmpty(stats.total_delay_incl_transport_ms);
+ ExpectEmpty(stats.time_between_rendered_frames_ms);
+ ExpectEmpty(stats.encode_frame_rate);
+ ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0);
+ ExpectEmpty(stats.decode_time_ms);
+ ExpectEmpty(stats.receive_to_render_time_ms);
+ ExpectEmpty(stats.skipped_between_rendered);
+ ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0);
+ ExpectEmpty(stats.time_between_freezes_ms);
+ ExpectEmpty(stats.resolution_of_decoded_frame);
+ ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1,
+ /*value=*/2000.0);
+ EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1));
+ ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2,
+ /*value=*/5.0);
+ ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1,
+ /*value=*/500.0);
+ ExpectEmpty(stats.recv_delta_frame_size_bytes);
+ EXPECT_EQ(stats.total_encoded_images_payload, 1000);
+ EXPECT_EQ(stats.num_send_key_frames, 1);
+ EXPECT_EQ(stats.num_recv_key_frames, 1);
+ // All frame in flight are not considered as dropped.
+ EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
+ {FrameDropPhase::kBeforeEncoder, 0},
+ {FrameDropPhase::kByEncoder, 0},
+ {FrameDropPhase::kTransport, 0},
+ {FrameDropPhase::kByDecoder, 0},
+ {FrameDropPhase::kAfterDecoder, 0}}));
+ EXPECT_EQ(stats.encoders,
+ std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
+ EXPECT_EQ(stats.decoders,
+ std::vector<StreamCodecInfo>{*frame_stats.used_decoder});
+}
+// Frame in flight end
+
+// Dropped frame start
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ CapturedOnlyDroppedFrameAccountedInStats) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer,
+ DefaultVideoQualityAnalyzerOptions());
+
+ Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ // Frame captured
+ FrameStats frame_stats(/*frame_id=*/1, captured_time);
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
+ captured_time, captured_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kDroppedFrame, frame_stats);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ EXPECT_EQ(comparator.stream_stats().size(), 1lu);
+ StreamStats stats = comparator.stream_stats().at(stats_key);
+ EXPECT_EQ(stats.stream_started_time, captured_time);
+ ExpectEmpty(stats.psnr);
+ ExpectEmpty(stats.ssim);
+ ExpectEmpty(stats.transport_time_ms);
+ ExpectEmpty(stats.total_delay_incl_transport_ms);
+ ExpectEmpty(stats.time_between_rendered_frames_ms);
+ ExpectEmpty(stats.encode_frame_rate);
+ ExpectEmpty(stats.encode_time_ms);
+ ExpectEmpty(stats.decode_time_ms);
+ ExpectEmpty(stats.receive_to_render_time_ms);
+ ExpectEmpty(stats.skipped_between_rendered);
+ ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0);
+ ExpectEmpty(stats.time_between_freezes_ms);
+ ExpectEmpty(stats.resolution_of_decoded_frame);
+ ExpectEmpty(stats.target_encode_bitrate);
+ EXPECT_THAT(stats.spatial_layers_qp, IsEmpty());
+ ExpectEmpty(stats.recv_key_frame_size_bytes);
+ ExpectEmpty(stats.recv_delta_frame_size_bytes);
+ EXPECT_EQ(stats.total_encoded_images_payload, 0);
+ EXPECT_EQ(stats.num_send_key_frames, 0);
+ EXPECT_EQ(stats.num_recv_key_frames, 0);
+ EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
+ {FrameDropPhase::kBeforeEncoder, 1},
+ {FrameDropPhase::kByEncoder, 0},
+ {FrameDropPhase::kTransport, 0},
+ {FrameDropPhase::kByDecoder, 0},
+ {FrameDropPhase::kAfterDecoder, 0}}));
+ EXPECT_THAT(stats.encoders, IsEmpty());
+ EXPECT_THAT(stats.decoders, IsEmpty());
+}
+
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ PreEncodedDroppedFrameAccountedInStats) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer,
+ DefaultVideoQualityAnalyzerOptions());
+
+ Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ // Frame captured
+ FrameStats frame_stats(/*frame_id=*/1, captured_time);
+ // Frame pre encoded
+ frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
+ captured_time, captured_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kDroppedFrame, frame_stats);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ EXPECT_EQ(comparator.stream_stats().size(), 1lu);
+ StreamStats stats = comparator.stream_stats().at(stats_key);
+ EXPECT_EQ(stats.stream_started_time, captured_time);
+ ExpectEmpty(stats.psnr);
+ ExpectEmpty(stats.ssim);
+ ExpectEmpty(stats.transport_time_ms);
+ ExpectEmpty(stats.total_delay_incl_transport_ms);
+ ExpectEmpty(stats.time_between_rendered_frames_ms);
+ ExpectEmpty(stats.encode_frame_rate);
+ ExpectEmpty(stats.encode_time_ms);
+ ExpectEmpty(stats.decode_time_ms);
+ ExpectEmpty(stats.receive_to_render_time_ms);
+ ExpectEmpty(stats.skipped_between_rendered);
+ ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0);
+ ExpectEmpty(stats.time_between_freezes_ms);
+ ExpectEmpty(stats.resolution_of_decoded_frame);
+ ExpectEmpty(stats.target_encode_bitrate);
+ EXPECT_THAT(stats.spatial_layers_qp, IsEmpty());
+ ExpectEmpty(stats.recv_key_frame_size_bytes);
+ ExpectEmpty(stats.recv_delta_frame_size_bytes);
+ EXPECT_EQ(stats.total_encoded_images_payload, 0);
+ EXPECT_EQ(stats.num_send_key_frames, 0);
+ EXPECT_EQ(stats.num_recv_key_frames, 0);
+ EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
+ {FrameDropPhase::kBeforeEncoder, 0},
+ {FrameDropPhase::kByEncoder, 1},
+ {FrameDropPhase::kTransport, 0},
+ {FrameDropPhase::kByDecoder, 0},
+ {FrameDropPhase::kAfterDecoder, 0}}));
+ EXPECT_THAT(stats.encoders, IsEmpty());
+ EXPECT_THAT(stats.decoders, IsEmpty());
+}
+
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ EncodedDroppedKeyFrameAccountedInStats) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer,
+ DefaultVideoQualityAnalyzerOptions());
+
+ Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
+ uint16_t frame_id = 1;
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ // Frame captured
+ FrameStats frame_stats(/*frame_id=*/1, captured_time);
+ // Frame pre encoded
+ frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
+ // Frame encoded
+ frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
+ frame_stats.used_encoder =
+ Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
+ frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
+ frame_stats.encoded_image_size = DataSize::Bytes(1000);
+ frame_stats.target_encode_bitrate = 2000;
+ frame_stats.spatial_layers_qp = {
+ {0, StatsCounter(
+ /*samples=*/{{5, Timestamp::Seconds(1)},
+ {5, Timestamp::Seconds(2)}})}};
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
+ captured_time, captured_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kDroppedFrame, frame_stats);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ EXPECT_EQ(comparator.stream_stats().size(), 1lu);
+ StreamStats stats = comparator.stream_stats().at(stats_key);
+ EXPECT_EQ(stats.stream_started_time, captured_time);
+ ExpectEmpty(stats.psnr);
+ ExpectEmpty(stats.ssim);
+ ExpectEmpty(stats.transport_time_ms);
+ ExpectEmpty(stats.total_delay_incl_transport_ms);
+ ExpectEmpty(stats.time_between_rendered_frames_ms);
+ ExpectEmpty(stats.encode_frame_rate);
+ ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0);
+ ExpectEmpty(stats.decode_time_ms);
+ ExpectEmpty(stats.receive_to_render_time_ms);
+ ExpectEmpty(stats.skipped_between_rendered);
+ ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0);
+ ExpectEmpty(stats.time_between_freezes_ms);
+ ExpectEmpty(stats.resolution_of_decoded_frame);
+ ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1,
+ /*value=*/2000.0);
+ EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1));
+ ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2,
+ /*value=*/5.0);
+ ExpectEmpty(stats.recv_key_frame_size_bytes);
+ ExpectEmpty(stats.recv_delta_frame_size_bytes);
+ EXPECT_EQ(stats.total_encoded_images_payload, 1000);
+ EXPECT_EQ(stats.num_send_key_frames, 1);
+ EXPECT_EQ(stats.num_recv_key_frames, 0);
+ EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
+ {FrameDropPhase::kBeforeEncoder, 0},
+ {FrameDropPhase::kByEncoder, 0},
+ {FrameDropPhase::kTransport, 1},
+ {FrameDropPhase::kByDecoder, 0},
+ {FrameDropPhase::kAfterDecoder, 0}}));
+ EXPECT_EQ(stats.encoders,
+ std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
+ EXPECT_THAT(stats.decoders, IsEmpty());
+}
+
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ EncodedDroppedDeltaFrameAccountedInStats) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer,
+ DefaultVideoQualityAnalyzerOptions());
+
+ Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
+ uint16_t frame_id = 1;
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ // Frame captured
+ FrameStats frame_stats(/*frame_id=*/1, captured_time);
+ // Frame pre encoded
+ frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
+ // Frame encoded
+ frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
+ frame_stats.used_encoder =
+ Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
+ frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameDelta;
+ frame_stats.encoded_image_size = DataSize::Bytes(1000);
+ frame_stats.target_encode_bitrate = 2000;
+ frame_stats.spatial_layers_qp = {
+ {0, StatsCounter(
+ /*samples=*/{{5, Timestamp::Seconds(1)},
+ {5, Timestamp::Seconds(2)}})}};
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
+ captured_time, captured_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kDroppedFrame, frame_stats);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ EXPECT_EQ(comparator.stream_stats().size(), 1lu);
+ StreamStats stats = comparator.stream_stats().at(stats_key);
+ EXPECT_EQ(stats.stream_started_time, captured_time);
+ ExpectEmpty(stats.psnr);
+ ExpectEmpty(stats.ssim);
+ ExpectEmpty(stats.transport_time_ms);
+ ExpectEmpty(stats.total_delay_incl_transport_ms);
+ ExpectEmpty(stats.time_between_rendered_frames_ms);
+ ExpectEmpty(stats.encode_frame_rate);
+ ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0);
+ ExpectEmpty(stats.decode_time_ms);
+ ExpectEmpty(stats.receive_to_render_time_ms);
+ ExpectEmpty(stats.skipped_between_rendered);
+ ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0);
+ ExpectEmpty(stats.time_between_freezes_ms);
+ ExpectEmpty(stats.resolution_of_decoded_frame);
+ ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1,
+ /*value=*/2000.0);
+ EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1));
+ ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2,
+ /*value=*/5.0);
+ ExpectEmpty(stats.recv_key_frame_size_bytes);
+ ExpectEmpty(stats.recv_delta_frame_size_bytes);
+ EXPECT_EQ(stats.total_encoded_images_payload, 1000);
+ EXPECT_EQ(stats.num_send_key_frames, 0);
+ EXPECT_EQ(stats.num_recv_key_frames, 0);
+ EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
+ {FrameDropPhase::kBeforeEncoder, 0},
+ {FrameDropPhase::kByEncoder, 0},
+ {FrameDropPhase::kTransport, 1},
+ {FrameDropPhase::kByDecoder, 0},
+ {FrameDropPhase::kAfterDecoder, 0}}));
+ EXPECT_EQ(stats.encoders,
+ std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
+ EXPECT_THAT(stats.decoders, IsEmpty());
+}
+
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ PreDecodedDroppedKeyFrameAccountedInStats) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer,
+ DefaultVideoQualityAnalyzerOptions());
+
+ Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
+ uint16_t frame_id = 1;
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ // Frame captured
+ FrameStats frame_stats(/*frame_id=*/1, captured_time);
+ // Frame pre encoded
+ frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
+ // Frame encoded
+ frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
+ frame_stats.used_encoder =
+ Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
+ frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
+ frame_stats.encoded_image_size = DataSize::Bytes(1000);
+ frame_stats.target_encode_bitrate = 2000;
+ // Frame pre decoded
+ frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey;
+ frame_stats.pre_decoded_image_size = DataSize::Bytes(500);
+ frame_stats.received_time = captured_time + TimeDelta::Millis(30);
+ frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
+ captured_time, captured_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kDroppedFrame, frame_stats);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ EXPECT_EQ(comparator.stream_stats().size(), 1lu);
+ StreamStats stats = comparator.stream_stats().at(stats_key);
+ EXPECT_EQ(stats.stream_started_time, captured_time);
+ ExpectEmpty(stats.psnr);
+ ExpectEmpty(stats.ssim);
+ ExpectEmpty(stats.transport_time_ms);
+ ExpectEmpty(stats.total_delay_incl_transport_ms);
+ ExpectEmpty(stats.time_between_rendered_frames_ms);
+ ExpectEmpty(stats.encode_frame_rate);
+ ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0);
+ ExpectEmpty(stats.decode_time_ms);
+ ExpectEmpty(stats.receive_to_render_time_ms);
+ ExpectEmpty(stats.skipped_between_rendered);
+ ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0);
+ ExpectEmpty(stats.time_between_freezes_ms);
+ ExpectEmpty(stats.resolution_of_decoded_frame);
+ ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1,
+ /*value=*/2000.0);
+ ExpectEmpty(stats.recv_key_frame_size_bytes);
+ ExpectEmpty(stats.recv_delta_frame_size_bytes);
+ EXPECT_EQ(stats.total_encoded_images_payload, 1000);
+ EXPECT_EQ(stats.num_send_key_frames, 1);
+ EXPECT_EQ(stats.num_recv_key_frames, 0);
+ EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
+ {FrameDropPhase::kBeforeEncoder, 0},
+ {FrameDropPhase::kByEncoder, 0},
+ {FrameDropPhase::kTransport, 0},
+ {FrameDropPhase::kByDecoder, 1},
+ {FrameDropPhase::kAfterDecoder, 0}}));
+ EXPECT_EQ(stats.encoders,
+ std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
+ EXPECT_THAT(stats.decoders, IsEmpty());
+}
+
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ DecodedDroppedKeyFrameAccountedInStats) {
+ // We don't really drop frames after decoder, so it's a bit unclear what is
+ // correct way to account such frames in stats, so this test just fixes some
+ // current way.
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer,
+ DefaultVideoQualityAnalyzerOptions());
+
+ Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
+ uint16_t frame_id = 1;
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ // Frame captured
+ FrameStats frame_stats(/*frame_id=*/1, captured_time);
+ // Frame pre encoded
+ frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
+ // Frame encoded
+ frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
+ frame_stats.used_encoder =
+ Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
+ frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
+ frame_stats.encoded_image_size = DataSize::Bytes(1000);
+ frame_stats.target_encode_bitrate = 2000;
+ frame_stats.spatial_layers_qp = {
+ {0, StatsCounter(
+ /*samples=*/{{5, Timestamp::Seconds(1)},
+ {5, Timestamp::Seconds(2)}})}};
+ // Frame pre decoded
+ frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey;
+ frame_stats.pre_decoded_image_size = DataSize::Bytes(500);
+ frame_stats.received_time = captured_time + TimeDelta::Millis(30);
+ frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
+ // Frame decoded
+ frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50);
+ frame_stats.used_decoder =
+ Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time);
+ frame_stats.decoded_frame_width = 200;
+ frame_stats.decoded_frame_height = 100;
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
+ captured_time, captured_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kDroppedFrame, frame_stats);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ EXPECT_EQ(comparator.stream_stats().size(), 1lu);
+ StreamStats stats = comparator.stream_stats().at(stats_key);
+ EXPECT_EQ(stats.stream_started_time, captured_time);
+ ExpectEmpty(stats.psnr);
+ ExpectEmpty(stats.ssim);
+ ExpectEmpty(stats.transport_time_ms);
+ ExpectEmpty(stats.total_delay_incl_transport_ms);
+ ExpectEmpty(stats.time_between_rendered_frames_ms);
+ ExpectEmpty(stats.encode_frame_rate);
+ ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0);
+ ExpectEmpty(stats.decode_time_ms);
+ ExpectEmpty(stats.receive_to_render_time_ms);
+ ExpectEmpty(stats.skipped_between_rendered);
+ ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0);
+ ExpectEmpty(stats.time_between_freezes_ms);
+ ExpectEmpty(stats.resolution_of_decoded_frame);
+ ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1,
+ /*value=*/2000.0);
+ EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1));
+ ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2,
+ /*value=*/5.0);
+ ExpectEmpty(stats.recv_key_frame_size_bytes);
+ ExpectEmpty(stats.recv_delta_frame_size_bytes);
+ EXPECT_EQ(stats.total_encoded_images_payload, 1000);
+ EXPECT_EQ(stats.num_send_key_frames, 1);
+ EXPECT_EQ(stats.num_recv_key_frames, 0);
+ EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
+ {FrameDropPhase::kBeforeEncoder, 0},
+ {FrameDropPhase::kByEncoder, 0},
+ {FrameDropPhase::kTransport, 0},
+ {FrameDropPhase::kByDecoder, 0},
+ {FrameDropPhase::kAfterDecoder, 1}}));
+ EXPECT_EQ(stats.encoders,
+ std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
+ EXPECT_EQ(stats.decoders,
+ std::vector<StreamCodecInfo>{*frame_stats.used_decoder});
+}
+
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ DecoderFailedDroppedKeyFrameAccountedInStats) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer,
+ DefaultVideoQualityAnalyzerOptions());
+
+ Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
+ uint16_t frame_id = 1;
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ // Frame captured
+ FrameStats frame_stats(/*frame_id=*/1, captured_time);
+ // Frame pre encoded
+ frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
+ // Frame encoded
+ frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
+ frame_stats.used_encoder =
+ Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
+ frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
+ frame_stats.encoded_image_size = DataSize::Bytes(1000);
+ frame_stats.target_encode_bitrate = 2000;
+ frame_stats.spatial_layers_qp = {
+ {0, StatsCounter(
+ /*samples=*/{{5, Timestamp::Seconds(1)},
+ {5, Timestamp::Seconds(2)}})}};
+ // Frame pre decoded
+ frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey;
+ frame_stats.pre_decoded_image_size = DataSize::Bytes(500);
+ frame_stats.received_time = captured_time + TimeDelta::Millis(30);
+ frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
+ // Frame decoded
+ frame_stats.decoder_failed = true;
+ frame_stats.used_decoder =
+ Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time);
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
+ captured_time, captured_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kDroppedFrame, frame_stats);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ EXPECT_EQ(comparator.stream_stats().size(), 1lu);
+ StreamStats stats = comparator.stream_stats().at(stats_key);
+ EXPECT_EQ(stats.stream_started_time, captured_time);
+ ExpectEmpty(stats.psnr);
+ ExpectEmpty(stats.ssim);
+ ExpectSizeAndAllElementsAre(stats.transport_time_ms, /*size=*/1,
+ /*value=*/20.0);
+ ExpectEmpty(stats.total_delay_incl_transport_ms);
+ ExpectEmpty(stats.time_between_rendered_frames_ms);
+ ExpectEmpty(stats.encode_frame_rate);
+ ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0);
+ ExpectEmpty(stats.decode_time_ms);
+ ExpectEmpty(stats.receive_to_render_time_ms);
+ ExpectEmpty(stats.skipped_between_rendered);
+ ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0);
+ ExpectEmpty(stats.time_between_freezes_ms);
+ ExpectEmpty(stats.resolution_of_decoded_frame);
+ ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1,
+ /*value=*/2000.0);
+ EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1));
+ ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2,
+ /*value=*/5.0);
+ ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1,
+ /*value=*/500.0);
+ ExpectEmpty(stats.recv_delta_frame_size_bytes);
+ EXPECT_EQ(stats.total_encoded_images_payload, 1000);
+ EXPECT_EQ(stats.num_send_key_frames, 1);
+ EXPECT_EQ(stats.num_recv_key_frames, 1);
+ EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
+ {FrameDropPhase::kBeforeEncoder, 0},
+ {FrameDropPhase::kByEncoder, 0},
+ {FrameDropPhase::kTransport, 0},
+ {FrameDropPhase::kByDecoder, 1},
+ {FrameDropPhase::kAfterDecoder, 0}}));
+ EXPECT_EQ(stats.encoders,
+ std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
+ EXPECT_EQ(stats.decoders,
+ std::vector<StreamCodecInfo>{*frame_stats.used_decoder});
+}
+// Dropped frame end
+
+// Regular frame start
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ RenderedKeyFrameAccountedInStats) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer,
+ DefaultVideoQualityAnalyzerOptions());
+
+ Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
+ uint16_t frame_id = 1;
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ // Frame captured
+ VideoFrame frame =
+ CreateFrame(frame_id, /*width=*/320, /*height=*/180, captured_time);
+ FrameStats frame_stats(/*frame_id=*/1, captured_time);
+ // Frame pre encoded
+ frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
+ // Frame encoded
+ frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
+ frame_stats.used_encoder =
+ Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
+ frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
+ frame_stats.encoded_image_size = DataSize::Bytes(1000);
+ frame_stats.target_encode_bitrate = 2000;
+ frame_stats.spatial_layers_qp = {
+ {0, StatsCounter(
+ /*samples=*/{{5, Timestamp::Seconds(1)},
+ {5, Timestamp::Seconds(2)}})}};
+ // Frame pre decoded
+ frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey;
+ frame_stats.pre_decoded_image_size = DataSize::Bytes(500);
+ frame_stats.received_time = captured_time + TimeDelta::Millis(30);
+ frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
+ // Frame decoded
+ frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50);
+ frame_stats.used_decoder =
+ Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time);
+ frame_stats.decoded_frame_width = 200;
+ frame_stats.decoded_frame_height = 100;
+ // Frame rendered
+ frame_stats.rendered_time = captured_time + TimeDelta::Millis(60);
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
+ captured_time, captured_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/frame,
+ /*rendered=*/frame, FrameComparisonType::kRegular,
+ frame_stats);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ EXPECT_EQ(comparator.stream_stats().size(), 1lu);
+ StreamStats stats = comparator.stream_stats().at(stats_key);
+ EXPECT_EQ(stats.stream_started_time, captured_time);
+ EXPECT_GE(GetFirstOrDie(stats.psnr), 20);
+ EXPECT_GE(GetFirstOrDie(stats.ssim), 0.5);
+ ExpectSizeAndAllElementsAre(stats.transport_time_ms, /*size=*/1,
+ /*value=*/20.0);
+ EXPECT_GE(GetFirstOrDie(stats.total_delay_incl_transport_ms), 60.0);
+ ExpectEmpty(stats.time_between_rendered_frames_ms);
+ ExpectEmpty(stats.encode_frame_rate);
+ ExpectSizeAndAllElementsAre(stats.encode_time_ms, /*size=*/1, /*value=*/10.0);
+ EXPECT_GE(GetFirstOrDie(stats.decode_time_ms), 10.0);
+ EXPECT_GE(GetFirstOrDie(stats.receive_to_render_time_ms), 30.0);
+ ExpectEmpty(stats.skipped_between_rendered);
+ ExpectSizeAndAllElementsAre(stats.freeze_time_ms, /*size=*/1, /*value=*/0);
+ ExpectEmpty(stats.time_between_freezes_ms);
+ EXPECT_GE(GetFirstOrDie(stats.resolution_of_decoded_frame), 200 * 100.0);
+ ExpectSizeAndAllElementsAre(stats.target_encode_bitrate, /*size=*/1,
+ /*value=*/2000.0);
+ EXPECT_THAT(stats.spatial_layers_qp, SizeIs(1));
+ ExpectSizeAndAllElementsAre(stats.spatial_layers_qp[0], /*size=*/2,
+ /*value=*/5.0);
+ ExpectSizeAndAllElementsAre(stats.recv_key_frame_size_bytes, /*size=*/1,
+ /*value=*/500.0);
+ ExpectEmpty(stats.recv_delta_frame_size_bytes);
+ EXPECT_EQ(stats.total_encoded_images_payload, 1000);
+ EXPECT_EQ(stats.num_send_key_frames, 1);
+ EXPECT_EQ(stats.num_recv_key_frames, 1);
+ EXPECT_THAT(stats.dropped_by_phase, Eq(std::map<FrameDropPhase, int64_t>{
+ {FrameDropPhase::kBeforeEncoder, 0},
+ {FrameDropPhase::kByEncoder, 0},
+ {FrameDropPhase::kTransport, 0},
+ {FrameDropPhase::kByDecoder, 0},
+ {FrameDropPhase::kAfterDecoder, 0}}));
+ EXPECT_EQ(stats.encoders,
+ std::vector<StreamCodecInfo>{*frame_stats.used_encoder});
+ EXPECT_EQ(stats.decoders,
+ std::vector<StreamCodecInfo>{*frame_stats.used_decoder});
+}
+
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest, AllStatsHaveMetadataSet) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer,
+ DefaultVideoQualityAnalyzerOptions());
+
+ Timestamp captured_time = Clock::GetRealTimeClock()->CurrentTime();
+ uint16_t frame_id = 1;
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ // Frame captured
+ VideoFrame frame =
+ CreateFrame(frame_id, /*width=*/320, /*height=*/180, captured_time);
+ FrameStats frame_stats(/*frame_id=*/1, captured_time);
+ // Frame pre encoded
+ frame_stats.pre_encode_time = captured_time + TimeDelta::Millis(10);
+ // Frame encoded
+ frame_stats.encoded_time = captured_time + TimeDelta::Millis(20);
+ frame_stats.used_encoder =
+ Vp8CodecForOneFrame(frame_id, frame_stats.encoded_time);
+ frame_stats.encoded_frame_type = VideoFrameType::kVideoFrameKey;
+ frame_stats.encoded_image_size = DataSize::Bytes(1000);
+ frame_stats.target_encode_bitrate = 2000;
+ frame_stats.spatial_layers_qp = {
+ {0, StatsCounter(
+ /*samples=*/{{5, Timestamp::Seconds(1)},
+ {5, Timestamp::Seconds(2)}})}};
+ // Frame pre decoded
+ frame_stats.pre_decoded_frame_type = VideoFrameType::kVideoFrameKey;
+ frame_stats.pre_decoded_image_size = DataSize::Bytes(500);
+ frame_stats.received_time = captured_time + TimeDelta::Millis(30);
+ frame_stats.decode_start_time = captured_time + TimeDelta::Millis(40);
+ // Frame decoded
+ frame_stats.decode_end_time = captured_time + TimeDelta::Millis(50);
+ frame_stats.used_decoder =
+ Vp8CodecForOneFrame(frame_id, frame_stats.decode_end_time);
+ // Frame rendered
+ frame_stats.rendered_time = captured_time + TimeDelta::Millis(60);
+ frame_stats.decoded_frame_width = 200;
+ frame_stats.decoded_frame_height = 100;
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, /*peers_count=*/2,
+ captured_time, captured_time);
+ comparator.AddComparison(stats_key,
+ /*captured=*/frame,
+ /*rendered=*/frame, FrameComparisonType::kRegular,
+ frame_stats);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ EXPECT_EQ(comparator.stream_stats().size(), 1lu);
+ StreamStats stats = comparator.stream_stats().at(stats_key);
+ AssertFirstMetadataHasField(stats.psnr, "frame_id", "1");
+ AssertFirstMetadataHasField(stats.ssim, "frame_id", "1");
+ AssertFirstMetadataHasField(stats.transport_time_ms, "frame_id", "1");
+ AssertFirstMetadataHasField(stats.total_delay_incl_transport_ms, "frame_id",
+ "1");
+ AssertFirstMetadataHasField(stats.encode_time_ms, "frame_id", "1");
+ AssertFirstMetadataHasField(stats.decode_time_ms, "frame_id", "1");
+ AssertFirstMetadataHasField(stats.receive_to_render_time_ms, "frame_id", "1");
+ AssertFirstMetadataHasField(stats.resolution_of_decoded_frame, "frame_id",
+ "1");
+ AssertFirstMetadataHasField(stats.target_encode_bitrate, "frame_id", "1");
+ AssertFirstMetadataHasField(stats.spatial_layers_qp[0], "frame_id", "1");
+ AssertFirstMetadataHasField(stats.recv_key_frame_size_bytes, "frame_id", "1");
+
+ ExpectEmpty(stats.recv_delta_frame_size_bytes);
+}
+// Regular frame end
+
+TEST(DefaultVideoQualityAnalyzerFramesComparatorTest,
+ FreezeStatsPresentedWithMetadataAfterAddFrameWithSkippedAndDelay) {
+ DefaultVideoQualityAnalyzerCpuMeasurer cpu_measurer;
+ DefaultVideoQualityAnalyzerFramesComparator comparator(
+ Clock::GetRealTimeClock(), cpu_measurer, AnalyzerOptionsForTest());
+
+ Timestamp stream_start_time = Clock::GetRealTimeClock()->CurrentTime();
+ size_t stream = 0;
+ size_t sender = 0;
+ size_t receiver = 1;
+ size_t peers_count = 2;
+ InternalStatsKey stats_key(stream, sender, receiver);
+
+ comparator.Start(/*max_threads_count=*/1);
+ comparator.EnsureStatsForStream(stream, sender, peers_count,
+ stream_start_time, stream_start_time);
+
+ // Add 5 frames which were rendered with 30 fps (~30ms between frames)
+ // Frame ids are in [1..5] and last frame is with 120ms offset from first.
+ Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity();
+ for (int i = 0; i < 5; ++i) {
+ FrameStats frame_stats = FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(
+ /*frame_id=*/i + 1, stream_start_time + TimeDelta::Millis(30 * i));
+ frame_stats.prev_frame_rendered_time = prev_frame_rendered_time;
+ prev_frame_rendered_time = frame_stats.rendered_time;
+
+ comparator.AddComparison(stats_key,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kRegular, frame_stats);
+ }
+
+ // Next frame was rendered with 4 frames skipped and delay 300ms after last
+ // frame.
+ FrameStats freeze_frame_stats =
+ FrameStatsWith10msDeltaBetweenPhasesAnd10x10Frame(
+ /*frame_id=*/10, stream_start_time + TimeDelta::Millis(120 + 300));
+ freeze_frame_stats.prev_frame_rendered_time = prev_frame_rendered_time;
+
+ comparator.AddComparison(stats_key,
+ /*skipped_between_rendered=*/4,
+ /*captured=*/absl::nullopt,
+ /*rendered=*/absl::nullopt,
+ FrameComparisonType::kRegular, freeze_frame_stats);
+ comparator.Stop(/*last_rendered_frame_times=*/{});
+
+ StreamStats stats = comparator.stream_stats().at(stats_key);
+ ASSERT_THAT(GetFirstOrDie(stats.skipped_between_rendered), Eq(4));
+ AssertFirstMetadataHasField(stats.skipped_between_rendered, "frame_id", "10");
+ ASSERT_THAT(GetFirstOrDie(stats.freeze_time_ms), Eq(300));
+ AssertFirstMetadataHasField(stats.freeze_time_ms, "frame_id", "10");
+ // 180ms is time from the stream start to the rendered time of the last frame
+ // among first 5 frames which were received before freeze.
+ ASSERT_THAT(GetFirstOrDie(stats.time_between_freezes_ms), Eq(180));
+ AssertFirstMetadataHasField(stats.time_between_freezes_ms, "frame_id", "10");
+}
+// Stats validation tests end.
+
+} // namespace
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.cc
new file mode 100644
index 0000000000..16f49ef154
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.cc
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
+
+#include "api/video/video_frame.h"
+#include "rtc_base/strings/string_builder.h"
+
+namespace webrtc {
+
+std::string InternalStatsKey::ToString() const {
+ rtc::StringBuilder out;
+ out << "stream=" << stream << "_sender=" << sender
+ << "_receiver=" << receiver;
+ return out.str();
+}
+
+bool operator<(const InternalStatsKey& a, const InternalStatsKey& b) {
+ if (a.stream != b.stream) {
+ return a.stream < b.stream;
+ }
+ if (a.sender != b.sender) {
+ return a.sender < b.sender;
+ }
+ return a.receiver < b.receiver;
+}
+
+bool operator==(const InternalStatsKey& a, const InternalStatsKey& b) {
+ return a.stream == b.stream && a.sender == b.sender &&
+ a.receiver == b.receiver;
+}
+
+FrameComparison::FrameComparison(InternalStatsKey stats_key,
+ absl::optional<VideoFrame> captured,
+ absl::optional<VideoFrame> rendered,
+ FrameComparisonType type,
+ FrameStats frame_stats,
+ OverloadReason overload_reason)
+ : stats_key(std::move(stats_key)),
+ captured(std::move(captured)),
+ rendered(std::move(rendered)),
+ type(type),
+ frame_stats(std::move(frame_stats)),
+ overload_reason(overload_reason) {}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h
new file mode 100644
index 0000000000..10f1314f46
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h
@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_INTERNAL_SHARED_OBJECTS_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_INTERNAL_SHARED_OBJECTS_H_
+
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/numerics/samples_stats_counter.h"
+#include "api/units/data_size.h"
+#include "api/units/timestamp.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_frame_type.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
+
+namespace webrtc {
+
+struct InternalStatsKey {
+ InternalStatsKey(size_t stream, size_t sender, size_t receiver)
+ : stream(stream), sender(sender), receiver(receiver) {}
+
+ std::string ToString() const;
+
+ size_t stream;
+ size_t sender;
+ size_t receiver;
+};
+
+// Required to use InternalStatsKey as std::map key.
+bool operator<(const InternalStatsKey& a, const InternalStatsKey& b);
+bool operator==(const InternalStatsKey& a, const InternalStatsKey& b);
+
+// Final stats computed for frame after it went through the whole video
+// pipeline from capturing to rendering or dropping.
+struct FrameStats {
+ FrameStats(uint16_t frame_id, Timestamp captured_time)
+ : frame_id(frame_id), captured_time(captured_time) {}
+
+ uint16_t frame_id;
+ // Frame events timestamp.
+ Timestamp captured_time;
+ Timestamp pre_encode_time = Timestamp::MinusInfinity();
+ Timestamp encoded_time = Timestamp::MinusInfinity();
+ // Time when last packet of a frame was received.
+ Timestamp received_time = Timestamp::MinusInfinity();
+ Timestamp decode_start_time = Timestamp::MinusInfinity();
+ Timestamp decode_end_time = Timestamp::MinusInfinity();
+ Timestamp rendered_time = Timestamp::MinusInfinity();
+ Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity();
+
+ VideoFrameType encoded_frame_type = VideoFrameType::kEmptyFrame;
+ DataSize encoded_image_size = DataSize::Bytes(0);
+ VideoFrameType pre_decoded_frame_type = VideoFrameType::kEmptyFrame;
+ DataSize pre_decoded_image_size = DataSize::Bytes(0);
+ uint32_t target_encode_bitrate = 0;
+ // Sender side qp values per spatial layer. In case when spatial layer is not
+ // set for `webrtc::EncodedImage`, 0 is used as default.
+ std::map<int, SamplesStatsCounter> spatial_layers_qp;
+
+ absl::optional<int> decoded_frame_width = absl::nullopt;
+ absl::optional<int> decoded_frame_height = absl::nullopt;
+
+ // Can be not set if frame was dropped by encoder.
+ absl::optional<StreamCodecInfo> used_encoder = absl::nullopt;
+ // Can be not set if frame was dropped in the network.
+ absl::optional<StreamCodecInfo> used_decoder = absl::nullopt;
+
+ bool decoder_failed = false;
+};
+
+// Describes why comparison was done in overloaded mode (without calculating
+// PSNR and SSIM).
+enum class OverloadReason {
+ kNone,
+ // Not enough CPU to process all incoming comparisons.
+ kCpu,
+ // Not enough memory to store captured frames for all comparisons.
+ kMemory
+};
+
+enum class FrameComparisonType {
+ // Comparison for captured and rendered frame.
+ kRegular,
+ // Comparison for captured frame that is known to be dropped somewhere in
+ // video pipeline.
+ kDroppedFrame,
+ // Comparison for captured frame that was still in the video pipeline when
+ // test was stopped. It's unknown is this frame dropped or would it be
+ // delivered if test continue.
+ kFrameInFlight
+};
+
+// Represents comparison between two VideoFrames. Contains video frames itself
+// and stats. Can be one of two types:
+// 1. Normal - in this case `captured` is presented and either `rendered` is
+// presented and `dropped` is false, either `rendered` is omitted and
+// `dropped` is true.
+// 2. Overloaded - in this case both `captured` and `rendered` are omitted
+// because there were too many comparisons in the queue. `dropped` can be
+// true or false showing was frame dropped or not.
+struct FrameComparison {
+ FrameComparison(InternalStatsKey stats_key,
+ absl::optional<VideoFrame> captured,
+ absl::optional<VideoFrame> rendered,
+ FrameComparisonType type,
+ FrameStats frame_stats,
+ OverloadReason overload_reason);
+
+ InternalStatsKey stats_key;
+ // Frames can be omitted if there too many computations waiting in the
+ // queue.
+ absl::optional<VideoFrame> captured;
+ absl::optional<VideoFrame> rendered;
+ FrameComparisonType type;
+ FrameStats frame_stats;
+ OverloadReason overload_reason;
+};
+
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_INTERNAL_SHARED_OBJECTS_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc
new file mode 100644
index 0000000000..f5029ac956
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_metric_names_test.cc
@@ -0,0 +1,682 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "api/rtp_packet_info.h"
+#include "api/rtp_packet_infos.h"
+#include "api/test/create_frame_generator.h"
+#include "api/test/metrics/metric.h"
+#include "api/test/metrics/metrics_logger.h"
+#include "api/test/metrics/stdout_metrics_exporter.h"
+#include "api/video/encoded_image.h"
+#include "api/video/i420_buffer.h"
+#include "api/video/video_frame.h"
+#include "common_video/libyuv/include/webrtc_libyuv.h"
+#include "rtc_tools/frame_analyzer/video_geometry_aligner.h"
+#include "system_wrappers/include/sleep.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
+
+namespace webrtc {
+namespace {
+
+using ::testing::Contains;
+using ::testing::SizeIs;
+using ::testing::UnorderedElementsAre;
+
+using ::webrtc::test::DefaultMetricsLogger;
+using ::webrtc::test::ImprovementDirection;
+using ::webrtc::test::Metric;
+using ::webrtc::test::MetricsExporter;
+using ::webrtc::test::StdoutMetricsExporter;
+using ::webrtc::test::Unit;
+
+constexpr int kAnalyzerMaxThreadsCount = 1;
+constexpr int kMaxFramesInFlightPerStream = 10;
+constexpr int kFrameWidth = 320;
+constexpr int kFrameHeight = 240;
+
+DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() {
+ DefaultVideoQualityAnalyzerOptions options;
+ options.compute_psnr = true;
+ options.compute_ssim = true;
+ options.adjust_cropping_before_comparing_frames = false;
+ options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream;
+ options.report_detailed_frame_stats = true;
+ return options;
+}
+
+VideoFrame NextFrame(test::FrameGeneratorInterface* frame_generator,
+ int64_t timestamp_us) {
+ test::FrameGeneratorInterface::VideoFrameData frame_data =
+ frame_generator->NextFrame();
+ return VideoFrame::Builder()
+ .set_video_frame_buffer(frame_data.buffer)
+ .set_update_rect(frame_data.update_rect)
+ .set_timestamp_us(timestamp_us)
+ .build();
+}
+
+EncodedImage FakeEncode(const VideoFrame& frame) {
+ EncodedImage image;
+ std::vector<RtpPacketInfo> packet_infos;
+ packet_infos.push_back(RtpPacketInfo(
+ /*ssrc=*/1,
+ /*csrcs=*/{},
+ /*rtp_timestamp=*/frame.timestamp(),
+ /*receive_time=*/Timestamp::Micros(frame.timestamp_us() + 10000)));
+ image.SetPacketInfos(RtpPacketInfos(packet_infos));
+ return image;
+}
+
+VideoFrame DeepCopy(const VideoFrame& frame) {
+ VideoFrame copy = frame;
+ copy.set_video_frame_buffer(
+ I420Buffer::Copy(*frame.video_frame_buffer()->ToI420()));
+ return copy;
+}
+
+void PassFramesThroughAnalyzer(DefaultVideoQualityAnalyzer& analyzer,
+ absl::string_view sender,
+ absl::string_view stream_label,
+ std::vector<absl::string_view> receivers,
+ int frames_count,
+ test::FrameGeneratorInterface& frame_generator,
+ int interframe_delay_ms = 0) {
+ for (int i = 0; i < frames_count; ++i) {
+ VideoFrame frame = NextFrame(&frame_generator, /*timestamp_us=*/1);
+ uint16_t frame_id =
+ analyzer.OnFrameCaptured(sender, std::string(stream_label), frame);
+ frame.set_id(frame_id);
+ analyzer.OnFramePreEncode(sender, frame);
+ analyzer.OnFrameEncoded(sender, frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(),
+ false);
+ for (absl::string_view receiver : receivers) {
+ VideoFrame received_frame = DeepCopy(frame);
+ analyzer.OnFramePreDecode(receiver, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(receiver, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(receiver, received_frame);
+ }
+ if (i < frames_count - 1 && interframe_delay_ms > 0) {
+ SleepMs(interframe_delay_ms);
+ }
+ }
+}
+
+// Metric fields to assert on
+struct MetricValidationInfo {
+ std::string test_case;
+ std::string name;
+ Unit unit;
+ ImprovementDirection improvement_direction;
+};
+
+bool operator==(const MetricValidationInfo& a, const MetricValidationInfo& b) {
+ return a.name == b.name && a.test_case == b.test_case && a.unit == b.unit &&
+ a.improvement_direction == b.improvement_direction;
+}
+
+std::ostream& operator<<(std::ostream& os, const MetricValidationInfo& m) {
+ os << "{ test_case=" << m.test_case << "; name=" << m.name
+ << "; unit=" << test::ToString(m.unit)
+ << "; improvement_direction=" << test::ToString(m.improvement_direction)
+ << " }";
+ return os;
+}
+
+std::vector<MetricValidationInfo> ToValidationInfo(
+ const std::vector<Metric>& metrics) {
+ std::vector<MetricValidationInfo> out;
+ for (const Metric& m : metrics) {
+ out.push_back(
+ MetricValidationInfo{.test_case = m.test_case,
+ .name = m.name,
+ .unit = m.unit,
+ .improvement_direction = m.improvement_direction});
+ }
+ return out;
+}
+
+std::vector<std::string> ToTestCases(const std::vector<Metric>& metrics) {
+ std::vector<std::string> out;
+ for (const Metric& m : metrics) {
+ out.push_back(m.test_case);
+ }
+ return out;
+}
+
+TEST(DefaultVideoQualityAnalyzerMetricNamesTest, MetricNamesForP2PAreCorrect) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ DefaultMetricsLogger metrics_logger(Clock::GetRealTimeClock());
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ &metrics_logger, options);
+ analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
+ kAnalyzerMaxThreadsCount);
+
+ PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"},
+ /*frames_count=*/5, *frame_generator,
+ /*interframe_delay_ms=*/50);
+ analyzer.Stop();
+
+ std::vector<MetricValidationInfo> metrics =
+ ToValidationInfo(metrics_logger.GetCollectedMetrics());
+ EXPECT_THAT(
+ metrics,
+ UnorderedElementsAre(
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "ssim",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "transport_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "total_delay_incl_transport",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "time_between_rendered_frames",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "harmonic_framerate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "encode_frame_rate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "encode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "time_between_freezes",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "freeze_time_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "pixels_per_frame",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "min_psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "decode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "receive_to_render_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "dropped_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "frames_in_flight",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "rendered_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "max_skipped",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "target_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "qp_sl0",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "actual_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "capture_frame_rate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "num_encoded_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "num_decoded_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "num_send_key_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "num_recv_key_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "recv_key_frame_size_bytes",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "recv_delta_frame_size_bytes",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{.test_case = "test_case",
+ .name = "cpu_usage_%",
+ .unit = Unit::kUnitless,
+ .improvement_direction =
+ ImprovementDirection::kSmallerIsBetter}));
+}
+
+TEST(DefaultVideoQualityAnalyzerMetricNamesTest,
+ MetricNamesFor3PeersAreCorrect) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ DefaultMetricsLogger metrics_logger(Clock::GetRealTimeClock());
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ &metrics_logger, options);
+ analyzer.Start("test_case",
+ std::vector<std::string>{"alice", "bob", "charlie"},
+ kAnalyzerMaxThreadsCount);
+
+ PassFramesThroughAnalyzer(analyzer, "alice", "alice_video",
+ {"bob", "charlie"},
+ /*frames_count=*/5, *frame_generator,
+ /*interframe_delay_ms=*/50);
+ analyzer.Stop();
+
+ std::vector<MetricValidationInfo> metrics =
+ ToValidationInfo(metrics_logger.GetCollectedMetrics());
+ EXPECT_THAT(
+ metrics,
+ UnorderedElementsAre(
+ // Bob
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "ssim",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "transport_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "total_delay_incl_transport",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "time_between_rendered_frames",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "harmonic_framerate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "encode_frame_rate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "encode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "time_between_freezes",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "freeze_time_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "pixels_per_frame",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "min_psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "decode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "receive_to_render_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "dropped_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "frames_in_flight",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "rendered_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "max_skipped",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "target_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "qp_sl0",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "actual_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "capture_frame_rate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "num_encoded_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "num_decoded_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "num_send_key_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "num_recv_key_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "recv_key_frame_size_bytes",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_bob",
+ .name = "recv_delta_frame_size_bytes",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+
+ // Charlie
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "ssim",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "transport_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "total_delay_incl_transport",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "time_between_rendered_frames",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "harmonic_framerate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "encode_frame_rate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "encode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "time_between_freezes",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "freeze_time_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "pixels_per_frame",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "min_psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "decode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "receive_to_render_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "dropped_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "frames_in_flight",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "rendered_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "max_skipped",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "target_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "qp_sl0",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "actual_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "capture_frame_rate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "num_encoded_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "num_decoded_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "num_send_key_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "num_recv_key_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "recv_key_frame_size_bytes",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video_alice_charlie",
+ .name = "recv_delta_frame_size_bytes",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter},
+ MetricValidationInfo{.test_case = "test_case",
+ .name = "cpu_usage_%",
+ .unit = Unit::kUnitless,
+ .improvement_direction =
+ ImprovementDirection::kSmallerIsBetter}));
+}
+
+TEST(DefaultVideoQualityAnalyzerMetricNamesTest,
+ TestCaseFor3PeerIsTheSameAfterAllPeersLeft) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ DefaultMetricsLogger metrics_logger(Clock::GetRealTimeClock());
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ &metrics_logger, options);
+ analyzer.Start("test_case",
+ std::vector<std::string>{"alice", "bob", "charlie"},
+ kAnalyzerMaxThreadsCount);
+
+ PassFramesThroughAnalyzer(analyzer, "alice", "alice_video",
+ {"bob", "charlie"},
+ /*frames_count=*/5, *frame_generator,
+ /*interframe_delay_ms=*/50);
+ analyzer.UnregisterParticipantInCall("alice");
+ analyzer.UnregisterParticipantInCall("bob");
+ analyzer.UnregisterParticipantInCall("charlie");
+ analyzer.Stop();
+
+ std::vector<std::string> metrics =
+ ToTestCases(metrics_logger.GetCollectedMetrics());
+ EXPECT_THAT(metrics, SizeIs(57));
+ EXPECT_THAT(metrics, Contains("test_case/alice_video_alice_bob").Times(28));
+ EXPECT_THAT(metrics,
+ Contains("test_case/alice_video_alice_charlie").Times(28));
+ EXPECT_THAT(metrics, Contains("test_case").Times(1));
+}
+
+} // namespace
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc
new file mode 100644
index 0000000000..79b9286e2d
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.cc
@@ -0,0 +1,172 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
+
+#include <algorithm>
+#include <iterator>
+#include <ostream>
+#include <string>
+
+#include "api/units/timestamp.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/strings/string_builder.h"
+
+namespace webrtc {
+namespace {
+
+constexpr int kMicrosPerSecond = 1000000;
+
+} // namespace
+
+std::string StreamCodecInfo::ToString() const {
+ rtc::StringBuilder out;
+ out << "{codec_name=" << codec_name << "; first_frame_id=" << first_frame_id
+ << "; last_frame_id=" << last_frame_id
+ << "; switched_on_at=" << webrtc::ToString(switched_on_at)
+ << "; switched_from_at=" << webrtc::ToString(switched_from_at) << " }";
+ return out.str();
+}
+
+std::ostream& operator<<(std::ostream& os, const StreamCodecInfo& state) {
+ return os << state.ToString();
+}
+
+rtc::StringBuilder& operator<<(rtc::StringBuilder& sb,
+ const StreamCodecInfo& state) {
+ return sb << state.ToString();
+}
+
+bool operator==(const StreamCodecInfo& a, const StreamCodecInfo& b) {
+ return a.codec_name == b.codec_name && a.first_frame_id == b.first_frame_id &&
+ a.last_frame_id == b.last_frame_id &&
+ a.switched_on_at == b.switched_on_at &&
+ a.switched_from_at == b.switched_from_at;
+}
+
+std::string ToString(FrameDropPhase phase) {
+ switch (phase) {
+ case FrameDropPhase::kBeforeEncoder:
+ return "kBeforeEncoder";
+ case FrameDropPhase::kByEncoder:
+ return "kByEncoder";
+ case FrameDropPhase::kTransport:
+ return "kTransport";
+ case FrameDropPhase::kByDecoder:
+ return "kByDecoder";
+ case FrameDropPhase::kAfterDecoder:
+ return "kAfterDecoder";
+ case FrameDropPhase::kLastValue:
+ return "kLastValue";
+ }
+}
+
+std::ostream& operator<<(std::ostream& os, FrameDropPhase phase) {
+ return os << ToString(phase);
+}
+rtc::StringBuilder& operator<<(rtc::StringBuilder& sb, FrameDropPhase phase) {
+ return sb << ToString(phase);
+}
+
+void SamplesRateCounter::AddEvent(Timestamp event_time) {
+ if (event_first_time_.IsMinusInfinity()) {
+ event_first_time_ = event_time;
+ }
+ event_last_time_ = event_time;
+ events_count_++;
+}
+
+double SamplesRateCounter::GetEventsPerSecond() const {
+ RTC_DCHECK(!IsEmpty());
+ // Divide on us and multiply on kMicrosPerSecond to correctly process cases
+ // where there were too small amount of events, so difference is less then 1
+ // sec. We can use us here, because Timestamp has us resolution.
+ return static_cast<double>(events_count_) /
+ (event_last_time_ - event_first_time_).us() * kMicrosPerSecond;
+}
+
+StreamStats::StreamStats(Timestamp stream_started_time)
+ : stream_started_time(stream_started_time) {
+ for (int i = static_cast<int>(FrameDropPhase::kBeforeEncoder);
+ i < static_cast<int>(FrameDropPhase::kLastValue); ++i) {
+ dropped_by_phase.emplace(static_cast<FrameDropPhase>(i), 0);
+ }
+}
+
+std::string StatsKey::ToString() const {
+ rtc::StringBuilder out;
+ out << stream_label << "_" << receiver;
+ return out.str();
+}
+
+bool operator<(const StatsKey& a, const StatsKey& b) {
+ if (a.stream_label != b.stream_label) {
+ return a.stream_label < b.stream_label;
+ }
+ return a.receiver < b.receiver;
+}
+
+bool operator==(const StatsKey& a, const StatsKey& b) {
+ return a.stream_label == b.stream_label && a.receiver == b.receiver;
+}
+
+VideoStreamsInfo::VideoStreamsInfo(
+ std::map<std::string, std::string> stream_to_sender,
+ std::map<std::string, std::set<std::string>> sender_to_streams,
+ std::map<std::string, std::set<std::string>> stream_to_receivers)
+ : stream_to_sender_(std::move(stream_to_sender)),
+ sender_to_streams_(std::move(sender_to_streams)),
+ stream_to_receivers_(std::move(stream_to_receivers)) {}
+
+std::set<StatsKey> VideoStreamsInfo::GetStatsKeys() const {
+ std::set<StatsKey> out;
+ for (const std::string& stream_label : GetStreams()) {
+ for (const std::string& receiver : GetReceivers(stream_label)) {
+ out.insert(StatsKey(stream_label, receiver));
+ }
+ }
+ return out;
+}
+
+std::set<std::string> VideoStreamsInfo::GetStreams() const {
+ std::set<std::string> out;
+ std::transform(stream_to_sender_.begin(), stream_to_sender_.end(),
+ std::inserter(out, out.end()),
+ [](auto map_entry) { return map_entry.first; });
+ return out;
+}
+
+std::set<std::string> VideoStreamsInfo::GetStreams(
+ absl::string_view sender_name) const {
+ auto it = sender_to_streams_.find(std::string(sender_name));
+ if (it == sender_to_streams_.end()) {
+ return {};
+ }
+ return it->second;
+}
+
+absl::optional<std::string> VideoStreamsInfo::GetSender(
+ absl::string_view stream_label) const {
+ auto it = stream_to_sender_.find(std::string(stream_label));
+ if (it == stream_to_sender_.end()) {
+ return absl::nullopt;
+ }
+ return it->second;
+}
+
+std::set<std::string> VideoStreamsInfo::GetReceivers(
+ absl::string_view stream_label) const {
+ auto it = stream_to_receivers_.find(std::string(stream_label));
+ if (it == stream_to_receivers_.end()) {
+ return {};
+ }
+ return it->second;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h
new file mode 100644
index 0000000000..175f777b68
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h
@@ -0,0 +1,284 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_SHARED_OBJECTS_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_SHARED_OBJECTS_H_
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <ostream>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/numerics/samples_stats_counter.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/strings/string_builder.h"
+
+namespace webrtc {
+
+// WebRTC will request a key frame after 3 seconds if no frames were received.
+// We assume max frame rate ~60 fps, so 270 frames will cover max freeze without
+// key frame request.
+constexpr size_t kDefaultMaxFramesInFlightPerStream = 270;
+
+class SamplesRateCounter {
+ public:
+ void AddEvent(Timestamp event_time);
+
+ bool IsEmpty() const { return event_first_time_ == event_last_time_; }
+
+ double GetEventsPerSecond() const;
+
+ private:
+ Timestamp event_first_time_ = Timestamp::MinusInfinity();
+ Timestamp event_last_time_ = Timestamp::MinusInfinity();
+ int64_t events_count_ = 0;
+};
+
+struct FrameCounters {
+ // Count of frames, that were passed into WebRTC pipeline by video stream
+ // source.
+ int64_t captured = 0;
+ // Count of frames that reached video encoder.
+ int64_t pre_encoded = 0;
+ // Count of encoded images that were produced by encoder for all requested
+ // spatial layers and simulcast streams.
+ int64_t encoded = 0;
+ // Count of encoded images received in decoder for all requested spatial
+ // layers and simulcast streams.
+ int64_t received = 0;
+ // Count of frames that were produced by decoder.
+ int64_t decoded = 0;
+ // Count of frames that went out from WebRTC pipeline to video sink.
+ int64_t rendered = 0;
+ // Count of frames that were dropped in any point between capturing and
+ // rendering.
+ int64_t dropped = 0;
+ // Count of frames for which decoder returned error when they were sent for
+ // decoding.
+ int64_t failed_to_decode = 0;
+};
+
+// Contains information about the codec that was used for encoding or decoding
+// the stream.
+struct StreamCodecInfo {
+ // Codec implementation name.
+ std::string codec_name;
+ // Id of the first frame for which this codec was used.
+ uint16_t first_frame_id;
+ // Id of the last frame for which this codec was used.
+ uint16_t last_frame_id;
+ // Timestamp when the first frame was handled by the encode/decoder.
+ Timestamp switched_on_at = Timestamp::PlusInfinity();
+ // Timestamp when this codec was used last time.
+ Timestamp switched_from_at = Timestamp::PlusInfinity();
+
+ std::string ToString() const;
+};
+
+std::ostream& operator<<(std::ostream& os, const StreamCodecInfo& state);
+rtc::StringBuilder& operator<<(rtc::StringBuilder& sb,
+ const StreamCodecInfo& state);
+bool operator==(const StreamCodecInfo& a, const StreamCodecInfo& b);
+
+// Represents phases where video frame can be dropped and such drop will be
+// detected by analyzer.
+enum class FrameDropPhase : int {
+ kBeforeEncoder,
+ kByEncoder,
+ kTransport,
+ kByDecoder,
+ kAfterDecoder,
+ // kLastValue must be the last value in this enumeration.
+ kLastValue
+};
+
+std::string ToString(FrameDropPhase phase);
+std::ostream& operator<<(std::ostream& os, FrameDropPhase phase);
+rtc::StringBuilder& operator<<(rtc::StringBuilder& sb, FrameDropPhase phase);
+
+struct StreamStats {
+ explicit StreamStats(Timestamp stream_started_time);
+
+ // The time when the first frame of this stream was captured.
+ Timestamp stream_started_time;
+
+ // Spatial quality metrics.
+ SamplesStatsCounter psnr;
+ SamplesStatsCounter ssim;
+
+ // Time from frame encoded (time point on exit from encoder) to the
+ // encoded image received in decoder (time point on entrance to decoder).
+ SamplesStatsCounter transport_time_ms;
+ // Time from frame was captured on device to time frame was displayed on
+ // device.
+ SamplesStatsCounter total_delay_incl_transport_ms;
+ // Time between frames out from renderer.
+ SamplesStatsCounter time_between_rendered_frames_ms;
+ SamplesRateCounter capture_frame_rate;
+ SamplesRateCounter encode_frame_rate;
+ SamplesStatsCounter encode_time_ms;
+ SamplesStatsCounter decode_time_ms;
+ // Time from last packet of frame is received until it's sent to the renderer.
+ SamplesStatsCounter receive_to_render_time_ms;
+ // Max frames skipped between two nearest.
+ SamplesStatsCounter skipped_between_rendered;
+ // In the next 2 metrics freeze is a pause that is longer, than maximum:
+ // 1. 150ms
+ // 2. 3 * average time between two sequential frames.
+ // Item 1 will cover high fps video and is a duration, that is noticeable by
+ // human eye. Item 2 will cover low fps video like screen sharing.
+ // Freeze duration.
+ SamplesStatsCounter freeze_time_ms;
+ // Mean time between one freeze end and next freeze start.
+ SamplesStatsCounter time_between_freezes_ms;
+ SamplesStatsCounter resolution_of_decoded_frame;
+ SamplesStatsCounter target_encode_bitrate;
+ // Sender side qp values per spatial layer. In case when spatial layer is not
+ // set for `webrtc::EncodedImage`, 0 is used as default.
+ std::map<int, SamplesStatsCounter> spatial_layers_qp;
+
+ int64_t total_encoded_images_payload = 0;
+ // Counters on which phase how many frames were dropped.
+ std::map<FrameDropPhase, int64_t> dropped_by_phase;
+
+ // Frame count metrics.
+ int64_t num_send_key_frames = 0;
+ int64_t num_recv_key_frames = 0;
+
+ // Encoded frame size (in bytes) metrics.
+ SamplesStatsCounter recv_key_frame_size_bytes;
+ SamplesStatsCounter recv_delta_frame_size_bytes;
+
+ // Vector of encoders used for this stream by sending client.
+ std::vector<StreamCodecInfo> encoders;
+ // Vectors of decoders used for this stream by receiving client.
+ std::vector<StreamCodecInfo> decoders;
+};
+
+struct AnalyzerStats {
+ // Size of analyzer internal comparisons queue, measured when new element
+ // id added to the queue.
+ SamplesStatsCounter comparisons_queue_size;
+ // Number of performed comparisons of 2 video frames from captured and
+ // rendered streams.
+ int64_t comparisons_done = 0;
+ // Number of cpu overloaded comparisons. Comparison is cpu overloaded if it is
+ // queued when there are too many not processed comparisons in the queue.
+ // Overloaded comparison doesn't include metrics like SSIM and PSNR that
+ // require heavy computations.
+ int64_t cpu_overloaded_comparisons_done = 0;
+ // Number of memory overloaded comparisons. Comparison is memory overloaded if
+ // it is queued when its captured frame was already removed due to high memory
+ // usage for that video stream.
+ int64_t memory_overloaded_comparisons_done = 0;
+ // Count of frames in flight in analyzer measured when new comparison is added
+ // and after analyzer was stopped.
+ SamplesStatsCounter frames_in_flight_left_count;
+
+ // Next metrics are collected and reported iff
+ // `DefaultVideoQualityAnalyzerOptions::report_infra_metrics` is true.
+ SamplesStatsCounter on_frame_captured_processing_time_ms;
+ SamplesStatsCounter on_frame_pre_encode_processing_time_ms;
+ SamplesStatsCounter on_frame_encoded_processing_time_ms;
+ SamplesStatsCounter on_frame_pre_decode_processing_time_ms;
+ SamplesStatsCounter on_frame_decoded_processing_time_ms;
+ SamplesStatsCounter on_frame_rendered_processing_time_ms;
+ SamplesStatsCounter on_decoder_error_processing_time_ms;
+};
+
+struct StatsKey {
+ StatsKey(std::string stream_label, std::string receiver)
+ : stream_label(std::move(stream_label)), receiver(std::move(receiver)) {}
+
+ std::string ToString() const;
+
+ // Label of video stream to which stats belongs to.
+ std::string stream_label;
+ // Name of the peer on which stream was received.
+ std::string receiver;
+};
+
+// Required to use StatsKey as std::map key.
+bool operator<(const StatsKey& a, const StatsKey& b);
+bool operator==(const StatsKey& a, const StatsKey& b);
+
+// Contains all metadata related to the video streams that were seen by the
+// video analyzer.
+class VideoStreamsInfo {
+ public:
+ std::set<StatsKey> GetStatsKeys() const;
+
+ // Returns all stream labels that are known to the video analyzer.
+ std::set<std::string> GetStreams() const;
+
+ // Returns set of the stream for specified `sender_name`. If sender didn't
+ // send any streams or `sender_name` isn't known to the video analyzer
+ // empty set will be returned.
+ std::set<std::string> GetStreams(absl::string_view sender_name) const;
+
+ // Returns sender name for specified `stream_label`. Returns `absl::nullopt`
+ // if provided `stream_label` isn't known to the video analyzer.
+ absl::optional<std::string> GetSender(absl::string_view stream_label) const;
+
+ // Returns set of the receivers for specified `stream_label`. If stream wasn't
+ // received by any peer or `stream_label` isn't known to the video analyzer
+ // empty set will be returned.
+ std::set<std::string> GetReceivers(absl::string_view stream_label) const;
+
+ protected:
+ friend class DefaultVideoQualityAnalyzer;
+ VideoStreamsInfo(
+ std::map<std::string, std::string> stream_to_sender,
+ std::map<std::string, std::set<std::string>> sender_to_streams,
+ std::map<std::string, std::set<std::string>> stream_to_receivers);
+
+ private:
+ std::map<std::string, std::string> stream_to_sender_;
+ std::map<std::string, std::set<std::string>> sender_to_streams_;
+ std::map<std::string, std::set<std::string>> stream_to_receivers_;
+};
+
+struct DefaultVideoQualityAnalyzerOptions {
+ // Tells DefaultVideoQualityAnalyzer if heavy metrics have to be computed.
+ bool compute_psnr = true;
+ bool compute_ssim = true;
+ // If true, weights the luma plane more than the chroma planes in the PSNR.
+ bool use_weighted_psnr = false;
+ // Tells DefaultVideoQualityAnalyzer if detailed frame stats should be
+ // reported.
+ bool report_detailed_frame_stats = false;
+ // Tells DefaultVideoQualityAnalyzer if infra metrics related to the
+ // performance and stability of the analyzer itself should be reported.
+ bool report_infra_metrics = false;
+ // If true DefaultVideoQualityAnalyzer will try to adjust frames before
+ // computing PSNR and SSIM for them. In some cases picture may be shifted by
+ // a few pixels after the encode/decode step. Those difference is invisible
+ // for a human eye, but it affects the metrics. So the adjustment is used to
+ // get metrics that are closer to how human perceive the video. This feature
+ // significantly slows down the comparison, so turn it on only when it is
+ // needed.
+ bool adjust_cropping_before_comparing_frames = false;
+ // Amount of frames that are queued in the DefaultVideoQualityAnalyzer from
+ // the point they were captured to the point they were rendered on all
+ // receivers per stream.
+ size_t max_frames_in_flight_per_stream_count =
+ kDefaultMaxFramesInFlightPerStream;
+ // If true, the analyzer will expect peers to receive their own video streams.
+ bool enable_receive_own_stream = false;
+};
+
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_SHARED_OBJECTS_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.cc
new file mode 100644
index 0000000000..d59ef12c63
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.cc
@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h"
+
+#include <map>
+#include <set>
+
+#include "absl/types/optional.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+namespace {
+
+template <typename T>
+absl::optional<T> MaybeGetValue(const std::map<size_t, T>& map, size_t key) {
+ auto it = map.find(key);
+ if (it == map.end()) {
+ return absl::nullopt;
+ }
+ return it->second;
+}
+
+} // namespace
+
+StreamState::StreamState(size_t sender,
+ std::set<size_t> receivers,
+ Timestamp stream_started_time)
+ : sender_(sender),
+ stream_started_time_(stream_started_time),
+ receivers_(receivers),
+ frame_ids_(std::move(receivers)) {
+ frame_ids_.AddReader(kAliveFramesQueueIndex);
+ RTC_CHECK_NE(sender_, kAliveFramesQueueIndex);
+ for (size_t receiver : receivers_) {
+ RTC_CHECK_NE(receiver, kAliveFramesQueueIndex);
+ }
+}
+
+uint16_t StreamState::PopFront(size_t peer) {
+ RTC_CHECK_NE(peer, kAliveFramesQueueIndex);
+ absl::optional<uint16_t> frame_id = frame_ids_.PopFront(peer);
+ RTC_DCHECK(frame_id.has_value());
+
+ // If alive's frame queue is longer than all others, than also pop frame from
+ // it, because that frame is received by all receivers.
+ size_t alive_size = frame_ids_.size(kAliveFramesQueueIndex);
+ size_t other_size = GetLongestReceiverQueue();
+ // Pops frame from alive queue if alive's queue is the longest one.
+ if (alive_size > other_size) {
+ absl::optional<uint16_t> alive_frame_id =
+ frame_ids_.PopFront(kAliveFramesQueueIndex);
+ RTC_DCHECK(alive_frame_id.has_value());
+ RTC_DCHECK_EQ(frame_id.value(), alive_frame_id.value());
+ }
+
+ return frame_id.value();
+}
+
+void StreamState::AddPeer(size_t peer) {
+ RTC_CHECK_NE(peer, kAliveFramesQueueIndex);
+ frame_ids_.AddReader(peer, kAliveFramesQueueIndex);
+ receivers_.insert(peer);
+}
+
+void StreamState::RemovePeer(size_t peer) {
+ RTC_CHECK_NE(peer, kAliveFramesQueueIndex);
+ frame_ids_.RemoveReader(peer);
+ receivers_.erase(peer);
+
+ // If we removed the last receiver for the alive frames, we need to pop them
+ // from the queue, because now they received by all receivers.
+ size_t alive_size = frame_ids_.size(kAliveFramesQueueIndex);
+ size_t other_size = GetLongestReceiverQueue();
+ while (alive_size > other_size) {
+ frame_ids_.PopFront(kAliveFramesQueueIndex);
+ alive_size--;
+ }
+}
+
+uint16_t StreamState::MarkNextAliveFrameAsDead() {
+ absl::optional<uint16_t> frame_id =
+ frame_ids_.PopFront(kAliveFramesQueueIndex);
+ RTC_DCHECK(frame_id.has_value());
+ return frame_id.value();
+}
+
+void StreamState::SetLastRenderedFrameTime(size_t peer, Timestamp time) {
+ auto it = last_rendered_frame_time_.find(peer);
+ if (it == last_rendered_frame_time_.end()) {
+ last_rendered_frame_time_.insert({peer, time});
+ } else {
+ it->second = time;
+ }
+}
+
+absl::optional<Timestamp> StreamState::last_rendered_frame_time(
+ size_t peer) const {
+ return MaybeGetValue(last_rendered_frame_time_, peer);
+}
+
+size_t StreamState::GetLongestReceiverQueue() const {
+ size_t max = 0;
+ for (size_t receiver : receivers_) {
+ size_t cur_size = frame_ids_.size(receiver);
+ if (cur_size > max) {
+ max = cur_size;
+ }
+ }
+ return max;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h
new file mode 100644
index 0000000000..829a79c7bf
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h
@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_STREAM_STATE_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_STREAM_STATE_H_
+
+#include <limits>
+#include <map>
+#include <set>
+
+#include "absl/types/optional.h"
+#include "api/units/timestamp.h"
+#include "test/pc/e2e/analyzer/video/multi_reader_queue.h"
+
+namespace webrtc {
+
+// Represents a current state of video stream inside
+// DefaultVideoQualityAnalyzer.
+//
+// Maintains the sequence of frames for each video stream and keeps track about
+// which frames were seen by each of the possible stream receiver.
+//
+// Keeps information about which frames are alive and which are dead. Frame is
+// alive if it contains VideoFrame payload for corresponding FrameInFlight
+// object inside DefaultVideoQualityAnalyzer, otherwise frame is considered
+// dead.
+//
+// Supports peer indexes from 0 to max(size_t) - 1.
+class StreamState {
+ public:
+ StreamState(size_t sender,
+ std::set<size_t> receivers,
+ Timestamp stream_started_time);
+
+ size_t sender() const { return sender_; }
+ Timestamp stream_started_time() const { return stream_started_time_; }
+
+ void PushBack(uint16_t frame_id) { frame_ids_.PushBack(frame_id); }
+ // Crash if state is empty.
+ uint16_t PopFront(size_t peer);
+ bool IsEmpty(size_t peer) const { return frame_ids_.IsEmpty(peer); }
+ // Crash if state is empty.
+ uint16_t Front(size_t peer) const { return frame_ids_.Front(peer).value(); }
+
+ // Adds a new peer to the state. All currently alive frames will be expected
+ // to be received by the newly added peer.
+ void AddPeer(size_t peer);
+
+ // Removes peer from the state. Frames that were expected to be received by
+ // this peer will be removed from it. On the other hand last rendered frame
+ // time for the removed peer will be preserved, because
+ // DefaultVideoQualityAnalyzer still may request it for stats processing.
+ void RemovePeer(size_t peer);
+
+ size_t GetAliveFramesCount() const {
+ return frame_ids_.size(kAliveFramesQueueIndex);
+ }
+ uint16_t MarkNextAliveFrameAsDead();
+
+ void SetLastRenderedFrameTime(size_t peer, Timestamp time);
+ absl::optional<Timestamp> last_rendered_frame_time(size_t peer) const;
+
+ private:
+ // Index of the `frame_ids_` queue which is used to track alive frames for
+ // this stream.
+ static constexpr size_t kAliveFramesQueueIndex =
+ std::numeric_limits<size_t>::max();
+
+ size_t GetLongestReceiverQueue() const;
+
+ // Index of the owner. Owner's queue in `frame_ids_` will keep alive frames.
+ const size_t sender_;
+ const Timestamp stream_started_time_;
+ std::set<size_t> receivers_;
+ // To correctly determine dropped frames we have to know sequence of frames
+ // in each stream so we will keep a list of frame ids inside the stream.
+ // This list is represented by multi head queue of frame ids with separate
+ // head for each receiver. When the frame is rendered, we will pop ids from
+ // the corresponding head until id will match with rendered one. All ids
+ // before matched one can be considered as dropped:
+ //
+ // | frame_id1 |->| frame_id2 |->| frame_id3 |->| frame_id4 |
+ //
+ // If we received frame with id frame_id3, then we will pop frame_id1 and
+ // frame_id2 and consider those frames as dropped and then compare received
+ // frame with the one from `FrameInFlight` with id frame_id3.
+ MultiReaderQueue<uint16_t> frame_ids_;
+ std::map<size_t, Timestamp> last_rendered_frame_time_;
+};
+
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_STREAM_STATE_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state_test.cc
new file mode 100644
index 0000000000..01a6aab28a
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state_test.cc
@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h"
+
+#include <set>
+
+#include "api/units/timestamp.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace {
+
+TEST(StreamStateTest, PopFrontAndFrontIndependentForEachPeer) {
+ StreamState state(/*sender=*/0,
+ /*receivers=*/std::set<size_t>{1, 2},
+ Timestamp::Seconds(1));
+ state.PushBack(/*frame_id=*/1);
+ state.PushBack(/*frame_id=*/2);
+
+ EXPECT_EQ(state.Front(/*peer=*/1), 1);
+ EXPECT_EQ(state.PopFront(/*peer=*/1), 1);
+ EXPECT_EQ(state.Front(/*peer=*/1), 2);
+ EXPECT_EQ(state.PopFront(/*peer=*/1), 2);
+ EXPECT_EQ(state.Front(/*peer=*/2), 1);
+ EXPECT_EQ(state.PopFront(/*peer=*/2), 1);
+ EXPECT_EQ(state.Front(/*peer=*/2), 2);
+ EXPECT_EQ(state.PopFront(/*peer=*/2), 2);
+}
+
+TEST(StreamStateTest, IsEmpty) {
+ StreamState state(/*sender=*/0,
+ /*receivers=*/std::set<size_t>{1, 2},
+ Timestamp::Seconds(1));
+ state.PushBack(/*frame_id=*/1);
+
+ EXPECT_FALSE(state.IsEmpty(/*peer=*/1));
+
+ state.PopFront(/*peer=*/1);
+
+ EXPECT_TRUE(state.IsEmpty(/*peer=*/1));
+}
+
+TEST(StreamStateTest, PopFrontForOnlyOnePeerDontChangeAliveFramesCount) {
+ StreamState state(/*sender=*/0,
+ /*receivers=*/std::set<size_t>{1, 2},
+ Timestamp::Seconds(1));
+ state.PushBack(/*frame_id=*/1);
+ state.PushBack(/*frame_id=*/2);
+
+ EXPECT_EQ(state.GetAliveFramesCount(), 2lu);
+
+ state.PopFront(/*peer=*/1);
+ state.PopFront(/*peer=*/1);
+
+ EXPECT_EQ(state.GetAliveFramesCount(), 2lu);
+}
+
+TEST(StreamStateTest, PopFrontForAllPeersReducesAliveFramesCount) {
+ StreamState state(/*sender=*/0,
+ /*receivers=*/std::set<size_t>{1, 2},
+ Timestamp::Seconds(1));
+ state.PushBack(/*frame_id=*/1);
+ state.PushBack(/*frame_id=*/2);
+
+ EXPECT_EQ(state.GetAliveFramesCount(), 2lu);
+
+ state.PopFront(/*peer=*/1);
+ state.PopFront(/*peer=*/2);
+
+ EXPECT_EQ(state.GetAliveFramesCount(), 1lu);
+}
+
+TEST(StreamStateTest, RemovePeerForLastExpectedReceiverUpdatesAliveFrames) {
+ StreamState state(/*sender=*/0,
+ /*receivers=*/std::set<size_t>{1, 2},
+ Timestamp::Seconds(1));
+ state.PushBack(/*frame_id=*/1);
+ state.PushBack(/*frame_id=*/2);
+
+ state.PopFront(/*peer=*/1);
+
+ EXPECT_EQ(state.GetAliveFramesCount(), 2lu);
+
+ state.RemovePeer(/*peer=*/2);
+
+ EXPECT_EQ(state.GetAliveFramesCount(), 1lu);
+}
+
+TEST(StreamStateTest, MarkNextAliveFrameAsDeadDecreseAliveFramesCount) {
+ StreamState state(/*sender=*/0,
+ /*receivers=*/std::set<size_t>{1, 2},
+ Timestamp::Seconds(1));
+ state.PushBack(/*frame_id=*/1);
+ state.PushBack(/*frame_id=*/2);
+
+ EXPECT_EQ(state.GetAliveFramesCount(), 2lu);
+
+ state.MarkNextAliveFrameAsDead();
+
+ EXPECT_EQ(state.GetAliveFramesCount(), 1lu);
+}
+
+TEST(StreamStateTest, MarkNextAliveFrameAsDeadDoesntAffectFrontFrameForPeer) {
+ StreamState state(/*sender=*/0,
+ /*receivers=*/std::set<size_t>{1, 2},
+ Timestamp::Seconds(1));
+ state.PushBack(/*frame_id=*/1);
+ state.PushBack(/*frame_id=*/2);
+
+ EXPECT_EQ(state.Front(/*peer=*/1), 1);
+
+ state.MarkNextAliveFrameAsDead();
+
+ EXPECT_EQ(state.Front(/*peer=*/1), 1);
+}
+
+} // namespace
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc
new file mode 100644
index 0000000000..fc970e1ea2
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc
@@ -0,0 +1,2204 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
+
+#include <algorithm>
+#include <map>
+#include <memory>
+#include <vector>
+
+#include "api/rtp_packet_info.h"
+#include "api/rtp_packet_infos.h"
+#include "api/test/create_frame_generator.h"
+#include "api/test/metrics/global_metrics_logger_and_exporter.h"
+#include "api/video/encoded_image.h"
+#include "api/video/i420_buffer.h"
+#include "api/video/video_frame.h"
+#include "common_video/libyuv/include/webrtc_libyuv.h"
+#include "rtc_base/strings/string_builder.h"
+#include "rtc_tools/frame_analyzer/video_geometry_aligner.h"
+#include "system_wrappers/include/sleep.h"
+#include "test/gtest.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
+
+namespace webrtc {
+namespace {
+
+using ::testing::TestWithParam;
+using ::testing::ValuesIn;
+
+using StatsSample = ::webrtc::SamplesStatsCounter::StatsSample;
+
+constexpr int kAnalyzerMaxThreadsCount = 1;
+constexpr int kMaxFramesInFlightPerStream = 10;
+constexpr int kFrameWidth = 320;
+constexpr int kFrameHeight = 240;
+constexpr double kMaxSsim = 1;
+constexpr char kStreamLabel[] = "video-stream";
+constexpr char kSenderPeerName[] = "alice";
+constexpr char kReceiverPeerName[] = "bob";
+
+DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() {
+ DefaultVideoQualityAnalyzerOptions options;
+ options.compute_psnr = false;
+ options.compute_ssim = false;
+ options.adjust_cropping_before_comparing_frames = false;
+ options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream;
+ return options;
+}
+
+VideoFrame NextFrame(test::FrameGeneratorInterface* frame_generator,
+ int64_t timestamp_us) {
+ test::FrameGeneratorInterface::VideoFrameData frame_data =
+ frame_generator->NextFrame();
+ return VideoFrame::Builder()
+ .set_video_frame_buffer(frame_data.buffer)
+ .set_update_rect(frame_data.update_rect)
+ .set_timestamp_us(timestamp_us)
+ .build();
+}
+
+EncodedImage FakeEncode(const VideoFrame& frame) {
+ EncodedImage image;
+ std::vector<RtpPacketInfo> packet_infos;
+ packet_infos.push_back(RtpPacketInfo(
+ /*ssrc=*/1,
+ /*csrcs=*/{},
+ /*rtp_timestamp=*/frame.timestamp(),
+ /*receive_time=*/Timestamp::Micros(frame.timestamp_us() + 10000)));
+ image.SetPacketInfos(RtpPacketInfos(packet_infos));
+ return image;
+}
+
+VideoFrame DeepCopy(const VideoFrame& frame) {
+ VideoFrame copy = frame;
+ copy.set_video_frame_buffer(
+ I420Buffer::Copy(*frame.video_frame_buffer()->ToI420()));
+ return copy;
+}
+
+std::vector<StatsSample> GetSortedSamples(const SamplesStatsCounter& counter) {
+ rtc::ArrayView<const StatsSample> view = counter.GetTimedSamples();
+ std::vector<StatsSample> out(view.begin(), view.end());
+ std::sort(out.begin(), out.end(),
+ [](const StatsSample& a, const StatsSample& b) {
+ return a.time < b.time;
+ });
+ return out;
+}
+
+std::string ToString(const std::vector<StatsSample>& values) {
+ rtc::StringBuilder out;
+ for (const auto& v : values) {
+ out << "{ time_ms=" << v.time.ms() << "; value=" << v.value << "}, ";
+ }
+ return out.str();
+}
+
+void FakeCPULoad() {
+ std::vector<int> temp(1000000);
+ for (size_t i = 0; i < temp.size(); ++i) {
+ temp[i] = rand();
+ }
+ std::sort(temp.begin(), temp.end());
+ ASSERT_TRUE(std::is_sorted(temp.begin(), temp.end()));
+}
+
+void PassFramesThroughAnalyzer(DefaultVideoQualityAnalyzer& analyzer,
+ absl::string_view sender,
+ absl::string_view stream_label,
+ std::vector<absl::string_view> receivers,
+ int frames_count,
+ test::FrameGeneratorInterface& frame_generator,
+ int interframe_delay_ms = 0) {
+ for (int i = 0; i < frames_count; ++i) {
+ VideoFrame frame = NextFrame(&frame_generator, /*timestamp_us=*/1);
+ uint16_t frame_id =
+ analyzer.OnFrameCaptured(sender, std::string(stream_label), frame);
+ frame.set_id(frame_id);
+ analyzer.OnFramePreEncode(sender, frame);
+ analyzer.OnFrameEncoded(sender, frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(),
+ false);
+ for (absl::string_view receiver : receivers) {
+ VideoFrame received_frame = DeepCopy(frame);
+ analyzer.OnFramePreDecode(receiver, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(receiver, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(receiver, received_frame);
+ }
+ if (i < frames_count - 1 && interframe_delay_ms > 0) {
+ SleepMs(interframe_delay_ms);
+ }
+ }
+}
+
+TEST(DefaultVideoQualityAnalyzerTest,
+ MemoryOverloadedAndThenAllFramesReceived) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(),
+ AnalyzerOptionsForTest());
+ analyzer.Start("test_case",
+ std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
+ kAnalyzerMaxThreadsCount);
+
+ std::map<uint16_t, VideoFrame> captured_frames;
+ std::vector<uint16_t> frames_order;
+ for (int i = 0; i < kMaxFramesInFlightPerStream * 2; ++i) {
+ VideoFrame frame = NextFrame(frame_generator.get(), i);
+ frame.set_id(
+ analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
+ frames_order.push_back(frame.id());
+ captured_frames.insert({frame.id(), frame});
+ analyzer.OnFramePreEncode(kSenderPeerName, frame);
+ analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(),
+ false);
+ }
+
+ for (const uint16_t& frame_id : frames_order) {
+ VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
+ analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
+ }
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ AnalyzerStats stats = analyzer.GetAnalyzerStats();
+ EXPECT_EQ(stats.memory_overloaded_comparisons_done,
+ kMaxFramesInFlightPerStream);
+ EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream * 2);
+ FrameCounters frame_counters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream * 2);
+ EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream * 2);
+ EXPECT_EQ(frame_counters.dropped, 0);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest,
+ FillMaxMemoryReceiveAllMemoryOverloadedAndThenAllFramesReceived) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(),
+ AnalyzerOptionsForTest());
+ analyzer.Start("test_case",
+ std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
+ kAnalyzerMaxThreadsCount);
+
+ std::map<uint16_t, VideoFrame> captured_frames;
+ std::vector<uint16_t> frames_order;
+ // Feel analyzer's memory up to limit
+ for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) {
+ VideoFrame frame = NextFrame(frame_generator.get(), i);
+ frame.set_id(
+ analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
+ frames_order.push_back(frame.id());
+ captured_frames.insert({frame.id(), frame});
+ analyzer.OnFramePreEncode(kSenderPeerName, frame);
+ analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(),
+ false);
+ }
+
+ // Receive all frames.
+ for (const uint16_t& frame_id : frames_order) {
+ VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
+ analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
+ }
+ frames_order.clear();
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+
+ // Overload analyzer's memory up to limit
+ for (int i = 0; i < 2 * kMaxFramesInFlightPerStream; ++i) {
+ VideoFrame frame = NextFrame(frame_generator.get(), i);
+ frame.set_id(
+ analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
+ frames_order.push_back(frame.id());
+ captured_frames.insert({frame.id(), frame});
+ analyzer.OnFramePreEncode(kSenderPeerName, frame);
+ analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(),
+ false);
+ }
+
+ // Receive all frames.
+ for (const uint16_t& frame_id : frames_order) {
+ VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
+ analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
+ }
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ AnalyzerStats stats = analyzer.GetAnalyzerStats();
+ EXPECT_EQ(stats.memory_overloaded_comparisons_done,
+ kMaxFramesInFlightPerStream);
+ EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream * 3);
+ FrameCounters frame_counters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream * 3);
+ EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream * 3);
+ EXPECT_EQ(frame_counters.dropped, 0);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest,
+ MemoryOverloadedHalfDroppedAndThenHalfFramesReceived) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(),
+ AnalyzerOptionsForTest());
+ analyzer.Start("test_case",
+ std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
+ kAnalyzerMaxThreadsCount);
+
+ std::map<uint16_t, VideoFrame> captured_frames;
+ std::vector<uint16_t> frames_order;
+ for (int i = 0; i < kMaxFramesInFlightPerStream * 2; ++i) {
+ VideoFrame frame = NextFrame(frame_generator.get(), i);
+ frame.set_id(
+ analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
+ frames_order.push_back(frame.id());
+ captured_frames.insert({frame.id(), frame});
+ analyzer.OnFramePreEncode(kSenderPeerName, frame);
+ analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(),
+ false);
+ }
+
+ for (size_t i = kMaxFramesInFlightPerStream; i < frames_order.size(); ++i) {
+ uint16_t frame_id = frames_order.at(i);
+ VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
+ analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
+ }
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ AnalyzerStats stats = analyzer.GetAnalyzerStats();
+ EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0);
+ EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream * 2);
+ FrameCounters frame_counters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream * 2);
+ EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream);
+ EXPECT_EQ(frame_counters.dropped, kMaxFramesInFlightPerStream);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest, NormalScenario) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(),
+ AnalyzerOptionsForTest());
+ analyzer.Start("test_case",
+ std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
+ kAnalyzerMaxThreadsCount);
+
+ std::map<uint16_t, VideoFrame> captured_frames;
+ std::vector<uint16_t> frames_order;
+ for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) {
+ VideoFrame frame = NextFrame(frame_generator.get(), i);
+ frame.set_id(
+ analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
+ frames_order.push_back(frame.id());
+ captured_frames.insert({frame.id(), frame});
+ analyzer.OnFramePreEncode(kSenderPeerName, frame);
+ analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(),
+ false);
+ }
+
+ for (size_t i = 1; i < frames_order.size(); i += 2) {
+ uint16_t frame_id = frames_order.at(i);
+ VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
+ analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
+ }
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ AnalyzerStats stats = analyzer.GetAnalyzerStats();
+ EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0);
+ EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream);
+
+ std::vector<StatsSample> frames_in_flight_sizes =
+ GetSortedSamples(stats.frames_in_flight_left_count);
+ EXPECT_EQ(frames_in_flight_sizes.back().value, 0)
+ << ToString(frames_in_flight_sizes);
+
+ FrameCounters frame_counters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream);
+ EXPECT_EQ(frame_counters.received, kMaxFramesInFlightPerStream / 2);
+ EXPECT_EQ(frame_counters.decoded, kMaxFramesInFlightPerStream / 2);
+ EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream / 2);
+ EXPECT_EQ(frame_counters.dropped, kMaxFramesInFlightPerStream / 2);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest, OneFrameReceivedTwice) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(),
+ AnalyzerOptionsForTest());
+ analyzer.Start("test_case",
+ std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
+ kAnalyzerMaxThreadsCount);
+
+ VideoFrame captured_frame = NextFrame(frame_generator.get(), 0);
+ captured_frame.set_id(
+ analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, captured_frame));
+ analyzer.OnFramePreEncode(kSenderPeerName, captured_frame);
+ analyzer.OnFrameEncoded(kSenderPeerName, captured_frame.id(),
+ FakeEncode(captured_frame),
+ VideoQualityAnalyzerInterface::EncoderStats(), false);
+
+ VideoFrame received_frame = DeepCopy(captured_frame);
+ analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
+
+ received_frame = DeepCopy(captured_frame);
+ analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ AnalyzerStats stats = analyzer.GetAnalyzerStats();
+ EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0);
+ EXPECT_EQ(stats.comparisons_done, 1);
+
+ FrameCounters frame_counters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(frame_counters.captured, 1);
+ EXPECT_EQ(frame_counters.received, 1);
+ EXPECT_EQ(frame_counters.decoded, 1);
+ EXPECT_EQ(frame_counters.rendered, 1);
+ EXPECT_EQ(frame_counters.dropped, 0);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest, NormalScenario2Receivers) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ constexpr char kAlice[] = "alice";
+ constexpr char kBob[] = "bob";
+ constexpr char kCharlie[] = "charlie";
+
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(),
+ AnalyzerOptionsForTest());
+ analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie},
+ kAnalyzerMaxThreadsCount);
+
+ std::map<uint16_t, VideoFrame> captured_frames;
+ std::vector<uint16_t> frames_order;
+ for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) {
+ VideoFrame frame = NextFrame(frame_generator.get(), i);
+ frame.set_id(analyzer.OnFrameCaptured(kAlice, kStreamLabel, frame));
+ frames_order.push_back(frame.id());
+ captured_frames.insert({frame.id(), frame});
+ analyzer.OnFramePreEncode(kAlice, frame);
+ SleepMs(20);
+ analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(),
+ false);
+ }
+
+ SleepMs(50);
+
+ for (size_t i = 1; i < frames_order.size(); i += 2) {
+ uint16_t frame_id = frames_order.at(i);
+ VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
+ analyzer.OnFramePreDecode(kBob, received_frame.id(),
+ FakeEncode(received_frame));
+ SleepMs(30);
+ analyzer.OnFrameDecoded(kBob, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ SleepMs(10);
+ analyzer.OnFrameRendered(kBob, received_frame);
+ }
+
+ for (size_t i = 1; i < frames_order.size(); i += 2) {
+ uint16_t frame_id = frames_order.at(i);
+ VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
+ analyzer.OnFramePreDecode(kCharlie, received_frame.id(),
+ FakeEncode(received_frame));
+ SleepMs(40);
+ analyzer.OnFrameDecoded(kCharlie, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ SleepMs(5);
+ analyzer.OnFrameRendered(kCharlie, received_frame);
+ }
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats();
+ EXPECT_EQ(analyzer_stats.memory_overloaded_comparisons_done, 0);
+ EXPECT_EQ(analyzer_stats.comparisons_done, kMaxFramesInFlightPerStream * 2);
+
+ FrameCounters frame_counters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream);
+ EXPECT_EQ(frame_counters.received, kMaxFramesInFlightPerStream);
+ EXPECT_EQ(frame_counters.decoded, kMaxFramesInFlightPerStream);
+ EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream);
+ EXPECT_EQ(frame_counters.dropped, kMaxFramesInFlightPerStream);
+
+ VideoStreamsInfo streams_info = analyzer.GetKnownStreams();
+ EXPECT_EQ(streams_info.GetStreams(), std::set<std::string>{kStreamLabel});
+ EXPECT_EQ(streams_info.GetStreams(kAlice),
+ std::set<std::string>{kStreamLabel});
+ EXPECT_EQ(streams_info.GetSender(kStreamLabel), kAlice);
+ EXPECT_EQ(streams_info.GetReceivers(kStreamLabel),
+ (std::set<std::string>{kBob, kCharlie}));
+
+ EXPECT_EQ(streams_info.GetStatsKeys().size(), 2lu);
+ for (auto stream_key : streams_info.GetStatsKeys()) {
+ FrameCounters stream_conters =
+ analyzer.GetPerStreamCounters().at(stream_key);
+ // On some devices the pipeline can be too slow, so we actually can't
+ // force real constraints here. Lets just check, that at least 1
+ // frame passed whole pipeline.
+ EXPECT_GE(stream_conters.captured, 10);
+ EXPECT_GE(stream_conters.pre_encoded, 10);
+ EXPECT_GE(stream_conters.encoded, 10);
+ EXPECT_GE(stream_conters.received, 5);
+ EXPECT_GE(stream_conters.decoded, 5);
+ EXPECT_GE(stream_conters.rendered, 5);
+ EXPECT_GE(stream_conters.dropped, 5);
+ }
+
+ std::map<StatsKey, StreamStats> stats = analyzer.GetStats();
+ const StatsKey kAliceBobStats(kStreamLabel, kBob);
+ const StatsKey kAliceCharlieStats(kStreamLabel, kCharlie);
+ EXPECT_EQ(stats.size(), 2lu);
+ {
+ auto it = stats.find(kAliceBobStats);
+ EXPECT_FALSE(it == stats.end());
+ ASSERT_FALSE(it->second.encode_time_ms.IsEmpty());
+ EXPECT_GE(it->second.encode_time_ms.GetMin(), 20);
+ ASSERT_FALSE(it->second.decode_time_ms.IsEmpty());
+ EXPECT_GE(it->second.decode_time_ms.GetMin(), 30);
+ ASSERT_FALSE(it->second.resolution_of_decoded_frame.IsEmpty());
+ EXPECT_GE(it->second.resolution_of_decoded_frame.GetMin(),
+ kFrameWidth * kFrameHeight - 1);
+ EXPECT_LE(it->second.resolution_of_decoded_frame.GetMax(),
+ kFrameWidth * kFrameHeight + 1);
+ }
+ {
+ auto it = stats.find(kAliceCharlieStats);
+ EXPECT_FALSE(it == stats.end());
+ ASSERT_FALSE(it->second.encode_time_ms.IsEmpty());
+ EXPECT_GE(it->second.encode_time_ms.GetMin(), 20);
+ ASSERT_FALSE(it->second.decode_time_ms.IsEmpty());
+ EXPECT_GE(it->second.decode_time_ms.GetMin(), 30);
+ ASSERT_FALSE(it->second.resolution_of_decoded_frame.IsEmpty());
+ EXPECT_GE(it->second.resolution_of_decoded_frame.GetMin(),
+ kFrameWidth * kFrameHeight - 1);
+ EXPECT_LE(it->second.resolution_of_decoded_frame.GetMax(),
+ kFrameWidth * kFrameHeight + 1);
+ }
+}
+
+// Test the case which can happen when SFU is switching from one layer to
+// another, so the same frame can be received twice by the same peer.
+TEST(DefaultVideoQualityAnalyzerTest,
+ OneFrameReceivedTwiceBySamePeerWith2Receivers) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ constexpr char kAlice[] = "alice";
+ constexpr char kBob[] = "bob";
+ constexpr char kCharlie[] = "charlie";
+
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(),
+ AnalyzerOptionsForTest());
+ analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie},
+ kAnalyzerMaxThreadsCount);
+
+ VideoFrame captured_frame = NextFrame(frame_generator.get(), 0);
+ captured_frame.set_id(
+ analyzer.OnFrameCaptured(kAlice, kStreamLabel, captured_frame));
+ analyzer.OnFramePreEncode(kAlice, captured_frame);
+ analyzer.OnFrameEncoded(kAlice, captured_frame.id(),
+ FakeEncode(captured_frame),
+ VideoQualityAnalyzerInterface::EncoderStats(), false);
+
+ VideoFrame received_frame = DeepCopy(captured_frame);
+ analyzer.OnFramePreDecode(kBob, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kBob, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kBob, received_frame);
+
+ received_frame = DeepCopy(captured_frame);
+ analyzer.OnFramePreDecode(kBob, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kBob, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kBob, received_frame);
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ AnalyzerStats stats = analyzer.GetAnalyzerStats();
+ EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0);
+ // We have 2 comparisons here because 1 for the frame received by Bob and
+ // 1 for the frame in flight from Alice to Charlie.
+ EXPECT_EQ(stats.comparisons_done, 2);
+
+ FrameCounters frame_counters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(frame_counters.captured, 1);
+ EXPECT_EQ(frame_counters.received, 1);
+ EXPECT_EQ(frame_counters.decoded, 1);
+ EXPECT_EQ(frame_counters.rendered, 1);
+ EXPECT_EQ(frame_counters.dropped, 0);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest, HeavyQualityMetricsFromEqualFrames) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions analyzer_options;
+ analyzer_options.compute_psnr = true;
+ analyzer_options.compute_ssim = true;
+ analyzer_options.adjust_cropping_before_comparing_frames = false;
+ analyzer_options.max_frames_in_flight_per_stream_count =
+ kMaxFramesInFlightPerStream;
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(),
+ analyzer_options);
+ analyzer.Start("test_case",
+ std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
+ kAnalyzerMaxThreadsCount);
+
+ for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) {
+ VideoFrame frame = NextFrame(frame_generator.get(), i);
+ frame.set_id(
+ analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
+ analyzer.OnFramePreEncode(kSenderPeerName, frame);
+ analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(),
+ false);
+
+ VideoFrame received_frame = DeepCopy(frame);
+ analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
+ }
+
+ // Give analyzer some time to process frames on async thread. Heavy metrics
+ // computation is turned on, so giving some extra time to be sure that
+ // computatio have ended.
+ SleepMs(500);
+ analyzer.Stop();
+
+ AnalyzerStats stats = analyzer.GetAnalyzerStats();
+ EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0);
+ EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream);
+
+ std::vector<StatsSample> frames_in_flight_sizes =
+ GetSortedSamples(stats.frames_in_flight_left_count);
+ EXPECT_EQ(frames_in_flight_sizes.back().value, 0)
+ << ToString(frames_in_flight_sizes);
+
+ std::map<StatsKey, StreamStats> stream_stats = analyzer.GetStats();
+ const StatsKey kAliceBobStats(kStreamLabel, kReceiverPeerName);
+ EXPECT_EQ(stream_stats.size(), 1lu);
+
+ auto it = stream_stats.find(kAliceBobStats);
+ EXPECT_GE(it->second.psnr.GetMin(), kPerfectPSNR);
+ EXPECT_GE(it->second.ssim.GetMin(), kMaxSsim);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest,
+ HeavyQualityMetricsFromShiftedFramesWithAdjustment) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions analyzer_options;
+ analyzer_options.compute_psnr = true;
+ analyzer_options.compute_ssim = true;
+ analyzer_options.adjust_cropping_before_comparing_frames = true;
+ analyzer_options.max_frames_in_flight_per_stream_count =
+ kMaxFramesInFlightPerStream;
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(),
+ analyzer_options);
+ analyzer.Start("test_case",
+ std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
+ kAnalyzerMaxThreadsCount);
+
+ for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) {
+ VideoFrame frame = NextFrame(frame_generator.get(), i);
+ frame.set_id(
+ analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
+ analyzer.OnFramePreEncode(kSenderPeerName, frame);
+ analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(),
+ false);
+
+ VideoFrame received_frame = frame;
+ // Shift frame by a few pixels.
+ test::CropRegion crop_region{0, 1, 3, 0};
+ rtc::scoped_refptr<VideoFrameBuffer> cropped_buffer =
+ CropAndZoom(crop_region, received_frame.video_frame_buffer()->ToI420());
+ received_frame.set_video_frame_buffer(cropped_buffer);
+
+ analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
+ }
+
+ // Give analyzer some time to process frames on async thread. Heavy metrics
+ // computation is turned on, so giving some extra time to be sure that
+ // computatio have ended.
+ SleepMs(500);
+ analyzer.Stop();
+
+ AnalyzerStats stats = analyzer.GetAnalyzerStats();
+ EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0);
+ EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream);
+
+ std::vector<StatsSample> frames_in_flight_sizes =
+ GetSortedSamples(stats.frames_in_flight_left_count);
+ EXPECT_EQ(frames_in_flight_sizes.back().value, 0)
+ << ToString(frames_in_flight_sizes);
+
+ std::map<StatsKey, StreamStats> stream_stats = analyzer.GetStats();
+ const StatsKey kAliceBobStats(kStreamLabel, kReceiverPeerName);
+ EXPECT_EQ(stream_stats.size(), 1lu);
+
+ auto it = stream_stats.find(kAliceBobStats);
+ EXPECT_GE(it->second.psnr.GetMin(), kPerfectPSNR);
+ EXPECT_GE(it->second.ssim.GetMin(), kMaxSsim);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest, CpuUsage) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(),
+ AnalyzerOptionsForTest());
+ analyzer.Start("test_case",
+ std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
+ kAnalyzerMaxThreadsCount);
+
+ std::map<uint16_t, VideoFrame> captured_frames;
+ std::vector<uint16_t> frames_order;
+ for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) {
+ VideoFrame frame = NextFrame(frame_generator.get(), i);
+ frame.set_id(
+ analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
+ frames_order.push_back(frame.id());
+ captured_frames.insert({frame.id(), frame});
+ analyzer.OnFramePreEncode(kSenderPeerName, frame);
+ analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(),
+ false);
+ }
+
+ // Windows CPU clock has low accuracy. We need to fake some additional load to
+ // be sure that the clock ticks (https://crbug.com/webrtc/12249).
+ FakeCPULoad();
+
+ for (size_t i = 1; i < frames_order.size(); i += 2) {
+ uint16_t frame_id = frames_order.at(i);
+ VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
+ analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
+ }
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ double cpu_usage = analyzer.GetCpuUsagePercent();
+ ASSERT_GT(cpu_usage, 0);
+
+ SleepMs(100);
+ analyzer.Stop();
+
+ EXPECT_EQ(analyzer.GetCpuUsagePercent(), cpu_usage);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest, RuntimeParticipantsAdding) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ constexpr char kAlice[] = "alice";
+ constexpr char kBob[] = "bob";
+ constexpr char kCharlie[] = "charlie";
+ constexpr char kKatie[] = "katie";
+
+ constexpr int kFramesCount = 9;
+ constexpr int kOneThirdFrames = kFramesCount / 3;
+ constexpr int kTwoThirdFrames = 2 * kOneThirdFrames;
+
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(),
+ AnalyzerOptionsForTest());
+ analyzer.Start("test_case", {}, kAnalyzerMaxThreadsCount);
+
+ std::map<uint16_t, VideoFrame> captured_frames;
+ std::vector<uint16_t> frames_order;
+ analyzer.RegisterParticipantInCall(kAlice);
+ analyzer.RegisterParticipantInCall(kBob);
+
+ // Alice is sending frames.
+ for (int i = 0; i < kFramesCount; ++i) {
+ VideoFrame frame = NextFrame(frame_generator.get(), i);
+ frame.set_id(analyzer.OnFrameCaptured(kAlice, kStreamLabel, frame));
+ frames_order.push_back(frame.id());
+ captured_frames.insert({frame.id(), frame});
+ analyzer.OnFramePreEncode(kAlice, frame);
+ analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(),
+ false);
+ }
+
+ // Bob receives one third of the sent frames.
+ for (int i = 0; i < kOneThirdFrames; ++i) {
+ uint16_t frame_id = frames_order.at(i);
+ VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
+ analyzer.OnFramePreDecode(kBob, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kBob, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kBob, received_frame);
+ }
+
+ analyzer.RegisterParticipantInCall(kCharlie);
+ analyzer.RegisterParticipantInCall(kKatie);
+
+ // New participants were dynamically added. Bob and Charlie receive second
+ // third of the sent frames. Katie drops the frames.
+ for (int i = kOneThirdFrames; i < kTwoThirdFrames; ++i) {
+ uint16_t frame_id = frames_order.at(i);
+ VideoFrame bob_received_frame = DeepCopy(captured_frames.at(frame_id));
+ analyzer.OnFramePreDecode(kBob, bob_received_frame.id(),
+ FakeEncode(bob_received_frame));
+ analyzer.OnFrameDecoded(kBob, bob_received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kBob, bob_received_frame);
+
+ VideoFrame charlie_received_frame = DeepCopy(captured_frames.at(frame_id));
+ analyzer.OnFramePreDecode(kCharlie, charlie_received_frame.id(),
+ FakeEncode(charlie_received_frame));
+ analyzer.OnFrameDecoded(kCharlie, charlie_received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kCharlie, charlie_received_frame);
+ }
+
+ // Bob, Charlie and Katie receive the rest of the sent frames.
+ for (int i = kTwoThirdFrames; i < kFramesCount; ++i) {
+ uint16_t frame_id = frames_order.at(i);
+ VideoFrame bob_received_frame = DeepCopy(captured_frames.at(frame_id));
+ analyzer.OnFramePreDecode(kBob, bob_received_frame.id(),
+ FakeEncode(bob_received_frame));
+ analyzer.OnFrameDecoded(kBob, bob_received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kBob, bob_received_frame);
+
+ VideoFrame charlie_received_frame = DeepCopy(captured_frames.at(frame_id));
+ analyzer.OnFramePreDecode(kCharlie, charlie_received_frame.id(),
+ FakeEncode(charlie_received_frame));
+ analyzer.OnFrameDecoded(kCharlie, charlie_received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kCharlie, charlie_received_frame);
+
+ VideoFrame katie_received_frame = DeepCopy(captured_frames.at(frame_id));
+ analyzer.OnFramePreDecode(kKatie, katie_received_frame.id(),
+ FakeEncode(katie_received_frame));
+ analyzer.OnFrameDecoded(kKatie, katie_received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kKatie, katie_received_frame);
+ }
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ AnalyzerStats stats = analyzer.GetAnalyzerStats();
+ EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0);
+ EXPECT_EQ(stats.comparisons_done, kFramesCount + 2 * kTwoThirdFrames);
+
+ std::vector<StatsSample> frames_in_flight_sizes =
+ GetSortedSamples(stats.frames_in_flight_left_count);
+ EXPECT_EQ(frames_in_flight_sizes.back().value, 0)
+ << ToString(frames_in_flight_sizes);
+
+ FrameCounters frame_counters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(frame_counters.captured, kFramesCount);
+ EXPECT_EQ(frame_counters.received, 2 * kFramesCount);
+ EXPECT_EQ(frame_counters.decoded, 2 * kFramesCount);
+ EXPECT_EQ(frame_counters.rendered, 2 * kFramesCount);
+ EXPECT_EQ(frame_counters.dropped, kOneThirdFrames);
+
+ const StatsKey kAliceBobStats(kStreamLabel, kBob);
+ const StatsKey kAliceCharlieStats(kStreamLabel, kCharlie);
+ const StatsKey kAliceKatieStats(kStreamLabel, kKatie);
+ EXPECT_EQ(analyzer.GetKnownStreams().GetStatsKeys(),
+ (std::set<StatsKey>{kAliceBobStats, kAliceCharlieStats,
+ kAliceKatieStats}));
+ {
+ FrameCounters stream_conters =
+ analyzer.GetPerStreamCounters().at(kAliceBobStats);
+ EXPECT_EQ(stream_conters.captured, kFramesCount);
+ EXPECT_EQ(stream_conters.pre_encoded, kFramesCount);
+ EXPECT_EQ(stream_conters.encoded, kFramesCount);
+ EXPECT_EQ(stream_conters.received, kFramesCount);
+ EXPECT_EQ(stream_conters.decoded, kFramesCount);
+ EXPECT_EQ(stream_conters.rendered, kFramesCount);
+ }
+ {
+ FrameCounters stream_conters =
+ analyzer.GetPerStreamCounters().at(kAliceCharlieStats);
+ EXPECT_EQ(stream_conters.captured, kFramesCount);
+ EXPECT_EQ(stream_conters.pre_encoded, kFramesCount);
+ EXPECT_EQ(stream_conters.encoded, kFramesCount);
+ EXPECT_EQ(stream_conters.received, kTwoThirdFrames);
+ EXPECT_EQ(stream_conters.decoded, kTwoThirdFrames);
+ EXPECT_EQ(stream_conters.rendered, kTwoThirdFrames);
+ }
+ {
+ FrameCounters stream_conters =
+ analyzer.GetPerStreamCounters().at(kAliceKatieStats);
+ EXPECT_EQ(stream_conters.captured, kFramesCount);
+ EXPECT_EQ(stream_conters.pre_encoded, kFramesCount);
+ EXPECT_EQ(stream_conters.encoded, kFramesCount);
+ EXPECT_EQ(stream_conters.received, kOneThirdFrames);
+ EXPECT_EQ(stream_conters.decoded, kOneThirdFrames);
+ EXPECT_EQ(stream_conters.rendered, kOneThirdFrames);
+ }
+}
+
+TEST(DefaultVideoQualityAnalyzerTest,
+ SimulcastFrameWasFullyReceivedByAllPeersBeforeEncodeFinish) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(),
+ AnalyzerOptionsForTest());
+ constexpr char kAlice[] = "alice";
+ constexpr char kBob[] = "bob";
+ constexpr char kCharlie[] = "charlie";
+ analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie},
+ kAnalyzerMaxThreadsCount);
+
+ VideoFrame frame = NextFrame(frame_generator.get(), 1);
+
+ frame.set_id(analyzer.OnFrameCaptured(kAlice, kStreamLabel, frame));
+ analyzer.OnFramePreEncode(kAlice, frame);
+ // Encode 1st simulcast layer
+ analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(), false);
+
+ // Receive by Bob
+ VideoFrame received_frame = DeepCopy(frame);
+ analyzer.OnFramePreDecode(kBob, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kBob, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kBob, received_frame);
+ // Receive by Charlie
+ received_frame = DeepCopy(frame);
+ analyzer.OnFramePreDecode(kCharlie, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kCharlie, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kCharlie, received_frame);
+
+ // Encode 2nd simulcast layer
+ analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(), false);
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ AnalyzerStats stats = analyzer.GetAnalyzerStats();
+ EXPECT_EQ(stats.comparisons_done, 2);
+
+ std::vector<StatsSample> frames_in_flight_sizes =
+ GetSortedSamples(stats.frames_in_flight_left_count);
+ EXPECT_EQ(frames_in_flight_sizes.back().value, 0)
+ << ToString(frames_in_flight_sizes);
+
+ FrameCounters frame_counters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(frame_counters.captured, 1);
+ EXPECT_EQ(frame_counters.rendered, 2);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest,
+ FrameCanBeReceivedBySenderAfterItWasReceivedByReceiver) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ options.enable_receive_own_stream = true;
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case",
+ std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
+ kAnalyzerMaxThreadsCount);
+
+ std::vector<VideoFrame> frames;
+ for (int i = 0; i < 3; ++i) {
+ VideoFrame frame = NextFrame(frame_generator.get(), 1);
+ frame.set_id(
+ analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
+ frames.push_back(frame);
+ analyzer.OnFramePreEncode(kSenderPeerName, frame);
+ analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(),
+ false);
+ }
+
+ // Receive by 2nd peer.
+ for (VideoFrame& frame : frames) {
+ VideoFrame received_frame = DeepCopy(frame);
+ analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
+ }
+
+ // Check that we still have that frame in flight.
+ AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats();
+ std::vector<StatsSample> frames_in_flight_sizes =
+ GetSortedSamples(analyzer_stats.frames_in_flight_left_count);
+ EXPECT_EQ(frames_in_flight_sizes.back().value, 3)
+ << "Expected that frame is still in flight, "
+ << "because it wasn't received by sender"
+ << ToString(frames_in_flight_sizes);
+
+ // Receive by sender
+ for (VideoFrame& frame : frames) {
+ VideoFrame received_frame = DeepCopy(frame);
+ analyzer.OnFramePreDecode(kSenderPeerName, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kSenderPeerName, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kSenderPeerName, received_frame);
+ }
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ analyzer_stats = analyzer.GetAnalyzerStats();
+ EXPECT_EQ(analyzer_stats.comparisons_done, 6);
+
+ frames_in_flight_sizes =
+ GetSortedSamples(analyzer_stats.frames_in_flight_left_count);
+ EXPECT_EQ(frames_in_flight_sizes.back().value, 0)
+ << ToString(frames_in_flight_sizes);
+
+ FrameCounters frame_counters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(frame_counters.captured, 3);
+ EXPECT_EQ(frame_counters.rendered, 6);
+
+ EXPECT_EQ(analyzer.GetStats().size(), 2lu);
+ {
+ FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
+ StatsKey(kStreamLabel, kReceiverPeerName));
+ EXPECT_EQ(stream_conters.captured, 3);
+ EXPECT_EQ(stream_conters.pre_encoded, 3);
+ EXPECT_EQ(stream_conters.encoded, 3);
+ EXPECT_EQ(stream_conters.received, 3);
+ EXPECT_EQ(stream_conters.decoded, 3);
+ EXPECT_EQ(stream_conters.rendered, 3);
+ }
+ {
+ FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
+ StatsKey(kStreamLabel, kSenderPeerName));
+ EXPECT_EQ(stream_conters.captured, 3);
+ EXPECT_EQ(stream_conters.pre_encoded, 3);
+ EXPECT_EQ(stream_conters.encoded, 3);
+ EXPECT_EQ(stream_conters.received, 3);
+ EXPECT_EQ(stream_conters.decoded, 3);
+ EXPECT_EQ(stream_conters.rendered, 3);
+ }
+}
+
+TEST(DefaultVideoQualityAnalyzerTest,
+ FrameCanBeReceivedByReceiverAfterItWasReceivedBySender) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ options.enable_receive_own_stream = true;
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case",
+ std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
+ kAnalyzerMaxThreadsCount);
+
+ std::vector<VideoFrame> frames;
+ for (int i = 0; i < 3; ++i) {
+ VideoFrame frame = NextFrame(frame_generator.get(), 1);
+ frame.set_id(
+ analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
+ frames.push_back(frame);
+ analyzer.OnFramePreEncode(kSenderPeerName, frame);
+ analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(),
+ false);
+ }
+
+ // Receive by sender
+ for (VideoFrame& frame : frames) {
+ VideoFrame received_frame = DeepCopy(frame);
+ analyzer.OnFramePreDecode(kSenderPeerName, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kSenderPeerName, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kSenderPeerName, received_frame);
+ }
+
+ // Check that we still have that frame in flight.
+ AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats();
+ std::vector<StatsSample> frames_in_flight_sizes =
+ GetSortedSamples(analyzer_stats.frames_in_flight_left_count);
+ EXPECT_EQ(frames_in_flight_sizes.back().value, 3)
+ << "Expected that frame is still in flight, "
+ << "because it wasn't received by sender"
+ << ToString(frames_in_flight_sizes);
+
+ // Receive by 2nd peer.
+ for (VideoFrame& frame : frames) {
+ VideoFrame received_frame = DeepCopy(frame);
+ analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
+ FakeEncode(received_frame));
+ analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
+ }
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ analyzer_stats = analyzer.GetAnalyzerStats();
+ EXPECT_EQ(analyzer_stats.comparisons_done, 6);
+
+ frames_in_flight_sizes =
+ GetSortedSamples(analyzer_stats.frames_in_flight_left_count);
+ EXPECT_EQ(frames_in_flight_sizes.back().value, 0)
+ << ToString(frames_in_flight_sizes);
+
+ FrameCounters frame_counters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(frame_counters.captured, 3);
+ EXPECT_EQ(frame_counters.rendered, 6);
+
+ EXPECT_EQ(analyzer.GetStats().size(), 2lu);
+ {
+ FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
+ StatsKey(kStreamLabel, kReceiverPeerName));
+ EXPECT_EQ(stream_conters.captured, 3);
+ EXPECT_EQ(stream_conters.pre_encoded, 3);
+ EXPECT_EQ(stream_conters.encoded, 3);
+ EXPECT_EQ(stream_conters.received, 3);
+ EXPECT_EQ(stream_conters.decoded, 3);
+ EXPECT_EQ(stream_conters.rendered, 3);
+ }
+ {
+ FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
+ StatsKey(kStreamLabel, kSenderPeerName));
+ EXPECT_EQ(stream_conters.captured, 3);
+ EXPECT_EQ(stream_conters.pre_encoded, 3);
+ EXPECT_EQ(stream_conters.encoded, 3);
+ EXPECT_EQ(stream_conters.received, 3);
+ EXPECT_EQ(stream_conters.decoded, 3);
+ EXPECT_EQ(stream_conters.rendered, 3);
+ }
+}
+
+TEST(DefaultVideoQualityAnalyzerTest, CodecTrackedCorrectly) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(),
+ AnalyzerOptionsForTest());
+ analyzer.Start("test_case",
+ std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
+ kAnalyzerMaxThreadsCount);
+
+ VideoQualityAnalyzerInterface::EncoderStats encoder_stats;
+ std::vector<std::string> codec_names = {"codec_1", "codec_2"};
+ std::vector<VideoFrame> frames;
+ // Send 3 frame for each codec.
+ for (size_t i = 0; i < codec_names.size(); ++i) {
+ for (size_t j = 0; j < 3; ++j) {
+ VideoFrame frame = NextFrame(frame_generator.get(), 3 * i + j);
+ frame.set_id(
+ analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
+ analyzer.OnFramePreEncode(kSenderPeerName, frame);
+ encoder_stats.encoder_name = codec_names[i];
+ analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
+ encoder_stats, false);
+ frames.push_back(std::move(frame));
+ }
+ }
+
+ // Receive 3 frame for each codec.
+ VideoQualityAnalyzerInterface::DecoderStats decoder_stats;
+ for (size_t i = 0; i < codec_names.size(); ++i) {
+ for (size_t j = 0; j < 3; ++j) {
+ VideoFrame received_frame = DeepCopy(frames[3 * i + j]);
+ analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
+ FakeEncode(received_frame));
+ decoder_stats.decoder_name = codec_names[i];
+ analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, decoder_stats);
+ analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
+ }
+ }
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ std::map<StatsKey, StreamStats> stats = analyzer.GetStats();
+ ASSERT_EQ(stats.size(), 1lu);
+ const StreamStats& stream_stats =
+ stats.at(StatsKey(kStreamLabel, kReceiverPeerName));
+ ASSERT_EQ(stream_stats.encoders.size(), 2lu);
+ EXPECT_EQ(stream_stats.encoders[0].codec_name, codec_names[0]);
+ EXPECT_EQ(stream_stats.encoders[0].first_frame_id, frames[0].id());
+ EXPECT_EQ(stream_stats.encoders[0].last_frame_id, frames[2].id());
+ EXPECT_EQ(stream_stats.encoders[1].codec_name, codec_names[1]);
+ EXPECT_EQ(stream_stats.encoders[1].first_frame_id, frames[3].id());
+ EXPECT_EQ(stream_stats.encoders[1].last_frame_id, frames[5].id());
+
+ ASSERT_EQ(stream_stats.decoders.size(), 2lu);
+ EXPECT_EQ(stream_stats.decoders[0].codec_name, codec_names[0]);
+ EXPECT_EQ(stream_stats.decoders[0].first_frame_id, frames[0].id());
+ EXPECT_EQ(stream_stats.decoders[0].last_frame_id, frames[2].id());
+ EXPECT_EQ(stream_stats.decoders[1].codec_name, codec_names[1]);
+ EXPECT_EQ(stream_stats.decoders[1].first_frame_id, frames[3].id());
+ EXPECT_EQ(stream_stats.decoders[1].last_frame_id, frames[5].id());
+}
+
+TEST(DefaultVideoQualityAnalyzerTest,
+ FramesInFlightAreCorrectlySentToTheComparatorAfterStop) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case",
+ std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
+ kAnalyzerMaxThreadsCount);
+
+ // There are 7 different timings inside frame stats: captured, pre_encode,
+ // encoded, received, decode_start, decode_end, rendered. captured is always
+ // set and received is set together with decode_start. So we create 6
+ // different frames, where for each frame next timings will be set
+ // * 1st - all of them set
+ // * 2nd - captured, pre_encode, encoded, received, decode_start, decode_end
+ // * 3rd - captured, pre_encode, encoded, received, decode_start
+ // * 4th - captured, pre_encode, encoded
+ // * 5th - captured, pre_encode
+ // * 6th - captured
+ std::vector<VideoFrame> frames;
+ // Sender side actions
+ for (int i = 0; i < 6; ++i) {
+ VideoFrame frame = NextFrame(frame_generator.get(), 1);
+ frame.set_id(
+ analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
+ frames.push_back(frame);
+ }
+ for (int i = 0; i < 5; ++i) {
+ analyzer.OnFramePreEncode(kSenderPeerName, frames[i]);
+ }
+ for (int i = 0; i < 4; ++i) {
+ analyzer.OnFrameEncoded(
+ kSenderPeerName, frames[i].id(), FakeEncode(frames[i]),
+ VideoQualityAnalyzerInterface::EncoderStats(), false);
+ }
+
+ // Receiver side actions
+ for (int i = 0; i < 3; ++i) {
+ analyzer.OnFramePreDecode(kReceiverPeerName, frames[i].id(),
+ FakeEncode(frames[i]));
+ }
+ for (int i = 0; i < 2; ++i) {
+ analyzer.OnFrameDecoded(kReceiverPeerName, DeepCopy(frames[i]),
+ VideoQualityAnalyzerInterface::DecoderStats());
+ }
+ for (int i = 0; i < 1; ++i) {
+ analyzer.OnFrameRendered(kReceiverPeerName, DeepCopy(frames[i]));
+ }
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats();
+ EXPECT_EQ(analyzer_stats.comparisons_done, 6);
+
+ // The last frames in flight size has to reflect the amount of frame in flight
+ // before all of them were sent to the comparison when Stop() was invoked.
+ std::vector<StatsSample> frames_in_flight_sizes =
+ GetSortedSamples(analyzer_stats.frames_in_flight_left_count);
+ EXPECT_EQ(frames_in_flight_sizes.back().value, 5)
+ << ToString(frames_in_flight_sizes);
+
+ FrameCounters frame_counters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(frame_counters.captured, 6);
+ EXPECT_EQ(frame_counters.pre_encoded, 5);
+ EXPECT_EQ(frame_counters.encoded, 4);
+ EXPECT_EQ(frame_counters.received, 3);
+ EXPECT_EQ(frame_counters.decoded, 2);
+ EXPECT_EQ(frame_counters.rendered, 1);
+
+ EXPECT_EQ(analyzer.GetStats().size(), 1lu);
+ {
+ FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
+ StatsKey(kStreamLabel, kReceiverPeerName));
+ EXPECT_EQ(stream_conters.captured, 6);
+ EXPECT_EQ(stream_conters.pre_encoded, 5);
+ EXPECT_EQ(stream_conters.encoded, 4);
+ EXPECT_EQ(stream_conters.received, 3);
+ EXPECT_EQ(stream_conters.decoded, 2);
+ EXPECT_EQ(stream_conters.rendered, 1);
+ }
+}
+
+TEST(
+ DefaultVideoQualityAnalyzerTest,
+ FramesInFlightAreCorrectlySentToTheComparatorAfterStopForSenderAndReceiver) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ options.enable_receive_own_stream = true;
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case",
+ std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
+ kAnalyzerMaxThreadsCount);
+
+ // There are 7 different timings inside frame stats: captured, pre_encode,
+ // encoded, received, decode_start, decode_end, rendered. captured is always
+ // set and received is set together with decode_start. So we create 6
+ // different frames, where for each frame next timings will be set
+ // * 1st - all of them set
+ // * 2nd - captured, pre_encode, encoded, received, decode_start, decode_end
+ // * 3rd - captured, pre_encode, encoded, received, decode_start
+ // * 4th - captured, pre_encode, encoded
+ // * 5th - captured, pre_encode
+ // * 6th - captured
+ std::vector<VideoFrame> frames;
+ // Sender side actions
+ for (int i = 0; i < 6; ++i) {
+ VideoFrame frame = NextFrame(frame_generator.get(), 1);
+ frame.set_id(
+ analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
+ frames.push_back(frame);
+ }
+ for (int i = 0; i < 5; ++i) {
+ analyzer.OnFramePreEncode(kSenderPeerName, frames[i]);
+ }
+ for (int i = 0; i < 4; ++i) {
+ analyzer.OnFrameEncoded(
+ kSenderPeerName, frames[i].id(), FakeEncode(frames[i]),
+ VideoQualityAnalyzerInterface::EncoderStats(), false);
+ }
+
+ // Receiver side actions
+ for (int i = 0; i < 3; ++i) {
+ analyzer.OnFramePreDecode(kSenderPeerName, frames[i].id(),
+ FakeEncode(frames[i]));
+ analyzer.OnFramePreDecode(kReceiverPeerName, frames[i].id(),
+ FakeEncode(frames[i]));
+ }
+ for (int i = 0; i < 2; ++i) {
+ analyzer.OnFrameDecoded(kSenderPeerName, DeepCopy(frames[i]),
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameDecoded(kReceiverPeerName, DeepCopy(frames[i]),
+ VideoQualityAnalyzerInterface::DecoderStats());
+ }
+ for (int i = 0; i < 1; ++i) {
+ analyzer.OnFrameRendered(kSenderPeerName, DeepCopy(frames[i]));
+ analyzer.OnFrameRendered(kReceiverPeerName, DeepCopy(frames[i]));
+ }
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats();
+ EXPECT_EQ(analyzer_stats.comparisons_done, 12);
+
+ // The last frames in flight size has to reflect the amount of frame in flight
+ // before all of them were sent to the comparison when Stop() was invoked.
+ std::vector<StatsSample> frames_in_flight_sizes =
+ GetSortedSamples(analyzer_stats.frames_in_flight_left_count);
+ EXPECT_EQ(frames_in_flight_sizes.back().value, 5)
+ << ToString(frames_in_flight_sizes);
+
+ FrameCounters frame_counters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(frame_counters.captured, 6);
+ EXPECT_EQ(frame_counters.pre_encoded, 5);
+ EXPECT_EQ(frame_counters.encoded, 4);
+ EXPECT_EQ(frame_counters.received, 6);
+ EXPECT_EQ(frame_counters.decoded, 4);
+ EXPECT_EQ(frame_counters.rendered, 2);
+
+ EXPECT_EQ(analyzer.GetStats().size(), 2lu);
+ {
+ FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
+ StatsKey(kStreamLabel, kReceiverPeerName));
+ EXPECT_EQ(stream_conters.captured, 6);
+ EXPECT_EQ(stream_conters.pre_encoded, 5);
+ EXPECT_EQ(stream_conters.encoded, 4);
+ EXPECT_EQ(stream_conters.received, 3);
+ EXPECT_EQ(stream_conters.decoded, 2);
+ EXPECT_EQ(stream_conters.rendered, 1);
+ }
+ {
+ FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
+ StatsKey(kStreamLabel, kSenderPeerName));
+ EXPECT_EQ(stream_conters.captured, 6);
+ EXPECT_EQ(stream_conters.pre_encoded, 5);
+ EXPECT_EQ(stream_conters.encoded, 4);
+ EXPECT_EQ(stream_conters.received, 3);
+ EXPECT_EQ(stream_conters.decoded, 2);
+ EXPECT_EQ(stream_conters.rendered, 1);
+ }
+}
+
+TEST(DefaultVideoQualityAnalyzerTest, GetStreamFrames) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
+ kAnalyzerMaxThreadsCount);
+
+ // The order in which peers captured frames and passed them to analyzer.
+ std::vector<std::string> frame_capturers_sequence{
+ "alice", "alice", "bob", "bob", "bob",
+ "bob", "bob", "alice", "alice", "alice",
+ };
+
+ std::map<std::string, std::vector<uint16_t>> stream_to_frame_ids;
+ stream_to_frame_ids.emplace("alice_video", std::vector<uint16_t>{});
+ stream_to_frame_ids.emplace("bob_video", std::vector<uint16_t>{});
+
+ std::vector<VideoFrame> frames;
+ for (const std::string& sender : frame_capturers_sequence) {
+ VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1);
+ uint16_t frame_id =
+ analyzer.OnFrameCaptured(sender, sender + "_video", frame);
+ frame.set_id(frame_id);
+ stream_to_frame_ids.find(sender + "_video")->second.push_back(frame_id);
+ frames.push_back(frame);
+ analyzer.OnFramePreEncode(sender, frame);
+ analyzer.OnFrameEncoded(sender, frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(),
+ false);
+ }
+ // We don't need to receive frames for stats to be gathered correctly.
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ EXPECT_EQ(analyzer.GetStreamFrames(), stream_to_frame_ids);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest, ReceiverReceivedFramesWhenSenderRemoved) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
+ kAnalyzerMaxThreadsCount);
+
+ VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1);
+ uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame);
+ frame.set_id(frame_id);
+ analyzer.OnFramePreEncode("alice", frame);
+ analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(), false);
+
+ analyzer.UnregisterParticipantInCall("alice");
+
+ analyzer.OnFramePreDecode("bob", frame.id(), FakeEncode(frame));
+ analyzer.OnFrameDecoded("bob", DeepCopy(frame),
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered("bob", DeepCopy(frame));
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ FrameCounters stream_conters =
+ analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob"));
+ EXPECT_EQ(stream_conters.captured, 1);
+ EXPECT_EQ(stream_conters.pre_encoded, 1);
+ EXPECT_EQ(stream_conters.encoded, 1);
+ EXPECT_EQ(stream_conters.received, 1);
+ EXPECT_EQ(stream_conters.decoded, 1);
+ EXPECT_EQ(stream_conters.rendered, 1);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest,
+ ReceiverReceivedFramesWhenSenderRemovedWithSelfview) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ options.enable_receive_own_stream = true;
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
+ kAnalyzerMaxThreadsCount);
+
+ VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1);
+ uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame);
+ frame.set_id(frame_id);
+ analyzer.OnFramePreEncode("alice", frame);
+ analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(), false);
+
+ analyzer.UnregisterParticipantInCall("alice");
+
+ analyzer.OnFramePreDecode("bob", frame.id(), FakeEncode(frame));
+ analyzer.OnFrameDecoded("bob", DeepCopy(frame),
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered("bob", DeepCopy(frame));
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ FrameCounters stream_conters =
+ analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob"));
+ EXPECT_EQ(stream_conters.captured, 1);
+ EXPECT_EQ(stream_conters.pre_encoded, 1);
+ EXPECT_EQ(stream_conters.encoded, 1);
+ EXPECT_EQ(stream_conters.received, 1);
+ EXPECT_EQ(stream_conters.decoded, 1);
+ EXPECT_EQ(stream_conters.rendered, 1);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest,
+ SenderReceivedFramesWhenReceiverRemovedWithSelfview) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ options.enable_receive_own_stream = true;
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
+ kAnalyzerMaxThreadsCount);
+
+ VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1);
+ uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame);
+ frame.set_id(frame_id);
+ analyzer.OnFramePreEncode("alice", frame);
+ analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(), false);
+
+ analyzer.UnregisterParticipantInCall("bob");
+
+ analyzer.OnFramePreDecode("alice", frame.id(), FakeEncode(frame));
+ analyzer.OnFrameDecoded("alice", DeepCopy(frame),
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered("alice", DeepCopy(frame));
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ FrameCounters stream_conters =
+ analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "alice"));
+ EXPECT_EQ(stream_conters.captured, 1);
+ EXPECT_EQ(stream_conters.pre_encoded, 1);
+ EXPECT_EQ(stream_conters.encoded, 1);
+ EXPECT_EQ(stream_conters.received, 1);
+ EXPECT_EQ(stream_conters.decoded, 1);
+ EXPECT_EQ(stream_conters.rendered, 1);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest,
+ SenderAndReceiverReceivedFramesWhenReceiverRemovedWithSelfview) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ options.enable_receive_own_stream = true;
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
+ kAnalyzerMaxThreadsCount);
+
+ VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1);
+ uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame);
+ frame.set_id(frame_id);
+ analyzer.OnFramePreEncode("alice", frame);
+ analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(), false);
+
+ analyzer.OnFramePreDecode("bob", frame.id(), FakeEncode(frame));
+ analyzer.OnFrameDecoded("bob", DeepCopy(frame),
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered("bob", DeepCopy(frame));
+
+ analyzer.UnregisterParticipantInCall("bob");
+
+ analyzer.OnFramePreDecode("alice", frame.id(), FakeEncode(frame));
+ analyzer.OnFrameDecoded("alice", DeepCopy(frame),
+ VideoQualityAnalyzerInterface::DecoderStats());
+ analyzer.OnFrameRendered("alice", DeepCopy(frame));
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ FrameCounters alice_alice_stream_conters =
+ analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "alice"));
+ EXPECT_EQ(alice_alice_stream_conters.captured, 1);
+ EXPECT_EQ(alice_alice_stream_conters.pre_encoded, 1);
+ EXPECT_EQ(alice_alice_stream_conters.encoded, 1);
+ EXPECT_EQ(alice_alice_stream_conters.received, 1);
+ EXPECT_EQ(alice_alice_stream_conters.decoded, 1);
+ EXPECT_EQ(alice_alice_stream_conters.rendered, 1);
+
+ FrameCounters alice_bob_stream_conters =
+ analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob"));
+ EXPECT_EQ(alice_bob_stream_conters.captured, 1);
+ EXPECT_EQ(alice_bob_stream_conters.pre_encoded, 1);
+ EXPECT_EQ(alice_bob_stream_conters.encoded, 1);
+ EXPECT_EQ(alice_bob_stream_conters.received, 1);
+ EXPECT_EQ(alice_bob_stream_conters.decoded, 1);
+ EXPECT_EQ(alice_bob_stream_conters.rendered, 1);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeCapturing2ndFrame) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
+ kAnalyzerMaxThreadsCount);
+
+ PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"},
+ /*frames_count=*/1, *frame_generator);
+ analyzer.UnregisterParticipantInCall("bob");
+ PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {},
+ /*frames_count=*/1, *frame_generator);
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ FrameCounters global_stream_conters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(global_stream_conters.captured, 2);
+ EXPECT_EQ(global_stream_conters.pre_encoded, 2);
+ EXPECT_EQ(global_stream_conters.encoded, 2);
+ EXPECT_EQ(global_stream_conters.received, 1);
+ EXPECT_EQ(global_stream_conters.decoded, 1);
+ EXPECT_EQ(global_stream_conters.rendered, 1);
+ FrameCounters stream_conters =
+ analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob"));
+ EXPECT_EQ(stream_conters.captured, 2);
+ EXPECT_EQ(stream_conters.pre_encoded, 2);
+ EXPECT_EQ(stream_conters.encoded, 2);
+ EXPECT_EQ(stream_conters.received, 1);
+ EXPECT_EQ(stream_conters.decoded, 1);
+ EXPECT_EQ(stream_conters.rendered, 1);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforePreEncoded) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
+ kAnalyzerMaxThreadsCount);
+
+ VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1);
+ uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame);
+ frame.set_id(frame_id);
+ analyzer.UnregisterParticipantInCall("bob");
+ analyzer.OnFramePreEncode("alice", frame);
+ analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(), false);
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ FrameCounters global_stream_conters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(global_stream_conters.captured, 1);
+ EXPECT_EQ(global_stream_conters.pre_encoded, 1);
+ EXPECT_EQ(global_stream_conters.encoded, 1);
+ EXPECT_EQ(global_stream_conters.received, 0);
+ EXPECT_EQ(global_stream_conters.decoded, 0);
+ EXPECT_EQ(global_stream_conters.rendered, 0);
+ FrameCounters stream_conters =
+ analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob"));
+ EXPECT_EQ(stream_conters.captured, 1);
+ EXPECT_EQ(stream_conters.pre_encoded, 1);
+ EXPECT_EQ(stream_conters.encoded, 1);
+ EXPECT_EQ(stream_conters.received, 0);
+ EXPECT_EQ(stream_conters.decoded, 0);
+ EXPECT_EQ(stream_conters.rendered, 0);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeEncoded) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
+ kAnalyzerMaxThreadsCount);
+
+ VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1);
+ uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame);
+ frame.set_id(frame_id);
+ analyzer.OnFramePreEncode("alice", frame);
+ analyzer.UnregisterParticipantInCall("bob");
+ analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(), false);
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ FrameCounters global_stream_conters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(global_stream_conters.captured, 1);
+ EXPECT_EQ(global_stream_conters.pre_encoded, 1);
+ EXPECT_EQ(global_stream_conters.encoded, 1);
+ EXPECT_EQ(global_stream_conters.received, 0);
+ EXPECT_EQ(global_stream_conters.decoded, 0);
+ EXPECT_EQ(global_stream_conters.rendered, 0);
+ FrameCounters stream_conters =
+ analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob"));
+ EXPECT_EQ(stream_conters.captured, 1);
+ EXPECT_EQ(stream_conters.pre_encoded, 1);
+ EXPECT_EQ(stream_conters.encoded, 1);
+ EXPECT_EQ(stream_conters.received, 0);
+ EXPECT_EQ(stream_conters.decoded, 0);
+ EXPECT_EQ(stream_conters.rendered, 0);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest,
+ ReceiverRemovedBetweenSimulcastLayersEncoded) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
+ kAnalyzerMaxThreadsCount);
+
+ VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1);
+ uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame);
+ frame.set_id(frame_id);
+ analyzer.OnFramePreEncode("alice", frame);
+ // 1st simulcast layer encoded
+ analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(), false);
+ analyzer.UnregisterParticipantInCall("bob");
+ // 2nd simulcast layer encoded
+ analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(), false);
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ FrameCounters global_stream_conters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(global_stream_conters.captured, 1);
+ EXPECT_EQ(global_stream_conters.pre_encoded, 1);
+ EXPECT_EQ(global_stream_conters.encoded, 1);
+ EXPECT_EQ(global_stream_conters.received, 0);
+ EXPECT_EQ(global_stream_conters.decoded, 0);
+ EXPECT_EQ(global_stream_conters.rendered, 0);
+ FrameCounters stream_conters =
+ analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob"));
+ EXPECT_EQ(stream_conters.captured, 1);
+ EXPECT_EQ(stream_conters.pre_encoded, 1);
+ EXPECT_EQ(stream_conters.encoded, 1);
+ EXPECT_EQ(stream_conters.received, 0);
+ EXPECT_EQ(stream_conters.decoded, 0);
+ EXPECT_EQ(stream_conters.rendered, 0);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest, UnregisterOneAndRegisterAnother) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case",
+ std::vector<std::string>{"alice", "bob", "charlie"},
+ kAnalyzerMaxThreadsCount);
+
+ PassFramesThroughAnalyzer(analyzer, "alice", "alice_video",
+ {"bob", "charlie"},
+ /*frames_count=*/2, *frame_generator);
+ analyzer.UnregisterParticipantInCall("bob");
+ analyzer.RegisterParticipantInCall("david");
+ PassFramesThroughAnalyzer(analyzer, "alice", "alice_video",
+ {"charlie", "david"},
+ /*frames_count=*/4, *frame_generator);
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ FrameCounters global_stream_conters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(global_stream_conters.captured, 6);
+ EXPECT_EQ(global_stream_conters.pre_encoded, 6);
+ EXPECT_EQ(global_stream_conters.encoded, 6);
+ EXPECT_EQ(global_stream_conters.received, 12);
+ EXPECT_EQ(global_stream_conters.decoded, 12);
+ EXPECT_EQ(global_stream_conters.rendered, 12);
+ FrameCounters alice_bob_stream_conters =
+ analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob"));
+ EXPECT_EQ(alice_bob_stream_conters.captured, 6);
+ EXPECT_EQ(alice_bob_stream_conters.pre_encoded, 6);
+ EXPECT_EQ(alice_bob_stream_conters.encoded, 6);
+ EXPECT_EQ(alice_bob_stream_conters.received, 2);
+ EXPECT_EQ(alice_bob_stream_conters.decoded, 2);
+ EXPECT_EQ(alice_bob_stream_conters.rendered, 2);
+ FrameCounters alice_charlie_stream_conters =
+ analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "charlie"));
+ EXPECT_EQ(alice_charlie_stream_conters.captured, 6);
+ EXPECT_EQ(alice_charlie_stream_conters.pre_encoded, 6);
+ EXPECT_EQ(alice_charlie_stream_conters.encoded, 6);
+ EXPECT_EQ(alice_charlie_stream_conters.received, 6);
+ EXPECT_EQ(alice_charlie_stream_conters.decoded, 6);
+ EXPECT_EQ(alice_charlie_stream_conters.rendered, 6);
+ FrameCounters alice_david_stream_conters =
+ analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "david"));
+ EXPECT_EQ(alice_david_stream_conters.captured, 6);
+ EXPECT_EQ(alice_david_stream_conters.pre_encoded, 6);
+ EXPECT_EQ(alice_david_stream_conters.encoded, 6);
+ EXPECT_EQ(alice_david_stream_conters.received, 4);
+ EXPECT_EQ(alice_david_stream_conters.decoded, 4);
+ EXPECT_EQ(alice_david_stream_conters.rendered, 4);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest,
+ UnregisterOneAndRegisterAnotherRegisterBack) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case",
+ std::vector<std::string>{"alice", "bob", "charlie"},
+ kAnalyzerMaxThreadsCount);
+
+ PassFramesThroughAnalyzer(analyzer, "alice", "alice_video",
+ {"bob", "charlie"},
+ /*frames_count=*/2, *frame_generator);
+ analyzer.UnregisterParticipantInCall("bob");
+ PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"charlie"},
+ /*frames_count=*/4, *frame_generator);
+ analyzer.RegisterParticipantInCall("bob");
+ PassFramesThroughAnalyzer(analyzer, "alice", "alice_video",
+ {"bob", "charlie"},
+ /*frames_count=*/6, *frame_generator);
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ FrameCounters global_stream_conters = analyzer.GetGlobalCounters();
+ EXPECT_EQ(global_stream_conters.captured, 12);
+ EXPECT_EQ(global_stream_conters.pre_encoded, 12);
+ EXPECT_EQ(global_stream_conters.encoded, 12);
+ EXPECT_EQ(global_stream_conters.received, 20);
+ EXPECT_EQ(global_stream_conters.decoded, 20);
+ EXPECT_EQ(global_stream_conters.rendered, 20);
+ FrameCounters alice_bob_stream_conters =
+ analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob"));
+ EXPECT_EQ(alice_bob_stream_conters.captured, 12);
+ EXPECT_EQ(alice_bob_stream_conters.pre_encoded, 12);
+ EXPECT_EQ(alice_bob_stream_conters.encoded, 12);
+ EXPECT_EQ(alice_bob_stream_conters.received, 8);
+ EXPECT_EQ(alice_bob_stream_conters.decoded, 8);
+ EXPECT_EQ(alice_bob_stream_conters.rendered, 8);
+ FrameCounters alice_charlie_stream_conters =
+ analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "charlie"));
+ EXPECT_EQ(alice_charlie_stream_conters.captured, 12);
+ EXPECT_EQ(alice_charlie_stream_conters.pre_encoded, 12);
+ EXPECT_EQ(alice_charlie_stream_conters.encoded, 12);
+ EXPECT_EQ(alice_charlie_stream_conters.received, 12);
+ EXPECT_EQ(alice_charlie_stream_conters.decoded, 12);
+ EXPECT_EQ(alice_charlie_stream_conters.rendered, 12);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest,
+ FramesInFlightAreAccountedForUnregisterPeers) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
+ kAnalyzerMaxThreadsCount);
+
+ // Add one frame in flight which has encode time >= 10ms.
+ VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1);
+ uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame);
+ frame.set_id(frame_id);
+ analyzer.OnFramePreEncode("alice", frame);
+ SleepMs(10);
+ analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame),
+ VideoQualityAnalyzerInterface::EncoderStats(), false);
+
+ analyzer.UnregisterParticipantInCall("bob");
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ StreamStats stats = analyzer.GetStats().at(StatsKey("alice_video", "bob"));
+ ASSERT_EQ(stats.encode_time_ms.NumSamples(), 1);
+ EXPECT_GE(stats.encode_time_ms.GetAverage(), 10);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest, InfraMetricsAreReportedWhenRequested) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ options.report_infra_metrics = true;
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
+ kAnalyzerMaxThreadsCount);
+
+ PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"},
+ /*frames_count=*/1, *frame_generator);
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ AnalyzerStats stats = analyzer.GetAnalyzerStats();
+ EXPECT_EQ(stats.on_frame_captured_processing_time_ms.NumSamples(), 1);
+ EXPECT_EQ(stats.on_frame_pre_encode_processing_time_ms.NumSamples(), 1);
+ EXPECT_EQ(stats.on_frame_encoded_processing_time_ms.NumSamples(), 1);
+ EXPECT_EQ(stats.on_frame_pre_decode_processing_time_ms.NumSamples(), 1);
+ EXPECT_EQ(stats.on_frame_decoded_processing_time_ms.NumSamples(), 1);
+ EXPECT_EQ(stats.on_frame_rendered_processing_time_ms.NumSamples(), 1);
+ EXPECT_EQ(stats.on_decoder_error_processing_time_ms.NumSamples(), 0);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest, InfraMetricsNotCollectedByDefault) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ options.report_infra_metrics = false;
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
+ kAnalyzerMaxThreadsCount);
+
+ PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"},
+ /*frames_count=*/1, *frame_generator);
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ AnalyzerStats stats = analyzer.GetAnalyzerStats();
+ EXPECT_EQ(stats.on_frame_captured_processing_time_ms.NumSamples(), 0);
+ EXPECT_EQ(stats.on_frame_pre_encode_processing_time_ms.NumSamples(), 0);
+ EXPECT_EQ(stats.on_frame_encoded_processing_time_ms.NumSamples(), 0);
+ EXPECT_EQ(stats.on_frame_pre_decode_processing_time_ms.NumSamples(), 0);
+ EXPECT_EQ(stats.on_frame_decoded_processing_time_ms.NumSamples(), 0);
+ EXPECT_EQ(stats.on_frame_rendered_processing_time_ms.NumSamples(), 0);
+ EXPECT_EQ(stats.on_decoder_error_processing_time_ms.NumSamples(), 0);
+}
+
+TEST(DefaultVideoQualityAnalyzerTest,
+ FrameDroppedByDecoderIsAccountedCorrectly) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ options.report_infra_metrics = false;
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
+ kAnalyzerMaxThreadsCount);
+
+ VideoFrame to_be_dropped_frame =
+ NextFrame(frame_generator.get(), /*timestamp_us=*/1);
+ uint16_t frame_id =
+ analyzer.OnFrameCaptured("alice", "alice_video", to_be_dropped_frame);
+ to_be_dropped_frame.set_id(frame_id);
+ analyzer.OnFramePreEncode("alice", to_be_dropped_frame);
+ analyzer.OnFrameEncoded("alice", to_be_dropped_frame.id(),
+ FakeEncode(to_be_dropped_frame),
+ VideoQualityAnalyzerInterface::EncoderStats(), false);
+ VideoFrame received_to_be_dropped_frame = DeepCopy(to_be_dropped_frame);
+ analyzer.OnFramePreDecode("bob", received_to_be_dropped_frame.id(),
+ FakeEncode(received_to_be_dropped_frame));
+ PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"},
+ /*frames_count=*/1, *frame_generator);
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(100);
+ analyzer.Stop();
+
+ StreamStats stats = analyzer.GetStats().at(StatsKey("alice_video", "bob"));
+ ASSERT_EQ(stats.dropped_by_phase[FrameDropPhase::kByDecoder], 1);
+}
+
+class DefaultVideoQualityAnalyzerTimeBetweenFreezesTest
+ : public TestWithParam<bool> {};
+
+TEST_P(DefaultVideoQualityAnalyzerTimeBetweenFreezesTest,
+ TimeBetweenFreezesIsEqualToStreamDurationWhenThereAreNoFeeezes) {
+ std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
+ test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
+ /*type=*/absl::nullopt,
+ /*num_squares=*/absl::nullopt);
+
+ DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
+ DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
+ test::GetGlobalMetricsLogger(), options);
+ analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
+ kAnalyzerMaxThreadsCount);
+
+ PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"},
+ /*frames_count=*/5, *frame_generator,
+ /*interframe_delay_ms=*/50);
+ if (GetParam()) {
+ analyzer.UnregisterParticipantInCall("bob");
+ }
+
+ // Give analyzer some time to process frames on async thread. The computations
+ // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
+ // means we have an issue!
+ SleepMs(50);
+ analyzer.Stop();
+
+ StreamStats stats = analyzer.GetStats().at(StatsKey("alice_video", "bob"));
+ ASSERT_EQ(stats.time_between_freezes_ms.NumSamples(), 1);
+ EXPECT_GE(stats.time_between_freezes_ms.GetAverage(), 200);
+}
+
+INSTANTIATE_TEST_SUITE_P(WithRegisteredAndUnregisteredPeerAtTheEndOfTheCall,
+ DefaultVideoQualityAnalyzerTimeBetweenFreezesTest,
+ ValuesIn({true, false}));
+
+} // namespace
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h
new file mode 100644
index 0000000000..384e901462
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_
+
+#include <cstdint>
+#include <utility>
+
+#include "absl/types/optional.h"
+#include "api/video/encoded_image.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+// Injects frame id into EncodedImage on encoder side
+class EncodedImageDataInjector {
+ public:
+ virtual ~EncodedImageDataInjector() = default;
+
+ // Return encoded image with specified `id` and `discard` flag injected into
+ // its payload. `discard` flag mean does analyzing decoder should discard this
+ // encoded image because it belongs to unnecessary simulcast stream or spatial
+ // layer.
+ virtual EncodedImage InjectData(uint16_t id,
+ bool discard,
+ const EncodedImage& source) = 0;
+};
+
+struct EncodedImageExtractionResult {
+ absl::optional<uint16_t> id;
+ EncodedImage image;
+ // Is true if encoded image should be discarded. It is used to filter out
+ // unnecessary spatial layers and simulcast streams.
+ bool discard;
+};
+
+// Extracts frame id from EncodedImage on decoder side.
+class EncodedImageDataExtractor {
+ public:
+ virtual ~EncodedImageDataExtractor() = default;
+
+ // Invoked by framework before any image will come to the extractor.
+ // `expected_receivers_count` is the expected amount of receivers for each
+ // encoded image.
+ virtual void Start(int expected_receivers_count) = 0;
+
+ // Invoked by framework when it is required to add one more receiver for
+ // frames. Will be invoked before that receiver will start receive data.
+ virtual void AddParticipantInCall() = 0;
+
+ // Invoked by framework when it is required to remove receiver for frames.
+ // Will be invoked after that receiver will stop receiving data.
+ virtual void RemoveParticipantInCall() = 0;
+
+ // Returns encoded image id, extracted from payload and also encoded image
+ // with its original payload. For concatenated spatial layers it should be the
+ // same id.
+ virtual EncodedImageExtractionResult ExtractData(
+ const EncodedImage& source) = 0;
+};
+
+class EncodedImageDataPropagator : public EncodedImageDataInjector,
+ public EncodedImageDataExtractor {
+ public:
+ ~EncodedImageDataPropagator() override = default;
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_ENCODED_IMAGE_DATA_INJECTOR_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc
new file mode 100644
index 0000000000..da9c53beb9
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc
@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/example_video_quality_analyzer.h"
+
+#include "api/array_view.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+
+ExampleVideoQualityAnalyzer::ExampleVideoQualityAnalyzer() = default;
+ExampleVideoQualityAnalyzer::~ExampleVideoQualityAnalyzer() = default;
+
+void ExampleVideoQualityAnalyzer::Start(
+ std::string test_case_name,
+ rtc::ArrayView<const std::string> peer_names,
+ int max_threads_count) {}
+
+uint16_t ExampleVideoQualityAnalyzer::OnFrameCaptured(
+ absl::string_view peer_name,
+ const std::string& stream_label,
+ const webrtc::VideoFrame& frame) {
+ MutexLock lock(&lock_);
+ uint16_t frame_id = next_frame_id_++;
+ if (frame_id == VideoFrame::kNotSetId) {
+ frame_id = next_frame_id_++;
+ }
+ auto it = frames_in_flight_.find(frame_id);
+ if (it == frames_in_flight_.end()) {
+ frames_in_flight_.insert(frame_id);
+ frames_to_stream_label_.insert({frame_id, stream_label});
+ } else {
+ RTC_LOG(LS_WARNING) << "Meet new frame with the same id: " << frame_id
+ << ". Assumes old one as dropped";
+ // We needn't insert frame to frames_in_flight_, because it is already
+ // there.
+ ++frames_dropped_;
+ auto stream_it = frames_to_stream_label_.find(frame_id);
+ RTC_CHECK(stream_it != frames_to_stream_label_.end());
+ stream_it->second = stream_label;
+ }
+ ++frames_captured_;
+ return frame_id;
+}
+
+void ExampleVideoQualityAnalyzer::OnFramePreEncode(
+ absl::string_view peer_name,
+ const webrtc::VideoFrame& frame) {
+ MutexLock lock(&lock_);
+ ++frames_pre_encoded_;
+}
+
+void ExampleVideoQualityAnalyzer::OnFrameEncoded(
+ absl::string_view peer_name,
+ uint16_t frame_id,
+ const webrtc::EncodedImage& encoded_image,
+ const EncoderStats& stats,
+ bool discarded) {
+ MutexLock lock(&lock_);
+ ++frames_encoded_;
+}
+
+void ExampleVideoQualityAnalyzer::OnFrameDropped(
+ absl::string_view peer_name,
+ webrtc::EncodedImageCallback::DropReason reason) {
+ RTC_LOG(LS_INFO) << "Frame dropped by encoder";
+ MutexLock lock(&lock_);
+ ++frames_dropped_;
+}
+
+void ExampleVideoQualityAnalyzer::OnFramePreDecode(
+ absl::string_view peer_name,
+ uint16_t frame_id,
+ const webrtc::EncodedImage& encoded_image) {
+ MutexLock lock(&lock_);
+ ++frames_received_;
+}
+
+void ExampleVideoQualityAnalyzer::OnFrameDecoded(
+ absl::string_view peer_name,
+ const webrtc::VideoFrame& frame,
+ const DecoderStats& stats) {
+ MutexLock lock(&lock_);
+ ++frames_decoded_;
+}
+
+void ExampleVideoQualityAnalyzer::OnFrameRendered(
+ absl::string_view peer_name,
+ const webrtc::VideoFrame& frame) {
+ MutexLock lock(&lock_);
+ frames_in_flight_.erase(frame.id());
+ ++frames_rendered_;
+}
+
+void ExampleVideoQualityAnalyzer::OnEncoderError(
+ absl::string_view peer_name,
+ const webrtc::VideoFrame& frame,
+ int32_t error_code) {
+ RTC_LOG(LS_ERROR) << "Failed to encode frame " << frame.id()
+ << ". Code: " << error_code;
+}
+
+void ExampleVideoQualityAnalyzer::OnDecoderError(absl::string_view peer_name,
+ uint16_t frame_id,
+ int32_t error_code,
+ const DecoderStats& stats) {
+ RTC_LOG(LS_ERROR) << "Failed to decode frame " << frame_id
+ << ". Code: " << error_code;
+}
+
+void ExampleVideoQualityAnalyzer::Stop() {
+ MutexLock lock(&lock_);
+ RTC_LOG(LS_INFO) << "There are " << frames_in_flight_.size()
+ << " frames in flight, assuming all of them are dropped";
+ frames_dropped_ += frames_in_flight_.size();
+}
+
+std::string ExampleVideoQualityAnalyzer::GetStreamLabel(uint16_t frame_id) {
+ MutexLock lock(&lock_);
+ auto it = frames_to_stream_label_.find(frame_id);
+ RTC_DCHECK(it != frames_to_stream_label_.end())
+ << "Unknown frame_id=" << frame_id;
+ return it->second;
+}
+
+uint64_t ExampleVideoQualityAnalyzer::frames_captured() const {
+ MutexLock lock(&lock_);
+ return frames_captured_;
+}
+
+uint64_t ExampleVideoQualityAnalyzer::frames_pre_encoded() const {
+ MutexLock lock(&lock_);
+ return frames_pre_encoded_;
+}
+
+uint64_t ExampleVideoQualityAnalyzer::frames_encoded() const {
+ MutexLock lock(&lock_);
+ return frames_encoded_;
+}
+
+uint64_t ExampleVideoQualityAnalyzer::frames_received() const {
+ MutexLock lock(&lock_);
+ return frames_received_;
+}
+
+uint64_t ExampleVideoQualityAnalyzer::frames_decoded() const {
+ MutexLock lock(&lock_);
+ return frames_decoded_;
+}
+
+uint64_t ExampleVideoQualityAnalyzer::frames_rendered() const {
+ MutexLock lock(&lock_);
+ return frames_rendered_;
+}
+
+uint64_t ExampleVideoQualityAnalyzer::frames_dropped() const {
+ MutexLock lock(&lock_);
+ return frames_dropped_;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h
new file mode 100644
index 0000000000..af4868a961
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/example_video_quality_analyzer.h
@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_EXAMPLE_VIDEO_QUALITY_ANALYZER_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_EXAMPLE_VIDEO_QUALITY_ANALYZER_H_
+
+#include <atomic>
+#include <map>
+#include <set>
+#include <string>
+
+#include "api/array_view.h"
+#include "api/test/video_quality_analyzer_interface.h"
+#include "api/video/encoded_image.h"
+#include "api/video/video_frame.h"
+#include "rtc_base/synchronization/mutex.h"
+
+namespace webrtc {
+
+// This class is an example implementation of
+// webrtc::VideoQualityAnalyzerInterface and calculates simple metrics
+// just to demonstration purposes. Assumed to be used in the single process
+// test cases, where both peers are in the same process.
+class ExampleVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
+ public:
+ ExampleVideoQualityAnalyzer();
+ ~ExampleVideoQualityAnalyzer() override;
+
+ void Start(std::string test_case_name,
+ rtc::ArrayView<const std::string> peer_names,
+ int max_threads_count) override;
+ uint16_t OnFrameCaptured(absl::string_view peer_name,
+ const std::string& stream_label,
+ const VideoFrame& frame) override;
+ void OnFramePreEncode(absl::string_view peer_name,
+ const VideoFrame& frame) override;
+ void OnFrameEncoded(absl::string_view peer_name,
+ uint16_t frame_id,
+ const EncodedImage& encoded_image,
+ const EncoderStats& stats,
+ bool discarded) override;
+ void OnFrameDropped(absl::string_view peer_name,
+ EncodedImageCallback::DropReason reason) override;
+ void OnFramePreDecode(absl::string_view peer_name,
+ uint16_t frame_id,
+ const EncodedImage& encoded_image) override;
+ void OnFrameDecoded(absl::string_view peer_name,
+ const VideoFrame& frame,
+ const DecoderStats& stats) override;
+ void OnFrameRendered(absl::string_view peer_name,
+ const VideoFrame& frame) override;
+ void OnEncoderError(absl::string_view peer_name,
+ const VideoFrame& frame,
+ int32_t error_code) override;
+ void OnDecoderError(absl::string_view peer_name,
+ uint16_t frame_id,
+ int32_t error_code,
+ const DecoderStats& stats) override;
+ void Stop() override;
+ std::string GetStreamLabel(uint16_t frame_id) override;
+
+ uint64_t frames_captured() const;
+ uint64_t frames_pre_encoded() const;
+ uint64_t frames_encoded() const;
+ uint64_t frames_received() const;
+ uint64_t frames_decoded() const;
+ uint64_t frames_rendered() const;
+ uint64_t frames_dropped() const;
+
+ private:
+ // When peer A captured the frame it will come into analyzer's OnFrameCaptured
+ // and will be stored in frames_in_flight_. It will be removed from there
+ // when it will be received in peer B, so we need to guard it with lock.
+ // Also because analyzer will serve for all video streams it can be called
+ // from different threads inside one peer.
+ mutable Mutex lock_;
+ // Stores frame ids, that are currently going from one peer to another. We
+ // need to keep them to correctly determine dropped frames and also correctly
+ // process frame id overlap.
+ std::set<uint16_t> frames_in_flight_ RTC_GUARDED_BY(lock_);
+ std::map<uint16_t, std::string> frames_to_stream_label_ RTC_GUARDED_BY(lock_);
+ uint16_t next_frame_id_ RTC_GUARDED_BY(lock_) = 1;
+ uint64_t frames_captured_ RTC_GUARDED_BY(lock_) = 0;
+ uint64_t frames_pre_encoded_ RTC_GUARDED_BY(lock_) = 0;
+ uint64_t frames_encoded_ RTC_GUARDED_BY(lock_) = 0;
+ uint64_t frames_received_ RTC_GUARDED_BY(lock_) = 0;
+ uint64_t frames_decoded_ RTC_GUARDED_BY(lock_) = 0;
+ uint64_t frames_rendered_ RTC_GUARDED_BY(lock_) = 0;
+ uint64_t frames_dropped_ RTC_GUARDED_BY(lock_) = 0;
+};
+
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_EXAMPLE_VIDEO_QUALITY_ANALYZER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue.h
new file mode 100644
index 0000000000..39d26b42bc
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue.h
@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_MULTI_READER_QUEUE_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_MULTI_READER_QUEUE_H_
+
+#include <deque>
+#include <memory>
+#include <set>
+#include <unordered_map>
+
+#include "absl/types/optional.h"
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+
+// Represents the queue which can be read by multiple readers. Each reader reads
+// from its own queue head. When an element is added it will become visible for
+// all readers. When an element will be removed by all the readers, the element
+// will be removed from the queue.
+template <typename T>
+class MultiReaderQueue {
+ public:
+ // Creates queue with exactly `readers_count` readers named from 0 to
+ // `readers_count - 1`.
+ explicit MultiReaderQueue(size_t readers_count) {
+ for (size_t i = 0; i < readers_count; ++i) {
+ heads_[i] = 0;
+ }
+ }
+ // Creates queue with specified readers.
+ explicit MultiReaderQueue(std::set<size_t> readers) {
+ for (size_t reader : readers) {
+ heads_[reader] = 0;
+ }
+ }
+
+ // Adds a new `reader`, initializing its reading position (the reader's head)
+ // equal to the one of `reader_to_copy`.
+ // Complexity O(MultiReaderQueue::size(reader_to_copy)).
+ void AddReader(size_t reader, size_t reader_to_copy) {
+ size_t pos = GetHeadPositionOrDie(reader_to_copy);
+
+ auto it = heads_.find(reader);
+ RTC_CHECK(it == heads_.end())
+ << "Reader " << reader << " is already in the queue";
+ heads_[reader] = heads_[reader_to_copy];
+ for (size_t i = pos; i < queue_.size(); ++i) {
+ in_queues_[i]++;
+ }
+ }
+
+ // Adds a new `reader`, initializing its reading position equal to first
+ // element in the queue.
+ // Complexity O(MultiReaderQueue::size()).
+ void AddReader(size_t reader) {
+ auto it = heads_.find(reader);
+ RTC_CHECK(it == heads_.end())
+ << "Reader " << reader << " is already in the queue";
+ heads_[reader] = removed_elements_count_;
+ for (size_t i = 0; i < queue_.size(); ++i) {
+ in_queues_[i]++;
+ }
+ }
+
+ // Removes specified `reader` from the queue.
+ // Complexity O(MultiReaderQueue::size(reader)).
+ void RemoveReader(size_t reader) {
+ size_t pos = GetHeadPositionOrDie(reader);
+ for (size_t i = pos; i < queue_.size(); ++i) {
+ in_queues_[i]--;
+ }
+ while (!in_queues_.empty() && in_queues_[0] == 0) {
+ PopFront();
+ }
+ heads_.erase(reader);
+ }
+
+ // Add value to the end of the queue. Complexity O(1).
+ void PushBack(T value) {
+ queue_.push_back(value);
+ in_queues_.push_back(heads_.size());
+ }
+
+ // Extract element from specified head. Complexity O(1).
+ absl::optional<T> PopFront(size_t reader) {
+ size_t pos = GetHeadPositionOrDie(reader);
+ if (pos >= queue_.size()) {
+ return absl::nullopt;
+ }
+
+ T out = queue_[pos];
+
+ in_queues_[pos]--;
+ heads_[reader]++;
+
+ if (in_queues_[pos] == 0) {
+ RTC_DCHECK_EQ(pos, 0);
+ PopFront();
+ }
+ return out;
+ }
+
+ // Returns element at specified head. Complexity O(1).
+ absl::optional<T> Front(size_t reader) const {
+ size_t pos = GetHeadPositionOrDie(reader);
+ if (pos >= queue_.size()) {
+ return absl::nullopt;
+ }
+ return queue_[pos];
+ }
+
+ // Returns true if for specified head there are no more elements in the queue
+ // or false otherwise. Complexity O(1).
+ bool IsEmpty(size_t reader) const {
+ size_t pos = GetHeadPositionOrDie(reader);
+ return pos >= queue_.size();
+ }
+
+ // Returns size of the longest queue between all readers.
+ // Complexity O(1).
+ size_t size() const { return queue_.size(); }
+
+ // Returns size of the specified queue. Complexity O(1).
+ size_t size(size_t reader) const {
+ size_t pos = GetHeadPositionOrDie(reader);
+ return queue_.size() - pos;
+ }
+
+ // Complexity O(1).
+ size_t readers_count() const { return heads_.size(); }
+
+ private:
+ size_t GetHeadPositionOrDie(size_t reader) const {
+ auto it = heads_.find(reader);
+ RTC_CHECK(it != heads_.end()) << "No queue for reader " << reader;
+ return it->second - removed_elements_count_;
+ }
+
+ void PopFront() {
+ RTC_DCHECK(!queue_.empty());
+ RTC_DCHECK_EQ(in_queues_[0], 0);
+ queue_.pop_front();
+ in_queues_.pop_front();
+ removed_elements_count_++;
+ }
+
+ // Number of the elements that were removed from the queue. It is used to
+ // subtract from each head to compute the right index inside `queue_`;
+ size_t removed_elements_count_ = 0;
+ std::deque<T> queue_;
+ // In how may queues the element at index `i` is. An element can be removed
+ // from the front if and only if it is in 0 queues.
+ std::deque<size_t> in_queues_;
+ // Map from the reader to the head position in the queue.
+ std::unordered_map<size_t, size_t> heads_;
+};
+
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_MULTI_READER_QUEUE_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue_test.cc
new file mode 100644
index 0000000000..ea6aa0a416
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/multi_reader_queue_test.cc
@@ -0,0 +1,206 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/multi_reader_queue.h"
+
+#include "absl/types/optional.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace {
+
+TEST(MultiReaderQueueTest, EmptyQueueEmptyForAllHeads) {
+ MultiReaderQueue<int> queue = MultiReaderQueue<int>(/*readers_count=*/10);
+ EXPECT_EQ(queue.size(), 0lu);
+ for (int i = 0; i < 10; ++i) {
+ EXPECT_TRUE(queue.IsEmpty(/*reader=*/i));
+ EXPECT_EQ(queue.size(/*reader=*/i), 0lu);
+ EXPECT_FALSE(queue.PopFront(/*reader=*/i).has_value());
+ EXPECT_FALSE(queue.Front(/*reader=*/i).has_value());
+ }
+}
+
+TEST(MultiReaderQueueTest, SizeIsEqualForAllHeadsAfterAddOnly) {
+ MultiReaderQueue<int> queue = MultiReaderQueue<int>(/*readers_count=*/10);
+ queue.PushBack(1);
+ queue.PushBack(2);
+ queue.PushBack(3);
+ EXPECT_EQ(queue.size(), 3lu);
+ for (int i = 0; i < 10; ++i) {
+ EXPECT_FALSE(queue.IsEmpty(/*reader=*/i));
+ EXPECT_EQ(queue.size(/*reader=*/i), 3lu);
+ }
+}
+
+TEST(MultiReaderQueueTest, SizeIsCorrectAfterRemoveFromOnlyOneHead) {
+ MultiReaderQueue<int> queue = MultiReaderQueue<int>(/*readers_count=*/10);
+ for (int i = 0; i < 5; ++i) {
+ queue.PushBack(i);
+ }
+ EXPECT_EQ(queue.size(), 5lu);
+ // Removing elements from queue #0
+ for (int i = 0; i < 5; ++i) {
+ EXPECT_EQ(queue.size(/*reader=*/0), static_cast<size_t>(5 - i));
+ EXPECT_EQ(queue.PopFront(/*reader=*/0), absl::optional<int>(i));
+ for (int j = 1; j < 10; ++j) {
+ EXPECT_EQ(queue.size(/*reader=*/j), 5lu);
+ }
+ }
+ EXPECT_EQ(queue.size(/*reader=*/0), 0lu);
+ EXPECT_TRUE(queue.IsEmpty(/*reader=*/0));
+}
+
+TEST(MultiReaderQueueTest, SingleHeadOneAddOneRemove) {
+ MultiReaderQueue<int> queue = MultiReaderQueue<int>(/*readers_count=*/1);
+ queue.PushBack(1);
+ EXPECT_EQ(queue.size(), 1lu);
+ EXPECT_TRUE(queue.Front(/*reader=*/0).has_value());
+ EXPECT_EQ(queue.Front(/*reader=*/0).value(), 1);
+ absl::optional<int> value = queue.PopFront(/*reader=*/0);
+ EXPECT_TRUE(value.has_value());
+ EXPECT_EQ(value.value(), 1);
+ EXPECT_EQ(queue.size(), 0lu);
+ EXPECT_TRUE(queue.IsEmpty(/*reader=*/0));
+}
+
+TEST(MultiReaderQueueTest, SingleHead) {
+ MultiReaderQueue<size_t> queue =
+ MultiReaderQueue<size_t>(/*readers_count=*/1);
+ for (size_t i = 0; i < 10; ++i) {
+ queue.PushBack(i);
+ EXPECT_EQ(queue.size(), i + 1);
+ }
+ for (size_t i = 0; i < 10; ++i) {
+ EXPECT_EQ(queue.Front(/*reader=*/0), absl::optional<size_t>(i));
+ EXPECT_EQ(queue.PopFront(/*reader=*/0), absl::optional<size_t>(i));
+ EXPECT_EQ(queue.size(), 10 - i - 1);
+ }
+}
+
+TEST(MultiReaderQueueTest, ThreeHeadsAddAllRemoveAllPerHead) {
+ MultiReaderQueue<size_t> queue =
+ MultiReaderQueue<size_t>(/*readers_count=*/3);
+ for (size_t i = 0; i < 10; ++i) {
+ queue.PushBack(i);
+ EXPECT_EQ(queue.size(), i + 1);
+ }
+ for (size_t i = 0; i < 10; ++i) {
+ absl::optional<size_t> value = queue.PopFront(/*reader=*/0);
+ EXPECT_EQ(queue.size(), 10lu);
+ ASSERT_TRUE(value.has_value());
+ EXPECT_EQ(value.value(), i);
+ }
+ for (size_t i = 0; i < 10; ++i) {
+ absl::optional<size_t> value = queue.PopFront(/*reader=*/1);
+ EXPECT_EQ(queue.size(), 10lu);
+ ASSERT_TRUE(value.has_value());
+ EXPECT_EQ(value.value(), i);
+ }
+ for (size_t i = 0; i < 10; ++i) {
+ absl::optional<size_t> value = queue.PopFront(/*reader=*/2);
+ EXPECT_EQ(queue.size(), 10 - i - 1);
+ ASSERT_TRUE(value.has_value());
+ EXPECT_EQ(value.value(), i);
+ }
+}
+
+TEST(MultiReaderQueueTest, ThreeHeadsAddAllRemoveAll) {
+ MultiReaderQueue<size_t> queue =
+ MultiReaderQueue<size_t>(/*readers_count=*/3);
+ for (size_t i = 0; i < 10; ++i) {
+ queue.PushBack(i);
+ EXPECT_EQ(queue.size(), i + 1);
+ }
+ for (size_t i = 0; i < 10; ++i) {
+ absl::optional<size_t> value1 = queue.PopFront(/*reader=*/0);
+ absl::optional<size_t> value2 = queue.PopFront(/*reader=*/1);
+ absl::optional<size_t> value3 = queue.PopFront(/*reader=*/2);
+ EXPECT_EQ(queue.size(), 10 - i - 1);
+ ASSERT_TRUE(value1.has_value());
+ ASSERT_TRUE(value2.has_value());
+ ASSERT_TRUE(value3.has_value());
+ EXPECT_EQ(value1.value(), i);
+ EXPECT_EQ(value2.value(), i);
+ EXPECT_EQ(value3.value(), i);
+ }
+}
+
+TEST(MultiReaderQueueTest, AddReaderSeeElementsOnlyFromReaderToCopy) {
+ MultiReaderQueue<size_t> queue =
+ MultiReaderQueue<size_t>(/*readers_count=*/2);
+ for (size_t i = 0; i < 10; ++i) {
+ queue.PushBack(i);
+ }
+ for (size_t i = 0; i < 5; ++i) {
+ queue.PopFront(0);
+ }
+
+ queue.AddReader(/*reader=*/2, /*reader_to_copy=*/0);
+
+ EXPECT_EQ(queue.readers_count(), 3lu);
+ for (size_t i = 5; i < 10; ++i) {
+ absl::optional<size_t> value = queue.PopFront(/*reader=*/2);
+ EXPECT_EQ(queue.size(/*reader=*/2), 10 - i - 1);
+ ASSERT_TRUE(value.has_value());
+ EXPECT_EQ(value.value(), i);
+ }
+}
+
+TEST(MultiReaderQueueTest, AddReaderWithoutReaderToCopySeeFullQueue) {
+ MultiReaderQueue<size_t> queue =
+ MultiReaderQueue<size_t>(/*readers_count=*/2);
+ for (size_t i = 0; i < 10; ++i) {
+ queue.PushBack(i);
+ }
+ for (size_t i = 0; i < 5; ++i) {
+ queue.PopFront(/*reader=*/0);
+ }
+
+ queue.AddReader(/*reader=*/2);
+
+ EXPECT_EQ(queue.readers_count(), 3lu);
+ for (size_t i = 0; i < 10; ++i) {
+ absl::optional<size_t> value = queue.PopFront(/*reader=*/2);
+ EXPECT_EQ(queue.size(/*reader=*/2), 10 - i - 1);
+ ASSERT_TRUE(value.has_value());
+ EXPECT_EQ(value.value(), i);
+ }
+}
+
+TEST(MultiReaderQueueTest, RemoveReaderWontChangeOthers) {
+ MultiReaderQueue<size_t> queue =
+ MultiReaderQueue<size_t>(/*readers_count=*/2);
+ for (size_t i = 0; i < 10; ++i) {
+ queue.PushBack(i);
+ }
+ EXPECT_EQ(queue.size(/*reader=*/1), 10lu);
+
+ queue.RemoveReader(0);
+
+ EXPECT_EQ(queue.readers_count(), 1lu);
+ EXPECT_EQ(queue.size(/*reader=*/1), 10lu);
+}
+
+TEST(MultiReaderQueueTest, RemoveLastReaderMakesQueueEmpty) {
+ MultiReaderQueue<size_t> queue =
+ MultiReaderQueue<size_t>(/*readers_count=*/1);
+ for (size_t i = 0; i < 10; ++i) {
+ queue.PushBack(i);
+ }
+ EXPECT_EQ(queue.size(), 10lu);
+
+ queue.RemoveReader(0);
+
+ EXPECT_EQ(queue.size(), 0lu);
+ EXPECT_EQ(queue.readers_count(), 0lu);
+}
+
+} // namespace
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.cc
new file mode 100644
index 0000000000..3ccab620f8
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.cc
@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/names_collection.h"
+
+#include <set>
+
+#include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
+
+namespace webrtc {
+
+NamesCollection::NamesCollection(rtc::ArrayView<const std::string> names) {
+ names_ = std::vector<std::string>(names.begin(), names.end());
+ for (size_t i = 0; i < names_.size(); ++i) {
+ index_.emplace(names_[i], i);
+ removed_.emplace_back(false);
+ }
+ size_ = names_.size();
+}
+
+bool NamesCollection::HasName(absl::string_view name) const {
+ auto it = index_.find(name);
+ if (it == index_.end()) {
+ return false;
+ }
+ return !removed_[it->second];
+}
+
+size_t NamesCollection::AddIfAbsent(absl::string_view name) {
+ auto it = index_.find(name);
+ if (it != index_.end()) {
+ // Name was registered in the collection before: we need to restore it.
+ size_t index = it->second;
+ if (removed_[index]) {
+ removed_[index] = false;
+ size_++;
+ }
+ return index;
+ }
+ size_t out = names_.size();
+ size_t old_capacity = names_.capacity();
+ names_.emplace_back(name);
+ removed_.emplace_back(false);
+ size_++;
+ size_t new_capacity = names_.capacity();
+
+ if (old_capacity == new_capacity) {
+ index_.emplace(names_[out], out);
+ } else {
+ // Reallocation happened in the vector, so we need to rebuild `index_` to
+ // fix absl::string_view internal references.
+ index_.clear();
+ for (size_t i = 0; i < names_.size(); ++i) {
+ index_.emplace(names_[i], i);
+ }
+ }
+ return out;
+}
+
+absl::optional<size_t> NamesCollection::RemoveIfPresent(
+ absl::string_view name) {
+ auto it = index_.find(name);
+ if (it == index_.end()) {
+ return absl::nullopt;
+ }
+ size_t index = it->second;
+ if (removed_[index]) {
+ return absl::nullopt;
+ }
+ removed_[index] = true;
+ size_--;
+ return index;
+}
+
+std::set<size_t> NamesCollection::GetPresentIndexes() const {
+ std::set<size_t> out;
+ for (size_t i = 0; i < removed_.size(); ++i) {
+ if (!removed_[i]) {
+ out.insert(i);
+ }
+ }
+ return out;
+}
+
+std::set<size_t> NamesCollection::GetAllIndexes() const {
+ std::set<size_t> out;
+ for (size_t i = 0; i < names_.size(); ++i) {
+ out.insert(i);
+ }
+ return out;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.h
new file mode 100644
index 0000000000..f9a13a2a11
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection.h
@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_NAMES_COLLECTION_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_NAMES_COLLECTION_H_
+
+#include <map>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
+#include "api/array_view.h"
+
+namespace webrtc {
+
+// Contains mapping between string names and unique size_t values (indexes).
+// Once the name is added to the collection it is guaranteed:
+// 1. Name will have the same index until collection will be destructed
+// 2. Adding, removing and re-adding name won't change its index
+//
+// The name is considered in the collection if it was added and wasn't removed.
+// Adding the name when it is in the collection won't change the collection, the
+// same as removing the name when it is removed.
+//
+// Collection will return name's index and name for the index independently from
+// was name removed or not. Once the name was added to the collection the index
+// will be allocated for it. To check if name is in collection right now user
+// has to explicitly call to `HasName` function.
+class NamesCollection {
+ public:
+ NamesCollection() = default;
+
+ explicit NamesCollection(rtc::ArrayView<const std::string> names);
+
+ // Returns amount of currently presented names in the collection.
+ size_t size() const { return size_; }
+
+ // Returns amount of all names known to this collection.
+ size_t GetKnownSize() const { return names_.size(); }
+
+ // Returns index of the `name` which was known to the collection. Crashes
+ // if `name` was never registered in the collection.
+ size_t index(absl::string_view name) const { return index_.at(name); }
+
+ // Returns name which was known to the collection for the specified `index`.
+ // Crashes if there was no any name registered in the collection for such
+ // `index`.
+ const std::string& name(size_t index) const { return names_.at(index); }
+
+ // Returns if `name` is currently presented in this collection.
+ bool HasName(absl::string_view name) const;
+
+ // Adds specified `name` to the collection if it isn't presented.
+ // Returns index which corresponds to specified `name`.
+ size_t AddIfAbsent(absl::string_view name);
+
+ // Removes specified `name` from the collection if it is presented.
+ //
+ // After name was removed, collection size will be decreased, but `name` index
+ // will be preserved. Collection will return false for `HasName(name)`, but
+ // will continue to return previously known index for `index(name)` and return
+ // `name` for `name(index(name))`.
+ //
+ // Returns the index of the removed value or absl::nullopt if no such `name`
+ // registered in the collection.
+ absl::optional<size_t> RemoveIfPresent(absl::string_view name);
+
+ // Returns a set of indexes for all currently present names in the
+ // collection.
+ std::set<size_t> GetPresentIndexes() const;
+
+ // Returns a set of all indexes known to the collection including indexes for
+ // names that were removed.
+ std::set<size_t> GetAllIndexes() const;
+
+ private:
+ std::vector<std::string> names_;
+ std::vector<bool> removed_;
+ std::map<absl::string_view, size_t> index_;
+ size_t size_ = 0;
+};
+
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_NAMES_COLLECTION_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection_test.cc
new file mode 100644
index 0000000000..6c52f96975
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/names_collection_test.cc
@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/names_collection.h"
+
+#include <string>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace {
+
+using ::testing::Eq;
+using ::testing::Ne;
+
+TEST(NamesCollectionTest, NamesFromCtorHasUniqueIndexes) {
+ NamesCollection collection(std::vector<std::string>{"alice", "bob"});
+
+ EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(2)));
+ EXPECT_TRUE(collection.HasName("alice"));
+ EXPECT_THAT(collection.name(collection.index("alice")), Eq("alice"));
+
+ EXPECT_TRUE(collection.HasName("bob"));
+ EXPECT_THAT(collection.name(collection.index("bob")), Eq("bob"));
+
+ EXPECT_THAT(collection.index("bob"), Ne(collection.index("alice")));
+}
+
+TEST(NamesCollectionTest, AddedNamesHasIndexes) {
+ NamesCollection collection(std::vector<std::string>{});
+ collection.AddIfAbsent("alice");
+
+ EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1)));
+ EXPECT_TRUE(collection.HasName("alice"));
+ EXPECT_THAT(collection.name(collection.index("alice")), Eq("alice"));
+}
+
+TEST(NamesCollectionTest, AddBobDoesNotChangeAliceIndex) {
+ NamesCollection collection(std::vector<std::string>{"alice"});
+
+ size_t alice_index = collection.index("alice");
+
+ collection.AddIfAbsent("bob");
+
+ EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(2)));
+ EXPECT_THAT(collection.index("alice"), Eq(alice_index));
+ EXPECT_THAT(collection.index("bob"), Ne(alice_index));
+}
+
+TEST(NamesCollectionTest, AddAliceSecondTimeDoesNotChangeIndex) {
+ NamesCollection collection(std::vector<std::string>{"alice"});
+
+ size_t alice_index = collection.index("alice");
+
+ EXPECT_THAT(collection.AddIfAbsent("alice"), Eq(alice_index));
+
+ EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1)));
+ EXPECT_THAT(collection.index("alice"), Eq(alice_index));
+}
+
+TEST(NamesCollectionTest, RemoveRemovesFromCollectionButNotIndex) {
+ NamesCollection collection(std::vector<std::string>{"alice", "bob"});
+
+ size_t bob_index = collection.index("bob");
+
+ EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(2)));
+
+ EXPECT_THAT(collection.RemoveIfPresent("bob"),
+ Eq(absl::optional<size_t>(bob_index)));
+
+ EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1)));
+ EXPECT_FALSE(collection.HasName("bob"));
+
+ EXPECT_THAT(collection.index("bob"), Eq(bob_index));
+ EXPECT_THAT(collection.name(bob_index), Eq("bob"));
+}
+
+TEST(NamesCollectionTest, RemoveOfAliceDoesNotChangeBobIndex) {
+ NamesCollection collection(std::vector<std::string>{"alice", "bob"});
+
+ size_t alice_index = collection.index("alice");
+ size_t bob_index = collection.index("bob");
+
+ EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(2)));
+
+ EXPECT_THAT(collection.RemoveIfPresent("alice"),
+ Eq(absl::optional<size_t>(alice_index)));
+
+ EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1)));
+ EXPECT_THAT(collection.index("bob"), Eq(bob_index));
+ EXPECT_THAT(collection.name(bob_index), Eq("bob"));
+}
+
+TEST(NamesCollectionTest, RemoveSecondTimeHasNoEffect) {
+ NamesCollection collection(std::vector<std::string>{"bob"});
+
+ size_t bob_index = collection.index("bob");
+
+ EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1)));
+ EXPECT_THAT(collection.RemoveIfPresent("bob"),
+ Eq(absl::optional<size_t>(bob_index)));
+
+ EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(0)));
+ EXPECT_THAT(collection.RemoveIfPresent("bob"), Eq(absl::nullopt));
+}
+
+TEST(NamesCollectionTest, RemoveOfNotExistingHasNoEffect) {
+ NamesCollection collection(std::vector<std::string>{"bob"});
+
+ EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1)));
+ EXPECT_THAT(collection.RemoveIfPresent("alice"), Eq(absl::nullopt));
+ EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1)));
+}
+
+TEST(NamesCollectionTest, AddRemoveAddPreserveTheIndex) {
+ NamesCollection collection(std::vector<std::string>{});
+
+ size_t alice_index = collection.AddIfAbsent("alice");
+ EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1)));
+
+ EXPECT_THAT(collection.RemoveIfPresent("alice"),
+ Eq(absl::optional<size_t>(alice_index)));
+ EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(0)));
+
+ EXPECT_THAT(collection.AddIfAbsent("alice"), Eq(alice_index));
+ EXPECT_THAT(collection.index("alice"), Eq(alice_index));
+ EXPECT_THAT(collection.size(), Eq(static_cast<size_t>(1)));
+}
+
+TEST(NamesCollectionTest, GetKnownSizeReturnsForRemovedNames) {
+ NamesCollection collection(std::vector<std::string>{});
+
+ size_t alice_index = collection.AddIfAbsent("alice");
+ EXPECT_THAT(collection.GetKnownSize(), Eq(static_cast<size_t>(1)));
+
+ EXPECT_THAT(collection.RemoveIfPresent("alice"),
+ Eq(absl::optional<size_t>(alice_index)));
+ EXPECT_THAT(collection.GetKnownSize(), Eq(static_cast<size_t>(1)));
+}
+
+} // namespace
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc
new file mode 100644
index 0000000000..b958f4d027
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc
@@ -0,0 +1,272 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h"
+
+#include <cstdint>
+#include <cstring>
+#include <memory>
+#include <utility>
+
+#include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
+#include "api/video/i420_buffer.h"
+#include "api/video/video_frame.h"
+#include "modules/video_coding/include/video_error_codes.h"
+#include "rtc_base/logging.h"
+#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+QualityAnalyzingVideoDecoder::QualityAnalyzingVideoDecoder(
+ absl::string_view peer_name,
+ std::unique_ptr<VideoDecoder> delegate,
+ EncodedImageDataExtractor* extractor,
+ VideoQualityAnalyzerInterface* analyzer)
+ : peer_name_(peer_name),
+ implementation_name_("AnalyzingDecoder-" +
+ std::string(delegate->ImplementationName())),
+ delegate_(std::move(delegate)),
+ extractor_(extractor),
+ analyzer_(analyzer) {
+ analyzing_callback_ = std::make_unique<DecoderCallback>(this);
+}
+QualityAnalyzingVideoDecoder::~QualityAnalyzingVideoDecoder() = default;
+
+bool QualityAnalyzingVideoDecoder::Configure(const Settings& settings) {
+ {
+ MutexLock lock(&mutex_);
+ codec_name_ = std::string(CodecTypeToPayloadString(settings.codec_type())) +
+ "_" + delegate_->GetDecoderInfo().implementation_name;
+ }
+ return delegate_->Configure(settings);
+}
+
+int32_t QualityAnalyzingVideoDecoder::Decode(const EncodedImage& input_image,
+ bool missing_frames,
+ int64_t render_time_ms) {
+ // Image extractor extracts id from provided EncodedImage and also returns
+ // the image with the original buffer. Buffer can be modified in place, so
+ // owner of original buffer will be responsible for deleting it, or extractor
+ // can create a new buffer. In such case extractor will be responsible for
+ // deleting it.
+ EncodedImageExtractionResult out = extractor_->ExtractData(input_image);
+
+ if (out.discard) {
+ // To partly emulate behavior of Selective Forwarding Unit (SFU) in the
+ // test, on receiver side we will "discard" frames from irrelevant streams.
+ // When all encoded images were marked to discarded, black frame have to be
+ // returned. Because simulcast streams will be received by receiver as 3
+ // different independent streams we don't want that irrelevant streams
+ // affect video quality metrics and also we don't want to use CPU time to
+ // decode them to prevent regressions on relevant streams. Also we can't
+ // just drop frame, because in such case, receiving part will be confused
+ // with all frames missing and will request a key frame, which will result
+ // into extra load on network and sender side. Because of it, discarded
+ // image will be always decoded as black frame and will be passed to
+ // callback directly without reaching decoder and video quality analyzer.
+ //
+ // For more details see QualityAnalyzingVideoEncoder.
+ return analyzing_callback_->IrrelevantSimulcastStreamDecoded(
+ out.id.value_or(VideoFrame::kNotSetId), input_image.Timestamp());
+ }
+
+ EncodedImage* origin_image;
+ {
+ MutexLock lock(&mutex_);
+ // Store id to be able to retrieve it in analyzing callback.
+ timestamp_to_frame_id_.insert({input_image.Timestamp(), out.id});
+ // Store encoded image to prevent its destruction while it is used in
+ // decoder.
+ origin_image = &(
+ decoding_images_.insert({input_image.Timestamp(), std::move(out.image)})
+ .first->second);
+ }
+ // We can safely dereference `origin_image`, because it can be removed from
+ // the map only after `delegate_` Decode method will be invoked. Image will
+ // be removed inside DecodedImageCallback, which can be done on separate
+ // thread.
+ analyzer_->OnFramePreDecode(
+ peer_name_, out.id.value_or(VideoFrame::kNotSetId), *origin_image);
+ int32_t result =
+ delegate_->Decode(*origin_image, missing_frames, render_time_ms);
+ if (result != WEBRTC_VIDEO_CODEC_OK) {
+ // If delegate decoder failed, then cleanup data for this image.
+ VideoQualityAnalyzerInterface::DecoderStats stats;
+ {
+ MutexLock lock(&mutex_);
+ timestamp_to_frame_id_.erase(input_image.Timestamp());
+ decoding_images_.erase(input_image.Timestamp());
+ stats.decoder_name = codec_name_;
+ }
+ analyzer_->OnDecoderError(
+ peer_name_, out.id.value_or(VideoFrame::kNotSetId), result, stats);
+ }
+ return result;
+}
+
+int32_t QualityAnalyzingVideoDecoder::RegisterDecodeCompleteCallback(
+ DecodedImageCallback* callback) {
+ analyzing_callback_->SetDelegateCallback(callback);
+ return delegate_->RegisterDecodeCompleteCallback(analyzing_callback_.get());
+}
+
+int32_t QualityAnalyzingVideoDecoder::Release() {
+ // Release decoder first. During release process it can still decode some
+ // frames, so we don't take a lock to prevent deadlock.
+ int32_t result = delegate_->Release();
+
+ MutexLock lock(&mutex_);
+ analyzing_callback_->SetDelegateCallback(nullptr);
+ timestamp_to_frame_id_.clear();
+ decoding_images_.clear();
+ return result;
+}
+
+VideoDecoder::DecoderInfo QualityAnalyzingVideoDecoder::GetDecoderInfo() const {
+ DecoderInfo info = delegate_->GetDecoderInfo();
+ info.implementation_name = implementation_name_;
+ return info;
+}
+
+const char* QualityAnalyzingVideoDecoder::ImplementationName() const {
+ return implementation_name_.c_str();
+}
+
+QualityAnalyzingVideoDecoder::DecoderCallback::DecoderCallback(
+ QualityAnalyzingVideoDecoder* decoder)
+ : decoder_(decoder), delegate_callback_(nullptr) {}
+QualityAnalyzingVideoDecoder::DecoderCallback::~DecoderCallback() = default;
+
+void QualityAnalyzingVideoDecoder::DecoderCallback::SetDelegateCallback(
+ DecodedImageCallback* delegate) {
+ MutexLock lock(&callback_mutex_);
+ delegate_callback_ = delegate;
+}
+
+// We have to implement all next 3 methods because we don't know which one
+// exactly is implemented in `delegate_callback_`, so we need to call the same
+// method on `delegate_callback_`, as was called on `this` callback.
+int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded(
+ VideoFrame& decodedImage) {
+ decoder_->OnFrameDecoded(&decodedImage, /*decode_time_ms=*/absl::nullopt,
+ /*qp=*/absl::nullopt);
+
+ MutexLock lock(&callback_mutex_);
+ RTC_DCHECK(delegate_callback_);
+ return delegate_callback_->Decoded(decodedImage);
+}
+
+int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded(
+ VideoFrame& decodedImage,
+ int64_t decode_time_ms) {
+ decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, /*qp=*/absl::nullopt);
+
+ MutexLock lock(&callback_mutex_);
+ RTC_DCHECK(delegate_callback_);
+ return delegate_callback_->Decoded(decodedImage, decode_time_ms);
+}
+
+void QualityAnalyzingVideoDecoder::DecoderCallback::Decoded(
+ VideoFrame& decodedImage,
+ absl::optional<int32_t> decode_time_ms,
+ absl::optional<uint8_t> qp) {
+ decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, qp);
+
+ MutexLock lock(&callback_mutex_);
+ RTC_DCHECK(delegate_callback_);
+ delegate_callback_->Decoded(decodedImage, decode_time_ms, qp);
+}
+
+int32_t
+QualityAnalyzingVideoDecoder::DecoderCallback::IrrelevantSimulcastStreamDecoded(
+ uint16_t frame_id,
+ uint32_t timestamp_ms) {
+ webrtc::VideoFrame dummy_frame =
+ webrtc::VideoFrame::Builder()
+ .set_video_frame_buffer(GetDummyFrameBuffer())
+ .set_timestamp_rtp(timestamp_ms)
+ .set_id(frame_id)
+ .build();
+ MutexLock lock(&callback_mutex_);
+ RTC_DCHECK(delegate_callback_);
+ delegate_callback_->Decoded(dummy_frame, absl::nullopt, absl::nullopt);
+ return WEBRTC_VIDEO_CODEC_OK;
+}
+
+rtc::scoped_refptr<webrtc::VideoFrameBuffer>
+QualityAnalyzingVideoDecoder::DecoderCallback::GetDummyFrameBuffer() {
+ if (!dummy_frame_buffer_) {
+ dummy_frame_buffer_ = CreateDummyFrameBuffer();
+ }
+
+ return dummy_frame_buffer_;
+}
+
+void QualityAnalyzingVideoDecoder::OnFrameDecoded(
+ VideoFrame* frame,
+ absl::optional<int32_t> decode_time_ms,
+ absl::optional<uint8_t> qp) {
+ absl::optional<uint16_t> frame_id;
+ std::string codec_name;
+ {
+ MutexLock lock(&mutex_);
+ auto it = timestamp_to_frame_id_.find(frame->timestamp());
+ if (it == timestamp_to_frame_id_.end()) {
+ // Ensure, that we have info about this frame. It can happen that for some
+ // reasons decoder response, that it failed to decode, when we were
+ // posting frame to it, but then call the callback for this frame.
+ RTC_LOG(LS_ERROR) << "QualityAnalyzingVideoDecoder::OnFrameDecoded: No "
+ "frame id for frame for frame->timestamp()="
+ << frame->timestamp();
+ return;
+ }
+ frame_id = it->second;
+ timestamp_to_frame_id_.erase(it);
+ decoding_images_.erase(frame->timestamp());
+ codec_name = codec_name_;
+ }
+ // Set frame id to the value, that was extracted from corresponding encoded
+ // image.
+ frame->set_id(frame_id.value_or(VideoFrame::kNotSetId));
+ VideoQualityAnalyzerInterface::DecoderStats stats;
+ stats.decoder_name = codec_name;
+ stats.decode_time_ms = decode_time_ms;
+ analyzer_->OnFrameDecoded(peer_name_, *frame, stats);
+}
+
+QualityAnalyzingVideoDecoderFactory::QualityAnalyzingVideoDecoderFactory(
+ absl::string_view peer_name,
+ std::unique_ptr<VideoDecoderFactory> delegate,
+ EncodedImageDataExtractor* extractor,
+ VideoQualityAnalyzerInterface* analyzer)
+ : peer_name_(peer_name),
+ delegate_(std::move(delegate)),
+ extractor_(extractor),
+ analyzer_(analyzer) {}
+QualityAnalyzingVideoDecoderFactory::~QualityAnalyzingVideoDecoderFactory() =
+ default;
+
+std::vector<SdpVideoFormat>
+QualityAnalyzingVideoDecoderFactory::GetSupportedFormats() const {
+ return delegate_->GetSupportedFormats();
+}
+
+std::unique_ptr<VideoDecoder>
+QualityAnalyzingVideoDecoderFactory::CreateVideoDecoder(
+ const SdpVideoFormat& format) {
+ std::unique_ptr<VideoDecoder> decoder = delegate_->CreateVideoDecoder(format);
+ return std::make_unique<QualityAnalyzingVideoDecoder>(
+ peer_name_, std::move(decoder), extractor_, analyzer_);
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h
new file mode 100644
index 0000000000..a86f4196b0
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h
@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
+#include "api/test/video_quality_analyzer_interface.h"
+#include "api/video/encoded_image.h"
+#include "api/video/video_frame.h"
+#include "api/video_codecs/sdp_video_format.h"
+#include "api/video_codecs/video_decoder.h"
+#include "api/video_codecs/video_decoder_factory.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+// QualityAnalyzingVideoDecoder is used to wrap origin video decoder and inject
+// VideoQualityAnalyzerInterface before and after decoder.
+//
+// QualityAnalyzingVideoDecoder propagates all calls to the origin decoder.
+// It registers its own DecodedImageCallback in the origin decoder and will
+// store user specified callback inside itself.
+//
+// When Decode(...) will be invoked, quality decoder first will extract frame id
+// from passed EncodedImage with EncodedImageIdExtracor that was specified in
+// constructor, then will call video quality analyzer, with correct
+// EncodedImage and only then will pass image to origin decoder.
+//
+// When origin decoder decodes the image it will call quality decoder's special
+// callback, where video analyzer will be called again and then decoded frame
+// will be passed to origin callback, provided by user.
+//
+// Quality decoder registers its own callback in origin decoder, at the same
+// time the user registers their callback in quality decoder.
+class QualityAnalyzingVideoDecoder : public VideoDecoder {
+ public:
+ QualityAnalyzingVideoDecoder(absl::string_view peer_name,
+ std::unique_ptr<VideoDecoder> delegate,
+ EncodedImageDataExtractor* extractor,
+ VideoQualityAnalyzerInterface* analyzer);
+ ~QualityAnalyzingVideoDecoder() override;
+
+ // Methods of VideoDecoder interface.
+ bool Configure(const Settings& settings) override;
+ int32_t Decode(const EncodedImage& input_image,
+ bool missing_frames,
+ int64_t render_time_ms) override;
+ int32_t RegisterDecodeCompleteCallback(
+ DecodedImageCallback* callback) override;
+ int32_t Release() override;
+ DecoderInfo GetDecoderInfo() const override;
+ const char* ImplementationName() const override;
+
+ private:
+ class DecoderCallback : public DecodedImageCallback {
+ public:
+ explicit DecoderCallback(QualityAnalyzingVideoDecoder* decoder);
+ ~DecoderCallback() override;
+
+ void SetDelegateCallback(DecodedImageCallback* delegate);
+
+ // Methods of DecodedImageCallback interface.
+ int32_t Decoded(VideoFrame& decodedImage) override;
+ int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) override;
+ void Decoded(VideoFrame& decodedImage,
+ absl::optional<int32_t> decode_time_ms,
+ absl::optional<uint8_t> qp) override;
+
+ int32_t IrrelevantSimulcastStreamDecoded(uint16_t frame_id,
+ uint32_t timestamp_ms);
+
+ private:
+ rtc::scoped_refptr<webrtc::VideoFrameBuffer> GetDummyFrameBuffer();
+
+ QualityAnalyzingVideoDecoder* const decoder_;
+
+ rtc::scoped_refptr<webrtc::VideoFrameBuffer> dummy_frame_buffer_;
+
+ Mutex callback_mutex_;
+ DecodedImageCallback* delegate_callback_ RTC_GUARDED_BY(callback_mutex_);
+ };
+
+ void OnFrameDecoded(VideoFrame* frame,
+ absl::optional<int32_t> decode_time_ms,
+ absl::optional<uint8_t> qp);
+
+ const std::string peer_name_;
+ const std::string implementation_name_;
+ std::unique_ptr<VideoDecoder> delegate_;
+ EncodedImageDataExtractor* const extractor_;
+ VideoQualityAnalyzerInterface* const analyzer_;
+ std::unique_ptr<DecoderCallback> analyzing_callback_;
+
+ // VideoDecoder interface assumes async delivery of decoded video frames.
+ // This lock is used to protect shared state, that have to be propagated
+ // from received EncodedImage to resulted VideoFrame.
+ Mutex mutex_;
+
+ // Name of the video codec type used. Ex: VP8, VP9, H264 etc.
+ std::string codec_name_ RTC_GUARDED_BY(mutex_);
+ std::map<uint32_t, absl::optional<uint16_t>> timestamp_to_frame_id_
+ RTC_GUARDED_BY(mutex_);
+ // Stores currently being decoded images by timestamp. Because
+ // EncodedImageDataExtractor can create new copy on EncodedImage we need to
+ // ensure, that this image won't be deleted during async decoding. To do it
+ // all images are putted into this map and removed from here inside callback.
+ std::map<uint32_t, EncodedImage> decoding_images_ RTC_GUARDED_BY(mutex_);
+};
+
+// Produces QualityAnalyzingVideoDecoder, which hold decoders, produced by
+// specified factory as delegates. Forwards all other calls to specified
+// factory.
+class QualityAnalyzingVideoDecoderFactory : public VideoDecoderFactory {
+ public:
+ QualityAnalyzingVideoDecoderFactory(
+ absl::string_view peer_name,
+ std::unique_ptr<VideoDecoderFactory> delegate,
+ EncodedImageDataExtractor* extractor,
+ VideoQualityAnalyzerInterface* analyzer);
+ ~QualityAnalyzingVideoDecoderFactory() override;
+
+ // Methods of VideoDecoderFactory interface.
+ std::vector<SdpVideoFormat> GetSupportedFormats() const override;
+ std::unique_ptr<VideoDecoder> CreateVideoDecoder(
+ const SdpVideoFormat& format) override;
+
+ private:
+ const std::string peer_name_;
+ std::unique_ptr<VideoDecoderFactory> delegate_;
+ EncodedImageDataExtractor* const extractor_;
+ VideoQualityAnalyzerInterface* const analyzer_;
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc
new file mode 100644
index 0000000000..e814ba88b7
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc
@@ -0,0 +1,403 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h"
+
+#include <cmath>
+#include <memory>
+#include <utility>
+
+#include "absl/strings/string_view.h"
+#include "api/video/video_codec_type.h"
+#include "api/video_codecs/video_encoder.h"
+#include "modules/video_coding/include/video_error_codes.h"
+#include "modules/video_coding/svc/scalability_mode_util.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+using EmulatedSFUConfigMap =
+ ::webrtc::webrtc_pc_e2e::QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap;
+
+constexpr size_t kMaxFrameInPipelineCount = 1000;
+constexpr double kNoMultiplier = 1.0;
+constexpr double kEps = 1e-6;
+
+std::pair<uint32_t, uint32_t> GetMinMaxBitratesBps(const VideoCodec& codec,
+ size_t spatial_idx) {
+ uint32_t min_bitrate = codec.minBitrate;
+ uint32_t max_bitrate = codec.maxBitrate;
+ if (spatial_idx < codec.numberOfSimulcastStreams &&
+ codec.codecType != VideoCodecType::kVideoCodecVP9) {
+ min_bitrate =
+ std::max(min_bitrate, codec.simulcastStream[spatial_idx].minBitrate);
+ max_bitrate =
+ std::min(max_bitrate, codec.simulcastStream[spatial_idx].maxBitrate);
+ }
+ if (codec.codecType == VideoCodecType::kVideoCodecVP9 &&
+ spatial_idx < codec.VP9().numberOfSpatialLayers) {
+ min_bitrate =
+ std::max(min_bitrate, codec.spatialLayers[spatial_idx].minBitrate);
+ max_bitrate =
+ std::min(max_bitrate, codec.spatialLayers[spatial_idx].maxBitrate);
+ }
+ RTC_DCHECK_GT(max_bitrate, min_bitrate);
+ return {min_bitrate * 1000, max_bitrate * 1000};
+}
+
+} // namespace
+
+QualityAnalyzingVideoEncoder::QualityAnalyzingVideoEncoder(
+ absl::string_view peer_name,
+ std::unique_ptr<VideoEncoder> delegate,
+ double bitrate_multiplier,
+ EmulatedSFUConfigMap stream_to_sfu_config,
+ EncodedImageDataInjector* injector,
+ VideoQualityAnalyzerInterface* analyzer)
+ : peer_name_(peer_name),
+ delegate_(std::move(delegate)),
+ bitrate_multiplier_(bitrate_multiplier),
+ stream_to_sfu_config_(std::move(stream_to_sfu_config)),
+ injector_(injector),
+ analyzer_(analyzer),
+ mode_(SimulcastMode::kNormal),
+ delegate_callback_(nullptr) {}
+QualityAnalyzingVideoEncoder::~QualityAnalyzingVideoEncoder() = default;
+
+void QualityAnalyzingVideoEncoder::SetFecControllerOverride(
+ FecControllerOverride* fec_controller_override) {
+ // Ignored.
+}
+
+int32_t QualityAnalyzingVideoEncoder::InitEncode(
+ const VideoCodec* codec_settings,
+ const Settings& settings) {
+ MutexLock lock(&mutex_);
+ codec_settings_ = *codec_settings;
+ mode_ = SimulcastMode::kNormal;
+ absl::optional<InterLayerPredMode> inter_layer_pred_mode;
+ if (codec_settings->GetScalabilityMode().has_value()) {
+ inter_layer_pred_mode = ScalabilityModeToInterLayerPredMode(
+ *codec_settings->GetScalabilityMode());
+ } else if (codec_settings->codecType == kVideoCodecVP9) {
+ if (codec_settings->VP9().numberOfSpatialLayers > 1) {
+ inter_layer_pred_mode = codec_settings->VP9().interLayerPred;
+ }
+ }
+ if (inter_layer_pred_mode.has_value()) {
+ switch (*inter_layer_pred_mode) {
+ case InterLayerPredMode::kOn:
+ mode_ = SimulcastMode::kSVC;
+ break;
+ case InterLayerPredMode::kOnKeyPic:
+ mode_ = SimulcastMode::kKSVC;
+ break;
+ case InterLayerPredMode::kOff:
+ mode_ = SimulcastMode::kSimulcast;
+ break;
+ default:
+ RTC_DCHECK_NOTREACHED()
+ << "Unknown InterLayerPredMode value " << *inter_layer_pred_mode;
+ break;
+ }
+ }
+ if (codec_settings->numberOfSimulcastStreams > 1) {
+ mode_ = SimulcastMode::kSimulcast;
+ }
+ return delegate_->InitEncode(codec_settings, settings);
+}
+
+int32_t QualityAnalyzingVideoEncoder::RegisterEncodeCompleteCallback(
+ EncodedImageCallback* callback) {
+ // We need to get a lock here because delegate_callback can be hypothetically
+ // accessed from different thread (encoder one) concurrently.
+ MutexLock lock(&mutex_);
+ delegate_callback_ = callback;
+ return delegate_->RegisterEncodeCompleteCallback(this);
+}
+
+int32_t QualityAnalyzingVideoEncoder::Release() {
+ // Release encoder first. During release process it can still encode some
+ // frames, so we don't take a lock to prevent deadlock.
+ int32_t result = delegate_->Release();
+
+ MutexLock lock(&mutex_);
+ delegate_callback_ = nullptr;
+ return result;
+}
+
+int32_t QualityAnalyzingVideoEncoder::Encode(
+ const VideoFrame& frame,
+ const std::vector<VideoFrameType>* frame_types) {
+ {
+ MutexLock lock(&mutex_);
+ // Store id to be able to retrieve it in analyzing callback.
+ timestamp_to_frame_id_list_.push_back({frame.timestamp(), frame.id()});
+ // If this list is growing, it means that we are not receiving new encoded
+ // images from encoder. So it should be a bug in setup on in the encoder.
+ RTC_DCHECK_LT(timestamp_to_frame_id_list_.size(), kMaxFrameInPipelineCount);
+ }
+ analyzer_->OnFramePreEncode(peer_name_, frame);
+ int32_t result = delegate_->Encode(frame, frame_types);
+ if (result != WEBRTC_VIDEO_CODEC_OK) {
+ // If origin encoder failed, then cleanup data for this frame.
+ {
+ MutexLock lock(&mutex_);
+ // The timestamp-frame_id pair can be not the last one, so we need to
+ // find it first and then remove. We will search from the end, because
+ // usually it will be the last or close to the last one.
+ auto it = timestamp_to_frame_id_list_.end();
+ while (it != timestamp_to_frame_id_list_.begin()) {
+ --it;
+ if (it->first == frame.timestamp()) {
+ timestamp_to_frame_id_list_.erase(it);
+ break;
+ }
+ }
+ }
+ analyzer_->OnEncoderError(peer_name_, frame, result);
+ }
+ return result;
+}
+
+void QualityAnalyzingVideoEncoder::SetRates(
+ const VideoEncoder::RateControlParameters& parameters) {
+ RTC_DCHECK_GT(bitrate_multiplier_, 0.0);
+ if (fabs(bitrate_multiplier_ - kNoMultiplier) < kEps) {
+ {
+ MutexLock lock(&mutex_);
+ bitrate_allocation_ = parameters.bitrate;
+ }
+ return delegate_->SetRates(parameters);
+ }
+
+ RateControlParameters adjusted_params = parameters;
+ {
+ MutexLock lock(&mutex_);
+ // Simulating encoder overshooting target bitrate, by configuring actual
+ // encoder too high. Take care not to adjust past limits of config,
+ // otherwise encoders may crash on DCHECK.
+ VideoBitrateAllocation multiplied_allocation;
+ for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
+ const uint32_t spatial_layer_bitrate_bps =
+ parameters.bitrate.GetSpatialLayerSum(si);
+ if (spatial_layer_bitrate_bps == 0) {
+ continue;
+ }
+
+ uint32_t min_bitrate_bps;
+ uint32_t max_bitrate_bps;
+ std::tie(min_bitrate_bps, max_bitrate_bps) =
+ GetMinMaxBitratesBps(codec_settings_, si);
+ double bitrate_multiplier = bitrate_multiplier_;
+ const uint32_t corrected_bitrate = rtc::checked_cast<uint32_t>(
+ bitrate_multiplier * spatial_layer_bitrate_bps);
+ if (corrected_bitrate < min_bitrate_bps) {
+ bitrate_multiplier = min_bitrate_bps / spatial_layer_bitrate_bps;
+ } else if (corrected_bitrate > max_bitrate_bps) {
+ bitrate_multiplier = max_bitrate_bps / spatial_layer_bitrate_bps;
+ }
+
+ for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
+ if (parameters.bitrate.HasBitrate(si, ti)) {
+ multiplied_allocation.SetBitrate(
+ si, ti,
+ rtc::checked_cast<uint32_t>(
+ bitrate_multiplier * parameters.bitrate.GetBitrate(si, ti)));
+ }
+ }
+ }
+
+ adjusted_params.bitrate = multiplied_allocation;
+ bitrate_allocation_ = adjusted_params.bitrate;
+ }
+ return delegate_->SetRates(adjusted_params);
+}
+
+VideoEncoder::EncoderInfo QualityAnalyzingVideoEncoder::GetEncoderInfo() const {
+ return delegate_->GetEncoderInfo();
+}
+
+// It is assumed, that encoded callback will be always invoked with encoded
+// images that correspond to the frames in the same sequence, that frames
+// arrived. In other words, assume we have frames F1, F2 and F3 and they have
+// corresponding encoded images I1, I2 and I3. In such case if we will call
+// encode first with F1, then with F2 and then with F3, then encoder callback
+// will be called first with all spatial layers for F1 (I1), then F2 (I2) and
+// then F3 (I3).
+//
+// Basing on it we will use a list of timestamp-frame_id pairs like this:
+// 1. If current encoded image timestamp is equals to timestamp in the front
+// pair - pick frame id from that pair
+// 2. If current encoded image timestamp isn't equals to timestamp in the front
+// pair - remove the front pair and got to the step 1.
+EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage(
+ const EncodedImage& encoded_image,
+ const CodecSpecificInfo* codec_specific_info) {
+ uint16_t frame_id;
+ bool discard = false;
+ uint32_t target_encode_bitrate = 0;
+ std::string codec_name;
+ {
+ MutexLock lock(&mutex_);
+ std::pair<uint32_t, uint16_t> timestamp_frame_id;
+ while (!timestamp_to_frame_id_list_.empty()) {
+ timestamp_frame_id = timestamp_to_frame_id_list_.front();
+ if (timestamp_frame_id.first == encoded_image.Timestamp()) {
+ break;
+ }
+ timestamp_to_frame_id_list_.pop_front();
+ }
+
+ // After the loop the first element should point to current `encoded_image`
+ // frame id. We don't remove it from the list, because there may be
+ // multiple spatial layers for this frame, so encoder can produce more
+ // encoded images with this timestamp. The first element will be removed
+ // when the next frame would be encoded and EncodedImageCallback would be
+ // called with the next timestamp.
+
+ if (timestamp_to_frame_id_list_.empty()) {
+ // Ensure, that we have info about this frame. It can happen that for some
+ // reasons encoder response, that he failed to decode, when we were
+ // posting frame to it, but then call the callback for this frame.
+ RTC_LOG(LS_ERROR) << "QualityAnalyzingVideoEncoder::OnEncodedImage: No "
+ "frame id for encoded_image.Timestamp()="
+ << encoded_image.Timestamp();
+ return EncodedImageCallback::Result(
+ EncodedImageCallback::Result::Error::OK);
+ }
+ frame_id = timestamp_frame_id.second;
+
+ discard = ShouldDiscard(frame_id, encoded_image);
+ if (!discard) {
+ target_encode_bitrate = bitrate_allocation_.GetSpatialLayerSum(
+ encoded_image.SpatialIndex().value_or(0));
+ }
+ codec_name =
+ std::string(CodecTypeToPayloadString(codec_settings_.codecType)) + "_" +
+ delegate_->GetEncoderInfo().implementation_name;
+ }
+
+ VideoQualityAnalyzerInterface::EncoderStats stats;
+ stats.encoder_name = codec_name;
+ stats.target_encode_bitrate = target_encode_bitrate;
+ stats.qp = encoded_image.qp_;
+ analyzer_->OnFrameEncoded(peer_name_, frame_id, encoded_image, stats,
+ discard);
+
+ // Image data injector injects frame id and discard flag into provided
+ // EncodedImage and returns the image with a) modified original buffer (in
+ // such case the current owner of the buffer will be responsible for deleting
+ // it) or b) a new buffer (in such case injector will be responsible for
+ // deleting it).
+ const EncodedImage& image =
+ injector_->InjectData(frame_id, discard, encoded_image);
+ {
+ MutexLock lock(&mutex_);
+ RTC_DCHECK(delegate_callback_);
+ return delegate_callback_->OnEncodedImage(image, codec_specific_info);
+ }
+}
+
+void QualityAnalyzingVideoEncoder::OnDroppedFrame(
+ EncodedImageCallback::DropReason reason) {
+ MutexLock lock(&mutex_);
+ analyzer_->OnFrameDropped(peer_name_, reason);
+ RTC_DCHECK(delegate_callback_);
+ delegate_callback_->OnDroppedFrame(reason);
+}
+
+bool QualityAnalyzingVideoEncoder::ShouldDiscard(
+ uint16_t frame_id,
+ const EncodedImage& encoded_image) {
+ std::string stream_label = analyzer_->GetStreamLabel(frame_id);
+ EmulatedSFUConfigMap::mapped_type emulated_sfu_config =
+ stream_to_sfu_config_[stream_label];
+
+ if (!emulated_sfu_config)
+ return false;
+
+ int cur_spatial_index = encoded_image.SpatialIndex().value_or(0);
+ int cur_temporal_index = encoded_image.TemporalIndex().value_or(0);
+
+ if (emulated_sfu_config->target_temporal_index &&
+ cur_temporal_index > *emulated_sfu_config->target_temporal_index)
+ return true;
+
+ if (emulated_sfu_config->target_layer_index) {
+ switch (mode_) {
+ case SimulcastMode::kSimulcast:
+ // In simulcast mode only encoded images with required spatial index are
+ // interested, so all others have to be discarded.
+ return cur_spatial_index != *emulated_sfu_config->target_layer_index;
+ case SimulcastMode::kSVC:
+ // In SVC mode encoded images with spatial indexes that are equal or
+ // less than required one are interesting, so all above have to be
+ // discarded.
+ return cur_spatial_index > *emulated_sfu_config->target_layer_index;
+ case SimulcastMode::kKSVC:
+ // In KSVC mode for key frame encoded images with spatial indexes that
+ // are equal or less than required one are interesting, so all above
+ // have to be discarded. For other frames only required spatial index
+ // is interesting, so all others except the ones depending on the
+ // keyframes can be discarded. There's no good test for that, so we keep
+ // all of temporal layer 0 for now.
+ if (encoded_image._frameType == VideoFrameType::kVideoFrameKey ||
+ cur_temporal_index == 0)
+ return cur_spatial_index > *emulated_sfu_config->target_layer_index;
+ return cur_spatial_index != *emulated_sfu_config->target_layer_index;
+ case SimulcastMode::kNormal:
+ RTC_DCHECK_NOTREACHED() << "Analyzing encoder is in kNormal mode, but "
+ "target_layer_index is set";
+ }
+ }
+ return false;
+}
+
+QualityAnalyzingVideoEncoderFactory::QualityAnalyzingVideoEncoderFactory(
+ absl::string_view peer_name,
+ std::unique_ptr<VideoEncoderFactory> delegate,
+ double bitrate_multiplier,
+ EmulatedSFUConfigMap stream_to_sfu_config,
+ EncodedImageDataInjector* injector,
+ VideoQualityAnalyzerInterface* analyzer)
+ : peer_name_(peer_name),
+ delegate_(std::move(delegate)),
+ bitrate_multiplier_(bitrate_multiplier),
+ stream_to_sfu_config_(std::move(stream_to_sfu_config)),
+ injector_(injector),
+ analyzer_(analyzer) {}
+QualityAnalyzingVideoEncoderFactory::~QualityAnalyzingVideoEncoderFactory() =
+ default;
+
+std::vector<SdpVideoFormat>
+QualityAnalyzingVideoEncoderFactory::GetSupportedFormats() const {
+ return delegate_->GetSupportedFormats();
+}
+
+VideoEncoderFactory::CodecSupport
+QualityAnalyzingVideoEncoderFactory::QueryCodecSupport(
+ const SdpVideoFormat& format,
+ absl::optional<std::string> scalability_mode) const {
+ return delegate_->QueryCodecSupport(format, scalability_mode);
+}
+
+std::unique_ptr<VideoEncoder>
+QualityAnalyzingVideoEncoderFactory::CreateVideoEncoder(
+ const SdpVideoFormat& format) {
+ return std::make_unique<QualityAnalyzingVideoEncoder>(
+ peer_name_, delegate_->CreateVideoEncoder(format), bitrate_multiplier_,
+ stream_to_sfu_config_, injector_, analyzer_);
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h
new file mode 100644
index 0000000000..4adeacc0cd
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h
@@ -0,0 +1,194 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_
+
+#include <list>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "absl/strings/string_view.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/video_quality_analyzer_interface.h"
+#include "api/video/video_frame.h"
+#include "api/video_codecs/sdp_video_format.h"
+#include "api/video_codecs/video_codec.h"
+#include "api/video_codecs/video_encoder.h"
+#include "api/video_codecs/video_encoder_factory.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+// QualityAnalyzingVideoEncoder is used to wrap origin video encoder and inject
+// VideoQualityAnalyzerInterface before and after encoder.
+//
+// QualityAnalyzingVideoEncoder propagates all calls to the origin encoder.
+// It registers its own EncodedImageCallback in the origin encoder and will
+// store user specified callback inside itself.
+//
+// When Encode(...) will be invoked, quality encoder first calls video quality
+// analyzer with original frame, then encodes frame with original encoder.
+//
+// When origin encoder encodes the image it will call quality encoder's special
+// callback, where video analyzer will be called again and then frame id will be
+// injected into EncodedImage with passed EncodedImageDataInjector. Then new
+// EncodedImage will be passed to origin callback, provided by user.
+//
+// Quality encoder registers its own callback in origin encoder, at the same
+// time the user registers their callback in quality encoder.
+class QualityAnalyzingVideoEncoder : public VideoEncoder,
+ public EncodedImageCallback {
+ public:
+ using EmulatedSFUConfigMap =
+ std::map<std::string, absl::optional<EmulatedSFUConfig>>;
+
+ QualityAnalyzingVideoEncoder(absl::string_view peer_name,
+ std::unique_ptr<VideoEncoder> delegate,
+ double bitrate_multiplier,
+ EmulatedSFUConfigMap stream_to_sfu_config,
+ EncodedImageDataInjector* injector,
+ VideoQualityAnalyzerInterface* analyzer);
+ ~QualityAnalyzingVideoEncoder() override;
+
+ // Methods of VideoEncoder interface.
+ void SetFecControllerOverride(
+ FecControllerOverride* fec_controller_override) override;
+ int32_t InitEncode(const VideoCodec* codec_settings,
+ const Settings& settings) override;
+ int32_t RegisterEncodeCompleteCallback(
+ EncodedImageCallback* callback) override;
+ int32_t Release() override;
+ int32_t Encode(const VideoFrame& frame,
+ const std::vector<VideoFrameType>* frame_types) override;
+ void SetRates(const VideoEncoder::RateControlParameters& parameters) override;
+ EncoderInfo GetEncoderInfo() const override;
+
+ // Methods of EncodedImageCallback interface.
+ EncodedImageCallback::Result OnEncodedImage(
+ const EncodedImage& encoded_image,
+ const CodecSpecificInfo* codec_specific_info) override;
+ void OnDroppedFrame(DropReason reason) override;
+
+ private:
+ enum SimulcastMode {
+ // In this mode encoder assumes not more than 1 encoded image per video
+ // frame
+ kNormal,
+
+ // Next modes are to test video conference behavior. For conference sender
+ // will send multiple spatial layers/simulcast streams for single video
+ // track and there is some Selective Forwarding Unit (SFU), that forwards
+ // only best one, that will pass through downlink to the receiver.
+ //
+ // Here this behavior will be partly emulated. Sender will send all spatial
+ // layers/simulcast streams and then some of them will be filtered out on
+ // the receiver side. During test setup user can specify which spatial
+ // layer/simulcast stream is required, what will simulated which spatial
+ // layer/simulcast stream will be chosen by SFU in the real world. Then
+ // sender will mark encoded images for all spatial layers above required or
+ // all simulcast streams except required as to be discarded and on receiver
+ // side they will be discarded in quality analyzing decoder and won't be
+ // passed into delegate decoder.
+ //
+ // If the sender for some reasons won't send specified spatial layer, then
+ // receiver still will fall back on lower spatial layers. But for simulcast
+ // streams if required one won't be sent, receiver will assume all frames
+ // in that period as dropped and will experience video freeze.
+ //
+ // Test based on this simulation will be used to evaluate video quality
+ // of concrete spatial layers/simulcast streams and also check distribution
+ // of bandwidth between spatial layers/simulcast streams by BWE.
+
+ // In this mode encoder assumes that for each frame simulcast encoded
+ // images will be produced. So all simulcast streams except required will
+ // be marked as to be discarded in decoder and won't reach video quality
+ // analyzer.
+ kSimulcast,
+ // In this mode encoder assumes that for each frame encoded images for
+ // different spatial layers will be produced. So all spatial layers above
+ // required will be marked to be discarded in decoder and won't reach
+ // video quality analyzer.
+ kSVC,
+ // In this mode encoder assumes that for each frame encoded images for
+ // different spatial layers will be produced. Compared to kSVC mode
+ // spatial layers that are above required will be marked to be discarded
+ // only for key frames and for regular frames all except required spatial
+ // layer will be marked as to be discarded in decoder and won't reach video
+ // quality analyzer.
+ kKSVC
+ };
+
+ bool ShouldDiscard(uint16_t frame_id, const EncodedImage& encoded_image)
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+
+ const std::string peer_name_;
+ std::unique_ptr<VideoEncoder> delegate_;
+ const double bitrate_multiplier_;
+ // Contains mapping from stream label to optional spatial index.
+ // If we have stream label "Foo" and mapping contains
+ // 1. `absl::nullopt` means all streams are required
+ // 2. Concrete value means that particular simulcast/SVC stream have to be
+ // analyzed.
+ EmulatedSFUConfigMap stream_to_sfu_config_;
+ EncodedImageDataInjector* const injector_;
+ VideoQualityAnalyzerInterface* const analyzer_;
+
+ // VideoEncoder interface assumes async delivery of encoded images.
+ // This lock is used to protect shared state, that have to be propagated
+ // from received VideoFrame to resulted EncodedImage.
+ Mutex mutex_;
+
+ VideoCodec codec_settings_ RTC_GUARDED_BY(mutex_);
+ SimulcastMode mode_ RTC_GUARDED_BY(mutex_);
+ EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(mutex_);
+ std::list<std::pair<uint32_t, uint16_t>> timestamp_to_frame_id_list_
+ RTC_GUARDED_BY(mutex_);
+ VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(mutex_);
+};
+
+// Produces QualityAnalyzingVideoEncoder, which hold decoders, produced by
+// specified factory as delegates. Forwards all other calls to specified
+// factory.
+class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory {
+ public:
+ QualityAnalyzingVideoEncoderFactory(
+ absl::string_view peer_name,
+ std::unique_ptr<VideoEncoderFactory> delegate,
+ double bitrate_multiplier,
+ QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap stream_to_sfu_config,
+ EncodedImageDataInjector* injector,
+ VideoQualityAnalyzerInterface* analyzer);
+ ~QualityAnalyzingVideoEncoderFactory() override;
+
+ // Methods of VideoEncoderFactory interface.
+ std::vector<SdpVideoFormat> GetSupportedFormats() const override;
+ VideoEncoderFactory::CodecSupport QueryCodecSupport(
+ const SdpVideoFormat& format,
+ absl::optional<std::string> scalability_mode) const override;
+ std::unique_ptr<VideoEncoder> CreateVideoEncoder(
+ const SdpVideoFormat& format) override;
+
+ private:
+ const std::string peer_name_;
+ std::unique_ptr<VideoEncoderFactory> delegate_;
+ const double bitrate_multiplier_;
+ QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap stream_to_sfu_config_;
+ EncodedImageDataInjector* const injector_;
+ VideoQualityAnalyzerInterface* const analyzer_;
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.cc
new file mode 100644
index 0000000000..7a73b9f4f1
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.cc
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h"
+
+#include "api/video/i420_buffer.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_frame_buffer.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+constexpr char kIrrelatedSimulcastStreamFrameData[] = "Dummy!";
+
+} // namespace
+
+rtc::scoped_refptr<webrtc::VideoFrameBuffer> CreateDummyFrameBuffer() {
+ // Use i420 buffer here as default one and supported by all codecs.
+ rtc::scoped_refptr<webrtc::I420Buffer> buffer =
+ webrtc::I420Buffer::Create(2, 2);
+ memcpy(buffer->MutableDataY(), kIrrelatedSimulcastStreamFrameData, 2);
+ memcpy(buffer->MutableDataY() + buffer->StrideY(),
+ kIrrelatedSimulcastStreamFrameData + 2, 2);
+ memcpy(buffer->MutableDataU(), kIrrelatedSimulcastStreamFrameData + 4, 1);
+ memcpy(buffer->MutableDataV(), kIrrelatedSimulcastStreamFrameData + 5, 1);
+ return buffer;
+}
+
+bool IsDummyFrame(const webrtc::VideoFrame& video_frame) {
+ if (video_frame.width() != 2 || video_frame.height() != 2) {
+ return false;
+ }
+ rtc::scoped_refptr<webrtc::I420BufferInterface> buffer =
+ video_frame.video_frame_buffer()->ToI420();
+ if (memcmp(buffer->DataY(), kIrrelatedSimulcastStreamFrameData, 2) != 0) {
+ return false;
+ }
+ if (memcmp(buffer->DataY() + buffer->StrideY(),
+ kIrrelatedSimulcastStreamFrameData + 2, 2) != 0) {
+ return false;
+ }
+ if (memcmp(buffer->DataU(), kIrrelatedSimulcastStreamFrameData + 4, 1) != 0) {
+ return false;
+ }
+ if (memcmp(buffer->DataV(), kIrrelatedSimulcastStreamFrameData + 5, 1) != 0) {
+ return false;
+ }
+ return true;
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h
new file mode 100644
index 0000000000..8ecfae7385
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_SIMULCAST_DUMMY_BUFFER_HELPER_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_SIMULCAST_DUMMY_BUFFER_HELPER_H_
+
+#include "api/video/video_frame.h"
+#include "api/video/video_frame_buffer.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+// Creates a special video frame buffer that should be used to create frames
+// during Selective Forwarding Unit (SFU) emulation. Such frames are used when
+// original was discarded and some frame is required to be passed upstream
+// to make WebRTC pipeline happy and not request key frame on the received
+// stream due to lack of incoming frames.
+rtc::scoped_refptr<webrtc::VideoFrameBuffer> CreateDummyFrameBuffer();
+
+// Tests if provided frame contains a buffer created by
+// `CreateDummyFrameBuffer`.
+bool IsDummyFrame(const webrtc::VideoFrame& video_frame);
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_SIMULCAST_DUMMY_BUFFER_HELPER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper_test.cc
new file mode 100644
index 0000000000..db1030232d
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper_test.cc
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h"
+
+#include "api/video/i420_buffer.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_frame_buffer.h"
+#include "rtc_base/random.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+uint8_t RandByte(Random& random) {
+ return random.Rand(255);
+}
+
+VideoFrame CreateRandom2x2VideoFrame(uint16_t id, Random& random) {
+ rtc::scoped_refptr<I420Buffer> buffer = I420Buffer::Create(2, 2);
+
+ uint8_t data[6] = {RandByte(random), RandByte(random), RandByte(random),
+ RandByte(random), RandByte(random), RandByte(random)};
+
+ memcpy(buffer->MutableDataY(), data, 2);
+ memcpy(buffer->MutableDataY() + buffer->StrideY(), data + 2, 2);
+ memcpy(buffer->MutableDataU(), data + 4, 1);
+ memcpy(buffer->MutableDataV(), data + 5, 1);
+
+ return VideoFrame::Builder()
+ .set_id(id)
+ .set_video_frame_buffer(buffer)
+ .set_timestamp_us(1)
+ .build();
+}
+
+TEST(CreateDummyFrameBufferTest, CreatedBufferIsDummy) {
+ VideoFrame dummy_frame = VideoFrame::Builder()
+ .set_video_frame_buffer(CreateDummyFrameBuffer())
+ .build();
+
+ EXPECT_TRUE(IsDummyFrame(dummy_frame));
+}
+
+TEST(IsDummyFrameTest, NotEveryFrameIsDummy) {
+ Random random(/*seed=*/100);
+ VideoFrame frame = CreateRandom2x2VideoFrame(1, random);
+ EXPECT_FALSE(IsDummyFrame(frame));
+}
+
+} // namespace
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc
new file mode 100644
index 0000000000..ccd2f03537
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc
@@ -0,0 +1,187 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h"
+
+#include <algorithm>
+#include <cstddef>
+
+#include "absl/memory/memory.h"
+#include "api/video/encoded_image.h"
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+SingleProcessEncodedImageDataInjector::SingleProcessEncodedImageDataInjector() =
+ default;
+SingleProcessEncodedImageDataInjector::
+ ~SingleProcessEncodedImageDataInjector() = default;
+
+EncodedImage SingleProcessEncodedImageDataInjector::InjectData(
+ uint16_t id,
+ bool discard,
+ const EncodedImage& source) {
+ RTC_CHECK(source.size() >= ExtractionInfo::kUsedBufferSize);
+
+ ExtractionInfo info;
+ info.discard = discard;
+ size_t insertion_pos = source.size() - ExtractionInfo::kUsedBufferSize;
+ memcpy(info.origin_data, &source.data()[insertion_pos],
+ ExtractionInfo::kUsedBufferSize);
+ {
+ MutexLock lock(&lock_);
+ // Will create new one if missed.
+ ExtractionInfoVector& ev = extraction_cache_[id];
+ info.sub_id = ev.next_sub_id++;
+ ev.infos[info.sub_id] = info;
+ }
+
+ auto buffer = EncodedImageBuffer::Create(source.data(), source.size());
+ buffer->data()[insertion_pos] = id & 0x00ff;
+ buffer->data()[insertion_pos + 1] = (id & 0xff00) >> 8;
+ buffer->data()[insertion_pos + 2] = info.sub_id;
+
+ EncodedImage out = source;
+ out.SetEncodedData(buffer);
+ return out;
+}
+
+void SingleProcessEncodedImageDataInjector::AddParticipantInCall() {
+ MutexLock crit(&lock_);
+ expected_receivers_count_++;
+}
+
+void SingleProcessEncodedImageDataInjector::RemoveParticipantInCall() {
+ MutexLock crit(&lock_);
+ expected_receivers_count_--;
+ // Now we need go over `extraction_cache_` and removed frames which have been
+ // received by `expected_receivers_count_`.
+ for (auto& [frame_id, extraction_infos] : extraction_cache_) {
+ for (auto it = extraction_infos.infos.begin();
+ it != extraction_infos.infos.end();) {
+ // Frame is received if `received_count` equals to
+ // `expected_receivers_count_`.
+ if (it->second.received_count == expected_receivers_count_) {
+ it = extraction_infos.infos.erase(it);
+ } else {
+ ++it;
+ }
+ }
+ }
+}
+
+EncodedImageExtractionResult SingleProcessEncodedImageDataInjector::ExtractData(
+ const EncodedImage& source) {
+ size_t size = source.size();
+ auto buffer = EncodedImageBuffer::Create(source.data(), source.size());
+ EncodedImage out = source;
+ out.SetEncodedData(buffer);
+
+ std::vector<size_t> frame_sizes;
+ std::vector<size_t> frame_sl_index;
+ size_t max_spatial_index = out.SpatialIndex().value_or(0);
+ for (size_t i = 0; i <= max_spatial_index; ++i) {
+ auto frame_size = source.SpatialLayerFrameSize(i);
+ if (frame_size.value_or(0)) {
+ frame_sl_index.push_back(i);
+ frame_sizes.push_back(frame_size.value());
+ }
+ }
+ if (frame_sizes.empty()) {
+ frame_sizes.push_back(size);
+ }
+
+ size_t prev_frames_size = 0;
+ absl::optional<uint16_t> id = absl::nullopt;
+ bool discard = true;
+ std::vector<ExtractionInfo> extraction_infos;
+ for (size_t frame_size : frame_sizes) {
+ size_t insertion_pos =
+ prev_frames_size + frame_size - ExtractionInfo::kUsedBufferSize;
+ // Extract frame id from first 2 bytes starting from insertion pos.
+ uint16_t next_id = buffer->data()[insertion_pos] +
+ (buffer->data()[insertion_pos + 1] << 8);
+ // Extract frame sub id from second 3 byte starting from insertion pos.
+ uint8_t sub_id = buffer->data()[insertion_pos + 2];
+ RTC_CHECK(!id || *id == next_id)
+ << "Different frames encoded into single encoded image: " << *id
+ << " vs " << next_id;
+ id = next_id;
+ ExtractionInfo info;
+ {
+ MutexLock lock(&lock_);
+ auto ext_vector_it = extraction_cache_.find(next_id);
+ RTC_CHECK(ext_vector_it != extraction_cache_.end())
+ << "Unknown frame_id=" << next_id;
+
+ auto info_it = ext_vector_it->second.infos.find(sub_id);
+ RTC_CHECK(info_it != ext_vector_it->second.infos.end())
+ << "Unknown sub_id=" << sub_id << " for frame_id=" << next_id;
+ info_it->second.received_count++;
+ info = info_it->second;
+ if (info.received_count == expected_receivers_count_) {
+ ext_vector_it->second.infos.erase(info_it);
+ }
+ }
+ // We need to discard encoded image only if all concatenated encoded images
+ // have to be discarded.
+ discard = discard && info.discard;
+
+ extraction_infos.push_back(info);
+ prev_frames_size += frame_size;
+ }
+ RTC_CHECK(id);
+
+ if (discard) {
+ out.set_size(0);
+ for (size_t i = 0; i <= max_spatial_index; ++i) {
+ out.SetSpatialLayerFrameSize(i, 0);
+ }
+ return EncodedImageExtractionResult{*id, out, true};
+ }
+
+ // Make a pass from begin to end to restore origin payload and erase discarded
+ // encoded images.
+ size_t pos = 0;
+ for (size_t frame_index = 0; frame_index < frame_sizes.size();
+ ++frame_index) {
+ RTC_CHECK(pos < size);
+ const size_t frame_size = frame_sizes[frame_index];
+ const ExtractionInfo& info = extraction_infos[frame_index];
+ if (info.discard) {
+ // If this encoded image is marked to be discarded - erase it's payload
+ // from the buffer.
+ memmove(&buffer->data()[pos], &buffer->data()[pos + frame_size],
+ size - pos - frame_size);
+ RTC_CHECK_LT(frame_index, frame_sl_index.size())
+ << "codec doesn't support discard option or the image, that was "
+ "supposed to be discarded, is lost";
+ out.SetSpatialLayerFrameSize(frame_sl_index[frame_index], 0);
+ size -= frame_size;
+ } else {
+ memcpy(
+ &buffer->data()[pos + frame_size - ExtractionInfo::kUsedBufferSize],
+ info.origin_data, ExtractionInfo::kUsedBufferSize);
+ pos += frame_size;
+ }
+ }
+ out.set_size(pos);
+
+ return EncodedImageExtractionResult{*id, out, discard};
+}
+
+SingleProcessEncodedImageDataInjector::ExtractionInfoVector::
+ ExtractionInfoVector() = default;
+SingleProcessEncodedImageDataInjector::ExtractionInfoVector::
+ ~ExtractionInfoVector() = default;
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h
new file mode 100644
index 0000000000..1082440e2f
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h
@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "api/video/encoded_image.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+// Based on assumption that all call participants are in the same OS process
+// and uses same QualityAnalyzingVideoContext to obtain
+// EncodedImageDataInjector.
+//
+// To inject frame id and discard flag into EncodedImage injector uses last 3rd
+// and 2nd bytes of EncodedImage payload. Then it uses last byte for frame
+// sub id, that is required to distinguish different spatial layers. The origin
+// data from these 3 bytes will be stored inside injector's internal storage and
+// then will be restored during extraction phase.
+//
+// This injector won't add any extra overhead into EncodedImage payload and
+// support frames with any size of payload. Also assumes that every EncodedImage
+// payload size is greater or equals to 3 bytes
+//
+// This injector doesn't support video frames/encoded images without frame ID.
+class SingleProcessEncodedImageDataInjector
+ : public EncodedImageDataPropagator {
+ public:
+ SingleProcessEncodedImageDataInjector();
+ ~SingleProcessEncodedImageDataInjector() override;
+
+ // Id and discard flag will be injected into EncodedImage buffer directly.
+ // This buffer won't be fully copied, so `source` image buffer will be also
+ // changed.
+ EncodedImage InjectData(uint16_t id,
+ bool discard,
+ const EncodedImage& source) override;
+
+ void Start(int expected_receivers_count) override {
+ MutexLock crit(&lock_);
+ expected_receivers_count_ = expected_receivers_count;
+ }
+ void AddParticipantInCall() override;
+ void RemoveParticipantInCall() override;
+ EncodedImageExtractionResult ExtractData(const EncodedImage& source) override;
+
+ private:
+ // Contains data required to extract frame id from EncodedImage and restore
+ // original buffer.
+ struct ExtractionInfo {
+ // Number of bytes from the beginning of the EncodedImage buffer that will
+ // be used to store frame id and sub id.
+ const static size_t kUsedBufferSize = 3;
+ // Frame sub id to distinguish encoded images for different spatial layers.
+ uint8_t sub_id;
+ // Flag to show is this encoded images should be discarded by analyzing
+ // decoder because of not required spatial layer/simulcast stream.
+ bool discard;
+ // Data from first 3 bytes of origin encoded image's payload.
+ uint8_t origin_data[ExtractionInfo::kUsedBufferSize];
+ // Count of how many times this frame was received.
+ int received_count = 0;
+ };
+
+ struct ExtractionInfoVector {
+ ExtractionInfoVector();
+ ~ExtractionInfoVector();
+
+ // Next sub id, that have to be used for this frame id.
+ uint8_t next_sub_id = 0;
+ std::map<uint8_t, ExtractionInfo> infos;
+ };
+
+ Mutex lock_;
+ int expected_receivers_count_ RTC_GUARDED_BY(lock_);
+ // Stores a mapping from frame id to extraction info for spatial layers
+ // for this frame id. There can be a lot of them, because if frame was
+ // dropped we can't clean it up, because we won't receive a signal on
+ // decoder side about that frame. In such case it will be replaced
+ // when sub id will overlap.
+ std::map<uint16_t, ExtractionInfoVector> extraction_cache_
+ RTC_GUARDED_BY(lock_);
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_SINGLE_PROCESS_ENCODED_IMAGE_DATA_INJECTOR_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc
new file mode 100644
index 0000000000..f6fa40455a
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector_unittest.cc
@@ -0,0 +1,445 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h"
+
+#include <utility>
+
+#include "api/video/encoded_image.h"
+#include "rtc_base/buffer.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+rtc::scoped_refptr<EncodedImageBuffer>
+CreateEncodedImageBufferOfSizeNFilledWithValuesFromX(size_t n, uint8_t x) {
+ auto buffer = EncodedImageBuffer::Create(n);
+ for (size_t i = 0; i < n; ++i) {
+ buffer->data()[i] = static_cast<uint8_t>(x + i);
+ }
+ return buffer;
+}
+
+EncodedImage CreateEncodedImageOfSizeNFilledWithValuesFromX(size_t n,
+ uint8_t x) {
+ EncodedImage image;
+ image.SetEncodedData(
+ CreateEncodedImageBufferOfSizeNFilledWithValuesFromX(n, x));
+ return image;
+}
+
+EncodedImage DeepCopyEncodedImage(const EncodedImage& source) {
+ EncodedImage copy = source;
+ copy.SetEncodedData(EncodedImageBuffer::Create(source.data(), source.size()));
+ return copy;
+}
+
+TEST(SingleProcessEncodedImageDataInjectorTest, InjectExtractDiscardFalse) {
+ SingleProcessEncodedImageDataInjector injector;
+ injector.Start(/*expected_receivers_count=*/1);
+
+ EncodedImage source =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
+ source.SetTimestamp(123456789);
+
+ EncodedImageExtractionResult out =
+ injector.ExtractData(injector.InjectData(512, false, source));
+ EXPECT_EQ(out.id, 512);
+ EXPECT_FALSE(out.discard);
+ EXPECT_EQ(out.image.size(), 10ul);
+ EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
+ for (int i = 0; i < 10; ++i) {
+ EXPECT_EQ(out.image.data()[i], i + 1);
+ }
+}
+
+TEST(SingleProcessEncodedImageDataInjectorTest, InjectExtractDiscardTrue) {
+ SingleProcessEncodedImageDataInjector injector;
+ injector.Start(/*expected_receivers_count=*/1);
+
+ EncodedImage source =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
+ source.SetTimestamp(123456789);
+
+ EncodedImageExtractionResult out =
+ injector.ExtractData(injector.InjectData(512, true, source));
+ EXPECT_EQ(out.id, 512);
+ EXPECT_TRUE(out.discard);
+ EXPECT_EQ(out.image.size(), 0ul);
+ EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
+}
+
+TEST(SingleProcessEncodedImageDataInjectorTest,
+ InjectWithUnsetSpatialLayerSizes) {
+ SingleProcessEncodedImageDataInjector injector;
+ injector.Start(/*expected_receivers_count=*/1);
+
+ EncodedImage source =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
+ source.SetTimestamp(123456789);
+
+ EncodedImage intermediate = injector.InjectData(512, false, source);
+ intermediate.SetSpatialIndex(2);
+
+ EncodedImageExtractionResult out = injector.ExtractData(intermediate);
+ EXPECT_EQ(out.id, 512);
+ EXPECT_FALSE(out.discard);
+ EXPECT_EQ(out.image.size(), 10ul);
+ for (int i = 0; i < 10; ++i) {
+ EXPECT_EQ(out.image.data()[i], i + 1);
+ }
+ EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2);
+ for (int i = 0; i < 3; ++i) {
+ EXPECT_EQ(out.image.SpatialLayerFrameSize(i).value_or(0), 0ul);
+ }
+}
+
+TEST(SingleProcessEncodedImageDataInjectorTest,
+ InjectWithZeroSpatialLayerSizes) {
+ SingleProcessEncodedImageDataInjector injector;
+ injector.Start(/*expected_receivers_count=*/1);
+
+ EncodedImage source =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
+ source.SetTimestamp(123456789);
+
+ EncodedImage intermediate = injector.InjectData(512, false, source);
+ intermediate.SetSpatialIndex(2);
+ intermediate.SetSpatialLayerFrameSize(0, 0);
+ intermediate.SetSpatialLayerFrameSize(1, 0);
+ intermediate.SetSpatialLayerFrameSize(2, 0);
+
+ EncodedImageExtractionResult out = injector.ExtractData(intermediate);
+ EXPECT_EQ(out.id, 512);
+ EXPECT_FALSE(out.discard);
+ EXPECT_EQ(out.image.size(), 10ul);
+ for (int i = 0; i < 10; ++i) {
+ EXPECT_EQ(out.image.data()[i], i + 1);
+ }
+ EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2);
+ for (int i = 0; i < 3; ++i) {
+ EXPECT_EQ(out.image.SpatialLayerFrameSize(i).value_or(0), 0ul);
+ }
+}
+
+TEST(SingleProcessEncodedImageDataInjectorTest, Inject3Extract3) {
+ SingleProcessEncodedImageDataInjector injector;
+ injector.Start(/*expected_receivers_count=*/1);
+
+ // 1st frame
+ EncodedImage source1 =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
+ source1.SetTimestamp(123456710);
+ // 2nd frame 1st spatial layer
+ EncodedImage source2 =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/11);
+ source2.SetTimestamp(123456720);
+ // 2nd frame 2nd spatial layer
+ EncodedImage source3 =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/21);
+ source3.SetTimestamp(123456720);
+
+ EncodedImage intermediate1 = injector.InjectData(510, false, source1);
+ EncodedImage intermediate2 = injector.InjectData(520, true, source2);
+ EncodedImage intermediate3 = injector.InjectData(520, false, source3);
+
+ // Extract ids in different order.
+ EncodedImageExtractionResult out3 = injector.ExtractData(intermediate3);
+ EncodedImageExtractionResult out1 = injector.ExtractData(intermediate1);
+ EncodedImageExtractionResult out2 = injector.ExtractData(intermediate2);
+
+ EXPECT_EQ(out1.id, 510);
+ EXPECT_FALSE(out1.discard);
+ EXPECT_EQ(out1.image.size(), 10ul);
+ EXPECT_EQ(out1.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
+ for (int i = 0; i < 10; ++i) {
+ EXPECT_EQ(out1.image.data()[i], i + 1);
+ }
+ EXPECT_EQ(out2.id, 520);
+ EXPECT_TRUE(out2.discard);
+ EXPECT_EQ(out2.image.size(), 0ul);
+ EXPECT_EQ(out2.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
+ EXPECT_EQ(out3.id, 520);
+ EXPECT_FALSE(out3.discard);
+ EXPECT_EQ(out3.image.size(), 10ul);
+ EXPECT_EQ(out3.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
+ for (int i = 0; i < 10; ++i) {
+ EXPECT_EQ(out3.image.data()[i], i + 21);
+ }
+}
+
+TEST(SingleProcessEncodedImageDataInjectorTest, InjectExtractFromConcatenated) {
+ SingleProcessEncodedImageDataInjector injector;
+ injector.Start(/*expected_receivers_count=*/1);
+
+ EncodedImage source1 =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
+ source1.SetTimestamp(123456710);
+ EncodedImage source2 =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/11);
+ source2.SetTimestamp(123456710);
+ EncodedImage source3 =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/21);
+ source3.SetTimestamp(123456710);
+
+ // Inject id into 3 images with same frame id.
+ EncodedImage intermediate1 = injector.InjectData(512, false, source1);
+ EncodedImage intermediate2 = injector.InjectData(512, true, source2);
+ EncodedImage intermediate3 = injector.InjectData(512, false, source3);
+
+ // Concatenate them into single encoded image, like it can be done in jitter
+ // buffer.
+ size_t concatenated_length =
+ intermediate1.size() + intermediate2.size() + intermediate3.size();
+ rtc::Buffer concatenated_buffer;
+ concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size());
+ concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size());
+ concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size());
+ EncodedImage concatenated;
+ concatenated.SetEncodedData(EncodedImageBuffer::Create(
+ concatenated_buffer.data(), concatenated_length));
+ concatenated.SetSpatialIndex(2);
+ concatenated.SetSpatialLayerFrameSize(0, intermediate1.size());
+ concatenated.SetSpatialLayerFrameSize(1, intermediate2.size());
+ concatenated.SetSpatialLayerFrameSize(2, intermediate3.size());
+
+ // Extract frame id from concatenated image
+ EncodedImageExtractionResult out = injector.ExtractData(concatenated);
+
+ EXPECT_EQ(out.id, 512);
+ EXPECT_FALSE(out.discard);
+ EXPECT_EQ(out.image.size(), 2 * 10ul);
+ for (int i = 0; i < 10; ++i) {
+ EXPECT_EQ(out.image.data()[i], i + 1);
+ EXPECT_EQ(out.image.data()[i + 10], i + 21);
+ }
+ EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2);
+ EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 10ul);
+ EXPECT_EQ(out.image.SpatialLayerFrameSize(1).value_or(0), 0ul);
+ EXPECT_EQ(out.image.SpatialLayerFrameSize(2).value_or(0), 10ul);
+}
+
+TEST(SingleProcessEncodedImageDataInjector,
+ InjectExtractFromConcatenatedAllDiscarded) {
+ SingleProcessEncodedImageDataInjector injector;
+ injector.Start(/*expected_receivers_count=*/1);
+
+ EncodedImage source1 =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
+ source1.SetTimestamp(123456710);
+ EncodedImage source2 =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/11);
+ source2.SetTimestamp(123456710);
+ EncodedImage source3 =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/21);
+ source3.SetTimestamp(123456710);
+
+ // Inject id into 3 images with same frame id.
+ EncodedImage intermediate1 = injector.InjectData(512, true, source1);
+ EncodedImage intermediate2 = injector.InjectData(512, true, source2);
+ EncodedImage intermediate3 = injector.InjectData(512, true, source3);
+
+ // Concatenate them into single encoded image, like it can be done in jitter
+ // buffer.
+ size_t concatenated_length =
+ intermediate1.size() + intermediate2.size() + intermediate3.size();
+ rtc::Buffer concatenated_buffer;
+ concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size());
+ concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size());
+ concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size());
+ EncodedImage concatenated;
+ concatenated.SetEncodedData(EncodedImageBuffer::Create(
+ concatenated_buffer.data(), concatenated_length));
+ concatenated.SetSpatialIndex(2);
+ concatenated.SetSpatialLayerFrameSize(0, intermediate1.size());
+ concatenated.SetSpatialLayerFrameSize(1, intermediate2.size());
+ concatenated.SetSpatialLayerFrameSize(2, intermediate3.size());
+
+ // Extract frame id from concatenated image
+ EncodedImageExtractionResult out = injector.ExtractData(concatenated);
+
+ EXPECT_EQ(out.id, 512);
+ EXPECT_TRUE(out.discard);
+ EXPECT_EQ(out.image.size(), 0ul);
+ EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2);
+ for (int i = 0; i < 3; ++i) {
+ EXPECT_EQ(out.image.SpatialLayerFrameSize(i).value_or(0), 0ul);
+ }
+}
+
+TEST(SingleProcessEncodedImageDataInjectorTest, InjectOnceExtractTwice) {
+ SingleProcessEncodedImageDataInjector injector;
+ injector.Start(/*expected_receivers_count=*/2);
+
+ EncodedImage source =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
+ source.SetTimestamp(123456789);
+
+ EncodedImageExtractionResult out = injector.ExtractData(
+ injector.InjectData(/*id=*/512, /*discard=*/false, source));
+ EXPECT_EQ(out.id, 512);
+ EXPECT_FALSE(out.discard);
+ EXPECT_EQ(out.image.size(), 10ul);
+ EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
+ for (int i = 0; i < 10; ++i) {
+ EXPECT_EQ(out.image.data()[i], i + 1);
+ }
+ out = injector.ExtractData(
+ injector.InjectData(/*id=*/512, /*discard=*/false, source));
+ EXPECT_EQ(out.id, 512);
+ EXPECT_FALSE(out.discard);
+ EXPECT_EQ(out.image.size(), 10ul);
+ EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
+ for (int i = 0; i < 10; ++i) {
+ EXPECT_EQ(out.image.data()[i], i + 1);
+ }
+}
+
+TEST(SingleProcessEncodedImageDataInjectorTest, Add1stReceiverAfterStart) {
+ SingleProcessEncodedImageDataInjector injector;
+ injector.Start(/*expected_receivers_count=*/0);
+
+ EncodedImage source =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
+ source.SetTimestamp(123456789);
+ EncodedImage modified_image = injector.InjectData(
+ /*id=*/512, /*discard=*/false, source);
+
+ injector.AddParticipantInCall();
+ EncodedImageExtractionResult out = injector.ExtractData(modified_image);
+
+ EXPECT_EQ(out.id, 512);
+ EXPECT_FALSE(out.discard);
+ EXPECT_EQ(out.image.size(), 10ul);
+ EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
+ for (int i = 0; i < 10; ++i) {
+ EXPECT_EQ(out.image.data()[i], i + 1);
+ }
+}
+
+TEST(SingleProcessEncodedImageDataInjectorTest, Add3rdReceiverAfterStart) {
+ SingleProcessEncodedImageDataInjector injector;
+ injector.Start(/*expected_receivers_count=*/2);
+
+ EncodedImage source =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
+ source.SetTimestamp(123456789);
+ EncodedImage modified_image = injector.InjectData(
+ /*id=*/512, /*discard=*/false, source);
+ injector.ExtractData(modified_image);
+
+ injector.AddParticipantInCall();
+ injector.ExtractData(modified_image);
+ EncodedImageExtractionResult out = injector.ExtractData(modified_image);
+
+ EXPECT_EQ(out.id, 512);
+ EXPECT_FALSE(out.discard);
+ EXPECT_EQ(out.image.size(), 10ul);
+ EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
+ for (int i = 0; i < 10; ++i) {
+ EXPECT_EQ(out.image.data()[i], i + 1);
+ }
+}
+
+TEST(SingleProcessEncodedImageDataInjectorTest,
+ RemoveReceiverRemovesOnlyFullyReceivedFrames) {
+ SingleProcessEncodedImageDataInjector injector;
+ injector.Start(/*expected_receivers_count=*/3);
+
+ EncodedImage source1 =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
+ source1.SetTimestamp(10);
+ EncodedImage source2 =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
+ source2.SetTimestamp(20);
+
+ EncodedImage modified_image1 = injector.InjectData(
+ /*id=*/512, /*discard=*/false, source1);
+ EncodedImage modified_image2 = injector.InjectData(
+ /*id=*/513, /*discard=*/false, source2);
+
+ // Out of 3 receivers 1st image received by 2 and 2nd image by 1
+ injector.ExtractData(DeepCopyEncodedImage(modified_image1));
+ injector.ExtractData(DeepCopyEncodedImage(modified_image1));
+ injector.ExtractData(DeepCopyEncodedImage(modified_image2));
+
+ // When we removed one receiver, 2nd image should still be available for
+ // extraction.
+ injector.RemoveParticipantInCall();
+
+ EncodedImageExtractionResult out =
+ injector.ExtractData(DeepCopyEncodedImage(modified_image2));
+
+ EXPECT_EQ(out.id, 513);
+ EXPECT_FALSE(out.discard);
+ EXPECT_EQ(out.image.size(), 10ul);
+ EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
+ for (int i = 0; i < 10; ++i) {
+ EXPECT_EQ(out.image.data()[i], i + 1);
+ }
+}
+
+// Death tests.
+// Disabled on Android because death tests misbehave on Android, see
+// base/test/gtest_util.h.
+#if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID)
+TEST(SingleProcessEncodedImageDataInjectorTestDeathTest,
+ InjectOnceExtractMoreThenExpected) {
+ SingleProcessEncodedImageDataInjector injector;
+ injector.Start(/*expected_receivers_count=*/2);
+
+ EncodedImage source =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
+ source.SetTimestamp(123456789);
+
+ EncodedImage modified =
+ injector.InjectData(/*id=*/512, /*discard=*/false, source);
+
+ injector.ExtractData(DeepCopyEncodedImage(modified));
+ injector.ExtractData(DeepCopyEncodedImage(modified));
+ EXPECT_DEATH(injector.ExtractData(DeepCopyEncodedImage(modified)),
+ "Unknown sub_id=0 for frame_id=512");
+}
+
+TEST(SingleProcessEncodedImageDataInjectorTestDeathTest,
+ RemoveReceiverRemovesOnlyFullyReceivedFramesVerifyFrameIsRemoved) {
+ SingleProcessEncodedImageDataInjector injector;
+ injector.Start(/*expected_receivers_count=*/3);
+
+ EncodedImage source1 =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
+ source1.SetTimestamp(10);
+ EncodedImage source2 =
+ CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
+ source2.SetTimestamp(20);
+
+ EncodedImage modified_image1 = injector.InjectData(
+ /*id=*/512, /*discard=*/false, source1);
+ EncodedImage modified_image2 = injector.InjectData(
+ /*id=*/513, /*discard=*/false, source2);
+
+ // Out of 3 receivers 1st image received by 2 and 2nd image by 1
+ injector.ExtractData(DeepCopyEncodedImage(modified_image1));
+ injector.ExtractData(DeepCopyEncodedImage(modified_image1));
+ injector.ExtractData(DeepCopyEncodedImage(modified_image2));
+
+ // When we removed one receiver 1st image should be removed.
+ injector.RemoveParticipantInCall();
+
+ EXPECT_DEATH(injector.ExtractData(DeepCopyEncodedImage(modified_image1)),
+ "Unknown sub_id=0 for frame_id=512");
+}
+#endif // RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID)
+
+} // namespace
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.cc
new file mode 100644
index 0000000000..4fec0a8f9e
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.cc
@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/analyzer/video/video_dumping.h"
+
+#include <stdio.h>
+
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/strings/string_view.h"
+#include "api/test/video/video_frame_writer.h"
+#include "api/video/video_frame.h"
+#include "rtc_base/logging.h"
+#include "system_wrappers/include/clock.h"
+#include "test/testsupport/video_frame_writer.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+class VideoFrameIdsWriter final : public test::VideoFrameWriter {
+ public:
+ explicit VideoFrameIdsWriter(absl::string_view file_name)
+ : file_name_(file_name) {
+ output_file_ = fopen(file_name_.c_str(), "wb");
+ RTC_LOG(LS_INFO) << "Writing VideoFrame IDs into " << file_name_;
+ RTC_CHECK(output_file_ != nullptr)
+ << "Failed to open file to dump frame ids for writing: " << file_name_;
+ }
+ ~VideoFrameIdsWriter() override { Close(); }
+
+ bool WriteFrame(const VideoFrame& frame) override {
+ RTC_CHECK(output_file_ != nullptr) << "Writer is already closed";
+ int chars_written = fprintf(output_file_, "%d\n", frame.id());
+ if (chars_written < 2) {
+ RTC_LOG(LS_ERROR) << "Failed to write frame id to the output file: "
+ << file_name_;
+ return false;
+ }
+ return true;
+ }
+
+ void Close() override {
+ if (output_file_ != nullptr) {
+ RTC_LOG(LS_INFO) << "Closing file for VideoFrame IDs: " << file_name_;
+ fclose(output_file_);
+ output_file_ = nullptr;
+ }
+ }
+
+ private:
+ const std::string file_name_;
+ FILE* output_file_;
+};
+
+// Broadcast received frame to multiple underlying frame writers.
+class BroadcastingFrameWriter final : public test::VideoFrameWriter {
+ public:
+ explicit BroadcastingFrameWriter(
+ std::vector<std::unique_ptr<test::VideoFrameWriter>> delegates)
+ : delegates_(std::move(delegates)) {}
+ ~BroadcastingFrameWriter() override { Close(); }
+
+ bool WriteFrame(const webrtc::VideoFrame& frame) override {
+ for (auto& delegate : delegates_) {
+ if (!delegate->WriteFrame(frame)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ void Close() override {
+ for (auto& delegate : delegates_) {
+ delegate->Close();
+ }
+ }
+
+ private:
+ std::vector<std::unique_ptr<test::VideoFrameWriter>> delegates_;
+};
+
+} // namespace
+
+VideoWriter::VideoWriter(test::VideoFrameWriter* video_writer,
+ int sampling_modulo)
+ : video_writer_(video_writer), sampling_modulo_(sampling_modulo) {}
+
+void VideoWriter::OnFrame(const VideoFrame& frame) {
+ if (frames_counter_++ % sampling_modulo_ != 0) {
+ return;
+ }
+ bool result = video_writer_->WriteFrame(frame);
+ RTC_CHECK(result) << "Failed to write frame";
+}
+
+std::unique_ptr<test::VideoFrameWriter> CreateVideoFrameWithIdsWriter(
+ std::unique_ptr<test::VideoFrameWriter> video_writer_delegate,
+ absl::string_view frame_ids_dump_file_name) {
+ std::vector<std::unique_ptr<test::VideoFrameWriter>> requested_writers;
+ requested_writers.push_back(std::move(video_writer_delegate));
+ requested_writers.push_back(
+ std::make_unique<VideoFrameIdsWriter>(frame_ids_dump_file_name));
+ return std::make_unique<BroadcastingFrameWriter>(
+ std::move(requested_writers));
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.h
new file mode 100644
index 0000000000..cad4e1bdbf
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_DUMPING_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_DUMPING_H_
+
+#include <memory>
+#include <string>
+
+#include "absl/strings/string_view.h"
+#include "api/test/video/video_frame_writer.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_sink_interface.h"
+#include "test/testsupport/video_frame_writer.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+// `VideoSinkInterface` to dump incoming video frames into specified video
+// writer.
+class VideoWriter final : public rtc::VideoSinkInterface<VideoFrame> {
+ public:
+ // Creates video writer. Caller keeps ownership of `video_writer` and is
+ // responsible for closing it after VideoWriter will be destroyed.
+ VideoWriter(test::VideoFrameWriter* video_writer, int sampling_modulo);
+ VideoWriter(const VideoWriter&) = delete;
+ VideoWriter& operator=(const VideoWriter&) = delete;
+ ~VideoWriter() override = default;
+
+ void OnFrame(const VideoFrame& frame) override;
+
+ private:
+ test::VideoFrameWriter* const video_writer_;
+ const int sampling_modulo_;
+
+ int64_t frames_counter_ = 0;
+};
+
+// Creates a `VideoFrameWriter` to dump video frames together with their ids.
+// It uses provided `video_writer_delegate` to write video itself. Frame ids
+// will be logged into the specified file.
+std::unique_ptr<test::VideoFrameWriter> CreateVideoFrameWithIdsWriter(
+ std::unique_ptr<test::VideoFrameWriter> video_writer_delegate,
+ absl::string_view frame_ids_dump_file_name);
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_DUMPING_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping_test.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping_test.cc
new file mode 100644
index 0000000000..5dd4021516
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_dumping_test.cc
@@ -0,0 +1,196 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/analyzer/video/video_dumping.h"
+
+#include <stdio.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/scoped_refptr.h"
+#include "api/video/i420_buffer.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_frame_buffer.h"
+#include "rtc_base/random.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/testsupport/file_utils.h"
+#include "test/testsupport/frame_reader.h"
+#include "test/testsupport/video_frame_writer.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+using ::testing::ElementsAreArray;
+using ::testing::Eq;
+using ::testing::Test;
+
+uint8_t RandByte(Random& random) {
+ return random.Rand(255);
+}
+
+VideoFrame CreateRandom2x2VideoFrame(uint16_t id, Random& random) {
+ rtc::scoped_refptr<I420Buffer> buffer = I420Buffer::Create(2, 2);
+
+ uint8_t data[6] = {RandByte(random), RandByte(random), RandByte(random),
+ RandByte(random), RandByte(random), RandByte(random)};
+
+ memcpy(buffer->MutableDataY(), data, 2);
+ memcpy(buffer->MutableDataY() + buffer->StrideY(), data + 2, 2);
+ memcpy(buffer->MutableDataU(), data + 4, 1);
+ memcpy(buffer->MutableDataV(), data + 5, 1);
+
+ return VideoFrame::Builder()
+ .set_id(id)
+ .set_video_frame_buffer(buffer)
+ .set_timestamp_us(1)
+ .build();
+}
+
+std::vector<uint8_t> AsVector(const uint8_t* data, size_t size) {
+ std::vector<uint8_t> out;
+ out.assign(data, data + size);
+ return out;
+}
+
+void AssertFramesEqual(rtc::scoped_refptr<webrtc::I420BufferInterface> actual,
+ rtc::scoped_refptr<VideoFrameBuffer> expected) {
+ ASSERT_THAT(actual->width(), Eq(expected->width()));
+ ASSERT_THAT(actual->height(), Eq(expected->height()));
+ rtc::scoped_refptr<webrtc::I420BufferInterface> expected_i420 =
+ expected->ToI420();
+
+ int height = actual->height();
+
+ EXPECT_THAT(AsVector(actual->DataY(), actual->StrideY() * height),
+ ElementsAreArray(expected_i420->DataY(),
+ expected_i420->StrideY() * height));
+ EXPECT_THAT(AsVector(actual->DataU(), actual->StrideU() * (height + 1) / 2),
+ ElementsAreArray(expected_i420->DataU(),
+ expected_i420->StrideU() * (height + 1) / 2));
+ EXPECT_THAT(AsVector(actual->DataV(), actual->StrideV() * (height + 1) / 2),
+ ElementsAreArray(expected_i420->DataV(),
+ expected_i420->StrideV() * (height + 1) / 2));
+}
+
+void AssertFrameIdsAre(const std::string& filename,
+ std::vector<std::string> expected_ids) {
+ FILE* file = fopen(filename.c_str(), "r");
+ ASSERT_TRUE(file != nullptr);
+ std::vector<std::string> actual_ids;
+ char buffer[8];
+ while (fgets(buffer, sizeof buffer, file) != nullptr) {
+ std::string current_id(buffer);
+ ASSERT_GE(current_id.size(), 2lu);
+ // Trim "\n" at the end.
+ actual_ids.push_back(current_id.substr(0, current_id.size() - 1));
+ }
+ EXPECT_THAT(actual_ids, ElementsAreArray(expected_ids));
+}
+
+class VideoDumpingTest : public Test {
+ protected:
+ ~VideoDumpingTest() override = default;
+
+ void SetUp() override {
+ video_filename_ = webrtc::test::TempFilename(webrtc::test::OutputPath(),
+ "video_dumping_test");
+ ids_filename_ = webrtc::test::TempFilename(webrtc::test::OutputPath(),
+ "video_dumping_test");
+ }
+
+ void TearDown() override {
+ remove(video_filename_.c_str());
+ remove(ids_filename_.c_str());
+ }
+
+ std::string video_filename_;
+ std::string ids_filename_;
+};
+
+using CreateVideoFrameWithIdsWriterTest = VideoDumpingTest;
+
+TEST_F(CreateVideoFrameWithIdsWriterTest, VideoIsWritenWithFrameIds) {
+ Random random(/*seed=*/100);
+ VideoFrame frame1 = CreateRandom2x2VideoFrame(1, random);
+ VideoFrame frame2 = CreateRandom2x2VideoFrame(2, random);
+
+ std::unique_ptr<test::VideoFrameWriter> writer =
+ CreateVideoFrameWithIdsWriter(
+ std::make_unique<test::Y4mVideoFrameWriterImpl>(
+ std::string(video_filename_),
+ /*width=*/2, /*height=*/2, /*fps=*/2),
+ ids_filename_);
+
+ ASSERT_TRUE(writer->WriteFrame(frame1));
+ ASSERT_TRUE(writer->WriteFrame(frame2));
+ writer->Close();
+
+ auto frame_reader = test::CreateY4mFrameReader(video_filename_);
+ EXPECT_THAT(frame_reader->num_frames(), Eq(2));
+ AssertFramesEqual(frame_reader->PullFrame(), frame1.video_frame_buffer());
+ AssertFramesEqual(frame_reader->PullFrame(), frame2.video_frame_buffer());
+ AssertFrameIdsAre(ids_filename_, {"1", "2"});
+}
+
+using VideoWriterTest = VideoDumpingTest;
+
+TEST_F(VideoWriterTest, AllFramesAreWrittenWithSamplingModulo1) {
+ Random random(/*seed=*/100);
+ VideoFrame frame1 = CreateRandom2x2VideoFrame(1, random);
+ VideoFrame frame2 = CreateRandom2x2VideoFrame(2, random);
+
+ {
+ test::Y4mVideoFrameWriterImpl frame_writer(std::string(video_filename_),
+ /*width=*/2, /*height=*/2,
+ /*fps=*/2);
+ VideoWriter writer(&frame_writer, /*sampling_modulo=*/1);
+
+ writer.OnFrame(frame1);
+ writer.OnFrame(frame2);
+ frame_writer.Close();
+ }
+
+ auto frame_reader = test::CreateY4mFrameReader(video_filename_);
+ EXPECT_THAT(frame_reader->num_frames(), Eq(2));
+ AssertFramesEqual(frame_reader->PullFrame(), frame1.video_frame_buffer());
+ AssertFramesEqual(frame_reader->PullFrame(), frame2.video_frame_buffer());
+}
+
+TEST_F(VideoWriterTest, OnlyEvery2ndFramesIsWrittenWithSamplingModulo2) {
+ Random random(/*seed=*/100);
+ VideoFrame frame1 = CreateRandom2x2VideoFrame(1, random);
+ VideoFrame frame2 = CreateRandom2x2VideoFrame(2, random);
+ VideoFrame frame3 = CreateRandom2x2VideoFrame(3, random);
+
+ {
+ test::Y4mVideoFrameWriterImpl frame_writer(std::string(video_filename_),
+ /*width=*/2, /*height=*/2,
+ /*fps=*/2);
+ VideoWriter writer(&frame_writer, /*sampling_modulo=*/2);
+
+ writer.OnFrame(frame1);
+ writer.OnFrame(frame2);
+ writer.OnFrame(frame3);
+ frame_writer.Close();
+ }
+
+ auto frame_reader = test::CreateY4mFrameReader(video_filename_);
+ EXPECT_THAT(frame_reader->num_frames(), Eq(2));
+ AssertFramesEqual(frame_reader->PullFrame(), frame1.video_frame_buffer());
+ AssertFramesEqual(frame_reader->PullFrame(), frame3.video_frame_buffer());
+}
+
+} // namespace
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.cc
new file mode 100644
index 0000000000..5a74d60250
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.cc
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h"
+
+#include "absl/memory/memory.h"
+#include "api/video/encoded_image.h"
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+EncodedImage VideoFrameTrackingIdInjector::InjectData(
+ uint16_t id,
+ bool unused_discard,
+ const EncodedImage& source) {
+ RTC_CHECK(!unused_discard);
+ EncodedImage out = source;
+ out.SetVideoFrameTrackingId(id);
+ return out;
+}
+
+EncodedImageExtractionResult VideoFrameTrackingIdInjector::ExtractData(
+ const EncodedImage& source) {
+ return EncodedImageExtractionResult{source.VideoFrameTrackingId(), source,
+ /*discard=*/false};
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h
new file mode 100644
index 0000000000..ecc3cd3f51
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_FRAME_TRACKING_ID_INJECTOR_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_FRAME_TRACKING_ID_INJECTOR_H_
+
+#include <cstdint>
+
+#include "api/video/encoded_image.h"
+#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+// This injector sets and retrieves the provided id in the EncodedImage
+// video_frame_tracking_id field. This is only possible with the RTP header
+// extension VideoFrameTrackingIdExtension that will propagate the input
+// tracking id to the received EncodedImage. This RTP header extension is
+// enabled with the field trial WebRTC-VideoFrameTrackingIdAdvertised
+// (http://www.webrtc.org/experiments/rtp-hdrext/video-frame-tracking-id).
+//
+// Note that this injector doesn't allow to discard frames.
+class VideoFrameTrackingIdInjector : public EncodedImageDataPropagator {
+ public:
+ EncodedImage InjectData(uint16_t id,
+ bool unused_discard,
+ const EncodedImage& source) override;
+
+ EncodedImageExtractionResult ExtractData(const EncodedImage& source) override;
+
+ void Start(int) override {}
+ void AddParticipantInCall() override {}
+ void RemoveParticipantInCall() override {}
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_FRAME_TRACKING_ID_INJECTOR_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector_unittest.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector_unittest.cc
new file mode 100644
index 0000000000..c7d453c4bb
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_frame_tracking_id_injector_unittest.cc
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h"
+
+#include "api/video/encoded_image.h"
+#include "rtc_base/buffer.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+EncodedImage CreateEncodedImageOfSizeN(size_t n) {
+ EncodedImage image;
+ rtc::scoped_refptr<EncodedImageBuffer> buffer = EncodedImageBuffer::Create(n);
+ for (size_t i = 0; i < n; ++i) {
+ buffer->data()[i] = static_cast<uint8_t>(i);
+ }
+ image.SetEncodedData(buffer);
+ return image;
+}
+
+TEST(VideoFrameTrackingIdInjectorTest, InjectExtractDiscardFalse) {
+ VideoFrameTrackingIdInjector injector;
+ EncodedImage source = CreateEncodedImageOfSizeN(10);
+ EncodedImageExtractionResult out =
+ injector.ExtractData(injector.InjectData(512, false, source));
+
+ ASSERT_TRUE(out.id.has_value());
+ EXPECT_EQ(*out.id, 512);
+ EXPECT_FALSE(out.discard);
+ EXPECT_EQ(out.image.size(), 10ul);
+ for (int i = 0; i < 10; ++i) {
+ EXPECT_EQ(source.data()[i], out.image.data()[i]);
+ }
+}
+
+#if GTEST_HAS_DEATH_TEST
+TEST(VideoFrameTrackingIdInjectorTest, InjectExtractDiscardTrue) {
+ VideoFrameTrackingIdInjector injector;
+ EncodedImage source = CreateEncodedImageOfSizeN(10);
+
+ EXPECT_DEATH(injector.InjectData(512, true, source), "");
+}
+#endif // GTEST_HAS_DEATH_TEST
+
+} // namespace
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc
new file mode 100644
index 0000000000..87c11886cc
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.cc
@@ -0,0 +1,264 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h"
+
+#include <stdio.h>
+
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "absl/strings/string_view.h"
+#include "api/array_view.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/video/i420_buffer.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/strings/string_builder.h"
+#include "system_wrappers/include/clock.h"
+#include "test/pc/e2e/analyzer/video/analyzing_video_sink.h"
+#include "test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h"
+#include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h"
+#include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h"
+#include "test/pc/e2e/analyzer/video/video_dumping.h"
+#include "test/testsupport/fixed_fps_video_frame_writer_adapter.h"
+#include "test/video_renderer.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+using webrtc::webrtc_pc_e2e::VideoConfig;
+using EmulatedSFUConfigMap =
+ ::webrtc::webrtc_pc_e2e::QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap;
+
+class AnalyzingFramePreprocessor
+ : public test::TestVideoCapturer::FramePreprocessor {
+ public:
+ AnalyzingFramePreprocessor(
+ absl::string_view peer_name,
+ absl::string_view stream_label,
+ VideoQualityAnalyzerInterface* analyzer,
+ std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> sinks)
+ : peer_name_(peer_name),
+ stream_label_(stream_label),
+ analyzer_(analyzer),
+ sinks_(std::move(sinks)) {}
+ ~AnalyzingFramePreprocessor() override = default;
+
+ VideoFrame Preprocess(const VideoFrame& source_frame) override {
+ // Copy VideoFrame to be able to set id on it.
+ VideoFrame frame = source_frame;
+ uint16_t frame_id =
+ analyzer_->OnFrameCaptured(peer_name_, stream_label_, frame);
+ frame.set_id(frame_id);
+
+ for (auto& sink : sinks_) {
+ sink->OnFrame(frame);
+ }
+ return frame;
+ }
+
+ private:
+ const std::string peer_name_;
+ const std::string stream_label_;
+ VideoQualityAnalyzerInterface* const analyzer_;
+ const std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>
+ sinks_;
+};
+
+} // namespace
+
+VideoQualityAnalyzerInjectionHelper::VideoQualityAnalyzerInjectionHelper(
+ Clock* clock,
+ std::unique_ptr<VideoQualityAnalyzerInterface> analyzer,
+ EncodedImageDataInjector* injector,
+ EncodedImageDataExtractor* extractor)
+ : clock_(clock),
+ analyzer_(std::move(analyzer)),
+ injector_(injector),
+ extractor_(extractor) {
+ RTC_DCHECK(clock_);
+ RTC_DCHECK(injector_);
+ RTC_DCHECK(extractor_);
+}
+VideoQualityAnalyzerInjectionHelper::~VideoQualityAnalyzerInjectionHelper() =
+ default;
+
+std::unique_ptr<VideoEncoderFactory>
+VideoQualityAnalyzerInjectionHelper::WrapVideoEncoderFactory(
+ absl::string_view peer_name,
+ std::unique_ptr<VideoEncoderFactory> delegate,
+ double bitrate_multiplier,
+ EmulatedSFUConfigMap stream_to_sfu_config) const {
+ return std::make_unique<QualityAnalyzingVideoEncoderFactory>(
+ peer_name, std::move(delegate), bitrate_multiplier,
+ std::move(stream_to_sfu_config), injector_, analyzer_.get());
+}
+
+std::unique_ptr<VideoDecoderFactory>
+VideoQualityAnalyzerInjectionHelper::WrapVideoDecoderFactory(
+ absl::string_view peer_name,
+ std::unique_ptr<VideoDecoderFactory> delegate) const {
+ return std::make_unique<QualityAnalyzingVideoDecoderFactory>(
+ peer_name, std::move(delegate), extractor_, analyzer_.get());
+}
+
+std::unique_ptr<test::TestVideoCapturer::FramePreprocessor>
+VideoQualityAnalyzerInjectionHelper::CreateFramePreprocessor(
+ absl::string_view peer_name,
+ const VideoConfig& config) {
+ std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> sinks;
+ if (config.input_dump_options.has_value()) {
+ std::unique_ptr<test::VideoFrameWriter> writer =
+ config.input_dump_options->CreateInputDumpVideoFrameWriter(
+ *config.stream_label, config.GetResolution());
+ sinks.push_back(std::make_unique<VideoWriter>(
+ writer.get(), config.input_dump_options->sampling_modulo()));
+ video_writers_.push_back(std::move(writer));
+ }
+ if (config.show_on_screen) {
+ sinks.push_back(absl::WrapUnique(
+ test::VideoRenderer::Create((*config.stream_label + "-capture").c_str(),
+ config.width, config.height)));
+ }
+ sinks_helper_.AddConfig(peer_name, config);
+ {
+ MutexLock lock(&mutex_);
+ known_video_configs_.insert({*config.stream_label, config});
+ }
+ return std::make_unique<AnalyzingFramePreprocessor>(
+ peer_name, std::move(*config.stream_label), analyzer_.get(),
+ std::move(sinks));
+}
+
+std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>
+VideoQualityAnalyzerInjectionHelper::CreateVideoSink(
+ absl::string_view peer_name) {
+ return std::make_unique<AnalyzingVideoSink2>(peer_name, this);
+}
+
+std::unique_ptr<AnalyzingVideoSink>
+VideoQualityAnalyzerInjectionHelper::CreateVideoSink(
+ absl::string_view peer_name,
+ const VideoSubscription& subscription,
+ bool report_infra_metrics) {
+ return std::make_unique<AnalyzingVideoSink>(peer_name, clock_, *analyzer_,
+ sinks_helper_, subscription,
+ report_infra_metrics);
+}
+
+void VideoQualityAnalyzerInjectionHelper::Start(
+ std::string test_case_name,
+ rtc::ArrayView<const std::string> peer_names,
+ int max_threads_count) {
+ analyzer_->Start(std::move(test_case_name), peer_names, max_threads_count);
+ extractor_->Start(peer_names.size());
+ MutexLock lock(&mutex_);
+ peers_count_ = peer_names.size();
+}
+
+void VideoQualityAnalyzerInjectionHelper::RegisterParticipantInCall(
+ absl::string_view peer_name) {
+ analyzer_->RegisterParticipantInCall(peer_name);
+ extractor_->AddParticipantInCall();
+ MutexLock lock(&mutex_);
+ peers_count_++;
+}
+
+void VideoQualityAnalyzerInjectionHelper::UnregisterParticipantInCall(
+ absl::string_view peer_name) {
+ analyzer_->UnregisterParticipantInCall(peer_name);
+ extractor_->RemoveParticipantInCall();
+ MutexLock lock(&mutex_);
+ peers_count_--;
+}
+
+void VideoQualityAnalyzerInjectionHelper::OnStatsReports(
+ absl::string_view pc_label,
+ const rtc::scoped_refptr<const RTCStatsReport>& report) {
+ analyzer_->OnStatsReports(pc_label, report);
+}
+
+void VideoQualityAnalyzerInjectionHelper::Stop() {
+ analyzer_->Stop();
+ for (const auto& video_writer : video_writers_) {
+ video_writer->Close();
+ }
+ video_writers_.clear();
+ sinks_helper_.Clear();
+}
+
+void VideoQualityAnalyzerInjectionHelper::OnFrame(absl::string_view peer_name,
+ const VideoFrame& frame) {
+ if (IsDummyFrame(frame)) {
+ // This is dummy frame, so we don't need to process it further.
+ return;
+ }
+ // Copy entire video frame including video buffer to ensure that analyzer
+ // won't hold any WebRTC internal buffers.
+ VideoFrame frame_copy = frame;
+ frame_copy.set_video_frame_buffer(
+ I420Buffer::Copy(*frame.video_frame_buffer()->ToI420()));
+ analyzer_->OnFrameRendered(peer_name, frame_copy);
+
+ if (frame.id() != VideoFrame::kNotSetId) {
+ std::string stream_label = analyzer_->GetStreamLabel(frame.id());
+ std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>* sinks =
+ PopulateSinks(ReceiverStream(peer_name, stream_label));
+ if (sinks == nullptr) {
+ return;
+ }
+ for (auto& sink : *sinks) {
+ sink->OnFrame(frame);
+ }
+ }
+}
+
+std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>*
+VideoQualityAnalyzerInjectionHelper::PopulateSinks(
+ const ReceiverStream& receiver_stream) {
+ MutexLock lock(&mutex_);
+ auto sinks_it = sinks_.find(receiver_stream);
+ if (sinks_it != sinks_.end()) {
+ return &sinks_it->second;
+ }
+ auto it = known_video_configs_.find(receiver_stream.stream_label);
+ RTC_DCHECK(it != known_video_configs_.end())
+ << "No video config for stream " << receiver_stream.stream_label;
+ const VideoConfig& config = it->second;
+
+ std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> sinks;
+ if (config.output_dump_options.has_value()) {
+ std::unique_ptr<test::VideoFrameWriter> writer =
+ config.output_dump_options->CreateOutputDumpVideoFrameWriter(
+ receiver_stream.stream_label, receiver_stream.peer_name,
+ config.GetResolution());
+ if (config.output_dump_use_fixed_framerate) {
+ writer = std::make_unique<test::FixedFpsVideoFrameWriterAdapter>(
+ config.fps, clock_, std::move(writer));
+ }
+ sinks.push_back(std::make_unique<VideoWriter>(
+ writer.get(), config.output_dump_options->sampling_modulo()));
+ video_writers_.push_back(std::move(writer));
+ }
+ if (config.show_on_screen) {
+ sinks.push_back(absl::WrapUnique(
+ test::VideoRenderer::Create((*config.stream_label + "-render").c_str(),
+ config.width, config.height)));
+ }
+ sinks_.insert({receiver_stream, std::move(sinks)});
+ return &(sinks_.find(receiver_stream)->second);
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h
new file mode 100644
index 0000000000..7421c8e4a7
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h
@@ -0,0 +1,170 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_ANALYZER_INJECTION_HELPER_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_ANALYZER_INJECTION_HELPER_H_
+
+#include <stdio.h>
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/strings/string_view.h"
+#include "api/array_view.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/stats_observer_interface.h"
+#include "api/test/video_quality_analyzer_interface.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_sink_interface.h"
+#include "api/video_codecs/video_decoder_factory.h"
+#include "api/video_codecs/video_encoder_factory.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "system_wrappers/include/clock.h"
+#include "test/pc/e2e/analyzer/video/analyzing_video_sink.h"
+#include "test/pc/e2e/analyzer/video/analyzing_video_sinks_helper.h"
+#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h"
+#include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h"
+#include "test/test_video_capturer.h"
+#include "test/testsupport/video_frame_writer.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+// Provides factory methods for components, that will be used to inject
+// VideoQualityAnalyzerInterface into PeerConnection pipeline.
+class VideoQualityAnalyzerInjectionHelper : public StatsObserverInterface {
+ public:
+ VideoQualityAnalyzerInjectionHelper(
+ Clock* clock,
+ std::unique_ptr<VideoQualityAnalyzerInterface> analyzer,
+ EncodedImageDataInjector* injector,
+ EncodedImageDataExtractor* extractor);
+ ~VideoQualityAnalyzerInjectionHelper() override;
+
+ // Wraps video encoder factory to give video quality analyzer access to frames
+ // before encoding and encoded images after.
+ std::unique_ptr<VideoEncoderFactory> WrapVideoEncoderFactory(
+ absl::string_view peer_name,
+ std::unique_ptr<VideoEncoderFactory> delegate,
+ double bitrate_multiplier,
+ QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap stream_to_sfu_config)
+ const;
+ // Wraps video decoder factory to give video quality analyzer access to
+ // received encoded images and frames, that were decoded from them.
+ std::unique_ptr<VideoDecoderFactory> WrapVideoDecoderFactory(
+ absl::string_view peer_name,
+ std::unique_ptr<VideoDecoderFactory> delegate) const;
+
+ // Creates VideoFrame preprocessor, that will allow video quality analyzer to
+ // get access to the captured frames. If provided config also specifies
+ // `input_dump_file_name`, video will be written into that file.
+ std::unique_ptr<test::TestVideoCapturer::FramePreprocessor>
+ CreateFramePreprocessor(absl::string_view peer_name,
+ const webrtc::webrtc_pc_e2e::VideoConfig& config);
+ // Creates sink, that will allow video quality analyzer to get access to
+ // the rendered frames. If corresponding video track has
+ // `output_dump_file_name` in its VideoConfig, which was used for
+ // CreateFramePreprocessor(...), then video also will be written
+ // into that file.
+ // TODO(titovartem): Remove method with `peer_name` only parameter.
+ std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> CreateVideoSink(
+ absl::string_view peer_name);
+ std::unique_ptr<AnalyzingVideoSink> CreateVideoSink(
+ absl::string_view peer_name,
+ const VideoSubscription& subscription,
+ bool report_infra_metrics);
+
+ void Start(std::string test_case_name,
+ rtc::ArrayView<const std::string> peer_names,
+ int max_threads_count = 1);
+
+ // Registers new call participant to the underlying video quality analyzer.
+ // The method should be called before the participant is actually added.
+ void RegisterParticipantInCall(absl::string_view peer_name);
+
+ // Will be called after test removed existing participant in the middle of the
+ // call.
+ void UnregisterParticipantInCall(absl::string_view peer_name);
+
+ // Forwards `stats_reports` for Peer Connection `pc_label` to
+ // `analyzer_`.
+ void OnStatsReports(
+ absl::string_view pc_label,
+ const rtc::scoped_refptr<const RTCStatsReport>& report) override;
+
+ // Stops VideoQualityAnalyzerInterface to populate final data and metrics.
+ // Should be invoked after analyzed video tracks are disposed.
+ void Stop();
+
+ private:
+ // Deprecated, to be removed when old API isn't used anymore.
+ class AnalyzingVideoSink2 final : public rtc::VideoSinkInterface<VideoFrame> {
+ public:
+ explicit AnalyzingVideoSink2(absl::string_view peer_name,
+ VideoQualityAnalyzerInjectionHelper* helper)
+ : peer_name_(peer_name), helper_(helper) {}
+ ~AnalyzingVideoSink2() override = default;
+
+ void OnFrame(const VideoFrame& frame) override {
+ helper_->OnFrame(peer_name_, frame);
+ }
+
+ private:
+ const std::string peer_name_;
+ VideoQualityAnalyzerInjectionHelper* const helper_;
+ };
+
+ struct ReceiverStream {
+ ReceiverStream(absl::string_view peer_name, absl::string_view stream_label)
+ : peer_name(peer_name), stream_label(stream_label) {}
+
+ std::string peer_name;
+ std::string stream_label;
+
+ // Define operators required to use ReceiverStream as std::map key.
+ bool operator==(const ReceiverStream& o) const {
+ return peer_name == o.peer_name && stream_label == o.stream_label;
+ }
+ bool operator<(const ReceiverStream& o) const {
+ return (peer_name == o.peer_name) ? stream_label < o.stream_label
+ : peer_name < o.peer_name;
+ }
+ };
+
+ // Creates a deep copy of the frame and passes it to the video analyzer, while
+ // passing real frame to the sinks
+ void OnFrame(absl::string_view peer_name, const VideoFrame& frame);
+ std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>*
+ PopulateSinks(const ReceiverStream& receiver_stream);
+
+ Clock* const clock_;
+ std::unique_ptr<VideoQualityAnalyzerInterface> analyzer_;
+ EncodedImageDataInjector* injector_;
+ EncodedImageDataExtractor* extractor_;
+
+ std::vector<std::unique_ptr<test::VideoFrameWriter>> video_writers_;
+
+ AnalyzingVideoSinksHelper sinks_helper_;
+ Mutex mutex_;
+ int peers_count_ RTC_GUARDED_BY(mutex_);
+ // Map from stream label to the video config.
+ std::map<std::string, webrtc::webrtc_pc_e2e::VideoConfig> known_video_configs_
+ RTC_GUARDED_BY(mutex_);
+ std::map<ReceiverStream,
+ std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>>
+ sinks_ RTC_GUARDED_BY(mutex_);
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_ANALYZER_INJECTION_HELPER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc
new file mode 100644
index 0000000000..8049af308e
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc
@@ -0,0 +1,162 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h"
+
+#include <map>
+#include <string>
+
+#include "api/stats/rtc_stats.h"
+#include "api/stats/rtcstats_objects.h"
+#include "api/test/metrics/metric.h"
+#include "api/units/data_rate.h"
+#include "api/units/time_delta.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/checks.h"
+#include "test/pc/e2e/metric_metadata_keys.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+using ::webrtc::test::ImprovementDirection;
+using ::webrtc::test::Unit;
+using ::webrtc::webrtc_pc_e2e::MetricMetadataKey;
+
+SamplesStatsCounter BytesPerSecondToKbps(const SamplesStatsCounter& counter) {
+ return counter * 0.008;
+}
+
+} // namespace
+
+VideoQualityMetricsReporter::VideoQualityMetricsReporter(
+ Clock* const clock,
+ test::MetricsLogger* const metrics_logger)
+ : clock_(clock), metrics_logger_(metrics_logger) {
+ RTC_CHECK(metrics_logger_);
+}
+
+void VideoQualityMetricsReporter::Start(
+ absl::string_view test_case_name,
+ const TrackIdStreamInfoMap* /*reporter_helper*/) {
+ test_case_name_ = std::string(test_case_name);
+ start_time_ = Now();
+}
+
+void VideoQualityMetricsReporter::OnStatsReports(
+ absl::string_view pc_label,
+ const rtc::scoped_refptr<const RTCStatsReport>& report) {
+ RTC_CHECK(start_time_)
+ << "Please invoke Start(...) method before calling OnStatsReports(...)";
+
+ auto transport_stats = report->GetStatsOfType<RTCTransportStats>();
+ if (transport_stats.size() == 0u ||
+ !transport_stats[0]->selected_candidate_pair_id.is_defined()) {
+ return;
+ }
+ RTC_DCHECK_EQ(transport_stats.size(), 1);
+ std::string selected_ice_id =
+ transport_stats[0]->selected_candidate_pair_id.ValueToString();
+ // Use the selected ICE candidate pair ID to get the appropriate ICE stats.
+ const RTCIceCandidatePairStats ice_candidate_pair_stats =
+ report->Get(selected_ice_id)->cast_to<const RTCIceCandidatePairStats>();
+
+ auto outbound_rtp_stats = report->GetStatsOfType<RTCOutboundRTPStreamStats>();
+ StatsSample sample;
+ for (auto& s : outbound_rtp_stats) {
+ if (!s->kind.is_defined()) {
+ continue;
+ }
+ if (!(*s->kind == RTCMediaStreamTrackKind::kVideo)) {
+ continue;
+ }
+ if (s->timestamp() > sample.sample_time) {
+ sample.sample_time = s->timestamp();
+ }
+ sample.retransmitted_bytes_sent +=
+ DataSize::Bytes(s->retransmitted_bytes_sent.ValueOrDefault(0ul));
+ sample.bytes_sent += DataSize::Bytes(s->bytes_sent.ValueOrDefault(0ul));
+ sample.header_bytes_sent +=
+ DataSize::Bytes(s->header_bytes_sent.ValueOrDefault(0ul));
+ }
+
+ MutexLock lock(&video_bwe_stats_lock_);
+ VideoBweStats& video_bwe_stats = video_bwe_stats_[std::string(pc_label)];
+ if (ice_candidate_pair_stats.available_outgoing_bitrate.is_defined()) {
+ video_bwe_stats.available_send_bandwidth.AddSample(
+ DataRate::BitsPerSec(
+ *ice_candidate_pair_stats.available_outgoing_bitrate)
+ .bytes_per_sec());
+ }
+
+ StatsSample prev_sample = last_stats_sample_[std::string(pc_label)];
+ if (prev_sample.sample_time.IsZero()) {
+ prev_sample.sample_time = start_time_.value();
+ }
+ last_stats_sample_[std::string(pc_label)] = sample;
+
+ TimeDelta time_between_samples = sample.sample_time - prev_sample.sample_time;
+ if (time_between_samples.IsZero()) {
+ return;
+ }
+
+ DataRate retransmission_bitrate =
+ (sample.retransmitted_bytes_sent - prev_sample.retransmitted_bytes_sent) /
+ time_between_samples;
+ video_bwe_stats.retransmission_bitrate.AddSample(
+ retransmission_bitrate.bytes_per_sec());
+ DataRate transmission_bitrate =
+ (sample.bytes_sent + sample.header_bytes_sent - prev_sample.bytes_sent -
+ prev_sample.header_bytes_sent) /
+ time_between_samples;
+ video_bwe_stats.transmission_bitrate.AddSample(
+ transmission_bitrate.bytes_per_sec());
+}
+
+void VideoQualityMetricsReporter::StopAndReportResults() {
+ MutexLock video_bwemutex_(&video_bwe_stats_lock_);
+ for (const auto& item : video_bwe_stats_) {
+ ReportVideoBweResults(item.first, item.second);
+ }
+}
+
+std::string VideoQualityMetricsReporter::GetTestCaseName(
+ const std::string& peer_name) const {
+ return test_case_name_ + "/" + peer_name;
+}
+
+void VideoQualityMetricsReporter::ReportVideoBweResults(
+ const std::string& peer_name,
+ const VideoBweStats& video_bwe_stats) {
+ std::string test_case_name = GetTestCaseName(peer_name);
+ // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey.
+ std::map<std::string, std::string> metric_metadata{
+ {MetricMetadataKey::kPeerMetadataKey, peer_name},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_case_name_}};
+
+ metrics_logger_->LogMetric(
+ "available_send_bandwidth", test_case_name,
+ BytesPerSecondToKbps(video_bwe_stats.available_send_bandwidth),
+ Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter,
+ metric_metadata);
+ metrics_logger_->LogMetric(
+ "transmission_bitrate", test_case_name,
+ BytesPerSecondToKbps(video_bwe_stats.transmission_bitrate),
+ Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter,
+ metric_metadata);
+ metrics_logger_->LogMetric(
+ "retransmission_bitrate", test_case_name,
+ BytesPerSecondToKbps(video_bwe_stats.retransmission_bitrate),
+ Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter,
+ metric_metadata);
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h
new file mode 100644
index 0000000000..d3d976343b
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_METRICS_REPORTER_H_
+#define TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_METRICS_REPORTER_H_
+
+#include <map>
+#include <string>
+
+#include "absl/strings/string_view.h"
+#include "api/numerics/samples_stats_counter.h"
+#include "api/test/metrics/metrics_logger.h"
+#include "api/test/peerconnection_quality_test_fixture.h"
+#include "api/test/track_id_stream_info_map.h"
+#include "api/units/data_size.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/synchronization/mutex.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+struct VideoBweStats {
+ SamplesStatsCounter available_send_bandwidth;
+ SamplesStatsCounter transmission_bitrate;
+ SamplesStatsCounter retransmission_bitrate;
+};
+
+class VideoQualityMetricsReporter
+ : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter {
+ public:
+ VideoQualityMetricsReporter(Clock* const clock,
+ test::MetricsLogger* const metrics_logger);
+ ~VideoQualityMetricsReporter() override = default;
+
+ void Start(absl::string_view test_case_name,
+ const TrackIdStreamInfoMap* reporter_helper) override;
+ void OnStatsReports(
+ absl::string_view pc_label,
+ const rtc::scoped_refptr<const RTCStatsReport>& report) override;
+ void StopAndReportResults() override;
+
+ private:
+ struct StatsSample {
+ DataSize bytes_sent = DataSize::Zero();
+ DataSize header_bytes_sent = DataSize::Zero();
+ DataSize retransmitted_bytes_sent = DataSize::Zero();
+
+ Timestamp sample_time = Timestamp::Zero();
+ };
+
+ std::string GetTestCaseName(const std::string& peer_name) const;
+ void ReportVideoBweResults(const std::string& peer_name,
+ const VideoBweStats& video_bwe_stats);
+ Timestamp Now() const { return clock_->CurrentTime(); }
+
+ Clock* const clock_;
+ test::MetricsLogger* const metrics_logger_;
+
+ std::string test_case_name_;
+ absl::optional<Timestamp> start_time_;
+
+ Mutex video_bwe_stats_lock_;
+ // Map between a peer connection label (provided by the framework) and
+ // its video BWE stats.
+ std::map<std::string, VideoBweStats> video_bwe_stats_
+ RTC_GUARDED_BY(video_bwe_stats_lock_);
+ std::map<std::string, StatsSample> last_stats_sample_
+ RTC_GUARDED_BY(video_bwe_stats_lock_);
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_VIDEO_VIDEO_QUALITY_METRICS_REPORTER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer_helper.cc b/third_party/libwebrtc/test/pc/e2e/analyzer_helper.cc
new file mode 100644
index 0000000000..76cd9a7c78
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer_helper.cc
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/analyzer_helper.h"
+
+#include <string>
+#include <utility>
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+AnalyzerHelper::AnalyzerHelper() {
+ signaling_sequence_checker_.Detach();
+}
+
+void AnalyzerHelper::AddTrackToStreamMapping(
+ absl::string_view track_id,
+ absl::string_view receiver_peer,
+ absl::string_view stream_label,
+ absl::optional<std::string> sync_group) {
+ RTC_DCHECK_RUN_ON(&signaling_sequence_checker_);
+ track_to_stream_map_.insert(
+ {std::string(track_id),
+ StreamInfo{.receiver_peer = std::string(receiver_peer),
+ .stream_label = std::string(stream_label),
+ .sync_group = sync_group.has_value()
+ ? *sync_group
+ : std::string(stream_label)}});
+}
+
+void AnalyzerHelper::AddTrackToStreamMapping(std::string track_id,
+ std::string stream_label) {
+ RTC_DCHECK_RUN_ON(&signaling_sequence_checker_);
+ track_to_stream_map_.insert(
+ {std::move(track_id), StreamInfo{stream_label, stream_label}});
+}
+
+void AnalyzerHelper::AddTrackToStreamMapping(std::string track_id,
+ std::string stream_label,
+ std::string sync_group) {
+ RTC_DCHECK_RUN_ON(&signaling_sequence_checker_);
+ track_to_stream_map_.insert(
+ {std::move(track_id),
+ StreamInfo{std::move(stream_label), std::move(sync_group)}});
+}
+
+AnalyzerHelper::StreamInfo AnalyzerHelper::GetStreamInfoFromTrackId(
+ absl::string_view track_id) const {
+ RTC_DCHECK_RUN_ON(&signaling_sequence_checker_);
+ auto track_to_stream_pair = track_to_stream_map_.find(std::string(track_id));
+ RTC_CHECK(track_to_stream_pair != track_to_stream_map_.end());
+ return track_to_stream_pair->second;
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/analyzer_helper.h b/third_party/libwebrtc/test/pc/e2e/analyzer_helper.h
new file mode 100644
index 0000000000..d0b47c4fb9
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/analyzer_helper.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ANALYZER_HELPER_H_
+#define TEST_PC_E2E_ANALYZER_HELPER_H_
+
+#include <map>
+#include <string>
+
+#include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
+#include "api/sequence_checker.h"
+#include "api/test/track_id_stream_info_map.h"
+#include "rtc_base/thread_annotations.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+// This class is a utility that provides bookkeeping capabilities that
+// are useful to associate stats reports track_ids to the remote stream info.
+// The framework will populate an instance of this class and it will pass
+// it to the Start method of Media Quality Analyzers.
+// An instance of AnalyzerHelper must only be accessed from a single
+// thread and since stats collection happens on the signaling thread,
+// AddTrackToStreamMapping, GetStreamLabelFromTrackId and
+// GetSyncGroupLabelFromTrackId must be invoked from the signaling thread. Get
+// methods should be invoked only after all data is added. Mixing Get methods
+// with adding new data may lead to undefined behavior.
+class AnalyzerHelper : public TrackIdStreamInfoMap {
+ public:
+ AnalyzerHelper();
+
+ void AddTrackToStreamMapping(absl::string_view track_id,
+ absl::string_view receiver_peer,
+ absl::string_view stream_label,
+ absl::optional<std::string> sync_group);
+ void AddTrackToStreamMapping(std::string track_id, std::string stream_label);
+ void AddTrackToStreamMapping(std::string track_id,
+ std::string stream_label,
+ std::string sync_group);
+
+ StreamInfo GetStreamInfoFromTrackId(
+ absl::string_view track_id) const override;
+
+ private:
+ SequenceChecker signaling_sequence_checker_;
+ std::map<std::string, StreamInfo> track_to_stream_map_
+ RTC_GUARDED_BY(signaling_sequence_checker_);
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ANALYZER_HELPER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.cc b/third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.cc
new file mode 100644
index 0000000000..0d4fe7478d
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.cc
@@ -0,0 +1,151 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/cross_media_metrics_reporter.h"
+
+#include <utility>
+#include <vector>
+
+#include "api/stats/rtc_stats.h"
+#include "api/stats/rtcstats_objects.h"
+#include "api/test/metrics/metric.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/event.h"
+#include "system_wrappers/include/field_trial.h"
+#include "test/pc/e2e/metric_metadata_keys.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+using ::webrtc::test::ImprovementDirection;
+using ::webrtc::test::Unit;
+
+CrossMediaMetricsReporter::CrossMediaMetricsReporter(
+ test::MetricsLogger* metrics_logger)
+ : metrics_logger_(metrics_logger) {
+ RTC_CHECK(metrics_logger_);
+}
+
+void CrossMediaMetricsReporter::Start(
+ absl::string_view test_case_name,
+ const TrackIdStreamInfoMap* reporter_helper) {
+ test_case_name_ = std::string(test_case_name);
+ reporter_helper_ = reporter_helper;
+}
+
+void CrossMediaMetricsReporter::OnStatsReports(
+ absl::string_view pc_label,
+ const rtc::scoped_refptr<const RTCStatsReport>& report) {
+ auto inbound_stats = report->GetStatsOfType<RTCInboundRTPStreamStats>();
+ std::map<std::string, std::vector<const RTCInboundRTPStreamStats*>>
+ sync_group_stats;
+ for (const auto& stat : inbound_stats) {
+ if (stat->estimated_playout_timestamp.ValueOrDefault(0.) > 0 &&
+ stat->track_identifier.is_defined()) {
+ sync_group_stats[reporter_helper_
+ ->GetStreamInfoFromTrackId(*stat->track_identifier)
+ .sync_group]
+ .push_back(stat);
+ }
+ }
+
+ MutexLock lock(&mutex_);
+ for (const auto& pair : sync_group_stats) {
+ // If there is less than two streams, it is not a sync group.
+ if (pair.second.size() < 2) {
+ continue;
+ }
+ auto sync_group = std::string(pair.first);
+ const RTCInboundRTPStreamStats* audio_stat = pair.second[0];
+ const RTCInboundRTPStreamStats* video_stat = pair.second[1];
+
+ RTC_CHECK(pair.second.size() == 2 && audio_stat->kind.is_defined() &&
+ video_stat->kind.is_defined() &&
+ *audio_stat->kind != *video_stat->kind)
+ << "Sync group should consist of one audio and one video stream.";
+
+ if (*audio_stat->kind == RTCMediaStreamTrackKind::kVideo) {
+ std::swap(audio_stat, video_stat);
+ }
+ // Stream labels of a sync group are same for all polls, so we need it add
+ // it only once.
+ if (stats_info_.find(sync_group) == stats_info_.end()) {
+ RTC_CHECK(audio_stat->track_identifier.is_defined());
+ RTC_CHECK(video_stat->track_identifier.is_defined());
+ stats_info_[sync_group].audio_stream_info =
+ reporter_helper_->GetStreamInfoFromTrackId(
+ *audio_stat->track_identifier);
+ stats_info_[sync_group].video_stream_info =
+ reporter_helper_->GetStreamInfoFromTrackId(
+ *video_stat->track_identifier);
+ }
+
+ double audio_video_playout_diff = *audio_stat->estimated_playout_timestamp -
+ *video_stat->estimated_playout_timestamp;
+ if (audio_video_playout_diff > 0) {
+ stats_info_[sync_group].audio_ahead_ms.AddSample(
+ audio_video_playout_diff);
+ stats_info_[sync_group].video_ahead_ms.AddSample(0);
+ } else {
+ stats_info_[sync_group].audio_ahead_ms.AddSample(0);
+ stats_info_[sync_group].video_ahead_ms.AddSample(
+ std::abs(audio_video_playout_diff));
+ }
+ }
+}
+
+void CrossMediaMetricsReporter::StopAndReportResults() {
+ MutexLock lock(&mutex_);
+ for (const auto& pair : stats_info_) {
+ const std::string& sync_group = pair.first;
+ // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey.
+ std::map<std::string, std::string> audio_metric_metadata{
+ {MetricMetadataKey::kPeerSyncGroupMetadataKey, sync_group},
+ {MetricMetadataKey::kAudioStreamMetadataKey,
+ pair.second.audio_stream_info.stream_label},
+ {MetricMetadataKey::kPeerMetadataKey,
+ pair.second.audio_stream_info.receiver_peer},
+ {MetricMetadataKey::kReceiverMetadataKey,
+ pair.second.audio_stream_info.receiver_peer},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_case_name_}};
+ metrics_logger_->LogMetric(
+ "audio_ahead_ms",
+ GetTestCaseName(pair.second.audio_stream_info.stream_label, sync_group),
+ pair.second.audio_ahead_ms, Unit::kMilliseconds,
+ webrtc::test::ImprovementDirection::kSmallerIsBetter,
+ std::move(audio_metric_metadata));
+
+ // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey.
+ std::map<std::string, std::string> video_metric_metadata{
+ {MetricMetadataKey::kPeerSyncGroupMetadataKey, sync_group},
+ {MetricMetadataKey::kAudioStreamMetadataKey,
+ pair.second.video_stream_info.stream_label},
+ {MetricMetadataKey::kPeerMetadataKey,
+ pair.second.video_stream_info.receiver_peer},
+ {MetricMetadataKey::kReceiverMetadataKey,
+ pair.second.video_stream_info.receiver_peer},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_case_name_}};
+ metrics_logger_->LogMetric(
+ "video_ahead_ms",
+ GetTestCaseName(pair.second.video_stream_info.stream_label, sync_group),
+ pair.second.video_ahead_ms, Unit::kMilliseconds,
+ webrtc::test::ImprovementDirection::kSmallerIsBetter,
+ std::move(video_metric_metadata));
+ }
+}
+
+std::string CrossMediaMetricsReporter::GetTestCaseName(
+ const std::string& stream_label,
+ const std::string& sync_group) const {
+ return test_case_name_ + "/" + sync_group + "_" + stream_label;
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.h b/third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.h
new file mode 100644
index 0000000000..2d51ebb20f
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/cross_media_metrics_reporter.h
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_CROSS_MEDIA_METRICS_REPORTER_H_
+#define TEST_PC_E2E_CROSS_MEDIA_METRICS_REPORTER_H_
+
+#include <map>
+#include <string>
+
+#include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
+#include "api/numerics/samples_stats_counter.h"
+#include "api/test/metrics/metrics_logger.h"
+#include "api/test/peerconnection_quality_test_fixture.h"
+#include "api/test/track_id_stream_info_map.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/synchronization/mutex.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+class CrossMediaMetricsReporter
+ : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter {
+ public:
+ explicit CrossMediaMetricsReporter(test::MetricsLogger* metrics_logger);
+ ~CrossMediaMetricsReporter() override = default;
+
+ void Start(absl::string_view test_case_name,
+ const TrackIdStreamInfoMap* reporter_helper) override;
+ void OnStatsReports(
+ absl::string_view pc_label,
+ const rtc::scoped_refptr<const RTCStatsReport>& report) override;
+ void StopAndReportResults() override;
+
+ private:
+ struct StatsInfo {
+ SamplesStatsCounter audio_ahead_ms;
+ SamplesStatsCounter video_ahead_ms;
+
+ TrackIdStreamInfoMap::StreamInfo audio_stream_info;
+ TrackIdStreamInfoMap::StreamInfo video_stream_info;
+ std::string audio_stream_label;
+ std::string video_stream_label;
+ };
+
+ std::string GetTestCaseName(const std::string& stream_label,
+ const std::string& sync_group) const;
+
+ test::MetricsLogger* const metrics_logger_;
+
+ std::string test_case_name_;
+ const TrackIdStreamInfoMap* reporter_helper_;
+
+ Mutex mutex_;
+ std::map<std::string, StatsInfo> stats_info_ RTC_GUARDED_BY(mutex_);
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_CROSS_MEDIA_METRICS_REPORTER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.cc b/third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.cc
new file mode 100644
index 0000000000..8fdabeb16f
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.cc
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/echo/echo_emulation.h"
+
+#include <limits>
+#include <utility>
+
+#include "api/test/pclf/media_configuration.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+constexpr int kSingleBufferDurationMs = 10;
+
+} // namespace
+
+EchoEmulatingCapturer::EchoEmulatingCapturer(
+ std::unique_ptr<TestAudioDeviceModule::Capturer> capturer,
+ EchoEmulationConfig config)
+ : delegate_(std::move(capturer)),
+ config_(config),
+ renderer_queue_(2 * config_.echo_delay.ms() / kSingleBufferDurationMs),
+ queue_input_(TestAudioDeviceModule::SamplesPerFrame(
+ delegate_->SamplingFrequency()) *
+ delegate_->NumChannels()),
+ queue_output_(TestAudioDeviceModule::SamplesPerFrame(
+ delegate_->SamplingFrequency()) *
+ delegate_->NumChannels()) {
+ renderer_thread_.Detach();
+ capturer_thread_.Detach();
+}
+
+void EchoEmulatingCapturer::OnAudioRendered(
+ rtc::ArrayView<const int16_t> data) {
+ RTC_DCHECK_RUN_ON(&renderer_thread_);
+ if (!recording_started_) {
+ // Because rendering can start before capturing in the beginning we can have
+ // a set of empty audio data frames. So we will skip them and will start
+ // fill the queue only after 1st non-empty audio data frame will arrive.
+ bool is_empty = true;
+ for (auto d : data) {
+ if (d != 0) {
+ is_empty = false;
+ break;
+ }
+ }
+ if (is_empty) {
+ return;
+ }
+ recording_started_ = true;
+ }
+ queue_input_.assign(data.begin(), data.end());
+ if (!renderer_queue_.Insert(&queue_input_)) {
+ RTC_LOG(LS_WARNING) << "Echo queue is full";
+ }
+}
+
+bool EchoEmulatingCapturer::Capture(rtc::BufferT<int16_t>* buffer) {
+ RTC_DCHECK_RUN_ON(&capturer_thread_);
+ bool result = delegate_->Capture(buffer);
+ // Now we have to reduce input signal to avoid saturation when mixing in the
+ // fake echo.
+ for (size_t i = 0; i < buffer->size(); ++i) {
+ (*buffer)[i] /= 2;
+ }
+
+ // When we accumulated enough delay in the echo buffer we will pop from
+ // that buffer on each ::Capture(...) call. If the buffer become empty it
+ // will mean some bug, so we will crash during removing item from the queue.
+ if (!delay_accumulated_) {
+ delay_accumulated_ =
+ renderer_queue_.SizeAtLeast() >=
+ static_cast<size_t>(config_.echo_delay.ms() / kSingleBufferDurationMs);
+ }
+
+ if (delay_accumulated_) {
+ RTC_CHECK(renderer_queue_.Remove(&queue_output_));
+ for (size_t i = 0; i < buffer->size() && i < queue_output_.size(); ++i) {
+ int32_t res = (*buffer)[i] + queue_output_[i];
+ if (res < std::numeric_limits<int16_t>::min()) {
+ res = std::numeric_limits<int16_t>::min();
+ }
+ if (res > std::numeric_limits<int16_t>::max()) {
+ res = std::numeric_limits<int16_t>::max();
+ }
+ (*buffer)[i] = static_cast<int16_t>(res);
+ }
+ }
+
+ return result;
+}
+
+EchoEmulatingRenderer::EchoEmulatingRenderer(
+ std::unique_ptr<TestAudioDeviceModule::Renderer> renderer,
+ EchoEmulatingCapturer* echo_emulating_capturer)
+ : delegate_(std::move(renderer)),
+ echo_emulating_capturer_(echo_emulating_capturer) {
+ RTC_DCHECK(echo_emulating_capturer_);
+}
+
+bool EchoEmulatingRenderer::Render(rtc::ArrayView<const int16_t> data) {
+ if (data.size() > 0) {
+ echo_emulating_capturer_->OnAudioRendered(data);
+ }
+ return delegate_->Render(data);
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.h b/third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.h
new file mode 100644
index 0000000000..359a481e46
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/echo/echo_emulation.h
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_ECHO_ECHO_EMULATION_H_
+#define TEST_PC_E2E_ECHO_ECHO_EMULATION_H_
+
+#include <atomic>
+#include <deque>
+#include <memory>
+#include <vector>
+
+#include "api/test/pclf/media_configuration.h"
+#include "modules/audio_device/include/test_audio_device.h"
+#include "rtc_base/swap_queue.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+// Reduces audio input strength from provided capturer twice and adds input
+// provided into EchoEmulatingCapturer::OnAudioRendered(...).
+class EchoEmulatingCapturer : public TestAudioDeviceModule::Capturer {
+ public:
+ EchoEmulatingCapturer(
+ std::unique_ptr<TestAudioDeviceModule::Capturer> capturer,
+ EchoEmulationConfig config);
+
+ void OnAudioRendered(rtc::ArrayView<const int16_t> data);
+
+ int SamplingFrequency() const override {
+ return delegate_->SamplingFrequency();
+ }
+ int NumChannels() const override { return delegate_->NumChannels(); }
+ bool Capture(rtc::BufferT<int16_t>* buffer) override;
+
+ private:
+ std::unique_ptr<TestAudioDeviceModule::Capturer> delegate_;
+ const EchoEmulationConfig config_;
+
+ SwapQueue<std::vector<int16_t>> renderer_queue_;
+
+ SequenceChecker renderer_thread_;
+ std::vector<int16_t> queue_input_ RTC_GUARDED_BY(renderer_thread_);
+ bool recording_started_ RTC_GUARDED_BY(renderer_thread_) = false;
+
+ SequenceChecker capturer_thread_;
+ std::vector<int16_t> queue_output_ RTC_GUARDED_BY(capturer_thread_);
+ bool delay_accumulated_ RTC_GUARDED_BY(capturer_thread_) = false;
+};
+
+// Renders output into provided renderer and also copy output into provided
+// EchoEmulationCapturer.
+class EchoEmulatingRenderer : public TestAudioDeviceModule::Renderer {
+ public:
+ EchoEmulatingRenderer(
+ std::unique_ptr<TestAudioDeviceModule::Renderer> renderer,
+ EchoEmulatingCapturer* echo_emulating_capturer);
+
+ int SamplingFrequency() const override {
+ return delegate_->SamplingFrequency();
+ }
+ int NumChannels() const override { return delegate_->NumChannels(); }
+ bool Render(rtc::ArrayView<const int16_t> data) override;
+
+ private:
+ std::unique_ptr<TestAudioDeviceModule::Renderer> delegate_;
+ EchoEmulatingCapturer* echo_emulating_capturer_;
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_ECHO_ECHO_EMULATION_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/architecture.md b/third_party/libwebrtc/test/pc/e2e/g3doc/architecture.md
new file mode 100644
index 0000000000..1b68c6db2c
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/g3doc/architecture.md
@@ -0,0 +1,209 @@
+<!-- go/cmark -->
+<!--* freshness: {owner: 'titovartem' reviewed: '2021-04-12'} *-->
+
+# PeerConnection level framework fixture architecture
+
+## Overview
+
+The main implementation of
+[`webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture`][1] is
+[`webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTest`][2]. Internally it owns
+the next main pieces:
+
+* [`MediaHelper`][3] - responsible for adding audio and video tracks to the
+ peers.
+* [`VideoQualityAnalyzerInjectionHelper`][4] and
+ [`SingleProcessEncodedImageDataInjector`][5] - used to inject video quality
+ analysis and properly match captured and rendered video frames. You can read
+ more about it in
+ [DefaultVideoQualityAnalyzer](default_video_quality_analyzer.md) section.
+* [`AudioQualityAnalyzerInterface`][6] - used to measure audio quality metrics
+* [`TestActivitiesExecutor`][7] - used to support [`ExecuteAt(...)`][8] and
+ [`ExecuteEvery(...)`][9] API of `PeerConnectionE2EQualityTestFixture` to run
+ any arbitrary action during test execution timely synchronized with a test
+ call.
+* A vector of [`QualityMetricsReporter`][10] added by the
+ `PeerConnectionE2EQualityTestFixture` user.
+* Two peers: Alice and Bob represented by instances of [`TestPeer`][11]
+ object.
+
+Also it keeps a reference to [`webrtc::TimeController`][12], which is used to
+create all required threads, task queues, task queue factories and time related
+objects.
+
+## TestPeer
+
+Call participants are represented by instances of `TestPeer` object.
+[`TestPeerFactory`][13] is used to create them. `TestPeer` owns all instances
+related to the `webrtc::PeerConnection`, including required listeners and
+callbacks. Also it provides an API to do offer/answer exchange and ICE candidate
+exchange. For this purposes internally it uses an instance of
+[`webrtc::PeerConnectionWrapper`][14].
+
+The `TestPeer` also owns the `PeerConnection` worker thread. The signaling
+thread for all `PeerConnection`'s is owned by
+`PeerConnectionE2EQualityTestFixture` and shared between all participants in the
+call. The network thread is owned by the network layer (it maybe either emulated
+network provided by [Network Emulation Framework][24] or network thread and
+`rtc::NetworkManager` provided by user) and provided when peer is added to the
+fixture via [`AddPeer(...)`][15] API.
+
+## GetStats API based metrics reporters
+
+`PeerConnectionE2EQualityTestFixture` gives the user ability to provide
+different `QualityMetricsReporter`s which will listen for `PeerConnection`
+[`GetStats`][16] API. Then such reporters will be able to report various metrics
+that user wants to measure.
+
+`PeerConnectionE2EQualityTestFixture` itself also uses this mechanism to
+measure:
+
+* Audio quality metrics
+* Audio/Video sync metrics (with help of [`CrossMediaMetricsReporter`][17])
+
+Also framework provides a [`StatsBasedNetworkQualityMetricsReporter`][18] to
+measure network related WebRTC metrics and print debug raw emulated network
+statistic. This reporter should be added by user via
+[`AddQualityMetricsReporter(...)`][19] API if requried.
+
+Internally stats gathering is done by [`StatsPoller`][20]. Stats are requested
+once per second for each `PeerConnection` and then resulted object is provided
+into each stats listener.
+
+## Offer/Answer exchange
+
+`PeerConnectionE2EQualityTest` provides ability to test Simulcast and SVC for
+video. These features aren't supported by P2P call and in general requires a
+Selective Forwarding Unit (SFU). So special logic is applied to mimic SFU
+behavior in P2P call. This logic is located inside [`SignalingInterceptor`][21],
+[`QualityAnalyzingVideoEncoder`][22] and [`QualityAnalyzingVideoDecoder`][23]
+and consist of SDP modification during offer/answer exchange and special
+handling of video frames from unrelated Simulcast/SVC streams during decoding.
+
+### Simulcast
+
+In case of Simulcast we have a video track, which internally contains multiple
+video streams, for example low resolution, medium resolution and high
+resolution. WebRTC client doesn't support receiving an offer with multiple
+streams in it, because usually SFU will keep only single stream for the client.
+To bypass it framework will modify offer by converting a single track with three
+video streams into three independent video tracks. Then sender will think that
+it send simulcast, but receiver will think that it receives 3 independent
+tracks.
+
+To achieve such behavior some extra tweaks are required:
+
+* MID RTP header extension from original offer have to be removed
+* RID RTP header extension from original offer is replaced with MID RTP header
+ extension, so the ID that sender uses for RID on receiver will be parsed as
+ MID.
+* Answer have to be modified in the opposite way.
+
+Described modifications are illustrated on the picture below.
+
+![VP8 Simulcast offer modification](vp8_simulcast_offer_modification.png "VP8 Simulcast offer modification")
+
+The exchange will look like this:
+
+1. Alice creates an offer
+2. Alice sets offer as local description
+3. Do described offer modification
+4. Alice sends modified offer to Bob
+5. Bob sets modified offer as remote description
+6. Bob creates answer
+7. Bob sets answer as local description
+8. Do reverse modifications on answer
+9. Bob sends modified answer to Alice
+10. Alice sets modified answer as remote description
+
+Such mechanism put a constraint that RTX streams are not supported, because they
+don't have RID RTP header extension in their packets.
+
+### SVC
+
+In case of SVC the framework will update the sender's offer before even setting
+it as local description on the sender side. Then no changes to answer will be
+required.
+
+`ssrc` is a 32 bit random value that is generated in RTP to denote a specific
+source used to send media in an RTP connection. In original offer video track
+section will look like this:
+
+```
+m=video 9 UDP/TLS/RTP/SAVPF 98 100 99 101
+...
+a=ssrc-group:FID <primary ssrc> <retransmission ssrc>
+a=ssrc:<primary ssrc> cname:...
+....
+a=ssrc:<retransmission ssrc> cname:...
+....
+```
+
+To enable SVC for such video track framework will add extra `ssrc`s for each SVC
+stream that is required like this:
+
+```
+a=ssrc-group:FID <Low resolution primary ssrc> <Low resolution retransmission ssrc>
+a=ssrc:<Low resolution primary ssrc> cname:...
+....
+a=ssrc:<Low resolution retransmission ssrc> cname:....
+...
+a=ssrc-group:FID <Medium resolution primary ssrc> <Medium resolution retransmission ssrc>
+a=ssrc:<Medium resolution primary ssrc> cname:...
+....
+a=ssrc:<Medium resolution retransmission ssrc> cname:....
+...
+a=ssrc-group:FID <High resolution primary ssrc> <High resolution retransmission ssrc>
+a=ssrc:<High resolution primary ssrc> cname:...
+....
+a=ssrc:<High resolution retransmission ssrc> cname:....
+...
+```
+
+The next line will also be added to the video track section of the offer:
+
+```
+a=ssrc-group:SIM <Low resolution primary ssrc> <Medium resolution primary ssrc> <High resolution primary ssrc>
+```
+
+It will tell PeerConnection that this track should be configured as SVC. It
+utilize WebRTC Plan B offer structure to achieve SVC behavior, also it modifies
+offer before setting it as local description which violates WebRTC standard.
+Also it adds limitations that on lossy networks only top resolution streams can
+be analyzed, because WebRTC won't try to restore low resolution streams in case
+of loss, because it still receives higher stream.
+
+### Handling in encoder/decoder
+
+In the encoder, the framework for each encoded video frame will propagate
+information requried for the fake SFU to know if it belongs to an interesting
+simulcast stream/spatial layer of if it should be "discarded".
+
+On the decoder side frames that should be "discarded" by fake SFU will be auto
+decoded into single pixel images and only the interesting simulcast
+stream/spatial layer will go into real decoder and then will be analyzed.
+
+[1]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=55;drc=484acf27231d931dbc99aedce85bc27e06486b96
+[2]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/peer_connection_quality_test.h;l=44;drc=6cc893ad778a0965e2b7a8e614f3c98aa81bee5b
+[3]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/media/media_helper.h;l=27;drc=d46db9f1523ae45909b4a6fdc90a140443068bc6
+[4]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h;l=38;drc=79020414fd5c71f9ec1f25445ea5f1c8001e1a49
+[5]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h;l=40;drc=79020414fd5c71f9ec1f25445ea5f1c8001e1a49
+[6]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/audio_quality_analyzer_interface.h;l=23;drc=20f45823e37fd7272aa841831c029c21f29742c2
+[7]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/test_activities_executor.h;l=28;drc=6cc893ad778a0965e2b7a8e614f3c98aa81bee5b
+[8]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=439;drc=484acf27231d931dbc99aedce85bc27e06486b96
+[9]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=445;drc=484acf27231d931dbc99aedce85bc27e06486b96
+[10]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=413;drc=9438fb3fff97c803d1ead34c0e4f223db168526f
+[11]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/test_activities_executor.h;l=28;drc=6cc893ad778a0965e2b7a8e614f3c98aa81bee5b
+[12]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/test_activities_executor.h;l=28;drc=6cc893ad778a0965e2b7a8e614f3c98aa81bee5b
+[13]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/test_peer_factory.h;l=46;drc=0ef4a2488a466a24ab97b31fdddde55440d451f9
+[14]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/pc/peer_connection_wrapper.h;l=47;drc=5ab79e62f691875a237ea28ca3975ea1f0ed62ec
+[15]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=459;drc=484acf27231d931dbc99aedce85bc27e06486b96
+[16]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/peer_connection_interface.h;l=886;drc=9438fb3fff97c803d1ead34c0e4f223db168526f
+[17]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/cross_media_metrics_reporter.h;l=29;drc=9d777620236ec76754cfce19f6e82dd18e52d22c
+[18]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/cross_media_metrics_reporter.h;l=29;drc=9d777620236ec76754cfce19f6e82dd18e52d22c
+[19]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=450;drc=484acf27231d931dbc99aedce85bc27e06486b96
+[20]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/stats_poller.h;l=52;drc=9b526180c9e9722d3fc7f8689da6ec094fc7fc0a
+[21]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/sdp/sdp_changer.h;l=79;drc=ee558dcca89fd8b105114ededf9e74d948da85e8
+[22]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h;l=54;drc=79020414fd5c71f9ec1f25445ea5f1c8001e1a49
+[23]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h;l=50;drc=79020414fd5c71f9ec1f25445ea5f1c8001e1a49
+[24]: /test/network/g3doc/index.md
diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/default_video_quality_analyzer.md b/third_party/libwebrtc/test/pc/e2e/g3doc/default_video_quality_analyzer.md
new file mode 100644
index 0000000000..67596777f2
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/g3doc/default_video_quality_analyzer.md
@@ -0,0 +1,197 @@
+<!-- go/cmark -->
+<!--* freshness: {owner: 'titovartem' reviewed: '2021-02-21'} *-->
+
+# DefaultVideoQualityAnalyzer
+
+## Audience
+
+This document is for users of
+[`webrtc::webrtc_pc_e2e::DefaultVideoQualityAnalyzer`][1].
+
+## Overview
+
+`DefaultVideoQualityAnalyzer` implements
+[`webrtc::VideoQualityAnalyzerInterface`][2] and is a main
+implementation of video quality analyzer for WebRTC. To operate correctly it
+requires to receive video frame on each step:
+
+1. On frame captured - analyzer will generate a unique ID for the frame, that
+ caller should attach to the it.
+2. Immediately before frame enter the encoder.
+3. Immediately after the frame was encoded.
+4. After the frame was received and immediately before it entered the decoder.
+5. Immediately after the frame was decoded.
+6. When the frame was rendered.
+
+![VideoQualityAnalyzerInterface pipeline](video_quality_analyzer_pipeline.png "VideoQualityAnalyzerInterface pipeline")
+
+The analyzer updates its internal metrics per frame when it was rendered and
+reports all of them after it was stopped through
+[WebRTC perf results reporting system][10].
+
+To properly inject `DefaultVideoQualityAnalyzer` into pipeline the following helpers can be used:
+
+### VideoQualityAnalyzerInjectionHelper
+
+[`webrtc::webrtc_pc_e2e::VideoQualityAnalyzerInjectionHelper`][3] provides
+factory methods for components, that will be used to inject
+`VideoQualityAnalyzerInterface` into the `PeerConnection` pipeline:
+
+* Wrappers for [`webrtc::VideoEncoderFactory`][4] and
+ [`webrtc::VideoDecodeFactory`][5] which will properly pass
+ [`webrtc::VideoFrame`][6] and [`webrtc::EncodedImage`][7] into analyzer
+ before and after real video encode and decoder.
+* [`webrtc::test::TestVideoCapturer::FramePreprocessor`][8] which is used to
+ pass generated frames into analyzer on capturing and then set the returned
+ frame ID. It also configures dumping of captured frames if requried.
+* [`rtc::VideoSinkInterface<VideoFrame>`][9] which is used to pass frames to
+ the analyzer before they will be rendered to compute per frame metrics. It
+ also configures dumping of rendered video if requried.
+
+Besides factories `VideoQualityAnalyzerInjectionHelper` has method to
+orchestrate `VideoQualityAnalyzerInterface` workflow:
+
+* `Start` - to start video analyzer, so it will be able to receive and analyze
+ video frames.
+* `RegisterParticipantInCall` - to add new participants after analyzer was
+ started.
+* `Stop` - to stop analyzer, compute all metrics for frames that were recevied
+ before and report them.
+
+Also `VideoQualityAnalyzerInjectionHelper` implements
+[`webrtc::webrtc_pc_e2e::StatsObserverInterface`][11] to propagate WebRTC stats
+to `VideoQualityAnalyzerInterface`.
+
+### EncodedImageDataInjector and EncodedImageDataExtractor
+
+[`webrtc::webrtc_pc_e2e::EncodedImageDataInjector`][14] and
+[`webrtc::webrtc_pc_e2e::EncodedImageDataInjector`][15] are used to inject and
+extract data into `webrtc::EncodedImage` to propagate frame ID and other
+required information through the network.
+
+By default [`webrtc::webrtc_pc_e2e::SingleProcessEncodedImageDataInjector`][16]
+is used. It assumes `webrtc::EncodedImage` payload as black box which is
+remaining unchanged from encoder to decoder and stores the information required
+for its work in the last 3 bytes of the payload, replacing the original data
+during injection and restoring it back during extraction. Also
+`SingleProcessEncodedImageDataInjector` requires that sender and receiver were
+inside single process.
+
+![SingleProcessEncodedImageDataInjector](single_process_encoded_image_data_injector.png "SingleProcessEncodedImageDataInjector")
+
+## Exported metrics
+
+Exported metrics are reported to WebRTC perf results reporting system.
+
+### General
+
+* *`cpu_usage`* - CPU usage excluding video analyzer
+
+### Video
+
+* *`psnr`* - peak signal-to-noise ratio:
+ [wikipedia](https://en.wikipedia.org/wiki/Peak_signal-to-noise_ratio)
+* *`ssim`* - structural similarity:
+ [wikipedia](https://en.wikipedia.org/wiki/Structural_similarity).
+* *`min_psnr`* - minimum value of psnr across all frames of video stream.
+* *`encode_time`* - time to encode a single frame.
+* *`decode_time`* - time to decode a single frame.
+* *`transport_time`* - time from frame encoded to frame received for decoding.
+* *`receive_to_render_time`* - time from frame received for decoding to frame
+ rendered.
+* *`total_delay_incl_transport`* - time from frame was captured on device to
+ time when frame was displayed on device.
+* *`encode_frame_rate`* - frame rate after encoder.
+* *`harmonic_framerate`* - video duration divided on squared sum of interframe
+ delays. Reflects render frame rate penalized by freezes.
+* *`time_between_rendered_frames`* - time between frames out to renderer.
+* *`dropped_frames`* - amount of frames that were sent, but weren't rendered
+ and are known not to be “on the way” from sender to receiver.
+
+Freeze is a pause when no new frames from decoder arrived for 150ms + avg time
+between frames or 3 * avg time between frames.
+
+* *`time_between_freezes`* - mean time from previous freeze end to new freeze
+ start.
+* *`freeze_time_ms`* - total freeze time in ms.
+* *`max_skipped`* - frames skipped between two nearest rendered.
+* *`pixels_per_frame`* - amount of pixels on frame (width * height).
+* *`target_encode_bitrate`* - target encode bitrate provided by BWE to
+ encoder.
+* *`actual_encode_bitrate -`* - actual encode bitrate produced by encoder.
+* *`available_send_bandwidth -`* - available send bandwidth estimated by BWE.
+* *`transmission_bitrate`* - bitrate of media in the emulated network, not
+ counting retransmissions FEC, and RTCP messages
+* *`retransmission_bitrate`* - bitrate of retransmission streams only.
+
+### Framework stability
+
+* *`frames_in_flight`* - amount of frames that were captured but wasn't seen
+ on receiver.
+
+## Debug metrics
+
+Debug metrics are not reported to WebRTC perf results reporting system, but are
+available through `DefaultVideoQualityAnalyzer` API.
+
+### [FrameCounters][12]
+
+Frame counters consist of next counters:
+
+* *`captured`* - count of frames, that were passed into WebRTC pipeline by
+ video stream source
+* *`pre_encoded`* - count of frames that reached video encoder.
+* *`encoded`* - count of encoded images that were produced by encoder for all
+ requested spatial layers and simulcast streams.
+* *`received`* - count of encoded images received in decoder for all requested
+ spatial layers and simulcast streams.
+* *`decoded`* - count of frames that were produced by decoder.
+* *`rendered`* - count of frames that went out from WebRTC pipeline to video
+ sink.
+* *`dropped`* - count of frames that were dropped in any point between
+ capturing and rendering.
+
+`DefaultVideoQualityAnalyzer` exports these frame counters:
+
+* *`GlobalCounters`* - frame counters for frames met on each stage of analysis
+ for all media streams.
+* *`PerStreamCounters`* - frame counters for frames met on each stage of
+ analysis separated per individual video track (single media section in the
+ SDP offer).
+
+### [AnalyzerStats][13]
+
+Contains metrics about internal state of video analyzer during its work
+
+* *`comparisons_queue_size`* - size of analyzer internal queue used to perform
+ captured and rendered frames comparisons measured when new element is added
+ to the queue.
+* *`comparisons_done`* - number of performed comparisons of 2 video frames
+ from captured and rendered streams.
+* *`cpu_overloaded_comparisons_done`* - number of cpu overloaded comparisons.
+ Comparison is cpu overloaded if it is queued when there are too many not
+ processed comparisons in the queue. Overloaded comparison doesn't include
+ metrics like SSIM and PSNR that require heavy computations.
+* *`memory_overloaded_comparisons_done`* - number of memory overloaded
+ comparisons. Comparison is memory overloaded if it is queued when its
+ captured frame was already removed due to high memory usage for that video
+ stream.
+* *`frames_in_flight_left_count`* - count of frames in flight in analyzer
+ measured when new comparison is added and after analyzer was stopped.
+
+[1]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h;l=188;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2
+[2]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/test/video_quality_analyzer_interface.h;l=56;drc=d7808f1c464a07c8f1e2f97ec7ee92fda998d590
+[3]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h;l=39;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2
+[4]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/video_codecs/video_encoder_factory.h;l=27;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2
+[5]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/video_codecs/video_decoder_factory.h;l=27;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2
+[6]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/video/video_frame.h;l=30;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2
+[7]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/video/encoded_image.h;l=71;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2
+[8]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/test_video_capturer.h;l=28;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2
+[9]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/video/video_sink_interface.h;l=19;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2
+[10]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/testsupport/perf_test.h;drc=0710b401b1e5b500b8e84946fb657656ba1b58b7
+[11]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/api/test/stats_observer_interface.h;l=21;drc=9b526180c9e9722d3fc7f8689da6ec094fc7fc0a
+[12]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h;l=57;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2
+[13]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/default_video_quality_analyzer.h;l=113;drc=08f46909a8735cf181b99ef2f7e1791c5a7531d2
+[14]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h;l=23;drc=c57089a97a3df454f4356d882cc8df173e8b3ead
+[15]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/encoded_image_data_injector.h;l=46;drc=c57089a97a3df454f4356d882cc8df173e8b3ead
+[16]: https://source.chromium.org/chromium/chromium/src/+/master:third_party/webrtc/test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h;l=40;drc=c57089a97a3df454f4356d882cc8df173e8b3ead
diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/in_test_psnr_plot.png b/third_party/libwebrtc/test/pc/e2e/g3doc/in_test_psnr_plot.png
new file mode 100644
index 0000000000..3f36725727
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/g3doc/in_test_psnr_plot.png
Binary files differ
diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/index.md b/third_party/libwebrtc/test/pc/e2e/g3doc/index.md
new file mode 100644
index 0000000000..678262bb2b
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/g3doc/index.md
@@ -0,0 +1,224 @@
+<!-- go/cmark -->
+<!--* freshness: {owner: 'titovartem' reviewed: '2021-04-12'} *-->
+
+# PeerConnection Level Framework
+
+## API
+
+* [Fixture][1]
+* [Fixture factory function][2]
+
+## Documentation
+
+The PeerConnection level framework is designed for end-to-end media quality
+testing through the PeerConnection level public API. The framework uses the
+*Unified plan* API to generate offers/answers during the signaling phase. The
+framework also wraps the video encoder/decoder and inject it into
+*`webrtc::PeerConnection`* to measure video quality, performing 1:1 frames
+matching between captured and rendered frames without any extra requirements to
+input video. For audio quality evaluation the standard `GetStats()` API from
+PeerConnection is used.
+
+The framework API is located in the namespace *`webrtc::webrtc_pc_e2e`*.
+
+### Supported features
+
+* Single or bidirectional media in the call
+* RTC Event log dump per peer
+* AEC dump per peer
+* Compatible with *`webrtc::TimeController`* for both real and simulated time
+* Media
+ * AV sync
+* Video
+ * Any amount of video tracks both from caller and callee sides
+ * Input video from
+ * Video generator
+ * Specified file
+ * Any instance of *`webrtc::test::FrameGeneratorInterface`*
+ * Dumping of captured/rendered video into file
+ * Screen sharing
+ * Vp8 simulcast from caller side
+ * Vp9 SVC from caller side
+ * Choosing of video codec (name and parameters), having multiple codecs
+ negotiated to support codec-switching testing.
+ * FEC (ULP or Flex)
+ * Forced codec overshooting (for encoder overshoot emulation on some
+ mobile devices, when hardware encoder can overshoot target bitrate)
+* Audio
+ * Up to 1 audio track both from caller and callee sides
+ * Generated audio
+ * Audio from specified file
+ * Dumping of captured/rendered audio into file
+ * Parameterizing of `cricket::AudioOptions`
+ * Echo emulation
+* Injection of various WebRTC components into underlying
+ *`webrtc::PeerConnection`* or *`webrtc::PeerConnectionFactory`*. You can see
+ the full list [here][11]
+* Scheduling of events, that can happen during the test, for example:
+ * Changes in network configuration
+ * User statistics measurements
+ * Custom defined actions
+* User defined statistics reporting via
+ *`webrtc::webrtc_pc_e2e::PeerConnectionE2EQualityTestFixture::QualityMetricsReporter`*
+ interface
+
+## Exported metrics
+
+### General
+
+* *`<peer_name>_connected`* - peer successfully established connection to
+ remote side
+* *`cpu_usage`* - CPU usage excluding video analyzer
+* *`audio_ahead_ms`* - Used to estimate how much audio and video is out of
+ sync when the two tracks were from the same source. Stats are polled
+ periodically during a call. The metric represents how much earlier was audio
+ played out on average over the call. If, during a stats poll, video is
+ ahead, then audio_ahead_ms will be equal to 0 for this poll.
+* *`video_ahead_ms`* - Used to estimate how much audio and video is out of
+ sync when the two tracks were from the same source. Stats are polled
+ periodically during a call. The metric represents how much earlier was video
+ played out on average over the call. If, during a stats poll, audio is
+ ahead, then video_ahead_ms will be equal to 0 for this poll.
+
+### Video
+
+See documentation for
+[*`DefaultVideoQualityAnalyzer`*](default_video_quality_analyzer.md#exported-metrics)
+
+### Audio
+
+* *`accelerate_rate`* - when playout is sped up, this counter is increased by
+ the difference between the number of samples received and the number of
+ samples played out. If speedup is achieved by removing samples, this will be
+ the count of samples removed. Rate is calculated as difference between
+ nearby samples divided on sample interval.
+* *`expand_rate`* - the total number of samples that are concealed samples
+ over time. A concealed sample is a sample that was replaced with synthesized
+ samples generated locally before being played out. Examples of samples that
+ have to be concealed are samples from lost packets or samples from packets
+ that arrive too late to be played out
+* *`speech_expand_rate`* - the total number of samples that are concealed
+ samples minus the total number of concealed samples inserted that are
+ "silent" over time. Playing out silent samples results in silence or comfort
+ noise.
+* *`preemptive_rate`* - when playout is slowed down, this counter is increased
+ by the difference between the number of samples received and the number of
+ samples played out. If playout is slowed down by inserting samples, this
+ will be the number of inserted samples. Rate is calculated as difference
+ between nearby samples divided on sample interval.
+* *`average_jitter_buffer_delay_ms`* - average size of NetEQ jitter buffer.
+* *`preferred_buffer_size_ms`* - preferred size of NetEQ jitter buffer.
+* *`visqol_mos`* - proxy for audio quality itself.
+* *`asdm_samples`* - measure of how much acceleration/deceleration was in the
+ signal.
+* *`word_error_rate`* - measure of how intelligible the audio was (percent of
+ words that could not be recognized in output audio).
+
+### Network
+
+* *`bytes_sent`* - represents the total number of payload bytes sent on this
+ PeerConnection, i.e., not including headers or padding
+* *`packets_sent`* - represents the total number of packets sent over this
+ PeerConnection’s transports.
+* *`average_send_rate`* - average send rate calculated on bytes_sent divided
+ by test duration.
+* *`payload_bytes_sent`* - total number of bytes sent for all SSRC plus total
+ number of RTP header and padding bytes sent for all SSRC. This does not
+ include the size of transport layer headers such as IP or UDP.
+* *`sent_packets_loss`* - packets_sent minus corresponding packets_received.
+* *`bytes_received`* - represents the total number of bytes received on this
+ PeerConnection, i.e., not including headers or padding.
+* *`packets_received`* - represents the total number of packets received on
+ this PeerConnection’s transports.
+* *`average_receive_rate`* - average receive rate calculated on bytes_received
+ divided by test duration.
+* *`payload_bytes_received`* - total number of bytes received for all SSRC
+ plus total number of RTP header and padding bytes received for all SSRC.
+ This does not include the size of transport layer headers such as IP or UDP.
+
+### Framework stability
+
+* *`frames_in_flight`* - amount of frames that were captured but wasn't seen
+ on receiver in the way that also all frames after also weren't seen on
+ receiver.
+* *`bytes_discarded_no_receiver`* - total number of bytes that were received
+ on network interfaces related to the peer, but destination port was closed.
+* *`packets_discarded_no_receiver`* - total number of packets that were
+ received on network interfaces related to the peer, but destination port was
+ closed.
+
+## Examples
+
+Examples can be found in
+
+* [peer_connection_e2e_smoke_test.cc][3]
+* [pc_full_stack_tests.cc][4]
+
+## Stats plotting
+
+### Description
+
+Stats plotting provides ability to plot statistic collected during the test.
+Right now it is used in PeerConnection level framework and give ability to see
+how video quality metrics changed during test execution.
+
+### Usage
+
+To make any metrics plottable you need:
+
+1. Collect metric data with [SamplesStatsCounter][5] which internally will
+ store all intermediate points and timestamps when these points were added.
+2. Then you need to report collected data with
+ [`webrtc::test::PrintResult(...)`][6]. By using these method you will also
+ specify name of the plottable metric.
+
+After these steps it will be possible to export your metric for plotting. There
+are several options how you can do this:
+
+1. Use [`webrtc::TestMain::Create()`][7] as `main` function implementation, for
+ example use [`test/test_main.cc`][8] as `main` function for your test.
+
+ In such case your binary will have flag `--plot`, where you can provide a
+ list of metrics, that you want to plot or specify `all` to plot all
+ available metrics.
+
+ If `--plot` is specified, the binary will output metrics data into `stdout`.
+ Then you need to pipe this `stdout` into python plotter script
+ [`rtc_tools/metrics_plotter.py`][9], which will plot data.
+
+ Examples:
+
+ ```shell
+ $ ./out/Default/test_support_unittests \
+ --gtest_filter=PeerConnectionE2EQualityTestSmokeTest.Svc \
+ --nologs \
+ --plot=all \
+ | python rtc_tools/metrics_plotter.py
+ ```
+
+ ```shell
+ $ ./out/Default/test_support_unittests \
+ --gtest_filter=PeerConnectionE2EQualityTestSmokeTest.Svc \
+ --nologs \
+ --plot=psnr,ssim \
+ | python rtc_tools/metrics_plotter.py
+ ```
+
+ Example chart: ![PSNR changes during the test](in_test_psnr_plot.png)
+
+2. Use API from [`test/testsupport/perf_test.h`][10] directly by invoking
+ `webrtc::test::PrintPlottableResults(const std::vector<std::string>&
+ desired_graphs)` to print plottable metrics to stdout. Then as in previous
+ option you need to pipe result into plotter script.
+
+[1]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;drc=cbe6e8a2589a925d4c91a2ac2c69201f03de9c39
+[2]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/create_peerconnection_quality_test_fixture.h;drc=cbe6e8a2589a925d4c91a2ac2c69201f03de9c39
+[3]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/pc/e2e/peer_connection_e2e_smoke_test.cc;drc=cbe6e8a2589a925d4c91a2ac2c69201f03de9c39
+[4]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/video/pc_full_stack_tests.cc;drc=cbe6e8a2589a925d4c91a2ac2c69201f03de9c39
+[5]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/numerics/samples_stats_counter.h;drc=cbe6e8a2589a925d4c91a2ac2c69201f03de9c39
+[6]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/testsupport/perf_test.h;l=86;drc=0710b401b1e5b500b8e84946fb657656ba1b58b7
+[7]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/test_main_lib.h;l=23;drc=bcb42f1e4be136c390986a40d9d5cb3ad0de260b
+[8]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/test_main.cc;drc=bcb42f1e4be136c390986a40d9d5cb3ad0de260b
+[9]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/rtc_tools/metrics_plotter.py;drc=8cc6695652307929edfc877cd64b75cd9ec2d615
+[10]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/test/testsupport/perf_test.h;l=105;drc=0710b401b1e5b500b8e84946fb657656ba1b58b7
+[11]: https://source.chromium.org/chromium/chromium/src/+/main:third_party/webrtc/api/test/peerconnection_quality_test_fixture.h;l=272;drc=484acf27231d931dbc99aedce85bc27e06486b96
diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/single_process_encoded_image_data_injector.png b/third_party/libwebrtc/test/pc/e2e/g3doc/single_process_encoded_image_data_injector.png
new file mode 100644
index 0000000000..73480bafbe
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/g3doc/single_process_encoded_image_data_injector.png
Binary files differ
diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/video_quality_analyzer_pipeline.png b/third_party/libwebrtc/test/pc/e2e/g3doc/video_quality_analyzer_pipeline.png
new file mode 100644
index 0000000000..6cddb91110
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/g3doc/video_quality_analyzer_pipeline.png
Binary files differ
diff --git a/third_party/libwebrtc/test/pc/e2e/g3doc/vp8_simulcast_offer_modification.png b/third_party/libwebrtc/test/pc/e2e/g3doc/vp8_simulcast_offer_modification.png
new file mode 100644
index 0000000000..c7eaa04c0e
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/g3doc/vp8_simulcast_offer_modification.png
Binary files differ
diff --git a/third_party/libwebrtc/test/pc/e2e/media/media_helper.cc b/third_party/libwebrtc/test/pc/e2e/media/media_helper.cc
new file mode 100644
index 0000000000..e945bd4dae
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/media/media_helper.cc
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/media/media_helper.h"
+
+#include <string>
+#include <utility>
+
+#include "absl/types/variant.h"
+#include "api/media_stream_interface.h"
+#include "api/test/create_frame_generator.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/pclf/peer_configurer.h"
+#include "test/frame_generator_capturer.h"
+#include "test/platform_video_capturer.h"
+#include "test/testsupport/file_utils.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+void MediaHelper::MaybeAddAudio(TestPeer* peer) {
+ if (!peer->params().audio_config) {
+ return;
+ }
+ const AudioConfig& audio_config = peer->params().audio_config.value();
+ rtc::scoped_refptr<webrtc::AudioSourceInterface> source =
+ peer->pc_factory()->CreateAudioSource(audio_config.audio_options);
+ rtc::scoped_refptr<AudioTrackInterface> track =
+ peer->pc_factory()->CreateAudioTrack(*audio_config.stream_label,
+ source.get());
+ std::string sync_group = audio_config.sync_group
+ ? audio_config.sync_group.value()
+ : audio_config.stream_label.value() + "-sync";
+ peer->AddTrack(track, {sync_group, *audio_config.stream_label});
+}
+
+std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>>
+MediaHelper::MaybeAddVideo(TestPeer* peer) {
+ // Params here valid because of pre-run validation.
+ const Params& params = peer->params();
+ const ConfigurableParams& configurable_params = peer->configurable_params();
+ std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>> out;
+ for (size_t i = 0; i < configurable_params.video_configs.size(); ++i) {
+ const VideoConfig& video_config = configurable_params.video_configs[i];
+ // Setup input video source into peer connection.
+ std::unique_ptr<test::TestVideoCapturer> capturer = CreateVideoCapturer(
+ video_config, peer->ReleaseVideoSource(i),
+ video_quality_analyzer_injection_helper_->CreateFramePreprocessor(
+ params.name.value(), video_config));
+ bool is_screencast =
+ video_config.content_hint == VideoTrackInterface::ContentHint::kText ||
+ video_config.content_hint ==
+ VideoTrackInterface::ContentHint::kDetailed;
+ rtc::scoped_refptr<TestVideoCapturerVideoTrackSource> source =
+ rtc::make_ref_counted<TestVideoCapturerVideoTrackSource>(
+ std::move(capturer), is_screencast);
+ out.push_back(source);
+ RTC_LOG(LS_INFO) << "Adding video with video_config.stream_label="
+ << video_config.stream_label.value();
+ rtc::scoped_refptr<VideoTrackInterface> track =
+ peer->pc_factory()->CreateVideoTrack(video_config.stream_label.value(),
+ source.get());
+ if (video_config.content_hint.has_value()) {
+ track->set_content_hint(video_config.content_hint.value());
+ }
+ std::string sync_group = video_config.sync_group
+ ? video_config.sync_group.value()
+ : video_config.stream_label.value() + "-sync";
+ RTCErrorOr<rtc::scoped_refptr<RtpSenderInterface>> sender =
+ peer->AddTrack(track, {sync_group, *video_config.stream_label});
+ RTC_CHECK(sender.ok());
+ if (video_config.temporal_layers_count ||
+ video_config.degradation_preference) {
+ RtpParameters rtp_parameters = sender.value()->GetParameters();
+ if (video_config.temporal_layers_count) {
+ for (auto& encoding_parameters : rtp_parameters.encodings) {
+ encoding_parameters.num_temporal_layers =
+ video_config.temporal_layers_count;
+ }
+ }
+ if (video_config.degradation_preference) {
+ rtp_parameters.degradation_preference =
+ video_config.degradation_preference;
+ }
+ RTCError res = sender.value()->SetParameters(rtp_parameters);
+ RTC_CHECK(res.ok()) << "Failed to set RTP parameters";
+ }
+ }
+ return out;
+}
+
+std::unique_ptr<test::TestVideoCapturer> MediaHelper::CreateVideoCapturer(
+ const VideoConfig& video_config,
+ PeerConfigurer::VideoSource source,
+ std::unique_ptr<test::TestVideoCapturer::FramePreprocessor>
+ frame_preprocessor) {
+ CapturingDeviceIndex* capturing_device_index =
+ absl::get_if<CapturingDeviceIndex>(&source);
+ if (capturing_device_index != nullptr) {
+ std::unique_ptr<test::TestVideoCapturer> capturer =
+ test::CreateVideoCapturer(video_config.width, video_config.height,
+ video_config.fps,
+ static_cast<size_t>(*capturing_device_index));
+ RTC_CHECK(capturer)
+ << "Failed to obtain input stream from capturing device #"
+ << *capturing_device_index;
+ capturer->SetFramePreprocessor(std::move(frame_preprocessor));
+ return capturer;
+ }
+
+ auto capturer = std::make_unique<test::FrameGeneratorCapturer>(
+ clock_,
+ absl::get<std::unique_ptr<test::FrameGeneratorInterface>>(
+ std::move(source)),
+ video_config.fps, *task_queue_factory_);
+ capturer->SetFramePreprocessor(std::move(frame_preprocessor));
+ capturer->Init();
+ return capturer;
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/media/media_helper.h b/third_party/libwebrtc/test/pc/e2e/media/media_helper.h
new file mode 100644
index 0000000000..2d163d009e
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/media/media_helper.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_MEDIA_MEDIA_HELPER_H_
+#define TEST_PC_E2E_MEDIA_MEDIA_HELPER_H_
+
+#include <memory>
+#include <vector>
+
+#include "api/test/frame_generator_interface.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/pclf/peer_configurer.h"
+#include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h"
+#include "test/pc/e2e/media/test_video_capturer_video_track_source.h"
+#include "test/pc/e2e/test_peer.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+class MediaHelper {
+ public:
+ MediaHelper(VideoQualityAnalyzerInjectionHelper*
+ video_quality_analyzer_injection_helper,
+ TaskQueueFactory* task_queue_factory,
+ Clock* clock)
+ : clock_(clock),
+ task_queue_factory_(task_queue_factory),
+ video_quality_analyzer_injection_helper_(
+ video_quality_analyzer_injection_helper) {}
+
+ void MaybeAddAudio(TestPeer* peer);
+
+ std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>>
+ MaybeAddVideo(TestPeer* peer);
+
+ private:
+ std::unique_ptr<test::TestVideoCapturer> CreateVideoCapturer(
+ const VideoConfig& video_config,
+ PeerConfigurer::VideoSource source,
+ std::unique_ptr<test::TestVideoCapturer::FramePreprocessor>
+ frame_preprocessor);
+
+ Clock* const clock_;
+ TaskQueueFactory* const task_queue_factory_;
+ VideoQualityAnalyzerInjectionHelper* video_quality_analyzer_injection_helper_;
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_MEDIA_MEDIA_HELPER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/media/test_video_capturer_video_track_source.h b/third_party/libwebrtc/test/pc/e2e/media/test_video_capturer_video_track_source.h
new file mode 100644
index 0000000000..c883a2e8e9
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/media/test_video_capturer_video_track_source.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_MEDIA_TEST_VIDEO_CAPTURER_VIDEO_TRACK_SOURCE_H_
+#define TEST_PC_E2E_MEDIA_TEST_VIDEO_CAPTURER_VIDEO_TRACK_SOURCE_H_
+
+#include <memory>
+#include <utility>
+
+#include "api/video/video_frame.h"
+#include "api/video/video_source_interface.h"
+#include "pc/video_track_source.h"
+#include "test/test_video_capturer.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+class TestVideoCapturerVideoTrackSource : public VideoTrackSource {
+ public:
+ TestVideoCapturerVideoTrackSource(
+ std::unique_ptr<test::TestVideoCapturer> video_capturer,
+ bool is_screencast)
+ : VideoTrackSource(/*remote=*/false),
+ video_capturer_(std::move(video_capturer)),
+ is_screencast_(is_screencast) {}
+
+ ~TestVideoCapturerVideoTrackSource() = default;
+
+ void Start() { SetState(kLive); }
+
+ void Stop() { SetState(kMuted); }
+
+ bool is_screencast() const override { return is_screencast_; }
+
+ protected:
+ rtc::VideoSourceInterface<VideoFrame>* source() override {
+ return video_capturer_.get();
+ }
+
+ private:
+ std::unique_ptr<test::TestVideoCapturer> video_capturer_;
+ const bool is_screencast_;
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_MEDIA_TEST_VIDEO_CAPTURER_VIDEO_TRACK_SOURCE_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/metric_metadata_keys.h b/third_party/libwebrtc/test/pc/e2e/metric_metadata_keys.h
new file mode 100644
index 0000000000..fbcd3b90fe
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/metric_metadata_keys.h
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef TEST_PC_E2E_METRIC_METADATA_KEYS_H_
+#define TEST_PC_E2E_METRIC_METADATA_KEYS_H_
+
+#include <string>
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+// All metadata fields are present only if applicable for particular metric.
+class MetricMetadataKey {
+ public:
+ // Represents on peer with whom the metric is associated.
+ static constexpr char kPeerMetadataKey[] = "peer";
+ // Represents sender of the media stream.
+ static constexpr char kSenderMetadataKey[] = "sender";
+ // Represents receiver of the media stream.
+ static constexpr char kReceiverMetadataKey[] = "receiver";
+ // Represents name of the audio stream.
+ static constexpr char kAudioStreamMetadataKey[] = "audio_stream";
+ // Represents name of the video stream.
+ static constexpr char kVideoStreamMetadataKey[] = "video_stream";
+ // Represents name of the sync group to which stream belongs.
+ static constexpr char kPeerSyncGroupMetadataKey[] = "peer_sync_group";
+ // Represents the test name (without any peer and stream data appended to it
+ // as it currently happens with the webrtc.test_metrics.Metric.test_case
+ // field). This metadata is temporary and it will be removed once this
+ // information is moved to webrtc.test_metrics.Metric.test_case.
+ // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey.
+ static constexpr char kExperimentalTestNameMetadataKey[] =
+ "experimental_test_name";
+ // Represents index of a video spatial layer to which metric belongs.
+ static constexpr char kSpatialLayerMetadataKey[] = "spatial_layer";
+
+ private:
+ MetricMetadataKey() = default;
+};
+
+// All metadata fields are presented only if applicable for particular metric.
+class SampleMetadataKey {
+ public:
+ // Represents a frame ID with which data point is associated.
+ static constexpr char kFrameIdMetadataKey[] = "frame_id";
+
+ private:
+ SampleMetadataKey() = default;
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_METRIC_METADATA_KEYS_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.cc b/third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.cc
new file mode 100644
index 0000000000..0bb28f0847
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.cc
@@ -0,0 +1,183 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/network_quality_metrics_reporter.h"
+
+#include <utility>
+
+#include "api/stats/rtc_stats.h"
+#include "api/stats/rtcstats_objects.h"
+#include "api/test/metrics/metric.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/event.h"
+#include "system_wrappers/include/field_trial.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+using ::webrtc::test::ImprovementDirection;
+using ::webrtc::test::Unit;
+
+constexpr TimeDelta kStatsWaitTimeout = TimeDelta::Seconds(1);
+
+// Field trial which controls whether to report standard-compliant bytes
+// sent/received per stream. If enabled, padding and headers are not included
+// in bytes sent or received.
+constexpr char kUseStandardBytesStats[] = "WebRTC-UseStandardBytesStats";
+
+} // namespace
+
+NetworkQualityMetricsReporter::NetworkQualityMetricsReporter(
+ EmulatedNetworkManagerInterface* alice_network,
+ EmulatedNetworkManagerInterface* bob_network,
+ test::MetricsLogger* metrics_logger)
+ : alice_network_(alice_network),
+ bob_network_(bob_network),
+ metrics_logger_(metrics_logger) {
+ RTC_CHECK(metrics_logger_);
+}
+
+void NetworkQualityMetricsReporter::Start(
+ absl::string_view test_case_name,
+ const TrackIdStreamInfoMap* /*reporter_helper*/) {
+ test_case_name_ = std::string(test_case_name);
+ // Check that network stats are clean before test execution.
+ EmulatedNetworkStats alice_stats = PopulateStats(alice_network_);
+ RTC_CHECK_EQ(alice_stats.overall_outgoing_stats.packets_sent, 0);
+ RTC_CHECK_EQ(alice_stats.overall_incoming_stats.packets_received, 0);
+ EmulatedNetworkStats bob_stats = PopulateStats(bob_network_);
+ RTC_CHECK_EQ(bob_stats.overall_outgoing_stats.packets_sent, 0);
+ RTC_CHECK_EQ(bob_stats.overall_incoming_stats.packets_received, 0);
+}
+
+void NetworkQualityMetricsReporter::OnStatsReports(
+ absl::string_view pc_label,
+ const rtc::scoped_refptr<const RTCStatsReport>& report) {
+ DataSize payload_received = DataSize::Zero();
+ DataSize payload_sent = DataSize::Zero();
+
+ auto inbound_stats = report->GetStatsOfType<RTCInboundRTPStreamStats>();
+ for (const auto& stat : inbound_stats) {
+ payload_received +=
+ DataSize::Bytes(stat->bytes_received.ValueOrDefault(0ul) +
+ stat->header_bytes_received.ValueOrDefault(0ul));
+ }
+
+ auto outbound_stats = report->GetStatsOfType<RTCOutboundRTPStreamStats>();
+ for (const auto& stat : outbound_stats) {
+ payload_sent +=
+ DataSize::Bytes(stat->bytes_sent.ValueOrDefault(0ul) +
+ stat->header_bytes_sent.ValueOrDefault(0ul));
+ }
+
+ MutexLock lock(&lock_);
+ PCStats& stats = pc_stats_[std::string(pc_label)];
+ stats.payload_received = payload_received;
+ stats.payload_sent = payload_sent;
+}
+
+void NetworkQualityMetricsReporter::StopAndReportResults() {
+ EmulatedNetworkStats alice_stats = PopulateStats(alice_network_);
+ EmulatedNetworkStats bob_stats = PopulateStats(bob_network_);
+ int64_t alice_packets_loss =
+ alice_stats.overall_outgoing_stats.packets_sent -
+ bob_stats.overall_incoming_stats.packets_received;
+ int64_t bob_packets_loss =
+ bob_stats.overall_outgoing_stats.packets_sent -
+ alice_stats.overall_incoming_stats.packets_received;
+ ReportStats("alice", alice_stats, alice_packets_loss);
+ ReportStats("bob", bob_stats, bob_packets_loss);
+
+ if (!webrtc::field_trial::IsEnabled(kUseStandardBytesStats)) {
+ RTC_LOG(LS_ERROR)
+ << "Non-standard GetStats; \"payload\" counts include RTP headers";
+ }
+
+ MutexLock lock(&lock_);
+ for (const auto& pair : pc_stats_) {
+ ReportPCStats(pair.first, pair.second);
+ }
+}
+
+EmulatedNetworkStats NetworkQualityMetricsReporter::PopulateStats(
+ EmulatedNetworkManagerInterface* network) {
+ rtc::Event wait;
+ EmulatedNetworkStats stats;
+ network->GetStats([&](EmulatedNetworkStats s) {
+ stats = std::move(s);
+ wait.Set();
+ });
+ bool stats_received = wait.Wait(kStatsWaitTimeout);
+ RTC_CHECK(stats_received);
+ return stats;
+}
+
+void NetworkQualityMetricsReporter::ReportStats(
+ const std::string& network_label,
+ const EmulatedNetworkStats& stats,
+ int64_t packet_loss) {
+ metrics_logger_->LogSingleValueMetric(
+ "bytes_sent", GetTestCaseName(network_label),
+ stats.overall_outgoing_stats.bytes_sent.bytes(), Unit::kBytes,
+ ImprovementDirection::kNeitherIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "packets_sent", GetTestCaseName(network_label),
+ stats.overall_outgoing_stats.packets_sent, Unit::kUnitless,
+ ImprovementDirection::kNeitherIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "average_send_rate", GetTestCaseName(network_label),
+ stats.overall_outgoing_stats.packets_sent >= 2
+ ? stats.overall_outgoing_stats.AverageSendRate().kbps<double>()
+ : 0,
+ Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "bytes_discarded_no_receiver", GetTestCaseName(network_label),
+ stats.overall_incoming_stats.bytes_discarded_no_receiver.bytes(),
+ Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "packets_discarded_no_receiver", GetTestCaseName(network_label),
+ stats.overall_incoming_stats.packets_discarded_no_receiver,
+ Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "bytes_received", GetTestCaseName(network_label),
+ stats.overall_incoming_stats.bytes_received.bytes(), Unit::kBytes,
+ ImprovementDirection::kNeitherIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "packets_received", GetTestCaseName(network_label),
+ stats.overall_incoming_stats.packets_received, Unit::kUnitless,
+ ImprovementDirection::kNeitherIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "average_receive_rate", GetTestCaseName(network_label),
+ stats.overall_incoming_stats.packets_received >= 2
+ ? stats.overall_incoming_stats.AverageReceiveRate().kbps<double>()
+ : 0,
+ Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "sent_packets_loss", GetTestCaseName(network_label), packet_loss,
+ Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
+}
+
+void NetworkQualityMetricsReporter::ReportPCStats(const std::string& pc_label,
+ const PCStats& stats) {
+ metrics_logger_->LogSingleValueMetric(
+ "payload_bytes_received", pc_label, stats.payload_received.bytes(),
+ Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
+ metrics_logger_->LogSingleValueMetric(
+ "payload_bytes_sent", pc_label, stats.payload_sent.bytes(), Unit::kBytes,
+ ImprovementDirection::kNeitherIsBetter);
+}
+
+std::string NetworkQualityMetricsReporter::GetTestCaseName(
+ const std::string& network_label) const {
+ return test_case_name_ + "/" + network_label;
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.h b/third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.h
new file mode 100644
index 0000000000..ed894bcf54
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/network_quality_metrics_reporter.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_
+#define TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_
+
+#include <memory>
+#include <string>
+
+#include "absl/strings/string_view.h"
+#include "api/test/metrics/metrics_logger.h"
+#include "api/test/network_emulation_manager.h"
+#include "api/test/peerconnection_quality_test_fixture.h"
+#include "api/test/track_id_stream_info_map.h"
+#include "api/units/data_size.h"
+#include "rtc_base/synchronization/mutex.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+class NetworkQualityMetricsReporter
+ : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter {
+ public:
+ NetworkQualityMetricsReporter(EmulatedNetworkManagerInterface* alice_network,
+ EmulatedNetworkManagerInterface* bob_network,
+ test::MetricsLogger* metrics_logger);
+ ~NetworkQualityMetricsReporter() override = default;
+
+ // Network stats must be empty when this method will be invoked.
+ void Start(absl::string_view test_case_name,
+ const TrackIdStreamInfoMap* reporter_helper) override;
+ void OnStatsReports(
+ absl::string_view pc_label,
+ const rtc::scoped_refptr<const RTCStatsReport>& report) override;
+ void StopAndReportResults() override;
+
+ private:
+ struct PCStats {
+ // TODO(nisse): Separate audio and video counters. Depends on standard stat
+ // counters, enabled by field trial "WebRTC-UseStandardBytesStats".
+ DataSize payload_received = DataSize::Zero();
+ DataSize payload_sent = DataSize::Zero();
+ };
+
+ static EmulatedNetworkStats PopulateStats(
+ EmulatedNetworkManagerInterface* network);
+ void ReportStats(const std::string& network_label,
+ const EmulatedNetworkStats& stats,
+ int64_t packet_loss);
+ void ReportPCStats(const std::string& pc_label, const PCStats& stats);
+ std::string GetTestCaseName(const std::string& network_label) const;
+
+ std::string test_case_name_;
+
+ EmulatedNetworkManagerInterface* const alice_network_;
+ EmulatedNetworkManagerInterface* const bob_network_;
+ test::MetricsLogger* const metrics_logger_;
+ Mutex lock_;
+ std::map<std::string, PCStats> pc_stats_ RTC_GUARDED_BY(lock_);
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_NETWORK_QUALITY_METRICS_REPORTER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/peer_connection_e2e_smoke_test.cc b/third_party/libwebrtc/test/pc/e2e/peer_connection_e2e_smoke_test.cc
new file mode 100644
index 0000000000..0e7993e5be
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/peer_connection_e2e_smoke_test.cc
@@ -0,0 +1,536 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <cstdint>
+#include <memory>
+#include <string>
+
+#include "api/media_stream_interface.h"
+#include "api/test/create_network_emulation_manager.h"
+#include "api/test/create_peer_connection_quality_test_frame_generator.h"
+#include "api/test/create_peerconnection_quality_test_fixture.h"
+#include "api/test/metrics/global_metrics_logger_and_exporter.h"
+#include "api/test/network_emulation_manager.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/pclf/media_quality_test_params.h"
+#include "api/test/pclf/peer_configurer.h"
+#include "api/test/peerconnection_quality_test_fixture.h"
+#include "call/simulated_network.h"
+#include "system_wrappers/include/field_trial.h"
+#include "test/field_trial.h"
+#include "test/gtest.h"
+#include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
+#include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h"
+#include "test/testsupport/file_utils.h"
+
+#if defined(WEBRTC_MAC) || defined(WEBRTC_IOS)
+#include "modules/video_coding/codecs/test/objc_codec_factory_helper.h" // nogncheck
+#endif
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+class PeerConnectionE2EQualityTestSmokeTest : public ::testing::Test {
+ public:
+ void SetUp() override {
+ network_emulation_ = CreateNetworkEmulationManager();
+ auto video_quality_analyzer = std::make_unique<DefaultVideoQualityAnalyzer>(
+ network_emulation_->time_controller()->GetClock(),
+ test::GetGlobalMetricsLogger());
+ video_quality_analyzer_ = video_quality_analyzer.get();
+ fixture_ = CreatePeerConnectionE2EQualityTestFixture(
+ testing::UnitTest::GetInstance()->current_test_info()->name(),
+ *network_emulation_->time_controller(),
+ /*audio_quality_analyzer=*/nullptr, std::move(video_quality_analyzer));
+ }
+
+ std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*>
+ CreateNetwork() {
+ EmulatedNetworkNode* alice_node = network_emulation_->CreateEmulatedNode(
+ std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig()));
+ EmulatedNetworkNode* bob_node = network_emulation_->CreateEmulatedNode(
+ std::make_unique<SimulatedNetwork>(BuiltInNetworkBehaviorConfig()));
+
+ EmulatedEndpoint* alice_endpoint =
+ network_emulation_->CreateEndpoint(EmulatedEndpointConfig());
+ EmulatedEndpoint* bob_endpoint =
+ network_emulation_->CreateEndpoint(EmulatedEndpointConfig());
+
+ network_emulation_->CreateRoute(alice_endpoint, {alice_node}, bob_endpoint);
+ network_emulation_->CreateRoute(bob_endpoint, {bob_node}, alice_endpoint);
+
+ EmulatedNetworkManagerInterface* alice_network =
+ network_emulation_->CreateEmulatedNetworkManagerInterface(
+ {alice_endpoint});
+ EmulatedNetworkManagerInterface* bob_network =
+ network_emulation_->CreateEmulatedNetworkManagerInterface(
+ {bob_endpoint});
+
+ return std::make_pair(alice_network, bob_network);
+ }
+
+ void AddPeer(EmulatedNetworkManagerInterface* network,
+ rtc::FunctionView<void(PeerConfigurer*)> update_configurer) {
+ auto configurer =
+ std::make_unique<PeerConfigurer>(network->network_dependencies());
+ update_configurer(configurer.get());
+ fixture_->AddPeer(std::move(configurer));
+ }
+
+ void RunAndCheckEachVideoStreamReceivedFrames(const RunParams& run_params) {
+ fixture_->Run(run_params);
+
+ EXPECT_GE(fixture_->GetRealTestDuration(), run_params.run_duration);
+ VideoStreamsInfo known_streams = video_quality_analyzer_->GetKnownStreams();
+ for (const StatsKey& stream_key : known_streams.GetStatsKeys()) {
+ FrameCounters stream_conters =
+ video_quality_analyzer_->GetPerStreamCounters().at(stream_key);
+ // On some devices the pipeline can be too slow, so we actually can't
+ // force real constraints here. Lets just check, that at least 1
+ // frame passed whole pipeline.
+ int64_t expected_min_fps = run_params.run_duration.seconds() * 15;
+ EXPECT_GE(stream_conters.captured, expected_min_fps)
+ << stream_key.ToString();
+ EXPECT_GE(stream_conters.pre_encoded, 1) << stream_key.ToString();
+ EXPECT_GE(stream_conters.encoded, 1) << stream_key.ToString();
+ EXPECT_GE(stream_conters.received, 1) << stream_key.ToString();
+ EXPECT_GE(stream_conters.decoded, 1) << stream_key.ToString();
+ EXPECT_GE(stream_conters.rendered, 1) << stream_key.ToString();
+ }
+ }
+
+ NetworkEmulationManager* network_emulation() {
+ return network_emulation_.get();
+ }
+
+ PeerConnectionE2EQualityTestFixture* fixture() { return fixture_.get(); }
+
+ private:
+ std::unique_ptr<NetworkEmulationManager> network_emulation_;
+ DefaultVideoQualityAnalyzer* video_quality_analyzer_;
+ std::unique_ptr<PeerConnectionE2EQualityTestFixture> fixture_;
+};
+
+// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
+#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
+#define MAYBE_Smoke DISABLED_Smoke
+#else
+#define MAYBE_Smoke Smoke
+#endif
+TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Smoke) {
+ std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*>
+ network_links = CreateNetwork();
+ AddPeer(network_links.first, [](PeerConfigurer* alice) {
+ VideoConfig video(160, 120, 15);
+ video.stream_label = "alice-video";
+ video.sync_group = "alice-media";
+ alice->AddVideoConfig(std::move(video));
+
+ AudioConfig audio;
+ audio.stream_label = "alice-audio";
+ audio.mode = AudioConfig::Mode::kFile;
+ audio.input_file_name =
+ test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
+ audio.sampling_frequency_in_hz = 48000;
+ audio.sync_group = "alice-media";
+ alice->SetAudioConfig(std::move(audio));
+ alice->SetVideoCodecs(
+ {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})});
+
+ alice->SetUseFlexFEC(true);
+ alice->SetUseUlpFEC(true);
+ alice->SetVideoEncoderBitrateMultiplier(1.1);
+ });
+ AddPeer(network_links.second, [](PeerConfigurer* charlie) {
+ charlie->SetName("charlie");
+ VideoConfig video(160, 120, 15);
+ video.stream_label = "charlie-video";
+ video.temporal_layers_count = 2;
+ charlie->AddVideoConfig(std::move(video));
+
+ AudioConfig audio;
+ audio.stream_label = "charlie-audio";
+ audio.mode = AudioConfig::Mode::kFile;
+ audio.input_file_name =
+ test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
+ charlie->SetAudioConfig(std::move(audio));
+ charlie->SetVideoCodecs(
+ {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})});
+
+ charlie->SetUseFlexFEC(true);
+ charlie->SetUseUlpFEC(true);
+ charlie->SetVideoEncoderBitrateMultiplier(1.1);
+ });
+ fixture()->AddQualityMetricsReporter(
+ std::make_unique<StatsBasedNetworkQualityMetricsReporter>(
+ std::map<std::string, std::vector<EmulatedEndpoint*>>(
+ {{"alice", network_links.first->endpoints()},
+ {"charlie", network_links.second->endpoints()}}),
+ network_emulation(), test::GetGlobalMetricsLogger()));
+ RunParams run_params(TimeDelta::Seconds(2));
+ run_params.enable_flex_fec_support = true;
+ RunAndCheckEachVideoStreamReceivedFrames(run_params);
+}
+
+// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
+#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
+#define MAYBE_Smoke DISABLED_Smoke
+#else
+#define MAYBE_SendAndReceivePacketsOnOneThread \
+ SmokeSendAndReceivePacketsOnOneThread
+#endif
+// Only use the network thread for sending and receiving packets.
+// The one and only network thread is used as a worker thread in all
+// PeerConnections. Pacing when sending packets is done on the worker thread.
+// See bugs.webrtc.org/14502.
+TEST_F(PeerConnectionE2EQualityTestSmokeTest,
+ MAYBE_SendAndReceivePacketsOnOneThread) {
+ test::ScopedFieldTrials trials(
+ std::string(field_trial::GetFieldTrialString()) +
+ "WebRTC-SendPacketsOnWorkerThread/Enabled/");
+
+ std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*>
+ network_links = CreateNetwork();
+ AddPeer(network_links.first, [](PeerConfigurer* alice) {
+ // Peerconnection use the network thread as the worker thread.
+ alice->SetUseNetworkThreadAsWorkerThread();
+ VideoConfig video(160, 120, 15);
+ video.stream_label = "alice-video";
+ video.sync_group = "alice-media";
+ alice->AddVideoConfig(std::move(video));
+
+ AudioConfig audio;
+ audio.stream_label = "alice-audio";
+ audio.mode = AudioConfig::Mode::kFile;
+ audio.input_file_name =
+ test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
+ audio.sampling_frequency_in_hz = 48000;
+ audio.sync_group = "alice-media";
+ alice->SetAudioConfig(std::move(audio));
+ alice->SetVideoCodecs(
+ {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})});
+ });
+ AddPeer(network_links.second, [](PeerConfigurer* charlie) {
+ // Peerconnection use the network thread as the worker thread.
+ charlie->SetUseNetworkThreadAsWorkerThread();
+ charlie->SetName("charlie");
+ VideoConfig video(160, 120, 15);
+ video.stream_label = "charlie-video";
+ video.temporal_layers_count = 2;
+ charlie->AddVideoConfig(std::move(video));
+
+ AudioConfig audio;
+ audio.stream_label = "charlie-audio";
+ audio.mode = AudioConfig::Mode::kFile;
+ audio.input_file_name =
+ test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
+ charlie->SetAudioConfig(std::move(audio));
+ charlie->SetVideoCodecs(
+ {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})});
+ charlie->SetVideoEncoderBitrateMultiplier(1.1);
+ });
+ fixture()->AddQualityMetricsReporter(
+ std::make_unique<StatsBasedNetworkQualityMetricsReporter>(
+ std::map<std::string, std::vector<EmulatedEndpoint*>>(
+ {{"alice", network_links.first->endpoints()},
+ {"charlie", network_links.second->endpoints()}}),
+ network_emulation(), test::GetGlobalMetricsLogger()));
+ RunParams run_params(TimeDelta::Seconds(2));
+ RunAndCheckEachVideoStreamReceivedFrames(run_params);
+}
+
+#if defined(WEBRTC_MAC) || defined(WEBRTC_IOS)
+TEST_F(PeerConnectionE2EQualityTestSmokeTest, SmokeH264) {
+ std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*>
+ network_links = CreateNetwork();
+
+ AddPeer(network_links.first, [](PeerConfigurer* alice) {
+ VideoConfig video(160, 120, 15);
+ video.stream_label = "alice-video";
+ video.sync_group = "alice-media";
+ alice->AddVideoConfig(std::move(video));
+
+ AudioConfig audio;
+ audio.stream_label = "alice-audio";
+ audio.mode = AudioConfig::Mode::kFile;
+ audio.input_file_name =
+ test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
+ audio.sampling_frequency_in_hz = 48000;
+ audio.sync_group = "alice-media";
+ alice->SetAudioConfig(std::move(audio));
+ alice->SetVideoCodecs({VideoCodecConfig(cricket::kH264CodecName)});
+ alice->SetVideoEncoderFactory(webrtc::test::CreateObjCEncoderFactory());
+ alice->SetVideoDecoderFactory(webrtc::test::CreateObjCDecoderFactory());
+ });
+ AddPeer(network_links.second, [](PeerConfigurer* charlie) {
+ charlie->SetName("charlie");
+ VideoConfig video(160, 120, 15);
+ video.stream_label = "charlie-video";
+ video.temporal_layers_count = 2;
+ charlie->AddVideoConfig(std::move(video));
+
+ AudioConfig audio;
+ audio.stream_label = "charlie-audio";
+ audio.mode = AudioConfig::Mode::kFile;
+ audio.input_file_name =
+ test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
+ charlie->SetAudioConfig(std::move(audio));
+ charlie->SetVideoCodecs({VideoCodecConfig(cricket::kH264CodecName)});
+ charlie->SetVideoEncoderFactory(webrtc::test::CreateObjCEncoderFactory());
+ charlie->SetVideoDecoderFactory(webrtc::test::CreateObjCDecoderFactory());
+ });
+
+ fixture()->AddQualityMetricsReporter(
+ std::make_unique<StatsBasedNetworkQualityMetricsReporter>(
+ std::map<std::string, std::vector<EmulatedEndpoint*>>(
+ {{"alice", network_links.first->endpoints()},
+ {"charlie", network_links.second->endpoints()}}),
+ network_emulation(), test::GetGlobalMetricsLogger()));
+ RunParams run_params(TimeDelta::Seconds(2));
+ run_params.enable_flex_fec_support = true;
+ RunAndCheckEachVideoStreamReceivedFrames(run_params);
+}
+#endif
+
+// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
+#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
+#define MAYBE_ChangeNetworkConditions DISABLED_ChangeNetworkConditions
+#else
+#define MAYBE_ChangeNetworkConditions ChangeNetworkConditions
+#endif
+TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_ChangeNetworkConditions) {
+ NetworkEmulationManager::SimulatedNetworkNode alice_node =
+ network_emulation()
+ ->NodeBuilder()
+ .config(BuiltInNetworkBehaviorConfig())
+ .Build();
+ NetworkEmulationManager::SimulatedNetworkNode bob_node =
+ network_emulation()
+ ->NodeBuilder()
+ .config(BuiltInNetworkBehaviorConfig())
+ .Build();
+
+ EmulatedEndpoint* alice_endpoint =
+ network_emulation()->CreateEndpoint(EmulatedEndpointConfig());
+ EmulatedEndpoint* bob_endpoint =
+ network_emulation()->CreateEndpoint(EmulatedEndpointConfig());
+
+ network_emulation()->CreateRoute(alice_endpoint, {alice_node.node},
+ bob_endpoint);
+ network_emulation()->CreateRoute(bob_endpoint, {bob_node.node},
+ alice_endpoint);
+
+ EmulatedNetworkManagerInterface* alice_network =
+ network_emulation()->CreateEmulatedNetworkManagerInterface(
+ {alice_endpoint});
+ EmulatedNetworkManagerInterface* bob_network =
+ network_emulation()->CreateEmulatedNetworkManagerInterface(
+ {bob_endpoint});
+
+ AddPeer(alice_network, [](PeerConfigurer* alice) {
+ VideoConfig video(160, 120, 15);
+ video.stream_label = "alice-video";
+ video.sync_group = "alice-media";
+ alice->AddVideoConfig(std::move(video));
+ alice->SetVideoCodecs(
+ {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})});
+
+ alice->SetUseFlexFEC(true);
+ alice->SetUseUlpFEC(true);
+ alice->SetVideoEncoderBitrateMultiplier(1.1);
+ });
+ AddPeer(bob_network, [](PeerConfigurer* bob) {
+ bob->SetVideoCodecs(
+ {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})});
+
+ bob->SetUseFlexFEC(true);
+ bob->SetUseUlpFEC(true);
+ bob->SetVideoEncoderBitrateMultiplier(1.1);
+ });
+ fixture()->AddQualityMetricsReporter(
+ std::make_unique<StatsBasedNetworkQualityMetricsReporter>(
+ std::map<std::string, std::vector<EmulatedEndpoint*>>(
+ {{"alice", alice_network->endpoints()},
+ {"bob", bob_network->endpoints()}}),
+ network_emulation(), test::GetGlobalMetricsLogger()));
+
+ fixture()->ExecuteAt(TimeDelta::Seconds(1), [alice_node](TimeDelta) {
+ BuiltInNetworkBehaviorConfig config;
+ config.loss_percent = 5;
+ alice_node.simulation->SetConfig(config);
+ });
+
+ RunParams run_params(TimeDelta::Seconds(2));
+ run_params.enable_flex_fec_support = true;
+ RunAndCheckEachVideoStreamReceivedFrames(run_params);
+}
+
+// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
+#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
+#define MAYBE_Screenshare DISABLED_Screenshare
+#else
+#define MAYBE_Screenshare Screenshare
+#endif
+TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Screenshare) {
+ std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*>
+ network_links = CreateNetwork();
+ AddPeer(network_links.first, [](PeerConfigurer* alice) {
+ VideoConfig screenshare(320, 180, 30);
+ screenshare.stream_label = "alice-screenshare";
+ screenshare.content_hint = VideoTrackInterface::ContentHint::kText;
+ ScreenShareConfig screen_share_config =
+ ScreenShareConfig(TimeDelta::Seconds(2));
+ screen_share_config.scrolling_params =
+ ScrollingParams{.duration = TimeDelta::Millis(1800)};
+ auto screen_share_frame_generator =
+ CreateScreenShareFrameGenerator(screenshare, screen_share_config);
+ alice->AddVideoConfig(std::move(screenshare),
+ std::move(screen_share_frame_generator));
+ });
+ AddPeer(network_links.second, [](PeerConfigurer* bob) {});
+ RunAndCheckEachVideoStreamReceivedFrames(RunParams(TimeDelta::Seconds(2)));
+}
+
+// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
+#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
+#define MAYBE_Echo DISABLED_Echo
+#else
+#define MAYBE_Echo Echo
+#endif
+TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Echo) {
+ std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*>
+ network_links = CreateNetwork();
+ AddPeer(network_links.first, [](PeerConfigurer* alice) {
+ AudioConfig audio;
+ audio.stream_label = "alice-audio";
+ audio.mode = AudioConfig::Mode::kFile;
+ audio.input_file_name =
+ test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
+ audio.sampling_frequency_in_hz = 48000;
+ alice->SetAudioConfig(std::move(audio));
+ });
+ AddPeer(network_links.second, [](PeerConfigurer* bob) {
+ AudioConfig audio;
+ audio.stream_label = "bob-audio";
+ audio.mode = AudioConfig::Mode::kFile;
+ audio.input_file_name =
+ test::ResourcePath("pc_quality_smoke_test_bob_source", "wav");
+ bob->SetAudioConfig(std::move(audio));
+ });
+ RunParams run_params(TimeDelta::Seconds(2));
+ run_params.echo_emulation_config = EchoEmulationConfig();
+ RunAndCheckEachVideoStreamReceivedFrames(run_params);
+}
+
+// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
+#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
+#define MAYBE_Simulcast DISABLED_Simulcast
+#else
+#define MAYBE_Simulcast Simulcast
+#endif
+TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Simulcast) {
+ std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*>
+ network_links = CreateNetwork();
+ AddPeer(network_links.first, [](PeerConfigurer* alice) {
+ VideoConfig simulcast(1280, 720, 15);
+ simulcast.stream_label = "alice-simulcast";
+ simulcast.simulcast_config = VideoSimulcastConfig(2);
+ simulcast.emulated_sfu_config = EmulatedSFUConfig(0);
+ alice->AddVideoConfig(std::move(simulcast));
+
+ AudioConfig audio;
+ audio.stream_label = "alice-audio";
+ audio.mode = AudioConfig::Mode::kFile;
+ audio.input_file_name =
+ test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
+ alice->SetAudioConfig(std::move(audio));
+ });
+ AddPeer(network_links.second, [](PeerConfigurer* bob) {});
+ RunParams run_params(TimeDelta::Seconds(2));
+ RunAndCheckEachVideoStreamReceivedFrames(run_params);
+}
+
+// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
+#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
+#define MAYBE_Svc DISABLED_Svc
+#else
+#define MAYBE_Svc Svc
+#endif
+TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_Svc) {
+ std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*>
+ network_links = CreateNetwork();
+ AddPeer(network_links.first, [](PeerConfigurer* alice) {
+ VideoConfig simulcast("alice-svc", 1280, 720, 15);
+ // Because we have network with packets loss we can analyze only the
+ // highest spatial layer in SVC mode.
+ simulcast.simulcast_config = VideoSimulcastConfig(2);
+ simulcast.emulated_sfu_config = EmulatedSFUConfig(1);
+ alice->AddVideoConfig(std::move(simulcast));
+
+ AudioConfig audio("alice-audio");
+ audio.mode = AudioConfig::Mode::kFile;
+ audio.input_file_name =
+ test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
+ alice->SetAudioConfig(std::move(audio));
+ alice->SetVideoCodecs({VideoCodecConfig(cricket::kVp9CodecName)});
+ });
+ AddPeer(network_links.second, [](PeerConfigurer* bob) {
+ bob->SetVideoCodecs({VideoCodecConfig(cricket::kVp9CodecName)});
+ });
+ RunParams run_params(TimeDelta::Seconds(2));
+ RunAndCheckEachVideoStreamReceivedFrames(run_params);
+}
+
+// IOS debug builds can be quite slow, disabling to avoid issues with timeouts.
+#if defined(WEBRTC_IOS) && defined(WEBRTC_ARCH_ARM64) && !defined(NDEBUG)
+#define MAYBE_HighBitrate DISABLED_HighBitrate
+#else
+#define MAYBE_HighBitrate HighBitrate
+#endif
+TEST_F(PeerConnectionE2EQualityTestSmokeTest, MAYBE_HighBitrate) {
+ std::pair<EmulatedNetworkManagerInterface*, EmulatedNetworkManagerInterface*>
+ network_links = CreateNetwork();
+ AddPeer(network_links.first, [](PeerConfigurer* alice) {
+ BitrateSettings bitrate_settings;
+ bitrate_settings.start_bitrate_bps = 3'000'000;
+ bitrate_settings.max_bitrate_bps = 3'000'000;
+ alice->SetBitrateSettings(bitrate_settings);
+ VideoConfig video(800, 600, 15);
+ video.stream_label = "alice-video";
+ RtpEncodingParameters encoding_parameters;
+ encoding_parameters.min_bitrate_bps = 500'000;
+ encoding_parameters.max_bitrate_bps = 3'000'000;
+ video.encoding_params.push_back(std::move(encoding_parameters));
+ alice->AddVideoConfig(std::move(video));
+
+ AudioConfig audio;
+ audio.stream_label = "alice-audio";
+ audio.mode = AudioConfig::Mode::kFile;
+ audio.input_file_name =
+ test::ResourcePath("pc_quality_smoke_test_alice_source", "wav");
+ audio.sampling_frequency_in_hz = 48000;
+ alice->SetAudioConfig(std::move(audio));
+ alice->SetVideoCodecs(
+ {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})});
+ });
+ AddPeer(network_links.second, [](PeerConfigurer* bob) {
+ bob->SetVideoCodecs(
+ {VideoCodecConfig(cricket::kVp9CodecName, {{"profile-id", "0"}})});
+ });
+ RunParams run_params(TimeDelta::Seconds(2));
+ RunAndCheckEachVideoStreamReceivedFrames(run_params);
+}
+
+} // namespace
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.cc b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.cc
new file mode 100644
index 0000000000..83613118f9
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.cc
@@ -0,0 +1,763 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/peer_connection_quality_test.h"
+
+#include <algorithm>
+#include <memory>
+#include <set>
+#include <utility>
+
+#include "absl/strings/string_view.h"
+#include "api/jsep.h"
+#include "api/media_stream_interface.h"
+#include "api/peer_connection_interface.h"
+#include "api/rtc_event_log/rtc_event_log.h"
+#include "api/rtc_event_log_output_file.h"
+#include "api/scoped_refptr.h"
+#include "api/test/metrics/metric.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/pclf/peer_configurer.h"
+#include "api/test/time_controller.h"
+#include "api/test/video_quality_analyzer_interface.h"
+#include "pc/sdp_utils.h"
+#include "pc/test/mock_peer_connection_observers.h"
+#include "rtc_base/gunit.h"
+#include "rtc_base/numerics/safe_conversions.h"
+#include "rtc_base/strings/string_builder.h"
+#include "rtc_base/task_queue_for_test.h"
+#include "system_wrappers/include/cpu_info.h"
+#include "system_wrappers/include/field_trial.h"
+#include "test/field_trial.h"
+#include "test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h"
+#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
+#include "test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h"
+#include "test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h"
+#include "test/pc/e2e/cross_media_metrics_reporter.h"
+#include "test/pc/e2e/metric_metadata_keys.h"
+#include "test/pc/e2e/peer_params_preprocessor.h"
+#include "test/pc/e2e/stats_poller.h"
+#include "test/pc/e2e/test_peer_factory.h"
+#include "test/testsupport/file_utils.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+using ::webrtc::test::ImprovementDirection;
+using ::webrtc::test::Unit;
+
+constexpr TimeDelta kDefaultTimeout = TimeDelta::Seconds(10);
+constexpr char kSignalThreadName[] = "signaling_thread";
+// 1 signaling, 2 network, 2 worker and 2 extra for codecs etc.
+constexpr int kPeerConnectionUsedThreads = 7;
+// Framework has extra thread for network layer and extra thread for peer
+// connection stats polling.
+constexpr int kFrameworkUsedThreads = 2;
+constexpr int kMaxVideoAnalyzerThreads = 8;
+
+constexpr TimeDelta kStatsUpdateInterval = TimeDelta::Seconds(1);
+
+constexpr TimeDelta kAliveMessageLogInterval = TimeDelta::Seconds(30);
+
+constexpr TimeDelta kQuickTestModeRunDuration = TimeDelta::Millis(100);
+
+// Field trials to enable Flex FEC advertising and receiving.
+constexpr char kFlexFecEnabledFieldTrials[] =
+ "WebRTC-FlexFEC-03-Advertised/Enabled/WebRTC-FlexFEC-03/Enabled/";
+constexpr char kUseStandardsBytesStats[] =
+ "WebRTC-UseStandardBytesStats/Enabled/";
+
+class FixturePeerConnectionObserver : public MockPeerConnectionObserver {
+ public:
+ // `on_track_callback` will be called when any new track will be added to peer
+ // connection.
+ // `on_connected_callback` will be called when peer connection will come to
+ // either connected or completed state. Client should notice that in the case
+ // of reconnect this callback can be called again, so it should be tolerant
+ // to such behavior.
+ FixturePeerConnectionObserver(
+ std::function<void(rtc::scoped_refptr<RtpTransceiverInterface>)>
+ on_track_callback,
+ std::function<void()> on_connected_callback)
+ : on_track_callback_(std::move(on_track_callback)),
+ on_connected_callback_(std::move(on_connected_callback)) {}
+
+ void OnTrack(
+ rtc::scoped_refptr<RtpTransceiverInterface> transceiver) override {
+ MockPeerConnectionObserver::OnTrack(transceiver);
+ on_track_callback_(transceiver);
+ }
+
+ void OnIceConnectionChange(
+ PeerConnectionInterface::IceConnectionState new_state) override {
+ MockPeerConnectionObserver::OnIceConnectionChange(new_state);
+ if (ice_connected_) {
+ on_connected_callback_();
+ }
+ }
+
+ private:
+ std::function<void(rtc::scoped_refptr<RtpTransceiverInterface>)>
+ on_track_callback_;
+ std::function<void()> on_connected_callback_;
+};
+
+void ValidateP2PSimulcastParams(
+ const std::vector<std::unique_ptr<PeerConfigurer>>& peers) {
+ for (size_t i = 0; i < peers.size(); ++i) {
+ Params* params = peers[i]->params();
+ ConfigurableParams* configurable_params = peers[i]->configurable_params();
+ for (const VideoConfig& video_config : configurable_params->video_configs) {
+ if (video_config.simulcast_config) {
+ // When we simulate SFU we support only one video codec.
+ RTC_CHECK_EQ(params->video_codecs.size(), 1)
+ << "Only 1 video codec is supported when simulcast is enabled in "
+ << "at least 1 video config";
+ }
+ }
+ }
+}
+
+} // namespace
+
+PeerConnectionE2EQualityTest::PeerConnectionE2EQualityTest(
+ std::string test_case_name,
+ TimeController& time_controller,
+ std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer,
+ std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer)
+ : PeerConnectionE2EQualityTest(std::move(test_case_name),
+ time_controller,
+ std::move(audio_quality_analyzer),
+ std::move(video_quality_analyzer),
+ /*metrics_logger_=*/nullptr) {}
+
+PeerConnectionE2EQualityTest::PeerConnectionE2EQualityTest(
+ std::string test_case_name,
+ TimeController& time_controller,
+ std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer,
+ std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer,
+ test::MetricsLogger* metrics_logger)
+ : time_controller_(time_controller),
+ task_queue_factory_(time_controller_.CreateTaskQueueFactory()),
+ test_case_name_(std::move(test_case_name)),
+ executor_(std::make_unique<TestActivitiesExecutor>(
+ time_controller_.GetClock())),
+ metrics_logger_(metrics_logger) {
+ // Create default video quality analyzer. We will always create an analyzer,
+ // even if there are no video streams, because it will be installed into video
+ // encoder/decoder factories.
+ if (video_quality_analyzer == nullptr) {
+ video_quality_analyzer = std::make_unique<DefaultVideoQualityAnalyzer>(
+ time_controller_.GetClock(), metrics_logger_);
+ }
+ if (field_trial::IsEnabled("WebRTC-VideoFrameTrackingIdAdvertised")) {
+ encoded_image_data_propagator_ =
+ std::make_unique<VideoFrameTrackingIdInjector>();
+ } else {
+ encoded_image_data_propagator_ =
+ std::make_unique<SingleProcessEncodedImageDataInjector>();
+ }
+ video_quality_analyzer_injection_helper_ =
+ std::make_unique<VideoQualityAnalyzerInjectionHelper>(
+ time_controller_.GetClock(), std::move(video_quality_analyzer),
+ encoded_image_data_propagator_.get(),
+ encoded_image_data_propagator_.get());
+
+ if (audio_quality_analyzer == nullptr) {
+ audio_quality_analyzer =
+ std::make_unique<DefaultAudioQualityAnalyzer>(metrics_logger_);
+ }
+ audio_quality_analyzer_.swap(audio_quality_analyzer);
+}
+
+void PeerConnectionE2EQualityTest::ExecuteAt(
+ TimeDelta target_time_since_start,
+ std::function<void(TimeDelta)> func) {
+ executor_->ScheduleActivity(target_time_since_start, absl::nullopt, func);
+}
+
+void PeerConnectionE2EQualityTest::ExecuteEvery(
+ TimeDelta initial_delay_since_start,
+ TimeDelta interval,
+ std::function<void(TimeDelta)> func) {
+ executor_->ScheduleActivity(initial_delay_since_start, interval, func);
+}
+
+void PeerConnectionE2EQualityTest::AddQualityMetricsReporter(
+ std::unique_ptr<QualityMetricsReporter> quality_metrics_reporter) {
+ quality_metrics_reporters_.push_back(std::move(quality_metrics_reporter));
+}
+
+PeerConnectionE2EQualityTest::PeerHandle* PeerConnectionE2EQualityTest::AddPeer(
+ std::unique_ptr<PeerConfigurer> configurer) {
+ peer_configurations_.push_back(std::move(configurer));
+ peer_handles_.push_back(PeerHandleImpl());
+ return &peer_handles_.back();
+}
+
+void PeerConnectionE2EQualityTest::Run(RunParams run_params) {
+ webrtc::webrtc_pc_e2e::PeerParamsPreprocessor params_preprocessor;
+ for (auto& peer_configuration : peer_configurations_) {
+ params_preprocessor.SetDefaultValuesForMissingParams(*peer_configuration);
+ params_preprocessor.ValidateParams(*peer_configuration);
+ }
+ ValidateP2PSimulcastParams(peer_configurations_);
+ RTC_CHECK_EQ(peer_configurations_.size(), 2)
+ << "Only peer to peer calls are allowed, please add 2 peers";
+
+ std::unique_ptr<PeerConfigurer> alice_configurer =
+ std::move(peer_configurations_[0]);
+ std::unique_ptr<PeerConfigurer> bob_configurer =
+ std::move(peer_configurations_[1]);
+ peer_configurations_.clear();
+
+ for (size_t i = 0;
+ i < bob_configurer->configurable_params()->video_configs.size(); ++i) {
+ // We support simulcast only from caller.
+ RTC_CHECK(!bob_configurer->configurable_params()
+ ->video_configs[i]
+ .simulcast_config)
+ << "Only simulcast stream from first peer is supported";
+ }
+
+ test::ScopedFieldTrials field_trials(GetFieldTrials(run_params));
+
+ // Print test summary
+ RTC_LOG(LS_INFO)
+ << "Media quality test: " << *alice_configurer->params()->name
+ << " will make a call to " << *bob_configurer->params()->name
+ << " with media video="
+ << !alice_configurer->configurable_params()->video_configs.empty()
+ << "; audio=" << alice_configurer->params()->audio_config.has_value()
+ << ". " << *bob_configurer->params()->name
+ << " will respond with media video="
+ << !bob_configurer->configurable_params()->video_configs.empty()
+ << "; audio=" << bob_configurer->params()->audio_config.has_value();
+
+ const std::unique_ptr<rtc::Thread> signaling_thread =
+ time_controller_.CreateThread(kSignalThreadName);
+ media_helper_ = std::make_unique<MediaHelper>(
+ video_quality_analyzer_injection_helper_.get(), task_queue_factory_.get(),
+ time_controller_.GetClock());
+
+ // Create a `task_queue_`.
+ task_queue_ = std::make_unique<webrtc::TaskQueueForTest>(
+ time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
+ "pc_e2e_quality_test", webrtc::TaskQueueFactory::Priority::NORMAL));
+
+ // Create call participants: Alice and Bob.
+ // Audio streams are intercepted in AudioDeviceModule, so if it is required to
+ // catch output of Alice's stream, Alice's output_dump_file_name should be
+ // passed to Bob's TestPeer setup as audio output file name.
+ absl::optional<RemotePeerAudioConfig> alice_remote_audio_config =
+ RemotePeerAudioConfig::Create(bob_configurer->params()->audio_config);
+ absl::optional<RemotePeerAudioConfig> bob_remote_audio_config =
+ RemotePeerAudioConfig::Create(alice_configurer->params()->audio_config);
+ // Copy Alice and Bob video configs, subscriptions and names to correctly pass
+ // them into lambdas.
+ VideoSubscription alice_subscription =
+ alice_configurer->configurable_params()->video_subscription;
+ std::vector<VideoConfig> alice_video_configs =
+ alice_configurer->configurable_params()->video_configs;
+ std::string alice_name = alice_configurer->params()->name.value();
+ VideoSubscription bob_subscription =
+ alice_configurer->configurable_params()->video_subscription;
+ std::vector<VideoConfig> bob_video_configs =
+ bob_configurer->configurable_params()->video_configs;
+ std::string bob_name = bob_configurer->params()->name.value();
+
+ TestPeerFactory test_peer_factory(
+ signaling_thread.get(), time_controller_,
+ video_quality_analyzer_injection_helper_.get(), task_queue_.get());
+ alice_ = test_peer_factory.CreateTestPeer(
+ std::move(alice_configurer),
+ std::make_unique<FixturePeerConnectionObserver>(
+ [this, alice_name, alice_subscription, bob_video_configs](
+ rtc::scoped_refptr<RtpTransceiverInterface> transceiver) {
+ OnTrackCallback(alice_name, alice_subscription, transceiver,
+ bob_video_configs);
+ },
+ [this]() { StartVideo(alice_video_sources_); }),
+ alice_remote_audio_config, run_params.echo_emulation_config);
+ bob_ = test_peer_factory.CreateTestPeer(
+ std::move(bob_configurer),
+ std::make_unique<FixturePeerConnectionObserver>(
+ [this, bob_name, bob_subscription, alice_video_configs](
+ rtc::scoped_refptr<RtpTransceiverInterface> transceiver) {
+ OnTrackCallback(bob_name, bob_subscription, transceiver,
+ alice_video_configs);
+ },
+ [this]() { StartVideo(bob_video_sources_); }),
+ bob_remote_audio_config, run_params.echo_emulation_config);
+
+ int num_cores = CpuInfo::DetectNumberOfCores();
+ RTC_DCHECK_GE(num_cores, 1);
+
+ int video_analyzer_threads =
+ num_cores - kPeerConnectionUsedThreads - kFrameworkUsedThreads;
+ if (video_analyzer_threads <= 0) {
+ video_analyzer_threads = 1;
+ }
+ video_analyzer_threads =
+ std::min(video_analyzer_threads, kMaxVideoAnalyzerThreads);
+ RTC_LOG(LS_INFO) << "video_analyzer_threads=" << video_analyzer_threads;
+ quality_metrics_reporters_.push_back(
+ std::make_unique<VideoQualityMetricsReporter>(time_controller_.GetClock(),
+ metrics_logger_));
+ quality_metrics_reporters_.push_back(
+ std::make_unique<CrossMediaMetricsReporter>(metrics_logger_));
+
+ video_quality_analyzer_injection_helper_->Start(
+ test_case_name_,
+ std::vector<std::string>{alice_->params().name.value(),
+ bob_->params().name.value()},
+ video_analyzer_threads);
+ audio_quality_analyzer_->Start(test_case_name_, &analyzer_helper_);
+ for (auto& reporter : quality_metrics_reporters_) {
+ reporter->Start(test_case_name_, &analyzer_helper_);
+ }
+
+ // Start RTCEventLog recording if requested.
+ if (alice_->params().rtc_event_log_path) {
+ auto alice_rtc_event_log = std::make_unique<webrtc::RtcEventLogOutputFile>(
+ alice_->params().rtc_event_log_path.value());
+ alice_->pc()->StartRtcEventLog(std::move(alice_rtc_event_log),
+ webrtc::RtcEventLog::kImmediateOutput);
+ }
+ if (bob_->params().rtc_event_log_path) {
+ auto bob_rtc_event_log = std::make_unique<webrtc::RtcEventLogOutputFile>(
+ bob_->params().rtc_event_log_path.value());
+ bob_->pc()->StartRtcEventLog(std::move(bob_rtc_event_log),
+ webrtc::RtcEventLog::kImmediateOutput);
+ }
+
+ // Setup alive logging. It is done to prevent test infra to think that test is
+ // dead.
+ RepeatingTaskHandle::DelayedStart(task_queue_->Get(),
+ kAliveMessageLogInterval, []() {
+ std::printf("Test is still running...\n");
+ return kAliveMessageLogInterval;
+ });
+
+ RTC_LOG(LS_INFO) << "Configuration is done. Now " << *alice_->params().name
+ << " is calling to " << *bob_->params().name << "...";
+
+ // Setup stats poller.
+ std::vector<StatsObserverInterface*> observers = {
+ audio_quality_analyzer_.get(),
+ video_quality_analyzer_injection_helper_.get()};
+ for (auto& reporter : quality_metrics_reporters_) {
+ observers.push_back(reporter.get());
+ }
+ StatsPoller stats_poller(observers,
+ std::map<std::string, StatsProvider*>{
+ {*alice_->params().name, alice_.get()},
+ {*bob_->params().name, bob_.get()}});
+ executor_->ScheduleActivity(TimeDelta::Zero(), kStatsUpdateInterval,
+ [&stats_poller](TimeDelta) {
+ stats_poller.PollStatsAndNotifyObservers();
+ });
+
+ // Setup call.
+ SendTask(signaling_thread.get(),
+ [this, &run_params] { SetupCallOnSignalingThread(run_params); });
+ std::unique_ptr<SignalingInterceptor> signaling_interceptor =
+ CreateSignalingInterceptor(run_params);
+ // Connect peers.
+ SendTask(signaling_thread.get(), [this, &signaling_interceptor] {
+ ExchangeOfferAnswer(signaling_interceptor.get());
+ });
+ WaitUntilIceCandidatesGathered(signaling_thread.get());
+
+ SendTask(signaling_thread.get(), [this, &signaling_interceptor] {
+ ExchangeIceCandidates(signaling_interceptor.get());
+ });
+ WaitUntilPeersAreConnected(signaling_thread.get());
+
+ executor_->Start(task_queue_.get());
+ Timestamp start_time = Now();
+
+ bool is_quick_test_enabled = field_trial::IsEnabled("WebRTC-QuickPerfTest");
+ if (is_quick_test_enabled) {
+ time_controller_.AdvanceTime(kQuickTestModeRunDuration);
+ } else {
+ time_controller_.AdvanceTime(run_params.run_duration);
+ }
+
+ RTC_LOG(LS_INFO) << "Test is done, initiating disconnect sequence.";
+
+ // Stop all client started tasks to prevent their access to any call related
+ // objects after these objects will be destroyed during call tear down.
+ executor_->Stop();
+ // There is no guarantee, that last stats collection will happen at the end
+ // of the call, so we force it after executor, which is among others is doing
+ // stats collection, was stopped.
+ task_queue_->SendTask([&stats_poller]() {
+ // Get final end-of-call stats.
+ stats_poller.PollStatsAndNotifyObservers();
+ });
+ // We need to detach AEC dumping from peers, because dump uses `task_queue_`
+ // inside.
+ alice_->DetachAecDump();
+ bob_->DetachAecDump();
+ // Tear down the call.
+ SendTask(signaling_thread.get(), [this] { TearDownCallOnSignalingThread(); });
+
+ Timestamp end_time = Now();
+ RTC_LOG(LS_INFO) << "All peers are disconnected.";
+ {
+ MutexLock lock(&lock_);
+ real_test_duration_ = end_time - start_time;
+ }
+
+ ReportGeneralTestResults();
+ audio_quality_analyzer_->Stop();
+ video_quality_analyzer_injection_helper_->Stop();
+ for (auto& reporter : quality_metrics_reporters_) {
+ reporter->StopAndReportResults();
+ }
+
+ // Reset `task_queue_` after test to cleanup.
+ task_queue_.reset();
+
+ alice_ = nullptr;
+ bob_ = nullptr;
+ // Ensuring that TestVideoCapturerVideoTrackSource are destroyed on the right
+ // thread.
+ RTC_CHECK(alice_video_sources_.empty());
+ RTC_CHECK(bob_video_sources_.empty());
+}
+
+std::string PeerConnectionE2EQualityTest::GetFieldTrials(
+ const RunParams& run_params) {
+ std::vector<absl::string_view> default_field_trials = {
+ kUseStandardsBytesStats};
+ if (run_params.enable_flex_fec_support) {
+ default_field_trials.push_back(kFlexFecEnabledFieldTrials);
+ }
+ rtc::StringBuilder sb;
+ sb << field_trial::GetFieldTrialString();
+ for (const absl::string_view& field_trial : default_field_trials) {
+ sb << field_trial;
+ }
+ return sb.Release();
+}
+
+void PeerConnectionE2EQualityTest::OnTrackCallback(
+ absl::string_view peer_name,
+ VideoSubscription peer_subscription,
+ rtc::scoped_refptr<RtpTransceiverInterface> transceiver,
+ std::vector<VideoConfig> remote_video_configs) {
+ const rtc::scoped_refptr<MediaStreamTrackInterface>& track =
+ transceiver->receiver()->track();
+ RTC_CHECK_EQ(transceiver->receiver()->stream_ids().size(), 2)
+ << "Expected 2 stream ids: 1st - sync group, 2nd - unique stream label";
+ std::string sync_group = transceiver->receiver()->stream_ids()[0];
+ std::string stream_label = transceiver->receiver()->stream_ids()[1];
+ analyzer_helper_.AddTrackToStreamMapping(track->id(), peer_name, stream_label,
+ sync_group);
+ if (track->kind() != MediaStreamTrackInterface::kVideoKind) {
+ return;
+ }
+
+ // It is safe to cast here, because it is checked above that
+ // track->kind() is kVideoKind.
+ auto* video_track = static_cast<VideoTrackInterface*>(track.get());
+ std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>> video_sink =
+ video_quality_analyzer_injection_helper_->CreateVideoSink(
+ peer_name, peer_subscription, /*report_infra_stats=*/false);
+ video_track->AddOrUpdateSink(video_sink.get(), rtc::VideoSinkWants());
+ output_video_sinks_.push_back(std::move(video_sink));
+}
+
+void PeerConnectionE2EQualityTest::SetupCallOnSignalingThread(
+ const RunParams& run_params) {
+ // We need receive-only transceivers for Bob's media stream, so there will
+ // be media section in SDP for that streams in Alice's offer, because it is
+ // forbidden to add new media sections in answer in Unified Plan.
+ RtpTransceiverInit receive_only_transceiver_init;
+ receive_only_transceiver_init.direction = RtpTransceiverDirection::kRecvOnly;
+ int alice_transceivers_counter = 0;
+ if (bob_->params().audio_config) {
+ // Setup receive audio transceiver if Bob has audio to send. If we'll need
+ // multiple audio streams, then we need transceiver for each Bob's audio
+ // stream.
+ RTCErrorOr<rtc::scoped_refptr<RtpTransceiverInterface>> result =
+ alice_->AddTransceiver(cricket::MediaType::MEDIA_TYPE_AUDIO,
+ receive_only_transceiver_init);
+ RTC_CHECK(result.ok());
+ alice_transceivers_counter++;
+ }
+
+ size_t alice_video_transceivers_non_simulcast_counter = 0;
+ for (auto& video_config : alice_->configurable_params().video_configs) {
+ RtpTransceiverInit transceiver_params;
+ if (video_config.simulcast_config) {
+ transceiver_params.direction = RtpTransceiverDirection::kSendOnly;
+ // Because simulcast enabled `alice_->params().video_codecs` has only 1
+ // element.
+ if (alice_->params().video_codecs[0].name == cricket::kVp8CodecName) {
+ // For Vp8 simulcast we need to add as many RtpEncodingParameters to the
+ // track as many simulcast streams requested. If they specified in
+ // `video_config.simulcast_config` it should be copied from there.
+ for (int i = 0;
+ i < video_config.simulcast_config->simulcast_streams_count; ++i) {
+ RtpEncodingParameters enc_params;
+ if (!video_config.encoding_params.empty()) {
+ enc_params = video_config.encoding_params[i];
+ }
+ // We need to be sure, that all rids will be unique with all mids.
+ enc_params.rid = std::to_string(alice_transceivers_counter) + "000" +
+ std::to_string(i);
+ transceiver_params.send_encodings.push_back(enc_params);
+ }
+ }
+ } else {
+ transceiver_params.direction = RtpTransceiverDirection::kSendRecv;
+ RtpEncodingParameters enc_params;
+ if (video_config.encoding_params.size() == 1) {
+ enc_params = video_config.encoding_params[0];
+ }
+ transceiver_params.send_encodings.push_back(enc_params);
+
+ alice_video_transceivers_non_simulcast_counter++;
+ }
+ RTCErrorOr<rtc::scoped_refptr<RtpTransceiverInterface>> result =
+ alice_->AddTransceiver(cricket::MediaType::MEDIA_TYPE_VIDEO,
+ transceiver_params);
+ RTC_CHECK(result.ok());
+
+ alice_transceivers_counter++;
+ }
+
+ // Add receive only transceivers in case Bob has more video_configs than
+ // Alice.
+ for (size_t i = alice_video_transceivers_non_simulcast_counter;
+ i < bob_->configurable_params().video_configs.size(); ++i) {
+ RTCErrorOr<rtc::scoped_refptr<RtpTransceiverInterface>> result =
+ alice_->AddTransceiver(cricket::MediaType::MEDIA_TYPE_VIDEO,
+ receive_only_transceiver_init);
+ RTC_CHECK(result.ok());
+ alice_transceivers_counter++;
+ }
+
+ // Then add media for Alice and Bob
+ media_helper_->MaybeAddAudio(alice_.get());
+ alice_video_sources_ = media_helper_->MaybeAddVideo(alice_.get());
+ media_helper_->MaybeAddAudio(bob_.get());
+ bob_video_sources_ = media_helper_->MaybeAddVideo(bob_.get());
+
+ SetPeerCodecPreferences(alice_.get());
+ SetPeerCodecPreferences(bob_.get());
+}
+
+void PeerConnectionE2EQualityTest::TearDownCallOnSignalingThread() {
+ TearDownCall();
+}
+
+void PeerConnectionE2EQualityTest::SetPeerCodecPreferences(TestPeer* peer) {
+ std::vector<RtpCodecCapability> with_rtx_video_capabilities =
+ FilterVideoCodecCapabilities(
+ peer->params().video_codecs, true, peer->params().use_ulp_fec,
+ peer->params().use_flex_fec,
+ peer->pc_factory()
+ ->GetRtpSenderCapabilities(cricket::MediaType::MEDIA_TYPE_VIDEO)
+ .codecs);
+ std::vector<RtpCodecCapability> without_rtx_video_capabilities =
+ FilterVideoCodecCapabilities(
+ peer->params().video_codecs, false, peer->params().use_ulp_fec,
+ peer->params().use_flex_fec,
+ peer->pc_factory()
+ ->GetRtpSenderCapabilities(cricket::MediaType::MEDIA_TYPE_VIDEO)
+ .codecs);
+
+ // Set codecs for transceivers
+ for (auto transceiver : peer->pc()->GetTransceivers()) {
+ if (transceiver->media_type() == cricket::MediaType::MEDIA_TYPE_VIDEO) {
+ if (transceiver->sender()->init_send_encodings().size() > 1) {
+ // If transceiver's sender has more then 1 send encodings, it means it
+ // has multiple simulcast streams, so we need disable RTX on it.
+ RTCError result =
+ transceiver->SetCodecPreferences(without_rtx_video_capabilities);
+ RTC_CHECK(result.ok());
+ } else {
+ RTCError result =
+ transceiver->SetCodecPreferences(with_rtx_video_capabilities);
+ RTC_CHECK(result.ok());
+ }
+ }
+ }
+}
+
+std::unique_ptr<SignalingInterceptor>
+PeerConnectionE2EQualityTest::CreateSignalingInterceptor(
+ const RunParams& run_params) {
+ std::map<std::string, int> stream_label_to_simulcast_streams_count;
+ // We add only Alice here, because simulcast/svc is supported only from the
+ // first peer.
+ for (auto& video_config : alice_->configurable_params().video_configs) {
+ if (video_config.simulcast_config) {
+ stream_label_to_simulcast_streams_count.insert(
+ {*video_config.stream_label,
+ video_config.simulcast_config->simulcast_streams_count});
+ }
+ }
+ PatchingParams patching_params(run_params.use_conference_mode,
+ stream_label_to_simulcast_streams_count);
+ return std::make_unique<SignalingInterceptor>(patching_params);
+}
+
+void PeerConnectionE2EQualityTest::WaitUntilIceCandidatesGathered(
+ rtc::Thread* signaling_thread) {
+ ASSERT_TRUE(time_controller_.Wait(
+ [&]() {
+ bool result;
+ SendTask(signaling_thread, [&]() {
+ result = alice_->IsIceGatheringDone() && bob_->IsIceGatheringDone();
+ });
+ return result;
+ },
+ 2 * kDefaultTimeout));
+}
+
+void PeerConnectionE2EQualityTest::WaitUntilPeersAreConnected(
+ rtc::Thread* signaling_thread) {
+ // This means that ICE and DTLS are connected.
+ alice_connected_ = time_controller_.Wait(
+ [&]() {
+ bool result;
+ SendTask(signaling_thread, [&] { result = alice_->IsIceConnected(); });
+ return result;
+ },
+ kDefaultTimeout);
+ bob_connected_ = time_controller_.Wait(
+ [&]() {
+ bool result;
+ SendTask(signaling_thread, [&] { result = bob_->IsIceConnected(); });
+ return result;
+ },
+ kDefaultTimeout);
+}
+
+void PeerConnectionE2EQualityTest::ExchangeOfferAnswer(
+ SignalingInterceptor* signaling_interceptor) {
+ std::string log_output;
+
+ auto offer = alice_->CreateOffer();
+ RTC_CHECK(offer);
+ offer->ToString(&log_output);
+ RTC_LOG(LS_INFO) << "Original offer: " << log_output;
+ LocalAndRemoteSdp patch_result = signaling_interceptor->PatchOffer(
+ std::move(offer), alice_->params().video_codecs[0]);
+ patch_result.local_sdp->ToString(&log_output);
+ RTC_LOG(LS_INFO) << "Offer to set as local description: " << log_output;
+ patch_result.remote_sdp->ToString(&log_output);
+ RTC_LOG(LS_INFO) << "Offer to set as remote description: " << log_output;
+
+ bool set_local_offer =
+ alice_->SetLocalDescription(std::move(patch_result.local_sdp));
+ RTC_CHECK(set_local_offer);
+ bool set_remote_offer =
+ bob_->SetRemoteDescription(std::move(patch_result.remote_sdp));
+ RTC_CHECK(set_remote_offer);
+ auto answer = bob_->CreateAnswer();
+ RTC_CHECK(answer);
+ answer->ToString(&log_output);
+ RTC_LOG(LS_INFO) << "Original answer: " << log_output;
+ patch_result = signaling_interceptor->PatchAnswer(
+ std::move(answer), bob_->params().video_codecs[0]);
+ patch_result.local_sdp->ToString(&log_output);
+ RTC_LOG(LS_INFO) << "Answer to set as local description: " << log_output;
+ patch_result.remote_sdp->ToString(&log_output);
+ RTC_LOG(LS_INFO) << "Answer to set as remote description: " << log_output;
+
+ bool set_local_answer =
+ bob_->SetLocalDescription(std::move(patch_result.local_sdp));
+ RTC_CHECK(set_local_answer);
+ bool set_remote_answer =
+ alice_->SetRemoteDescription(std::move(patch_result.remote_sdp));
+ RTC_CHECK(set_remote_answer);
+}
+
+void PeerConnectionE2EQualityTest::ExchangeIceCandidates(
+ SignalingInterceptor* signaling_interceptor) {
+ // Connect an ICE candidate pairs.
+ std::vector<std::unique_ptr<IceCandidateInterface>> alice_candidates =
+ signaling_interceptor->PatchOffererIceCandidates(
+ alice_->observer()->GetAllCandidates());
+ for (auto& candidate : alice_candidates) {
+ std::string candidate_str;
+ RTC_CHECK(candidate->ToString(&candidate_str));
+ RTC_LOG(LS_INFO) << *alice_->params().name
+ << " ICE candidate(mid= " << candidate->sdp_mid()
+ << "): " << candidate_str;
+ }
+ ASSERT_TRUE(bob_->AddIceCandidates(std::move(alice_candidates)));
+ std::vector<std::unique_ptr<IceCandidateInterface>> bob_candidates =
+ signaling_interceptor->PatchAnswererIceCandidates(
+ bob_->observer()->GetAllCandidates());
+ for (auto& candidate : bob_candidates) {
+ std::string candidate_str;
+ RTC_CHECK(candidate->ToString(&candidate_str));
+ RTC_LOG(LS_INFO) << *bob_->params().name
+ << " ICE candidate(mid= " << candidate->sdp_mid()
+ << "): " << candidate_str;
+ }
+ ASSERT_TRUE(alice_->AddIceCandidates(std::move(bob_candidates)));
+}
+
+void PeerConnectionE2EQualityTest::StartVideo(
+ const std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>>&
+ sources) {
+ for (auto& source : sources) {
+ if (source->state() != MediaSourceInterface::SourceState::kLive) {
+ source->Start();
+ }
+ }
+}
+
+void PeerConnectionE2EQualityTest::TearDownCall() {
+ for (const auto& video_source : alice_video_sources_) {
+ video_source->Stop();
+ }
+ for (const auto& video_source : bob_video_sources_) {
+ video_source->Stop();
+ }
+
+ alice_video_sources_.clear();
+ bob_video_sources_.clear();
+
+ alice_->Close();
+ bob_->Close();
+
+ media_helper_ = nullptr;
+}
+
+void PeerConnectionE2EQualityTest::ReportGeneralTestResults() {
+ // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey.
+ metrics_logger_->LogSingleValueMetric(
+ *alice_->params().name + "_connected", test_case_name_, alice_connected_,
+ Unit::kUnitless, ImprovementDirection::kBiggerIsBetter,
+ {{MetricMetadataKey::kPeerMetadataKey, *alice_->params().name},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_case_name_}});
+ // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey.
+ metrics_logger_->LogSingleValueMetric(
+ *bob_->params().name + "_connected", test_case_name_, bob_connected_,
+ Unit::kUnitless, ImprovementDirection::kBiggerIsBetter,
+ {{MetricMetadataKey::kPeerMetadataKey, *bob_->params().name},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_case_name_}});
+}
+
+Timestamp PeerConnectionE2EQualityTest::Now() const {
+ return time_controller_.GetClock()->CurrentTime();
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.h b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.h
new file mode 100644
index 0000000000..6cbf232874
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test.h
@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef TEST_PC_E2E_PEER_CONNECTION_QUALITY_TEST_H_
+#define TEST_PC_E2E_PEER_CONNECTION_QUALITY_TEST_H_
+
+#include <memory>
+#include <queue>
+#include <string>
+#include <vector>
+
+#include "absl/strings/string_view.h"
+#include "api/task_queue/task_queue_factory.h"
+#include "api/test/audio_quality_analyzer_interface.h"
+#include "api/test/metrics/metrics_logger.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/pclf/media_quality_test_params.h"
+#include "api/test/pclf/peer_configurer.h"
+#include "api/test/peerconnection_quality_test_fixture.h"
+#include "api/test/time_controller.h"
+#include "api/units/time_delta.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "rtc_base/task_queue_for_test.h"
+#include "rtc_base/thread.h"
+#include "rtc_base/thread_annotations.h"
+#include "system_wrappers/include/clock.h"
+#include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h"
+#include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h"
+#include "test/pc/e2e/analyzer_helper.h"
+#include "test/pc/e2e/media/media_helper.h"
+#include "test/pc/e2e/sdp/sdp_changer.h"
+#include "test/pc/e2e/test_activities_executor.h"
+#include "test/pc/e2e/test_peer.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+class PeerConnectionE2EQualityTest
+ : public PeerConnectionE2EQualityTestFixture {
+ public:
+ using QualityMetricsReporter =
+ PeerConnectionE2EQualityTestFixture::QualityMetricsReporter;
+
+ PeerConnectionE2EQualityTest(
+ std::string test_case_name,
+ TimeController& time_controller,
+ std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer,
+ std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer);
+ PeerConnectionE2EQualityTest(
+ std::string test_case_name,
+ TimeController& time_controller,
+ std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer,
+ std::unique_ptr<VideoQualityAnalyzerInterface> video_quality_analyzer,
+ test::MetricsLogger* metrics_logger);
+
+ ~PeerConnectionE2EQualityTest() override = default;
+
+ void ExecuteAt(TimeDelta target_time_since_start,
+ std::function<void(TimeDelta)> func) override;
+ void ExecuteEvery(TimeDelta initial_delay_since_start,
+ TimeDelta interval,
+ std::function<void(TimeDelta)> func) override;
+
+ void AddQualityMetricsReporter(std::unique_ptr<QualityMetricsReporter>
+ quality_metrics_reporter) override;
+
+ PeerHandle* AddPeer(std::unique_ptr<PeerConfigurer> configurer) override;
+ void Run(RunParams run_params) override;
+
+ TimeDelta GetRealTestDuration() const override {
+ MutexLock lock(&lock_);
+ RTC_CHECK_NE(real_test_duration_, TimeDelta::Zero());
+ return real_test_duration_;
+ }
+
+ private:
+ class PeerHandleImpl : public PeerHandle {
+ public:
+ ~PeerHandleImpl() override = default;
+ };
+
+ // For some functionality some field trials have to be enabled, they will be
+ // enabled in Run().
+ std::string GetFieldTrials(const RunParams& run_params);
+ void OnTrackCallback(absl::string_view peer_name,
+ VideoSubscription peer_subscription,
+ rtc::scoped_refptr<RtpTransceiverInterface> transceiver,
+ std::vector<VideoConfig> remote_video_configs);
+ // Have to be run on the signaling thread.
+ void SetupCallOnSignalingThread(const RunParams& run_params);
+ void TearDownCallOnSignalingThread();
+ void SetPeerCodecPreferences(TestPeer* peer);
+ std::unique_ptr<SignalingInterceptor> CreateSignalingInterceptor(
+ const RunParams& run_params);
+ void WaitUntilIceCandidatesGathered(rtc::Thread* signaling_thread);
+ void WaitUntilPeersAreConnected(rtc::Thread* signaling_thread);
+ void ExchangeOfferAnswer(SignalingInterceptor* signaling_interceptor);
+ void ExchangeIceCandidates(SignalingInterceptor* signaling_interceptor);
+ void StartVideo(
+ const std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>>&
+ sources);
+ void TearDownCall();
+ void ReportGeneralTestResults();
+ Timestamp Now() const;
+
+ TimeController& time_controller_;
+ const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
+ std::string test_case_name_;
+ std::unique_ptr<VideoQualityAnalyzerInjectionHelper>
+ video_quality_analyzer_injection_helper_;
+ std::unique_ptr<MediaHelper> media_helper_;
+ std::unique_ptr<EncodedImageDataPropagator> encoded_image_data_propagator_;
+ std::unique_ptr<AudioQualityAnalyzerInterface> audio_quality_analyzer_;
+ std::unique_ptr<TestActivitiesExecutor> executor_;
+ test::MetricsLogger* const metrics_logger_;
+
+ std::vector<std::unique_ptr<PeerConfigurer>> peer_configurations_;
+ std::vector<PeerHandleImpl> peer_handles_;
+
+ std::unique_ptr<TestPeer> alice_;
+ std::unique_ptr<TestPeer> bob_;
+ std::vector<std::unique_ptr<QualityMetricsReporter>>
+ quality_metrics_reporters_;
+
+ std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>>
+ alice_video_sources_;
+ std::vector<rtc::scoped_refptr<TestVideoCapturerVideoTrackSource>>
+ bob_video_sources_;
+ std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>>
+ output_video_sinks_;
+ AnalyzerHelper analyzer_helper_;
+
+ mutable Mutex lock_;
+ TimeDelta real_test_duration_ RTC_GUARDED_BY(lock_) = TimeDelta::Zero();
+
+ // Task queue, that is used for running activities during test call.
+ // This task queue will be created before call set up and will be destroyed
+ // immediately before call tear down.
+ std::unique_ptr<TaskQueueForTest> task_queue_;
+
+ bool alice_connected_ = false;
+ bool bob_connected_ = false;
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_PEER_CONNECTION_QUALITY_TEST_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc
new file mode 100644
index 0000000000..8a47e108e0
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc
@@ -0,0 +1,1102 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <map>
+#include <memory>
+#include <string>
+
+#include "api/test/create_network_emulation_manager.h"
+#include "api/test/create_peer_connection_quality_test_frame_generator.h"
+#include "api/test/metrics/metrics_logger.h"
+#include "api/test/metrics/stdout_metrics_exporter.h"
+#include "api/test/network_emulation_manager.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/pclf/media_quality_test_params.h"
+#include "api/test/pclf/peer_configurer.h"
+#include "api/test/peerconnection_quality_test_fixture.h"
+#include "api/units/time_delta.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/pc/e2e/metric_metadata_keys.h"
+#include "test/pc/e2e/peer_connection_quality_test.h"
+#include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+using ::testing::UnorderedElementsAre;
+
+using ::webrtc::test::DefaultMetricsLogger;
+using ::webrtc::test::ImprovementDirection;
+using ::webrtc::test::Metric;
+using ::webrtc::test::MetricsExporter;
+using ::webrtc::test::StdoutMetricsExporter;
+using ::webrtc::test::Unit;
+using ::webrtc::webrtc_pc_e2e::PeerConfigurer;
+
+// Adds a peer with some audio and video (the client should not care about
+// details about audio and video configs).
+void AddDefaultAudioVideoPeer(
+ absl::string_view peer_name,
+ absl::string_view audio_stream_label,
+ absl::string_view video_stream_label,
+ const PeerNetworkDependencies& network_dependencies,
+ PeerConnectionE2EQualityTestFixture& fixture) {
+ AudioConfig audio{std::string(audio_stream_label)};
+ audio.sync_group = std::string(peer_name);
+ VideoConfig video(std::string(video_stream_label), 320, 180, 15);
+ video.sync_group = std::string(peer_name);
+ auto peer = std::make_unique<PeerConfigurer>(network_dependencies);
+ peer->SetName(peer_name);
+ peer->SetAudioConfig(std::move(audio));
+ peer->AddVideoConfig(std::move(video));
+ peer->SetVideoCodecs({VideoCodecConfig(cricket::kVp8CodecName)});
+ fixture.AddPeer(std::move(peer));
+}
+
+// Metric fields to assert on
+struct MetricValidationInfo {
+ std::string test_case;
+ std::string name;
+ Unit unit;
+ ImprovementDirection improvement_direction;
+ std::map<std::string, std::string> metadata;
+};
+
+bool operator==(const MetricValidationInfo& a, const MetricValidationInfo& b) {
+ return a.name == b.name && a.test_case == b.test_case && a.unit == b.unit &&
+ a.improvement_direction == b.improvement_direction &&
+ a.metadata == b.metadata;
+}
+
+std::ostream& operator<<(std::ostream& os, const MetricValidationInfo& m) {
+ os << "{ test_case=" << m.test_case << "; name=" << m.name
+ << "; unit=" << test::ToString(m.unit)
+ << "; improvement_direction=" << test::ToString(m.improvement_direction)
+ << "; metadata={ ";
+ for (const auto& [key, value] : m.metadata) {
+ os << "{ key=" << key << "; value=" << value << " }";
+ }
+ os << " }}";
+ return os;
+}
+
+std::vector<MetricValidationInfo> ToValidationInfo(
+ const std::vector<Metric>& metrics) {
+ std::vector<MetricValidationInfo> out;
+ for (const Metric& m : metrics) {
+ out.push_back(
+ MetricValidationInfo{.test_case = m.test_case,
+ .name = m.name,
+ .unit = m.unit,
+ .improvement_direction = m.improvement_direction,
+ .metadata = m.metric_metadata});
+ }
+ return out;
+}
+
+TEST(PeerConnectionE2EQualityTestMetricNamesTest,
+ ExportedMetricsHasCorrectNamesAndAnnotation) {
+ std::unique_ptr<NetworkEmulationManager> network_emulation =
+ CreateNetworkEmulationManager(TimeMode::kSimulated);
+ DefaultMetricsLogger metrics_logger(
+ network_emulation->time_controller()->GetClock());
+ PeerConnectionE2EQualityTest fixture(
+ "test_case", *network_emulation->time_controller(),
+ /*audio_quality_analyzer=*/nullptr, /*video_quality_analyzer=*/nullptr,
+ &metrics_logger);
+
+ EmulatedEndpoint* alice_endpoint =
+ network_emulation->CreateEndpoint(EmulatedEndpointConfig());
+ EmulatedEndpoint* bob_endpoint =
+ network_emulation->CreateEndpoint(EmulatedEndpointConfig());
+
+ network_emulation->CreateRoute(
+ alice_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()},
+ bob_endpoint);
+ network_emulation->CreateRoute(
+ bob_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()},
+ alice_endpoint);
+
+ EmulatedNetworkManagerInterface* alice_network =
+ network_emulation->CreateEmulatedNetworkManagerInterface(
+ {alice_endpoint});
+ EmulatedNetworkManagerInterface* bob_network =
+ network_emulation->CreateEmulatedNetworkManagerInterface({bob_endpoint});
+
+ AddDefaultAudioVideoPeer("alice", "alice_audio", "alice_video",
+ alice_network->network_dependencies(), fixture);
+ AddDefaultAudioVideoPeer("bob", "bob_audio", "bob_video",
+ bob_network->network_dependencies(), fixture);
+ fixture.AddQualityMetricsReporter(
+ std::make_unique<StatsBasedNetworkQualityMetricsReporter>(
+ std::map<std::string, std::vector<EmulatedEndpoint*>>(
+ {{"alice", alice_network->endpoints()},
+ {"bob", bob_network->endpoints()}}),
+ network_emulation.get(), &metrics_logger));
+
+ // Run for at least 7 seconds, so AV-sync metrics will be collected.
+ fixture.Run(RunParams(TimeDelta::Seconds(7)));
+
+ std::vector<MetricValidationInfo> metrics =
+ ToValidationInfo(metrics_logger.GetCollectedMetrics());
+ EXPECT_THAT(
+ metrics,
+ UnorderedElementsAre(
+ // Metrics from PeerConnectionE2EQualityTest
+ MetricValidationInfo{
+ .test_case = "test_case",
+ .name = "alice_connected",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case",
+ .name = "bob_connected",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+
+ // Metrics from DefaultAudioQualityAnalyzer
+ MetricValidationInfo{
+ .test_case = "test_case/alice_audio",
+ .name = "expand_rate",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "alice_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_audio",
+ .name = "accelerate_rate",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "alice_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_audio",
+ .name = "preemptive_rate",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "alice_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_audio",
+ .name = "speech_expand_rate",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "alice_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_audio",
+ .name = "average_jitter_buffer_delay_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "alice_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_audio",
+ .name = "preferred_buffer_size_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "alice_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_audio",
+ .name = "expand_rate",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "bob_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_audio",
+ .name = "accelerate_rate",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "bob_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_audio",
+ .name = "preemptive_rate",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "bob_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_audio",
+ .name = "speech_expand_rate",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "bob_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_audio",
+ .name = "average_jitter_buffer_delay_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "bob_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_audio",
+ .name = "preferred_buffer_size_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "bob_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+
+ // Metrics from DefaultVideoQualityAnalyzer
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "ssim",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "transport_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "total_delay_incl_transport",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "time_between_rendered_frames",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "harmonic_framerate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "encode_frame_rate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "encode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "time_between_freezes",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "freeze_time_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "pixels_per_frame",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "min_psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "decode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "receive_to_render_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "dropped_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "frames_in_flight",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "rendered_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "max_skipped",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "target_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "qp_sl0",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kSpatialLayerMetadataKey, "0"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "actual_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "ssim",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "transport_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "total_delay_incl_transport",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "time_between_rendered_frames",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "harmonic_framerate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "encode_frame_rate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "encode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "time_between_freezes",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "freeze_time_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "pixels_per_frame",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "min_psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "decode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "receive_to_render_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "dropped_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "frames_in_flight",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "rendered_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "max_skipped",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "target_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "actual_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "qp_sl0",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kSpatialLayerMetadataKey, "0"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case",
+ .name = "cpu_usage_%",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+
+ // Metrics from StatsBasedNetworkQualityMetricsReporter
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "bytes_discarded_no_receiver",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "packets_discarded_no_receiver",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "payload_bytes_received",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "payload_bytes_sent",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "bytes_sent",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "packets_sent",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "average_send_rate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "bytes_received",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "packets_received",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "average_receive_rate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "sent_packets_loss",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "bytes_discarded_no_receiver",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "packets_discarded_no_receiver",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "payload_bytes_received",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "payload_bytes_sent",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "bytes_sent",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "packets_sent",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "average_send_rate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "bytes_received",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "packets_received",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "average_receive_rate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "sent_packets_loss",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+
+ // Metrics from VideoQualityMetricsReporter
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "available_send_bandwidth",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "transmission_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "retransmission_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "available_send_bandwidth",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "transmission_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "retransmission_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+
+ // Metrics from CrossMediaMetricsReporter
+ MetricValidationInfo{
+ .test_case = "test_case/alice_alice_audio",
+ .name = "audio_ahead_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata =
+ {{MetricMetadataKey::kAudioStreamMetadataKey, "alice_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kPeerSyncGroupMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_alice_video",
+ .name = "video_ahead_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata =
+ {{MetricMetadataKey::kAudioStreamMetadataKey, "alice_video"},
+ {MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kPeerSyncGroupMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_bob_audio",
+ .name = "audio_ahead_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata =
+ {{MetricMetadataKey::kAudioStreamMetadataKey, "bob_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kPeerSyncGroupMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_bob_video",
+ .name = "video_ahead_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {
+ {MetricMetadataKey::kAudioStreamMetadataKey, "bob_video"},
+ {MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kPeerSyncGroupMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}}));
+}
+
+} // namespace
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_test.cc b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_test.cc
new file mode 100644
index 0000000000..066fe7d8ee
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_test.cc
@@ -0,0 +1,139 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/peer_connection_quality_test.h"
+
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "api/test/create_network_emulation_manager.h"
+#include "api/test/metrics/global_metrics_logger_and_exporter.h"
+#include "api/test/network_emulation_manager.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/pclf/media_quality_test_params.h"
+#include "api/test/pclf/peer_configurer.h"
+#include "api/test/peerconnection_quality_test_fixture.h"
+#include "api/units/time_delta.h"
+#include "rtc_base/time_utils.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/testsupport/file_utils.h"
+#include "test/testsupport/frame_reader.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+using ::testing::Eq;
+using ::testing::Test;
+
+using ::webrtc::webrtc_pc_e2e::PeerConfigurer;
+
+// Remove files and directories in a directory non-recursively.
+void CleanDir(absl::string_view dir, size_t expected_output_files_count) {
+ absl::optional<std::vector<std::string>> dir_content =
+ test::ReadDirectory(dir);
+ if (expected_output_files_count == 0) {
+ ASSERT_FALSE(dir_content.has_value()) << "Empty directory is expected";
+ } else {
+ ASSERT_TRUE(dir_content.has_value()) << "Test directory is empty!";
+ EXPECT_EQ(dir_content->size(), expected_output_files_count);
+ for (const auto& entry : *dir_content) {
+ if (test::DirExists(entry)) {
+ EXPECT_TRUE(test::RemoveDir(entry))
+ << "Failed to remove sub directory: " << entry;
+ } else if (test::FileExists(entry)) {
+ EXPECT_TRUE(test::RemoveFile(entry))
+ << "Failed to remove file: " << entry;
+ } else {
+ FAIL() << "Can't remove unknown file type: " << entry;
+ }
+ }
+ }
+ EXPECT_TRUE(test::RemoveDir(dir)) << "Failed to remove directory: " << dir;
+}
+
+class PeerConnectionE2EQualityTestTest : public Test {
+ protected:
+ ~PeerConnectionE2EQualityTestTest() override = default;
+
+ void SetUp() override {
+ // Create an empty temporary directory for this test.
+ test_directory_ = test::JoinFilename(
+ test::OutputPath(),
+ "TestDir_PeerConnectionE2EQualityTestTest_" +
+ std::string(
+ testing::UnitTest::GetInstance()->current_test_info()->name()));
+ test::CreateDir(test_directory_);
+ }
+
+ void TearDown() override {
+ CleanDir(test_directory_, expected_output_files_count_);
+ }
+
+ void ExpectOutputFilesCount(size_t count) {
+ expected_output_files_count_ = count;
+ }
+
+ std::string test_directory_;
+ size_t expected_output_files_count_ = 0;
+};
+
+TEST_F(PeerConnectionE2EQualityTestTest, OutputVideoIsDumpedWhenRequested) {
+ std::unique_ptr<NetworkEmulationManager> network_emulation =
+ CreateNetworkEmulationManager(TimeMode::kSimulated);
+ PeerConnectionE2EQualityTest fixture(
+ "test_case", *network_emulation->time_controller(),
+ /*audio_quality_analyzer=*/nullptr, /*video_quality_analyzer=*/nullptr,
+ test::GetGlobalMetricsLogger());
+
+ EmulatedEndpoint* alice_endpoint =
+ network_emulation->CreateEndpoint(EmulatedEndpointConfig());
+ EmulatedEndpoint* bob_endpoint =
+ network_emulation->CreateEndpoint(EmulatedEndpointConfig());
+
+ network_emulation->CreateRoute(
+ alice_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()},
+ bob_endpoint);
+ network_emulation->CreateRoute(
+ bob_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()},
+ alice_endpoint);
+
+ EmulatedNetworkManagerInterface* alice_network =
+ network_emulation->CreateEmulatedNetworkManagerInterface(
+ {alice_endpoint});
+ EmulatedNetworkManagerInterface* bob_network =
+ network_emulation->CreateEmulatedNetworkManagerInterface({bob_endpoint});
+
+ VideoConfig alice_video("alice_video", 320, 180, 15);
+ alice_video.output_dump_options = VideoDumpOptions(test_directory_);
+ PeerConfigurer alice(alice_network->network_dependencies());
+ alice.SetName("alice");
+ alice.AddVideoConfig(std::move(alice_video));
+ fixture.AddPeer(std::make_unique<PeerConfigurer>(std::move(alice)));
+
+ PeerConfigurer bob(bob_network->network_dependencies());
+ bob.SetName("bob");
+ fixture.AddPeer(std::make_unique<PeerConfigurer>(std::move(bob)));
+
+ fixture.Run(RunParams(TimeDelta::Seconds(2)));
+
+ auto frame_reader = test::CreateY4mFrameReader(
+ test::JoinFilename(test_directory_, "alice_video_bob_320x180_15.y4m"));
+ EXPECT_THAT(frame_reader->num_frames(), Eq(31)); // 2 seconds 15 fps + 1
+
+ ExpectOutputFilesCount(1);
+}
+
+} // namespace
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/peer_params_preprocessor.cc b/third_party/libwebrtc/test/pc/e2e/peer_params_preprocessor.cc
new file mode 100644
index 0000000000..05372125d2
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/peer_params_preprocessor.cc
@@ -0,0 +1,217 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/peer_params_preprocessor.h"
+
+#include <set>
+#include <string>
+
+#include "absl/strings/string_view.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/pclf/media_quality_test_params.h"
+#include "api/test/pclf/peer_configurer.h"
+#include "api/test/peer_network_dependencies.h"
+#include "modules/video_coding/svc/create_scalability_structure.h"
+#include "modules/video_coding/svc/scalability_mode_util.h"
+#include "rtc_base/arraysize.h"
+#include "test/testsupport/file_utils.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+// List of default names of generic participants according to
+// https://en.wikipedia.org/wiki/Alice_and_Bob
+constexpr absl::string_view kDefaultNames[] = {"alice", "bob", "charlie",
+ "david", "erin", "frank"};
+
+} // namespace
+
+class PeerParamsPreprocessor::DefaultNamesProvider {
+ public:
+ // Caller have to ensure that default names array will outlive names provider
+ // instance.
+ explicit DefaultNamesProvider(
+ absl::string_view prefix,
+ rtc::ArrayView<const absl::string_view> default_names = {})
+ : prefix_(prefix), default_names_(default_names) {}
+
+ void MaybeSetName(absl::optional<std::string>& name) {
+ if (name.has_value()) {
+ known_names_.insert(name.value());
+ } else {
+ name = GenerateName();
+ }
+ }
+
+ private:
+ std::string GenerateName() {
+ std::string name;
+ do {
+ name = GenerateNameInternal();
+ } while (!known_names_.insert(name).second);
+ return name;
+ }
+
+ std::string GenerateNameInternal() {
+ if (counter_ < default_names_.size()) {
+ return std::string(default_names_[counter_++]);
+ }
+ return prefix_ + std::to_string(counter_++);
+ }
+
+ const std::string prefix_;
+ const rtc::ArrayView<const absl::string_view> default_names_;
+
+ std::set<std::string> known_names_;
+ size_t counter_ = 0;
+};
+
+PeerParamsPreprocessor::PeerParamsPreprocessor()
+ : peer_names_provider_(
+ std::make_unique<DefaultNamesProvider>("peer_", kDefaultNames)) {}
+PeerParamsPreprocessor::~PeerParamsPreprocessor() = default;
+
+void PeerParamsPreprocessor::SetDefaultValuesForMissingParams(
+ PeerConfigurer& peer) {
+ Params* params = peer.params();
+ ConfigurableParams* configurable_params = peer.configurable_params();
+ peer_names_provider_->MaybeSetName(params->name);
+ DefaultNamesProvider video_stream_names_provider(*params->name +
+ "_auto_video_stream_label_");
+ for (VideoConfig& config : configurable_params->video_configs) {
+ video_stream_names_provider.MaybeSetName(config.stream_label);
+ }
+ if (params->audio_config) {
+ DefaultNamesProvider audio_stream_names_provider(
+ *params->name + "_auto_audio_stream_label_");
+ audio_stream_names_provider.MaybeSetName(
+ params->audio_config->stream_label);
+ }
+
+ if (params->video_codecs.empty()) {
+ params->video_codecs.push_back(VideoCodecConfig(cricket::kVp8CodecName));
+ }
+}
+
+void PeerParamsPreprocessor::ValidateParams(const PeerConfigurer& peer) {
+ const Params& p = peer.params();
+ RTC_CHECK_GT(p.video_encoder_bitrate_multiplier, 0.0);
+ // Each peer should at least support 1 video codec.
+ RTC_CHECK_GE(p.video_codecs.size(), 1);
+
+ {
+ RTC_CHECK(p.name);
+ bool inserted = peer_names_.insert(p.name.value()).second;
+ RTC_CHECK(inserted) << "Duplicate name=" << p.name.value();
+ }
+
+ // Validate that all video stream labels are unique and sync groups are
+ // valid.
+ for (const VideoConfig& video_config :
+ peer.configurable_params().video_configs) {
+ RTC_CHECK(video_config.stream_label);
+ bool inserted =
+ video_labels_.insert(video_config.stream_label.value()).second;
+ RTC_CHECK(inserted) << "Duplicate video_config.stream_label="
+ << video_config.stream_label.value();
+
+ // TODO(bugs.webrtc.org/4762): remove this check after synchronization of
+ // more than two streams is supported.
+ if (video_config.sync_group.has_value()) {
+ bool sync_group_inserted =
+ video_sync_groups_.insert(video_config.sync_group.value()).second;
+ RTC_CHECK(sync_group_inserted)
+ << "Sync group shouldn't consist of more than two streams (one "
+ "video and one audio). Duplicate video_config.sync_group="
+ << video_config.sync_group.value();
+ }
+
+ if (video_config.simulcast_config) {
+ if (!video_config.encoding_params.empty()) {
+ RTC_CHECK_EQ(video_config.simulcast_config->simulcast_streams_count,
+ video_config.encoding_params.size())
+ << "|encoding_params| have to be specified for each simulcast "
+ << "stream in |video_config|.";
+ }
+ } else {
+ RTC_CHECK_LE(video_config.encoding_params.size(), 1)
+ << "|encoding_params| has multiple values but simulcast is not "
+ "enabled.";
+ }
+
+ if (video_config.emulated_sfu_config) {
+ if (video_config.simulcast_config &&
+ video_config.emulated_sfu_config->target_layer_index) {
+ RTC_CHECK_LT(*video_config.emulated_sfu_config->target_layer_index,
+ video_config.simulcast_config->simulcast_streams_count);
+ }
+ if (!video_config.encoding_params.empty()) {
+ bool is_svc = false;
+ for (const auto& encoding_param : video_config.encoding_params) {
+ if (!encoding_param.scalability_mode)
+ continue;
+
+ absl::optional<ScalabilityMode> scalability_mode =
+ ScalabilityModeFromString(*encoding_param.scalability_mode);
+ RTC_CHECK(scalability_mode) << "Unknown scalability_mode requested";
+
+ absl::optional<ScalableVideoController::StreamLayersConfig>
+ stream_layers_config =
+ ScalabilityStructureConfig(*scalability_mode);
+ is_svc |= stream_layers_config->num_spatial_layers > 1;
+ RTC_CHECK(stream_layers_config->num_spatial_layers == 1 ||
+ video_config.encoding_params.size() == 1)
+ << "Can't enable SVC modes with multiple spatial layers ("
+ << stream_layers_config->num_spatial_layers
+ << " layers) or simulcast ("
+ << video_config.encoding_params.size() << " layers)";
+ if (video_config.emulated_sfu_config->target_layer_index) {
+ RTC_CHECK_LT(*video_config.emulated_sfu_config->target_layer_index,
+ stream_layers_config->num_spatial_layers);
+ }
+ }
+ if (!is_svc && video_config.emulated_sfu_config->target_layer_index) {
+ RTC_CHECK_LT(*video_config.emulated_sfu_config->target_layer_index,
+ video_config.encoding_params.size());
+ }
+ }
+ }
+ }
+ if (p.audio_config) {
+ bool inserted =
+ audio_labels_.insert(p.audio_config->stream_label.value()).second;
+ RTC_CHECK(inserted) << "Duplicate audio_config.stream_label="
+ << p.audio_config->stream_label.value();
+ // TODO(bugs.webrtc.org/4762): remove this check after synchronization of
+ // more than two streams is supported.
+ if (p.audio_config->sync_group.has_value()) {
+ bool sync_group_inserted =
+ audio_sync_groups_.insert(p.audio_config->sync_group.value()).second;
+ RTC_CHECK(sync_group_inserted)
+ << "Sync group shouldn't consist of more than two streams (one "
+ "video and one audio). Duplicate audio_config.sync_group="
+ << p.audio_config->sync_group.value();
+ }
+ // Check that if mode input file name specified only if mode is kFile.
+ if (p.audio_config.value().mode == AudioConfig::Mode::kGenerated) {
+ RTC_CHECK(!p.audio_config.value().input_file_name);
+ }
+ if (p.audio_config.value().mode == AudioConfig::Mode::kFile) {
+ RTC_CHECK(p.audio_config.value().input_file_name);
+ RTC_CHECK(
+ test::FileExists(p.audio_config.value().input_file_name.value()))
+ << p.audio_config.value().input_file_name.value() << " doesn't exist";
+ }
+ }
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/peer_params_preprocessor.h b/third_party/libwebrtc/test/pc/e2e/peer_params_preprocessor.h
new file mode 100644
index 0000000000..c222811546
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/peer_params_preprocessor.h
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef TEST_PC_E2E_PEER_PARAMS_PREPROCESSOR_H_
+#define TEST_PC_E2E_PEER_PARAMS_PREPROCESSOR_H_
+
+#include <memory>
+#include <set>
+#include <string>
+
+#include "api/test/pclf/peer_configurer.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+class PeerParamsPreprocessor {
+ public:
+ PeerParamsPreprocessor();
+ ~PeerParamsPreprocessor();
+
+ // Set missing params to default values if it is required:
+ // * Generate video stream labels if some of them are missing
+ // * Generate audio stream labels if some of them are missing
+ // * Set video source generation mode if it is not specified
+ // * Video codecs under test
+ void SetDefaultValuesForMissingParams(PeerConfigurer& peer);
+
+ // Validate peer's parameters, also ensure uniqueness of all video stream
+ // labels.
+ void ValidateParams(const PeerConfigurer& peer);
+
+ private:
+ class DefaultNamesProvider;
+ std::unique_ptr<DefaultNamesProvider> peer_names_provider_;
+
+ std::set<std::string> peer_names_;
+ std::set<std::string> video_labels_;
+ std::set<std::string> audio_labels_;
+ std::set<std::string> video_sync_groups_;
+ std::set<std::string> audio_sync_groups_;
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_PEER_PARAMS_PREPROCESSOR_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.cc b/third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.cc
new file mode 100644
index 0000000000..af55f29175
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.cc
@@ -0,0 +1,601 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/sdp/sdp_changer.h"
+
+#include <utility>
+
+#include "absl/memory/memory.h"
+#include "api/jsep_session_description.h"
+#include "api/test/pclf/media_configuration.h"
+#include "media/base/media_constants.h"
+#include "p2p/base/p2p_constants.h"
+#include "pc/sdp_utils.h"
+#include "rtc_base/strings/string_builder.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+std::string CodecRequiredParamsToString(
+ const std::map<std::string, std::string>& codec_required_params) {
+ rtc::StringBuilder out;
+ for (const auto& entry : codec_required_params) {
+ out << entry.first << "=" << entry.second << ";";
+ }
+ return out.str();
+}
+
+std::string SupportedCodecsToString(
+ rtc::ArrayView<const RtpCodecCapability> supported_codecs) {
+ rtc::StringBuilder out;
+ for (const auto& codec : supported_codecs) {
+ out << codec.name;
+ if (!codec.parameters.empty()) {
+ out << "(";
+ for (const auto& param : codec.parameters) {
+ out << param.first << "=" << param.second << ";";
+ }
+ out << ")";
+ }
+ out << "; ";
+ }
+ return out.str();
+}
+
+} // namespace
+
+std::vector<RtpCodecCapability> FilterVideoCodecCapabilities(
+ rtc::ArrayView<const VideoCodecConfig> video_codecs,
+ bool use_rtx,
+ bool use_ulpfec,
+ bool use_flexfec,
+ rtc::ArrayView<const RtpCodecCapability> supported_codecs) {
+ std::vector<RtpCodecCapability> output_codecs;
+ // Find requested codecs among supported and add them to output in the order
+ // they were requested.
+ for (auto& codec_request : video_codecs) {
+ size_t size_before = output_codecs.size();
+ for (auto& codec : supported_codecs) {
+ if (codec.name != codec_request.name) {
+ continue;
+ }
+ bool parameters_matched = true;
+ for (const auto& item : codec_request.required_params) {
+ auto it = codec.parameters.find(item.first);
+ if (it == codec.parameters.end()) {
+ parameters_matched = false;
+ break;
+ }
+ if (item.second != it->second) {
+ parameters_matched = false;
+ break;
+ }
+ }
+ if (parameters_matched) {
+ output_codecs.push_back(codec);
+ }
+ }
+ RTC_CHECK_GT(output_codecs.size(), size_before)
+ << "Codec with name=" << codec_request.name << " and params {"
+ << CodecRequiredParamsToString(codec_request.required_params)
+ << "} is unsupported for this peer connection. Supported codecs are: "
+ << SupportedCodecsToString(supported_codecs);
+ }
+
+ // Add required FEC and RTX codecs to output.
+ for (auto& codec : supported_codecs) {
+ if (codec.name == cricket::kRtxCodecName && use_rtx) {
+ output_codecs.push_back(codec);
+ } else if (codec.name == cricket::kFlexfecCodecName && use_flexfec) {
+ output_codecs.push_back(codec);
+ } else if ((codec.name == cricket::kRedCodecName ||
+ codec.name == cricket::kUlpfecCodecName) &&
+ use_ulpfec) {
+ // Red and ulpfec should be enabled or disabled together.
+ output_codecs.push_back(codec);
+ }
+ }
+ return output_codecs;
+}
+
+// If offer has no simulcast video sections - do nothing.
+//
+// If offer has simulcast video sections - for each section creates
+// SimulcastSectionInfo and put it into `context_`.
+void SignalingInterceptor::FillSimulcastContext(
+ SessionDescriptionInterface* offer) {
+ for (auto& content : offer->description()->contents()) {
+ cricket::MediaContentDescription* media_desc = content.media_description();
+ if (media_desc->type() != cricket::MediaType::MEDIA_TYPE_VIDEO) {
+ continue;
+ }
+ if (media_desc->HasSimulcast()) {
+ // We support only single stream simulcast sections with rids.
+ RTC_CHECK_EQ(media_desc->mutable_streams().size(), 1);
+ RTC_CHECK(media_desc->mutable_streams()[0].has_rids());
+
+ // Create SimulcastSectionInfo for this video section.
+ SimulcastSectionInfo info(content.mid(), content.type,
+ media_desc->mutable_streams()[0].rids());
+
+ // Set new rids basing on created SimulcastSectionInfo.
+ std::vector<cricket::RidDescription> rids;
+ cricket::SimulcastDescription simulcast_description;
+ for (std::string& rid : info.rids) {
+ rids.emplace_back(rid, cricket::RidDirection::kSend);
+ simulcast_description.send_layers().AddLayer(
+ cricket::SimulcastLayer(rid, false));
+ }
+ media_desc->mutable_streams()[0].set_rids(rids);
+ media_desc->set_simulcast_description(simulcast_description);
+
+ info.simulcast_description = media_desc->simulcast_description();
+ for (const auto& extension : media_desc->rtp_header_extensions()) {
+ if (extension.uri == RtpExtension::kMidUri) {
+ info.mid_extension = extension;
+ } else if (extension.uri == RtpExtension::kRidUri) {
+ info.rid_extension = extension;
+ } else if (extension.uri == RtpExtension::kRepairedRidUri) {
+ info.rrid_extension = extension;
+ }
+ }
+ RTC_CHECK_NE(info.rid_extension.id, 0);
+ RTC_CHECK_NE(info.mid_extension.id, 0);
+ bool transport_description_found = false;
+ for (auto& transport_info : offer->description()->transport_infos()) {
+ if (transport_info.content_name == info.mid) {
+ info.transport_description = transport_info.description;
+ transport_description_found = true;
+ break;
+ }
+ }
+ RTC_CHECK(transport_description_found);
+
+ context_.AddSimulcastInfo(info);
+ }
+ }
+}
+
+LocalAndRemoteSdp SignalingInterceptor::PatchOffer(
+ std::unique_ptr<SessionDescriptionInterface> offer,
+ const VideoCodecConfig& first_codec) {
+ for (auto& content : offer->description()->contents()) {
+ context_.mids_order.push_back(content.mid());
+ cricket::MediaContentDescription* media_desc = content.media_description();
+ if (media_desc->type() != cricket::MediaType::MEDIA_TYPE_VIDEO) {
+ continue;
+ }
+ if (content.media_description()->streams().empty()) {
+ // It means that this media section describes receive only media section
+ // in SDP.
+ RTC_CHECK_EQ(content.media_description()->direction(),
+ RtpTransceiverDirection::kRecvOnly);
+ continue;
+ }
+ media_desc->set_conference_mode(params_.use_conference_mode);
+ }
+
+ if (!params_.stream_label_to_simulcast_streams_count.empty()) {
+ // Because simulcast enabled `params_.video_codecs` has only 1 element.
+ if (first_codec.name == cricket::kVp8CodecName) {
+ return PatchVp8Offer(std::move(offer));
+ }
+
+ if (first_codec.name == cricket::kVp9CodecName) {
+ return PatchVp9Offer(std::move(offer));
+ }
+ }
+
+ auto offer_for_remote = CloneSessionDescription(offer.get());
+ return LocalAndRemoteSdp(std::move(offer), std::move(offer_for_remote));
+}
+
+LocalAndRemoteSdp SignalingInterceptor::PatchVp8Offer(
+ std::unique_ptr<SessionDescriptionInterface> offer) {
+ FillSimulcastContext(offer.get());
+ if (!context_.HasSimulcast()) {
+ auto offer_for_remote = CloneSessionDescription(offer.get());
+ return LocalAndRemoteSdp(std::move(offer), std::move(offer_for_remote));
+ }
+
+ // Clone original offer description. We mustn't access original offer after
+ // this point.
+ std::unique_ptr<cricket::SessionDescription> desc =
+ offer->description()->Clone();
+
+ for (auto& info : context_.simulcast_infos) {
+ // For each simulcast section we have to perform:
+ // 1. Swap MID and RID header extensions
+ // 2. Remove RIDs from streams and remove SimulcastDescription
+ // 3. For each RID duplicate media section
+ cricket::ContentInfo* simulcast_content = desc->GetContentByName(info.mid);
+
+ // Now we need to prepare common prototype for "m=video" sections, in which
+ // single simulcast section will be converted. Do it before removing content
+ // because otherwise description will be deleted.
+ std::unique_ptr<cricket::MediaContentDescription> prototype_media_desc =
+ simulcast_content->media_description()->Clone();
+
+ // Remove simulcast video section from offer.
+ RTC_CHECK(desc->RemoveContentByName(simulcast_content->mid()));
+ // Clear `simulcast_content`, because now it is pointing to removed object.
+ simulcast_content = nullptr;
+
+ // Swap mid and rid extensions, so remote peer will understand rid as mid.
+ // Also remove rid extension.
+ std::vector<webrtc::RtpExtension> extensions =
+ prototype_media_desc->rtp_header_extensions();
+ for (auto ext_it = extensions.begin(); ext_it != extensions.end();) {
+ if (ext_it->uri == RtpExtension::kRidUri) {
+ // We don't need rid extension for remote peer.
+ ext_it = extensions.erase(ext_it);
+ continue;
+ }
+ if (ext_it->uri == RtpExtension::kRepairedRidUri) {
+ // We don't support RTX in simulcast.
+ ext_it = extensions.erase(ext_it);
+ continue;
+ }
+ if (ext_it->uri == RtpExtension::kMidUri) {
+ ext_it->id = info.rid_extension.id;
+ }
+ ++ext_it;
+ }
+
+ prototype_media_desc->ClearRtpHeaderExtensions();
+ prototype_media_desc->set_rtp_header_extensions(extensions);
+
+ // We support only single stream inside video section with simulcast
+ RTC_CHECK_EQ(prototype_media_desc->mutable_streams().size(), 1);
+ // This stream must have rids.
+ RTC_CHECK(prototype_media_desc->mutable_streams()[0].has_rids());
+
+ // Remove rids and simulcast description from media description.
+ prototype_media_desc->mutable_streams()[0].set_rids({});
+ prototype_media_desc->set_simulcast_description(
+ cricket::SimulcastDescription());
+
+ // For each rid add separate video section.
+ for (std::string& rid : info.rids) {
+ desc->AddContent(rid, info.media_protocol_type,
+ prototype_media_desc->Clone());
+ }
+ }
+
+ // Now we need to add bundle line to have all media bundled together.
+ cricket::ContentGroup bundle_group(cricket::GROUP_TYPE_BUNDLE);
+ for (auto& content : desc->contents()) {
+ bundle_group.AddContentName(content.mid());
+ }
+ if (desc->HasGroup(cricket::GROUP_TYPE_BUNDLE)) {
+ desc->RemoveGroupByName(cricket::GROUP_TYPE_BUNDLE);
+ }
+ desc->AddGroup(bundle_group);
+
+ // Update transport_infos to add TransportInfo for each new media section.
+ std::vector<cricket::TransportInfo> transport_infos = desc->transport_infos();
+ transport_infos.erase(std::remove_if(
+ transport_infos.begin(), transport_infos.end(),
+ [this](const cricket::TransportInfo& ti) {
+ // Remove transport infos that correspond to simulcast video sections.
+ return context_.simulcast_infos_by_mid.find(ti.content_name) !=
+ context_.simulcast_infos_by_mid.end();
+ }));
+ for (auto& info : context_.simulcast_infos) {
+ for (auto& rid : info.rids) {
+ transport_infos.emplace_back(rid, info.transport_description);
+ }
+ }
+ desc->set_transport_infos(transport_infos);
+
+ // Create patched offer.
+ auto patched_offer =
+ std::make_unique<JsepSessionDescription>(SdpType::kOffer);
+ patched_offer->Initialize(std::move(desc), offer->session_id(),
+ offer->session_version());
+ return LocalAndRemoteSdp(std::move(offer), std::move(patched_offer));
+}
+
+LocalAndRemoteSdp SignalingInterceptor::PatchVp9Offer(
+ std::unique_ptr<SessionDescriptionInterface> offer) {
+ rtc::UniqueRandomIdGenerator ssrcs_generator;
+ for (auto& content : offer->description()->contents()) {
+ for (auto& stream : content.media_description()->streams()) {
+ for (auto& ssrc : stream.ssrcs) {
+ ssrcs_generator.AddKnownId(ssrc);
+ }
+ }
+ }
+
+ for (auto& content : offer->description()->contents()) {
+ if (content.media_description()->type() !=
+ cricket::MediaType::MEDIA_TYPE_VIDEO) {
+ // We are interested in only video tracks
+ continue;
+ }
+ if (content.media_description()->direction() ==
+ RtpTransceiverDirection::kRecvOnly) {
+ // If direction is receive only, then there is no media in this track from
+ // sender side, so we needn't to do anything with this track.
+ continue;
+ }
+ RTC_CHECK_EQ(content.media_description()->streams().size(), 1);
+ cricket::StreamParams& stream =
+ content.media_description()->mutable_streams()[0];
+ RTC_CHECK_EQ(stream.stream_ids().size(), 2)
+ << "Expected 2 stream ids in video stream: 1st - sync_group, 2nd - "
+ "unique label";
+ std::string stream_label = stream.stream_ids()[1];
+
+ auto it =
+ params_.stream_label_to_simulcast_streams_count.find(stream_label);
+ if (it == params_.stream_label_to_simulcast_streams_count.end()) {
+ continue;
+ }
+ int svc_layers_count = it->second;
+
+ RTC_CHECK(stream.has_ssrc_groups()) << "Only SVC with RTX is supported";
+ RTC_CHECK_EQ(stream.ssrc_groups.size(), 1)
+ << "Too many ssrc groups in the track";
+ std::vector<uint32_t> primary_ssrcs;
+ stream.GetPrimarySsrcs(&primary_ssrcs);
+ RTC_CHECK(primary_ssrcs.size() == 1);
+ for (int i = 1; i < svc_layers_count; ++i) {
+ uint32_t ssrc = ssrcs_generator.GenerateId();
+ primary_ssrcs.push_back(ssrc);
+ stream.add_ssrc(ssrc);
+ stream.AddFidSsrc(ssrc, ssrcs_generator.GenerateId());
+ }
+ stream.ssrc_groups.push_back(
+ cricket::SsrcGroup(cricket::kSimSsrcGroupSemantics, primary_ssrcs));
+ }
+ auto offer_for_remote = CloneSessionDescription(offer.get());
+ return LocalAndRemoteSdp(std::move(offer), std::move(offer_for_remote));
+}
+
+LocalAndRemoteSdp SignalingInterceptor::PatchAnswer(
+ std::unique_ptr<SessionDescriptionInterface> answer,
+ const VideoCodecConfig& first_codec) {
+ for (auto& content : answer->description()->contents()) {
+ cricket::MediaContentDescription* media_desc = content.media_description();
+ if (media_desc->type() != cricket::MediaType::MEDIA_TYPE_VIDEO) {
+ continue;
+ }
+ if (content.media_description()->direction() !=
+ RtpTransceiverDirection::kRecvOnly) {
+ continue;
+ }
+ media_desc->set_conference_mode(params_.use_conference_mode);
+ }
+
+ if (!params_.stream_label_to_simulcast_streams_count.empty()) {
+ // Because simulcast enabled `params_.video_codecs` has only 1 element.
+ if (first_codec.name == cricket::kVp8CodecName) {
+ return PatchVp8Answer(std::move(answer));
+ }
+
+ if (first_codec.name == cricket::kVp9CodecName) {
+ return PatchVp9Answer(std::move(answer));
+ }
+ }
+
+ auto answer_for_remote = CloneSessionDescription(answer.get());
+ return LocalAndRemoteSdp(std::move(answer), std::move(answer_for_remote));
+}
+
+LocalAndRemoteSdp SignalingInterceptor::PatchVp8Answer(
+ std::unique_ptr<SessionDescriptionInterface> answer) {
+ if (!context_.HasSimulcast()) {
+ auto answer_for_remote = CloneSessionDescription(answer.get());
+ return LocalAndRemoteSdp(std::move(answer), std::move(answer_for_remote));
+ }
+
+ std::unique_ptr<cricket::SessionDescription> desc =
+ answer->description()->Clone();
+
+ for (auto& info : context_.simulcast_infos) {
+ cricket::ContentInfo* simulcast_content =
+ desc->GetContentByName(info.rids[0]);
+ RTC_CHECK(simulcast_content);
+
+ // Get media description, which will be converted to simulcast answer.
+ std::unique_ptr<cricket::MediaContentDescription> media_desc =
+ simulcast_content->media_description()->Clone();
+ // Set `simulcast_content` to nullptr, because then it will be removed, so
+ // it will point to deleted object.
+ simulcast_content = nullptr;
+
+ // Remove separate media sections for simulcast streams.
+ for (auto& rid : info.rids) {
+ RTC_CHECK(desc->RemoveContentByName(rid));
+ }
+
+ // Patch `media_desc` to make it simulcast answer description.
+ // Restore mid/rid rtp header extensions
+ std::vector<webrtc::RtpExtension> extensions =
+ media_desc->rtp_header_extensions();
+ // First remove existing rid/mid header extensions.
+ extensions.erase(std::remove_if(extensions.begin(), extensions.end(),
+ [](const webrtc::RtpExtension& e) {
+ return e.uri == RtpExtension::kMidUri ||
+ e.uri == RtpExtension::kRidUri ||
+ e.uri ==
+ RtpExtension::kRepairedRidUri;
+ }));
+
+ // Then add right ones.
+ extensions.push_back(info.mid_extension);
+ extensions.push_back(info.rid_extension);
+ // extensions.push_back(info.rrid_extension);
+ media_desc->ClearRtpHeaderExtensions();
+ media_desc->set_rtp_header_extensions(extensions);
+
+ // Add StreamParams with rids for receive.
+ RTC_CHECK_EQ(media_desc->mutable_streams().size(), 0);
+ std::vector<cricket::RidDescription> rids;
+ for (auto& rid : info.rids) {
+ rids.emplace_back(rid, cricket::RidDirection::kReceive);
+ }
+ cricket::StreamParams stream_params;
+ stream_params.set_rids(rids);
+ media_desc->mutable_streams().push_back(stream_params);
+
+ // Restore SimulcastDescription. It should correspond to one from offer,
+ // but it have to have receive layers instead of send. So we need to put
+ // send layers from offer to receive layers in answer.
+ cricket::SimulcastDescription simulcast_description;
+ for (const auto& layer : info.simulcast_description.send_layers()) {
+ simulcast_description.receive_layers().AddLayerWithAlternatives(layer);
+ }
+ media_desc->set_simulcast_description(simulcast_description);
+
+ // Add simulcast media section.
+ desc->AddContent(info.mid, info.media_protocol_type, std::move(media_desc));
+ }
+
+ desc = RestoreMediaSectionsOrder(std::move(desc));
+
+ // Now we need to add bundle line to have all media bundled together.
+ cricket::ContentGroup bundle_group(cricket::GROUP_TYPE_BUNDLE);
+ for (auto& content : desc->contents()) {
+ bundle_group.AddContentName(content.mid());
+ }
+ if (desc->HasGroup(cricket::GROUP_TYPE_BUNDLE)) {
+ desc->RemoveGroupByName(cricket::GROUP_TYPE_BUNDLE);
+ }
+ desc->AddGroup(bundle_group);
+
+ // Fix transport_infos: it have to have single info for simulcast section.
+ std::vector<cricket::TransportInfo> transport_infos = desc->transport_infos();
+ std::map<std::string, cricket::TransportDescription>
+ mid_to_transport_description;
+ for (auto info_it = transport_infos.begin();
+ info_it != transport_infos.end();) {
+ auto it = context_.simulcast_infos_by_rid.find(info_it->content_name);
+ if (it != context_.simulcast_infos_by_rid.end()) {
+ // This transport info correspond to some extra added media section.
+ mid_to_transport_description.insert(
+ {it->second->mid, info_it->description});
+ info_it = transport_infos.erase(info_it);
+ } else {
+ ++info_it;
+ }
+ }
+ for (auto& info : context_.simulcast_infos) {
+ transport_infos.emplace_back(info.mid,
+ mid_to_transport_description.at(info.mid));
+ }
+ desc->set_transport_infos(transport_infos);
+
+ auto patched_answer =
+ std::make_unique<JsepSessionDescription>(SdpType::kAnswer);
+ patched_answer->Initialize(std::move(desc), answer->session_id(),
+ answer->session_version());
+ return LocalAndRemoteSdp(std::move(answer), std::move(patched_answer));
+}
+
+std::unique_ptr<cricket::SessionDescription>
+SignalingInterceptor::RestoreMediaSectionsOrder(
+ std::unique_ptr<cricket::SessionDescription> source) {
+ std::unique_ptr<cricket::SessionDescription> out = source->Clone();
+ for (auto& mid : context_.mids_order) {
+ RTC_CHECK(out->RemoveContentByName(mid));
+ }
+ RTC_CHECK_EQ(out->contents().size(), 0);
+ for (auto& mid : context_.mids_order) {
+ cricket::ContentInfo* content = source->GetContentByName(mid);
+ RTC_CHECK(content);
+ out->AddContent(mid, content->type, content->media_description()->Clone());
+ }
+ return out;
+}
+
+LocalAndRemoteSdp SignalingInterceptor::PatchVp9Answer(
+ std::unique_ptr<SessionDescriptionInterface> answer) {
+ auto answer_for_remote = CloneSessionDescription(answer.get());
+ return LocalAndRemoteSdp(std::move(answer), std::move(answer_for_remote));
+}
+
+std::vector<std::unique_ptr<IceCandidateInterface>>
+SignalingInterceptor::PatchOffererIceCandidates(
+ rtc::ArrayView<const IceCandidateInterface* const> candidates) {
+ std::vector<std::unique_ptr<IceCandidateInterface>> out;
+ for (auto* candidate : candidates) {
+ auto simulcast_info_it =
+ context_.simulcast_infos_by_mid.find(candidate->sdp_mid());
+ if (simulcast_info_it != context_.simulcast_infos_by_mid.end()) {
+ // This is candidate for simulcast section, so it should be transformed
+ // into candidates for replicated sections. The sdpMLineIndex is set to
+ // -1 and ignored if the rid is present.
+ for (const std::string& rid : simulcast_info_it->second->rids) {
+ out.push_back(CreateIceCandidate(rid, -1, candidate->candidate()));
+ }
+ } else {
+ out.push_back(CreateIceCandidate(candidate->sdp_mid(),
+ candidate->sdp_mline_index(),
+ candidate->candidate()));
+ }
+ }
+ RTC_CHECK_GT(out.size(), 0);
+ return out;
+}
+
+std::vector<std::unique_ptr<IceCandidateInterface>>
+SignalingInterceptor::PatchAnswererIceCandidates(
+ rtc::ArrayView<const IceCandidateInterface* const> candidates) {
+ std::vector<std::unique_ptr<IceCandidateInterface>> out;
+ for (auto* candidate : candidates) {
+ auto simulcast_info_it =
+ context_.simulcast_infos_by_rid.find(candidate->sdp_mid());
+ if (simulcast_info_it != context_.simulcast_infos_by_rid.end()) {
+ // This is candidate for replicated section, created from single simulcast
+ // section, so it should be transformed into candidates for simulcast
+ // section.
+ out.push_back(CreateIceCandidate(simulcast_info_it->second->mid, 0,
+ candidate->candidate()));
+ } else if (!context_.simulcast_infos_by_rid.empty()) {
+ // When using simulcast and bundle, put everything on the first m-line.
+ out.push_back(CreateIceCandidate("", 0, candidate->candidate()));
+ } else {
+ out.push_back(CreateIceCandidate(candidate->sdp_mid(),
+ candidate->sdp_mline_index(),
+ candidate->candidate()));
+ }
+ }
+ RTC_CHECK_GT(out.size(), 0);
+ return out;
+}
+
+SignalingInterceptor::SimulcastSectionInfo::SimulcastSectionInfo(
+ const std::string& mid,
+ cricket::MediaProtocolType media_protocol_type,
+ const std::vector<cricket::RidDescription>& rids_desc)
+ : mid(mid), media_protocol_type(media_protocol_type) {
+ for (auto& rid : rids_desc) {
+ rids.push_back(rid.rid);
+ }
+}
+
+void SignalingInterceptor::SignalingContext::AddSimulcastInfo(
+ const SimulcastSectionInfo& info) {
+ simulcast_infos.push_back(info);
+ bool inserted =
+ simulcast_infos_by_mid.insert({info.mid, &simulcast_infos.back()}).second;
+ RTC_CHECK(inserted);
+ for (auto& rid : info.rids) {
+ inserted =
+ simulcast_infos_by_rid.insert({rid, &simulcast_infos.back()}).second;
+ RTC_CHECK(inserted);
+ }
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.h b/third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.h
new file mode 100644
index 0000000000..6f68d03f52
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/sdp/sdp_changer.h
@@ -0,0 +1,146 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_SDP_SDP_CHANGER_H_
+#define TEST_PC_E2E_SDP_SDP_CHANGER_H_
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
+#include "api/array_view.h"
+#include "api/jsep.h"
+#include "api/rtp_parameters.h"
+#include "api/test/pclf/media_configuration.h"
+#include "media/base/rid_description.h"
+#include "pc/session_description.h"
+#include "pc/simulcast_description.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+// Creates list of capabilities, which can be set on RtpTransceiverInterface via
+// RtpTransceiverInterface::SetCodecPreferences(...) to negotiate use of codecs
+// from list of `supported_codecs` which will match `video_codecs`. If flags
+// `ulpfec` or `flexfec` set to true corresponding FEC codec will be added.
+// FEC and RTX codecs will be added after required codecs.
+//
+// All codecs will be added only if they exists in the list of
+// `supported_codecs`. If multiple codecs from this list will match
+// `video_codecs`, then all of them will be added to the output
+// vector and they will be added in the same order, as they were in
+// `supported_codecs`.
+std::vector<RtpCodecCapability> FilterVideoCodecCapabilities(
+ rtc::ArrayView<const VideoCodecConfig> video_codecs,
+ bool use_rtx,
+ bool use_ulpfec,
+ bool use_flexfec,
+ rtc::ArrayView<const RtpCodecCapability> supported_codecs);
+
+struct LocalAndRemoteSdp {
+ LocalAndRemoteSdp(std::unique_ptr<SessionDescriptionInterface> local_sdp,
+ std::unique_ptr<SessionDescriptionInterface> remote_sdp)
+ : local_sdp(std::move(local_sdp)), remote_sdp(std::move(remote_sdp)) {}
+
+ // Sdp, that should be as local description on the peer, that created it.
+ std::unique_ptr<SessionDescriptionInterface> local_sdp;
+ // Sdp, that should be set as remote description on the peer opposite to the
+ // one, who created it.
+ std::unique_ptr<SessionDescriptionInterface> remote_sdp;
+};
+
+struct PatchingParams {
+ PatchingParams(
+ bool use_conference_mode,
+ std::map<std::string, int> stream_label_to_simulcast_streams_count)
+ : use_conference_mode(use_conference_mode),
+ stream_label_to_simulcast_streams_count(
+ stream_label_to_simulcast_streams_count) {}
+
+ bool use_conference_mode;
+ std::map<std::string, int> stream_label_to_simulcast_streams_count;
+};
+
+class SignalingInterceptor {
+ public:
+ explicit SignalingInterceptor(PatchingParams params) : params_(params) {}
+
+ LocalAndRemoteSdp PatchOffer(
+ std::unique_ptr<SessionDescriptionInterface> offer,
+ const VideoCodecConfig& first_codec);
+ LocalAndRemoteSdp PatchAnswer(
+ std::unique_ptr<SessionDescriptionInterface> answer,
+ const VideoCodecConfig& first_codec);
+
+ std::vector<std::unique_ptr<IceCandidateInterface>> PatchOffererIceCandidates(
+ rtc::ArrayView<const IceCandidateInterface* const> candidates);
+ std::vector<std::unique_ptr<IceCandidateInterface>>
+ PatchAnswererIceCandidates(
+ rtc::ArrayView<const IceCandidateInterface* const> candidates);
+
+ private:
+ // Contains information about simulcast section, that is required to perform
+ // modified offer/answer and ice candidates exchange.
+ struct SimulcastSectionInfo {
+ SimulcastSectionInfo(const std::string& mid,
+ cricket::MediaProtocolType media_protocol_type,
+ const std::vector<cricket::RidDescription>& rids_desc);
+
+ const std::string mid;
+ const cricket::MediaProtocolType media_protocol_type;
+ std::vector<std::string> rids;
+ cricket::SimulcastDescription simulcast_description;
+ webrtc::RtpExtension mid_extension;
+ webrtc::RtpExtension rid_extension;
+ webrtc::RtpExtension rrid_extension;
+ cricket::TransportDescription transport_description;
+ };
+
+ struct SignalingContext {
+ SignalingContext() = default;
+ // SignalingContext is not copyable and movable.
+ SignalingContext(SignalingContext&) = delete;
+ SignalingContext& operator=(SignalingContext&) = delete;
+ SignalingContext(SignalingContext&&) = delete;
+ SignalingContext& operator=(SignalingContext&&) = delete;
+
+ void AddSimulcastInfo(const SimulcastSectionInfo& info);
+ bool HasSimulcast() const { return !simulcast_infos.empty(); }
+
+ std::vector<SimulcastSectionInfo> simulcast_infos;
+ std::map<std::string, SimulcastSectionInfo*> simulcast_infos_by_mid;
+ std::map<std::string, SimulcastSectionInfo*> simulcast_infos_by_rid;
+
+ std::vector<std::string> mids_order;
+ };
+
+ LocalAndRemoteSdp PatchVp8Offer(
+ std::unique_ptr<SessionDescriptionInterface> offer);
+ LocalAndRemoteSdp PatchVp9Offer(
+ std::unique_ptr<SessionDescriptionInterface> offer);
+ LocalAndRemoteSdp PatchVp8Answer(
+ std::unique_ptr<SessionDescriptionInterface> answer);
+ LocalAndRemoteSdp PatchVp9Answer(
+ std::unique_ptr<SessionDescriptionInterface> answer);
+
+ void FillSimulcastContext(SessionDescriptionInterface* offer);
+ std::unique_ptr<cricket::SessionDescription> RestoreMediaSectionsOrder(
+ std::unique_ptr<cricket::SessionDescription> source);
+
+ PatchingParams params_;
+ SignalingContext context_;
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_SDP_SDP_CHANGER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.cc b/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.cc
new file mode 100644
index 0000000000..65dca5b518
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.cc
@@ -0,0 +1,592 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h"
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <set>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "absl/strings/string_view.h"
+#include "api/array_view.h"
+#include "api/scoped_refptr.h"
+#include "api/sequence_checker.h"
+#include "api/stats/rtc_stats.h"
+#include "api/stats/rtcstats_objects.h"
+#include "api/test/metrics/metric.h"
+#include "api/test/network_emulation/network_emulation_interfaces.h"
+#include "api/test/network_emulation_manager.h"
+#include "api/units/data_rate.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/event.h"
+#include "rtc_base/ip_address.h"
+#include "rtc_base/strings/string_builder.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "rtc_base/system/no_unique_address.h"
+#include "system_wrappers/include/field_trial.h"
+#include "test/pc/e2e/metric_metadata_keys.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+using ::webrtc::test::ImprovementDirection;
+using ::webrtc::test::Unit;
+
+using NetworkLayerStats =
+ StatsBasedNetworkQualityMetricsReporter::NetworkLayerStats;
+
+constexpr TimeDelta kStatsWaitTimeout = TimeDelta::Seconds(1);
+
+// Field trial which controls whether to report standard-compliant bytes
+// sent/received per stream. If enabled, padding and headers are not included
+// in bytes sent or received.
+constexpr char kUseStandardBytesStats[] = "WebRTC-UseStandardBytesStats";
+
+EmulatedNetworkStats PopulateStats(std::vector<EmulatedEndpoint*> endpoints,
+ NetworkEmulationManager* network_emulation) {
+ rtc::Event stats_loaded;
+ EmulatedNetworkStats stats;
+ network_emulation->GetStats(endpoints, [&](EmulatedNetworkStats s) {
+ stats = std::move(s);
+ stats_loaded.Set();
+ });
+ bool stats_received = stats_loaded.Wait(kStatsWaitTimeout);
+ RTC_CHECK(stats_received);
+ return stats;
+}
+
+std::map<rtc::IPAddress, std::string> PopulateIpToPeer(
+ const std::map<std::string, std::vector<EmulatedEndpoint*>>&
+ peer_endpoints) {
+ std::map<rtc::IPAddress, std::string> out;
+ for (const auto& entry : peer_endpoints) {
+ for (const EmulatedEndpoint* const endpoint : entry.second) {
+ RTC_CHECK(out.find(endpoint->GetPeerLocalAddress()) == out.end())
+ << "Two peers can't share the same endpoint";
+ out.emplace(endpoint->GetPeerLocalAddress(), entry.first);
+ }
+ }
+ return out;
+}
+
+// Accumulates emulated network stats being executed on the network thread.
+// When all stats are collected stores it in thread safe variable.
+class EmulatedNetworkStatsAccumulator {
+ public:
+ // `expected_stats_count` - the number of calls to
+ // AddEndpointStats/AddUplinkStats/AddDownlinkStats the accumulator is going
+ // to wait. If called more than expected, the program will crash.
+ explicit EmulatedNetworkStatsAccumulator(size_t expected_stats_count)
+ : not_collected_stats_count_(expected_stats_count) {
+ RTC_DCHECK_GE(not_collected_stats_count_, 0);
+ if (not_collected_stats_count_ == 0) {
+ all_stats_collected_.Set();
+ }
+ sequence_checker_.Detach();
+ }
+
+ // Has to be executed on network thread.
+ void AddEndpointStats(std::string peer_name, EmulatedNetworkStats stats) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
+ n_stats_[peer_name].endpoints_stats = std::move(stats);
+ DecrementNotCollectedStatsCount();
+ }
+
+ // Has to be executed on network thread.
+ void AddUplinkStats(std::string peer_name, EmulatedNetworkNodeStats stats) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
+ n_stats_[peer_name].uplink_stats = std::move(stats);
+ DecrementNotCollectedStatsCount();
+ }
+
+ // Has to be executed on network thread.
+ void AddDownlinkStats(std::string peer_name, EmulatedNetworkNodeStats stats) {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
+ n_stats_[peer_name].downlink_stats = std::move(stats);
+ DecrementNotCollectedStatsCount();
+ }
+
+ // Can be executed on any thread.
+ // Returns true if count down was completed and false if timeout elapsed
+ // before.
+ bool Wait(TimeDelta timeout) { return all_stats_collected_.Wait(timeout); }
+
+ // Can be called once. Returns all collected stats by moving underlying
+ // object.
+ std::map<std::string, NetworkLayerStats> ReleaseStats() {
+ RTC_DCHECK(!stats_released_);
+ stats_released_ = true;
+ MutexLock lock(&mutex_);
+ return std::move(stats_);
+ }
+
+ private:
+ void DecrementNotCollectedStatsCount() {
+ RTC_DCHECK_RUN_ON(&sequence_checker_);
+ RTC_CHECK_GT(not_collected_stats_count_, 0)
+ << "All stats are already collected";
+ not_collected_stats_count_--;
+ if (not_collected_stats_count_ == 0) {
+ MutexLock lock(&mutex_);
+ stats_ = std::move(n_stats_);
+ all_stats_collected_.Set();
+ }
+ }
+
+ RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_;
+ size_t not_collected_stats_count_ RTC_GUARDED_BY(sequence_checker_);
+ // Collected on the network thread. Moved into `stats_` after all stats are
+ // collected.
+ std::map<std::string, NetworkLayerStats> n_stats_
+ RTC_GUARDED_BY(sequence_checker_);
+
+ rtc::Event all_stats_collected_;
+ Mutex mutex_;
+ std::map<std::string, NetworkLayerStats> stats_ RTC_GUARDED_BY(mutex_);
+ bool stats_released_ = false;
+};
+
+} // namespace
+
+StatsBasedNetworkQualityMetricsReporter::
+ StatsBasedNetworkQualityMetricsReporter(
+ std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints,
+ NetworkEmulationManager* network_emulation,
+ test::MetricsLogger* metrics_logger)
+ : collector_(std::move(peer_endpoints), network_emulation),
+ clock_(network_emulation->time_controller()->GetClock()),
+ metrics_logger_(metrics_logger) {
+ RTC_CHECK(metrics_logger_);
+}
+
+StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector::
+ NetworkLayerStatsCollector(
+ std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints,
+ NetworkEmulationManager* network_emulation)
+ : peer_endpoints_(std::move(peer_endpoints)),
+ ip_to_peer_(PopulateIpToPeer(peer_endpoints_)),
+ network_emulation_(network_emulation) {}
+
+void StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector::
+ Start() {
+ MutexLock lock(&mutex_);
+ // Check that network stats are clean before test execution.
+ for (const auto& entry : peer_endpoints_) {
+ EmulatedNetworkStats stats =
+ PopulateStats(entry.second, network_emulation_);
+ RTC_CHECK_EQ(stats.overall_outgoing_stats.packets_sent, 0);
+ RTC_CHECK_EQ(stats.overall_incoming_stats.packets_received, 0);
+ }
+}
+
+void StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector::
+ AddPeer(absl::string_view peer_name,
+ std::vector<EmulatedEndpoint*> endpoints,
+ std::vector<EmulatedNetworkNode*> uplink,
+ std::vector<EmulatedNetworkNode*> downlink) {
+ MutexLock lock(&mutex_);
+ // When new peer is added not in the constructor, don't check if it has empty
+ // stats, because their endpoint could be used for traffic before.
+ peer_endpoints_.emplace(peer_name, std::move(endpoints));
+ peer_uplinks_.emplace(peer_name, std::move(uplink));
+ peer_downlinks_.emplace(peer_name, std::move(downlink));
+ for (const EmulatedEndpoint* const endpoint : endpoints) {
+ RTC_CHECK(ip_to_peer_.find(endpoint->GetPeerLocalAddress()) ==
+ ip_to_peer_.end())
+ << "Two peers can't share the same endpoint";
+ ip_to_peer_.emplace(endpoint->GetPeerLocalAddress(), peer_name);
+ }
+}
+
+std::map<std::string, NetworkLayerStats>
+StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector::
+ GetStats() {
+ MutexLock lock(&mutex_);
+ EmulatedNetworkStatsAccumulator stats_accumulator(
+ peer_endpoints_.size() + peer_uplinks_.size() + peer_downlinks_.size());
+ for (const auto& entry : peer_endpoints_) {
+ network_emulation_->GetStats(
+ entry.second, [&stats_accumulator,
+ peer = entry.first](EmulatedNetworkStats s) mutable {
+ stats_accumulator.AddEndpointStats(std::move(peer), std::move(s));
+ });
+ }
+ for (const auto& entry : peer_uplinks_) {
+ network_emulation_->GetStats(
+ entry.second, [&stats_accumulator,
+ peer = entry.first](EmulatedNetworkNodeStats s) mutable {
+ stats_accumulator.AddUplinkStats(std::move(peer), std::move(s));
+ });
+ }
+ for (const auto& entry : peer_downlinks_) {
+ network_emulation_->GetStats(
+ entry.second, [&stats_accumulator,
+ peer = entry.first](EmulatedNetworkNodeStats s) mutable {
+ stats_accumulator.AddDownlinkStats(std::move(peer), std::move(s));
+ });
+ }
+ bool stats_collected = stats_accumulator.Wait(kStatsWaitTimeout);
+ RTC_CHECK(stats_collected);
+ std::map<std::string, NetworkLayerStats> peer_to_stats =
+ stats_accumulator.ReleaseStats();
+ std::map<std::string, std::vector<std::string>> sender_to_receivers;
+ for (const auto& entry : peer_endpoints_) {
+ const std::string& peer_name = entry.first;
+ const NetworkLayerStats& stats = peer_to_stats[peer_name];
+ for (const auto& income_stats_entry :
+ stats.endpoints_stats.incoming_stats_per_source) {
+ const rtc::IPAddress& source_ip = income_stats_entry.first;
+ auto it = ip_to_peer_.find(source_ip);
+ if (it == ip_to_peer_.end()) {
+ // Source IP is unknown for this collector, so will be skipped.
+ continue;
+ }
+ sender_to_receivers[it->second].push_back(peer_name);
+ }
+ }
+ for (auto& entry : peer_to_stats) {
+ const std::vector<std::string>& receivers =
+ sender_to_receivers[entry.first];
+ entry.second.receivers =
+ std::set<std::string>(receivers.begin(), receivers.end());
+ }
+ return peer_to_stats;
+}
+
+void StatsBasedNetworkQualityMetricsReporter::AddPeer(
+ absl::string_view peer_name,
+ std::vector<EmulatedEndpoint*> endpoints) {
+ collector_.AddPeer(peer_name, std::move(endpoints), /*uplink=*/{},
+ /*downlink=*/{});
+}
+
+void StatsBasedNetworkQualityMetricsReporter::AddPeer(
+ absl::string_view peer_name,
+ std::vector<EmulatedEndpoint*> endpoints,
+ std::vector<EmulatedNetworkNode*> uplink,
+ std::vector<EmulatedNetworkNode*> downlink) {
+ collector_.AddPeer(peer_name, std::move(endpoints), std::move(uplink),
+ std::move(downlink));
+}
+
+void StatsBasedNetworkQualityMetricsReporter::Start(
+ absl::string_view test_case_name,
+ const TrackIdStreamInfoMap* reporter_helper) {
+ test_case_name_ = std::string(test_case_name);
+ collector_.Start();
+ start_time_ = clock_->CurrentTime();
+}
+
+void StatsBasedNetworkQualityMetricsReporter::OnStatsReports(
+ absl::string_view pc_label,
+ const rtc::scoped_refptr<const RTCStatsReport>& report) {
+ PCStats cur_stats;
+
+ auto inbound_stats = report->GetStatsOfType<RTCInboundRTPStreamStats>();
+ for (const auto& stat : inbound_stats) {
+ cur_stats.payload_received +=
+ DataSize::Bytes(stat->bytes_received.ValueOrDefault(0ul) +
+ stat->header_bytes_received.ValueOrDefault(0ul));
+ }
+
+ auto outbound_stats = report->GetStatsOfType<RTCOutboundRTPStreamStats>();
+ for (const auto& stat : outbound_stats) {
+ cur_stats.payload_sent +=
+ DataSize::Bytes(stat->bytes_sent.ValueOrDefault(0ul) +
+ stat->header_bytes_sent.ValueOrDefault(0ul));
+ }
+
+ auto candidate_pairs_stats = report->GetStatsOfType<RTCTransportStats>();
+ for (const auto& stat : candidate_pairs_stats) {
+ cur_stats.total_received +=
+ DataSize::Bytes(stat->bytes_received.ValueOrDefault(0ul));
+ cur_stats.total_sent +=
+ DataSize::Bytes(stat->bytes_sent.ValueOrDefault(0ul));
+ cur_stats.packets_received += stat->packets_received.ValueOrDefault(0ul);
+ cur_stats.packets_sent += stat->packets_sent.ValueOrDefault(0ul);
+ }
+
+ MutexLock lock(&mutex_);
+ pc_stats_[std::string(pc_label)] = cur_stats;
+}
+
+void StatsBasedNetworkQualityMetricsReporter::StopAndReportResults() {
+ Timestamp end_time = clock_->CurrentTime();
+
+ if (!webrtc::field_trial::IsEnabled(kUseStandardBytesStats)) {
+ RTC_LOG(LS_ERROR)
+ << "Non-standard GetStats; \"payload\" counts include RTP headers";
+ }
+
+ std::map<std::string, NetworkLayerStats> stats = collector_.GetStats();
+ for (const auto& entry : stats) {
+ LogNetworkLayerStats(entry.first, entry.second);
+ }
+ MutexLock lock(&mutex_);
+ for (const auto& pair : pc_stats_) {
+ auto it = stats.find(pair.first);
+ RTC_CHECK(it != stats.end())
+ << "Peer name used for PeerConnection stats collection and peer name "
+ "used for endpoints naming doesn't match. No endpoints found for "
+ "peer "
+ << pair.first;
+ const NetworkLayerStats& network_layer_stats = it->second;
+ int64_t total_packets_received = 0;
+ bool found = false;
+ for (const auto& dest_peer : network_layer_stats.receivers) {
+ auto pc_stats_it = pc_stats_.find(dest_peer);
+ if (pc_stats_it == pc_stats_.end()) {
+ continue;
+ }
+ found = true;
+ total_packets_received += pc_stats_it->second.packets_received;
+ }
+ int64_t packet_loss = -1;
+ if (found) {
+ packet_loss = pair.second.packets_sent - total_packets_received;
+ }
+ ReportStats(pair.first, pair.second, network_layer_stats, packet_loss,
+ end_time);
+ }
+}
+
+void StatsBasedNetworkQualityMetricsReporter::ReportStats(
+ const std::string& pc_label,
+ const PCStats& pc_stats,
+ const NetworkLayerStats& network_layer_stats,
+ int64_t packet_loss,
+ const Timestamp& end_time) {
+ // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey.
+ std::map<std::string, std::string> metric_metadata{
+ {MetricMetadataKey::kPeerMetadataKey, pc_label},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_case_name_}};
+ metrics_logger_->LogSingleValueMetric(
+ "bytes_discarded_no_receiver", GetTestCaseName(pc_label),
+ network_layer_stats.endpoints_stats.overall_incoming_stats
+ .bytes_discarded_no_receiver.bytes(),
+ Unit::kBytes, ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ metrics_logger_->LogSingleValueMetric(
+ "packets_discarded_no_receiver", GetTestCaseName(pc_label),
+ network_layer_stats.endpoints_stats.overall_incoming_stats
+ .packets_discarded_no_receiver,
+ Unit::kUnitless, ImprovementDirection::kNeitherIsBetter, metric_metadata);
+
+ metrics_logger_->LogSingleValueMetric(
+ "payload_bytes_received", GetTestCaseName(pc_label),
+ pc_stats.payload_received.bytes(), Unit::kBytes,
+ ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ metrics_logger_->LogSingleValueMetric(
+ "payload_bytes_sent", GetTestCaseName(pc_label),
+ pc_stats.payload_sent.bytes(), Unit::kBytes,
+ ImprovementDirection::kNeitherIsBetter, metric_metadata);
+
+ metrics_logger_->LogSingleValueMetric(
+ "bytes_sent", GetTestCaseName(pc_label), pc_stats.total_sent.bytes(),
+ Unit::kBytes, ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ metrics_logger_->LogSingleValueMetric(
+ "packets_sent", GetTestCaseName(pc_label), pc_stats.packets_sent,
+ Unit::kUnitless, ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ metrics_logger_->LogSingleValueMetric(
+ "average_send_rate", GetTestCaseName(pc_label),
+ (pc_stats.total_sent / (end_time - start_time_)).kbps<double>(),
+ Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter,
+ metric_metadata);
+ metrics_logger_->LogSingleValueMetric(
+ "bytes_received", GetTestCaseName(pc_label),
+ pc_stats.total_received.bytes(), Unit::kBytes,
+ ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ metrics_logger_->LogSingleValueMetric(
+ "packets_received", GetTestCaseName(pc_label), pc_stats.packets_received,
+ Unit::kUnitless, ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ metrics_logger_->LogSingleValueMetric(
+ "average_receive_rate", GetTestCaseName(pc_label),
+ (pc_stats.total_received / (end_time - start_time_)).kbps<double>(),
+ Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter,
+ metric_metadata);
+ metrics_logger_->LogSingleValueMetric(
+ "sent_packets_loss", GetTestCaseName(pc_label), packet_loss,
+ Unit::kUnitless, ImprovementDirection::kNeitherIsBetter, metric_metadata);
+}
+
+std::string StatsBasedNetworkQualityMetricsReporter::GetTestCaseName(
+ absl::string_view network_label) const {
+ rtc::StringBuilder builder;
+ builder << test_case_name_ << "/" << network_label.data();
+ return builder.str();
+}
+
+void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats(
+ const std::string& peer_name,
+ const NetworkLayerStats& stats) const {
+ DataRate average_send_rate =
+ stats.endpoints_stats.overall_outgoing_stats.packets_sent >= 2
+ ? stats.endpoints_stats.overall_outgoing_stats.AverageSendRate()
+ : DataRate::Zero();
+ DataRate average_receive_rate =
+ stats.endpoints_stats.overall_incoming_stats.packets_received >= 2
+ ? stats.endpoints_stats.overall_incoming_stats.AverageReceiveRate()
+ : DataRate::Zero();
+ // TODO(bugs.webrtc.org/14757): Remove kExperimentalTestNameMetadataKey.
+ std::map<std::string, std::string> metric_metadata{
+ {MetricMetadataKey::kPeerMetadataKey, peer_name},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey, test_case_name_}};
+ rtc::StringBuilder log;
+ log << "Raw network layer statistic for [" << peer_name << "]:\n"
+ << "Local IPs:\n";
+ for (size_t i = 0; i < stats.endpoints_stats.local_addresses.size(); ++i) {
+ log << " " << stats.endpoints_stats.local_addresses[i].ToString() << "\n";
+ }
+ if (!stats.endpoints_stats.overall_outgoing_stats.sent_packets_size
+ .IsEmpty()) {
+ metrics_logger_->LogMetric(
+ "sent_packets_size", GetTestCaseName(peer_name),
+ stats.endpoints_stats.overall_outgoing_stats.sent_packets_size,
+ Unit::kBytes, ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ }
+ if (!stats.endpoints_stats.overall_incoming_stats.received_packets_size
+ .IsEmpty()) {
+ metrics_logger_->LogMetric(
+ "received_packets_size", GetTestCaseName(peer_name),
+ stats.endpoints_stats.overall_incoming_stats.received_packets_size,
+ Unit::kBytes, ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ }
+ if (!stats.endpoints_stats.overall_incoming_stats
+ .packets_discarded_no_receiver_size.IsEmpty()) {
+ metrics_logger_->LogMetric(
+ "packets_discarded_no_receiver_size", GetTestCaseName(peer_name),
+ stats.endpoints_stats.overall_incoming_stats
+ .packets_discarded_no_receiver_size,
+ Unit::kBytes, ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ }
+ if (!stats.endpoints_stats.sent_packets_queue_wait_time_us.IsEmpty()) {
+ metrics_logger_->LogMetric(
+ "sent_packets_queue_wait_time_us", GetTestCaseName(peer_name),
+ stats.endpoints_stats.sent_packets_queue_wait_time_us, Unit::kUnitless,
+ ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ }
+
+ log << "Send statistic:\n"
+ << " packets: "
+ << stats.endpoints_stats.overall_outgoing_stats.packets_sent << " bytes: "
+ << stats.endpoints_stats.overall_outgoing_stats.bytes_sent.bytes()
+ << " avg_rate (bytes/sec): " << average_send_rate.bytes_per_sec()
+ << " avg_rate (bps): " << average_send_rate.bps() << "\n"
+ << "Send statistic per destination:\n";
+
+ for (const auto& entry :
+ stats.endpoints_stats.outgoing_stats_per_destination) {
+ DataRate source_average_send_rate = entry.second.packets_sent >= 2
+ ? entry.second.AverageSendRate()
+ : DataRate::Zero();
+ log << "(" << entry.first.ToString() << "):\n"
+ << " packets: " << entry.second.packets_sent
+ << " bytes: " << entry.second.bytes_sent.bytes()
+ << " avg_rate (bytes/sec): " << source_average_send_rate.bytes_per_sec()
+ << " avg_rate (bps): " << source_average_send_rate.bps() << "\n";
+ if (!entry.second.sent_packets_size.IsEmpty()) {
+ metrics_logger_->LogMetric(
+ "sent_packets_size",
+ GetTestCaseName(peer_name + "/" + entry.first.ToString()),
+ entry.second.sent_packets_size, Unit::kBytes,
+ ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ }
+ }
+
+ if (!stats.uplink_stats.packet_transport_time.IsEmpty()) {
+ log << "[Debug stats] packet_transport_time=("
+ << stats.uplink_stats.packet_transport_time.GetAverage() << ", "
+ << stats.uplink_stats.packet_transport_time.GetStandardDeviation()
+ << ")\n";
+ metrics_logger_->LogMetric(
+ "uplink_packet_transport_time", GetTestCaseName(peer_name),
+ stats.uplink_stats.packet_transport_time, Unit::kMilliseconds,
+ ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ }
+ if (!stats.uplink_stats.size_to_packet_transport_time.IsEmpty()) {
+ log << "[Debug stats] size_to_packet_transport_time=("
+ << stats.uplink_stats.size_to_packet_transport_time.GetAverage() << ", "
+ << stats.uplink_stats.size_to_packet_transport_time
+ .GetStandardDeviation()
+ << ")\n";
+ metrics_logger_->LogMetric(
+ "uplink_size_to_packet_transport_time", GetTestCaseName(peer_name),
+ stats.uplink_stats.size_to_packet_transport_time, Unit::kUnitless,
+ ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ }
+
+ log << "Receive statistic:\n"
+ << " packets: "
+ << stats.endpoints_stats.overall_incoming_stats.packets_received
+ << " bytes: "
+ << stats.endpoints_stats.overall_incoming_stats.bytes_received.bytes()
+ << " avg_rate (bytes/sec): " << average_receive_rate.bytes_per_sec()
+ << " avg_rate (bps): " << average_receive_rate.bps() << "\n"
+ << "Receive statistic per source:\n";
+
+ for (const auto& entry : stats.endpoints_stats.incoming_stats_per_source) {
+ DataRate source_average_receive_rate =
+ entry.second.packets_received >= 2 ? entry.second.AverageReceiveRate()
+ : DataRate::Zero();
+ log << "(" << entry.first.ToString() << "):\n"
+ << " packets: " << entry.second.packets_received
+ << " bytes: " << entry.second.bytes_received.bytes()
+ << " avg_rate (bytes/sec): "
+ << source_average_receive_rate.bytes_per_sec()
+ << " avg_rate (bps): " << source_average_receive_rate.bps() << "\n";
+ if (!entry.second.received_packets_size.IsEmpty()) {
+ metrics_logger_->LogMetric(
+ "received_packets_size",
+ GetTestCaseName(peer_name + "/" + entry.first.ToString()),
+ entry.second.received_packets_size, Unit::kBytes,
+ ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ }
+ if (!entry.second.packets_discarded_no_receiver_size.IsEmpty()) {
+ metrics_logger_->LogMetric(
+ "packets_discarded_no_receiver_size",
+ GetTestCaseName(peer_name + "/" + entry.first.ToString()),
+ entry.second.packets_discarded_no_receiver_size, Unit::kBytes,
+ ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ }
+ }
+ if (!stats.downlink_stats.packet_transport_time.IsEmpty()) {
+ log << "[Debug stats] packet_transport_time=("
+ << stats.downlink_stats.packet_transport_time.GetAverage() << ", "
+ << stats.downlink_stats.packet_transport_time.GetStandardDeviation()
+ << ")\n";
+ metrics_logger_->LogMetric(
+ "downlink_packet_transport_time", GetTestCaseName(peer_name),
+ stats.downlink_stats.packet_transport_time, Unit::kMilliseconds,
+ ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ }
+ if (!stats.downlink_stats.size_to_packet_transport_time.IsEmpty()) {
+ log << "[Debug stats] size_to_packet_transport_time=("
+ << stats.downlink_stats.size_to_packet_transport_time.GetAverage()
+ << ", "
+ << stats.downlink_stats.size_to_packet_transport_time
+ .GetStandardDeviation()
+ << ")\n";
+ metrics_logger_->LogMetric(
+ "downlink_size_to_packet_transport_time", GetTestCaseName(peer_name),
+ stats.downlink_stats.size_to_packet_transport_time, Unit::kUnitless,
+ ImprovementDirection::kNeitherIsBetter, metric_metadata);
+ }
+
+ RTC_LOG(LS_INFO) << log.str();
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.h b/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.h
new file mode 100644
index 0000000000..60daf40c8c
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter.h
@@ -0,0 +1,136 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_STATS_BASED_NETWORK_QUALITY_METRICS_REPORTER_H_
+#define TEST_PC_E2E_STATS_BASED_NETWORK_QUALITY_METRICS_REPORTER_H_
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/strings/string_view.h"
+#include "api/numerics/samples_stats_counter.h"
+#include "api/test/metrics/metrics_logger.h"
+#include "api/test/network_emulation/network_emulation_interfaces.h"
+#include "api/test/network_emulation_manager.h"
+#include "api/test/peerconnection_quality_test_fixture.h"
+#include "api/units/data_size.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/ip_address.h"
+#include "rtc_base/synchronization/mutex.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+// TODO(titovartem): make this class testable and add tests.
+class StatsBasedNetworkQualityMetricsReporter
+ : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter {
+ public:
+ // Emulated network layer stats for single peer.
+ struct NetworkLayerStats {
+ EmulatedNetworkStats endpoints_stats;
+ EmulatedNetworkNodeStats uplink_stats;
+ EmulatedNetworkNodeStats downlink_stats;
+ std::set<std::string> receivers;
+ };
+
+ // `networks` map peer name to network to report network layer stability stats
+ // and to log network layer metrics.
+ StatsBasedNetworkQualityMetricsReporter(
+ std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints,
+ NetworkEmulationManager* network_emulation,
+ test::MetricsLogger* metrics_logger);
+ ~StatsBasedNetworkQualityMetricsReporter() override = default;
+
+ void AddPeer(absl::string_view peer_name,
+ std::vector<EmulatedEndpoint*> endpoints);
+ void AddPeer(absl::string_view peer_name,
+ std::vector<EmulatedEndpoint*> endpoints,
+ std::vector<EmulatedNetworkNode*> uplink,
+ std::vector<EmulatedNetworkNode*> downlink);
+
+ // Network stats must be empty when this method will be invoked.
+ void Start(absl::string_view test_case_name,
+ const TrackIdStreamInfoMap* reporter_helper) override;
+ void OnStatsReports(
+ absl::string_view pc_label,
+ const rtc::scoped_refptr<const RTCStatsReport>& report) override;
+ void StopAndReportResults() override;
+
+ private:
+ struct PCStats {
+ // TODO(bugs.webrtc.org/10525): Separate audio and video counters. Depends
+ // on standard stat counters, enabled by field trial
+ // "WebRTC-UseStandardBytesStats".
+ DataSize payload_received = DataSize::Zero();
+ DataSize payload_sent = DataSize::Zero();
+
+ // Total bytes/packets sent/received in all RTCTransport's.
+ DataSize total_received = DataSize::Zero();
+ DataSize total_sent = DataSize::Zero();
+ int64_t packets_received = 0;
+ int64_t packets_sent = 0;
+ };
+
+ class NetworkLayerStatsCollector {
+ public:
+ NetworkLayerStatsCollector(
+ std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints,
+ NetworkEmulationManager* network_emulation);
+
+ void Start();
+
+ void AddPeer(absl::string_view peer_name,
+ std::vector<EmulatedEndpoint*> endpoints,
+ std::vector<EmulatedNetworkNode*> uplink,
+ std::vector<EmulatedNetworkNode*> downlink);
+
+ std::map<std::string, NetworkLayerStats> GetStats();
+
+ private:
+ Mutex mutex_;
+ std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints_
+ RTC_GUARDED_BY(mutex_);
+ std::map<std::string, std::vector<EmulatedNetworkNode*>> peer_uplinks_
+ RTC_GUARDED_BY(mutex_);
+ std::map<std::string, std::vector<EmulatedNetworkNode*>> peer_downlinks_
+ RTC_GUARDED_BY(mutex_);
+ std::map<rtc::IPAddress, std::string> ip_to_peer_ RTC_GUARDED_BY(mutex_);
+ NetworkEmulationManager* const network_emulation_;
+ };
+
+ void ReportStats(const std::string& pc_label,
+ const PCStats& pc_stats,
+ const NetworkLayerStats& network_layer_stats,
+ int64_t packet_loss,
+ const Timestamp& end_time);
+ std::string GetTestCaseName(absl::string_view network_label) const;
+ void LogNetworkLayerStats(const std::string& peer_name,
+ const NetworkLayerStats& stats) const;
+
+ NetworkLayerStatsCollector collector_;
+ Clock* const clock_;
+ test::MetricsLogger* const metrics_logger_;
+
+ std::string test_case_name_;
+ Timestamp start_time_ = Timestamp::MinusInfinity();
+
+ Mutex mutex_;
+ std::map<std::string, PCStats> pc_stats_ RTC_GUARDED_BY(mutex_);
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_STATS_BASED_NETWORK_QUALITY_METRICS_REPORTER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter_test.cc b/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter_test.cc
new file mode 100644
index 0000000000..be55149482
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/stats_based_network_quality_metrics_reporter_test.cc
@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h"
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
+#include "api/array_view.h"
+#include "api/test/create_network_emulation_manager.h"
+#include "api/test/create_peer_connection_quality_test_frame_generator.h"
+#include "api/test/metrics/metrics_logger.h"
+#include "api/test/metrics/stdout_metrics_exporter.h"
+#include "api/test/network_emulation_manager.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/pclf/media_quality_test_params.h"
+#include "api/test/pclf/peer_configurer.h"
+#include "api/test/peerconnection_quality_test_fixture.h"
+#include "api/units/time_delta.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/pc/e2e/metric_metadata_keys.h"
+#include "test/pc/e2e/peer_connection_quality_test.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+using ::testing::UnorderedElementsAre;
+
+using ::webrtc::test::DefaultMetricsLogger;
+using ::webrtc::test::ImprovementDirection;
+using ::webrtc::test::Metric;
+using ::webrtc::test::Unit;
+using ::webrtc::webrtc_pc_e2e::PeerConfigurer;
+
+// Adds a peer with some audio and video (the client should not care about
+// details about audio and video configs).
+void AddDefaultAudioVideoPeer(
+ absl::string_view peer_name,
+ absl::string_view audio_stream_label,
+ absl::string_view video_stream_label,
+ const PeerNetworkDependencies& network_dependencies,
+ PeerConnectionE2EQualityTestFixture& fixture) {
+ AudioConfig audio{std::string(audio_stream_label)};
+ audio.sync_group = std::string(peer_name);
+ VideoConfig video(std::string(video_stream_label), 320, 180, 15);
+ video.sync_group = std::string(peer_name);
+ auto peer = std::make_unique<PeerConfigurer>(network_dependencies);
+ peer->SetName(peer_name);
+ peer->SetAudioConfig(std::move(audio));
+ peer->AddVideoConfig(std::move(video));
+ peer->SetVideoCodecs({VideoCodecConfig(cricket::kVp8CodecName)});
+ fixture.AddPeer(std::move(peer));
+}
+
+absl::optional<Metric> FindMeetricByName(absl::string_view name,
+ rtc::ArrayView<const Metric> metrics) {
+ for (const Metric& metric : metrics) {
+ if (metric.name == name) {
+ return metric;
+ }
+ }
+ return absl::nullopt;
+}
+
+TEST(StatsBasedNetworkQualityMetricsReporterTest, DebugStatsAreCollected) {
+ std::unique_ptr<NetworkEmulationManager> network_emulation =
+ CreateNetworkEmulationManager(TimeMode::kSimulated,
+ EmulatedNetworkStatsGatheringMode::kDebug);
+ DefaultMetricsLogger metrics_logger(
+ network_emulation->time_controller()->GetClock());
+ PeerConnectionE2EQualityTest fixture(
+ "test_case", *network_emulation->time_controller(),
+ /*audio_quality_analyzer=*/nullptr, /*video_quality_analyzer=*/nullptr,
+ &metrics_logger);
+
+ EmulatedEndpoint* alice_endpoint =
+ network_emulation->CreateEndpoint(EmulatedEndpointConfig());
+ EmulatedEndpoint* bob_endpoint =
+ network_emulation->CreateEndpoint(EmulatedEndpointConfig());
+
+ EmulatedNetworkNode* alice_link = network_emulation->CreateEmulatedNode(
+ BuiltInNetworkBehaviorConfig{.link_capacity_kbps = 500});
+ network_emulation->CreateRoute(alice_endpoint, {alice_link}, bob_endpoint);
+ EmulatedNetworkNode* bob_link = network_emulation->CreateEmulatedNode(
+ BuiltInNetworkBehaviorConfig{.link_capacity_kbps = 500});
+ network_emulation->CreateRoute(bob_endpoint, {bob_link}, alice_endpoint);
+
+ EmulatedNetworkManagerInterface* alice_network =
+ network_emulation->CreateEmulatedNetworkManagerInterface(
+ {alice_endpoint});
+ EmulatedNetworkManagerInterface* bob_network =
+ network_emulation->CreateEmulatedNetworkManagerInterface({bob_endpoint});
+
+ AddDefaultAudioVideoPeer("alice", "alice_audio", "alice_video",
+ alice_network->network_dependencies(), fixture);
+ AddDefaultAudioVideoPeer("bob", "bob_audio", "bob_video",
+ bob_network->network_dependencies(), fixture);
+
+ auto network_stats_reporter =
+ std::make_unique<StatsBasedNetworkQualityMetricsReporter>(
+ /*peer_endpoints=*/std::map<std::string,
+ std::vector<EmulatedEndpoint*>>{},
+ network_emulation.get(), &metrics_logger);
+ network_stats_reporter->AddPeer("alice", alice_network->endpoints(),
+ /*uplink=*/{alice_link},
+ /*downlink=*/{bob_link});
+ network_stats_reporter->AddPeer("bob", bob_network->endpoints(),
+ /*uplink=*/{bob_link},
+ /*downlink=*/{alice_link});
+ fixture.AddQualityMetricsReporter(std::move(network_stats_reporter));
+
+ fixture.Run(RunParams(TimeDelta::Seconds(4)));
+
+ std::vector<Metric> metrics = metrics_logger.GetCollectedMetrics();
+ absl::optional<Metric> uplink_packet_transport_time =
+ FindMeetricByName("uplink_packet_transport_time", metrics);
+ ASSERT_TRUE(uplink_packet_transport_time.has_value());
+ ASSERT_FALSE(uplink_packet_transport_time->time_series.samples.empty());
+ absl::optional<Metric> uplink_size_to_packet_transport_time =
+ FindMeetricByName("uplink_size_to_packet_transport_time", metrics);
+ ASSERT_TRUE(uplink_size_to_packet_transport_time.has_value());
+ ASSERT_FALSE(
+ uplink_size_to_packet_transport_time->time_series.samples.empty());
+ absl::optional<Metric> downlink_packet_transport_time =
+ FindMeetricByName("downlink_packet_transport_time", metrics);
+ ASSERT_TRUE(downlink_packet_transport_time.has_value());
+ ASSERT_FALSE(downlink_packet_transport_time->time_series.samples.empty());
+ absl::optional<Metric> downlink_size_to_packet_transport_time =
+ FindMeetricByName("downlink_size_to_packet_transport_time", metrics);
+ ASSERT_TRUE(downlink_size_to_packet_transport_time.has_value());
+ ASSERT_FALSE(
+ downlink_size_to_packet_transport_time->time_series.samples.empty());
+}
+
+} // namespace
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/stats_poller.cc b/third_party/libwebrtc/test/pc/e2e/stats_poller.cc
new file mode 100644
index 0000000000..c04805fb20
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/stats_poller.cc
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/stats_poller.h"
+
+#include <utility>
+
+#include "rtc_base/logging.h"
+#include "rtc_base/synchronization/mutex.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+void InternalStatsObserver::PollStats() {
+ peer_->GetStats(this);
+}
+
+void InternalStatsObserver::OnStatsDelivered(
+ const rtc::scoped_refptr<const RTCStatsReport>& report) {
+ for (auto* observer : observers_) {
+ observer->OnStatsReports(pc_label_, report);
+ }
+}
+
+StatsPoller::StatsPoller(std::vector<StatsObserverInterface*> observers,
+ std::map<std::string, StatsProvider*> peers)
+ : observers_(std::move(observers)) {
+ webrtc::MutexLock lock(&mutex_);
+ for (auto& peer : peers) {
+ pollers_.push_back(rtc::make_ref_counted<InternalStatsObserver>(
+ peer.first, peer.second, observers_));
+ }
+}
+
+StatsPoller::StatsPoller(std::vector<StatsObserverInterface*> observers,
+ std::map<std::string, TestPeer*> peers)
+ : observers_(std::move(observers)) {
+ webrtc::MutexLock lock(&mutex_);
+ for (auto& peer : peers) {
+ pollers_.push_back(rtc::make_ref_counted<InternalStatsObserver>(
+ peer.first, peer.second, observers_));
+ }
+}
+
+void StatsPoller::PollStatsAndNotifyObservers() {
+ webrtc::MutexLock lock(&mutex_);
+ for (auto& poller : pollers_) {
+ poller->PollStats();
+ }
+}
+
+void StatsPoller::RegisterParticipantInCall(absl::string_view peer_name,
+ StatsProvider* peer) {
+ webrtc::MutexLock lock(&mutex_);
+ pollers_.push_back(rtc::make_ref_counted<InternalStatsObserver>(
+ peer_name, peer, observers_));
+}
+
+bool StatsPoller::UnregisterParticipantInCall(absl::string_view peer_name) {
+ webrtc::MutexLock lock(&mutex_);
+ for (auto it = pollers_.begin(); it != pollers_.end(); ++it) {
+ if ((*it)->pc_label() == peer_name) {
+ pollers_.erase(it);
+ return true;
+ }
+ }
+ return false;
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/stats_poller.h b/third_party/libwebrtc/test/pc/e2e/stats_poller.h
new file mode 100644
index 0000000000..3576f1bf05
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/stats_poller.h
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_STATS_POLLER_H_
+#define TEST_PC_E2E_STATS_POLLER_H_
+
+#include <map>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "api/peer_connection_interface.h"
+#include "api/stats/rtc_stats_collector_callback.h"
+#include "api/test/stats_observer_interface.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "rtc_base/thread_annotations.h"
+#include "test/pc/e2e/stats_provider.h"
+#include "test/pc/e2e/test_peer.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+// Helper class that will notify all the webrtc::test::StatsObserverInterface
+// objects subscribed.
+class InternalStatsObserver : public RTCStatsCollectorCallback {
+ public:
+ InternalStatsObserver(absl::string_view pc_label,
+ StatsProvider* peer,
+ std::vector<StatsObserverInterface*> observers)
+ : pc_label_(pc_label), peer_(peer), observers_(std::move(observers)) {}
+
+ std::string pc_label() const { return pc_label_; }
+
+ void PollStats();
+
+ void OnStatsDelivered(
+ const rtc::scoped_refptr<const RTCStatsReport>& report) override;
+
+ private:
+ std::string pc_label_;
+ StatsProvider* peer_;
+ std::vector<StatsObserverInterface*> observers_;
+};
+
+// Helper class to invoke GetStats on a PeerConnection by passing a
+// webrtc::StatsObserver that will notify all the
+// webrtc::test::StatsObserverInterface subscribed.
+class StatsPoller {
+ public:
+ StatsPoller(std::vector<StatsObserverInterface*> observers,
+ std::map<std::string, StatsProvider*> peers_to_observe);
+ StatsPoller(std::vector<StatsObserverInterface*> observers,
+ std::map<std::string, TestPeer*> peers_to_observe);
+
+ void PollStatsAndNotifyObservers();
+
+ void RegisterParticipantInCall(absl::string_view peer_name,
+ StatsProvider* peer);
+ // Unregister participant from stats poller. Returns true if participant was
+ // removed and false if participant wasn't found.
+ bool UnregisterParticipantInCall(absl::string_view peer_name);
+
+ private:
+ const std::vector<StatsObserverInterface*> observers_;
+ webrtc::Mutex mutex_;
+ std::vector<rtc::scoped_refptr<InternalStatsObserver>> pollers_
+ RTC_GUARDED_BY(mutex_);
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_STATS_POLLER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/stats_poller_test.cc b/third_party/libwebrtc/test/pc/e2e/stats_poller_test.cc
new file mode 100644
index 0000000000..02a323127b
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/stats_poller_test.cc
@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/stats_poller.h"
+
+#include "api/stats/rtc_stats_collector_callback.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+using ::testing::Eq;
+
+class TestStatsProvider : public StatsProvider {
+ public:
+ ~TestStatsProvider() override = default;
+
+ void GetStats(RTCStatsCollectorCallback* callback) override {
+ stats_collections_count_++;
+ }
+
+ int stats_collections_count() const { return stats_collections_count_; }
+
+ private:
+ int stats_collections_count_ = 0;
+};
+
+class MockStatsObserver : public StatsObserverInterface {
+ public:
+ ~MockStatsObserver() override = default;
+
+ MOCK_METHOD(void,
+ OnStatsReports,
+ (absl::string_view pc_label,
+ const rtc::scoped_refptr<const RTCStatsReport>& report));
+};
+
+TEST(StatsPollerTest, UnregisterParticipantAddedInCtor) {
+ TestStatsProvider alice;
+ TestStatsProvider bob;
+
+ MockStatsObserver stats_observer;
+
+ StatsPoller poller(/*observers=*/{&stats_observer},
+ /*peers_to_observe=*/{{"alice", &alice}, {"bob", &bob}});
+ poller.PollStatsAndNotifyObservers();
+
+ EXPECT_THAT(alice.stats_collections_count(), Eq(1));
+ EXPECT_THAT(bob.stats_collections_count(), Eq(1));
+
+ poller.UnregisterParticipantInCall("bob");
+ poller.PollStatsAndNotifyObservers();
+
+ EXPECT_THAT(alice.stats_collections_count(), Eq(2));
+ EXPECT_THAT(bob.stats_collections_count(), Eq(1));
+}
+
+TEST(StatsPollerTest, UnregisterParticipantRegisteredInCall) {
+ TestStatsProvider alice;
+ TestStatsProvider bob;
+
+ MockStatsObserver stats_observer;
+
+ StatsPoller poller(/*observers=*/{&stats_observer},
+ /*peers_to_observe=*/{{"alice", &alice}});
+ poller.RegisterParticipantInCall("bob", &bob);
+ poller.PollStatsAndNotifyObservers();
+
+ EXPECT_THAT(alice.stats_collections_count(), Eq(1));
+ EXPECT_THAT(bob.stats_collections_count(), Eq(1));
+
+ poller.UnregisterParticipantInCall("bob");
+ poller.PollStatsAndNotifyObservers();
+
+ EXPECT_THAT(alice.stats_collections_count(), Eq(2));
+ EXPECT_THAT(bob.stats_collections_count(), Eq(1));
+}
+
+} // namespace
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/stats_provider.h b/third_party/libwebrtc/test/pc/e2e/stats_provider.h
new file mode 100644
index 0000000000..eef62d779c
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/stats_provider.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_STATS_PROVIDER_H_
+#define TEST_PC_E2E_STATS_PROVIDER_H_
+
+#include "api/stats/rtc_stats_collector_callback.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+class StatsProvider {
+ public:
+ virtual ~StatsProvider() = default;
+
+ virtual void GetStats(RTCStatsCollectorCallback* callback) = 0;
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_STATS_PROVIDER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/test_activities_executor.cc b/third_party/libwebrtc/test/pc/e2e/test_activities_executor.cc
new file mode 100644
index 0000000000..7bcf7dd6c3
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/test_activities_executor.cc
@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "test/pc/e2e/test_activities_executor.h"
+
+#include <memory>
+#include <utility>
+
+#include "absl/memory/memory.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/task_queue_for_test.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+void TestActivitiesExecutor::Start(TaskQueueBase* task_queue) {
+ RTC_DCHECK(task_queue);
+ task_queue_ = task_queue;
+ MutexLock lock(&lock_);
+ start_time_ = Now();
+ while (!scheduled_activities_.empty()) {
+ PostActivity(std::move(scheduled_activities_.front()));
+ scheduled_activities_.pop();
+ }
+}
+
+void TestActivitiesExecutor::Stop() {
+ if (task_queue_ == nullptr) {
+ // Already stopped or not started.
+ return;
+ }
+ SendTask(task_queue_, [this]() {
+ MutexLock lock(&lock_);
+ for (auto& handle : repeating_task_handles_) {
+ handle.Stop();
+ }
+ });
+ task_queue_ = nullptr;
+}
+
+void TestActivitiesExecutor::ScheduleActivity(
+ TimeDelta initial_delay_since_start,
+ absl::optional<TimeDelta> interval,
+ std::function<void(TimeDelta)> func) {
+ RTC_CHECK(initial_delay_since_start.IsFinite() &&
+ initial_delay_since_start >= TimeDelta::Zero());
+ RTC_CHECK(!interval ||
+ (interval->IsFinite() && *interval > TimeDelta::Zero()));
+ MutexLock lock(&lock_);
+ ScheduledActivity activity(initial_delay_since_start, interval, func);
+ if (start_time_.IsInfinite()) {
+ scheduled_activities_.push(std::move(activity));
+ } else {
+ PostActivity(std::move(activity));
+ }
+}
+
+void TestActivitiesExecutor::PostActivity(ScheduledActivity activity) {
+ // Because start_time_ will never change at this point copy it to local
+ // variable to capture in in lambda without requirement to hold a lock.
+ Timestamp start_time = start_time_;
+
+ TimeDelta remaining_delay =
+ activity.initial_delay_since_start == TimeDelta::Zero()
+ ? TimeDelta::Zero()
+ : activity.initial_delay_since_start - (Now() - start_time);
+ if (remaining_delay < TimeDelta::Zero()) {
+ RTC_LOG(LS_WARNING) << "Executing late task immediately, late by="
+ << ToString(remaining_delay.Abs());
+ remaining_delay = TimeDelta::Zero();
+ }
+
+ if (activity.interval) {
+ if (remaining_delay == TimeDelta::Zero()) {
+ repeating_task_handles_.push_back(RepeatingTaskHandle::Start(
+ task_queue_, [activity, start_time, this]() {
+ activity.func(Now() - start_time);
+ return *activity.interval;
+ }));
+ return;
+ }
+ repeating_task_handles_.push_back(RepeatingTaskHandle::DelayedStart(
+ task_queue_, remaining_delay, [activity, start_time, this]() {
+ activity.func(Now() - start_time);
+ return *activity.interval;
+ }));
+ return;
+ }
+
+ if (remaining_delay == TimeDelta::Zero()) {
+ task_queue_->PostTask(
+ [activity, start_time, this]() { activity.func(Now() - start_time); });
+ return;
+ }
+
+ task_queue_->PostDelayedTask(
+ [activity, start_time, this]() { activity.func(Now() - start_time); },
+ remaining_delay);
+}
+
+Timestamp TestActivitiesExecutor::Now() const {
+ return clock_->CurrentTime();
+}
+
+TestActivitiesExecutor::ScheduledActivity::ScheduledActivity(
+ TimeDelta initial_delay_since_start,
+ absl::optional<TimeDelta> interval,
+ std::function<void(TimeDelta)> func)
+ : initial_delay_since_start(initial_delay_since_start),
+ interval(interval),
+ func(std::move(func)) {}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/test_activities_executor.h b/third_party/libwebrtc/test/pc/e2e/test_activities_executor.h
new file mode 100644
index 0000000000..2469ac7f36
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/test_activities_executor.h
@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_TEST_ACTIVITIES_EXECUTOR_H_
+#define TEST_PC_E2E_TEST_ACTIVITIES_EXECUTOR_H_
+
+#include <queue>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/task_queue/task_queue_base.h"
+#include "api/units/time_delta.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "rtc_base/task_queue_for_test.h"
+#include "rtc_base/task_utils/repeating_task.h"
+#include "system_wrappers/include/clock.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+class TestActivitiesExecutor {
+ public:
+ explicit TestActivitiesExecutor(Clock* clock) : clock_(clock) {}
+ ~TestActivitiesExecutor() { Stop(); }
+
+ // Starts scheduled activities according to their schedule. All activities
+ // that will be scheduled after Start(...) was invoked will be executed
+ // immediately according to their schedule.
+ void Start(TaskQueueForTest* task_queue) { Start(task_queue->Get()); }
+ void Start(TaskQueueBase* task_queue);
+ void Stop();
+
+ // Schedule activity to be executed. If test isn't started yet, then activity
+ // will be executed according to its schedule after Start() will be invoked.
+ // If test is started, then it will be executed immediately according to its
+ // schedule.
+ void ScheduleActivity(TimeDelta initial_delay_since_start,
+ absl::optional<TimeDelta> interval,
+ std::function<void(TimeDelta)> func);
+
+ private:
+ struct ScheduledActivity {
+ ScheduledActivity(TimeDelta initial_delay_since_start,
+ absl::optional<TimeDelta> interval,
+ std::function<void(TimeDelta)> func);
+
+ TimeDelta initial_delay_since_start;
+ absl::optional<TimeDelta> interval;
+ std::function<void(TimeDelta)> func;
+ };
+
+ void PostActivity(ScheduledActivity activity)
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_);
+ Timestamp Now() const;
+
+ Clock* const clock_;
+
+ TaskQueueBase* task_queue_;
+
+ Mutex lock_;
+ // Time when test was started. Minus infinity means that it wasn't started
+ // yet.
+ Timestamp start_time_ RTC_GUARDED_BY(lock_) = Timestamp::MinusInfinity();
+ // Queue of activities that were added before test was started.
+ // Activities from this queue will be posted on the `task_queue_` after test
+ // will be set up and then this queue will be unused.
+ std::queue<ScheduledActivity> scheduled_activities_ RTC_GUARDED_BY(lock_);
+ // List of task handles for activities, that are posted on `task_queue_` as
+ // repeated during the call.
+ std::vector<RepeatingTaskHandle> repeating_task_handles_
+ RTC_GUARDED_BY(lock_);
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_TEST_ACTIVITIES_EXECUTOR_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/test_peer.cc b/third_party/libwebrtc/test/pc/e2e/test_peer.cc
new file mode 100644
index 0000000000..b3a9e1c164
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/test_peer.cc
@@ -0,0 +1,151 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/test_peer.h"
+
+#include <string>
+#include <utility>
+
+#include "absl/memory/memory.h"
+#include "absl/strings/string_view.h"
+#include "api/scoped_refptr.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/pclf/peer_configurer.h"
+#include "modules/audio_processing/include/audio_processing.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+class SetRemoteDescriptionCallback
+ : public webrtc::SetRemoteDescriptionObserverInterface {
+ public:
+ void OnSetRemoteDescriptionComplete(webrtc::RTCError error) override {
+ is_called_ = true;
+ error_ = error;
+ }
+
+ bool is_called() const { return is_called_; }
+
+ webrtc::RTCError error() const { return error_; }
+
+ private:
+ bool is_called_ = false;
+ webrtc::RTCError error_;
+};
+
+} // namespace
+
+ConfigurableParams TestPeer::configurable_params() const {
+ MutexLock lock(&mutex_);
+ return configurable_params_;
+}
+
+void TestPeer::AddVideoConfig(VideoConfig config) {
+ MutexLock lock(&mutex_);
+ configurable_params_.video_configs.push_back(std::move(config));
+}
+
+void TestPeer::RemoveVideoConfig(absl::string_view stream_label) {
+ MutexLock lock(&mutex_);
+ bool config_removed = false;
+ for (auto it = configurable_params_.video_configs.begin();
+ it != configurable_params_.video_configs.end(); ++it) {
+ if (*it->stream_label == stream_label) {
+ configurable_params_.video_configs.erase(it);
+ config_removed = true;
+ break;
+ }
+ }
+ RTC_CHECK(config_removed) << *params_.name << ": No video config with label ["
+ << stream_label << "] was found";
+}
+
+void TestPeer::SetVideoSubscription(VideoSubscription subscription) {
+ MutexLock lock(&mutex_);
+ configurable_params_.video_subscription = std::move(subscription);
+}
+
+void TestPeer::GetStats(RTCStatsCollectorCallback* callback) {
+ pc()->signaling_thread()->PostTask(
+ SafeTask(signaling_thread_task_safety_,
+ [this, callback]() { pc()->GetStats(callback); }));
+}
+
+bool TestPeer::SetRemoteDescription(
+ std::unique_ptr<SessionDescriptionInterface> desc,
+ std::string* error_out) {
+ RTC_CHECK(wrapper_) << "TestPeer is already closed";
+
+ auto observer = rtc::make_ref_counted<SetRemoteDescriptionCallback>();
+ // We're assuming (and asserting) that the PeerConnection implementation of
+ // SetRemoteDescription is synchronous when called on the signaling thread.
+ pc()->SetRemoteDescription(std::move(desc), observer);
+ RTC_CHECK(observer->is_called());
+ if (!observer->error().ok()) {
+ RTC_LOG(LS_ERROR) << *params_.name << ": Failed to set remote description: "
+ << observer->error().message();
+ if (error_out) {
+ *error_out = observer->error().message();
+ }
+ }
+ return observer->error().ok();
+}
+
+bool TestPeer::AddIceCandidates(
+ std::vector<std::unique_ptr<IceCandidateInterface>> candidates) {
+ RTC_CHECK(wrapper_) << "TestPeer is already closed";
+ bool success = true;
+ for (auto& candidate : candidates) {
+ if (!pc()->AddIceCandidate(candidate.get())) {
+ std::string candidate_str;
+ bool res = candidate->ToString(&candidate_str);
+ RTC_CHECK(res);
+ RTC_LOG(LS_ERROR) << "Failed to add ICE candidate, candidate_str="
+ << candidate_str;
+ success = false;
+ } else {
+ remote_ice_candidates_.push_back(std::move(candidate));
+ }
+ }
+ return success;
+}
+
+void TestPeer::Close() {
+ signaling_thread_task_safety_->SetNotAlive();
+ wrapper_->pc()->Close();
+ remote_ice_candidates_.clear();
+ audio_processing_ = nullptr;
+ video_sources_.clear();
+ wrapper_ = nullptr;
+ worker_thread_ = nullptr;
+}
+
+TestPeer::TestPeer(
+ rtc::scoped_refptr<PeerConnectionFactoryInterface> pc_factory,
+ rtc::scoped_refptr<PeerConnectionInterface> pc,
+ std::unique_ptr<MockPeerConnectionObserver> observer,
+ Params params,
+ ConfigurableParams configurable_params,
+ std::vector<PeerConfigurer::VideoSource> video_sources,
+ rtc::scoped_refptr<AudioProcessing> audio_processing,
+ std::unique_ptr<rtc::Thread> worker_thread)
+ : params_(std::move(params)),
+ configurable_params_(std::move(configurable_params)),
+ worker_thread_(std::move(worker_thread)),
+ wrapper_(std::make_unique<PeerConnectionWrapper>(std::move(pc_factory),
+ std::move(pc),
+ std::move(observer))),
+ video_sources_(std::move(video_sources)),
+ audio_processing_(audio_processing) {
+ signaling_thread_task_safety_ = PendingTaskSafetyFlag::CreateDetached();
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/test_peer.h b/third_party/libwebrtc/test/pc/e2e/test_peer.h
new file mode 100644
index 0000000000..1088871817
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/test_peer.h
@@ -0,0 +1,188 @@
+/*
+ * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_TEST_PEER_H_
+#define TEST_PC_E2E_TEST_PEER_H_
+
+#include <memory>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "absl/strings/string_view.h"
+#include "api/function_view.h"
+#include "api/scoped_refptr.h"
+#include "api/sequence_checker.h"
+#include "api/set_remote_description_observer_interface.h"
+#include "api/task_queue/pending_task_safety_flag.h"
+#include "api/test/frame_generator_interface.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/pclf/media_quality_test_params.h"
+#include "api/test/pclf/peer_configurer.h"
+#include "pc/peer_connection_wrapper.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "test/pc/e2e/stats_provider.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+// Describes a single participant in the call.
+class TestPeer final : public StatsProvider {
+ public:
+ ~TestPeer() override = default;
+
+ const Params& params() const { return params_; }
+
+ ConfigurableParams configurable_params() const;
+ void AddVideoConfig(VideoConfig config);
+ // Removes video config with specified name. Crashes if the config with
+ // specified name isn't found.
+ void RemoveVideoConfig(absl::string_view stream_label);
+ void SetVideoSubscription(VideoSubscription subscription);
+
+ void GetStats(RTCStatsCollectorCallback* callback) override;
+
+ PeerConfigurer::VideoSource ReleaseVideoSource(size_t i) {
+ RTC_CHECK(wrapper_) << "TestPeer is already closed";
+ return std::move(video_sources_[i]);
+ }
+
+ PeerConnectionFactoryInterface* pc_factory() {
+ RTC_CHECK(wrapper_) << "TestPeer is already closed";
+ return wrapper_->pc_factory();
+ }
+ PeerConnectionInterface* pc() {
+ RTC_CHECK(wrapper_) << "TestPeer is already closed";
+ return wrapper_->pc();
+ }
+ MockPeerConnectionObserver* observer() {
+ RTC_CHECK(wrapper_) << "TestPeer is already closed";
+ return wrapper_->observer();
+ }
+
+ // Tell underlying `PeerConnection` to create an Offer.
+ // `observer` will be invoked on the signaling thread when offer is created.
+ void CreateOffer(
+ rtc::scoped_refptr<CreateSessionDescriptionObserver> observer) {
+ RTC_CHECK(wrapper_) << "TestPeer is already closed";
+ pc()->CreateOffer(observer.get(), params_.rtc_offer_answer_options);
+ }
+ std::unique_ptr<SessionDescriptionInterface> CreateOffer() {
+ RTC_CHECK(wrapper_) << "TestPeer is already closed";
+ return wrapper_->CreateOffer(params_.rtc_offer_answer_options);
+ }
+
+ std::unique_ptr<SessionDescriptionInterface> CreateAnswer() {
+ RTC_CHECK(wrapper_) << "TestPeer is already closed";
+ return wrapper_->CreateAnswer();
+ }
+
+ bool SetLocalDescription(std::unique_ptr<SessionDescriptionInterface> desc,
+ std::string* error_out = nullptr) {
+ RTC_CHECK(wrapper_) << "TestPeer is already closed";
+ return wrapper_->SetLocalDescription(std::move(desc), error_out);
+ }
+
+ // `error_out` will be set only if returned value is false.
+ bool SetRemoteDescription(std::unique_ptr<SessionDescriptionInterface> desc,
+ std::string* error_out = nullptr);
+
+ rtc::scoped_refptr<RtpTransceiverInterface> AddTransceiver(
+ cricket::MediaType media_type,
+ const RtpTransceiverInit& init) {
+ RTC_CHECK(wrapper_) << "TestPeer is already closed";
+ return wrapper_->AddTransceiver(media_type, init);
+ }
+
+ rtc::scoped_refptr<RtpSenderInterface> AddTrack(
+ rtc::scoped_refptr<MediaStreamTrackInterface> track,
+ const std::vector<std::string>& stream_ids = {}) {
+ RTC_CHECK(wrapper_) << "TestPeer is already closed";
+ return wrapper_->AddTrack(track, stream_ids);
+ }
+
+ rtc::scoped_refptr<DataChannelInterface> CreateDataChannel(
+ const std::string& label) {
+ RTC_CHECK(wrapper_) << "TestPeer is already closed";
+ return wrapper_->CreateDataChannel(label);
+ }
+
+ PeerConnectionInterface::SignalingState signaling_state() {
+ RTC_CHECK(wrapper_) << "TestPeer is already closed";
+ return wrapper_->signaling_state();
+ }
+
+ bool IsIceGatheringDone() {
+ RTC_CHECK(wrapper_) << "TestPeer is already closed";
+ return wrapper_->IsIceGatheringDone();
+ }
+
+ bool IsIceConnected() {
+ RTC_CHECK(wrapper_) << "TestPeer is already closed";
+ return wrapper_->IsIceConnected();
+ }
+
+ rtc::scoped_refptr<const RTCStatsReport> GetStats() {
+ RTC_CHECK(wrapper_) << "TestPeer is already closed";
+ return wrapper_->GetStats();
+ }
+
+ void DetachAecDump() {
+ RTC_CHECK(wrapper_) << "TestPeer is already closed";
+ if (audio_processing_) {
+ audio_processing_->DetachAecDump();
+ }
+ }
+
+ // Adds provided `candidates` to the owned peer connection.
+ bool AddIceCandidates(
+ std::vector<std::unique_ptr<IceCandidateInterface>> candidates);
+
+ // Closes underlying peer connection and destroys all related objects freeing
+ // up related resources.
+ void Close();
+
+ protected:
+ friend class TestPeerFactory;
+ TestPeer(rtc::scoped_refptr<PeerConnectionFactoryInterface> pc_factory,
+ rtc::scoped_refptr<PeerConnectionInterface> pc,
+ std::unique_ptr<MockPeerConnectionObserver> observer,
+ Params params,
+ ConfigurableParams configurable_params,
+ std::vector<PeerConfigurer::VideoSource> video_sources,
+ rtc::scoped_refptr<AudioProcessing> audio_processing,
+ std::unique_ptr<rtc::Thread> worker_thread);
+
+ private:
+ const Params params_;
+
+ mutable Mutex mutex_;
+ ConfigurableParams configurable_params_ RTC_GUARDED_BY(mutex_);
+
+ // Safety flag to protect all tasks posted on the signaling thread to not be
+ // executed after `wrapper_` object is destructed.
+ rtc::scoped_refptr<PendingTaskSafetyFlag> signaling_thread_task_safety_ =
+ nullptr;
+
+ // Keeps ownership of worker thread. It has to be destroyed after `wrapper_`.
+ // `worker_thread_`can be null if the Peer use only one thread as both the
+ // worker thread and network thread.
+ std::unique_ptr<rtc::Thread> worker_thread_;
+ std::unique_ptr<PeerConnectionWrapper> wrapper_;
+ std::vector<PeerConfigurer::VideoSource> video_sources_;
+ rtc::scoped_refptr<AudioProcessing> audio_processing_;
+
+ std::vector<std::unique_ptr<IceCandidateInterface>> remote_ice_candidates_;
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_TEST_PEER_H_
diff --git a/third_party/libwebrtc/test/pc/e2e/test_peer_factory.cc b/third_party/libwebrtc/test/pc/e2e/test_peer_factory.cc
new file mode 100644
index 0000000000..7fc12f2c11
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/test_peer_factory.cc
@@ -0,0 +1,374 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+#include "test/pc/e2e/test_peer_factory.h"
+
+#include <utility>
+
+#include "absl/memory/memory.h"
+#include "absl/strings/string_view.h"
+#include "api/task_queue/default_task_queue_factory.h"
+#include "api/test/create_time_controller.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/pclf/peer_configurer.h"
+#include "api/test/time_controller.h"
+#include "api/transport/field_trial_based_config.h"
+#include "api/video_codecs/builtin_video_decoder_factory.h"
+#include "api/video_codecs/builtin_video_encoder_factory.h"
+#include "media/engine/webrtc_media_engine.h"
+#include "media/engine/webrtc_media_engine_defaults.h"
+#include "modules/audio_processing/aec_dump/aec_dump_factory.h"
+#include "p2p/client/basic_port_allocator.h"
+#include "rtc_base/thread.h"
+#include "test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h"
+#include "test/pc/e2e/echo/echo_emulation.h"
+#include "test/testsupport/copy_to_file_audio_capturer.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+using EmulatedSFUConfigMap =
+ ::webrtc::webrtc_pc_e2e::QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap;
+
+constexpr int16_t kGeneratedAudioMaxAmplitude = 32000;
+constexpr int kDefaultSamplingFrequencyInHz = 48000;
+
+// Sets mandatory entities in injectable components like `pcf_dependencies`
+// and `pc_dependencies` if they are omitted. Also setup required
+// dependencies, that won't be specially provided by factory and will be just
+// transferred to peer connection creation code.
+void SetMandatoryEntities(InjectableComponents* components,
+ TimeController& time_controller) {
+ RTC_DCHECK(components->pcf_dependencies);
+ RTC_DCHECK(components->pc_dependencies);
+
+ // Setup required peer connection factory dependencies.
+ if (components->pcf_dependencies->task_queue_factory == nullptr) {
+ components->pcf_dependencies->task_queue_factory =
+ time_controller.CreateTaskQueueFactory();
+ }
+ if (components->pcf_dependencies->call_factory == nullptr) {
+ components->pcf_dependencies->call_factory =
+ CreateTimeControllerBasedCallFactory(&time_controller);
+ }
+ if (components->pcf_dependencies->event_log_factory == nullptr) {
+ components->pcf_dependencies->event_log_factory =
+ std::make_unique<RtcEventLogFactory>(
+ components->pcf_dependencies->task_queue_factory.get());
+ }
+ if (!components->pcf_dependencies->trials) {
+ components->pcf_dependencies->trials =
+ std::make_unique<FieldTrialBasedConfig>();
+ }
+}
+
+// Returns mapping from stream label to optional spatial index.
+// If we have stream label "Foo" and mapping contains
+// 1. `absl::nullopt` means all simulcast/SVC streams are required
+// 2. Concrete value means that particular simulcast/SVC stream have to be
+// analyzed.
+EmulatedSFUConfigMap CalculateRequiredSpatialIndexPerStream(
+ const std::vector<VideoConfig>& video_configs) {
+ EmulatedSFUConfigMap result;
+ for (auto& video_config : video_configs) {
+ // Stream label should be set by fixture implementation here.
+ RTC_DCHECK(video_config.stream_label);
+ bool res = result
+ .insert({*video_config.stream_label,
+ video_config.emulated_sfu_config})
+ .second;
+ RTC_DCHECK(res) << "Duplicate video_config.stream_label="
+ << *video_config.stream_label;
+ }
+ return result;
+}
+
+std::unique_ptr<TestAudioDeviceModule::Renderer> CreateAudioRenderer(
+ const absl::optional<RemotePeerAudioConfig>& config) {
+ if (!config) {
+ // Return default renderer because we always require some renderer.
+ return TestAudioDeviceModule::CreateDiscardRenderer(
+ kDefaultSamplingFrequencyInHz);
+ }
+ if (config->output_file_name) {
+ return TestAudioDeviceModule::CreateBoundedWavFileWriter(
+ config->output_file_name.value(), config->sampling_frequency_in_hz);
+ }
+ return TestAudioDeviceModule::CreateDiscardRenderer(
+ config->sampling_frequency_in_hz);
+}
+
+std::unique_ptr<TestAudioDeviceModule::Capturer> CreateAudioCapturer(
+ const absl::optional<AudioConfig>& audio_config) {
+ if (!audio_config) {
+ // If we have no audio config we still need to provide some audio device.
+ // In such case use generated capturer. Despite of we provided audio here,
+ // in test media setup audio stream won't be added into peer connection.
+ return TestAudioDeviceModule::CreatePulsedNoiseCapturer(
+ kGeneratedAudioMaxAmplitude, kDefaultSamplingFrequencyInHz);
+ }
+
+ switch (audio_config->mode) {
+ case AudioConfig::Mode::kGenerated:
+ return TestAudioDeviceModule::CreatePulsedNoiseCapturer(
+ kGeneratedAudioMaxAmplitude, audio_config->sampling_frequency_in_hz);
+ case AudioConfig::Mode::kFile:
+ RTC_DCHECK(audio_config->input_file_name);
+ return TestAudioDeviceModule::CreateWavFileReader(
+ audio_config->input_file_name.value(), /*repeat=*/true);
+ }
+}
+
+rtc::scoped_refptr<AudioDeviceModule> CreateAudioDeviceModule(
+ absl::optional<AudioConfig> audio_config,
+ absl::optional<RemotePeerAudioConfig> remote_audio_config,
+ absl::optional<EchoEmulationConfig> echo_emulation_config,
+ TaskQueueFactory* task_queue_factory) {
+ std::unique_ptr<TestAudioDeviceModule::Renderer> renderer =
+ CreateAudioRenderer(remote_audio_config);
+ std::unique_ptr<TestAudioDeviceModule::Capturer> capturer =
+ CreateAudioCapturer(audio_config);
+ RTC_DCHECK(renderer);
+ RTC_DCHECK(capturer);
+
+ // Setup echo emulation if required.
+ if (echo_emulation_config) {
+ capturer = std::make_unique<EchoEmulatingCapturer>(std::move(capturer),
+ *echo_emulation_config);
+ renderer = std::make_unique<EchoEmulatingRenderer>(
+ std::move(renderer),
+ static_cast<EchoEmulatingCapturer*>(capturer.get()));
+ }
+
+ // Setup input stream dumping if required.
+ if (audio_config && audio_config->input_dump_file_name) {
+ capturer = std::make_unique<test::CopyToFileAudioCapturer>(
+ std::move(capturer), audio_config->input_dump_file_name.value());
+ }
+
+ return TestAudioDeviceModule::Create(task_queue_factory, std::move(capturer),
+ std::move(renderer), /*speed=*/1.f);
+}
+
+std::unique_ptr<cricket::MediaEngineInterface> CreateMediaEngine(
+ PeerConnectionFactoryComponents* pcf_dependencies,
+ rtc::scoped_refptr<AudioDeviceModule> audio_device_module) {
+ cricket::MediaEngineDependencies media_deps;
+ media_deps.task_queue_factory = pcf_dependencies->task_queue_factory.get();
+ media_deps.adm = audio_device_module;
+ media_deps.audio_processing = pcf_dependencies->audio_processing;
+ media_deps.audio_mixer = pcf_dependencies->audio_mixer;
+ media_deps.video_encoder_factory =
+ std::move(pcf_dependencies->video_encoder_factory);
+ media_deps.video_decoder_factory =
+ std::move(pcf_dependencies->video_decoder_factory);
+ webrtc::SetMediaEngineDefaults(&media_deps);
+ RTC_DCHECK(pcf_dependencies->trials);
+ media_deps.trials = pcf_dependencies->trials.get();
+
+ return cricket::CreateMediaEngine(std::move(media_deps));
+}
+
+void WrapVideoEncoderFactory(
+ absl::string_view peer_name,
+ double bitrate_multiplier,
+ EmulatedSFUConfigMap stream_to_sfu_config,
+ PeerConnectionFactoryComponents* pcf_dependencies,
+ VideoQualityAnalyzerInjectionHelper* video_analyzer_helper) {
+ std::unique_ptr<VideoEncoderFactory> video_encoder_factory;
+ if (pcf_dependencies->video_encoder_factory != nullptr) {
+ video_encoder_factory = std::move(pcf_dependencies->video_encoder_factory);
+ } else {
+ video_encoder_factory = CreateBuiltinVideoEncoderFactory();
+ }
+ pcf_dependencies->video_encoder_factory =
+ video_analyzer_helper->WrapVideoEncoderFactory(
+ peer_name, std::move(video_encoder_factory), bitrate_multiplier,
+ std::move(stream_to_sfu_config));
+}
+
+void WrapVideoDecoderFactory(
+ absl::string_view peer_name,
+ PeerConnectionFactoryComponents* pcf_dependencies,
+ VideoQualityAnalyzerInjectionHelper* video_analyzer_helper) {
+ std::unique_ptr<VideoDecoderFactory> video_decoder_factory;
+ if (pcf_dependencies->video_decoder_factory != nullptr) {
+ video_decoder_factory = std::move(pcf_dependencies->video_decoder_factory);
+ } else {
+ video_decoder_factory = CreateBuiltinVideoDecoderFactory();
+ }
+ pcf_dependencies->video_decoder_factory =
+ video_analyzer_helper->WrapVideoDecoderFactory(
+ peer_name, std::move(video_decoder_factory));
+}
+
+// Creates PeerConnectionFactoryDependencies objects, providing entities
+// from InjectableComponents::PeerConnectionFactoryComponents.
+PeerConnectionFactoryDependencies CreatePCFDependencies(
+ std::unique_ptr<PeerConnectionFactoryComponents> pcf_dependencies,
+ std::unique_ptr<cricket::MediaEngineInterface> media_engine,
+ rtc::Thread* signaling_thread,
+ rtc::Thread* worker_thread,
+ rtc::Thread* network_thread) {
+ PeerConnectionFactoryDependencies pcf_deps;
+ pcf_deps.signaling_thread = signaling_thread;
+ pcf_deps.worker_thread = worker_thread;
+ pcf_deps.network_thread = network_thread;
+ pcf_deps.media_engine = std::move(media_engine);
+
+ pcf_deps.call_factory = std::move(pcf_dependencies->call_factory);
+ pcf_deps.event_log_factory = std::move(pcf_dependencies->event_log_factory);
+ pcf_deps.task_queue_factory = std::move(pcf_dependencies->task_queue_factory);
+
+ if (pcf_dependencies->fec_controller_factory != nullptr) {
+ pcf_deps.fec_controller_factory =
+ std::move(pcf_dependencies->fec_controller_factory);
+ }
+ if (pcf_dependencies->network_controller_factory != nullptr) {
+ pcf_deps.network_controller_factory =
+ std::move(pcf_dependencies->network_controller_factory);
+ }
+ if (pcf_dependencies->neteq_factory != nullptr) {
+ pcf_deps.neteq_factory = std::move(pcf_dependencies->neteq_factory);
+ }
+ if (pcf_dependencies->trials != nullptr) {
+ pcf_deps.trials = std::move(pcf_dependencies->trials);
+ }
+
+ return pcf_deps;
+}
+
+// Creates PeerConnectionDependencies objects, providing entities
+// from InjectableComponents::PeerConnectionComponents.
+PeerConnectionDependencies CreatePCDependencies(
+ MockPeerConnectionObserver* observer,
+ uint32_t port_allocator_extra_flags,
+ std::unique_ptr<PeerConnectionComponents> pc_dependencies) {
+ PeerConnectionDependencies pc_deps(observer);
+
+ auto port_allocator = std::make_unique<cricket::BasicPortAllocator>(
+ pc_dependencies->network_manager, pc_dependencies->packet_socket_factory);
+
+ // This test does not support TCP
+ int flags = port_allocator_extra_flags | cricket::PORTALLOCATOR_DISABLE_TCP;
+ port_allocator->set_flags(port_allocator->flags() | flags);
+
+ pc_deps.allocator = std::move(port_allocator);
+
+ if (pc_dependencies->async_resolver_factory != nullptr) {
+ pc_deps.async_resolver_factory =
+ std::move(pc_dependencies->async_resolver_factory);
+ }
+ if (pc_dependencies->cert_generator != nullptr) {
+ pc_deps.cert_generator = std::move(pc_dependencies->cert_generator);
+ }
+ if (pc_dependencies->tls_cert_verifier != nullptr) {
+ pc_deps.tls_cert_verifier = std::move(pc_dependencies->tls_cert_verifier);
+ }
+ if (pc_dependencies->ice_transport_factory != nullptr) {
+ pc_deps.ice_transport_factory =
+ std::move(pc_dependencies->ice_transport_factory);
+ }
+ return pc_deps;
+}
+
+} // namespace
+
+absl::optional<RemotePeerAudioConfig> RemotePeerAudioConfig::Create(
+ absl::optional<AudioConfig> config) {
+ if (!config) {
+ return absl::nullopt;
+ }
+ return RemotePeerAudioConfig(config.value());
+}
+
+std::unique_ptr<TestPeer> TestPeerFactory::CreateTestPeer(
+ std::unique_ptr<PeerConfigurer> configurer,
+ std::unique_ptr<MockPeerConnectionObserver> observer,
+ absl::optional<RemotePeerAudioConfig> remote_audio_config,
+ absl::optional<EchoEmulationConfig> echo_emulation_config) {
+ std::unique_ptr<InjectableComponents> components =
+ configurer->ReleaseComponents();
+ std::unique_ptr<Params> params = configurer->ReleaseParams();
+ std::unique_ptr<ConfigurableParams> configurable_params =
+ configurer->ReleaseConfigurableParams();
+ std::vector<PeerConfigurer::VideoSource> video_sources =
+ configurer->ReleaseVideoSources();
+ RTC_DCHECK(components);
+ RTC_DCHECK(params);
+ RTC_DCHECK(configurable_params);
+ RTC_DCHECK_EQ(configurable_params->video_configs.size(),
+ video_sources.size());
+ SetMandatoryEntities(components.get(), time_controller_);
+ params->rtc_configuration.sdp_semantics = SdpSemantics::kUnifiedPlan;
+
+ // Create peer connection factory.
+ if (components->pcf_dependencies->audio_processing == nullptr) {
+ components->pcf_dependencies->audio_processing =
+ webrtc::AudioProcessingBuilder().Create();
+ }
+ if (params->aec_dump_path) {
+ components->pcf_dependencies->audio_processing->CreateAndAttachAecDump(
+ *params->aec_dump_path, -1, task_queue_);
+ }
+ rtc::scoped_refptr<AudioDeviceModule> audio_device_module =
+ CreateAudioDeviceModule(
+ params->audio_config, remote_audio_config, echo_emulation_config,
+ components->pcf_dependencies->task_queue_factory.get());
+ WrapVideoEncoderFactory(
+ params->name.value(), params->video_encoder_bitrate_multiplier,
+ CalculateRequiredSpatialIndexPerStream(
+ configurable_params->video_configs),
+ components->pcf_dependencies.get(), video_analyzer_helper_);
+ WrapVideoDecoderFactory(params->name.value(),
+ components->pcf_dependencies.get(),
+ video_analyzer_helper_);
+ std::unique_ptr<cricket::MediaEngineInterface> media_engine =
+ CreateMediaEngine(components->pcf_dependencies.get(),
+ audio_device_module);
+
+ std::unique_ptr<rtc::Thread> owned_worker_thread =
+ components->worker_thread != nullptr
+ ? nullptr
+ : time_controller_.CreateThread("worker_thread");
+ if (components->worker_thread == nullptr) {
+ components->worker_thread = owned_worker_thread.get();
+ }
+
+ // Store `webrtc::AudioProcessing` into local variable before move of
+ // `components->pcf_dependencies`
+ rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing =
+ components->pcf_dependencies->audio_processing;
+ PeerConnectionFactoryDependencies pcf_deps = CreatePCFDependencies(
+ std::move(components->pcf_dependencies), std::move(media_engine),
+ signaling_thread_, components->worker_thread, components->network_thread);
+ rtc::scoped_refptr<PeerConnectionFactoryInterface> peer_connection_factory =
+ CreateModularPeerConnectionFactory(std::move(pcf_deps));
+
+ // Create peer connection.
+ PeerConnectionDependencies pc_deps =
+ CreatePCDependencies(observer.get(), params->port_allocator_extra_flags,
+ std::move(components->pc_dependencies));
+ rtc::scoped_refptr<PeerConnectionInterface> peer_connection =
+ peer_connection_factory
+ ->CreatePeerConnectionOrError(params->rtc_configuration,
+ std::move(pc_deps))
+ .MoveValue();
+ peer_connection->SetBitrate(params->bitrate_settings);
+
+ return absl::WrapUnique(
+ new TestPeer(peer_connection_factory, peer_connection,
+ std::move(observer), std::move(*params),
+ std::move(*configurable_params), std::move(video_sources),
+ audio_processing, std::move(owned_worker_thread)));
+}
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
diff --git a/third_party/libwebrtc/test/pc/e2e/test_peer_factory.h b/third_party/libwebrtc/test/pc/e2e/test_peer_factory.h
new file mode 100644
index 0000000000..f2698e2a15
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/test_peer_factory.h
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_E2E_TEST_PEER_FACTORY_H_
+#define TEST_PC_E2E_TEST_PEER_FACTORY_H_
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "absl/strings/string_view.h"
+#include "api/rtc_event_log/rtc_event_log_factory.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/pclf/media_quality_test_params.h"
+#include "api/test/pclf/peer_configurer.h"
+#include "api/test/time_controller.h"
+#include "modules/audio_device/include/test_audio_device.h"
+#include "rtc_base/task_queue.h"
+#include "test/pc/e2e/analyzer/video/video_quality_analyzer_injection_helper.h"
+#include "test/pc/e2e/test_peer.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+
+struct RemotePeerAudioConfig {
+ explicit RemotePeerAudioConfig(AudioConfig config)
+ : sampling_frequency_in_hz(config.sampling_frequency_in_hz),
+ output_file_name(config.output_dump_file_name) {}
+
+ static absl::optional<RemotePeerAudioConfig> Create(
+ absl::optional<AudioConfig> config);
+
+ int sampling_frequency_in_hz;
+ absl::optional<std::string> output_file_name;
+};
+
+class TestPeerFactory {
+ public:
+ // Creates a test peer factory.
+ // `signaling_thread` will be used as a signaling thread for all peers created
+ // by this factory.
+ // `time_controller` will be used to create required threads, task queue
+ // factories and call factory.
+ // `video_analyzer_helper` will be used to setup video quality analysis for
+ // created peers.
+ // `task_queue` will be used for AEC dump if it is requested.
+ TestPeerFactory(rtc::Thread* signaling_thread,
+ TimeController& time_controller,
+ VideoQualityAnalyzerInjectionHelper* video_analyzer_helper,
+ rtc::TaskQueue* task_queue)
+ : signaling_thread_(signaling_thread),
+ time_controller_(time_controller),
+ video_analyzer_helper_(video_analyzer_helper),
+ task_queue_(task_queue) {}
+
+ // Setups all components, that should be provided to WebRTC
+ // PeerConnectionFactory and PeerConnection creation methods,
+ // also will setup dependencies, that are required for media analyzers
+ // injection.
+ std::unique_ptr<TestPeer> CreateTestPeer(
+ std::unique_ptr<PeerConfigurer> configurer,
+ std::unique_ptr<MockPeerConnectionObserver> observer,
+ absl::optional<RemotePeerAudioConfig> remote_audio_config,
+ absl::optional<EchoEmulationConfig> echo_emulation_config);
+
+ private:
+ rtc::Thread* signaling_thread_;
+ TimeController& time_controller_;
+ VideoQualityAnalyzerInjectionHelper* video_analyzer_helper_;
+ rtc::TaskQueue* task_queue_;
+};
+
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
+
+#endif // TEST_PC_E2E_TEST_PEER_FACTORY_H_
diff --git a/third_party/libwebrtc/test/pc/sctp/BUILD.gn b/third_party/libwebrtc/test/pc/sctp/BUILD.gn
new file mode 100644
index 0000000000..f088a5b20c
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/sctp/BUILD.gn
@@ -0,0 +1,18 @@
+# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import("../../../webrtc.gni")
+
+rtc_source_set("fake_sctp_transport") {
+ visibility = [ "*" ]
+ sources = [ "fake_sctp_transport.h" ]
+ deps = [
+ "../../../api/transport:sctp_transport_factory_interface",
+ "../../../media:rtc_data_sctp_transport_internal",
+ ]
+}
diff --git a/third_party/libwebrtc/test/pc/sctp/fake_sctp_transport.h b/third_party/libwebrtc/test/pc/sctp/fake_sctp_transport.h
new file mode 100644
index 0000000000..a1bb0e219c
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/sctp/fake_sctp_transport.h
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_PC_SCTP_FAKE_SCTP_TRANSPORT_H_
+#define TEST_PC_SCTP_FAKE_SCTP_TRANSPORT_H_
+
+#include <memory>
+
+#include "api/transport/sctp_transport_factory_interface.h"
+#include "media/sctp/sctp_transport_internal.h"
+
+// Used for tests in this file to verify that PeerConnection responds to signals
+// from the SctpTransport correctly, and calls Start with the correct
+// local/remote ports.
+class FakeSctpTransport : public cricket::SctpTransportInternal {
+ public:
+ void SetOnConnectedCallback(std::function<void()> callback) override {}
+ void SetDataChannelSink(webrtc::DataChannelSink* sink) override {}
+ void SetDtlsTransport(rtc::PacketTransportInternal* transport) override {}
+ bool Start(int local_port, int remote_port, int max_message_size) override {
+ local_port_.emplace(local_port);
+ remote_port_.emplace(remote_port);
+ max_message_size_ = max_message_size;
+ return true;
+ }
+ bool OpenStream(int sid) override { return true; }
+ bool ResetStream(int sid) override { return true; }
+ bool SendData(int sid,
+ const webrtc::SendDataParams& params,
+ const rtc::CopyOnWriteBuffer& payload,
+ cricket::SendDataResult* result = nullptr) override {
+ return true;
+ }
+ bool ReadyToSendData() override { return true; }
+ void set_debug_name_for_testing(const char* debug_name) override {}
+
+ int max_message_size() const { return max_message_size_; }
+ absl::optional<int> max_outbound_streams() const { return absl::nullopt; }
+ absl::optional<int> max_inbound_streams() const { return absl::nullopt; }
+ int local_port() const {
+ RTC_DCHECK(local_port_);
+ return *local_port_;
+ }
+ int remote_port() const {
+ RTC_DCHECK(remote_port_);
+ return *remote_port_;
+ }
+
+ private:
+ absl::optional<int> local_port_;
+ absl::optional<int> remote_port_;
+ int max_message_size_;
+};
+
+class FakeSctpTransportFactory : public webrtc::SctpTransportFactoryInterface {
+ public:
+ std::unique_ptr<cricket::SctpTransportInternal> CreateSctpTransport(
+ rtc::PacketTransportInternal*) override {
+ last_fake_sctp_transport_ = new FakeSctpTransport();
+ return std::unique_ptr<cricket::SctpTransportInternal>(
+ last_fake_sctp_transport_);
+ }
+
+ FakeSctpTransport* last_fake_sctp_transport() {
+ return last_fake_sctp_transport_;
+ }
+
+ private:
+ FakeSctpTransport* last_fake_sctp_transport_ = nullptr;
+};
+
+#endif // TEST_PC_SCTP_FAKE_SCTP_TRANSPORT_H_