summaryrefslogtreecommitdiffstats
path: root/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc1102
1 files changed, 1102 insertions, 0 deletions
diff --git a/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc
new file mode 100644
index 0000000000..8a47e108e0
--- /dev/null
+++ b/third_party/libwebrtc/test/pc/e2e/peer_connection_quality_test_metric_names_test.cc
@@ -0,0 +1,1102 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <map>
+#include <memory>
+#include <string>
+
+#include "api/test/create_network_emulation_manager.h"
+#include "api/test/create_peer_connection_quality_test_frame_generator.h"
+#include "api/test/metrics/metrics_logger.h"
+#include "api/test/metrics/stdout_metrics_exporter.h"
+#include "api/test/network_emulation_manager.h"
+#include "api/test/pclf/media_configuration.h"
+#include "api/test/pclf/media_quality_test_params.h"
+#include "api/test/pclf/peer_configurer.h"
+#include "api/test/peerconnection_quality_test_fixture.h"
+#include "api/units/time_delta.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/pc/e2e/metric_metadata_keys.h"
+#include "test/pc/e2e/peer_connection_quality_test.h"
+#include "test/pc/e2e/stats_based_network_quality_metrics_reporter.h"
+
+namespace webrtc {
+namespace webrtc_pc_e2e {
+namespace {
+
+using ::testing::UnorderedElementsAre;
+
+using ::webrtc::test::DefaultMetricsLogger;
+using ::webrtc::test::ImprovementDirection;
+using ::webrtc::test::Metric;
+using ::webrtc::test::MetricsExporter;
+using ::webrtc::test::StdoutMetricsExporter;
+using ::webrtc::test::Unit;
+using ::webrtc::webrtc_pc_e2e::PeerConfigurer;
+
+// Adds a peer with some audio and video (the client should not care about
+// details about audio and video configs).
+void AddDefaultAudioVideoPeer(
+ absl::string_view peer_name,
+ absl::string_view audio_stream_label,
+ absl::string_view video_stream_label,
+ const PeerNetworkDependencies& network_dependencies,
+ PeerConnectionE2EQualityTestFixture& fixture) {
+ AudioConfig audio{std::string(audio_stream_label)};
+ audio.sync_group = std::string(peer_name);
+ VideoConfig video(std::string(video_stream_label), 320, 180, 15);
+ video.sync_group = std::string(peer_name);
+ auto peer = std::make_unique<PeerConfigurer>(network_dependencies);
+ peer->SetName(peer_name);
+ peer->SetAudioConfig(std::move(audio));
+ peer->AddVideoConfig(std::move(video));
+ peer->SetVideoCodecs({VideoCodecConfig(cricket::kVp8CodecName)});
+ fixture.AddPeer(std::move(peer));
+}
+
+// Metric fields to assert on
+struct MetricValidationInfo {
+ std::string test_case;
+ std::string name;
+ Unit unit;
+ ImprovementDirection improvement_direction;
+ std::map<std::string, std::string> metadata;
+};
+
+bool operator==(const MetricValidationInfo& a, const MetricValidationInfo& b) {
+ return a.name == b.name && a.test_case == b.test_case && a.unit == b.unit &&
+ a.improvement_direction == b.improvement_direction &&
+ a.metadata == b.metadata;
+}
+
+std::ostream& operator<<(std::ostream& os, const MetricValidationInfo& m) {
+ os << "{ test_case=" << m.test_case << "; name=" << m.name
+ << "; unit=" << test::ToString(m.unit)
+ << "; improvement_direction=" << test::ToString(m.improvement_direction)
+ << "; metadata={ ";
+ for (const auto& [key, value] : m.metadata) {
+ os << "{ key=" << key << "; value=" << value << " }";
+ }
+ os << " }}";
+ return os;
+}
+
+std::vector<MetricValidationInfo> ToValidationInfo(
+ const std::vector<Metric>& metrics) {
+ std::vector<MetricValidationInfo> out;
+ for (const Metric& m : metrics) {
+ out.push_back(
+ MetricValidationInfo{.test_case = m.test_case,
+ .name = m.name,
+ .unit = m.unit,
+ .improvement_direction = m.improvement_direction,
+ .metadata = m.metric_metadata});
+ }
+ return out;
+}
+
+TEST(PeerConnectionE2EQualityTestMetricNamesTest,
+ ExportedMetricsHasCorrectNamesAndAnnotation) {
+ std::unique_ptr<NetworkEmulationManager> network_emulation =
+ CreateNetworkEmulationManager(TimeMode::kSimulated);
+ DefaultMetricsLogger metrics_logger(
+ network_emulation->time_controller()->GetClock());
+ PeerConnectionE2EQualityTest fixture(
+ "test_case", *network_emulation->time_controller(),
+ /*audio_quality_analyzer=*/nullptr, /*video_quality_analyzer=*/nullptr,
+ &metrics_logger);
+
+ EmulatedEndpoint* alice_endpoint =
+ network_emulation->CreateEndpoint(EmulatedEndpointConfig());
+ EmulatedEndpoint* bob_endpoint =
+ network_emulation->CreateEndpoint(EmulatedEndpointConfig());
+
+ network_emulation->CreateRoute(
+ alice_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()},
+ bob_endpoint);
+ network_emulation->CreateRoute(
+ bob_endpoint, {network_emulation->CreateUnconstrainedEmulatedNode()},
+ alice_endpoint);
+
+ EmulatedNetworkManagerInterface* alice_network =
+ network_emulation->CreateEmulatedNetworkManagerInterface(
+ {alice_endpoint});
+ EmulatedNetworkManagerInterface* bob_network =
+ network_emulation->CreateEmulatedNetworkManagerInterface({bob_endpoint});
+
+ AddDefaultAudioVideoPeer("alice", "alice_audio", "alice_video",
+ alice_network->network_dependencies(), fixture);
+ AddDefaultAudioVideoPeer("bob", "bob_audio", "bob_video",
+ bob_network->network_dependencies(), fixture);
+ fixture.AddQualityMetricsReporter(
+ std::make_unique<StatsBasedNetworkQualityMetricsReporter>(
+ std::map<std::string, std::vector<EmulatedEndpoint*>>(
+ {{"alice", alice_network->endpoints()},
+ {"bob", bob_network->endpoints()}}),
+ network_emulation.get(), &metrics_logger));
+
+ // Run for at least 7 seconds, so AV-sync metrics will be collected.
+ fixture.Run(RunParams(TimeDelta::Seconds(7)));
+
+ std::vector<MetricValidationInfo> metrics =
+ ToValidationInfo(metrics_logger.GetCollectedMetrics());
+ EXPECT_THAT(
+ metrics,
+ UnorderedElementsAre(
+ // Metrics from PeerConnectionE2EQualityTest
+ MetricValidationInfo{
+ .test_case = "test_case",
+ .name = "alice_connected",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case",
+ .name = "bob_connected",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+
+ // Metrics from DefaultAudioQualityAnalyzer
+ MetricValidationInfo{
+ .test_case = "test_case/alice_audio",
+ .name = "expand_rate",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "alice_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_audio",
+ .name = "accelerate_rate",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "alice_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_audio",
+ .name = "preemptive_rate",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "alice_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_audio",
+ .name = "speech_expand_rate",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "alice_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_audio",
+ .name = "average_jitter_buffer_delay_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "alice_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_audio",
+ .name = "preferred_buffer_size_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "alice_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_audio",
+ .name = "expand_rate",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "bob_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_audio",
+ .name = "accelerate_rate",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "bob_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_audio",
+ .name = "preemptive_rate",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "bob_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_audio",
+ .name = "speech_expand_rate",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "bob_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_audio",
+ .name = "average_jitter_buffer_delay_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "bob_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_audio",
+ .name = "preferred_buffer_size_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
+ "bob_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+
+ // Metrics from DefaultVideoQualityAnalyzer
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "ssim",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "transport_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "total_delay_incl_transport",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "time_between_rendered_frames",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "harmonic_framerate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "encode_frame_rate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "encode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "time_between_freezes",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "freeze_time_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "pixels_per_frame",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "min_psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "decode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "receive_to_render_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "dropped_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "frames_in_flight",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "rendered_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "max_skipped",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "target_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "qp_sl0",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kSpatialLayerMetadataKey, "0"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_video",
+ .name = "actual_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "alice_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "ssim",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "transport_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "total_delay_incl_transport",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "time_between_rendered_frames",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "harmonic_framerate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "encode_frame_rate",
+ .unit = Unit::kHertz,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "encode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "time_between_freezes",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "freeze_time_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "pixels_per_frame",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "min_psnr_dB",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "decode_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "receive_to_render_time",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "dropped_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "frames_in_flight",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "rendered_frames",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kBiggerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "max_skipped",
+ .unit = Unit::kCount,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "target_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "actual_encode_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_video",
+ .name = "qp_sl0",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kVideoStreamMetadataKey,
+ "bob_video"},
+ {MetricMetadataKey::kSenderMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kSpatialLayerMetadataKey, "0"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case",
+ .name = "cpu_usage_%",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {{MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+
+ // Metrics from StatsBasedNetworkQualityMetricsReporter
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "bytes_discarded_no_receiver",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "packets_discarded_no_receiver",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "payload_bytes_received",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "payload_bytes_sent",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "bytes_sent",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "packets_sent",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "average_send_rate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "bytes_received",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "packets_received",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "average_receive_rate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "sent_packets_loss",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "bytes_discarded_no_receiver",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "packets_discarded_no_receiver",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "payload_bytes_received",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "payload_bytes_sent",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "bytes_sent",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "packets_sent",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "average_send_rate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "bytes_received",
+ .unit = Unit::kBytes,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "packets_received",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "average_receive_rate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "sent_packets_loss",
+ .unit = Unit::kUnitless,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+
+ // Metrics from VideoQualityMetricsReporter
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "available_send_bandwidth",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "transmission_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice",
+ .name = "retransmission_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "available_send_bandwidth",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "transmission_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob",
+ .name = "retransmission_bitrate",
+ .unit = Unit::kKilobitsPerSecond,
+ .improvement_direction = ImprovementDirection::kNeitherIsBetter,
+ .metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+
+ // Metrics from CrossMediaMetricsReporter
+ MetricValidationInfo{
+ .test_case = "test_case/alice_alice_audio",
+ .name = "audio_ahead_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata =
+ {{MetricMetadataKey::kAudioStreamMetadataKey, "alice_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kPeerSyncGroupMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/alice_alice_video",
+ .name = "video_ahead_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata =
+ {{MetricMetadataKey::kAudioStreamMetadataKey, "alice_video"},
+ {MetricMetadataKey::kPeerMetadataKey, "bob"},
+ {MetricMetadataKey::kPeerSyncGroupMetadataKey, "alice"},
+ {MetricMetadataKey::kReceiverMetadataKey, "bob"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_bob_audio",
+ .name = "audio_ahead_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata =
+ {{MetricMetadataKey::kAudioStreamMetadataKey, "bob_audio"},
+ {MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kPeerSyncGroupMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}},
+ MetricValidationInfo{
+ .test_case = "test_case/bob_bob_video",
+ .name = "video_ahead_ms",
+ .unit = Unit::kMilliseconds,
+ .improvement_direction = ImprovementDirection::kSmallerIsBetter,
+ .metadata = {
+ {MetricMetadataKey::kAudioStreamMetadataKey, "bob_video"},
+ {MetricMetadataKey::kPeerMetadataKey, "alice"},
+ {MetricMetadataKey::kPeerSyncGroupMetadataKey, "bob"},
+ {MetricMetadataKey::kReceiverMetadataKey, "alice"},
+ {MetricMetadataKey::kExperimentalTestNameMetadataKey,
+ "test_case"}}}));
+}
+
+} // namespace
+} // namespace webrtc_pc_e2e
+} // namespace webrtc
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-26 20:21:06 +0000