1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
/*
* Copyright 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "test/scenario/stats_collection.h"
#include "test/gtest.h"
#include "test/scenario/scenario.h"
namespace webrtc {
namespace test {
namespace {
void CreateAnalyzedStream(Scenario* s,
NetworkSimulationConfig network_config,
VideoQualityAnalyzer* analyzer,
CallStatsCollectors* collectors) {
VideoStreamConfig config;
config.encoder.codec = VideoStreamConfig::Encoder::Codec::kVideoCodecVP8;
config.encoder.implementation =
VideoStreamConfig::Encoder::Implementation::kSoftware;
config.hooks.frame_pair_handlers = {analyzer->Handler()};
auto* caller = s->CreateClient("caller", CallClientConfig());
auto* callee = s->CreateClient("callee", CallClientConfig());
auto route =
s->CreateRoutes(caller, {s->CreateSimulationNode(network_config)}, callee,
{s->CreateSimulationNode(NetworkSimulationConfig())});
VideoStreamPair* video = s->CreateVideoStream(route->forward(), config);
auto* audio = s->CreateAudioStream(route->forward(), AudioStreamConfig());
s->Every(TimeDelta::Seconds(1), [=] {
collectors->call.AddStats(caller->GetStats());
VideoSendStream::Stats send_stats;
caller->SendTask([&]() { send_stats = video->send()->GetStats(); });
collectors->video_send.AddStats(send_stats, s->Now());
AudioReceiveStreamInterface::Stats receive_stats;
caller->SendTask([&]() { receive_stats = audio->receive()->GetStats(); });
collectors->audio_receive.AddStats(receive_stats);
// Querying the video stats from within the expected runtime environment
// (i.e. the TQ that belongs to the CallClient, not the Scenario TQ that
// we're currently on).
VideoReceiveStreamInterface::Stats video_receive_stats;
auto* video_stream = video->receive();
callee->SendTask([&video_stream, &video_receive_stats]() {
video_receive_stats = video_stream->GetStats();
});
collectors->video_receive.AddStats(video_receive_stats);
});
}
} // namespace
TEST(ScenarioAnalyzerTest, PsnrIsHighWhenNetworkIsGood) {
VideoQualityAnalyzer analyzer;
CallStatsCollectors stats;
{
Scenario s;
NetworkSimulationConfig good_network;
good_network.bandwidth = DataRate::KilobitsPerSec(1000);
CreateAnalyzedStream(&s, good_network, &analyzer, &stats);
s.RunFor(TimeDelta::Seconds(3));
}
// This is a change detecting test, the targets are based on previous runs and
// might change due to changes in configuration and encoder etc. The main
// purpose is to show how the stats can be used. To avoid being overly
// sensistive to change, the ranges are chosen to be quite large.
EXPECT_NEAR(analyzer.stats().psnr_with_freeze.Mean(), 43, 10);
EXPECT_NEAR(stats.call.stats().target_rate.Mean().kbps(), 700, 300);
EXPECT_NEAR(stats.video_send.stats().media_bitrate.Mean().kbps(), 500, 200);
EXPECT_NEAR(stats.video_receive.stats().resolution.Mean(), 180, 10);
EXPECT_NEAR(stats.audio_receive.stats().jitter_buffer.Mean().ms(), 40, 20);
}
TEST(ScenarioAnalyzerTest, PsnrIsLowWhenNetworkIsBad) {
VideoQualityAnalyzer analyzer;
CallStatsCollectors stats;
{
Scenario s;
NetworkSimulationConfig bad_network;
bad_network.bandwidth = DataRate::KilobitsPerSec(100);
bad_network.loss_rate = 0.02;
CreateAnalyzedStream(&s, bad_network, &analyzer, &stats);
s.RunFor(TimeDelta::Seconds(3));
}
// This is a change detecting test, the targets are based on previous runs and
// might change due to changes in configuration and encoder etc.
EXPECT_NEAR(analyzer.stats().psnr_with_freeze.Mean(), 20, 10);
EXPECT_NEAR(stats.call.stats().target_rate.Mean().kbps(), 75, 50);
EXPECT_NEAR(stats.video_send.stats().media_bitrate.Mean().kbps(), 70, 30);
EXPECT_NEAR(stats.video_receive.stats().resolution.Mean(), 180, 10);
EXPECT_NEAR(stats.audio_receive.stats().jitter_buffer.Mean().ms(), 250, 200);
}
TEST(ScenarioAnalyzerTest, CountsCapturedButNotRendered) {
VideoQualityAnalyzer analyzer;
CallStatsCollectors stats;
{
Scenario s;
NetworkSimulationConfig long_delays;
long_delays.delay = TimeDelta::Seconds(5);
CreateAnalyzedStream(&s, long_delays, &analyzer, &stats);
// Enough time to send frames but not enough to deliver.
s.RunFor(TimeDelta::Millis(100));
}
EXPECT_GE(analyzer.stats().capture.count, 1);
EXPECT_EQ(analyzer.stats().render.count, 0);
}
} // namespace test
} // namespace webrtc
|
