1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
|
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_
#define TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_
#include <list>
#include <memory>
#include <utility>
#include <vector>
#include "absl/strings/string_view.h"
#include "api/test/pclf/media_configuration.h"
#include "api/test/video_quality_analyzer_interface.h"
#include "api/video/video_frame.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "rtc_base/synchronization/mutex.h"
#include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h"
namespace webrtc {
namespace webrtc_pc_e2e {
// QualityAnalyzingVideoEncoder is used to wrap origin video encoder and inject
// VideoQualityAnalyzerInterface before and after encoder.
//
// QualityAnalyzingVideoEncoder propagates all calls to the origin encoder.
// It registers its own EncodedImageCallback in the origin encoder and will
// store user specified callback inside itself.
//
// When Encode(...) will be invoked, quality encoder first calls video quality
// analyzer with original frame, then encodes frame with original encoder.
//
// When origin encoder encodes the image it will call quality encoder's special
// callback, where video analyzer will be called again and then frame id will be
// injected into EncodedImage with passed EncodedImageDataInjector. Then new
// EncodedImage will be passed to origin callback, provided by user.
//
// Quality encoder registers its own callback in origin encoder, at the same
// time the user registers their callback in quality encoder.
class QualityAnalyzingVideoEncoder : public VideoEncoder,
public EncodedImageCallback {
public:
using EmulatedSFUConfigMap =
std::map<std::string, absl::optional<EmulatedSFUConfig>>;
QualityAnalyzingVideoEncoder(absl::string_view peer_name,
std::unique_ptr<VideoEncoder> delegate,
double bitrate_multiplier,
EmulatedSFUConfigMap stream_to_sfu_config,
EncodedImageDataInjector* injector,
VideoQualityAnalyzerInterface* analyzer);
~QualityAnalyzingVideoEncoder() override;
// Methods of VideoEncoder interface.
void SetFecControllerOverride(
FecControllerOverride* fec_controller_override) override;
int32_t InitEncode(const VideoCodec* codec_settings,
const Settings& settings) override;
int32_t RegisterEncodeCompleteCallback(
EncodedImageCallback* callback) override;
int32_t Release() override;
int32_t Encode(const VideoFrame& frame,
const std::vector<VideoFrameType>* frame_types) override;
void SetRates(const VideoEncoder::RateControlParameters& parameters) override;
EncoderInfo GetEncoderInfo() const override;
// Methods of EncodedImageCallback interface.
EncodedImageCallback::Result OnEncodedImage(
const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific_info) override;
void OnDroppedFrame(DropReason reason) override;
private:
enum SimulcastMode {
// In this mode encoder assumes not more than 1 encoded image per video
// frame
kNormal,
// Next modes are to test video conference behavior. For conference sender
// will send multiple spatial layers/simulcast streams for single video
// track and there is some Selective Forwarding Unit (SFU), that forwards
// only best one, that will pass through downlink to the receiver.
//
// Here this behavior will be partly emulated. Sender will send all spatial
// layers/simulcast streams and then some of them will be filtered out on
// the receiver side. During test setup user can specify which spatial
// layer/simulcast stream is required, what will simulated which spatial
// layer/simulcast stream will be chosen by SFU in the real world. Then
// sender will mark encoded images for all spatial layers above required or
// all simulcast streams except required as to be discarded and on receiver
// side they will be discarded in quality analyzing decoder and won't be
// passed into delegate decoder.
//
// If the sender for some reasons won't send specified spatial layer, then
// receiver still will fall back on lower spatial layers. But for simulcast
// streams if required one won't be sent, receiver will assume all frames
// in that period as dropped and will experience video freeze.
//
// Test based on this simulation will be used to evaluate video quality
// of concrete spatial layers/simulcast streams and also check distribution
// of bandwidth between spatial layers/simulcast streams by BWE.
// In this mode encoder assumes that for each frame simulcast encoded
// images will be produced. So all simulcast streams except required will
// be marked as to be discarded in decoder and won't reach video quality
// analyzer.
kSimulcast,
// In this mode encoder assumes that for each frame encoded images for
// different spatial layers will be produced. So all spatial layers above
// required will be marked to be discarded in decoder and won't reach
// video quality analyzer.
kSVC,
// In this mode encoder assumes that for each frame encoded images for
// different spatial layers will be produced. Compared to kSVC mode
// spatial layers that are above required will be marked to be discarded
// only for key frames and for regular frames all except required spatial
// layer will be marked as to be discarded in decoder and won't reach video
// quality analyzer.
kKSVC
};
bool ShouldDiscard(uint16_t frame_id, const EncodedImage& encoded_image)
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
const std::string peer_name_;
std::unique_ptr<VideoEncoder> delegate_;
const double bitrate_multiplier_;
// Contains mapping from stream label to optional spatial index.
// If we have stream label "Foo" and mapping contains
// 1. `absl::nullopt` means all streams are required
// 2. Concrete value means that particular simulcast/SVC stream have to be
// analyzed.
EmulatedSFUConfigMap stream_to_sfu_config_;
EncodedImageDataInjector* const injector_;
VideoQualityAnalyzerInterface* const analyzer_;
// VideoEncoder interface assumes async delivery of encoded images.
// This lock is used to protect shared state, that have to be propagated
// from received VideoFrame to resulted EncodedImage.
Mutex mutex_;
VideoCodec codec_settings_ RTC_GUARDED_BY(mutex_);
SimulcastMode mode_ RTC_GUARDED_BY(mutex_);
EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(mutex_);
std::list<std::pair<uint32_t, uint16_t>> timestamp_to_frame_id_list_
RTC_GUARDED_BY(mutex_);
VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(mutex_);
};
// Produces QualityAnalyzingVideoEncoder, which hold decoders, produced by
// specified factory as delegates. Forwards all other calls to specified
// factory.
class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory {
public:
QualityAnalyzingVideoEncoderFactory(
absl::string_view peer_name,
std::unique_ptr<VideoEncoderFactory> delegate,
double bitrate_multiplier,
QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap stream_to_sfu_config,
EncodedImageDataInjector* injector,
VideoQualityAnalyzerInterface* analyzer);
~QualityAnalyzingVideoEncoderFactory() override;
// Methods of VideoEncoderFactory interface.
std::vector<SdpVideoFormat> GetSupportedFormats() const override;
VideoEncoderFactory::CodecSupport QueryCodecSupport(
const SdpVideoFormat& format,
absl::optional<std::string> scalability_mode) const override;
std::unique_ptr<VideoEncoder> CreateVideoEncoder(
const SdpVideoFormat& format) override;
private:
const std::string peer_name_;
std::unique_ptr<VideoEncoderFactory> delegate_;
const double bitrate_multiplier_;
QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap stream_to_sfu_config_;
EncodedImageDataInjector* const injector_;
VideoQualityAnalyzerInterface* const analyzer_;
};
} // namespace webrtc_pc_e2e
} // namespace webrtc
#endif // TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_
|
