1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
|
/*
* Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef API_TEST_PCLF_MEDIA_CONFIGURATION_H_
#define API_TEST_PCLF_MEDIA_CONFIGURATION_H_
#include <stddef.h>
#include <stdint.h>
#include <functional>
#include <map>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "absl/memory/memory.h"
#include "absl/strings/string_view.h"
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/async_resolver_factory.h"
#include "api/audio/audio_mixer.h"
#include "api/audio_options.h"
#include "api/call/call_factory_interface.h"
#include "api/fec_controller.h"
#include "api/function_view.h"
#include "api/media_stream_interface.h"
#include "api/peer_connection_interface.h"
#include "api/rtc_event_log/rtc_event_log_factory_interface.h"
#include "api/rtp_parameters.h"
#include "api/task_queue/task_queue_factory.h"
#include "api/test/audio_quality_analyzer_interface.h"
#include "api/test/frame_generator_interface.h"
#include "api/test/peer_network_dependencies.h"
#include "api/test/simulated_network.h"
#include "api/test/stats_observer_interface.h"
#include "api/test/track_id_stream_info_map.h"
#include "api/test/video/video_frame_writer.h"
#include "api/test/video_quality_analyzer_interface.h"
#include "api/transport/network_control.h"
#include "api/units/time_delta.h"
#include "api/video_codecs/video_decoder_factory.h"
#include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "modules/audio_processing/include/audio_processing.h"
#include "rtc_base/checks.h"
#include "rtc_base/network.h"
#include "rtc_base/rtc_certificate_generator.h"
#include "rtc_base/ssl_certificate.h"
#include "rtc_base/thread.h"
namespace webrtc {
namespace webrtc_pc_e2e {
constexpr size_t kDefaultSlidesWidth = 1850;
constexpr size_t kDefaultSlidesHeight = 1110;
// The index of required capturing device in OS provided list of video
// devices. On Linux and Windows the list will be obtained via
// webrtc::VideoCaptureModule::DeviceInfo, on Mac OS via
// [RTCCameraVideoCapturer captureDevices].
enum class CapturingDeviceIndex : size_t {};
// Contains parameters for screen share scrolling.
//
// If scrolling is enabled, then it will be done by putting sliding window
// on source video and moving this window from top left corner to the
// bottom right corner of the picture.
//
// In such case source dimensions must be greater or equal to the sliding
// window dimensions. So `source_width` and `source_height` are the dimensions
// of the source frame, while `VideoConfig::width` and `VideoConfig::height`
// are the dimensions of the sliding window.
//
// Because `source_width` and `source_height` are dimensions of the source
// frame, they have to be width and height of videos from
// `ScreenShareConfig::slides_yuv_file_names`.
//
// Because scrolling have to be done on single slide it also requires, that
// `duration` must be less or equal to
// `ScreenShareConfig::slide_change_interval`.
struct ScrollingParams {
// Duration of scrolling.
TimeDelta duration;
// Width of source slides video.
size_t source_width = kDefaultSlidesWidth;
// Height of source slides video.
size_t source_height = kDefaultSlidesHeight;
};
// Contains screen share video stream properties.
struct ScreenShareConfig {
explicit ScreenShareConfig(TimeDelta slide_change_interval);
// Shows how long one slide should be presented on the screen during
// slide generation.
TimeDelta slide_change_interval;
// If true, slides will be generated programmatically. No scrolling params
// will be applied in such case.
bool generate_slides = false;
// If present scrolling will be applied. Please read extra requirement on
// `slides_yuv_file_names` for scrolling.
absl::optional<ScrollingParams> scrolling_params;
// Contains list of yuv files with slides.
//
// If empty, default set of slides will be used. In such case
// `VideoConfig::width` must be equal to `kDefaultSlidesWidth` and
// `VideoConfig::height` must be equal to `kDefaultSlidesHeight` or if
// `scrolling_params` are specified, then `ScrollingParams::source_width`
// must be equal to `kDefaultSlidesWidth` and
// `ScrollingParams::source_height` must be equal to `kDefaultSlidesHeight`.
std::vector<std::string> slides_yuv_file_names;
};
// Config for Vp8 simulcast or non-standard Vp9 SVC testing.
//
// To configure standard SVC setting, use `scalability_mode` in the
// `encoding_params` array.
// This configures Vp9 SVC by requesting simulcast layers, the request is
// internally converted to a request for SVC layers.
//
// SVC support is limited:
// During SVC testing there is no SFU, so framework will try to emulate SFU
// behavior in regular p2p call. Because of it there are such limitations:
// * if `target_spatial_index` is not equal to the highest spatial layer
// then no packet/frame drops are allowed.
//
// If there will be any drops, that will affect requested layer, then
// WebRTC SVC implementation will continue decoding only the highest
// available layer and won't restore lower layers, so analyzer won't
// receive required data which will cause wrong results or test failures.
struct VideoSimulcastConfig {
explicit VideoSimulcastConfig(int simulcast_streams_count);
// Specified amount of simulcast streams/SVC layers, depending on which
// encoder is used.
int simulcast_streams_count;
};
// Configuration for the emulated Selective Forward Unit (SFU)
//
// The framework can optionally filter out frames that are decoded
// using an emulated SFU.
// When using simulcast or SVC, it's not always desirable to receive
// all frames. In a real world call, a SFU will only forward a subset
// of the frames.
// The emulated SFU is not able to change its configuration dynamically,
// if adaptation happens during the call, layers may be dropped and the
// analyzer won't receive the required data which will cause wrong results or
// test failures.
struct EmulatedSFUConfig {
EmulatedSFUConfig() = default;
explicit EmulatedSFUConfig(int target_layer_index);
EmulatedSFUConfig(absl::optional<int> target_layer_index,
absl::optional<int> target_temporal_index);
// Specifies simulcast or spatial index of the video stream to analyze.
// There are 2 cases:
// 1. simulcast encoding is used:
// in such case `target_layer_index` will specify the index of
// simulcast stream, that should be analyzed. Other streams will be
// dropped.
// 2. SVC encoding is used:
// in such case `target_layer_index` will specify the top interesting
// spatial layer and all layers below, including target one will be
// processed. All layers above target one will be dropped.
// If not specified then all streams will be received and analyzed.
// When set, it instructs the framework to create an emulated Selective
// Forwarding Unit (SFU) that will propagate only the requested layers.
absl::optional<int> target_layer_index;
// Specifies the index of the maximum temporal unit to keep.
// If not specified then all temporal layers will be received and analyzed.
// When set, it instructs the framework to create an emulated Selective
// Forwarding Unit (SFU) that will propagate only up to the requested layer.
absl::optional<int> target_temporal_index;
};
class VideoResolution {
public:
// Determines special resolutions, which can't be expressed in terms of
// width, height and fps.
enum class Spec {
// No extra spec set. It describes a regular resolution described by
// width, height and fps.
kNone,
// Describes resolution which contains max value among all sender's
// video streams in each dimension (width, height, fps).
kMaxFromSender
};
VideoResolution(size_t width, size_t height, int32_t fps);
explicit VideoResolution(Spec spec = Spec::kNone);
bool operator==(const VideoResolution& other) const;
bool operator!=(const VideoResolution& other) const;
size_t width() const { return width_; }
void set_width(size_t width) { width_ = width; }
size_t height() const { return height_; }
void set_height(size_t height) { height_ = height; }
int32_t fps() const { return fps_; }
void set_fps(int32_t fps) { fps_ = fps; }
// Returns if it is a regular resolution or not. The resolution is regular
// if it's spec is `Spec::kNone`.
bool IsRegular() const;
std::string ToString() const;
private:
size_t width_ = 0;
size_t height_ = 0;
int32_t fps_ = 0;
Spec spec_ = Spec::kNone;
};
class VideoDumpOptions {
public:
static constexpr int kDefaultSamplingModulo = 1;
// output_directory - the output directory where stream will be dumped. The
// output files' names will be constructed as
// <stream_name>_<receiver_name>_<resolution>.<extension> for output dumps
// and <stream_name>_<resolution>.<extension> for input dumps.
// By default <extension> is "y4m". Resolution is in the format
// <width>x<height>_<fps>.
// sampling_modulo - the module for the video frames to be dumped. Modulo
// equals X means every Xth frame will be written to the dump file. The
// value must be greater than 0. (Default: 1)
// export_frame_ids - specifies if frame ids should be exported together
// with content of the stream. If true, an output file with the same name as
// video dump and suffix ".frame_ids.txt" will be created. It will contain
// the frame ids in the same order as original frames in the output
// file with stream content. File will contain one frame id per line.
// (Default: false)
// `video_frame_writer_factory` - factory function to create a video frame
// writer for input and output video files. (Default: Y4M video writer
// factory).
explicit VideoDumpOptions(
absl::string_view output_directory,
int sampling_modulo = kDefaultSamplingModulo,
bool export_frame_ids = false,
std::function<std::unique_ptr<test::VideoFrameWriter>(
absl::string_view file_name_prefix,
const VideoResolution& resolution)> video_frame_writer_factory =
Y4mVideoFrameWriterFactory);
VideoDumpOptions(absl::string_view output_directory, bool export_frame_ids);
VideoDumpOptions(const VideoDumpOptions&) = default;
VideoDumpOptions& operator=(const VideoDumpOptions&) = default;
VideoDumpOptions(VideoDumpOptions&&) = default;
VideoDumpOptions& operator=(VideoDumpOptions&&) = default;
std::string output_directory() const { return output_directory_; }
int sampling_modulo() const { return sampling_modulo_; }
bool export_frame_ids() const { return export_frame_ids_; }
std::unique_ptr<test::VideoFrameWriter> CreateInputDumpVideoFrameWriter(
absl::string_view stream_label,
const VideoResolution& resolution) const;
std::unique_ptr<test::VideoFrameWriter> CreateOutputDumpVideoFrameWriter(
absl::string_view stream_label,
absl::string_view receiver,
const VideoResolution& resolution) const;
std::string ToString() const;
private:
static std::unique_ptr<test::VideoFrameWriter> Y4mVideoFrameWriterFactory(
absl::string_view file_name_prefix,
const VideoResolution& resolution);
std::string GetInputDumpFileName(absl::string_view stream_label,
const VideoResolution& resolution) const;
// Returns file name for input frame ids dump if `export_frame_ids()` is
// true, absl::nullopt otherwise.
absl::optional<std::string> GetInputFrameIdsDumpFileName(
absl::string_view stream_label,
const VideoResolution& resolution) const;
std::string GetOutputDumpFileName(absl::string_view stream_label,
absl::string_view receiver,
const VideoResolution& resolution) const;
// Returns file name for output frame ids dump if `export_frame_ids()` is
// true, absl::nullopt otherwise.
absl::optional<std::string> GetOutputFrameIdsDumpFileName(
absl::string_view stream_label,
absl::string_view receiver,
const VideoResolution& resolution) const;
std::string output_directory_;
int sampling_modulo_ = 1;
bool export_frame_ids_ = false;
std::function<std::unique_ptr<test::VideoFrameWriter>(
absl::string_view file_name_prefix,
const VideoResolution& resolution)>
video_frame_writer_factory_;
};
// Contains properties of single video stream.
struct VideoConfig {
explicit VideoConfig(const VideoResolution& resolution);
VideoConfig(size_t width, size_t height, int32_t fps);
VideoConfig(std::string stream_label,
size_t width,
size_t height,
int32_t fps);
// Video stream width.
size_t width;
// Video stream height.
size_t height;
int32_t fps;
VideoResolution GetResolution() const {
return VideoResolution(width, height, fps);
}
// Have to be unique among all specified configs for all peers in the call.
// Will be auto generated if omitted.
absl::optional<std::string> stream_label;
// Will be set for current video track. If equals to kText or kDetailed -
// screencast in on.
absl::optional<VideoTrackInterface::ContentHint> content_hint;
// If presented video will be transfered in simulcast/SVC mode depending on
// which encoder is used.
//
// Simulcast is supported only from 1st added peer. For VP8 simulcast only
// without RTX is supported so it will be automatically disabled for all
// simulcast tracks. For VP9 simulcast enables VP9 SVC mode and support RTX,
// but only on non-lossy networks. See more in documentation to
// VideoSimulcastConfig.
absl::optional<VideoSimulcastConfig> simulcast_config;
// Configuration for the emulated Selective Forward Unit (SFU).
absl::optional<EmulatedSFUConfig> emulated_sfu_config;
// Encoding parameters for both singlecast and per simulcast layer.
// If singlecast is used, if not empty, a single value can be provided.
// If simulcast is used, if not empty, `encoding_params` size have to be
// equal to `simulcast_config.simulcast_streams_count`. Will be used to set
// transceiver send encoding params for each layer.
// RtpEncodingParameters::rid may be changed by fixture implementation to
// ensure signaling correctness.
std::vector<RtpEncodingParameters> encoding_params;
// Count of temporal layers for video stream. This value will be set into
// each RtpEncodingParameters of RtpParameters of corresponding
// RtpSenderInterface for this video stream.
absl::optional<int> temporal_layers_count;
// If specified defines how input should be dumped. It is actually one of
// the test's output file, which contains copy of what was captured during
// the test for this video stream on sender side. It is useful when
// generator is used as input.
absl::optional<VideoDumpOptions> input_dump_options;
// If specified defines how output should be dumped on the receiver side for
// this stream. The produced files contain what was rendered for this video
// stream on receiver side per each receiver.
absl::optional<VideoDumpOptions> output_dump_options;
// If set to true uses fixed frame rate while dumping output video to the
// file. Requested `VideoSubscription::fps()` will be used as frame rate.
bool output_dump_use_fixed_framerate = false;
// If true will display input and output video on the user's screen.
bool show_on_screen = false;
// If specified, determines a sync group to which this video stream belongs.
// According to bugs.webrtc.org/4762 WebRTC supports synchronization only
// for pair of single audio and single video stream.
absl::optional<std::string> sync_group;
// If specified, it will be set into RtpParameters of corresponding
// RtpSenderInterface for this video stream.
// Note that this setting takes precedence over `content_hint`.
absl::optional<DegradationPreference> degradation_preference;
};
// Contains properties for audio in the call.
struct AudioConfig {
enum Mode {
kGenerated,
kFile,
};
AudioConfig() = default;
explicit AudioConfig(std::string stream_label);
// Have to be unique among all specified configs for all peers in the call.
// Will be auto generated if omitted.
absl::optional<std::string> stream_label;
Mode mode = kGenerated;
// Have to be specified only if mode = kFile
absl::optional<std::string> input_file_name;
// If specified the input stream will be also copied to specified file.
absl::optional<std::string> input_dump_file_name;
// If specified the output stream will be copied to specified file.
absl::optional<std::string> output_dump_file_name;
// Audio options to use.
cricket::AudioOptions audio_options;
// Sampling frequency of input audio data (from file or generated).
int sampling_frequency_in_hz = 48000;
// If specified, determines a sync group to which this audio stream belongs.
// According to bugs.webrtc.org/4762 WebRTC supports synchronization only
// for pair of single audio and single video stream.
absl::optional<std::string> sync_group;
};
struct VideoCodecConfig {
explicit VideoCodecConfig(std::string name);
VideoCodecConfig(std::string name,
std::map<std::string, std::string> required_params);
// Next two fields are used to specify concrete video codec, that should be
// used in the test. Video code will be negotiated in SDP during offer/
// answer exchange.
// Video codec name. You can find valid names in
// media/base/media_constants.h
std::string name;
// Map of parameters, that have to be specified on SDP codec. Each parameter
// is described by key and value. Codec parameters will match the specified
// map if and only if for each key from `required_params` there will be
// a parameter with name equal to this key and parameter value will be equal
// to the value from `required_params` for this key.
// If empty then only name will be used to match the codec.
std::map<std::string, std::string> required_params;
};
// Subscription to the remote video streams. It declares which remote stream
// peer should receive and in which resolution (width x height x fps).
class VideoSubscription {
public:
// Returns the resolution constructed as maximum from all resolution
// dimensions: width, height and fps.
static absl::optional<VideoResolution> GetMaxResolution(
rtc::ArrayView<const VideoConfig> video_configs);
static absl::optional<VideoResolution> GetMaxResolution(
rtc::ArrayView<const VideoResolution> resolutions);
bool operator==(const VideoSubscription& other) const;
bool operator!=(const VideoSubscription& other) const;
// Subscribes receiver to all streams sent by the specified peer with
// specified resolution. It will override any resolution that was used in
// `SubscribeToAll` independently from methods call order.
VideoSubscription& SubscribeToPeer(
absl::string_view peer_name,
VideoResolution resolution =
VideoResolution(VideoResolution::Spec::kMaxFromSender));
// Subscribes receiver to the all sent streams with specified resolution.
// If any stream was subscribed to with `SubscribeTo` method that will
// override resolution passed to this function independently from methods
// call order.
VideoSubscription& SubscribeToAllPeers(
VideoResolution resolution =
VideoResolution(VideoResolution::Spec::kMaxFromSender));
// Returns resolution for specific sender. If no specific resolution was
// set for this sender, then will return resolution used for all streams.
// If subscription doesn't subscribe to all streams, `absl::nullopt` will be
// returned.
absl::optional<VideoResolution> GetResolutionForPeer(
absl::string_view peer_name) const;
// Returns a maybe empty list of senders for which peer explicitly
// subscribed to with specific resolution.
std::vector<std::string> GetSubscribedPeers() const;
std::string ToString() const;
private:
absl::optional<VideoResolution> default_resolution_ = absl::nullopt;
std::map<std::string, VideoResolution> peers_resolution_;
};
// Contains configuration for echo emulator.
struct EchoEmulationConfig {
// Delay which represents the echo path delay, i.e. how soon rendered signal
// should reach capturer.
TimeDelta echo_delay = TimeDelta::Millis(50);
};
} // namespace webrtc_pc_e2e
} // namespace webrtc
#endif // API_TEST_PCLF_MEDIA_CONFIGURATION_H_
|