1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
|
/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "api/video_codecs/video_encoder_software_fallback_wrapper.h"
#include <stdint.h>
#include <cstdio>
#include <memory>
#include <string>
#include <vector>
#include "absl/strings/match.h"
#include "absl/types/optional.h"
#include "api/fec_controller_override.h"
#include "api/transport/field_trial_based_config.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_bitrate_allocation.h"
#include "api/video/video_frame.h"
#include "api/video_codecs/video_codec.h"
#include "api/video_codecs/video_encoder.h"
#include "media/base/video_common.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "modules/video_coding/utility/simulcast_utility.h"
#include "rtc_base/checks.h"
#include "rtc_base/experiments/field_trial_parser.h"
#include "rtc_base/logging.h"
#include "system_wrappers/include/field_trial.h"
namespace webrtc {
namespace {
// If forced fallback is allowed, either:
//
// 1) The forced fallback is requested if the resolution is less than or equal
// to `max_pixels_`. The resolution is allowed to be scaled down to
// `min_pixels_`.
//
// 2) The forced fallback is requested if temporal support is preferred and the
// SW fallback supports temporal layers while the HW encoder does not.
struct ForcedFallbackParams {
public:
bool SupportsResolutionBasedSwitch(const VideoCodec& codec) const {
if (!enable_resolution_based_switch ||
codec.width * codec.height > max_pixels) {
return false;
}
if (vp8_specific_resolution_switch &&
(codec.codecType != kVideoCodecVP8 ||
codec.numberOfSimulcastStreams > 1)) {
return false;
}
return true;
}
bool SupportsTemporalBasedSwitch(const VideoCodec& codec) const {
return enable_temporal_based_switch &&
SimulcastUtility::NumberOfTemporalLayers(codec, 0) != 1;
}
bool enable_temporal_based_switch = false;
bool enable_resolution_based_switch = false;
bool vp8_specific_resolution_switch = false;
int min_pixels = kDefaultMinPixelsPerFrame;
int max_pixels = 320 * 240;
};
const char kVp8ForceFallbackEncoderFieldTrial[] =
"WebRTC-VP8-Forced-Fallback-Encoder-v2";
absl::optional<ForcedFallbackParams> ParseFallbackParamsFromFieldTrials(
const VideoEncoder& main_encoder) {
// Ignore WebRTC-VP8-Forced-Fallback-Encoder-v2 if
// WebRTC-Video-EncoderFallbackSettings is present.
FieldTrialOptional<int> resolution_threshold_px("resolution_threshold_px");
ParseFieldTrial(
{&resolution_threshold_px},
FieldTrialBasedConfig().Lookup("WebRTC-Video-EncoderFallbackSettings"));
if (resolution_threshold_px) {
ForcedFallbackParams params;
params.enable_resolution_based_switch = true;
params.max_pixels = resolution_threshold_px.Value();
return params;
}
const std::string field_trial =
webrtc::field_trial::FindFullName(kVp8ForceFallbackEncoderFieldTrial);
if (!absl::StartsWith(field_trial, "Enabled")) {
return absl::nullopt;
}
int max_pixels_lower_bound =
main_encoder.GetEncoderInfo().scaling_settings.min_pixels_per_frame - 1;
ForcedFallbackParams params;
params.enable_resolution_based_switch = true;
int min_bps = 0;
if (sscanf(field_trial.c_str(), "Enabled-%d,%d,%d", ¶ms.min_pixels,
¶ms.max_pixels, &min_bps) != 3) {
RTC_LOG(LS_WARNING)
<< "Invalid number of forced fallback parameters provided.";
return absl::nullopt;
} else if (params.min_pixels <= 0 ||
params.max_pixels < max_pixels_lower_bound ||
params.max_pixels < params.min_pixels || min_bps <= 0) {
RTC_LOG(LS_WARNING) << "Invalid forced fallback parameter value provided.";
return absl::nullopt;
}
params.vp8_specific_resolution_switch = true;
return params;
}
absl::optional<ForcedFallbackParams> GetForcedFallbackParams(
bool prefer_temporal_support,
const VideoEncoder& main_encoder) {
absl::optional<ForcedFallbackParams> params =
ParseFallbackParamsFromFieldTrials(main_encoder);
if (prefer_temporal_support) {
if (!params.has_value()) {
params.emplace();
}
params->enable_temporal_based_switch = true;
}
return params;
}
class VideoEncoderSoftwareFallbackWrapper final : public VideoEncoder {
public:
VideoEncoderSoftwareFallbackWrapper(
std::unique_ptr<webrtc::VideoEncoder> sw_encoder,
std::unique_ptr<webrtc::VideoEncoder> hw_encoder,
bool prefer_temporal_support);
~VideoEncoderSoftwareFallbackWrapper() override;
void SetFecControllerOverride(
FecControllerOverride* fec_controller_override) override;
int32_t InitEncode(const VideoCodec* codec_settings,
const VideoEncoder::Settings& settings) override;
int32_t RegisterEncodeCompleteCallback(
EncodedImageCallback* callback) override;
int32_t Release() override;
int32_t Encode(const VideoFrame& frame,
const std::vector<VideoFrameType>* frame_types) override;
void OnPacketLossRateUpdate(float packet_loss_rate) override;
void OnRttUpdate(int64_t rtt_ms) override;
void OnLossNotification(const LossNotification& loss_notification) override;
void SetRates(const RateControlParameters& parameters) override;
EncoderInfo GetEncoderInfo() const override;
private:
bool InitFallbackEncoder(bool is_forced);
bool TryInitForcedFallbackEncoder();
bool IsFallbackActive() const;
VideoEncoder* current_encoder() {
switch (encoder_state_) {
case EncoderState::kUninitialized:
RTC_LOG(LS_WARNING)
<< "Trying to access encoder in uninitialized fallback wrapper.";
// Return main encoder to preserve previous behavior.
[[fallthrough]];
case EncoderState::kMainEncoderUsed:
return encoder_.get();
case EncoderState::kFallbackDueToFailure:
case EncoderState::kForcedFallback:
return fallback_encoder_.get();
}
RTC_CHECK_NOTREACHED();
}
// Updates encoder with last observed parameters, such as callbacks, rates,
// etc.
void PrimeEncoder(VideoEncoder* encoder) const;
// Settings used in the last InitEncode call and used if a dynamic fallback to
// software is required.
VideoCodec codec_settings_;
absl::optional<VideoEncoder::Settings> encoder_settings_;
// The last rate control settings, if set.
absl::optional<RateControlParameters> rate_control_parameters_;
// The last channel parameters set.
absl::optional<float> packet_loss_;
absl::optional<int64_t> rtt_;
absl::optional<LossNotification> loss_notification_;
enum class EncoderState {
kUninitialized,
kMainEncoderUsed,
kFallbackDueToFailure,
kForcedFallback
};
EncoderState encoder_state_;
const std::unique_ptr<webrtc::VideoEncoder> encoder_;
const std::unique_ptr<webrtc::VideoEncoder> fallback_encoder_;
EncodedImageCallback* callback_;
const absl::optional<ForcedFallbackParams> fallback_params_;
int32_t EncodeWithMainEncoder(const VideoFrame& frame,
const std::vector<VideoFrameType>* frame_types);
};
VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper(
std::unique_ptr<webrtc::VideoEncoder> sw_encoder,
std::unique_ptr<webrtc::VideoEncoder> hw_encoder,
bool prefer_temporal_support)
: encoder_state_(EncoderState::kUninitialized),
encoder_(std::move(hw_encoder)),
fallback_encoder_(std::move(sw_encoder)),
callback_(nullptr),
fallback_params_(
GetForcedFallbackParams(prefer_temporal_support, *encoder_)) {
RTC_DCHECK(fallback_encoder_);
}
VideoEncoderSoftwareFallbackWrapper::~VideoEncoderSoftwareFallbackWrapper() =
default;
void VideoEncoderSoftwareFallbackWrapper::PrimeEncoder(
VideoEncoder* encoder) const {
RTC_DCHECK(encoder);
// Replay callback, rates, and channel parameters.
if (callback_) {
encoder->RegisterEncodeCompleteCallback(callback_);
}
if (rate_control_parameters_) {
encoder->SetRates(*rate_control_parameters_);
}
if (rtt_.has_value()) {
encoder->OnRttUpdate(rtt_.value());
}
if (packet_loss_.has_value()) {
encoder->OnPacketLossRateUpdate(packet_loss_.value());
}
if (loss_notification_.has_value()) {
encoder->OnLossNotification(loss_notification_.value());
}
}
bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder(bool is_forced) {
RTC_LOG(LS_WARNING) << "Encoder falling back to software encoding.";
RTC_DCHECK(encoder_settings_.has_value());
const int ret = fallback_encoder_->InitEncode(&codec_settings_,
encoder_settings_.value());
if (ret != WEBRTC_VIDEO_CODEC_OK) {
RTC_LOG(LS_ERROR) << "Failed to initialize software-encoder fallback.";
fallback_encoder_->Release();
return false;
}
if (encoder_state_ == EncoderState::kMainEncoderUsed) {
// Since we're switching to the fallback encoder, Release the real encoder.
// It may be re-initialized via InitEncode later, and it will continue to
// get Set calls for rates and channel parameters in the meantime.
encoder_->Release();
}
if (is_forced) {
encoder_state_ = EncoderState::kForcedFallback;
} else {
encoder_state_ = EncoderState::kFallbackDueToFailure;
}
return true;
}
void VideoEncoderSoftwareFallbackWrapper::SetFecControllerOverride(
FecControllerOverride* fec_controller_override) {
// It is important that only one of those would ever interact with the
// `fec_controller_override` at a given time. This is the responsibility
// of `this` to maintain.
encoder_->SetFecControllerOverride(fec_controller_override);
fallback_encoder_->SetFecControllerOverride(fec_controller_override);
}
int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode(
const VideoCodec* codec_settings,
const VideoEncoder::Settings& settings) {
// Store settings, in case we need to dynamically switch to the fallback
// encoder after a failed Encode call.
codec_settings_ = *codec_settings;
encoder_settings_ = settings;
// Clear stored rate/channel parameters.
rate_control_parameters_ = absl::nullopt;
RTC_DCHECK_EQ(encoder_state_, EncoderState::kUninitialized)
<< "InitEncode() should never be called on an active instance!";
// Try to init forced software codec if it should be used.
if (TryInitForcedFallbackEncoder()) {
PrimeEncoder(current_encoder());
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t ret = encoder_->InitEncode(codec_settings, settings);
if (ret == WEBRTC_VIDEO_CODEC_OK) {
encoder_state_ = EncoderState::kMainEncoderUsed;
PrimeEncoder(current_encoder());
return ret;
}
// Try to instantiate software codec.
if (InitFallbackEncoder(/*is_forced=*/false)) {
PrimeEncoder(current_encoder());
return WEBRTC_VIDEO_CODEC_OK;
}
// Software encoder failed too, use original return code.
encoder_state_ = EncoderState::kUninitialized;
return ret;
}
int32_t VideoEncoderSoftwareFallbackWrapper::RegisterEncodeCompleteCallback(
EncodedImageCallback* callback) {
callback_ = callback;
return current_encoder()->RegisterEncodeCompleteCallback(callback);
}
int32_t VideoEncoderSoftwareFallbackWrapper::Release() {
if (encoder_state_ == EncoderState::kUninitialized) {
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t ret = current_encoder()->Release();
encoder_state_ = EncoderState::kUninitialized;
return ret;
}
int32_t VideoEncoderSoftwareFallbackWrapper::Encode(
const VideoFrame& frame,
const std::vector<VideoFrameType>* frame_types) {
switch (encoder_state_) {
case EncoderState::kUninitialized:
return WEBRTC_VIDEO_CODEC_ERROR;
case EncoderState::kMainEncoderUsed: {
return EncodeWithMainEncoder(frame, frame_types);
}
case EncoderState::kFallbackDueToFailure:
case EncoderState::kForcedFallback:
return fallback_encoder_->Encode(frame, frame_types);
}
RTC_CHECK_NOTREACHED();
}
int32_t VideoEncoderSoftwareFallbackWrapper::EncodeWithMainEncoder(
const VideoFrame& frame,
const std::vector<VideoFrameType>* frame_types) {
int32_t ret = encoder_->Encode(frame, frame_types);
// If requested, try a software fallback.
bool fallback_requested = (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE);
if (fallback_requested && InitFallbackEncoder(/*is_forced=*/false)) {
// Start using the fallback with this frame.
PrimeEncoder(current_encoder());
if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative &&
fallback_encoder_->GetEncoderInfo().supports_native_handle) {
return fallback_encoder_->Encode(frame, frame_types);
} else {
RTC_LOG(LS_INFO) << "Fallback encoder does not support native handle - "
"converting frame to I420";
rtc::scoped_refptr<I420BufferInterface> src_buffer =
frame.video_frame_buffer()->ToI420();
if (!src_buffer) {
RTC_LOG(LS_ERROR) << "Failed to convert from to I420";
return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
}
rtc::scoped_refptr<VideoFrameBuffer> dst_buffer =
src_buffer->Scale(codec_settings_.width, codec_settings_.height);
if (!dst_buffer) {
RTC_LOG(LS_ERROR) << "Failed to scale video frame.";
return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
}
VideoFrame scaled_frame = frame;
scaled_frame.set_video_frame_buffer(dst_buffer);
scaled_frame.set_update_rect(VideoFrame::UpdateRect{
0, 0, scaled_frame.width(), scaled_frame.height()});
return fallback_encoder_->Encode(scaled_frame, frame_types);
}
}
// Fallback encoder failed too, return original error code.
return ret;
}
void VideoEncoderSoftwareFallbackWrapper::SetRates(
const RateControlParameters& parameters) {
rate_control_parameters_ = parameters;
return current_encoder()->SetRates(parameters);
}
void VideoEncoderSoftwareFallbackWrapper::OnPacketLossRateUpdate(
float packet_loss_rate) {
packet_loss_ = packet_loss_rate;
current_encoder()->OnPacketLossRateUpdate(packet_loss_rate);
}
void VideoEncoderSoftwareFallbackWrapper::OnRttUpdate(int64_t rtt_ms) {
rtt_ = rtt_ms;
current_encoder()->OnRttUpdate(rtt_ms);
}
void VideoEncoderSoftwareFallbackWrapper::OnLossNotification(
const LossNotification& loss_notification) {
loss_notification_ = loss_notification;
current_encoder()->OnLossNotification(loss_notification);
}
VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo()
const {
EncoderInfo fallback_encoder_info = fallback_encoder_->GetEncoderInfo();
EncoderInfo default_encoder_info = encoder_->GetEncoderInfo();
EncoderInfo info =
IsFallbackActive() ? fallback_encoder_info : default_encoder_info;
info.requested_resolution_alignment = cricket::LeastCommonMultiple(
fallback_encoder_info.requested_resolution_alignment,
default_encoder_info.requested_resolution_alignment);
info.apply_alignment_to_all_simulcast_layers =
fallback_encoder_info.apply_alignment_to_all_simulcast_layers ||
default_encoder_info.apply_alignment_to_all_simulcast_layers;
if (fallback_params_ && fallback_params_->vp8_specific_resolution_switch) {
info.scaling_settings.min_pixels_per_frame = fallback_params_->min_pixels;
}
return info;
}
bool VideoEncoderSoftwareFallbackWrapper::IsFallbackActive() const {
return encoder_state_ == EncoderState::kForcedFallback ||
encoder_state_ == EncoderState::kFallbackDueToFailure;
}
bool VideoEncoderSoftwareFallbackWrapper::TryInitForcedFallbackEncoder() {
if (!fallback_params_) {
return false;
}
RTC_DCHECK_EQ(encoder_state_, EncoderState::kUninitialized);
if (fallback_params_->SupportsResolutionBasedSwitch(codec_settings_)) {
// Settings valid, try to instantiate software codec.
RTC_LOG(LS_INFO) << "Request forced SW encoder fallback: "
<< codec_settings_.width << "x" << codec_settings_.height;
return InitFallbackEncoder(/*is_forced=*/true);
}
if (fallback_params_->SupportsTemporalBasedSwitch(codec_settings_)) {
// First init main encoder to see if that supports temporal layers.
if (encoder_->InitEncode(&codec_settings_, encoder_settings_.value()) ==
WEBRTC_VIDEO_CODEC_OK) {
encoder_state_ = EncoderState::kMainEncoderUsed;
}
if (encoder_state_ == EncoderState::kMainEncoderUsed &&
encoder_->GetEncoderInfo().fps_allocation[0].size() != 1) {
// Primary encoder already supports temporal layers, use that instead.
return true;
}
// Try to initialize fallback and check if it supports temporal layers.
if (fallback_encoder_->InitEncode(&codec_settings_,
encoder_settings_.value()) ==
WEBRTC_VIDEO_CODEC_OK) {
if (fallback_encoder_->GetEncoderInfo().fps_allocation[0].size() != 1) {
// Fallback encoder available and supports temporal layers, use it!
if (encoder_state_ == EncoderState::kMainEncoderUsed) {
// Main encoder initialized but does not support temporal layers,
// release it again.
encoder_->Release();
}
encoder_state_ = EncoderState::kForcedFallback;
RTC_LOG(LS_INFO)
<< "Forced switch to SW encoder due to temporal support.";
return true;
} else {
// Fallback encoder intialization succeeded, but it does not support
// temporal layers either - release it.
fallback_encoder_->Release();
}
}
if (encoder_state_ == EncoderState::kMainEncoderUsed) {
// Main encoder already initialized - make use of it.
RTC_LOG(LS_INFO)
<< "Cannot fall back for temporal support since fallback that "
"supports is not available. Using main encoder instead.";
return true;
}
}
// Neither forced fallback mode supported.
return false;
}
} // namespace
std::unique_ptr<VideoEncoder> CreateVideoEncoderSoftwareFallbackWrapper(
std::unique_ptr<VideoEncoder> sw_fallback_encoder,
std::unique_ptr<VideoEncoder> hw_encoder,
bool prefer_temporal_support) {
return std::make_unique<VideoEncoderSoftwareFallbackWrapper>(
std::move(sw_fallback_encoder), std::move(hw_encoder),
prefer_temporal_support);
}
} // namespace webrtc
|