1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
|
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/svc/svc_rate_allocator.h"
#include <algorithm>
#include <cmath>
#include <cstddef>
#include <numeric>
#include <vector>
#include "absl/container/inlined_vector.h"
#include "modules/video_coding/svc/create_scalability_structure.h"
#include "rtc_base/checks.h"
namespace webrtc {
namespace {
constexpr float kSpatialLayeringRateScalingFactor = 0.55f;
constexpr float kTemporalLayeringRateScalingFactor = 0.55f;
struct ActiveSpatialLayers {
size_t first = 0;
size_t num = 0;
};
ActiveSpatialLayers GetActiveSpatialLayers(const VideoCodec& codec,
size_t num_spatial_layers) {
ActiveSpatialLayers active;
for (active.first = 0; active.first < num_spatial_layers; ++active.first) {
if (codec.spatialLayers[active.first].active) {
break;
}
}
size_t last_active_layer = active.first;
for (; last_active_layer < num_spatial_layers; ++last_active_layer) {
if (!codec.spatialLayers[last_active_layer].active) {
break;
}
}
active.num = last_active_layer - active.first;
return active;
}
std::vector<DataRate> AdjustAndVerify(
const VideoCodec& codec,
size_t first_active_layer,
const std::vector<DataRate>& spatial_layer_rates) {
std::vector<DataRate> adjusted_spatial_layer_rates;
// Keep track of rate that couldn't be applied to the previous layer due to
// max bitrate constraint, try to pass it forward to the next one.
DataRate excess_rate = DataRate::Zero();
for (size_t sl_idx = 0; sl_idx < spatial_layer_rates.size(); ++sl_idx) {
DataRate min_rate = DataRate::KilobitsPerSec(
codec.spatialLayers[first_active_layer + sl_idx].minBitrate);
DataRate max_rate = DataRate::KilobitsPerSec(
codec.spatialLayers[first_active_layer + sl_idx].maxBitrate);
DataRate layer_rate = spatial_layer_rates[sl_idx] + excess_rate;
if (layer_rate < min_rate) {
// Not enough rate to reach min bitrate for desired number of layers,
// abort allocation.
if (spatial_layer_rates.size() == 1) {
return spatial_layer_rates;
}
return adjusted_spatial_layer_rates;
}
if (layer_rate <= max_rate) {
excess_rate = DataRate::Zero();
adjusted_spatial_layer_rates.push_back(layer_rate);
} else {
excess_rate = layer_rate - max_rate;
adjusted_spatial_layer_rates.push_back(max_rate);
}
}
return adjusted_spatial_layer_rates;
}
static std::vector<DataRate> SplitBitrate(size_t num_layers,
DataRate total_bitrate,
float rate_scaling_factor) {
std::vector<DataRate> bitrates;
double denominator = 0.0;
for (size_t layer_idx = 0; layer_idx < num_layers; ++layer_idx) {
denominator += std::pow(rate_scaling_factor, layer_idx);
}
double numerator = std::pow(rate_scaling_factor, num_layers - 1);
for (size_t layer_idx = 0; layer_idx < num_layers; ++layer_idx) {
bitrates.push_back(numerator * total_bitrate / denominator);
numerator /= rate_scaling_factor;
}
const DataRate sum =
std::accumulate(bitrates.begin(), bitrates.end(), DataRate::Zero());
// Keep the sum of split bitrates equal to the total bitrate by adding or
// subtracting bits, which were lost due to rounding, to the latest layer.
if (total_bitrate > sum) {
bitrates.back() += total_bitrate - sum;
} else if (total_bitrate < sum) {
bitrates.back() -= sum - total_bitrate;
}
return bitrates;
}
// Returns the minimum bitrate needed for `num_active_layers` spatial layers to
// become active using the configuration specified by `codec`.
DataRate FindLayerTogglingThreshold(const VideoCodec& codec,
size_t first_active_layer,
size_t num_active_layers) {
if (num_active_layers == 1) {
return DataRate::KilobitsPerSec(codec.spatialLayers[0].minBitrate);
}
if (codec.mode == VideoCodecMode::kRealtimeVideo) {
DataRate lower_bound = DataRate::Zero();
DataRate upper_bound = DataRate::Zero();
if (num_active_layers > 1) {
for (size_t i = 0; i < num_active_layers - 1; ++i) {
lower_bound += DataRate::KilobitsPerSec(
codec.spatialLayers[first_active_layer + i].minBitrate);
upper_bound += DataRate::KilobitsPerSec(
codec.spatialLayers[first_active_layer + i].maxBitrate);
}
}
upper_bound += DataRate::KilobitsPerSec(
codec.spatialLayers[first_active_layer + num_active_layers - 1]
.minBitrate);
// Do a binary search until upper and lower bound is the highest bitrate for
// `num_active_layers` - 1 layers and lowest bitrate for `num_active_layers`
// layers respectively.
while (upper_bound - lower_bound > DataRate::BitsPerSec(1)) {
DataRate try_rate = (lower_bound + upper_bound) / 2;
if (AdjustAndVerify(codec, first_active_layer,
SplitBitrate(num_active_layers, try_rate,
kSpatialLayeringRateScalingFactor))
.size() == num_active_layers) {
upper_bound = try_rate;
} else {
lower_bound = try_rate;
}
}
return upper_bound;
} else {
DataRate toggling_rate = DataRate::Zero();
for (size_t i = 0; i < num_active_layers - 1; ++i) {
toggling_rate += DataRate::KilobitsPerSec(
codec.spatialLayers[first_active_layer + i].targetBitrate);
}
toggling_rate += DataRate::KilobitsPerSec(
codec.spatialLayers[first_active_layer + num_active_layers - 1]
.minBitrate);
return toggling_rate;
}
}
} // namespace
SvcRateAllocator::NumLayers SvcRateAllocator::GetNumLayers(
const VideoCodec& codec) {
NumLayers layers;
if (absl::optional<ScalabilityMode> scalability_mode =
codec.GetScalabilityMode();
scalability_mode.has_value()) {
if (auto structure = CreateScalabilityStructure(*scalability_mode)) {
ScalableVideoController::StreamLayersConfig config =
structure->StreamConfig();
layers.spatial = config.num_spatial_layers;
layers.temporal = config.num_temporal_layers;
return layers;
}
}
if (codec.codecType == kVideoCodecVP9) {
layers.spatial = codec.VP9().numberOfSpatialLayers;
layers.temporal = codec.VP9().numberOfTemporalLayers;
return layers;
}
layers.spatial = 1;
layers.temporal = 1;
return layers;
}
SvcRateAllocator::SvcRateAllocator(const VideoCodec& codec)
: codec_(codec),
num_layers_(GetNumLayers(codec)),
experiment_settings_(StableTargetRateExperiment::ParseFromFieldTrials()),
cumulative_layer_start_bitrates_(GetLayerStartBitrates(codec)),
last_active_layer_count_(0) {
RTC_DCHECK_GT(num_layers_.spatial, 0);
RTC_DCHECK_LE(num_layers_.spatial, kMaxSpatialLayers);
RTC_DCHECK_GT(num_layers_.temporal, 0);
RTC_DCHECK_LE(num_layers_.temporal, 3);
for (size_t layer_idx = 0; layer_idx < num_layers_.spatial; ++layer_idx) {
// Verify min <= target <= max.
if (codec.spatialLayers[layer_idx].active) {
RTC_DCHECK_GT(codec.spatialLayers[layer_idx].maxBitrate, 0);
RTC_DCHECK_GE(codec.spatialLayers[layer_idx].maxBitrate,
codec.spatialLayers[layer_idx].minBitrate);
RTC_DCHECK_GE(codec.spatialLayers[layer_idx].targetBitrate,
codec.spatialLayers[layer_idx].minBitrate);
RTC_DCHECK_GE(codec.spatialLayers[layer_idx].maxBitrate,
codec.spatialLayers[layer_idx].targetBitrate);
}
}
}
VideoBitrateAllocation SvcRateAllocator::Allocate(
VideoBitrateAllocationParameters parameters) {
DataRate total_bitrate = parameters.total_bitrate;
if (codec_.maxBitrate != 0) {
total_bitrate =
std::min(total_bitrate, DataRate::KilobitsPerSec(codec_.maxBitrate));
}
if (codec_.spatialLayers[0].targetBitrate == 0) {
// Delegate rate distribution to encoder wrapper if bitrate thresholds
// are not set.
VideoBitrateAllocation bitrate_allocation;
bitrate_allocation.SetBitrate(0, 0, total_bitrate.bps());
return bitrate_allocation;
}
const ActiveSpatialLayers active_layers =
GetActiveSpatialLayers(codec_, num_layers_.spatial);
size_t num_spatial_layers = active_layers.num;
if (num_spatial_layers == 0) {
return VideoBitrateAllocation(); // All layers are deactivated.
}
// Figure out how many spatial layers should be active.
if (experiment_settings_.IsEnabled() &&
parameters.stable_bitrate > DataRate::Zero()) {
double hysteresis_factor;
if (codec_.mode == VideoCodecMode::kScreensharing) {
hysteresis_factor = experiment_settings_.GetScreenshareHysteresisFactor();
} else {
hysteresis_factor = experiment_settings_.GetVideoHysteresisFactor();
}
DataRate stable_rate = std::min(total_bitrate, parameters.stable_bitrate);
// First check if bitrate has grown large enough to enable new layers.
size_t num_enabled_with_hysteresis =
FindNumEnabledLayers(stable_rate / hysteresis_factor);
if (num_enabled_with_hysteresis >= last_active_layer_count_) {
num_spatial_layers = num_enabled_with_hysteresis;
} else {
// We could not enable new layers, check if any should be disabled.
num_spatial_layers =
std::min(last_active_layer_count_, FindNumEnabledLayers(stable_rate));
}
} else {
num_spatial_layers = FindNumEnabledLayers(total_bitrate);
}
last_active_layer_count_ = num_spatial_layers;
VideoBitrateAllocation allocation;
if (codec_.mode == VideoCodecMode::kRealtimeVideo) {
allocation = GetAllocationNormalVideo(total_bitrate, active_layers.first,
num_spatial_layers);
} else {
allocation = GetAllocationScreenSharing(total_bitrate, active_layers.first,
num_spatial_layers);
}
allocation.set_bw_limited(num_spatial_layers < active_layers.num);
return allocation;
}
VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo(
DataRate total_bitrate,
size_t first_active_layer,
size_t num_spatial_layers) const {
std::vector<DataRate> spatial_layer_rates;
if (num_spatial_layers == 0) {
// Not enough rate for even the base layer. Force allocation at the total
// bitrate anyway.
num_spatial_layers = 1;
spatial_layer_rates.push_back(total_bitrate);
} else {
spatial_layer_rates =
AdjustAndVerify(codec_, first_active_layer,
SplitBitrate(num_spatial_layers, total_bitrate,
kSpatialLayeringRateScalingFactor));
RTC_DCHECK_EQ(spatial_layer_rates.size(), num_spatial_layers);
}
VideoBitrateAllocation bitrate_allocation;
for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
std::vector<DataRate> temporal_layer_rates =
SplitBitrate(num_layers_.temporal, spatial_layer_rates[sl_idx],
kTemporalLayeringRateScalingFactor);
// Distribute rate across temporal layers. Allocate more bits to lower
// layers since they are used for prediction of higher layers and their
// references are far apart.
if (num_layers_.temporal == 1) {
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
temporal_layer_rates[0].bps());
} else if (num_layers_.temporal == 2) {
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
temporal_layer_rates[1].bps());
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1,
temporal_layer_rates[0].bps());
} else {
RTC_CHECK_EQ(num_layers_.temporal, 3);
// In case of three temporal layers the high layer has two frames and the
// middle layer has one frame within GOP (in between two consecutive low
// layer frames). Thus high layer requires more bits (comparing pure
// bitrate of layer, excluding bitrate of base layers) to keep quality on
// par with lower layers.
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
temporal_layer_rates[2].bps());
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1,
temporal_layer_rates[0].bps());
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 2,
temporal_layer_rates[1].bps());
}
}
return bitrate_allocation;
}
// Bit-rate is allocated in such a way, that the highest enabled layer will have
// between min and max bitrate, and all others will have exactly target
// bit-rate allocated.
VideoBitrateAllocation SvcRateAllocator::GetAllocationScreenSharing(
DataRate total_bitrate,
size_t first_active_layer,
size_t num_spatial_layers) const {
VideoBitrateAllocation bitrate_allocation;
if (num_spatial_layers == 0 ||
total_bitrate <
DataRate::KilobitsPerSec(
codec_.spatialLayers[first_active_layer].minBitrate)) {
// Always enable at least one layer.
bitrate_allocation.SetBitrate(first_active_layer, 0, total_bitrate.bps());
return bitrate_allocation;
}
DataRate allocated_rate = DataRate::Zero();
DataRate top_layer_rate = DataRate::Zero();
size_t sl_idx;
for (sl_idx = first_active_layer;
sl_idx < first_active_layer + num_spatial_layers; ++sl_idx) {
const DataRate min_rate =
DataRate::KilobitsPerSec(codec_.spatialLayers[sl_idx].minBitrate);
const DataRate target_rate =
DataRate::KilobitsPerSec(codec_.spatialLayers[sl_idx].targetBitrate);
if (allocated_rate + min_rate > total_bitrate) {
// Use stable rate to determine if layer should be enabled.
break;
}
top_layer_rate = std::min(target_rate, total_bitrate - allocated_rate);
bitrate_allocation.SetBitrate(sl_idx, 0, top_layer_rate.bps());
allocated_rate += top_layer_rate;
}
if (sl_idx > 0 && total_bitrate - allocated_rate > DataRate::Zero()) {
// Add leftover to the last allocated layer.
top_layer_rate = std::min(
top_layer_rate + (total_bitrate - allocated_rate),
DataRate::KilobitsPerSec(codec_.spatialLayers[sl_idx - 1].maxBitrate));
bitrate_allocation.SetBitrate(sl_idx - 1, 0, top_layer_rate.bps());
}
return bitrate_allocation;
}
size_t SvcRateAllocator::FindNumEnabledLayers(DataRate target_rate) const {
if (cumulative_layer_start_bitrates_.empty()) {
return 0;
}
size_t num_enabled_layers = 0;
for (DataRate start_rate : cumulative_layer_start_bitrates_) {
// First layer is always enabled.
if (num_enabled_layers == 0 || start_rate <= target_rate) {
++num_enabled_layers;
} else {
break;
}
}
return num_enabled_layers;
}
DataRate SvcRateAllocator::GetMaxBitrate(const VideoCodec& codec) {
const NumLayers num_layers = GetNumLayers(codec);
const ActiveSpatialLayers active_layers =
GetActiveSpatialLayers(codec, num_layers.spatial);
DataRate max_bitrate = DataRate::Zero();
for (size_t sl_idx = 0; sl_idx < active_layers.num; ++sl_idx) {
max_bitrate += DataRate::KilobitsPerSec(
codec.spatialLayers[active_layers.first + sl_idx].maxBitrate);
}
if (codec.maxBitrate != 0) {
max_bitrate =
std::min(max_bitrate, DataRate::KilobitsPerSec(codec.maxBitrate));
}
return max_bitrate;
}
DataRate SvcRateAllocator::GetPaddingBitrate(const VideoCodec& codec) {
auto start_bitrate = GetLayerStartBitrates(codec);
if (start_bitrate.empty()) {
return DataRate::Zero(); // All layers are deactivated.
}
return start_bitrate.back();
}
absl::InlinedVector<DataRate, kMaxSpatialLayers>
SvcRateAllocator::GetLayerStartBitrates(const VideoCodec& codec) {
absl::InlinedVector<DataRate, kMaxSpatialLayers> start_bitrates;
const NumLayers num_layers = GetNumLayers(codec);
const ActiveSpatialLayers active_layers =
GetActiveSpatialLayers(codec, num_layers.spatial);
DataRate last_rate = DataRate::Zero();
for (size_t i = 1; i <= active_layers.num; ++i) {
DataRate layer_toggling_rate =
FindLayerTogglingThreshold(codec, active_layers.first, i);
start_bitrates.push_back(layer_toggling_rate);
RTC_DCHECK_LE(last_rate, layer_toggling_rate);
last_rate = layer_toggling_rate;
}
return start_bitrates;
}
} // namespace webrtc
|