summaryrefslogtreecommitdiffstats
path: root/third_party/libwebrtc/video/encoder_bitrate_adjuster.cc
blob: 8ed16a7565ed9b375792b0ead497da594c991f0b (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
/*
 *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "video/encoder_bitrate_adjuster.h"

#include <algorithm>
#include <memory>
#include <vector>

#include "rtc_base/experiments/rate_control_settings.h"
#include "rtc_base/logging.h"
#include "rtc_base/time_utils.h"

namespace webrtc {
namespace {
// Helper struct with metadata for a single spatial layer.
struct LayerRateInfo {
  double link_utilization_factor = 0.0;
  double media_utilization_factor = 0.0;
  DataRate target_rate = DataRate::Zero();

  DataRate WantedOvershoot() const {
    // If there is headroom, allow bitrate to go up to media rate limit.
    // Still limit media utilization to 1.0, so we don't overshoot over long
    // runs even if we have headroom.
    const double max_media_utilization =
        std::max(1.0, media_utilization_factor);
    if (link_utilization_factor > max_media_utilization) {
      return (link_utilization_factor - max_media_utilization) * target_rate;
    }
    return DataRate::Zero();
  }
};
}  // namespace
constexpr int64_t EncoderBitrateAdjuster::kWindowSizeMs;
constexpr size_t EncoderBitrateAdjuster::kMinFramesSinceLayoutChange;
constexpr double EncoderBitrateAdjuster::kDefaultUtilizationFactor;

EncoderBitrateAdjuster::EncoderBitrateAdjuster(const VideoCodec& codec_settings)
    : utilize_bandwidth_headroom_(RateControlSettings::ParseFromFieldTrials()
                                      .BitrateAdjusterCanUseNetworkHeadroom()),
      frames_since_layout_change_(0),
      min_bitrates_bps_{} {
  if (codec_settings.codecType == VideoCodecType::kVideoCodecVP9) {
    for (size_t si = 0; si < codec_settings.VP9().numberOfSpatialLayers; ++si) {
      if (codec_settings.spatialLayers[si].active) {
        min_bitrates_bps_[si] =
            std::max(codec_settings.minBitrate * 1000,
                     codec_settings.spatialLayers[si].minBitrate * 1000);
      }
    }
  } else {
    for (size_t si = 0; si < codec_settings.numberOfSimulcastStreams; ++si) {
      if (codec_settings.simulcastStream[si].active) {
        min_bitrates_bps_[si] =
            std::max(codec_settings.minBitrate * 1000,
                     codec_settings.simulcastStream[si].minBitrate * 1000);
      }
    }
  }
}

EncoderBitrateAdjuster::~EncoderBitrateAdjuster() = default;

VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation(
    const VideoEncoder::RateControlParameters& rates) {
  current_rate_control_parameters_ = rates;

  // First check that overshoot detectors exist, and store per spatial layer
  // how many active temporal layers we have.
  size_t active_tls_[kMaxSpatialLayers] = {};
  for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
    active_tls_[si] = 0;
    for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
      // Layer is enabled iff it has both positive bitrate and framerate target.
      if (rates.bitrate.GetBitrate(si, ti) > 0 &&
          current_fps_allocation_[si].size() > ti &&
          current_fps_allocation_[si][ti] > 0) {
        ++active_tls_[si];
        if (!overshoot_detectors_[si][ti]) {
          overshoot_detectors_[si][ti] =
              std::make_unique<EncoderOvershootDetector>(kWindowSizeMs);
          frames_since_layout_change_ = 0;
        }
      } else if (overshoot_detectors_[si][ti]) {
        // Layer removed, destroy overshoot detector.
        overshoot_detectors_[si][ti].reset();
        frames_since_layout_change_ = 0;
      }
    }
  }

  // Next poll the overshoot detectors and populate the adjusted allocation.
  const int64_t now_ms = rtc::TimeMillis();
  VideoBitrateAllocation adjusted_allocation;
  std::vector<LayerRateInfo> layer_infos;
  DataRate wanted_overshoot_sum = DataRate::Zero();

  for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
    layer_infos.emplace_back();
    LayerRateInfo& layer_info = layer_infos.back();

    layer_info.target_rate =
        DataRate::BitsPerSec(rates.bitrate.GetSpatialLayerSum(si));

    // Adjustment is done per spatial layer only (not per temporal layer).
    if (frames_since_layout_change_ < kMinFramesSinceLayoutChange) {
      layer_info.link_utilization_factor = kDefaultUtilizationFactor;
      layer_info.media_utilization_factor = kDefaultUtilizationFactor;
    } else if (active_tls_[si] == 0 ||
               layer_info.target_rate == DataRate::Zero()) {
      // No signaled temporal layers, or no bitrate set. Could either be unused
      // spatial layer or bitrate dynamic mode; pass bitrate through without any
      // change.
      layer_info.link_utilization_factor = 1.0;
      layer_info.media_utilization_factor = 1.0;
    } else if (active_tls_[si] == 1) {
      // A single active temporal layer, this might mean single layer or that
      // encoder does not support temporal layers. Merge target bitrates for
      // this spatial layer.
      RTC_DCHECK(overshoot_detectors_[si][0]);
      layer_info.link_utilization_factor =
          overshoot_detectors_[si][0]
              ->GetNetworkRateUtilizationFactor(now_ms)
              .value_or(kDefaultUtilizationFactor);
      layer_info.media_utilization_factor =
          overshoot_detectors_[si][0]
              ->GetMediaRateUtilizationFactor(now_ms)
              .value_or(kDefaultUtilizationFactor);
    } else if (layer_info.target_rate > DataRate::Zero()) {
      // Multiple temporal layers enabled for this spatial layer. Update rate
      // for each of them and make a weighted average of utilization factors,
      // with bitrate fraction used as weight.
      // If any layer is missing a utilization factor, fall back to default.
      layer_info.link_utilization_factor = 0.0;
      layer_info.media_utilization_factor = 0.0;
      for (size_t ti = 0; ti < active_tls_[si]; ++ti) {
        RTC_DCHECK(overshoot_detectors_[si][ti]);
        const absl::optional<double> ti_link_utilization_factor =
            overshoot_detectors_[si][ti]->GetNetworkRateUtilizationFactor(
                now_ms);
        const absl::optional<double> ti_media_utilization_factor =
            overshoot_detectors_[si][ti]->GetMediaRateUtilizationFactor(now_ms);
        if (!ti_link_utilization_factor || !ti_media_utilization_factor) {
          layer_info.link_utilization_factor = kDefaultUtilizationFactor;
          layer_info.media_utilization_factor = kDefaultUtilizationFactor;
          break;
        }
        const double weight =
            static_cast<double>(rates.bitrate.GetBitrate(si, ti)) /
            layer_info.target_rate.bps();
        layer_info.link_utilization_factor +=
            weight * ti_link_utilization_factor.value();
        layer_info.media_utilization_factor +=
            weight * ti_media_utilization_factor.value();
      }
    } else {
      RTC_DCHECK_NOTREACHED();
    }

    if (layer_info.link_utilization_factor < 1.0) {
      // TODO(sprang): Consider checking underuse and allowing it to cancel some
      // potential overuse by other streams.

      // Don't boost target bitrate if encoder is under-using.
      layer_info.link_utilization_factor = 1.0;
    } else {
      // Don't reduce encoder target below 50%, in which case the frame dropper
      // should kick in instead.
      layer_info.link_utilization_factor =
          std::min(layer_info.link_utilization_factor, 2.0);

      // Keep track of sum of desired overshoot bitrate.
      wanted_overshoot_sum += layer_info.WantedOvershoot();
    }
  }

  // Available link headroom that can be used to fill wanted overshoot.
  DataRate available_headroom = DataRate::Zero();
  if (utilize_bandwidth_headroom_) {
    available_headroom = rates.bandwidth_allocation -
                         DataRate::BitsPerSec(rates.bitrate.get_sum_bps());
  }

  // All wanted overshoots are satisfied in the same proportion based on
  // available headroom.
  const double granted_overshoot_ratio =
      wanted_overshoot_sum == DataRate::Zero()
          ? 0.0
          : std::min(1.0, available_headroom.bps<double>() /
                              wanted_overshoot_sum.bps());

  for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
    LayerRateInfo& layer_info = layer_infos[si];
    double utilization_factor = layer_info.link_utilization_factor;
    DataRate allowed_overshoot =
        granted_overshoot_ratio * layer_info.WantedOvershoot();
    if (allowed_overshoot > DataRate::Zero()) {
      // Pretend the target bitrate is higher by the allowed overshoot.
      // Since utilization_factor = actual_bitrate / target_bitrate, it can be
      // done by multiplying by old_target_bitrate / new_target_bitrate.
      utilization_factor *= layer_info.target_rate.bps<double>() /
                            (allowed_overshoot.bps<double>() +
                             layer_info.target_rate.bps<double>());
    }

    if (min_bitrates_bps_[si] > 0 &&
        layer_info.target_rate > DataRate::Zero() &&
        DataRate::BitsPerSec(min_bitrates_bps_[si]) < layer_info.target_rate) {
      // Make sure rate adjuster doesn't push target bitrate below minimum.
      utilization_factor =
          std::min(utilization_factor, layer_info.target_rate.bps<double>() /
                                           min_bitrates_bps_[si]);
    }

    if (layer_info.target_rate > DataRate::Zero()) {
      RTC_LOG(LS_VERBOSE) << "Utilization factors for spatial index " << si
                          << ": link = " << layer_info.link_utilization_factor
                          << ", media = " << layer_info.media_utilization_factor
                          << ", wanted overshoot = "
                          << layer_info.WantedOvershoot().bps()
                          << " bps, available headroom = "
                          << available_headroom.bps()
                          << " bps, total utilization factor = "
                          << utilization_factor;
    }

    // Populate the adjusted allocation with determined utilization factor.
    if (active_tls_[si] == 1 &&
        layer_info.target_rate >
            DataRate::BitsPerSec(rates.bitrate.GetBitrate(si, 0))) {
      // Bitrate allocation indicates temporal layer usage, but encoder
      // does not seem to support it. Pipe all bitrate into a single
      // overshoot detector.
      uint32_t adjusted_layer_bitrate_bps =
          std::min(static_cast<uint32_t>(
                       layer_info.target_rate.bps() / utilization_factor + 0.5),
                   layer_info.target_rate.bps<uint32_t>());
      adjusted_allocation.SetBitrate(si, 0, adjusted_layer_bitrate_bps);
    } else {
      for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
        if (rates.bitrate.HasBitrate(si, ti)) {
          uint32_t adjusted_layer_bitrate_bps = std::min(
              static_cast<uint32_t>(
                  rates.bitrate.GetBitrate(si, ti) / utilization_factor + 0.5),
              rates.bitrate.GetBitrate(si, ti));
          adjusted_allocation.SetBitrate(si, ti, adjusted_layer_bitrate_bps);
        }
      }
    }

    // In case of rounding errors, add bitrate to TL0 until min bitrate
    // constraint has been met.
    const uint32_t adjusted_spatial_layer_sum =
        adjusted_allocation.GetSpatialLayerSum(si);
    if (layer_info.target_rate > DataRate::Zero() &&
        adjusted_spatial_layer_sum < min_bitrates_bps_[si]) {
      adjusted_allocation.SetBitrate(si, 0,
                                     adjusted_allocation.GetBitrate(si, 0) +
                                         min_bitrates_bps_[si] -
                                         adjusted_spatial_layer_sum);
    }

    // Update all detectors with the new adjusted bitrate targets.
    for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
      const uint32_t layer_bitrate_bps = adjusted_allocation.GetBitrate(si, ti);
      // Overshoot detector may not exist, eg for ScreenshareLayers case.
      if (layer_bitrate_bps > 0 && overshoot_detectors_[si][ti]) {
        // Number of frames in this layer alone is not cumulative, so
        // subtract fps from any low temporal layer.
        const double fps_fraction =
            static_cast<double>(
                current_fps_allocation_[si][ti] -
                (ti == 0 ? 0 : current_fps_allocation_[si][ti - 1])) /
            VideoEncoder::EncoderInfo::kMaxFramerateFraction;

        if (fps_fraction <= 0.0) {
          RTC_LOG(LS_WARNING)
              << "Encoder config has temporal layer with non-zero bitrate "
                 "allocation but zero framerate allocation.";
          continue;
        }

        overshoot_detectors_[si][ti]->SetTargetRate(
            DataRate::BitsPerSec(layer_bitrate_bps),
            fps_fraction * rates.framerate_fps, now_ms);
      }
    }
  }

  // Since no spatial layers or streams are toggled by the adjustment
  // bw-limited flag stays the same.
  adjusted_allocation.set_bw_limited(rates.bitrate.is_bw_limited());

  return adjusted_allocation;
}

void EncoderBitrateAdjuster::OnEncoderInfo(
    const VideoEncoder::EncoderInfo& encoder_info) {
  // Copy allocation into current state and re-allocate.
  for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
    current_fps_allocation_[si] = encoder_info.fps_allocation[si];
  }

  // Trigger re-allocation so that overshoot detectors have correct targets.
  AdjustRateAllocation(current_rate_control_parameters_);
}

void EncoderBitrateAdjuster::OnEncodedFrame(DataSize size,
                                            int spatial_index,
                                            int temporal_index) {
  ++frames_since_layout_change_;
  // Detectors may not exist, for instance if ScreenshareLayers is used.
  auto& detector = overshoot_detectors_[spatial_index][temporal_index];
  if (detector) {
    detector->OnEncodedFrame(size.bytes(), rtc::TimeMillis());
  }
}

void EncoderBitrateAdjuster::Reset() {
  for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
    for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
      overshoot_detectors_[si][ti].reset();
    }
  }
  // Call AdjustRateAllocation() with the last know bitrate allocation, so that
  // the appropriate overuse detectors are immediately re-created.
  AdjustRateAllocation(current_rate_control_parameters_);
}

}  // namespace webrtc