From 6bf0a5cb5034a7e684dcc3500e841785237ce2dd Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 19:32:43 +0200 Subject: Adding upstream version 1:115.7.0. Signed-off-by: Daniel Baumann --- third_party/libwebrtc/video/config/simulcast.cc | 497 ++++++++++++++++++++++++ 1 file changed, 497 insertions(+) create mode 100644 third_party/libwebrtc/video/config/simulcast.cc (limited to 'third_party/libwebrtc/video/config/simulcast.cc') diff --git a/third_party/libwebrtc/video/config/simulcast.cc b/third_party/libwebrtc/video/config/simulcast.cc new file mode 100644 index 0000000000..2bd4ac04c3 --- /dev/null +++ b/third_party/libwebrtc/video/config/simulcast.cc @@ -0,0 +1,497 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/config/simulcast.h" + +#include +#include + +#include +#include +#include + +#include "absl/strings/match.h" +#include "absl/types/optional.h" +#include "api/video/video_codec_constants.h" +#include "media/base/media_constants.h" +#include "modules/video_coding/utility/simulcast_rate_allocator.h" +#include "rtc_base/checks.h" +#include "rtc_base/experiments/field_trial_parser.h" +#include "rtc_base/experiments/min_video_bitrate_experiment.h" +#include "rtc_base/experiments/normalize_simulcast_size_experiment.h" +#include "rtc_base/experiments/rate_control_settings.h" +#include "rtc_base/logging.h" + +namespace cricket { + +namespace { + +constexpr char kUseLegacySimulcastLayerLimitFieldTrial[] = + "WebRTC-LegacySimulcastLayerLimit"; + +constexpr double kDefaultMaxRoundupRate = 0.1; + +// Limits for legacy conference screensharing mode. Currently used for the +// lower of the two simulcast streams. +constexpr webrtc::DataRate kScreenshareDefaultTl0Bitrate = + webrtc::DataRate::KilobitsPerSec(200); +constexpr webrtc::DataRate kScreenshareDefaultTl1Bitrate = + webrtc::DataRate::KilobitsPerSec(1000); + +// Min/max bitrate for the higher one of the two simulcast stream used for +// screen content. +constexpr webrtc::DataRate kScreenshareHighStreamMinBitrate = + webrtc::DataRate::KilobitsPerSec(600); +constexpr webrtc::DataRate kScreenshareHighStreamMaxBitrate = + webrtc::DataRate::KilobitsPerSec(1250); + +constexpr int kDefaultNumTemporalLayers = 3; +constexpr int kScreenshareMaxSimulcastLayers = 2; +constexpr int kScreenshareTemporalLayers = 2; + +struct SimulcastFormat { + int width; + int height; + // The maximum number of simulcast layers can be used for + // resolutions at `widthxheight` for legacy applications. + size_t max_layers; + // The maximum bitrate for encoding stream at `widthxheight`, when we are + // not sending the next higher spatial stream. + webrtc::DataRate max_bitrate; + // The target bitrate for encoding stream at `widthxheight`, when this layer + // is not the highest layer (i.e., when we are sending another higher spatial + // stream). + webrtc::DataRate target_bitrate; + // The minimum bitrate needed for encoding stream at `widthxheight`. + webrtc::DataRate min_bitrate; +}; + +// These tables describe from which resolution we can use how many +// simulcast layers at what bitrates (maximum, target, and minimum). +// Important!! Keep this table from high resolution to low resolution. +constexpr const SimulcastFormat kSimulcastFormats[] = { + {1920, 1080, 3, webrtc::DataRate::KilobitsPerSec(5000), + webrtc::DataRate::KilobitsPerSec(4000), + webrtc::DataRate::KilobitsPerSec(800)}, + {1280, 720, 3, webrtc::DataRate::KilobitsPerSec(2500), + webrtc::DataRate::KilobitsPerSec(2500), + webrtc::DataRate::KilobitsPerSec(600)}, + {960, 540, 3, webrtc::DataRate::KilobitsPerSec(1200), + webrtc::DataRate::KilobitsPerSec(1200), + webrtc::DataRate::KilobitsPerSec(350)}, + {640, 360, 2, webrtc::DataRate::KilobitsPerSec(700), + webrtc::DataRate::KilobitsPerSec(500), + webrtc::DataRate::KilobitsPerSec(150)}, + {480, 270, 2, webrtc::DataRate::KilobitsPerSec(450), + webrtc::DataRate::KilobitsPerSec(350), + webrtc::DataRate::KilobitsPerSec(150)}, + {320, 180, 1, webrtc::DataRate::KilobitsPerSec(200), + webrtc::DataRate::KilobitsPerSec(150), + webrtc::DataRate::KilobitsPerSec(30)}, + // As the resolution goes down, interpolate the target and max bitrates down + // towards zero. The min bitrate is still limited at 30 kbps and the target + // and the max will be capped from below accordingly. + {0, 0, 1, webrtc::DataRate::KilobitsPerSec(0), + webrtc::DataRate::KilobitsPerSec(0), + webrtc::DataRate::KilobitsPerSec(30)}}; + +constexpr webrtc::DataRate Interpolate(const webrtc::DataRate& a, + const webrtc::DataRate& b, + float rate) { + return a * (1.0 - rate) + b * rate; +} + +// TODO(webrtc:12415): Flip this to a kill switch when this feature launches. +bool EnableLowresBitrateInterpolation(const webrtc::FieldTrialsView& trials) { + return absl::StartsWith( + trials.Lookup("WebRTC-LowresSimulcastBitrateInterpolation"), "Enabled"); +} + +std::vector GetSimulcastFormats( + bool enable_lowres_bitrate_interpolation) { + std::vector formats; + formats.insert(formats.begin(), std::begin(kSimulcastFormats), + std::end(kSimulcastFormats)); + if (!enable_lowres_bitrate_interpolation) { + RTC_CHECK_GE(formats.size(), 2u); + SimulcastFormat& format0x0 = formats[formats.size() - 1]; + const SimulcastFormat& format_prev = formats[formats.size() - 2]; + format0x0.max_bitrate = format_prev.max_bitrate; + format0x0.target_bitrate = format_prev.target_bitrate; + format0x0.min_bitrate = format_prev.min_bitrate; + } + return formats; +} + +// Multiway: Number of temporal layers for each simulcast stream. +int DefaultNumberOfTemporalLayers(const webrtc::FieldTrialsView& trials) { + const std::string group_name = + trials.Lookup("WebRTC-VP8ConferenceTemporalLayers"); + if (group_name.empty()) + return kDefaultNumTemporalLayers; + + int num_temporal_layers = kDefaultNumTemporalLayers; + if (sscanf(group_name.c_str(), "%d", &num_temporal_layers) == 1 && + num_temporal_layers > 0 && + num_temporal_layers <= webrtc::kMaxTemporalStreams) { + return num_temporal_layers; + } + + RTC_LOG(LS_WARNING) << "Attempt to set number of temporal layers to " + "incorrect value: " + << group_name; + + return kDefaultNumTemporalLayers; +} + +int FindSimulcastFormatIndex(int width, + int height, + bool enable_lowres_bitrate_interpolation) { + RTC_DCHECK_GE(width, 0); + RTC_DCHECK_GE(height, 0); + const auto formats = GetSimulcastFormats(enable_lowres_bitrate_interpolation); + for (uint32_t i = 0; i < formats.size(); ++i) { + if (width * height >= formats[i].width * formats[i].height) { + return i; + } + } + RTC_DCHECK_NOTREACHED(); + return -1; +} + +} // namespace + +// Round size to nearest simulcast-friendly size. +// Simulcast stream width and height must both be dividable by +// |2 ^ (simulcast_layers - 1)|. +int NormalizeSimulcastSize(int size, size_t simulcast_layers) { + int base2_exponent = static_cast(simulcast_layers) - 1; + const absl::optional experimental_base2_exponent = + webrtc::NormalizeSimulcastSizeExperiment::GetBase2Exponent(); + if (experimental_base2_exponent && + (size > (1 << *experimental_base2_exponent))) { + base2_exponent = *experimental_base2_exponent; + } + return ((size >> base2_exponent) << base2_exponent); +} + +SimulcastFormat InterpolateSimulcastFormat( + int width, + int height, + absl::optional max_roundup_rate, + bool enable_lowres_bitrate_interpolation) { + const auto formats = GetSimulcastFormats(enable_lowres_bitrate_interpolation); + const int index = FindSimulcastFormatIndex( + width, height, enable_lowres_bitrate_interpolation); + if (index == 0) + return formats[index]; + const int total_pixels_up = + formats[index - 1].width * formats[index - 1].height; + const int total_pixels_down = formats[index].width * formats[index].height; + const int total_pixels = width * height; + const float rate = (total_pixels_up - total_pixels) / + static_cast(total_pixels_up - total_pixels_down); + + // Use upper resolution if `rate` is below the configured threshold. + size_t max_layers = (rate < max_roundup_rate.value_or(kDefaultMaxRoundupRate)) + ? formats[index - 1].max_layers + : formats[index].max_layers; + webrtc::DataRate max_bitrate = Interpolate(formats[index - 1].max_bitrate, + formats[index].max_bitrate, rate); + webrtc::DataRate target_bitrate = Interpolate( + formats[index - 1].target_bitrate, formats[index].target_bitrate, rate); + webrtc::DataRate min_bitrate = Interpolate(formats[index - 1].min_bitrate, + formats[index].min_bitrate, rate); + + return {width, height, max_layers, max_bitrate, target_bitrate, min_bitrate}; +} + +SimulcastFormat InterpolateSimulcastFormat( + int width, + int height, + bool enable_lowres_bitrate_interpolation) { + return InterpolateSimulcastFormat(width, height, absl::nullopt, + enable_lowres_bitrate_interpolation); +} + +webrtc::DataRate FindSimulcastMaxBitrate( + int width, + int height, + bool enable_lowres_bitrate_interpolation) { + return InterpolateSimulcastFormat(width, height, + enable_lowres_bitrate_interpolation) + .max_bitrate; +} + +webrtc::DataRate FindSimulcastTargetBitrate( + int width, + int height, + bool enable_lowres_bitrate_interpolation) { + return InterpolateSimulcastFormat(width, height, + enable_lowres_bitrate_interpolation) + .target_bitrate; +} + +webrtc::DataRate FindSimulcastMinBitrate( + int width, + int height, + bool enable_lowres_bitrate_interpolation) { + return InterpolateSimulcastFormat(width, height, + enable_lowres_bitrate_interpolation) + .min_bitrate; +} + +void BoostMaxSimulcastLayer(webrtc::DataRate max_bitrate, + std::vector* layers) { + if (layers->empty()) + return; + + const webrtc::DataRate total_bitrate = GetTotalMaxBitrate(*layers); + + // We're still not using all available bits. + if (total_bitrate < max_bitrate) { + // Spend additional bits to boost the max layer. + const webrtc::DataRate bitrate_left = max_bitrate - total_bitrate; + layers->back().max_bitrate_bps += bitrate_left.bps(); + } +} + +webrtc::DataRate GetTotalMaxBitrate( + const std::vector& layers) { + if (layers.empty()) + return webrtc::DataRate::Zero(); + + int total_max_bitrate_bps = 0; + for (size_t s = 0; s < layers.size() - 1; ++s) { + total_max_bitrate_bps += layers[s].target_bitrate_bps; + } + total_max_bitrate_bps += layers.back().max_bitrate_bps; + return webrtc::DataRate::BitsPerSec(total_max_bitrate_bps); +} + +size_t LimitSimulcastLayerCount(int width, + int height, + size_t need_layers, + size_t layer_count, + const webrtc::FieldTrialsView& trials) { + if (!absl::StartsWith(trials.Lookup(kUseLegacySimulcastLayerLimitFieldTrial), + "Disabled")) { + // Max layers from one higher resolution in kSimulcastFormats will be used + // if the ratio (pixels_up - pixels) / (pixels_up - pixels_down) is less + // than configured `max_ratio`. pixels_down is the selected index in + // kSimulcastFormats based on pixels. + webrtc::FieldTrialOptional max_ratio("max_ratio"); + webrtc::ParseFieldTrial({&max_ratio}, + trials.Lookup("WebRTC-SimulcastLayerLimitRoundUp")); + + const bool enable_lowres_bitrate_interpolation = + EnableLowresBitrateInterpolation(trials); + size_t adaptive_layer_count = std::max( + need_layers, + InterpolateSimulcastFormat(width, height, max_ratio.GetOptional(), + enable_lowres_bitrate_interpolation) + .max_layers); + if (layer_count > adaptive_layer_count) { + RTC_LOG(LS_WARNING) << "Reducing simulcast layer count from " + << layer_count << " to " << adaptive_layer_count; + layer_count = adaptive_layer_count; + } + } + return layer_count; +} + +std::vector GetSimulcastConfig( + size_t min_layers, + size_t max_layers, + int width, + int height, + double bitrate_priority, + int max_qp, + bool is_screenshare_with_conference_mode, + bool temporal_layers_supported, + const webrtc::FieldTrialsView& trials) { + RTC_DCHECK_LE(min_layers, max_layers); + RTC_DCHECK(max_layers > 1 || is_screenshare_with_conference_mode); + + const bool base_heavy_tl3_rate_alloc = + webrtc::RateControlSettings::ParseFromKeyValueConfig(&trials) + .Vp8BaseHeavyTl3RateAllocation(); + if (is_screenshare_with_conference_mode) { + return GetScreenshareLayers(max_layers, width, height, bitrate_priority, + max_qp, temporal_layers_supported, + base_heavy_tl3_rate_alloc, trials); + } else { + // Some applications rely on the old behavior limiting the simulcast layer + // count based on the resolution automatically, which they can get through + // the WebRTC-LegacySimulcastLayerLimit field trial until they update. + max_layers = + LimitSimulcastLayerCount(width, height, min_layers, max_layers, trials); + + return GetNormalSimulcastLayers(max_layers, width, height, bitrate_priority, + max_qp, temporal_layers_supported, + base_heavy_tl3_rate_alloc, trials); + } +} + +std::vector GetNormalSimulcastLayers( + size_t layer_count, + int width, + int height, + double bitrate_priority, + int max_qp, + bool temporal_layers_supported, + bool base_heavy_tl3_rate_alloc, + const webrtc::FieldTrialsView& trials) { + std::vector layers(layer_count); + + const bool enable_lowres_bitrate_interpolation = + EnableLowresBitrateInterpolation(trials); + + // Format width and height has to be divisible by |2 ^ num_simulcast_layers - + // 1|. + width = NormalizeSimulcastSize(width, layer_count); + height = NormalizeSimulcastSize(height, layer_count); + // Add simulcast streams, from highest resolution (`s` = num_simulcast_layers + // -1) to lowest resolution at `s` = 0. + for (size_t s = layer_count - 1;; --s) { + layers[s].width = width; + layers[s].height = height; + // TODO(pbos): Fill actual temporal-layer bitrate thresholds. + layers[s].max_qp = max_qp; + layers[s].num_temporal_layers = + temporal_layers_supported ? DefaultNumberOfTemporalLayers(trials) : 1; + layers[s].max_bitrate_bps = + FindSimulcastMaxBitrate(width, height, + enable_lowres_bitrate_interpolation) + .bps(); + layers[s].target_bitrate_bps = + FindSimulcastTargetBitrate(width, height, + enable_lowres_bitrate_interpolation) + .bps(); + int num_temporal_layers = DefaultNumberOfTemporalLayers(trials); + if (s == 0) { + // If alternative temporal rate allocation is selected, adjust the + // bitrate of the lowest simulcast stream so that absolute bitrate for + // the base temporal layer matches the bitrate for the base temporal + // layer with the default 3 simulcast streams. Otherwise we risk a + // higher threshold for receiving a feed at all. + float rate_factor = 1.0; + if (num_temporal_layers == 3) { + if (base_heavy_tl3_rate_alloc) { + // Base heavy allocation increases TL0 bitrate from 40% to 60%. + rate_factor = 0.4 / 0.6; + } + } else { + rate_factor = + webrtc::SimulcastRateAllocator::GetTemporalRateAllocation( + 3, 0, /*base_heavy_tl3_rate_alloc=*/false) / + webrtc::SimulcastRateAllocator::GetTemporalRateAllocation( + num_temporal_layers, 0, /*base_heavy_tl3_rate_alloc=*/false); + } + + layers[s].max_bitrate_bps = + static_cast(layers[s].max_bitrate_bps * rate_factor); + layers[s].target_bitrate_bps = + static_cast(layers[s].target_bitrate_bps * rate_factor); + } + layers[s].min_bitrate_bps = + FindSimulcastMinBitrate(width, height, + enable_lowres_bitrate_interpolation) + .bps(); + + // Ensure consistency. + layers[s].max_bitrate_bps = + std::max(layers[s].min_bitrate_bps, layers[s].max_bitrate_bps); + layers[s].target_bitrate_bps = + std::max(layers[s].min_bitrate_bps, layers[s].target_bitrate_bps); + + layers[s].max_framerate = kDefaultVideoMaxFramerate; + + width /= 2; + height /= 2; + + if (s == 0) { + break; + } + } + // Currently the relative bitrate priority of the sender is controlled by + // the value of the lowest VideoStream. + // TODO(bugs.webrtc.org/8630): The web specification describes being able to + // control relative bitrate for each individual simulcast layer, but this + // is currently just implemented per rtp sender. + layers[0].bitrate_priority = bitrate_priority; + return layers; +} + +std::vector GetScreenshareLayers( + size_t max_layers, + int width, + int height, + double bitrate_priority, + int max_qp, + bool temporal_layers_supported, + bool base_heavy_tl3_rate_alloc, + const webrtc::FieldTrialsView& trials) { + size_t num_simulcast_layers = + std::min(max_layers, kScreenshareMaxSimulcastLayers); + + std::vector layers(num_simulcast_layers); + // For legacy screenshare in conference mode, tl0 and tl1 bitrates are + // piggybacked on the VideoCodec struct as target and max bitrates, + // respectively. See eg. webrtc::LibvpxVp8Encoder::SetRates(). + layers[0].width = width; + layers[0].height = height; + layers[0].max_qp = max_qp; + layers[0].max_framerate = 5; + layers[0].min_bitrate_bps = webrtc::kDefaultMinVideoBitrateBps; + layers[0].target_bitrate_bps = kScreenshareDefaultTl0Bitrate.bps(); + layers[0].max_bitrate_bps = kScreenshareDefaultTl1Bitrate.bps(); + layers[0].num_temporal_layers = temporal_layers_supported ? 2 : 1; + + // With simulcast enabled, add another spatial layer. This one will have a + // more normal layout, with the regular 3 temporal layer pattern and no fps + // restrictions. The base simulcast layer will still use legacy setup. + if (num_simulcast_layers == kScreenshareMaxSimulcastLayers) { + // Add optional upper simulcast layer. + int max_bitrate_bps; + bool using_boosted_bitrate = false; + if (!temporal_layers_supported) { + // Set the max bitrate to where the base layer would have been if temporal + // layers were enabled. + max_bitrate_bps = static_cast( + kScreenshareHighStreamMaxBitrate.bps() * + webrtc::SimulcastRateAllocator::GetTemporalRateAllocation( + kScreenshareTemporalLayers, 0, base_heavy_tl3_rate_alloc)); + } else { + // Experimental temporal layer mode used, use increased max bitrate. + max_bitrate_bps = kScreenshareHighStreamMaxBitrate.bps(); + using_boosted_bitrate = true; + } + + layers[1].width = width; + layers[1].height = height; + layers[1].max_qp = max_qp; + layers[1].max_framerate = kDefaultVideoMaxFramerate; + layers[1].num_temporal_layers = + temporal_layers_supported ? kScreenshareTemporalLayers : 1; + layers[1].min_bitrate_bps = using_boosted_bitrate + ? kScreenshareHighStreamMinBitrate.bps() + : layers[0].target_bitrate_bps * 2; + layers[1].target_bitrate_bps = max_bitrate_bps; + layers[1].max_bitrate_bps = max_bitrate_bps; + } + + // The bitrate priority currently implemented on a per-sender level, so we + // just set it for the first simulcast layer. + layers[0].bitrate_priority = bitrate_priority; + return layers; +} + +} // namespace cricket -- cgit v1.2.3