diff options
Diffstat (limited to 'third_party/libwebrtc/video/config')
11 files changed, 2694 insertions, 0 deletions
diff --git a/third_party/libwebrtc/video/config/BUILD.gn b/third_party/libwebrtc/video/config/BUILD.gn new file mode 100644 index 0000000000..96e254e76b --- /dev/null +++ b/third_party/libwebrtc/video/config/BUILD.gn @@ -0,0 +1,99 @@ +# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") + +rtc_library("streams_config") { + sources = [ + "encoder_stream_factory.cc", + "encoder_stream_factory.h", + "simulcast.cc", + "simulcast.h", + ] + + deps = [ + ":encoder_config", + "../../api:field_trials_view", + "../../api/transport:field_trial_based_config", + "../../api/units:data_rate", + "../../api/video:video_codec_constants", + "../../api/video_codecs:video_codecs_api", + "../../call/adaptation:resource_adaptation", + "../../media:media_constants", + "../../media:rtc_media_base", + "../../modules/video_coding:video_coding_utility", + "../../modules/video_coding:webrtc_vp9_helpers", + "../../rtc_base:checks", + "../../rtc_base:logging", + "../../rtc_base/experiments:field_trial_parser", + "../../rtc_base/experiments:min_video_bitrate_experiment", + "../../rtc_base/experiments:normalize_simulcast_size_experiment", + "../../rtc_base/experiments:rate_control_settings", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("encoder_config") { + sources = [ + "video_encoder_config.cc", + "video_encoder_config.h", + ] + + deps = [ + "../../api:scoped_refptr", + "../../api/video:resolution", + "../../api/video_codecs:scalability_mode", + "../../api/video_codecs:video_codecs_api", + "../../rtc_base:checks", + "../../rtc_base:refcount", + "../../rtc_base:stringutils", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +if (rtc_include_tests) { + rtc_library("video_config_tests") { + testonly = true + + defines = [] + sources = [ + "encoder_stream_factory_unittest.cc", + "simulcast_unittest.cc", + ] + deps = [ + ":streams_config", + "../../api/transport:field_trial_based_config", + "../../call/adaptation:resource_adaptation", + "../../media:media_constants", + "../../test:field_trial", + "../../test:test_support", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/functional:any_invocable", + "//third_party/abseil-cpp/absl/functional:bind_front", + "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + "//third_party/abseil-cpp/absl/types:variant", + ] + if (!build_with_mozilla) { + deps += [ "../../media:rtc_media_base" ] + } + } +} diff --git a/third_party/libwebrtc/video/config/encoder_config_gn/moz.build b/third_party/libwebrtc/video/config/encoder_config_gn/moz.build new file mode 100644 index 0000000000..1c2c9ef835 --- /dev/null +++ b/third_party/libwebrtc/video/config/encoder_config_gn/moz.build @@ -0,0 +1,232 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/video/config/video_encoder_config.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_ENABLE_LIBEVENT"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_ENABLE_LIBEVENT"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_ENABLE_LIBEVENT"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["RTC_ENABLE_WIN_WGC"] = True + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["TARGET_CPU"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["TARGET_CPU"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["TARGET_CPU"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["TARGET_CPU"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["TARGET_CPU"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["TARGET_CPU"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "arm": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "x86": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "x86_64": + + DEFINES["_GNU_SOURCE"] = True + +Library("encoder_config_gn") diff --git a/third_party/libwebrtc/video/config/encoder_stream_factory.cc b/third_party/libwebrtc/video/config/encoder_stream_factory.cc new file mode 100644 index 0000000000..a4e41ad62d --- /dev/null +++ b/third_party/libwebrtc/video/config/encoder_stream_factory.cc @@ -0,0 +1,496 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "video/config/encoder_stream_factory.h" + +#include <algorithm> +#include <limits> +#include <set> +#include <string> +#include <utility> + +#include "absl/algorithm/container.h" +#include "absl/strings/match.h" +#include "api/video/video_codec_constants.h" +#include "media/base/media_constants.h" +#include "media/base/video_adapter.h" +#include "modules/video_coding/codecs/vp9/svc_config.h" +#include "rtc_base/experiments/min_video_bitrate_experiment.h" +#include "rtc_base/experiments/normalize_simulcast_size_experiment.h" +#include "rtc_base/logging.h" +#include "video/config/simulcast.h" + +namespace cricket { +namespace { + +const int kMinLayerSize = 16; + +int ScaleDownResolution(int resolution, + double scale_down_by, + int min_resolution) { + // Resolution is never scalied down to smaller than min_resolution. + // If the input resolution is already smaller than min_resolution, + // no scaling should be done at all. + if (resolution <= min_resolution) + return resolution; + return std::max(static_cast<int>(resolution / scale_down_by + 0.5), + min_resolution); +} + +bool PowerOfTwo(int value) { + return (value > 0) && ((value & (value - 1)) == 0); +} + +bool IsScaleFactorsPowerOfTwo(const webrtc::VideoEncoderConfig& config) { + for (const auto& layer : config.simulcast_layers) { + double scale = std::max(layer.scale_resolution_down_by, 1.0); + if (std::round(scale) != scale || !PowerOfTwo(scale)) { + return false; + } + } + return true; +} + +bool IsTemporalLayersSupported(const std::string& codec_name) { + return absl::EqualsIgnoreCase(codec_name, kVp8CodecName) || + absl::EqualsIgnoreCase(codec_name, kVp9CodecName) || + absl::EqualsIgnoreCase(codec_name, kAv1CodecName); +} + +size_t FindRequiredActiveLayers( + const webrtc::VideoEncoderConfig& encoder_config) { + // Need enough layers so that at least the first active one is present. + for (size_t i = 0; i < encoder_config.number_of_streams; ++i) { + if (encoder_config.simulcast_layers[i].active) { + return i + 1; + } + } + return 0; +} + +// The selected thresholds for QVGA and VGA corresponded to a QP around 10. +// The change in QP declined above the selected bitrates. +static int GetMaxDefaultVideoBitrateKbps(int width, + int height, + bool is_screenshare) { + int max_bitrate; + if (width * height <= 320 * 240) { + max_bitrate = 600; + } else if (width * height <= 640 * 480) { + max_bitrate = 1700; + } else if (width * height <= 960 * 540) { + max_bitrate = 2000; + } else { + max_bitrate = 2500; + } + if (is_screenshare) + max_bitrate = std::max(max_bitrate, 1200); + return max_bitrate; +} + +} // namespace + +// TODO(bugs.webrtc.org/8785): Consider removing max_qp as member of +// EncoderStreamFactory and instead set this value individually for each stream +// in the VideoEncoderConfig.simulcast_layers. +EncoderStreamFactory::EncoderStreamFactory(std::string codec_name, + int max_qp, + bool is_screenshare, + bool conference_mode) + : codec_name_(codec_name), + max_qp_(max_qp), + is_screenshare_(is_screenshare), + conference_mode_(conference_mode), + trials_(fallback_trials_), + encoder_info_requested_resolution_alignment_(1) {} + +EncoderStreamFactory::EncoderStreamFactory( + std::string codec_name, + int max_qp, + bool is_screenshare, + bool conference_mode, + const webrtc::VideoEncoder::EncoderInfo& encoder_info, + absl::optional<webrtc::VideoSourceRestrictions> restrictions, + const webrtc::FieldTrialsView* trials) + : codec_name_(codec_name), + max_qp_(max_qp), + is_screenshare_(is_screenshare), + conference_mode_(conference_mode), + trials_(trials ? *trials : fallback_trials_), + encoder_info_requested_resolution_alignment_( + encoder_info.requested_resolution_alignment), + restrictions_(restrictions) {} + +std::vector<webrtc::VideoStream> EncoderStreamFactory::CreateEncoderStreams( + int frame_width, + int frame_height, + const webrtc::VideoEncoderConfig& encoder_config) { + RTC_DCHECK_GT(encoder_config.number_of_streams, 0); + RTC_DCHECK_GE(encoder_config.simulcast_layers.size(), + encoder_config.number_of_streams); + + const absl::optional<webrtc::DataRate> experimental_min_bitrate = + GetExperimentalMinVideoBitrate(encoder_config.codec_type); + + bool is_simulcast = (encoder_config.number_of_streams > 1); + // If scalability mode was specified, don't treat {active,inactive,inactive} + // as simulcast since the simulcast configuration assumes very low bitrates + // on the first layer. This would prevent rampup of multiple spatial layers. + // See https://crbug.com/webrtc/15041. + if (is_simulcast && + encoder_config.simulcast_layers[0].scalability_mode.has_value()) { + // Require at least one non-first layer to be active for is_simulcast=true. + is_simulcast = false; + for (size_t i = 1; i < encoder_config.simulcast_layers.size(); ++i) { + if (encoder_config.simulcast_layers[i].active) { + is_simulcast = true; + break; + } + } + } + + if (is_simulcast || ((absl::EqualsIgnoreCase(codec_name_, kVp8CodecName) || + absl::EqualsIgnoreCase(codec_name_, kH264CodecName)) && + is_screenshare_ && conference_mode_)) { + return CreateSimulcastOrConferenceModeScreenshareStreams( + frame_width, frame_height, encoder_config, experimental_min_bitrate); + } + + return CreateDefaultVideoStreams(frame_width, frame_height, encoder_config, + experimental_min_bitrate); +} + +std::vector<webrtc::VideoStream> +EncoderStreamFactory::CreateDefaultVideoStreams( + int width, + int height, + const webrtc::VideoEncoderConfig& encoder_config, + const absl::optional<webrtc::DataRate>& experimental_min_bitrate) const { + std::vector<webrtc::VideoStream> layers; + + // The max bitrate specified by the API. + // - `encoder_config.simulcast_layers[0].max_bitrate_bps` comes from the first + // RtpEncodingParamters, which is the encoding of this stream. + // - `encoder_config.max_bitrate_bps` comes from SDP; "b=AS" or conditionally + // "x-google-max-bitrate". + // If `api_max_bitrate_bps` has a value then it is positive. + absl::optional<int> api_max_bitrate_bps; + if (encoder_config.simulcast_layers[0].max_bitrate_bps > 0) { + api_max_bitrate_bps = encoder_config.simulcast_layers[0].max_bitrate_bps; + } + if (encoder_config.max_bitrate_bps > 0) { + api_max_bitrate_bps = + api_max_bitrate_bps.has_value() + ? std::min(encoder_config.max_bitrate_bps, *api_max_bitrate_bps) + : encoder_config.max_bitrate_bps; + } + + // For unset max bitrates set default bitrate for non-simulcast. + int max_bitrate_bps = + api_max_bitrate_bps.has_value() + ? *api_max_bitrate_bps + : GetMaxDefaultVideoBitrateKbps(width, height, is_screenshare_) * + 1000; + + int min_bitrate_bps = + experimental_min_bitrate + ? rtc::saturated_cast<int>(experimental_min_bitrate->bps()) + : webrtc::kDefaultMinVideoBitrateBps; + if (encoder_config.simulcast_layers[0].min_bitrate_bps > 0) { + // Use set min bitrate. + min_bitrate_bps = encoder_config.simulcast_layers[0].min_bitrate_bps; + // If only min bitrate is configured, make sure max is above min. + if (!api_max_bitrate_bps.has_value()) + max_bitrate_bps = std::max(min_bitrate_bps, max_bitrate_bps); + } + int max_framerate = (encoder_config.simulcast_layers[0].max_framerate > 0) + ? encoder_config.simulcast_layers[0].max_framerate + : kDefaultVideoMaxFramerate; + + webrtc::VideoStream layer; + layer.width = width; + layer.height = height; + layer.max_framerate = max_framerate; + layer.requested_resolution = + encoder_config.simulcast_layers[0].requested_resolution; + // Note: VP9 seems to have be sending if any layer is active, + // (see `UpdateSendState`) and still use parameters only from + // encoder_config.simulcast_layers[0]. + layer.active = absl::c_any_of(encoder_config.simulcast_layers, + [](const auto& layer) { return layer.active; }); + + if (encoder_config.simulcast_layers[0].requested_resolution) { + auto res = GetLayerResolutionFromRequestedResolution( + width, height, + *encoder_config.simulcast_layers[0].requested_resolution); + layer.width = res.width; + layer.height = res.height; + } else if (encoder_config.simulcast_layers[0].scale_resolution_down_by > 1.) { + layer.width = ScaleDownResolution( + layer.width, + encoder_config.simulcast_layers[0].scale_resolution_down_by, + kMinLayerSize); + layer.height = ScaleDownResolution( + layer.height, + encoder_config.simulcast_layers[0].scale_resolution_down_by, + kMinLayerSize); + } + + if (absl::EqualsIgnoreCase(codec_name_, kVp9CodecName)) { + RTC_DCHECK(encoder_config.encoder_specific_settings); + // Use VP9 SVC layering from codec settings which might be initialized + // though field trial in ConfigureVideoEncoderSettings. + webrtc::VideoCodecVP9 vp9_settings; + encoder_config.encoder_specific_settings->FillVideoCodecVp9(&vp9_settings); + layer.num_temporal_layers = vp9_settings.numberOfTemporalLayers; + + // Number of spatial layers is signalled differently from different call + // sites (sigh), pick the max as we are interested in the upper bound. + int num_spatial_layers = + std::max({encoder_config.simulcast_layers.size(), + encoder_config.spatial_layers.size(), + size_t{vp9_settings.numberOfSpatialLayers}}); + + if (width * height > 0 && + (layer.num_temporal_layers > 1u || num_spatial_layers > 1)) { + // In SVC mode, the VP9 max bitrate is determined by SvcConfig, instead of + // GetMaxDefaultVideoBitrateKbps(). + std::vector<webrtc::SpatialLayer> svc_layers = + webrtc::GetSvcConfig(width, height, max_framerate, + /*first_active_layer=*/0, num_spatial_layers, + *layer.num_temporal_layers, is_screenshare_); + int sum_max_bitrates_kbps = 0; + for (const webrtc::SpatialLayer& spatial_layer : svc_layers) { + sum_max_bitrates_kbps += spatial_layer.maxBitrate; + } + RTC_DCHECK_GE(sum_max_bitrates_kbps, 0); + if (!api_max_bitrate_bps.has_value()) { + max_bitrate_bps = sum_max_bitrates_kbps * 1000; + } else { + max_bitrate_bps = + std::min(max_bitrate_bps, sum_max_bitrates_kbps * 1000); + } + max_bitrate_bps = std::max(min_bitrate_bps, max_bitrate_bps); + } + } + + // In the case that the application sets a max bitrate that's lower than the + // min bitrate, we adjust it down (see bugs.webrtc.org/9141). + layer.min_bitrate_bps = std::min(min_bitrate_bps, max_bitrate_bps); + if (encoder_config.simulcast_layers[0].target_bitrate_bps <= 0) { + layer.target_bitrate_bps = max_bitrate_bps; + } else { + layer.target_bitrate_bps = std::min( + encoder_config.simulcast_layers[0].target_bitrate_bps, max_bitrate_bps); + } + layer.max_bitrate_bps = max_bitrate_bps; + layer.max_qp = max_qp_; + layer.bitrate_priority = encoder_config.bitrate_priority; + + if (IsTemporalLayersSupported(codec_name_)) { + // Use configured number of temporal layers if set. + if (encoder_config.simulcast_layers[0].num_temporal_layers) { + layer.num_temporal_layers = + *encoder_config.simulcast_layers[0].num_temporal_layers; + } + } + layer.scalability_mode = encoder_config.simulcast_layers[0].scalability_mode; + layers.push_back(layer); + return layers; +} + +std::vector<webrtc::VideoStream> +EncoderStreamFactory::CreateSimulcastOrConferenceModeScreenshareStreams( + int width, + int height, + const webrtc::VideoEncoderConfig& encoder_config, + const absl::optional<webrtc::DataRate>& experimental_min_bitrate) const { + std::vector<webrtc::VideoStream> layers; + + const bool temporal_layers_supported = IsTemporalLayersSupported(codec_name_); + // Use legacy simulcast screenshare if conference mode is explicitly enabled + // or use the regular simulcast configuration path which is generic. + layers = GetSimulcastConfig(FindRequiredActiveLayers(encoder_config), + encoder_config.number_of_streams, width, height, + encoder_config.bitrate_priority, max_qp_, + is_screenshare_ && conference_mode_, + temporal_layers_supported, trials_); + // Allow an experiment to override the minimum bitrate for the lowest + // spatial layer. The experiment's configuration has the lowest priority. + if (experimental_min_bitrate) { + layers[0].min_bitrate_bps = + rtc::saturated_cast<int>(experimental_min_bitrate->bps()); + } + // Update the active simulcast layers and configured bitrates. + bool is_highest_layer_max_bitrate_configured = false; + const bool has_scale_resolution_down_by = absl::c_any_of( + encoder_config.simulcast_layers, [](const webrtc::VideoStream& layer) { + return layer.scale_resolution_down_by != -1.; + }); + + bool default_scale_factors_used = true; + if (has_scale_resolution_down_by) { + default_scale_factors_used = IsScaleFactorsPowerOfTwo(encoder_config); + } + const bool norm_size_configured = + webrtc::NormalizeSimulcastSizeExperiment::GetBase2Exponent().has_value(); + const int normalized_width = + (default_scale_factors_used || norm_size_configured) && + (width >= kMinLayerSize) + ? NormalizeSimulcastSize(width, encoder_config.number_of_streams) + : width; + const int normalized_height = + (default_scale_factors_used || norm_size_configured) && + (height >= kMinLayerSize) + ? NormalizeSimulcastSize(height, encoder_config.number_of_streams) + : height; + for (size_t i = 0; i < layers.size(); ++i) { + layers[i].active = encoder_config.simulcast_layers[i].active; + layers[i].scalability_mode = + encoder_config.simulcast_layers[i].scalability_mode; + layers[i].requested_resolution = + encoder_config.simulcast_layers[i].requested_resolution; + // Update with configured num temporal layers if supported by codec. + if (encoder_config.simulcast_layers[i].num_temporal_layers && + IsTemporalLayersSupported(codec_name_)) { + layers[i].num_temporal_layers = + *encoder_config.simulcast_layers[i].num_temporal_layers; + } + if (encoder_config.simulcast_layers[i].max_framerate > 0) { + layers[i].max_framerate = + encoder_config.simulcast_layers[i].max_framerate; + } + if (encoder_config.simulcast_layers[i].requested_resolution.has_value()) { + auto res = GetLayerResolutionFromRequestedResolution( + normalized_width, normalized_height, + *encoder_config.simulcast_layers[i].requested_resolution); + layers[i].width = res.width; + layers[i].height = res.height; + } else if (has_scale_resolution_down_by) { + const double scale_resolution_down_by = std::max( + encoder_config.simulcast_layers[i].scale_resolution_down_by, 1.0); + layers[i].width = ScaleDownResolution( + normalized_width, scale_resolution_down_by, kMinLayerSize); + layers[i].height = ScaleDownResolution( + normalized_height, scale_resolution_down_by, kMinLayerSize); + } + // Update simulcast bitrates with configured min and max bitrate. + if (encoder_config.simulcast_layers[i].min_bitrate_bps > 0) { + layers[i].min_bitrate_bps = + encoder_config.simulcast_layers[i].min_bitrate_bps; + } + if (encoder_config.simulcast_layers[i].max_bitrate_bps > 0) { + layers[i].max_bitrate_bps = + encoder_config.simulcast_layers[i].max_bitrate_bps; + } + if (encoder_config.simulcast_layers[i].target_bitrate_bps > 0) { + layers[i].target_bitrate_bps = + encoder_config.simulcast_layers[i].target_bitrate_bps; + } + if (encoder_config.simulcast_layers[i].min_bitrate_bps > 0 && + encoder_config.simulcast_layers[i].max_bitrate_bps > 0) { + // Min and max bitrate are configured. + // Set target to 3/4 of the max bitrate (or to max if below min). + if (encoder_config.simulcast_layers[i].target_bitrate_bps <= 0) + layers[i].target_bitrate_bps = layers[i].max_bitrate_bps * 3 / 4; + if (layers[i].target_bitrate_bps < layers[i].min_bitrate_bps) + layers[i].target_bitrate_bps = layers[i].max_bitrate_bps; + } else if (encoder_config.simulcast_layers[i].min_bitrate_bps > 0) { + // Only min bitrate is configured, make sure target/max are above min. + layers[i].target_bitrate_bps = + std::max(layers[i].target_bitrate_bps, layers[i].min_bitrate_bps); + layers[i].max_bitrate_bps = + std::max(layers[i].max_bitrate_bps, layers[i].min_bitrate_bps); + } else if (encoder_config.simulcast_layers[i].max_bitrate_bps > 0) { + // Only max bitrate is configured, make sure min/target are below max. + // Keep target bitrate if it is set explicitly in encoding config. + // Otherwise set target bitrate to 3/4 of the max bitrate + // or the one calculated from GetSimulcastConfig() which is larger. + layers[i].min_bitrate_bps = + std::min(layers[i].min_bitrate_bps, layers[i].max_bitrate_bps); + if (encoder_config.simulcast_layers[i].target_bitrate_bps <= 0) { + layers[i].target_bitrate_bps = std::max( + layers[i].target_bitrate_bps, layers[i].max_bitrate_bps * 3 / 4); + } + layers[i].target_bitrate_bps = std::max( + std::min(layers[i].target_bitrate_bps, layers[i].max_bitrate_bps), + layers[i].min_bitrate_bps); + } + if (i == layers.size() - 1) { + is_highest_layer_max_bitrate_configured = + encoder_config.simulcast_layers[i].max_bitrate_bps > 0; + } + } + if (!is_screenshare_ && !is_highest_layer_max_bitrate_configured && + encoder_config.max_bitrate_bps > 0) { + // No application-configured maximum for the largest layer. + // If there is bitrate leftover, give it to the largest layer. + BoostMaxSimulcastLayer( + webrtc::DataRate::BitsPerSec(encoder_config.max_bitrate_bps), &layers); + } + + // Sort the layers by max_bitrate_bps, they might not always be from + // smallest to biggest + std::vector<size_t> index(layers.size()); + std::iota(index.begin(), index.end(), 0); + std::stable_sort(index.begin(), index.end(), [&layers](size_t a, size_t b) { + return layers[a].max_bitrate_bps < layers[b].max_bitrate_bps; + }); + + if (!layers[index[0]].active) { + // Adjust min bitrate of the first active layer to allow it to go as low as + // the lowest (now inactive) layer could. + // Otherwise, if e.g. a single HD stream is active, it would have 600kbps + // min bitrate, which would always be allocated to the stream. + // This would lead to congested network, dropped frames and overall bad + // experience. + + const int min_configured_bitrate = layers[index[0]].min_bitrate_bps; + for (size_t i = 0; i < layers.size(); ++i) { + if (layers[index[i]].active) { + layers[index[i]].min_bitrate_bps = min_configured_bitrate; + break; + } + } + } + + return layers; +} + +webrtc::Resolution +EncoderStreamFactory::GetLayerResolutionFromRequestedResolution( + int frame_width, + int frame_height, + webrtc::Resolution requested_resolution) const { + VideoAdapter adapter(encoder_info_requested_resolution_alignment_); + adapter.OnOutputFormatRequest(requested_resolution.ToPair(), + requested_resolution.PixelCount(), + absl::nullopt); + if (restrictions_) { + rtc::VideoSinkWants wants; + wants.is_active = true; + wants.target_pixel_count = restrictions_->target_pixels_per_frame(); + wants.max_pixel_count = + rtc::dchecked_cast<int>(restrictions_->max_pixels_per_frame().value_or( + std::numeric_limits<int>::max())); + wants.aggregates.emplace(rtc::VideoSinkWants::Aggregates()); + wants.resolution_alignment = encoder_info_requested_resolution_alignment_; + adapter.OnSinkWants(wants); + } + int cropped_width, cropped_height; + int out_width = 0, out_height = 0; + if (!adapter.AdaptFrameResolution(frame_width, frame_height, 0, + &cropped_width, &cropped_height, &out_width, + &out_height)) { + RTC_LOG(LS_ERROR) << "AdaptFrameResolution returned false!"; + } + return {.width = out_width, .height = out_height}; +} + +} // namespace cricket diff --git a/third_party/libwebrtc/video/config/encoder_stream_factory.h b/third_party/libwebrtc/video/config/encoder_stream_factory.h new file mode 100644 index 0000000000..37abb93876 --- /dev/null +++ b/third_party/libwebrtc/video/config/encoder_stream_factory.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef VIDEO_CONFIG_ENCODER_STREAM_FACTORY_H_ +#define VIDEO_CONFIG_ENCODER_STREAM_FACTORY_H_ + +#include <string> +#include <vector> + +#include "api/transport/field_trial_based_config.h" +#include "api/units/data_rate.h" +#include "api/video_codecs/video_encoder.h" +#include "call/adaptation/video_source_restrictions.h" +#include "video/config/video_encoder_config.h" + +namespace cricket { + +class EncoderStreamFactory + : public webrtc::VideoEncoderConfig::VideoStreamFactoryInterface { + public: + // Note: this constructor is used by testcase in downstream. + EncoderStreamFactory(std::string codec_name, + int max_qp, + bool is_screenshare, + bool conference_mode); + + EncoderStreamFactory(std::string codec_name, + int max_qp, + bool is_screenshare, + bool conference_mode, + const webrtc::VideoEncoder::EncoderInfo& encoder_info, + absl::optional<webrtc::VideoSourceRestrictions> + restrictions = absl::nullopt, + const webrtc::FieldTrialsView* trials = nullptr); + + std::vector<webrtc::VideoStream> CreateEncoderStreams( + int width, + int height, + const webrtc::VideoEncoderConfig& encoder_config) override; + + private: + std::vector<webrtc::VideoStream> CreateDefaultVideoStreams( + int width, + int height, + const webrtc::VideoEncoderConfig& encoder_config, + const absl::optional<webrtc::DataRate>& experimental_min_bitrate) const; + + std::vector<webrtc::VideoStream> + CreateSimulcastOrConferenceModeScreenshareStreams( + int width, + int height, + const webrtc::VideoEncoderConfig& encoder_config, + const absl::optional<webrtc::DataRate>& experimental_min_bitrate) const; + + webrtc::Resolution GetLayerResolutionFromRequestedResolution( + int in_frame_width, + int in_frame_height, + webrtc::Resolution requested_resolution) const; + + const std::string codec_name_; + const int max_qp_; + const bool is_screenshare_; + // Allows a screenshare specific configuration, which enables temporal + // layering and various settings. + const bool conference_mode_; + const webrtc::FieldTrialBasedConfig fallback_trials_; + const webrtc::FieldTrialsView& trials_; + const int encoder_info_requested_resolution_alignment_; + const absl::optional<webrtc::VideoSourceRestrictions> restrictions_; +}; + +} // namespace cricket + +#endif // VIDEO_CONFIG_ENCODER_STREAM_FACTORY_H_ diff --git a/third_party/libwebrtc/video/config/encoder_stream_factory_unittest.cc b/third_party/libwebrtc/video/config/encoder_stream_factory_unittest.cc new file mode 100644 index 0000000000..b37b300c96 --- /dev/null +++ b/third_party/libwebrtc/video/config/encoder_stream_factory_unittest.cc @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/config/encoder_stream_factory.h" + +#include "call/adaptation/video_source_restrictions.h" +#include "test/gtest.h" + +namespace webrtc { + +using cricket::EncoderStreamFactory; +constexpr int kMaxQp = 48; + +namespace { + +std::vector<Resolution> GetStreamResolutions( + const std::vector<VideoStream>& streams) { + std::vector<Resolution> res; + for (const auto& s : streams) { + if (s.active) { + res.push_back( + {rtc::checked_cast<int>(s.width), rtc::checked_cast<int>(s.height)}); + } + } + return res; +} + +VideoStream LayerWithRequestedResolution(Resolution res) { + VideoStream s; + s.requested_resolution = res; + return s; +} + +} // namespace + +TEST(EncoderStreamFactory, SinglecastRequestedResolution) { + VideoEncoder::EncoderInfo encoder_info; + auto factory = rtc::make_ref_counted<EncoderStreamFactory>( + "VP8", kMaxQp, + /* is_screenshare= */ false, + /* conference_mode= */ false, encoder_info); + VideoEncoderConfig encoder_config; + encoder_config.number_of_streams = 1; + encoder_config.simulcast_layers.push_back( + LayerWithRequestedResolution({.width = 640, .height = 360})); + auto streams = factory->CreateEncoderStreams(1280, 720, encoder_config); + EXPECT_EQ(streams[0].requested_resolution, + (Resolution{.width = 640, .height = 360})); + EXPECT_EQ(GetStreamResolutions(streams), (std::vector<Resolution>{ + {.width = 640, .height = 360}, + })); +} + +TEST(EncoderStreamFactory, SinglecastRequestedResolutionWithAdaptation) { + VideoSourceRestrictions restrictions( + /* max_pixels_per_frame= */ (320 * 320), + /* target_pixels_per_frame= */ absl::nullopt, + /* max_frame_rate= */ absl::nullopt); + VideoEncoder::EncoderInfo encoder_info; + auto factory = rtc::make_ref_counted<EncoderStreamFactory>( + "VP8", kMaxQp, + /* is_screenshare= */ false, + /* conference_mode= */ false, encoder_info, restrictions); + VideoEncoderConfig encoder_config; + encoder_config.number_of_streams = 1; + encoder_config.simulcast_layers.push_back( + LayerWithRequestedResolution({.width = 640, .height = 360})); + auto streams = factory->CreateEncoderStreams(1280, 720, encoder_config); + EXPECT_EQ(streams[0].requested_resolution, + (Resolution{.width = 640, .height = 360})); + EXPECT_EQ(GetStreamResolutions(streams), (std::vector<Resolution>{ + {.width = 320, .height = 180}, + })); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/video/config/simulcast.cc b/third_party/libwebrtc/video/config/simulcast.cc new file mode 100644 index 0000000000..2bd4ac04c3 --- /dev/null +++ b/third_party/libwebrtc/video/config/simulcast.cc @@ -0,0 +1,497 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/config/simulcast.h" + +#include <stdint.h> +#include <stdio.h> + +#include <algorithm> +#include <string> +#include <vector> + +#include "absl/strings/match.h" +#include "absl/types/optional.h" +#include "api/video/video_codec_constants.h" +#include "media/base/media_constants.h" +#include "modules/video_coding/utility/simulcast_rate_allocator.h" +#include "rtc_base/checks.h" +#include "rtc_base/experiments/field_trial_parser.h" +#include "rtc_base/experiments/min_video_bitrate_experiment.h" +#include "rtc_base/experiments/normalize_simulcast_size_experiment.h" +#include "rtc_base/experiments/rate_control_settings.h" +#include "rtc_base/logging.h" + +namespace cricket { + +namespace { + +constexpr char kUseLegacySimulcastLayerLimitFieldTrial[] = + "WebRTC-LegacySimulcastLayerLimit"; + +constexpr double kDefaultMaxRoundupRate = 0.1; + +// Limits for legacy conference screensharing mode. Currently used for the +// lower of the two simulcast streams. +constexpr webrtc::DataRate kScreenshareDefaultTl0Bitrate = + webrtc::DataRate::KilobitsPerSec(200); +constexpr webrtc::DataRate kScreenshareDefaultTl1Bitrate = + webrtc::DataRate::KilobitsPerSec(1000); + +// Min/max bitrate for the higher one of the two simulcast stream used for +// screen content. +constexpr webrtc::DataRate kScreenshareHighStreamMinBitrate = + webrtc::DataRate::KilobitsPerSec(600); +constexpr webrtc::DataRate kScreenshareHighStreamMaxBitrate = + webrtc::DataRate::KilobitsPerSec(1250); + +constexpr int kDefaultNumTemporalLayers = 3; +constexpr int kScreenshareMaxSimulcastLayers = 2; +constexpr int kScreenshareTemporalLayers = 2; + +struct SimulcastFormat { + int width; + int height; + // The maximum number of simulcast layers can be used for + // resolutions at `widthxheight` for legacy applications. + size_t max_layers; + // The maximum bitrate for encoding stream at `widthxheight`, when we are + // not sending the next higher spatial stream. + webrtc::DataRate max_bitrate; + // The target bitrate for encoding stream at `widthxheight`, when this layer + // is not the highest layer (i.e., when we are sending another higher spatial + // stream). + webrtc::DataRate target_bitrate; + // The minimum bitrate needed for encoding stream at `widthxheight`. + webrtc::DataRate min_bitrate; +}; + +// These tables describe from which resolution we can use how many +// simulcast layers at what bitrates (maximum, target, and minimum). +// Important!! Keep this table from high resolution to low resolution. +constexpr const SimulcastFormat kSimulcastFormats[] = { + {1920, 1080, 3, webrtc::DataRate::KilobitsPerSec(5000), + webrtc::DataRate::KilobitsPerSec(4000), + webrtc::DataRate::KilobitsPerSec(800)}, + {1280, 720, 3, webrtc::DataRate::KilobitsPerSec(2500), + webrtc::DataRate::KilobitsPerSec(2500), + webrtc::DataRate::KilobitsPerSec(600)}, + {960, 540, 3, webrtc::DataRate::KilobitsPerSec(1200), + webrtc::DataRate::KilobitsPerSec(1200), + webrtc::DataRate::KilobitsPerSec(350)}, + {640, 360, 2, webrtc::DataRate::KilobitsPerSec(700), + webrtc::DataRate::KilobitsPerSec(500), + webrtc::DataRate::KilobitsPerSec(150)}, + {480, 270, 2, webrtc::DataRate::KilobitsPerSec(450), + webrtc::DataRate::KilobitsPerSec(350), + webrtc::DataRate::KilobitsPerSec(150)}, + {320, 180, 1, webrtc::DataRate::KilobitsPerSec(200), + webrtc::DataRate::KilobitsPerSec(150), + webrtc::DataRate::KilobitsPerSec(30)}, + // As the resolution goes down, interpolate the target and max bitrates down + // towards zero. The min bitrate is still limited at 30 kbps and the target + // and the max will be capped from below accordingly. + {0, 0, 1, webrtc::DataRate::KilobitsPerSec(0), + webrtc::DataRate::KilobitsPerSec(0), + webrtc::DataRate::KilobitsPerSec(30)}}; + +constexpr webrtc::DataRate Interpolate(const webrtc::DataRate& a, + const webrtc::DataRate& b, + float rate) { + return a * (1.0 - rate) + b * rate; +} + +// TODO(webrtc:12415): Flip this to a kill switch when this feature launches. +bool EnableLowresBitrateInterpolation(const webrtc::FieldTrialsView& trials) { + return absl::StartsWith( + trials.Lookup("WebRTC-LowresSimulcastBitrateInterpolation"), "Enabled"); +} + +std::vector<SimulcastFormat> GetSimulcastFormats( + bool enable_lowres_bitrate_interpolation) { + std::vector<SimulcastFormat> formats; + formats.insert(formats.begin(), std::begin(kSimulcastFormats), + std::end(kSimulcastFormats)); + if (!enable_lowres_bitrate_interpolation) { + RTC_CHECK_GE(formats.size(), 2u); + SimulcastFormat& format0x0 = formats[formats.size() - 1]; + const SimulcastFormat& format_prev = formats[formats.size() - 2]; + format0x0.max_bitrate = format_prev.max_bitrate; + format0x0.target_bitrate = format_prev.target_bitrate; + format0x0.min_bitrate = format_prev.min_bitrate; + } + return formats; +} + +// Multiway: Number of temporal layers for each simulcast stream. +int DefaultNumberOfTemporalLayers(const webrtc::FieldTrialsView& trials) { + const std::string group_name = + trials.Lookup("WebRTC-VP8ConferenceTemporalLayers"); + if (group_name.empty()) + return kDefaultNumTemporalLayers; + + int num_temporal_layers = kDefaultNumTemporalLayers; + if (sscanf(group_name.c_str(), "%d", &num_temporal_layers) == 1 && + num_temporal_layers > 0 && + num_temporal_layers <= webrtc::kMaxTemporalStreams) { + return num_temporal_layers; + } + + RTC_LOG(LS_WARNING) << "Attempt to set number of temporal layers to " + "incorrect value: " + << group_name; + + return kDefaultNumTemporalLayers; +} + +int FindSimulcastFormatIndex(int width, + int height, + bool enable_lowres_bitrate_interpolation) { + RTC_DCHECK_GE(width, 0); + RTC_DCHECK_GE(height, 0); + const auto formats = GetSimulcastFormats(enable_lowres_bitrate_interpolation); + for (uint32_t i = 0; i < formats.size(); ++i) { + if (width * height >= formats[i].width * formats[i].height) { + return i; + } + } + RTC_DCHECK_NOTREACHED(); + return -1; +} + +} // namespace + +// Round size to nearest simulcast-friendly size. +// Simulcast stream width and height must both be dividable by +// |2 ^ (simulcast_layers - 1)|. +int NormalizeSimulcastSize(int size, size_t simulcast_layers) { + int base2_exponent = static_cast<int>(simulcast_layers) - 1; + const absl::optional<int> experimental_base2_exponent = + webrtc::NormalizeSimulcastSizeExperiment::GetBase2Exponent(); + if (experimental_base2_exponent && + (size > (1 << *experimental_base2_exponent))) { + base2_exponent = *experimental_base2_exponent; + } + return ((size >> base2_exponent) << base2_exponent); +} + +SimulcastFormat InterpolateSimulcastFormat( + int width, + int height, + absl::optional<double> max_roundup_rate, + bool enable_lowres_bitrate_interpolation) { + const auto formats = GetSimulcastFormats(enable_lowres_bitrate_interpolation); + const int index = FindSimulcastFormatIndex( + width, height, enable_lowres_bitrate_interpolation); + if (index == 0) + return formats[index]; + const int total_pixels_up = + formats[index - 1].width * formats[index - 1].height; + const int total_pixels_down = formats[index].width * formats[index].height; + const int total_pixels = width * height; + const float rate = (total_pixels_up - total_pixels) / + static_cast<float>(total_pixels_up - total_pixels_down); + + // Use upper resolution if `rate` is below the configured threshold. + size_t max_layers = (rate < max_roundup_rate.value_or(kDefaultMaxRoundupRate)) + ? formats[index - 1].max_layers + : formats[index].max_layers; + webrtc::DataRate max_bitrate = Interpolate(formats[index - 1].max_bitrate, + formats[index].max_bitrate, rate); + webrtc::DataRate target_bitrate = Interpolate( + formats[index - 1].target_bitrate, formats[index].target_bitrate, rate); + webrtc::DataRate min_bitrate = Interpolate(formats[index - 1].min_bitrate, + formats[index].min_bitrate, rate); + + return {width, height, max_layers, max_bitrate, target_bitrate, min_bitrate}; +} + +SimulcastFormat InterpolateSimulcastFormat( + int width, + int height, + bool enable_lowres_bitrate_interpolation) { + return InterpolateSimulcastFormat(width, height, absl::nullopt, + enable_lowres_bitrate_interpolation); +} + +webrtc::DataRate FindSimulcastMaxBitrate( + int width, + int height, + bool enable_lowres_bitrate_interpolation) { + return InterpolateSimulcastFormat(width, height, + enable_lowres_bitrate_interpolation) + .max_bitrate; +} + +webrtc::DataRate FindSimulcastTargetBitrate( + int width, + int height, + bool enable_lowres_bitrate_interpolation) { + return InterpolateSimulcastFormat(width, height, + enable_lowres_bitrate_interpolation) + .target_bitrate; +} + +webrtc::DataRate FindSimulcastMinBitrate( + int width, + int height, + bool enable_lowres_bitrate_interpolation) { + return InterpolateSimulcastFormat(width, height, + enable_lowres_bitrate_interpolation) + .min_bitrate; +} + +void BoostMaxSimulcastLayer(webrtc::DataRate max_bitrate, + std::vector<webrtc::VideoStream>* layers) { + if (layers->empty()) + return; + + const webrtc::DataRate total_bitrate = GetTotalMaxBitrate(*layers); + + // We're still not using all available bits. + if (total_bitrate < max_bitrate) { + // Spend additional bits to boost the max layer. + const webrtc::DataRate bitrate_left = max_bitrate - total_bitrate; + layers->back().max_bitrate_bps += bitrate_left.bps(); + } +} + +webrtc::DataRate GetTotalMaxBitrate( + const std::vector<webrtc::VideoStream>& layers) { + if (layers.empty()) + return webrtc::DataRate::Zero(); + + int total_max_bitrate_bps = 0; + for (size_t s = 0; s < layers.size() - 1; ++s) { + total_max_bitrate_bps += layers[s].target_bitrate_bps; + } + total_max_bitrate_bps += layers.back().max_bitrate_bps; + return webrtc::DataRate::BitsPerSec(total_max_bitrate_bps); +} + +size_t LimitSimulcastLayerCount(int width, + int height, + size_t need_layers, + size_t layer_count, + const webrtc::FieldTrialsView& trials) { + if (!absl::StartsWith(trials.Lookup(kUseLegacySimulcastLayerLimitFieldTrial), + "Disabled")) { + // Max layers from one higher resolution in kSimulcastFormats will be used + // if the ratio (pixels_up - pixels) / (pixels_up - pixels_down) is less + // than configured `max_ratio`. pixels_down is the selected index in + // kSimulcastFormats based on pixels. + webrtc::FieldTrialOptional<double> max_ratio("max_ratio"); + webrtc::ParseFieldTrial({&max_ratio}, + trials.Lookup("WebRTC-SimulcastLayerLimitRoundUp")); + + const bool enable_lowres_bitrate_interpolation = + EnableLowresBitrateInterpolation(trials); + size_t adaptive_layer_count = std::max( + need_layers, + InterpolateSimulcastFormat(width, height, max_ratio.GetOptional(), + enable_lowres_bitrate_interpolation) + .max_layers); + if (layer_count > adaptive_layer_count) { + RTC_LOG(LS_WARNING) << "Reducing simulcast layer count from " + << layer_count << " to " << adaptive_layer_count; + layer_count = adaptive_layer_count; + } + } + return layer_count; +} + +std::vector<webrtc::VideoStream> GetSimulcastConfig( + size_t min_layers, + size_t max_layers, + int width, + int height, + double bitrate_priority, + int max_qp, + bool is_screenshare_with_conference_mode, + bool temporal_layers_supported, + const webrtc::FieldTrialsView& trials) { + RTC_DCHECK_LE(min_layers, max_layers); + RTC_DCHECK(max_layers > 1 || is_screenshare_with_conference_mode); + + const bool base_heavy_tl3_rate_alloc = + webrtc::RateControlSettings::ParseFromKeyValueConfig(&trials) + .Vp8BaseHeavyTl3RateAllocation(); + if (is_screenshare_with_conference_mode) { + return GetScreenshareLayers(max_layers, width, height, bitrate_priority, + max_qp, temporal_layers_supported, + base_heavy_tl3_rate_alloc, trials); + } else { + // Some applications rely on the old behavior limiting the simulcast layer + // count based on the resolution automatically, which they can get through + // the WebRTC-LegacySimulcastLayerLimit field trial until they update. + max_layers = + LimitSimulcastLayerCount(width, height, min_layers, max_layers, trials); + + return GetNormalSimulcastLayers(max_layers, width, height, bitrate_priority, + max_qp, temporal_layers_supported, + base_heavy_tl3_rate_alloc, trials); + } +} + +std::vector<webrtc::VideoStream> GetNormalSimulcastLayers( + size_t layer_count, + int width, + int height, + double bitrate_priority, + int max_qp, + bool temporal_layers_supported, + bool base_heavy_tl3_rate_alloc, + const webrtc::FieldTrialsView& trials) { + std::vector<webrtc::VideoStream> layers(layer_count); + + const bool enable_lowres_bitrate_interpolation = + EnableLowresBitrateInterpolation(trials); + + // Format width and height has to be divisible by |2 ^ num_simulcast_layers - + // 1|. + width = NormalizeSimulcastSize(width, layer_count); + height = NormalizeSimulcastSize(height, layer_count); + // Add simulcast streams, from highest resolution (`s` = num_simulcast_layers + // -1) to lowest resolution at `s` = 0. + for (size_t s = layer_count - 1;; --s) { + layers[s].width = width; + layers[s].height = height; + // TODO(pbos): Fill actual temporal-layer bitrate thresholds. + layers[s].max_qp = max_qp; + layers[s].num_temporal_layers = + temporal_layers_supported ? DefaultNumberOfTemporalLayers(trials) : 1; + layers[s].max_bitrate_bps = + FindSimulcastMaxBitrate(width, height, + enable_lowres_bitrate_interpolation) + .bps(); + layers[s].target_bitrate_bps = + FindSimulcastTargetBitrate(width, height, + enable_lowres_bitrate_interpolation) + .bps(); + int num_temporal_layers = DefaultNumberOfTemporalLayers(trials); + if (s == 0) { + // If alternative temporal rate allocation is selected, adjust the + // bitrate of the lowest simulcast stream so that absolute bitrate for + // the base temporal layer matches the bitrate for the base temporal + // layer with the default 3 simulcast streams. Otherwise we risk a + // higher threshold for receiving a feed at all. + float rate_factor = 1.0; + if (num_temporal_layers == 3) { + if (base_heavy_tl3_rate_alloc) { + // Base heavy allocation increases TL0 bitrate from 40% to 60%. + rate_factor = 0.4 / 0.6; + } + } else { + rate_factor = + webrtc::SimulcastRateAllocator::GetTemporalRateAllocation( + 3, 0, /*base_heavy_tl3_rate_alloc=*/false) / + webrtc::SimulcastRateAllocator::GetTemporalRateAllocation( + num_temporal_layers, 0, /*base_heavy_tl3_rate_alloc=*/false); + } + + layers[s].max_bitrate_bps = + static_cast<int>(layers[s].max_bitrate_bps * rate_factor); + layers[s].target_bitrate_bps = + static_cast<int>(layers[s].target_bitrate_bps * rate_factor); + } + layers[s].min_bitrate_bps = + FindSimulcastMinBitrate(width, height, + enable_lowres_bitrate_interpolation) + .bps(); + + // Ensure consistency. + layers[s].max_bitrate_bps = + std::max(layers[s].min_bitrate_bps, layers[s].max_bitrate_bps); + layers[s].target_bitrate_bps = + std::max(layers[s].min_bitrate_bps, layers[s].target_bitrate_bps); + + layers[s].max_framerate = kDefaultVideoMaxFramerate; + + width /= 2; + height /= 2; + + if (s == 0) { + break; + } + } + // Currently the relative bitrate priority of the sender is controlled by + // the value of the lowest VideoStream. + // TODO(bugs.webrtc.org/8630): The web specification describes being able to + // control relative bitrate for each individual simulcast layer, but this + // is currently just implemented per rtp sender. + layers[0].bitrate_priority = bitrate_priority; + return layers; +} + +std::vector<webrtc::VideoStream> GetScreenshareLayers( + size_t max_layers, + int width, + int height, + double bitrate_priority, + int max_qp, + bool temporal_layers_supported, + bool base_heavy_tl3_rate_alloc, + const webrtc::FieldTrialsView& trials) { + size_t num_simulcast_layers = + std::min<int>(max_layers, kScreenshareMaxSimulcastLayers); + + std::vector<webrtc::VideoStream> layers(num_simulcast_layers); + // For legacy screenshare in conference mode, tl0 and tl1 bitrates are + // piggybacked on the VideoCodec struct as target and max bitrates, + // respectively. See eg. webrtc::LibvpxVp8Encoder::SetRates(). + layers[0].width = width; + layers[0].height = height; + layers[0].max_qp = max_qp; + layers[0].max_framerate = 5; + layers[0].min_bitrate_bps = webrtc::kDefaultMinVideoBitrateBps; + layers[0].target_bitrate_bps = kScreenshareDefaultTl0Bitrate.bps(); + layers[0].max_bitrate_bps = kScreenshareDefaultTl1Bitrate.bps(); + layers[0].num_temporal_layers = temporal_layers_supported ? 2 : 1; + + // With simulcast enabled, add another spatial layer. This one will have a + // more normal layout, with the regular 3 temporal layer pattern and no fps + // restrictions. The base simulcast layer will still use legacy setup. + if (num_simulcast_layers == kScreenshareMaxSimulcastLayers) { + // Add optional upper simulcast layer. + int max_bitrate_bps; + bool using_boosted_bitrate = false; + if (!temporal_layers_supported) { + // Set the max bitrate to where the base layer would have been if temporal + // layers were enabled. + max_bitrate_bps = static_cast<int>( + kScreenshareHighStreamMaxBitrate.bps() * + webrtc::SimulcastRateAllocator::GetTemporalRateAllocation( + kScreenshareTemporalLayers, 0, base_heavy_tl3_rate_alloc)); + } else { + // Experimental temporal layer mode used, use increased max bitrate. + max_bitrate_bps = kScreenshareHighStreamMaxBitrate.bps(); + using_boosted_bitrate = true; + } + + layers[1].width = width; + layers[1].height = height; + layers[1].max_qp = max_qp; + layers[1].max_framerate = kDefaultVideoMaxFramerate; + layers[1].num_temporal_layers = + temporal_layers_supported ? kScreenshareTemporalLayers : 1; + layers[1].min_bitrate_bps = using_boosted_bitrate + ? kScreenshareHighStreamMinBitrate.bps() + : layers[0].target_bitrate_bps * 2; + layers[1].target_bitrate_bps = max_bitrate_bps; + layers[1].max_bitrate_bps = max_bitrate_bps; + } + + // The bitrate priority currently implemented on a per-sender level, so we + // just set it for the first simulcast layer. + layers[0].bitrate_priority = bitrate_priority; + return layers; +} + +} // namespace cricket diff --git a/third_party/libwebrtc/video/config/simulcast.h b/third_party/libwebrtc/video/config/simulcast.h new file mode 100644 index 0000000000..32af168bcd --- /dev/null +++ b/third_party/libwebrtc/video/config/simulcast.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_CONFIG_SIMULCAST_H_ +#define VIDEO_CONFIG_SIMULCAST_H_ + +#include <stddef.h> + +#include <vector> + +#include "api/field_trials_view.h" +#include "api/units/data_rate.h" +#include "video/config/video_encoder_config.h" + +namespace cricket { + +// Gets the total maximum bitrate for the `streams`. +webrtc::DataRate GetTotalMaxBitrate( + const std::vector<webrtc::VideoStream>& streams); + +// Adds any bitrate of `max_bitrate` that is above the total maximum bitrate for +// the `layers` to the highest quality layer. +void BoostMaxSimulcastLayer(webrtc::DataRate max_bitrate, + std::vector<webrtc::VideoStream>* layers); + +// Round size to nearest simulcast-friendly size +int NormalizeSimulcastSize(int size, size_t simulcast_layers); + +// Gets simulcast settings. +std::vector<webrtc::VideoStream> GetSimulcastConfig( + size_t min_layers, + size_t max_layers, + int width, + int height, + double bitrate_priority, + int max_qp, + bool is_screenshare_with_conference_mode, + bool temporal_layers_supported, + const webrtc::FieldTrialsView& trials); + +// Gets the simulcast config layers for a non-screensharing case. +std::vector<webrtc::VideoStream> GetNormalSimulcastLayers( + size_t max_layers, + int width, + int height, + double bitrate_priority, + int max_qp, + bool temporal_layers_supported, + bool base_heavy_tl3_rate_alloc, + const webrtc::FieldTrialsView& trials); + +// Gets simulcast config layers for screenshare settings. +std::vector<webrtc::VideoStream> GetScreenshareLayers( + size_t max_layers, + int width, + int height, + double bitrate_priority, + int max_qp, + bool temporal_layers_supported, + bool base_heavy_tl3_rate_alloc, + const webrtc::FieldTrialsView& trials); + +} // namespace cricket + +#endif // VIDEO_CONFIG_SIMULCAST_H_ diff --git a/third_party/libwebrtc/video/config/simulcast_unittest.cc b/third_party/libwebrtc/video/config/simulcast_unittest.cc new file mode 100644 index 0000000000..152a0f9525 --- /dev/null +++ b/third_party/libwebrtc/video/config/simulcast_unittest.cc @@ -0,0 +1,525 @@ +/* + * Copyright 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/config/simulcast.h" + +#include "api/transport/field_trial_based_config.h" +#include "media/base/media_constants.h" +#include "test/field_trial.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { +constexpr int kQpMax = 55; +constexpr double kBitratePriority = 2.0; +constexpr bool kScreenshare = true; +constexpr int kDefaultTemporalLayers = 3; // Value from simulcast.cc. + +// Values from kSimulcastConfigs in simulcast.cc. +const std::vector<VideoStream> GetSimulcastBitrates720p() { + std::vector<VideoStream> streams(3); + streams[0].min_bitrate_bps = 30000; + streams[0].target_bitrate_bps = 150000; + streams[0].max_bitrate_bps = 200000; + streams[1].min_bitrate_bps = 150000; + streams[1].target_bitrate_bps = 500000; + streams[1].max_bitrate_bps = 700000; + streams[2].min_bitrate_bps = 600000; + streams[2].target_bitrate_bps = 2500000; + streams[2].max_bitrate_bps = 2500000; + return streams; +} +} // namespace + +TEST(SimulcastTest, TotalMaxBitrateIsZeroForNoStreams) { + std::vector<VideoStream> streams; + EXPECT_EQ(0, cricket::GetTotalMaxBitrate(streams).bps()); +} + +TEST(SimulcastTest, GetTotalMaxBitrateForSingleStream) { + std::vector<VideoStream> streams(1); + streams[0].max_bitrate_bps = 100000; + EXPECT_EQ(100000, cricket::GetTotalMaxBitrate(streams).bps()); +} + +TEST(SimulcastTest, GetTotalMaxBitrateForMultipleStreams) { + std::vector<VideoStream> streams(3); + streams[0].target_bitrate_bps = 100000; + streams[1].target_bitrate_bps = 200000; + streams[2].max_bitrate_bps = 400000; + EXPECT_EQ(700000, cricket::GetTotalMaxBitrate(streams).bps()); +} + +TEST(SimulcastTest, BandwidthAboveTotalMaxBitrateGivenToHighestStream) { + std::vector<VideoStream> streams(3); + streams[0].target_bitrate_bps = 100000; + streams[1].target_bitrate_bps = 200000; + streams[2].max_bitrate_bps = 400000; + + const webrtc::DataRate one_bps = webrtc::DataRate::BitsPerSec(1); + + // No bitrate above the total max to give to the highest stream. + const webrtc::DataRate max_total_bitrate = + cricket::GetTotalMaxBitrate(streams); + cricket::BoostMaxSimulcastLayer(max_total_bitrate, &streams); + EXPECT_EQ(400000, streams[2].max_bitrate_bps); + EXPECT_EQ(max_total_bitrate, cricket::GetTotalMaxBitrate(streams)); + + // The bitrate above the total max should be given to the highest stream. + cricket::BoostMaxSimulcastLayer(max_total_bitrate + one_bps, &streams); + EXPECT_EQ(400000 + 1, streams[2].max_bitrate_bps); + EXPECT_EQ(max_total_bitrate + one_bps, cricket::GetTotalMaxBitrate(streams)); +} + +TEST(SimulcastTest, GetConfig) { + const std::vector<VideoStream> kExpected = GetSimulcastBitrates720p(); + const FieldTrialBasedConfig trials; + + const size_t kMinLayers = 1; + const size_t kMaxLayers = 3; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 1280, 720, kBitratePriority, kQpMax, + !kScreenshare, true, trials); + + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(320u, streams[0].width); + EXPECT_EQ(180u, streams[0].height); + EXPECT_EQ(640u, streams[1].width); + EXPECT_EQ(360u, streams[1].height); + EXPECT_EQ(1280u, streams[2].width); + EXPECT_EQ(720u, streams[2].height); + + for (size_t i = 0; i < streams.size(); ++i) { + EXPECT_EQ(size_t{kDefaultTemporalLayers}, streams[i].num_temporal_layers); + EXPECT_EQ(cricket::kDefaultVideoMaxFramerate, streams[i].max_framerate); + EXPECT_EQ(kQpMax, streams[i].max_qp); + EXPECT_EQ(kExpected[i].min_bitrate_bps, streams[i].min_bitrate_bps); + EXPECT_EQ(kExpected[i].target_bitrate_bps, streams[i].target_bitrate_bps); + EXPECT_EQ(kExpected[i].max_bitrate_bps, streams[i].max_bitrate_bps); + EXPECT_TRUE(streams[i].active); + } + // Currently set on lowest stream. + EXPECT_EQ(kBitratePriority, streams[0].bitrate_priority); + EXPECT_FALSE(streams[1].bitrate_priority); + EXPECT_FALSE(streams[2].bitrate_priority); +} + +TEST(SimulcastTest, GetConfigWithBaseHeavyVP8TL3RateAllocation) { + test::ScopedFieldTrials field_trials( + "WebRTC-UseBaseHeavyVP8TL3RateAllocation/Enabled/"); + FieldTrialBasedConfig trials; + + const std::vector<VideoStream> kExpected = GetSimulcastBitrates720p(); + + const size_t kMinLayers = 1; + const size_t kMaxLayers = 3; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 1280, 720, kBitratePriority, kQpMax, + !kScreenshare, true, trials); + + EXPECT_EQ(kExpected[0].min_bitrate_bps, streams[0].min_bitrate_bps); + EXPECT_EQ(static_cast<int>(0.4 * kExpected[0].target_bitrate_bps / 0.6), + streams[0].target_bitrate_bps); + EXPECT_EQ(static_cast<int>(0.4 * kExpected[0].max_bitrate_bps / 0.6), + streams[0].max_bitrate_bps); + for (size_t i = 1; i < streams.size(); ++i) { + EXPECT_EQ(kExpected[i].min_bitrate_bps, streams[i].min_bitrate_bps); + EXPECT_EQ(kExpected[i].target_bitrate_bps, streams[i].target_bitrate_bps); + EXPECT_EQ(kExpected[i].max_bitrate_bps, streams[i].max_bitrate_bps); + } +} + +TEST(SimulcastTest, GetConfigWithLimitedMaxLayers) { + const size_t kMinLayers = 1; + const size_t kMaxLayers = 2; + FieldTrialBasedConfig trials; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 1280, 720, kBitratePriority, kQpMax, + !kScreenshare, true, trials); + + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(640u, streams[0].width); + EXPECT_EQ(360u, streams[0].height); + EXPECT_EQ(1280u, streams[1].width); + EXPECT_EQ(720u, streams[1].height); +} + +TEST(SimulcastTest, GetConfigWithLimitedMaxLayersForResolution) { + test::ScopedFieldTrials field_trials( + "WebRTC-LegacySimulcastLayerLimit/Enabled/"); + FieldTrialBasedConfig trials; + const size_t kMinLayers = 1; + const size_t kMaxLayers = 3; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 800, 600, kBitratePriority, kQpMax, !kScreenshare, + true, trials); + + EXPECT_EQ(2u, streams.size()); + EXPECT_EQ(400u, streams[0].width); + EXPECT_EQ(300u, streams[0].height); + EXPECT_EQ(800u, streams[1].width); + EXPECT_EQ(600u, streams[1].height); +} + +TEST(SimulcastTest, GetConfigWithLowResolutionScreenshare) { + test::ScopedFieldTrials field_trials( + "WebRTC-LegacySimulcastLayerLimit/Enabled/"); + FieldTrialBasedConfig trials; + const size_t kMinLayers = 1; + const size_t kMaxLayers = 3; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 100, 100, kBitratePriority, kQpMax, kScreenshare, + true, trials); + + // Simulcast streams number is never decreased for screenshare, + // even for very low resolution. + EXPECT_GT(streams.size(), 1u); +} + +TEST(SimulcastTest, GetConfigWithNotLimitedMaxLayersForResolution) { + test::ScopedFieldTrials field_trials( + "WebRTC-LegacySimulcastLayerLimit/Disabled/"); + FieldTrialBasedConfig trials; + const size_t kMinLayers = 1; + const size_t kMaxLayers = 3; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 800, 600, kBitratePriority, kQpMax, !kScreenshare, + true, trials); + + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(200u, streams[0].width); + EXPECT_EQ(150u, streams[0].height); + EXPECT_EQ(400u, streams[1].width); + EXPECT_EQ(300u, streams[1].height); + EXPECT_EQ(800u, streams[2].width); + EXPECT_EQ(600u, streams[2].height); +} + +TEST(SimulcastTest, GetConfigWithNormalizedResolution) { + FieldTrialBasedConfig trials; + const size_t kMinLayers = 1; + const size_t kMaxLayers = 2; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 640 + 1, 360 + 1, kBitratePriority, kQpMax, + !kScreenshare, true, trials); + + // Must be divisible by |2 ^ (num_layers - 1)|. + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(320u, streams[0].width); + EXPECT_EQ(180u, streams[0].height); + EXPECT_EQ(640u, streams[1].width); + EXPECT_EQ(360u, streams[1].height); +} + +TEST(SimulcastTest, GetConfigWithNormalizedResolutionDivisibleBy4) { + test::ScopedFieldTrials field_trials( + "WebRTC-NormalizeSimulcastResolution/Enabled-2/"); + FieldTrialBasedConfig trials; + + const size_t kMinLayers = 1; + const size_t kMaxLayers = 2; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 709, 501, kBitratePriority, kQpMax, !kScreenshare, + true, trials); + + // Must be divisible by |2 ^ 2|. + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(354u, streams[0].width); + EXPECT_EQ(250u, streams[0].height); + EXPECT_EQ(708u, streams[1].width); + EXPECT_EQ(500u, streams[1].height); +} + +TEST(SimulcastTest, GetConfigWithNormalizedResolutionDivisibleBy8) { + test::ScopedFieldTrials field_trials( + "WebRTC-NormalizeSimulcastResolution/Enabled-3/"); + FieldTrialBasedConfig trials; + + const size_t kMinLayers = 1; + const size_t kMaxLayers = 2; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 709, 501, kBitratePriority, kQpMax, !kScreenshare, + true, trials); + + // Must be divisible by |2 ^ 3|. + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(352u, streams[0].width); + EXPECT_EQ(248u, streams[0].height); + EXPECT_EQ(704u, streams[1].width); + EXPECT_EQ(496u, streams[1].height); +} + +TEST(SimulcastTest, GetConfigForLegacyLayerLimit) { + test::ScopedFieldTrials field_trials( + "WebRTC-LegacySimulcastLayerLimit/Enabled/"); + FieldTrialBasedConfig trials; + + const size_t kMinLayers = 1; + const int kMaxLayers = 3; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 320, 180, kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(1u, streams.size()); + + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 640, 360, + kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(2u, streams.size()); + + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 1920, 1080, + kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(3u, streams.size()); +} + +TEST(SimulcastTest, GetConfigForLegacyLayerLimitWithRequiredHD) { + test::ScopedFieldTrials field_trials( + "WebRTC-LegacySimulcastLayerLimit/Enabled/"); + FieldTrialBasedConfig trials; + + const size_t kMinLayers = 3; // "HD" layer must be present! + const int kMaxLayers = 3; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 320, 180, kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(3u, streams.size()); + + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 640, 360, + kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(3u, streams.size()); + + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 1920, 1080, + kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(3u, streams.size()); +} + +TEST(SimulcastTest, GetConfigForScreenshareSimulcast) { + FieldTrialBasedConfig trials; + const size_t kMinLayers = 1; + const size_t kMaxLayers = 3; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 1400, 800, kBitratePriority, kQpMax, kScreenshare, + true, trials); + + EXPECT_GT(streams.size(), 1u); + for (size_t i = 0; i < streams.size(); ++i) { + EXPECT_EQ(1400u, streams[i].width) << "Screen content never scaled."; + EXPECT_EQ(800u, streams[i].height) << "Screen content never scaled."; + EXPECT_EQ(kQpMax, streams[i].max_qp); + EXPECT_TRUE(streams[i].active); + EXPECT_GT(streams[i].num_temporal_layers, size_t{1}); + EXPECT_GT(streams[i].max_framerate, 0); + EXPECT_GT(streams[i].min_bitrate_bps, 0); + EXPECT_GT(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps); + EXPECT_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps); + } +} + +TEST(SimulcastTest, GetConfigForScreenshareSimulcastWithLimitedMaxLayers) { + FieldTrialBasedConfig trials; + const size_t kMinLayers = 1; + const size_t kMaxLayers = 1; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 1400, 800, kBitratePriority, kQpMax, kScreenshare, + true, trials); + + EXPECT_EQ(kMaxLayers, streams.size()); +} + +TEST(SimulcastTest, AveragesBitratesForNonStandardResolution) { + FieldTrialBasedConfig trials; + const size_t kMinLayers = 1; + const size_t kMaxLayers = 3; + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, 900, 800, kBitratePriority, kQpMax, !kScreenshare, + true, trials); + + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(900u, streams[2].width); + EXPECT_EQ(800u, streams[2].height); + EXPECT_EQ(1850000, streams[2].max_bitrate_bps); + EXPECT_EQ(1850000, streams[2].target_bitrate_bps); + EXPECT_EQ(475000, streams[2].min_bitrate_bps); +} + +TEST(SimulcastTest, BitratesForCloseToStandardResolution) { + const size_t kMinLayers = 1; + const size_t kMaxLayers = 3; + // Resolution very close to 720p in number of pixels + const size_t kWidth = 1280; + const size_t kHeight = 716; + const std::vector<VideoStream> kExpectedNear = GetSimulcastBitrates720p(); + FieldTrialBasedConfig trials; + + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + kMinLayers, kMaxLayers, kWidth, kHeight, kBitratePriority, kQpMax, + !kScreenshare, true, trials); + + EXPECT_EQ(kMaxLayers, streams.size()); + EXPECT_EQ(kWidth, streams[2].width); + EXPECT_EQ(kHeight, streams[2].height); + for (size_t i = 0; i < streams.size(); ++i) { + EXPECT_NEAR(kExpectedNear[i].max_bitrate_bps, streams[i].max_bitrate_bps, + 20000); + EXPECT_NEAR(kExpectedNear[i].target_bitrate_bps, + streams[i].target_bitrate_bps, 20000); + EXPECT_NEAR(kExpectedNear[i].min_bitrate_bps, streams[i].min_bitrate_bps, + 20000); + } +} + +TEST(SimulcastTest, MaxLayersWithRoundUpDisabled) { + test::ScopedFieldTrials field_trials( + "WebRTC-SimulcastLayerLimitRoundUp/max_ratio:0.0/"); + FieldTrialBasedConfig trials; + const size_t kMinLayers = 1; + const int kMaxLayers = 3; + + std::vector<VideoStream> streams; + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 960, 540, + kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(3u, streams.size()); + // <960x540: 2 layers + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 960, 539, + kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(2u, streams.size()); + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 480, 270, + kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(2u, streams.size()); + // <480x270: 1 layer + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 480, 269, + kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(1u, streams.size()); +} + +TEST(SimulcastTest, MaxLayersWithDefaultRoundUpRatio) { + // Default: "WebRTC-SimulcastLayerLimitRoundUp/max_ratio:0.1/" + FieldTrialBasedConfig trials; + const size_t kMinLayers = 1; + const int kMaxLayers = 3; + + std::vector<VideoStream> streams; + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 960, 540, + kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(3u, streams.size()); + // Lowest cropped height where max layers from higher resolution is used. + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 960, 512, + kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(3u, streams.size()); + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 960, 508, + kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(2u, streams.size()); + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 480, 270, + kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(2u, streams.size()); + // Lowest cropped height where max layers from higher resolution is used. + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 480, 256, + kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(2u, streams.size()); + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 480, 254, + kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(1u, streams.size()); +} + +TEST(SimulcastTest, MaxLayersWithRoundUpRatio) { + test::ScopedFieldTrials field_trials( + "WebRTC-SimulcastLayerLimitRoundUp/max_ratio:0.13/"); + FieldTrialBasedConfig trials; + const size_t kMinLayers = 1; + const int kMaxLayers = 3; + + std::vector<VideoStream> streams; + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 480, 270, + kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(2u, streams.size()); + // Lowest cropped height where max layers from higher resolution is used. + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 480, 252, + kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(2u, streams.size()); + streams = cricket::GetSimulcastConfig(kMinLayers, kMaxLayers, 480, 250, + kBitratePriority, kQpMax, !kScreenshare, + true, trials); + EXPECT_EQ(1u, streams.size()); +} + +TEST(SimulcastTest, BitratesInterpolatedForResBelow180p) { + // TODO(webrtc:12415): Remove when feature launches. + test::ScopedFieldTrials field_trials( + "WebRTC-LowresSimulcastBitrateInterpolation/Enabled/"); + + const size_t kMaxLayers = 3; + FieldTrialBasedConfig trials; + + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + /* min_layers = */ 1, kMaxLayers, /* width = */ 960, /* height = */ 540, + kBitratePriority, kQpMax, !kScreenshare, true, trials); + + ASSERT_EQ(streams.size(), kMaxLayers); + EXPECT_EQ(240u, streams[0].width); + EXPECT_EQ(135u, streams[0].height); + EXPECT_EQ(streams[0].max_bitrate_bps, 112500); + EXPECT_EQ(streams[0].target_bitrate_bps, 84375); + EXPECT_EQ(streams[0].min_bitrate_bps, 30000); +} + +TEST(SimulcastTest, BitratesConsistentForVerySmallRes) { + // TODO(webrtc:12415): Remove when feature launches. + test::ScopedFieldTrials field_trials( + "WebRTC-LowresSimulcastBitrateInterpolation/Enabled/"); + + FieldTrialBasedConfig trials; + + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + /* min_layers = */ 1, /* max_layers = */ 3, /* width = */ 1, + /* height = */ 1, kBitratePriority, kQpMax, !kScreenshare, true, trials); + + ASSERT_TRUE(!streams.empty()); + EXPECT_EQ(1u, streams[0].width); + EXPECT_EQ(1u, streams[0].height); + EXPECT_EQ(streams[0].max_bitrate_bps, 30000); + EXPECT_EQ(streams[0].target_bitrate_bps, 30000); + EXPECT_EQ(streams[0].min_bitrate_bps, 30000); +} + +TEST(SimulcastTest, + BitratesNotInterpolatedForResBelow180pWhenDisabledTrialSet) { + test::ScopedFieldTrials field_trials( + "WebRTC-LowresSimulcastBitrateInterpolation/Disabled/"); + + const size_t kMaxLayers = 3; + FieldTrialBasedConfig trials; + + std::vector<VideoStream> streams = cricket::GetSimulcastConfig( + /* min_layers = */ 1, kMaxLayers, /* width = */ 960, /* height = */ 540, + kBitratePriority, kQpMax, !kScreenshare, true, trials); + + ASSERT_EQ(streams.size(), kMaxLayers); + EXPECT_EQ(240u, streams[0].width); + EXPECT_EQ(135u, streams[0].height); + EXPECT_EQ(streams[0].max_bitrate_bps, 200000); + EXPECT_EQ(streams[0].target_bitrate_bps, 150000); + EXPECT_EQ(streams[0].min_bitrate_bps, 30000); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/video/config/streams_config_gn/moz.build b/third_party/libwebrtc/video/config/streams_config_gn/moz.build new file mode 100644 index 0000000000..e2d2f9963d --- /dev/null +++ b/third_party/libwebrtc/video/config/streams_config_gn/moz.build @@ -0,0 +1,238 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/video/config/encoder_stream_factory.cc", + "/third_party/libwebrtc/video/config/simulcast.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_ENABLE_LIBEVENT"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "GLESv2", + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_ENABLE_LIBEVENT"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_ENABLE_LIBEVENT"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["RTC_ENABLE_WIN_WGC"] = True + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["TARGET_CPU"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["TARGET_CPU"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["TARGET_CPU"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["TARGET_CPU"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["TARGET_CPU"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["TARGET_CPU"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "arm": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "x86": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "x86_64": + + DEFINES["_GNU_SOURCE"] = True + +Library("streams_config_gn") diff --git a/third_party/libwebrtc/video/config/video_encoder_config.cc b/third_party/libwebrtc/video/config/video_encoder_config.cc new file mode 100644 index 0000000000..84442aeddf --- /dev/null +++ b/third_party/libwebrtc/video/config/video_encoder_config.cc @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "video/config/video_encoder_config.h" + +#include <string> + +#include "rtc_base/checks.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { +VideoStream::VideoStream() + : width(0), + height(0), + max_framerate(-1), + min_bitrate_bps(-1), + target_bitrate_bps(-1), + max_bitrate_bps(-1), + scale_resolution_down_by(-1.), + max_qp(-1), + num_temporal_layers(absl::nullopt), + active(true) {} +VideoStream::VideoStream(const VideoStream& other) = default; + +VideoStream::~VideoStream() = default; + +std::string VideoStream::ToString() const { + char buf[1024]; + rtc::SimpleStringBuilder ss(buf); + ss << "{width: " << width; + ss << ", height: " << height; + ss << ", max_framerate: " << max_framerate; + ss << ", min_bitrate_bps:" << min_bitrate_bps; + ss << ", target_bitrate_bps:" << target_bitrate_bps; + ss << ", max_bitrate_bps:" << max_bitrate_bps; + ss << ", max_qp: " << max_qp; + ss << ", num_temporal_layers: " << num_temporal_layers.value_or(1); + ss << ", bitrate_priority: " << bitrate_priority.value_or(0); + ss << ", active: " << active; + ss << ", scale_down_by: " << scale_resolution_down_by; + + return ss.str(); +} + +VideoEncoderConfig::VideoEncoderConfig() + : codec_type(kVideoCodecGeneric), + video_format("Unset"), + content_type(ContentType::kRealtimeVideo), + frame_drop_enabled(false), + encoder_specific_settings(nullptr), + min_transmit_bitrate_bps(0), + max_bitrate_bps(0), + bitrate_priority(1.0), + number_of_streams(0), + legacy_conference_mode(false), + is_quality_scaling_allowed(false) {} + +VideoEncoderConfig::VideoEncoderConfig(VideoEncoderConfig&&) = default; + +VideoEncoderConfig::~VideoEncoderConfig() = default; + +std::string VideoEncoderConfig::ToString() const { + char buf[1024]; + rtc::SimpleStringBuilder ss(buf); + ss << "{codec_type: " << CodecTypeToPayloadString(codec_type); + ss << ", content_type: "; + switch (content_type) { + case ContentType::kRealtimeVideo: + ss << "kRealtimeVideo"; + break; + case ContentType::kScreen: + ss << "kScreenshare"; + break; + } + ss << ", frame_drop_enabled: " << frame_drop_enabled; + ss << ", encoder_specific_settings: "; + ss << (encoder_specific_settings != nullptr ? "(ptr)" : "NULL"); + + ss << ", min_transmit_bitrate_bps: " << min_transmit_bitrate_bps; + ss << '}'; + return ss.str(); +} + +VideoEncoderConfig::VideoEncoderConfig(const VideoEncoderConfig&) = default; + +void VideoEncoderConfig::EncoderSpecificSettings::FillEncoderSpecificSettings( + VideoCodec* codec) const { + if (codec->codecType == kVideoCodecVP8) { + FillVideoCodecVp8(codec->VP8()); + } else if (codec->codecType == kVideoCodecVP9) { + FillVideoCodecVp9(codec->VP9()); + } else if (codec->codecType == kVideoCodecAV1) { + FillVideoCodecAv1(codec->AV1()); + } else { + RTC_DCHECK_NOTREACHED() + << "Encoder specifics set/used for unknown codec type."; + } +} + +void VideoEncoderConfig::EncoderSpecificSettings::FillVideoCodecVp8( + VideoCodecVP8* vp8_settings) const { + RTC_DCHECK_NOTREACHED(); +} + +void VideoEncoderConfig::EncoderSpecificSettings::FillVideoCodecVp9( + VideoCodecVP9* vp9_settings) const { + RTC_DCHECK_NOTREACHED(); +} + +void VideoEncoderConfig::EncoderSpecificSettings::FillVideoCodecAv1( + VideoCodecAV1* av1_settings) const { + RTC_DCHECK_NOTREACHED(); +} + +VideoEncoderConfig::Vp8EncoderSpecificSettings::Vp8EncoderSpecificSettings( + const VideoCodecVP8& specifics) + : specifics_(specifics) {} + +void VideoEncoderConfig::Vp8EncoderSpecificSettings::FillVideoCodecVp8( + VideoCodecVP8* vp8_settings) const { + *vp8_settings = specifics_; +} + +VideoEncoderConfig::Vp9EncoderSpecificSettings::Vp9EncoderSpecificSettings( + const VideoCodecVP9& specifics) + : specifics_(specifics) {} + +void VideoEncoderConfig::Vp9EncoderSpecificSettings::FillVideoCodecVp9( + VideoCodecVP9* vp9_settings) const { + *vp9_settings = specifics_; +} + +VideoEncoderConfig::Av1EncoderSpecificSettings::Av1EncoderSpecificSettings( + const VideoCodecAV1& specifics) + : specifics_(specifics) {} + +void VideoEncoderConfig::Av1EncoderSpecificSettings::FillVideoCodecAv1( + VideoCodecAV1* av1_settings) const { + *av1_settings = specifics_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/video/config/video_encoder_config.h b/third_party/libwebrtc/video/config/video_encoder_config.h new file mode 100644 index 0000000000..cb0644a7fd --- /dev/null +++ b/third_party/libwebrtc/video/config/video_encoder_config.h @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_CONFIG_VIDEO_ENCODER_CONFIG_H_ +#define VIDEO_CONFIG_VIDEO_ENCODER_CONFIG_H_ + +#include <stddef.h> + +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "api/video/resolution.h" +#include "api/video_codecs/scalability_mode.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" +#include "rtc_base/ref_count.h" + +namespace webrtc { + +// The `VideoStream` struct describes a simulcast layer, or "stream". +struct VideoStream { + VideoStream(); + ~VideoStream(); + VideoStream(const VideoStream& other); + std::string ToString() const; + + // Width/Height in pixels. + // This is the actual width and height used to configure encoder, + // which might be less than `requested_resolution` due to adaptation + // or due to the source providing smaller frames than requested. + size_t width; + size_t height; + + // Frame rate in fps. + int max_framerate; + + // Bitrate, in bps, for the stream. + int min_bitrate_bps; + int target_bitrate_bps; + int max_bitrate_bps; + + // Scaling factor applied to the stream size. + // `width` and `height` values are already scaled down. + double scale_resolution_down_by; + + // Maximum Quantization Parameter to use when encoding the stream. + int max_qp; + + // Determines the number of temporal layers that the stream should be + // encoded with. This value should be greater than zero. + // TODO(brandtr): This class is used both for configuring the encoder + // (meaning that this field _must_ be set), and for signaling the app-level + // encoder settings (meaning that the field _may_ be set). We should separate + // this and remove this optional instead. + absl::optional<size_t> num_temporal_layers; + + // The priority of this stream, to be used when allocating resources + // between multiple streams. + absl::optional<double> bitrate_priority; + + absl::optional<ScalabilityMode> scalability_mode; + + // If this stream is enabled by the user, or not. + bool active; + + // An optional user supplied max_frame_resolution + // than can be set independently of (adapted) VideoSource. + // This value is set from RtpEncodingParameters::requested_resolution + // (i.e. used for signaling app-level settings). + // + // The actual encode resolution is in `width` and `height`, + // which can be lower than requested_resolution, + // e.g. if source only provides lower resolution or + // if resource adaptation is active. + absl::optional<Resolution> requested_resolution; +}; + +class VideoEncoderConfig { + public: + // These are reference counted to permit copying VideoEncoderConfig and be + // kept alive until all encoder_specific_settings go out of scope. + // TODO(kthelgason): Consider removing the need for copying VideoEncoderConfig + // and use absl::optional for encoder_specific_settings instead. + class EncoderSpecificSettings : public rtc::RefCountInterface { + public: + // TODO(pbos): Remove FillEncoderSpecificSettings as soon as VideoCodec is + // not in use and encoder implementations ask for codec-specific structs + // directly. + void FillEncoderSpecificSettings(VideoCodec* codec_struct) const; + + virtual void FillVideoCodecVp8(VideoCodecVP8* vp8_settings) const; + virtual void FillVideoCodecVp9(VideoCodecVP9* vp9_settings) const; + virtual void FillVideoCodecAv1(VideoCodecAV1* av1_settings) const; + + private: + ~EncoderSpecificSettings() override {} + friend class VideoEncoderConfig; + }; + + class Vp8EncoderSpecificSettings : public EncoderSpecificSettings { + public: + explicit Vp8EncoderSpecificSettings(const VideoCodecVP8& specifics); + void FillVideoCodecVp8(VideoCodecVP8* vp8_settings) const override; + + private: + VideoCodecVP8 specifics_; + }; + + class Vp9EncoderSpecificSettings : public EncoderSpecificSettings { + public: + explicit Vp9EncoderSpecificSettings(const VideoCodecVP9& specifics); + void FillVideoCodecVp9(VideoCodecVP9* vp9_settings) const override; + + private: + VideoCodecVP9 specifics_; + }; + + class Av1EncoderSpecificSettings : public EncoderSpecificSettings { + public: + explicit Av1EncoderSpecificSettings(const VideoCodecAV1& specifics); + void FillVideoCodecAv1(VideoCodecAV1* av1_settings) const override; + + private: + VideoCodecAV1 specifics_; + }; + + enum class ContentType { + kRealtimeVideo, + kScreen, + }; + + class VideoStreamFactoryInterface : public rtc::RefCountInterface { + public: + // An implementation should return a std::vector<VideoStream> with the + // wanted VideoStream settings for the given video resolution. + // The size of the vector may not be larger than + // `encoder_config.number_of_streams`. + virtual std::vector<VideoStream> CreateEncoderStreams( + int frame_width, + int frame_height, + const VideoEncoderConfig& encoder_config) = 0; + + protected: + ~VideoStreamFactoryInterface() override {} + }; + + VideoEncoderConfig& operator=(VideoEncoderConfig&&) = default; + VideoEncoderConfig& operator=(const VideoEncoderConfig&) = delete; + + // Mostly used by tests. Avoid creating copies if you can. + VideoEncoderConfig Copy() const { return VideoEncoderConfig(*this); } + + VideoEncoderConfig(); + VideoEncoderConfig(VideoEncoderConfig&&); + ~VideoEncoderConfig(); + std::string ToString() const; + + // TODO(bugs.webrtc.org/6883): Consolidate on one of these. + VideoCodecType codec_type; + SdpVideoFormat video_format; + + // Note: This factory can be unset, and VideoStreamEncoder will + // then use the EncoderStreamFactory. The factory is only set by + // tests. + rtc::scoped_refptr<VideoStreamFactoryInterface> video_stream_factory; + std::vector<SpatialLayer> spatial_layers; + ContentType content_type; + bool frame_drop_enabled; + rtc::scoped_refptr<const EncoderSpecificSettings> encoder_specific_settings; + + // Padding will be used up to this bitrate regardless of the bitrate produced + // by the encoder. Padding above what's actually produced by the encoder helps + // maintaining a higher bitrate estimate. Padding will however not be sent + // unless the estimated bandwidth indicates that the link can handle it. + int min_transmit_bitrate_bps; + int max_bitrate_bps; + // The bitrate priority used for all VideoStreams. + double bitrate_priority; + + // The simulcast layer's configurations set by the application for this video + // sender. These are modified by the video_stream_factory before being passed + // down to lower layers for the video encoding. + // `simulcast_layers` is also used for configuring non-simulcast (when there + // is a single VideoStream). + // We have the same number of `simulcast_layers` as we have negotiated + // encodings, for example 3 are used in both simulcast and legacy kSVC. + std::vector<VideoStream> simulcast_layers; + + // Max number of encoded VideoStreams to produce. + // This is the same as the number of encodings negotiated (i.e. SSRCs), + // whether or not those encodings are `active`, except for when legacy kSVC + // is used. In this case we have three SSRCs but `number_of_streams` is + // changed to 1 to tell lower layers to limit the number of streams. + size_t number_of_streams; + + // Legacy Google conference mode flag for simulcast screenshare + bool legacy_conference_mode; + + // Indicates whether quality scaling can be used or not. + bool is_quality_scaling_allowed; + + // Maximum Quantization Parameter. + // This value is fed into EncoderStreamFactory that + // apply it to all simulcast layers/spatial layers. + int max_qp; + + private: + // Access to the copy constructor is private to force use of the Copy() + // method for those exceptional cases where we do use it. + VideoEncoderConfig(const VideoEncoderConfig&); +}; + +} // namespace webrtc + +#endif // VIDEO_CONFIG_VIDEO_ENCODER_CONFIG_H_ |