diff options
Diffstat (limited to 'third_party/libwebrtc/api/video_codecs')
65 files changed, 8540 insertions, 0 deletions
diff --git a/third_party/libwebrtc/api/video_codecs/BUILD.gn b/third_party/libwebrtc/api/video_codecs/BUILD.gn new file mode 100644 index 0000000000..101848a2e4 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/BUILD.gn @@ -0,0 +1,314 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../webrtc.gni") +if (is_android) { + import("//build/config/android/config.gni") + import("//build/config/android/rules.gni") +} + +rtc_source_set("scalability_mode") { + visibility = [ "*" ] + sources = [ + "scalability_mode.cc", + "scalability_mode.h", + ] + deps = [ + "../../rtc_base:checks", + "../../rtc_base/system:rtc_export", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] +} + +rtc_source_set("scalability_mode_helper") { + visibility = [ "*" ] + sources = [ + "scalability_mode_helper.cc", + "scalability_mode_helper.h", + ] + deps = [ "../../modules/video_coding/svc:scalability_mode_util" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_library("video_codecs_api") { + visibility = [ "*" ] + sources = [ + "av1_profile.cc", + "av1_profile.h", + "h264_profile_level_id.cc", + "h264_profile_level_id.h", + "sdp_video_format.cc", + "sdp_video_format.h", + "simulcast_stream.cc", + "simulcast_stream.h", + "spatial_layer.cc", + "spatial_layer.h", + "video_codec.cc", + "video_codec.h", + "video_decoder.cc", + "video_decoder.h", + "video_decoder_factory.h", + "video_encoder.cc", + "video_encoder.h", + "video_encoder_factory.h", + "vp8_frame_buffer_controller.h", + "vp8_frame_config.cc", + "vp8_frame_config.h", + "vp8_temporal_layers.cc", + "vp8_temporal_layers.h", + "vp9_profile.cc", + "vp9_profile.h", + ] + + deps = [ + ":scalability_mode", + "..:fec_controller_api", + "..:scoped_refptr", + "../../api:array_view", + "../../modules/video_coding:codec_globals_headers", + "../../rtc_base:checks", + "../../rtc_base:logging", + "../../rtc_base:macromagic", + "../../rtc_base:refcount", + "../../rtc_base:stringutils", + "../../rtc_base/system:rtc_export", + "../units:data_rate", + "../video:encoded_image", + "../video:render_resolution", + "../video:resolution", + "../video:video_bitrate_allocation", + "../video:video_codec_constants", + "../video:video_frame", + "../video:video_rtp_headers", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/container:inlined_vector", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_source_set("bitstream_parser_api") { + visibility = [ "*" ] + sources = [ "bitstream_parser.h" ] + deps = [ "..:array_view" ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("builtin_video_decoder_factory") { + visibility = [ "*" ] + allow_poison = [ + "audio_codecs", # TODO(bugs.webrtc.org/8396): Remove. + "software_video_codecs", + ] + sources = [ + "builtin_video_decoder_factory.cc", + "builtin_video_decoder_factory.h", + ] + + deps = [ + ":video_codecs_api", + "../../api:scoped_refptr", + "../../media:rtc_internal_video_codecs", + "../../rtc_base/system:rtc_export", + ] +} + +rtc_library("builtin_video_encoder_factory") { + visibility = [ "*" ] + allow_poison = [ + "audio_codecs", # TODO(bugs.webrtc.org/8396): Remove. + "software_video_codecs", + ] + sources = [ + "builtin_video_encoder_factory.cc", + "builtin_video_encoder_factory.h", + ] + + deps = [ + ":video_codecs_api", + "../../api:scoped_refptr", + "../../media:codec", + "../../media:media_constants", + "../../media:rtc_encoder_simulcast_proxy", + "../../media:rtc_internal_video_codecs", + "../../media:rtc_media_base", + "../../rtc_base:checks", + "../../rtc_base/system:rtc_export", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_source_set("video_encoder_factory_template") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + public = [ "video_encoder_factory_template.h" ] + + deps = [ + ":video_codecs_api", + "../../api:array_view", + "../../modules/video_coding/svc:scalability_mode_util", + ] + + absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container" ] +} + +rtc_source_set("video_encoder_factory_template_libvpx_vp8_adapter") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + public = [ "video_encoder_factory_template_libvpx_vp8_adapter.h" ] + + deps = [ + ":video_codecs_api", + "../../modules/video_coding:webrtc_vp8", + "../../modules/video_coding:webrtc_vp8_scalability", + ] + + absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ] +} + +rtc_source_set("video_encoder_factory_template_libvpx_vp9_adapter") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + public = [ "video_encoder_factory_template_libvpx_vp9_adapter.h" ] + + deps = [ "../../modules/video_coding:webrtc_vp9" ] +} + +rtc_source_set("video_encoder_factory_template_open_h264_adapter") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + public = [ "video_encoder_factory_template_open_h264_adapter.h" ] + + deps = [ "../../modules/video_coding:webrtc_h264" ] +} + +rtc_source_set("video_encoder_factory_template_libaom_av1_adapter") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + public = [ "video_encoder_factory_template_libaom_av1_adapter.h" ] + + deps = [ + ":scalability_mode", + ":video_codecs_api", + "../../modules/video_coding/codecs/av1:av1_svc_config", + "../../modules/video_coding/codecs/av1:libaom_av1_encoder", + "../../modules/video_coding/svc:scalability_mode_util", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ] +} + +rtc_source_set("video_decoder_factory_template") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + public = [ "video_decoder_factory_template.h" ] + + deps = [ + ":video_codecs_api", + "../../api:array_view", + ] + + absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container" ] +} + +rtc_source_set("video_decoder_factory_template_libvpx_vp8_adapter") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + public = [ "video_decoder_factory_template_libvpx_vp8_adapter.h" ] + + deps = [ + ":video_codecs_api", + "../../modules/video_coding:webrtc_vp8", + ] +} + +rtc_source_set("video_decoder_factory_template_libvpx_vp9_adapter") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + public = [ "video_decoder_factory_template_libvpx_vp9_adapter.h" ] + + deps = [ "../../modules/video_coding:webrtc_vp9" ] +} + +rtc_source_set("video_decoder_factory_template_open_h264_adapter") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + public = [ "video_decoder_factory_template_open_h264_adapter.h" ] + + deps = [ "../../modules/video_coding:webrtc_h264" ] +} + +rtc_source_set("video_decoder_factory_template_dav1d_adapter") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + public = [ "video_decoder_factory_template_dav1d_adapter.h" ] + + deps = [ + ":video_codecs_api", + "../../modules/video_coding/codecs/av1:dav1d_decoder", + ] +} + +rtc_library("vp8_temporal_layers_factory") { + visibility = [ "*" ] + allow_poison = [ "software_video_codecs" ] + sources = [ + "vp8_temporal_layers_factory.cc", + "vp8_temporal_layers_factory.h", + ] + + deps = [ + ":video_codecs_api", + "../:fec_controller_api", + "../../modules/video_coding:video_coding_utility", + "../../modules/video_coding:webrtc_vp8_temporal_layers", + "../../rtc_base:checks", + ] +} + +rtc_library("rtc_software_fallback_wrappers") { + visibility = [ "*" ] + + sources = [ + "video_decoder_software_fallback_wrapper.cc", + "video_decoder_software_fallback_wrapper.h", + "video_encoder_software_fallback_wrapper.cc", + "video_encoder_software_fallback_wrapper.h", + ] + + deps = [ + ":video_codecs_api", + "..:fec_controller_api", + "../../api/video:video_frame", + "../../media:rtc_media_base", + "../../modules/video_coding:video_codec_interface", + "../../modules/video_coding:video_coding_utility", + "../../rtc_base:checks", + "../../rtc_base:event_tracer", + "../../rtc_base:logging", + "../../rtc_base/system:rtc_export", + "../../system_wrappers:field_trial", + "../../system_wrappers:metrics", + "../video:encoded_image", + "../video:video_bitrate_allocation", + "../video:video_frame", + "../video:video_rtp_headers", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/base:core_headers", + "//third_party/abseil-cpp/absl/strings", + "//third_party/abseil-cpp/absl/types:optional", + ] +} diff --git a/third_party/libwebrtc/api/video_codecs/OWNERS b/third_party/libwebrtc/api/video_codecs/OWNERS new file mode 100644 index 0000000000..f73b04f829 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/OWNERS @@ -0,0 +1,4 @@ +magjed@webrtc.org +sprang@webrtc.org +brandtr@webrtc.org +philipel@webrtc.org diff --git a/third_party/libwebrtc/api/video_codecs/av1_profile.cc b/third_party/libwebrtc/api/video_codecs/av1_profile.cc new file mode 100644 index 0000000000..eefe166d80 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/av1_profile.cc @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/av1_profile.h" + +#include <map> +#include <utility> + +#include "rtc_base/string_to_number.h" + +namespace webrtc { + +// Parameter name in the format parameter map for AV1 video. +const char kAV1FmtpProfile[] = "profile"; + +absl::string_view AV1ProfileToString(AV1Profile profile) { + switch (profile) { + case AV1Profile::kProfile0: + return "0"; + case AV1Profile::kProfile1: + return "1"; + case AV1Profile::kProfile2: + return "2"; + } + return "0"; +} + +absl::optional<AV1Profile> StringToAV1Profile(absl::string_view str) { + const absl::optional<int> i = rtc::StringToNumber<int>(str); + if (!i.has_value()) + return absl::nullopt; + + switch (i.value()) { + case 0: + return AV1Profile::kProfile0; + case 1: + return AV1Profile::kProfile1; + case 2: + return AV1Profile::kProfile2; + default: + return absl::nullopt; + } +} + +absl::optional<AV1Profile> ParseSdpForAV1Profile( + const SdpVideoFormat::Parameters& params) { + const auto profile_it = params.find(kAV1FmtpProfile); + if (profile_it == params.end()) + return AV1Profile::kProfile0; + const std::string& profile_str = profile_it->second; + return StringToAV1Profile(profile_str); +} + +bool AV1IsSameProfile(const SdpVideoFormat::Parameters& params1, + const SdpVideoFormat::Parameters& params2) { + const absl::optional<AV1Profile> profile = ParseSdpForAV1Profile(params1); + const absl::optional<AV1Profile> other_profile = + ParseSdpForAV1Profile(params2); + return profile && other_profile && profile == other_profile; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/av1_profile.h b/third_party/libwebrtc/api/video_codecs/av1_profile.h new file mode 100644 index 0000000000..2254d5ecd3 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/av1_profile.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_AV1_PROFILE_H_ +#define API_VIDEO_CODECS_AV1_PROFILE_H_ + +#include <string> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "api/video_codecs/sdp_video_format.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// Profile information for AV1 video. +extern RTC_EXPORT const char kAV1FmtpProfile[]; + +// Profiles can be found at: +// https://aomedia.org/av1/specification/annex-a/#profiles +// The enum values match the number specified in the SDP. +enum class AV1Profile { + kProfile0 = 0, + kProfile1 = 1, + kProfile2 = 2, +}; + +// Helper function which converts an AV1Profile to std::string. Returns "0" if +// an unknown value is passed in. +RTC_EXPORT absl::string_view AV1ProfileToString(AV1Profile profile); + +// Helper function which converts a std::string to AV1Profile. Returns null if +// |profile| is not a valid profile string. +absl::optional<AV1Profile> StringToAV1Profile(absl::string_view profile); + +// Parses an SDP key-value map of format parameters to retrive an AV1 profile. +// Returns an AV1Profile if one has been specified, `kProfile0` if no profile is +// specified and an empty value if the profile key is present but contains an +// invalid value. +RTC_EXPORT absl::optional<AV1Profile> ParseSdpForAV1Profile( + const SdpVideoFormat::Parameters& params); + +// Returns true if the parameters have the same AV1 profile or neither contains +// an AV1 profile, otherwise false. +bool AV1IsSameProfile(const SdpVideoFormat::Parameters& params1, + const SdpVideoFormat::Parameters& params2); + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_AV1_PROFILE_H_ diff --git a/third_party/libwebrtc/api/video_codecs/bitstream_parser.h b/third_party/libwebrtc/api/video_codecs/bitstream_parser.h new file mode 100644 index 0000000000..86ce192e49 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/bitstream_parser.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_BITSTREAM_PARSER_H_ +#define API_VIDEO_CODECS_BITSTREAM_PARSER_H_ + +#include <stddef.h> +#include <stdint.h> + +#include "absl/types/optional.h" +#include "api/array_view.h" + +namespace webrtc { + +// This class is an interface for bitstream parsers. +class BitstreamParser { + public: + virtual ~BitstreamParser() = default; + + // Parse an additional chunk of the bitstream. + virtual void ParseBitstream(rtc::ArrayView<const uint8_t> bitstream) = 0; + + // Get the last extracted QP value from the parsed bitstream. If no QP + // value could be parsed, returns absl::nullopt. + virtual absl::optional<int> GetLastSliceQp() const = 0; +}; + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_BITSTREAM_PARSER_H_ diff --git a/third_party/libwebrtc/api/video_codecs/bitstream_parser_api_gn/moz.build b/third_party/libwebrtc/api/video_codecs/bitstream_parser_api_gn/moz.build new file mode 100644 index 0000000000..dcf5d2273c --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/bitstream_parser_api_gn/moz.build @@ -0,0 +1,205 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("bitstream_parser_api_gn") diff --git a/third_party/libwebrtc/api/video_codecs/builtin_video_decoder_factory.cc b/third_party/libwebrtc/api/video_codecs/builtin_video_decoder_factory.cc new file mode 100644 index 0000000000..f831905189 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/builtin_video_decoder_factory.cc @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/builtin_video_decoder_factory.h" + +#include <memory> + +#include "media/engine/internal_decoder_factory.h" + +namespace webrtc { + +std::unique_ptr<VideoDecoderFactory> CreateBuiltinVideoDecoderFactory() { + return std::make_unique<InternalDecoderFactory>(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/builtin_video_decoder_factory.h b/third_party/libwebrtc/api/video_codecs/builtin_video_decoder_factory.h new file mode 100644 index 0000000000..d516077d99 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/builtin_video_decoder_factory.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_BUILTIN_VIDEO_DECODER_FACTORY_H_ +#define API_VIDEO_CODECS_BUILTIN_VIDEO_DECODER_FACTORY_H_ + +#include <memory> + +#include "api/video_codecs/video_decoder_factory.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// Creates a new factory that can create the built-in types of video decoders. +RTC_EXPORT std::unique_ptr<VideoDecoderFactory> +CreateBuiltinVideoDecoderFactory(); + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_BUILTIN_VIDEO_DECODER_FACTORY_H_ diff --git a/third_party/libwebrtc/api/video_codecs/builtin_video_encoder_factory.cc b/third_party/libwebrtc/api/video_codecs/builtin_video_encoder_factory.cc new file mode 100644 index 0000000000..7c5f35b216 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/builtin_video_encoder_factory.cc @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/builtin_video_encoder_factory.h" + +#include <memory> +#include <string> +#include <vector> + +#include "absl/strings/match.h" +#include "absl/types/optional.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#include "media/base/codec.h" +#include "media/base/media_constants.h" +#include "media/engine/encoder_simulcast_proxy.h" +#include "media/engine/internal_encoder_factory.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +namespace { + +// This class wraps the internal factory and adds simulcast. +class BuiltinVideoEncoderFactory : public VideoEncoderFactory { + public: + BuiltinVideoEncoderFactory() + : internal_encoder_factory_(new InternalEncoderFactory()) {} + + std::unique_ptr<VideoEncoder> CreateVideoEncoder( + const SdpVideoFormat& format) override { + // Try creating internal encoder. + std::unique_ptr<VideoEncoder> internal_encoder; + if (format.IsCodecInList( + internal_encoder_factory_->GetSupportedFormats())) { + internal_encoder = std::make_unique<EncoderSimulcastProxy>( + internal_encoder_factory_.get(), format); + } + + return internal_encoder; + } + + std::vector<SdpVideoFormat> GetSupportedFormats() const override { + return internal_encoder_factory_->GetSupportedFormats(); + } + + CodecSupport QueryCodecSupport( + const SdpVideoFormat& format, + absl::optional<std::string> scalability_mode) const override { + return internal_encoder_factory_->QueryCodecSupport(format, + scalability_mode); + } + + private: + const std::unique_ptr<VideoEncoderFactory> internal_encoder_factory_; +}; + +} // namespace + +std::unique_ptr<VideoEncoderFactory> CreateBuiltinVideoEncoderFactory() { + return std::make_unique<BuiltinVideoEncoderFactory>(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/builtin_video_encoder_factory.h b/third_party/libwebrtc/api/video_codecs/builtin_video_encoder_factory.h new file mode 100644 index 0000000000..2c4537205c --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/builtin_video_encoder_factory.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_BUILTIN_VIDEO_ENCODER_FACTORY_H_ +#define API_VIDEO_CODECS_BUILTIN_VIDEO_ENCODER_FACTORY_H_ + +#include <memory> + +#include "api/video_codecs/video_encoder_factory.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// Creates a new factory that can create the built-in types of video encoders. +// The factory has simulcast support for VP8. +RTC_EXPORT std::unique_ptr<VideoEncoderFactory> +CreateBuiltinVideoEncoderFactory(); + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_BUILTIN_VIDEO_ENCODER_FACTORY_H_ diff --git a/third_party/libwebrtc/api/video_codecs/h264_profile_level_id.cc b/third_party/libwebrtc/api/video_codecs/h264_profile_level_id.cc new file mode 100644 index 0000000000..02b43ba4f2 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/h264_profile_level_id.cc @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/h264_profile_level_id.h" + +#include <cstdio> +#include <cstdlib> +#include <string> + +#include "rtc_base/arraysize.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +namespace { + +const char kProfileLevelId[] = "profile-level-id"; + +// For level_idc=11 and profile_idc=0x42, 0x4D, or 0x58, the constraint set3 +// flag specifies if level 1b or level 1.1 is used. +const uint8_t kConstraintSet3Flag = 0x10; + +// Convert a string of 8 characters into a byte where the positions containing +// character c will have their bit set. For example, c = 'x', str = "x1xx0000" +// will return 0b10110000. constexpr is used so that the pattern table in +// kProfilePatterns is statically initialized. +constexpr uint8_t ByteMaskString(char c, const char (&str)[9]) { + return (str[0] == c) << 7 | (str[1] == c) << 6 | (str[2] == c) << 5 | + (str[3] == c) << 4 | (str[4] == c) << 3 | (str[5] == c) << 2 | + (str[6] == c) << 1 | (str[7] == c) << 0; +} + +// Class for matching bit patterns such as "x1xx0000" where 'x' is allowed to be +// either 0 or 1. +class BitPattern { + public: + explicit constexpr BitPattern(const char (&str)[9]) + : mask_(~ByteMaskString('x', str)), + masked_value_(ByteMaskString('1', str)) {} + + bool IsMatch(uint8_t value) const { return masked_value_ == (value & mask_); } + + private: + const uint8_t mask_; + const uint8_t masked_value_; +}; + +// Table for converting between profile_idc/profile_iop to H264Profile. +struct ProfilePattern { + const uint8_t profile_idc; + const BitPattern profile_iop; + const H264Profile profile; +}; + +// This is from https://tools.ietf.org/html/rfc6184#section-8.1. +constexpr ProfilePattern kProfilePatterns[] = { + {0x42, BitPattern("x1xx0000"), H264Profile::kProfileConstrainedBaseline}, + {0x4D, BitPattern("1xxx0000"), H264Profile::kProfileConstrainedBaseline}, + {0x58, BitPattern("11xx0000"), H264Profile::kProfileConstrainedBaseline}, + {0x42, BitPattern("x0xx0000"), H264Profile::kProfileBaseline}, + {0x58, BitPattern("10xx0000"), H264Profile::kProfileBaseline}, + {0x4D, BitPattern("0x0x0000"), H264Profile::kProfileMain}, + {0x64, BitPattern("00000000"), H264Profile::kProfileHigh}, + {0x64, BitPattern("00001100"), H264Profile::kProfileConstrainedHigh}, + {0xF4, BitPattern("00000000"), H264Profile::kProfilePredictiveHigh444}}; + +struct LevelConstraint { + const int max_macroblocks_per_second; + const int max_macroblock_frame_size; + const H264Level level; +}; + +// This is from ITU-T H.264 (02/2016) Table A-1 – Level limits. +static constexpr LevelConstraint kLevelConstraints[] = { + {1485, 99, H264Level::kLevel1}, + {1485, 99, H264Level::kLevel1_b}, + {3000, 396, H264Level::kLevel1_1}, + {6000, 396, H264Level::kLevel1_2}, + {11880, 396, H264Level::kLevel1_3}, + {11880, 396, H264Level::kLevel2}, + {19800, 792, H264Level::kLevel2_1}, + {20250, 1620, H264Level::kLevel2_2}, + {40500, 1620, H264Level::kLevel3}, + {108000, 3600, H264Level::kLevel3_1}, + {216000, 5120, H264Level::kLevel3_2}, + {245760, 8192, H264Level::kLevel4}, + {245760, 8192, H264Level::kLevel4_1}, + {522240, 8704, H264Level::kLevel4_2}, + {589824, 22080, H264Level::kLevel5}, + {983040, 36864, H264Level::kLevel5_1}, + {2073600, 36864, H264Level::kLevel5_2}, +}; + +} // anonymous namespace + +absl::optional<H264ProfileLevelId> ParseH264ProfileLevelId(const char* str) { + // The string should consist of 3 bytes in hexadecimal format. + if (strlen(str) != 6u) + return absl::nullopt; + const uint32_t profile_level_id_numeric = strtol(str, nullptr, 16); + if (profile_level_id_numeric == 0) + return absl::nullopt; + + // Separate into three bytes. + const uint8_t level_idc = + static_cast<uint8_t>(profile_level_id_numeric & 0xFF); + const uint8_t profile_iop = + static_cast<uint8_t>((profile_level_id_numeric >> 8) & 0xFF); + const uint8_t profile_idc = + static_cast<uint8_t>((profile_level_id_numeric >> 16) & 0xFF); + + // Parse level based on level_idc and constraint set 3 flag. + H264Level level_casted = static_cast<H264Level>(level_idc); + H264Level level; + + switch (level_casted) { + case H264Level::kLevel1_1: + level = (profile_iop & kConstraintSet3Flag) != 0 ? H264Level::kLevel1_b + : H264Level::kLevel1_1; + break; + case H264Level::kLevel1: + case H264Level::kLevel1_2: + case H264Level::kLevel1_3: + case H264Level::kLevel2: + case H264Level::kLevel2_1: + case H264Level::kLevel2_2: + case H264Level::kLevel3: + case H264Level::kLevel3_1: + case H264Level::kLevel3_2: + case H264Level::kLevel4: + case H264Level::kLevel4_1: + case H264Level::kLevel4_2: + case H264Level::kLevel5: + case H264Level::kLevel5_1: + case H264Level::kLevel5_2: + level = level_casted; + break; + default: + // Unrecognized level_idc. + return absl::nullopt; + } + + // Parse profile_idc/profile_iop into a Profile enum. + for (const ProfilePattern& pattern : kProfilePatterns) { + if (profile_idc == pattern.profile_idc && + pattern.profile_iop.IsMatch(profile_iop)) { + return H264ProfileLevelId(pattern.profile, level); + } + } + + // Unrecognized profile_idc/profile_iop combination. + return absl::nullopt; +} + +absl::optional<H264Level> H264SupportedLevel(int max_frame_pixel_count, + float max_fps) { + static const int kPixelsPerMacroblock = 16 * 16; + + for (int i = arraysize(kLevelConstraints) - 1; i >= 0; --i) { + const LevelConstraint& level_constraint = kLevelConstraints[i]; + if (level_constraint.max_macroblock_frame_size * kPixelsPerMacroblock <= + max_frame_pixel_count && + level_constraint.max_macroblocks_per_second <= + max_fps * level_constraint.max_macroblock_frame_size) { + return level_constraint.level; + } + } + + // No level supported. + return absl::nullopt; +} + +absl::optional<H264ProfileLevelId> ParseSdpForH264ProfileLevelId( + const SdpVideoFormat::Parameters& params) { + // TODO(magjed): The default should really be kProfileBaseline and kLevel1 + // according to the spec: https://tools.ietf.org/html/rfc6184#section-8.1. In + // order to not break backwards compatibility with older versions of WebRTC + // where external codecs don't have any parameters, use + // kProfileConstrainedBaseline kLevel3_1 instead. This workaround will only be + // done in an interim period to allow external clients to update their code. + // http://crbug/webrtc/6337. + static const H264ProfileLevelId kDefaultProfileLevelId( + H264Profile::kProfileConstrainedBaseline, H264Level::kLevel3_1); + + const auto profile_level_id_it = params.find(kProfileLevelId); + return (profile_level_id_it == params.end()) + ? kDefaultProfileLevelId + : ParseH264ProfileLevelId(profile_level_id_it->second.c_str()); +} + +absl::optional<std::string> H264ProfileLevelIdToString( + const H264ProfileLevelId& profile_level_id) { + // Handle special case level == 1b. + if (profile_level_id.level == H264Level::kLevel1_b) { + switch (profile_level_id.profile) { + case H264Profile::kProfileConstrainedBaseline: + return {"42f00b"}; + case H264Profile::kProfileBaseline: + return {"42100b"}; + case H264Profile::kProfileMain: + return {"4d100b"}; + // Level 1b is not allowed for other profiles. + default: + return absl::nullopt; + } + } + + const char* profile_idc_iop_string; + switch (profile_level_id.profile) { + case H264Profile::kProfileConstrainedBaseline: + profile_idc_iop_string = "42e0"; + break; + case H264Profile::kProfileBaseline: + profile_idc_iop_string = "4200"; + break; + case H264Profile::kProfileMain: + profile_idc_iop_string = "4d00"; + break; + case H264Profile::kProfileConstrainedHigh: + profile_idc_iop_string = "640c"; + break; + case H264Profile::kProfileHigh: + profile_idc_iop_string = "6400"; + break; + case H264Profile::kProfilePredictiveHigh444: + profile_idc_iop_string = "f400"; + break; + // Unrecognized profile. + default: + return absl::nullopt; + } + + char str[7]; + snprintf(str, 7u, "%s%02x", profile_idc_iop_string, profile_level_id.level); + return {str}; +} + +bool H264IsSameProfile(const SdpVideoFormat::Parameters& params1, + const SdpVideoFormat::Parameters& params2) { + const absl::optional<H264ProfileLevelId> profile_level_id = + ParseSdpForH264ProfileLevelId(params1); + const absl::optional<H264ProfileLevelId> other_profile_level_id = + ParseSdpForH264ProfileLevelId(params2); + // Compare H264 profiles, but not levels. + return profile_level_id && other_profile_level_id && + profile_level_id->profile == other_profile_level_id->profile; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/h264_profile_level_id.h b/third_party/libwebrtc/api/video_codecs/h264_profile_level_id.h new file mode 100644 index 0000000000..4b46ad329d --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/h264_profile_level_id.h @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_H264_PROFILE_LEVEL_ID_H_ +#define API_VIDEO_CODECS_H264_PROFILE_LEVEL_ID_H_ + +#include <string> + +#include "absl/types/optional.h" +#include "api/video_codecs/sdp_video_format.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +enum class H264Profile { + kProfileConstrainedBaseline, + kProfileBaseline, + kProfileMain, + kProfileConstrainedHigh, + kProfileHigh, + kProfilePredictiveHigh444, +}; + +// All values are equal to ten times the level number, except level 1b which is +// special. +enum class H264Level { + kLevel1_b = 0, + kLevel1 = 10, + kLevel1_1 = 11, + kLevel1_2 = 12, + kLevel1_3 = 13, + kLevel2 = 20, + kLevel2_1 = 21, + kLevel2_2 = 22, + kLevel3 = 30, + kLevel3_1 = 31, + kLevel3_2 = 32, + kLevel4 = 40, + kLevel4_1 = 41, + kLevel4_2 = 42, + kLevel5 = 50, + kLevel5_1 = 51, + kLevel5_2 = 52 +}; + +struct H264ProfileLevelId { + constexpr H264ProfileLevelId(H264Profile profile, H264Level level) + : profile(profile), level(level) {} + H264Profile profile; + H264Level level; +}; + +// Parse profile level id that is represented as a string of 3 hex bytes. +// Nothing will be returned if the string is not a recognized H264 +// profile level id. +absl::optional<H264ProfileLevelId> ParseH264ProfileLevelId(const char* str); + +// Parse profile level id that is represented as a string of 3 hex bytes +// contained in an SDP key-value map. A default profile level id will be +// returned if the profile-level-id key is missing. Nothing will be returned if +// the key is present but the string is invalid. +RTC_EXPORT absl::optional<H264ProfileLevelId> ParseSdpForH264ProfileLevelId( + const SdpVideoFormat::Parameters& params); + +// Given that a decoder supports up to a given frame size (in pixels) at up to a +// given number of frames per second, return the highest H.264 level where it +// can guarantee that it will be able to support all valid encoded streams that +// are within that level. +RTC_EXPORT absl::optional<H264Level> H264SupportedLevel( + int max_frame_pixel_count, + float max_fps); + +// Returns canonical string representation as three hex bytes of the profile +// level id, or returns nothing for invalid profile level ids. +RTC_EXPORT absl::optional<std::string> H264ProfileLevelIdToString( + const H264ProfileLevelId& profile_level_id); + +// Returns true if the parameters have the same H264 profile (Baseline, High, +// etc). +RTC_EXPORT bool H264IsSameProfile(const SdpVideoFormat::Parameters& params1, + const SdpVideoFormat::Parameters& params2); + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_H264_PROFILE_LEVEL_ID_H_ diff --git a/third_party/libwebrtc/api/video_codecs/rtc_software_fallback_wrappers_gn/moz.build b/third_party/libwebrtc/api/video_codecs/rtc_software_fallback_wrappers_gn/moz.build new file mode 100644 index 0000000000..987a01bd2f --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/rtc_software_fallback_wrappers_gn/moz.build @@ -0,0 +1,234 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/api/video_codecs/video_decoder_software_fallback_wrapper.cc", + "/third_party/libwebrtc/api/video_codecs/video_encoder_software_fallback_wrapper.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "GLESv2", + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("rtc_software_fallback_wrappers_gn") diff --git a/third_party/libwebrtc/api/video_codecs/scalability_mode.cc b/third_party/libwebrtc/api/video_codecs/scalability_mode.cc new file mode 100644 index 0000000000..c449b4217e --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/scalability_mode.cc @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/scalability_mode.h" + +#include "rtc_base/checks.h" + +namespace webrtc { + +absl::string_view ScalabilityModeToString(ScalabilityMode scalability_mode) { + switch (scalability_mode) { + case ScalabilityMode::kL1T1: + return "L1T1"; + case ScalabilityMode::kL1T2: + return "L1T2"; + case ScalabilityMode::kL1T3: + return "L1T3"; + case ScalabilityMode::kL2T1: + return "L2T1"; + case ScalabilityMode::kL2T1h: + return "L2T1h"; + case ScalabilityMode::kL2T1_KEY: + return "L2T1_KEY"; + case ScalabilityMode::kL2T2: + return "L2T2"; + case ScalabilityMode::kL2T2h: + return "L2T2h"; + case ScalabilityMode::kL2T2_KEY: + return "L2T2_KEY"; + case ScalabilityMode::kL2T2_KEY_SHIFT: + return "L2T2_KEY_SHIFT"; + case ScalabilityMode::kL2T3: + return "L2T3"; + case ScalabilityMode::kL2T3h: + return "L2T3h"; + case ScalabilityMode::kL2T3_KEY: + return "L2T3_KEY"; + case ScalabilityMode::kL3T1: + return "L3T1"; + case ScalabilityMode::kL3T1h: + return "L3T1h"; + case ScalabilityMode::kL3T1_KEY: + return "L3T1_KEY"; + case ScalabilityMode::kL3T2: + return "L3T2"; + case ScalabilityMode::kL3T2h: + return "L3T2h"; + case ScalabilityMode::kL3T2_KEY: + return "L3T2_KEY"; + case ScalabilityMode::kL3T3: + return "L3T3"; + case ScalabilityMode::kL3T3h: + return "L3T3h"; + case ScalabilityMode::kL3T3_KEY: + return "L3T3_KEY"; + case ScalabilityMode::kS2T1: + return "S2T1"; + case ScalabilityMode::kS2T1h: + return "S2T1h"; + case ScalabilityMode::kS2T2: + return "S2T2"; + case ScalabilityMode::kS2T2h: + return "S2T2h"; + case ScalabilityMode::kS2T3: + return "S2T3"; + case ScalabilityMode::kS2T3h: + return "S2T3h"; + case ScalabilityMode::kS3T1: + return "S3T1"; + case ScalabilityMode::kS3T1h: + return "S3T1h"; + case ScalabilityMode::kS3T2: + return "S3T2"; + case ScalabilityMode::kS3T2h: + return "S3T2h"; + case ScalabilityMode::kS3T3: + return "S3T3"; + case ScalabilityMode::kS3T3h: + return "S3T3h"; + } + RTC_CHECK_NOTREACHED(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/scalability_mode.h b/third_party/libwebrtc/api/video_codecs/scalability_mode.h new file mode 100644 index 0000000000..b26f32eb22 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/scalability_mode.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_SCALABILITY_MODE_H_ +#define API_VIDEO_CODECS_SCALABILITY_MODE_H_ + +#include <stddef.h> +#include <stdint.h> + +#include "absl/strings/string_view.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// Supported scalability modes. Most applications should use the +// PeerConnection-level apis where scalability mode is represented as a string. +// This list of currently recognized modes is intended for the api boundary +// between webrtc and injected encoders. Any application usage outside of +// injected encoders is strongly discouraged. +enum class ScalabilityMode : uint8_t { + kL1T1, + kL1T2, + kL1T3, + kL2T1, + kL2T1h, + kL2T1_KEY, + kL2T2, + kL2T2h, + kL2T2_KEY, + kL2T2_KEY_SHIFT, + kL2T3, + kL2T3h, + kL2T3_KEY, + kL3T1, + kL3T1h, + kL3T1_KEY, + kL3T2, + kL3T2h, + kL3T2_KEY, + kL3T3, + kL3T3h, + kL3T3_KEY, + kS2T1, + kS2T1h, + kS2T2, + kS2T2h, + kS2T3, + kS2T3h, + kS3T1, + kS3T1h, + kS3T2, + kS3T2h, + kS3T3, + kS3T3h, +}; + +inline constexpr ScalabilityMode kAllScalabilityModes[] = { + // clang-format off + ScalabilityMode::kL1T1, + ScalabilityMode::kL1T2, + ScalabilityMode::kL1T3, + ScalabilityMode::kL2T1, + ScalabilityMode::kL2T1h, + ScalabilityMode::kL2T1_KEY, + ScalabilityMode::kL2T2, + ScalabilityMode::kL2T2h, + ScalabilityMode::kL2T2_KEY, + ScalabilityMode::kL2T2_KEY_SHIFT, + ScalabilityMode::kL2T3, + ScalabilityMode::kL2T3h, + ScalabilityMode::kL2T3_KEY, + ScalabilityMode::kL3T1, + ScalabilityMode::kL3T1h, + ScalabilityMode::kL3T1_KEY, + ScalabilityMode::kL3T2, + ScalabilityMode::kL3T2h, + ScalabilityMode::kL3T2_KEY, + ScalabilityMode::kL3T3, + ScalabilityMode::kL3T3h, + ScalabilityMode::kL3T3_KEY, + ScalabilityMode::kS2T1, + ScalabilityMode::kS2T1h, + ScalabilityMode::kS2T2, + ScalabilityMode::kS2T2h, + ScalabilityMode::kS2T3, + ScalabilityMode::kS2T3h, + ScalabilityMode::kS3T1, + ScalabilityMode::kS3T1h, + ScalabilityMode::kS3T2, + ScalabilityMode::kS3T2h, + ScalabilityMode::kS3T3, + ScalabilityMode::kS3T3h, + // clang-format on +}; + +inline constexpr size_t kScalabilityModeCount = + sizeof(kAllScalabilityModes) / sizeof(ScalabilityMode); + +RTC_EXPORT +absl::string_view ScalabilityModeToString(ScalabilityMode scalability_mode); + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_SCALABILITY_MODE_H_ diff --git a/third_party/libwebrtc/api/video_codecs/scalability_mode_gn/moz.build b/third_party/libwebrtc/api/video_codecs/scalability_mode_gn/moz.build new file mode 100644 index 0000000000..a308bfc58f --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/scalability_mode_gn/moz.build @@ -0,0 +1,221 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/api/video_codecs/scalability_mode.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("scalability_mode_gn") diff --git a/third_party/libwebrtc/api/video_codecs/scalability_mode_helper.cc b/third_party/libwebrtc/api/video_codecs/scalability_mode_helper.cc new file mode 100644 index 0000000000..b4571632d9 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/scalability_mode_helper.cc @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/scalability_mode_helper.h" + +#include "modules/video_coding/svc/scalability_mode_util.h" + +namespace webrtc { + +absl::optional<int> ScalabilityModeStringToNumSpatialLayers( + absl::string_view scalability_mode_string) { + absl::optional<ScalabilityMode> scalability_mode = + ScalabilityModeFromString(scalability_mode_string); + if (!scalability_mode.has_value()) { + return absl::nullopt; + } + return ScalabilityModeToNumSpatialLayers(*scalability_mode); +} + +absl::optional<int> ScalabilityModeStringToNumTemporalLayers( + absl::string_view scalability_mode_string) { + absl::optional<ScalabilityMode> scalability_mode = + ScalabilityModeFromString(scalability_mode_string); + if (!scalability_mode.has_value()) { + return absl::nullopt; + } + return ScalabilityModeToNumTemporalLayers(*scalability_mode); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/scalability_mode_helper.h b/third_party/libwebrtc/api/video_codecs/scalability_mode_helper.h new file mode 100644 index 0000000000..a8b060d079 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/scalability_mode_helper.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_SCALABILITY_MODE_HELPER_H_ +#define API_VIDEO_CODECS_SCALABILITY_MODE_HELPER_H_ + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" + +namespace webrtc { + +// Returns the number of spatial layers from the `scalability_mode_string` +// or nullopt if the given mode is unknown. +absl::optional<int> ScalabilityModeStringToNumSpatialLayers( + absl::string_view scalability_mode_string); + +// Returns the number of temporal layers from the `scalability_mode_string` +// or nullopt if the given mode is unknown. +absl::optional<int> ScalabilityModeStringToNumTemporalLayers( + absl::string_view scalability_mode_string); + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_SCALABILITY_MODE_HELPER_H_ diff --git a/third_party/libwebrtc/api/video_codecs/sdp_video_format.cc b/third_party/libwebrtc/api/video_codecs/sdp_video_format.cc new file mode 100644 index 0000000000..cb7e98a682 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/sdp_video_format.cc @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/sdp_video_format.h" + +#include "absl/strings/match.h" +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/video_codecs/av1_profile.h" +#include "api/video_codecs/h264_profile_level_id.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/vp9_profile.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { + +namespace { + +std::string H264GetPacketizationModeOrDefault( + const SdpVideoFormat::Parameters& params) { + constexpr char kH264FmtpPacketizationMode[] = "packetization-mode"; + const auto it = params.find(kH264FmtpPacketizationMode); + if (it != params.end()) { + return it->second; + } + // If packetization-mode is not present, default to "0". + // https://tools.ietf.org/html/rfc6184#section-6.2 + return "0"; +} + +bool H264IsSamePacketizationMode(const SdpVideoFormat::Parameters& left, + const SdpVideoFormat::Parameters& right) { + return H264GetPacketizationModeOrDefault(left) == + H264GetPacketizationModeOrDefault(right); +} + +// Some (video) codecs are actually families of codecs and rely on parameters +// to distinguish different incompatible family members. +bool IsSameCodecSpecific(const SdpVideoFormat& format1, + const SdpVideoFormat& format2) { + // The assumption when calling this function is that the two formats have the + // same name. + RTC_DCHECK(absl::EqualsIgnoreCase(format1.name, format2.name)); + + VideoCodecType codec_type = PayloadStringToCodecType(format1.name); + switch (codec_type) { + case kVideoCodecH264: + return H264IsSameProfile(format1.parameters, format2.parameters) && + H264IsSamePacketizationMode(format1.parameters, + format2.parameters); + case kVideoCodecVP9: + return VP9IsSameProfile(format1.parameters, format2.parameters); + case kVideoCodecAV1: + return AV1IsSameProfile(format1.parameters, format2.parameters); + default: + return true; + } +} +} // namespace + +SdpVideoFormat::SdpVideoFormat(const std::string& name) : name(name) {} + +SdpVideoFormat::SdpVideoFormat(const std::string& name, + const Parameters& parameters) + : name(name), parameters(parameters) {} + +SdpVideoFormat::SdpVideoFormat( + const std::string& name, + const Parameters& parameters, + const absl::InlinedVector<ScalabilityMode, kScalabilityModeCount>& + scalability_modes) + : name(name), + parameters(parameters), + scalability_modes(scalability_modes) {} + +SdpVideoFormat::SdpVideoFormat(const SdpVideoFormat&) = default; +SdpVideoFormat::SdpVideoFormat(SdpVideoFormat&&) = default; +SdpVideoFormat& SdpVideoFormat::operator=(const SdpVideoFormat&) = default; +SdpVideoFormat& SdpVideoFormat::operator=(SdpVideoFormat&&) = default; + +SdpVideoFormat::~SdpVideoFormat() = default; + +std::string SdpVideoFormat::ToString() const { + rtc::StringBuilder builder; + builder << "Codec name: " << name << ", parameters: {"; + for (const auto& kv : parameters) { + builder << " " << kv.first << "=" << kv.second; + } + + builder << " }"; + if (!scalability_modes.empty()) { + builder << ", scalability_modes: ["; + bool first = true; + for (const auto scalability_mode : scalability_modes) { + if (first) { + first = false; + } else { + builder << ", "; + } + builder << ScalabilityModeToString(scalability_mode); + } + builder << "]"; + } + + return builder.str(); +} + +bool SdpVideoFormat::IsSameCodec(const SdpVideoFormat& other) const { + // Two codecs are considered the same if the name matches (case insensitive) + // and certain codec-specific parameters match. + return absl::EqualsIgnoreCase(name, other.name) && + IsSameCodecSpecific(*this, other); +} + +bool SdpVideoFormat::IsCodecInList( + rtc::ArrayView<const webrtc::SdpVideoFormat> formats) const { + for (const auto& format : formats) { + if (IsSameCodec(format)) { + return true; + } + } + return false; +} + +bool operator==(const SdpVideoFormat& a, const SdpVideoFormat& b) { + return a.name == b.name && a.parameters == b.parameters && + a.scalability_modes == b.scalability_modes; +} + +absl::optional<SdpVideoFormat> FuzzyMatchSdpVideoFormat( + rtc::ArrayView<const SdpVideoFormat> supported_formats, + const SdpVideoFormat& format) { + absl::optional<SdpVideoFormat> res; + int best_parameter_match = 0; + for (const auto& supported_format : supported_formats) { + if (absl::EqualsIgnoreCase(supported_format.name, format.name)) { + int matching_parameters = 0; + for (const auto& kv : supported_format.parameters) { + auto it = format.parameters.find(kv.first); + if (it != format.parameters.end() && it->second == kv.second) { + matching_parameters += 1; + } + } + + if (!res || matching_parameters > best_parameter_match) { + res = supported_format; + best_parameter_match = matching_parameters; + } + } + } + + if (!res) { + RTC_LOG(LS_INFO) << "Failed to match SdpVideoFormat " << format.ToString(); + } else if (*res != format) { + RTC_LOG(LS_INFO) << "Matched SdpVideoFormat " << format.ToString() + << " with " << res->ToString(); + } + + return res; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/sdp_video_format.h b/third_party/libwebrtc/api/video_codecs/sdp_video_format.h new file mode 100644 index 0000000000..faaa66c241 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/sdp_video_format.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_SDP_VIDEO_FORMAT_H_ +#define API_VIDEO_CODECS_SDP_VIDEO_FORMAT_H_ + +#include <map> +#include <string> + +#include "absl/container/inlined_vector.h" +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/video_codecs/scalability_mode.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// SDP specification for a single video codec. +// NOTE: This class is still under development and may change without notice. +struct RTC_EXPORT SdpVideoFormat { + using Parameters = std::map<std::string, std::string>; + + explicit SdpVideoFormat(const std::string& name); + SdpVideoFormat(const std::string& name, const Parameters& parameters); + SdpVideoFormat( + const std::string& name, + const Parameters& parameters, + const absl::InlinedVector<ScalabilityMode, kScalabilityModeCount>& + scalability_modes); + SdpVideoFormat(const SdpVideoFormat&); + SdpVideoFormat(SdpVideoFormat&&); + SdpVideoFormat& operator=(const SdpVideoFormat&); + SdpVideoFormat& operator=(SdpVideoFormat&&); + + ~SdpVideoFormat(); + + // Returns true if the SdpVideoFormats have the same names as well as codec + // specific parameters. Please note that two SdpVideoFormats can represent the + // same codec even though not all parameters are the same. + bool IsSameCodec(const SdpVideoFormat& other) const; + bool IsCodecInList( + rtc::ArrayView<const webrtc::SdpVideoFormat> formats) const; + + std::string ToString() const; + + friend RTC_EXPORT bool operator==(const SdpVideoFormat& a, + const SdpVideoFormat& b); + friend RTC_EXPORT bool operator!=(const SdpVideoFormat& a, + const SdpVideoFormat& b) { + return !(a == b); + } + + std::string name; + Parameters parameters; + absl::InlinedVector<ScalabilityMode, kScalabilityModeCount> scalability_modes; +}; + +// For not so good reasons sometimes additional parameters are added to an +// SdpVideoFormat, which makes instances that should compare equal to not match +// anymore. Until we stop misusing SdpVideoFormats provide this convenience +// function to perform fuzzy matching. +absl::optional<SdpVideoFormat> FuzzyMatchSdpVideoFormat( + rtc::ArrayView<const SdpVideoFormat> supported_formats, + const SdpVideoFormat& format); + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_SDP_VIDEO_FORMAT_H_ diff --git a/third_party/libwebrtc/api/video_codecs/simulcast_stream.cc b/third_party/libwebrtc/api/video_codecs/simulcast_stream.cc new file mode 100644 index 0000000000..312429ef9f --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/simulcast_stream.cc @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/simulcast_stream.h" + +#include "rtc_base/checks.h" + +namespace webrtc { + +unsigned char SimulcastStream::GetNumberOfTemporalLayers() const { + return numberOfTemporalLayers; +} +void SimulcastStream::SetNumberOfTemporalLayers(unsigned char n) { + RTC_DCHECK_GE(n, 1); + RTC_DCHECK_LE(n, 3); + numberOfTemporalLayers = n; +} + +ScalabilityMode SimulcastStream::GetScalabilityMode() const { + RTC_CHECK_GE(numberOfTemporalLayers, 1); + RTC_CHECK_LE(numberOfTemporalLayers, 3); + static const ScalabilityMode scalability_modes[3] = { + ScalabilityMode::kL1T1, + ScalabilityMode::kL1T2, + ScalabilityMode::kL1T3, + }; + return scalability_modes[numberOfTemporalLayers - 1]; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/simulcast_stream.h b/third_party/libwebrtc/api/video_codecs/simulcast_stream.h new file mode 100644 index 0000000000..7c0dd5d786 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/simulcast_stream.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_SIMULCAST_STREAM_H_ +#define API_VIDEO_CODECS_SIMULCAST_STREAM_H_ + +#include "api/video_codecs/scalability_mode.h" + +namespace webrtc { + +// TODO(bugs.webrtc.org/6883): Unify with struct VideoStream, part of +// VideoEncoderConfig. +struct SimulcastStream { + // Temporary utility methods for transition from numberOfTemporalLayers + // setting to ScalabilityMode. + unsigned char GetNumberOfTemporalLayers() const; + ScalabilityMode GetScalabilityMode() const; + void SetNumberOfTemporalLayers(unsigned char n); + + int width = 0; + int height = 0; + float maxFramerate = 0; // fps. + unsigned char numberOfTemporalLayers = 1; + unsigned int maxBitrate = 0; // kilobits/sec. + unsigned int targetBitrate = 0; // kilobits/sec. + unsigned int minBitrate = 0; // kilobits/sec. + unsigned int qpMax = 0; // minimum quality + bool active = false; // encoded and sent. +}; + +} // namespace webrtc +#endif // API_VIDEO_CODECS_SIMULCAST_STREAM_H_ diff --git a/third_party/libwebrtc/api/video_codecs/spatial_layer.cc b/third_party/libwebrtc/api/video_codecs/spatial_layer.cc new file mode 100644 index 0000000000..25ccdfeb48 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/spatial_layer.cc @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/spatial_layer.h" + +namespace webrtc { + +bool SpatialLayer::operator==(const SpatialLayer& other) const { + return (width == other.width && height == other.height && + maxFramerate == other.maxFramerate && + numberOfTemporalLayers == other.numberOfTemporalLayers && + maxBitrate == other.maxBitrate && + targetBitrate == other.targetBitrate && + minBitrate == other.minBitrate && qpMax == other.qpMax && + active == other.active); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/spatial_layer.h b/third_party/libwebrtc/api/video_codecs/spatial_layer.h new file mode 100644 index 0000000000..5a1b425427 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/spatial_layer.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_SPATIAL_LAYER_H_ +#define API_VIDEO_CODECS_SPATIAL_LAYER_H_ + +namespace webrtc { + +struct SpatialLayer { + bool operator==(const SpatialLayer& other) const; + bool operator!=(const SpatialLayer& other) const { return !(*this == other); } + + unsigned short width; // NOLINT(runtime/int) + unsigned short height; // NOLINT(runtime/int) + float maxFramerate; // fps. + unsigned char numberOfTemporalLayers; + unsigned int maxBitrate; // kilobits/sec. + unsigned int targetBitrate; // kilobits/sec. + unsigned int minBitrate; // kilobits/sec. + unsigned int qpMax; // minimum quality + bool active; // encoded and sent. +}; + +} // namespace webrtc +#endif // API_VIDEO_CODECS_SPATIAL_LAYER_H_ diff --git a/third_party/libwebrtc/api/video_codecs/test/BUILD.gn b/third_party/libwebrtc/api/video_codecs/test/BUILD.gn new file mode 100644 index 0000000000..47d5ff9aa3 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/test/BUILD.gn @@ -0,0 +1,81 @@ +# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../webrtc.gni") + +if (rtc_include_tests) { + rtc_library("video_codecs_api_unittests") { + testonly = true + sources = [ + "builtin_video_encoder_factory_unittest.cc", + "h264_profile_level_id_unittest.cc", + "sdp_video_format_unittest.cc", + "video_decoder_software_fallback_wrapper_unittest.cc", + "video_encoder_software_fallback_wrapper_unittest.cc", + ] + + deps = [ + ":video_decoder_factory_template_tests", + ":video_encoder_factory_template_tests", + "..:builtin_video_encoder_factory", + "..:rtc_software_fallback_wrappers", + "..:video_codecs_api", + "../..:fec_controller_api", + "../..:mock_video_encoder", + "../../../api:scoped_refptr", + "../../../media:media_constants", + "../../../media:rtc_media_base", + "../../../modules/video_coding:video_codec_interface", + "../../../modules/video_coding:video_coding_utility", + "../../../modules/video_coding:webrtc_vp8", + "../../../rtc_base:checks", + "../../../rtc_base:rtc_base_tests_utils", + "../../../test:field_trial", + "../../../test:test_support", + "../../../test:video_test_common", + "../../video:encoded_image", + "../../video:video_bitrate_allocation", + "../../video:video_frame", + "../../video:video_rtp_headers", + "//testing/gtest", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + + rtc_library("video_encoder_factory_template_tests") { + testonly = true + sources = [ "video_encoder_factory_template_tests.cc" ] + + deps = [ + "..:video_encoder_factory_template", + "..:video_encoder_factory_template_libaom_av1_adapter", + "..:video_encoder_factory_template_libvpx_vp8_adapter", + "..:video_encoder_factory_template_libvpx_vp9_adapter", + "..:video_encoder_factory_template_open_h264_adapter", + "../../:mock_video_encoder", + "../../../test:test_support", + "//testing/gtest", + ] + } + + rtc_library("video_decoder_factory_template_tests") { + testonly = true + sources = [ "video_decoder_factory_template_tests.cc" ] + + deps = [ + "..:video_decoder_factory_template", + "..:video_decoder_factory_template_dav1d_adapter", + "..:video_decoder_factory_template_libvpx_vp8_adapter", + "..:video_decoder_factory_template_libvpx_vp9_adapter", + "..:video_decoder_factory_template_open_h264_adapter", + "../../:mock_video_decoder", + "../../../test:test_support", + "//testing/gtest", + ] + } +} diff --git a/third_party/libwebrtc/api/video_codecs/test/builtin_video_encoder_factory_unittest.cc b/third_party/libwebrtc/api/video_codecs/test/builtin_video_encoder_factory_unittest.cc new file mode 100644 index 0000000000..84fd594b4c --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/test/builtin_video_encoder_factory_unittest.cc @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/builtin_video_encoder_factory.h" + +#include <memory> +#include <string> +#include <vector> + +#include "api/video_codecs/sdp_video_format.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(BuiltinVideoEncoderFactoryTest, AnnouncesVp9AccordingToBuildFlags) { + std::unique_ptr<VideoEncoderFactory> factory = + CreateBuiltinVideoEncoderFactory(); + bool claims_vp9_support = false; + for (const SdpVideoFormat& format : factory->GetSupportedFormats()) { + if (format.name == "VP9") { + claims_vp9_support = true; + break; + } + } +#if defined(RTC_ENABLE_VP9) + EXPECT_TRUE(claims_vp9_support); +#else + EXPECT_FALSE(claims_vp9_support); +#endif // defined(RTC_ENABLE_VP9) +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/test/h264_profile_level_id_unittest.cc b/third_party/libwebrtc/api/video_codecs/test/h264_profile_level_id_unittest.cc new file mode 100644 index 0000000000..47098d2682 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/test/h264_profile_level_id_unittest.cc @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/h264_profile_level_id.h" + +#include <map> +#include <string> + +#include "absl/types/optional.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(H264ProfileLevelId, TestParsingInvalid) { + // Malformed strings. + EXPECT_FALSE(ParseH264ProfileLevelId("")); + EXPECT_FALSE(ParseH264ProfileLevelId(" 42e01f")); + EXPECT_FALSE(ParseH264ProfileLevelId("4242e01f")); + EXPECT_FALSE(ParseH264ProfileLevelId("e01f")); + EXPECT_FALSE(ParseH264ProfileLevelId("gggggg")); + + // Invalid level. + EXPECT_FALSE(ParseH264ProfileLevelId("42e000")); + EXPECT_FALSE(ParseH264ProfileLevelId("42e00f")); + EXPECT_FALSE(ParseH264ProfileLevelId("42e0ff")); + + // Invalid profile. + EXPECT_FALSE(ParseH264ProfileLevelId("42e11f")); + EXPECT_FALSE(ParseH264ProfileLevelId("58601f")); + EXPECT_FALSE(ParseH264ProfileLevelId("64e01f")); +} + +TEST(H264ProfileLevelId, TestParsingLevel) { + EXPECT_EQ(H264Level::kLevel3_1, ParseH264ProfileLevelId("42e01f")->level); + EXPECT_EQ(H264Level::kLevel1_1, ParseH264ProfileLevelId("42e00b")->level); + EXPECT_EQ(H264Level::kLevel1_b, ParseH264ProfileLevelId("42f00b")->level); + EXPECT_EQ(H264Level::kLevel4_2, ParseH264ProfileLevelId("42C02A")->level); + EXPECT_EQ(H264Level::kLevel5_2, ParseH264ProfileLevelId("640c34")->level); +} + +TEST(H264ProfileLevelId, TestParsingConstrainedBaseline) { + EXPECT_EQ(H264Profile::kProfileConstrainedBaseline, + ParseH264ProfileLevelId("42e01f")->profile); + EXPECT_EQ(H264Profile::kProfileConstrainedBaseline, + ParseH264ProfileLevelId("42C02A")->profile); + EXPECT_EQ(H264Profile::kProfileConstrainedBaseline, + ParseH264ProfileLevelId("4de01f")->profile); + EXPECT_EQ(H264Profile::kProfileConstrainedBaseline, + ParseH264ProfileLevelId("58f01f")->profile); +} + +TEST(H264ProfileLevelId, TestParsingBaseline) { + EXPECT_EQ(H264Profile::kProfileBaseline, + ParseH264ProfileLevelId("42a01f")->profile); + EXPECT_EQ(H264Profile::kProfileBaseline, + ParseH264ProfileLevelId("58A01F")->profile); +} + +TEST(H264ProfileLevelId, TestParsingMain) { + EXPECT_EQ(H264Profile::kProfileMain, + ParseH264ProfileLevelId("4D401f")->profile); +} + +TEST(H264ProfileLevelId, TestParsingHigh) { + EXPECT_EQ(H264Profile::kProfileHigh, + ParseH264ProfileLevelId("64001f")->profile); +} + +TEST(H264ProfileLevelId, TestParsingConstrainedHigh) { + EXPECT_EQ(H264Profile::kProfileConstrainedHigh, + ParseH264ProfileLevelId("640c1f")->profile); +} + +TEST(H264ProfileLevelId, TestSupportedLevel) { + EXPECT_EQ(H264Level::kLevel2_1, *H264SupportedLevel(640 * 480, 25)); + EXPECT_EQ(H264Level::kLevel3_1, *H264SupportedLevel(1280 * 720, 30)); + EXPECT_EQ(H264Level::kLevel4_2, *H264SupportedLevel(1920 * 1280, 60)); +} + +// Test supported level below level 1 requirements. +TEST(H264ProfileLevelId, TestSupportedLevelInvalid) { + EXPECT_FALSE(H264SupportedLevel(0, 0)); + // All levels support fps > 5. + EXPECT_FALSE(H264SupportedLevel(1280 * 720, 5)); + // All levels support frame sizes > 183 * 137. + EXPECT_FALSE(H264SupportedLevel(183 * 137, 30)); +} + +TEST(H264ProfileLevelId, TestToString) { + EXPECT_EQ("42e01f", *H264ProfileLevelIdToString(H264ProfileLevelId( + H264Profile::kProfileConstrainedBaseline, + H264Level::kLevel3_1))); + EXPECT_EQ("42000a", *H264ProfileLevelIdToString(H264ProfileLevelId( + H264Profile::kProfileBaseline, H264Level::kLevel1))); + EXPECT_EQ("4d001f", H264ProfileLevelIdToString(H264ProfileLevelId( + H264Profile::kProfileMain, H264Level::kLevel3_1))); + EXPECT_EQ("640c2a", + *H264ProfileLevelIdToString(H264ProfileLevelId( + H264Profile::kProfileConstrainedHigh, H264Level::kLevel4_2))); + EXPECT_EQ("64002a", *H264ProfileLevelIdToString(H264ProfileLevelId( + H264Profile::kProfileHigh, H264Level::kLevel4_2))); +} + +TEST(H264ProfileLevelId, TestToStringLevel1b) { + EXPECT_EQ("42f00b", *H264ProfileLevelIdToString(H264ProfileLevelId( + H264Profile::kProfileConstrainedBaseline, + H264Level::kLevel1_b))); + EXPECT_EQ("42100b", + *H264ProfileLevelIdToString(H264ProfileLevelId( + H264Profile::kProfileBaseline, H264Level::kLevel1_b))); + EXPECT_EQ("4d100b", *H264ProfileLevelIdToString(H264ProfileLevelId( + H264Profile::kProfileMain, H264Level::kLevel1_b))); +} + +TEST(H264ProfileLevelId, TestToStringRoundTrip) { + EXPECT_EQ("42e01f", + *H264ProfileLevelIdToString(*ParseH264ProfileLevelId("42e01f"))); + EXPECT_EQ("42e01f", + *H264ProfileLevelIdToString(*ParseH264ProfileLevelId("42E01F"))); + EXPECT_EQ("4d100b", + *H264ProfileLevelIdToString(*ParseH264ProfileLevelId("4d100b"))); + EXPECT_EQ("4d100b", + *H264ProfileLevelIdToString(*ParseH264ProfileLevelId("4D100B"))); + EXPECT_EQ("640c2a", + *H264ProfileLevelIdToString(*ParseH264ProfileLevelId("640c2a"))); + EXPECT_EQ("640c2a", + *H264ProfileLevelIdToString(*ParseH264ProfileLevelId("640C2A"))); +} + +TEST(H264ProfileLevelId, TestToStringInvalid) { + EXPECT_FALSE(H264ProfileLevelIdToString( + H264ProfileLevelId(H264Profile::kProfileHigh, H264Level::kLevel1_b))); + EXPECT_FALSE(H264ProfileLevelIdToString(H264ProfileLevelId( + H264Profile::kProfileConstrainedHigh, H264Level::kLevel1_b))); + EXPECT_FALSE(H264ProfileLevelIdToString( + H264ProfileLevelId(static_cast<H264Profile>(255), H264Level::kLevel3_1))); +} + +TEST(H264ProfileLevelId, TestParseSdpProfileLevelIdEmpty) { + const absl::optional<H264ProfileLevelId> profile_level_id = + ParseSdpForH264ProfileLevelId(SdpVideoFormat::Parameters()); + EXPECT_TRUE(profile_level_id); + EXPECT_EQ(H264Profile::kProfileConstrainedBaseline, + profile_level_id->profile); + EXPECT_EQ(H264Level::kLevel3_1, profile_level_id->level); +} + +TEST(H264ProfileLevelId, TestParseSdpProfileLevelIdConstrainedHigh) { + SdpVideoFormat::Parameters params; + params["profile-level-id"] = "640c2a"; + const absl::optional<H264ProfileLevelId> profile_level_id = + ParseSdpForH264ProfileLevelId(params); + EXPECT_TRUE(profile_level_id); + EXPECT_EQ(H264Profile::kProfileConstrainedHigh, profile_level_id->profile); + EXPECT_EQ(H264Level::kLevel4_2, profile_level_id->level); +} + +TEST(H264ProfileLevelId, TestParseSdpProfileLevelIdInvalid) { + SdpVideoFormat::Parameters params; + params["profile-level-id"] = "foobar"; + EXPECT_FALSE(ParseSdpForH264ProfileLevelId(params)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/test/sdp_video_format_unittest.cc b/third_party/libwebrtc/api/video_codecs/test/sdp_video_format_unittest.cc new file mode 100644 index 0000000000..bb158aeb95 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/test/sdp_video_format_unittest.cc @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/sdp_video_format.h" + +#include <stdint.h> + +#include "media/base/media_constants.h" +#include "test/gtest.h" + +namespace webrtc { + +typedef SdpVideoFormat Sdp; +typedef SdpVideoFormat::Parameters Params; + +TEST(SdpVideoFormatTest, SameCodecNameNoParameters) { + EXPECT_TRUE(Sdp("H264").IsSameCodec(Sdp("h264"))); + EXPECT_TRUE(Sdp("VP8").IsSameCodec(Sdp("vp8"))); + EXPECT_TRUE(Sdp("VP9").IsSameCodec(Sdp("vp9"))); + EXPECT_TRUE(Sdp("AV1").IsSameCodec(Sdp("Av1"))); +} + +TEST(SdpVideoFormatTest, DifferentCodecNameNoParameters) { + EXPECT_FALSE(Sdp("H264").IsSameCodec(Sdp("VP8"))); + EXPECT_FALSE(Sdp("VP8").IsSameCodec(Sdp("VP9"))); + EXPECT_FALSE(Sdp("AV1").IsSameCodec(Sdp("VP8"))); +} + +TEST(SdpVideoFormatTest, SameCodecNameSameParameters) { + EXPECT_TRUE(Sdp("VP9").IsSameCodec(Sdp("VP9", Params{{"profile-id", "0"}}))); + EXPECT_TRUE(Sdp("VP9", Params{{"profile-id", "0"}}) + .IsSameCodec(Sdp("VP9", Params{{"profile-id", "0"}}))); + EXPECT_TRUE(Sdp("VP9", Params{{"profile-id", "2"}}) + .IsSameCodec(Sdp("VP9", Params{{"profile-id", "2"}}))); + EXPECT_TRUE( + Sdp("H264", Params{{"profile-level-id", "42e01f"}}) + .IsSameCodec(Sdp("H264", Params{{"profile-level-id", "42e01f"}}))); + EXPECT_TRUE( + Sdp("H264", Params{{"profile-level-id", "640c34"}}) + .IsSameCodec(Sdp("H264", Params{{"profile-level-id", "640c34"}}))); + EXPECT_TRUE(Sdp("AV1").IsSameCodec(Sdp("AV1", Params{{"profile", "0"}}))); + EXPECT_TRUE(Sdp("AV1", Params{{"profile", "0"}}) + .IsSameCodec(Sdp("AV1", Params{{"profile", "0"}}))); + EXPECT_TRUE(Sdp("AV1", Params{{"profile", "2"}}) + .IsSameCodec(Sdp("AV1", Params{{"profile", "2"}}))); +} + +TEST(SdpVideoFormatTest, SameCodecNameDifferentParameters) { + EXPECT_FALSE(Sdp("VP9").IsSameCodec(Sdp("VP9", Params{{"profile-id", "2"}}))); + EXPECT_FALSE(Sdp("VP9", Params{{"profile-id", "0"}}) + .IsSameCodec(Sdp("VP9", Params{{"profile-id", "1"}}))); + EXPECT_FALSE(Sdp("VP9", Params{{"profile-id", "2"}}) + .IsSameCodec(Sdp("VP9", Params{{"profile-id", "0"}}))); + EXPECT_FALSE( + Sdp("H264", Params{{"profile-level-id", "42e01f"}}) + .IsSameCodec(Sdp("H264", Params{{"profile-level-id", "640c34"}}))); + EXPECT_FALSE( + Sdp("H264", Params{{"profile-level-id", "640c34"}}) + .IsSameCodec(Sdp("H264", Params{{"profile-level-id", "42f00b"}}))); + EXPECT_FALSE(Sdp("AV1").IsSameCodec(Sdp("AV1", Params{{"profile", "1"}}))); + EXPECT_FALSE(Sdp("AV1", Params{{"profile", "0"}}) + .IsSameCodec(Sdp("AV1", Params{{"profile", "1"}}))); + EXPECT_FALSE(Sdp("AV1", Params{{"profile", "1"}}) + .IsSameCodec(Sdp("AV1", Params{{"profile", "2"}}))); +} + +TEST(SdpVideoFormatTest, DifferentCodecNameSameParameters) { + EXPECT_FALSE(Sdp("VP9", Params{{"profile-id", "0"}}) + .IsSameCodec(Sdp("H264", Params{{"profile-id", "0"}}))); + EXPECT_FALSE(Sdp("VP9", Params{{"profile-id", "2"}}) + .IsSameCodec(Sdp("VP8", Params{{"profile-id", "2"}}))); + EXPECT_FALSE( + Sdp("H264", Params{{"profile-level-id", "42e01f"}}) + .IsSameCodec(Sdp("VP9", Params{{"profile-level-id", "42e01f"}}))); + EXPECT_FALSE( + Sdp("H264", Params{{"profile-level-id", "640c34"}}) + .IsSameCodec(Sdp("VP8", Params{{"profile-level-id", "640c34"}}))); + EXPECT_FALSE(Sdp("AV1", Params{{"profile", "0"}}) + .IsSameCodec(Sdp("H264", Params{{"profile", "0"}}))); + EXPECT_FALSE(Sdp("AV1", Params{{"profile", "2"}}) + .IsSameCodec(Sdp("VP9", Params{{"profile", "2"}}))); +} + +TEST(SdpVideoFormatTest, H264PacketizationMode) { + // The default packetization mode is 0. + EXPECT_TRUE(Sdp("H264", Params{{cricket::kH264FmtpPacketizationMode, "0"}}) + .IsSameCodec(Sdp("H264"))); + EXPECT_FALSE(Sdp("H264", Params{{cricket::kH264FmtpPacketizationMode, "1"}}) + .IsSameCodec(Sdp("H264"))); + + EXPECT_TRUE( + Sdp("H264", Params{{cricket::kH264FmtpPacketizationMode, "1"}}) + .IsSameCodec( + Sdp("H264", Params{{cricket::kH264FmtpPacketizationMode, "1"}}))); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/test/video_decoder_factory_template_tests.cc b/third_party/libwebrtc/api/video_codecs/test/video_decoder_factory_template_tests.cc new file mode 100644 index 0000000000..e9d7052501 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/test/video_decoder_factory_template_tests.cc @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/mock_video_decoder.h" +#include "api/video_codecs/video_decoder_factory_template.h" +#include "api/video_codecs/video_decoder_factory_template_dav1d_adapter.h" +#include "api/video_codecs/video_decoder_factory_template_libvpx_vp8_adapter.h" +#include "api/video_codecs/video_decoder_factory_template_libvpx_vp9_adapter.h" +#include "api/video_codecs/video_decoder_factory_template_open_h264_adapter.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using ::testing::Contains; +using ::testing::Each; +using ::testing::Eq; +using ::testing::Field; +using ::testing::IsEmpty; +using ::testing::Ne; +using ::testing::Not; +using ::testing::UnorderedElementsAre; + +namespace webrtc { +namespace { +const SdpVideoFormat kFooSdp("Foo"); +const SdpVideoFormat kBarLowSdp("Bar", {{"profile", "low"}}); +const SdpVideoFormat kBarHighSdp("Bar", {{"profile", "high"}}); + +struct FooDecoderTemplateAdapter { + static std::vector<SdpVideoFormat> SupportedFormats() { return {kFooSdp}; } + + static std::unique_ptr<VideoDecoder> CreateDecoder( + const SdpVideoFormat& format) { + auto decoder = std::make_unique<testing::StrictMock<MockVideoDecoder>>(); + EXPECT_CALL(*decoder, Destruct); + return decoder; + } +}; + +struct BarDecoderTemplateAdapter { + static std::vector<SdpVideoFormat> SupportedFormats() { + return {kBarLowSdp, kBarHighSdp}; + } + + static std::unique_ptr<VideoDecoder> CreateDecoder( + const SdpVideoFormat& format) { + auto decoder = std::make_unique<testing::StrictMock<MockVideoDecoder>>(); + EXPECT_CALL(*decoder, Destruct); + return decoder; + } +}; + +TEST(VideoDecoderFactoryTemplate, OneTemplateAdapterCreateDecoder) { + VideoDecoderFactoryTemplate<FooDecoderTemplateAdapter> factory; + EXPECT_THAT(factory.GetSupportedFormats(), UnorderedElementsAre(kFooSdp)); + EXPECT_THAT(factory.CreateVideoDecoder(kFooSdp), Ne(nullptr)); + EXPECT_THAT(factory.CreateVideoDecoder(SdpVideoFormat("FooX")), Eq(nullptr)); +} + +TEST(VideoDecoderFactoryTemplate, TwoTemplateAdaptersNoDuplicates) { + VideoDecoderFactoryTemplate<FooDecoderTemplateAdapter, + FooDecoderTemplateAdapter> + factory; + EXPECT_THAT(factory.GetSupportedFormats(), UnorderedElementsAre(kFooSdp)); +} + +TEST(VideoDecoderFactoryTemplate, TwoTemplateAdaptersCreateDecoders) { + VideoDecoderFactoryTemplate<FooDecoderTemplateAdapter, + BarDecoderTemplateAdapter> + factory; + EXPECT_THAT(factory.GetSupportedFormats(), + UnorderedElementsAre(kFooSdp, kBarLowSdp, kBarHighSdp)); + EXPECT_THAT(factory.CreateVideoDecoder(kFooSdp), Ne(nullptr)); + EXPECT_THAT(factory.CreateVideoDecoder(kBarLowSdp), Ne(nullptr)); + EXPECT_THAT(factory.CreateVideoDecoder(kBarHighSdp), Ne(nullptr)); + EXPECT_THAT(factory.CreateVideoDecoder(SdpVideoFormat("FooX")), Eq(nullptr)); + EXPECT_THAT(factory.CreateVideoDecoder(SdpVideoFormat("Bar")), Eq(nullptr)); +} + +TEST(VideoDecoderFactoryTemplate, LibvpxVp8) { + VideoDecoderFactoryTemplate<LibvpxVp8DecoderTemplateAdapter> factory; + auto formats = factory.GetSupportedFormats(); + EXPECT_THAT(formats.size(), 1); + EXPECT_THAT(formats[0], Field(&SdpVideoFormat::name, "VP8")); + EXPECT_THAT(factory.CreateVideoDecoder(formats[0]), Ne(nullptr)); +} + +TEST(VideoDecoderFactoryTemplate, LibvpxVp9) { + VideoDecoderFactoryTemplate<LibvpxVp9DecoderTemplateAdapter> factory; + auto formats = factory.GetSupportedFormats(); + EXPECT_THAT(formats, Not(IsEmpty())); + EXPECT_THAT(formats, Each(Field(&SdpVideoFormat::name, "VP9"))); + EXPECT_THAT(factory.CreateVideoDecoder(formats[0]), Ne(nullptr)); +} + +// TODO(bugs.webrtc.org/13573): When OpenH264 is no longer a conditional build +// target remove this #ifdef. +#if defined(WEBRTC_USE_H264) +TEST(VideoDecoderFactoryTemplate, OpenH264) { + VideoDecoderFactoryTemplate<OpenH264DecoderTemplateAdapter> factory; + auto formats = factory.GetSupportedFormats(); + EXPECT_THAT(formats, Not(IsEmpty())); + EXPECT_THAT(formats, Each(Field(&SdpVideoFormat::name, "H264"))); + EXPECT_THAT(factory.CreateVideoDecoder(formats[0]), Ne(nullptr)); +} +#endif // defined(WEBRTC_USE_H264) + +TEST(VideoDecoderFactoryTemplate, Dav1d) { + VideoDecoderFactoryTemplate<Dav1dDecoderTemplateAdapter> factory; + auto formats = factory.GetSupportedFormats(); + EXPECT_THAT(formats.size(), 1); + EXPECT_THAT(formats[0], Field(&SdpVideoFormat::name, "AV1")); + EXPECT_THAT(factory.CreateVideoDecoder(formats[0]), Ne(nullptr)); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/test/video_decoder_software_fallback_wrapper_unittest.cc b/third_party/libwebrtc/api/video_codecs/test/video_decoder_software_fallback_wrapper_unittest.cc new file mode 100644 index 0000000000..73dedc8395 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/test/video_decoder_software_fallback_wrapper_unittest.cc @@ -0,0 +1,305 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/video_decoder_software_fallback_wrapper.h" + +#include <stdint.h> + +#include "absl/types/optional.h" +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/video_decoder.h" +#include "modules/video_coding/codecs/vp8/include/vp8.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/checks.h" +#include "test/field_trial.h" +#include "test/gtest.h" + +namespace webrtc { + +class VideoDecoderSoftwareFallbackWrapperTest : public ::testing::Test { + protected: + VideoDecoderSoftwareFallbackWrapperTest() + : VideoDecoderSoftwareFallbackWrapperTest("") {} + explicit VideoDecoderSoftwareFallbackWrapperTest( + const std::string& field_trials) + : override_field_trials_(field_trials), + fake_decoder_(new CountingFakeDecoder()), + fallback_wrapper_(CreateVideoDecoderSoftwareFallbackWrapper( + std::unique_ptr<VideoDecoder>(VP8Decoder::Create()), + std::unique_ptr<VideoDecoder>(fake_decoder_))) {} + + class CountingFakeDecoder : public VideoDecoder { + public: + bool Configure(const Settings& settings) override { + ++configure_count_; + return configure_return_value_; + } + + int32_t Decode(const EncodedImage& input_image, + bool missing_frames, + int64_t render_time_ms) override { + ++decode_count_; + return decode_return_code_; + } + + int32_t RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) override { + decode_complete_callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Release() override { + ++release_count_; + return WEBRTC_VIDEO_CODEC_OK; + } + + const char* ImplementationName() const override { return "fake-decoder"; } + + int configure_count_ = 0; + int decode_count_ = 0; + bool configure_return_value_ = true; + int32_t decode_return_code_ = WEBRTC_VIDEO_CODEC_OK; + DecodedImageCallback* decode_complete_callback_ = nullptr; + int release_count_ = 0; + int reset_count_ = 0; + }; + test::ScopedFieldTrials override_field_trials_; + // `fake_decoder_` is owned and released by `fallback_wrapper_`. + CountingFakeDecoder* fake_decoder_; + std::unique_ptr<VideoDecoder> fallback_wrapper_; +}; + +TEST_F(VideoDecoderSoftwareFallbackWrapperTest, InitializesDecoder) { + fallback_wrapper_->Configure({}); + EXPECT_EQ(1, fake_decoder_->configure_count_); + + EncodedImage encoded_image; + encoded_image._frameType = VideoFrameType::kVideoFrameKey; + fallback_wrapper_->Decode(encoded_image, false, -1); + EXPECT_EQ(1, fake_decoder_->configure_count_) + << "Initialized decoder should not be reinitialized."; + EXPECT_EQ(1, fake_decoder_->decode_count_); +} + +TEST_F(VideoDecoderSoftwareFallbackWrapperTest, + UsesFallbackDecoderAfterAnyInitDecodeFailure) { + fake_decoder_->configure_return_value_ = false; + fallback_wrapper_->Configure({}); + EXPECT_EQ(1, fake_decoder_->configure_count_); + + EncodedImage encoded_image; + encoded_image._frameType = VideoFrameType::kVideoFrameKey; + fallback_wrapper_->Decode(encoded_image, false, -1); + EXPECT_EQ(1, fake_decoder_->configure_count_) + << "Should not have attempted reinitializing the fallback decoder on " + "keyframe."; + // Unfortunately faking a VP8 frame is hard. Rely on no Decode -> using SW + // decoder. + EXPECT_EQ(0, fake_decoder_->decode_count_) + << "Decoder used even though no InitDecode had succeeded."; +} + +TEST_F(VideoDecoderSoftwareFallbackWrapperTest, IsSoftwareFallbackSticky) { + fallback_wrapper_->Configure({}); + + fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + EncodedImage encoded_image; + fallback_wrapper_->Decode(encoded_image, false, -1); + EXPECT_EQ(1, fake_decoder_->decode_count_); + + // Software fallback should be sticky, fake_decoder_ shouldn't be used. + encoded_image._frameType = VideoFrameType::kVideoFrameKey; + fallback_wrapper_->Decode(encoded_image, false, -1); + EXPECT_EQ(1, fake_decoder_->decode_count_) + << "Decoder shouldn't be used after failure."; + + // fake_decoder_ should have only been initialized once during the test. + EXPECT_EQ(1, fake_decoder_->configure_count_); +} + +TEST_F(VideoDecoderSoftwareFallbackWrapperTest, DoesNotFallbackOnEveryError) { + fallback_wrapper_->Configure({}); + fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR; + EncodedImage encoded_image; + EXPECT_EQ(fake_decoder_->decode_return_code_, + fallback_wrapper_->Decode(encoded_image, false, -1)); + EXPECT_EQ(1, fake_decoder_->decode_count_); + + fallback_wrapper_->Decode(encoded_image, false, -1); + EXPECT_EQ(2, fake_decoder_->decode_count_) + << "Decoder should be active even though previous decode failed."; +} + +TEST_F(VideoDecoderSoftwareFallbackWrapperTest, UsesHwDecoderAfterReinit) { + fallback_wrapper_->Configure({}); + + fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + EncodedImage encoded_image; + fallback_wrapper_->Decode(encoded_image, false, -1); + EXPECT_EQ(1, fake_decoder_->decode_count_); + + fallback_wrapper_->Release(); + fallback_wrapper_->Configure({}); + + fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_OK; + fallback_wrapper_->Decode(encoded_image, false, -1); + EXPECT_EQ(2, fake_decoder_->decode_count_) + << "Should not be using fallback after reinit."; +} + +TEST_F(VideoDecoderSoftwareFallbackWrapperTest, ForwardsReleaseCall) { + fallback_wrapper_->Configure({}); + fallback_wrapper_->Release(); + EXPECT_EQ(1, fake_decoder_->release_count_); + + fallback_wrapper_->Configure({}); + fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + EncodedImage encoded_image; + fallback_wrapper_->Decode(encoded_image, false, -1); + EXPECT_EQ(2, fake_decoder_->release_count_) + << "Decoder should be released during fallback."; + fallback_wrapper_->Release(); + EXPECT_EQ(2, fake_decoder_->release_count_); +} + +// TODO(pbos): Fake a VP8 frame well enough to actually receive a callback from +// the software decoder. +TEST_F(VideoDecoderSoftwareFallbackWrapperTest, + ForwardsRegisterDecodeCompleteCallback) { + class FakeDecodedImageCallback : public DecodedImageCallback { + int32_t Decoded(VideoFrame& decodedImage) override { return 0; } + int32_t Decoded(webrtc::VideoFrame& decodedImage, + int64_t decode_time_ms) override { + RTC_DCHECK_NOTREACHED(); + return -1; + } + void Decoded(webrtc::VideoFrame& decodedImage, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp) override { + RTC_DCHECK_NOTREACHED(); + } + } callback; + + fallback_wrapper_->Configure({}); + fallback_wrapper_->RegisterDecodeCompleteCallback(&callback); + EXPECT_EQ(&callback, fake_decoder_->decode_complete_callback_); +} + +TEST_F(VideoDecoderSoftwareFallbackWrapperTest, + ReportsFallbackImplementationName) { + fallback_wrapper_->Configure({}); + + fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + EncodedImage encoded_image; + fallback_wrapper_->Decode(encoded_image, false, -1); + // Hard coded expected value since libvpx is the software implementation name + // for VP8. Change accordingly if the underlying implementation does. + EXPECT_STREQ("libvpx (fallback from: fake-decoder)", + fallback_wrapper_->ImplementationName()); + fallback_wrapper_->Release(); +} + +TEST_F(VideoDecoderSoftwareFallbackWrapperTest, FallbacksOnTooManyErrors) { + fallback_wrapper_->Configure({}); + + fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR; + EncodedImage encoded_image; + encoded_image._frameType = VideoFrameType::kVideoFrameKey; + // Doesn't fallback from a single error. + fallback_wrapper_->Decode(encoded_image, false, -1); + EXPECT_STREQ("fake-decoder", fallback_wrapper_->ImplementationName()); + + // However, many frames with the same error, fallback should happen. + const int kNumFramesToEncode = 10; + for (int i = 0; i < kNumFramesToEncode; ++i) { + fallback_wrapper_->Decode(encoded_image, false, -1); + } + // Hard coded expected value since libvpx is the software implementation name + // for VP8. Change accordingly if the underlying implementation does. + EXPECT_STREQ("libvpx (fallback from: fake-decoder)", + fallback_wrapper_->ImplementationName()); + fallback_wrapper_->Release(); +} + +TEST_F(VideoDecoderSoftwareFallbackWrapperTest, + DoesNotFallbackOnDeltaFramesErrors) { + fallback_wrapper_->Configure({}); + + fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR; + EncodedImage encoded_image; + encoded_image._frameType = VideoFrameType::kVideoFrameDelta; + + // Many decoded frames with the same error + const int kNumFramesToEncode = 10; + for (int i = 0; i < kNumFramesToEncode; ++i) { + fallback_wrapper_->Decode(encoded_image, false, -1); + } + EXPECT_STREQ("fake-decoder", fallback_wrapper_->ImplementationName()); + + fallback_wrapper_->Release(); +} + +TEST_F(VideoDecoderSoftwareFallbackWrapperTest, + DoesNotFallbacksOnNonConsequtiveErrors) { + fallback_wrapper_->Configure({}); + + EncodedImage encoded_image; + encoded_image._frameType = VideoFrameType::kVideoFrameKey; + + const int kNumFramesToEncode = 10; + for (int i = 0; i < kNumFramesToEncode; ++i) { + // Interleaved errors and successful decodes. + fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR; + fallback_wrapper_->Decode(encoded_image, false, -1); + fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_OK; + fallback_wrapper_->Decode(encoded_image, false, -1); + } + EXPECT_STREQ("fake-decoder", fallback_wrapper_->ImplementationName()); + fallback_wrapper_->Release(); +} + +class ForcedSoftwareDecoderFallbackTest + : public VideoDecoderSoftwareFallbackWrapperTest { + public: + ForcedSoftwareDecoderFallbackTest() + : VideoDecoderSoftwareFallbackWrapperTest( + "WebRTC-Video-ForcedSwDecoderFallback/Enabled/") { + fake_decoder_ = new CountingFakeDecoder(); + sw_fallback_decoder_ = new CountingFakeDecoder(); + fallback_wrapper_ = CreateVideoDecoderSoftwareFallbackWrapper( + std::unique_ptr<VideoDecoder>(sw_fallback_decoder_), + std::unique_ptr<VideoDecoder>(fake_decoder_)); + } + + CountingFakeDecoder* sw_fallback_decoder_; +}; + +TEST_F(ForcedSoftwareDecoderFallbackTest, UsesForcedFallback) { + fallback_wrapper_->Configure({}); + EXPECT_EQ(1, sw_fallback_decoder_->configure_count_); + + EncodedImage encoded_image; + encoded_image._frameType = VideoFrameType::kVideoFrameKey; + fallback_wrapper_->Decode(encoded_image, false, -1); + EXPECT_EQ(1, sw_fallback_decoder_->configure_count_); + EXPECT_EQ(1, sw_fallback_decoder_->decode_count_); + + fallback_wrapper_->Release(); + EXPECT_EQ(1, sw_fallback_decoder_->release_count_); + + // Only fallback decoder should have been used. + EXPECT_EQ(0, fake_decoder_->configure_count_); + EXPECT_EQ(0, fake_decoder_->decode_count_); + EXPECT_EQ(0, fake_decoder_->release_count_); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/test/video_encoder_factory_template_tests.cc b/third_party/libwebrtc/api/video_codecs/test/video_encoder_factory_template_tests.cc new file mode 100644 index 0000000000..4c3d0cd24e --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/test/video_encoder_factory_template_tests.cc @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/test/mock_video_encoder.h" +#include "api/video_codecs/video_encoder_factory_template.h" +#include "api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h" +#include "api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h" +#include "api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h" +#include "api/video_codecs/video_encoder_factory_template_open_h264_adapter.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using ::testing::Contains; +using ::testing::Each; +using ::testing::Eq; +using ::testing::Field; +using ::testing::IsEmpty; +using ::testing::Ne; +using ::testing::Not; +using ::testing::UnorderedElementsAre; + +namespace webrtc { +namespace { +using CodecSupport = VideoEncoderFactory::CodecSupport; +const SdpVideoFormat kFooSdp("Foo"); +const SdpVideoFormat kBarLowSdp("Bar", {{"profile", "low"}}); +const SdpVideoFormat kBarHighSdp("Bar", {{"profile", "high"}}); + +struct FooEncoderTemplateAdapter { + static std::vector<SdpVideoFormat> SupportedFormats() { return {kFooSdp}; } + + static std::unique_ptr<VideoEncoder> CreateEncoder( + const SdpVideoFormat& format) { + return std::make_unique<testing::StrictMock<MockVideoEncoder>>(); + } + + static bool IsScalabilityModeSupported(ScalabilityMode scalability_mode) { + return scalability_mode == ScalabilityMode::kL1T2 || + scalability_mode == ScalabilityMode::kL1T3; + } +}; + +struct BarEncoderTemplateAdapter { + static std::vector<SdpVideoFormat> SupportedFormats() { + return {kBarLowSdp, kBarHighSdp}; + } + + static std::unique_ptr<VideoEncoder> CreateEncoder( + const SdpVideoFormat& format) { + return std::make_unique<testing::StrictMock<MockVideoEncoder>>(); + } + + static bool IsScalabilityModeSupported(ScalabilityMode scalability_mode) { + return scalability_mode == ScalabilityMode::kL1T2 || + scalability_mode == ScalabilityMode::kL1T3 || + scalability_mode == ScalabilityMode::kS2T1 || + scalability_mode == ScalabilityMode::kS3T3; + } +}; + +TEST(VideoEncoderFactoryTemplate, OneTemplateAdapterCreateEncoder) { + VideoEncoderFactoryTemplate<FooEncoderTemplateAdapter> factory; + EXPECT_THAT(factory.GetSupportedFormats(), UnorderedElementsAre(kFooSdp)); + EXPECT_THAT(factory.CreateVideoEncoder(kFooSdp), Ne(nullptr)); + EXPECT_THAT(factory.CreateVideoEncoder(SdpVideoFormat("FooX")), Eq(nullptr)); +} + +TEST(VideoEncoderFactoryTemplate, OneTemplateAdapterCodecSupport) { + VideoEncoderFactoryTemplate<FooEncoderTemplateAdapter> factory; + EXPECT_THAT(factory.QueryCodecSupport(kFooSdp, absl::nullopt), + Field(&CodecSupport::is_supported, true)); + EXPECT_THAT(factory.QueryCodecSupport(kFooSdp, "L1T2"), + Field(&CodecSupport::is_supported, true)); + EXPECT_THAT(factory.QueryCodecSupport(kFooSdp, "S3T3"), + Field(&CodecSupport::is_supported, false)); + EXPECT_THAT(factory.QueryCodecSupport(SdpVideoFormat("FooX"), absl::nullopt), + Field(&CodecSupport::is_supported, false)); +} + +TEST(VideoEncoderFactoryTemplate, TwoTemplateAdaptersNoDuplicates) { + VideoEncoderFactoryTemplate<FooEncoderTemplateAdapter, + FooEncoderTemplateAdapter> + factory; + EXPECT_THAT(factory.GetSupportedFormats(), UnorderedElementsAre(kFooSdp)); +} + +TEST(VideoEncoderFactoryTemplate, TwoTemplateAdaptersCreateEncoders) { + VideoEncoderFactoryTemplate<FooEncoderTemplateAdapter, + BarEncoderTemplateAdapter> + factory; + EXPECT_THAT(factory.GetSupportedFormats(), + UnorderedElementsAre(kFooSdp, kBarLowSdp, kBarHighSdp)); + EXPECT_THAT(factory.CreateVideoEncoder(kFooSdp), Ne(nullptr)); + EXPECT_THAT(factory.CreateVideoEncoder(kBarLowSdp), Ne(nullptr)); + EXPECT_THAT(factory.CreateVideoEncoder(kBarHighSdp), Ne(nullptr)); + EXPECT_THAT(factory.CreateVideoEncoder(SdpVideoFormat("FooX")), Eq(nullptr)); + EXPECT_THAT(factory.CreateVideoEncoder(SdpVideoFormat("Bar")), Eq(nullptr)); +} + +TEST(VideoEncoderFactoryTemplate, TwoTemplateAdaptersCodecSupport) { + VideoEncoderFactoryTemplate<FooEncoderTemplateAdapter, + BarEncoderTemplateAdapter> + factory; + EXPECT_THAT(factory.QueryCodecSupport(kFooSdp, absl::nullopt), + Field(&CodecSupport::is_supported, true)); + EXPECT_THAT(factory.QueryCodecSupport(kFooSdp, "L1T2"), + Field(&CodecSupport::is_supported, true)); + EXPECT_THAT(factory.QueryCodecSupport(kFooSdp, "S3T3"), + Field(&CodecSupport::is_supported, false)); + EXPECT_THAT(factory.QueryCodecSupport(kBarLowSdp, absl::nullopt), + Field(&CodecSupport::is_supported, true)); + EXPECT_THAT(factory.QueryCodecSupport(kBarHighSdp, absl::nullopt), + Field(&CodecSupport::is_supported, true)); + EXPECT_THAT(factory.QueryCodecSupport(kBarLowSdp, "S2T1"), + Field(&CodecSupport::is_supported, true)); + EXPECT_THAT(factory.QueryCodecSupport(kBarHighSdp, "S3T2"), + Field(&CodecSupport::is_supported, false)); +} + +TEST(VideoEncoderFactoryTemplate, LibvpxVp8) { + VideoEncoderFactoryTemplate<LibvpxVp8EncoderTemplateAdapter> factory; + auto formats = factory.GetSupportedFormats(); + EXPECT_THAT(formats.size(), 1); + EXPECT_THAT(formats[0], Field(&SdpVideoFormat::name, "VP8")); + EXPECT_THAT(formats[0], Field(&SdpVideoFormat::scalability_modes, + Contains(ScalabilityMode::kL1T3))); + EXPECT_THAT(factory.CreateVideoEncoder(formats[0]), Ne(nullptr)); +} + +TEST(VideoEncoderFactoryTemplate, LibvpxVp9) { + VideoEncoderFactoryTemplate<LibvpxVp9EncoderTemplateAdapter> factory; + auto formats = factory.GetSupportedFormats(); + EXPECT_THAT(formats, Not(IsEmpty())); + EXPECT_THAT(formats, Each(Field(&SdpVideoFormat::name, "VP9"))); + EXPECT_THAT(formats, Each(Field(&SdpVideoFormat::scalability_modes, + Contains(ScalabilityMode::kL3T3_KEY)))); + EXPECT_THAT(factory.CreateVideoEncoder(formats[0]), Ne(nullptr)); +} + +// TODO(bugs.webrtc.org/13573): When OpenH264 is no longer a conditional build +// target remove this #ifdef. +#if defined(WEBRTC_USE_H264) +TEST(VideoEncoderFactoryTemplate, OpenH264) { + VideoEncoderFactoryTemplate<OpenH264EncoderTemplateAdapter> factory; + auto formats = factory.GetSupportedFormats(); + EXPECT_THAT(formats, Not(IsEmpty())); + EXPECT_THAT(formats, Each(Field(&SdpVideoFormat::name, "H264"))); + EXPECT_THAT(formats, Each(Field(&SdpVideoFormat::scalability_modes, + Contains(ScalabilityMode::kL1T3)))); + EXPECT_THAT(factory.CreateVideoEncoder(formats[0]), Ne(nullptr)); +} +#endif // defined(WEBRTC_USE_H264) + +TEST(VideoEncoderFactoryTemplate, LibaomAv1) { + VideoEncoderFactoryTemplate<LibaomAv1EncoderTemplateAdapter> factory; + auto formats = factory.GetSupportedFormats(); + EXPECT_THAT(formats.size(), 1); + EXPECT_THAT(formats[0], Field(&SdpVideoFormat::name, "AV1")); + EXPECT_THAT(formats[0], Field(&SdpVideoFormat::scalability_modes, + Contains(ScalabilityMode::kL3T3_KEY))); + EXPECT_THAT(factory.CreateVideoEncoder(formats[0]), Ne(nullptr)); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc b/third_party/libwebrtc/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc new file mode 100644 index 0000000000..b213356e7b --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc @@ -0,0 +1,1055 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/video_encoder_software_fallback_wrapper.h" + +#include <stddef.h> +#include <stdint.h> + +#include <memory> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/fec_controller_override.h" +#include "api/scoped_refptr.h" +#include "api/test/mock_video_encoder.h" +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_bitrate_allocation.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_buffer.h" +#include "api/video/video_rotation.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/codecs/vp8/include/vp8.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/utility/simulcast_rate_allocator.h" +#include "rtc_base/fake_clock.h" +#include "test/fake_texture_frame.h" +#include "test/field_trial.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +using ::testing::_; +using ::testing::Return; + +namespace { +const int kWidth = 320; +const int kHeight = 240; +const int kNumCores = 2; +const uint32_t kFramerate = 30; +const size_t kMaxPayloadSize = 800; +const int kLowThreshold = 10; +const int kHighThreshold = 20; + +const VideoEncoder::Capabilities kCapabilities(false); +const VideoEncoder::Settings kSettings(kCapabilities, + kNumCores, + kMaxPayloadSize); + +VideoEncoder::EncoderInfo GetEncoderInfoWithTrustedRateController( + bool trusted_rate_controller) { + VideoEncoder::EncoderInfo info; + info.has_trusted_rate_controller = trusted_rate_controller; + return info; +} + +VideoEncoder::EncoderInfo GetEncoderInfoWithHardwareAccelerated( + bool hardware_accelerated) { + VideoEncoder::EncoderInfo info; + info.is_hardware_accelerated = hardware_accelerated; + return info; +} + +class FakeEncodedImageCallback : public EncodedImageCallback { + public: + Result OnEncodedImage(const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info) override { + ++callback_count_; + return Result(Result::OK, callback_count_); + } + int callback_count_ = 0; +}; +} // namespace + +class VideoEncoderSoftwareFallbackWrapperTestBase : public ::testing::Test { + protected: + VideoEncoderSoftwareFallbackWrapperTestBase( + const std::string& field_trials, + std::unique_ptr<VideoEncoder> sw_encoder) + : override_field_trials_(field_trials), + fake_encoder_(new CountingFakeEncoder()), + wrapper_initialized_(false), + fallback_wrapper_(CreateVideoEncoderSoftwareFallbackWrapper( + std::move(sw_encoder), + std::unique_ptr<VideoEncoder>(fake_encoder_), + false)) {} + + class CountingFakeEncoder : public VideoEncoder { + public: + void SetFecControllerOverride( + FecControllerOverride* fec_controller_override) override { + // Ignored. + } + + int32_t InitEncode(const VideoCodec* codec_settings, + const VideoEncoder::Settings& settings) override { + ++init_encode_count_; + return init_encode_return_code_; + } + + int32_t Encode(const VideoFrame& frame, + const std::vector<VideoFrameType>* frame_types) override { + ++encode_count_; + last_video_frame_ = frame; + if (encode_complete_callback_ && + encode_return_code_ == WEBRTC_VIDEO_CODEC_OK) { + encode_complete_callback_->OnEncodedImage(EncodedImage(), nullptr); + } + return encode_return_code_; + } + + int32_t RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) override { + encode_complete_callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Release() override { + ++release_count_; + return WEBRTC_VIDEO_CODEC_OK; + } + + void SetRates(const RateControlParameters& parameters) override {} + + EncoderInfo GetEncoderInfo() const override { + ++supports_native_handle_count_; + EncoderInfo info; + info.scaling_settings = ScalingSettings(kLowThreshold, kHighThreshold); + info.supports_native_handle = supports_native_handle_; + info.implementation_name = implementation_name_; + if (is_qp_trusted_) + info.is_qp_trusted = is_qp_trusted_; + return info; + } + + int init_encode_count_ = 0; + int32_t init_encode_return_code_ = WEBRTC_VIDEO_CODEC_OK; + int32_t encode_return_code_ = WEBRTC_VIDEO_CODEC_OK; + int encode_count_ = 0; + EncodedImageCallback* encode_complete_callback_ = nullptr; + int release_count_ = 0; + mutable int supports_native_handle_count_ = 0; + bool supports_native_handle_ = false; + bool is_qp_trusted_ = false; + std::string implementation_name_ = "fake-encoder"; + absl::optional<VideoFrame> last_video_frame_; + }; + + void InitEncode(); + void UtilizeFallbackEncoder(); + void FallbackFromEncodeRequest(); + void EncodeFrame(); + void EncodeFrame(int expected_ret); + void CheckLastEncoderName(const char* expected_name) { + EXPECT_EQ(expected_name, + fallback_wrapper_->GetEncoderInfo().implementation_name); + } + + test::ScopedFieldTrials override_field_trials_; + FakeEncodedImageCallback callback_; + // `fake_encoder_` is owned and released by `fallback_wrapper_`. + CountingFakeEncoder* fake_encoder_; + CountingFakeEncoder* fake_sw_encoder_; + bool wrapper_initialized_; + std::unique_ptr<VideoEncoder> fallback_wrapper_; + VideoCodec codec_ = {}; + std::unique_ptr<VideoFrame> frame_; + std::unique_ptr<SimulcastRateAllocator> rate_allocator_; +}; + +class VideoEncoderSoftwareFallbackWrapperTest + : public VideoEncoderSoftwareFallbackWrapperTestBase { + protected: + VideoEncoderSoftwareFallbackWrapperTest() + : VideoEncoderSoftwareFallbackWrapperTest(new CountingFakeEncoder()) {} + explicit VideoEncoderSoftwareFallbackWrapperTest( + CountingFakeEncoder* fake_sw_encoder) + : VideoEncoderSoftwareFallbackWrapperTestBase( + "", + std::unique_ptr<VideoEncoder>(fake_sw_encoder)), + fake_sw_encoder_(fake_sw_encoder) { + fake_sw_encoder_->implementation_name_ = "fake_sw_encoder"; + } + + CountingFakeEncoder* fake_sw_encoder_; +}; + +void VideoEncoderSoftwareFallbackWrapperTestBase::EncodeFrame() { + EncodeFrame(WEBRTC_VIDEO_CODEC_OK); +} + +void VideoEncoderSoftwareFallbackWrapperTestBase::EncodeFrame( + int expected_ret) { + rtc::scoped_refptr<I420Buffer> buffer = + I420Buffer::Create(codec_.width, codec_.height); + I420Buffer::SetBlack(buffer.get()); + std::vector<VideoFrameType> types(1, VideoFrameType::kVideoFrameKey); + + frame_ = + std::make_unique<VideoFrame>(VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(0) + .build()); + EXPECT_EQ(expected_ret, fallback_wrapper_->Encode(*frame_, &types)); +} + +void VideoEncoderSoftwareFallbackWrapperTestBase::InitEncode() { + if (!wrapper_initialized_) { + fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_); + EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_); + } + + // Register fake encoder as main. + codec_.codecType = kVideoCodecVP8; + codec_.maxFramerate = kFramerate; + codec_.width = kWidth; + codec_.height = kHeight; + codec_.VP8()->numberOfTemporalLayers = 1; + rate_allocator_.reset(new SimulcastRateAllocator(codec_)); + + if (wrapper_initialized_) { + fallback_wrapper_->Release(); + } + + fake_encoder_->init_encode_return_code_ = WEBRTC_VIDEO_CODEC_OK; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_->InitEncode(&codec_, kSettings)); + + if (!wrapper_initialized_) { + fallback_wrapper_->SetRates(VideoEncoder::RateControlParameters( + rate_allocator_->Allocate( + VideoBitrateAllocationParameters(300000, kFramerate)), + kFramerate)); + } + wrapper_initialized_ = true; +} + +void VideoEncoderSoftwareFallbackWrapperTestBase::UtilizeFallbackEncoder() { + if (!wrapper_initialized_) { + fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_); + EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_); + } + + // Register with failing fake encoder. Should succeed with VP8 fallback. + codec_.codecType = kVideoCodecVP8; + codec_.maxFramerate = kFramerate; + codec_.width = kWidth; + codec_.height = kHeight; + codec_.VP8()->numberOfTemporalLayers = 1; + rate_allocator_.reset(new SimulcastRateAllocator(codec_)); + + if (wrapper_initialized_) { + fallback_wrapper_->Release(); + } + + fake_encoder_->init_encode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR; + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_->InitEncode(&codec_, kSettings)); + fallback_wrapper_->SetRates(VideoEncoder::RateControlParameters( + rate_allocator_->Allocate( + VideoBitrateAllocationParameters(300000, kFramerate)), + kFramerate)); + + int callback_count = callback_.callback_count_; + int encode_count = fake_encoder_->encode_count_; + EncodeFrame(); + EXPECT_EQ(encode_count, fake_encoder_->encode_count_); + EXPECT_EQ(callback_count + 1, callback_.callback_count_); +} + +void VideoEncoderSoftwareFallbackWrapperTestBase::FallbackFromEncodeRequest() { + fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_); + codec_.codecType = kVideoCodecVP8; + codec_.maxFramerate = kFramerate; + codec_.width = kWidth; + codec_.height = kHeight; + codec_.VP8()->numberOfTemporalLayers = 1; + rate_allocator_.reset(new SimulcastRateAllocator(codec_)); + if (wrapper_initialized_) { + fallback_wrapper_->Release(); + } + fallback_wrapper_->InitEncode(&codec_, kSettings); + fallback_wrapper_->SetRates(VideoEncoder::RateControlParameters( + rate_allocator_->Allocate( + VideoBitrateAllocationParameters(300000, kFramerate)), + kFramerate)); + EXPECT_EQ(1, fake_encoder_->init_encode_count_); + + // Have the non-fallback encoder request a software fallback. + fake_encoder_->encode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + int callback_count = callback_.callback_count_; + int encode_count = fake_encoder_->encode_count_; + EncodeFrame(); + // Single encode request, which returned failure. + EXPECT_EQ(encode_count + 1, fake_encoder_->encode_count_); + EXPECT_EQ(callback_count + 1, callback_.callback_count_); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, InitializesEncoder) { + VideoCodec codec = {}; + fallback_wrapper_->InitEncode(&codec, kSettings); + EXPECT_EQ(1, fake_encoder_->init_encode_count_); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, EncodeRequestsFallback) { + FallbackFromEncodeRequest(); + // After fallback, further encodes shouldn't hit the fake encoder. + int encode_count = fake_encoder_->encode_count_; + EncodeFrame(); + EXPECT_EQ(encode_count, fake_encoder_->encode_count_); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, CanUtilizeFallbackEncoder) { + UtilizeFallbackEncoder(); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release()); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + InternalEncoderReleasedDuringFallback) { + EXPECT_EQ(0, fake_encoder_->init_encode_count_); + EXPECT_EQ(0, fake_encoder_->release_count_); + + InitEncode(); + + EXPECT_EQ(1, fake_encoder_->init_encode_count_); + EXPECT_EQ(0, fake_encoder_->release_count_); + + UtilizeFallbackEncoder(); + + // One successful InitEncode(), one failed. + EXPECT_EQ(2, fake_encoder_->init_encode_count_); + EXPECT_EQ(1, fake_encoder_->release_count_); + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release()); + + // No extra release when the fallback is released. + EXPECT_EQ(2, fake_encoder_->init_encode_count_); + EXPECT_EQ(1, fake_encoder_->release_count_); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + InternalEncoderNotEncodingDuringFallback) { + UtilizeFallbackEncoder(); + int encode_count = fake_encoder_->encode_count_; + EncodeFrame(); + EXPECT_EQ(encode_count, fake_encoder_->encode_count_); + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release()); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + CanRegisterCallbackWhileUsingFallbackEncoder) { + InitEncode(); + EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_); + + UtilizeFallbackEncoder(); + + // Registering an encode-complete callback will now pass to the fallback + // instead of the main encoder. + FakeEncodedImageCallback callback2; + fallback_wrapper_->RegisterEncodeCompleteCallback(&callback2); + EXPECT_EQ(&callback_, fake_encoder_->encode_complete_callback_); + + // Encoding a frame using the fallback should arrive at the new callback. + std::vector<VideoFrameType> types(1, VideoFrameType::kVideoFrameKey); + frame_->set_timestamp(frame_->timestamp() + 1000); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Encode(*frame_, &types)); + EXPECT_EQ(callback2.callback_count_, 1); + + // Re-initialize to use the main encoder, the new callback should be in use. + InitEncode(); + EXPECT_EQ(&callback2, fake_encoder_->encode_complete_callback_); + + frame_->set_timestamp(frame_->timestamp() + 2000); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Encode(*frame_, &types)); + EXPECT_EQ(callback2.callback_count_, 2); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + SupportsNativeHandleForwardedWithoutFallback) { + fallback_wrapper_->GetEncoderInfo(); + EXPECT_EQ(1, fake_encoder_->supports_native_handle_count_); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + SupportsNativeHandleNotForwardedDuringFallback) { + // Fake encoder signals support for native handle, default (libvpx) does not. + fake_encoder_->supports_native_handle_ = true; + EXPECT_TRUE(fallback_wrapper_->GetEncoderInfo().supports_native_handle); + UtilizeFallbackEncoder(); + EXPECT_FALSE(fallback_wrapper_->GetEncoderInfo().supports_native_handle); + // Both times, both encoders are queried. + EXPECT_EQ(2, fake_encoder_->supports_native_handle_count_); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release()); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, ReportsImplementationName) { + codec_.width = kWidth; + codec_.height = kHeight; + fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_); + fallback_wrapper_->InitEncode(&codec_, kSettings); + EncodeFrame(); + CheckLastEncoderName("fake-encoder"); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + IsQpTrustedNotForwardedDuringFallback) { + // Fake encoder signals trusted QP, default (libvpx) does not. + fake_encoder_->is_qp_trusted_ = true; + EXPECT_TRUE(fake_encoder_->GetEncoderInfo().is_qp_trusted.value_or(false)); + UtilizeFallbackEncoder(); + EXPECT_FALSE(fallback_wrapper_->GetEncoderInfo().is_qp_trusted.has_value()); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release()); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + ReportsFallbackImplementationName) { + UtilizeFallbackEncoder(); + CheckLastEncoderName(fake_sw_encoder_->implementation_name_.c_str()); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + OnEncodeFallbackNativeFrameScaledIfFallbackDoesNotSupportNativeFrames) { + fake_encoder_->supports_native_handle_ = true; + fake_sw_encoder_->supports_native_handle_ = false; + InitEncode(); + int width = codec_.width * 2; + int height = codec_.height * 2; + VideoFrame native_frame = test::FakeNativeBuffer::CreateFrame( + width, height, 0, 0, VideoRotation::kVideoRotation_0); + std::vector<VideoFrameType> types(1, VideoFrameType::kVideoFrameKey); + fake_encoder_->encode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_->Encode(native_frame, &types)); + EXPECT_EQ(1, fake_sw_encoder_->encode_count_); + ASSERT_TRUE(fake_sw_encoder_->last_video_frame_.has_value()); + EXPECT_NE(VideoFrameBuffer::Type::kNative, + fake_sw_encoder_->last_video_frame_->video_frame_buffer()->type()); + EXPECT_EQ(codec_.width, fake_sw_encoder_->last_video_frame_->width()); + EXPECT_EQ(codec_.height, fake_sw_encoder_->last_video_frame_->height()); +} + +TEST_F(VideoEncoderSoftwareFallbackWrapperTest, + OnEncodeFallbackNativeFrameForwardedToFallbackIfItSupportsNativeFrames) { + fake_encoder_->supports_native_handle_ = true; + fake_sw_encoder_->supports_native_handle_ = true; + InitEncode(); + int width = codec_.width * 2; + int height = codec_.height * 2; + VideoFrame native_frame = test::FakeNativeBuffer::CreateFrame( + width, height, 0, 0, VideoRotation::kVideoRotation_0); + std::vector<VideoFrameType> types(1, VideoFrameType::kVideoFrameKey); + fake_encoder_->encode_return_code_ = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_->Encode(native_frame, &types)); + EXPECT_EQ(1, fake_sw_encoder_->encode_count_); + ASSERT_TRUE(fake_sw_encoder_->last_video_frame_.has_value()); + EXPECT_EQ(VideoFrameBuffer::Type::kNative, + fake_sw_encoder_->last_video_frame_->video_frame_buffer()->type()); + EXPECT_EQ(native_frame.width(), fake_sw_encoder_->last_video_frame_->width()); + EXPECT_EQ(native_frame.height(), + fake_sw_encoder_->last_video_frame_->height()); +} + +namespace { +const int kBitrateKbps = 200; +const int kMinPixelsPerFrame = 1; +const char kFieldTrial[] = "WebRTC-VP8-Forced-Fallback-Encoder-v2"; +} // namespace + +class ForcedFallbackTest : public VideoEncoderSoftwareFallbackWrapperTestBase { + public: + explicit ForcedFallbackTest(const std::string& field_trials) + : VideoEncoderSoftwareFallbackWrapperTestBase(field_trials, + VP8Encoder::Create()) {} + + ~ForcedFallbackTest() override {} + + protected: + void SetUp() override { + clock_.SetTime(Timestamp::Micros(1234)); + ConfigureVp8Codec(); + } + + void TearDown() override { + if (wrapper_initialized_) { + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_->Release()); + } + } + + void ConfigureVp8Codec() { + codec_.codecType = kVideoCodecVP8; + codec_.maxFramerate = kFramerate; + codec_.width = kWidth; + codec_.height = kHeight; + codec_.VP8()->numberOfTemporalLayers = 1; + codec_.VP8()->automaticResizeOn = true; + codec_.SetFrameDropEnabled(true); + rate_allocator_.reset(new SimulcastRateAllocator(codec_)); + } + + void InitEncode(int width, int height) { + codec_.width = width; + codec_.height = height; + if (wrapper_initialized_) { + fallback_wrapper_->Release(); + } + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_->InitEncode(&codec_, kSettings)); + fallback_wrapper_->RegisterEncodeCompleteCallback(&callback_); + wrapper_initialized_ = true; + SetRateAllocation(kBitrateKbps); + } + + void SetRateAllocation(uint32_t bitrate_kbps) { + fallback_wrapper_->SetRates(VideoEncoder::RateControlParameters( + rate_allocator_->Allocate( + VideoBitrateAllocationParameters(bitrate_kbps * 1000, kFramerate)), + kFramerate)); + } + + void EncodeFrameAndVerifyLastName(const char* expected_name) { + EncodeFrameAndVerifyLastName(expected_name, WEBRTC_VIDEO_CODEC_OK); + } + + void EncodeFrameAndVerifyLastName(const char* expected_name, + int expected_ret) { + EncodeFrame(expected_ret); + CheckLastEncoderName(expected_name); + } + + rtc::ScopedFakeClock clock_; +}; + +class ForcedFallbackTestEnabled : public ForcedFallbackTest { + public: + ForcedFallbackTestEnabled() + : ForcedFallbackTest(std::string(kFieldTrial) + "/Enabled-" + + std::to_string(kMinPixelsPerFrame) + "," + + std::to_string(kWidth * kHeight) + ",30000/") {} +}; + +class ForcedFallbackTestDisabled : public ForcedFallbackTest { + public: + ForcedFallbackTestDisabled() + : ForcedFallbackTest(std::string(kFieldTrial) + "/Disabled/") {} +}; + +TEST_F(ForcedFallbackTestDisabled, NoFallbackWithoutFieldTrial) { + // Resolution above max threshold. + InitEncode(kWidth + 1, kHeight); + EXPECT_EQ(1, fake_encoder_->init_encode_count_); + EncodeFrameAndVerifyLastName("fake-encoder"); + + // Resolution at max threshold. + InitEncode(kWidth, kHeight); + EncodeFrameAndVerifyLastName("fake-encoder"); +} + +TEST_F(ForcedFallbackTestEnabled, FallbackIfAtMaxResolutionLimit) { + // Resolution above max threshold. + InitEncode(kWidth + 1, kHeight); + EXPECT_EQ(1, fake_encoder_->init_encode_count_); + EncodeFrameAndVerifyLastName("fake-encoder"); + + // Resolution at max threshold. + InitEncode(kWidth, kHeight); + EncodeFrameAndVerifyLastName("libvpx"); +} + +TEST_F(ForcedFallbackTestEnabled, FallbackIsKeptWhenInitEncodeIsCalled) { + // Resolution above max threshold. + InitEncode(kWidth + 1, kHeight); + EXPECT_EQ(1, fake_encoder_->init_encode_count_); + EncodeFrameAndVerifyLastName("fake-encoder"); + + // Resolution at max threshold. + InitEncode(kWidth, kHeight); + EncodeFrameAndVerifyLastName("libvpx"); + + // Re-initialize encoder, still expect fallback. + InitEncode(kWidth / 2, kHeight / 2); + EXPECT_EQ(1, fake_encoder_->init_encode_count_); // No change. + EncodeFrameAndVerifyLastName("libvpx"); +} + +TEST_F(ForcedFallbackTestEnabled, FallbackIsEndedWhenResolutionIsTooLarge) { + // Resolution above max threshold. + InitEncode(kWidth + 1, kHeight); + EXPECT_EQ(1, fake_encoder_->init_encode_count_); + EncodeFrameAndVerifyLastName("fake-encoder"); + + // Resolution at max threshold. + InitEncode(kWidth, kHeight); + EncodeFrameAndVerifyLastName("libvpx"); + + // Re-initialize encoder with a larger resolution, expect no fallback. + InitEncode(kWidth + 1, kHeight); + EXPECT_EQ(2, fake_encoder_->init_encode_count_); + EncodeFrameAndVerifyLastName("fake-encoder"); +} + +TEST_F(ForcedFallbackTestEnabled, FallbackIsEndedForNonValidSettings) { + // Resolution at max threshold. + InitEncode(kWidth, kHeight); + EncodeFrameAndVerifyLastName("libvpx"); + + // Re-initialize encoder with invalid setting, expect no fallback. + codec_.numberOfSimulcastStreams = 2; + InitEncode(kWidth, kHeight); + EXPECT_EQ(1, fake_encoder_->init_encode_count_); + EncodeFrameAndVerifyLastName("fake-encoder"); + + // Re-initialize encoder with valid setting. + codec_.numberOfSimulcastStreams = 1; + InitEncode(kWidth, kHeight); + EXPECT_EQ(1, fake_encoder_->init_encode_count_); + EncodeFrameAndVerifyLastName("libvpx"); +} + +TEST_F(ForcedFallbackTestEnabled, MultipleStartEndFallback) { + const int kNumRuns = 5; + for (int i = 1; i <= kNumRuns; ++i) { + // Resolution at max threshold. + InitEncode(kWidth, kHeight); + EncodeFrameAndVerifyLastName("libvpx"); + // Resolution above max threshold. + InitEncode(kWidth + 1, kHeight); + EXPECT_EQ(i, fake_encoder_->init_encode_count_); + EncodeFrameAndVerifyLastName("fake-encoder"); + } +} + +TEST_F(ForcedFallbackTestDisabled, GetScaleSettings) { + // Resolution above max threshold. + InitEncode(kWidth + 1, kHeight); + EXPECT_EQ(1, fake_encoder_->init_encode_count_); + EncodeFrameAndVerifyLastName("fake-encoder"); + + // Default min pixels per frame should be used. + const auto settings = fallback_wrapper_->GetEncoderInfo().scaling_settings; + EXPECT_TRUE(settings.thresholds.has_value()); + EXPECT_EQ(kDefaultMinPixelsPerFrame, settings.min_pixels_per_frame); +} + +TEST_F(ForcedFallbackTestEnabled, GetScaleSettingsWithNoFallback) { + // Resolution above max threshold. + InitEncode(kWidth + 1, kHeight); + EncodeFrameAndVerifyLastName("fake-encoder"); + + // Configured min pixels per frame should be used. + const auto settings = fallback_wrapper_->GetEncoderInfo().scaling_settings; + EXPECT_EQ(kMinPixelsPerFrame, settings.min_pixels_per_frame); + ASSERT_TRUE(settings.thresholds); + EXPECT_EQ(kLowThreshold, settings.thresholds->low); + EXPECT_EQ(kHighThreshold, settings.thresholds->high); +} + +TEST_F(ForcedFallbackTestEnabled, GetScaleSettingsWithFallback) { + // Resolution at max threshold. + InitEncode(kWidth, kHeight); + EncodeFrameAndVerifyLastName("libvpx"); + + // Configured min pixels per frame should be used. + const auto settings = fallback_wrapper_->GetEncoderInfo().scaling_settings; + EXPECT_TRUE(settings.thresholds.has_value()); + EXPECT_EQ(kMinPixelsPerFrame, settings.min_pixels_per_frame); +} + +TEST_F(ForcedFallbackTestEnabled, ScalingDisabledIfResizeOff) { + // Resolution at max threshold. + codec_.VP8()->automaticResizeOn = false; + InitEncode(kWidth, kHeight); + EncodeFrameAndVerifyLastName("libvpx"); + + // Should be disabled for automatic resize off. + const auto settings = fallback_wrapper_->GetEncoderInfo().scaling_settings; + EXPECT_FALSE(settings.thresholds.has_value()); +} + +TEST(SoftwareFallbackEncoderTest, BothRateControllersNotTrusted) { + auto* sw_encoder = new ::testing::NiceMock<MockVideoEncoder>(); + auto* hw_encoder = new ::testing::NiceMock<MockVideoEncoder>(); + + EXPECT_CALL(*sw_encoder, GetEncoderInfo()) + .WillRepeatedly(Return(GetEncoderInfoWithTrustedRateController(false))); + EXPECT_CALL(*hw_encoder, GetEncoderInfo()) + .WillRepeatedly(Return(GetEncoderInfoWithTrustedRateController(false))); + + std::unique_ptr<VideoEncoder> wrapper = + CreateVideoEncoderSoftwareFallbackWrapper( + std::unique_ptr<VideoEncoder>(sw_encoder), + std::unique_ptr<VideoEncoder>(hw_encoder)); + EXPECT_FALSE(wrapper->GetEncoderInfo().has_trusted_rate_controller); +} + +TEST(SoftwareFallbackEncoderTest, SwRateControllerTrusted) { + auto* sw_encoder = new ::testing::NiceMock<MockVideoEncoder>(); + auto* hw_encoder = new ::testing::NiceMock<MockVideoEncoder>(); + EXPECT_CALL(*sw_encoder, GetEncoderInfo()) + .WillRepeatedly(Return(GetEncoderInfoWithTrustedRateController(true))); + EXPECT_CALL(*hw_encoder, GetEncoderInfo()) + .WillRepeatedly(Return(GetEncoderInfoWithTrustedRateController(false))); + + std::unique_ptr<VideoEncoder> wrapper = + CreateVideoEncoderSoftwareFallbackWrapper( + std::unique_ptr<VideoEncoder>(sw_encoder), + std::unique_ptr<VideoEncoder>(hw_encoder)); + EXPECT_FALSE(wrapper->GetEncoderInfo().has_trusted_rate_controller); +} + +TEST(SoftwareFallbackEncoderTest, HwRateControllerTrusted) { + auto* sw_encoder = new ::testing::NiceMock<MockVideoEncoder>(); + auto* hw_encoder = new ::testing::NiceMock<MockVideoEncoder>(); + EXPECT_CALL(*sw_encoder, GetEncoderInfo()) + .WillRepeatedly(Return(GetEncoderInfoWithTrustedRateController(false))); + EXPECT_CALL(*hw_encoder, GetEncoderInfo()) + .WillRepeatedly(Return(GetEncoderInfoWithTrustedRateController(true))); + + std::unique_ptr<VideoEncoder> wrapper = + CreateVideoEncoderSoftwareFallbackWrapper( + std::unique_ptr<VideoEncoder>(sw_encoder), + std::unique_ptr<VideoEncoder>(hw_encoder)); + EXPECT_TRUE(wrapper->GetEncoderInfo().has_trusted_rate_controller); + + VideoCodec codec_ = {}; + codec_.width = 100; + codec_.height = 100; + wrapper->InitEncode(&codec_, kSettings); + + // Trigger fallback to software. + EXPECT_CALL(*hw_encoder, Encode) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE)); + VideoFrame frame = VideoFrame::Builder() + .set_video_frame_buffer(I420Buffer::Create(100, 100)) + .build(); + wrapper->Encode(frame, nullptr); + + EXPECT_FALSE(wrapper->GetEncoderInfo().has_trusted_rate_controller); +} + +TEST(SoftwareFallbackEncoderTest, BothRateControllersTrusted) { + auto* sw_encoder = new ::testing::NiceMock<MockVideoEncoder>(); + auto* hw_encoder = new ::testing::NiceMock<MockVideoEncoder>(); + EXPECT_CALL(*sw_encoder, GetEncoderInfo()) + .WillRepeatedly(Return(GetEncoderInfoWithTrustedRateController(true))); + EXPECT_CALL(*hw_encoder, GetEncoderInfo()) + .WillRepeatedly(Return(GetEncoderInfoWithTrustedRateController(true))); + + std::unique_ptr<VideoEncoder> wrapper = + CreateVideoEncoderSoftwareFallbackWrapper( + std::unique_ptr<VideoEncoder>(sw_encoder), + std::unique_ptr<VideoEncoder>(hw_encoder)); + EXPECT_TRUE(wrapper->GetEncoderInfo().has_trusted_rate_controller); +} + +TEST(SoftwareFallbackEncoderTest, ReportsHardwareAccelerated) { + auto* sw_encoder = new ::testing::NiceMock<MockVideoEncoder>(); + auto* hw_encoder = new ::testing::NiceMock<MockVideoEncoder>(); + EXPECT_CALL(*sw_encoder, GetEncoderInfo()) + .WillRepeatedly(Return(GetEncoderInfoWithHardwareAccelerated(false))); + EXPECT_CALL(*hw_encoder, GetEncoderInfo()) + .WillRepeatedly(Return(GetEncoderInfoWithHardwareAccelerated(true))); + + std::unique_ptr<VideoEncoder> wrapper = + CreateVideoEncoderSoftwareFallbackWrapper( + std::unique_ptr<VideoEncoder>(sw_encoder), + std::unique_ptr<VideoEncoder>(hw_encoder)); + EXPECT_TRUE(wrapper->GetEncoderInfo().is_hardware_accelerated); + + VideoCodec codec_ = {}; + codec_.width = 100; + codec_.height = 100; + wrapper->InitEncode(&codec_, kSettings); + + // Trigger fallback to software. + EXPECT_CALL(*hw_encoder, Encode) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE)); + VideoFrame frame = VideoFrame::Builder() + .set_video_frame_buffer(I420Buffer::Create(100, 100)) + .build(); + wrapper->Encode(frame, nullptr); + EXPECT_FALSE(wrapper->GetEncoderInfo().is_hardware_accelerated); +} + +class PreferTemporalLayersFallbackTest : public ::testing::Test { + public: + PreferTemporalLayersFallbackTest() {} + void SetUp() override { + sw_ = new ::testing::NiceMock<MockVideoEncoder>(); + sw_info_.implementation_name = "sw"; + EXPECT_CALL(*sw_, GetEncoderInfo).WillRepeatedly([&]() { + return sw_info_; + }); + EXPECT_CALL(*sw_, InitEncode(_, _, _)) + .WillRepeatedly(Return(WEBRTC_VIDEO_CODEC_OK)); + + hw_ = new ::testing::NiceMock<MockVideoEncoder>(); + hw_info_.implementation_name = "hw"; + EXPECT_CALL(*hw_, GetEncoderInfo()).WillRepeatedly([&]() { + return hw_info_; + }); + EXPECT_CALL(*hw_, InitEncode(_, _, _)) + .WillRepeatedly(Return(WEBRTC_VIDEO_CODEC_OK)); + + wrapper_ = CreateVideoEncoderSoftwareFallbackWrapper( + std::unique_ptr<VideoEncoder>(sw_), std::unique_ptr<VideoEncoder>(hw_), + /*prefer_temporal_support=*/true); + + codec_settings.codecType = kVideoCodecVP8; + codec_settings.maxFramerate = kFramerate; + codec_settings.width = kWidth; + codec_settings.height = kHeight; + codec_settings.numberOfSimulcastStreams = 1; + codec_settings.VP8()->numberOfTemporalLayers = 1; + } + + protected: + void SetSupportsLayers(VideoEncoder::EncoderInfo* info, bool tl_enabled) { + int num_layers = 1; + if (tl_enabled) { + num_layers = codec_settings.VP8()->numberOfTemporalLayers; + } + SetNumLayers(info, num_layers); + } + + void SetNumLayers(VideoEncoder::EncoderInfo* info, int num_layers) { + info->fps_allocation[0].clear(); + for (int i = 0; i < num_layers; ++i) { + info->fps_allocation[0].push_back( + VideoEncoder::EncoderInfo::kMaxFramerateFraction >> + (num_layers - i - 1)); + } + } + + VideoCodec codec_settings; + ::testing::NiceMock<MockVideoEncoder>* sw_; + ::testing::NiceMock<MockVideoEncoder>* hw_; + VideoEncoder::EncoderInfo sw_info_; + VideoEncoder::EncoderInfo hw_info_; + std::unique_ptr<VideoEncoder> wrapper_; +}; + +TEST_F(PreferTemporalLayersFallbackTest, UsesMainWhenLayersNotUsed) { + codec_settings.VP8()->numberOfTemporalLayers = 1; + SetSupportsLayers(&hw_info_, true); + SetSupportsLayers(&sw_info_, true); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + wrapper_->InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw"); +} + +TEST_F(PreferTemporalLayersFallbackTest, UsesMainWhenLayersSupported) { + codec_settings.VP8()->numberOfTemporalLayers = 2; + SetSupportsLayers(&hw_info_, true); + SetSupportsLayers(&sw_info_, true); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + wrapper_->InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw"); +} + +TEST_F(PreferTemporalLayersFallbackTest, + UsesFallbackWhenLayersNotSupportedOnMain) { + codec_settings.VP8()->numberOfTemporalLayers = 2; + SetSupportsLayers(&hw_info_, false); + SetSupportsLayers(&sw_info_, true); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + wrapper_->InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "sw"); +} + +TEST_F(PreferTemporalLayersFallbackTest, UsesMainWhenNeitherSupportsTemporal) { + codec_settings.VP8()->numberOfTemporalLayers = 2; + SetSupportsLayers(&hw_info_, false); + SetSupportsLayers(&sw_info_, false); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + wrapper_->InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw"); +} + +TEST_F(PreferTemporalLayersFallbackTest, UsesFallbackWhenLayersAreUndefined) { + codec_settings.VP8()->numberOfTemporalLayers = 2; + SetNumLayers(&hw_info_, 1); + SetNumLayers(&sw_info_, 0); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + wrapper_->InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "sw"); +} + +TEST_F(PreferTemporalLayersFallbackTest, PrimesEncoderOnSwitch) { + codec_settings.VP8()->numberOfTemporalLayers = 2; + // Both support temporal layers, will use main one. + SetSupportsLayers(&hw_info_, true); + SetSupportsLayers(&sw_info_, true); + + // On first InitEncode most params have no state and will not be + // called to update. + EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback).Times(0); + EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback).Times(0); + + EXPECT_CALL(*hw_, SetFecControllerOverride).Times(0); + EXPECT_CALL(*sw_, SetFecControllerOverride).Times(0); + + EXPECT_CALL(*hw_, SetRates).Times(0); + EXPECT_CALL(*hw_, SetRates).Times(0); + + EXPECT_CALL(*hw_, OnPacketLossRateUpdate).Times(0); + EXPECT_CALL(*sw_, OnPacketLossRateUpdate).Times(0); + + EXPECT_CALL(*hw_, OnRttUpdate).Times(0); + EXPECT_CALL(*sw_, OnRttUpdate).Times(0); + + EXPECT_CALL(*hw_, OnLossNotification).Times(0); + EXPECT_CALL(*sw_, OnLossNotification).Times(0); + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + wrapper_->InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw"); + + FakeEncodedImageCallback callback1; + class DummyFecControllerOverride : public FecControllerOverride { + public: + void SetFecAllowed(bool fec_allowed) override {} + }; + DummyFecControllerOverride fec_controller_override1; + VideoEncoder::RateControlParameters rate_params1; + float packet_loss1 = 0.1; + int64_t rtt1 = 1; + VideoEncoder::LossNotification lntf1; + + EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback(&callback1)); + EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback).Times(0); + wrapper_->RegisterEncodeCompleteCallback(&callback1); + + EXPECT_CALL(*hw_, SetFecControllerOverride(&fec_controller_override1)); + EXPECT_CALL(*sw_, SetFecControllerOverride).Times(1); + wrapper_->SetFecControllerOverride(&fec_controller_override1); + + EXPECT_CALL(*hw_, SetRates(rate_params1)); + EXPECT_CALL(*sw_, SetRates).Times(0); + wrapper_->SetRates(rate_params1); + + EXPECT_CALL(*hw_, OnPacketLossRateUpdate(packet_loss1)); + EXPECT_CALL(*sw_, OnPacketLossRateUpdate).Times(0); + wrapper_->OnPacketLossRateUpdate(packet_loss1); + + EXPECT_CALL(*hw_, OnRttUpdate(rtt1)); + EXPECT_CALL(*sw_, OnRttUpdate).Times(0); + wrapper_->OnRttUpdate(rtt1); + + EXPECT_CALL(*hw_, OnLossNotification).Times(1); + EXPECT_CALL(*sw_, OnLossNotification).Times(0); + wrapper_->OnLossNotification(lntf1); + + // Release and re-init, with fallback to software. This should trigger + // the software encoder to be primed with the current state. + wrapper_->Release(); + EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback(&callback1)); + EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback).Times(0); + + EXPECT_CALL(*sw_, SetFecControllerOverride).Times(0); + EXPECT_CALL(*hw_, SetFecControllerOverride).Times(0); + + // Rate control parameters are cleared on InitEncode. + EXPECT_CALL(*sw_, SetRates).Times(0); + EXPECT_CALL(*hw_, SetRates).Times(0); + + EXPECT_CALL(*sw_, OnPacketLossRateUpdate(packet_loss1)); + EXPECT_CALL(*hw_, OnPacketLossRateUpdate).Times(0); + + EXPECT_CALL(*sw_, OnRttUpdate(rtt1)); + EXPECT_CALL(*hw_, OnRttUpdate).Times(0); + + EXPECT_CALL(*sw_, OnLossNotification).Times(1); + EXPECT_CALL(*hw_, OnLossNotification).Times(0); + + SetSupportsLayers(&hw_info_, false); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + wrapper_->InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "sw"); + + // Update with all-new params for the software encoder. + FakeEncodedImageCallback callback2; + DummyFecControllerOverride fec_controller_override2; + VideoEncoder::RateControlParameters rate_params2; + float packet_loss2 = 0.2; + int64_t rtt2 = 2; + VideoEncoder::LossNotification lntf2; + + EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback(&callback2)); + EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback).Times(0); + wrapper_->RegisterEncodeCompleteCallback(&callback2); + + EXPECT_CALL(*sw_, SetFecControllerOverride(&fec_controller_override2)); + EXPECT_CALL(*hw_, SetFecControllerOverride).Times(1); + wrapper_->SetFecControllerOverride(&fec_controller_override2); + + EXPECT_CALL(*sw_, SetRates(rate_params2)); + EXPECT_CALL(*hw_, SetRates).Times(0); + wrapper_->SetRates(rate_params2); + + EXPECT_CALL(*sw_, OnPacketLossRateUpdate(packet_loss2)); + EXPECT_CALL(*hw_, OnPacketLossRateUpdate).Times(0); + wrapper_->OnPacketLossRateUpdate(packet_loss2); + + EXPECT_CALL(*sw_, OnRttUpdate(rtt2)); + EXPECT_CALL(*hw_, OnRttUpdate).Times(0); + wrapper_->OnRttUpdate(rtt2); + + EXPECT_CALL(*sw_, OnLossNotification).Times(1); + EXPECT_CALL(*hw_, OnLossNotification).Times(0); + wrapper_->OnLossNotification(lntf2); + + // Release and re-init, back to main encoder. This should trigger + // the main encoder to be primed with the current state. + wrapper_->Release(); + EXPECT_CALL(*hw_, RegisterEncodeCompleteCallback(&callback2)); + EXPECT_CALL(*sw_, RegisterEncodeCompleteCallback).Times(0); + + EXPECT_CALL(*hw_, SetFecControllerOverride).Times(0); + EXPECT_CALL(*sw_, SetFecControllerOverride).Times(0); + + // Rate control parameters are cleared on InitEncode. + EXPECT_CALL(*sw_, SetRates).Times(0); + EXPECT_CALL(*hw_, SetRates).Times(0); + + EXPECT_CALL(*hw_, OnPacketLossRateUpdate(packet_loss2)); + EXPECT_CALL(*sw_, OnPacketLossRateUpdate).Times(0); + + EXPECT_CALL(*hw_, OnRttUpdate(rtt2)); + EXPECT_CALL(*sw_, OnRttUpdate).Times(0); + + EXPECT_CALL(*hw_, OnLossNotification).Times(1); + EXPECT_CALL(*sw_, OnLossNotification).Times(0); + + SetSupportsLayers(&hw_info_, true); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + wrapper_->InitEncode(&codec_settings, kSettings)); + EXPECT_EQ(wrapper_->GetEncoderInfo().implementation_name, "hw"); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/video_codec.cc b/third_party/libwebrtc/api/video_codecs/video_codec.cc new file mode 100644 index 0000000000..c6122d3f6a --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_codec.cc @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/video_codec.h" + +#include <string.h> + +#include <string> + +#include "absl/strings/match.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { +constexpr char kPayloadNameVp8[] = "VP8"; +constexpr char kPayloadNameVp9[] = "VP9"; +constexpr char kPayloadNameAv1[] = "AV1"; +// TODO(bugs.webrtc.org/13166): Remove AV1X when backwards compatibility is not +// needed. +constexpr char kPayloadNameAv1x[] = "AV1X"; +constexpr char kPayloadNameH264[] = "H264"; +constexpr char kPayloadNameGeneric[] = "Generic"; +constexpr char kPayloadNameMultiplex[] = "Multiplex"; +} // namespace + +bool VideoCodecVP8::operator==(const VideoCodecVP8& other) const { + return (numberOfTemporalLayers == other.numberOfTemporalLayers && + denoisingOn == other.denoisingOn && + automaticResizeOn == other.automaticResizeOn && + keyFrameInterval == other.keyFrameInterval); +} + +bool VideoCodecVP9::operator==(const VideoCodecVP9& other) const { + return (numberOfTemporalLayers == other.numberOfTemporalLayers && + denoisingOn == other.denoisingOn && + keyFrameInterval == other.keyFrameInterval && + adaptiveQpMode == other.adaptiveQpMode && + automaticResizeOn == other.automaticResizeOn && + numberOfSpatialLayers == other.numberOfSpatialLayers && + flexibleMode == other.flexibleMode); +} + +bool VideoCodecH264::operator==(const VideoCodecH264& other) const { + return (keyFrameInterval == other.keyFrameInterval && + numberOfTemporalLayers == other.numberOfTemporalLayers); +} + +VideoCodec::VideoCodec() + : codecType(kVideoCodecGeneric), + width(0), + height(0), + startBitrate(0), + maxBitrate(0), + minBitrate(0), + maxFramerate(0), + active(true), + qpMax(0), + numberOfSimulcastStreams(0), + simulcastStream(), + spatialLayers(), + mode(VideoCodecMode::kRealtimeVideo), + expect_encode_from_texture(false), + timing_frame_thresholds({0, 0}), + legacy_conference_mode(false), + codec_specific_(), + complexity_(VideoCodecComplexity::kComplexityNormal) {} + +VideoCodecVP8* VideoCodec::VP8() { + RTC_DCHECK_EQ(codecType, kVideoCodecVP8); + return &codec_specific_.VP8; +} + +const VideoCodecVP8& VideoCodec::VP8() const { + RTC_DCHECK_EQ(codecType, kVideoCodecVP8); + return codec_specific_.VP8; +} + +VideoCodecVP9* VideoCodec::VP9() { + RTC_DCHECK_EQ(codecType, kVideoCodecVP9); + return &codec_specific_.VP9; +} + +const VideoCodecVP9& VideoCodec::VP9() const { + RTC_DCHECK_EQ(codecType, kVideoCodecVP9); + return codec_specific_.VP9; +} + +VideoCodecH264* VideoCodec::H264() { + RTC_DCHECK_EQ(codecType, kVideoCodecH264); + return &codec_specific_.H264; +} + +const VideoCodecH264& VideoCodec::H264() const { + RTC_DCHECK_EQ(codecType, kVideoCodecH264); + return codec_specific_.H264; +} + +const char* CodecTypeToPayloadString(VideoCodecType type) { + switch (type) { + case kVideoCodecVP8: + return kPayloadNameVp8; + case kVideoCodecVP9: + return kPayloadNameVp9; + case kVideoCodecAV1: + return kPayloadNameAv1; + case kVideoCodecH264: + return kPayloadNameH264; + case kVideoCodecMultiplex: + return kPayloadNameMultiplex; + case kVideoCodecGeneric: + return kPayloadNameGeneric; + } + RTC_CHECK_NOTREACHED(); +} + +VideoCodecType PayloadStringToCodecType(const std::string& name) { + if (absl::EqualsIgnoreCase(name, kPayloadNameVp8)) + return kVideoCodecVP8; + if (absl::EqualsIgnoreCase(name, kPayloadNameVp9)) + return kVideoCodecVP9; + if (absl::EqualsIgnoreCase(name, kPayloadNameAv1) || + absl::EqualsIgnoreCase(name, kPayloadNameAv1x)) + return kVideoCodecAV1; + if (absl::EqualsIgnoreCase(name, kPayloadNameH264)) + return kVideoCodecH264; + if (absl::EqualsIgnoreCase(name, kPayloadNameMultiplex)) + return kVideoCodecMultiplex; + return kVideoCodecGeneric; +} + +VideoCodecComplexity VideoCodec::GetVideoEncoderComplexity() const { + return complexity_; +} + +void VideoCodec::SetVideoEncoderComplexity( + VideoCodecComplexity complexity_setting) { + complexity_ = complexity_setting; +} + +bool VideoCodec::GetFrameDropEnabled() const { + return frame_drop_enabled_; +} + +void VideoCodec::SetFrameDropEnabled(bool enabled) { + frame_drop_enabled_ = enabled; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/video_codec.h b/third_party/libwebrtc/api/video_codecs/video_codec.h new file mode 100644 index 0000000000..10bceda0d2 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_codec.h @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VIDEO_CODEC_H_ +#define API_VIDEO_CODECS_VIDEO_CODEC_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <string> + +#include "absl/strings/string_view.h" +#include "api/video/video_bitrate_allocation.h" +#include "api/video/video_codec_type.h" +#include "api/video_codecs/scalability_mode.h" +#include "api/video_codecs/simulcast_stream.h" +#include "api/video_codecs/spatial_layer.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// The VideoCodec class represents an old defacto-apis, which we're migrating +// away from slowly. + +// Video codec +enum class VideoCodecComplexity { + kComplexityLow = -1, + kComplexityNormal = 0, + kComplexityHigh = 1, + kComplexityHigher = 2, + kComplexityMax = 3 +}; + +// VP8 specific +struct VideoCodecVP8 { + bool operator==(const VideoCodecVP8& other) const; + bool operator!=(const VideoCodecVP8& other) const { + return !(*this == other); + } + // Temporary utility method for transition deleting numberOfTemporalLayers + // setting (replaced by ScalabilityMode). + void SetNumberOfTemporalLayers(unsigned char n) { + numberOfTemporalLayers = n; + } + unsigned char numberOfTemporalLayers; + bool denoisingOn; + bool automaticResizeOn; + int keyFrameInterval; +}; + +enum class InterLayerPredMode : int { + kOff = 0, // Inter-layer prediction is disabled. + kOn = 1, // Inter-layer prediction is enabled. + kOnKeyPic = 2 // Inter-layer prediction is enabled but limited to key frames. +}; + +// VP9 specific. +struct VideoCodecVP9 { + bool operator==(const VideoCodecVP9& other) const; + bool operator!=(const VideoCodecVP9& other) const { + return !(*this == other); + } + // Temporary utility method for transition deleting numberOfTemporalLayers + // setting (replaced by ScalabilityMode). + void SetNumberOfTemporalLayers(unsigned char n) { + numberOfTemporalLayers = n; + } + unsigned char numberOfTemporalLayers; + bool denoisingOn; + int keyFrameInterval; + bool adaptiveQpMode; + bool automaticResizeOn; + unsigned char numberOfSpatialLayers; + bool flexibleMode; + InterLayerPredMode interLayerPred; +}; + +// H264 specific. +struct VideoCodecH264 { + bool operator==(const VideoCodecH264& other) const; + bool operator!=(const VideoCodecH264& other) const { + return !(*this == other); + } + // Temporary utility method for transition deleting numberOfTemporalLayers + // setting (replaced by ScalabilityMode). + void SetNumberOfTemporalLayers(unsigned char n) { + numberOfTemporalLayers = n; + } + int keyFrameInterval; + uint8_t numberOfTemporalLayers; +}; + +// Translates from name of codec to codec type and vice versa. +RTC_EXPORT const char* CodecTypeToPayloadString(VideoCodecType type); +RTC_EXPORT VideoCodecType PayloadStringToCodecType(const std::string& name); + +union VideoCodecUnion { + VideoCodecVP8 VP8; + VideoCodecVP9 VP9; + VideoCodecH264 H264; +}; + +enum class VideoCodecMode { kRealtimeVideo, kScreensharing }; + +// Common video codec properties +class RTC_EXPORT VideoCodec { + public: + VideoCodec(); + + // Scalability mode as described in + // https://www.w3.org/TR/webrtc-svc/#scalabilitymodes* + absl::optional<ScalabilityMode> GetScalabilityMode() const { + return scalability_mode_; + } + void SetScalabilityMode(ScalabilityMode scalability_mode) { + scalability_mode_ = scalability_mode; + } + void UnsetScalabilityMode() { scalability_mode_ = absl::nullopt; } + + VideoCodecComplexity GetVideoEncoderComplexity() const; + void SetVideoEncoderComplexity(VideoCodecComplexity complexity_setting); + + bool GetFrameDropEnabled() const; + void SetFrameDropEnabled(bool enabled); + + // Public variables. TODO(hta): Make them private with accessors. + VideoCodecType codecType; + + // TODO(nisse): Change to int, for consistency. + uint16_t width; + uint16_t height; + + unsigned int startBitrate; // kilobits/sec. + unsigned int maxBitrate; // kilobits/sec. + unsigned int minBitrate; // kilobits/sec. + + uint32_t maxFramerate; + + // This enables/disables encoding and sending when there aren't multiple + // simulcast streams,by allocating 0 bitrate if inactive. + bool active; + + unsigned int qpMax; + unsigned char numberOfSimulcastStreams; + SimulcastStream simulcastStream[kMaxSimulcastStreams]; + SpatialLayer spatialLayers[kMaxSpatialLayers]; + + VideoCodecMode mode; + bool expect_encode_from_texture; + + // Timing frames configuration. There is delay of delay_ms between two + // consequent timing frames, excluding outliers. Frame is always made a + // timing frame if it's at least outlier_ratio in percent of "ideal" average + // frame given bitrate and framerate, i.e. if it's bigger than + // |outlier_ratio / 100.0 * bitrate_bps / fps| in bits. This way, timing + // frames will not be sent too often usually. Yet large frames will always + // have timing information for debug purposes because they are more likely to + // cause extra delays. + struct TimingFrameTriggerThresholds { + int64_t delay_ms; + uint16_t outlier_ratio_percent; + } timing_frame_thresholds; + + // Legacy Google conference mode flag for simulcast screenshare + bool legacy_conference_mode; + + bool operator==(const VideoCodec& other) const = delete; + bool operator!=(const VideoCodec& other) const = delete; + + // Accessors for codec specific information. + // There is a const version of each that returns a reference, + // and a non-const version that returns a pointer, in order + // to allow modification of the parameters. + VideoCodecVP8* VP8(); + const VideoCodecVP8& VP8() const; + VideoCodecVP9* VP9(); + const VideoCodecVP9& VP9() const; + VideoCodecH264* H264(); + const VideoCodecH264& H264() const; + + private: + // TODO(hta): Consider replacing the union with a pointer type. + // This will allow removing the VideoCodec* types from this file. + VideoCodecUnion codec_specific_; + absl::optional<ScalabilityMode> scalability_mode_; + // 'complexity_' indicates the CPU capability of the client. It's used to + // determine encoder CPU complexity (e.g., cpu_used for VP8, VP9. and AV1). + VideoCodecComplexity complexity_; + bool frame_drop_enabled_ = false; +}; + +} // namespace webrtc +#endif // API_VIDEO_CODECS_VIDEO_CODEC_H_ diff --git a/third_party/libwebrtc/api/video_codecs/video_codecs_api_gn/moz.build b/third_party/libwebrtc/api/video_codecs/video_codecs_api_gn/moz.build new file mode 100644 index 0000000000..eccdc6cfc7 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_codecs_api_gn/moz.build @@ -0,0 +1,235 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/api/video_codecs/av1_profile.cc", + "/third_party/libwebrtc/api/video_codecs/h264_profile_level_id.cc", + "/third_party/libwebrtc/api/video_codecs/sdp_video_format.cc", + "/third_party/libwebrtc/api/video_codecs/simulcast_stream.cc", + "/third_party/libwebrtc/api/video_codecs/spatial_layer.cc", + "/third_party/libwebrtc/api/video_codecs/video_codec.cc", + "/third_party/libwebrtc/api/video_codecs/video_decoder.cc", + "/third_party/libwebrtc/api/video_codecs/video_encoder.cc", + "/third_party/libwebrtc/api/video_codecs/vp8_frame_config.cc", + "/third_party/libwebrtc/api/video_codecs/vp8_temporal_layers.cc", + "/third_party/libwebrtc/api/video_codecs/vp9_profile.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("video_codecs_api_gn") diff --git a/third_party/libwebrtc/api/video_codecs/video_decoder.cc b/third_party/libwebrtc/api/video_codecs/video_decoder.cc new file mode 100644 index 0000000000..c8f40cee7f --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_decoder.cc @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/video_decoder.h" + +#include "absl/types/optional.h" +#include "api/video/render_resolution.h" +#include "api/video/video_codec_type.h" +#include "rtc_base/checks.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { + +int32_t DecodedImageCallback::Decoded(VideoFrame& decodedImage, + int64_t decode_time_ms) { + // The default implementation ignores custom decode time value. + return Decoded(decodedImage); +} + +void DecodedImageCallback::Decoded(VideoFrame& decodedImage, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp) { + Decoded(decodedImage, decode_time_ms.value_or(-1)); +} + +VideoDecoder::DecoderInfo VideoDecoder::GetDecoderInfo() const { + DecoderInfo info; + info.implementation_name = ImplementationName(); + return info; +} + +const char* VideoDecoder::ImplementationName() const { + return "unknown"; +} + +std::string VideoDecoder::DecoderInfo::ToString() const { + char string_buf[2048]; + rtc::SimpleStringBuilder oss(string_buf); + + oss << "DecoderInfo { " + << "prefers_late_decoding = " + << "implementation_name = '" << implementation_name << "', " + << "is_hardware_accelerated = " + << (is_hardware_accelerated ? "true" : "false") << " }"; + return oss.str(); +} + +bool VideoDecoder::DecoderInfo::operator==(const DecoderInfo& rhs) const { + return is_hardware_accelerated == rhs.is_hardware_accelerated && + implementation_name == rhs.implementation_name; +} + +void VideoDecoder::Settings::set_number_of_cores(int value) { + RTC_DCHECK_GT(value, 0); + number_of_cores_ = value; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/video_decoder.h b/third_party/libwebrtc/api/video_codecs/video_decoder.h new file mode 100644 index 0000000000..aa7ee24307 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_decoder.h @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VIDEO_DECODER_H_ +#define API_VIDEO_CODECS_VIDEO_DECODER_H_ + +#include <memory> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/video/encoded_image.h" +#include "api/video/render_resolution.h" +#include "api/video/video_codec_type.h" +#include "api/video/video_frame.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +class RTC_EXPORT DecodedImageCallback { + public: + virtual ~DecodedImageCallback() {} + + virtual int32_t Decoded(VideoFrame& decodedImage) = 0; + // Provides an alternative interface that allows the decoder to specify the + // decode time excluding waiting time for any previous pending frame to + // return. This is necessary for breaking positive feedback in the delay + // estimation when the decoder has a single output buffer. + virtual int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms); + + // TODO(sakal): Remove other implementations when upstream projects have been + // updated. + virtual void Decoded(VideoFrame& decodedImage, + absl::optional<int32_t> decode_time_ms, + absl::optional<uint8_t> qp); +}; + +class RTC_EXPORT VideoDecoder { + public: + struct DecoderInfo { + // Descriptive name of the decoder implementation. + std::string implementation_name; + + // True if the decoder is backed by hardware acceleration. + bool is_hardware_accelerated = false; + + std::string ToString() const; + bool operator==(const DecoderInfo& rhs) const; + bool operator!=(const DecoderInfo& rhs) const { return !(*this == rhs); } + }; + + class Settings { + public: + Settings() = default; + Settings(const Settings&) = default; + Settings& operator=(const Settings&) = default; + ~Settings() = default; + + // The size of pool which is used to store video frame buffers inside + // decoder. If value isn't present some codec-default value will be used. If + // value is present and decoder doesn't have buffer pool the value will be + // ignored. + absl::optional<int> buffer_pool_size() const; + void set_buffer_pool_size(absl::optional<int> value); + + // When valid, user of the VideoDecoder interface shouldn't `Decode` + // encoded images with render resolution larger than width and height + // specified here. + RenderResolution max_render_resolution() const; + void set_max_render_resolution(RenderResolution value); + + // Maximum number of cpu cores the decoder is allowed to use in parallel. + // Must be positive. + int number_of_cores() const { return number_of_cores_; } + void set_number_of_cores(int value); + + // Codec of encoded images user of the VideoDecoder interface will `Decode`. + VideoCodecType codec_type() const { return codec_type_; } + void set_codec_type(VideoCodecType value) { codec_type_ = value; } + + private: + absl::optional<int> buffer_pool_size_; + RenderResolution max_resolution_; + int number_of_cores_ = 1; + VideoCodecType codec_type_ = kVideoCodecGeneric; + }; + + virtual ~VideoDecoder() = default; + + // Prepares decoder to handle incoming encoded frames. Can be called multiple + // times, in such case only latest `settings` are in effect. + virtual bool Configure(const Settings& settings) = 0; + + virtual int32_t Decode(const EncodedImage& input_image, + bool missing_frames, + int64_t render_time_ms) = 0; + + virtual int32_t RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) = 0; + + virtual int32_t Release() = 0; + + virtual DecoderInfo GetDecoderInfo() const; + + // Deprecated, use GetDecoderInfo().implementation_name instead. + virtual const char* ImplementationName() const; +}; + +inline absl::optional<int> VideoDecoder::Settings::buffer_pool_size() const { + return buffer_pool_size_; +} + +inline void VideoDecoder::Settings::set_buffer_pool_size( + absl::optional<int> value) { + buffer_pool_size_ = value; +} + +inline RenderResolution VideoDecoder::Settings::max_render_resolution() const { + return max_resolution_; +} + +inline void VideoDecoder::Settings::set_max_render_resolution( + RenderResolution value) { + max_resolution_ = value; +} + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VIDEO_DECODER_H_ diff --git a/third_party/libwebrtc/api/video_codecs/video_decoder_factory.h b/third_party/libwebrtc/api/video_codecs/video_decoder_factory.h new file mode 100644 index 0000000000..7e1d2ee883 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_decoder_factory.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VIDEO_DECODER_FACTORY_H_ +#define API_VIDEO_CODECS_VIDEO_DECODER_FACTORY_H_ + +#include <memory> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/video_codecs/sdp_video_format.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +class VideoDecoder; + +// A factory that creates VideoDecoders. +// NOTE: This class is still under development and may change without notice. +class RTC_EXPORT VideoDecoderFactory { + public: + struct CodecSupport { + bool is_supported = false; + bool is_power_efficient = false; + }; + + // Returns a list of supported video formats in order of preference, to use + // for signaling etc. + virtual std::vector<SdpVideoFormat> GetSupportedFormats() const = 0; + + // Query whether the specifed format is supported or not and if it will be + // power efficient, which is currently interpreted as if there is support for + // hardware acceleration. + // The parameter `reference_scaling` is used to query support for prediction + // across spatial layers. An example where support for reference scaling is + // needed is if the video stream is produced with a scalability mode that has + // a dependency between the spatial layers. See + // https://w3c.github.io/webrtc-svc/#scalabilitymodes* for a specification of + // different scalabilty modes. NOTE: QueryCodecSupport is currently an + // experimental feature that is subject to change without notice. + virtual CodecSupport QueryCodecSupport(const SdpVideoFormat& format, + bool reference_scaling) const { + // Default implementation, query for supported formats and check if the + // specified format is supported. Returns false if `reference_scaling` is + // true. + CodecSupport codec_support; + codec_support.is_supported = + !reference_scaling && format.IsCodecInList(GetSupportedFormats()); + return codec_support; + } + + // Creates a VideoDecoder for the specified format. + virtual std::unique_ptr<VideoDecoder> CreateVideoDecoder( + const SdpVideoFormat& format) = 0; + + virtual ~VideoDecoderFactory() {} +}; + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VIDEO_DECODER_FACTORY_H_ diff --git a/third_party/libwebrtc/api/video_codecs/video_decoder_factory_template.h b/third_party/libwebrtc/api/video_codecs/video_decoder_factory_template.h new file mode 100644 index 0000000000..703ae11664 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_decoder_factory_template.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VIDEO_DECODER_FACTORY_TEMPLATE_H_ +#define API_VIDEO_CODECS_VIDEO_DECODER_FACTORY_TEMPLATE_H_ + +#include <memory> +#include <vector> + +#include "absl/algorithm/container.h" +#include "api/array_view.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" + +namespace webrtc { +// The VideoDecoderFactoryTemplate supports decoder implementations given as +// template arguments. +// +// To include a decoder in the factory it requires two static members +// functions to be defined: +// +// // Returns the supported SdpVideoFormats this decoder can decode. +// static std::vector<SdpVideoFormat> SupportedFormats(); +// +// // Creates a decoder instance for the given format. +// static std::unique_ptr<VideoDecoder> +// CreateDecoder(const SdpVideoFormat& format); +// +// Note that the order of the template arguments matter as the factory will +// return the first decoder implementation supporting the given SdpVideoFormat. +template <typename... Ts> +class VideoDecoderFactoryTemplate : public VideoDecoderFactory { + public: + std::vector<SdpVideoFormat> GetSupportedFormats() const override { + return GetSupportedFormatsInternal<Ts...>(); + } + + std::unique_ptr<VideoDecoder> CreateVideoDecoder( + const SdpVideoFormat& format) override { + return CreateVideoDecoderInternal<Ts...>(format); + } + + private: + bool IsFormatInList( + const SdpVideoFormat& format, + rtc::ArrayView<const SdpVideoFormat> supported_formats) const { + return absl::c_any_of( + supported_formats, [&](const SdpVideoFormat& supported_format) { + return supported_format.name == format.name && + supported_format.parameters == format.parameters; + }); + } + + template <typename V, typename... Vs> + std::vector<SdpVideoFormat> GetSupportedFormatsInternal() const { + auto supported_formats = V::SupportedFormats(); + + if constexpr (sizeof...(Vs) > 0) { + // Supported formats may overlap between implementations, so duplicates + // should be filtered out. + for (const auto& other_format : GetSupportedFormatsInternal<Vs...>()) { + if (!IsFormatInList(other_format, supported_formats)) { + supported_formats.push_back(other_format); + } + } + } + + return supported_formats; + } + + template <typename V, typename... Vs> + std::unique_ptr<VideoDecoder> CreateVideoDecoderInternal( + const SdpVideoFormat& format) { + if (IsFormatInList(format, V::SupportedFormats())) { + return V::CreateDecoder(format); + } + + if constexpr (sizeof...(Vs) > 0) { + return CreateVideoDecoderInternal<Vs...>(format); + } + + return nullptr; + } +}; + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VIDEO_DECODER_FACTORY_TEMPLATE_H_ diff --git a/third_party/libwebrtc/api/video_codecs/video_decoder_factory_template_dav1d_adapter.h b/third_party/libwebrtc/api/video_codecs/video_decoder_factory_template_dav1d_adapter.h new file mode 100644 index 0000000000..6d80cadf83 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_decoder_factory_template_dav1d_adapter.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VIDEO_DECODER_FACTORY_TEMPLATE_DAV1D_ADAPTER_H_ +#define API_VIDEO_CODECS_VIDEO_DECODER_FACTORY_TEMPLATE_DAV1D_ADAPTER_H_ + +#include <memory> +#include <vector> + +#include "api/video_codecs/sdp_video_format.h" +#include "modules/video_coding/codecs/av1/dav1d_decoder.h" + +namespace webrtc { +struct Dav1dDecoderTemplateAdapter { + static std::vector<SdpVideoFormat> SupportedFormats() { + return {SdpVideoFormat("AV1")}; + } + + static std::unique_ptr<VideoDecoder> CreateDecoder( + const SdpVideoFormat& format) { + return CreateDav1dDecoder(); + } +}; + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VIDEO_DECODER_FACTORY_TEMPLATE_DAV1D_ADAPTER_H_ diff --git a/third_party/libwebrtc/api/video_codecs/video_decoder_factory_template_libvpx_vp8_adapter.h b/third_party/libwebrtc/api/video_codecs/video_decoder_factory_template_libvpx_vp8_adapter.h new file mode 100644 index 0000000000..0c45a4b622 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_decoder_factory_template_libvpx_vp8_adapter.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VIDEO_DECODER_FACTORY_TEMPLATE_LIBVPX_VP8_ADAPTER_H_ +#define API_VIDEO_CODECS_VIDEO_DECODER_FACTORY_TEMPLATE_LIBVPX_VP8_ADAPTER_H_ + +#include <memory> +#include <vector> + +#include "api/video_codecs/sdp_video_format.h" +#include "modules/video_coding/codecs/vp8/include/vp8.h" + +namespace webrtc { +struct LibvpxVp8DecoderTemplateAdapter { + static std::vector<SdpVideoFormat> SupportedFormats() { + return {SdpVideoFormat("VP8")}; + } + + static std::unique_ptr<VideoDecoder> CreateDecoder( + const SdpVideoFormat& format) { + return VP8Decoder::Create(); + } +}; +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VIDEO_DECODER_FACTORY_TEMPLATE_LIBVPX_VP8_ADAPTER_H_ diff --git a/third_party/libwebrtc/api/video_codecs/video_decoder_factory_template_libvpx_vp9_adapter.h b/third_party/libwebrtc/api/video_codecs/video_decoder_factory_template_libvpx_vp9_adapter.h new file mode 100644 index 0000000000..e0ec0010be --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_decoder_factory_template_libvpx_vp9_adapter.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VIDEO_DECODER_FACTORY_TEMPLATE_LIBVPX_VP9_ADAPTER_H_ +#define API_VIDEO_CODECS_VIDEO_DECODER_FACTORY_TEMPLATE_LIBVPX_VP9_ADAPTER_H_ + +#include <memory> +#include <vector> + +#include "modules/video_coding/codecs/vp9/include/vp9.h" + +namespace webrtc { +struct LibvpxVp9DecoderTemplateAdapter { + static std::vector<SdpVideoFormat> SupportedFormats() { + return SupportedVP9DecoderCodecs(); + } + + static std::unique_ptr<VideoDecoder> CreateDecoder( + const SdpVideoFormat& format) { + return VP9Decoder::Create(); + } +}; +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VIDEO_DECODER_FACTORY_TEMPLATE_LIBVPX_VP9_ADAPTER_H_ diff --git a/third_party/libwebrtc/api/video_codecs/video_decoder_factory_template_open_h264_adapter.h b/third_party/libwebrtc/api/video_codecs/video_decoder_factory_template_open_h264_adapter.h new file mode 100644 index 0000000000..2746bde132 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_decoder_factory_template_open_h264_adapter.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VIDEO_DECODER_FACTORY_TEMPLATE_OPEN_H264_ADAPTER_H_ +#define API_VIDEO_CODECS_VIDEO_DECODER_FACTORY_TEMPLATE_OPEN_H264_ADAPTER_H_ + +#include <memory> +#include <vector> + +#include "modules/video_coding/codecs/h264/include/h264.h" + +namespace webrtc { +// TODO(bugs.webrtc.org/13573): When OpenH264 is no longer a conditional build +// target remove #ifdefs. +struct OpenH264DecoderTemplateAdapter { + static std::vector<SdpVideoFormat> SupportedFormats() { +#if defined(WEBRTC_USE_H264) + + return SupportedH264DecoderCodecs(); +#else + return {}; +#endif + } + + static std::unique_ptr<VideoDecoder> CreateDecoder( + const SdpVideoFormat& format) { +#if defined(WEBRTC_USE_H264) + + return H264Decoder::Create(); +#else + return nullptr; +#endif + } +}; +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VIDEO_DECODER_FACTORY_TEMPLATE_OPEN_H264_ADAPTER_H_ diff --git a/third_party/libwebrtc/api/video_codecs/video_decoder_software_fallback_wrapper.cc b/third_party/libwebrtc/api/video_codecs/video_decoder_software_fallback_wrapper.cc new file mode 100644 index 0000000000..cf6f823b92 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_decoder_software_fallback_wrapper.cc @@ -0,0 +1,284 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/video_decoder_software_fallback_wrapper.h" + +#include <stdint.h> + +#include <memory> +#include <string> +#include <utility> + +#include "api/video/encoded_image.h" +#include "api/video_codecs/video_decoder.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/trace_event.h" +#include "system_wrappers/include/field_trial.h" +#include "system_wrappers/include/metrics.h" + +namespace webrtc { + +namespace { + +constexpr size_t kMaxConsequtiveHwErrors = 4; + +class VideoDecoderSoftwareFallbackWrapper final : public VideoDecoder { + public: + VideoDecoderSoftwareFallbackWrapper( + std::unique_ptr<VideoDecoder> sw_fallback_decoder, + std::unique_ptr<VideoDecoder> hw_decoder); + ~VideoDecoderSoftwareFallbackWrapper() override; + + bool Configure(const Settings& settings) override; + + int32_t Decode(const EncodedImage& input_image, + bool missing_frames, + int64_t render_time_ms) override; + + int32_t RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) override; + + int32_t Release() override; + + DecoderInfo GetDecoderInfo() const override; + const char* ImplementationName() const override; + + private: + bool InitFallbackDecoder(); + void UpdateFallbackDecoderHistograms(); + + bool InitHwDecoder(); + + VideoDecoder& active_decoder() const; + + // Determines if we are trying to use the HW or SW decoder. + enum class DecoderType { + kNone, + kHardware, + kFallback, + } decoder_type_; + std::unique_ptr<VideoDecoder> hw_decoder_; + + Settings decoder_settings_; + const std::unique_ptr<VideoDecoder> fallback_decoder_; + const std::string fallback_implementation_name_; + DecodedImageCallback* callback_; + int32_t hw_decoded_frames_since_last_fallback_; + size_t hw_consequtive_generic_errors_; +}; + +VideoDecoderSoftwareFallbackWrapper::VideoDecoderSoftwareFallbackWrapper( + std::unique_ptr<VideoDecoder> sw_fallback_decoder, + std::unique_ptr<VideoDecoder> hw_decoder) + : decoder_type_(DecoderType::kNone), + hw_decoder_(std::move(hw_decoder)), + fallback_decoder_(std::move(sw_fallback_decoder)), + fallback_implementation_name_( + fallback_decoder_->GetDecoderInfo().implementation_name + + " (fallback from: " + + hw_decoder_->GetDecoderInfo().implementation_name + ")"), + callback_(nullptr), + hw_decoded_frames_since_last_fallback_(0), + hw_consequtive_generic_errors_(0) {} +VideoDecoderSoftwareFallbackWrapper::~VideoDecoderSoftwareFallbackWrapper() = + default; + +bool VideoDecoderSoftwareFallbackWrapper::Configure(const Settings& settings) { + decoder_settings_ = settings; + + if (webrtc::field_trial::IsEnabled("WebRTC-Video-ForcedSwDecoderFallback")) { + RTC_LOG(LS_INFO) << "Forced software decoder fallback enabled."; + RTC_DCHECK(decoder_type_ == DecoderType::kNone); + return InitFallbackDecoder(); + } + if (InitHwDecoder()) { + return true; + } + + RTC_DCHECK(decoder_type_ == DecoderType::kNone); + return InitFallbackDecoder(); +} + +bool VideoDecoderSoftwareFallbackWrapper::InitHwDecoder() { + RTC_DCHECK(decoder_type_ == DecoderType::kNone); + if (!hw_decoder_->Configure(decoder_settings_)) { + return false; + } + + decoder_type_ = DecoderType::kHardware; + if (callback_) + hw_decoder_->RegisterDecodeCompleteCallback(callback_); + return true; +} + +bool VideoDecoderSoftwareFallbackWrapper::InitFallbackDecoder() { + RTC_DCHECK(decoder_type_ == DecoderType::kNone || + decoder_type_ == DecoderType::kHardware); + RTC_LOG(LS_WARNING) << "Decoder falling back to software decoding."; + if (!fallback_decoder_->Configure(decoder_settings_)) { + RTC_LOG(LS_ERROR) << "Failed to initialize software-decoder fallback."; + return false; + } + + UpdateFallbackDecoderHistograms(); + + if (decoder_type_ == DecoderType::kHardware) { + hw_decoder_->Release(); + } + decoder_type_ = DecoderType::kFallback; + + if (callback_) + fallback_decoder_->RegisterDecodeCompleteCallback(callback_); + return true; +} + +void VideoDecoderSoftwareFallbackWrapper::UpdateFallbackDecoderHistograms() { + const std::string kFallbackHistogramsUmaPrefix = + "WebRTC.Video.HardwareDecodedFramesBetweenSoftwareFallbacks."; + // Each histogram needs its own code path for this to work otherwise the + // histogram names will be mixed up by the optimization that takes place. + switch (decoder_settings_.codec_type()) { + case kVideoCodecGeneric: + RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Generic", + hw_decoded_frames_since_last_fallback_); + break; + case kVideoCodecVP8: + RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Vp8", + hw_decoded_frames_since_last_fallback_); + break; + case kVideoCodecVP9: + RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Vp9", + hw_decoded_frames_since_last_fallback_); + break; + case kVideoCodecAV1: + RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Av1", + hw_decoded_frames_since_last_fallback_); + break; + case kVideoCodecH264: + RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "H264", + hw_decoded_frames_since_last_fallback_); + break; + case kVideoCodecMultiplex: + RTC_HISTOGRAM_COUNTS_100000(kFallbackHistogramsUmaPrefix + "Multiplex", + hw_decoded_frames_since_last_fallback_); + break; + } +} + +int32_t VideoDecoderSoftwareFallbackWrapper::Decode( + const EncodedImage& input_image, + bool missing_frames, + int64_t render_time_ms) { + TRACE_EVENT0("webrtc", "VideoDecoderSoftwareFallbackWrapper::Decode"); + switch (decoder_type_) { + case DecoderType::kNone: + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + case DecoderType::kHardware: { + int32_t ret = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; + ret = hw_decoder_->Decode(input_image, missing_frames, render_time_ms); + if (ret != WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE) { + if (ret != WEBRTC_VIDEO_CODEC_ERROR) { + ++hw_decoded_frames_since_last_fallback_; + hw_consequtive_generic_errors_ = 0; + return ret; + } + if (input_image._frameType == VideoFrameType::kVideoFrameKey) { + // Only count errors on key-frames, since generic errors can happen + // with hw decoder due to many arbitrary reasons. + // However, requesting a key-frame is supposed to fix the issue. + ++hw_consequtive_generic_errors_; + } + if (hw_consequtive_generic_errors_ < kMaxConsequtiveHwErrors) { + return ret; + } + } + + // HW decoder returned WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE or + // too many generic errors on key-frames encountered. + if (!InitFallbackDecoder()) { + return ret; + } + + // Fallback decoder initialized, fall-through. + [[fallthrough]]; + } + case DecoderType::kFallback: + return fallback_decoder_->Decode(input_image, missing_frames, + render_time_ms); + default: + RTC_DCHECK_NOTREACHED(); + return WEBRTC_VIDEO_CODEC_ERROR; + } +} + +int32_t VideoDecoderSoftwareFallbackWrapper::RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) { + callback_ = callback; + return active_decoder().RegisterDecodeCompleteCallback(callback); +} + +int32_t VideoDecoderSoftwareFallbackWrapper::Release() { + int32_t status; + switch (decoder_type_) { + case DecoderType::kHardware: + status = hw_decoder_->Release(); + break; + case DecoderType::kFallback: + RTC_LOG(LS_INFO) << "Releasing software fallback decoder."; + status = fallback_decoder_->Release(); + break; + case DecoderType::kNone: + status = WEBRTC_VIDEO_CODEC_OK; + break; + default: + RTC_DCHECK_NOTREACHED(); + status = WEBRTC_VIDEO_CODEC_ERROR; + } + + decoder_type_ = DecoderType::kNone; + return status; +} + +VideoDecoder::DecoderInfo VideoDecoderSoftwareFallbackWrapper::GetDecoderInfo() + const { + DecoderInfo info = active_decoder().GetDecoderInfo(); + if (decoder_type_ == DecoderType::kFallback) { + // Cached "A (fallback from B)" string. + info.implementation_name = fallback_implementation_name_; + } + return info; +} + +const char* VideoDecoderSoftwareFallbackWrapper::ImplementationName() const { + if (decoder_type_ == DecoderType::kFallback) { + // Cached "A (fallback from B)" string. + return fallback_implementation_name_.c_str(); + } else { + return hw_decoder_->ImplementationName(); + } +} + +VideoDecoder& VideoDecoderSoftwareFallbackWrapper::active_decoder() const { + return decoder_type_ == DecoderType::kFallback ? *fallback_decoder_ + : *hw_decoder_; +} + +} // namespace + +std::unique_ptr<VideoDecoder> CreateVideoDecoderSoftwareFallbackWrapper( + std::unique_ptr<VideoDecoder> sw_fallback_decoder, + std::unique_ptr<VideoDecoder> hw_decoder) { + return std::make_unique<VideoDecoderSoftwareFallbackWrapper>( + std::move(sw_fallback_decoder), std::move(hw_decoder)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/video_decoder_software_fallback_wrapper.h b/third_party/libwebrtc/api/video_codecs/video_decoder_software_fallback_wrapper.h new file mode 100644 index 0000000000..3f44e02b26 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_decoder_software_fallback_wrapper.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VIDEO_DECODER_SOFTWARE_FALLBACK_WRAPPER_H_ +#define API_VIDEO_CODECS_VIDEO_DECODER_SOFTWARE_FALLBACK_WRAPPER_H_ + +#include <memory> + +#include "api/video_codecs/video_decoder.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// Used to wrap external VideoDecoders to provide a fallback option on +// software decoding when a hardware decoder fails to decode a stream due to +// hardware restrictions, such as max resolution. +RTC_EXPORT std::unique_ptr<VideoDecoder> +CreateVideoDecoderSoftwareFallbackWrapper( + std::unique_ptr<VideoDecoder> sw_fallback_decoder, + std::unique_ptr<VideoDecoder> hw_decoder); + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VIDEO_DECODER_SOFTWARE_FALLBACK_WRAPPER_H_ diff --git a/third_party/libwebrtc/api/video_codecs/video_encoder.cc b/third_party/libwebrtc/api/video_codecs/video_encoder.cc new file mode 100644 index 0000000000..deb4fdc637 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_encoder.cc @@ -0,0 +1,345 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/video_encoder.h" + +#include <string.h> +#include <algorithm> + +#include "rtc_base/checks.h" +#include "rtc_base/strings/string_builder.h" + +namespace webrtc { + +// TODO(mflodman): Add default complexity for VP9 and VP9. +VideoCodecVP8 VideoEncoder::GetDefaultVp8Settings() { + VideoCodecVP8 vp8_settings; + memset(&vp8_settings, 0, sizeof(vp8_settings)); + + vp8_settings.numberOfTemporalLayers = 1; + vp8_settings.denoisingOn = true; + vp8_settings.automaticResizeOn = false; + vp8_settings.keyFrameInterval = 3000; + + return vp8_settings; +} + +VideoCodecVP9 VideoEncoder::GetDefaultVp9Settings() { + VideoCodecVP9 vp9_settings; + memset(&vp9_settings, 0, sizeof(vp9_settings)); + + vp9_settings.numberOfTemporalLayers = 1; + vp9_settings.denoisingOn = true; + vp9_settings.keyFrameInterval = 3000; + vp9_settings.adaptiveQpMode = true; + vp9_settings.automaticResizeOn = true; + vp9_settings.numberOfSpatialLayers = 1; + vp9_settings.flexibleMode = false; + vp9_settings.interLayerPred = InterLayerPredMode::kOn; + + return vp9_settings; +} + +VideoCodecH264 VideoEncoder::GetDefaultH264Settings() { + VideoCodecH264 h264_settings; + memset(&h264_settings, 0, sizeof(h264_settings)); + + h264_settings.keyFrameInterval = 3000; + h264_settings.numberOfTemporalLayers = 1; + + return h264_settings; +} + +VideoEncoder::ScalingSettings::ScalingSettings() = default; + +VideoEncoder::ScalingSettings::ScalingSettings(KOff) : ScalingSettings() {} + +VideoEncoder::ScalingSettings::ScalingSettings(int low, int high) + : thresholds(QpThresholds(low, high)) {} + +VideoEncoder::ScalingSettings::ScalingSettings(int low, + int high, + int min_pixels) + : thresholds(QpThresholds(low, high)), min_pixels_per_frame(min_pixels) {} + +VideoEncoder::ScalingSettings::ScalingSettings(const ScalingSettings&) = + default; + +VideoEncoder::ScalingSettings::~ScalingSettings() {} + +// static +constexpr VideoEncoder::ScalingSettings::KOff + VideoEncoder::ScalingSettings::kOff; +// static +constexpr uint8_t VideoEncoder::EncoderInfo::kMaxFramerateFraction; + +bool VideoEncoder::ResolutionBitrateLimits::operator==( + const ResolutionBitrateLimits& rhs) const { + return frame_size_pixels == rhs.frame_size_pixels && + min_start_bitrate_bps == rhs.min_start_bitrate_bps && + min_bitrate_bps == rhs.min_bitrate_bps && + max_bitrate_bps == rhs.max_bitrate_bps; +} + +VideoEncoder::EncoderInfo::EncoderInfo() + : scaling_settings(VideoEncoder::ScalingSettings::kOff), + requested_resolution_alignment(1), + apply_alignment_to_all_simulcast_layers(false), + supports_native_handle(false), + implementation_name("unknown"), + has_trusted_rate_controller(false), + is_hardware_accelerated(true), + fps_allocation{absl::InlinedVector<uint8_t, kMaxTemporalStreams>( + 1, + kMaxFramerateFraction)}, + supports_simulcast(false), + preferred_pixel_formats{VideoFrameBuffer::Type::kI420} {} + +VideoEncoder::EncoderInfo::EncoderInfo(const EncoderInfo&) = default; + +VideoEncoder::EncoderInfo::~EncoderInfo() = default; + +std::string VideoEncoder::EncoderInfo::ToString() const { + char string_buf[2048]; + rtc::SimpleStringBuilder oss(string_buf); + + oss << "EncoderInfo { " + "ScalingSettings { "; + if (scaling_settings.thresholds) { + oss << "Thresholds { " + "low = " + << scaling_settings.thresholds->low + << ", high = " << scaling_settings.thresholds->high << "}, "; + } + oss << "min_pixels_per_frame = " << scaling_settings.min_pixels_per_frame + << " }"; + oss << ", requested_resolution_alignment = " << requested_resolution_alignment + << ", apply_alignment_to_all_simulcast_layers = " + << apply_alignment_to_all_simulcast_layers + << ", supports_native_handle = " << supports_native_handle + << ", implementation_name = '" << implementation_name + << "'" + ", has_trusted_rate_controller = " + << has_trusted_rate_controller + << ", is_hardware_accelerated = " << is_hardware_accelerated + << ", fps_allocation = ["; + size_t num_spatial_layer_with_fps_allocation = 0; + for (size_t i = 0; i < kMaxSpatialLayers; ++i) { + if (!fps_allocation[i].empty()) { + num_spatial_layer_with_fps_allocation = i + 1; + } + } + bool first = true; + for (size_t i = 0; i < num_spatial_layer_with_fps_allocation; ++i) { + if (fps_allocation[i].empty()) { + break; + } + if (!first) { + oss << ", "; + } + const absl::InlinedVector<uint8_t, kMaxTemporalStreams>& fractions = + fps_allocation[i]; + if (!fractions.empty()) { + first = false; + oss << "[ "; + for (size_t i = 0; i < fractions.size(); ++i) { + if (i > 0) { + oss << ", "; + } + oss << (static_cast<double>(fractions[i]) / kMaxFramerateFraction); + } + oss << "] "; + } + } + oss << "]"; + oss << ", resolution_bitrate_limits = ["; + for (size_t i = 0; i < resolution_bitrate_limits.size(); ++i) { + if (i > 0) { + oss << ", "; + } + ResolutionBitrateLimits l = resolution_bitrate_limits[i]; + oss << "Limits { " + "frame_size_pixels = " + << l.frame_size_pixels + << ", min_start_bitrate_bps = " << l.min_start_bitrate_bps + << ", min_bitrate_bps = " << l.min_bitrate_bps + << ", max_bitrate_bps = " << l.max_bitrate_bps << "} "; + } + oss << "] " + ", supports_simulcast = " + << supports_simulcast; + oss << ", preferred_pixel_formats = ["; + for (size_t i = 0; i < preferred_pixel_formats.size(); ++i) { + if (i > 0) + oss << ", "; +#if defined(WEBRTC_MOZILLA_BUILD) + // This could assert, as opposed to throw using the form in the + // else, but since we're in a for loop that uses .size() we can + // be fairly sure that this is safe without doing a further + // check to make sure 'i' is in-range. + oss << VideoFrameBufferTypeToString(preferred_pixel_formats[i]); +#else + oss << VideoFrameBufferTypeToString(preferred_pixel_formats.at(i)); +#endif + } + oss << "]"; + if (is_qp_trusted.has_value()) { + oss << ", is_qp_trusted = " << is_qp_trusted.value(); + } + oss << "}"; + return oss.str(); +} + +bool VideoEncoder::EncoderInfo::operator==(const EncoderInfo& rhs) const { + if (scaling_settings.thresholds.has_value() != + rhs.scaling_settings.thresholds.has_value()) { + return false; + } + if (scaling_settings.thresholds.has_value()) { + QpThresholds l = *scaling_settings.thresholds; + QpThresholds r = *rhs.scaling_settings.thresholds; + if (l.low != r.low || l.high != r.high) { + return false; + } + } + if (scaling_settings.min_pixels_per_frame != + rhs.scaling_settings.min_pixels_per_frame) { + return false; + } + + if (supports_native_handle != rhs.supports_native_handle || + implementation_name != rhs.implementation_name || + has_trusted_rate_controller != rhs.has_trusted_rate_controller || + is_hardware_accelerated != rhs.is_hardware_accelerated) { + return false; + } + + for (size_t i = 0; i < kMaxSpatialLayers; ++i) { + if (fps_allocation[i] != rhs.fps_allocation[i]) { + return false; + } + } + + if (resolution_bitrate_limits != rhs.resolution_bitrate_limits || + supports_simulcast != rhs.supports_simulcast) { + return false; + } + + return true; +} + +absl::optional<VideoEncoder::ResolutionBitrateLimits> +VideoEncoder::EncoderInfo::GetEncoderBitrateLimitsForResolution( + int frame_size_pixels) const { + std::vector<ResolutionBitrateLimits> bitrate_limits = + resolution_bitrate_limits; + + // Sort the list of bitrate limits by resolution. + sort(bitrate_limits.begin(), bitrate_limits.end(), + [](const ResolutionBitrateLimits& lhs, + const ResolutionBitrateLimits& rhs) { + return lhs.frame_size_pixels < rhs.frame_size_pixels; + }); + + for (size_t i = 0; i < bitrate_limits.size(); ++i) { + RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps, 0); + RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps, 0); + RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps, + bitrate_limits[i].min_bitrate_bps); + if (i > 0) { + // The bitrate limits aren't expected to decrease with resolution. + RTC_DCHECK_GE(bitrate_limits[i].min_bitrate_bps, + bitrate_limits[i - 1].min_bitrate_bps); + RTC_DCHECK_GE(bitrate_limits[i].min_start_bitrate_bps, + bitrate_limits[i - 1].min_start_bitrate_bps); + RTC_DCHECK_GE(bitrate_limits[i].max_bitrate_bps, + bitrate_limits[i - 1].max_bitrate_bps); + } + + if (bitrate_limits[i].frame_size_pixels >= frame_size_pixels) { + return absl::optional<ResolutionBitrateLimits>(bitrate_limits[i]); + } + } + + return absl::nullopt; +} + +VideoEncoder::RateControlParameters::RateControlParameters() + : bitrate(VideoBitrateAllocation()), + framerate_fps(0.0), + bandwidth_allocation(DataRate::Zero()) {} + +VideoEncoder::RateControlParameters::RateControlParameters( + const VideoBitrateAllocation& bitrate, + double framerate_fps) + : bitrate(bitrate), + framerate_fps(framerate_fps), + bandwidth_allocation(DataRate::BitsPerSec(bitrate.get_sum_bps())) {} + +VideoEncoder::RateControlParameters::RateControlParameters( + const VideoBitrateAllocation& bitrate, + double framerate_fps, + DataRate bandwidth_allocation) + : bitrate(bitrate), + framerate_fps(framerate_fps), + bandwidth_allocation(bandwidth_allocation) {} + +bool VideoEncoder::RateControlParameters::operator==( + const VideoEncoder::RateControlParameters& rhs) const { + return std::tie(bitrate, framerate_fps, bandwidth_allocation) == + std::tie(rhs.bitrate, rhs.framerate_fps, rhs.bandwidth_allocation); +} + +bool VideoEncoder::RateControlParameters::operator!=( + const VideoEncoder::RateControlParameters& rhs) const { + return !(rhs == *this); +} + +VideoEncoder::RateControlParameters::~RateControlParameters() = default; + +void VideoEncoder::SetFecControllerOverride( + FecControllerOverride* fec_controller_override) {} + +int32_t VideoEncoder::InitEncode(const VideoCodec* codec_settings, + int32_t number_of_cores, + size_t max_payload_size) { + const VideoEncoder::Capabilities capabilities(/* loss_notification= */ false); + const VideoEncoder::Settings settings(capabilities, number_of_cores, + max_payload_size); + // In theory, this and the other version of InitEncode() could end up calling + // each other in a loop until we get a stack overflow. + // In practice, any subclass of VideoEncoder would overload at least one + // of these, and we have a TODO in the header file to make this pure virtual. + return InitEncode(codec_settings, settings); +} + +int VideoEncoder::InitEncode(const VideoCodec* codec_settings, + const VideoEncoder::Settings& settings) { + // In theory, this and the other version of InitEncode() could end up calling + // each other in a loop until we get a stack overflow. + // In practice, any subclass of VideoEncoder would overload at least one + // of these, and we have a TODO in the header file to make this pure virtual. + return InitEncode(codec_settings, settings.number_of_cores, + settings.max_payload_size); +} + +void VideoEncoder::OnPacketLossRateUpdate(float packet_loss_rate) {} + +void VideoEncoder::OnRttUpdate(int64_t rtt_ms) {} + +void VideoEncoder::OnLossNotification( + const LossNotification& loss_notification) {} + +// TODO(webrtc:9722): Remove and make pure virtual. +VideoEncoder::EncoderInfo VideoEncoder::GetEncoderInfo() const { + return EncoderInfo(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/video_encoder.h b/third_party/libwebrtc/api/video_codecs/video_encoder.h new file mode 100644 index 0000000000..395a87e089 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_encoder.h @@ -0,0 +1,426 @@ +/* + * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VIDEO_ENCODER_H_ +#define API_VIDEO_CODECS_VIDEO_ENCODER_H_ + +#include <limits> +#include <memory> +#include <string> +#include <vector> + +#include "absl/container/inlined_vector.h" +#include "absl/types/optional.h" +#include "api/fec_controller_override.h" +#include "api/units/data_rate.h" +#include "api/video/encoded_image.h" +#include "api/video/video_bitrate_allocation.h" +#include "api/video/video_codec_constants.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/video_codec.h" +#include "rtc_base/checks.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// TODO(pbos): Expose these through a public (root) header or change these APIs. +struct CodecSpecificInfo; + +constexpr int kDefaultMinPixelsPerFrame = 320 * 180; + +class RTC_EXPORT EncodedImageCallback { + public: + virtual ~EncodedImageCallback() {} + + struct Result { + enum Error { + OK, + + // Failed to send the packet. + ERROR_SEND_FAILED, + }; + + explicit Result(Error error) : error(error) {} + Result(Error error, uint32_t frame_id) : error(error), frame_id(frame_id) {} + + Error error; + + // Frame ID assigned to the frame. The frame ID should be the same as the ID + // seen by the receiver for this frame. RTP timestamp of the frame is used + // as frame ID when RTP is used to send video. Must be used only when + // error=OK. + uint32_t frame_id = 0; + + // Tells the encoder that the next frame is should be dropped. + bool drop_next_frame = false; + }; + + // Used to signal the encoder about reason a frame is dropped. + // kDroppedByMediaOptimizations - dropped by MediaOptimizations (for rate + // limiting purposes). + // kDroppedByEncoder - dropped by encoder's internal rate limiter. + // TODO(bugs.webrtc.org/10164): Delete this enum? It duplicates the more + // general VideoStreamEncoderObserver::DropReason. Also, + // kDroppedByMediaOptimizations is not produced by any encoder, but by + // VideoStreamEncoder. + enum class DropReason : uint8_t { + kDroppedByMediaOptimizations, + kDroppedByEncoder + }; + + // Callback function which is called when an image has been encoded. + virtual Result OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info) = 0; + + virtual void OnDroppedFrame(DropReason reason) {} +}; + +class RTC_EXPORT VideoEncoder { + public: + struct QpThresholds { + QpThresholds(int l, int h) : low(l), high(h) {} + QpThresholds() : low(-1), high(-1) {} + int low; + int high; + }; + + // Quality scaling is enabled if thresholds are provided. + struct RTC_EXPORT ScalingSettings { + private: + // Private magic type for kOff, implicitly convertible to + // ScalingSettings. + struct KOff {}; + + public: + // TODO(bugs.webrtc.org/9078): Since absl::optional should be trivially copy + // constructible, this magic value can likely be replaced by a constexpr + // ScalingSettings value. + static constexpr KOff kOff = {}; + + ScalingSettings(int low, int high); + ScalingSettings(int low, int high, int min_pixels); + ScalingSettings(const ScalingSettings&); + ScalingSettings(KOff); // NOLINT(runtime/explicit) + ~ScalingSettings(); + + absl::optional<QpThresholds> thresholds; + + // We will never ask for a resolution lower than this. + // TODO(kthelgason): Lower this limit when better testing + // on MediaCodec and fallback implementations are in place. + // See https://bugs.chromium.org/p/webrtc/issues/detail?id=7206 + int min_pixels_per_frame = kDefaultMinPixelsPerFrame; + + private: + // Private constructor; to get an object without thresholds, use + // the magic constant ScalingSettings::kOff. + ScalingSettings(); + }; + + // Bitrate limits for resolution. + struct ResolutionBitrateLimits { + ResolutionBitrateLimits(int frame_size_pixels, + int min_start_bitrate_bps, + int min_bitrate_bps, + int max_bitrate_bps) + : frame_size_pixels(frame_size_pixels), + min_start_bitrate_bps(min_start_bitrate_bps), + min_bitrate_bps(min_bitrate_bps), + max_bitrate_bps(max_bitrate_bps) {} + // Size of video frame, in pixels, the bitrate thresholds are intended for. + int frame_size_pixels = 0; + // Recommended minimum bitrate to start encoding. + int min_start_bitrate_bps = 0; + // Recommended minimum bitrate. + int min_bitrate_bps = 0; + // Recommended maximum bitrate. + int max_bitrate_bps = 0; + + bool operator==(const ResolutionBitrateLimits& rhs) const; + bool operator!=(const ResolutionBitrateLimits& rhs) const { + return !(*this == rhs); + } + }; + + // Struct containing metadata about the encoder implementing this interface. + struct RTC_EXPORT EncoderInfo { + static constexpr uint8_t kMaxFramerateFraction = + std::numeric_limits<uint8_t>::max(); + + EncoderInfo(); + EncoderInfo(const EncoderInfo&); + + ~EncoderInfo(); + + std::string ToString() const; + bool operator==(const EncoderInfo& rhs) const; + bool operator!=(const EncoderInfo& rhs) const { return !(*this == rhs); } + + // Any encoder implementation wishing to use the WebRTC provided + // quality scaler must populate this field. + ScalingSettings scaling_settings; + + // The width and height of the incoming video frames should be divisible + // by `requested_resolution_alignment`. If they are not, the encoder may + // drop the incoming frame. + // For example: With I420, this value would be a multiple of 2. + // Note that this field is unrelated to any horizontal or vertical stride + // requirements the encoder has on the incoming video frame buffers. + uint32_t requested_resolution_alignment; + + // Same as above but if true, each simulcast layer should also be divisible + // by `requested_resolution_alignment`. + // Note that scale factors `scale_resolution_down_by` may be adjusted so a + // common multiple is not too large to avoid largely cropped frames and + // possibly with an aspect ratio far from the original. + // Warning: large values of scale_resolution_down_by could be changed + // considerably, especially if `requested_resolution_alignment` is large. + bool apply_alignment_to_all_simulcast_layers; + + // If true, encoder supports working with a native handle (e.g. texture + // handle for hw codecs) rather than requiring a raw I420 buffer. + bool supports_native_handle; + + // The name of this particular encoder implementation, e.g. "libvpx". + std::string implementation_name; + + // If this field is true, the encoder rate controller must perform + // well even in difficult situations, and produce close to the specified + // target bitrate seen over a reasonable time window, drop frames if + // necessary in order to keep the rate correct, and react quickly to + // changing bitrate targets. If this method returns true, we disable the + // frame dropper in the media optimization module and rely entirely on the + // encoder to produce media at a bitrate that closely matches the target. + // Any overshooting may result in delay buildup. If this method returns + // false (default behavior), the media opt frame dropper will drop input + // frames if it suspect encoder misbehavior. Misbehavior is common, + // especially in hardware codecs. Disable media opt at your own risk. + bool has_trusted_rate_controller; + + // If this field is true, the encoder uses hardware support and different + // thresholds will be used in CPU adaptation. + bool is_hardware_accelerated; + + // For each spatial layer (simulcast stream or SVC layer), represented as an + // element in `fps_allocation` a vector indicates how many temporal layers + // the encoder is using for that spatial layer. + // For each spatial/temporal layer pair, the frame rate fraction is given as + // an 8bit unsigned integer where 0 = 0% and 255 = 100%. + // + // If the vector is empty for a given spatial layer, it indicates that frame + // rates are not defined and we can't count on any specific frame rate to be + // generated. Likely this indicates Vp8TemporalLayersType::kBitrateDynamic. + // + // The encoder may update this on a per-frame basis in response to both + // internal and external signals. + // + // Spatial layers are treated independently, but temporal layers are + // cumulative. For instance, if: + // fps_allocation[0][0] = kFullFramerate / 2; + // fps_allocation[0][1] = kFullFramerate; + // Then half of the frames are in the base layer and half is in TL1, but + // since TL1 is assumed to depend on the base layer, the frame rate is + // indicated as the full 100% for the top layer. + // + // Defaults to a single spatial layer containing a single temporal layer + // with a 100% frame rate fraction. + absl::InlinedVector<uint8_t, kMaxTemporalStreams> + fps_allocation[kMaxSpatialLayers]; + + // Recommended bitrate limits for different resolutions. + std::vector<ResolutionBitrateLimits> resolution_bitrate_limits; + + // Obtains the limits from `resolution_bitrate_limits` that best matches the + // `frame_size_pixels`. + absl::optional<ResolutionBitrateLimits> + GetEncoderBitrateLimitsForResolution(int frame_size_pixels) const; + + // If true, this encoder has internal support for generating simulcast + // streams. Otherwise, an adapter class will be needed. + // Even if true, the config provided to InitEncode() might not be supported, + // in such case the encoder should return + // WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED. + bool supports_simulcast; + + // The list of pixel formats preferred by the encoder. It is assumed that if + // the list is empty and supports_native_handle is false, then {I420} is the + // preferred pixel format. The order of the formats does not matter. + absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats> + preferred_pixel_formats; + + // Indicates whether or not QP value encoder writes into frame/slice/tile + // header can be interpreted as average frame/slice/tile QP. + absl::optional<bool> is_qp_trusted; + }; + + struct RTC_EXPORT RateControlParameters { + RateControlParameters(); + RateControlParameters(const VideoBitrateAllocation& bitrate, + double framerate_fps); + RateControlParameters(const VideoBitrateAllocation& bitrate, + double framerate_fps, + DataRate bandwidth_allocation); + virtual ~RateControlParameters(); + + // Target bitrate, per spatial/temporal layer. + // A target bitrate of 0bps indicates a layer should not be encoded at all. + VideoBitrateAllocation target_bitrate; + // Adjusted target bitrate, per spatial/temporal layer. May be lower or + // higher than the target depending on encoder behaviour. + VideoBitrateAllocation bitrate; + // Target framerate, in fps. A value <= 0.0 is invalid and should be + // interpreted as framerate target not available. In this case the encoder + // should fall back to the max framerate specified in `codec_settings` of + // the last InitEncode() call. + double framerate_fps; + // The network bandwidth available for video. This is at least + // `bitrate.get_sum_bps()`, but may be higher if the application is not + // network constrained. + DataRate bandwidth_allocation; + + bool operator==(const RateControlParameters& rhs) const; + bool operator!=(const RateControlParameters& rhs) const; + }; + + struct LossNotification { + // The timestamp of the last decodable frame *prior* to the last received. + // (The last received - described below - might itself be decodable or not.) + uint32_t timestamp_of_last_decodable; + // The timestamp of the last received frame. + uint32_t timestamp_of_last_received; + // Describes whether the dependencies of the last received frame were + // all decodable. + // `false` if some dependencies were undecodable, `true` if all dependencies + // were decodable, and `nullopt` if the dependencies are unknown. + absl::optional<bool> dependencies_of_last_received_decodable; + // Describes whether the received frame was decodable. + // `false` if some dependency was undecodable or if some packet belonging + // to the last received frame was missed. + // `true` if all dependencies were decodable and all packets belonging + // to the last received frame were received. + // `nullopt` if no packet belonging to the last frame was missed, but the + // last packet in the frame was not yet received. + absl::optional<bool> last_received_decodable; + }; + + // Negotiated capabilities which the VideoEncoder may expect the other + // side to use. + struct Capabilities { + explicit Capabilities(bool loss_notification) + : loss_notification(loss_notification) {} + bool loss_notification; + }; + + struct Settings { + Settings(const Capabilities& capabilities, + int number_of_cores, + size_t max_payload_size) + : capabilities(capabilities), + number_of_cores(number_of_cores), + max_payload_size(max_payload_size) {} + + Capabilities capabilities; + int number_of_cores; + size_t max_payload_size; + }; + + static VideoCodecVP8 GetDefaultVp8Settings(); + static VideoCodecVP9 GetDefaultVp9Settings(); + static VideoCodecH264 GetDefaultH264Settings(); + + virtual ~VideoEncoder() {} + + // Set a FecControllerOverride, through which the encoder may override + // decisions made by FecController. + // TODO(bugs.webrtc.org/10769): Update downstream, then make pure-virtual. + virtual void SetFecControllerOverride( + FecControllerOverride* fec_controller_override); + + // Initialize the encoder with the information from the codecSettings + // + // Input: + // - codec_settings : Codec settings + // - settings : Settings affecting the encoding itself. + // Input for deprecated version: + // - number_of_cores : Number of cores available for the encoder + // - max_payload_size : The maximum size each payload is allowed + // to have. Usually MTU - overhead. + // + // Return value : Set bit rate if OK + // <0 - Errors: + // WEBRTC_VIDEO_CODEC_ERR_PARAMETER + // WEBRTC_VIDEO_CODEC_ERR_SIZE + // WEBRTC_VIDEO_CODEC_MEMORY + // WEBRTC_VIDEO_CODEC_ERROR + // TODO(bugs.webrtc.org/10720): After updating downstream projects and posting + // an announcement to discuss-webrtc, remove the three-parameters variant + // and make the two-parameters variant pure-virtual. + /* ABSL_DEPRECATED("bugs.webrtc.org/10720") */ virtual int32_t InitEncode( + const VideoCodec* codec_settings, + int32_t number_of_cores, + size_t max_payload_size); + virtual int InitEncode(const VideoCodec* codec_settings, + const VideoEncoder::Settings& settings); + + // Register an encode complete callback object. + // + // Input: + // - callback : Callback object which handles encoded images. + // + // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. + virtual int32_t RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) = 0; + + // Free encoder memory. + // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. + virtual int32_t Release() = 0; + + // Encode an image (as a part of a video stream). The encoded image + // will be returned to the user through the encode complete callback. + // + // Input: + // - frame : Image to be encoded + // - frame_types : Frame type to be generated by the encoder. + // + // Return value : WEBRTC_VIDEO_CODEC_OK if OK + // <0 - Errors: + // WEBRTC_VIDEO_CODEC_ERR_PARAMETER + // WEBRTC_VIDEO_CODEC_MEMORY + // WEBRTC_VIDEO_CODEC_ERROR + virtual int32_t Encode(const VideoFrame& frame, + const std::vector<VideoFrameType>* frame_types) = 0; + + // Sets rate control parameters: bitrate, framerate, etc. These settings are + // instantaneous (i.e. not moving averages) and should apply from now until + // the next call to SetRates(). + virtual void SetRates(const RateControlParameters& parameters) = 0; + + // Inform the encoder when the packet loss rate changes. + // + // Input: - packet_loss_rate : The packet loss rate (0.0 to 1.0). + virtual void OnPacketLossRateUpdate(float packet_loss_rate); + + // Inform the encoder when the round trip time changes. + // + // Input: - rtt_ms : The new RTT, in milliseconds. + virtual void OnRttUpdate(int64_t rtt_ms); + + // Called when a loss notification is received. + virtual void OnLossNotification(const LossNotification& loss_notification); + + // Returns meta-data about the encoder, such as implementation name. + // The output of this method may change during runtime. For instance if a + // hardware encoder fails, it may fall back to doing software encoding using + // an implementation with different characteristics. + virtual EncoderInfo GetEncoderInfo() const; +}; +} // namespace webrtc +#endif // API_VIDEO_CODECS_VIDEO_ENCODER_H_ diff --git a/third_party/libwebrtc/api/video_codecs/video_encoder_factory.h b/third_party/libwebrtc/api/video_codecs/video_encoder_factory.h new file mode 100644 index 0000000000..d28a2a4035 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_encoder_factory.h @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VIDEO_ENCODER_FACTORY_H_ +#define API_VIDEO_CODECS_VIDEO_ENCODER_FACTORY_H_ + +#include <memory> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/units/data_rate.h" +#include "api/video/render_resolution.h" +#include "api/video_codecs/sdp_video_format.h" + +namespace webrtc { + +class VideoEncoder; + +// A factory that creates VideoEncoders. +// NOTE: This class is still under development and may change without notice. +class VideoEncoderFactory { + public: + struct CodecSupport { + bool is_supported = false; + bool is_power_efficient = false; + }; + + // An injectable class that is continuously updated with encoding conditions + // and selects the best encoder given those conditions. An implementation is + // typically stateful to avoid toggling between different encoders, which is + // costly due to recreation of objects, a new codec will always start with a + // key-frame. + class EncoderSelectorInterface { + public: + virtual ~EncoderSelectorInterface() {} + + // Informs the encoder selector about which encoder that is currently being + // used. + virtual void OnCurrentEncoder(const SdpVideoFormat& format) = 0; + + // Called every time the available bitrate is updated. Should return a + // non-empty if an encoder switch should be performed. + virtual absl::optional<SdpVideoFormat> OnAvailableBitrate( + const DataRate& rate) = 0; + + // Called every time the encoder input resolution change. Should return a + // non-empty if an encoder switch should be performed. + virtual absl::optional<SdpVideoFormat> OnResolutionChange( + const RenderResolution& resolution) { + return absl::nullopt; + } + + // Called if the currently used encoder reports itself as broken. Should + // return a non-empty if an encoder switch should be performed. + virtual absl::optional<SdpVideoFormat> OnEncoderBroken() = 0; + }; + + // Returns a list of supported video formats in order of preference, to use + // for signaling etc. + virtual std::vector<SdpVideoFormat> GetSupportedFormats() const = 0; + + // Returns a list of supported video formats in order of preference, that can + // also be tagged with additional information to allow the VideoEncoderFactory + // to separate between different implementations when CreateVideoEncoder is + // called. + virtual std::vector<SdpVideoFormat> GetImplementations() const { + return GetSupportedFormats(); + } + + // Query whether the specifed format is supported or not and if it will be + // power efficient, which is currently interpreted as if there is support for + // hardware acceleration. + // See https://w3c.github.io/webrtc-svc/#scalabilitymodes* for a specification + // of valid values for `scalability_mode`. + // NOTE: QueryCodecSupport is currently an experimental feature that is + // subject to change without notice. + virtual CodecSupport QueryCodecSupport( + const SdpVideoFormat& format, + absl::optional<std::string> scalability_mode) const { + // Default implementation, query for supported formats and check if the + // specified format is supported. Returns false if scalability_mode is + // specified. + CodecSupport codec_support; + if (!scalability_mode) { + codec_support.is_supported = format.IsCodecInList(GetSupportedFormats()); + } + return codec_support; + } + + // Creates a VideoEncoder for the specified format. + virtual std::unique_ptr<VideoEncoder> CreateVideoEncoder( + const SdpVideoFormat& format) = 0; + + // This method creates a EncoderSelector to use for a VideoSendStream. + // (and hence should probably been called CreateEncoderSelector()). + // + // Note: This method is unsuitable if encoding several streams that + // are using same VideoEncoderFactory (either by several streams in one + // PeerConnection or streams with different PeerConnection but same + // PeerConnectionFactory). This is due to the fact that the method is not + // given any stream identifier, nor is the EncoderSelectorInterface given any + // stream identifiers, i.e one does not know which stream is being encoded + // with help of the selector. + // + // In such scenario, the `RtpSenderInterface::SetEncoderSelector` is + // recommended. + // + // TODO(bugs.webrtc.org:14122): Deprecate and remove in favor of + // `RtpSenderInterface::SetEncoderSelector`. + virtual std::unique_ptr<EncoderSelectorInterface> GetEncoderSelector() const { + return nullptr; + } + + virtual ~VideoEncoderFactory() {} +}; + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VIDEO_ENCODER_FACTORY_H_ diff --git a/third_party/libwebrtc/api/video_codecs/video_encoder_factory_template.h b/third_party/libwebrtc/api/video_codecs/video_encoder_factory_template.h new file mode 100644 index 0000000000..643096dbbb --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_encoder_factory_template.h @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VIDEO_ENCODER_FACTORY_TEMPLATE_H_ +#define API_VIDEO_CODECS_VIDEO_ENCODER_FACTORY_TEMPLATE_H_ + +#include <memory> +#include <string> +#include <vector> + +#include "absl/algorithm/container.h" +#include "api/array_view.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/video_coding/svc/scalability_mode_util.h" + +namespace webrtc { +// The VideoEncoderFactoryTemplate supports encoders implementations given as +// template arguments. +// +// To include an encoder in the factory it requires three static members +// functions to be defined: +// +// // Returns the supported SdpVideoFormats this encoder can produce. +// static std::vector<SdpVideoFormat> SupportedFormats(); +// +// // Creates an encoder instance for the given format. +// static std::unique_ptr<VideoEncoder> +// CreateEncoder(const SdpVideoFormat& format); +// +// // Returns true if the encoder supports the given scalability mode. +// static bool +// IsScalabilityModeSupported(ScalabilityMode scalability_mode); +// +// Note that the order of the template arguments matter as the factory will +// query/return the first encoder implementation supporting the given +// SdpVideoFormat. +template <typename... Ts> +class VideoEncoderFactoryTemplate : public VideoEncoderFactory { + public: + std::vector<SdpVideoFormat> GetSupportedFormats() const override { + return GetSupportedFormatsInternal<Ts...>(); + } + + std::unique_ptr<VideoEncoder> CreateVideoEncoder( + const SdpVideoFormat& format) override { + return CreateVideoEncoderInternal<Ts...>(format); + } + + CodecSupport QueryCodecSupport( + const SdpVideoFormat& format, + absl::optional<std::string> scalability_mode) const override { + return QueryCodecSupportInternal<Ts...>(format, scalability_mode); + } + + private: + bool IsFormatInList( + const SdpVideoFormat& format, + rtc::ArrayView<const SdpVideoFormat> supported_formats) const { + return absl::c_any_of( + supported_formats, [&](const SdpVideoFormat& supported_format) { + return supported_format.name == format.name && + supported_format.parameters == format.parameters; + }); + } + + template <typename V> + bool IsScalabilityModeSupported( + const absl::optional<std::string>& scalability_mode_string) const { + if (!scalability_mode_string.has_value()) { + return true; + } + absl::optional<ScalabilityMode> scalability_mode = + ScalabilityModeFromString(*scalability_mode_string); + return scalability_mode.has_value() && + V::IsScalabilityModeSupported(*scalability_mode); + } + + template <typename V, typename... Vs> + std::vector<SdpVideoFormat> GetSupportedFormatsInternal() const { + auto supported_formats = V::SupportedFormats(); + + if constexpr (sizeof...(Vs) > 0) { + // Supported formats may overlap between implementations, so duplicates + // should be filtered out. + for (const auto& other_format : GetSupportedFormatsInternal<Vs...>()) { + if (!IsFormatInList(other_format, supported_formats)) { + supported_formats.push_back(other_format); + } + } + } + + return supported_formats; + } + + template <typename V, typename... Vs> + std::unique_ptr<VideoEncoder> CreateVideoEncoderInternal( + const SdpVideoFormat& format) { + if (IsFormatInList(format, V::SupportedFormats())) { + return V::CreateEncoder(format); + } + + if constexpr (sizeof...(Vs) > 0) { + return CreateVideoEncoderInternal<Vs...>(format); + } + + return nullptr; + } + + template <typename V, typename... Vs> + CodecSupport QueryCodecSupportInternal( + const SdpVideoFormat& format, + const absl::optional<std::string>& scalability_mode) const { + if (IsFormatInList(format, V::SupportedFormats())) { + return {.is_supported = IsScalabilityModeSupported<V>(scalability_mode)}; + } + + if constexpr (sizeof...(Vs) > 0) { + return QueryCodecSupportInternal<Vs...>(format, scalability_mode); + } + + return {.is_supported = false}; + } +}; + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VIDEO_ENCODER_FACTORY_TEMPLATE_H_ diff --git a/third_party/libwebrtc/api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h b/third_party/libwebrtc/api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h new file mode 100644 index 0000000000..417df1e192 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VIDEO_ENCODER_FACTORY_TEMPLATE_LIBAOM_AV1_ADAPTER_H_ +#define API_VIDEO_CODECS_VIDEO_ENCODER_FACTORY_TEMPLATE_LIBAOM_AV1_ADAPTER_H_ + +#include <memory> +#include <vector> + +#include "absl/container/inlined_vector.h" +#include "api/video_codecs/sdp_video_format.h" +#include "modules/video_coding/codecs/av1/av1_svc_config.h" +#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h" + +namespace webrtc { +struct LibaomAv1EncoderTemplateAdapter { + static std::vector<SdpVideoFormat> SupportedFormats() { + absl::InlinedVector<ScalabilityMode, kScalabilityModeCount> + scalability_modes = LibaomAv1EncoderSupportedScalabilityModes(); + return { + SdpVideoFormat("AV1", SdpVideoFormat::Parameters(), scalability_modes)}; + } + + static std::unique_ptr<VideoEncoder> CreateEncoder( + const SdpVideoFormat& format) { + return CreateLibaomAv1Encoder(); + } + + static bool IsScalabilityModeSupported(ScalabilityMode scalability_mode) { + return LibaomAv1EncoderSupportsScalabilityMode(scalability_mode); + } +}; + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VIDEO_ENCODER_FACTORY_TEMPLATE_LIBAOM_AV1_ADAPTER_H_ diff --git a/third_party/libwebrtc/api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h b/third_party/libwebrtc/api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h new file mode 100644 index 0000000000..0f0a9bacd5 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VIDEO_ENCODER_FACTORY_TEMPLATE_LIBVPX_VP8_ADAPTER_H_ +#define API_VIDEO_CODECS_VIDEO_ENCODER_FACTORY_TEMPLATE_LIBVPX_VP8_ADAPTER_H_ + +#include <memory> +#include <vector> + +#include "absl/container/inlined_vector.h" +#include "api/video_codecs/sdp_video_format.h" +#include "modules/video_coding/codecs/vp8/include/vp8.h" +#include "modules/video_coding/codecs/vp8/vp8_scalability.h" + +namespace webrtc { +struct LibvpxVp8EncoderTemplateAdapter { + static std::vector<SdpVideoFormat> SupportedFormats() { + absl::InlinedVector<ScalabilityMode, kScalabilityModeCount> + scalability_modes; + for (const auto scalability_mode : kVP8SupportedScalabilityModes) { + scalability_modes.push_back(scalability_mode); + } + + return { + SdpVideoFormat("VP8", SdpVideoFormat::Parameters(), scalability_modes)}; + } + + static std::unique_ptr<VideoEncoder> CreateEncoder( + const SdpVideoFormat& format) { + return VP8Encoder::Create(); + } + + static bool IsScalabilityModeSupported(ScalabilityMode scalability_mode) { + return VP8SupportsScalabilityMode(scalability_mode); + } +}; +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VIDEO_ENCODER_FACTORY_TEMPLATE_LIBVPX_VP8_ADAPTER_H_ diff --git a/third_party/libwebrtc/api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h b/third_party/libwebrtc/api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h new file mode 100644 index 0000000000..c10fda4dc2 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VIDEO_ENCODER_FACTORY_TEMPLATE_LIBVPX_VP9_ADAPTER_H_ +#define API_VIDEO_CODECS_VIDEO_ENCODER_FACTORY_TEMPLATE_LIBVPX_VP9_ADAPTER_H_ + +#include <memory> +#include <vector> + +#include "modules/video_coding/codecs/vp9/include/vp9.h" + +namespace webrtc { +struct LibvpxVp9EncoderTemplateAdapter { + static std::vector<SdpVideoFormat> SupportedFormats() { + return SupportedVP9Codecs(/*add_scalability_modes=*/true); + } + + static std::unique_ptr<VideoEncoder> CreateEncoder( + const SdpVideoFormat& format) { + return VP9Encoder::Create(cricket::VideoCodec(format)); + } + + static bool IsScalabilityModeSupported(ScalabilityMode scalability_mode) { + return VP9Encoder::SupportsScalabilityMode(scalability_mode); + } +}; +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VIDEO_ENCODER_FACTORY_TEMPLATE_LIBVPX_VP9_ADAPTER_H_ diff --git a/third_party/libwebrtc/api/video_codecs/video_encoder_factory_template_open_h264_adapter.h b/third_party/libwebrtc/api/video_codecs/video_encoder_factory_template_open_h264_adapter.h new file mode 100644 index 0000000000..0830460cdb --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_encoder_factory_template_open_h264_adapter.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VIDEO_ENCODER_FACTORY_TEMPLATE_OPEN_H264_ADAPTER_H_ +#define API_VIDEO_CODECS_VIDEO_ENCODER_FACTORY_TEMPLATE_OPEN_H264_ADAPTER_H_ + +#include <memory> +#include <vector> + +#include "modules/video_coding/codecs/h264/include/h264.h" + +namespace webrtc { +// TODO(bugs.webrtc.org/13573): When OpenH264 is no longer a conditional build +// target remove #ifdefs. +struct OpenH264EncoderTemplateAdapter { + static std::vector<SdpVideoFormat> SupportedFormats() { +#if defined(WEBRTC_USE_H264) + return SupportedH264Codecs(/*add_scalability_modes=*/true); +#else + return {}; +#endif + } + + static std::unique_ptr<VideoEncoder> CreateEncoder( + const SdpVideoFormat& format) { +#if defined(WEBRTC_USE_H264) + return H264Encoder::Create(cricket::VideoCodec(format)); +#else + return nullptr; +#endif + } + + static bool IsScalabilityModeSupported(ScalabilityMode scalability_mode) { +#if defined(WEBRTC_USE_H264) + return H264Encoder::SupportsScalabilityMode(scalability_mode); +#else + return false; +#endif + } +}; +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VIDEO_ENCODER_FACTORY_TEMPLATE_OPEN_H264_ADAPTER_H_ diff --git a/third_party/libwebrtc/api/video_codecs/video_encoder_software_fallback_wrapper.cc b/third_party/libwebrtc/api/video_codecs/video_encoder_software_fallback_wrapper.cc new file mode 100644 index 0000000000..39c52a0081 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_encoder_software_fallback_wrapper.cc @@ -0,0 +1,519 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/video_encoder_software_fallback_wrapper.h" + +#include <stdint.h> + +#include <cstdio> +#include <memory> +#include <string> +#include <vector> + +#include "absl/strings/match.h" +#include "absl/types/optional.h" +#include "api/fec_controller_override.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_bitrate_allocation.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "media/base/video_common.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/utility/simulcast_utility.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { + +namespace { + +// If forced fallback is allowed, either: +// +// 1) The forced fallback is requested if the resolution is less than or equal +// to `max_pixels_`. The resolution is allowed to be scaled down to +// `min_pixels_`. +// +// 2) The forced fallback is requested if temporal support is preferred and the +// SW fallback supports temporal layers while the HW encoder does not. + +struct ForcedFallbackParams { + public: + bool SupportsResolutionBasedSwitch(const VideoCodec& codec) const { + return enable_resolution_based_switch && + codec.codecType == kVideoCodecVP8 && + codec.numberOfSimulcastStreams <= 1 && + codec.width * codec.height <= max_pixels; + } + + bool SupportsTemporalBasedSwitch(const VideoCodec& codec) const { + return enable_temporal_based_switch && + SimulcastUtility::NumberOfTemporalLayers(codec, 0) != 1; + } + + bool enable_temporal_based_switch = false; + bool enable_resolution_based_switch = false; + int min_pixels = 320 * 180; + int max_pixels = 320 * 240; +}; + +const char kVp8ForceFallbackEncoderFieldTrial[] = + "WebRTC-VP8-Forced-Fallback-Encoder-v2"; + +absl::optional<ForcedFallbackParams> ParseFallbackParamsFromFieldTrials( + const VideoEncoder& main_encoder) { + const std::string field_trial = + webrtc::field_trial::FindFullName(kVp8ForceFallbackEncoderFieldTrial); + if (!absl::StartsWith(field_trial, "Enabled")) { + return absl::nullopt; + } + + int max_pixels_lower_bound = + main_encoder.GetEncoderInfo().scaling_settings.min_pixels_per_frame - 1; + + ForcedFallbackParams params; + params.enable_resolution_based_switch = true; + + int min_bps = 0; + if (sscanf(field_trial.c_str(), "Enabled-%d,%d,%d", ¶ms.min_pixels, + ¶ms.max_pixels, &min_bps) != 3) { + RTC_LOG(LS_WARNING) + << "Invalid number of forced fallback parameters provided."; + return absl::nullopt; + } else if (params.min_pixels <= 0 || + params.max_pixels < max_pixels_lower_bound || + params.max_pixels < params.min_pixels || min_bps <= 0) { + RTC_LOG(LS_WARNING) << "Invalid forced fallback parameter value provided."; + return absl::nullopt; + } + + return params; +} + +absl::optional<ForcedFallbackParams> GetForcedFallbackParams( + bool prefer_temporal_support, + const VideoEncoder& main_encoder) { + absl::optional<ForcedFallbackParams> params = + ParseFallbackParamsFromFieldTrials(main_encoder); + if (prefer_temporal_support) { + if (!params.has_value()) { + params.emplace(); + } + params->enable_temporal_based_switch = prefer_temporal_support; + } + return params; +} + +class VideoEncoderSoftwareFallbackWrapper final : public VideoEncoder { + public: + VideoEncoderSoftwareFallbackWrapper( + std::unique_ptr<webrtc::VideoEncoder> sw_encoder, + std::unique_ptr<webrtc::VideoEncoder> hw_encoder, + bool prefer_temporal_support); + ~VideoEncoderSoftwareFallbackWrapper() override; + + void SetFecControllerOverride( + FecControllerOverride* fec_controller_override) override; + + int32_t InitEncode(const VideoCodec* codec_settings, + const VideoEncoder::Settings& settings) override; + + int32_t RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) override; + + int32_t Release() override; + + int32_t Encode(const VideoFrame& frame, + const std::vector<VideoFrameType>* frame_types) override; + + void OnPacketLossRateUpdate(float packet_loss_rate) override; + + void OnRttUpdate(int64_t rtt_ms) override; + + void OnLossNotification(const LossNotification& loss_notification) override; + + void SetRates(const RateControlParameters& parameters) override; + + EncoderInfo GetEncoderInfo() const override; + + private: + bool InitFallbackEncoder(bool is_forced); + bool TryInitForcedFallbackEncoder(); + bool IsFallbackActive() const; + + VideoEncoder* current_encoder() { + switch (encoder_state_) { + case EncoderState::kUninitialized: + RTC_LOG(LS_WARNING) + << "Trying to access encoder in uninitialized fallback wrapper."; + // Return main encoder to preserve previous behavior. + [[fallthrough]]; + case EncoderState::kMainEncoderUsed: + return encoder_.get(); + case EncoderState::kFallbackDueToFailure: + case EncoderState::kForcedFallback: + return fallback_encoder_.get(); + } + RTC_CHECK_NOTREACHED(); + } + + // Updates encoder with last observed parameters, such as callbacks, rates, + // etc. + void PrimeEncoder(VideoEncoder* encoder) const; + + // Settings used in the last InitEncode call and used if a dynamic fallback to + // software is required. + VideoCodec codec_settings_; + absl::optional<VideoEncoder::Settings> encoder_settings_; + + // The last rate control settings, if set. + absl::optional<RateControlParameters> rate_control_parameters_; + + // The last channel parameters set. + absl::optional<float> packet_loss_; + absl::optional<int64_t> rtt_; + absl::optional<LossNotification> loss_notification_; + + enum class EncoderState { + kUninitialized, + kMainEncoderUsed, + kFallbackDueToFailure, + kForcedFallback + }; + + EncoderState encoder_state_; + const std::unique_ptr<webrtc::VideoEncoder> encoder_; + const std::unique_ptr<webrtc::VideoEncoder> fallback_encoder_; + + EncodedImageCallback* callback_; + + const absl::optional<ForcedFallbackParams> fallback_params_; + int32_t EncodeWithMainEncoder(const VideoFrame& frame, + const std::vector<VideoFrameType>* frame_types); +}; + +VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper( + std::unique_ptr<webrtc::VideoEncoder> sw_encoder, + std::unique_ptr<webrtc::VideoEncoder> hw_encoder, + bool prefer_temporal_support) + : encoder_state_(EncoderState::kUninitialized), + encoder_(std::move(hw_encoder)), + fallback_encoder_(std::move(sw_encoder)), + callback_(nullptr), + fallback_params_( + GetForcedFallbackParams(prefer_temporal_support, *encoder_)) { + RTC_DCHECK(fallback_encoder_); +} + +VideoEncoderSoftwareFallbackWrapper::~VideoEncoderSoftwareFallbackWrapper() = + default; + +void VideoEncoderSoftwareFallbackWrapper::PrimeEncoder( + VideoEncoder* encoder) const { + RTC_DCHECK(encoder); + // Replay callback, rates, and channel parameters. + if (callback_) { + encoder->RegisterEncodeCompleteCallback(callback_); + } + if (rate_control_parameters_) { + encoder->SetRates(*rate_control_parameters_); + } + if (rtt_.has_value()) { + encoder->OnRttUpdate(rtt_.value()); + } + if (packet_loss_.has_value()) { + encoder->OnPacketLossRateUpdate(packet_loss_.value()); + } + + if (loss_notification_.has_value()) { + encoder->OnLossNotification(loss_notification_.value()); + } +} + +bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder(bool is_forced) { + RTC_LOG(LS_WARNING) << "Encoder falling back to software encoding."; + + RTC_DCHECK(encoder_settings_.has_value()); + const int ret = fallback_encoder_->InitEncode(&codec_settings_, + encoder_settings_.value()); + + if (ret != WEBRTC_VIDEO_CODEC_OK) { + RTC_LOG(LS_ERROR) << "Failed to initialize software-encoder fallback."; + fallback_encoder_->Release(); + return false; + } + + if (encoder_state_ == EncoderState::kMainEncoderUsed) { + // Since we're switching to the fallback encoder, Release the real encoder. + // It may be re-initialized via InitEncode later, and it will continue to + // get Set calls for rates and channel parameters in the meantime. + encoder_->Release(); + } + + if (is_forced) { + encoder_state_ = EncoderState::kForcedFallback; + } else { + encoder_state_ = EncoderState::kFallbackDueToFailure; + } + + return true; +} + +void VideoEncoderSoftwareFallbackWrapper::SetFecControllerOverride( + FecControllerOverride* fec_controller_override) { + // It is important that only one of those would ever interact with the + // `fec_controller_override` at a given time. This is the responsibility + // of `this` to maintain. + + encoder_->SetFecControllerOverride(fec_controller_override); + fallback_encoder_->SetFecControllerOverride(fec_controller_override); +} + +int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode( + const VideoCodec* codec_settings, + const VideoEncoder::Settings& settings) { + // Store settings, in case we need to dynamically switch to the fallback + // encoder after a failed Encode call. + codec_settings_ = *codec_settings; + encoder_settings_ = settings; + // Clear stored rate/channel parameters. + rate_control_parameters_ = absl::nullopt; + + RTC_DCHECK_EQ(encoder_state_, EncoderState::kUninitialized) + << "InitEncode() should never be called on an active instance!"; + + // Try to init forced software codec if it should be used. + if (TryInitForcedFallbackEncoder()) { + PrimeEncoder(current_encoder()); + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t ret = encoder_->InitEncode(codec_settings, settings); + if (ret == WEBRTC_VIDEO_CODEC_OK) { + encoder_state_ = EncoderState::kMainEncoderUsed; + PrimeEncoder(current_encoder()); + return ret; + } + + // Try to instantiate software codec. + if (InitFallbackEncoder(/*is_forced=*/false)) { + PrimeEncoder(current_encoder()); + return WEBRTC_VIDEO_CODEC_OK; + } + + // Software encoder failed too, use original return code. + encoder_state_ = EncoderState::kUninitialized; + return ret; +} + +int32_t VideoEncoderSoftwareFallbackWrapper::RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) { + callback_ = callback; + return current_encoder()->RegisterEncodeCompleteCallback(callback); +} + +int32_t VideoEncoderSoftwareFallbackWrapper::Release() { + if (encoder_state_ == EncoderState::kUninitialized) { + return WEBRTC_VIDEO_CODEC_OK; + } + int32_t ret = current_encoder()->Release(); + encoder_state_ = EncoderState::kUninitialized; + return ret; +} + +int32_t VideoEncoderSoftwareFallbackWrapper::Encode( + const VideoFrame& frame, + const std::vector<VideoFrameType>* frame_types) { + switch (encoder_state_) { + case EncoderState::kUninitialized: + return WEBRTC_VIDEO_CODEC_ERROR; + case EncoderState::kMainEncoderUsed: { + return EncodeWithMainEncoder(frame, frame_types); + } + case EncoderState::kFallbackDueToFailure: + case EncoderState::kForcedFallback: + return fallback_encoder_->Encode(frame, frame_types); + } + RTC_CHECK_NOTREACHED(); +} + +int32_t VideoEncoderSoftwareFallbackWrapper::EncodeWithMainEncoder( + const VideoFrame& frame, + const std::vector<VideoFrameType>* frame_types) { + int32_t ret = encoder_->Encode(frame, frame_types); + // If requested, try a software fallback. + bool fallback_requested = (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE); + if (fallback_requested && InitFallbackEncoder(/*is_forced=*/false)) { + // Start using the fallback with this frame. + PrimeEncoder(current_encoder()); + if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative && + fallback_encoder_->GetEncoderInfo().supports_native_handle) { + return fallback_encoder_->Encode(frame, frame_types); + } else { + RTC_LOG(LS_INFO) << "Fallback encoder does not support native handle - " + "converting frame to I420"; + rtc::scoped_refptr<I420BufferInterface> src_buffer = + frame.video_frame_buffer()->ToI420(); + if (!src_buffer) { + RTC_LOG(LS_ERROR) << "Failed to convert from to I420"; + return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } + rtc::scoped_refptr<VideoFrameBuffer> dst_buffer = + src_buffer->Scale(codec_settings_.width, codec_settings_.height); + if (!dst_buffer) { + RTC_LOG(LS_ERROR) << "Failed to scale video frame."; + return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } + VideoFrame scaled_frame = frame; + scaled_frame.set_video_frame_buffer(dst_buffer); + scaled_frame.set_update_rect(VideoFrame::UpdateRect{ + 0, 0, scaled_frame.width(), scaled_frame.height()}); + return fallback_encoder_->Encode(scaled_frame, frame_types); + } + } + // Fallback encoder failed too, return original error code. + return ret; +} + +void VideoEncoderSoftwareFallbackWrapper::SetRates( + const RateControlParameters& parameters) { + rate_control_parameters_ = parameters; + return current_encoder()->SetRates(parameters); +} + +void VideoEncoderSoftwareFallbackWrapper::OnPacketLossRateUpdate( + float packet_loss_rate) { + packet_loss_ = packet_loss_rate; + current_encoder()->OnPacketLossRateUpdate(packet_loss_rate); +} + +void VideoEncoderSoftwareFallbackWrapper::OnRttUpdate(int64_t rtt_ms) { + rtt_ = rtt_ms; + current_encoder()->OnRttUpdate(rtt_ms); +} + +void VideoEncoderSoftwareFallbackWrapper::OnLossNotification( + const LossNotification& loss_notification) { + loss_notification_ = loss_notification; + current_encoder()->OnLossNotification(loss_notification); +} + +VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo() + const { + EncoderInfo fallback_encoder_info = fallback_encoder_->GetEncoderInfo(); + EncoderInfo default_encoder_info = encoder_->GetEncoderInfo(); + + EncoderInfo info = + IsFallbackActive() ? fallback_encoder_info : default_encoder_info; + + info.requested_resolution_alignment = cricket::LeastCommonMultiple( + fallback_encoder_info.requested_resolution_alignment, + default_encoder_info.requested_resolution_alignment); + info.apply_alignment_to_all_simulcast_layers = + fallback_encoder_info.apply_alignment_to_all_simulcast_layers || + default_encoder_info.apply_alignment_to_all_simulcast_layers; + + if (fallback_params_.has_value()) { + const auto settings = (encoder_state_ == EncoderState::kForcedFallback) + ? fallback_encoder_info.scaling_settings + : default_encoder_info.scaling_settings; + info.scaling_settings = + settings.thresholds + ? VideoEncoder::ScalingSettings(settings.thresholds->low, + settings.thresholds->high, + fallback_params_->min_pixels) + : VideoEncoder::ScalingSettings::kOff; + } else { + info.scaling_settings = default_encoder_info.scaling_settings; + } + + return info; +} + +bool VideoEncoderSoftwareFallbackWrapper::IsFallbackActive() const { + return encoder_state_ == EncoderState::kForcedFallback || + encoder_state_ == EncoderState::kFallbackDueToFailure; +} + +bool VideoEncoderSoftwareFallbackWrapper::TryInitForcedFallbackEncoder() { + if (!fallback_params_) { + return false; + } + + RTC_DCHECK_EQ(encoder_state_, EncoderState::kUninitialized); + + if (fallback_params_->SupportsResolutionBasedSwitch(codec_settings_)) { + // Settings valid, try to instantiate software codec. + RTC_LOG(LS_INFO) << "Request forced SW encoder fallback: " + << codec_settings_.width << "x" << codec_settings_.height; + return InitFallbackEncoder(/*is_forced=*/true); + } + + if (fallback_params_->SupportsTemporalBasedSwitch(codec_settings_)) { + // First init main encoder to see if that supports temporal layers. + if (encoder_->InitEncode(&codec_settings_, encoder_settings_.value()) == + WEBRTC_VIDEO_CODEC_OK) { + encoder_state_ = EncoderState::kMainEncoderUsed; + } + + if (encoder_state_ == EncoderState::kMainEncoderUsed && + encoder_->GetEncoderInfo().fps_allocation[0].size() != 1) { + // Primary encoder already supports temporal layers, use that instead. + return true; + } + + // Try to initialize fallback and check if it supports temporal layers. + if (fallback_encoder_->InitEncode(&codec_settings_, + encoder_settings_.value()) == + WEBRTC_VIDEO_CODEC_OK) { + if (fallback_encoder_->GetEncoderInfo().fps_allocation[0].size() != 1) { + // Fallback encoder available and supports temporal layers, use it! + if (encoder_state_ == EncoderState::kMainEncoderUsed) { + // Main encoder initialized but does not support temporal layers, + // release it again. + encoder_->Release(); + } + encoder_state_ = EncoderState::kForcedFallback; + RTC_LOG(LS_INFO) + << "Forced switch to SW encoder due to temporal support."; + return true; + } else { + // Fallback encoder intialization succeeded, but it does not support + // temporal layers either - release it. + fallback_encoder_->Release(); + } + } + + if (encoder_state_ == EncoderState::kMainEncoderUsed) { + // Main encoder already initialized - make use of it. + RTC_LOG(LS_INFO) + << "Cannot fall back for temporal support since fallback that " + "supports is not available. Using main encoder instead."; + return true; + } + } + + // Neither forced fallback mode supported. + return false; +} + +} // namespace + +std::unique_ptr<VideoEncoder> CreateVideoEncoderSoftwareFallbackWrapper( + std::unique_ptr<VideoEncoder> sw_fallback_encoder, + std::unique_ptr<VideoEncoder> hw_encoder, + bool prefer_temporal_support) { + return std::make_unique<VideoEncoderSoftwareFallbackWrapper>( + std::move(sw_fallback_encoder), std::move(hw_encoder), + prefer_temporal_support); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/video_encoder_software_fallback_wrapper.h b/third_party/libwebrtc/api/video_codecs/video_encoder_software_fallback_wrapper.h new file mode 100644 index 0000000000..6e6902eb3f --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/video_encoder_software_fallback_wrapper.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VIDEO_ENCODER_SOFTWARE_FALLBACK_WRAPPER_H_ +#define API_VIDEO_CODECS_VIDEO_ENCODER_SOFTWARE_FALLBACK_WRAPPER_H_ + +#include <memory> +#include <utility> + +#include "api/video_codecs/video_encoder.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// Used to wrap external VideoEncoders to provide a fallback option on +// software encoding when a hardware encoder fails to encode a stream due to +// hardware restrictions, such as max resolution. +// |bool prefer_temporal_support| indicates that if the software fallback +// encoder supports temporal layers but the hardware encoder does not, a +// fallback should be forced even if the encoder otherwise works. +RTC_EXPORT std::unique_ptr<VideoEncoder> +CreateVideoEncoderSoftwareFallbackWrapper( + std::unique_ptr<VideoEncoder> sw_fallback_encoder, + std::unique_ptr<VideoEncoder> hw_encoder, + bool prefer_temporal_support); + +// Default fallback for call-sites not yet updated with +// `prefer_temporal_support`. +// TODO(sprang): Remove when usage is gone. +RTC_EXPORT inline std::unique_ptr<VideoEncoder> +CreateVideoEncoderSoftwareFallbackWrapper( + std::unique_ptr<VideoEncoder> sw_fallback_encoder, + std::unique_ptr<VideoEncoder> hw_encoder) { + return CreateVideoEncoderSoftwareFallbackWrapper( + std::move(sw_fallback_encoder), std::move(hw_encoder), false); +} + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VIDEO_ENCODER_SOFTWARE_FALLBACK_WRAPPER_H_ diff --git a/third_party/libwebrtc/api/video_codecs/vp8_frame_buffer_controller.h b/third_party/libwebrtc/api/video_codecs/vp8_frame_buffer_controller.h new file mode 100644 index 0000000000..fc494f7293 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/vp8_frame_buffer_controller.h @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VP8_FRAME_BUFFER_CONTROLLER_H_ +#define API_VIDEO_CODECS_VP8_FRAME_BUFFER_CONTROLLER_H_ + +#include <array> +#include <memory> +#include <vector> + +#include "absl/types/optional.h" +#include "api/fec_controller_override.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/vp8_frame_config.h" + +namespace webrtc { + +// Some notes on the prerequisites of the TemporalLayers interface. +// * Vp8FrameBufferController is not thread safe, synchronization is the +// caller's responsibility. +// * The encoder is assumed to encode all frames in order, and callbacks to +// PopulateCodecSpecific() / OnEncodeDone() must happen in the same order. +// +// This means that in the case of pipelining encoders, it is OK to have a chain +// of calls such as this: +// - NextFrameConfig(timestampA) +// - NextFrameConfig(timestampB) +// - PopulateCodecSpecific(timestampA, ...) +// - NextFrameConfig(timestampC) +// - OnEncodeDone(timestampA, 1234, ...) +// - NextFrameConfig(timestampC) +// - OnEncodeDone(timestampB, 0, ...) +// - OnEncodeDone(timestampC, 1234, ...) +// Note that NextFrameConfig() for a new frame can happen before +// OnEncodeDone() for a previous one, but calls themselves must be both +// synchronized (e.g. run on a task queue) and in order (per type). +// +// TODO(eladalon): Revise comment (referring to PopulateCodecSpecific in this +// context is not very meaningful). + +struct CodecSpecificInfo; + +// Each member represents an override of the VPX configuration if the optional +// value is set. +struct Vp8EncoderConfig { + struct TemporalLayerConfig { + bool operator!=(const TemporalLayerConfig& other) const { + return ts_number_layers != other.ts_number_layers || + ts_target_bitrate != other.ts_target_bitrate || + ts_rate_decimator != other.ts_rate_decimator || + ts_periodicity != other.ts_periodicity || + ts_layer_id != other.ts_layer_id; + } + + static constexpr size_t kMaxPeriodicity = 16; + static constexpr size_t kMaxLayers = 5; + + // Number of active temporal layers. Set to 0 if not used. + uint32_t ts_number_layers; + + // Arrays of length `ts_number_layers`, indicating (cumulative) target + // bitrate and rate decimator (e.g. 4 if every 4th frame is in the given + // layer) for each active temporal layer, starting with temporal id 0. + std::array<uint32_t, kMaxLayers> ts_target_bitrate; + std::array<uint32_t, kMaxLayers> ts_rate_decimator; + + // The periodicity of the temporal pattern. Set to 0 if not used. + uint32_t ts_periodicity; + + // Array of length `ts_periodicity` indicating the sequence of temporal id's + // to assign to incoming frames. + std::array<uint32_t, kMaxPeriodicity> ts_layer_id; + }; + + absl::optional<TemporalLayerConfig> temporal_layer_config; + + // Target bitrate, in bps. + absl::optional<uint32_t> rc_target_bitrate; + + // Clamp QP to max. Use 0 to disable clamping. + absl::optional<uint32_t> rc_max_quantizer; + + // Error resilience mode. + absl::optional<uint32_t> g_error_resilient; + + // If set to true, all previous configuration overrides should be reset. + bool reset_previous_configuration_overrides = false; +}; + +// This interface defines a way of delegating the logic of buffer management. +// Multiple streams may be controlled by a single controller, demuxing between +// them using stream_index. +class Vp8FrameBufferController { + public: + virtual ~Vp8FrameBufferController() = default; + + // Set limits on QP. + // The limits are suggestion-only; the controller is allowed to exceed them. + virtual void SetQpLimits(size_t stream_index, int min_qp, int max_qp) = 0; + + // Number of streamed controlled by `this`. + virtual size_t StreamCount() const = 0; + + // If this method returns true, the encoder is free to drop frames for + // instance in an effort to uphold encoding bitrate. + // If this return false, the encoder must not drop any frames unless: + // 1. Requested to do so via Vp8FrameConfig.drop_frame + // 2. The frame to be encoded is requested to be a keyframe + // 3. The encoder detected a large overshoot and decided to drop and then + // re-encode the image at a low bitrate. In this case the encoder should + // call OnFrameDropped() once to indicate drop, and then call + // OnEncodeDone() again when the frame has actually been encoded. + virtual bool SupportsEncoderFrameDropping(size_t stream_index) const = 0; + + // New target bitrate for a stream (each entry in + // `bitrates_bps` is for another temporal layer). + virtual void OnRatesUpdated(size_t stream_index, + const std::vector<uint32_t>& bitrates_bps, + int framerate_fps) = 0; + + // Called by the encoder before encoding a frame. Returns a set of overrides + // the controller wishes to enact in the encoder's configuration. + // If a value is not overridden, previous overrides are still in effect. + // However, if `Vp8EncoderConfig::reset_previous_configuration_overrides` + // is set to `true`, all previous overrides are reset. + virtual Vp8EncoderConfig UpdateConfiguration(size_t stream_index) = 0; + + // Returns the recommended VP8 encode flags needed. + // The timestamp may be used as both a time and a unique identifier, and so + // the caller must make sure no two frames use the same timestamp. + // The timestamp uses a 90kHz RTP clock. + // After calling this method, first call the actual encoder with the provided + // frame configuration, and then OnEncodeDone() below. + virtual Vp8FrameConfig NextFrameConfig(size_t stream_index, + uint32_t rtp_timestamp) = 0; + + // Called after the encode step is done. `rtp_timestamp` must match the + // parameter use in the NextFrameConfig() call. + // `is_keyframe` must be true iff the encoder decided to encode this frame as + // a keyframe. + // If `info` is not null, the encoder may update `info` with codec specific + // data such as temporal id. `qp` should indicate the frame-level QP this + // frame was encoded at. If the encoder does not support extracting this, `qp` + // should be set to 0. + virtual void OnEncodeDone(size_t stream_index, + uint32_t rtp_timestamp, + size_t size_bytes, + bool is_keyframe, + int qp, + CodecSpecificInfo* info) = 0; + + // Called when a frame is dropped by the encoder. + virtual void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) = 0; + + // Called by the encoder when the packet loss rate changes. + // `packet_loss_rate` runs between 0.0 (no loss) and 1.0 (everything lost). + virtual void OnPacketLossRateUpdate(float packet_loss_rate) = 0; + + // Called by the encoder when the round trip time changes. + virtual void OnRttUpdate(int64_t rtt_ms) = 0; + + // Called when a loss notification is received. + virtual void OnLossNotification( + const VideoEncoder::LossNotification& loss_notification) = 0; +}; + +// Interface for a factory of Vp8FrameBufferController instances. +class Vp8FrameBufferControllerFactory { + public: + virtual ~Vp8FrameBufferControllerFactory() = default; + + // Clones oneself. (Avoids Vp8FrameBufferControllerFactoryFactory.) + virtual std::unique_ptr<Vp8FrameBufferControllerFactory> Clone() const = 0; + + // Create a Vp8FrameBufferController instance. + virtual std::unique_ptr<Vp8FrameBufferController> Create( + const VideoCodec& codec, + const VideoEncoder::Settings& settings, + FecControllerOverride* fec_controller_override) = 0; +}; + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VP8_FRAME_BUFFER_CONTROLLER_H_ diff --git a/third_party/libwebrtc/api/video_codecs/vp8_frame_config.cc b/third_party/libwebrtc/api/video_codecs/vp8_frame_config.cc new file mode 100644 index 0000000000..05e1911bb7 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/vp8_frame_config.cc @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/vp8_frame_config.h" + +#include "modules/video_coding/codecs/interface/common_constants.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +Vp8FrameConfig::Vp8FrameConfig() : Vp8FrameConfig(kNone, kNone, kNone, false) {} + +Vp8FrameConfig::Vp8FrameConfig(BufferFlags last, + BufferFlags golden, + BufferFlags arf) + : Vp8FrameConfig(last, golden, arf, false) {} + +Vp8FrameConfig::Vp8FrameConfig(BufferFlags last, + BufferFlags golden, + BufferFlags arf, + FreezeEntropy) + : Vp8FrameConfig(last, golden, arf, true) {} + +Vp8FrameConfig::Vp8FrameConfig(BufferFlags last, + BufferFlags golden, + BufferFlags arf, + bool freeze_entropy) + : drop_frame(last == BufferFlags::kNone && golden == BufferFlags::kNone && + arf == BufferFlags::kNone), + last_buffer_flags(last), + golden_buffer_flags(golden), + arf_buffer_flags(arf), + encoder_layer_id(0), + packetizer_temporal_idx(kNoTemporalIdx), + layer_sync(false), + freeze_entropy(freeze_entropy), + first_reference(Vp8BufferReference::kNone), + second_reference(Vp8BufferReference::kNone), + retransmission_allowed(true) {} + +bool Vp8FrameConfig::References(Buffer buffer) const { + switch (buffer) { + case Buffer::kLast: + return (last_buffer_flags & kReference) != 0; + case Buffer::kGolden: + return (golden_buffer_flags & kReference) != 0; + case Buffer::kArf: + return (arf_buffer_flags & kReference) != 0; + case Buffer::kCount: + break; + } + RTC_DCHECK_NOTREACHED(); + return false; +} + +bool Vp8FrameConfig::Updates(Buffer buffer) const { + switch (buffer) { + case Buffer::kLast: + return (last_buffer_flags & kUpdate) != 0; + case Buffer::kGolden: + return (golden_buffer_flags & kUpdate) != 0; + case Buffer::kArf: + return (arf_buffer_flags & kUpdate) != 0; + case Buffer::kCount: + break; + } + RTC_DCHECK_NOTREACHED(); + return false; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/vp8_frame_config.h b/third_party/libwebrtc/api/video_codecs/vp8_frame_config.h new file mode 100644 index 0000000000..5369bf58bc --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/vp8_frame_config.h @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VP8_FRAME_CONFIG_H_ +#define API_VIDEO_CODECS_VP8_FRAME_CONFIG_H_ + +#include <stdint.h> + +namespace webrtc { + +// Configuration of a VP8 frame - which buffers are to be referenced +// by it, which buffers should be updated, etc. +struct Vp8FrameConfig { + static Vp8FrameConfig GetIntraFrameConfig() { + Vp8FrameConfig frame_config = Vp8FrameConfig( + BufferFlags::kUpdate, BufferFlags::kUpdate, BufferFlags::kUpdate); + frame_config.packetizer_temporal_idx = 0; + return frame_config; + } + + enum BufferFlags : int { + kNone = 0, + kReference = 1, + kUpdate = 2, + kReferenceAndUpdate = kReference | kUpdate, + }; + + enum FreezeEntropy { kFreezeEntropy }; + + // Defined bit-maskable reference to the three buffers available in VP8. + enum class Vp8BufferReference : uint8_t { + kNone = 0, + kLast = 1, + kGolden = 2, + kAltref = 4 + }; + + Vp8FrameConfig(); + + Vp8FrameConfig(BufferFlags last, BufferFlags golden, BufferFlags arf); + Vp8FrameConfig(BufferFlags last, + BufferFlags golden, + BufferFlags arf, + FreezeEntropy); + + enum class Buffer : int { kLast = 0, kGolden = 1, kArf = 2, kCount }; + + bool References(Buffer buffer) const; + + bool Updates(Buffer buffer) const; + + bool IntraFrame() const { + // Intra frames do not reference any buffers, and update all buffers. + return last_buffer_flags == kUpdate && golden_buffer_flags == kUpdate && + arf_buffer_flags == kUpdate; + } + + bool drop_frame; + BufferFlags last_buffer_flags; + BufferFlags golden_buffer_flags; + BufferFlags arf_buffer_flags; + + // The encoder layer ID is used to utilize the correct bitrate allocator + // inside the encoder. It does not control references nor determine which + // "actual" temporal layer this is. The packetizer temporal index determines + // which layer the encoded frame should be packetized into. + // Normally these are the same, but current temporal-layer strategies for + // screenshare use one bitrate allocator for all layers, but attempt to + // packetize / utilize references to split a stream into multiple layers, + // with different quantizer settings, to hit target bitrate. + // TODO(sprang): Screenshare layers are being reconsidered at the time of + // writing, we might be able to remove this distinction, and have a temporal + // layer imply both (the normal case). + int encoder_layer_id; + // TODO(eladalon/sprang): Move out of this class. + int packetizer_temporal_idx; + + // TODO(eladalon/sprang): Move out of this class. + bool layer_sync; + + bool freeze_entropy; + + // Indicates in which order the encoder should search the reference buffers + // when doing motion prediction. Set to kNone to use unspecified order. Any + // buffer indicated here must not have the corresponding no_ref bit set. + // If all three buffers can be reference, the one not listed here should be + // searched last. + Vp8BufferReference first_reference; + Vp8BufferReference second_reference; + + // Whether this frame is eligible for retransmission. + bool retransmission_allowed; + + private: + Vp8FrameConfig(BufferFlags last, + BufferFlags golden, + BufferFlags arf, + bool freeze_entropy); +}; + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VP8_FRAME_CONFIG_H_ diff --git a/third_party/libwebrtc/api/video_codecs/vp8_temporal_layers.cc b/third_party/libwebrtc/api/video_codecs/vp8_temporal_layers.cc new file mode 100644 index 0000000000..dd75c616d8 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/vp8_temporal_layers.cc @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/vp8_temporal_layers.h" + +#include <utility> + +#include "absl/algorithm/container.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +Vp8TemporalLayers::Vp8TemporalLayers( + std::vector<std::unique_ptr<Vp8FrameBufferController>>&& controllers, + FecControllerOverride* fec_controller_override) + : controllers_(std::move(controllers)) { + RTC_DCHECK(!controllers_.empty()); + RTC_DCHECK(absl::c_none_of( + controllers_, + [](const std::unique_ptr<Vp8FrameBufferController>& controller) { + return controller.get() == nullptr; + })); + if (fec_controller_override) { + fec_controller_override->SetFecAllowed(true); + } +} + +void Vp8TemporalLayers::SetQpLimits(size_t stream_index, + int min_qp, + int max_qp) { + RTC_DCHECK_LT(stream_index, controllers_.size()); + return controllers_[stream_index]->SetQpLimits(0, min_qp, max_qp); +} + +size_t Vp8TemporalLayers::StreamCount() const { + return controllers_.size(); +} + +bool Vp8TemporalLayers::SupportsEncoderFrameDropping( + size_t stream_index) const { + RTC_DCHECK_LT(stream_index, controllers_.size()); + return controllers_[stream_index]->SupportsEncoderFrameDropping(0); +} + +void Vp8TemporalLayers::OnRatesUpdated( + size_t stream_index, + const std::vector<uint32_t>& bitrates_bps, + int framerate_fps) { + RTC_DCHECK_LT(stream_index, controllers_.size()); + return controllers_[stream_index]->OnRatesUpdated(0, bitrates_bps, + framerate_fps); +} + +Vp8EncoderConfig Vp8TemporalLayers::UpdateConfiguration(size_t stream_index) { + RTC_DCHECK_LT(stream_index, controllers_.size()); + return controllers_[stream_index]->UpdateConfiguration(0); +} + +Vp8FrameConfig Vp8TemporalLayers::NextFrameConfig(size_t stream_index, + uint32_t rtp_timestamp) { + RTC_DCHECK_LT(stream_index, controllers_.size()); + return controllers_[stream_index]->NextFrameConfig(0, rtp_timestamp); +} + +void Vp8TemporalLayers::OnEncodeDone(size_t stream_index, + uint32_t rtp_timestamp, + size_t size_bytes, + bool is_keyframe, + int qp, + CodecSpecificInfo* info) { + RTC_DCHECK_LT(stream_index, controllers_.size()); + return controllers_[stream_index]->OnEncodeDone(0, rtp_timestamp, size_bytes, + is_keyframe, qp, info); +} + +void Vp8TemporalLayers::OnFrameDropped(size_t stream_index, + uint32_t rtp_timestamp) { + RTC_DCHECK_LT(stream_index, controllers_.size()); + controllers_[stream_index]->OnFrameDropped(stream_index, rtp_timestamp); +} + +void Vp8TemporalLayers::OnPacketLossRateUpdate(float packet_loss_rate) { + for (auto& controller : controllers_) { + controller->OnPacketLossRateUpdate(packet_loss_rate); + } +} + +void Vp8TemporalLayers::OnRttUpdate(int64_t rtt_ms) { + for (auto& controller : controllers_) { + controller->OnRttUpdate(rtt_ms); + } +} + +void Vp8TemporalLayers::OnLossNotification( + const VideoEncoder::LossNotification& loss_notification) { + for (auto& controller : controllers_) { + controller->OnLossNotification(loss_notification); + } +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/vp8_temporal_layers.h b/third_party/libwebrtc/api/video_codecs/vp8_temporal_layers.h new file mode 100644 index 0000000000..2ffe6eacdf --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/vp8_temporal_layers.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VP8_TEMPORAL_LAYERS_H_ +#define API_VIDEO_CODECS_VP8_TEMPORAL_LAYERS_H_ + +#include <memory> +#include <vector> + +#include "api/fec_controller_override.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/vp8_frame_buffer_controller.h" +#include "api/video_codecs/vp8_frame_config.h" + +namespace webrtc { + +// Two different flavors of temporal layers are currently available: +// kFixedPattern uses a fixed repeating pattern of 1-4 layers. +// kBitrateDynamic can allocate frames dynamically to 1 or 2 layers, based on +// the bitrate produced. +// TODO(eladalon): Remove this enum. +enum class Vp8TemporalLayersType { kFixedPattern, kBitrateDynamic }; + +// This interface defines a way of getting the encoder settings needed to +// realize a temporal layer structure. +class Vp8TemporalLayers final : public Vp8FrameBufferController { + public: + Vp8TemporalLayers( + std::vector<std::unique_ptr<Vp8FrameBufferController>>&& controllers, + FecControllerOverride* fec_controller_override); + ~Vp8TemporalLayers() override = default; + + void SetQpLimits(size_t stream_index, int min_qp, int max_qp) override; + + size_t StreamCount() const override; + + bool SupportsEncoderFrameDropping(size_t stream_index) const override; + + void OnRatesUpdated(size_t stream_index, + const std::vector<uint32_t>& bitrates_bps, + int framerate_fps) override; + + Vp8EncoderConfig UpdateConfiguration(size_t stream_index) override; + + Vp8FrameConfig NextFrameConfig(size_t stream_index, + uint32_t rtp_timestamp) override; + + void OnEncodeDone(size_t stream_index, + uint32_t rtp_timestamp, + size_t size_bytes, + bool is_keyframe, + int qp, + CodecSpecificInfo* info) override; + + void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) override; + + void OnPacketLossRateUpdate(float packet_loss_rate) override; + + void OnRttUpdate(int64_t rtt_ms) override; + + void OnLossNotification( + const VideoEncoder::LossNotification& loss_notification) override; + + private: + std::vector<std::unique_ptr<Vp8FrameBufferController>> controllers_; +}; + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VP8_TEMPORAL_LAYERS_H_ diff --git a/third_party/libwebrtc/api/video_codecs/vp8_temporal_layers_factory.cc b/third_party/libwebrtc/api/video_codecs/vp8_temporal_layers_factory.cc new file mode 100644 index 0000000000..193494d71d --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/vp8_temporal_layers_factory.cc @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/vp8_temporal_layers_factory.h" + +#include <algorithm> +#include <memory> +#include <utility> +#include <vector> + +#include "api/fec_controller_override.h" +#include "modules/video_coding/codecs/vp8/default_temporal_layers.h" +#include "modules/video_coding/codecs/vp8/screenshare_layers.h" +#include "modules/video_coding/utility/simulcast_utility.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +std::unique_ptr<Vp8FrameBufferController> Vp8TemporalLayersFactory::Create( + const VideoCodec& codec, + const VideoEncoder::Settings& settings, + FecControllerOverride* fec_controller_override) { + std::vector<std::unique_ptr<Vp8FrameBufferController>> controllers; + const int num_streams = SimulcastUtility::NumberOfSimulcastStreams(codec); + RTC_DCHECK_GE(num_streams, 1); + controllers.reserve(num_streams); + + for (int i = 0; i < num_streams; ++i) { + int num_temporal_layers = + SimulcastUtility::NumberOfTemporalLayers(codec, i); + RTC_DCHECK_GE(num_temporal_layers, 1); + if (SimulcastUtility::IsConferenceModeScreenshare(codec) && i == 0) { + // Legacy screenshare layers supports max 2 layers. + num_temporal_layers = std::max(2, num_temporal_layers); + controllers.push_back( + std::make_unique<ScreenshareLayers>(num_temporal_layers)); + } else { + controllers.push_back( + std::make_unique<DefaultTemporalLayers>(num_temporal_layers)); + } + } + + return std::make_unique<Vp8TemporalLayers>(std::move(controllers), + fec_controller_override); +} + +std::unique_ptr<Vp8FrameBufferControllerFactory> +Vp8TemporalLayersFactory::Clone() const { + return std::make_unique<Vp8TemporalLayersFactory>(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/vp8_temporal_layers_factory.h b/third_party/libwebrtc/api/video_codecs/vp8_temporal_layers_factory.h new file mode 100644 index 0000000000..7a146f1d4f --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/vp8_temporal_layers_factory.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VP8_TEMPORAL_LAYERS_FACTORY_H_ +#define API_VIDEO_CODECS_VP8_TEMPORAL_LAYERS_FACTORY_H_ + +#include <memory> + +#include "api/video_codecs/vp8_temporal_layers.h" + +namespace webrtc { + +class Vp8TemporalLayersFactory : public Vp8FrameBufferControllerFactory { + public: + ~Vp8TemporalLayersFactory() override = default; + + std::unique_ptr<Vp8FrameBufferControllerFactory> Clone() const override; + + std::unique_ptr<Vp8FrameBufferController> Create( + const VideoCodec& codec, + const VideoEncoder::Settings& settings, + FecControllerOverride* fec_controller_override) override; +}; + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VP8_TEMPORAL_LAYERS_FACTORY_H_ diff --git a/third_party/libwebrtc/api/video_codecs/vp8_temporal_layers_factory_gn/moz.build b/third_party/libwebrtc/api/video_codecs/vp8_temporal_layers_factory_gn/moz.build new file mode 100644 index 0000000000..2769bef0eb --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/vp8_temporal_layers_factory_gn/moz.build @@ -0,0 +1,232 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" +DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "!/third_party/libwebrtc/gen", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/api/video_codecs/vp8_temporal_layers_factory.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "mips32": + + DEFINES["MIPS32_LE"] = True + DEFINES["MIPS_FPU_LE"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "mips64": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["CPU_ARCH"] == "x86_64": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + CXXFLAGS += [ + "-msse2" + ] + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + CXXFLAGS += [ + "-msse2" + ] + + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_GNU_SOURCE"] = True + +Library("vp8_temporal_layers_factory_gn") diff --git a/third_party/libwebrtc/api/video_codecs/vp9_profile.cc b/third_party/libwebrtc/api/video_codecs/vp9_profile.cc new file mode 100644 index 0000000000..7e627cc080 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/vp9_profile.cc @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/vp9_profile.h" + +#include <map> +#include <utility> + +#include "rtc_base/string_to_number.h" + +namespace webrtc { + +// Profile information for VP9 video. +const char kVP9FmtpProfileId[] = "profile-id"; + +std::string VP9ProfileToString(VP9Profile profile) { + switch (profile) { + case VP9Profile::kProfile0: + return "0"; + case VP9Profile::kProfile1: + return "1"; + case VP9Profile::kProfile2: + return "2"; + case VP9Profile::kProfile3: + return "3"; + } + return "0"; +} + +absl::optional<VP9Profile> StringToVP9Profile(const std::string& str) { + const absl::optional<int> i = rtc::StringToNumber<int>(str); + if (!i.has_value()) + return absl::nullopt; + + switch (i.value()) { + case 0: + return VP9Profile::kProfile0; + case 1: + return VP9Profile::kProfile1; + case 2: + return VP9Profile::kProfile2; + case 3: + return VP9Profile::kProfile3; + default: + return absl::nullopt; + } +} + +absl::optional<VP9Profile> ParseSdpForVP9Profile( + const SdpVideoFormat::Parameters& params) { + const auto profile_it = params.find(kVP9FmtpProfileId); + if (profile_it == params.end()) + return VP9Profile::kProfile0; + const std::string& profile_str = profile_it->second; + return StringToVP9Profile(profile_str); +} + +bool VP9IsSameProfile(const SdpVideoFormat::Parameters& params1, + const SdpVideoFormat::Parameters& params2) { + const absl::optional<VP9Profile> profile = ParseSdpForVP9Profile(params1); + const absl::optional<VP9Profile> other_profile = + ParseSdpForVP9Profile(params2); + return profile && other_profile && profile == other_profile; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/api/video_codecs/vp9_profile.h b/third_party/libwebrtc/api/video_codecs/vp9_profile.h new file mode 100644 index 0000000000..b570bc3bb6 --- /dev/null +++ b/third_party/libwebrtc/api/video_codecs/vp9_profile.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_CODECS_VP9_PROFILE_H_ +#define API_VIDEO_CODECS_VP9_PROFILE_H_ + +#include <string> + +#include "absl/types/optional.h" +#include "api/video_codecs/sdp_video_format.h" +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// Profile information for VP9 video. +extern RTC_EXPORT const char kVP9FmtpProfileId[]; + +enum class VP9Profile { + kProfile0, + kProfile1, + kProfile2, + kProfile3, +}; + +// Helper functions to convert VP9Profile to std::string. Returns "0" by +// default. +RTC_EXPORT std::string VP9ProfileToString(VP9Profile profile); + +// Helper functions to convert std::string to VP9Profile. Returns null if given +// an invalid profile string. +absl::optional<VP9Profile> StringToVP9Profile(const std::string& str); + +// Parse profile that is represented as a string of single digit contained in an +// SDP key-value map. A default profile(kProfile0) will be returned if the +// profile key is missing. Nothing will be returned if the key is present but +// the string is invalid. +RTC_EXPORT absl::optional<VP9Profile> ParseSdpForVP9Profile( + const SdpVideoFormat::Parameters& params); + +// Returns true if the parameters have the same VP9 profile, or neither contains +// VP9 profile. +bool VP9IsSameProfile(const SdpVideoFormat::Parameters& params1, + const SdpVideoFormat::Parameters& params2); + +} // namespace webrtc + +#endif // API_VIDEO_CODECS_VP9_PROFILE_H_ |