From 8dd16259287f58f9273002717ec4d27e97127719 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 12 Jun 2024 07:43:14 +0200 Subject: Merging upstream version 127.0. Signed-off-by: Daniel Baumann --- .../video_coding/codecs/av1/libaom_av1_encoder.cc | 18 +++- .../codecs/av1/libaom_av1_encoder_unittest.cc | 25 +++++ .../video_coding/codecs/av1/libaom_av1_unittest.cc | 1 + .../multiplex/include/multiplex_decoder_adapter.h | 5 +- .../codecs/multiplex/multiplex_decoder_adapter.cc | 7 +- .../multiplex/test/multiplex_adapter_unittest.cc | 12 ++- .../video_coding/codecs/test/video_codec_test.cc | 20 +++- .../video_encoder_decoder_instantiation_tests.cc | 8 +- .../codecs/test/videocodec_test_fixture_impl.cc | 6 +- .../modules/video_coding/codecs/vp8/include/vp8.h | 5 + .../video_coding/codecs/vp8/libvpx_vp8_decoder.cc | 30 ++++-- .../video_coding/codecs/vp8/libvpx_vp8_decoder.h | 6 ++ .../codecs/vp8/libvpx_vp8_simulcast_test.cc | 4 +- .../codecs/vp8/test/vp8_impl_unittest.cc | 3 +- .../video_coding/codecs/vp9/libvpx_vp9_encoder.cc | 48 +++++++-- .../video_coding/codecs/vp9/libvpx_vp9_encoder.h | 8 ++ .../modules/video_coding/codecs/vp9/svc_config.cc | 3 + .../video_coding/codecs/vp9/svc_config_unittest.cc | 20 ++++ .../codecs/vp9/test/vp9_impl_unittest.cc | 109 +++++++++++++++++++++ 19 files changed, 306 insertions(+), 32 deletions(-) (limited to 'third_party/libwebrtc/modules/video_coding/codecs') diff --git a/third_party/libwebrtc/modules/video_coding/codecs/av1/libaom_av1_encoder.cc b/third_party/libwebrtc/modules/video_coding/codecs/av1/libaom_av1_encoder.cc index 4ff22bfe34..03bb367fe0 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/av1/libaom_av1_encoder.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/av1/libaom_av1_encoder.cc @@ -133,6 +133,7 @@ class LibaomAv1Encoder final : public VideoEncoder { // TODO(webrtc:15225): Kill switch for disabling frame dropping. Remove it // after frame dropping is fully rolled out. bool disable_frame_dropping_; + int max_consec_frame_drop_; }; int32_t VerifyCodecSettings(const VideoCodec& codec_settings) { @@ -163,6 +164,14 @@ int32_t VerifyCodecSettings(const VideoCodec& codec_settings) { return WEBRTC_VIDEO_CODEC_OK; } +int GetMaxConsecutiveFrameDrop(const FieldTrialsView& field_trials) { + webrtc::FieldTrialParameter maxdrop("maxdrop", 0); + webrtc::ParseFieldTrial( + {&maxdrop}, + field_trials.Lookup("WebRTC-LibaomAv1Encoder-MaxConsecFrameDrop")); + return maxdrop; +} + LibaomAv1Encoder::LibaomAv1Encoder( const absl::optional& aux_config, const FieldTrialsView& trials) @@ -174,7 +183,8 @@ LibaomAv1Encoder::LibaomAv1Encoder( timestamp_(0), disable_frame_dropping_(absl::StartsWith( trials.Lookup("WebRTC-LibaomAv1Encoder-DisableFrameDropping"), - "Enabled")) {} + "Enabled")), + max_consec_frame_drop_(GetMaxConsecutiveFrameDrop(trials)) {} LibaomAv1Encoder::~LibaomAv1Encoder() { Release(); @@ -297,6 +307,12 @@ int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings, SET_ENCODER_PARAM_OR_RETURN_ERROR(AV1E_SET_ENABLE_PALETTE, 0); } + if (codec_settings->mode == VideoCodecMode::kRealtimeVideo && + encoder_settings_.GetFrameDropEnabled() && max_consec_frame_drop_ > 0) { + SET_ENCODER_PARAM_OR_RETURN_ERROR(AV1E_SET_MAX_CONSEC_FRAME_DROP_CBR, + max_consec_frame_drop_); + } + if (cfg_.g_threads == 8) { // Values passed to AV1E_SET_TILE_ROWS and AV1E_SET_TILE_COLUMNS are log2() // based. diff --git a/third_party/libwebrtc/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc b/third_party/libwebrtc/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc index 04ee9162ba..127aadb275 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc @@ -188,6 +188,31 @@ TEST(LibaomAv1EncoderTest, CheckOddDimensionsWithSpatialLayers) { ASSERT_THAT(encoded_frames, SizeIs(6)); } +TEST(LibaomAv1EncoderTest, WithMaximumConsecutiveFrameDrop) { + test::ScopedFieldTrials field_trials( + "WebRTC-LibaomAv1Encoder-MaxConsecFrameDrop/maxdrop:2/"); + VideoBitrateAllocation allocation; + allocation.SetBitrate(0, 0, 1000); // some very low bitrate + std::unique_ptr encoder = CreateLibaomAv1Encoder(); + VideoCodec codec_settings = DefaultCodecSettings(); + codec_settings.SetFrameDropEnabled(true); + codec_settings.SetScalabilityMode(ScalabilityMode::kL1T1); + codec_settings.startBitrate = allocation.get_sum_kbps(); + ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()), + WEBRTC_VIDEO_CODEC_OK); + encoder->SetRates(VideoEncoder::RateControlParameters( + allocation, codec_settings.maxFramerate)); + EncodedVideoFrameProducer evfp(*encoder); + evfp.SetResolution( + RenderResolution{codec_settings.width, codec_settings.height}); + // We should code the first frame, skip two, then code another frame. + std::vector encoded_frames = + evfp.SetNumInputFrames(4).Encode(); + ASSERT_THAT(encoded_frames, SizeIs(2)); + // The 4 frames have default Rtp-timestamps of 1000, 4000, 7000, 10000. + ASSERT_THAT(encoded_frames[1].encoded_image.RtpTimestamp(), 10000); +} + TEST(LibaomAv1EncoderTest, EncoderInfoWithoutResolutionBitrateLimits) { std::unique_ptr encoder = CreateLibaomAv1Encoder(); EXPECT_TRUE(encoder->GetEncoderInfo().resolution_bitrate_limits.empty()); diff --git a/third_party/libwebrtc/modules/video_coding/codecs/av1/libaom_av1_unittest.cc b/third_party/libwebrtc/modules/video_coding/codecs/av1/libaom_av1_unittest.cc index d486c1d062..6a135e2bab 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/av1/libaom_av1_unittest.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/av1/libaom_av1_unittest.cc @@ -62,6 +62,7 @@ VideoCodec DefaultCodecSettings() { codec_settings.height = kHeight; codec_settings.maxFramerate = kFramerate; codec_settings.maxBitrate = 1000; + codec_settings.startBitrate = 1; codec_settings.qpMax = 63; return codec_settings; } diff --git a/third_party/libwebrtc/modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h b/third_party/libwebrtc/modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h index d58981e4b2..ed02f2d72b 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h +++ b/third_party/libwebrtc/modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h @@ -15,6 +15,7 @@ #include #include +#include "api/environment/environment.h" #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_decoder.h" #include "api/video_codecs/video_decoder_factory.h" @@ -25,7 +26,8 @@ namespace webrtc { class MultiplexDecoderAdapter : public VideoDecoder { public: // `factory` is not owned and expected to outlive this class. - MultiplexDecoderAdapter(VideoDecoderFactory* factory, + MultiplexDecoderAdapter(const Environment& env, + VideoDecoderFactory* factory, const SdpVideoFormat& associated_format, bool supports_augmenting_data = false); virtual ~MultiplexDecoderAdapter(); @@ -62,6 +64,7 @@ class MultiplexDecoderAdapter : public VideoDecoder { std::unique_ptr augmenting_data, uint16_t augmenting_data_length); + const Environment env_; VideoDecoderFactory* const factory_; const SdpVideoFormat associated_format_; std::vector> decoders_; diff --git a/third_party/libwebrtc/modules/video_coding/codecs/multiplex/multiplex_decoder_adapter.cc b/third_party/libwebrtc/modules/video_coding/codecs/multiplex/multiplex_decoder_adapter.cc index 551a9490b0..7cebbe14d0 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/multiplex/multiplex_decoder_adapter.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/multiplex/multiplex_decoder_adapter.cc @@ -10,6 +10,7 @@ #include "modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h" +#include "api/environment/environment.h" #include "api/video/encoded_image.h" #include "api/video/i420_buffer.h" #include "api/video/video_frame_buffer.h" @@ -93,10 +94,12 @@ struct MultiplexDecoderAdapter::AugmentingData { }; MultiplexDecoderAdapter::MultiplexDecoderAdapter( + const Environment& env, VideoDecoderFactory* factory, const SdpVideoFormat& associated_format, bool supports_augmenting_data) - : factory_(factory), + : env_(env), + factory_(factory), associated_format_(associated_format), supports_augmenting_data_(supports_augmenting_data) {} @@ -111,7 +114,7 @@ bool MultiplexDecoderAdapter::Configure(const Settings& settings) { PayloadStringToCodecType(associated_format_.name)); for (size_t i = 0; i < kAlphaCodecStreams; ++i) { std::unique_ptr decoder = - factory_->CreateVideoDecoder(associated_format_); + factory_->Create(env_, associated_format_); if (!decoder->Configure(associated_settings)) { return false; } diff --git a/third_party/libwebrtc/modules/video_coding/codecs/multiplex/test/multiplex_adapter_unittest.cc b/third_party/libwebrtc/modules/video_coding/codecs/multiplex/test/multiplex_adapter_unittest.cc index a2f36a306d..9c6300e368 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/multiplex/test/multiplex_adapter_unittest.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/multiplex/test/multiplex_adapter_unittest.cc @@ -16,6 +16,8 @@ #include #include "absl/types/optional.h" +#include "api/environment/environment.h" +#include "api/environment/environment_factory.h" #include "api/scoped_refptr.h" #include "api/test/mock_video_decoder_factory.h" #include "api/test/mock_video_encoder_factory.h" @@ -63,7 +65,8 @@ class TestMultiplexAdapter : public VideoCodecUnitTest, protected: std::unique_ptr CreateDecoder() override { return std::make_unique( - decoder_factory_.get(), SdpVideoFormat(kMultiplexAssociatedCodecName), + env_, decoder_factory_.get(), + SdpVideoFormat(kMultiplexAssociatedCodecName), supports_augmenting_data_); } @@ -182,9 +185,9 @@ class TestMultiplexAdapter : public VideoCodecUnitTest, EXPECT_CALL(*decoder_factory_, Die); // The decoders/encoders will be owned by the caller of // CreateVideoDecoder()/CreateVideoEncoder(). - EXPECT_CALL(*decoder_factory_, CreateVideoDecoder) - .Times(2) - .WillRepeatedly([] { return VP9Decoder::Create(); }); + EXPECT_CALL(*decoder_factory_, Create).Times(2).WillRepeatedly([] { + return VP9Decoder::Create(); + }); EXPECT_CALL(*encoder_factory_, Die); EXPECT_CALL(*encoder_factory_, CreateVideoEncoder) @@ -194,6 +197,7 @@ class TestMultiplexAdapter : public VideoCodecUnitTest, VideoCodecUnitTest::SetUp(); } + const Environment env_ = CreateEnvironment(); const std::unique_ptr decoder_factory_; const std::unique_ptr encoder_factory_; const bool supports_augmenting_data_; diff --git a/third_party/libwebrtc/modules/video_coding/codecs/test/video_codec_test.cc b/third_party/libwebrtc/modules/video_coding/codecs/test/video_codec_test.cc index 2ab1106a59..0811685e33 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/test/video_codec_test.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/test/video_codec_test.cc @@ -14,6 +14,8 @@ #include "absl/flags/flag.h" #include "absl/functional/any_invocable.h" +#include "api/environment/environment.h" +#include "api/environment/environment_factory.h" #include "api/test/metrics/global_metrics_logger_and_exporter.h" #include "api/units/data_rate.h" #include "api/units/frequency.h" @@ -26,6 +28,8 @@ #include "modules/video_coding/svc/scalability_mode_util.h" #include "rtc_base/logging.h" #include "rtc_base/strings/string_builder.h" +#include "test/explicit_key_value_config.h" +#include "test/field_trial.h" #include "test/gtest.h" #include "test/test_flags.h" #include "test/testsupport/file_utils.h" @@ -58,6 +62,7 @@ ABSL_FLAG(double, 30.0, "Encode target frame rate of the top temporal layer in fps."); ABSL_FLAG(int, num_frames, 300, "Number of frames to encode and/or decode."); +ABSL_FLAG(std::string, field_trials, "", "Field trials to apply."); ABSL_FLAG(std::string, test_name, "", "Test name."); ABSL_FLAG(bool, dump_decoder_input, false, "Dump decoder input."); ABSL_FLAG(bool, dump_decoder_output, false, "Dump decoder output."); @@ -178,6 +183,7 @@ std::string TestOutputPath() { } // namespace std::unique_ptr RunEncodeDecodeTest( + const Environment& env, std::string encoder_impl, std::string decoder_impl, const VideoInfo& video_info, @@ -247,7 +253,7 @@ std::unique_ptr RunEncodeDecodeTest( } return VideoCodecTester::RunEncodeDecodeTest( - source_settings, encoder_factory.get(), decoder_factory.get(), + env, source_settings, encoder_factory.get(), decoder_factory.get(), encoder_settings, decoder_settings, encoding_settings); } @@ -313,6 +319,7 @@ class SpatialQualityTest : public ::testing::TestWithParam stats = - RunEncodeDecodeTest(codec_impl, codec_impl, video_info, frames_settings); + std::unique_ptr stats = RunEncodeDecodeTest( + env, codec_impl, codec_impl, video_info, frames_settings); VideoCodecStats::Stream stream; if (stats != nullptr) { @@ -527,6 +534,11 @@ INSTANTIATE_TEST_SUITE_P( FramerateAdaptationTest::TestParamsToString); TEST(VideoCodecTest, DISABLED_EncodeDecode) { + ScopedFieldTrials field_trials(absl::GetFlag(FLAGS_field_trials)); + const Environment env = + CreateEnvironment(std::make_unique( + absl::GetFlag(FLAGS_field_trials))); + std::vector bitrate_str = absl::GetFlag(FLAGS_bitrate_kbps); std::vector bitrate_kbps; std::transform(bitrate_str.begin(), bitrate_str.end(), @@ -544,7 +556,7 @@ TEST(VideoCodecTest, DISABLED_EncodeDecode) { // logged test name (implies lossing history in the chromeperf dashboard). // Sync with changes in Stream::LogMetrics (see TODOs there). std::unique_ptr stats = RunEncodeDecodeTest( - CodecNameToCodecImpl(absl::GetFlag(FLAGS_encoder)), + env, CodecNameToCodecImpl(absl::GetFlag(FLAGS_encoder)), CodecNameToCodecImpl(absl::GetFlag(FLAGS_decoder)), kRawVideos.at(absl::GetFlag(FLAGS_video_name)), frames_settings); ASSERT_NE(nullptr, stats); diff --git a/third_party/libwebrtc/modules/video_coding/codecs/test/video_encoder_decoder_instantiation_tests.cc b/third_party/libwebrtc/modules/video_coding/codecs/test/video_encoder_decoder_instantiation_tests.cc index 41f2304748..581750768d 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/test/video_encoder_decoder_instantiation_tests.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/test/video_encoder_decoder_instantiation_tests.cc @@ -11,6 +11,8 @@ #include #include +#include "api/environment/environment.h" +#include "api/environment/environment_factory.h" #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_decoder.h" #include "api/video_codecs/video_decoder_factory.h" @@ -86,6 +88,8 @@ class VideoEncoderDecoderInstantiationTest } } + const Environment env_ = CreateEnvironment(); + const SdpVideoFormat vp8_format_; const SdpVideoFormat vp9_format_; const SdpVideoFormat h264cbp_format_; @@ -126,7 +130,7 @@ TEST_P(VideoEncoderDecoderInstantiationTest, DISABLED_InstantiateVp8Codecs) { for (int i = 0; i < num_decoders_; ++i) { std::unique_ptr decoder = - decoder_factory_->CreateVideoDecoder(vp8_format_); + decoder_factory_->Create(env_, vp8_format_); ASSERT_THAT(decoder, NotNull()); EXPECT_TRUE(decoder->Configure(DecoderSettings(kVideoCodecVP8))); decoders_.emplace_back(std::move(decoder)); @@ -144,7 +148,7 @@ TEST_P(VideoEncoderDecoderInstantiationTest, for (int i = 0; i < num_decoders_; ++i) { std::unique_ptr decoder = - decoder_factory_->CreateVideoDecoder(h264cbp_format_); + decoder_factory_->Create(env_, h264cbp_format_); ASSERT_THAT(decoder, NotNull()); EXPECT_TRUE(decoder->Configure(DecoderSettings(kVideoCodecH264))); decoders_.push_back(std::move(decoder)); diff --git a/third_party/libwebrtc/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc b/third_party/libwebrtc/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc index 35355d4387..508ac384b0 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc @@ -24,6 +24,8 @@ #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "api/array_view.h" +#include "api/environment/environment.h" +#include "api/environment/environment_factory.h" #include "api/test/metrics/global_metrics_logger_and_exporter.h" #include "api/test/metrics/metric.h" #include "api/transport/field_trial_based_config.h" @@ -685,6 +687,8 @@ void VideoCodecTestFixtureImpl::VerifyVideoStatistic( } bool VideoCodecTestFixtureImpl::CreateEncoderAndDecoder() { + const Environment env = CreateEnvironment(); + SdpVideoFormat encoder_format(CreateSdpVideoFormat(config_)); SdpVideoFormat decoder_format = encoder_format; @@ -709,7 +713,7 @@ bool VideoCodecTestFixtureImpl::CreateEncoderAndDecoder() { config_.NumberOfSimulcastStreams(), config_.NumberOfSpatialLayers()); for (size_t i = 0; i < num_simulcast_or_spatial_layers; ++i) { std::unique_ptr decoder = - decoder_factory_->CreateVideoDecoder(decoder_format); + decoder_factory_->Create(env, decoder_format); EXPECT_TRUE(decoder) << "Decoder not successfully created."; if (decoder == nullptr) { return false; diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/vp8.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/vp8.h index 2fc647874f..45b7cee00a 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/vp8.h +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/include/vp8.h @@ -14,6 +14,7 @@ #include #include +#include "api/environment/environment.h" #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/vp8_frame_buffer_controller.h" #include "modules/video_coding/include/video_codec_interface.h" @@ -40,11 +41,15 @@ class VP8Encoder { static std::unique_ptr Create(Settings settings); }; +// TODO: bugs.webrtc.org/15791 - Deprecate and delete in favor of the +// CreateVp8Decoder function. class VP8Decoder { public: static std::unique_ptr Create(); }; +std::unique_ptr CreateVp8Decoder(const Environment& env); + } // namespace webrtc #endif // MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_H_ diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc index 9b77388f10..4c06aca5ad 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc @@ -18,7 +18,10 @@ #include #include "absl/types/optional.h" +#include "api/environment/environment.h" +#include "api/field_trials_view.h" #include "api/scoped_refptr.h" +#include "api/transport/field_trial_based_config.h" #include "api/video/i420_buffer.h" #include "api/video/video_frame.h" #include "api/video/video_frame_buffer.h" @@ -28,7 +31,6 @@ #include "rtc_base/checks.h" #include "rtc_base/numerics/exp_filter.h" #include "rtc_base/time_utils.h" -#include "system_wrappers/include/field_trial.h" #include "system_wrappers/include/metrics.h" #include "third_party/libyuv/include/libyuv/convert.h" #include "vpx/vp8.h" @@ -59,9 +61,9 @@ absl::optional DefaultDeblockParams() { } absl::optional -GetPostProcParamsFromFieldTrialGroup() { - std::string group = webrtc::field_trial::FindFullName( - kIsArm ? kVp8PostProcArmFieldTrial : kVp8PostProcFieldTrial); +GetPostProcParamsFromFieldTrialGroup(const FieldTrialsView& field_trials) { + std::string group = field_trials.Lookup(kIsArm ? kVp8PostProcArmFieldTrial + : kVp8PostProcFieldTrial); if (group.empty()) { return DefaultDeblockParams(); } @@ -89,6 +91,10 @@ std::unique_ptr VP8Decoder::Create() { return std::make_unique(); } +std::unique_ptr CreateVp8Decoder(const Environment& env) { + return std::make_unique(env); +} + class LibvpxVp8Decoder::QpSmoother { public: QpSmoother() : last_sample_ms_(rtc::TimeMillis()), smoother_(kAlpha) {} @@ -114,9 +120,14 @@ class LibvpxVp8Decoder::QpSmoother { }; LibvpxVp8Decoder::LibvpxVp8Decoder() - : use_postproc_( - kIsArm ? webrtc::field_trial::IsEnabled(kVp8PostProcArmFieldTrial) - : true), + : LibvpxVp8Decoder(FieldTrialBasedConfig()) {} + +LibvpxVp8Decoder::LibvpxVp8Decoder(const Environment& env) + : LibvpxVp8Decoder(env.field_trials()) {} + +LibvpxVp8Decoder::LibvpxVp8Decoder(const FieldTrialsView& field_trials) + : use_postproc_(kIsArm ? field_trials.IsEnabled(kVp8PostProcArmFieldTrial) + : true), buffer_pool_(false, 300 /* max_number_of_buffers*/), decode_complete_callback_(NULL), inited_(false), @@ -124,8 +135,9 @@ LibvpxVp8Decoder::LibvpxVp8Decoder() last_frame_width_(0), last_frame_height_(0), key_frame_required_(true), - deblock_params_(use_postproc_ ? GetPostProcParamsFromFieldTrialGroup() - : absl::nullopt), + deblock_params_(use_postproc_ + ? GetPostProcParamsFromFieldTrialGroup(field_trials) + : absl::nullopt), qp_smoother_(use_postproc_ ? new QpSmoother() : nullptr) {} LibvpxVp8Decoder::~LibvpxVp8Decoder() { diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h index 74f4dc7c89..8ed8e7ca88 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h @@ -14,6 +14,8 @@ #include #include "absl/types/optional.h" +#include "api/environment/environment.h" +#include "api/field_trials_view.h" #include "api/video/encoded_image.h" #include "api/video_codecs/video_decoder.h" #include "common_video/include/video_frame_buffer_pool.h" @@ -26,7 +28,10 @@ namespace webrtc { class LibvpxVp8Decoder : public VideoDecoder { public: + // TODO: bugs.webrtc.org/15791 - Delete default constructor when + // Environment is always propagated. LibvpxVp8Decoder(); + explicit LibvpxVp8Decoder(const Environment& env); ~LibvpxVp8Decoder() override; bool Configure(const Settings& settings) override; @@ -56,6 +61,7 @@ class LibvpxVp8Decoder : public VideoDecoder { private: class QpSmoother; + explicit LibvpxVp8Decoder(const FieldTrialsView& field_trials); int ReturnFrame(const vpx_image_t* img, uint32_t timeStamp, int qp, diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_simulcast_test.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_simulcast_test.cc index 4ca3de20d5..3f13066892 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_simulcast_test.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/libvpx_vp8_simulcast_test.cc @@ -27,7 +27,9 @@ std::unique_ptr CreateSpecificSimulcastTestFixture() { []() { return VP8Encoder::Create(); }); std::unique_ptr decoder_factory = std::make_unique( - []() { return VP8Decoder::Create(); }); + [](const Environment& env, const SdpVideoFormat& format) { + return CreateVp8Decoder(env); + }); return CreateSimulcastTestFixture(std::move(encoder_factory), std::move(decoder_factory), SdpVideoFormat("VP8")); diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc index a6f570f855..514d3d7e1d 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc @@ -13,6 +13,7 @@ #include #include +#include "api/environment/environment_factory.h" #include "api/test/create_frame_generator.h" #include "api/test/frame_generator_interface.h" #include "api/test/mock_video_decoder.h" @@ -70,7 +71,7 @@ class TestVp8Impl : public VideoCodecUnitTest { } std::unique_ptr CreateDecoder() override { - return VP8Decoder::Create(); + return CreateVp8Decoder(CreateEnvironment()); } void ModifyCodecSettings(VideoCodec* codec_settings) override { diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc index 5330eb7e8c..edbe781639 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.cc @@ -267,7 +267,8 @@ LibvpxVp9Encoder::LibvpxVp9Encoder(const cricket::VideoCodec& codec, "Disabled")), performance_flags_(ParsePerformanceFlagsFromTrials(trials)), num_steady_state_frames_(0), - config_changed_(true) { + config_changed_(true), + svc_frame_drop_config_(ParseSvcFrameDropConfig(trials)) { codec_ = {}; memset(&svc_params_, 0, sizeof(vpx_svc_extra_cfg_t)); } @@ -838,6 +839,8 @@ int LibvpxVp9Encoder::InitAndSetControlSettings(const VideoCodec* inst) { // 1:2 scaling in each dimension. svc_params_.scaling_factor_num[i] = scaling_factor_num; svc_params_.scaling_factor_den[i] = 256; + if (inst->mode != VideoCodecMode::kScreensharing) + scaling_factor_num /= 2; } } @@ -924,11 +927,24 @@ int LibvpxVp9Encoder::InitAndSetControlSettings(const VideoCodec* inst) { svc_drop_frame_.framedrop_thresh[i] = config_->rc_dropframe_thresh; } } else { - // Configure encoder to drop entire superframe whenever it needs to drop - // a layer. This mode is preferred over per-layer dropping which causes - // quality flickering and is not compatible with RTP non-flexible mode. - svc_drop_frame_.framedrop_mode = FULL_SUPERFRAME_DROP; - svc_drop_frame_.max_consec_drop = std::numeric_limits::max(); + if (svc_frame_drop_config_.enabled && + svc_frame_drop_config_.layer_drop_mode == LAYER_DROP && + is_flexible_mode_ && svc_controller_ && + (inter_layer_pred_ == InterLayerPredMode::kOff || + inter_layer_pred_ == InterLayerPredMode::kOnKeyPic)) { + // SVC controller is required since it properly accounts for dropped + // refs (unlike SetReferences(), which assumes full superframe drop). + svc_drop_frame_.framedrop_mode = LAYER_DROP; + } else { + // Configure encoder to drop entire superframe whenever it needs to drop + // a layer. This mode is preferred over per-layer dropping which causes + // quality flickering and is not compatible with RTP non-flexible mode. + svc_drop_frame_.framedrop_mode = FULL_SUPERFRAME_DROP; + } + svc_drop_frame_.max_consec_drop = + svc_frame_drop_config_.enabled + ? svc_frame_drop_config_.max_consec_drop + : std::numeric_limits::max(); for (size_t i = 0; i < num_spatial_layers_; ++i) { svc_drop_frame_.framedrop_thresh[i] = config_->rc_dropframe_thresh; } @@ -1960,6 +1976,26 @@ LibvpxVp9Encoder::ParseQualityScalerConfig(const FieldTrialsView& trials) { return config; } +LibvpxVp9Encoder::SvcFrameDropConfig LibvpxVp9Encoder::ParseSvcFrameDropConfig( + const FieldTrialsView& trials) { + FieldTrialFlag enabled = FieldTrialFlag("Enabled"); + FieldTrialParameter layer_drop_mode("layer_drop_mode", + FULL_SUPERFRAME_DROP); + FieldTrialParameter max_consec_drop("max_consec_drop", + std::numeric_limits::max()); + ParseFieldTrial({&enabled, &layer_drop_mode, &max_consec_drop}, + trials.Lookup("WebRTC-LibvpxVp9Encoder-SvcFrameDropConfig")); + SvcFrameDropConfig config; + config.enabled = enabled.Get(); + config.layer_drop_mode = layer_drop_mode.Get(); + config.max_consec_drop = max_consec_drop.Get(); + RTC_LOG(LS_INFO) << "Libvpx VP9 encoder SVC frame drop config: " + << (config.enabled ? "enabled" : "disabled") + << " layer_drop_mode " << config.layer_drop_mode + << " max_consec_drop " << config.max_consec_drop; + return config; +} + void LibvpxVp9Encoder::UpdatePerformanceFlags() { flat_map params_by_resolution; if (codec_.GetVideoEncoderComplexity() == diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.h b/third_party/libwebrtc/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.h index 1953923f81..ea4e5810ac 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.h +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp9/libvpx_vp9_encoder.h @@ -240,6 +240,14 @@ class LibvpxVp9Encoder : public VP9Encoder { bool config_changed_; const LibvpxVp9EncoderInfoSettings encoder_info_override_; + + const struct SvcFrameDropConfig { + bool enabled; + int layer_drop_mode; // SVC_LAYER_DROP_MODE + int max_consec_drop; + } svc_frame_drop_config_; + static SvcFrameDropConfig ParseSvcFrameDropConfig( + const FieldTrialsView& trials); }; } // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp9/svc_config.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp9/svc_config.cc index 7af8cab3cb..555af835a5 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/vp9/svc_config.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp9/svc_config.cc @@ -190,6 +190,9 @@ std::vector GetVp9SvcConfig(VideoCodec& codec) { codec.SetScalabilityMode(limited_scalability_mode); } + codec.VP9()->interLayerPred = + ScalabilityModeToInterLayerPredMode(*scalability_mode); + absl::optional info = ScalabilityStructureConfig(*scalability_mode); if (!info.has_value()) { diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp9/svc_config_unittest.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp9/svc_config_unittest.cc index 1b1abe0f6d..2515b1ce4b 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/vp9/svc_config_unittest.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp9/svc_config_unittest.cc @@ -13,6 +13,7 @@ #include #include +#include "api/video_codecs/video_encoder.h" #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" #include "test/gmock.h" #include "test/gtest.h" @@ -65,6 +66,25 @@ TEST(SvcConfig, NumSpatialLayersWithScalabilityMode) { EXPECT_EQ(codec.GetScalabilityMode(), ScalabilityMode::kL3T3_KEY); } +TEST(SvcConfig, UpdatesInterLayerPredModeBasedOnScalabilityMode) { + VideoCodec codec; + codec.codecType = kVideoCodecVP9; + codec.width = 1280; + codec.height = 720; + codec.SetScalabilityMode(ScalabilityMode::kL3T3_KEY); + + std::vector spatial_layers = GetVp9SvcConfig(codec); + EXPECT_EQ(codec.VP9()->interLayerPred, InterLayerPredMode::kOnKeyPic); + + codec.SetScalabilityMode(ScalabilityMode::kL3T3); + spatial_layers = GetVp9SvcConfig(codec); + EXPECT_EQ(codec.VP9()->interLayerPred, InterLayerPredMode::kOn); + + codec.SetScalabilityMode(ScalabilityMode::kS3T3); + spatial_layers = GetVp9SvcConfig(codec); + EXPECT_EQ(codec.VP9()->interLayerPred, InterLayerPredMode::kOff); +} + TEST(SvcConfig, NumSpatialLayersLimitedWithScalabilityMode) { VideoCodec codec; codec.codecType = kVideoCodecVP9; diff --git a/third_party/libwebrtc/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc b/third_party/libwebrtc/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc index 993fd245ad..50e9cf2369 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc @@ -2459,4 +2459,113 @@ TEST(Vp9SpeedSettingsTrialsTest, DefaultPerLayerFlagsWithSvc) { } } +struct SvcFrameDropConfigTestParameters { + bool flexible_mode; + absl::optional scalability_mode; + std::string field_trial; + int expected_framedrop_mode; + int expected_max_consec_drop; +}; + +class TestVp9ImplSvcFrameDropConfig + : public ::testing::TestWithParam {}; + +TEST_P(TestVp9ImplSvcFrameDropConfig, SvcFrameDropConfig) { + SvcFrameDropConfigTestParameters test_params = GetParam(); + auto* const vpx = new NiceMock(); + LibvpxVp9Encoder encoder( + cricket::CreateVideoCodec(cricket::kVp9CodecName), + absl::WrapUnique(vpx), + test::ExplicitKeyValueConfig(test_params.field_trial)); + + vpx_image_t img; + ON_CALL(*vpx, img_wrap).WillByDefault(GetWrapImageFunction(&img)); + + EXPECT_CALL(*vpx, + codec_control(_, VP9E_SET_SVC_FRAME_DROP_LAYER, + SafeMatcherCast(AllOf( + Field(&vpx_svc_frame_drop_t::framedrop_mode, + test_params.expected_framedrop_mode), + Field(&vpx_svc_frame_drop_t::max_consec_drop, + test_params.expected_max_consec_drop))))); + + VideoCodec settings = DefaultCodecSettings(); + settings.VP9()->flexibleMode = test_params.flexible_mode; + if (test_params.scalability_mode.has_value()) { + settings.SetScalabilityMode(*test_params.scalability_mode); + } + settings.VP9()->numberOfSpatialLayers = + 3; // to execute SVC code paths even when scalability_mode is not set. + + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder.InitEncode(&settings, kSettings)); +} + +INSTANTIATE_TEST_SUITE_P( + All, + TestVp9ImplSvcFrameDropConfig, + ::testing::Values( + // Flexible mode is disabled. Layer drop is not allowed. Ignore + // layer_drop_mode from field trial. + SvcFrameDropConfigTestParameters{ + .flexible_mode = false, + .scalability_mode = ScalabilityMode::kL3T3_KEY, + .field_trial = "WebRTC-LibvpxVp9Encoder-SvcFrameDropConfig/" + "Enabled,layer_drop_mode:1,max_consec_drop:7/", + .expected_framedrop_mode = FULL_SUPERFRAME_DROP, + .expected_max_consec_drop = 7}, + // Flexible mode is enabled but the field trial is not set. Use default + // settings. + SvcFrameDropConfigTestParameters{ + .flexible_mode = true, + .scalability_mode = ScalabilityMode::kL3T3_KEY, + .field_trial = "", + .expected_framedrop_mode = FULL_SUPERFRAME_DROP, + .expected_max_consec_drop = std::numeric_limits::max()}, + // Flexible mode is enabled but the field trial is disabled. Use default + // settings. + SvcFrameDropConfigTestParameters{ + .flexible_mode = true, + .scalability_mode = ScalabilityMode::kL3T3_KEY, + .field_trial = "WebRTC-LibvpxVp9Encoder-SvcFrameDropConfig/" + "Disabled,layer_drop_mode:1,max_consec_drop:7/", + .expected_framedrop_mode = FULL_SUPERFRAME_DROP, + .expected_max_consec_drop = std::numeric_limits::max()}, + // Flexible mode is enabled, layer drop is enabled, KSVC. Apply config + // from field trial. + SvcFrameDropConfigTestParameters{ + .flexible_mode = true, + .scalability_mode = ScalabilityMode::kL3T3_KEY, + .field_trial = "WebRTC-LibvpxVp9Encoder-SvcFrameDropConfig/" + "Enabled,layer_drop_mode:1,max_consec_drop:7/", + .expected_framedrop_mode = LAYER_DROP, + .expected_max_consec_drop = 7}, + // Flexible mode is enabled, layer drop is enabled, simulcast. Apply + // config from field trial. + SvcFrameDropConfigTestParameters{ + .flexible_mode = true, + .scalability_mode = ScalabilityMode::kS3T3, + .field_trial = "WebRTC-LibvpxVp9Encoder-SvcFrameDropConfig/" + "Enabled,layer_drop_mode:1,max_consec_drop:7/", + .expected_framedrop_mode = LAYER_DROP, + .expected_max_consec_drop = 7}, + // Flexible mode is enabled, layer drop is enabled, full SVC. Apply + // config from field trial. + SvcFrameDropConfigTestParameters{ + .flexible_mode = false, + .scalability_mode = ScalabilityMode::kL3T3, + .field_trial = "WebRTC-LibvpxVp9Encoder-SvcFrameDropConfig/" + "Enabled,layer_drop_mode:1,max_consec_drop:7/", + .expected_framedrop_mode = FULL_SUPERFRAME_DROP, + .expected_max_consec_drop = 7}, + // Flexible mode is enabled, layer-drop is enabled, scalability mode is + // not set (i.e., SVC controller is not enabled). Ignore layer_drop_mode + // from field trial. + SvcFrameDropConfigTestParameters{ + .flexible_mode = true, + .scalability_mode = absl::nullopt, + .field_trial = "WebRTC-LibvpxVp9Encoder-SvcFrameDropConfig/" + "Enabled,layer_drop_mode:1,max_consec_drop:7/", + .expected_framedrop_mode = FULL_SUPERFRAME_DROP, + .expected_max_consec_drop = 7})); + } // namespace webrtc -- cgit v1.2.3