diff options
Diffstat (limited to 'third_party/libwebrtc/test')
28 files changed, 2312 insertions, 234 deletions
diff --git a/third_party/libwebrtc/test/BUILD.gn b/third_party/libwebrtc/test/BUILD.gn index f7980f941d..854530c01e 100644 --- a/third_party/libwebrtc/test/BUILD.gn +++ b/third_party/libwebrtc/test/BUILD.gn @@ -724,6 +724,7 @@ if (rtc_include_tests) { ":test_main", ":test_support", ":test_support_test_artifacts", + ":video_codec_tester", ":video_test_common", ":video_test_support", ":y4m_frame_generator", @@ -731,11 +732,15 @@ if (rtc_include_tests) { "../api:create_frame_generator", "../api:create_simulcast_test_fixture_api", "../api:frame_generator_api", + "../api:mock_video_codec_factory", + "../api:mock_video_decoder", + "../api:mock_video_encoder", "../api:scoped_refptr", "../api:simulcast_test_fixture_api", "../api/task_queue:task_queue_test", "../api/test/video:function_video_factory", "../api/test/video:video_frame_writer", + "../api/units:data_rate", "../api/units:time_delta", "../api/video:encoded_image", "../api/video:video_frame", @@ -752,6 +757,7 @@ if (rtc_include_tests) { "../modules/video_coding:webrtc_h264", "../modules/video_coding:webrtc_vp8", "../modules/video_coding:webrtc_vp9", + "../modules/video_coding/svc:scalability_mode_util", "../rtc_base:criticalsection", "../rtc_base:rtc_event", "../rtc_base:rtc_task_queue", @@ -765,6 +771,7 @@ if (rtc_include_tests) { "scenario:scenario_unittests", "time_controller:time_controller", "time_controller:time_controller_unittests", + "//third_party/libyuv", ] absl_deps = [ "//third_party/abseil-cpp/absl/flags:flag", @@ -789,6 +796,7 @@ if (rtc_include_tests) { "testsupport/y4m_frame_writer_unittest.cc", "testsupport/yuv_frame_reader_unittest.cc", "testsupport/yuv_frame_writer_unittest.cc", + "video_codec_tester_unittest.cc", ] if (rtc_enable_protobuf) { @@ -1287,9 +1295,10 @@ if (!build_with_chromium) { "../api:transport_api", "../api/audio_codecs:builtin_audio_decoder_factory", "../api/audio_codecs:builtin_audio_encoder_factory", + "../api/environment", + "../api/environment:environment_factory", "../api/rtc_event_log", "../api/task_queue", - "../api/task_queue:default_task_queue_factory", "../api/test/video:function_video_factory", "../api/transport:field_trial_based_config", "../api/units:time_delta", @@ -1378,3 +1387,47 @@ rtc_library("fake_encoded_frame") { ] absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } + +rtc_library("video_codec_tester") { + testonly = true + sources = [ + "video_codec_tester.cc", + "video_codec_tester.h", + ] + deps = [ + "../api:array_view", + "../api/numerics:numerics", + "../api/test/metrics:metric", + "../api/test/metrics:metrics_logger", + "../api/units:data_rate", + "../api/units:data_size", + "../api/units:frequency", + "../api/units:time_delta", + "../api/units:timestamp", + "../api/video:builtin_video_bitrate_allocator_factory", + "../api/video:encoded_image", + "../api/video:resolution", + "../api/video:video_bitrate_allocator", + "../api/video:video_frame", + "../api/video_codecs:video_codecs_api", + "../media:media_constants", + "../modules/video_coding:video_codec_interface", + "../modules/video_coding:video_coding_utility", + "../modules/video_coding:webrtc_vp9_helpers", + "../modules/video_coding/codecs/av1:av1_svc_config", + "../modules/video_coding/svc:scalability_mode_util", + "../rtc_base:checks", + "../rtc_base:logging", + "../rtc_base:rtc_event", + "../rtc_base:stringutils", + "../rtc_base:task_queue_for_test", + "../rtc_base:timeutils", + "../rtc_base/synchronization:mutex", + "../system_wrappers", + "../test:fileutils", + "../test:video_test_support", + "//third_party/libyuv", + ] + + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} diff --git a/third_party/libwebrtc/test/call_test.cc b/third_party/libwebrtc/test/call_test.cc index 2b71608ac6..09099cccd6 100644 --- a/third_party/libwebrtc/test/call_test.cc +++ b/third_party/libwebrtc/test/call_test.cc @@ -15,7 +15,8 @@ #include "api/audio_codecs/builtin_audio_decoder_factory.h" #include "api/audio_codecs/builtin_audio_encoder_factory.h" -#include "api/task_queue/default_task_queue_factory.h" +#include "api/environment/environment.h" +#include "api/environment/environment_factory.h" #include "api/task_queue/task_queue_base.h" #include "api/test/create_frame_generator.h" #include "api/video/builtin_video_bitrate_allocator_factory.h" @@ -38,19 +39,18 @@ namespace webrtc { namespace test { CallTest::CallTest() - : clock_(Clock::GetRealTimeClock()), - task_queue_factory_(CreateDefaultTaskQueueFactory()), - send_event_log_(std::make_unique<RtcEventLogNull>()), - recv_event_log_(std::make_unique<RtcEventLogNull>()), + : env_(CreateEnvironment(&field_trials_)), + send_env_(env_), + recv_env_(env_), audio_send_config_(/*send_transport=*/nullptr), audio_send_stream_(nullptr), frame_generator_capturer_(nullptr), fake_encoder_factory_([this]() { std::unique_ptr<FakeEncoder> fake_encoder; if (video_encoder_configs_[0].codec_type == kVideoCodecVP8) { - fake_encoder = std::make_unique<FakeVp8Encoder>(clock_); + fake_encoder = std::make_unique<FakeVp8Encoder>(&env_.clock()); } else { - fake_encoder = std::make_unique<FakeEncoder>(clock_); + fake_encoder = std::make_unique<FakeEncoder>(&env_.clock()); } fake_encoder->SetMaxBitrate(fake_encoder_max_bitrate_); return fake_encoder; @@ -62,12 +62,24 @@ CallTest::CallTest() num_flexfec_streams_(0), audio_decoder_factory_(CreateBuiltinAudioDecoderFactory()), audio_encoder_factory_(CreateBuiltinAudioEncoderFactory()), - task_queue_(task_queue_factory_->CreateTaskQueue( + task_queue_(env_.task_queue_factory().CreateTaskQueue( "CallTestTaskQueue", TaskQueueFactory::Priority::NORMAL)) {} CallTest::~CallTest() = default; +void CallTest::SetSendEventLog(std::unique_ptr<RtcEventLog> event_log) { + EnvironmentFactory f(env_); + f.Set(std::move(event_log)); + send_env_ = f.Create(); +} + +void CallTest::SetRecvEventLog(std::unique_ptr<RtcEventLog> event_log) { + EnvironmentFactory f(env_); + f.Set(std::move(event_log)); + recv_env_ = f.Create(); +} + void CallTest::RegisterRtpExtension(const RtpExtension& extension) { for (const RtpExtension& registered_extension : rtp_extensions_) { if (registered_extension.id == extension.id) { @@ -97,7 +109,7 @@ void CallTest::RunBaseTest(BaseTest* test) { num_audio_streams_ = test->GetNumAudioStreams(); num_flexfec_streams_ = test->GetNumFlexfecStreams(); RTC_DCHECK(num_video_streams_ > 0 || num_audio_streams_ > 0); - CallConfig send_config(send_event_log_.get()); + CallConfig send_config = SendCallConfig(); test->ModifySenderBitrateConfig(&send_config.bitrate_config); if (num_audio_streams_ > 0) { CreateFakeAudioDevices(test->CreateCapturer(), test->CreateRenderer()); @@ -117,7 +129,7 @@ void CallTest::RunBaseTest(BaseTest* test) { } CreateSenderCall(send_config); if (test->ShouldCreateReceivers()) { - CallConfig recv_config(recv_event_log_.get()); + CallConfig recv_config = RecvCallConfig(); test->ModifyReceiverBitrateConfig(&recv_config.bitrate_config); if (num_audio_streams_ > 0) { AudioState::Config audio_state_config; @@ -206,9 +218,20 @@ void CallTest::RunBaseTest(BaseTest* test) { }); } +CallConfig CallTest::SendCallConfig() const { + CallConfig sender_config(send_env_); + sender_config.network_state_predictor_factory = + network_state_predictor_factory_.get(); + sender_config.network_controller_factory = network_controller_factory_.get(); + return sender_config; +} + +CallConfig CallTest::RecvCallConfig() const { + return CallConfig(recv_env_); +} + void CallTest::CreateCalls() { - CreateCalls(CallConfig(send_event_log_.get()), - CallConfig(recv_event_log_.get())); + CreateCalls(SendCallConfig(), RecvCallConfig()); } void CallTest::CreateCalls(const CallConfig& sender_config, @@ -218,24 +241,15 @@ void CallTest::CreateCalls(const CallConfig& sender_config, } void CallTest::CreateSenderCall() { - CreateSenderCall(CallConfig(send_event_log_.get())); + CreateSenderCall(SendCallConfig()); } void CallTest::CreateSenderCall(const CallConfig& config) { - auto sender_config = config; - sender_config.task_queue_factory = task_queue_factory_.get(); - sender_config.network_state_predictor_factory = - network_state_predictor_factory_.get(); - sender_config.network_controller_factory = network_controller_factory_.get(); - sender_config.trials = &field_trials_; - sender_call_ = Call::Create(sender_config); + sender_call_ = Call::Create(config); } void CallTest::CreateReceiverCall(const CallConfig& config) { - auto receiver_config = config; - receiver_config.task_queue_factory = task_queue_factory_.get(); - receiver_config.trials = &field_trials_; - receiver_call_ = Call::Create(receiver_config); + receiver_call_ = Call::Create(config); } void CallTest::DestroyCalls() { @@ -489,7 +503,7 @@ void CallTest::CreateFrameGeneratorCapturerWithDrift(Clock* clock, clock, test::CreateSquareFrameGenerator(width, height, absl::nullopt, absl::nullopt), - framerate * speed, *task_queue_factory_); + framerate * speed, env_.task_queue_factory()); frame_generator_capturer_ = frame_generator_capturer.get(); frame_generator_capturer->Init(); video_sources_.push_back(std::move(frame_generator_capturer)); @@ -502,10 +516,10 @@ void CallTest::CreateFrameGeneratorCapturer(int framerate, video_sources_.clear(); auto frame_generator_capturer = std::make_unique<test::FrameGeneratorCapturer>( - clock_, + &env_.clock(), test::CreateSquareFrameGenerator(width, height, absl::nullopt, absl::nullopt), - framerate, *task_queue_factory_); + framerate, env_.task_queue_factory()); frame_generator_capturer_ = frame_generator_capturer.get(); frame_generator_capturer->Init(); video_sources_.push_back(std::move(frame_generator_capturer)); @@ -516,9 +530,9 @@ void CallTest::CreateFakeAudioDevices( std::unique_ptr<TestAudioDeviceModule::Capturer> capturer, std::unique_ptr<TestAudioDeviceModule::Renderer> renderer) { fake_send_audio_device_ = TestAudioDeviceModule::Create( - task_queue_factory_.get(), std::move(capturer), nullptr, 1.f); + &env_.task_queue_factory(), std::move(capturer), nullptr, 1.f); fake_recv_audio_device_ = TestAudioDeviceModule::Create( - task_queue_factory_.get(), nullptr, std::move(renderer), 1.f); + &env_.task_queue_factory(), nullptr, std::move(renderer), 1.f); } void CallTest::CreateVideoStreams() { diff --git a/third_party/libwebrtc/test/call_test.h b/third_party/libwebrtc/test/call_test.h index 8d2b001f72..decf02f20b 100644 --- a/third_party/libwebrtc/test/call_test.h +++ b/third_party/libwebrtc/test/call_test.h @@ -17,6 +17,7 @@ #include "absl/types/optional.h" #include "api/array_view.h" +#include "api/environment/environment.h" #include "api/rtc_event_log/rtc_event_log.h" #include "api/task_queue/task_queue_base.h" #include "api/task_queue/task_queue_factory.h" @@ -52,6 +53,11 @@ class CallTest : public ::testing::Test, public RtpPacketSinkInterface { static const std::map<uint8_t, MediaType> payload_type_map_; protected: + const Environment& env() const { return env_; } + + void SetSendEventLog(std::unique_ptr<RtcEventLog> event_log); + void SetRecvEventLog(std::unique_ptr<RtcEventLog> event_log); + void RegisterRtpExtension(const RtpExtension& extension); // Returns header extensions that can be parsed by the transport. rtc::ArrayView<const RtpExtension> GetRegisteredExtensions() { @@ -62,6 +68,9 @@ class CallTest : public ::testing::Test, public RtpPacketSinkInterface { // to simplify test code. void RunBaseTest(BaseTest* test); + CallConfig SendCallConfig() const; + CallConfig RecvCallConfig() const; + void CreateCalls(); void CreateCalls(const CallConfig& sender_config, const CallConfig& receiver_config); @@ -185,13 +194,11 @@ class CallTest : public ::testing::Test, public RtpPacketSinkInterface { void OnRtpPacket(const RtpPacketReceived& packet) override; test::RunLoop loop_; - - Clock* const clock_; test::ScopedKeyValueConfig field_trials_; + Environment env_; + Environment send_env_; + Environment recv_env_; - std::unique_ptr<TaskQueueFactory> task_queue_factory_; - std::unique_ptr<webrtc::RtcEventLog> send_event_log_; - std::unique_ptr<webrtc::RtcEventLog> recv_event_log_; std::unique_ptr<Call> sender_call_; std::unique_ptr<PacketTransport> send_transport_; SimulatedNetworkInterface* send_simulated_network_ = nullptr; diff --git a/third_party/libwebrtc/test/fuzzers/BUILD.gn b/third_party/libwebrtc/test/fuzzers/BUILD.gn index 3d2ac7a7e7..43e9a5e922 100644 --- a/third_party/libwebrtc/test/fuzzers/BUILD.gn +++ b/third_party/libwebrtc/test/fuzzers/BUILD.gn @@ -12,6 +12,7 @@ import("../../webrtc.gni") rtc_library("webrtc_fuzzer_main") { sources = [ "webrtc_fuzzer_main.cc" ] + testonly = true deps = [ "../../rtc_base:logging", "//testing/libfuzzer:fuzzing_engine_main", diff --git a/third_party/libwebrtc/test/fuzzers/forward_error_correction_fuzzer.cc b/third_party/libwebrtc/test/fuzzers/forward_error_correction_fuzzer.cc index 04a459bc71..87b76858bb 100644 --- a/third_party/libwebrtc/test/fuzzers/forward_error_correction_fuzzer.cc +++ b/third_party/libwebrtc/test/fuzzers/forward_error_correction_fuzzer.cc @@ -34,7 +34,7 @@ void FuzzOneInput(const uint8_t* data, size_t size) { ForwardErrorCorrection::CreateFlexfec(kFecSsrc, kMediaSsrc); // Entropy from fuzzer. - rtc::ByteBufferReader fuzz_buffer(reinterpret_cast<const char*>(data), size); + rtc::ByteBufferReader fuzz_buffer(rtc::MakeArrayView(data, size)); // Initial stream state. uint16_t media_seqnum; @@ -75,8 +75,8 @@ void FuzzOneInput(const uint8_t* data, size_t size) { uint8_t packet_type; uint8_t packet_loss; while (true) { - if (!fuzz_buffer.ReadBytes(reinterpret_cast<char*>(packet_buffer), - kPacketSize)) { + if (!fuzz_buffer.ReadBytes( + rtc::ArrayView<uint8_t>(packet_buffer, kPacketSize))) { return; } if (!fuzz_buffer.ReadUInt8(&reordering)) diff --git a/third_party/libwebrtc/test/fuzzers/rtp_frame_reference_finder_fuzzer.cc b/third_party/libwebrtc/test/fuzzers/rtp_frame_reference_finder_fuzzer.cc index ea66c4a1c8..93673f0b8e 100644 --- a/third_party/libwebrtc/test/fuzzers/rtp_frame_reference_finder_fuzzer.cc +++ b/third_party/libwebrtc/test/fuzzers/rtp_frame_reference_finder_fuzzer.cc @@ -23,7 +23,7 @@ class DataReader { template <typename T> void CopyTo(T* object) { - static_assert(std::is_pod<T>(), ""); + static_assert(std::is_trivial_v<T> && std::is_standard_layout_v<T>, ""); uint8_t* destination = reinterpret_cast<uint8_t*>(object); size_t object_size = sizeof(T); size_t num_bytes = std::min(size_ - offset_, object_size); diff --git a/third_party/libwebrtc/test/fuzzers/stun_parser_fuzzer.cc b/third_party/libwebrtc/test/fuzzers/stun_parser_fuzzer.cc index 6ca9eac8b2..c8aad8c232 100644 --- a/third_party/libwebrtc/test/fuzzers/stun_parser_fuzzer.cc +++ b/third_party/libwebrtc/test/fuzzers/stun_parser_fuzzer.cc @@ -15,14 +15,13 @@ namespace webrtc { void FuzzOneInput(const uint8_t* data, size_t size) { - const char* message = reinterpret_cast<const char*>(data); // Normally we'd check the integrity first, but those checks are // fuzzed separately in stun_validator_fuzzer.cc. We still want to // fuzz this target since the integrity checks could be forged by a // malicious adversary who receives a call. std::unique_ptr<cricket::IceMessage> stun_msg(new cricket::IceMessage()); - rtc::ByteBufferReader buf(message, size); + rtc::ByteBufferReader buf(rtc::MakeArrayView(data, size)); stun_msg->Read(&buf); stun_msg->ValidateMessageIntegrity(""); } diff --git a/third_party/libwebrtc/test/fuzzers/utils/BUILD.gn b/third_party/libwebrtc/test/fuzzers/utils/BUILD.gn index dfb617857c..895e76e939 100644 --- a/third_party/libwebrtc/test/fuzzers/utils/BUILD.gn +++ b/third_party/libwebrtc/test/fuzzers/utils/BUILD.gn @@ -15,10 +15,8 @@ rtc_library("rtp_replayer") { "rtp_replayer.h", ] deps = [ - "../../../api/rtc_event_log", - "../../../api/task_queue:default_task_queue_factory", + "../../../api/environment:environment_factory", "../../../api/test/video:function_video_factory", - "../../../api/transport:field_trial_based_config", "../../../api/units:timestamp", "../../../api/video_codecs:video_codecs_api", "../../../call", diff --git a/third_party/libwebrtc/test/fuzzers/utils/rtp_replayer.cc b/third_party/libwebrtc/test/fuzzers/utils/rtp_replayer.cc index f6d7119338..1b1b96803f 100644 --- a/third_party/libwebrtc/test/fuzzers/utils/rtp_replayer.cc +++ b/third_party/libwebrtc/test/fuzzers/utils/rtp_replayer.cc @@ -16,8 +16,7 @@ #include <utility> #include "absl/memory/memory.h" -#include "api/task_queue/default_task_queue_factory.h" -#include "api/transport/field_trial_based_config.h" +#include "api/environment/environment_factory.h" #include "api/units/timestamp.h" #include "modules/rtp_rtcp/source/rtp_packet.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" @@ -73,13 +72,7 @@ void RtpReplayer::Replay( } // Setup the video streams based on the configuration. - webrtc::RtcEventLogNull event_log; - std::unique_ptr<TaskQueueFactory> task_queue_factory = - CreateDefaultTaskQueueFactory(); - CallConfig call_config(&event_log); - call_config.task_queue_factory = task_queue_factory.get(); - FieldTrialBasedConfig field_trials; - call_config.trials = &field_trials; + CallConfig call_config(CreateEnvironment()); std::unique_ptr<Call> call(Call::Create(call_config)); SetupVideoStreams(&receive_stream_configs, stream_state.get(), call.get()); diff --git a/third_party/libwebrtc/test/fuzzers/utils/rtp_replayer.h b/third_party/libwebrtc/test/fuzzers/utils/rtp_replayer.h index ae94a640a5..73220eecfa 100644 --- a/third_party/libwebrtc/test/fuzzers/utils/rtp_replayer.h +++ b/third_party/libwebrtc/test/fuzzers/utils/rtp_replayer.h @@ -18,7 +18,6 @@ #include <string> #include <vector> -#include "api/rtc_event_log/rtc_event_log.h" #include "api/test/video/function_video_decoder_factory.h" #include "api/video_codecs/video_decoder.h" #include "call/call.h" diff --git a/third_party/libwebrtc/test/network/BUILD.gn b/third_party/libwebrtc/test/network/BUILD.gn index 5a6cb31f4b..b8255d35fd 100644 --- a/third_party/libwebrtc/test/network/BUILD.gn +++ b/third_party/libwebrtc/test/network/BUILD.gn @@ -76,6 +76,7 @@ rtc_library("emulated_network") { "../../rtc_base:task_queue_for_test", "../../rtc_base:threading", "../../rtc_base/memory:always_valid_pointer", + "../../rtc_base/network:received_packet", "../../rtc_base/synchronization:mutex", "../../rtc_base/system:no_unique_address", "../../rtc_base/task_utils:repeating_task", @@ -117,6 +118,7 @@ if (rtc_include_tests && !build_with_chromium) { ":emulated_network", "../:test_support", "../../api:callfactory_api", + "../../api:enable_media_with_defaults", "../../api:libjingle_peerconnection_api", "../../api:scoped_refptr", "../../api:simulated_network_api", @@ -125,7 +127,6 @@ if (rtc_include_tests && !build_with_chromium) { "../../api/transport:field_trial_based_config", "../../call:simulated_network", "../../media:rtc_audio_video", - "../../media:rtc_media_engine_defaults", "../../modules/audio_device:test_audio_device_module", "../../p2p:rtc_p2p", "../../pc:pc_test_utils", diff --git a/third_party/libwebrtc/test/network/emulated_turn_server.cc b/third_party/libwebrtc/test/network/emulated_turn_server.cc index 0bc7ec6e2a..93724ca8a3 100644 --- a/third_party/libwebrtc/test/network/emulated_turn_server.cc +++ b/third_party/libwebrtc/test/network/emulated_turn_server.cc @@ -14,6 +14,7 @@ #include <utility> #include "api/packet_socket_factory.h" +#include "rtc_base/network/received_packet.h" #include "rtc_base/strings/string_builder.h" #include "rtc_base/task_queue_for_test.h" @@ -22,10 +23,56 @@ namespace { static const char kTestRealm[] = "example.org"; static const char kTestSoftware[] = "TestTurnServer"; +// A wrapper class for cricket::TurnServer to allocate sockets. +class PacketSocketFactoryWrapper : public rtc::PacketSocketFactory { + public: + explicit PacketSocketFactoryWrapper( + webrtc::test::EmulatedTURNServer* turn_server) + : turn_server_(turn_server) {} + ~PacketSocketFactoryWrapper() override {} + + // This method is called from TurnServer when making a TURN ALLOCATION. + // It will create a socket on the `peer_` endpoint. + rtc::AsyncPacketSocket* CreateUdpSocket(const rtc::SocketAddress& address, + uint16_t min_port, + uint16_t max_port) override { + return turn_server_->CreatePeerSocket(); + } + + rtc::AsyncListenSocket* CreateServerTcpSocket( + const rtc::SocketAddress& local_address, + uint16_t min_port, + uint16_t max_port, + int opts) override { + return nullptr; + } + rtc::AsyncPacketSocket* CreateClientTcpSocket( + const rtc::SocketAddress& local_address, + const rtc::SocketAddress& remote_address, + const rtc::ProxyInfo& proxy_info, + const std::string& user_agent, + const rtc::PacketSocketTcpOptions& tcp_options) override { + return nullptr; + } + std::unique_ptr<webrtc::AsyncDnsResolverInterface> CreateAsyncDnsResolver() + override { + return nullptr; + } + + private: + webrtc::test::EmulatedTURNServer* turn_server_; +}; + +} // namespace + +namespace webrtc { +namespace test { + // A wrapper class for copying data between an AsyncPacketSocket and a // EmulatedEndpoint. This is used by the cricket::TurnServer when // sending data back into the emulated network. -class AsyncPacketSocketWrapper : public rtc::AsyncPacketSocket { +class EmulatedTURNServer::AsyncPacketSocketWrapper + : public rtc::AsyncPacketSocket { public: AsyncPacketSocketWrapper(webrtc::test::EmulatedTURNServer* turn_server, webrtc::EmulatedEndpoint* endpoint, @@ -56,6 +103,9 @@ class AsyncPacketSocketWrapper : public rtc::AsyncPacketSocket { return cb; } int Close() override { return 0; } + void NotifyPacketReceived(const rtc::ReceivedPacket& packet) { + rtc::AsyncPacketSocket::NotifyPacketReceived(packet); + } rtc::AsyncPacketSocket::State GetState() const override { return rtc::AsyncPacketSocket::STATE_BOUND; @@ -71,51 +121,6 @@ class AsyncPacketSocketWrapper : public rtc::AsyncPacketSocket { const rtc::SocketAddress local_address_; }; -// A wrapper class for cricket::TurnServer to allocate sockets. -class PacketSocketFactoryWrapper : public rtc::PacketSocketFactory { - public: - explicit PacketSocketFactoryWrapper( - webrtc::test::EmulatedTURNServer* turn_server) - : turn_server_(turn_server) {} - ~PacketSocketFactoryWrapper() override {} - - // This method is called from TurnServer when making a TURN ALLOCATION. - // It will create a socket on the `peer_` endpoint. - rtc::AsyncPacketSocket* CreateUdpSocket(const rtc::SocketAddress& address, - uint16_t min_port, - uint16_t max_port) override { - return turn_server_->CreatePeerSocket(); - } - - rtc::AsyncListenSocket* CreateServerTcpSocket( - const rtc::SocketAddress& local_address, - uint16_t min_port, - uint16_t max_port, - int opts) override { - return nullptr; - } - rtc::AsyncPacketSocket* CreateClientTcpSocket( - const rtc::SocketAddress& local_address, - const rtc::SocketAddress& remote_address, - const rtc::ProxyInfo& proxy_info, - const std::string& user_agent, - const rtc::PacketSocketTcpOptions& tcp_options) override { - return nullptr; - } - std::unique_ptr<webrtc::AsyncDnsResolverInterface> CreateAsyncDnsResolver() - override { - return nullptr; - } - - private: - webrtc::test::EmulatedTURNServer* turn_server_; -}; - -} // namespace - -namespace webrtc { -namespace test { - EmulatedTURNServer::EmulatedTURNServer(std::unique_ptr<rtc::Thread> thread, EmulatedEndpoint* client, EmulatedEndpoint* peer) @@ -170,9 +175,8 @@ void EmulatedTURNServer::OnPacketReceived(webrtc::EmulatedIpPacket packet) { RTC_DCHECK_RUN_ON(thread_.get()); auto it = sockets_.find(packet.to); if (it != sockets_.end()) { - it->second->SignalReadPacket( - it->second, reinterpret_cast<const char*>(packet.cdata()), - packet.size(), packet.from, packet.arrival_time.ms()); + it->second->NotifyPacketReceived( + rtc::ReceivedPacket(packet.data, packet.from, packet.arrival_time)); } }); } diff --git a/third_party/libwebrtc/test/network/emulated_turn_server.h b/third_party/libwebrtc/test/network/emulated_turn_server.h index 9cb0ceabf6..de5d266897 100644 --- a/third_party/libwebrtc/test/network/emulated_turn_server.h +++ b/third_party/libwebrtc/test/network/emulated_turn_server.h @@ -84,7 +84,8 @@ class EmulatedTURNServer : public EmulatedTURNServerInterface, EmulatedEndpoint* const client_; EmulatedEndpoint* const peer_; std::unique_ptr<cricket::TurnServer> turn_server_ RTC_GUARDED_BY(&thread_); - std::map<rtc::SocketAddress, rtc::AsyncPacketSocket*> sockets_ + class AsyncPacketSocketWrapper; + std::map<rtc::SocketAddress, AsyncPacketSocketWrapper*> sockets_ RTC_GUARDED_BY(&thread_); // Wraps a EmulatedEndpoint in a AsyncPacketSocket to bridge interaction diff --git a/third_party/libwebrtc/test/network/network_emulation_pc_unittest.cc b/third_party/libwebrtc/test/network/network_emulation_pc_unittest.cc index 51a45a8234..09d3946747 100644 --- a/third_party/libwebrtc/test/network/network_emulation_pc_unittest.cc +++ b/third_party/libwebrtc/test/network/network_emulation_pc_unittest.cc @@ -11,7 +11,7 @@ #include <cstdint> #include <memory> -#include "api/call/call_factory_interface.h" +#include "api/enable_media_with_defaults.h" #include "api/peer_connection_interface.h" #include "api/rtc_event_log/rtc_event_log_factory.h" #include "api/scoped_refptr.h" @@ -19,7 +19,6 @@ #include "api/transport/field_trial_based_config.h" #include "call/simulated_network.h" #include "media/engine/webrtc_media_engine.h" -#include "media/engine/webrtc_media_engine_defaults.h" #include "modules/audio_device/include/test_audio_device.h" #include "p2p/base/basic_packet_socket_factory.h" #include "p2p/client/basic_port_allocator.h" @@ -57,23 +56,18 @@ rtc::scoped_refptr<PeerConnectionFactoryInterface> CreatePeerConnectionFactory( rtc::Thread* network_thread) { PeerConnectionFactoryDependencies pcf_deps; pcf_deps.task_queue_factory = CreateDefaultTaskQueueFactory(); - pcf_deps.call_factory = CreateCallFactory(); pcf_deps.event_log_factory = std::make_unique<RtcEventLogFactory>(pcf_deps.task_queue_factory.get()); pcf_deps.network_thread = network_thread; pcf_deps.signaling_thread = signaling_thread; pcf_deps.trials = std::make_unique<FieldTrialBasedConfig>(); - cricket::MediaEngineDependencies media_deps; - media_deps.task_queue_factory = pcf_deps.task_queue_factory.get(); - media_deps.adm = TestAudioDeviceModule::Create( - media_deps.task_queue_factory, + pcf_deps.adm = TestAudioDeviceModule::Create( + pcf_deps.task_queue_factory.get(), TestAudioDeviceModule::CreatePulsedNoiseCapturer(kMaxAptitude, kSamplingFrequency), TestAudioDeviceModule::CreateDiscardRenderer(kSamplingFrequency), /*speed=*/1.f); - media_deps.trials = pcf_deps.trials.get(); - SetMediaEngineDefaults(&media_deps); - pcf_deps.media_engine = cricket::CreateMediaEngine(std::move(media_deps)); + EnableMediaWithDefaults(pcf_deps); return CreateModularPeerConnectionFactory(std::move(pcf_deps)); } diff --git a/third_party/libwebrtc/test/pc/e2e/BUILD.gn b/third_party/libwebrtc/test/pc/e2e/BUILD.gn index 86a9110f58..9d1c1d437f 100644 --- a/third_party/libwebrtc/test/pc/e2e/BUILD.gn +++ b/third_party/libwebrtc/test/pc/e2e/BUILD.gn @@ -96,6 +96,7 @@ if (!build_with_chromium) { ":test_peer", "../..:copy_to_file_audio_capturer", "../../../api:create_time_controller", + "../../../api:enable_media_with_defaults", "../../../api:time_controller", "../../../api/rtc_event_log:rtc_event_log_factory", "../../../api/task_queue:default_task_queue_factory", @@ -105,8 +106,6 @@ if (!build_with_chromium) { "../../../api/transport:field_trial_based_config", "../../../api/video_codecs:builtin_video_decoder_factory", "../../../api/video_codecs:builtin_video_encoder_factory", - "../../../media:rtc_audio_video", - "../../../media:rtc_media_engine_defaults", "../../../modules/audio_device:test_audio_device_module", "../../../modules/audio_processing/aec_dump", "../../../p2p:rtc_p2p", diff --git a/third_party/libwebrtc/test/pc/e2e/test_peer_factory.cc b/third_party/libwebrtc/test/pc/e2e/test_peer_factory.cc index 41f7533c3a..dd900027ee 100644 --- a/third_party/libwebrtc/test/pc/e2e/test_peer_factory.cc +++ b/third_party/libwebrtc/test/pc/e2e/test_peer_factory.cc @@ -13,6 +13,7 @@ #include "absl/memory/memory.h" #include "absl/strings/string_view.h" +#include "api/enable_media_with_defaults.h" #include "api/task_queue/default_task_queue_factory.h" #include "api/test/create_time_controller.h" #include "api/test/pclf/media_configuration.h" @@ -21,8 +22,6 @@ #include "api/transport/field_trial_based_config.h" #include "api/video_codecs/builtin_video_decoder_factory.h" #include "api/video_codecs/builtin_video_encoder_factory.h" -#include "media/engine/webrtc_media_engine.h" -#include "media/engine/webrtc_media_engine_defaults.h" #include "modules/audio_processing/aec_dump/aec_dump_factory.h" #include "p2p/client/basic_port_allocator.h" #include "rtc_base/thread.h" @@ -146,28 +145,6 @@ rtc::scoped_refptr<AudioDeviceModule> CreateAudioDeviceModule( std::move(renderer), /*speed=*/1.f); } -std::unique_ptr<cricket::MediaEngineInterface> CreateMediaEngine( - TaskQueueFactory* task_queue_factory, - PeerConnectionFactoryComponents* pcf_dependencies, - rtc::scoped_refptr<AudioDeviceModule> audio_device_module) { - cricket::MediaEngineDependencies media_deps; - media_deps.task_queue_factory = task_queue_factory; - media_deps.adm = audio_device_module; - media_deps.audio_processing = pcf_dependencies->audio_processing; - media_deps.audio_mixer = pcf_dependencies->audio_mixer; - media_deps.video_encoder_factory = - std::move(pcf_dependencies->video_encoder_factory); - media_deps.video_decoder_factory = - std::move(pcf_dependencies->video_decoder_factory); - media_deps.audio_encoder_factory = pcf_dependencies->audio_encoder_factory; - media_deps.audio_decoder_factory = pcf_dependencies->audio_decoder_factory; - webrtc::SetMediaEngineDefaults(&media_deps); - RTC_DCHECK(pcf_dependencies->trials); - media_deps.trials = pcf_dependencies->trials.get(); - - return cricket::CreateMediaEngine(std::move(media_deps)); -} - void WrapVideoEncoderFactory( absl::string_view peer_name, double bitrate_multiplier, @@ -206,7 +183,7 @@ void WrapVideoDecoderFactory( PeerConnectionFactoryDependencies CreatePCFDependencies( std::unique_ptr<PeerConnectionFactoryComponents> pcf_dependencies, TimeController& time_controller, - std::unique_ptr<cricket::MediaEngineInterface> media_engine, + rtc::scoped_refptr<AudioDeviceModule> audio_device_module, rtc::Thread* signaling_thread, rtc::Thread* worker_thread, rtc::Thread* network_thread) { @@ -214,10 +191,7 @@ PeerConnectionFactoryDependencies CreatePCFDependencies( pcf_deps.signaling_thread = signaling_thread; pcf_deps.worker_thread = worker_thread; pcf_deps.network_thread = network_thread; - pcf_deps.media_engine = std::move(media_engine); - pcf_deps.call_factory = - CreateTimeControllerBasedCallFactory(&time_controller); pcf_deps.event_log_factory = std::move(pcf_dependencies->event_log_factory); pcf_deps.task_queue_factory = time_controller.CreateTaskQueueFactory(); @@ -236,6 +210,18 @@ PeerConnectionFactoryDependencies CreatePCFDependencies( pcf_deps.trials = std::move(pcf_dependencies->trials); } + // Media dependencies + pcf_deps.adm = std::move(audio_device_module); + pcf_deps.audio_processing = pcf_dependencies->audio_processing; + pcf_deps.audio_mixer = pcf_dependencies->audio_mixer; + pcf_deps.video_encoder_factory = + std::move(pcf_dependencies->video_encoder_factory); + pcf_deps.video_decoder_factory = + std::move(pcf_dependencies->video_decoder_factory); + pcf_deps.audio_encoder_factory = pcf_dependencies->audio_encoder_factory; + pcf_deps.audio_decoder_factory = pcf_dependencies->audio_decoder_factory; + EnableMediaWithDefaultsAndTimeController(time_controller, pcf_deps); + return pcf_deps; } @@ -324,10 +310,6 @@ std::unique_ptr<TestPeer> TestPeerFactory::CreateTestPeer( WrapVideoDecoderFactory(params->name.value(), components->pcf_dependencies.get(), video_analyzer_helper_); - std::unique_ptr<cricket::MediaEngineInterface> media_engine = - CreateMediaEngine(time_controller_.GetTaskQueueFactory(), - components->pcf_dependencies.get(), - audio_device_module); std::unique_ptr<rtc::Thread> owned_worker_thread = components->worker_thread != nullptr @@ -343,8 +325,8 @@ std::unique_ptr<TestPeer> TestPeerFactory::CreateTestPeer( components->pcf_dependencies->audio_processing; PeerConnectionFactoryDependencies pcf_deps = CreatePCFDependencies( std::move(components->pcf_dependencies), time_controller_, - std::move(media_engine), signaling_thread_, components->worker_thread, - components->network_thread); + std::move(audio_device_module), signaling_thread_, + components->worker_thread, components->network_thread); rtc::scoped_refptr<PeerConnectionFactoryInterface> peer_connection_factory = CreateModularPeerConnectionFactory(std::move(pcf_deps)); diff --git a/third_party/libwebrtc/test/peer_scenario/peer_scenario_client.cc b/third_party/libwebrtc/test/peer_scenario/peer_scenario_client.cc index 697bf055a7..60f2ea7f2e 100644 --- a/third_party/libwebrtc/test/peer_scenario/peer_scenario_client.cc +++ b/third_party/libwebrtc/test/peer_scenario/peer_scenario_client.cc @@ -246,17 +246,13 @@ PeerScenarioClient::PeerScenarioClient( pcf_deps.network_thread = manager->network_thread(); pcf_deps.signaling_thread = signaling_thread_; pcf_deps.worker_thread = worker_thread_.get(); - pcf_deps.call_factory = - CreateTimeControllerBasedCallFactory(net->time_controller()); pcf_deps.task_queue_factory = net->time_controller()->CreateTaskQueueFactory(); pcf_deps.event_log_factory = std::make_unique<RtcEventLogFactory>(task_queue_factory_); pcf_deps.trials = std::make_unique<FieldTrialBasedConfig>(); - cricket::MediaEngineDependencies media_deps; - media_deps.task_queue_factory = task_queue_factory_; - media_deps.adm = TestAudioDeviceModule::Create( + pcf_deps.adm = TestAudioDeviceModule::Create( task_queue_factory_, TestAudioDeviceModule::CreatePulsedNoiseCapturer( config.audio.pulsed_noise->amplitude * @@ -264,28 +260,24 @@ PeerScenarioClient::PeerScenarioClient( config.audio.sample_rate, config.audio.channels), TestAudioDeviceModule::CreateDiscardRenderer(config.audio.sample_rate)); - media_deps.audio_processing = AudioProcessingBuilder().Create(); if (config.video.use_fake_codecs) { - media_deps.video_encoder_factory = - std::make_unique<FakeVideoEncoderFactory>( - net->time_controller()->GetClock()); - media_deps.video_decoder_factory = + pcf_deps.video_encoder_factory = std::make_unique<FakeVideoEncoderFactory>( + net->time_controller()->GetClock()); + pcf_deps.video_decoder_factory = std::make_unique<FakeVideoDecoderFactory>(); } else { - media_deps.video_encoder_factory = + pcf_deps.video_encoder_factory = std::make_unique<VideoEncoderFactoryTemplate< LibvpxVp8EncoderTemplateAdapter, LibvpxVp9EncoderTemplateAdapter, OpenH264EncoderTemplateAdapter, LibaomAv1EncoderTemplateAdapter>>(); - media_deps.video_decoder_factory = + pcf_deps.video_decoder_factory = std::make_unique<VideoDecoderFactoryTemplate< LibvpxVp8DecoderTemplateAdapter, LibvpxVp9DecoderTemplateAdapter, OpenH264DecoderTemplateAdapter, Dav1dDecoderTemplateAdapter>>(); } - media_deps.audio_encoder_factory = CreateBuiltinAudioEncoderFactory(); - media_deps.audio_decoder_factory = CreateBuiltinAudioDecoderFactory(); - media_deps.trials = pcf_deps.trials.get(); - pcf_deps.media_engine = cricket::CreateMediaEngine(std::move(media_deps)); + EnableMediaWithDefaultsAndTimeController(*net->time_controller(), pcf_deps); + pcf_deps.fec_controller_factory = nullptr; pcf_deps.network_controller_factory = nullptr; pcf_deps.network_state_predictor_factory = nullptr; diff --git a/third_party/libwebrtc/test/peer_scenario/scenario_connection.cc b/third_party/libwebrtc/test/peer_scenario/scenario_connection.cc index 66eca275d1..8b2081a4c3 100644 --- a/third_party/libwebrtc/test/peer_scenario/scenario_connection.cc +++ b/third_party/libwebrtc/test/peer_scenario/scenario_connection.cc @@ -173,21 +173,14 @@ void ScenarioIceConnectionImpl::SetRemoteSdp(SdpType type, }); auto res = jsep_controller_->SetRemoteDescription( - remote_description_->GetType(), remote_description_->description()); + remote_description_->GetType(), + local_description_ ? local_description_->description() : nullptr, + remote_description_->description()); RTC_CHECK(res.ok()) << res.message(); RtpDemuxerCriteria criteria; for (const auto& content : remote_description_->description()->contents()) { - if (content.media_description()->as_audio()) { - for (const auto& codec : - content.media_description()->as_audio()->codecs()) { - criteria.payload_types().insert(codec.id); - } - } - if (content.media_description()->as_video()) { - for (const auto& codec : - content.media_description()->as_video()->codecs()) { - criteria.payload_types().insert(codec.id); - } + for (const auto& codec : content.media_description()->codecs()) { + criteria.payload_types().insert(codec.id); } } @@ -203,7 +196,8 @@ void ScenarioIceConnectionImpl::SetLocalSdp(SdpType type, RTC_DCHECK_RUN_ON(signaling_thread_); local_description_ = webrtc::CreateSessionDescription(type, local_sdp); auto res = jsep_controller_->SetLocalDescription( - local_description_->GetType(), local_description_->description()); + local_description_->GetType(), local_description_->description(), + remote_description_ ? remote_description_->description() : nullptr); RTC_CHECK(res.ok()) << res.message(); jsep_controller_->MaybeStartGathering(); } diff --git a/third_party/libwebrtc/test/rtp_test_utils_gn/moz.build b/third_party/libwebrtc/test/rtp_test_utils_gn/moz.build index f3ffb448cd..5f400c69f9 100644 --- a/third_party/libwebrtc/test/rtp_test_utils_gn/moz.build +++ b/third_party/libwebrtc/test/rtp_test_utils_gn/moz.build @@ -192,16 +192,9 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] -if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": - - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/test/scenario/BUILD.gn b/third_party/libwebrtc/test/scenario/BUILD.gn index c3b8847fd1..7ac06c6822 100644 --- a/third_party/libwebrtc/test/scenario/BUILD.gn +++ b/third_party/libwebrtc/test/scenario/BUILD.gn @@ -86,10 +86,11 @@ if (rtc_include_tests && !build_with_chromium) { "../../api:rtp_parameters", "../../api:sequence_checker", "../../api:time_controller", - "../../api:time_controller", "../../api:transport_api", "../../api/audio_codecs:builtin_audio_decoder_factory", "../../api/audio_codecs:builtin_audio_encoder_factory", + "../../api/environment", + "../../api/environment:environment_factory", "../../api/rtc_event_log", "../../api/rtc_event_log:rtc_event_log_factory", "../../api/task_queue", @@ -146,7 +147,6 @@ if (rtc_include_tests && !build_with_chromium) { "../../rtc_base/synchronization:mutex", "../../rtc_base/task_utils:repeating_task", "../../system_wrappers", - "../../system_wrappers:field_trial", "../../video/config:streams_config", "../logging:log_writer", "../network:emulated_network", diff --git a/third_party/libwebrtc/test/scenario/audio_stream.cc b/third_party/libwebrtc/test/scenario/audio_stream.cc index 7715555e23..5f7db7acdf 100644 --- a/third_party/libwebrtc/test/scenario/audio_stream.cc +++ b/third_party/libwebrtc/test/scenario/audio_stream.cc @@ -14,13 +14,11 @@ #include "test/video_test_constants.h" #if WEBRTC_ENABLE_PROTOBUF -RTC_PUSH_IGNORING_WUNDEF() #ifdef WEBRTC_ANDROID_PLATFORM_BUILD #include "external/webrtc/webrtc/modules/audio_coding/audio_network_adaptor/config.pb.h" #else #include "modules/audio_coding/audio_network_adaptor/config.pb.h" #endif -RTC_POP_IGNORING_WUNDEF() #endif namespace webrtc { diff --git a/third_party/libwebrtc/test/scenario/call_client.cc b/third_party/libwebrtc/test/scenario/call_client.cc index fdf36dee08..8845ad6b0f 100644 --- a/third_party/libwebrtc/test/scenario/call_client.cc +++ b/third_party/libwebrtc/test/scenario/call_client.cc @@ -13,6 +13,8 @@ #include <memory> #include <utility> +#include "api/environment/environment.h" +#include "api/environment/environment_factory.h" #include "api/media_types.h" #include "api/rtc_event_log/rtc_event_log.h" #include "api/rtc_event_log/rtc_event_log_factory.h" @@ -60,38 +62,27 @@ CallClientFakeAudio InitAudio(TimeController* time_controller) { } std::unique_ptr<Call> CreateCall( - TimeController* time_controller, - RtcEventLog* event_log, + const Environment& env, CallClientConfig config, LoggingNetworkControllerFactory* network_controller_factory, rtc::scoped_refptr<AudioState> audio_state) { - CallConfig call_config(event_log); + CallConfig call_config(env); call_config.bitrate_config.max_bitrate_bps = config.transport.rates.max_rate.bps_or(-1); call_config.bitrate_config.min_bitrate_bps = config.transport.rates.min_rate.bps(); call_config.bitrate_config.start_bitrate_bps = config.transport.rates.start_rate.bps(); - call_config.task_queue_factory = time_controller->GetTaskQueueFactory(); call_config.network_controller_factory = network_controller_factory; call_config.audio_state = audio_state; - call_config.pacer_burst_interval = config.pacer_burst_interval; - call_config.trials = config.field_trials; - Clock* clock = time_controller->GetClock(); - return Call::Create(call_config, clock, - RtpTransportControllerSendFactory().Create( - call_config.ExtractTransportConfig(), clock)); + return Call::Create(call_config); } std::unique_ptr<RtcEventLog> CreateEventLog( - TaskQueueFactory* task_queue_factory, - LogWriterFactoryInterface* log_writer_factory) { - if (!log_writer_factory) { - return std::make_unique<RtcEventLogNull>(); - } - auto event_log = RtcEventLogFactory(task_queue_factory) - .CreateRtcEventLog(RtcEventLog::EncodingType::NewFormat); - bool success = event_log->StartLogging(log_writer_factory->Create(".rtc.dat"), + const Environment& env, + LogWriterFactoryInterface& log_writer_factory) { + auto event_log = RtcEventLogFactory().Create(env); + bool success = event_log->StartLogging(log_writer_factory.Create(".rtc.dat"), kEventLogOutputIntervalMs); RTC_CHECK(success); return event_log; @@ -219,22 +210,25 @@ CallClient::CallClient( std::unique_ptr<LogWriterFactoryInterface> log_writer_factory, CallClientConfig config) : time_controller_(time_controller), - clock_(time_controller->GetClock()), + env_(CreateEnvironment(time_controller_->CreateTaskQueueFactory(), + time_controller_->GetClock())), log_writer_factory_(std::move(log_writer_factory)), network_controller_factory_(log_writer_factory_.get(), config.transport), - task_queue_(time_controller->GetTaskQueueFactory()->CreateTaskQueue( + task_queue_(env_.task_queue_factory().CreateTaskQueue( "CallClient", TaskQueueFactory::Priority::NORMAL)) { - config.field_trials = &field_trials_; SendTask([this, config] { - event_log_ = CreateEventLog(time_controller_->GetTaskQueueFactory(), - log_writer_factory_.get()); + if (log_writer_factory_ != nullptr) { + EnvironmentFactory env_factory(env_); + env_factory.Set(CreateEventLog(env_, *log_writer_factory_)); + env_ = env_factory.Create(); + } fake_audio_setup_ = InitAudio(time_controller_); - call_ = - CreateCall(time_controller_, event_log_.get(), config, - &network_controller_factory_, fake_audio_setup_.audio_state); - transport_ = std::make_unique<NetworkNodeTransport>(clock_, call_.get()); + call_ = CreateCall(env_, config, &network_controller_factory_, + fake_audio_setup_.audio_state); + transport_ = + std::make_unique<NetworkNodeTransport>(&env_.clock(), call_.get()); }); } @@ -243,9 +237,8 @@ CallClient::~CallClient() { call_.reset(); fake_audio_setup_ = {}; rtc::Event done; - event_log_->StopLogging([&done] { done.Set(); }); + env_.event_log().StopLogging([&done] { done.Set(); }); done.Wait(rtc::Event::kForever); - event_log_.reset(); }); } @@ -283,7 +276,7 @@ DataRate CallClient::padding_rate() const { void CallClient::SetRemoteBitrate(DataRate bitrate) { RemoteBitrateReport msg; msg.bandwidth = bitrate; - msg.receive_time = clock_->CurrentTime(); + msg.receive_time = env_.clock().CurrentTime(); network_controller_factory_.SetRemoteBitrateEstimate(msg); } diff --git a/third_party/libwebrtc/test/scenario/call_client.h b/third_party/libwebrtc/test/scenario/call_client.h index 3717a7e796..f3c483ef28 100644 --- a/third_party/libwebrtc/test/scenario/call_client.h +++ b/third_party/libwebrtc/test/scenario/call_client.h @@ -17,6 +17,7 @@ #include <vector> #include "api/array_view.h" +#include "api/environment/environment.h" #include "api/rtc_event_log/rtc_event_log.h" #include "api/rtp_parameters.h" #include "api/test/time_controller.h" @@ -156,9 +157,8 @@ class CallClient : public EmulatedNetworkReceiverInterface { void UnBind(); TimeController* const time_controller_; - Clock* clock_; + Environment env_; const std::unique_ptr<LogWriterFactoryInterface> log_writer_factory_; - std::unique_ptr<RtcEventLog> event_log_; LoggingNetworkControllerFactory network_controller_factory_; CallClientFakeAudio fake_audio_setup_; std::unique_ptr<Call> call_; @@ -175,8 +175,6 @@ class CallClient : public EmulatedNetworkReceiverInterface { std::map<uint32_t, MediaType> ssrc_media_types_; // Defined last so it's destroyed first. TaskQueueForTest task_queue_; - - const FieldTrialBasedConfig field_trials_; }; class CallClientPair { diff --git a/third_party/libwebrtc/test/scenario/scenario_config.h b/third_party/libwebrtc/test/scenario/scenario_config.h index 9ce99401d7..50845cd677 100644 --- a/third_party/libwebrtc/test/scenario/scenario_config.h +++ b/third_party/libwebrtc/test/scenario/scenario_config.h @@ -57,7 +57,6 @@ struct CallClientConfig { // The number of bites that can be sent in one burst is pacer_burst_interval * // current bwe. 40ms is the default Chrome setting. TimeDelta pacer_burst_interval = TimeDelta::Millis(40); - const FieldTrialsView* field_trials = nullptr; }; struct PacketStreamConfig { diff --git a/third_party/libwebrtc/test/scenario/video_stream.cc b/third_party/libwebrtc/test/scenario/video_stream.cc index 38e42c88e0..eb20f8dbc7 100644 --- a/third_party/libwebrtc/test/scenario/video_stream.cc +++ b/third_party/libwebrtc/test/scenario/video_stream.cc @@ -372,9 +372,9 @@ SendVideoStream::SendVideoStream(CallClient* sender, VideoFrameMatcher* matcher) : sender_(sender), config_(config) { video_capturer_ = std::make_unique<FrameGeneratorCapturer>( - sender_->clock_, CreateFrameGenerator(sender_->clock_, config.source), - config.source.framerate, - *sender->time_controller_->GetTaskQueueFactory()); + &sender_->env_.clock(), + CreateFrameGenerator(&sender_->env_.clock(), config.source), + config.source.framerate, sender_->env_.task_queue_factory()); video_capturer_->Init(); using Encoder = VideoStreamConfig::Encoder; @@ -386,9 +386,11 @@ SendVideoStream::SendVideoStream(CallClient* sender, MutexLock lock(&mutex_); std::unique_ptr<FakeEncoder> encoder; if (config_.encoder.codec == Codec::kVideoCodecVP8) { - encoder = std::make_unique<test::FakeVp8Encoder>(sender_->clock_); + encoder = std::make_unique<test::FakeVp8Encoder>( + &sender_->env_.clock()); } else if (config_.encoder.codec == Codec::kVideoCodecGeneric) { - encoder = std::make_unique<test::FakeEncoder>(sender_->clock_); + encoder = + std::make_unique<test::FakeEncoder>(&sender_->env_.clock()); } else { RTC_DCHECK_NOTREACHED(); } @@ -436,7 +438,7 @@ SendVideoStream::SendVideoStream(CallClient* sender, if (matcher->Active()) { frame_tap_ = std::make_unique<ForwardingCapturedFrameTap>( - sender_->clock_, matcher, video_capturer_.get()); + &sender_->env_.clock(), matcher, video_capturer_.get()); send_stream_->SetSource(frame_tap_.get(), config.encoder.degradation_preference); } else { @@ -565,8 +567,8 @@ ReceiveVideoStream::ReceiveVideoStream(CallClient* receiver, for (size_t i = 0; i < num_streams; ++i) { rtc::VideoSinkInterface<VideoFrame>* renderer = &fake_renderer_; if (matcher->Active()) { - render_taps_.emplace_back( - std::make_unique<DecodedFrameTap>(receiver_->clock_, matcher, i)); + render_taps_.emplace_back(std::make_unique<DecodedFrameTap>( + &receiver_->env_.clock(), matcher, i)); renderer = render_taps_.back().get(); } auto recv_config = CreateVideoReceiveStreamConfig( diff --git a/third_party/libwebrtc/test/video_codec_tester.cc b/third_party/libwebrtc/test/video_codec_tester.cc new file mode 100644 index 0000000000..9453c3a7ef --- /dev/null +++ b/third_party/libwebrtc/test/video_codec_tester.cc @@ -0,0 +1,1324 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/video_codec_tester.h" + +#include <algorithm> +#include <set> +#include <tuple> +#include <utility> + +#include "api/array_view.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "api/video/builtin_video_bitrate_allocator_factory.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_bitrate_allocator.h" +#include "api/video/video_codec_type.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_encoder.h" +#include "media/base/media_constants.h" +#include "modules/video_coding/codecs/av1/av1_svc_config.h" +#include "modules/video_coding/codecs/vp9/svc_config.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/svc/scalability_mode_util.h" +#include "modules/video_coding/utility/ivf_file_writer.h" +#include "rtc_base/event.h" +#include "rtc_base/logging.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/task_queue_for_test.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/sleep.h" +#include "test/testsupport/file_utils.h" +#include "test/testsupport/frame_reader.h" +#include "test/testsupport/video_frame_writer.h" +#include "third_party/libyuv/include/libyuv/compare.h" + +namespace webrtc { +namespace test { + +namespace { +using CodedVideoSource = VideoCodecTester::CodedVideoSource; +using VideoSourceSettings = VideoCodecTester::VideoSourceSettings; +using EncodingSettings = VideoCodecTester::EncodingSettings; +using LayerSettings = EncodingSettings::LayerSettings; +using LayerId = VideoCodecTester::LayerId; +using EncoderSettings = VideoCodecTester::EncoderSettings; +using DecoderSettings = VideoCodecTester::DecoderSettings; +using PacingSettings = VideoCodecTester::PacingSettings; +using PacingMode = PacingSettings::PacingMode; +using VideoCodecStats = VideoCodecTester::VideoCodecStats; +using DecodeCallback = + absl::AnyInvocable<void(const VideoFrame& decoded_frame)>; +using webrtc::test::ImprovementDirection; + +constexpr Frequency k90kHz = Frequency::Hertz(90000); + +const std::set<ScalabilityMode> kFullSvcScalabilityModes{ + ScalabilityMode::kL2T1, ScalabilityMode::kL2T1h, ScalabilityMode::kL2T2, + ScalabilityMode::kL2T2h, ScalabilityMode::kL2T3, ScalabilityMode::kL2T3h, + ScalabilityMode::kL3T1, ScalabilityMode::kL3T1h, ScalabilityMode::kL3T2, + ScalabilityMode::kL3T2h, ScalabilityMode::kL3T3, ScalabilityMode::kL3T3h}; + +const std::set<ScalabilityMode> kKeySvcScalabilityModes{ + ScalabilityMode::kL2T1_KEY, ScalabilityMode::kL2T2_KEY, + ScalabilityMode::kL2T2_KEY_SHIFT, ScalabilityMode::kL2T3_KEY, + ScalabilityMode::kL3T1_KEY, ScalabilityMode::kL3T2_KEY, + ScalabilityMode::kL3T3_KEY}; + +// A thread-safe raw video frame reader. +class VideoSource { + public: + explicit VideoSource(VideoSourceSettings source_settings) + : source_settings_(source_settings) { + MutexLock lock(&mutex_); + frame_reader_ = CreateYuvFrameReader( + source_settings_.file_path, source_settings_.resolution, + YuvFrameReaderImpl::RepeatMode::kPingPong); + RTC_CHECK(frame_reader_); + } + + // Pulls next frame. + VideoFrame PullFrame(uint32_t timestamp_rtp, + Resolution resolution, + Frequency framerate) { + MutexLock lock(&mutex_); + int frame_num; + auto buffer = frame_reader_->PullFrame( + &frame_num, resolution, + {.num = framerate.millihertz<int>(), + .den = source_settings_.framerate.millihertz<int>()}); + RTC_CHECK(buffer) << "Can not pull frame. RTP timestamp " << timestamp_rtp; + frame_num_[timestamp_rtp] = frame_num; + return VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp_rtp) + .set_timestamp_us((timestamp_rtp / k90kHz).us()) + .build(); + } + + // Reads frame specified by `timestamp_rtp`, scales it to `resolution` and + // returns. Frame with the given `timestamp_rtp` is expected to be pulled + // before. + VideoFrame ReadFrame(uint32_t timestamp_rtp, Resolution resolution) { + MutexLock lock(&mutex_); + RTC_CHECK(frame_num_.find(timestamp_rtp) != frame_num_.end()) + << "Frame with RTP timestamp " << timestamp_rtp + << " was not pulled before"; + auto buffer = + frame_reader_->ReadFrame(frame_num_.at(timestamp_rtp), resolution); + return VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(timestamp_rtp) + .build(); + } + + private: + VideoSourceSettings source_settings_; + std::unique_ptr<FrameReader> frame_reader_ RTC_GUARDED_BY(mutex_); + std::map<uint32_t, int> frame_num_ RTC_GUARDED_BY(mutex_); + Mutex mutex_; +}; + +// Pacer calculates delay necessary to keep frame encode or decode call spaced +// from the previous calls by the pacing time. `Schedule` is expected to be +// called as close as possible to posting frame encode or decode task. This +// class is not thread safe. +class Pacer { + public: + explicit Pacer(PacingSettings settings) + : settings_(settings), delay_(TimeDelta::Zero()) {} + + Timestamp Schedule(Timestamp timestamp) { + Timestamp now = Timestamp::Micros(rtc::TimeMicros()); + if (settings_.mode == PacingMode::kNoPacing) { + return now; + } + + Timestamp scheduled = now; + if (prev_scheduled_) { + scheduled = *prev_scheduled_ + PacingTime(timestamp); + if (scheduled < now) { + scheduled = now; + } + } + + prev_timestamp_ = timestamp; + prev_scheduled_ = scheduled; + return scheduled; + } + + private: + TimeDelta PacingTime(Timestamp timestamp) { + if (settings_.mode == PacingMode::kRealTime) { + return timestamp - *prev_timestamp_; + } + RTC_CHECK_EQ(PacingMode::kConstantRate, settings_.mode); + return 1 / settings_.constant_rate; + } + + PacingSettings settings_; + absl::optional<Timestamp> prev_timestamp_; + absl::optional<Timestamp> prev_scheduled_; + TimeDelta delay_; +}; + +class LimitedTaskQueue { + public: + // The codec tester reads frames from video source in the main thread. + // Encoding and decoding are done in separate threads. If encoding or + // decoding is slow, the reading may go far ahead and may buffer too many + // frames in memory. To prevent this we limit the encoding/decoding queue + // size. When the queue is full, the main thread and, hence, reading frames + // from video source is blocked until a previously posted encoding/decoding + // task starts. + static constexpr int kMaxTaskQueueSize = 3; + + LimitedTaskQueue() : queue_size_(0) {} + + void PostScheduledTask(absl::AnyInvocable<void() &&> task, Timestamp start) { + ++queue_size_; + task_queue_.PostTask([this, task = std::move(task), start]() mutable { + // `TaskQueue` doesn't guarantee FIFO order of execution for delayed + // tasks. + int64_t wait_ms = (start - Timestamp::Millis(rtc::TimeMillis())).ms(); + if (wait_ms > 0) { + RTC_CHECK_LT(wait_ms, 10000) << "Too high wait_ms " << wait_ms; + SleepMs(wait_ms); + } + std::move(task)(); + --queue_size_; + task_executed_.Set(); + }); + + task_executed_.Reset(); + if (queue_size_ > kMaxTaskQueueSize) { + task_executed_.Wait(rtc::Event::kForever); + RTC_CHECK(queue_size_ <= kMaxTaskQueueSize); + } + } + + void PostTaskAndWait(absl::AnyInvocable<void() &&> task) { + PostScheduledTask(std::move(task), Timestamp::Millis(rtc::TimeMillis())); + task_queue_.WaitForPreviouslyPostedTasks(); + } + + private: + TaskQueueForTest task_queue_; + std::atomic_int queue_size_; + rtc::Event task_executed_; +}; + +class TesterY4mWriter { + public: + explicit TesterY4mWriter(absl::string_view base_path) + : base_path_(base_path) {} + + ~TesterY4mWriter() { + task_queue_.SendTask([] {}); + } + + void Write(const VideoFrame& frame, int spatial_idx) { + task_queue_.PostTask([this, frame, spatial_idx] { + if (y4m_writers_.find(spatial_idx) == y4m_writers_.end()) { + std::string file_path = + base_path_ + "-s" + std::to_string(spatial_idx) + ".y4m"; + Y4mVideoFrameWriterImpl* y4m_writer = new Y4mVideoFrameWriterImpl( + file_path, frame.width(), frame.height(), /*fps=*/30); + RTC_CHECK(y4m_writer); + + y4m_writers_[spatial_idx] = + std::unique_ptr<VideoFrameWriter>(y4m_writer); + } + + y4m_writers_.at(spatial_idx)->WriteFrame(frame); + }); + } + + private: + std::string base_path_; + std::map<int, std::unique_ptr<VideoFrameWriter>> y4m_writers_; + TaskQueueForTest task_queue_; +}; + +class TesterIvfWriter { + public: + explicit TesterIvfWriter(absl::string_view base_path) + : base_path_(base_path) {} + + ~TesterIvfWriter() { + task_queue_.SendTask([] {}); + } + + void Write(const EncodedImage& encoded_frame) { + task_queue_.PostTask([this, encoded_frame] { + int spatial_idx = encoded_frame.SimulcastIndex().value_or(0); + if (ivf_file_writers_.find(spatial_idx) == ivf_file_writers_.end()) { + std::string ivf_path = + base_path_ + "-s" + std::to_string(spatial_idx) + ".ivf"; + FileWrapper ivf_file = FileWrapper::OpenWriteOnly(ivf_path); + RTC_CHECK(ivf_file.is_open()); + + std::unique_ptr<IvfFileWriter> ivf_writer = + IvfFileWriter::Wrap(std::move(ivf_file), /*byte_limit=*/0); + RTC_CHECK(ivf_writer); + + ivf_file_writers_[spatial_idx] = std::move(ivf_writer); + } + + // To play: ffplay -vcodec vp8|vp9|av1|hevc|h264 filename + ivf_file_writers_.at(spatial_idx) + ->WriteFrame(encoded_frame, VideoCodecType::kVideoCodecGeneric); + }); + } + + private: + std::string base_path_; + std::map<int, std::unique_ptr<IvfFileWriter>> ivf_file_writers_; + TaskQueueForTest task_queue_; +}; + +class LeakyBucket { + public: + LeakyBucket() : level_bits_(0) {} + + // Updates bucket level and returns its current level in bits. Data is remove + // from bucket with rate equal to target bitrate of previous frame. Bucket + // level is tracked with floating point precision. Returned value of bucket + // level is rounded up. + int Update(const VideoCodecStats::Frame& frame) { + RTC_CHECK(frame.target_bitrate) << "Bitrate must be specified."; + if (prev_frame_) { + RTC_CHECK_GT(frame.timestamp_rtp, prev_frame_->timestamp_rtp) + << "Timestamp must increase."; + TimeDelta passed = + (frame.timestamp_rtp - prev_frame_->timestamp_rtp) / k90kHz; + level_bits_ -= + prev_frame_->target_bitrate->bps<double>() * passed.seconds<double>(); + level_bits_ = std::max(level_bits_, 0.0); + } + prev_frame_ = frame; + level_bits_ += frame.frame_size.bytes() * 8; + return static_cast<int>(std::ceil(level_bits_)); + } + + private: + absl::optional<VideoCodecStats::Frame> prev_frame_; + double level_bits_; +}; + +class VideoCodecAnalyzer : public VideoCodecTester::VideoCodecStats { + public: + explicit VideoCodecAnalyzer(VideoSource* video_source) + : video_source_(video_source) {} + + void StartEncode(const VideoFrame& video_frame, + const EncodingSettings& encoding_settings) { + int64_t encode_start_us = rtc::TimeMicros(); + task_queue_.PostTask([this, timestamp_rtp = video_frame.timestamp(), + encoding_settings, encode_start_us]() { + RTC_CHECK(frames_.find(timestamp_rtp) == frames_.end()) + << "Duplicate frame. Frame with timestamp " << timestamp_rtp + << " was seen before"; + + Frame frame; + frame.timestamp_rtp = timestamp_rtp; + frame.encode_start = Timestamp::Micros(encode_start_us), + frames_.emplace(timestamp_rtp, + std::map<int, Frame>{{/*spatial_idx=*/0, frame}}); + encoding_settings_.emplace(timestamp_rtp, encoding_settings); + }); + } + + void FinishEncode(const EncodedImage& encoded_frame) { + int64_t encode_finished_us = rtc::TimeMicros(); + task_queue_.PostTask( + [this, timestamp_rtp = encoded_frame.RtpTimestamp(), + spatial_idx = encoded_frame.SpatialIndex().value_or(0), + temporal_idx = encoded_frame.TemporalIndex().value_or(0), + width = encoded_frame._encodedWidth, + height = encoded_frame._encodedHeight, + frame_type = encoded_frame._frameType, + frame_size_bytes = encoded_frame.size(), qp = encoded_frame.qp_, + encode_finished_us]() { + if (spatial_idx > 0) { + RTC_CHECK(frames_.find(timestamp_rtp) != frames_.end()) + << "Spatial layer 0 frame with timestamp " << timestamp_rtp + << " was not seen before"; + const Frame& base_frame = + frames_.at(timestamp_rtp).at(/*spatial_idx=*/0); + frames_.at(timestamp_rtp).emplace(spatial_idx, base_frame); + } + + Frame& frame = frames_.at(timestamp_rtp).at(spatial_idx); + frame.layer_id = {.spatial_idx = spatial_idx, + .temporal_idx = temporal_idx}; + frame.width = width; + frame.height = height; + frame.frame_size = DataSize::Bytes(frame_size_bytes); + frame.qp = qp; + frame.keyframe = frame_type == VideoFrameType::kVideoFrameKey; + frame.encode_time = + Timestamp::Micros(encode_finished_us) - frame.encode_start; + frame.encoded = true; + }); + } + + void StartDecode(const EncodedImage& encoded_frame) { + int64_t decode_start_us = rtc::TimeMicros(); + task_queue_.PostTask( + [this, timestamp_rtp = encoded_frame.RtpTimestamp(), + spatial_idx = encoded_frame.SpatialIndex().value_or(0), + frame_size_bytes = encoded_frame.size(), decode_start_us]() { + if (frames_.find(timestamp_rtp) == frames_.end() || + frames_.at(timestamp_rtp).find(spatial_idx) == + frames_.at(timestamp_rtp).end()) { + Frame frame; + frame.timestamp_rtp = timestamp_rtp; + frame.layer_id = {.spatial_idx = spatial_idx}; + frame.frame_size = DataSize::Bytes(frame_size_bytes); + frames_.emplace(timestamp_rtp, + std::map<int, Frame>{{spatial_idx, frame}}); + } + + Frame& frame = frames_.at(timestamp_rtp).at(spatial_idx); + frame.decode_start = Timestamp::Micros(decode_start_us); + }); + } + + void FinishDecode(const VideoFrame& decoded_frame, int spatial_idx) { + int64_t decode_finished_us = rtc::TimeMicros(); + task_queue_.PostTask([this, timestamp_rtp = decoded_frame.timestamp(), + spatial_idx, width = decoded_frame.width(), + height = decoded_frame.height(), + decode_finished_us]() { + Frame& frame = frames_.at(timestamp_rtp).at(spatial_idx); + frame.decode_time = + Timestamp::Micros(decode_finished_us) - frame.decode_start; + if (!frame.encoded) { + frame.width = width; + frame.height = height; + } + frame.decoded = true; + }); + + if (video_source_ != nullptr) { + // Copy hardware-backed frame into main memory to release output buffers + // which number may be limited in hardware decoders. + rtc::scoped_refptr<I420BufferInterface> decoded_buffer = + decoded_frame.video_frame_buffer()->ToI420(); + + task_queue_.PostTask([this, decoded_buffer, + timestamp_rtp = decoded_frame.timestamp(), + spatial_idx]() { + VideoFrame ref_frame = video_source_->ReadFrame( + timestamp_rtp, {.width = decoded_buffer->width(), + .height = decoded_buffer->height()}); + rtc::scoped_refptr<I420BufferInterface> ref_buffer = + ref_frame.video_frame_buffer()->ToI420(); + Frame& frame = frames_.at(timestamp_rtp).at(spatial_idx); + frame.psnr = CalcPsnr(*decoded_buffer, *ref_buffer); + }); + } + } + + std::vector<Frame> Slice(Filter filter, bool merge) const { + std::vector<Frame> slice; + for (const auto& [timestamp_rtp, temporal_unit_frames] : frames_) { + if (temporal_unit_frames.empty()) { + continue; + } + + bool is_svc = false; + if (!encoding_settings_.empty()) { + ScalabilityMode scalability_mode = + encoding_settings_.at(timestamp_rtp).scalability_mode; + if (kFullSvcScalabilityModes.count(scalability_mode) > 0 || + (kKeySvcScalabilityModes.count(scalability_mode) > 0 && + temporal_unit_frames.at(0).keyframe)) { + is_svc = true; + } + } + + std::vector<Frame> subframes; + for (const auto& [spatial_idx, frame] : temporal_unit_frames) { + if (frame.timestamp_rtp < filter.min_timestamp_rtp || + frame.timestamp_rtp > filter.max_timestamp_rtp) { + continue; + } + if (filter.layer_id) { + if (is_svc && + frame.layer_id.spatial_idx > filter.layer_id->spatial_idx) { + continue; + } + if (!is_svc && + frame.layer_id.spatial_idx != filter.layer_id->spatial_idx) { + continue; + } + if (frame.layer_id.temporal_idx > filter.layer_id->temporal_idx) { + continue; + } + } + subframes.push_back(frame); + } + + if (subframes.empty()) { + continue; + } + + if (!merge) { + std::copy(subframes.begin(), subframes.end(), + std::back_inserter(slice)); + continue; + } + + Frame superframe = subframes.back(); + for (const Frame& frame : + rtc::ArrayView<Frame>(subframes).subview(0, subframes.size() - 1)) { + superframe.frame_size += frame.frame_size; + superframe.keyframe |= frame.keyframe; + superframe.encode_time = + std::max(superframe.encode_time, frame.encode_time); + superframe.decode_time = + std::max(superframe.decode_time, frame.decode_time); + } + + if (!encoding_settings_.empty()) { + RTC_CHECK(encoding_settings_.find(superframe.timestamp_rtp) != + encoding_settings_.end()) + << "No encoding settings for frame " << superframe.timestamp_rtp; + const EncodingSettings& es = + encoding_settings_.at(superframe.timestamp_rtp); + superframe.target_bitrate = GetTargetBitrate(es, filter.layer_id); + superframe.target_framerate = GetTargetFramerate(es, filter.layer_id); + } + + slice.push_back(superframe); + } + return slice; + } + + Stream Aggregate(Filter filter) const { + std::vector<Frame> frames = Slice(filter, /*merge=*/true); + Stream stream; + LeakyBucket leaky_bucket; + for (const Frame& frame : frames) { + Timestamp time = Timestamp::Micros((frame.timestamp_rtp / k90kHz).us()); + if (!frame.frame_size.IsZero()) { + stream.width.AddSample(StatsSample(frame.width, time)); + stream.height.AddSample(StatsSample(frame.height, time)); + stream.frame_size_bytes.AddSample( + StatsSample(frame.frame_size.bytes(), time)); + stream.keyframe.AddSample(StatsSample(frame.keyframe, time)); + if (frame.qp) { + stream.qp.AddSample(StatsSample(*frame.qp, time)); + } + } + if (frame.encoded) { + stream.encode_time_ms.AddSample( + StatsSample(frame.encode_time.ms(), time)); + } + if (frame.decoded) { + stream.decode_time_ms.AddSample( + StatsSample(frame.decode_time.ms(), time)); + } + if (frame.psnr) { + stream.psnr.y.AddSample(StatsSample(frame.psnr->y, time)); + stream.psnr.u.AddSample(StatsSample(frame.psnr->u, time)); + stream.psnr.v.AddSample(StatsSample(frame.psnr->v, time)); + } + if (frame.target_framerate) { + stream.target_framerate_fps.AddSample( + StatsSample(frame.target_framerate->hertz<double>(), time)); + } + if (frame.target_bitrate) { + stream.target_bitrate_kbps.AddSample( + StatsSample(frame.target_bitrate->kbps<double>(), time)); + int buffer_level_bits = leaky_bucket.Update(frame); + stream.transmission_time_ms.AddSample(StatsSample( + 1000 * buffer_level_bits / frame.target_bitrate->bps<double>(), + time)); + } + } + + int num_encoded_frames = stream.frame_size_bytes.NumSamples(); + const Frame& first_frame = frames.front(); + + Filter filter_all_layers{.min_timestamp_rtp = filter.min_timestamp_rtp, + .max_timestamp_rtp = filter.max_timestamp_rtp}; + std::vector<Frame> frames_all_layers = + Slice(filter_all_layers, /*merge=*/true); + const Frame& last_frame = frames_all_layers.back(); + TimeDelta duration = + (last_frame.timestamp_rtp - first_frame.timestamp_rtp) / k90kHz; + if (last_frame.target_framerate) { + duration += 1 / *last_frame.target_framerate; + } + + DataRate encoded_bitrate = + DataSize::Bytes(stream.frame_size_bytes.GetSum()) / duration; + Frequency encoded_framerate = num_encoded_frames / duration; + + double bitrate_mismatch_pct = 0.0; + if (const auto& target_bitrate = first_frame.target_bitrate; + target_bitrate) { + bitrate_mismatch_pct = 100 * (encoded_bitrate / *target_bitrate - 1); + } + double framerate_mismatch_pct = 0.0; + if (const auto& target_framerate = first_frame.target_framerate; + target_framerate) { + framerate_mismatch_pct = + 100 * (encoded_framerate / *target_framerate - 1); + } + + for (Frame& frame : frames) { + Timestamp time = Timestamp::Micros((frame.timestamp_rtp / k90kHz).us()); + stream.encoded_bitrate_kbps.AddSample( + StatsSample(encoded_bitrate.kbps<double>(), time)); + stream.encoded_framerate_fps.AddSample( + StatsSample(encoded_framerate.hertz<double>(), time)); + stream.bitrate_mismatch_pct.AddSample( + StatsSample(bitrate_mismatch_pct, time)); + stream.framerate_mismatch_pct.AddSample( + StatsSample(framerate_mismatch_pct, time)); + } + + return stream; + } + + void LogMetrics(absl::string_view csv_path, + std::vector<Frame> frames, + std::map<std::string, std::string> metadata) const { + RTC_LOG(LS_INFO) << "Write metrics to " << csv_path; + FILE* csv_file = fopen(csv_path.data(), "w"); + const std::string delimiter = ";"; + rtc::StringBuilder header; + header + << "timestamp_rtp;spatial_idx;temporal_idx;width;height;frame_size_" + "bytes;keyframe;qp;encode_time_us;decode_time_us;psnr_y_db;psnr_u_" + "db;psnr_v_db;target_bitrate_kbps;target_framerate_fps"; + for (const auto& data : metadata) { + header << ";" << data.first; + } + fwrite(header.str().c_str(), 1, header.size(), csv_file); + + for (const Frame& f : frames) { + rtc::StringBuilder row; + row << "\n" << f.timestamp_rtp; + row << ";" << f.layer_id.spatial_idx; + row << ";" << f.layer_id.temporal_idx; + row << ";" << f.width; + row << ";" << f.height; + row << ";" << f.frame_size.bytes(); + row << ";" << f.keyframe; + row << ";"; + if (f.qp) { + row << *f.qp; + } + row << ";" << f.encode_time.us(); + row << ";" << f.decode_time.us(); + if (f.psnr) { + row << ";" << f.psnr->y; + row << ";" << f.psnr->u; + row << ";" << f.psnr->v; + } else { + row << ";;;"; + } + + const auto& es = encoding_settings_.at(f.timestamp_rtp); + row << ";" + << f.target_bitrate.value_or(GetTargetBitrate(es, f.layer_id)).kbps(); + row << ";" + << f.target_framerate.value_or(GetTargetFramerate(es, f.layer_id)) + .hertz<double>(); + + for (const auto& data : metadata) { + row << ";" << data.second; + } + fwrite(row.str().c_str(), 1, row.size(), csv_file); + } + + fclose(csv_file); + } + + void Flush() { task_queue_.WaitForPreviouslyPostedTasks(); } + + private: + struct FrameId { + uint32_t timestamp_rtp; + int spatial_idx; + + bool operator==(const FrameId& o) const { + return timestamp_rtp == o.timestamp_rtp && spatial_idx == o.spatial_idx; + } + bool operator<(const FrameId& o) const { + return timestamp_rtp < o.timestamp_rtp || + (timestamp_rtp == o.timestamp_rtp && spatial_idx < o.spatial_idx); + } + }; + + Frame::Psnr CalcPsnr(const I420BufferInterface& ref_buffer, + const I420BufferInterface& dec_buffer) { + RTC_CHECK_EQ(ref_buffer.width(), dec_buffer.width()); + RTC_CHECK_EQ(ref_buffer.height(), dec_buffer.height()); + + uint64_t sse_y = libyuv::ComputeSumSquareErrorPlane( + dec_buffer.DataY(), dec_buffer.StrideY(), ref_buffer.DataY(), + ref_buffer.StrideY(), dec_buffer.width(), dec_buffer.height()); + + uint64_t sse_u = libyuv::ComputeSumSquareErrorPlane( + dec_buffer.DataU(), dec_buffer.StrideU(), ref_buffer.DataU(), + ref_buffer.StrideU(), dec_buffer.width() / 2, dec_buffer.height() / 2); + + uint64_t sse_v = libyuv::ComputeSumSquareErrorPlane( + dec_buffer.DataV(), dec_buffer.StrideV(), ref_buffer.DataV(), + ref_buffer.StrideV(), dec_buffer.width() / 2, dec_buffer.height() / 2); + + int num_y_samples = dec_buffer.width() * dec_buffer.height(); + Frame::Psnr psnr; + psnr.y = libyuv::SumSquareErrorToPsnr(sse_y, num_y_samples); + psnr.u = libyuv::SumSquareErrorToPsnr(sse_u, num_y_samples / 4); + psnr.v = libyuv::SumSquareErrorToPsnr(sse_v, num_y_samples / 4); + return psnr; + } + + DataRate GetTargetBitrate(const EncodingSettings& encoding_settings, + absl::optional<LayerId> layer_id) const { + int base_spatial_idx; + if (layer_id.has_value()) { + bool is_svc = + kFullSvcScalabilityModes.count(encoding_settings.scalability_mode); + base_spatial_idx = is_svc ? 0 : layer_id->spatial_idx; + } else { + int num_spatial_layers = + ScalabilityModeToNumSpatialLayers(encoding_settings.scalability_mode); + int num_temporal_layers = ScalabilityModeToNumTemporalLayers( + encoding_settings.scalability_mode); + layer_id = LayerId({.spatial_idx = num_spatial_layers - 1, + .temporal_idx = num_temporal_layers - 1}); + base_spatial_idx = 0; + } + + DataRate bitrate = DataRate::Zero(); + for (int sidx = base_spatial_idx; sidx <= layer_id->spatial_idx; ++sidx) { + for (int tidx = 0; tidx <= layer_id->temporal_idx; ++tidx) { + auto layer_settings = encoding_settings.layers_settings.find( + {.spatial_idx = sidx, .temporal_idx = tidx}); + RTC_CHECK(layer_settings != encoding_settings.layers_settings.end()) + << "bitrate is not specified for layer sidx=" << sidx + << " tidx=" << tidx; + bitrate += layer_settings->second.bitrate; + } + } + return bitrate; + } + + Frequency GetTargetFramerate(const EncodingSettings& encoding_settings, + absl::optional<LayerId> layer_id) const { + if (layer_id.has_value()) { + auto layer_settings = encoding_settings.layers_settings.find( + {.spatial_idx = layer_id->spatial_idx, + .temporal_idx = layer_id->temporal_idx}); + RTC_CHECK(layer_settings != encoding_settings.layers_settings.end()) + << "framerate is not specified for layer sidx=" + << layer_id->spatial_idx << " tidx=" << layer_id->temporal_idx; + return layer_settings->second.framerate; + } + return encoding_settings.layers_settings.rbegin()->second.framerate; + } + + SamplesStatsCounter::StatsSample StatsSample(double value, + Timestamp time) const { + return SamplesStatsCounter::StatsSample{value, time}; + } + + VideoSource* const video_source_; + TaskQueueForTest task_queue_; + // RTP timestamp -> spatial layer -> Frame + std::map<uint32_t, std::map<int, Frame>> frames_; + std::map<uint32_t, EncodingSettings> encoding_settings_; +}; + +class Decoder : public DecodedImageCallback { + public: + Decoder(VideoDecoderFactory* decoder_factory, + const DecoderSettings& decoder_settings, + VideoCodecAnalyzer* analyzer) + : decoder_factory_(decoder_factory), + analyzer_(analyzer), + pacer_(decoder_settings.pacing_settings) { + RTC_CHECK(analyzer_) << "Analyzer must be provided"; + + if (decoder_settings.decoder_input_base_path) { + ivf_writer_ = std::make_unique<TesterIvfWriter>( + *decoder_settings.decoder_input_base_path); + } + + if (decoder_settings.decoder_output_base_path) { + y4m_writer_ = std::make_unique<TesterY4mWriter>( + *decoder_settings.decoder_output_base_path); + } + } + + void Initialize(const SdpVideoFormat& sdp_video_format) { + decoder_ = decoder_factory_->CreateVideoDecoder(sdp_video_format); + RTC_CHECK(decoder_) << "Could not create decoder for video format " + << sdp_video_format.ToString(); + + task_queue_.PostTaskAndWait([this, &sdp_video_format] { + decoder_->RegisterDecodeCompleteCallback(this); + + VideoDecoder::Settings ds; + ds.set_codec_type(PayloadStringToCodecType(sdp_video_format.name)); + ds.set_number_of_cores(1); + ds.set_max_render_resolution({1280, 720}); + bool result = decoder_->Configure(ds); + RTC_CHECK(result) << "Failed to configure decoder"; + }); + } + + void Decode(const EncodedImage& encoded_frame) { + Timestamp pts = + Timestamp::Micros((encoded_frame.RtpTimestamp() / k90kHz).us()); + + task_queue_.PostScheduledTask( + [this, encoded_frame] { + analyzer_->StartDecode(encoded_frame); + int error = decoder_->Decode(encoded_frame, /*render_time_ms*/ 0); + if (error != 0) { + RTC_LOG(LS_WARNING) + << "Decode failed with error code " << error + << " RTP timestamp " << encoded_frame.RtpTimestamp(); + } + }, + pacer_.Schedule(pts)); + + if (ivf_writer_) { + ivf_writer_->Write(encoded_frame); + } + } + + void Flush() { + // TODO(webrtc:14852): Add Flush() to VideoDecoder API. + task_queue_.PostTaskAndWait([this] { decoder_->Release(); }); + } + + private: + int Decoded(VideoFrame& decoded_frame) override { + analyzer_->FinishDecode(decoded_frame, /*spatial_idx=*/0); + + if (y4m_writer_) { + y4m_writer_->Write(decoded_frame, /*spatial_idx=*/0); + } + + return WEBRTC_VIDEO_CODEC_OK; + } + + VideoDecoderFactory* decoder_factory_; + std::unique_ptr<VideoDecoder> decoder_; + VideoCodecAnalyzer* const analyzer_; + Pacer pacer_; + LimitedTaskQueue task_queue_; + std::unique_ptr<TesterIvfWriter> ivf_writer_; + std::unique_ptr<TesterY4mWriter> y4m_writer_; +}; + +class Encoder : public EncodedImageCallback { + public: + using EncodeCallback = + absl::AnyInvocable<void(const EncodedImage& encoded_frame)>; + + Encoder(VideoEncoderFactory* encoder_factory, + const EncoderSettings& encoder_settings, + VideoCodecAnalyzer* analyzer) + : encoder_factory_(encoder_factory), + analyzer_(analyzer), + pacer_(encoder_settings.pacing_settings) { + RTC_CHECK(analyzer_) << "Analyzer must be provided"; + + if (encoder_settings.encoder_input_base_path) { + y4m_writer_ = std::make_unique<TesterY4mWriter>( + *encoder_settings.encoder_input_base_path); + } + + if (encoder_settings.encoder_output_base_path) { + ivf_writer_ = std::make_unique<TesterIvfWriter>( + *encoder_settings.encoder_output_base_path); + } + } + + void Initialize(const EncodingSettings& encoding_settings) { + encoder_ = encoder_factory_->CreateVideoEncoder( + encoding_settings.sdp_video_format); + RTC_CHECK(encoder_) << "Could not create encoder for video format " + << encoding_settings.sdp_video_format.ToString(); + + task_queue_.PostTaskAndWait([this, encoding_settings] { + encoder_->RegisterEncodeCompleteCallback(this); + Configure(encoding_settings); + SetRates(encoding_settings); + }); + } + + void Encode(const VideoFrame& input_frame, + const EncodingSettings& encoding_settings, + EncodeCallback callback) { + { + MutexLock lock(&mutex_); + callbacks_[input_frame.timestamp()] = std::move(callback); + } + + Timestamp pts = Timestamp::Micros((input_frame.timestamp() / k90kHz).us()); + + task_queue_.PostScheduledTask( + [this, input_frame, encoding_settings] { + analyzer_->StartEncode(input_frame, encoding_settings); + + if (!last_encoding_settings_ || + !IsSameRate(encoding_settings, *last_encoding_settings_)) { + SetRates(encoding_settings); + } + + int error = encoder_->Encode(input_frame, /*frame_types=*/nullptr); + if (error != 0) { + RTC_LOG(LS_WARNING) << "Encode failed with error code " << error + << " RTP timestamp " << input_frame.timestamp(); + } + + last_encoding_settings_ = encoding_settings; + }, + pacer_.Schedule(pts)); + + if (y4m_writer_) { + y4m_writer_->Write(input_frame, /*spatial_idx=*/0); + } + } + + void Flush() { + task_queue_.PostTaskAndWait([this] { encoder_->Release(); }); + } + + private: + Result OnEncodedImage(const EncodedImage& encoded_frame, + const CodecSpecificInfo* codec_specific_info) override { + analyzer_->FinishEncode(encoded_frame); + + { + MutexLock lock(&mutex_); + auto it = callbacks_.find(encoded_frame.RtpTimestamp()); + RTC_CHECK(it != callbacks_.end()); + it->second(encoded_frame); + callbacks_.erase(callbacks_.begin(), it); + } + + if (ivf_writer_ != nullptr) { + ivf_writer_->Write(encoded_frame); + } + + return Result(Result::Error::OK); + } + + void Configure(const EncodingSettings& es) { + const LayerSettings& layer_settings = es.layers_settings.rbegin()->second; + const DataRate& bitrate = layer_settings.bitrate; + + VideoCodec vc; + vc.width = layer_settings.resolution.width; + vc.height = layer_settings.resolution.height; + vc.startBitrate = bitrate.kbps(); + vc.maxBitrate = bitrate.kbps(); + vc.minBitrate = 0; + vc.maxFramerate = layer_settings.framerate.hertz<uint32_t>(); + vc.active = true; + vc.numberOfSimulcastStreams = 0; + vc.mode = webrtc::VideoCodecMode::kRealtimeVideo; + vc.SetFrameDropEnabled(true); + vc.SetScalabilityMode(es.scalability_mode); + vc.SetVideoEncoderComplexity(VideoCodecComplexity::kComplexityNormal); + + vc.codecType = PayloadStringToCodecType(es.sdp_video_format.name); + switch (vc.codecType) { + case kVideoCodecVP8: + *(vc.VP8()) = VideoEncoder::GetDefaultVp8Settings(); + vc.VP8()->SetNumberOfTemporalLayers( + ScalabilityModeToNumTemporalLayers(es.scalability_mode)); + vc.qpMax = cricket::kDefaultVideoMaxQpVpx; + // TODO(webrtc:14852): Configure simulcast. + break; + case kVideoCodecVP9: + *(vc.VP9()) = VideoEncoder::GetDefaultVp9Settings(); + // See LibvpxVp9Encoder::ExplicitlyConfiguredSpatialLayers. + vc.spatialLayers[0].targetBitrate = vc.maxBitrate; + vc.qpMax = cricket::kDefaultVideoMaxQpVpx; + break; + case kVideoCodecAV1: + vc.qpMax = cricket::kDefaultVideoMaxQpVpx; + break; + case kVideoCodecH264: + *(vc.H264()) = VideoEncoder::GetDefaultH264Settings(); + vc.qpMax = cricket::kDefaultVideoMaxQpH26x; + break; + case kVideoCodecH265: + vc.qpMax = cricket::kDefaultVideoMaxQpH26x; + break; + case kVideoCodecGeneric: + case kVideoCodecMultiplex: + RTC_CHECK_NOTREACHED(); + break; + } + + VideoEncoder::Settings ves( + VideoEncoder::Capabilities(/*loss_notification=*/false), + /*number_of_cores=*/1, + /*max_payload_size=*/1440); + + int result = encoder_->InitEncode(&vc, ves); + RTC_CHECK(result == WEBRTC_VIDEO_CODEC_OK); + + SetRates(es); + } + + void SetRates(const EncodingSettings& es) { + VideoEncoder::RateControlParameters rc; + int num_spatial_layers = + ScalabilityModeToNumSpatialLayers(es.scalability_mode); + int num_temporal_layers = + ScalabilityModeToNumTemporalLayers(es.scalability_mode); + for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { + for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { + auto layers_settings = es.layers_settings.find( + {.spatial_idx = sidx, .temporal_idx = tidx}); + RTC_CHECK(layers_settings != es.layers_settings.end()) + << "Bitrate for layer S=" << sidx << " T=" << tidx << " is not set"; + rc.bitrate.SetBitrate(sidx, tidx, + layers_settings->second.bitrate.bps()); + } + } + rc.framerate_fps = + es.layers_settings.rbegin()->second.framerate.hertz<double>(); + encoder_->SetRates(rc); + } + + bool IsSameRate(const EncodingSettings& a, const EncodingSettings& b) const { + for (auto [layer_id, layer] : a.layers_settings) { + const auto& other_layer = b.layers_settings.at(layer_id); + if (layer.bitrate != other_layer.bitrate || + layer.framerate != other_layer.framerate) { + return false; + } + } + + return true; + } + + VideoEncoderFactory* const encoder_factory_; + std::unique_ptr<VideoEncoder> encoder_; + VideoCodecAnalyzer* const analyzer_; + Pacer pacer_; + absl::optional<EncodingSettings> last_encoding_settings_; + std::unique_ptr<VideoBitrateAllocator> bitrate_allocator_; + LimitedTaskQueue task_queue_; + std::unique_ptr<TesterY4mWriter> y4m_writer_; + std::unique_ptr<TesterIvfWriter> ivf_writer_; + std::map<uint32_t, int> sidx_ RTC_GUARDED_BY(mutex_); + std::map<uint32_t, EncodeCallback> callbacks_ RTC_GUARDED_BY(mutex_); + Mutex mutex_; +}; + +std::tuple<std::vector<DataRate>, ScalabilityMode> +SplitBitrateAndUpdateScalabilityMode(std::string codec_type, + ScalabilityMode scalability_mode, + int width, + int height, + std::vector<int> bitrates_kbps, + double framerate_fps) { + int num_spatial_layers = ScalabilityModeToNumSpatialLayers(scalability_mode); + int num_temporal_layers = + ScalabilityModeToNumTemporalLayers(scalability_mode); + + if (bitrates_kbps.size() > 1 || + (num_spatial_layers == 1 && num_temporal_layers == 1)) { + RTC_CHECK(bitrates_kbps.size() == + static_cast<size_t>(num_spatial_layers * num_temporal_layers)) + << "bitrates must be provided for all layers"; + std::vector<DataRate> bitrates; + for (const auto& bitrate_kbps : bitrates_kbps) { + bitrates.push_back(DataRate::KilobitsPerSec(bitrate_kbps)); + } + return std::make_tuple(bitrates, scalability_mode); + } + + VideoCodec vc; + vc.codecType = PayloadStringToCodecType(codec_type); + vc.width = width; + vc.height = height; + vc.startBitrate = bitrates_kbps.front(); + vc.maxBitrate = bitrates_kbps.front(); + vc.minBitrate = 0; + vc.maxFramerate = static_cast<uint32_t>(framerate_fps); + vc.numberOfSimulcastStreams = 0; + vc.mode = webrtc::VideoCodecMode::kRealtimeVideo; + vc.SetScalabilityMode(scalability_mode); + + switch (vc.codecType) { + case kVideoCodecVP8: + // TODO(webrtc:14852): Configure simulcast. + *(vc.VP8()) = VideoEncoder::GetDefaultVp8Settings(); + vc.VP8()->SetNumberOfTemporalLayers(num_temporal_layers); + vc.simulcastStream[0].width = vc.width; + vc.simulcastStream[0].height = vc.height; + break; + case kVideoCodecVP9: { + *(vc.VP9()) = VideoEncoder::GetDefaultVp9Settings(); + vc.VP9()->SetNumberOfTemporalLayers(num_temporal_layers); + const std::vector<SpatialLayer> spatialLayers = GetVp9SvcConfig(vc); + for (size_t i = 0; i < spatialLayers.size(); ++i) { + vc.spatialLayers[i] = spatialLayers[i]; + vc.spatialLayers[i].active = true; + } + } break; + case kVideoCodecAV1: { + bool result = + SetAv1SvcConfig(vc, num_spatial_layers, num_temporal_layers); + RTC_CHECK(result) << "SetAv1SvcConfig failed"; + } break; + case kVideoCodecH264: { + *(vc.H264()) = VideoEncoder::GetDefaultH264Settings(); + vc.H264()->SetNumberOfTemporalLayers(num_temporal_layers); + } break; + case kVideoCodecH265: + break; + case kVideoCodecGeneric: + case kVideoCodecMultiplex: + RTC_CHECK_NOTREACHED(); + } + + if (*vc.GetScalabilityMode() != scalability_mode) { + RTC_LOG(LS_WARNING) << "Scalability mode changed from " + << ScalabilityModeToString(scalability_mode) << " to " + << ScalabilityModeToString(*vc.GetScalabilityMode()); + num_spatial_layers = + ScalabilityModeToNumSpatialLayers(*vc.GetScalabilityMode()); + num_temporal_layers = + ScalabilityModeToNumTemporalLayers(*vc.GetScalabilityMode()); + } + + std::unique_ptr<VideoBitrateAllocator> bitrate_allocator = + CreateBuiltinVideoBitrateAllocatorFactory()->CreateVideoBitrateAllocator( + vc); + VideoBitrateAllocation bitrate_allocation = + bitrate_allocator->Allocate(VideoBitrateAllocationParameters( + 1000 * bitrates_kbps.front(), framerate_fps)); + + std::vector<DataRate> bitrates; + for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { + for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { + int bitrate_bps = bitrate_allocation.GetBitrate(sidx, tidx); + bitrates.push_back(DataRate::BitsPerSec(bitrate_bps)); + } + } + + return std::make_tuple(bitrates, *vc.GetScalabilityMode()); +} + +} // namespace + +void VideoCodecStats::Stream::LogMetrics( + MetricsLogger* logger, + std::string test_case_name, + std::string prefix, + std::map<std::string, std::string> metadata) const { + logger->LogMetric(prefix + "width", test_case_name, width, Unit::kCount, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric(prefix + "height", test_case_name, height, Unit::kCount, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric(prefix + "frame_size_bytes", test_case_name, + frame_size_bytes, Unit::kBytes, + ImprovementDirection::kNeitherIsBetter, metadata); + logger->LogMetric(prefix + "keyframe", test_case_name, keyframe, Unit::kCount, + ImprovementDirection::kSmallerIsBetter, metadata); + logger->LogMetric(prefix + "qp", test_case_name, qp, Unit::kUnitless, + ImprovementDirection::kSmallerIsBetter, metadata); + // TODO(webrtc:14852): Change to us or even ns. + logger->LogMetric(prefix + "encode_time_ms", test_case_name, encode_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter, + metadata); + logger->LogMetric(prefix + "decode_time_ms", test_case_name, decode_time_ms, + Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter, + metadata); + // TODO(webrtc:14852): Change to kUnitLess. kKilobitsPerSecond are converted + // to bytes per second in Chromeperf dash. + logger->LogMetric(prefix + "target_bitrate_kbps", test_case_name, + target_bitrate_kbps, Unit::kKilobitsPerSecond, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric(prefix + "target_framerate_fps", test_case_name, + target_framerate_fps, Unit::kHertz, + ImprovementDirection::kBiggerIsBetter, metadata); + // TODO(webrtc:14852): Change to kUnitLess. kKilobitsPerSecond are converted + // to bytes per second in Chromeperf dash. + logger->LogMetric(prefix + "encoded_bitrate_kbps", test_case_name, + encoded_bitrate_kbps, Unit::kKilobitsPerSecond, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric(prefix + "encoded_framerate_fps", test_case_name, + encoded_framerate_fps, Unit::kHertz, + ImprovementDirection::kBiggerIsBetter, metadata); + logger->LogMetric(prefix + "bitrate_mismatch_pct", test_case_name, + bitrate_mismatch_pct, Unit::kPercent, + ImprovementDirection::kNeitherIsBetter, metadata); + logger->LogMetric(prefix + "framerate_mismatch_pct", test_case_name, + framerate_mismatch_pct, Unit::kPercent, + ImprovementDirection::kNeitherIsBetter, metadata); + logger->LogMetric(prefix + "transmission_time_ms", test_case_name, + transmission_time_ms, Unit::kMilliseconds, + ImprovementDirection::kSmallerIsBetter, metadata); + logger->LogMetric(prefix + "psnr_y_db", test_case_name, psnr.y, + Unit::kUnitless, ImprovementDirection::kBiggerIsBetter, + metadata); + logger->LogMetric(prefix + "psnr_u_db", test_case_name, psnr.u, + Unit::kUnitless, ImprovementDirection::kBiggerIsBetter, + metadata); + logger->LogMetric(prefix + "psnr_v_db", test_case_name, psnr.v, + Unit::kUnitless, ImprovementDirection::kBiggerIsBetter, + metadata); +} + +// TODO(ssilkin): use Frequency and DataRate for framerate and bitrate. +std::map<uint32_t, EncodingSettings> VideoCodecTester::CreateEncodingSettings( + std::string codec_type, + std::string scalability_name, + int width, + int height, + std::vector<int> layer_bitrates_kbps, + double framerate_fps, + int num_frames, + uint32_t first_timestamp_rtp) { + auto [layer_bitrates, scalability_mode] = + SplitBitrateAndUpdateScalabilityMode( + codec_type, *ScalabilityModeFromString(scalability_name), width, + height, layer_bitrates_kbps, framerate_fps); + + int num_spatial_layers = ScalabilityModeToNumSpatialLayers(scalability_mode); + int num_temporal_layers = + ScalabilityModeToNumTemporalLayers(scalability_mode); + + std::map<LayerId, LayerSettings> layers_settings; + for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { + int layer_width = width >> (num_spatial_layers - sidx - 1); + int layer_height = height >> (num_spatial_layers - sidx - 1); + for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { + double layer_framerate_fps = + framerate_fps / (1 << (num_temporal_layers - tidx - 1)); + layers_settings.emplace( + LayerId{.spatial_idx = sidx, .temporal_idx = tidx}, + LayerSettings{ + .resolution = {.width = layer_width, .height = layer_height}, + .framerate = Frequency::MilliHertz(1000 * layer_framerate_fps), + .bitrate = layer_bitrates[sidx * num_temporal_layers + tidx]}); + } + } + + std::map<uint32_t, EncodingSettings> frames_settings; + uint32_t timestamp_rtp = first_timestamp_rtp; + for (int frame_num = 0; frame_num < num_frames; ++frame_num) { + frames_settings.emplace( + timestamp_rtp, + EncodingSettings{.sdp_video_format = SdpVideoFormat(codec_type), + .scalability_mode = scalability_mode, + .layers_settings = layers_settings}); + + timestamp_rtp += k90kHz / Frequency::MilliHertz(1000 * framerate_fps); + } + + return frames_settings; +} + +std::unique_ptr<VideoCodecTester::VideoCodecStats> +VideoCodecTester::RunDecodeTest(CodedVideoSource* video_source, + VideoDecoderFactory* decoder_factory, + const DecoderSettings& decoder_settings, + const SdpVideoFormat& sdp_video_format) { + std::unique_ptr<VideoCodecAnalyzer> analyzer = + std::make_unique<VideoCodecAnalyzer>(/*video_source=*/nullptr); + Decoder decoder(decoder_factory, decoder_settings, analyzer.get()); + decoder.Initialize(sdp_video_format); + + while (auto frame = video_source->PullFrame()) { + decoder.Decode(*frame); + } + + decoder.Flush(); + analyzer->Flush(); + return std::move(analyzer); +} + +std::unique_ptr<VideoCodecTester::VideoCodecStats> +VideoCodecTester::RunEncodeTest( + const VideoSourceSettings& source_settings, + VideoEncoderFactory* encoder_factory, + const EncoderSettings& encoder_settings, + const std::map<uint32_t, EncodingSettings>& encoding_settings) { + VideoSource video_source(source_settings); + std::unique_ptr<VideoCodecAnalyzer> analyzer = + std::make_unique<VideoCodecAnalyzer>(/*video_source=*/nullptr); + Encoder encoder(encoder_factory, encoder_settings, analyzer.get()); + encoder.Initialize(encoding_settings.begin()->second); + + for (const auto& [timestamp_rtp, frame_settings] : encoding_settings) { + const EncodingSettings::LayerSettings& top_layer = + frame_settings.layers_settings.rbegin()->second; + VideoFrame source_frame = video_source.PullFrame( + timestamp_rtp, top_layer.resolution, top_layer.framerate); + encoder.Encode(source_frame, frame_settings, + [](const EncodedImage& encoded_frame) {}); + } + + encoder.Flush(); + analyzer->Flush(); + return std::move(analyzer); +} + +std::unique_ptr<VideoCodecTester::VideoCodecStats> +VideoCodecTester::RunEncodeDecodeTest( + const VideoSourceSettings& source_settings, + VideoEncoderFactory* encoder_factory, + VideoDecoderFactory* decoder_factory, + const EncoderSettings& encoder_settings, + const DecoderSettings& decoder_settings, + const std::map<uint32_t, EncodingSettings>& encoding_settings) { + VideoSource video_source(source_settings); + std::unique_ptr<VideoCodecAnalyzer> analyzer = + std::make_unique<VideoCodecAnalyzer>(&video_source); + Decoder decoder(decoder_factory, decoder_settings, analyzer.get()); + Encoder encoder(encoder_factory, encoder_settings, analyzer.get()); + encoder.Initialize(encoding_settings.begin()->second); + decoder.Initialize(encoding_settings.begin()->second.sdp_video_format); + + for (const auto& [timestamp_rtp, frame_settings] : encoding_settings) { + const EncodingSettings::LayerSettings& top_layer = + frame_settings.layers_settings.rbegin()->second; + VideoFrame source_frame = video_source.PullFrame( + timestamp_rtp, top_layer.resolution, top_layer.framerate); + encoder.Encode(source_frame, frame_settings, + [&decoder](const EncodedImage& encoded_frame) { + decoder.Decode(encoded_frame); + }); + } + + encoder.Flush(); + decoder.Flush(); + analyzer->Flush(); + return std::move(analyzer); +} + +} // namespace test +} // namespace webrtc diff --git a/third_party/libwebrtc/test/video_codec_tester.h b/third_party/libwebrtc/test/video_codec_tester.h new file mode 100644 index 0000000000..87cc5f76f8 --- /dev/null +++ b/third_party/libwebrtc/test/video_codec_tester.h @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_VIDEO_CODEC_TESTER_H_ +#define TEST_VIDEO_CODEC_TESTER_H_ + +#include <limits> +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/numerics/samples_stats_counter.h" +#include "api/test/metrics/metric.h" +#include "api/test/metrics/metrics_logger.h" +#include "api/units/data_rate.h" +#include "api/units/data_size.h" +#include "api/units/frequency.h" +#include "api/video/encoded_image.h" +#include "api/video/resolution.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" + +namespace webrtc { +namespace test { + +class VideoCodecTester { + public: + struct LayerId { + int spatial_idx = 0; + int temporal_idx = 0; + + bool operator==(const LayerId& o) const { + return spatial_idx == o.spatial_idx && temporal_idx == o.temporal_idx; + } + bool operator<(const LayerId& o) const { + return spatial_idx < o.spatial_idx || + (spatial_idx == o.spatial_idx && temporal_idx < o.temporal_idx); + } + }; + + struct EncodingSettings { + SdpVideoFormat sdp_video_format = SdpVideoFormat("VP8"); + ScalabilityMode scalability_mode = ScalabilityMode::kL1T1; + + struct LayerSettings { + Resolution resolution; + Frequency framerate; + DataRate bitrate; + }; + std::map<LayerId, LayerSettings> layers_settings; + }; + + class VideoCodecStats { + public: + struct Filter { + uint32_t min_timestamp_rtp = std::numeric_limits<uint32_t>::min(); + uint32_t max_timestamp_rtp = std::numeric_limits<uint32_t>::max(); + absl::optional<LayerId> layer_id; + }; + + struct Frame { + uint32_t timestamp_rtp = 0; + LayerId layer_id; + bool encoded = false; + bool decoded = false; + int width = 0; + int height = 0; + DataSize frame_size = DataSize::Zero(); + bool keyframe = false; + absl::optional<int> qp; + Timestamp encode_start = Timestamp::Zero(); + TimeDelta encode_time = TimeDelta::Zero(); + Timestamp decode_start = Timestamp::Zero(); + TimeDelta decode_time = TimeDelta::Zero(); + absl::optional<DataRate> target_bitrate; + absl::optional<Frequency> target_framerate; + + struct Psnr { + double y = 0.0; + double u = 0.0; + double v = 0.0; + }; + absl::optional<Psnr> psnr; + }; + + struct Stream { + SamplesStatsCounter width; + SamplesStatsCounter height; + SamplesStatsCounter frame_size_bytes; + SamplesStatsCounter keyframe; + SamplesStatsCounter qp; + SamplesStatsCounter encode_time_ms; + SamplesStatsCounter decode_time_ms; + SamplesStatsCounter target_bitrate_kbps; + SamplesStatsCounter target_framerate_fps; + SamplesStatsCounter encoded_bitrate_kbps; + SamplesStatsCounter encoded_framerate_fps; + SamplesStatsCounter bitrate_mismatch_pct; + SamplesStatsCounter framerate_mismatch_pct; + SamplesStatsCounter transmission_time_ms; + + struct Psnr { + SamplesStatsCounter y; + SamplesStatsCounter u; + SamplesStatsCounter v; + } psnr; + + // Logs `Stream` metrics to provided `MetricsLogger`. + void LogMetrics(MetricsLogger* logger, + std::string test_case_name, + std::string prefix, + std::map<std::string, std::string> metadata = {}) const; + }; + + virtual ~VideoCodecStats() = default; + + // Returns frames for the slice specified by `filter`. If `merge` is true, + // also merges frames belonging to the same temporal unit into one + // superframe. + virtual std::vector<Frame> Slice(Filter filter, bool merge) const = 0; + + // Returns video statistics aggregated for the slice specified by `filter`. + virtual Stream Aggregate(Filter filter) const = 0; + + // Write metrics to a CSV file. + virtual void LogMetrics( + absl::string_view csv_path, + std::vector<Frame> frames, + std::map<std::string, std::string> metadata) const = 0; + }; + + // Pacing settings for codec input. + struct PacingSettings { + enum PacingMode { + // Pacing is not used. Frames are sent to codec back-to-back. + kNoPacing, + // Pace with the rate equal to the target video frame rate. Pacing time is + // derived from RTP timestamp. + kRealTime, + // Pace with the explicitly provided rate. + kConstantRate, + }; + PacingMode mode = PacingMode::kNoPacing; + // Pacing rate for `kConstantRate` mode. + Frequency constant_rate = Frequency::Zero(); + }; + + struct VideoSourceSettings { + std::string file_path; + Resolution resolution; + Frequency framerate; + }; + + struct DecoderSettings { + PacingSettings pacing_settings; + absl::optional<std::string> decoder_input_base_path; + absl::optional<std::string> decoder_output_base_path; + }; + + struct EncoderSettings { + PacingSettings pacing_settings; + absl::optional<std::string> encoder_input_base_path; + absl::optional<std::string> encoder_output_base_path; + }; + + virtual ~VideoCodecTester() = default; + + // Interface for a coded video frames source. + class CodedVideoSource { + public: + virtual ~CodedVideoSource() = default; + + // Returns next frame. Returns `absl::nullopt` if the end-of-stream is + // reached. Frames should have RTP timestamps representing desired frame + // rate. + virtual absl::optional<EncodedImage> PullFrame() = 0; + }; + + // A helper function that creates `EncodingSettings` for `num_frames` frames, + // wraps the settings into RTP timestamp -> settings map and returns the map. + static std::map<uint32_t, EncodingSettings> CreateEncodingSettings( + std::string codec_type, + std::string scalability_name, + int width, + int height, + std::vector<int> bitrates_kbps, + double framerate_fps, + int num_frames, + uint32_t first_timestamp_rtp = 90000); + + // Decodes video, collects and returns decode metrics. + static std::unique_ptr<VideoCodecStats> RunDecodeTest( + CodedVideoSource* video_source, + VideoDecoderFactory* decoder_factory, + const DecoderSettings& decoder_settings, + const SdpVideoFormat& sdp_video_format); + + // Encodes video, collects and returns encode metrics. + static std::unique_ptr<VideoCodecStats> RunEncodeTest( + const VideoSourceSettings& source_settings, + VideoEncoderFactory* encoder_factory, + const EncoderSettings& encoder_settings, + const std::map<uint32_t, EncodingSettings>& encoding_settings); + + // Encodes and decodes video, collects and returns encode and decode metrics. + static std::unique_ptr<VideoCodecStats> RunEncodeDecodeTest( + const VideoSourceSettings& source_settings, + VideoEncoderFactory* encoder_factory, + VideoDecoderFactory* decoder_factory, + const EncoderSettings& encoder_settings, + const DecoderSettings& decoder_settings, + const std::map<uint32_t, EncodingSettings>& encoding_settings); +}; + +} // namespace test +} // namespace webrtc + +#endif // TEST_VIDEO_CODEC_TESTER_H_ diff --git a/third_party/libwebrtc/test/video_codec_tester_unittest.cc b/third_party/libwebrtc/test/video_codec_tester_unittest.cc new file mode 100644 index 0000000000..af31fe2c13 --- /dev/null +++ b/third_party/libwebrtc/test/video_codec_tester_unittest.cc @@ -0,0 +1,513 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/video_codec_tester.h" + +#include <map> +#include <memory> +#include <string> +#include <tuple> +#include <utility> +#include <vector> + +#include "api/test/mock_video_decoder.h" +#include "api/test/mock_video_decoder_factory.h" +#include "api/test/mock_video_encoder.h" +#include "api/test/mock_video_encoder_factory.h" +#include "api/units/data_rate.h" +#include "api/units/time_delta.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_frame.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/svc/scalability_mode_util.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" +#include "third_party/libyuv/include/libyuv/planar_functions.h" + +namespace webrtc { +namespace test { + +namespace { +using ::testing::_; +using ::testing::ElementsAre; +using ::testing::Field; +using ::testing::Invoke; +using ::testing::InvokeWithoutArgs; +using ::testing::NiceMock; +using ::testing::Return; +using ::testing::SizeIs; + +using VideoCodecStats = VideoCodecTester::VideoCodecStats; +using VideoSourceSettings = VideoCodecTester::VideoSourceSettings; +using CodedVideoSource = VideoCodecTester::CodedVideoSource; +using EncodingSettings = VideoCodecTester::EncodingSettings; +using LayerSettings = EncodingSettings::LayerSettings; +using LayerId = VideoCodecTester::LayerId; +using DecoderSettings = VideoCodecTester::DecoderSettings; +using EncoderSettings = VideoCodecTester::EncoderSettings; +using PacingSettings = VideoCodecTester::PacingSettings; +using PacingMode = PacingSettings::PacingMode; +using Filter = VideoCodecStats::Filter; +using Frame = VideoCodecTester::VideoCodecStats::Frame; +using Stream = VideoCodecTester::VideoCodecStats::Stream; + +constexpr int kWidth = 2; +constexpr int kHeight = 2; +const DataRate kTargetLayerBitrate = DataRate::BytesPerSec(100); +const Frequency kTargetFramerate = Frequency::Hertz(30); +constexpr Frequency k90kHz = Frequency::Hertz(90000); + +rtc::scoped_refptr<I420Buffer> CreateYuvBuffer(uint8_t y = 0, + uint8_t u = 0, + uint8_t v = 0) { + rtc::scoped_refptr<I420Buffer> buffer(I420Buffer::Create(2, 2)); + + libyuv::I420Rect(buffer->MutableDataY(), buffer->StrideY(), + buffer->MutableDataU(), buffer->StrideU(), + buffer->MutableDataV(), buffer->StrideV(), 0, 0, + buffer->width(), buffer->height(), y, u, v); + return buffer; +} + +std::string CreateYuvFile(int width, int height, int num_frames) { + std::string path = webrtc::test::TempFilename(webrtc::test::OutputPath(), + "video_codec_tester_unittest"); + FILE* file = fopen(path.c_str(), "wb"); + for (int frame_num = 0; frame_num < num_frames; ++frame_num) { + uint8_t y = (frame_num + 0) & 255; + uint8_t u = (frame_num + 1) & 255; + uint8_t v = (frame_num + 2) & 255; + rtc::scoped_refptr<I420Buffer> buffer = CreateYuvBuffer(y, u, v); + fwrite(buffer->DataY(), 1, width * height, file); + int chroma_size_bytes = (width + 1) / 2 * (height + 1) / 2; + fwrite(buffer->DataU(), 1, chroma_size_bytes, file); + fwrite(buffer->DataV(), 1, chroma_size_bytes, file); + } + fclose(file); + return path; +} + +std::unique_ptr<VideoCodecStats> RunTest(std::vector<std::vector<Frame>> frames, + ScalabilityMode scalability_mode) { + int num_frames = static_cast<int>(frames.size()); + std::string source_yuv_path = CreateYuvFile(kWidth, kHeight, num_frames); + VideoSourceSettings source_settings{ + .file_path = source_yuv_path, + .resolution = {.width = kWidth, .height = kHeight}, + .framerate = kTargetFramerate}; + + int num_encoded_frames = 0; + EncodedImageCallback* encoded_frame_callback; + NiceMock<MockVideoEncoderFactory> encoder_factory; + ON_CALL(encoder_factory, CreateVideoEncoder) + .WillByDefault([&](const SdpVideoFormat&) { + auto encoder = std::make_unique<NiceMock<MockVideoEncoder>>(); + ON_CALL(*encoder, RegisterEncodeCompleteCallback) + .WillByDefault([&](EncodedImageCallback* callback) { + encoded_frame_callback = callback; + return WEBRTC_VIDEO_CODEC_OK; + }); + ON_CALL(*encoder, Encode) + .WillByDefault([&](const VideoFrame& input_frame, + const std::vector<VideoFrameType>*) { + for (const Frame& frame : frames[num_encoded_frames]) { + EncodedImage encoded_frame; + encoded_frame._encodedWidth = frame.width; + encoded_frame._encodedHeight = frame.height; + encoded_frame.SetFrameType( + frame.keyframe ? VideoFrameType::kVideoFrameKey + : VideoFrameType::kVideoFrameDelta); + encoded_frame.SetRtpTimestamp(input_frame.timestamp()); + encoded_frame.SetSpatialIndex(frame.layer_id.spatial_idx); + encoded_frame.SetTemporalIndex(frame.layer_id.temporal_idx); + encoded_frame.SetEncodedData( + EncodedImageBuffer::Create(frame.frame_size.bytes())); + encoded_frame_callback->OnEncodedImage( + encoded_frame, + /*codec_specific_info=*/nullptr); + } + ++num_encoded_frames; + return WEBRTC_VIDEO_CODEC_OK; + }); + return encoder; + }); + + int num_decoded_frames = 0; + DecodedImageCallback* decode_callback; + NiceMock<MockVideoDecoderFactory> decoder_factory; + ON_CALL(decoder_factory, CreateVideoDecoder) + .WillByDefault([&](const SdpVideoFormat&) { + auto decoder = std::make_unique<NiceMock<MockVideoDecoder>>(); + ON_CALL(*decoder, RegisterDecodeCompleteCallback) + .WillByDefault([&](DecodedImageCallback* callback) { + decode_callback = callback; + return WEBRTC_VIDEO_CODEC_OK; + }); + ON_CALL(*decoder, Decode(_, _)) + .WillByDefault([&](const EncodedImage& encoded_frame, int64_t) { + // Make values to be different from source YUV generated in + // `CreateYuvFile`. + uint8_t y = ((num_decoded_frames + 1) * 2) & 255; + uint8_t u = ((num_decoded_frames + 2) * 2) & 255; + uint8_t v = ((num_decoded_frames + 3) * 2) & 255; + rtc::scoped_refptr<I420Buffer> frame_buffer = + CreateYuvBuffer(y, u, v); + VideoFrame decoded_frame = + VideoFrame::Builder() + .set_video_frame_buffer(frame_buffer) + .set_timestamp_rtp(encoded_frame.RtpTimestamp()) + .build(); + decode_callback->Decoded(decoded_frame); + ++num_decoded_frames; + return WEBRTC_VIDEO_CODEC_OK; + }); + return decoder; + }); + + int num_spatial_layers = ScalabilityModeToNumSpatialLayers(scalability_mode); + int num_temporal_layers = + ScalabilityModeToNumTemporalLayers(scalability_mode); + + std::map<uint32_t, EncodingSettings> encoding_settings; + for (int frame_num = 0; frame_num < num_frames; ++frame_num) { + std::map<LayerId, LayerSettings> layers_settings; + for (int sidx = 0; sidx < num_spatial_layers; ++sidx) { + for (int tidx = 0; tidx < num_temporal_layers; ++tidx) { + layers_settings.emplace( + LayerId{.spatial_idx = sidx, .temporal_idx = tidx}, + LayerSettings{.resolution = {.width = kWidth, .height = kHeight}, + .framerate = kTargetFramerate / + (1 << (num_temporal_layers - 1 - tidx)), + .bitrate = kTargetLayerBitrate}); + } + } + encoding_settings.emplace( + frames[frame_num][0].timestamp_rtp, + EncodingSettings{.scalability_mode = scalability_mode, + .layers_settings = layers_settings}); + } + + EncoderSettings encoder_settings; + DecoderSettings decoder_settings; + std::unique_ptr<VideoCodecStats> stats = + VideoCodecTester::RunEncodeDecodeTest( + source_settings, &encoder_factory, &decoder_factory, encoder_settings, + decoder_settings, encoding_settings); + remove(source_yuv_path.c_str()); + return stats; +} + +EncodedImage CreateEncodedImage(uint32_t timestamp_rtp) { + EncodedImage encoded_image; + encoded_image.SetRtpTimestamp(timestamp_rtp); + return encoded_image; +} + +class MockCodedVideoSource : public CodedVideoSource { + public: + MockCodedVideoSource(int num_frames, Frequency framerate) + : num_frames_(num_frames), frame_num_(0), framerate_(framerate) {} + + absl::optional<EncodedImage> PullFrame() override { + if (frame_num_ >= num_frames_) { + return absl::nullopt; + } + uint32_t timestamp_rtp = frame_num_ * k90kHz / framerate_; + ++frame_num_; + return CreateEncodedImage(timestamp_rtp); + } + + private: + int num_frames_; + int frame_num_; + Frequency framerate_; +}; + +} // namespace + +TEST(VideoCodecTester, Slice) { + std::unique_ptr<VideoCodecStats> stats = RunTest( + {{{.timestamp_rtp = 0, .layer_id = {.spatial_idx = 0, .temporal_idx = 0}}, + {.timestamp_rtp = 0, + .layer_id = {.spatial_idx = 1, .temporal_idx = 0}}}, + {{.timestamp_rtp = 1, + .layer_id = {.spatial_idx = 0, .temporal_idx = 1}}}}, + ScalabilityMode::kL2T2); + std::vector<Frame> slice = stats->Slice(Filter{}, /*merge=*/false); + EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 0), + Field(&Frame::timestamp_rtp, 0), + Field(&Frame::timestamp_rtp, 1))); + + slice = stats->Slice({.min_timestamp_rtp = 1}, /*merge=*/false); + EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 1))); + + slice = stats->Slice({.max_timestamp_rtp = 0}, /*merge=*/false); + EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 0), + Field(&Frame::timestamp_rtp, 0))); + + slice = stats->Slice({.layer_id = {{.spatial_idx = 0, .temporal_idx = 0}}}, + /*merge=*/false); + EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 0))); + + slice = stats->Slice({.layer_id = {{.spatial_idx = 0, .temporal_idx = 1}}}, + /*merge=*/false); + EXPECT_THAT(slice, ElementsAre(Field(&Frame::timestamp_rtp, 0), + Field(&Frame::timestamp_rtp, 1))); +} + +TEST(VideoCodecTester, Merge) { + std::unique_ptr<VideoCodecStats> stats = + RunTest({{{.timestamp_rtp = 0, + .layer_id = {.spatial_idx = 0, .temporal_idx = 0}, + .frame_size = DataSize::Bytes(1), + .keyframe = true}, + {.timestamp_rtp = 0, + .layer_id = {.spatial_idx = 1, .temporal_idx = 0}, + .frame_size = DataSize::Bytes(2)}}, + {{.timestamp_rtp = 1, + .layer_id = {.spatial_idx = 0, .temporal_idx = 1}, + .frame_size = DataSize::Bytes(4)}, + {.timestamp_rtp = 1, + .layer_id = {.spatial_idx = 1, .temporal_idx = 1}, + .frame_size = DataSize::Bytes(8)}}}, + ScalabilityMode::kL2T2_KEY); + + std::vector<Frame> slice = stats->Slice(Filter{}, /*merge=*/true); + EXPECT_THAT( + slice, + ElementsAre( + AllOf(Field(&Frame::timestamp_rtp, 0), Field(&Frame::keyframe, true), + Field(&Frame::frame_size, DataSize::Bytes(3))), + AllOf(Field(&Frame::timestamp_rtp, 1), Field(&Frame::keyframe, false), + Field(&Frame::frame_size, DataSize::Bytes(12))))); +} + +struct AggregationTestParameters { + Filter filter; + double expected_keyframe_sum; + double expected_encoded_bitrate_kbps; + double expected_encoded_framerate_fps; + double expected_bitrate_mismatch_pct; + double expected_framerate_mismatch_pct; +}; + +class VideoCodecTesterTestAggregation + : public ::testing::TestWithParam<AggregationTestParameters> {}; + +TEST_P(VideoCodecTesterTestAggregation, Aggregate) { + AggregationTestParameters test_params = GetParam(); + std::unique_ptr<VideoCodecStats> stats = + RunTest({{// L0T0 + {.timestamp_rtp = 0, + .layer_id = {.spatial_idx = 0, .temporal_idx = 0}, + .frame_size = DataSize::Bytes(1), + .keyframe = true}, + // L1T0 + {.timestamp_rtp = 0, + .layer_id = {.spatial_idx = 1, .temporal_idx = 0}, + .frame_size = DataSize::Bytes(2)}}, + // Emulate frame drop (frame_size = 0). + {{.timestamp_rtp = 3000, + .layer_id = {.spatial_idx = 0, .temporal_idx = 0}, + .frame_size = DataSize::Zero()}}, + {// L0T1 + {.timestamp_rtp = 87000, + .layer_id = {.spatial_idx = 0, .temporal_idx = 1}, + .frame_size = DataSize::Bytes(4)}, + // L1T1 + {.timestamp_rtp = 87000, + .layer_id = {.spatial_idx = 1, .temporal_idx = 1}, + .frame_size = DataSize::Bytes(8)}}}, + ScalabilityMode::kL2T2_KEY); + + Stream stream = stats->Aggregate(test_params.filter); + EXPECT_EQ(stream.keyframe.GetSum(), test_params.expected_keyframe_sum); + EXPECT_EQ(stream.encoded_bitrate_kbps.GetAverage(), + test_params.expected_encoded_bitrate_kbps); + EXPECT_EQ(stream.encoded_framerate_fps.GetAverage(), + test_params.expected_encoded_framerate_fps); + EXPECT_EQ(stream.bitrate_mismatch_pct.GetAverage(), + test_params.expected_bitrate_mismatch_pct); + EXPECT_EQ(stream.framerate_mismatch_pct.GetAverage(), + test_params.expected_framerate_mismatch_pct); +} + +INSTANTIATE_TEST_SUITE_P( + All, + VideoCodecTesterTestAggregation, + ::testing::Values( + // No filtering. + AggregationTestParameters{ + .filter = {}, + .expected_keyframe_sum = 1, + .expected_encoded_bitrate_kbps = + DataRate::BytesPerSec(15).kbps<double>(), + .expected_encoded_framerate_fps = 2, + .expected_bitrate_mismatch_pct = + 100 * (15.0 / (kTargetLayerBitrate.bytes_per_sec() * 4) - 1), + .expected_framerate_mismatch_pct = + 100 * (2.0 / kTargetFramerate.hertz() - 1)}, + // L0T0 + AggregationTestParameters{ + .filter = {.layer_id = {{.spatial_idx = 0, .temporal_idx = 0}}}, + .expected_keyframe_sum = 1, + .expected_encoded_bitrate_kbps = + DataRate::BytesPerSec(1).kbps<double>(), + .expected_encoded_framerate_fps = 1, + .expected_bitrate_mismatch_pct = + 100 * (1.0 / kTargetLayerBitrate.bytes_per_sec() - 1), + .expected_framerate_mismatch_pct = + 100 * (1.0 / (kTargetFramerate.hertz() / 2) - 1)}, + // L0T1 + AggregationTestParameters{ + .filter = {.layer_id = {{.spatial_idx = 0, .temporal_idx = 1}}}, + .expected_keyframe_sum = 1, + .expected_encoded_bitrate_kbps = + DataRate::BytesPerSec(5).kbps<double>(), + .expected_encoded_framerate_fps = 2, + .expected_bitrate_mismatch_pct = + 100 * (5.0 / (kTargetLayerBitrate.bytes_per_sec() * 2) - 1), + .expected_framerate_mismatch_pct = + 100 * (2.0 / kTargetFramerate.hertz() - 1)}, + // L1T0 + AggregationTestParameters{ + .filter = {.layer_id = {{.spatial_idx = 1, .temporal_idx = 0}}}, + .expected_keyframe_sum = 1, + .expected_encoded_bitrate_kbps = + DataRate::BytesPerSec(3).kbps<double>(), + .expected_encoded_framerate_fps = 1, + .expected_bitrate_mismatch_pct = + 100 * (3.0 / kTargetLayerBitrate.bytes_per_sec() - 1), + .expected_framerate_mismatch_pct = + 100 * (1.0 / (kTargetFramerate.hertz() / 2) - 1)}, + // L1T1 + AggregationTestParameters{ + .filter = {.layer_id = {{.spatial_idx = 1, .temporal_idx = 1}}}, + .expected_keyframe_sum = 1, + .expected_encoded_bitrate_kbps = + DataRate::BytesPerSec(11).kbps<double>(), + .expected_encoded_framerate_fps = 2, + .expected_bitrate_mismatch_pct = + 100 * (11.0 / (kTargetLayerBitrate.bytes_per_sec() * 2) - 1), + .expected_framerate_mismatch_pct = + 100 * (2.0 / kTargetFramerate.hertz() - 1)})); + +TEST(VideoCodecTester, Psnr) { + std::unique_ptr<VideoCodecStats> stats = + RunTest({{{.timestamp_rtp = 0, .frame_size = DataSize::Bytes(1)}}, + {{.timestamp_rtp = 3000, .frame_size = DataSize::Bytes(1)}}}, + ScalabilityMode::kL1T1); + + std::vector<Frame> slice = stats->Slice(Filter{}, /*merge=*/false); + ASSERT_THAT(slice, SizeIs(2)); + ASSERT_TRUE(slice[0].psnr.has_value()); + ASSERT_TRUE(slice[1].psnr.has_value()); + EXPECT_NEAR(slice[0].psnr->y, 42, 1); + EXPECT_NEAR(slice[0].psnr->u, 38, 1); + EXPECT_NEAR(slice[0].psnr->v, 36, 1); + EXPECT_NEAR(slice[1].psnr->y, 38, 1); + EXPECT_NEAR(slice[1].psnr->u, 36, 1); + EXPECT_NEAR(slice[1].psnr->v, 34, 1); +} + +class VideoCodecTesterTestPacing + : public ::testing::TestWithParam<std::tuple<PacingSettings, int>> { + public: + const int kSourceWidth = 2; + const int kSourceHeight = 2; + const int kNumFrames = 3; + const int kTargetLayerBitrateKbps = 128; + const Frequency kTargetFramerate = Frequency::Hertz(10); + + void SetUp() override { + source_yuv_file_path_ = webrtc::test::TempFilename( + webrtc::test::OutputPath(), "video_codec_tester_impl_unittest"); + FILE* file = fopen(source_yuv_file_path_.c_str(), "wb"); + for (int i = 0; i < 3 * kSourceWidth * kSourceHeight / 2; ++i) { + fwrite("x", 1, 1, file); + } + fclose(file); + } + + protected: + std::string source_yuv_file_path_; +}; + +TEST_P(VideoCodecTesterTestPacing, PaceEncode) { + auto [pacing_settings, expected_delta_ms] = GetParam(); + VideoSourceSettings video_source{ + .file_path = source_yuv_file_path_, + .resolution = {.width = kSourceWidth, .height = kSourceHeight}, + .framerate = kTargetFramerate}; + + NiceMock<MockVideoEncoderFactory> encoder_factory; + ON_CALL(encoder_factory, CreateVideoEncoder(_)) + .WillByDefault([](const SdpVideoFormat&) { + return std::make_unique<NiceMock<MockVideoEncoder>>(); + }); + + std::map<uint32_t, EncodingSettings> encoding_settings = + VideoCodecTester::CreateEncodingSettings( + "VP8", "L1T1", kSourceWidth, kSourceHeight, {kTargetLayerBitrateKbps}, + kTargetFramerate.hertz(), kNumFrames); + + EncoderSettings encoder_settings; + encoder_settings.pacing_settings = pacing_settings; + std::vector<Frame> frames = + VideoCodecTester::RunEncodeTest(video_source, &encoder_factory, + encoder_settings, encoding_settings) + ->Slice(/*filter=*/{}, /*merge=*/false); + ASSERT_THAT(frames, SizeIs(kNumFrames)); + EXPECT_NEAR((frames[1].encode_start - frames[0].encode_start).ms(), + expected_delta_ms, 10); + EXPECT_NEAR((frames[2].encode_start - frames[1].encode_start).ms(), + expected_delta_ms, 10); +} + +TEST_P(VideoCodecTesterTestPacing, PaceDecode) { + auto [pacing_settings, expected_delta_ms] = GetParam(); + MockCodedVideoSource video_source(kNumFrames, kTargetFramerate); + + NiceMock<MockVideoDecoderFactory> decoder_factory; + ON_CALL(decoder_factory, CreateVideoDecoder(_)) + .WillByDefault([](const SdpVideoFormat&) { + return std::make_unique<NiceMock<MockVideoDecoder>>(); + }); + + DecoderSettings decoder_settings; + decoder_settings.pacing_settings = pacing_settings; + std::vector<Frame> frames = + VideoCodecTester::RunDecodeTest(&video_source, &decoder_factory, + decoder_settings, SdpVideoFormat("VP8")) + ->Slice(/*filter=*/{}, /*merge=*/false); + ASSERT_THAT(frames, SizeIs(kNumFrames)); + EXPECT_NEAR((frames[1].decode_start - frames[0].decode_start).ms(), + expected_delta_ms, 10); + EXPECT_NEAR((frames[2].decode_start - frames[1].decode_start).ms(), + expected_delta_ms, 10); +} + +INSTANTIATE_TEST_SUITE_P( + DISABLED_All, + VideoCodecTesterTestPacing, + ::testing::Values( + // No pacing. + std::make_tuple(PacingSettings{.mode = PacingMode::kNoPacing}, + /*expected_delta_ms=*/0), + // Real-time pacing. + std::make_tuple(PacingSettings{.mode = PacingMode::kRealTime}, + /*expected_delta_ms=*/100), + // Pace with specified constant rate. + std::make_tuple(PacingSettings{.mode = PacingMode::kConstantRate, + .constant_rate = Frequency::Hertz(20)}, + /*expected_delta_ms=*/50))); +} // namespace test +} // namespace webrtc |