From d8bbc7858622b6d9c278469aab701ca0b609cddf Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 15 May 2024 05:35:49 +0200 Subject: Merging upstream version 126.0. Signed-off-by: Daniel Baumann --- .../builtin_audio_decoder_factory_unittest.cc | 2 +- .../codecs/opus/audio_encoder_opus_unittest.cc | 2 +- .../neteq/test/neteq_opus_quality_test.cc | 4 +- .../modules/audio_coding/test/TestStereo.cc | 2 +- .../libwebrtc/modules/audio_device/BUILD.gn | 3 +- .../modules/audio_device/fine_audio_buffer.cc | 7 +- .../modules/audio_device/fine_audio_buffer.h | 11 +- .../audio_device/fine_audio_buffer_unittest.cc | 2 +- .../audio_device/include/test_audio_device.cc | 2 +- .../audio_device/mock_audio_device_buffer.h | 5 +- .../modules/audio_device/test_audio_device_impl.cc | 8 +- .../modules/audio_device/test_audio_device_impl.h | 4 +- .../libwebrtc/modules/audio_processing/BUILD.gn | 10 +- .../modules/audio_processing/aec_dump/BUILD.gn | 18 +- .../audio_processing/aec_dump/aec_dump_factory.h | 36 ++-- .../audio_processing/aec_dump/aec_dump_impl.cc | 26 +-- .../audio_processing/aec_dump/aec_dump_impl.h | 6 +- .../audio_processing/aec_dump/aec_dump_unittest.cc | 4 +- .../aec_dump/null_aec_dump_factory.cc | 23 ++- .../audio_processing/audio_processing_impl.cc | 16 +- .../audio_processing/audio_processing_impl.h | 16 +- .../audio_processing/audio_processing_unittest.cc | 10 +- .../audio_processing/include/audio_processing.h | 20 +-- .../include/mock_audio_processing.h | 6 +- .../test/audio_processing_simulator.cc | 2 +- .../audio_processing/test/debug_dump_test.cc | 2 +- .../goog_cc/delay_based_bwe_unittest_helper.cc | 14 +- .../goog_cc/delay_based_bwe_unittest_helper.h | 9 +- .../goog_cc/send_side_bandwidth_estimation.cc | 3 +- .../modules/congestion_controller/rtp/BUILD.gn | 1 - .../congestion_controller/rtp/control_handler.cc | 18 +- .../congestion_controller/rtp/control_handler.h | 6 +- third_party/libwebrtc/modules/pacing/BUILD.gn | 1 + .../libwebrtc/modules/pacing/pacing_controller.cc | 22 +-- .../libwebrtc/modules/pacing/pacing_controller.h | 44 ++++- .../modules/pacing/pacing_controller_unittest.cc | 38 +++++ .../modules/pacing/prioritized_packet_queue.cc | 149 ++++++++++++++--- .../modules/pacing/prioritized_packet_queue.h | 26 ++- .../pacing/prioritized_packet_queue_unittest.cc | 171 +++++++++++++++++-- .../remote_estimator_proxy.cc | 7 +- .../rtp_rtcp/source/receive_statistics_impl.cc | 8 +- .../rtp_rtcp/source/receive_statistics_unittest.cc | 17 ++ .../modules/rtp_rtcp/source/rtcp_receiver.cc | 4 +- .../source/rtp_dependency_descriptor_extension.cc | 13 ++ .../source/rtp_dependency_descriptor_extension.h | 10 ++ .../modules/rtp_rtcp/source/rtp_format_h264.cc | 27 +-- .../rtp_rtcp/source/rtp_format_h264_unittest.cc | 186 +++++++++++++++++++++ .../modules/rtp_rtcp/source/rtp_format_vp8.cc | 4 +- .../rtp_rtcp/source/rtp_format_vp8_unittest.cc | 12 ++ .../modules/rtp_rtcp/source/rtp_format_vp9.cc | 5 +- .../rtp_rtcp/source/rtp_format_vp9_unittest.cc | 5 + .../libwebrtc/modules/rtp_rtcp/source/rtp_packet.h | 8 +- .../modules/rtp_rtcp/source/rtp_packet_to_send.cc | 11 ++ .../modules/rtp_rtcp/source/rtp_packet_to_send.h | 10 +- .../modules/rtp_rtcp/source/rtp_packet_unittest.cc | 35 ++++ .../rtp_rtcp/source/rtp_packetizer_av1_unittest.cc | 6 + .../libwebrtc/modules/video_capture/BUILD.gn | 1 + .../video_capture/test/video_capture_unittest.cc | 21 +-- .../libwebrtc/modules/video_coding/BUILD.gn | 3 +- .../video_coding/codecs/av1/dav1d_decoder.cc | 2 + .../video_coding/codecs/av1/libaom_av1_unittest.cc | 3 +- .../video_coding/codecs/h264/h264_decoder_impl.cc | 6 +- .../video_coding/codecs/test/video_codec_test.cc | 34 ++-- .../codecs/test/videocodec_test_fixture_impl.cc | 2 +- .../modules/video_coding/fec_controller_default.cc | 30 ++-- .../modules/video_coding/fec_controller_default.h | 11 +- .../video_coding/fec_controller_unittest.cc | 4 +- .../modules/video_coding/nack_requester.cc | 56 +------ .../modules/video_coding/nack_requester.h | 10 +- .../video_coding/nack_requester_unittest.cc | 160 ++++-------------- .../modules/video_coding/utility/qp_parser.cc | 16 +- .../modules/video_coding/utility/qp_parser.h | 18 ++ 72 files changed, 1041 insertions(+), 453 deletions(-) (limited to 'third_party/libwebrtc/modules') diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/builtin_audio_decoder_factory_unittest.cc b/third_party/libwebrtc/modules/audio_coding/codecs/builtin_audio_decoder_factory_unittest.cc index bd8d1cc341..998e78e1d3 100644 --- a/third_party/libwebrtc/modules/audio_coding/codecs/builtin_audio_decoder_factory_unittest.cc +++ b/third_party/libwebrtc/modules/audio_coding/codecs/builtin_audio_decoder_factory_unittest.cc @@ -146,7 +146,7 @@ TEST(AudioDecoderFactoryTest, CreateOpus) { for (int hz : {8000, 16000, 32000, 48000}) { for (int channels : {0, 1, 2, 3}) { for (std::string stereo : {"XX", "0", "1", "2"}) { - SdpAudioFormat::Parameters params; + CodecParameterMap params; if (stereo != "XX") { params["stereo"] = stereo; } diff --git a/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc index a2ebe43bbe..f82ef965db 100644 --- a/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc +++ b/third_party/libwebrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc @@ -39,7 +39,7 @@ constexpr int kDefaultOpusPacSize = 960; constexpr int64_t kInitialTimeUs = 12345678; AudioEncoderOpusConfig CreateConfigWithParameters( - const SdpAudioFormat::Parameters& params) { + const CodecParameterMap& params) { const SdpAudioFormat format("opus", 48000, 2, params); return *AudioEncoderOpus::SdpToConfig(format); } diff --git a/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_opus_quality_test.cc b/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_opus_quality_test.cc index 5a2df24ef6..eddacd3680 100644 --- a/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_opus_quality_test.cc +++ b/third_party/libwebrtc/modules/audio_coding/neteq/test/neteq_opus_quality_test.cc @@ -106,8 +106,8 @@ NetEqOpusQualityTest::NetEqOpusQualityTest() // Redefine decoder type if input is stereo. if (channels_ > 1) { - audio_format_ = SdpAudioFormat("opus", 48000, 2, - SdpAudioFormat::Parameters{{"stereo", "1"}}); + audio_format_ = + SdpAudioFormat("opus", 48000, 2, CodecParameterMap{{"stereo", "1"}}); } application_ = absl::GetFlag(FLAGS_application); } diff --git a/third_party/libwebrtc/modules/audio_coding/test/TestStereo.cc b/third_party/libwebrtc/modules/audio_coding/test/TestStereo.cc index 94a1576026..1e65e4a219 100644 --- a/third_party/libwebrtc/modules/audio_coding/test/TestStereo.cc +++ b/third_party/libwebrtc/modules/audio_coding/test/TestStereo.cc @@ -469,7 +469,7 @@ void TestStereo::RegisterSendCodec(char side, : sampling_freq_hz; const std::string ptime = rtc::ToString(rtc::CheckedDivExact( pack_size, rtc::CheckedDivExact(sampling_freq_hz, 1000))); - SdpAudioFormat::Parameters params = {{"ptime", ptime}}; + CodecParameterMap params = {{"ptime", ptime}}; RTC_CHECK(channels == 1 || channels == 2); if (absl::EqualsIgnoreCase(codec_name, "opus")) { if (channels == 2) { diff --git a/third_party/libwebrtc/modules/audio_device/BUILD.gn b/third_party/libwebrtc/modules/audio_device/BUILD.gn index a135f042db..1672be3f95 100644 --- a/third_party/libwebrtc/modules/audio_device/BUILD.gn +++ b/third_party/libwebrtc/modules/audio_device/BUILD.gn @@ -91,6 +91,7 @@ if (!build_with_mozilla) { # See Bug 1820869. "../../system_wrappers", "../../system_wrappers:metrics", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } } @@ -213,7 +214,6 @@ if (!build_with_chromium) { "../../rtc_base:platform_thread", "../../rtc_base:random", "../../rtc_base:rtc_event", - "../../rtc_base:rtc_task_queue", "../../rtc_base:safe_conversions", "../../rtc_base:timeutils", "../../rtc_base/synchronization:mutex", @@ -461,6 +461,7 @@ rtc_source_set("mock_audio_device") { "../../api:make_ref_counted", "../../test:test_support", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } } diff --git a/third_party/libwebrtc/modules/audio_device/fine_audio_buffer.cc b/third_party/libwebrtc/modules/audio_device/fine_audio_buffer.cc index 86240da196..f483b8dc79 100644 --- a/third_party/libwebrtc/modules/audio_device/fine_audio_buffer.cc +++ b/third_party/libwebrtc/modules/audio_device/fine_audio_buffer.cc @@ -13,6 +13,7 @@ #include #include +#include "api/array_view.h" #include "modules/audio_device/audio_device_buffer.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" @@ -107,7 +108,8 @@ void FineAudioBuffer::GetPlayoutData(rtc::ArrayView audio_buffer, void FineAudioBuffer::DeliverRecordedData( rtc::ArrayView audio_buffer, - int record_delay_ms) { + int record_delay_ms, + absl::optional capture_time_ns) { RTC_DCHECK(IsReadyForRecord()); // Always append new data and grow the buffer when needed. record_buffer_.AppendData(audio_buffer.data(), audio_buffer.size()); @@ -118,7 +120,8 @@ void FineAudioBuffer::DeliverRecordedData( record_channels_ * record_samples_per_channel_10ms_; while (record_buffer_.size() >= num_elements_10ms) { audio_device_buffer_->SetRecordedBuffer(record_buffer_.data(), - record_samples_per_channel_10ms_); + record_samples_per_channel_10ms_, + capture_time_ns); audio_device_buffer_->SetVQEData(playout_delay_ms_, record_delay_ms); audio_device_buffer_->DeliverRecordedData(); memmove(record_buffer_.data(), record_buffer_.data() + num_elements_10ms, diff --git a/third_party/libwebrtc/modules/audio_device/fine_audio_buffer.h b/third_party/libwebrtc/modules/audio_device/fine_audio_buffer.h index a6c3042bb2..7af41d3b21 100644 --- a/third_party/libwebrtc/modules/audio_device/fine_audio_buffer.h +++ b/third_party/libwebrtc/modules/audio_device/fine_audio_buffer.h @@ -11,6 +11,10 @@ #ifndef MODULES_AUDIO_DEVICE_FINE_AUDIO_BUFFER_H_ #define MODULES_AUDIO_DEVICE_FINE_AUDIO_BUFFER_H_ +#include +#include + +#include "absl/types/optional.h" #include "api/array_view.h" #include "rtc_base/buffer.h" @@ -61,7 +65,12 @@ class FineAudioBuffer { // 5ms of data and sends a total of 10ms to WebRTC and clears the internal // cache. Call #3 restarts the scheme above. void DeliverRecordedData(rtc::ArrayView audio_buffer, - int record_delay_ms); + int record_delay_ms) { + DeliverRecordedData(audio_buffer, record_delay_ms, absl::nullopt); + } + void DeliverRecordedData(rtc::ArrayView audio_buffer, + int record_delay_ms, + absl::optional capture_time_ns); private: // Device buffer that works with 10ms chunks of data both for playout and diff --git a/third_party/libwebrtc/modules/audio_device/fine_audio_buffer_unittest.cc b/third_party/libwebrtc/modules/audio_device/fine_audio_buffer_unittest.cc index 36ea85f7dd..bb9fe63922 100644 --- a/third_party/libwebrtc/modules/audio_device/fine_audio_buffer_unittest.cc +++ b/third_party/libwebrtc/modules/audio_device/fine_audio_buffer_unittest.cc @@ -113,7 +113,7 @@ void RunFineBufferTest(int frame_size_in_samples) { { InSequence s; for (int j = 0; j < kNumberOfUpdateBufferCalls - 1; ++j) { - EXPECT_CALL(audio_device_buffer, SetRecordedBuffer(_, kSamplesPer10Ms)) + EXPECT_CALL(audio_device_buffer, SetRecordedBuffer(_, kSamplesPer10Ms, _)) .WillOnce(VerifyInputBuffer(j, kChannels * kSamplesPer10Ms)) .RetiresOnSaturation(); } diff --git a/third_party/libwebrtc/modules/audio_device/include/test_audio_device.cc b/third_party/libwebrtc/modules/audio_device/include/test_audio_device.cc index 4c29c98f2c..b3923ae67d 100644 --- a/third_party/libwebrtc/modules/audio_device/include/test_audio_device.cc +++ b/third_party/libwebrtc/modules/audio_device/include/test_audio_device.cc @@ -21,6 +21,7 @@ #include "absl/strings/string_view.h" #include "api/array_view.h" #include "api/make_ref_counted.h" +#include "api/task_queue/task_queue_factory.h" #include "common_audio/wav_file.h" #include "modules/audio_device/audio_device_impl.h" #include "modules/audio_device/include/audio_device_default.h" @@ -33,7 +34,6 @@ #include "rtc_base/platform_thread.h" #include "rtc_base/random.h" #include "rtc_base/synchronization/mutex.h" -#include "rtc_base/task_queue.h" #include "rtc_base/task_utils/repeating_task.h" #include "rtc_base/thread_annotations.h" #include "rtc_base/time_utils.h" diff --git a/third_party/libwebrtc/modules/audio_device/mock_audio_device_buffer.h b/third_party/libwebrtc/modules/audio_device/mock_audio_device_buffer.h index b0f54c20ff..0b276185da 100644 --- a/third_party/libwebrtc/modules/audio_device/mock_audio_device_buffer.h +++ b/third_party/libwebrtc/modules/audio_device/mock_audio_device_buffer.h @@ -11,6 +11,7 @@ #ifndef MODULES_AUDIO_DEVICE_MOCK_AUDIO_DEVICE_BUFFER_H_ #define MODULES_AUDIO_DEVICE_MOCK_AUDIO_DEVICE_BUFFER_H_ +#include "absl/types/optional.h" #include "modules/audio_device/audio_device_buffer.h" #include "test/gmock.h" @@ -24,7 +25,9 @@ class MockAudioDeviceBuffer : public AudioDeviceBuffer { MOCK_METHOD(int32_t, GetPlayoutData, (void* audioBuffer), (override)); MOCK_METHOD(int32_t, SetRecordedBuffer, - (const void* audioBuffer, size_t nSamples), + (const void* audioBuffer, + size_t nSamples, + absl::optional capture_time_ns), (override)); MOCK_METHOD(void, SetVQEData, (int playDelayMS, int recDelayMS), (override)); MOCK_METHOD(int32_t, DeliverRecordedData, (), (override)); diff --git a/third_party/libwebrtc/modules/audio_device/test_audio_device_impl.cc b/third_party/libwebrtc/modules/audio_device/test_audio_device_impl.cc index 627e68b36f..a3742ea581 100644 --- a/third_party/libwebrtc/modules/audio_device/test_audio_device_impl.cc +++ b/third_party/libwebrtc/modules/audio_device/test_audio_device_impl.cc @@ -19,7 +19,6 @@ #include "modules/audio_device/include/test_audio_device.h" #include "rtc_base/checks.h" #include "rtc_base/synchronization/mutex.h" -#include "rtc_base/task_queue.h" #include "rtc_base/task_utils/repeating_task.h" namespace webrtc { @@ -59,11 +58,10 @@ TestAudioDevice::TestAudioDevice( } AudioDeviceGeneric::InitStatus TestAudioDevice::Init() { - task_queue_ = - std::make_unique(task_queue_factory_->CreateTaskQueue( - "TestAudioDeviceModuleImpl", TaskQueueFactory::Priority::NORMAL)); + task_queue_ = task_queue_factory_->CreateTaskQueue( + "TestAudioDeviceModuleImpl", TaskQueueFactory::Priority::NORMAL); - RepeatingTaskHandle::Start(task_queue_->Get(), [this]() { + RepeatingTaskHandle::Start(task_queue_.get(), [this]() { ProcessAudio(); return TimeDelta::Micros(process_interval_us_); }); diff --git a/third_party/libwebrtc/modules/audio_device/test_audio_device_impl.h b/third_party/libwebrtc/modules/audio_device/test_audio_device_impl.h index 36192b7f7f..84b48948ba 100644 --- a/third_party/libwebrtc/modules/audio_device/test_audio_device_impl.h +++ b/third_party/libwebrtc/modules/audio_device/test_audio_device_impl.h @@ -14,6 +14,7 @@ #include #include +#include "api/task_queue/task_queue_base.h" #include "api/task_queue/task_queue_factory.h" #include "modules/audio_device/audio_device_buffer.h" #include "modules/audio_device/audio_device_generic.h" @@ -22,7 +23,6 @@ #include "modules/audio_device/include/test_audio_device.h" #include "rtc_base/buffer.h" #include "rtc_base/synchronization/mutex.h" -#include "rtc_base/task_queue.h" namespace webrtc { @@ -190,7 +190,7 @@ class TestAudioDevice : public AudioDeviceGeneric { std::vector playout_buffer_ RTC_GUARDED_BY(lock_); rtc::BufferT recording_buffer_ RTC_GUARDED_BY(lock_); - std::unique_ptr task_queue_; + std::unique_ptr task_queue_; }; } // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_processing/BUILD.gn b/third_party/libwebrtc/modules/audio_processing/BUILD.gn index 6aca7dee46..817a3515b0 100644 --- a/third_party/libwebrtc/modules/audio_processing/BUILD.gn +++ b/third_party/libwebrtc/modules/audio_processing/BUILD.gn @@ -34,6 +34,7 @@ rtc_library("api") { "../../api/audio:aec3_config", "../../api/audio:audio_frame_api", "../../api/audio:echo_control", + "../../api/task_queue", "../../rtc_base:macromagic", "../../rtc_base:refcount", "../../rtc_base:stringutils", @@ -43,6 +44,7 @@ rtc_library("api") { "agc:gain_control_interface", ] absl_deps = [ + "//third_party/abseil-cpp/absl/base:nullability", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -185,6 +187,7 @@ rtc_library("audio_processing") { "../../api/audio:aec3_config", "../../api/audio:audio_frame_api", "../../api/audio:echo_control", + "../../api/task_queue", "../../audio/utility:audio_frame_operations", "../../common_audio:common_audio_c", "../../common_audio/third_party/ooura:fft_size_256", @@ -217,6 +220,7 @@ rtc_library("audio_processing") { "vad", ] absl_deps = [ + "//third_party/abseil-cpp/absl/base:nullability", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -337,9 +341,13 @@ if (rtc_include_tests) { ":audio_buffer", ":audio_processing", ":audio_processing_statistics", + "../../api/task_queue", "../../test:test_support", ] - absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/base:nullability", + "//third_party/abseil-cpp/absl/strings", + ] } if (!build_with_chromium) { diff --git a/third_party/libwebrtc/modules/audio_processing/aec_dump/BUILD.gn b/third_party/libwebrtc/modules/audio_processing/aec_dump/BUILD.gn index 78bae56835..5193e28dff 100644 --- a/third_party/libwebrtc/modules/audio_processing/aec_dump/BUILD.gn +++ b/third_party/libwebrtc/modules/audio_processing/aec_dump/BUILD.gn @@ -14,10 +14,14 @@ rtc_source_set("aec_dump") { deps = [ "..:aec_dump_interface", + "../../../api/task_queue", "../../../rtc_base/system:file_wrapper", "../../../rtc_base/system:rtc_export", ] - absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/base:nullability", + "//third_party/abseil-cpp/absl/strings", + ] } if (rtc_include_tests) { @@ -71,11 +75,13 @@ if (rtc_enable_protobuf) { "../../../rtc_base:protobuf_utils", "../../../rtc_base:race_checker", "../../../rtc_base:rtc_event", - "../../../rtc_base:rtc_task_queue", "../../../rtc_base/system:file_wrapper", "../../../system_wrappers", ] - absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/base:nullability", + "//third_party/abseil-cpp/absl/strings", + ] deps += [ "../:audioproc_debug_proto" ] } @@ -106,6 +112,10 @@ rtc_library("null_aec_dump_factory") { deps = [ ":aec_dump", "..:aec_dump_interface", + "../../../api/task_queue", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/base:nullability", + "//third_party/abseil-cpp/absl/strings", ] - absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } diff --git a/third_party/libwebrtc/modules/audio_processing/aec_dump/aec_dump_factory.h b/third_party/libwebrtc/modules/audio_processing/aec_dump/aec_dump_factory.h index 20718c3d7f..0d258a9ebc 100644 --- a/third_party/libwebrtc/modules/audio_processing/aec_dump/aec_dump_factory.h +++ b/third_party/libwebrtc/modules/audio_processing/aec_dump/aec_dump_factory.h @@ -13,34 +13,34 @@ #include +#include "absl/base/nullability.h" #include "absl/strings/string_view.h" +#include "api/task_queue/task_queue_base.h" #include "modules/audio_processing/include/aec_dump.h" #include "rtc_base/system/file_wrapper.h" #include "rtc_base/system/rtc_export.h" -namespace rtc { -class TaskQueue; -} // namespace rtc - namespace webrtc { class RTC_EXPORT AecDumpFactory { public: - // The `worker_queue` may not be null and must outlive the created - // AecDump instance. `max_log_size_bytes == -1` means the log size - // will be unlimited. `handle` may not be null. The AecDump takes - // responsibility for `handle` and closes it in the destructor. A - // non-null return value indicates that the file has been + // The `worker_queue` must outlive the created AecDump instance. + // `max_log_size_bytes == -1` means the log size will be unlimited. + // The AecDump takes responsibility for `handle` and closes it in the + // destructor. A non-null return value indicates that the file has been // sucessfully opened. - static std::unique_ptr Create(webrtc::FileWrapper file, - int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue); - static std::unique_ptr Create(absl::string_view file_name, - int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue); - static std::unique_ptr Create(FILE* handle, - int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue); + static absl::Nullable> Create( + FileWrapper file, + int64_t max_log_size_bytes, + absl::Nonnull worker_queue); + static absl::Nullable> Create( + absl::string_view file_name, + int64_t max_log_size_bytes, + absl::Nonnull worker_queue); + static absl::Nullable> Create( + absl::Nonnull handle, + int64_t max_log_size_bytes, + absl::Nonnull worker_queue); }; } // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_processing/aec_dump/aec_dump_impl.cc b/third_party/libwebrtc/modules/audio_processing/aec_dump/aec_dump_impl.cc index 94c24048e0..8484fcc6e1 100644 --- a/third_party/libwebrtc/modules/audio_processing/aec_dump/aec_dump_impl.cc +++ b/third_party/libwebrtc/modules/audio_processing/aec_dump/aec_dump_impl.cc @@ -13,11 +13,12 @@ #include #include +#include "absl/base/nullability.h" #include "absl/strings/string_view.h" +#include "api/task_queue/task_queue_base.h" #include "modules/audio_processing/aec_dump/aec_dump_factory.h" #include "rtc_base/checks.h" #include "rtc_base/event.h" -#include "rtc_base/task_queue.h" namespace webrtc { @@ -59,7 +60,7 @@ void CopyFromConfigToEvent(const webrtc::InternalAPMConfig& config, AecDumpImpl::AecDumpImpl(FileWrapper debug_file, int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue) + absl::Nonnull worker_queue) : debug_file_(std::move(debug_file)), num_bytes_left_for_log_(max_log_size_bytes), worker_queue_(worker_queue) {} @@ -254,9 +255,10 @@ void AecDumpImpl::PostWriteToFileTask(std::unique_ptr event) { }); } -std::unique_ptr AecDumpFactory::Create(webrtc::FileWrapper file, - int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue) { +absl::Nullable> AecDumpFactory::Create( + FileWrapper file, + int64_t max_log_size_bytes, + absl::Nonnull worker_queue) { RTC_DCHECK(worker_queue); if (!file.is_open()) return nullptr; @@ -265,16 +267,18 @@ std::unique_ptr AecDumpFactory::Create(webrtc::FileWrapper file, worker_queue); } -std::unique_ptr AecDumpFactory::Create(absl::string_view file_name, - int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue) { +absl::Nullable> AecDumpFactory::Create( + absl::string_view file_name, + int64_t max_log_size_bytes, + absl::Nonnull worker_queue) { return Create(FileWrapper::OpenWriteOnly(file_name), max_log_size_bytes, worker_queue); } -std::unique_ptr AecDumpFactory::Create(FILE* handle, - int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue) { +absl::Nullable> AecDumpFactory::Create( + absl::Nonnull handle, + int64_t max_log_size_bytes, + absl::Nonnull worker_queue) { return Create(FileWrapper(handle), max_log_size_bytes, worker_queue); } diff --git a/third_party/libwebrtc/modules/audio_processing/aec_dump/aec_dump_impl.h b/third_party/libwebrtc/modules/audio_processing/aec_dump/aec_dump_impl.h index 429808f9af..d5af31b01e 100644 --- a/third_party/libwebrtc/modules/audio_processing/aec_dump/aec_dump_impl.h +++ b/third_party/libwebrtc/modules/audio_processing/aec_dump/aec_dump_impl.h @@ -15,11 +15,11 @@ #include #include +#include "api/task_queue/task_queue_base.h" #include "modules/audio_processing/aec_dump/capture_stream_info.h" #include "modules/audio_processing/include/aec_dump.h" #include "rtc_base/race_checker.h" #include "rtc_base/system/file_wrapper.h" -#include "rtc_base/task_queue.h" #include "rtc_base/thread_annotations.h" // Files generated at build-time by the protobuf compiler. @@ -39,7 +39,7 @@ class AecDumpImpl : public AecDump { // `max_log_size_bytes == -1` means the log size will be unlimited. AecDumpImpl(FileWrapper debug_file, int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue); + absl::Nonnull worker_queue); AecDumpImpl(const AecDumpImpl&) = delete; AecDumpImpl& operator=(const AecDumpImpl&) = delete; ~AecDumpImpl() override; @@ -74,7 +74,7 @@ class AecDumpImpl : public AecDump { FileWrapper debug_file_; int64_t num_bytes_left_for_log_ = 0; rtc::RaceChecker race_checker_; - rtc::TaskQueue* worker_queue_; + absl::Nonnull worker_queue_; CaptureStreamInfo capture_stream_info_; }; } // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_processing/aec_dump/aec_dump_unittest.cc b/third_party/libwebrtc/modules/audio_processing/aec_dump/aec_dump_unittest.cc index 62f896fe14..2a8110c4fc 100644 --- a/third_party/libwebrtc/modules/audio_processing/aec_dump/aec_dump_unittest.cc +++ b/third_party/libwebrtc/modules/audio_processing/aec_dump/aec_dump_unittest.cc @@ -28,7 +28,7 @@ TEST(AecDumper, APICallsDoNotCrash) { { std::unique_ptr aec_dump = - webrtc::AecDumpFactory::Create(filename, -1, &file_writer_queue); + webrtc::AecDumpFactory::Create(filename, -1, file_writer_queue.Get()); constexpr int kNumChannels = 1; constexpr int kNumSamplesPerChannel = 160; @@ -63,7 +63,7 @@ TEST(AecDumper, WriteToFile) { { std::unique_ptr aec_dump = - webrtc::AecDumpFactory::Create(filename, -1, &file_writer_queue); + webrtc::AecDumpFactory::Create(filename, -1, file_writer_queue.Get()); constexpr int kNumChannels = 1; constexpr int kNumSamplesPerChannel = 160; diff --git a/third_party/libwebrtc/modules/audio_processing/aec_dump/null_aec_dump_factory.cc b/third_party/libwebrtc/modules/audio_processing/aec_dump/null_aec_dump_factory.cc index 9bd9745069..63929afac4 100644 --- a/third_party/libwebrtc/modules/audio_processing/aec_dump/null_aec_dump_factory.cc +++ b/third_party/libwebrtc/modules/audio_processing/aec_dump/null_aec_dump_factory.cc @@ -8,27 +8,32 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "absl/base/nullability.h" #include "absl/strings/string_view.h" +#include "api/task_queue/task_queue_base.h" #include "modules/audio_processing/aec_dump/aec_dump_factory.h" #include "modules/audio_processing/include/aec_dump.h" namespace webrtc { -std::unique_ptr AecDumpFactory::Create(webrtc::FileWrapper file, - int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue) { +absl::Nullable> AecDumpFactory::Create( + FileWrapper file, + int64_t max_log_size_bytes, + absl::Nonnull worker_queue) { return nullptr; } -std::unique_ptr AecDumpFactory::Create(absl::string_view file_name, - int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue) { +absl::Nullable> AecDumpFactory::Create( + absl::string_view file_name, + int64_t max_log_size_bytes, + absl::Nonnull worker_queue) { return nullptr; } -std::unique_ptr AecDumpFactory::Create(FILE* handle, - int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue) { +absl::Nullable> AecDumpFactory::Create( + absl::Nonnull handle, + int64_t max_log_size_bytes, + absl::Nonnull worker_queue) { return nullptr; } } // namespace webrtc diff --git a/third_party/libwebrtc/modules/audio_processing/audio_processing_impl.cc b/third_party/libwebrtc/modules/audio_processing/audio_processing_impl.cc index c304453388..4ac074526c 100644 --- a/third_party/libwebrtc/modules/audio_processing/audio_processing_impl.cc +++ b/third_party/libwebrtc/modules/audio_processing/audio_processing_impl.cc @@ -18,11 +18,13 @@ #include #include +#include "absl/base/nullability.h" #include "absl/strings/match.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "api/array_view.h" #include "api/audio/audio_frame.h" +#include "api/task_queue/task_queue_base.h" #include "common_audio/audio_converter.h" #include "common_audio/include/audio_util.h" #include "modules/audio_processing/aec_dump/aec_dump_factory.h" @@ -2083,9 +2085,10 @@ void AudioProcessingImpl::UpdateRecommendedInputVolumeLocked() { capture_.recommended_input_volume = capture_.applied_input_volume; } -bool AudioProcessingImpl::CreateAndAttachAecDump(absl::string_view file_name, - int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue) { +bool AudioProcessingImpl::CreateAndAttachAecDump( + absl::string_view file_name, + int64_t max_log_size_bytes, + absl::Nonnull worker_queue) { std::unique_ptr aec_dump = AecDumpFactory::Create(file_name, max_log_size_bytes, worker_queue); if (!aec_dump) { @@ -2096,9 +2099,10 @@ bool AudioProcessingImpl::CreateAndAttachAecDump(absl::string_view file_name, return true; } -bool AudioProcessingImpl::CreateAndAttachAecDump(FILE* handle, - int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue) { +bool AudioProcessingImpl::CreateAndAttachAecDump( + FILE* handle, + int64_t max_log_size_bytes, + absl::Nonnull worker_queue) { std::unique_ptr aec_dump = AecDumpFactory::Create(handle, max_log_size_bytes, worker_queue); if (!aec_dump) { diff --git a/third_party/libwebrtc/modules/audio_processing/audio_processing_impl.h b/third_party/libwebrtc/modules/audio_processing/audio_processing_impl.h index 1e058b5a32..2c0ab198db 100644 --- a/third_party/libwebrtc/modules/audio_processing/audio_processing_impl.h +++ b/third_party/libwebrtc/modules/audio_processing/audio_processing_impl.h @@ -19,10 +19,12 @@ #include #include +#include "absl/base/nullability.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "api/array_view.h" #include "api/function_view.h" +#include "api/task_queue/task_queue_base.h" #include "modules/audio_processing/aec3/echo_canceller3.h" #include "modules/audio_processing/agc/agc_manager_direct.h" #include "modules/audio_processing/agc/gain_control.h" @@ -71,12 +73,14 @@ class AudioProcessingImpl : public AudioProcessing { int Initialize() override; int Initialize(const ProcessingConfig& processing_config) override; void ApplyConfig(const AudioProcessing::Config& config) override; - bool CreateAndAttachAecDump(absl::string_view file_name, - int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue) override; - bool CreateAndAttachAecDump(FILE* handle, - int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue) override; + bool CreateAndAttachAecDump( + absl::string_view file_name, + int64_t max_log_size_bytes, + absl::Nonnull worker_queue) override; + bool CreateAndAttachAecDump( + FILE* handle, + int64_t max_log_size_bytes, + absl::Nonnull worker_queue) override; // TODO(webrtc:5298) Deprecated variant. void AttachAecDump(std::unique_ptr aec_dump) override; void DetachAecDump() override; diff --git a/third_party/libwebrtc/modules/audio_processing/audio_processing_unittest.cc b/third_party/libwebrtc/modules/audio_processing/audio_processing_unittest.cc index c2bedb2da4..2d3684e9b5 100644 --- a/third_party/libwebrtc/modules/audio_processing/audio_processing_unittest.cc +++ b/third_party/libwebrtc/modules/audio_processing/audio_processing_unittest.cc @@ -1485,8 +1485,8 @@ void ApmTest::ProcessDebugDump(absl::string_view in_filename, if (first_init) { // AttachAecDump() writes an additional init message. Don't start // recording until after the first init to avoid the extra message. - auto aec_dump = - AecDumpFactory::Create(out_filename, max_size_bytes, &worker_queue); + auto aec_dump = AecDumpFactory::Create(out_filename, max_size_bytes, + worker_queue.Get()); EXPECT_TRUE(aec_dump); apm_->AttachAecDump(std::move(aec_dump)); first_init = false; @@ -1632,7 +1632,7 @@ TEST_F(ApmTest, DebugDump) { const std::string filename = test::TempFilename(test::OutputPath(), "debug_aec"); { - auto aec_dump = AecDumpFactory::Create("", -1, &worker_queue); + auto aec_dump = AecDumpFactory::Create("", -1, worker_queue.Get()); EXPECT_FALSE(aec_dump); } @@ -1640,7 +1640,7 @@ TEST_F(ApmTest, DebugDump) { // Stopping without having started should be OK. apm_->DetachAecDump(); - auto aec_dump = AecDumpFactory::Create(filename, -1, &worker_queue); + auto aec_dump = AecDumpFactory::Create(filename, -1, worker_queue.Get()); EXPECT_TRUE(aec_dump); apm_->AttachAecDump(std::move(aec_dump)); EXPECT_EQ(apm_->kNoError, @@ -1683,7 +1683,7 @@ TEST_F(ApmTest, DebugDumpFromFileHandle) { // Stopping without having started should be OK. apm_->DetachAecDump(); - auto aec_dump = AecDumpFactory::Create(std::move(f), -1, &worker_queue); + auto aec_dump = AecDumpFactory::Create(std::move(f), -1, worker_queue.Get()); EXPECT_TRUE(aec_dump); apm_->AttachAecDump(std::move(aec_dump)); EXPECT_EQ(apm_->kNoError, diff --git a/third_party/libwebrtc/modules/audio_processing/include/audio_processing.h b/third_party/libwebrtc/modules/audio_processing/include/audio_processing.h index e3223513af..dd484be4f1 100644 --- a/third_party/libwebrtc/modules/audio_processing/include/audio_processing.h +++ b/third_party/libwebrtc/modules/audio_processing/include/audio_processing.h @@ -23,6 +23,7 @@ #include +#include "absl/base/nullability.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "api/array_view.h" @@ -30,15 +31,12 @@ #include "api/audio/echo_control.h" #include "api/ref_count.h" #include "api/scoped_refptr.h" +#include "api/task_queue/task_queue_base.h" #include "modules/audio_processing/include/audio_processing_statistics.h" #include "rtc_base/arraysize.h" #include "rtc_base/system/file_wrapper.h" #include "rtc_base/system/rtc_export.h" -namespace rtc { -class TaskQueue; -} // namespace rtc - namespace webrtc { class AecDump; @@ -632,12 +630,14 @@ class RTC_EXPORT AudioProcessing : public RefCountInterface { // return value of true indicates that the file has been // sucessfully opened, while a value of false indicates that // opening the file failed. - virtual bool CreateAndAttachAecDump(absl::string_view file_name, - int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue) = 0; - virtual bool CreateAndAttachAecDump(FILE* handle, - int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue) = 0; + virtual bool CreateAndAttachAecDump( + absl::string_view file_name, + int64_t max_log_size_bytes, + absl::Nonnull worker_queue) = 0; + virtual bool CreateAndAttachAecDump( + absl::Nonnull handle, + int64_t max_log_size_bytes, + absl::Nonnull worker_queue) = 0; // TODO(webrtc:5298) Deprecated variant. // Attaches provided webrtc::AecDump for recording debugging diff --git a/third_party/libwebrtc/modules/audio_processing/include/mock_audio_processing.h b/third_party/libwebrtc/modules/audio_processing/include/mock_audio_processing.h index 2ea1a865c3..dfe7d84e07 100644 --- a/third_party/libwebrtc/modules/audio_processing/include/mock_audio_processing.h +++ b/third_party/libwebrtc/modules/audio_processing/include/mock_audio_processing.h @@ -13,7 +13,9 @@ #include +#include "absl/base/nullability.h" #include "absl/strings/string_view.h" +#include "api/task_queue/task_queue_base.h" #include "modules/audio_processing/include/aec_dump.h" #include "modules/audio_processing/include/audio_processing.h" #include "modules/audio_processing/include/audio_processing_statistics.h" @@ -155,13 +157,13 @@ class MockAudioProcessing : public AudioProcessing { CreateAndAttachAecDump, (absl::string_view file_name, int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue), + absl::Nonnull worker_queue), (override)); MOCK_METHOD(bool, CreateAndAttachAecDump, (FILE * handle, int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue), + absl::Nonnull worker_queue), (override)); MOCK_METHOD(void, AttachAecDump, (std::unique_ptr), (override)); MOCK_METHOD(void, DetachAecDump, (), (override)); diff --git a/third_party/libwebrtc/modules/audio_processing/test/audio_processing_simulator.cc b/third_party/libwebrtc/modules/audio_processing/test/audio_processing_simulator.cc index 7bd6da0133..500005f26a 100644 --- a/third_party/libwebrtc/modules/audio_processing/test/audio_processing_simulator.cc +++ b/third_party/libwebrtc/modules/audio_processing/test/audio_processing_simulator.cc @@ -622,7 +622,7 @@ void AudioProcessingSimulator::ConfigureAudioProcessor() { if (settings_.aec_dump_output_filename) { ap_->AttachAecDump(AecDumpFactory::Create( - *settings_.aec_dump_output_filename, -1, &worker_queue_)); + *settings_.aec_dump_output_filename, -1, worker_queue_.Get())); } } diff --git a/third_party/libwebrtc/modules/audio_processing/test/debug_dump_test.cc b/third_party/libwebrtc/modules/audio_processing/test/debug_dump_test.cc index cded5de217..0d3eefa94a 100644 --- a/third_party/libwebrtc/modules/audio_processing/test/debug_dump_test.cc +++ b/third_party/libwebrtc/modules/audio_processing/test/debug_dump_test.cc @@ -197,7 +197,7 @@ void DebugDumpGenerator::SetOutputChannels(int channels) { void DebugDumpGenerator::StartRecording() { apm_->AttachAecDump( - AecDumpFactory::Create(dump_file_name_.c_str(), -1, &worker_queue_)); + AecDumpFactory::Create(dump_file_name_.c_str(), -1, worker_queue_.Get())); } void DebugDumpGenerator::Process(size_t num_blocks) { diff --git a/third_party/libwebrtc/modules/congestion_controller/goog_cc/delay_based_bwe_unittest_helper.cc b/third_party/libwebrtc/modules/congestion_controller/goog_cc/delay_based_bwe_unittest_helper.cc index 16bd4153a6..e56aa4a09c 100644 --- a/third_party/libwebrtc/modules/congestion_controller/goog_cc/delay_based_bwe_unittest_helper.cc +++ b/third_party/libwebrtc/modules/congestion_controller/goog_cc/delay_based_bwe_unittest_helper.cc @@ -26,6 +26,7 @@ #include "modules/congestion_controller/goog_cc/probe_bitrate_estimator.h" #include "rtc_base/checks.h" #include "test/field_trial.h" +#include "test/gtest.h" namespace webrtc { constexpr size_t kMtu = 1200; @@ -52,6 +53,7 @@ RtpStream::RtpStream(int fps, int bitrate_bps) // previous frame, no frame will be generated. The frame is split into // packets. int64_t RtpStream::GenerateFrame(int64_t time_now_us, + int64_t* next_sequence_number, std::vector* packets) { if (time_now_us < next_rtp_time_) { return next_rtp_time_; @@ -66,6 +68,7 @@ int64_t RtpStream::GenerateFrame(int64_t time_now_us, packet.sent_packet.send_time = Timestamp::Micros(time_now_us + kSendSideOffsetUs); packet.sent_packet.size = DataSize::Bytes(payload_size); + packet.sent_packet.sequence_number = (*next_sequence_number)++; packets->push_back(packet); } next_rtp_time_ = time_now_us + (1000000 + fps_ / 2) / fps_; @@ -131,14 +134,15 @@ void StreamGenerator::SetBitrateBps(int bitrate_bps) { // TODO(holmer): Break out the channel simulation part from this class to make // it possible to simulate different types of channels. -int64_t StreamGenerator::GenerateFrame(std::vector* packets, - int64_t time_now_us) { +int64_t StreamGenerator::GenerateFrame(int64_t time_now_us, + int64_t* next_sequence_number, + std::vector* packets) { RTC_CHECK(packets != NULL); RTC_CHECK(packets->empty()); RTC_CHECK_GT(capacity_, 0); auto it = std::min_element(streams_.begin(), streams_.end(), RtpStream::Compare); - (*it)->GenerateFrame(time_now_us, packets); + (*it)->GenerateFrame(time_now_us, next_sequence_number, packets); for (PacketResult& packet : *packets) { int capacity_bpus = capacity_ / 1000; int64_t required_network_time_us = @@ -233,8 +237,8 @@ bool DelayBasedBweTest::GenerateAndProcessFrame(uint32_t ssrc, stream_generator_->SetBitrateBps(bitrate_bps); std::vector packets; - int64_t next_time_us = - stream_generator_->GenerateFrame(&packets, clock_.TimeInMicroseconds()); + int64_t next_time_us = stream_generator_->GenerateFrame( + clock_.TimeInMicroseconds(), &next_sequence_number_, &packets); if (packets.empty()) return false; diff --git a/third_party/libwebrtc/modules/congestion_controller/goog_cc/delay_based_bwe_unittest_helper.h b/third_party/libwebrtc/modules/congestion_controller/goog_cc/delay_based_bwe_unittest_helper.h index 89eb1a353f..634e6a4d82 100644 --- a/third_party/libwebrtc/modules/congestion_controller/goog_cc/delay_based_bwe_unittest_helper.h +++ b/third_party/libwebrtc/modules/congestion_controller/goog_cc/delay_based_bwe_unittest_helper.h @@ -15,12 +15,11 @@ #include #include -#include #include -#include "absl/strings/string_view.h" #include "api/transport/field_trial_based_config.h" #include "api/transport/network_types.h" +#include "api/units/timestamp.h" #include "modules/congestion_controller/goog_cc/acknowledged_bitrate_estimator.h" #include "modules/congestion_controller/goog_cc/delay_based_bwe.h" #include "system_wrappers/include/clock.h" @@ -61,6 +60,7 @@ class RtpStream { // previous frame, no frame will be generated. The frame is split into // packets. int64_t GenerateFrame(int64_t time_now_us, + int64_t* next_sequence_number, std::vector* packets); // The send-side time when the next frame can be generated. @@ -102,8 +102,9 @@ class StreamGenerator { // TODO(holmer): Break out the channel simulation part from this class to make // it possible to simulate different types of channels. - int64_t GenerateFrame(std::vector* packets, - int64_t time_now_us); + int64_t GenerateFrame(int64_t time_now_us, + int64_t* next_sequence_number, + std::vector* packets); private: // Capacity of the simulated channel in bits per second. diff --git a/third_party/libwebrtc/modules/congestion_controller/goog_cc/send_side_bandwidth_estimation.cc b/third_party/libwebrtc/modules/congestion_controller/goog_cc/send_side_bandwidth_estimation.cc index 22693d67e9..211d86c95d 100644 --- a/third_party/libwebrtc/modules/congestion_controller/goog_cc/send_side_bandwidth_estimation.cc +++ b/third_party/libwebrtc/modules/congestion_controller/goog_cc/send_side_bandwidth_estimation.cc @@ -281,7 +281,8 @@ void SendSideBandwidthEstimation::OnRouteChange() { uma_update_state_ = kNoUpdate; uma_rtt_state_ = kNoUpdate; last_rtc_event_log_ = Timestamp::MinusInfinity(); - if (loss_based_bandwidth_estimator_v2_->UseInStartPhase()) { + if (LossBasedBandwidthEstimatorV2Enabled() && + loss_based_bandwidth_estimator_v2_->UseInStartPhase()) { loss_based_bandwidth_estimator_v2_.reset( new LossBasedBweV2(key_value_config_)); } diff --git a/third_party/libwebrtc/modules/congestion_controller/rtp/BUILD.gn b/third_party/libwebrtc/modules/congestion_controller/rtp/BUILD.gn index cd13332b7f..8ec755e1aa 100644 --- a/third_party/libwebrtc/modules/congestion_controller/rtp/BUILD.gn +++ b/third_party/libwebrtc/modules/congestion_controller/rtp/BUILD.gn @@ -34,7 +34,6 @@ rtc_library("control_handler") { "../../../rtc_base:safe_conversions", "../../../rtc_base:safe_minmax", "../../../rtc_base/system:no_unique_address", - "../../../system_wrappers:field_trial", "../../pacing", ] absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] diff --git a/third_party/libwebrtc/modules/congestion_controller/rtp/control_handler.cc b/third_party/libwebrtc/modules/congestion_controller/rtp/control_handler.cc index da6451c97e..357a0f0798 100644 --- a/third_party/libwebrtc/modules/congestion_controller/rtp/control_handler.cc +++ b/third_party/libwebrtc/modules/congestion_controller/rtp/control_handler.cc @@ -18,21 +18,8 @@ #include "rtc_base/logging.h" #include "rtc_base/numerics/safe_conversions.h" #include "rtc_base/numerics/safe_minmax.h" -#include "system_wrappers/include/field_trial.h" namespace webrtc { -namespace { - -// By default, pacer emergency stops encoder when buffer reaches a high level. -bool IsPacerEmergencyStopDisabled() { - return field_trial::IsEnabled("WebRTC-DisablePacerEmergencyStop"); -} - -} // namespace -CongestionControlHandler::CongestionControlHandler() - : disable_pacer_emergency_stop_(IsPacerEmergencyStopDisabled()) {} - -CongestionControlHandler::~CongestionControlHandler() {} void CongestionControlHandler::SetTargetRate( TargetTransferRate new_target_rate) { @@ -60,9 +47,8 @@ absl::optional CongestionControlHandler::GetUpdate() { bool pause_encoding = false; if (!network_available_) { pause_encoding = true; - } else if (!disable_pacer_emergency_stop_ && - pacer_expected_queue_ms_ > - PacingController::kMaxExpectedQueueLength.ms()) { + } else if (pacer_expected_queue_ms_ > + PacingController::kMaxExpectedQueueLength.ms()) { pause_encoding = true; } if (pause_encoding) diff --git a/third_party/libwebrtc/modules/congestion_controller/rtp/control_handler.h b/third_party/libwebrtc/modules/congestion_controller/rtp/control_handler.h index d8e7263a02..649d2f911e 100644 --- a/third_party/libwebrtc/modules/congestion_controller/rtp/control_handler.h +++ b/third_party/libwebrtc/modules/congestion_controller/rtp/control_handler.h @@ -28,12 +28,13 @@ namespace webrtc { // destruction unless members are properly ordered. class CongestionControlHandler { public: - CongestionControlHandler(); - ~CongestionControlHandler(); + CongestionControlHandler() = default; CongestionControlHandler(const CongestionControlHandler&) = delete; CongestionControlHandler& operator=(const CongestionControlHandler&) = delete; + ~CongestionControlHandler() = default; + void SetTargetRate(TargetTransferRate new_target_rate); void SetNetworkAvailability(bool network_available); void SetPacerQueue(TimeDelta expected_queue_time); @@ -45,7 +46,6 @@ class CongestionControlHandler { bool network_available_ = true; bool encoder_paused_in_last_report_ = false; - const bool disable_pacer_emergency_stop_; int64_t pacer_expected_queue_ms_ = 0; RTC_NO_UNIQUE_ADDRESS SequenceChecker sequenced_checker_; diff --git a/third_party/libwebrtc/modules/pacing/BUILD.gn b/third_party/libwebrtc/modules/pacing/BUILD.gn index ea80c8c819..87498817b6 100644 --- a/third_party/libwebrtc/modules/pacing/BUILD.gn +++ b/third_party/libwebrtc/modules/pacing/BUILD.gn @@ -63,6 +63,7 @@ rtc_library("pacing") { ] absl_deps = [ "//third_party/abseil-cpp/absl/cleanup", + "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/memory", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", diff --git a/third_party/libwebrtc/modules/pacing/pacing_controller.cc b/third_party/libwebrtc/modules/pacing/pacing_controller.cc index 5b81207d56..41f97a37fb 100644 --- a/third_party/libwebrtc/modules/pacing/pacing_controller.cc +++ b/third_party/libwebrtc/modules/pacing/pacing_controller.cc @@ -19,11 +19,11 @@ #include "absl/strings/match.h" #include "api/units/data_size.h" #include "api/units/time_delta.h" +#include "api/units/timestamp.h" #include "modules/pacing/bitrate_prober.h" -#include "modules/pacing/interval_budget.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" -#include "rtc_base/time_utils.h" #include "system_wrappers/include/clock.h" namespace webrtc { @@ -44,8 +44,6 @@ bool IsEnabled(const FieldTrialsView& field_trials, absl::string_view key) { } // namespace -const TimeDelta PacingController::kMaxExpectedQueueLength = - TimeDelta::Millis(2000); const TimeDelta PacingController::kPausedProcessInterval = kCongestedPacketInterval; const TimeDelta PacingController::kMinSleepTime = TimeDelta::Millis(1); @@ -57,11 +55,13 @@ const TimeDelta PacingController::kMaxEarlyProbeProcessing = PacingController::PacingController(Clock* clock, PacketSender* packet_sender, - const FieldTrialsView& field_trials) + const FieldTrialsView& field_trials, + Configuration configuration) : clock_(clock), packet_sender_(packet_sender), field_trials_(field_trials), drain_large_queues_( + configuration.drain_large_queues && !IsDisabled(field_trials_, "WebRTC-Pacer-DrainQueue")), send_padding_if_silent_( IsEnabled(field_trials_, "WebRTC-Pacer-PadInSilence")), @@ -71,9 +71,10 @@ PacingController::PacingController(Clock* clock, fast_retransmissions_( IsEnabled(field_trials_, "WebRTC-Pacer-FastRetransmissions")), keyframe_flushing_( + configuration.keyframe_flushing || IsEnabled(field_trials_, "WebRTC-Pacer-KeyframeFlushing")), transport_overhead_per_packet_(DataSize::Zero()), - send_burst_interval_(kDefaultBurstInterval), + send_burst_interval_(configuration.send_burst_interval), last_timestamp_(clock_->CurrentTime()), paused_(false), media_debt_(DataSize::Zero()), @@ -86,9 +87,11 @@ PacingController::PacingController(Clock* clock, last_process_time_(clock->CurrentTime()), last_send_time_(last_process_time_), seen_first_packet_(false), - packet_queue_(/*creation_time=*/last_process_time_), + packet_queue_(/*creation_time=*/last_process_time_, + configuration.prioritize_audio_retransmission, + configuration.packet_queue_ttl), congested_(false), - queue_time_limit_(kMaxExpectedQueueLength), + queue_time_limit_(configuration.queue_time_limit), account_for_audio_(false), include_overhead_(false), circuit_breaker_threshold_(1 << 16) { @@ -710,8 +713,7 @@ Timestamp PacingController::NextUnpacedSendTime() const { } if (fast_retransmissions_) { Timestamp leading_retransmission_send_time = - packet_queue_.LeadingPacketEnqueueTime( - RtpPacketMediaType::kRetransmission); + packet_queue_.LeadingPacketEnqueueTimeForRetransmission(); if (leading_retransmission_send_time.IsFinite()) { return leading_retransmission_send_time; } diff --git a/third_party/libwebrtc/modules/pacing/pacing_controller.h b/third_party/libwebrtc/modules/pacing/pacing_controller.h index 04e0a820f9..fe6ee737a9 100644 --- a/third_party/libwebrtc/modules/pacing/pacing_controller.h +++ b/third_party/libwebrtc/modules/pacing/pacing_controller.h @@ -67,11 +67,6 @@ class PacingController { } }; - // Expected max pacer delay. If ExpectedQueueTime() is higher than - // this value, the packet producers should wait (eg drop frames rather than - // encoding them). Bitrate sent may temporarily exceed target set by - // UpdateBitrate() so that this limit will be upheld. - static const TimeDelta kMaxExpectedQueueLength; // If no media or paused, wake up at least every `kPausedProcessIntervalMs` in // order to send a keep-alive packet so we don't get stuck in a bad state due // to lack of feedback. @@ -93,14 +88,45 @@ class PacingController { // the send burst interval. // Ex: max send burst interval = 63Kb / 10Mbit/s = 50ms. static constexpr DataSize kMaxBurstSize = DataSize::Bytes(63 * 1000); - // The pacer is allowed to send enqued packets in bursts and can build up a - // packet "debt" that correspond to approximately the send rate during - // the burst interval. + + // Configuration default values. static constexpr TimeDelta kDefaultBurstInterval = TimeDelta::Millis(40); + static constexpr TimeDelta kMaxExpectedQueueLength = TimeDelta::Millis(2000); + + struct Configuration { + // If the pacer queue grows longer than the configured max queue limit, + // pacer sends at the minimum rate needed to keep the max queue limit and + // ignore the current bandwidth estimate. + bool drain_large_queues = true; + // Expected max pacer delay. If ExpectedQueueTime() is higher than + // this value, the packet producers should wait (eg drop frames rather than + // encoding them). Bitrate sent may temporarily exceed target set by + // SetPacingRates() so that this limit will be upheld if + // `drain_large_queues` is set. + TimeDelta queue_time_limit = kMaxExpectedQueueLength; + // If the first packet of a keyframe is enqueued on a RTP stream, pacer + // skips forward to that packet and drops other enqueued packets on that + // stream, unless a keyframe is already being paced. + bool keyframe_flushing = false; + // Audio retransmission is prioritized before video retransmission packets. + bool prioritize_audio_retransmission = false; + // Configure separate timeouts per priority. After a timeout, a packet of + // that sort will not be paced and instead dropped. + // Note: to set TTL on audio retransmission, + // `prioritize_audio_retransmission` must be true. + PacketQueueTTL packet_queue_ttl; + // The pacer is allowed to send enqueued packets in bursts and can build up + // a packet "debt" that correspond to approximately the send rate during the + // burst interval. + TimeDelta send_burst_interval = kDefaultBurstInterval; + }; + + static Configuration DefaultConfiguration() { return Configuration{}; } PacingController(Clock* clock, PacketSender* packet_sender, - const FieldTrialsView& field_trials); + const FieldTrialsView& field_trials, + Configuration configuration = DefaultConfiguration()); ~PacingController(); diff --git a/third_party/libwebrtc/modules/pacing/pacing_controller_unittest.cc b/third_party/libwebrtc/modules/pacing/pacing_controller_unittest.cc index 9e6ede6dc0..2c3a71b369 100644 --- a/third_party/libwebrtc/modules/pacing/pacing_controller_unittest.cc +++ b/third_party/libwebrtc/modules/pacing/pacing_controller_unittest.cc @@ -2348,5 +2348,43 @@ TEST_F(PacingControllerTest, FlushesPacketsOnKeyFrames) { pacer->ProcessPackets(); } +TEST_F(PacingControllerTest, CanControlQueueSizeUsingTtl) { + const uint32_t kSsrc = 12345; + const uint32_t kAudioSsrc = 2345; + uint16_t sequence_number = 1234; + + PacingController::Configuration config; + config.drain_large_queues = false; + config.packet_queue_ttl.video = TimeDelta::Millis(500); + auto pacer = + std::make_unique(&clock_, &callback_, trials_, config); + pacer->SetPacingRates(DataRate::BitsPerSec(100'000), DataRate::Zero()); + + Timestamp send_time = Timestamp::Zero(); + for (int i = 0; i < 100; ++i) { + // Enqueue a new audio and video frame every 33ms. + if (clock_.CurrentTime() - send_time > TimeDelta::Millis(33)) { + for (int j = 0; j < 3; ++j) { + auto packet = BuildPacket(RtpPacketMediaType::kVideo, kSsrc, + /*sequence_number=*/++sequence_number, + /*capture_time_ms=*/2, + /*size_bytes=*/1000); + pacer->EnqueuePacket(std::move(packet)); + } + auto packet = BuildPacket(RtpPacketMediaType::kAudio, kAudioSsrc, + /*sequence_number=*/++sequence_number, + /*capture_time_ms=*/2, + /*size_bytes=*/100); + pacer->EnqueuePacket(std::move(packet)); + send_time = clock_.CurrentTime(); + } + + EXPECT_LE(clock_.CurrentTime() - pacer->OldestPacketEnqueueTime(), + TimeDelta::Millis(500)); + clock_.AdvanceTime(pacer->NextSendTime() - clock_.CurrentTime()); + pacer->ProcessPackets(); + } +} + } // namespace } // namespace webrtc diff --git a/third_party/libwebrtc/modules/pacing/prioritized_packet_queue.cc b/third_party/libwebrtc/modules/pacing/prioritized_packet_queue.cc index ea211ea683..2d0d829648 100644 --- a/third_party/libwebrtc/modules/pacing/prioritized_packet_queue.cc +++ b/third_party/libwebrtc/modules/pacing/prioritized_packet_queue.cc @@ -10,41 +10,70 @@ #include "modules/pacing/prioritized_packet_queue.h" +#include +#include #include +#include "absl/container/inlined_vector.h" +#include "absl/types/optional.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/rtp_packet_to_send.h" #include "rtc_base/checks.h" +#include "rtc_base/logging.h" namespace webrtc { namespace { constexpr int kAudioPrioLevel = 0; -int GetPriorityForType(RtpPacketMediaType type) { +int GetPriorityForType( + RtpPacketMediaType type, + absl::optional original_type) { // Lower number takes priority over higher. switch (type) { case RtpPacketMediaType::kAudio: // Audio is always prioritized over other packet types. return kAudioPrioLevel; case RtpPacketMediaType::kRetransmission: - // Send retransmissions before new media. + // Send retransmissions before new media. If original_type is set, audio + // retransmission is prioritized more than video retransmission. + if (original_type == RtpPacketToSend::OriginalType::kVideo) { + return kAudioPrioLevel + 2; + } return kAudioPrioLevel + 1; case RtpPacketMediaType::kVideo: case RtpPacketMediaType::kForwardErrorCorrection: // Video has "normal" priority, in the old speak. // Send redundancy concurrently to video. If it is delayed it might have a // lower chance of being useful. - return kAudioPrioLevel + 2; + return kAudioPrioLevel + 3; case RtpPacketMediaType::kPadding: // Packets that are in themselves likely useless, only sent to keep the // BWE high. - return kAudioPrioLevel + 3; + return kAudioPrioLevel + 4; } RTC_CHECK_NOTREACHED(); } } // namespace +absl::InlinedVector +PrioritizedPacketQueue::ToTtlPerPrio(PacketQueueTTL packet_queue_ttl) { + absl::InlinedVector + ttl_per_prio(kNumPriorityLevels, TimeDelta::PlusInfinity()); + ttl_per_prio[GetPriorityForType(RtpPacketMediaType::kRetransmission, + RtpPacketToSend::OriginalType::kAudio)] = + packet_queue_ttl.audio_retransmission; + ttl_per_prio[GetPriorityForType(RtpPacketMediaType::kRetransmission, + RtpPacketToSend::OriginalType::kVideo)] = + packet_queue_ttl.video_retransmission; + ttl_per_prio[GetPriorityForType(RtpPacketMediaType::kVideo, absl::nullopt)] = + packet_queue_ttl.video; + return ttl_per_prio; +} + DataSize PrioritizedPacketQueue::QueuedPacket::PacketSize() const { return DataSize::Bytes(packet->payload_size() + packet->padding_size()); } @@ -109,8 +138,13 @@ PrioritizedPacketQueue::StreamQueue::DequeueAll() { return packets_by_prio; } -PrioritizedPacketQueue::PrioritizedPacketQueue(Timestamp creation_time) - : queue_time_sum_(TimeDelta::Zero()), +PrioritizedPacketQueue::PrioritizedPacketQueue( + Timestamp creation_time, + bool prioritize_audio_retransmission, + PacketQueueTTL packet_queue_ttl) + : prioritize_audio_retransmission_(prioritize_audio_retransmission), + time_to_live_per_prio_(ToTtlPerPrio(packet_queue_ttl)), + queue_time_sum_(TimeDelta::Zero()), pause_time_sum_(TimeDelta::Zero()), size_packets_(0), size_packets_per_media_type_({}), @@ -133,7 +167,11 @@ void PrioritizedPacketQueue::Push(Timestamp enqueue_time, enqueue_times_.insert(enqueue_times_.end(), enqueue_time); RTC_DCHECK(packet->packet_type().has_value()); RtpPacketMediaType packet_type = packet->packet_type().value(); - int prio_level = GetPriorityForType(packet_type); + int prio_level = + GetPriorityForType(packet_type, prioritize_audio_retransmission_ + ? packet->original_packet_type() + : absl::nullopt); + PurgeOldPacketsAtPriorityLevel(prio_level, enqueue_time); RTC_DCHECK_GE(prio_level, 0); RTC_DCHECK_LT(prio_level, kNumPriorityLevels); QueuedPacket queued_packed = {.packet = std::move(packet), @@ -214,7 +252,8 @@ PrioritizedPacketQueue::SizeInPacketsPerRtpPacketMediaType() const { Timestamp PrioritizedPacketQueue::LeadingPacketEnqueueTime( RtpPacketMediaType type) const { - const int priority_level = GetPriorityForType(type); + RTC_DCHECK(type != RtpPacketMediaType::kRetransmission); + const int priority_level = GetPriorityForType(type, absl::nullopt); if (streams_by_prio_[priority_level].empty()) { return Timestamp::MinusInfinity(); } @@ -222,6 +261,39 @@ Timestamp PrioritizedPacketQueue::LeadingPacketEnqueueTime( priority_level); } +Timestamp PrioritizedPacketQueue::LeadingPacketEnqueueTimeForRetransmission() + const { + if (!prioritize_audio_retransmission_) { + const int priority_level = + GetPriorityForType(RtpPacketMediaType::kRetransmission, absl::nullopt); + if (streams_by_prio_[priority_level].empty()) { + return Timestamp::PlusInfinity(); + } + return streams_by_prio_[priority_level].front()->LeadingPacketEnqueueTime( + priority_level); + } + const int audio_priority_level = + GetPriorityForType(RtpPacketMediaType::kRetransmission, + RtpPacketToSend::OriginalType::kAudio); + const int video_priority_level = + GetPriorityForType(RtpPacketMediaType::kRetransmission, + RtpPacketToSend::OriginalType::kVideo); + + Timestamp next_audio = + streams_by_prio_[audio_priority_level].empty() + ? Timestamp::PlusInfinity() + : streams_by_prio_[audio_priority_level] + .front() + ->LeadingPacketEnqueueTime(audio_priority_level); + Timestamp next_video = + streams_by_prio_[video_priority_level].empty() + ? Timestamp::PlusInfinity() + : streams_by_prio_[video_priority_level] + .front() + ->LeadingPacketEnqueueTime(video_priority_level); + return std::min(next_audio, next_video); +} + Timestamp PrioritizedPacketQueue::OldestEnqueueTime() const { return enqueue_times_.empty() ? Timestamp::MinusInfinity() : enqueue_times_.front(); @@ -283,9 +355,6 @@ void PrioritizedPacketQueue::RemovePacketsForSsrc(uint32_t ssrc) { // Update the global top prio level if neccessary. RTC_DCHECK(streams_by_prio_[i].front() == &queue); streams_by_prio_[i].pop_front(); - if (i == top_active_prio_level_) { - MaybeUpdateTopPrioLevel(); - } } else { // More than stream had packets at this prio level, filter this one out. std::deque filtered_queue; @@ -298,6 +367,7 @@ void PrioritizedPacketQueue::RemovePacketsForSsrc(uint32_t ssrc) { } } } + MaybeUpdateTopPrioLevel(); } bool PrioritizedPacketQueue::HasKeyframePackets(uint32_t ssrc) const { @@ -340,18 +410,53 @@ void PrioritizedPacketQueue::DequeuePacketInternal(QueuedPacket& packet) { } void PrioritizedPacketQueue::MaybeUpdateTopPrioLevel() { - if (streams_by_prio_[top_active_prio_level_].empty()) { - // No stream queues have packets at this prio level, find top priority - // that is not empty. - if (size_packets_ == 0) { - top_active_prio_level_ = -1; + if (top_active_prio_level_ != -1 && + !streams_by_prio_[top_active_prio_level_].empty()) { + return; + } + // No stream queues have packets at top_active_prio_level_, find top priority + // that is not empty. + for (int i = 0; i < kNumPriorityLevels; ++i) { + PurgeOldPacketsAtPriorityLevel(i, last_update_time_); + if (!streams_by_prio_[i].empty()) { + top_active_prio_level_ = i; + break; + } + } + if (size_packets_ == 0) { + // There are no packets left to send. Last packet may have been purged. Prio + // will change when a new packet is pushed. + top_active_prio_level_ = -1; + } +} + +void PrioritizedPacketQueue::PurgeOldPacketsAtPriorityLevel(int prio_level, + Timestamp now) { + RTC_DCHECK(prio_level >= 0 && prio_level < kNumPriorityLevels); + TimeDelta time_to_live = time_to_live_per_prio_[prio_level]; + if (time_to_live.IsInfinite()) { + return; + } + + std::deque& queues = streams_by_prio_[prio_level]; + auto iter = queues.begin(); + while (iter != queues.end()) { + StreamQueue* queue_ptr = *iter; + while (queue_ptr->HasPacketsAtPrio(prio_level) && + (now - queue_ptr->LeadingPacketEnqueueTime(prio_level)) > + time_to_live) { + QueuedPacket packet = queue_ptr->DequeuePacket(prio_level); + RTC_LOG(LS_INFO) << "Dropping old packet on SSRC: " + << packet.packet->Ssrc() + << " seq:" << packet.packet->SequenceNumber() + << " time in queue:" << (now - packet.enqueue_time).ms() + << " ms"; + DequeuePacketInternal(packet); + } + if (!queue_ptr->HasPacketsAtPrio(prio_level)) { + iter = queues.erase(iter); } else { - for (int i = 0; i < kNumPriorityLevels; ++i) { - if (!streams_by_prio_[i].empty()) { - top_active_prio_level_ = i; - break; - } - } + ++iter; } } } diff --git a/third_party/libwebrtc/modules/pacing/prioritized_packet_queue.h b/third_party/libwebrtc/modules/pacing/prioritized_packet_queue.h index 935c530027..179ef104fe 100644 --- a/third_party/libwebrtc/modules/pacing/prioritized_packet_queue.h +++ b/third_party/libwebrtc/modules/pacing/prioritized_packet_queue.h @@ -18,8 +18,8 @@ #include #include #include -#include +#include "absl/container/inlined_vector.h" #include "api/units/data_size.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" @@ -27,9 +27,19 @@ namespace webrtc { +// Describes how long time a packet may stay in the queue before being dropped. +struct PacketQueueTTL { + TimeDelta audio_retransmission = TimeDelta::PlusInfinity(); + TimeDelta video_retransmission = TimeDelta::PlusInfinity(); + TimeDelta video = TimeDelta::PlusInfinity(); +}; + class PrioritizedPacketQueue { public: - explicit PrioritizedPacketQueue(Timestamp creation_time); + explicit PrioritizedPacketQueue( + Timestamp creation_time, + bool prioritize_audio_retransmission = false, + PacketQueueTTL packet_queue_ttl = PacketQueueTTL()); PrioritizedPacketQueue(const PrioritizedPacketQueue&) = delete; PrioritizedPacketQueue& operator=(const PrioritizedPacketQueue&) = delete; @@ -63,6 +73,7 @@ class PrioritizedPacketQueue { // method, for the given packet type. If queue has no packets, of that type, // returns Timestamp::MinusInfinity(). Timestamp LeadingPacketEnqueueTime(RtpPacketMediaType type) const; + Timestamp LeadingPacketEnqueueTimeForRetransmission() const; // Enqueue time of the oldest packet in the queue, // Timestamp::MinusInfinity() if queue is empty. @@ -90,7 +101,7 @@ class PrioritizedPacketQueue { bool HasKeyframePackets(uint32_t ssrc) const; private: - static constexpr int kNumPriorityLevels = 4; + static constexpr int kNumPriorityLevels = 5; class QueuedPacket { public: @@ -139,6 +150,15 @@ class PrioritizedPacketQueue { // if so move it to the lowest non-empty index. void MaybeUpdateTopPrioLevel(); + void PurgeOldPacketsAtPriorityLevel(int prio_level, Timestamp now); + + static absl::InlinedVector ToTtlPerPrio( + PacketQueueTTL); + + const bool prioritize_audio_retransmission_; + const absl::InlinedVector + time_to_live_per_prio_; + // Cumulative sum, over all packets, of time spent in the queue. TimeDelta queue_time_sum_; // Cumulative sum of time the queue has spent in a paused state. diff --git a/third_party/libwebrtc/modules/pacing/prioritized_packet_queue_unittest.cc b/third_party/libwebrtc/modules/pacing/prioritized_packet_queue_unittest.cc index 9ed19642c7..76c31036b3 100644 --- a/third_party/libwebrtc/modules/pacing/prioritized_packet_queue_unittest.cc +++ b/third_party/libwebrtc/modules/pacing/prioritized_packet_queue_unittest.cc @@ -10,6 +10,7 @@ #include "modules/pacing/prioritized_packet_queue.h" +#include #include #include "api/units/time_delta.h" @@ -26,18 +27,39 @@ constexpr uint32_t kDefaultSsrc = 123; constexpr int kDefaultPayloadSize = 789; std::unique_ptr CreatePacket(RtpPacketMediaType type, - uint16_t sequence_number, + uint16_t seq, uint32_t ssrc = kDefaultSsrc, bool is_key_frame = false) { auto packet = std::make_unique(/*extensions=*/nullptr); packet->set_packet_type(type); packet->SetSsrc(ssrc); - packet->SetSequenceNumber(sequence_number); + packet->SetSequenceNumber(seq); packet->SetPayloadSize(kDefaultPayloadSize); packet->set_is_key_frame(is_key_frame); return packet; } +std::unique_ptr CreateRetransmissionPacket( + RtpPacketMediaType original_type, + uint16_t seq, + uint32_t ssrc = kDefaultSsrc) { + auto packet = std::make_unique(/*extensions=*/nullptr); + packet->set_packet_type(original_type); + packet->set_packet_type(RtpPacketMediaType::kRetransmission); + RTC_DCHECK(packet->packet_type() == RtpPacketMediaType::kRetransmission); + if (original_type == RtpPacketMediaType::kVideo) { + RTC_DCHECK(packet->original_packet_type() == + RtpPacketToSend::OriginalType::kVideo); + } else { + RTC_DCHECK(packet->original_packet_type() == + RtpPacketToSend::OriginalType::kAudio); + } + packet->SetSsrc(ssrc); + packet->SetSequenceNumber(seq); + packet->SetPayloadSize(kDefaultPayloadSize); + return packet; +} + } // namespace TEST(PrioritizedPacketQueue, ReturnsPacketsInPrioritizedOrder) { @@ -49,18 +71,42 @@ TEST(PrioritizedPacketQueue, ReturnsPacketsInPrioritizedOrder) { queue.Push(now, CreatePacket(RtpPacketMediaType::kVideo, /*seq=*/2)); queue.Push(now, CreatePacket(RtpPacketMediaType::kForwardErrorCorrection, /*seq=*/3)); - queue.Push(now, CreatePacket(RtpPacketMediaType::kRetransmission, /*seq=*/4)); - queue.Push(now, CreatePacket(RtpPacketMediaType::kAudio, /*seq=*/5)); + queue.Push(now, + CreateRetransmissionPacket(RtpPacketMediaType::kVideo, /*seq=*/4)); + queue.Push(now, + CreateRetransmissionPacket(RtpPacketMediaType::kAudio, /*seq=*/5)); + queue.Push(now, CreatePacket(RtpPacketMediaType::kAudio, /*seq=*/6)); // Packets should be returned in high to low order. - EXPECT_EQ(queue.Pop()->SequenceNumber(), 5); + EXPECT_EQ(queue.Pop()->SequenceNumber(), 6); + // Audio and video retransmission has same prio, but video was enqueued first. EXPECT_EQ(queue.Pop()->SequenceNumber(), 4); + EXPECT_EQ(queue.Pop()->SequenceNumber(), 5); // Video and FEC prioritized equally - but video was enqueued first. EXPECT_EQ(queue.Pop()->SequenceNumber(), 2); EXPECT_EQ(queue.Pop()->SequenceNumber(), 3); EXPECT_EQ(queue.Pop()->SequenceNumber(), 1); } +TEST(PrioritizedPacketQueue, + PrioritizeAudioRetransmissionBeforeVideoRetransmissionIfConfigured) { + Timestamp now = Timestamp::Zero(); + PrioritizedPacketQueue queue(now, /*prioritize_audio_retransmission=*/true); + + // Add packets in low to high packet order. + queue.Push(now, CreatePacket(RtpPacketMediaType::kVideo, /*seq=*/3)); + queue.Push(now, + CreateRetransmissionPacket(RtpPacketMediaType::kVideo, /*seq=*/4)); + queue.Push(now, + CreateRetransmissionPacket(RtpPacketMediaType::kAudio, /*seq=*/5)); + queue.Push(now, CreatePacket(RtpPacketMediaType::kAudio, /*seq=*/6)); + + // Packets should be returned in high to low order. + EXPECT_EQ(queue.Pop()->SequenceNumber(), 6); + EXPECT_EQ(queue.Pop()->SequenceNumber(), 5); + EXPECT_EQ(queue.Pop()->SequenceNumber(), 4); +} + TEST(PrioritizedPacketQueue, ReturnsEqualPrioPacketsInRoundRobinOrder) { Timestamp now = Timestamp::Zero(); PrioritizedPacketQueue queue(now); @@ -251,6 +297,26 @@ TEST(PrioritizedPacketQueue, ReportsLeadingPacketEnqueueTime) { Timestamp::MinusInfinity()); } +TEST(PrioritizedPacketQueue, ReportsLeadingPacketEnqueueTimeForRetransmission) { + PrioritizedPacketQueue queue(/*creation_time=*/Timestamp::Zero(), + /*prioritize_audio_retransmission=*/true); + EXPECT_EQ(queue.LeadingPacketEnqueueTimeForRetransmission(), + Timestamp::PlusInfinity()); + + queue.Push(Timestamp::Millis(10), + CreateRetransmissionPacket(RtpPacketMediaType::kVideo, /*seq=*/1)); + queue.Push(Timestamp::Millis(11), + CreateRetransmissionPacket(RtpPacketMediaType::kAudio, /*seq=*/2)); + EXPECT_EQ(queue.LeadingPacketEnqueueTimeForRetransmission(), + Timestamp::Millis(10)); + queue.Pop(); // Pop audio retransmission since it has higher prio. + EXPECT_EQ(queue.LeadingPacketEnqueueTimeForRetransmission(), + Timestamp::Millis(10)); + queue.Pop(); // Pop video retransmission. + EXPECT_EQ(queue.LeadingPacketEnqueueTimeForRetransmission(), + Timestamp::PlusInfinity()); +} + TEST(PrioritizedPacketQueue, PushAndPopUpdatesSizeInPacketsPerRtpPacketMediaType) { Timestamp now = Timestamp::Zero(); @@ -272,7 +338,7 @@ TEST(PrioritizedPacketQueue, RtpPacketMediaType::kVideo)], 1); - queue.Push(now, CreatePacket(RtpPacketMediaType::kRetransmission, 3)); + queue.Push(now, CreateRetransmissionPacket(RtpPacketMediaType::kVideo, 3)); EXPECT_EQ(queue.SizeInPacketsPerRtpPacketMediaType()[static_cast( RtpPacketMediaType::kRetransmission)], 1); @@ -326,6 +392,8 @@ TEST(PrioritizedPacketQueue, ClearsPackets) { // Remove all of them. queue.RemovePacketsForSsrc(kSsrc); EXPECT_TRUE(queue.Empty()); + queue.RemovePacketsForSsrc(kSsrc); + EXPECT_TRUE(queue.Empty()); } TEST(PrioritizedPacketQueue, ClearPacketsAffectsOnlySpecifiedSsrc) { @@ -338,16 +406,16 @@ TEST(PrioritizedPacketQueue, ClearPacketsAffectsOnlySpecifiedSsrc) { // ensuring they are first in line. queue.Push( now, CreatePacket(RtpPacketMediaType::kAudio, /*seq=*/1, kRemovingSsrc)); - queue.Push(now, CreatePacket(RtpPacketMediaType::kRetransmission, /*seq=*/2, - kRemovingSsrc)); + queue.Push(now, CreateRetransmissionPacket(RtpPacketMediaType::kVideo, + /*seq=*/2, kRemovingSsrc)); // Add a video packet and a retransmission for the SSRC that will remain. // The retransmission packets now both have pointers to their respective qeues // from the same prio level. queue.Push(now, CreatePacket(RtpPacketMediaType::kVideo, /*seq=*/3, kStayingSsrc)); - queue.Push(now, CreatePacket(RtpPacketMediaType::kRetransmission, /*seq=*/4, - kStayingSsrc)); + queue.Push(now, CreateRetransmissionPacket(RtpPacketMediaType::kVideo, + /*seq=*/4, kStayingSsrc)); EXPECT_EQ(queue.SizeInPackets(), 4); @@ -413,4 +481,87 @@ TEST(PrioritizedPacketQueue, ReportsKeyframePackets) { EXPECT_FALSE(queue.HasKeyframePackets(kVideoSsrc2)); } +TEST(PrioritizedPacketQueue, PacketsDroppedIfNotPulledWithinTttl) { + Timestamp now = Timestamp::Zero(); + PacketQueueTTL ttls; + ttls.audio_retransmission = TimeDelta::Millis(200); + PrioritizedPacketQueue queue(now, /*prioritize_audio_retransmission=*/true, + ttls); + + queue.Push(now, + CreateRetransmissionPacket(RtpPacketMediaType::kAudio, /*seq=*/1)); + now += ttls.audio_retransmission + TimeDelta::Millis(1); + EXPECT_EQ(queue.SizeInPackets(), 1); + queue.Push(now, + CreateRetransmissionPacket(RtpPacketMediaType::kAudio, /*seq=*/2)); + EXPECT_EQ(queue.SizeInPackets(), 1); + EXPECT_EQ(queue.Pop()->SequenceNumber(), 2); +} + +TEST(PrioritizedPacketQueue, DontSendPacketsAfterTttl) { + Timestamp now = Timestamp::Zero(); + PacketQueueTTL ttls; + ttls.audio_retransmission = TimeDelta::Millis(200); + PrioritizedPacketQueue queue(now, /*prioritize_audio_retransmission=*/true, + ttls); + + queue.Push(now, + CreateRetransmissionPacket(RtpPacketMediaType::kAudio, /*seq=*/1)); + now += ttls.audio_retransmission + TimeDelta::Millis(1); + EXPECT_EQ(queue.SizeInPackets(), 1); + queue.Push(now, CreatePacket(RtpPacketMediaType::kVideo, /*seq=*/2)); + queue.Push(now, CreatePacket(RtpPacketMediaType::kAudio, /*seq=*/3)); + // Expect the old packet to have been removed since it was not popped in time. + EXPECT_EQ(queue.SizeInPackets(), 3); + EXPECT_EQ(queue.Pop()->SequenceNumber(), 3); + EXPECT_EQ(queue.SizeInPackets(), 1); + EXPECT_EQ(queue.Pop()->SequenceNumber(), 2); + EXPECT_EQ(queue.SizeInPackets(), 0); +} + +TEST(PrioritizedPacketQueue, SendsNewVideoPacketAfterPurgingLastOldRtxPacket) { + Timestamp now = Timestamp::Zero(); + PacketQueueTTL ttls; + ttls.video_retransmission = TimeDelta::Millis(400); + PrioritizedPacketQueue queue(now, /*prioritize_audio_retransmission=*/true, + ttls); + + queue.Push(now, + CreateRetransmissionPacket(RtpPacketMediaType::kVideo, /*seq=*/1)); + now += ttls.video_retransmission + TimeDelta::Millis(1); + queue.Push(now, CreatePacket(RtpPacketMediaType::kAudio, /*seq=*/2)); + EXPECT_EQ(queue.SizeInPackets(), 2); + // Expect the audio packet to be send and the video retransmission packet to + // be dropped since it is old. + EXPECT_EQ(queue.Pop()->SequenceNumber(), 2); + EXPECT_EQ(queue.SizeInPackets(), 0); + + queue.Push(now, CreatePacket(RtpPacketMediaType::kVideo, /*seq=*/3)); + EXPECT_EQ(queue.SizeInPackets(), 1); + EXPECT_EQ(queue.Pop()->SequenceNumber(), 3); + EXPECT_EQ(queue.SizeInPackets(), 0); +} + +TEST(PrioritizedPacketQueue, + SendsPacketsAfterTttlIfPrioHigherThanPushedPackets) { + Timestamp now = Timestamp::Zero(); + PacketQueueTTL ttls; + ttls.audio_retransmission = TimeDelta::Millis(200); + PrioritizedPacketQueue queue(now, /*prioritize_audio_retransmission=*/true, + ttls); + + queue.Push(now, + CreateRetransmissionPacket(RtpPacketMediaType::kAudio, /*seq=*/1)); + now += ttls.audio_retransmission + TimeDelta::Millis(1); + EXPECT_EQ(queue.SizeInPackets(), 1); + queue.Push(now, CreatePacket(RtpPacketMediaType::kVideo, /*seq=*/2)); + + // This test just show that TTL is not enforced strictly. If a new audio + // packet had been queued before a packet was popped, the audio retransmission + // packet would have been dropped. + EXPECT_EQ(queue.SizeInPackets(), 2); + EXPECT_EQ(queue.Pop()->SequenceNumber(), 1); + EXPECT_EQ(queue.SizeInPackets(), 1); +} + } // namespace webrtc diff --git a/third_party/libwebrtc/modules/remote_bitrate_estimator/remote_estimator_proxy.cc b/third_party/libwebrtc/modules/remote_bitrate_estimator/remote_estimator_proxy.cc index e100995b8e..6953ec8400 100644 --- a/third_party/libwebrtc/modules/remote_bitrate_estimator/remote_estimator_proxy.cc +++ b/third_party/libwebrtc/modules/remote_bitrate_estimator/remote_estimator_proxy.cc @@ -260,8 +260,11 @@ void RemoteEstimatorProxy::SendFeedbackOnRequest( feedback_request.include_timestamps, first_sequence_number, sequence_number + 1, /*is_periodic_update=*/false); - // This is called when a packet has just been added. - RTC_DCHECK(feedback_packet != nullptr); + // Even though this is called when a packet has just been added, + // no feedback may be produced when that new packet is too old. + if (feedback_packet == nullptr) { + return; + } // Clear up to the first packet that is included in this feedback packet. packet_arrival_times_.EraseTo(first_sequence_number); diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/receive_statistics_impl.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/receive_statistics_impl.cc index 0e5e40f502..1dc56bb96f 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/receive_statistics_impl.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/receive_statistics_impl.cc @@ -18,6 +18,7 @@ #include "api/units/time_delta.h" #include "modules/remote_bitrate_estimator/test/bwe_test_logging.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/rtcp_packet/report_block.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" #include "modules/rtp_rtcp/source/rtp_rtcp_config.h" @@ -159,16 +160,15 @@ void StreamStatisticianImpl::UpdateJitter(const RtpPacketReceived& packet, int32_t time_diff_samples = receive_diff_rtp - (packet.Timestamp() - last_received_timestamp_); - time_diff_samples = std::abs(time_diff_samples); - ReviseFrequencyAndJitter(packet.payload_type_frequency()); // lib_jingle sometimes deliver crazy jumps in TS for the same stream. // If this happens, don't update jitter value. Use 5 secs video frequency // as the threshold. - if (time_diff_samples < 450000) { + if (time_diff_samples < 5 * kVideoPayloadTypeFrequency && + time_diff_samples > -5 * kVideoPayloadTypeFrequency) { // Note we calculate in Q4 to avoid using float. - int32_t jitter_diff_q4 = (time_diff_samples << 4) - jitter_q4_; + int32_t jitter_diff_q4 = (std::abs(time_diff_samples) << 4) - jitter_q4_; jitter_q4_ += ((jitter_diff_q4 + 8) >> 4); } } diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/receive_statistics_unittest.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/receive_statistics_unittest.cc index a2558545f0..8b31912f0f 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/receive_statistics_unittest.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/receive_statistics_unittest.cc @@ -898,5 +898,22 @@ TEST(ReviseJitterTest, EXPECT_EQ(GetJitter(*statistics), 172U); } +TEST(ReviseJitterTest, TwoPacketsWithMaximumRtpTimestampDifference) { + SimulatedClock clock(0); + std::unique_ptr statistics = + ReceiveStatistics::Create(&clock); + RtpPacketReceived packet1 = MakeRtpPacket(/*payload_type_frequency=*/90'000, + /*timestamp=*/0x01234567); + RtpPacketReceived packet2 = + MakeNextRtpPacket(packet1, + /*payload_type_frequency=*/90'000, + /*timestamp=*/0x81234567); + statistics->OnRtpPacket(packet1); + statistics->OnRtpPacket(packet2); + + // Expect large jump in RTP timestamp is ignored for jitter calculation. + EXPECT_EQ(GetJitter(*statistics), 0U); +} + } // namespace } // namespace webrtc diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtcp_receiver.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtcp_receiver.cc index bda6ad9a52..756136866d 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtcp_receiver.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtcp_receiver.cc @@ -1190,11 +1190,11 @@ std::vector RTCPReceiver::TmmbrReceived() { MutexLock lock(&rtcp_receiver_lock_); std::vector candidates; - Timestamp timeout = clock_->CurrentTime() - kTmmbrTimeoutInterval; + Timestamp now = clock_->CurrentTime(); for (auto& kv : tmmbr_infos_) { for (auto it = kv.second.tmmbr.begin(); it != kv.second.tmmbr.end();) { - if (it->second.last_updated < timeout) { + if (now - it->second.last_updated > kTmmbrTimeoutInterval) { // Erase timeout entries. it = kv.second.tmmbr.erase(it); } else { diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.cc index fd42b798d4..27b0420926 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.cc @@ -52,4 +52,17 @@ bool RtpDependencyDescriptorExtension::Write( return writer.Write(); } +bool RtpDependencyDescriptorExtensionMandatory::Parse( + rtc::ArrayView data, + DependencyDescriptorMandatory* descriptor) { + if (data.size() < 3) { + return false; + } + descriptor->set_first_packet_in_frame(data[0] & 0b1000'0000); + descriptor->set_last_packet_in_frame(data[0] & 0b0100'0000); + descriptor->set_template_id(data[0] & 0b0011'1111); + descriptor->set_frame_number((uint16_t{data[1]} << 8) | data[2]); + return true; +} + } // namespace webrtc diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h index 8d6e4b8d37..a3e415917c 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h @@ -54,6 +54,16 @@ class RtpDependencyDescriptorExtension { static constexpr std::bitset<32> kAllChainsAreActive = ~uint32_t{0}; }; +// Trait to only read the mandatory part of the descriptor. +class RtpDependencyDescriptorExtensionMandatory { + public: + static constexpr webrtc::RTPExtensionType kId = + webrtc::RtpDependencyDescriptorExtension::kId; + + static bool Parse(rtc::ArrayView data, + DependencyDescriptorMandatory* descriptor); +}; + } // namespace webrtc #endif // MODULES_RTP_RTCP_SOURCE_RTP_DEPENDENCY_DESCRIPTOR_EXTENSION_H_ diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_h264.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_h264.cc index d066bafb90..9c1dc4edb8 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_h264.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_h264.cc @@ -19,6 +19,7 @@ #include #include +#include "absl/algorithm/container.h" #include "absl/types/optional.h" #include "absl/types/variant.h" #include "common_video/h264/h264_common.h" @@ -52,11 +53,14 @@ RtpPacketizerH264::RtpPacketizerH264(rtc::ArrayView payload, input_fragments_.push_back( payload.subview(nalu.payload_start_offset, nalu.payload_size)); } - - if (!GeneratePackets(packetization_mode)) { - // If failed to generate all the packets, discard already generated - // packets in case the caller would ignore return value and still try to - // call NextPacket(). + bool has_empty_fragments = absl::c_any_of( + input_fragments_, [](const rtc::ArrayView fragment) { + return fragment.empty(); + }); + if (has_empty_fragments || !GeneratePackets(packetization_mode)) { + // If empty fragments were found or we failed to generate all the packets, + // discard already generated packets in case the caller would ignore the + // return value and still try to call NextPacket(). num_packets_left_ = 0; while (!packets_.empty()) { packets_.pop(); @@ -73,6 +77,7 @@ size_t RtpPacketizerH264::NumPackets() const { bool RtpPacketizerH264::GeneratePackets( H264PacketizationMode packetization_mode) { for (size_t i = 0; i < input_fragments_.size();) { + RTC_DCHECK(!input_fragments_[i].empty()); switch (packetization_mode) { case H264PacketizationMode::SingleNalUnit: if (!PacketizeSingleNalu(i)) @@ -173,11 +178,11 @@ size_t RtpPacketizerH264::PacketizeStapA(size_t fragment_index) { return fragment_size; } }; - while (payload_size_left >= payload_size_needed()) { RTC_CHECK_GT(fragment.size(), 0); - packets_.push(PacketUnit(fragment, aggregated_fragments == 0, false, true, - fragment[0])); + + packets_.push(PacketUnit(fragment, /*first=*/aggregated_fragments == 0, + /*last=*/false, /*aggregated=*/true, fragment[0])); payload_size_left -= fragment.size(); payload_size_left -= fragment_headers_length; @@ -218,9 +223,9 @@ bool RtpPacketizerH264::PacketizeSingleNalu(size_t fragment_index) { << limits_.max_payload_len; return false; } - RTC_CHECK_GT(fragment.size(), 0u); - packets_.push(PacketUnit(fragment, true /* first */, true /* last */, - false /* aggregated */, fragment[0])); + RTC_CHECK(!fragment.empty()); + packets_.push(PacketUnit(fragment, /*first=*/true, /*last=*/true, + /*aggregated=*/false, fragment[0])); ++num_packets_left_; return true; } diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_h264_unittest.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_h264_unittest.cc index 18311c6e8c..3920c4acd5 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_h264_unittest.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_h264_unittest.cc @@ -116,6 +116,7 @@ std::vector FetchAllPackets(RtpPacketizerH264* packetizer) { RtpPacketToSend packet(kNoExtensions); while (packetizer->NextPacket(&packet)) { result.push_back(packet); + packet.Clear(); } EXPECT_THAT(result, SizeIs(num_packets)); return result; @@ -527,5 +528,190 @@ TEST(RtpPacketizerH264Test, RejectsOverlongDataInPacketizationMode0) { EXPECT_THAT(packets, IsEmpty()); } + +TEST(RtpPacketizerH264Test, DoesNotPacketizeWithEmptyNalUnit) { + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = kMaxPayloadSize; + + uint8_t empty_nal_input[] = {0x00, 0x00, 0x01, /* empty NAL unit data */ + 0x00, 0x00, 0x01, 0x01}; + RtpPacketizerH264 packetizer(empty_nal_input, limits, + H264PacketizationMode::NonInterleaved); + EXPECT_EQ(packetizer.NumPackets(), 0u); +} + +TEST(RtpPacketizerH264Test, MultipleStapA) { + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = kMaxPayloadSize; + limits.first_packet_reduction_len = 0; + limits.last_packet_reduction_len = 0; + limits.single_packet_reduction_len = 0; + // A lot of small NAL units that will result in two STAP-A being generated. + // Input data must exceed the size of a single RTP packet. + uint8_t long_input[] = { + 0x19, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, + 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x04, + 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, + 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, + 0x00, 0x01, 0x00, 0xaf, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, + 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, + 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, 0xaf, 0x01, + 0x04, 0x00, 0x19, 0x00, 0x00, 0x01, 0xf9, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0xf7, 0x01, + 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, 0x0b, 0x01, 0x04, 0x00, + 0x19, 0x00, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, 0xaf, 0x01, 0x04, 0x00, 0x19, 0x00, + 0x00, 0x01, 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0xaf, 0x01, 0x04, 0x00, 0x19, 0x00, 0x00, + 0x01, 0x01, 0x02, 0x00, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, + 0x19, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, + 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x04, + 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x19, + 0x7a, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, + 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x04, + 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x01, 0x00, + 0xaf, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, + 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x04, + 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0xaf, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, + 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, + 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, + 0x00, 0x00, 0x11, 0xd4, 0x00, 0x19, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, + 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, + 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, 0xaf, 0x00, 0x00, 0x01, + 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, + 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, + 0x00, 0x01, 0x00, 0xaf, 0x01, 0x04, 0x00, 0x19, 0x00, 0x01, 0x00, 0xaf, + 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x04, 0x00, + 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x10, 0x00, 0x00, + 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, 0xaf, 0x01, 0x04, 0x00, 0x19, 0x00, + 0x00, 0x01, 0xf9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, + 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0x08, 0xfe, 0xfb, 0xff, 0xff, 0xf4, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, + 0x00, 0x01, 0x00, 0xaf, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, + 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, + 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0xaf, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, + 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x11, 0xd4, 0x00, 0x19, 0x00, 0x00, + 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x01, + 0x04, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, + 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, + 0xaf, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, + 0x00, 0x00, 0x00, 0x00, 0x11, 0xd4, 0x00, 0x19, 0x00, 0x00, 0x01, 0x04, + 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x01, 0x04, 0x00, + 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, 0xaf, 0x00, + 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, + 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x04, 0x00, 0x00, 0x01, 0x00, 0xaf, 0x01, 0x04, 0x00, 0x19, 0x00, 0x00, + 0x01, 0xf9, 0x01, 0x00, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0xf7, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, + 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, + 0x00, 0x01, 0x00, 0xaf, 0x01, 0x04, 0x00, 0x19, 0x00, 0x00, 0x01, 0x01, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0xaf, 0x01, 0x04, 0x00, 0x19, 0x00, 0x00, 0x01, 0x01, 0x02, + 0x00, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0xf7, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, + 0x00, 0xf7, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x26, 0x00, 0x00, 0x01, 0x00, + 0xaf, 0x01, 0x04, 0x00, 0x19, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, + 0x01, 0x00, 0xaf, 0x01, 0x04, 0x00, 0x19, 0x00, 0x00, 0x01, 0x00, 0x01, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, + 0xaf, 0x01, 0x04, 0x00, 0x19, 0x00, 0x00, 0x01, 0xf9, 0x01, 0x00, 0x00, + 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0xf7, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, 0x0b, 0x2c, + 0x01, 0x04, 0x00, 0x19, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, 0xaf, 0x01, 0x04, 0x00, 0x19, 0x00, + 0x00, 0x01, 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0xaf, 0x01, 0x04, 0x00, 0x19, 0x00, 0x00, + 0x01, 0x01, 0x02, 0x00, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, + 0x19, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, + 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x04, + 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x19, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, + 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, + 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, 0xaf, + 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x04, 0x00, + 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x10, 0x00, 0x00, + 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, 0xaf, 0x01, 0x04, 0x00, 0x19, 0x00, + 0x00, 0x01, 0xf9, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, + 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0x08, 0xfe, 0xfb, 0xff, 0xff, 0xf4, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, + 0x00, 0x01, 0x00, 0xaf, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, + 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, + 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0xaf, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, + 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x04, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, + 0x8e, 0x00, 0x04, 0x00, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x01, 0x00, + 0xaf, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x00, 0x00, 0x11, 0xd4, 0x00, + 0x19, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, + 0x8e, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, + 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, + 0x00, 0x01, 0x00, 0xaf, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, + 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, + 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, 0xaf, 0x01, + 0x04, 0x00, 0x19, 0x00, 0x00, 0x01, 0xf9, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0xf7, 0x01, + 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, 0xaf, 0x01, 0x04, 0x00, + 0x19, 0x00, 0x00, 0x01, 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0xaf, 0x01, 0x04, 0x00, 0x19, + 0x00, 0x00, 0x01, 0x01, 0x02, 0x00, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0xf7, 0x01, 0x04, 0x00, 0x00, + 0x01, 0x01, 0x00, 0x00, 0x01, 0x00, 0xf7, 0x01, 0x04, 0x00, 0x00, 0x01, + 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, + 0x26, 0x00, 0x00, 0x01, 0x00, 0xaf, 0x01, 0x04, 0x00, 0x19, 0x00, 0x00, + 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, 0xaf, 0x01, 0x04, 0x00, 0x19, + 0x00, 0x00, 0x01, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x04, 0x00, 0x00, 0xaf, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, + 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, + 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, + 0x00, 0x00, 0x00, 0x11, 0xd4, 0x00, 0x19, 0x00, 0x00, 0x01, 0x04, 0x00, + 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x01, 0x04, 0x00, 0x00, + 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x00, 0xaf, 0x00, 0x00, + 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, + 0x01, 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, + 0x00, 0xaf, 0x01, 0x04, 0x00, 0x19, 0x00, 0x00, 0x01, 0xf9, 0x01, 0x00, + 0x00, 0x00, 0x01, 0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0xf7, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, + 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x8e, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x01, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x01}; + RtpPacketizerH264 packetizer(long_input, limits, + H264PacketizationMode::NonInterleaved); + EXPECT_EQ(packetizer.NumPackets(), 2u); + EXPECT_THAT(FetchAllPackets(&packetizer), SizeIs(2)); +} + } // namespace } // namespace webrtc diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_vp8.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_vp8.cc index ae5f4e50a4..34b3fd9d2f 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_vp8.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_vp8.cc @@ -63,7 +63,9 @@ RtpPacketizerVp8::RtpPacketizerVp8(rtc::ArrayView payload, const RTPVideoHeaderVP8& hdr_info) : hdr_(BuildHeader(hdr_info)), remaining_payload_(payload) { limits.max_payload_len -= hdr_.size(); - payload_sizes_ = SplitAboutEqually(payload.size(), limits); + if (!payload.empty()) { + payload_sizes_ = SplitAboutEqually(payload.size(), limits); + } current_packet_ = payload_sizes_.begin(); } diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_vp8_unittest.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_vp8_unittest.cc index 7934ff8ea9..cab7fc4422 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_vp8_unittest.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_vp8_unittest.cc @@ -21,6 +21,18 @@ namespace { constexpr RtpPacketizer::PayloadSizeLimits kNoSizeLimits; +TEST(RtpPacketizerVp8Test, EmptyPayload) { + RTPVideoHeaderVP8 hdr_info; + hdr_info.InitRTPVideoHeaderVP8(); + hdr_info.pictureId = 200; + RtpFormatVp8TestHelper helper(&hdr_info, /*payload_len=*/30); + + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = 12; // Small enough to produce 4 packets. + RtpPacketizerVp8 packetizer({}, limits, hdr_info); + EXPECT_EQ(packetizer.NumPackets(), 0u); +} + TEST(RtpPacketizerVp8Test, ResultPacketsAreAlmostEqualSize) { RTPVideoHeaderVP8 hdr_info; hdr_info.InitRTPVideoHeaderVP8(); diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_vp9.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_vp9.cc index 9ad4aa97c3..5e037859ce 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_vp9.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_vp9.cc @@ -321,8 +321,9 @@ RtpPacketizerVp9::RtpPacketizerVp9(rtc::ArrayView payload, limits.max_payload_len -= header_size_; limits.first_packet_reduction_len += first_packet_extra_header_size_; limits.single_packet_reduction_len += first_packet_extra_header_size_; - - payload_sizes_ = SplitAboutEqually(payload.size(), limits); + if (!payload.empty()) { + payload_sizes_ = SplitAboutEqually(payload.size(), limits); + } current_packet_ = payload_sizes_.begin(); } diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc index e18b8a803f..948bcf3bdb 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format_vp9_unittest.cc @@ -186,6 +186,11 @@ class RtpPacketizerVp9Test : public ::testing::Test { } }; +TEST_F(RtpPacketizerVp9Test, EmptyPayload) { + RTPVideoHeader video_header; + VideoRtpDepacketizerVp9::ParseRtpPayload({}, &video_header); +} + TEST_F(RtpPacketizerVp9Test, TestEqualSizedMode_OnePacket) { const size_t kFrameSize = 25; const size_t kPacketSize = 26; diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packet.h b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packet.h index e91ec6368b..c002e51de6 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packet.h +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packet.h @@ -11,6 +11,7 @@ #define MODULES_RTP_RTCP_SOURCE_RTP_PACKET_H_ #include +#include #include #include "absl/types/optional.h" @@ -127,7 +128,7 @@ class RtpPacket { bool IsRegistered() const; template - bool GetExtension(FirstValue, Values...) const; + bool GetExtension(FirstValue&&, Values&&...) const; template absl::optional GetExtension() const; @@ -231,11 +232,12 @@ bool RtpPacket::IsRegistered() const { } template -bool RtpPacket::GetExtension(FirstValue first, Values... values) const { +bool RtpPacket::GetExtension(FirstValue&& first, Values&&... values) const { auto raw = FindExtension(Extension::kId); if (raw.empty()) return false; - return Extension::Parse(raw, first, values...); + return Extension::Parse(raw, std::forward(first), + std::forward(values)...); } template diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packet_to_send.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packet_to_send.cc index b55e74aaf0..691a243c5f 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packet_to_send.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packet_to_send.cc @@ -12,6 +12,8 @@ #include +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" + namespace webrtc { RtpPacketToSend::RtpPacketToSend(const ExtensionManager* extensions) @@ -28,4 +30,13 @@ RtpPacketToSend& RtpPacketToSend::operator=(RtpPacketToSend&& packet) = default; RtpPacketToSend::~RtpPacketToSend() = default; +void RtpPacketToSend::set_packet_type(RtpPacketMediaType type) { + if (packet_type_ == RtpPacketMediaType::kAudio) { + original_packet_type_ = OriginalType::kAudio; + } else if (packet_type_ == RtpPacketMediaType::kVideo) { + original_packet_type_ = OriginalType::kVideo; + } + packet_type_ = type; +} + } // namespace webrtc diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h index 438ca354ed..64f9ee1ab1 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h @@ -49,11 +49,18 @@ class RtpPacketToSend : public RtpPacket { webrtc::Timestamp capture_time() const { return capture_time_; } void set_capture_time(webrtc::Timestamp time) { capture_time_ = time; } - void set_packet_type(RtpPacketMediaType type) { packet_type_ = type; } + void set_packet_type(RtpPacketMediaType type); + absl::optional packet_type() const { return packet_type_; } + enum class OriginalType { kAudio, kVideo }; + // Original type does not change if packet type is changed to kRetransmission. + absl::optional original_packet_type() const { + return original_packet_type_; + } + // If this is a retransmission, indicates the sequence number of the original // media packet that this packet represents. If RTX is used this will likely // be different from SequenceNumber(). @@ -133,6 +140,7 @@ class RtpPacketToSend : public RtpPacket { private: webrtc::Timestamp capture_time_ = webrtc::Timestamp::Zero(); absl::optional packet_type_; + absl::optional original_packet_type_; bool allow_retransmission_ = false; absl::optional retransmitted_sequence_number_; rtc::scoped_refptr additional_data_; diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packet_unittest.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packet_unittest.cc index b3a9452df9..44f1a9e742 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packet_unittest.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packet_unittest.cc @@ -933,6 +933,41 @@ TEST(RtpPacketTest, GetUncopyableExtension) { EXPECT_TRUE(rtp_packet.GetExtension(&value2)); } +struct ParseByReferenceExtension { + static constexpr RTPExtensionType kId = kRtpExtensionDependencyDescriptor; + static constexpr absl::string_view Uri() { return "uri"; } + + static size_t ValueSize(uint8_t value1, uint8_t value2) { return 2; } + static bool Write(rtc::ArrayView data, + uint8_t value1, + uint8_t value2) { + data[0] = value1; + data[1] = value2; + return true; + } + static bool Parse(rtc::ArrayView data, + uint8_t& value1, + uint8_t& value2) { + value1 = data[0]; + value2 = data[1]; + return true; + } +}; + +TEST(RtpPacketTest, GetExtensionByReference) { + RtpHeaderExtensionMap extensions; + extensions.Register(1); + RtpPacket rtp_packet(&extensions); + rtp_packet.SetExtension(13, 42); + + uint8_t value1 = 1; + uint8_t value2 = 1; + EXPECT_TRUE( + rtp_packet.GetExtension(value1, value2)); + EXPECT_EQ(int{value1}, 13); + EXPECT_EQ(int{value2}, 42); +} + TEST(RtpPacketTest, CreateAndParseTimingFrameExtension) { // Create a packet with video frame timing extension populated. RtpPacketToSend::ExtensionManager send_extensions; diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packetizer_av1_unittest.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packetizer_av1_unittest.cc index 2151a59295..83a2be24ea 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packetizer_av1_unittest.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packetizer_av1_unittest.cc @@ -99,6 +99,12 @@ Av1Frame ReassembleFrame(rtc::ArrayView rtp_payloads) { return Av1Frame(VideoRtpDepacketizerAv1().AssembleFrame(payloads)); } +TEST(RtpPacketizerAv1Test, EmptyPayload) { + RtpPacketizer::PayloadSizeLimits limits; + RtpPacketizerAv1 packetizer({}, limits, VideoFrameType::kVideoFrameKey, true); + EXPECT_EQ(packetizer.NumPackets(), 0u); +} + TEST(RtpPacketizerAv1Test, PacketizeOneObuWithoutSizeAndExtension) { auto kFrame = BuildAv1Frame({Av1Obu(kAv1ObuTypeFrame) .WithoutSize() diff --git a/third_party/libwebrtc/modules/video_capture/BUILD.gn b/third_party/libwebrtc/modules/video_capture/BUILD.gn index 45a0272eee..3132e452ba 100644 --- a/third_party/libwebrtc/modules/video_capture/BUILD.gn +++ b/third_party/libwebrtc/modules/video_capture/BUILD.gn @@ -173,6 +173,7 @@ if (!build_with_chromium || is_linux || is_chromeos) { "../../api/video:video_frame", "../../api/video:video_rtp_headers", "../../common_video", + "../../rtc_base:gunit_helpers", "../../rtc_base:timeutils", "../../rtc_base/synchronization:mutex", "../../system_wrappers", diff --git a/third_party/libwebrtc/modules/video_capture/test/video_capture_unittest.cc b/third_party/libwebrtc/modules/video_capture/test/video_capture_unittest.cc index c8af222b57..dec8de70cb 100644 --- a/third_party/libwebrtc/modules/video_capture/test/video_capture_unittest.cc +++ b/third_party/libwebrtc/modules/video_capture/test/video_capture_unittest.cc @@ -22,35 +22,16 @@ #include "api/video/video_frame.h" #include "common_video/libyuv/include/webrtc_libyuv.h" #include "modules/video_capture/video_capture_factory.h" +#include "rtc_base/gunit.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/time_utils.h" -#include "system_wrappers/include/sleep.h" #include "test/frame_utils.h" #include "test/gtest.h" -using webrtc::SleepMs; using webrtc::VideoCaptureCapability; using webrtc::VideoCaptureFactory; using webrtc::VideoCaptureModule; -#define WAIT_(ex, timeout, res) \ - do { \ - res = (ex); \ - int64_t start = rtc::TimeMillis(); \ - while (!res && rtc::TimeMillis() < start + timeout) { \ - SleepMs(5); \ - res = (ex); \ - } \ - } while (0) - -#define EXPECT_TRUE_WAIT(ex, timeout) \ - do { \ - bool res; \ - WAIT_(ex, timeout, res); \ - if (!res) \ - EXPECT_TRUE(ex); \ - } while (0) - static const int kTimeOut = 5000; #ifdef WEBRTC_MAC static const int kTestHeight = 288; diff --git a/third_party/libwebrtc/modules/video_coding/BUILD.gn b/third_party/libwebrtc/modules/video_coding/BUILD.gn index d9e614ff81..0457b818c3 100644 --- a/third_party/libwebrtc/modules/video_coding/BUILD.gn +++ b/third_party/libwebrtc/modules/video_coding/BUILD.gn @@ -218,6 +218,7 @@ rtc_library("video_coding") { "../../api:rtp_packet_info", "../../api:scoped_refptr", "../../api:sequence_checker", + "../../api/environment", "../../api/task_queue", "../../api/units:data_rate", "../../api/units:data_size", @@ -258,7 +259,6 @@ rtc_library("video_coding") { "../../rtc_base/task_utils:repeating_task", "../../rtc_base/third_party/base64", "../../system_wrappers", - "../../system_wrappers:field_trial", "../../system_wrappers:metrics", "../../video/config:encoder_config", "../rtp_rtcp", @@ -1217,6 +1217,7 @@ if (rtc_include_tests) { "../../api:scoped_refptr", "../../api:simulcast_test_fixture_api", "../../api:videocodec_test_fixture_api", + "../../api/environment:environment_factory", "../../api/task_queue", "../../api/task_queue:default_task_queue_factory", "../../api/test/video:function_video_factory", diff --git a/third_party/libwebrtc/modules/video_coding/codecs/av1/dav1d_decoder.cc b/third_party/libwebrtc/modules/video_coding/codecs/av1/dav1d_decoder.cc index 6a787ff935..d658e401e8 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/av1/dav1d_decoder.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/av1/dav1d_decoder.cc @@ -87,6 +87,8 @@ bool Dav1dDecoder::Configure(const Settings& settings) { s.n_threads = std::max(2, settings.number_of_cores()); s.max_frame_delay = 1; // For low latency decoding. s.all_layers = 0; // Don't output a frame for every spatial layer. + // Limit max frame size to avoid OOM'ing fuzzers. crbug.com/325284120. + s.frame_size_limit = 16384 * 16384; s.operating_point = 31; // Decode all operating points. return dav1d_open(&context_, &s) == 0; diff --git a/third_party/libwebrtc/modules/video_coding/codecs/av1/libaom_av1_unittest.cc b/third_party/libwebrtc/modules/video_coding/codecs/av1/libaom_av1_unittest.cc index 766b7660e4..d486c1d062 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/av1/libaom_av1_unittest.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/av1/libaom_av1_unittest.cc @@ -350,7 +350,8 @@ INSTANTIATE_TEST_SUITE_P( SvcTestParam{"L3T1", /*num_frames_to_generate=*/3}, SvcTestParam{"L3T3", /*num_frames_to_generate=*/8}, SvcTestParam{"S2T1", /*num_frames_to_generate=*/3}, - SvcTestParam{"S3T3", /*num_frames_to_generate=*/8}, + // TODO: bugs.webrtc.org/15715 - Re-enable once AV1 is fixed. + // SvcTestParam{"S3T3", /*num_frames_to_generate=*/8}, SvcTestParam{"L2T2", /*num_frames_to_generate=*/4}, SvcTestParam{"L2T2_KEY", /*num_frames_to_generate=*/4}, SvcTestParam{"L2T2_KEY_SHIFT", diff --git a/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_decoder_impl.cc b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_decoder_impl.cc index a9e9926c4f..c6446c25ce 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_decoder_impl.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/h264/h264_decoder_impl.cc @@ -80,7 +80,11 @@ int H264DecoderImpl::AVGetBuffer2(AVCodecContext* context, kPixelFormatsSupported.begin(), kPixelFormatsSupported.end(), [context](AVPixelFormat format) { return context->pix_fmt == format; }); - RTC_CHECK(pixelFormatSupported != kPixelFormatsSupported.end()); + if (pixelFormatSupported == kPixelFormatsSupported.end()) { + RTC_LOG(LS_ERROR) << "Unsupported pixel format: " << context->pix_fmt; + decoder->ReportError(); + return -1; + } // `av_frame->width` and `av_frame->height` are set by FFmpeg. These are the // actual image's dimensions and may be different from `context->width` and diff --git a/third_party/libwebrtc/modules/video_coding/codecs/test/video_codec_test.cc b/third_party/libwebrtc/modules/video_coding/codecs/test/video_codec_test.cc index 60c2fcbb6e..2ab1106a59 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/test/video_codec_test.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/test/video_codec_test.cc @@ -178,7 +178,8 @@ std::string TestOutputPath() { } // namespace std::unique_ptr RunEncodeDecodeTest( - std::string codec_impl, + std::string encoder_impl, + std::string decoder_impl, const VideoInfo& video_info, const std::map& encoding_settings) { VideoSourceSettings source_settings{ @@ -190,28 +191,34 @@ std::unique_ptr RunEncodeDecodeTest( encoding_settings.begin()->second.sdp_video_format; std::unique_ptr encoder_factory = - CreateEncoderFactory(codec_impl); + CreateEncoderFactory(encoder_impl); if (!encoder_factory ->QueryCodecSupport(sdp_video_format, /*scalability_mode=*/absl::nullopt) .is_supported) { - RTC_LOG(LS_WARNING) << "No encoder for video format " + RTC_LOG(LS_WARNING) << "No " << encoder_impl << " encoder for video format " << sdp_video_format.ToString(); return nullptr; } std::unique_ptr decoder_factory = - CreateDecoderFactory(codec_impl); + CreateDecoderFactory(decoder_impl); if (!decoder_factory ->QueryCodecSupport(sdp_video_format, /*reference_scaling=*/false) .is_supported) { + RTC_LOG(LS_WARNING) << "No " << decoder_impl << " decoder for video format " + << sdp_video_format.ToString() + << ". Trying built-in decoder."; + // TODO(ssilkin): No H264 support in ffmpeg on ARM. Consider trying HW + // decoder. decoder_factory = CreateDecoderFactory("builtin"); if (!decoder_factory ->QueryCodecSupport(sdp_video_format, /*reference_scaling=*/false) .is_supported) { - RTC_LOG(LS_WARNING) << "No decoder for video format " + RTC_LOG(LS_WARNING) << "No " << decoder_impl + << " decoder for video format " << sdp_video_format.ToString(); return nullptr; } @@ -221,7 +228,7 @@ std::unique_ptr RunEncodeDecodeTest( VideoCodecTester::EncoderSettings encoder_settings; encoder_settings.pacing_settings.mode = - codec_impl == "builtin" ? PacingMode::kNoPacing : PacingMode::kRealTime; + encoder_impl == "builtin" ? PacingMode::kNoPacing : PacingMode::kRealTime; if (absl::GetFlag(FLAGS_dump_encoder_input)) { encoder_settings.encoder_input_base_path = output_path + "_enc_input"; } @@ -231,7 +238,7 @@ std::unique_ptr RunEncodeDecodeTest( VideoCodecTester::DecoderSettings decoder_settings; decoder_settings.pacing_settings.mode = - codec_impl == "builtin" ? PacingMode::kNoPacing : PacingMode::kRealTime; + decoder_impl == "builtin" ? PacingMode::kNoPacing : PacingMode::kRealTime; if (absl::GetFlag(FLAGS_dump_decoder_input)) { decoder_settings.decoder_input_base_path = output_path + "_dec_input"; } @@ -318,7 +325,7 @@ TEST_P(SpatialQualityTest, SpatialQuality) { {bitrate_kbps}, framerate_fps, num_frames); std::unique_ptr stats = - RunEncodeDecodeTest(codec_impl, video_info, frames_settings); + RunEncodeDecodeTest(codec_impl, codec_impl, video_info, frames_settings); VideoCodecStats::Stream stream; if (stats != nullptr) { @@ -348,15 +355,15 @@ INSTANTIATE_TEST_SUITE_P( Values("builtin"), #endif Values(kRawVideos.at("FourPeople_1280x720_30")), - Values(std::make_tuple(320, 180, 30, 32, 28), - std::make_tuple(320, 180, 30, 64, 30), - std::make_tuple(320, 180, 30, 128, 33), + Values(std::make_tuple(320, 180, 30, 32, 26), + std::make_tuple(320, 180, 30, 64, 29), + std::make_tuple(320, 180, 30, 128, 32), std::make_tuple(320, 180, 30, 256, 36), - std::make_tuple(640, 360, 30, 128, 31), + std::make_tuple(640, 360, 30, 128, 29), std::make_tuple(640, 360, 30, 256, 33), std::make_tuple(640, 360, 30, 384, 35), std::make_tuple(640, 360, 30, 512, 36), - std::make_tuple(1280, 720, 30, 256, 32), + std::make_tuple(1280, 720, 30, 256, 30), std::make_tuple(1280, 720, 30, 512, 34), std::make_tuple(1280, 720, 30, 1024, 37), std::make_tuple(1280, 720, 30, 2048, 39))), @@ -538,6 +545,7 @@ TEST(VideoCodecTest, DISABLED_EncodeDecode) { // Sync with changes in Stream::LogMetrics (see TODOs there). std::unique_ptr stats = RunEncodeDecodeTest( CodecNameToCodecImpl(absl::GetFlag(FLAGS_encoder)), + CodecNameToCodecImpl(absl::GetFlag(FLAGS_decoder)), kRawVideos.at(absl::GetFlag(FLAGS_video_name)), frames_settings); ASSERT_NE(nullptr, stats); diff --git a/third_party/libwebrtc/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc b/third_party/libwebrtc/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc index eb264e5285..35355d4387 100644 --- a/third_party/libwebrtc/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc +++ b/third_party/libwebrtc/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc @@ -167,7 +167,7 @@ SdpVideoFormat CreateSdpVideoFormat( H264PacketizationMode::NonInterleaved ? "1" : "0"; - SdpVideoFormat::Parameters codec_params = { + CodecParameterMap codec_params = { {cricket::kH264FmtpProfileLevelId, *H264ProfileLevelIdToString(H264ProfileLevelId( config.h264_codec_settings.profile, H264Level::kLevel3_1))}, diff --git a/third_party/libwebrtc/modules/video_coding/fec_controller_default.cc b/third_party/libwebrtc/modules/video_coding/fec_controller_default.cc index f204b01c7c..d548d6580c 100644 --- a/third_party/libwebrtc/modules/video_coding/fec_controller_default.cc +++ b/third_party/libwebrtc/modules/video_coding/fec_controller_default.cc @@ -15,30 +15,28 @@ #include #include +#include "api/environment/environment.h" +#include "api/field_trials_view.h" #include "modules/include/module_fec_types.h" #include "rtc_base/logging.h" -#include "system_wrappers/include/field_trial.h" +#include "system_wrappers/include/clock.h" namespace webrtc { const float kProtectionOverheadRateThreshold = 0.5; FecControllerDefault::FecControllerDefault( - Clock* clock, + const Environment& env, VCMProtectionCallback* protection_callback) - : clock_(clock), + : env_(env), protection_callback_(protection_callback), loss_prot_logic_(new media_optimization::VCMLossProtectionLogic( - clock_->TimeInMilliseconds())), + env_.clock().TimeInMilliseconds())), max_payload_size_(1460), overhead_threshold_(GetProtectionOverheadRateThreshold()) {} -FecControllerDefault::FecControllerDefault(Clock* clock) - : clock_(clock), - loss_prot_logic_(new media_optimization::VCMLossProtectionLogic( - clock_->TimeInMilliseconds())), - max_payload_size_(1460), - overhead_threshold_(GetProtectionOverheadRateThreshold()) {} +FecControllerDefault::FecControllerDefault(const Environment& env) + : FecControllerDefault(env, nullptr) {} FecControllerDefault::~FecControllerDefault(void) { loss_prot_logic_->Release(); @@ -61,8 +59,8 @@ void FecControllerDefault::SetEncodingData(size_t width, float FecControllerDefault::GetProtectionOverheadRateThreshold() { float overhead_threshold = - strtof(webrtc::field_trial::FindFullName( - "WebRTC-ProtectionOverheadRateThreshold") + strtof(env_.field_trials() + .Lookup("WebRTC-ProtectionOverheadRateThreshold") .c_str(), nullptr); if (overhead_threshold > 0 && overhead_threshold <= 1) { @@ -107,7 +105,7 @@ uint32_t FecControllerDefault::UpdateFecRates( media_optimization::FilterPacketLossMode filter_mode = media_optimization::kMaxFilter; uint8_t packet_loss_enc = loss_prot_logic_->FilteredLoss( - clock_->TimeInMilliseconds(), filter_mode, fraction_lost); + env_.clock().TimeInMilliseconds(), filter_mode, fraction_lost); // For now use the filtered loss for computing the robustness settings. loss_prot_logic_->UpdateFilteredLossPr(packet_loss_enc); if (loss_prot_logic_->SelectedType() == media_optimization::kNone) { @@ -191,11 +189,11 @@ void FecControllerDefault::UpdateWithEncodedData( const float min_packets_per_frame = encoded_length / static_cast(max_payload_size_); if (delta_frame) { - loss_prot_logic_->UpdatePacketsPerFrame(min_packets_per_frame, - clock_->TimeInMilliseconds()); + loss_prot_logic_->UpdatePacketsPerFrame( + min_packets_per_frame, env_.clock().TimeInMilliseconds()); } else { loss_prot_logic_->UpdatePacketsPerFrameKey( - min_packets_per_frame, clock_->TimeInMilliseconds()); + min_packets_per_frame, env_.clock().TimeInMilliseconds()); } } if (!delta_frame && encoded_length > 0) { diff --git a/third_party/libwebrtc/modules/video_coding/fec_controller_default.h b/third_party/libwebrtc/modules/video_coding/fec_controller_default.h index a97dea011b..f60302b2d8 100644 --- a/third_party/libwebrtc/modules/video_coding/fec_controller_default.h +++ b/third_party/libwebrtc/modules/video_coding/fec_controller_default.h @@ -17,24 +17,25 @@ #include #include +#include "api/environment/environment.h" #include "api/fec_controller.h" #include "modules/video_coding/media_opt_util.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/thread_annotations.h" -#include "system_wrappers/include/clock.h" namespace webrtc { class FecControllerDefault : public FecController { public: - FecControllerDefault(Clock* clock, + FecControllerDefault(const Environment& env, VCMProtectionCallback* protection_callback); - explicit FecControllerDefault(Clock* clock); - ~FecControllerDefault() override; + explicit FecControllerDefault(const Environment& env); FecControllerDefault(const FecControllerDefault&) = delete; FecControllerDefault& operator=(const FecControllerDefault&) = delete; + ~FecControllerDefault() override; + void SetProtectionCallback( VCMProtectionCallback* protection_callback) override; void SetProtectionMethod(bool enable_fec, bool enable_nack) override; @@ -54,7 +55,7 @@ class FecControllerDefault : public FecController { private: enum { kBitrateAverageWinMs = 1000 }; - Clock* const clock_; + const Environment env_; VCMProtectionCallback* protection_callback_; Mutex mutex_; std::unique_ptr loss_prot_logic_ diff --git a/third_party/libwebrtc/modules/video_coding/fec_controller_unittest.cc b/third_party/libwebrtc/modules/video_coding/fec_controller_unittest.cc index fa3cc7a93b..c4a0a08fd1 100644 --- a/third_party/libwebrtc/modules/video_coding/fec_controller_unittest.cc +++ b/third_party/libwebrtc/modules/video_coding/fec_controller_unittest.cc @@ -14,6 +14,7 @@ #include +#include "api/environment/environment_factory.h" #include "modules/include/module_fec_types.h" #include "modules/video_coding/fec_controller_default.h" #include "system_wrappers/include/clock.h" @@ -50,7 +51,8 @@ class ProtectionBitrateCalculatorTest : public ::testing::Test { // Note: simulated clock starts at 1 seconds, since parts of webrtc use 0 as // a special case (e.g. frame rate in media optimization). ProtectionBitrateCalculatorTest() - : clock_(1000), fec_controller_(&clock_, &protection_callback_) {} + : clock_(1000), + fec_controller_(CreateEnvironment(&clock_), &protection_callback_) {} SimulatedClock clock_; ProtectionCallback protection_callback_; diff --git a/third_party/libwebrtc/modules/video_coding/nack_requester.cc b/third_party/libwebrtc/modules/video_coding/nack_requester.cc index 008420f4da..b3e928d05e 100644 --- a/third_party/libwebrtc/modules/video_coding/nack_requester.cc +++ b/third_party/libwebrtc/modules/video_coding/nack_requester.cc @@ -141,13 +141,12 @@ void NackRequester::ProcessNacks() { } } -int NackRequester::OnReceivedPacket(uint16_t seq_num, bool is_keyframe) { +int NackRequester::OnReceivedPacket(uint16_t seq_num) { RTC_DCHECK_RUN_ON(worker_thread_); - return OnReceivedPacket(seq_num, is_keyframe, false); + return OnReceivedPacket(seq_num, false); } int NackRequester::OnReceivedPacket(uint16_t seq_num, - bool is_keyframe, bool is_recovered) { RTC_DCHECK_RUN_ON(worker_thread_); // TODO(philipel): When the packet includes information whether it is @@ -158,8 +157,6 @@ int NackRequester::OnReceivedPacket(uint16_t seq_num, if (!initialized_) { newest_seq_num_ = seq_num; - if (is_keyframe) - keyframe_list_.insert(seq_num); initialized_ = true; return 0; } @@ -182,15 +179,6 @@ int NackRequester::OnReceivedPacket(uint16_t seq_num, return nacks_sent_for_packet; } - // Keep track of new keyframes. - if (is_keyframe) - keyframe_list_.insert(seq_num); - - // And remove old ones so we don't accumulate keyframes. - auto it = keyframe_list_.lower_bound(seq_num - kMaxPacketAge); - if (it != keyframe_list_.begin()) - keyframe_list_.erase(keyframe_list_.begin(), it); - if (is_recovered) { recovered_list_.insert(seq_num); @@ -225,8 +213,6 @@ void NackRequester::ClearUpTo(uint16_t seq_num) { // thread. RTC_DCHECK_RUN_ON(worker_thread_); nack_list_.erase(nack_list_.begin(), nack_list_.lower_bound(seq_num)); - keyframe_list_.erase(keyframe_list_.begin(), - keyframe_list_.lower_bound(seq_num)); recovered_list_.erase(recovered_list_.begin(), recovered_list_.lower_bound(seq_num)); } @@ -236,25 +222,6 @@ void NackRequester::UpdateRtt(int64_t rtt_ms) { rtt_ = TimeDelta::Millis(rtt_ms); } -bool NackRequester::RemovePacketsUntilKeyFrame() { - // Called on worker_thread_. - while (!keyframe_list_.empty()) { - auto it = nack_list_.lower_bound(*keyframe_list_.begin()); - - if (it != nack_list_.begin()) { - // We have found a keyframe that actually is newer than at least one - // packet in the nack list. - nack_list_.erase(nack_list_.begin(), it); - return true; - } - - // If this keyframe is so old it does not remove any packets from the list, - // remove it from the list of keyframes and try the next keyframe. - keyframe_list_.erase(keyframe_list_.begin()); - } - return false; -} - void NackRequester::AddPacketsToNack(uint16_t seq_num_start, uint16_t seq_num_end) { // Called on worker_thread_. @@ -262,22 +229,13 @@ void NackRequester::AddPacketsToNack(uint16_t seq_num_start, auto it = nack_list_.lower_bound(seq_num_end - kMaxPacketAge); nack_list_.erase(nack_list_.begin(), it); - // If the nack list is too large, remove packets from the nack list until - // the latest first packet of a keyframe. If the list is still too large, - // clear it and request a keyframe. uint16_t num_new_nacks = ForwardDiff(seq_num_start, seq_num_end); if (nack_list_.size() + num_new_nacks > kMaxNackPackets) { - while (RemovePacketsUntilKeyFrame() && - nack_list_.size() + num_new_nacks > kMaxNackPackets) { - } - - if (nack_list_.size() + num_new_nacks > kMaxNackPackets) { - nack_list_.clear(); - RTC_LOG(LS_WARNING) << "NACK list full, clearing NACK" - " list and requesting keyframe."; - keyframe_request_sender_->RequestKeyFrame(); - return; - } + nack_list_.clear(); + RTC_LOG(LS_WARNING) << "NACK list full, clearing NACK" + " list and requesting keyframe."; + keyframe_request_sender_->RequestKeyFrame(); + return; } for (uint16_t seq_num = seq_num_start; seq_num != seq_num_end; ++seq_num) { diff --git a/third_party/libwebrtc/modules/video_coding/nack_requester.h b/third_party/libwebrtc/modules/video_coding/nack_requester.h index c860787dcf..b5ef30f01b 100644 --- a/third_party/libwebrtc/modules/video_coding/nack_requester.h +++ b/third_party/libwebrtc/modules/video_coding/nack_requester.h @@ -78,8 +78,8 @@ class NackRequester final : public NackRequesterBase { void ProcessNacks() override; - int OnReceivedPacket(uint16_t seq_num, bool is_keyframe); - int OnReceivedPacket(uint16_t seq_num, bool is_keyframe, bool is_recovered); + int OnReceivedPacket(uint16_t seq_num); + int OnReceivedPacket(uint16_t seq_num, bool is_recovered); void ClearUpTo(uint16_t seq_num); void UpdateRtt(int64_t rtt_ms); @@ -108,10 +108,6 @@ class NackRequester final : public NackRequesterBase { void AddPacketsToNack(uint16_t seq_num_start, uint16_t seq_num_end) RTC_EXCLUSIVE_LOCKS_REQUIRED(worker_thread_); - // Removes packets from the nack list until the next keyframe. Returns true - // if packets were removed. - bool RemovePacketsUntilKeyFrame() - RTC_EXCLUSIVE_LOCKS_REQUIRED(worker_thread_); std::vector GetNackBatch(NackFilterOptions options) RTC_EXCLUSIVE_LOCKS_REQUIRED(worker_thread_); @@ -134,8 +130,6 @@ class NackRequester final : public NackRequesterBase { // synchronized access. std::map> nack_list_ RTC_GUARDED_BY(worker_thread_); - std::set> keyframe_list_ - RTC_GUARDED_BY(worker_thread_); std::set> recovered_list_ RTC_GUARDED_BY(worker_thread_); video_coding::Histogram reordering_histogram_ RTC_GUARDED_BY(worker_thread_); diff --git a/third_party/libwebrtc/modules/video_coding/nack_requester_unittest.cc b/third_party/libwebrtc/modules/video_coding/nack_requester_unittest.cc index 6f11cb6e91..2f24df2aef 100644 --- a/third_party/libwebrtc/modules/video_coding/nack_requester_unittest.cc +++ b/third_party/libwebrtc/modules/video_coding/nack_requester_unittest.cc @@ -102,90 +102,25 @@ class TestNackRequester : public ::testing::Test, TEST_F(TestNackRequester, NackOnePacket) { NackRequester& nack_module = CreateNackModule(); - nack_module.OnReceivedPacket(1, false, false); - nack_module.OnReceivedPacket(3, false, false); + nack_module.OnReceivedPacket(1); + nack_module.OnReceivedPacket(3); ASSERT_EQ(1u, sent_nacks_.size()); EXPECT_EQ(2, sent_nacks_[0]); } TEST_F(TestNackRequester, WrappingSeqNum) { NackRequester& nack_module = CreateNackModule(); - nack_module.OnReceivedPacket(0xfffe, false, false); - nack_module.OnReceivedPacket(1, false, false); + nack_module.OnReceivedPacket(0xfffe); + nack_module.OnReceivedPacket(1); ASSERT_EQ(2u, sent_nacks_.size()); EXPECT_EQ(0xffff, sent_nacks_[0]); EXPECT_EQ(0, sent_nacks_[1]); } -TEST_F(TestNackRequester, WrappingSeqNumClearToKeyframe) { - NackRequester& nack_module = CreateNackModule(TimeDelta::Millis(10)); - nack_module.OnReceivedPacket(0xfffe, false, false); - nack_module.OnReceivedPacket(1, false, false); - ASSERT_EQ(2u, sent_nacks_.size()); - EXPECT_EQ(0xffff, sent_nacks_[0]); - EXPECT_EQ(0, sent_nacks_[1]); - - sent_nacks_.clear(); - nack_module.OnReceivedPacket(2, true, false); - ASSERT_EQ(0u, sent_nacks_.size()); - - nack_module.OnReceivedPacket(501, true, false); - ASSERT_EQ(498u, sent_nacks_.size()); - for (int seq_num = 3; seq_num < 501; ++seq_num) - EXPECT_EQ(seq_num, sent_nacks_[seq_num - 3]); - - sent_nacks_.clear(); - nack_module.OnReceivedPacket(1001, false, false); - EXPECT_EQ(499u, sent_nacks_.size()); - for (int seq_num = 502; seq_num < 1001; ++seq_num) - EXPECT_EQ(seq_num, sent_nacks_[seq_num - 502]); - - sent_nacks_.clear(); - clock_->AdvanceTimeMilliseconds(100); - ASSERT_TRUE(WaitForSendNack()); - ASSERT_EQ(999u, sent_nacks_.size()); - EXPECT_EQ(0xffff, sent_nacks_[0]); - EXPECT_EQ(0, sent_nacks_[1]); - for (int seq_num = 3; seq_num < 501; ++seq_num) - EXPECT_EQ(seq_num, sent_nacks_[seq_num - 1]); - for (int seq_num = 502; seq_num < 1001; ++seq_num) - EXPECT_EQ(seq_num, sent_nacks_[seq_num - 2]); - - // Adding packet 1004 will cause the nack list to reach it's max limit. - // It will then clear all nacks up to the next keyframe (seq num 2), - // thus removing 0xffff and 0 from the nack list. - sent_nacks_.clear(); - nack_module.OnReceivedPacket(1004, false, false); - ASSERT_EQ(2u, sent_nacks_.size()); - EXPECT_EQ(1002, sent_nacks_[0]); - EXPECT_EQ(1003, sent_nacks_[1]); - - sent_nacks_.clear(); - clock_->AdvanceTimeMilliseconds(100); - ASSERT_TRUE(WaitForSendNack()); - ASSERT_EQ(999u, sent_nacks_.size()); - for (int seq_num = 3; seq_num < 501; ++seq_num) - EXPECT_EQ(seq_num, sent_nacks_[seq_num - 3]); - for (int seq_num = 502; seq_num < 1001; ++seq_num) - EXPECT_EQ(seq_num, sent_nacks_[seq_num - 4]); - - // Adding packet 1007 will cause the nack module to overflow again, thus - // clearing everything up to 501 which is the next keyframe. - nack_module.OnReceivedPacket(1007, false, false); - sent_nacks_.clear(); - clock_->AdvanceTimeMilliseconds(100); - ASSERT_TRUE(WaitForSendNack()); - ASSERT_EQ(503u, sent_nacks_.size()); - for (int seq_num = 502; seq_num < 1001; ++seq_num) - EXPECT_EQ(seq_num, sent_nacks_[seq_num - 502]); - EXPECT_EQ(1005, sent_nacks_[501]); - EXPECT_EQ(1006, sent_nacks_[502]); -} - TEST_F(TestNackRequester, ResendNack) { NackRequester& nack_module = CreateNackModule(TimeDelta::Millis(1)); - nack_module.OnReceivedPacket(1, false, false); - nack_module.OnReceivedPacket(3, false, false); + nack_module.OnReceivedPacket(1); + nack_module.OnReceivedPacket(3); size_t expected_nacks_sent = 1; ASSERT_EQ(expected_nacks_sent, sent_nacks_.size()); EXPECT_EQ(2, sent_nacks_[0]); @@ -225,8 +160,8 @@ TEST_F(TestNackRequester, ResendNack) { TEST_F(TestNackRequester, ResendPacketMaxRetries) { NackRequester& nack_module = CreateNackModule(TimeDelta::Millis(1)); - nack_module.OnReceivedPacket(1, false, false); - nack_module.OnReceivedPacket(3, false, false); + nack_module.OnReceivedPacket(1); + nack_module.OnReceivedPacket(3); ASSERT_EQ(1u, sent_nacks_.size()); EXPECT_EQ(2, sent_nacks_[0]); @@ -246,37 +181,22 @@ TEST_F(TestNackRequester, ResendPacketMaxRetries) { TEST_F(TestNackRequester, TooLargeNackList) { NackRequester& nack_module = CreateNackModule(); - nack_module.OnReceivedPacket(0, false, false); - nack_module.OnReceivedPacket(1001, false, false); + nack_module.OnReceivedPacket(0); + nack_module.OnReceivedPacket(1001); EXPECT_EQ(1000u, sent_nacks_.size()); EXPECT_EQ(0, keyframes_requested_); - nack_module.OnReceivedPacket(1003, false, false); + nack_module.OnReceivedPacket(1003); EXPECT_EQ(1000u, sent_nacks_.size()); EXPECT_EQ(1, keyframes_requested_); - nack_module.OnReceivedPacket(1004, false, false); - EXPECT_EQ(1000u, sent_nacks_.size()); - EXPECT_EQ(1, keyframes_requested_); -} - -TEST_F(TestNackRequester, TooLargeNackListWithKeyFrame) { - NackRequester& nack_module = CreateNackModule(); - nack_module.OnReceivedPacket(0, false, false); - nack_module.OnReceivedPacket(1, true, false); - nack_module.OnReceivedPacket(1001, false, false); - EXPECT_EQ(999u, sent_nacks_.size()); - EXPECT_EQ(0, keyframes_requested_); - nack_module.OnReceivedPacket(1003, false, false); - EXPECT_EQ(1000u, sent_nacks_.size()); - EXPECT_EQ(0, keyframes_requested_); - nack_module.OnReceivedPacket(1005, false, false); + nack_module.OnReceivedPacket(1004); EXPECT_EQ(1000u, sent_nacks_.size()); EXPECT_EQ(1, keyframes_requested_); } TEST_F(TestNackRequester, ClearUpTo) { NackRequester& nack_module = CreateNackModule(TimeDelta::Millis(1)); - nack_module.OnReceivedPacket(0, false, false); - nack_module.OnReceivedPacket(100, false, false); + nack_module.OnReceivedPacket(0); + nack_module.OnReceivedPacket(100); EXPECT_EQ(99u, sent_nacks_.size()); sent_nacks_.clear(); @@ -289,8 +209,8 @@ TEST_F(TestNackRequester, ClearUpTo) { TEST_F(TestNackRequester, ClearUpToWrap) { NackRequester& nack_module = CreateNackModule(); - nack_module.OnReceivedPacket(0xfff0, false, false); - nack_module.OnReceivedPacket(0xf, false, false); + nack_module.OnReceivedPacket(0xfff0); + nack_module.OnReceivedPacket(0xf); EXPECT_EQ(30u, sent_nacks_.size()); sent_nacks_.clear(); @@ -303,13 +223,13 @@ TEST_F(TestNackRequester, ClearUpToWrap) { TEST_F(TestNackRequester, PacketNackCount) { NackRequester& nack_module = CreateNackModule(TimeDelta::Millis(1)); - EXPECT_EQ(0, nack_module.OnReceivedPacket(0, false, false)); - EXPECT_EQ(0, nack_module.OnReceivedPacket(2, false, false)); - EXPECT_EQ(1, nack_module.OnReceivedPacket(1, false, false)); + EXPECT_EQ(0, nack_module.OnReceivedPacket(0)); + EXPECT_EQ(0, nack_module.OnReceivedPacket(2)); + EXPECT_EQ(1, nack_module.OnReceivedPacket(1)); sent_nacks_.clear(); nack_module.UpdateRtt(100); - EXPECT_EQ(0, nack_module.OnReceivedPacket(5, false, false)); + EXPECT_EQ(0, nack_module.OnReceivedPacket(5)); clock_->AdvanceTimeMilliseconds(100); WaitForSendNack(); EXPECT_EQ(4u, sent_nacks_.size()); @@ -319,40 +239,24 @@ TEST_F(TestNackRequester, PacketNackCount) { EXPECT_EQ(6u, sent_nacks_.size()); - EXPECT_EQ(3, nack_module.OnReceivedPacket(3, false, false)); - EXPECT_EQ(3, nack_module.OnReceivedPacket(4, false, false)); - EXPECT_EQ(0, nack_module.OnReceivedPacket(4, false, false)); -} - -TEST_F(TestNackRequester, NackListFullAndNoOverlapWithKeyframes) { - NackRequester& nack_module = CreateNackModule(); - const int kMaxNackPackets = 1000; - const unsigned int kFirstGap = kMaxNackPackets - 20; - const unsigned int kSecondGap = 200; - uint16_t seq_num = 0; - nack_module.OnReceivedPacket(seq_num++, true, false); - seq_num += kFirstGap; - nack_module.OnReceivedPacket(seq_num++, true, false); - EXPECT_EQ(kFirstGap, sent_nacks_.size()); - sent_nacks_.clear(); - seq_num += kSecondGap; - nack_module.OnReceivedPacket(seq_num, true, false); - EXPECT_EQ(kSecondGap, sent_nacks_.size()); + EXPECT_EQ(3, nack_module.OnReceivedPacket(3)); + EXPECT_EQ(3, nack_module.OnReceivedPacket(4)); + EXPECT_EQ(0, nack_module.OnReceivedPacket(4)); } TEST_F(TestNackRequester, HandleFecRecoveredPacket) { NackRequester& nack_module = CreateNackModule(); - nack_module.OnReceivedPacket(1, false, false); - nack_module.OnReceivedPacket(4, false, true); + nack_module.OnReceivedPacket(1); + nack_module.OnReceivedPacket(4, /*is_recovered=*/true); EXPECT_EQ(0u, sent_nacks_.size()); - nack_module.OnReceivedPacket(5, false, false); + nack_module.OnReceivedPacket(5); EXPECT_EQ(2u, sent_nacks_.size()); } TEST_F(TestNackRequester, SendNackWithoutDelay) { NackRequester& nack_module = CreateNackModule(); - nack_module.OnReceivedPacket(0, false, false); - nack_module.OnReceivedPacket(100, false, false); + nack_module.OnReceivedPacket(0); + nack_module.OnReceivedPacket(100); EXPECT_EQ(99u, sent_nacks_.size()); } @@ -389,14 +293,14 @@ class TestNackRequesterWithFieldTrial : public ::testing::Test, }; TEST_F(TestNackRequesterWithFieldTrial, SendNackWithDelay) { - nack_module_.OnReceivedPacket(0, false, false); - nack_module_.OnReceivedPacket(100, false, false); + nack_module_.OnReceivedPacket(0); + nack_module_.OnReceivedPacket(100); EXPECT_EQ(0u, sent_nacks_.size()); clock_->AdvanceTimeMilliseconds(10); - nack_module_.OnReceivedPacket(106, false, false); + nack_module_.OnReceivedPacket(106); EXPECT_EQ(99u, sent_nacks_.size()); clock_->AdvanceTimeMilliseconds(10); - nack_module_.OnReceivedPacket(109, false, false); + nack_module_.OnReceivedPacket(109); EXPECT_EQ(104u, sent_nacks_.size()); } } // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/qp_parser.cc b/third_party/libwebrtc/modules/video_coding/utility/qp_parser.cc index 3b9aaa377e..a531b54a7e 100644 --- a/third_party/libwebrtc/modules/video_coding/utility/qp_parser.cc +++ b/third_party/libwebrtc/modules/video_coding/utility/qp_parser.cc @@ -37,7 +37,10 @@ absl::optional QpParser::Parse(VideoCodecType codec_type, } else if (codec_type == kVideoCodecH264) { return h264_parsers_[spatial_idx].Parse(frame_data, frame_size); } else if (codec_type == kVideoCodecH265) { - // TODO(bugs.webrtc.org/13485) + // H.265 bitstream parser is conditionally built. +#ifdef RTC_ENABLE_H265 + return h265_parsers_[spatial_idx].Parse(frame_data, frame_size); +#endif } return absl::nullopt; @@ -52,4 +55,15 @@ absl::optional QpParser::H264QpParser::Parse( return bitstream_parser_.GetLastSliceQp(); } +#ifdef RTC_ENABLE_H265 +absl::optional QpParser::H265QpParser::Parse( + const uint8_t* frame_data, + size_t frame_size) { + MutexLock lock(&mutex_); + bitstream_parser_.ParseBitstream( + rtc::ArrayView(frame_data, frame_size)); + return bitstream_parser_.GetLastSliceQp(); +} +#endif + } // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/utility/qp_parser.h b/third_party/libwebrtc/modules/video_coding/utility/qp_parser.h index f132ff9337..210fe02bc3 100644 --- a/third_party/libwebrtc/modules/video_coding/utility/qp_parser.h +++ b/third_party/libwebrtc/modules/video_coding/utility/qp_parser.h @@ -15,6 +15,9 @@ #include "api/video/video_codec_constants.h" #include "api/video/video_codec_type.h" #include "common_video/h264/h264_bitstream_parser.h" +#ifdef RTC_ENABLE_H265 +#include "common_video/h265/h265_bitstream_parser.h" +#endif #include "rtc_base/synchronization/mutex.h" namespace webrtc { @@ -38,6 +41,21 @@ class QpParser { }; H264QpParser h264_parsers_[kMaxSimulcastStreams]; + +#ifdef RTC_ENABLE_H265 + // A thread safe wrapper for H.265 bitstream parser. + class H265QpParser { + public: + absl::optional Parse(const uint8_t* frame_data, + size_t frame_size); + + private: + Mutex mutex_; + H265BitstreamParser bitstream_parser_ RTC_GUARDED_BY(mutex_); + }; + + H265QpParser h265_parsers_[kMaxSimulcastStreams]; +#endif }; } // namespace webrtc -- cgit v1.2.3