diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 01:13:27 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 01:13:27 +0000 |
| commit | 40a355a42d4a9444dc753c04c6608dade2f06a23 (patch) | |
| tree | 871fc667d2de662f171103ce5ec067014ef85e61 /third_party/libwebrtc/video | |
| parent | Adding upstream version 124.0.1. (diff) | |
| download | firefox-upstream/125.0.1.tar.xz firefox-upstream/125.0.1.zip | |
Adding upstream version 125.0.1.upstream/125.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/libwebrtc/video')
32 files changed, 463 insertions, 306 deletions
diff --git a/third_party/libwebrtc/video/BUILD.gn b/third_party/libwebrtc/video/BUILD.gn index e84b73a352..0a930053c0 100644 --- a/third_party/libwebrtc/video/BUILD.gn +++ b/third_party/libwebrtc/video/BUILD.gn @@ -836,6 +836,8 @@ if (rtc_include_tests) { "../api:transport_api", "../api/adaptation:resource_adaptation_api", "../api/crypto:options", + "../api/environment", + "../api/environment:environment_factory", "../api/metronome/test:fake_metronome", "../api/rtc_event_log", "../api/task_queue", @@ -843,7 +845,6 @@ if (rtc_include_tests) { "../api/test/metrics:global_metrics_logger_and_exporter", "../api/test/metrics:metric", "../api/test/video:function_video_factory", - "../api/transport:field_trial_based_config", "../api/units:data_rate", "../api/units:frequency", "../api/units:time_delta", diff --git a/third_party/libwebrtc/video/adaptation/video_adaptation_gn/moz.build b/third_party/libwebrtc/video/adaptation/video_adaptation_gn/moz.build index 2d1fd69ed0..18d29da556 100644 --- a/third_party/libwebrtc/video/adaptation/video_adaptation_gn/moz.build +++ b/third_party/libwebrtc/video/adaptation/video_adaptation_gn/moz.build @@ -209,7 +209,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] @@ -219,10 +218,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": "-msse2" ] - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/video/config/encoder_config_gn/moz.build b/third_party/libwebrtc/video/config/encoder_config_gn/moz.build index 1c2c9ef835..4dbe7bf32c 100644 --- a/third_party/libwebrtc/video/config/encoder_config_gn/moz.build +++ b/third_party/libwebrtc/video/config/encoder_config_gn/moz.build @@ -195,7 +195,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] @@ -205,10 +204,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": "-msse2" ] - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/video/config/streams_config_gn/moz.build b/third_party/libwebrtc/video/config/streams_config_gn/moz.build index e2d2f9963d..c34883fb37 100644 --- a/third_party/libwebrtc/video/config/streams_config_gn/moz.build +++ b/third_party/libwebrtc/video/config/streams_config_gn/moz.build @@ -201,7 +201,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] @@ -211,10 +210,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": "-msse2" ] - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/video/decode_synchronizer_gn/moz.build b/third_party/libwebrtc/video/decode_synchronizer_gn/moz.build index d0e4adeda2..0aff4a854c 100644 --- a/third_party/libwebrtc/video/decode_synchronizer_gn/moz.build +++ b/third_party/libwebrtc/video/decode_synchronizer_gn/moz.build @@ -199,7 +199,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] @@ -209,10 +208,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": "-msse2" ] - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/video/end_to_end_tests/multi_stream_tester.cc b/third_party/libwebrtc/video/end_to_end_tests/multi_stream_tester.cc index 1eb388cc76..759b75ccf9 100644 --- a/third_party/libwebrtc/video/end_to_end_tests/multi_stream_tester.cc +++ b/third_party/libwebrtc/video/end_to_end_tests/multi_stream_tester.cc @@ -14,8 +14,8 @@ #include <utility> #include <vector> -#include "api/rtc_event_log/rtc_event_log.h" -#include "api/task_queue/default_task_queue_factory.h" +#include "api/environment/environment.h" +#include "api/environment/environment_factory.h" #include "api/task_queue/task_queue_base.h" #include "api/test/create_frame_generator.h" #include "api/test/simulated_network.h" @@ -42,17 +42,13 @@ MultiStreamTester::MultiStreamTester() { MultiStreamTester::~MultiStreamTester() = default; void MultiStreamTester::RunTest() { - webrtc::RtcEventLogNull event_log; - auto task_queue_factory = CreateDefaultTaskQueueFactory(); + Environment env = CreateEnvironment(); // Use high prioirity since this task_queue used for fake network delivering // at correct time. Those test tasks should be prefered over code under test // to make test more stable. - auto task_queue = task_queue_factory->CreateTaskQueue( + auto task_queue = env.task_queue_factory().CreateTaskQueue( "TaskQueue", TaskQueueFactory::Priority::HIGH); - CallConfig config(&event_log); - test::ScopedKeyValueConfig field_trials; - config.trials = &field_trials; - config.task_queue_factory = task_queue_factory.get(); + CallConfig config(env); std::unique_ptr<Call> sender_call; std::unique_ptr<Call> receiver_call; std::unique_ptr<test::DirectTransport> sender_transport; @@ -115,10 +111,10 @@ void MultiStreamTester::RunTest() { receive_streams[i]->Start(); auto* frame_generator = new test::FrameGeneratorCapturer( - Clock::GetRealTimeClock(), + &env.clock(), test::CreateSquareFrameGenerator(width, height, absl::nullopt, absl::nullopt), - 30, *task_queue_factory); + 30, env.task_queue_factory()); frame_generators[i] = frame_generator; send_streams[i]->SetSource(frame_generator, DegradationPreference::MAINTAIN_FRAMERATE); diff --git a/third_party/libwebrtc/video/end_to_end_tests/network_state_tests.cc b/third_party/libwebrtc/video/end_to_end_tests/network_state_tests.cc index 7bc9f1493e..7e9c86362a 100644 --- a/third_party/libwebrtc/video/end_to_end_tests/network_state_tests.cc +++ b/third_party/libwebrtc/video/end_to_end_tests/network_state_tests.cc @@ -94,7 +94,7 @@ void NetworkStateEndToEndTest::VerifyNewVideoSendStreamsRespectNetworkState( SendTask(task_queue(), [this, network_to_bring_up, &encoder_factory, transport]() { - CreateSenderCall(CallConfig(send_event_log_.get())); + CreateSenderCall(); sender_call_->SignalChannelNetworkState(network_to_bring_up, kNetworkUp); CreateSendConfig(1, 0, 0, transport); diff --git a/third_party/libwebrtc/video/end_to_end_tests/stats_tests.cc b/third_party/libwebrtc/video/end_to_end_tests/stats_tests.cc index cc0b328b2b..d6820eeac2 100644 --- a/third_party/libwebrtc/video/end_to_end_tests/stats_tests.cc +++ b/third_party/libwebrtc/video/end_to_end_tests/stats_tests.cc @@ -518,9 +518,9 @@ TEST_F(StatsEndToEndTest, MAYBE_ContentTypeSwitches) { metrics::Reset(); - CallConfig send_config(send_event_log_.get()); + CallConfig send_config = SendCallConfig(); test.ModifySenderBitrateConfig(&send_config.bitrate_config); - CallConfig recv_config(recv_event_log_.get()); + CallConfig recv_config = RecvCallConfig(); test.ModifyReceiverBitrateConfig(&recv_config.bitrate_config); VideoEncoderConfig encoder_config_with_screenshare; @@ -732,13 +732,13 @@ TEST_F(StatsEndToEndTest, CallReportsRttForSender) { Start(); }); - int64_t start_time_ms = clock_->TimeInMilliseconds(); + int64_t start_time_ms = env().clock().TimeInMilliseconds(); while (true) { Call::Stats stats; SendTask(task_queue(), [this, &stats]() { stats = sender_call_->GetStats(); }); ASSERT_GE(start_time_ms + test::VideoTestConstants::kDefaultTimeout.ms(), - clock_->TimeInMilliseconds()) + env().clock().TimeInMilliseconds()) << "No RTT stats before timeout!"; if (stats.rtt_ms != -1) { // To avoid failures caused by rounding or minor ntp clock adjustments, diff --git a/third_party/libwebrtc/video/frame_cadence_adapter.cc b/third_party/libwebrtc/video/frame_cadence_adapter.cc index b09986847b..2c4acdd6c2 100644 --- a/third_party/libwebrtc/video/frame_cadence_adapter.cc +++ b/third_party/libwebrtc/video/frame_cadence_adapter.cc @@ -50,7 +50,7 @@ class AdapterMode { // Called on the worker thread for every frame that enters. virtual void OnFrame(Timestamp post_time, - int frames_scheduled_for_processing, + bool queue_overload, const VideoFrame& frame) = 0; // Returns the currently estimated input framerate. @@ -71,10 +71,10 @@ class PassthroughAdapterMode : public AdapterMode { // Adapter overrides. void OnFrame(Timestamp post_time, - int frames_scheduled_for_processing, + bool queue_overload, const VideoFrame& frame) override { RTC_DCHECK_RUN_ON(&sequence_checker_); - callback_->OnFrame(post_time, frames_scheduled_for_processing, frame); + callback_->OnFrame(post_time, queue_overload, frame); } absl::optional<uint32_t> GetInputFrameRateFps() override { @@ -119,7 +119,7 @@ class ZeroHertzAdapterMode : public AdapterMode { // Adapter overrides. void OnFrame(Timestamp post_time, - int frames_scheduled_for_processing, + bool queue_overload, const VideoFrame& frame) override; absl::optional<uint32_t> GetInputFrameRateFps() override; void UpdateFrameRate() override {} @@ -131,6 +131,10 @@ class ZeroHertzAdapterMode : public AdapterMode { // Callback::RequestRefreshFrame. void ProcessKeyFrameRequest(); + // Updates the restrictions of max frame rate for the video source. + // Always called during construction using latest `restricted_frame_delay_`. + void UpdateVideoSourceRestrictions(absl::optional<double> max_frame_rate); + private: // The tracking state of each spatial layer. Used for determining when to // stop repeating frames. @@ -171,20 +175,22 @@ class ZeroHertzAdapterMode : public AdapterMode { // after this call. void ResetQualityConvergenceInfo() RTC_RUN_ON(sequence_checker_); // Processes incoming frames on a delayed cadence. - void ProcessOnDelayedCadence() RTC_RUN_ON(sequence_checker_); - // Schedules a later repeat with delay depending on state of layer trackers. + void ProcessOnDelayedCadence(Timestamp post_time) + RTC_RUN_ON(sequence_checker_); + // Schedules a later repeat with delay depending on state of layer trackers + // and if UpdateVideoSourceRestrictions has been called or not. // If true is passed in `idle_repeat`, the repeat is going to be - // kZeroHertzIdleRepeatRatePeriod. Otherwise it'll be the value of - // `frame_delay`. + // kZeroHertzIdleRepeatRatePeriod. Otherwise it'll be the maximum value of + // `frame_delay` or `restricted_frame_delay_` if it has been set. void ScheduleRepeat(int frame_id, bool idle_repeat) RTC_RUN_ON(sequence_checker_); - // Repeats a frame in the abscence of incoming frames. Slows down when quality + // Repeats a frame in the absence of incoming frames. Slows down when quality // convergence is attained, and stops the cadence terminally when new frames // have arrived. void ProcessRepeatedFrameOnDelayedCadence(int frame_id) RTC_RUN_ON(sequence_checker_); // Sends a frame, updating the timestamp to the current time. - void SendFrameNow(const VideoFrame& frame) const + void SendFrameNow(Timestamp post_time, const VideoFrame& frame) const RTC_RUN_ON(sequence_checker_); // Returns the repeat duration depending on if it's an idle repeat or not. TimeDelta RepeatDuration(bool idle_repeat) const @@ -220,6 +226,10 @@ class ZeroHertzAdapterMode : public AdapterMode { // they can be dropped in various places in the capture pipeline. RepeatingTaskHandle refresh_frame_requester_ RTC_GUARDED_BY(sequence_checker_); + // Can be set by UpdateVideoSourceRestrictions when the video source restricts + // the max frame rate. + absl::optional<TimeDelta> restricted_frame_delay_ + RTC_GUARDED_BY(sequence_checker_); ScopedTaskSafety safety_; }; @@ -240,6 +250,8 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface { void UpdateLayerQualityConvergence(size_t spatial_index, bool quality_converged) override; void UpdateLayerStatus(size_t spatial_index, bool enabled) override; + void UpdateVideoSourceRestrictions( + absl::optional<double> max_frame_rate) override; void ProcessKeyFrameRequest() override; // VideoFrameSink overrides. @@ -249,9 +261,9 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface { const VideoTrackSourceConstraints& constraints) override; private: - // Called from OnFrame in zero-hertz mode. + // Called from OnFrame in both pass-through and zero-hertz mode. void OnFrameOnMainQueue(Timestamp post_time, - int frames_scheduled_for_processing, + bool queue_overload, const VideoFrame& frame) RTC_RUN_ON(queue_); // Returns true under all of the following conditions: @@ -276,6 +288,7 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface { absl::optional<ZeroHertzAdapterMode> zero_hertz_adapter_; // If set, zero-hertz mode has been enabled. absl::optional<ZeroHertzModeParams> zero_hertz_params_; + std::atomic<bool> zero_hertz_adapter_is_active_{false}; // Cache for the current adapter mode. AdapterMode* current_adapter_mode_ = nullptr; @@ -290,10 +303,14 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface { absl::optional<VideoTrackSourceConstraints> source_constraints_ RTC_GUARDED_BY(queue_); + // Stores the latest restriction in max frame rate set by + // UpdateVideoSourceRestrictions. Ensures that a previously set restriction + // can be maintained during reconstructions of the adapter. + absl::optional<double> restricted_max_frame_rate_ RTC_GUARDED_BY(queue_); + // Race checker for incoming frames. This is the network thread in chromium, // but may vary from test contexts. rtc::RaceChecker incoming_frame_race_checker_; - bool has_reported_screenshare_frame_rate_umas_ RTC_GUARDED_BY(queue_) = false; // Number of frames that are currently scheduled for processing on the // `queue_`. @@ -355,7 +372,7 @@ void ZeroHertzAdapterMode::UpdateLayerStatus(size_t spatial_index, } void ZeroHertzAdapterMode::OnFrame(Timestamp post_time, - int frames_scheduled_for_processing, + bool queue_overload, const VideoFrame& frame) { RTC_DCHECK_RUN_ON(&sequence_checker_); TRACE_EVENT0("webrtc", "ZeroHertzAdapterMode::OnFrame"); @@ -382,7 +399,7 @@ void ZeroHertzAdapterMode::OnFrame(Timestamp post_time, frame_id); queue_->PostDelayedHighPrecisionTask( SafeTask(safety_.flag(), - [this, frame_id, frame] { + [this, post_time, frame_id, frame] { RTC_UNUSED(frame_id); RTC_DCHECK_RUN_ON(&sequence_checker_); TRACE_EVENT_ASYNC_END0(TRACE_DISABLED_BY_DEFAULT("webrtc"), @@ -390,7 +407,7 @@ void ZeroHertzAdapterMode::OnFrame(Timestamp post_time, TRACE_EVENT_ASYNC_END0(TRACE_DISABLED_BY_DEFAULT("webrtc"), "OnFrameToEncode", frame.video_frame_buffer().get()); - ProcessOnDelayedCadence(); + ProcessOnDelayedCadence(post_time); }), std::max(frame_delay_ - time_spent_since_post, TimeDelta::Zero())); } @@ -411,6 +428,20 @@ absl::optional<uint32_t> ZeroHertzAdapterMode::GetInputFrameRateFps() { return max_fps_; } +void ZeroHertzAdapterMode::UpdateVideoSourceRestrictions( + absl::optional<double> max_frame_rate) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + TRACE_EVENT_INSTANT1(TRACE_DISABLED_BY_DEFAULT("webrtc"), __func__, + "max_frame_rate", max_frame_rate.value_or(-1)); + if (max_frame_rate.value_or(0) > 0) { + // Set new, validated (> 0) and restricted frame rate. + restricted_frame_delay_ = TimeDelta::Seconds(1) / *max_frame_rate; + } else { + // Source reports that the frame rate is now unrestricted. + restricted_frame_delay_ = absl::nullopt; + } +} + void ZeroHertzAdapterMode::ProcessKeyFrameRequest() { RTC_DCHECK_RUN_ON(&sequence_checker_); TRACE_EVENT_INSTANT0("webrtc", __func__); @@ -475,13 +506,13 @@ void ZeroHertzAdapterMode::ResetQualityConvergenceInfo() { } } -void ZeroHertzAdapterMode::ProcessOnDelayedCadence() { +void ZeroHertzAdapterMode::ProcessOnDelayedCadence(Timestamp post_time) { RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK(!queued_frames_.empty()); TRACE_EVENT0("webrtc", __func__); // Avoid sending the front frame for encoding (which could take a long time) - // until we schedule a repeate. + // until we schedule a repeat. VideoFrame front_frame = queued_frames_.front(); // If there were two or more frames stored, we do not have to schedule repeats @@ -494,7 +525,7 @@ void ZeroHertzAdapterMode::ProcessOnDelayedCadence() { // arrive. ScheduleRepeat(current_frame_id_, HasQualityConverged()); } - SendFrameNow(front_frame); + SendFrameNow(post_time, front_frame); } void ZeroHertzAdapterMode::ScheduleRepeat(int frame_id, bool idle_repeat) { @@ -551,23 +582,36 @@ void ZeroHertzAdapterMode::ProcessRepeatedFrameOnDelayedCadence(int frame_id) { // Schedule another repeat before sending the frame off which could take time. ScheduleRepeat(frame_id, HasQualityConverged()); - SendFrameNow(frame); + // Mark `post_time` with 0 to signal that this is a repeated frame. + SendFrameNow(Timestamp::Zero(), frame); } -void ZeroHertzAdapterMode::SendFrameNow(const VideoFrame& frame) const { +void ZeroHertzAdapterMode::SendFrameNow(Timestamp post_time, + const VideoFrame& frame) const { RTC_DCHECK_RUN_ON(&sequence_checker_); TRACE_EVENT0("webrtc", __func__); - // TODO(crbug.com/1255737): figure out if frames_scheduled_for_processing - // makes sense to compute in this implementation. - callback_->OnFrame(/*post_time=*/clock_->CurrentTime(), - /*frames_scheduled_for_processing=*/1, frame); + Timestamp now = clock_->CurrentTime(); + // Exclude repeated frames which are marked with zero as post time. + if (post_time != Timestamp::Zero()) { + TimeDelta delay = (now - post_time); + RTC_HISTOGRAM_COUNTS_10000("WebRTC.Screenshare.ZeroHz.DelayMs", delay.ms()); + } + // TODO(crbug.com/1255737): ensure queue_overload is computed from current + // conditions on the encoder queue. + callback_->OnFrame(/*post_time=*/now, + /*queue_overload=*/false, frame); } TimeDelta ZeroHertzAdapterMode::RepeatDuration(bool idle_repeat) const { RTC_DCHECK_RUN_ON(&sequence_checker_); + // By default use `frame_delay_` in non-idle repeat mode but use the + // restricted frame delay instead if it is set in + // UpdateVideoSourceRestrictions. + TimeDelta frame_delay = + std::max(frame_delay_, restricted_frame_delay_.value_or(frame_delay_)); return idle_repeat ? FrameCadenceAdapterInterface::kZeroHertzIdleRepeatRatePeriod - : frame_delay_; + : frame_delay; } void ZeroHertzAdapterMode::MaybeStartRefreshFrameRequester() { @@ -609,8 +653,6 @@ void FrameCadenceAdapterImpl::SetZeroHertzModeEnabled( absl::optional<ZeroHertzModeParams> params) { RTC_DCHECK_RUN_ON(queue_); bool was_zero_hertz_enabled = zero_hertz_params_.has_value(); - if (params.has_value() && !was_zero_hertz_enabled) - has_reported_screenshare_frame_rate_umas_ = false; zero_hertz_params_ = params; MaybeReconfigureAdapters(was_zero_hertz_enabled); } @@ -642,6 +684,17 @@ void FrameCadenceAdapterImpl::UpdateLayerStatus(size_t spatial_index, zero_hertz_adapter_->UpdateLayerStatus(spatial_index, enabled); } +void FrameCadenceAdapterImpl::UpdateVideoSourceRestrictions( + absl::optional<double> max_frame_rate) { + RTC_DCHECK_RUN_ON(queue_); + // Store the restriction to ensure that it can be reapplied in possible + // future adapter creations on configuration changes. + restricted_max_frame_rate_ = max_frame_rate; + if (zero_hertz_adapter_) { + zero_hertz_adapter_->UpdateVideoSourceRestrictions(max_frame_rate); + } +} + void FrameCadenceAdapterImpl::ProcessKeyFrameRequest() { RTC_DCHECK_RUN_ON(queue_); if (zero_hertz_adapter_) @@ -657,14 +710,21 @@ void FrameCadenceAdapterImpl::OnFrame(const VideoFrame& frame) { // Local time in webrtc time base. Timestamp post_time = clock_->CurrentTime(); frames_scheduled_for_processing_.fetch_add(1, std::memory_order_relaxed); - TRACE_EVENT_ASYNC_BEGIN0(TRACE_DISABLED_BY_DEFAULT("webrtc"), - "OnFrameToEncode", frame.video_frame_buffer().get()); - TRACE_EVENT_ASYNC_BEGIN0(TRACE_DISABLED_BY_DEFAULT("webrtc"), - "OnFrameToQueue", frame.video_frame_buffer().get()); + if (zero_hertz_adapter_is_active_.load(std::memory_order_relaxed)) { + TRACE_EVENT_ASYNC_BEGIN0(TRACE_DISABLED_BY_DEFAULT("webrtc"), + "OnFrameToEncode", + frame.video_frame_buffer().get()); + TRACE_EVENT_ASYNC_BEGIN0(TRACE_DISABLED_BY_DEFAULT("webrtc"), + "OnFrameToQueue", + frame.video_frame_buffer().get()); + } queue_->PostTask(SafeTask(safety_.flag(), [this, post_time, frame] { RTC_DCHECK_RUN_ON(queue_); - TRACE_EVENT_ASYNC_END0(TRACE_DISABLED_BY_DEFAULT("webrtc"), - "OnFrameToQueue", frame.video_frame_buffer().get()); + if (zero_hertz_adapter_is_active_.load(std::memory_order_relaxed)) { + TRACE_EVENT_ASYNC_END0(TRACE_DISABLED_BY_DEFAULT("webrtc"), + "OnFrameToQueue", + frame.video_frame_buffer().get()); + } if (zero_hertz_adapter_created_timestamp_.has_value()) { TimeDelta time_until_first_frame = clock_->CurrentTime() - *zero_hertz_adapter_created_timestamp_; @@ -677,7 +737,7 @@ void FrameCadenceAdapterImpl::OnFrame(const VideoFrame& frame) { const int frames_scheduled_for_processing = frames_scheduled_for_processing_.fetch_sub(1, std::memory_order_relaxed); - OnFrameOnMainQueue(post_time, frames_scheduled_for_processing, + OnFrameOnMainQueue(post_time, frames_scheduled_for_processing > 1, std::move(frame)); })); } @@ -705,13 +765,11 @@ void FrameCadenceAdapterImpl::OnConstraintsChanged( })); } -void FrameCadenceAdapterImpl::OnFrameOnMainQueue( - Timestamp post_time, - int frames_scheduled_for_processing, - const VideoFrame& frame) { +void FrameCadenceAdapterImpl::OnFrameOnMainQueue(Timestamp post_time, + bool queue_overload, + const VideoFrame& frame) { RTC_DCHECK_RUN_ON(queue_); - current_adapter_mode_->OnFrame(post_time, frames_scheduled_for_processing, - frame); + current_adapter_mode_->OnFrame(post_time, queue_overload, frame); } bool FrameCadenceAdapterImpl::IsZeroHertzScreenshareEnabled() const { @@ -727,17 +785,25 @@ void FrameCadenceAdapterImpl::MaybeReconfigureAdapters( RTC_DCHECK_RUN_ON(queue_); bool is_zero_hertz_enabled = IsZeroHertzScreenshareEnabled(); if (is_zero_hertz_enabled) { - if (!was_zero_hertz_enabled) { + bool max_fps_has_changed = GetInputFrameRateFps().value_or(-1) != + source_constraints_->max_fps.value_or(-1); + if (!was_zero_hertz_enabled || max_fps_has_changed) { + RTC_LOG(LS_INFO) << "Zero hertz mode enabled (max_fps=" + << source_constraints_->max_fps.value() << ")"; zero_hertz_adapter_.emplace(queue_, clock_, callback_, source_constraints_->max_fps.value()); - RTC_LOG(LS_INFO) << "Zero hertz mode activated."; + zero_hertz_adapter_->UpdateVideoSourceRestrictions( + restricted_max_frame_rate_); zero_hertz_adapter_created_timestamp_ = clock_->CurrentTime(); } zero_hertz_adapter_->ReconfigureParameters(zero_hertz_params_.value()); current_adapter_mode_ = &zero_hertz_adapter_.value(); } else { - if (was_zero_hertz_enabled) + if (was_zero_hertz_enabled) { zero_hertz_adapter_ = absl::nullopt; + zero_hertz_adapter_is_active_.store(false, std::memory_order_relaxed); + RTC_LOG(LS_INFO) << "Zero hertz mode disabled."; + } current_adapter_mode_ = &passthrough_adapter_.value(); } } diff --git a/third_party/libwebrtc/video/frame_cadence_adapter.h b/third_party/libwebrtc/video/frame_cadence_adapter.h index d0eab7e770..2b62bb26cd 100644 --- a/third_party/libwebrtc/video/frame_cadence_adapter.h +++ b/third_party/libwebrtc/video/frame_cadence_adapter.h @@ -60,14 +60,11 @@ class FrameCadenceAdapterInterface // The |post_time| parameter indicates the current time sampled when // FrameCadenceAdapterInterface::OnFrame was called. // - // |frames_scheduled_for_processing| indicates how many frames that have - // been scheduled for processing. During sequential conditions where - // FrameCadenceAdapterInterface::OnFrame is invoked and subsequently ending - // up in this callback, this value will read 1. Otherwise if the - // |queue| gets stalled for some reason, the value will increase - // beyond 1. + // |queue_overload| is true if the frame cadence adapter notices it's + // not able to deliver the incoming |frame| to the |queue| in the expected + // time. virtual void OnFrame(Timestamp post_time, - int frames_scheduled_for_processing, + bool queue_overload, const VideoFrame& frame) = 0; // Called when the source has discarded a frame. @@ -112,6 +109,12 @@ class FrameCadenceAdapterInterface // Updates spatial layer enabled status. virtual void UpdateLayerStatus(size_t spatial_index, bool enabled) = 0; + // Updates the restrictions of max frame rate for the video source. + // The new `max_frame_rate` will only affect the cadence of Callback::OnFrame + // for non-idle (non converged) repeated frames. + virtual void UpdateVideoSourceRestrictions( + absl::optional<double> max_frame_rate) = 0; + // Conditionally requests a refresh frame via // Callback::RequestRefreshFrame. virtual void ProcessKeyFrameRequest() = 0; diff --git a/third_party/libwebrtc/video/frame_cadence_adapter_gn/moz.build b/third_party/libwebrtc/video/frame_cadence_adapter_gn/moz.build index 51ec86b2cf..a860d550ad 100644 --- a/third_party/libwebrtc/video/frame_cadence_adapter_gn/moz.build +++ b/third_party/libwebrtc/video/frame_cadence_adapter_gn/moz.build @@ -199,7 +199,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] @@ -209,10 +208,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": "-msse2" ] - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/video/frame_cadence_adapter_unittest.cc b/third_party/libwebrtc/video/frame_cadence_adapter_unittest.cc index 052b0a6c61..0fef2400f0 100644 --- a/third_party/libwebrtc/video/frame_cadence_adapter_unittest.cc +++ b/third_party/libwebrtc/video/frame_cadence_adapter_unittest.cc @@ -70,7 +70,7 @@ std::unique_ptr<FrameCadenceAdapterInterface> CreateAdapter( class MockCallback : public FrameCadenceAdapterInterface::Callback { public: - MOCK_METHOD(void, OnFrame, (Timestamp, int, const VideoFrame&), (override)); + MOCK_METHOD(void, OnFrame, (Timestamp, bool, const VideoFrame&), (override)); MOCK_METHOD(void, OnDiscardedFrame, (), (override)); MOCK_METHOD(void, RequestRefreshFrame, (), (override)); }; @@ -115,13 +115,13 @@ TEST(FrameCadenceAdapterTest, CountsOutstandingFramesToProcess) { MockCallback callback; auto adapter = CreateAdapter(no_field_trials, time_controller.GetClock()); adapter->Initialize(&callback); - EXPECT_CALL(callback, OnFrame(_, 2, _)).Times(1); - EXPECT_CALL(callback, OnFrame(_, 1, _)).Times(1); + EXPECT_CALL(callback, OnFrame(_, true, _)).Times(1); + EXPECT_CALL(callback, OnFrame(_, false, _)).Times(1); auto frame = CreateFrame(); adapter->OnFrame(frame); adapter->OnFrame(frame); time_controller.AdvanceTime(TimeDelta::Zero()); - EXPECT_CALL(callback, OnFrame(_, 1, _)).Times(1); + EXPECT_CALL(callback, OnFrame(_, false, _)).Times(1); adapter->OnFrame(frame); time_controller.AdvanceTime(TimeDelta::Zero()); } @@ -184,6 +184,29 @@ TEST(FrameCadenceAdapterTest, FrameRateFollowsMaxFpsWhenZeroHertzActivated) { } } +TEST(FrameCadenceAdapterTest, ZeroHertzAdapterSupportsMaxFpsChange) { + ZeroHertzFieldTrialEnabler enabler; + GlobalSimulatedTimeController time_controller(Timestamp::Zero()); + auto adapter = CreateAdapter(enabler, time_controller.GetClock()); + MockCallback callback; + adapter->Initialize(&callback); + adapter->SetZeroHertzModeEnabled( + FrameCadenceAdapterInterface::ZeroHertzModeParams{}); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1}); + time_controller.AdvanceTime(TimeDelta::Zero()); + EXPECT_EQ(adapter->GetInputFrameRateFps(), 1u); + adapter->OnFrame(CreateFrame()); + time_controller.AdvanceTime(TimeDelta::Seconds(1)); + adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 2}); + time_controller.AdvanceTime(TimeDelta::Zero()); + EXPECT_EQ(adapter->GetInputFrameRateFps(), 2u); + adapter->OnFrame(CreateFrame()); + // Ensure that the max_fps has been changed from 1 to 2 fps even if it was + // changed while zero hertz was already active. + EXPECT_CALL(callback, OnFrame); + time_controller.AdvanceTime(TimeDelta::Millis(500)); +} + TEST(FrameCadenceAdapterTest, FrameRateFollowsRateStatisticsAfterZeroHertzDeactivated) { ZeroHertzFieldTrialEnabler enabler; @@ -230,7 +253,7 @@ TEST(FrameCadenceAdapterTest, ForwardsFramesDelayed) { EXPECT_CALL(callback, OnFrame).Times(0); adapter->OnFrame(frame); EXPECT_CALL(callback, OnFrame) - .WillOnce(Invoke([&](Timestamp post_time, int, + .WillOnce(Invoke([&](Timestamp post_time, bool, const VideoFrame& frame) { EXPECT_EQ(post_time, time_controller.GetClock()->CurrentTime()); EXPECT_EQ(frame.timestamp_us(), @@ -309,7 +332,7 @@ TEST(FrameCadenceAdapterTest, RepeatsFramesDelayed) { adapter->OnFrame(frame); EXPECT_CALL(callback, OnFrame) - .WillOnce(Invoke([&](Timestamp post_time, int, const VideoFrame& frame) { + .WillOnce(Invoke([&](Timestamp post_time, bool, const VideoFrame& frame) { EXPECT_EQ(post_time, time_controller.GetClock()->CurrentTime()); EXPECT_EQ(frame.timestamp_us(), original_timestamp_us); EXPECT_EQ(frame.ntp_time_ms(), original_ntp_time.ToMs()); @@ -318,7 +341,7 @@ TEST(FrameCadenceAdapterTest, RepeatsFramesDelayed) { Mock::VerifyAndClearExpectations(&callback); EXPECT_CALL(callback, OnFrame) - .WillOnce(Invoke([&](Timestamp post_time, int, const VideoFrame& frame) { + .WillOnce(Invoke([&](Timestamp post_time, bool, const VideoFrame& frame) { EXPECT_EQ(post_time, time_controller.GetClock()->CurrentTime()); EXPECT_EQ(frame.timestamp_us(), original_timestamp_us + rtc::kNumMicrosecsPerSec); @@ -329,7 +352,7 @@ TEST(FrameCadenceAdapterTest, RepeatsFramesDelayed) { Mock::VerifyAndClearExpectations(&callback); EXPECT_CALL(callback, OnFrame) - .WillOnce(Invoke([&](Timestamp post_time, int, const VideoFrame& frame) { + .WillOnce(Invoke([&](Timestamp post_time, bool, const VideoFrame& frame) { EXPECT_EQ(post_time, time_controller.GetClock()->CurrentTime()); EXPECT_EQ(frame.timestamp_us(), original_timestamp_us + 2 * rtc::kNumMicrosecsPerSec); @@ -359,7 +382,7 @@ TEST(FrameCadenceAdapterTest, // Send one frame, expect a repeat. adapter->OnFrame(CreateFrame()); EXPECT_CALL(callback, OnFrame) - .WillOnce(Invoke([&](Timestamp post_time, int, const VideoFrame& frame) { + .WillOnce(Invoke([&](Timestamp post_time, bool, const VideoFrame& frame) { EXPECT_EQ(post_time, time_controller.GetClock()->CurrentTime()); EXPECT_EQ(frame.timestamp_us(), 0); EXPECT_EQ(frame.ntp_time_ms(), 0); @@ -367,7 +390,7 @@ TEST(FrameCadenceAdapterTest, time_controller.AdvanceTime(TimeDelta::Seconds(1)); Mock::VerifyAndClearExpectations(&callback); EXPECT_CALL(callback, OnFrame) - .WillOnce(Invoke([&](Timestamp post_time, int, const VideoFrame& frame) { + .WillOnce(Invoke([&](Timestamp post_time, bool, const VideoFrame& frame) { EXPECT_EQ(post_time, time_controller.GetClock()->CurrentTime()); EXPECT_EQ(frame.timestamp_us(), 0); EXPECT_EQ(frame.ntp_time_ms(), 0); @@ -399,7 +422,7 @@ TEST(FrameCadenceAdapterTest, StopsRepeatingFramesDelayed) { // Send the new frame at 2.5s, which should appear after 3.5s. adapter->OnFrame(CreateFrameWithTimestamps(&time_controller)); EXPECT_CALL(callback, OnFrame) - .WillOnce(Invoke([&](Timestamp, int, const VideoFrame& frame) { + .WillOnce(Invoke([&](Timestamp, bool, const VideoFrame& frame) { EXPECT_EQ(frame.timestamp_us(), 5 * rtc::kNumMicrosecsPerSec / 2); EXPECT_EQ(frame.ntp_time_ms(), original_ntp_time.ToMs() + 5u * rtc::kNumMillisecsPerSec / 2); @@ -708,6 +731,8 @@ class ZeroHertzLayerQualityConvergenceTest : public ::testing::Test { static constexpr TimeDelta kMinFrameDelay = TimeDelta::Millis(100); static constexpr TimeDelta kIdleFrameDelay = FrameCadenceAdapterInterface::kZeroHertzIdleRepeatRatePeriod; + // Restricts non-idle repeat rate to 5 fps (default is 10 fps); + static constexpr int kRestrictedMaxFps = 5; ZeroHertzLayerQualityConvergenceTest() { adapter_->Initialize(&callback_); @@ -725,7 +750,7 @@ class ZeroHertzLayerQualityConvergenceTest : public ::testing::Test { std::initializer_list<TimeDelta> list) { Timestamp origin = time_controller_.GetClock()->CurrentTime(); for (auto delay : list) { - EXPECT_CALL(callback_, OnFrame(origin + delay, _, _)); + EXPECT_CALL(callback_, OnFrame(origin + delay, false, _)); time_controller_.AdvanceTime(origin + delay - time_controller_.GetClock()->CurrentTime()); } @@ -810,6 +835,100 @@ TEST_F(ZeroHertzLayerQualityConvergenceTest, }); } +TEST_F(ZeroHertzLayerQualityConvergenceTest, + UnconvergedRepeatRateAdaptsDownWhenRestricted) { + PassFrame(); + ScheduleDelayed(1.5 * kMinFrameDelay, [&] { + adapter_->UpdateVideoSourceRestrictions(kRestrictedMaxFps); + }); + ExpectFrameEntriesAtDelaysFromNow({ + 1 * kMinFrameDelay, // Original frame emitted at non-restricted rate. + + // 1.5 * kMinFrameDelay: restricts max fps to 5 fps which should result + // in a new non-idle repeat delay of 2 * kMinFrameDelay. + 2 * kMinFrameDelay, // Unconverged repeat at non-restricted rate. + 4 * kMinFrameDelay, // Unconverged repeats at restricted rate. This + // happens 2 * kMinFrameDelay after the last frame. + 6 * kMinFrameDelay, // ... + }); +} + +TEST_F(ZeroHertzLayerQualityConvergenceTest, + UnconvergedRepeatRateAdaptsUpWhenGoingFromRestrictedToUnrestricted) { + PassFrame(); + ScheduleDelayed(1.5 * kMinFrameDelay, [&] { + adapter_->UpdateVideoSourceRestrictions(kRestrictedMaxFps); + }); + ScheduleDelayed(5.5 * kMinFrameDelay, [&] { + adapter_->UpdateVideoSourceRestrictions(absl::nullopt); + }); + ExpectFrameEntriesAtDelaysFromNow({ + 1 * kMinFrameDelay, // Original frame emitted at non-restricted rate. + + // 1.5 * kMinFrameDelay: restricts max fps to 5 fps which should result + // in a new non-idle repeat delay of 2 * kMinFrameDelay. + 2 * kMinFrameDelay, // Unconverged repeat at non-restricted rate. + 4 * kMinFrameDelay, // Unconverged repeat at restricted rate. + + // 5.5 * kMinFrameDelay: removes frame-rate restriction and we should + // then go back to 10 fps as unconverged repeat rate. + 6 * kMinFrameDelay, // Last unconverged repeat at restricted rate. + 7 * kMinFrameDelay, // Back to unconverged repeat at non-restricted rate. + 8 * kMinFrameDelay, // We are now unrestricted. + 9 * kMinFrameDelay, // ... + }); +} + +TEST_F(ZeroHertzLayerQualityConvergenceTest, + UnconvergedRepeatRateMaintainsRestrictionOnReconfigureToHigherMaxFps) { + PassFrame(); + ScheduleDelayed(1.5 * kMinFrameDelay, [&] { + adapter_->UpdateVideoSourceRestrictions(kRestrictedMaxFps); + }); + ScheduleDelayed(2.5 * kMinFrameDelay, [&] { + adapter_->OnConstraintsChanged(VideoTrackSourceConstraints{ + /*min_fps=*/0, /*max_fps=*/2 * TimeDelta::Seconds(1) / kMinFrameDelay}); + }); + ScheduleDelayed(3 * kMinFrameDelay, [&] { PassFrame(); }); + ScheduleDelayed(8 * kMinFrameDelay, [&] { + adapter_->OnConstraintsChanged(VideoTrackSourceConstraints{ + /*min_fps=*/0, + /*max_fps=*/0.2 * TimeDelta::Seconds(1) / kMinFrameDelay}); + }); + ScheduleDelayed(9 * kMinFrameDelay, [&] { PassFrame(); }); + ExpectFrameEntriesAtDelaysFromNow({ + 1 * kMinFrameDelay, // Original frame emitted at non-restricted rate. + + // 1.5 * kMinFrameDelay: restricts max fps to 5 fps which should result + // in a new non-idle repeat delay of 2 * kMinFrameDelay. + 2 * kMinFrameDelay, // Unconverged repeat at non-restricted rate. + + // 2.5 * kMinFrameDelay: new constraint asks for max rate of 20 fps. + // The 0Hz adapter is reconstructed for 20 fps but inherits the current + // restriction for rate of non-converged frames of 5 fps. + + // A new frame is passed at 3 * kMinFrameDelay. The previous repeat + // cadence was stopped by the change in constraints. + 3.5 * kMinFrameDelay, // Original frame emitted at non-restricted 20 fps. + // The delay is 0.5 * kMinFrameDelay. + 5.5 * kMinFrameDelay, // Unconverged repeat at restricted rate. + // The delay is 2 * kMinFrameDelay when restricted. + 7.5 * kMinFrameDelay, // ... + + // 8 * kMinFrameDelay: new constraint asks for max rate of 2 fps. + // The 0Hz adapter is reconstructed for 2 fps and will therefore not obey + // the current restriction for rate of non-converged frames of 5 fps + // since the new max rate is lower. + + // A new frame is passed at 9 * kMinFrameDelay. The previous repeat + // cadence was stopped by the change in constraints. + 14 * kMinFrameDelay, // Original frame emitted at non-restricted 2 fps. + // The delay is 5 * kMinFrameDelay. + 19 * kMinFrameDelay, // Unconverged repeat at non-restricted rate. + 24 * kMinFrameDelay, // ... + }); +} + class FrameCadenceAdapterMetricsTest : public ::testing::Test { public: FrameCadenceAdapterMetricsTest() : time_controller_(Timestamp::Millis(1)) { @@ -869,7 +988,7 @@ TEST(FrameCadenceAdapterRealTimeTest, TimestampsDoNotDrift) { constexpr int kSleepMs = rtc::kNumMillisecsPerSec / 2; EXPECT_CALL(callback, OnFrame) .WillRepeatedly( - Invoke([&](Timestamp, int, const VideoFrame& incoming_frame) { + Invoke([&](Timestamp, bool, const VideoFrame& incoming_frame) { ++frame_counter; // Avoid the first OnFrame and sleep on the second. if (frame_counter == 2) { diff --git a/third_party/libwebrtc/video/frame_decode_scheduler_gn/moz.build b/third_party/libwebrtc/video/frame_decode_scheduler_gn/moz.build index a53fde81f2..d3fb5a4710 100644 --- a/third_party/libwebrtc/video/frame_decode_scheduler_gn/moz.build +++ b/third_party/libwebrtc/video/frame_decode_scheduler_gn/moz.build @@ -191,16 +191,9 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] -if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": - - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/video/frame_decode_timing_gn/moz.build b/third_party/libwebrtc/video/frame_decode_timing_gn/moz.build index ca09fac23b..9161f27d71 100644 --- a/third_party/libwebrtc/video/frame_decode_timing_gn/moz.build +++ b/third_party/libwebrtc/video/frame_decode_timing_gn/moz.build @@ -199,7 +199,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] @@ -209,10 +208,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": "-msse2" ] - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/video/frame_dumping_decoder_gn/moz.build b/third_party/libwebrtc/video/frame_dumping_decoder_gn/moz.build index 39e9878bd2..309c0b159e 100644 --- a/third_party/libwebrtc/video/frame_dumping_decoder_gn/moz.build +++ b/third_party/libwebrtc/video/frame_dumping_decoder_gn/moz.build @@ -200,7 +200,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] @@ -210,10 +209,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": "-msse2" ] - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/video/frame_dumping_encoder_gn/moz.build b/third_party/libwebrtc/video/frame_dumping_encoder_gn/moz.build index 615033eb1d..6719245825 100644 --- a/third_party/libwebrtc/video/frame_dumping_encoder_gn/moz.build +++ b/third_party/libwebrtc/video/frame_dumping_encoder_gn/moz.build @@ -200,7 +200,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] @@ -210,10 +209,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": "-msse2" ] - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/video/render/incoming_video_stream_gn/moz.build b/third_party/libwebrtc/video/render/incoming_video_stream_gn/moz.build index ad1d7fbfb3..36d087a36a 100644 --- a/third_party/libwebrtc/video/render/incoming_video_stream_gn/moz.build +++ b/third_party/libwebrtc/video/render/incoming_video_stream_gn/moz.build @@ -195,7 +195,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] @@ -205,10 +204,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": "-msse2" ] - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/video/render/video_render_frames_gn/moz.build b/third_party/libwebrtc/video/render/video_render_frames_gn/moz.build index d5de8cd178..b0b4923e34 100644 --- a/third_party/libwebrtc/video/render/video_render_frames_gn/moz.build +++ b/third_party/libwebrtc/video/render/video_render_frames_gn/moz.build @@ -195,7 +195,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] @@ -205,10 +204,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": "-msse2" ] - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/video/send_statistics_proxy.cc b/third_party/libwebrtc/video/send_statistics_proxy.cc index f8ba31beff..324a5d671d 100644 --- a/third_party/libwebrtc/video/send_statistics_proxy.cc +++ b/third_party/libwebrtc/video/send_statistics_proxy.cc @@ -658,6 +658,10 @@ void SendStatisticsProxy::UmaSamplesContainer::UpdateHistograms( << current_stats.frames_dropped_by_capturer << "\n"; RTC_HISTOGRAMS_COUNTS_1000(kIndex, uma_prefix_ + "DroppedFrames.Capturer", current_stats.frames_dropped_by_capturer); + log_stream << uma_prefix_ << "DroppedFrames.BadTimestamp" + << current_stats.frames_dropped_by_bad_timestamp << "\n"; + RTC_HISTOGRAMS_COUNTS_1000(kIndex, uma_prefix_ + "DroppedFrames.BadTimestamp", + current_stats.frames_dropped_by_bad_timestamp); log_stream << uma_prefix_ << "DroppedFrames.EncoderQueue " << current_stats.frames_dropped_by_encoder_queue << "\n"; RTC_HISTOGRAMS_COUNTS_1000(kIndex, uma_prefix_ + "DroppedFrames.EncoderQueue", @@ -1097,6 +1101,9 @@ void SendStatisticsProxy::OnFrameDropped(DropReason reason) { case DropReason::kSource: ++stats_.frames_dropped_by_capturer; break; + case DropReason::kBadTimestamp: + ++stats_.frames_dropped_by_bad_timestamp; + break; case DropReason::kEncoderQueue: ++stats_.frames_dropped_by_encoder_queue; break; diff --git a/third_party/libwebrtc/video/task_queue_frame_decode_scheduler_gn/moz.build b/third_party/libwebrtc/video/task_queue_frame_decode_scheduler_gn/moz.build index 2651f290bc..302f8420c9 100644 --- a/third_party/libwebrtc/video/task_queue_frame_decode_scheduler_gn/moz.build +++ b/third_party/libwebrtc/video/task_queue_frame_decode_scheduler_gn/moz.build @@ -199,7 +199,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] @@ -209,10 +208,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": "-msse2" ] - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/video/unique_timestamp_counter_gn/moz.build b/third_party/libwebrtc/video/unique_timestamp_counter_gn/moz.build index 24b3c5a30b..e38fabc4cc 100644 --- a/third_party/libwebrtc/video/unique_timestamp_counter_gn/moz.build +++ b/third_party/libwebrtc/video/unique_timestamp_counter_gn/moz.build @@ -184,7 +184,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] @@ -194,10 +193,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": "-msse2" ] - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/video/video_gn/moz.build b/third_party/libwebrtc/video/video_gn/moz.build index e8ac15dc40..5e3d75d621 100644 --- a/third_party/libwebrtc/video/video_gn/moz.build +++ b/third_party/libwebrtc/video/video_gn/moz.build @@ -220,7 +220,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] @@ -230,10 +229,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": "-msse2" ] - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/video/video_quality_test.cc b/third_party/libwebrtc/video/video_quality_test.cc index 971c129329..99b722322a 100644 --- a/third_party/libwebrtc/video/video_quality_test.cc +++ b/third_party/libwebrtc/video/video_quality_test.cc @@ -375,7 +375,6 @@ VideoQualityTest::VideoQualityTest( std::unique_ptr<InjectionComponents> injection_components) : clock_(Clock::GetRealTimeClock()), task_queue_factory_(CreateDefaultTaskQueueFactory()), - rtc_event_log_factory_(task_queue_factory_.get()), video_decoder_factory_([this](const SdpVideoFormat& format) { return this->CreateVideoDecoder(format); }), @@ -1221,10 +1220,10 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { } if (!params.logging.rtc_event_log_name.empty()) { - send_event_log_ = rtc_event_log_factory_.CreateRtcEventLog( - RtcEventLog::EncodingType::NewFormat); - recv_event_log_ = rtc_event_log_factory_.CreateRtcEventLog( - RtcEventLog::EncodingType::NewFormat); + std::unique_ptr<RtcEventLog> send_event_log = + rtc_event_log_factory_.Create(env()); + std::unique_ptr<RtcEventLog> recv_event_log = + rtc_event_log_factory_.Create(env()); std::unique_ptr<RtcEventLogOutputFile> send_output( std::make_unique<RtcEventLogOutputFile>( params.logging.rtc_event_log_name + "_send", @@ -1234,19 +1233,18 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { params.logging.rtc_event_log_name + "_recv", RtcEventLog::kUnlimitedOutput)); bool event_log_started = - send_event_log_->StartLogging(std::move(send_output), - RtcEventLog::kImmediateOutput) && - recv_event_log_->StartLogging(std::move(recv_output), - RtcEventLog::kImmediateOutput); + send_event_log->StartLogging(std::move(send_output), + RtcEventLog::kImmediateOutput) && + recv_event_log->StartLogging(std::move(recv_output), + RtcEventLog::kImmediateOutput); RTC_DCHECK(event_log_started); - } else { - send_event_log_ = std::make_unique<RtcEventLogNull>(); - recv_event_log_ = std::make_unique<RtcEventLogNull>(); + SetSendEventLog(std::move(send_event_log)); + SetRecvEventLog(std::move(recv_event_log)); } SendTask(task_queue(), [this, ¶ms, &send_transport, &recv_transport]() { - CallConfig send_call_config(send_event_log_.get()); - CallConfig recv_call_config(recv_event_log_.get()); + CallConfig send_call_config = SendCallConfig(); + CallConfig recv_call_config = RecvCallConfig(); send_call_config.bitrate_config = params.call.call_bitrate_config; recv_call_config.bitrate_config = params.call.call_bitrate_config; if (params_.audio.enabled) @@ -1444,10 +1442,10 @@ void VideoQualityTest::RunWithRenderers(const Params& params) { std::vector<std::unique_ptr<test::VideoRenderer>> loopback_renderers; if (!params.logging.rtc_event_log_name.empty()) { - send_event_log_ = rtc_event_log_factory_.CreateRtcEventLog( - RtcEventLog::EncodingType::NewFormat); - recv_event_log_ = rtc_event_log_factory_.CreateRtcEventLog( - RtcEventLog::EncodingType::NewFormat); + std::unique_ptr<RtcEventLog> send_event_log = + rtc_event_log_factory_.Create(env()); + std::unique_ptr<RtcEventLog> recv_event_log = + rtc_event_log_factory_.Create(env()); std::unique_ptr<RtcEventLogOutputFile> send_output( std::make_unique<RtcEventLogOutputFile>( params.logging.rtc_event_log_name + "_send", @@ -1457,14 +1455,13 @@ void VideoQualityTest::RunWithRenderers(const Params& params) { params.logging.rtc_event_log_name + "_recv", RtcEventLog::kUnlimitedOutput)); bool event_log_started = - send_event_log_->StartLogging(std::move(send_output), - /*output_period_ms=*/5000) && - recv_event_log_->StartLogging(std::move(recv_output), - /*output_period_ms=*/5000); + send_event_log->StartLogging(std::move(send_output), + /*output_period_ms=*/5000) && + recv_event_log->StartLogging(std::move(recv_output), + /*output_period_ms=*/5000); RTC_DCHECK(event_log_started); - } else { - send_event_log_ = std::make_unique<RtcEventLogNull>(); - recv_event_log_ = std::make_unique<RtcEventLogNull>(); + SetSendEventLog(std::move(send_event_log)); + SetRecvEventLog(std::move(recv_event_log)); } SendTask(task_queue(), [&]() { @@ -1473,9 +1470,9 @@ void VideoQualityTest::RunWithRenderers(const Params& params) { // TODO(ivica): Remove bitrate_config and use the default CallConfig(), to // match the full stack tests. - CallConfig send_call_config(send_event_log_.get()); + CallConfig send_call_config = SendCallConfig(); send_call_config.bitrate_config = params_.call.call_bitrate_config; - CallConfig recv_call_config(recv_event_log_.get()); + CallConfig recv_call_config = RecvCallConfig(); if (params_.audio.enabled) InitializeAudioDevice(&send_call_config, &recv_call_config, diff --git a/third_party/libwebrtc/video/video_receive_stream2.cc b/third_party/libwebrtc/video/video_receive_stream2.cc index 85cc0aa0dc..33e2f39ced 100644 --- a/third_party/libwebrtc/video/video_receive_stream2.cc +++ b/third_party/libwebrtc/video/video_receive_stream2.cc @@ -568,10 +568,7 @@ VideoReceiveStreamInterface::Stats VideoReceiveStream2::GetStats() const { rtp_receive_statistics_->GetStatistician(rtx_ssrc()); if (rtx_statistician) { stats.total_bitrate_bps += rtx_statistician->BitrateReceived(); - // TODO(bugs.webrtc.org/15096): remove kill-switch after rollout. - if (!call_->trials().IsDisabled("WebRTC-Stats-RtxReceiveStats")) { - stats.rtx_rtp_stats = rtx_statistician->GetStats(); - } + stats.rtx_rtp_stats = rtx_statistician->GetStats(); } } @@ -917,6 +914,22 @@ int VideoReceiveStream2::DecodeAndMaybeDispatchEncodedFrame( } int decode_result = video_receiver_.Decode(frame_ptr); + if (decode_result < WEBRTC_VIDEO_CODEC_OK) { + // Asynchronous decoders may delay error reporting, potentially resulting in + // error reports reflecting issues that occurred several frames back. + RTC_LOG(LS_WARNING) + << "Failed to decode frame. Return code: " << decode_result + << ", SSRC: " << remote_ssrc() + << ", frame RTP timestamp: " << frame_ptr->RtpTimestamp() + << ", type: " << VideoFrameTypeToString(frame_ptr->FrameType()) + << ", size: " << frame_ptr->size() + << ", width: " << frame_ptr->_encodedWidth + << ", height: " << frame_ptr->_encodedHeight + << ", spatial idx: " << frame_ptr->SpatialIndex().value_or(-1) + << ", temporal idx: " << frame_ptr->TemporalIndex().value_or(-1) + << ", id: " << frame_ptr->Id(); + } + if (encoded_frame_output_enabled) { absl::optional<RecordableEncodedFrame::EncodedResolution> pending_resolution; diff --git a/third_party/libwebrtc/video/video_receive_stream_timeout_tracker_gn/moz.build b/third_party/libwebrtc/video/video_receive_stream_timeout_tracker_gn/moz.build index 8fb9a9c737..f7e5be24c0 100644 --- a/third_party/libwebrtc/video/video_receive_stream_timeout_tracker_gn/moz.build +++ b/third_party/libwebrtc/video/video_receive_stream_timeout_tracker_gn/moz.build @@ -199,7 +199,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] @@ -209,10 +208,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": "-msse2" ] - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/video/video_stream_buffer_controller_gn/moz.build b/third_party/libwebrtc/video/video_stream_buffer_controller_gn/moz.build index e60151d967..7da76222b1 100644 --- a/third_party/libwebrtc/video/video_stream_buffer_controller_gn/moz.build +++ b/third_party/libwebrtc/video/video_stream_buffer_controller_gn/moz.build @@ -200,7 +200,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] @@ -210,10 +209,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": "-msse2" ] - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/video/video_stream_encoder.cc b/third_party/libwebrtc/video/video_stream_encoder.cc index c5f086e155..669f165635 100644 --- a/third_party/libwebrtc/video/video_stream_encoder.cc +++ b/third_party/libwebrtc/video/video_stream_encoder.cc @@ -94,6 +94,18 @@ int GetNumSpatialLayers(const VideoCodec& codec) { } } +absl::optional<EncodedImageCallback::DropReason> MaybeConvertDropReason( + VideoStreamEncoderObserver::DropReason reason) { + switch (reason) { + case VideoStreamEncoderObserver::DropReason::kMediaOptimization: + return EncodedImageCallback::DropReason::kDroppedByMediaOptimizations; + case VideoStreamEncoderObserver::DropReason::kEncoder: + return EncodedImageCallback::DropReason::kDroppedByEncoder; + default: + return absl::nullopt; + } +} + bool RequiresEncoderReset(const VideoCodec& prev_send_codec, const VideoCodec& new_send_codec, bool was_encode_called_since_last_initialization) { @@ -647,7 +659,6 @@ VideoStreamEncoder::VideoStreamEncoder( : field_trials_(field_trials), worker_queue_(TaskQueueBase::Current()), number_of_cores_(number_of_cores), - sink_(nullptr), settings_(settings), allocation_cb_type_(allocation_cb_type), rate_control_settings_( @@ -661,39 +672,12 @@ VideoStreamEncoder::VideoStreamEncoder( ? encoder_selector_from_constructor_ : encoder_selector_from_factory_.get()), encoder_stats_observer_(encoder_stats_observer), - cadence_callback_(*this), frame_cadence_adapter_(std::move(frame_cadence_adapter)), - encoder_initialized_(false), - max_framerate_(-1), - pending_encoder_reconfiguration_(false), - pending_encoder_creation_(false), - crop_width_(0), - crop_height_(0), - encoder_target_bitrate_bps_(absl::nullopt), - max_data_payload_length_(0), - encoder_paused_and_dropped_frame_(false), - was_encode_called_since_last_initialization_(false), - encoder_failed_(false), clock_(clock), - last_captured_timestamp_(0), delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - clock_->TimeInMilliseconds()), last_frame_log_ms_(clock_->TimeInMilliseconds()), - captured_frame_count_(0), - dropped_frame_cwnd_pushback_count_(0), - dropped_frame_encoder_block_count_(0), - pending_frame_post_time_us_(0), - accumulated_update_rect_{0, 0, 0, 0}, - accumulated_update_rect_is_valid_(true), - animation_start_time_(Timestamp::PlusInfinity()), - cap_resolution_due_to_video_content_(false), - expect_resize_state_(ExpectResizeState::kNoResize), - fec_controller_override_(nullptr), - force_disable_frame_dropper_(false), - pending_frame_drops_(0), - cwnd_frame_counter_(0), next_frame_types_(1, VideoFrameType::kVideoFrameDelta), - frame_encode_metadata_writer_(this), automatic_animation_detection_experiment_( ParseAutomatincAnimationDetectionFieldTrial()), input_state_provider_(encoder_stats_observer), @@ -704,7 +688,6 @@ VideoStreamEncoder::VideoStreamEncoder( degradation_preference_manager_( std::make_unique<DegradationPreferenceManager>( video_stream_adapter_.get())), - adaptation_constraints_(), stream_resource_manager_(&input_state_provider_, encoder_stats_observer, clock_, @@ -1499,7 +1482,7 @@ void VideoStreamEncoder::OnEncoderSettingsChanged() { } void VideoStreamEncoder::OnFrame(Timestamp post_time, - int frames_scheduled_for_processing, + bool queue_overload, const VideoFrame& video_frame) { RTC_DCHECK_RUN_ON(&encoder_queue_); VideoFrame incoming_frame = video_frame; @@ -1538,11 +1521,8 @@ void VideoStreamEncoder::OnFrame(Timestamp post_time, << incoming_frame.ntp_time_ms() << " <= " << last_captured_timestamp_ << ") for incoming frame. Dropping."; - encoder_queue_.PostTask([this, incoming_frame]() { - RTC_DCHECK_RUN_ON(&encoder_queue_); - accumulated_update_rect_.Union(incoming_frame.update_rect()); - accumulated_update_rect_is_valid_ &= incoming_frame.has_update_rect(); - }); + ProcessDroppedFrame(incoming_frame, + VideoStreamEncoderObserver::DropReason::kBadTimestamp); return; } @@ -1561,25 +1541,24 @@ void VideoStreamEncoder::OnFrame(Timestamp post_time, bool cwnd_frame_drop = cwnd_frame_drop_interval_ && (cwnd_frame_counter_++ % cwnd_frame_drop_interval_.value() == 0); - if (frames_scheduled_for_processing == 1 && !cwnd_frame_drop) { + if (!queue_overload && !cwnd_frame_drop) { MaybeEncodeVideoFrame(incoming_frame, post_time.us()); } else { if (cwnd_frame_drop) { // Frame drop by congestion window pushback. Do not encode this // frame. ++dropped_frame_cwnd_pushback_count_; - encoder_stats_observer_->OnFrameDropped( - VideoStreamEncoderObserver::DropReason::kCongestionWindow); } else { // There is a newer frame in flight. Do not encode this frame. RTC_LOG(LS_VERBOSE) << "Incoming frame dropped due to that the encoder is blocked."; ++dropped_frame_encoder_block_count_; - encoder_stats_observer_->OnFrameDropped( - VideoStreamEncoderObserver::DropReason::kEncoderQueue); } - accumulated_update_rect_.Union(incoming_frame.update_rect()); - accumulated_update_rect_is_valid_ &= incoming_frame.has_update_rect(); + ProcessDroppedFrame( + incoming_frame, + cwnd_frame_drop + ? VideoStreamEncoderObserver::DropReason::kCongestionWindow + : VideoStreamEncoderObserver::DropReason::kEncoderQueue); } if (log_stats) { RTC_LOG(LS_INFO) << "Number of frames: captured " << captured_frame_count_ @@ -1816,10 +1795,8 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, // Because pending frame will be dropped in any case, we need to // remember its updated region. if (pending_frame_) { - encoder_stats_observer_->OnFrameDropped( - VideoStreamEncoderObserver::DropReason::kEncoderQueue); - accumulated_update_rect_.Union(pending_frame_->update_rect()); - accumulated_update_rect_is_valid_ &= pending_frame_->has_update_rect(); + ProcessDroppedFrame(*pending_frame_, + VideoStreamEncoderObserver::DropReason::kEncoderQueue); } if (DropDueToSize(video_frame.size())) { @@ -1833,10 +1810,8 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, } else { // Ensure that any previously stored frame is dropped. pending_frame_.reset(); - accumulated_update_rect_.Union(video_frame.update_rect()); - accumulated_update_rect_is_valid_ &= video_frame.has_update_rect(); - encoder_stats_observer_->OnFrameDropped( - VideoStreamEncoderObserver::DropReason::kEncoderQueue); + ProcessDroppedFrame( + video_frame, VideoStreamEncoderObserver::DropReason::kEncoderQueue); } return; } @@ -1854,10 +1829,8 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, // Ensure that any previously stored frame is dropped. pending_frame_.reset(); TraceFrameDropStart(); - accumulated_update_rect_.Union(video_frame.update_rect()); - accumulated_update_rect_is_valid_ &= video_frame.has_update_rect(); - encoder_stats_observer_->OnFrameDropped( - VideoStreamEncoderObserver::DropReason::kEncoderQueue); + ProcessDroppedFrame( + video_frame, VideoStreamEncoderObserver::DropReason::kEncoderQueue); } return; } @@ -1879,10 +1852,9 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, ? last_encoder_rate_settings_->encoder_target.bps() : 0) << ", input frame rate " << framerate_fps; - OnDroppedFrame( - EncodedImageCallback::DropReason::kDroppedByMediaOptimizations); - accumulated_update_rect_.Union(video_frame.update_rect()); - accumulated_update_rect_is_valid_ &= video_frame.has_update_rect(); + ProcessDroppedFrame( + video_frame, + VideoStreamEncoderObserver::DropReason::kMediaOptimization); return; } @@ -2228,16 +2200,6 @@ EncodedImageCallback::Result VideoStreamEncoder::OnEncodedImage( } void VideoStreamEncoder::OnDroppedFrame(DropReason reason) { - switch (reason) { - case DropReason::kDroppedByMediaOptimizations: - encoder_stats_observer_->OnFrameDropped( - VideoStreamEncoderObserver::DropReason::kMediaOptimization); - break; - case DropReason::kDroppedByEncoder: - encoder_stats_observer_->OnFrameDropped( - VideoStreamEncoderObserver::DropReason::kEncoder); - break; - } sink_->OnDroppedFrame(reason); encoder_queue_.PostTask([this, reason] { RTC_DCHECK_RUN_ON(&encoder_queue_); @@ -2396,6 +2358,11 @@ void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated( << (reason ? reason->Name() : std::string("<null>")) << " to " << restrictions.ToString(); + if (frame_cadence_adapter_) { + frame_cadence_adapter_->UpdateVideoSourceRestrictions( + restrictions.max_frame_rate()); + } + // TODO(webrtc:14451) Split video_source_sink_controller_ // so that ownership on restrictions/wants is kept on &encoder_queue_ latest_restrictions_ = restrictions; @@ -2620,4 +2587,16 @@ void VideoStreamEncoder::RemoveRestrictionsListenerForTesting( event.Wait(rtc::Event::kForever); } +// RTC_RUN_ON(&encoder_queue_) +void VideoStreamEncoder::ProcessDroppedFrame( + const VideoFrame& frame, + VideoStreamEncoderObserver::DropReason reason) { + accumulated_update_rect_.Union(frame.update_rect()); + accumulated_update_rect_is_valid_ &= frame.has_update_rect(); + if (auto converted_reason = MaybeConvertDropReason(reason)) { + OnDroppedFrame(*converted_reason); + } + encoder_stats_observer_->OnFrameDropped(reason); +} + } // namespace webrtc diff --git a/third_party/libwebrtc/video/video_stream_encoder.h b/third_party/libwebrtc/video/video_stream_encoder.h index 68f008afc0..f2c21c12b0 100644 --- a/third_party/libwebrtc/video/video_stream_encoder.h +++ b/third_party/libwebrtc/video/video_stream_encoder.h @@ -132,6 +132,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, double cwnd_reduce_ratio); protected: + friend class VideoStreamEncoderFrameCadenceRestrictionTest; + // Used for testing. For example the `ScalingObserverInterface` methods must // be called on `encoder_queue_`. TaskQueueBase* encoder_queue() { return encoder_queue_.Get(); } @@ -160,10 +162,9 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, : video_stream_encoder_(video_stream_encoder) {} // FrameCadenceAdapterInterface::Callback overrides. void OnFrame(Timestamp post_time, - int frames_scheduled_for_processing, + bool queue_overload, const VideoFrame& frame) override { - video_stream_encoder_.OnFrame(post_time, frames_scheduled_for_processing, - frame); + video_stream_encoder_.OnFrame(post_time, queue_overload, frame); } void OnDiscardedFrame() override { video_stream_encoder_.OnDiscardedFrame(); @@ -212,7 +213,7 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, void ReconfigureEncoder() RTC_RUN_ON(&encoder_queue_); void OnEncoderSettingsChanged() RTC_RUN_ON(&encoder_queue_); void OnFrame(Timestamp post_time, - int frames_scheduled_for_processing, + bool queue_overload, const VideoFrame& video_frame); void OnDiscardedFrame(); void RequestRefreshFrame(); @@ -266,12 +267,16 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, const EncodedImage& encoded_image, const CodecSpecificInfo* codec_specific_info); + void ProcessDroppedFrame(const VideoFrame& frame, + VideoStreamEncoderObserver::DropReason reason) + RTC_RUN_ON(&encoder_queue_); + const FieldTrialsView& field_trials_; TaskQueueBase* const worker_queue_; const int number_of_cores_; - EncoderSink* sink_; + EncoderSink* sink_ = nullptr; const VideoStreamEncoderSettings settings_; const BitrateAllocationCallbackType allocation_cb_type_; const RateControlSettings rate_control_settings_; @@ -287,7 +292,7 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, VideoStreamEncoderObserver* const encoder_stats_observer_; // Adapter that avoids public inheritance of the cadence adapter's callback // interface. - CadenceCallback cadence_callback_; + CadenceCallback cadence_callback_{*this}; // Frame cadence encoder adapter. Frames enter this adapter first, and it then // forwards them to our OnFrame method. std::unique_ptr<FrameCadenceAdapterInterface> frame_cadence_adapter_ @@ -296,70 +301,74 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, VideoEncoderConfig encoder_config_ RTC_GUARDED_BY(&encoder_queue_); std::unique_ptr<VideoEncoder> encoder_ RTC_GUARDED_BY(&encoder_queue_) RTC_PT_GUARDED_BY(&encoder_queue_); - bool encoder_initialized_; + bool encoder_initialized_ = false; std::unique_ptr<VideoBitrateAllocator> rate_allocator_ RTC_GUARDED_BY(&encoder_queue_) RTC_PT_GUARDED_BY(&encoder_queue_); - int max_framerate_ RTC_GUARDED_BY(&encoder_queue_); + int max_framerate_ RTC_GUARDED_BY(&encoder_queue_) = -1; // Set when ConfigureEncoder has been called in order to lazy reconfigure the // encoder on the next frame. - bool pending_encoder_reconfiguration_ RTC_GUARDED_BY(&encoder_queue_); + bool pending_encoder_reconfiguration_ RTC_GUARDED_BY(&encoder_queue_) = false; // Set when configuration must create a new encoder object, e.g., // because of a codec change. - bool pending_encoder_creation_ RTC_GUARDED_BY(&encoder_queue_); + bool pending_encoder_creation_ RTC_GUARDED_BY(&encoder_queue_) = false; absl::InlinedVector<SetParametersCallback, 2> encoder_configuration_callbacks_ RTC_GUARDED_BY(&encoder_queue_); absl::optional<VideoFrameInfo> last_frame_info_ RTC_GUARDED_BY(&encoder_queue_); - int crop_width_ RTC_GUARDED_BY(&encoder_queue_); - int crop_height_ RTC_GUARDED_BY(&encoder_queue_); + int crop_width_ RTC_GUARDED_BY(&encoder_queue_) = 0; + int crop_height_ RTC_GUARDED_BY(&encoder_queue_) = 0; absl::optional<uint32_t> encoder_target_bitrate_bps_ RTC_GUARDED_BY(&encoder_queue_); - size_t max_data_payload_length_ RTC_GUARDED_BY(&encoder_queue_); + size_t max_data_payload_length_ RTC_GUARDED_BY(&encoder_queue_) = 0; absl::optional<EncoderRateSettings> last_encoder_rate_settings_ RTC_GUARDED_BY(&encoder_queue_); - bool encoder_paused_and_dropped_frame_ RTC_GUARDED_BY(&encoder_queue_); + bool encoder_paused_and_dropped_frame_ RTC_GUARDED_BY(&encoder_queue_) = + false; // Set to true if at least one frame was sent to encoder since last encoder // initialization. bool was_encode_called_since_last_initialization_ - RTC_GUARDED_BY(&encoder_queue_); + RTC_GUARDED_BY(&encoder_queue_) = false; - bool encoder_failed_ RTC_GUARDED_BY(&encoder_queue_); + bool encoder_failed_ RTC_GUARDED_BY(&encoder_queue_) = false; Clock* const clock_; // Used to make sure incoming time stamp is increasing for every frame. - int64_t last_captured_timestamp_ RTC_GUARDED_BY(&encoder_queue_); + int64_t last_captured_timestamp_ RTC_GUARDED_BY(&encoder_queue_) = 0; // Delta used for translating between NTP and internal timestamps. const int64_t delta_ntp_internal_ms_ RTC_GUARDED_BY(&encoder_queue_); int64_t last_frame_log_ms_ RTC_GUARDED_BY(&encoder_queue_); - int captured_frame_count_ RTC_GUARDED_BY(&encoder_queue_); - int dropped_frame_cwnd_pushback_count_ RTC_GUARDED_BY(&encoder_queue_); - int dropped_frame_encoder_block_count_ RTC_GUARDED_BY(&encoder_queue_); + int captured_frame_count_ RTC_GUARDED_BY(&encoder_queue_) = 0; + int dropped_frame_cwnd_pushback_count_ RTC_GUARDED_BY(&encoder_queue_) = 0; + int dropped_frame_encoder_block_count_ RTC_GUARDED_BY(&encoder_queue_) = 0; absl::optional<VideoFrame> pending_frame_ RTC_GUARDED_BY(&encoder_queue_); - int64_t pending_frame_post_time_us_ RTC_GUARDED_BY(&encoder_queue_); + int64_t pending_frame_post_time_us_ RTC_GUARDED_BY(&encoder_queue_) = 0; VideoFrame::UpdateRect accumulated_update_rect_ RTC_GUARDED_BY(&encoder_queue_); - bool accumulated_update_rect_is_valid_ RTC_GUARDED_BY(&encoder_queue_); + bool accumulated_update_rect_is_valid_ RTC_GUARDED_BY(&encoder_queue_) = true; // Used for automatic content type detection. absl::optional<VideoFrame::UpdateRect> last_update_rect_ RTC_GUARDED_BY(&encoder_queue_); - Timestamp animation_start_time_ RTC_GUARDED_BY(&encoder_queue_); - bool cap_resolution_due_to_video_content_ RTC_GUARDED_BY(&encoder_queue_); + Timestamp animation_start_time_ RTC_GUARDED_BY(&encoder_queue_) = + Timestamp::PlusInfinity(); + bool cap_resolution_due_to_video_content_ RTC_GUARDED_BY(&encoder_queue_) = + false; // Used to correctly ignore changes in update_rect introduced by // resize triggered by animation detection. enum class ExpectResizeState { kNoResize, // Normal operation. kResize, // Resize was triggered by the animation detection. kFirstFrameAfterResize // Resize observed. - } expect_resize_state_ RTC_GUARDED_BY(&encoder_queue_); + } expect_resize_state_ RTC_GUARDED_BY(&encoder_queue_) = + ExpectResizeState::kNoResize; FecControllerOverride* fec_controller_override_ - RTC_GUARDED_BY(&encoder_queue_); + RTC_GUARDED_BY(&encoder_queue_) = nullptr; absl::optional<int64_t> last_parameters_update_ms_ RTC_GUARDED_BY(&encoder_queue_); absl::optional<int64_t> last_encode_info_ms_ RTC_GUARDED_BY(&encoder_queue_); @@ -372,18 +381,18 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, // disabled if VideoEncoder::GetEncoderInfo() indicates that the encoder has a // trusted rate controller. This is determined on a per-frame basis, as the // encoder behavior might dynamically change. - bool force_disable_frame_dropper_ RTC_GUARDED_BY(&encoder_queue_); + bool force_disable_frame_dropper_ RTC_GUARDED_BY(&encoder_queue_) = false; // Incremented on worker thread whenever `frame_dropper_` determines that a // frame should be dropped. Decremented on whichever thread runs // OnEncodedImage(), which is only called by one thread but not necessarily // the worker thread. - std::atomic<int> pending_frame_drops_; + std::atomic<int> pending_frame_drops_{0}; // Congestion window frame drop ratio (drop 1 in every // cwnd_frame_drop_interval_ frames). absl::optional<int> cwnd_frame_drop_interval_ RTC_GUARDED_BY(&encoder_queue_); // Frame counter for congestion window frame drop. - int cwnd_frame_counter_ RTC_GUARDED_BY(&encoder_queue_); + int cwnd_frame_counter_ RTC_GUARDED_BY(&encoder_queue_) = 0; std::unique_ptr<EncoderBitrateAdjuster> bitrate_adjuster_ RTC_GUARDED_BY(&encoder_queue_); @@ -392,7 +401,7 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, // turn this into a simple bool `pending_keyframe_request_`. std::vector<VideoFrameType> next_frame_types_ RTC_GUARDED_BY(&encoder_queue_); - FrameEncodeMetadataWriter frame_encode_metadata_writer_; + FrameEncodeMetadataWriter frame_encode_metadata_writer_{this}; struct AutomaticAnimationDetectionExperiment { bool enabled = false; diff --git a/third_party/libwebrtc/video/video_stream_encoder_impl_gn/moz.build b/third_party/libwebrtc/video/video_stream_encoder_impl_gn/moz.build index 2ea2fce60b..af4e29ee0a 100644 --- a/third_party/libwebrtc/video/video_stream_encoder_impl_gn/moz.build +++ b/third_party/libwebrtc/video/video_stream_encoder_impl_gn/moz.build @@ -205,7 +205,6 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] @@ -215,10 +214,6 @@ if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": "-msse2" ] - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/video/video_stream_encoder_interface_gn/moz.build b/third_party/libwebrtc/video/video_stream_encoder_interface_gn/moz.build index 0d40ef9c8f..beeb7f9117 100644 --- a/third_party/libwebrtc/video/video_stream_encoder_interface_gn/moz.build +++ b/third_party/libwebrtc/video/video_stream_encoder_interface_gn/moz.build @@ -187,16 +187,9 @@ if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "arm": OS_LIBS += [ - "android_support", "unwind" ] -if CONFIG["OS_TARGET"] == "Android" and CONFIG["TARGET_CPU"] == "x86": - - OS_LIBS += [ - "android_support" - ] - if CONFIG["OS_TARGET"] == "Linux" and CONFIG["TARGET_CPU"] == "aarch64": DEFINES["_GNU_SOURCE"] = True diff --git a/third_party/libwebrtc/video/video_stream_encoder_observer.h b/third_party/libwebrtc/video/video_stream_encoder_observer.h index c10412181d..95ca5fa887 100644 --- a/third_party/libwebrtc/video/video_stream_encoder_observer.h +++ b/third_party/libwebrtc/video/video_stream_encoder_observer.h @@ -58,6 +58,7 @@ class VideoStreamEncoderObserver : public CpuOveruseMetricsObserver { enum class DropReason { kSource, + kBadTimestamp, kEncoderQueue, kEncoder, kMediaOptimization, diff --git a/third_party/libwebrtc/video/video_stream_encoder_unittest.cc b/third_party/libwebrtc/video/video_stream_encoder_unittest.cc index fa28368d68..6fa99081cd 100644 --- a/third_party/libwebrtc/video/video_stream_encoder_unittest.cc +++ b/third_party/libwebrtc/video/video_stream_encoder_unittest.cc @@ -136,8 +136,8 @@ void PassAFrame( FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback, int64_t ntp_time_ms) { encoder_queue->PostTask([video_stream_encoder_callback, ntp_time_ms] { - video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms), 1, - CreateSimpleNV12Frame()); + video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms), + false, CreateSimpleNV12Frame()); }); } @@ -783,6 +783,10 @@ class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface { UpdateLayerStatus, (size_t spatial_index, bool enabled), (override)); + MOCK_METHOD(void, + UpdateVideoSourceRestrictions, + (absl::optional<double>), + (override)); MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override)); }; @@ -9580,4 +9584,60 @@ TEST(VideoStreamEncoderFrameCadenceTest, kMaxFps); } +class VideoStreamEncoderFrameCadenceRestrictionTest : public ::testing::Test { + public: + VideoStreamEncoderFrameCadenceRestrictionTest() + : adapter_ptr_(adapter_.get()), + fake_resource_(FakeResource::Create("FakeResource")), + video_stream_encoder_( + factory_.Create(std::move(adapter_), &encoder_queue_)) {} + + ~VideoStreamEncoderFrameCadenceRestrictionTest() { + factory_.DepleteTaskQueues(); + } + + void UpdateVideoSourceRestrictions(VideoSourceRestrictions restrictions) { + encoder_queue_->PostTask([this, restrictions] { + RTC_DCHECK_RUN_ON(encoder_queue_); + video_stream_encoder_->OnVideoSourceRestrictionsUpdated( + restrictions, VideoAdaptationCounters(), fake_resource_, + VideoSourceRestrictions()); + }); + } + + protected: + SimpleVideoStreamEncoderFactory factory_; + std::unique_ptr<NiceMock<MockFrameCadenceAdapter>> adapter_{ + std::make_unique<NiceMock<MockFrameCadenceAdapter>>()}; + NiceMock<MockFrameCadenceAdapter>* adapter_ptr_; + TaskQueueBase* encoder_queue_{nullptr}; + rtc::scoped_refptr<FakeResource> fake_resource_; + VideoSourceRestrictions restrictions_; + std::unique_ptr<SimpleVideoStreamEncoderFactory::AdaptedVideoStreamEncoder> + video_stream_encoder_; +}; + +TEST_F(VideoStreamEncoderFrameCadenceRestrictionTest, + UpdatesVideoSourceRestrictionsUnRestricted) { + EXPECT_CALL(*adapter_ptr_, UpdateVideoSourceRestrictions(Eq(absl::nullopt))); + UpdateVideoSourceRestrictions(VideoSourceRestrictions()); +} + +TEST_F(VideoStreamEncoderFrameCadenceRestrictionTest, + UpdatesVideoSourceRestrictionsWithMaxFrameRateRestriction) { + restrictions_.set_max_frame_rate(20); + EXPECT_CALL(*adapter_ptr_, UpdateVideoSourceRestrictions(Optional(20))); + UpdateVideoSourceRestrictions(restrictions_); +} + +TEST_F(VideoStreamEncoderFrameCadenceRestrictionTest, + UpdatesVideoSourceRestrictionsWithoutMaxFrameRateRestriction) { + // Restrictions in resolution count as restriction updated, even though the + // FPS is unlimited. + restrictions_.set_max_pixels_per_frame(99); + restrictions_.set_target_pixels_per_frame(101); + EXPECT_CALL(*adapter_ptr_, UpdateVideoSourceRestrictions(Eq(absl::nullopt))); + UpdateVideoSourceRestrictions(restrictions_); +} + } // namespace webrtc |