diff options
Diffstat (limited to 'third_party/libwebrtc/modules/video_coding/timing')
24 files changed, 3833 insertions, 0 deletions
diff --git a/third_party/libwebrtc/modules/video_coding/timing/BUILD.gn b/third_party/libwebrtc/modules/video_coding/timing/BUILD.gn new file mode 100644 index 0000000000..b130f92154 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/BUILD.gn @@ -0,0 +1,132 @@ +# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../webrtc.gni") + +rtc_library("codec_timer") { + sources = [ + "codec_timer.cc", + "codec_timer.h", + ] + deps = [ "../../../rtc_base:rtc_numerics" ] +} + +rtc_library("inter_frame_delay") { + sources = [ + "inter_frame_delay.cc", + "inter_frame_delay.h", + ] + deps = [ + "../..:module_api_public", + "../../../api/units:frequency", + "../../../api/units:time_delta", + "../../../api/units:timestamp", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("frame_delay_delta_kalman_filter") { + sources = [ + "frame_delay_delta_kalman_filter.cc", + "frame_delay_delta_kalman_filter.h", + ] + deps = [ + "../../../api/units:data_size", + "../../../api/units:time_delta", + ] + visibility = [ + ":jitter_estimator", + ":timing_unittests", + ] +} + +rtc_library("jitter_estimator") { + sources = [ + "jitter_estimator.cc", + "jitter_estimator.h", + ] + deps = [ + ":frame_delay_delta_kalman_filter", + ":rtt_filter", + "../../../api:field_trials_view", + "../../../api/units:data_size", + "../../../api/units:frequency", + "../../../api/units:time_delta", + "../../../api/units:timestamp", + "../../../rtc_base", + "../../../rtc_base:safe_conversions", + "../../../system_wrappers", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +rtc_library("rtt_filter") { + sources = [ + "rtt_filter.cc", + "rtt_filter.h", + ] + deps = [ "../../../api/units:time_delta" ] + absl_deps = [ + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/container:inlined_vector", + ] +} + +rtc_library("timing_module") { + sources = [ + "timing.cc", + "timing.h", + ] + deps = [ + ":codec_timer", + "../../../api:field_trials_view", + "../../../api/units:time_delta", + "../../../api/video:video_frame", + "../../../api/video:video_rtp_headers", + "../../../rtc_base:logging", + "../../../rtc_base:macromagic", + "../../../rtc_base:rtc_numerics", + "../../../rtc_base/experiments:field_trial_parser", + "../../../rtc_base/synchronization:mutex", + "../../../rtc_base/time:timestamp_extrapolator", + "../../../system_wrappers", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + +if (!build_with_mozilla) { +rtc_library("timing_unittests") { + testonly = true + sources = [ + "frame_delay_delta_kalman_filter_unittest.cc", + "inter_frame_delay_unittest.cc", + "jitter_estimator_unittest.cc", + "rtt_filter_unittest.cc", + "timing_unittest.cc", + ] + deps = [ + ":frame_delay_delta_kalman_filter", + ":inter_frame_delay", + ":jitter_estimator", + ":rtt_filter", + ":timing_module", + "../../../api:array_view", + "../../../api/units:data_size", + "../../../api/units:frequency", + "../../../api/units:time_delta", + "../../../api/units:timestamp", + "../../../rtc_base:histogram_percentile_counter", + "../../../rtc_base:stringutils", + "../../../rtc_base:timeutils", + "../../../system_wrappers:system_wrappers", + "../../../test:scoped_key_value_config", + "../../../test:test_support", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} +} diff --git a/third_party/libwebrtc/modules/video_coding/timing/codec_timer.cc b/third_party/libwebrtc/modules/video_coding/timing/codec_timer.cc new file mode 100644 index 0000000000..f57d42d40a --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/codec_timer.cc @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/codec_timer.h" + +#include <cstdint> + +namespace webrtc { + +namespace { + +// The first kIgnoredSampleCount samples will be ignored. +const int kIgnoredSampleCount = 5; +// Return the `kPercentile` value in RequiredDecodeTimeMs(). +const float kPercentile = 0.95f; +// The window size in ms. +const int64_t kTimeLimitMs = 10000; + +} // anonymous namespace + +CodecTimer::CodecTimer() : ignored_sample_count_(0), filter_(kPercentile) {} +CodecTimer::~CodecTimer() = default; + +void CodecTimer::AddTiming(int64_t decode_time_ms, int64_t now_ms) { + // Ignore the first `kIgnoredSampleCount` samples. + if (ignored_sample_count_ < kIgnoredSampleCount) { + ++ignored_sample_count_; + return; + } + + // Insert new decode time value. + filter_.Insert(decode_time_ms); + history_.emplace(decode_time_ms, now_ms); + + // Pop old decode time values. + while (!history_.empty() && + now_ms - history_.front().sample_time_ms > kTimeLimitMs) { + filter_.Erase(history_.front().decode_time_ms); + history_.pop(); + } +} + +// Get the 95th percentile observed decode time within a time window. +int64_t CodecTimer::RequiredDecodeTimeMs() const { + return filter_.GetPercentileValue(); +} + +CodecTimer::Sample::Sample(int64_t decode_time_ms, int64_t sample_time_ms) + : decode_time_ms(decode_time_ms), sample_time_ms(sample_time_ms) {} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/codec_timer.h b/third_party/libwebrtc/modules/video_coding/timing/codec_timer.h new file mode 100644 index 0000000000..9f12d82e98 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/codec_timer.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_TIMING_CODEC_TIMER_H_ +#define MODULES_VIDEO_CODING_TIMING_CODEC_TIMER_H_ + +#include <queue> + +#include "rtc_base/numerics/percentile_filter.h" + +namespace webrtc { + +class CodecTimer { + public: + CodecTimer(); + ~CodecTimer(); + + // Add a new decode time to the filter. + void AddTiming(int64_t new_decode_time_ms, int64_t now_ms); + + // Get the required decode time in ms. It is the 95th percentile observed + // decode time within a time window. + int64_t RequiredDecodeTimeMs() const; + + private: + struct Sample { + Sample(int64_t decode_time_ms, int64_t sample_time_ms); + int64_t decode_time_ms; + int64_t sample_time_ms; + }; + + // The number of samples ignored so far. + int ignored_sample_count_; + // Queue with history of latest decode time values. + std::queue<Sample> history_; + // `filter_` contains the same values as `history_`, but in a data structure + // that allows efficient retrieval of the percentile value. + PercentileFilter<int64_t> filter_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_TIMING_CODEC_TIMER_H_ diff --git a/third_party/libwebrtc/modules/video_coding/timing/codec_timer_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/codec_timer_gn/moz.build new file mode 100644 index 0000000000..fe230f262d --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/codec_timer_gn/moz.build @@ -0,0 +1,201 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/video_coding/timing/codec_timer.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +Library("codec_timer_gn") diff --git a/third_party/libwebrtc/modules/video_coding/timing/frame_delay_delta_kalman_filter.cc b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_delta_kalman_filter.cc new file mode 100644 index 0000000000..69af4e25e4 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_delta_kalman_filter.cc @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/frame_delay_delta_kalman_filter.h" + +#include "api/units/data_size.h" +#include "api/units/time_delta.h" + +namespace webrtc { + +namespace { +// TODO(brandtr): The value below corresponds to 8 Gbps. Is that reasonable? +constexpr double kMaxBandwidth = 0.000001; // Unit: [1 / bytes per ms]. +} + +FrameDelayDeltaKalmanFilter::FrameDelayDeltaKalmanFilter() { + // TODO(brandtr): Is there a factor 1000 missing here? + estimate_[0] = 1 / (512e3 / 8); // Unit: [1 / bytes per ms] + estimate_[1] = 0; // Unit: [ms] + + // Initial estimate covariance. + estimate_cov_[0][0] = 1e-4; // Unit: [(1 / bytes per ms)^2] + estimate_cov_[1][1] = 1e2; // Unit: [ms^2] + estimate_cov_[0][1] = estimate_cov_[1][0] = 0; + + // Process noise covariance. + process_noise_cov_diag_[0] = 2.5e-10; // Unit: [(1 / bytes per ms)^2] + process_noise_cov_diag_[1] = 1e-10; // Unit: [ms^2] +} + +void FrameDelayDeltaKalmanFilter::PredictAndUpdate( + TimeDelta frame_delay_variation, + double frame_size_variation_bytes, + DataSize max_frame_size, + double var_noise) { + // Sanity checks. + if (max_frame_size < DataSize::Bytes(1)) { + return; + } + if (var_noise <= 0.0) { + return; + } + + // This member function follows the data flow in + // https://en.wikipedia.org/wiki/Kalman_filter#Details. + + // 1) Estimate prediction: `x = F*x`. + // For this model, there is no need to explicitly predict the estimate, since + // the state transition matrix is the identity. + + // 2) Estimate covariance prediction: `P = F*P*F' + Q`. + // Again, since the state transition matrix is the identity, this update + // is performed by simply adding the process noise covariance. + estimate_cov_[0][0] += process_noise_cov_diag_[0]; + estimate_cov_[1][1] += process_noise_cov_diag_[1]; + + // 3) Innovation: `y = z - H*x`. + // This is the part of the measurement that cannot be explained by the current + // estimate. + double innovation = + frame_delay_variation.ms() - + GetFrameDelayVariationEstimateTotal(frame_size_variation_bytes); + + // 4) Innovation variance: `s = H*P*H' + r`. + double estim_cov_times_obs[2]; + estim_cov_times_obs[0] = + estimate_cov_[0][0] * frame_size_variation_bytes + estimate_cov_[0][1]; + estim_cov_times_obs[1] = + estimate_cov_[1][0] * frame_size_variation_bytes + estimate_cov_[1][1]; + double observation_noise_stddev = + (300.0 * exp(-fabs(frame_size_variation_bytes) / + (1e0 * max_frame_size.bytes())) + + 1) * + sqrt(var_noise); + if (observation_noise_stddev < 1.0) { + observation_noise_stddev = 1.0; + } + // TODO(brandtr): Shouldn't we add observation_noise_stddev^2 here? Otherwise, + // the dimensional analysis fails. + double innovation_var = frame_size_variation_bytes * estim_cov_times_obs[0] + + estim_cov_times_obs[1] + observation_noise_stddev; + if ((innovation_var < 1e-9 && innovation_var >= 0) || + (innovation_var > -1e-9 && innovation_var <= 0)) { + RTC_DCHECK_NOTREACHED(); + return; + } + + // 5) Optimal Kalman gain: `K = P*H'/s`. + // How much to trust the model vs. how much to trust the measurement. + double kalman_gain[2]; + kalman_gain[0] = estim_cov_times_obs[0] / innovation_var; + kalman_gain[1] = estim_cov_times_obs[1] / innovation_var; + + // 6) Estimate update: `x = x + K*y`. + // Optimally weight the new information in the innovation and add it to the + // old estimate. + estimate_[0] += kalman_gain[0] * innovation; + estimate_[1] += kalman_gain[1] * innovation; + + // (This clamping is not part of the linear Kalman filter.) + if (estimate_[0] < kMaxBandwidth) { + estimate_[0] = kMaxBandwidth; + } + + // 7) Estimate covariance update: `P = (I - K*H)*P` + double t00 = estimate_cov_[0][0]; + double t01 = estimate_cov_[0][1]; + estimate_cov_[0][0] = + (1 - kalman_gain[0] * frame_size_variation_bytes) * t00 - + kalman_gain[0] * estimate_cov_[1][0]; + estimate_cov_[0][1] = + (1 - kalman_gain[0] * frame_size_variation_bytes) * t01 - + kalman_gain[0] * estimate_cov_[1][1]; + estimate_cov_[1][0] = estimate_cov_[1][0] * (1 - kalman_gain[1]) - + kalman_gain[1] * frame_size_variation_bytes * t00; + estimate_cov_[1][1] = estimate_cov_[1][1] * (1 - kalman_gain[1]) - + kalman_gain[1] * frame_size_variation_bytes * t01; + + // Covariance matrix, must be positive semi-definite. + RTC_DCHECK(estimate_cov_[0][0] + estimate_cov_[1][1] >= 0 && + estimate_cov_[0][0] * estimate_cov_[1][1] - + estimate_cov_[0][1] * estimate_cov_[1][0] >= + 0 && + estimate_cov_[0][0] >= 0); +} + +double FrameDelayDeltaKalmanFilter::GetFrameDelayVariationEstimateSizeBased( + double frame_size_variation_bytes) const { + // Unit: [1 / bytes per millisecond] * [bytes] = [milliseconds]. + return estimate_[0] * frame_size_variation_bytes; +} + +double FrameDelayDeltaKalmanFilter::GetFrameDelayVariationEstimateTotal( + double frame_size_variation_bytes) const { + double frame_transmission_delay_ms = + GetFrameDelayVariationEstimateSizeBased(frame_size_variation_bytes); + double link_queuing_delay_ms = estimate_[1]; + return frame_transmission_delay_ms + link_queuing_delay_ms; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/frame_delay_delta_kalman_filter.h b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_delta_kalman_filter.h new file mode 100644 index 0000000000..1612ef3aa2 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_delta_kalman_filter.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_TIMING_FRAME_DELAY_DELTA_KALMAN_FILTER_H_ +#define MODULES_VIDEO_CODING_TIMING_FRAME_DELAY_DELTA_KALMAN_FILTER_H_ + +#include "api/units/data_size.h" +#include "api/units/time_delta.h" + +namespace webrtc { + +// This class uses a linear Kalman filter (see +// https://en.wikipedia.org/wiki/Kalman_filter) to estimate the frame delay +// variation (i.e., the difference in transmission time between a frame and the +// prior frame) for a frame, given its size variation in bytes (i.e., the +// difference in size between a frame and the prior frame). The idea is that, +// given a fixed link bandwidth, a larger frame (in bytes) would take +// proportionally longer to arrive than a correspondingly smaller frame. Using +// the variations of frame delay and frame size, the underlying bandwidth and +// queuing delay variation of the network link can be estimated. +// +// The filter takes as input the frame delay variation, the difference between +// the actual inter-frame arrival time and the expected inter-frame arrival time +// (based on RTP timestamp), and frame size variation, the inter-frame size +// delta for a single frame. The frame delay variation is seen as the +// measurement and the frame size variation is used in the observation model. +// The hidden state of the filter is the link bandwidth and queuing delay +// buildup. The estimated state can be used to get the expected frame delay +// variation for a frame, given its frame size variation. This information can +// then be used to estimate the frame delay variation coming from network +// jitter. +// +// Mathematical details: +// * The state (`x` in Wikipedia notation) is a 2x1 vector comprising the +// reciprocal of link bandwidth [1 / bytes per ms] and the +// link queuing delay buildup [ms]. +// * The state transition matrix (`F`) is the 2x2 identity matrix, meaning that +// link bandwidth and link queuing delay buildup are modeled as independent. +// * The measurement (`z`) is the (scalar) frame delay variation [ms]. +// * The observation matrix (`H`) is a 1x2 vector set as +// `{frame_size_variation [bytes], 1.0}`. +// * The state estimate covariance (`P`) is a symmetric 2x2 matrix. +// * The process noise covariance (`Q`) is a constant 2x2 diagonal matrix +// [(1 / bytes per ms)^2, ms^2]. +// * The observation noise covariance (`r`) is a scalar [ms^2] that is +// determined externally to this class. +class FrameDelayDeltaKalmanFilter { + public: + FrameDelayDeltaKalmanFilter(); + ~FrameDelayDeltaKalmanFilter() = default; + + // Predicts and updates the filter, given a new pair of frame delay variation + // and frame size variation. + // + // Inputs: + // `frame_delay_variation`: + // Frame delay variation as calculated by the `InterFrameDelay` estimator. + // + // `frame_size_variation_bytes`: + // Frame size variation, i.e., the current frame size minus the previous + // frame size (in bytes). Note that this quantity may be negative. + // + // `max_frame_size`: + // Filtered largest frame size received since the last reset. + // + // `var_noise`: + // Variance of the estimated random jitter. + void PredictAndUpdate(TimeDelta frame_delay_variation, + double frame_size_variation_bytes, + DataSize max_frame_size, + double var_noise); + + // Given a frame size variation, returns the estimated frame delay variation + // explained by the link bandwidth alone. + double GetFrameDelayVariationEstimateSizeBased( + double frame_size_variation_bytes) const; + + // Given a frame size variation, returns the estimated frame delay variation + // explained by both link bandwidth and link queuing delay buildup. + double GetFrameDelayVariationEstimateTotal( + double frame_size_variation_bytes) const; + + private: + // State estimate (bandwidth [1 / bytes per ms], queue buildup [ms]). + double estimate_[2]; + double estimate_cov_[2][2]; // Estimate covariance. + + // Process noise covariance. This is a diagonal matrix, so we only store the + // diagonal entries. + double process_noise_cov_diag_[2]; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_TIMING_FRAME_DELAY_DELTA_KALMAN_FILTER_H_ diff --git a/third_party/libwebrtc/modules/video_coding/timing/frame_delay_delta_kalman_filter_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_delta_kalman_filter_gn/moz.build new file mode 100644 index 0000000000..35e13f8678 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_delta_kalman_filter_gn/moz.build @@ -0,0 +1,201 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/video_coding/timing/frame_delay_delta_kalman_filter.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +Library("frame_delay_delta_kalman_filter_gn") diff --git a/third_party/libwebrtc/modules/video_coding/timing/frame_delay_delta_kalman_filter_unittest.cc b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_delta_kalman_filter_unittest.cc new file mode 100644 index 0000000000..d4c1fbdba0 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_delta_kalman_filter_unittest.cc @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/frame_delay_delta_kalman_filter.h" + +#include "api/units/data_size.h" +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +// This test verifies that the initial filter state (link bandwidth, link +// propagation delay) is such that a frame of size zero would take no time to +// propagate. +TEST(FrameDelayDeltaKalmanFilterTest, + InitializedFilterWithZeroSizeFrameTakesNoTimeToPropagate) { + FrameDelayDeltaKalmanFilter filter; + + // A zero-sized frame... + double frame_size_variation_bytes = 0.0; + + // ...should take no time to propagate due to it's size... + EXPECT_EQ(filter.GetFrameDelayVariationEstimateSizeBased( + frame_size_variation_bytes), + 0.0); + + // ...and no time due to the initial link propagation delay being zero. + EXPECT_EQ( + filter.GetFrameDelayVariationEstimateTotal(frame_size_variation_bytes), + 0.0); +} + +// TODO(brandtr): Look into if there is a factor 1000 missing here? It seems +// unreasonable to have an initial link bandwidth of 512 _mega_bits per second? +TEST(FrameDelayDeltaKalmanFilterTest, + InitializedFilterWithSmallSizeFrameTakesFixedTimeToPropagate) { + FrameDelayDeltaKalmanFilter filter; + + // A 1000-byte frame... + double frame_size_variation_bytes = 1000.0; + // ...should take around `1000.0 / (512e3 / 8.0) = 0.015625 ms` to transmit. + double expected_frame_delay_variation_estimate_ms = 1000.0 / (512e3 / 8.0); + + EXPECT_EQ(filter.GetFrameDelayVariationEstimateSizeBased( + frame_size_variation_bytes), + expected_frame_delay_variation_estimate_ms); + EXPECT_EQ( + filter.GetFrameDelayVariationEstimateTotal(frame_size_variation_bytes), + expected_frame_delay_variation_estimate_ms); +} + +TEST(FrameDelayDeltaKalmanFilterTest, + NegativeNoiseVarianceDoesNotUpdateFilter) { + FrameDelayDeltaKalmanFilter filter; + + // Negative variance... + double var_noise = -0.1; + filter.PredictAndUpdate(/*frame_delay_variation=*/TimeDelta::Millis(3), + /*frame_size_variation_bytes=*/200.0, + /*max_frame_size=*/DataSize::Bytes(2000), var_noise); + + // ...does _not_ update the filter. + EXPECT_EQ(filter.GetFrameDelayVariationEstimateTotal( + /*frame_size_variation_bytes=*/0.0), + 0.0); + + // Positive variance... + var_noise = 0.1; + filter.PredictAndUpdate(/*frame_delay_variation=*/TimeDelta::Millis(3), + /*frame_size_variation_bytes=*/200.0, + /*max_frame_size=*/DataSize::Bytes(2000), var_noise); + + // ...does update the filter. + EXPECT_GT(filter.GetFrameDelayVariationEstimateTotal( + /*frame_size_variation_bytes=*/0.0), + 0.0); +} + +TEST(FrameDelayDeltaKalmanFilterTest, + VerifyConvergenceWithAlternatingDeviations) { + FrameDelayDeltaKalmanFilter filter; + + // One frame every 33 ms. + int framerate_fps = 30; + // Let's assume approximately 10% delay variation. + TimeDelta frame_delay_variation = TimeDelta::Millis(3); + // With a bitrate of 512 kbps, each frame will be around 2000 bytes. + DataSize max_frame_size = DataSize::Bytes(2000); + // And again, let's assume 10% size deviation. + double frame_size_variation_bytes = 200; + double var_noise = 0.1; + int test_duration_s = 60; + + for (int i = 0; i < test_duration_s * framerate_fps; ++i) { + // For simplicity, assume alternating variations. + double sign = (i % 2 == 0) ? 1.0 : -1.0; + filter.PredictAndUpdate(sign * frame_delay_variation, + sign * frame_size_variation_bytes, max_frame_size, + var_noise); + } + + // Verify that the filter has converged within a margin of 0.1 ms. + EXPECT_NEAR( + filter.GetFrameDelayVariationEstimateTotal(frame_size_variation_bytes), + frame_delay_variation.ms(), 0.1); +} + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.cc b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.cc new file mode 100644 index 0000000000..bed9f875ee --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.cc @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/inter_frame_delay.h" + +#include "absl/types/optional.h" +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "modules/include/module_common_types_public.h" + +namespace webrtc { + +namespace { +constexpr Frequency k90kHz = Frequency::KiloHertz(90); +} + +InterFrameDelay::InterFrameDelay() { + Reset(); +} + +// Resets the delay estimate. +void InterFrameDelay::Reset() { + prev_wall_clock_ = absl::nullopt; + prev_rtp_timestamp_unwrapped_ = 0; +} + +// Calculates the delay of a frame with the given timestamp. +// This method is called when the frame is complete. +absl::optional<TimeDelta> InterFrameDelay::CalculateDelay( + uint32_t rtp_timestamp, + Timestamp now) { + int64_t rtp_timestamp_unwrapped = unwrapper_.Unwrap(rtp_timestamp); + if (!prev_wall_clock_) { + // First set of data, initialization, wait for next frame. + prev_wall_clock_ = now; + prev_rtp_timestamp_unwrapped_ = rtp_timestamp_unwrapped; + return TimeDelta::Zero(); + } + + // Account for reordering in jitter variance estimate in the future? + // Note that this also captures incomplete frames which are grabbed for + // decoding after a later frame has been complete, i.e. real packet losses. + uint32_t cropped_last = static_cast<uint32_t>(prev_rtp_timestamp_unwrapped_); + if (rtp_timestamp_unwrapped < prev_rtp_timestamp_unwrapped_ || + !IsNewerTimestamp(rtp_timestamp, cropped_last)) { + return absl::nullopt; + } + + // Compute the compensated timestamp difference. + int64_t d_rtp_ticks = rtp_timestamp_unwrapped - prev_rtp_timestamp_unwrapped_; + TimeDelta dts = d_rtp_ticks / k90kHz; + TimeDelta dt = now - *prev_wall_clock_; + + // frameDelay is the difference of dT and dTS -- i.e. the difference of the + // wall clock time difference and the timestamp difference between two + // following frames. + TimeDelta delay = dt - dts; + + prev_rtp_timestamp_unwrapped_ = rtp_timestamp_unwrapped; + prev_wall_clock_ = now; + return delay; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.h b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.h new file mode 100644 index 0000000000..579a488cb1 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_TIMING_INTER_FRAME_DELAY_H_ +#define MODULES_VIDEO_CODING_TIMING_INTER_FRAME_DELAY_H_ + +#include <stdint.h> + +#include "absl/types/optional.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "modules/include/module_common_types_public.h" + +namespace webrtc { + +class InterFrameDelay { + public: + InterFrameDelay(); + + // Resets the estimate. Zeros are given as parameters. + void Reset(); + + // Calculates the delay of a frame with the given timestamp. + // This method is called when the frame is complete. + absl::optional<TimeDelta> CalculateDelay(uint32_t rtp_timestamp, + Timestamp now); + + private: + // The previous rtp timestamp passed to the delay estimate + int64_t prev_rtp_timestamp_unwrapped_; + TimestampUnwrapper unwrapper_; + + // The previous wall clock timestamp used by the delay estimate + absl::optional<Timestamp> prev_wall_clock_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_TIMING_INTER_FRAME_DELAY_H_ diff --git a/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_gn/moz.build new file mode 100644 index 0000000000..84a87f2a49 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_gn/moz.build @@ -0,0 +1,201 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +Library("inter_frame_delay_gn") diff --git a/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_unittest.cc b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_unittest.cc new file mode 100644 index 0000000000..183b378ced --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_unittest.cc @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/inter_frame_delay.h" + +#include <limits> + +#include "absl/types/optional.h" +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "system_wrappers/include/clock.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +// Test is for frames at 30fps. At 30fps, RTP timestamps will increase by +// 90000 / 30 = 3000 ticks per frame. +constexpr Frequency k30Fps = Frequency::Hertz(30); +constexpr TimeDelta kFrameDelay = 1 / k30Fps; +constexpr uint32_t kRtpTicksPerFrame = Frequency::KiloHertz(90) / k30Fps; +constexpr Timestamp kStartTime = Timestamp::Millis(1337); + +} // namespace + +using ::testing::Eq; +using ::testing::Optional; + +TEST(InterFrameDelayTest, OldRtpTimestamp) { + InterFrameDelay inter_frame_delay; + EXPECT_THAT(inter_frame_delay.CalculateDelay(180000, kStartTime), + Optional(TimeDelta::Zero())); + EXPECT_THAT(inter_frame_delay.CalculateDelay(90000, kStartTime), + Eq(absl::nullopt)); +} + +TEST(InterFrameDelayTest, NegativeWrapAroundIsSameAsOldRtpTimestamp) { + InterFrameDelay inter_frame_delay; + uint32_t rtp = 1500; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, kStartTime), + Optional(TimeDelta::Zero())); + // RTP has wrapped around backwards. + rtp -= 3000; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, kStartTime), + Eq(absl::nullopt)); +} + +TEST(InterFrameDelayTest, CorrectDelayForFrames) { + InterFrameDelay inter_frame_delay; + // Use a fake clock to simplify time keeping. + SimulatedClock clock(kStartTime); + + // First frame is always delay 0. + uint32_t rtp = 90000; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Zero())); + + // Perfectly timed frame has 0 delay. + clock.AdvanceTime(kFrameDelay); + rtp += kRtpTicksPerFrame; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Zero())); + + // Slightly early frame will have a negative delay. + clock.AdvanceTime(kFrameDelay - TimeDelta::Millis(3)); + rtp += kRtpTicksPerFrame; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(-TimeDelta::Millis(3))); + + // Slightly late frame will have positive delay. + clock.AdvanceTime(kFrameDelay + TimeDelta::Micros(5125)); + rtp += kRtpTicksPerFrame; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Micros(5125))); + + // Simulate faster frame RTP at the same clock delay. The frame arrives late, + // since the RTP timestamp is faster than the delay, and thus is positive. + clock.AdvanceTime(kFrameDelay); + rtp += kRtpTicksPerFrame / 2; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(kFrameDelay / 2.0)); + + // Simulate slower frame RTP at the same clock delay. The frame is early, + // since the RTP timestamp advanced more than the delay, and thus is negative. + clock.AdvanceTime(kFrameDelay); + rtp += 1.5 * kRtpTicksPerFrame; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(-kFrameDelay / 2.0)); +} + +TEST(InterFrameDelayTest, PositiveWrapAround) { + InterFrameDelay inter_frame_delay; + // Use a fake clock to simplify time keeping. + SimulatedClock clock(kStartTime); + + // First frame is behind the max RTP by 1500. + uint32_t rtp = std::numeric_limits<uint32_t>::max() - 1500; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Zero())); + + // Rtp wraps around, now 1499. + rtp += kRtpTicksPerFrame; + + // Frame delay should be as normal, in this case simulated as 1ms late. + clock.AdvanceTime(kFrameDelay + TimeDelta::Millis(1)); + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Millis(1))); +} + +TEST(InterFrameDelayTest, MultipleWrapArounds) { + // Simulate a long pauses which cause wrap arounds multiple times. + constexpr Frequency k90Khz = Frequency::KiloHertz(90); + constexpr uint32_t kHalfRtp = std::numeric_limits<uint32_t>::max() / 2; + constexpr TimeDelta kWrapAroundDelay = kHalfRtp / k90Khz; + + InterFrameDelay inter_frame_delay; + // Use a fake clock to simplify time keeping. + SimulatedClock clock(kStartTime); + uint32_t rtp = 0; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Zero())); + + rtp += kHalfRtp; + clock.AdvanceTime(kWrapAroundDelay); + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Zero())); + // 1st wrap around. + rtp += kHalfRtp + 1; + clock.AdvanceTime(kWrapAroundDelay + TimeDelta::Millis(1)); + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Millis(1) - (1 / k90Khz))); + + rtp += kHalfRtp; + clock.AdvanceTime(kWrapAroundDelay); + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Zero())); + // 2nd wrap arounds. + rtp += kHalfRtp + 1; + clock.AdvanceTime(kWrapAroundDelay - TimeDelta::Millis(1)); + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(-TimeDelta::Millis(1) - (1 / k90Khz))); + + // Ensure short delay (large RTP delay) between wrap-arounds has correct + // jitter. + rtp += kHalfRtp; + clock.AdvanceTime(TimeDelta::Millis(10)); + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(-(kWrapAroundDelay - TimeDelta::Millis(10)))); + // 3nd wrap arounds, this time with large RTP delay. + rtp += kHalfRtp + 1; + clock.AdvanceTime(TimeDelta::Millis(10)); + EXPECT_THAT( + inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(-(kWrapAroundDelay - TimeDelta::Millis(10) + (1 / k90Khz)))); +} + +TEST(InterFrameDelayTest, NegativeWrapAroundAfterPositiveWrapAround) { + InterFrameDelay inter_frame_delay; + // Use a fake clock to simplify time keeping. + SimulatedClock clock(kStartTime); + uint32_t rtp = std::numeric_limits<uint32_t>::max() - 1500; + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Zero())); + + // Rtp wraps around, now 1499. + rtp += kRtpTicksPerFrame; + // Frame delay should be as normal, in this case simulated as 1ms late. + clock.AdvanceTime(kFrameDelay); + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Optional(TimeDelta::Zero())); + + // Wrap back. + rtp -= kRtpTicksPerFrame; + // Frame delay should be as normal, in this case simulated as 1ms late. + clock.AdvanceTime(kFrameDelay); + EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()), + Eq(absl::nullopt)); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.cc b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.cc new file mode 100644 index 0000000000..7c5c7fdc06 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.cc @@ -0,0 +1,314 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/jitter_estimator.h" + +#include <math.h> +#include <string.h> + +#include <algorithm> +#include <cstdint> + +#include "absl/types/optional.h" +#include "api/field_trials_view.h" +#include "api/units/data_size.h" +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "modules/video_coding/timing/rtt_filter.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +namespace { +static constexpr uint32_t kStartupDelaySamples = 30; +static constexpr int64_t kFsAccuStartupSamples = 5; +static constexpr Frequency kMaxFramerateEstimate = Frequency::Hertz(200); +static constexpr TimeDelta kNackCountTimeout = TimeDelta::Seconds(60); +static constexpr double kDefaultMaxTimestampDeviationInSigmas = 3.5; + +constexpr double kPhi = 0.97; +constexpr double kPsi = 0.9999; +constexpr uint32_t kAlphaCountMax = 400; +constexpr uint32_t kNackLimit = 3; +constexpr int32_t kNumStdDevDelayOutlier = 15; +constexpr int32_t kNumStdDevFrameSizeOutlier = 3; +// ~Less than 1% chance (look up in normal distribution table)... +constexpr double kNoiseStdDevs = 2.33; +// ...of getting 30 ms freezes +constexpr double kNoiseStdDevOffset = 30.0; + +} // namespace + +JitterEstimator::JitterEstimator(Clock* clock, + const FieldTrialsView& field_trials) + : fps_counter_(30), // TODO(sprang): Use an estimator with limit based on + // time, rather than number of samples. + clock_(clock) { + Reset(); +} + +JitterEstimator::~JitterEstimator() = default; + +// Resets the JitterEstimate. +void JitterEstimator::Reset() { + var_noise_ = 4.0; + + avg_frame_size_ = kDefaultAvgAndMaxFrameSize; + max_frame_size_ = kDefaultAvgAndMaxFrameSize; + var_frame_size_ = 100; + last_update_time_ = absl::nullopt; + prev_estimate_ = absl::nullopt; + prev_frame_size_ = absl::nullopt; + avg_noise_ = 0.0; + alpha_count_ = 1; + filter_jitter_estimate_ = TimeDelta::Zero(); + latest_nack_ = Timestamp::Zero(); + nack_count_ = 0; + frame_size_sum_ = DataSize::Zero(); + frame_size_count_ = 0; + startup_count_ = 0; + rtt_filter_.Reset(); + fps_counter_.Reset(); + + kalman_filter_ = FrameDelayDeltaKalmanFilter(); +} + +// Updates the estimates with the new measurements. +void JitterEstimator::UpdateEstimate(TimeDelta frame_delay, + DataSize frame_size) { + if (frame_size.IsZero()) { + return; + } + // Can't use DataSize since this can be negative. + double delta_frame_bytes = + frame_size.bytes() - prev_frame_size_.value_or(DataSize::Zero()).bytes(); + if (frame_size_count_ < kFsAccuStartupSamples) { + frame_size_sum_ += frame_size; + frame_size_count_++; + } else if (frame_size_count_ == kFsAccuStartupSamples) { + // Give the frame size filter. + avg_frame_size_ = frame_size_sum_ / static_cast<double>(frame_size_count_); + frame_size_count_++; + } + + DataSize avg_frame_size = kPhi * avg_frame_size_ + (1 - kPhi) * frame_size; + DataSize deviation_size = DataSize::Bytes(2 * sqrt(var_frame_size_)); + if (frame_size < avg_frame_size_ + deviation_size) { + // Only update the average frame size if this sample wasn't a key frame. + avg_frame_size_ = avg_frame_size; + } + + double delta_bytes = frame_size.bytes() - avg_frame_size.bytes(); + var_frame_size_ = std::max( + kPhi * var_frame_size_ + (1 - kPhi) * (delta_bytes * delta_bytes), 1.0); + + // Update max_frame_size_ estimate. + max_frame_size_ = std::max(kPsi * max_frame_size_, frame_size); + + if (!prev_frame_size_) { + prev_frame_size_ = frame_size; + return; + } + prev_frame_size_ = frame_size; + + // Cap frame_delay based on the current time deviation noise. + TimeDelta max_time_deviation = TimeDelta::Millis( + kDefaultMaxTimestampDeviationInSigmas * sqrt(var_noise_) + 0.5); + frame_delay.Clamp(-max_time_deviation, max_time_deviation); + + // Only update the Kalman filter if the sample is not considered an extreme + // outlier. Even if it is an extreme outlier from a delay point of view, if + // the frame size also is large the deviation is probably due to an incorrect + // line slope. + double deviation = + frame_delay.ms() - + kalman_filter_.GetFrameDelayVariationEstimateTotal(delta_frame_bytes); + + if (fabs(deviation) < kNumStdDevDelayOutlier * sqrt(var_noise_) || + frame_size.bytes() > + avg_frame_size_.bytes() + + kNumStdDevFrameSizeOutlier * sqrt(var_frame_size_)) { + // Update the variance of the deviation from the line given by the Kalman + // filter. + EstimateRandomJitter(deviation); + // Prevent updating with frames which have been congested by a large frame, + // and therefore arrives almost at the same time as that frame. + // This can occur when we receive a large frame (key frame) which has been + // delayed. The next frame is of normal size (delta frame), and thus deltaFS + // will be << 0. This removes all frame samples which arrives after a key + // frame. + if (delta_frame_bytes > -0.25 * max_frame_size_.bytes()) { + // Update the Kalman filter with the new data + kalman_filter_.PredictAndUpdate(frame_delay, delta_frame_bytes, + max_frame_size_, var_noise_); + } + } else { + int nStdDev = + (deviation >= 0) ? kNumStdDevDelayOutlier : -kNumStdDevDelayOutlier; + EstimateRandomJitter(nStdDev * sqrt(var_noise_)); + } + // Post process the total estimated jitter + if (startup_count_ >= kStartupDelaySamples) { + PostProcessEstimate(); + } else { + startup_count_++; + } +} + +// Updates the nack/packet ratio. +void JitterEstimator::FrameNacked() { + if (nack_count_ < kNackLimit) { + nack_count_++; + } + latest_nack_ = clock_->CurrentTime(); +} + +// Estimates the random jitter by calculating the variance of the sample +// distance from the line given by theta. +void JitterEstimator::EstimateRandomJitter(double d_dT) { + Timestamp now = clock_->CurrentTime(); + if (last_update_time_.has_value()) { + fps_counter_.AddSample((now - *last_update_time_).us()); + } + last_update_time_ = now; + + if (alpha_count_ == 0) { + RTC_DCHECK_NOTREACHED(); + return; + } + double alpha = + static_cast<double>(alpha_count_ - 1) / static_cast<double>(alpha_count_); + alpha_count_++; + if (alpha_count_ > kAlphaCountMax) + alpha_count_ = kAlphaCountMax; + + // In order to avoid a low frame rate stream to react slower to changes, + // scale the alpha weight relative a 30 fps stream. + Frequency fps = GetFrameRate(); + if (fps > Frequency::Zero()) { + constexpr Frequency k30Fps = Frequency::Hertz(30); + double rate_scale = k30Fps / fps; + // At startup, there can be a lot of noise in the fps estimate. + // Interpolate rate_scale linearly, from 1.0 at sample #1, to 30.0 / fps + // at sample #kStartupDelaySamples. + if (alpha_count_ < kStartupDelaySamples) { + rate_scale = + (alpha_count_ * rate_scale + (kStartupDelaySamples - alpha_count_)) / + kStartupDelaySamples; + } + alpha = pow(alpha, rate_scale); + } + + double avgNoise = alpha * avg_noise_ + (1 - alpha) * d_dT; + double varNoise = alpha * var_noise_ + + (1 - alpha) * (d_dT - avg_noise_) * (d_dT - avg_noise_); + avg_noise_ = avgNoise; + var_noise_ = varNoise; + if (var_noise_ < 1.0) { + // The variance should never be zero, since we might get stuck and consider + // all samples as outliers. + var_noise_ = 1.0; + } +} + +double JitterEstimator::NoiseThreshold() const { + double noiseThreshold = kNoiseStdDevs * sqrt(var_noise_) - kNoiseStdDevOffset; + if (noiseThreshold < 1.0) { + noiseThreshold = 1.0; + } + return noiseThreshold; +} + +// Calculates the current jitter estimate from the filtered estimates. +TimeDelta JitterEstimator::CalculateEstimate() { + double retMs = kalman_filter_.GetFrameDelayVariationEstimateSizeBased( + max_frame_size_.bytes() - avg_frame_size_.bytes()) + + NoiseThreshold(); + + TimeDelta ret = TimeDelta::Millis(retMs); + + constexpr TimeDelta kMinEstimate = TimeDelta::Millis(1); + constexpr TimeDelta kMaxEstimate = TimeDelta::Seconds(10); + // A very low estimate (or negative) is neglected. + if (ret < kMinEstimate) { + ret = prev_estimate_.value_or(kMinEstimate); + // Sanity check to make sure that no other method has set `prev_estimate_` + // to a value lower than `kMinEstimate`. + RTC_DCHECK_GE(ret, kMinEstimate); + } else if (ret > kMaxEstimate) { // Sanity + ret = kMaxEstimate; + } + prev_estimate_ = ret; + return ret; +} + +void JitterEstimator::PostProcessEstimate() { + filter_jitter_estimate_ = CalculateEstimate(); +} + +void JitterEstimator::UpdateRtt(TimeDelta rtt) { + rtt_filter_.Update(rtt); +} + +// Returns the current filtered estimate if available, +// otherwise tries to calculate an estimate. +TimeDelta JitterEstimator::GetJitterEstimate( + double rtt_multiplier, + absl::optional<TimeDelta> rtt_mult_add_cap) { + TimeDelta jitter = CalculateEstimate() + OPERATING_SYSTEM_JITTER; + Timestamp now = clock_->CurrentTime(); + + if (now - latest_nack_ > kNackCountTimeout) + nack_count_ = 0; + + if (filter_jitter_estimate_ > jitter) + jitter = filter_jitter_estimate_; + if (nack_count_ >= kNackLimit) { + if (rtt_mult_add_cap.has_value()) { + jitter += std::min(rtt_filter_.Rtt() * rtt_multiplier, + rtt_mult_add_cap.value()); + } else { + jitter += rtt_filter_.Rtt() * rtt_multiplier; + } + } + + static const Frequency kJitterScaleLowThreshold = Frequency::Hertz(5); + static const Frequency kJitterScaleHighThreshold = Frequency::Hertz(10); + Frequency fps = GetFrameRate(); + // Ignore jitter for very low fps streams. + if (fps < kJitterScaleLowThreshold) { + if (fps.IsZero()) { + return std::max(TimeDelta::Zero(), jitter); + } + return TimeDelta::Zero(); + } + + // Semi-low frame rate; scale by factor linearly interpolated from 0.0 at + // kJitterScaleLowThreshold to 1.0 at kJitterScaleHighThreshold. + if (fps < kJitterScaleHighThreshold) { + jitter = (1.0 / (kJitterScaleHighThreshold - kJitterScaleLowThreshold)) * + (fps - kJitterScaleLowThreshold) * jitter; + } + + return std::max(TimeDelta::Zero(), jitter); +} + +Frequency JitterEstimator::GetFrameRate() const { + TimeDelta mean_frame_period = TimeDelta::Micros(fps_counter_.ComputeMean()); + if (mean_frame_period <= TimeDelta::Zero()) + return Frequency::Zero(); + + Frequency fps = 1 / mean_frame_period; + // Sanity check. + RTC_DCHECK_GE(fps, Frequency::Zero()); + return std::min(fps, kMaxFramerateEstimate); +} +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.h b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.h new file mode 100644 index 0000000000..ec1e696b68 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_TIMING_JITTER_ESTIMATOR_H_ +#define MODULES_VIDEO_CODING_TIMING_JITTER_ESTIMATOR_H_ + +#include "absl/types/optional.h" +#include "api/field_trials_view.h" +#include "api/units/data_size.h" +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "modules/video_coding/timing/frame_delay_delta_kalman_filter.h" +#include "modules/video_coding/timing/rtt_filter.h" +#include "rtc_base/rolling_accumulator.h" + +namespace webrtc { + +class Clock; + +class JitterEstimator { + public: + explicit JitterEstimator(Clock* clock, const FieldTrialsView& field_trials); + ~JitterEstimator(); + JitterEstimator(const JitterEstimator&) = delete; + JitterEstimator& operator=(const JitterEstimator&) = delete; + + // Resets the estimate to the initial state. + void Reset(); + + // Updates the jitter estimate with the new data. + // + // Input: + // - frame_delay : Delay-delta calculated by UTILDelayEstimate. + // - frame_size : Frame size of the current frame. + void UpdateEstimate(TimeDelta frame_delay, DataSize frame_size); + + // Returns the current jitter estimate and adds an RTT dependent term in cases + // of retransmission. + // Input: + // - rtt_multiplier : RTT param multiplier (when applicable). + // - rtt_mult_add_cap : Multiplier cap from the RTTMultExperiment. + // + // Return value : Jitter estimate. + TimeDelta GetJitterEstimate(double rtt_multiplier, + absl::optional<TimeDelta> rtt_mult_add_cap); + + // Updates the nack counter. + void FrameNacked(); + + // Updates the RTT filter. + // + // Input: + // - rtt : Round trip time. + void UpdateRtt(TimeDelta rtt); + + // A constant describing the delay from the jitter buffer to the delay on the + // receiving side which is not accounted for by the jitter buffer nor the + // decoding delay estimate. + static constexpr TimeDelta OPERATING_SYSTEM_JITTER = TimeDelta::Millis(10); + + private: + double var_noise_; // Variance of the time-deviation from the line + + // Updates the random jitter estimate, i.e. the variance of the time + // deviations from the line given by the Kalman filter. + // + // Input: + // - d_dT : The deviation from the kalman estimate. + void EstimateRandomJitter(double d_dT); + + double NoiseThreshold() const; + + // Calculates the current jitter estimate. + // + // Return value : The current jitter estimate. + TimeDelta CalculateEstimate(); + + // Post process the calculated estimate. + void PostProcessEstimate(); + + Frequency GetFrameRate() const; + + // Filters the {frame_delay_delta, frame_size_delta} measurements through + // a linear Kalman filter. + FrameDelayDeltaKalmanFilter kalman_filter_; + + static constexpr DataSize kDefaultAvgAndMaxFrameSize = DataSize::Bytes(500); + DataSize avg_frame_size_ = kDefaultAvgAndMaxFrameSize; // Average frame size + double var_frame_size_; // Frame size variance. Unit is bytes^2. + // Largest frame size received (descending with a factor kPsi) + DataSize max_frame_size_ = kDefaultAvgAndMaxFrameSize; + DataSize frame_size_sum_ = DataSize::Zero(); + uint32_t frame_size_count_; + + absl::optional<Timestamp> last_update_time_; + // The previously returned jitter estimate + absl::optional<TimeDelta> prev_estimate_; + // Frame size of the previous frame + absl::optional<DataSize> prev_frame_size_; + // Average of the random jitter + double avg_noise_; + uint32_t alpha_count_; + // The filtered sum of jitter estimates + TimeDelta filter_jitter_estimate_ = TimeDelta::Zero(); + + uint32_t startup_count_; + // Time when the latest nack was seen + Timestamp latest_nack_ = Timestamp::Zero(); + // Keeps track of the number of nacks received, but never goes above + // kNackLimit. + uint32_t nack_count_; + RttFilter rtt_filter_; + + // Tracks frame rates in microseconds. + rtc::RollingAccumulator<uint64_t> fps_counter_; + Clock* clock_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_TIMING_JITTER_ESTIMATOR_H_ diff --git a/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_gn/moz.build new file mode 100644 index 0000000000..f45f6f072f --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_gn/moz.build @@ -0,0 +1,214 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "GLESv2", + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "dl", + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +Library("jitter_estimator_gn") diff --git a/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_unittest.cc b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_unittest.cc new file mode 100644 index 0000000000..f442dbb62d --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_unittest.cc @@ -0,0 +1,113 @@ +/* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/jitter_estimator.h" + +#include <stdint.h> + +#include <memory> +#include <utility> +#include <vector> + +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/units/data_size.h" +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "rtc_base/numerics/histogram_percentile_counter.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/clock.h" +#include "test/gtest.h" +#include "test/scoped_key_value_config.h" + +namespace webrtc { + +class TestJitterEstimator : public ::testing::Test { + protected: + TestJitterEstimator() : fake_clock_(0) {} + + virtual void SetUp() { + estimator_ = std::make_unique<JitterEstimator>(&fake_clock_, field_trials_); + } + + SimulatedClock fake_clock_; + test::ScopedKeyValueConfig field_trials_; + std::unique_ptr<JitterEstimator> estimator_; +}; + +// Generates some simple test data in the form of a sawtooth wave. +class ValueGenerator { + public: + explicit ValueGenerator(int32_t amplitude) + : amplitude_(amplitude), counter_(0) {} + + virtual ~ValueGenerator() = default; + + TimeDelta Delay() const { + return TimeDelta::Millis((counter_ % 11) - 5) * amplitude_; + } + + DataSize FrameSize() const { + return DataSize::Bytes(1000 + Delay().ms() / 5); + } + + void Advance() { ++counter_; } + + private: + const int32_t amplitude_; + int64_t counter_; +}; + +TEST_F(TestJitterEstimator, TestLowRate) { + ValueGenerator gen(10); + // At 5 fps, we disable jitter delay altogether. + TimeDelta time_delta = 1 / Frequency::Hertz(5); + for (int i = 0; i < 60; ++i) { + estimator_->UpdateEstimate(gen.Delay(), gen.FrameSize()); + fake_clock_.AdvanceTime(time_delta); + if (i > 2) + EXPECT_EQ(estimator_->GetJitterEstimate(0, absl::nullopt), + TimeDelta::Zero()); + gen.Advance(); + } +} + +TEST_F(TestJitterEstimator, RttMultAddCap) { + std::vector<std::pair<TimeDelta, rtc::HistogramPercentileCounter>> + jitter_by_rtt_mult_cap; + jitter_by_rtt_mult_cap.emplace_back( + /*rtt_mult_add_cap=*/TimeDelta::Millis(10), /*long_tail_boundary=*/1000); + jitter_by_rtt_mult_cap.emplace_back( + /*rtt_mult_add_cap=*/TimeDelta::Millis(200), /*long_tail_boundary=*/1000); + + for (auto& [rtt_mult_add_cap, jitter] : jitter_by_rtt_mult_cap) { + SetUp(); + + ValueGenerator gen(50); + TimeDelta time_delta = 1 / Frequency::Hertz(30); + constexpr TimeDelta kRtt = TimeDelta::Millis(250); + for (int i = 0; i < 100; ++i) { + estimator_->UpdateEstimate(gen.Delay(), gen.FrameSize()); + fake_clock_.AdvanceTime(time_delta); + estimator_->FrameNacked(); + estimator_->UpdateRtt(kRtt); + jitter.Add( + estimator_->GetJitterEstimate(/*rtt_mult=*/1.0, rtt_mult_add_cap) + .ms()); + gen.Advance(); + } + } + + // 200ms cap should result in at least 25% higher max compared to 10ms. + EXPECT_GT(*jitter_by_rtt_mult_cap[1].second.GetPercentile(1.0), + *jitter_by_rtt_mult_cap[0].second.GetPercentile(1.0) * 1.25); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.cc b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.cc new file mode 100644 index 0000000000..6962224d61 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.cc @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/rtt_filter.h" + +#include <math.h> +#include <stdlib.h> +#include <string.h> + +#include <algorithm> + +#include "absl/algorithm/container.h" +#include "absl/container/inlined_vector.h" +#include "api/units/time_delta.h" + +namespace webrtc { + +namespace { + +constexpr TimeDelta kMaxRtt = TimeDelta::Seconds(3); +constexpr uint32_t kFilterFactorMax = 35; +constexpr double kJumpStddev = 2.5; +constexpr double kDriftStdDev = 3.5; + +} // namespace + +RttFilter::RttFilter() + : avg_rtt_(TimeDelta::Zero()), + var_rtt_(0), + max_rtt_(TimeDelta::Zero()), + jump_buf_(kMaxDriftJumpCount, TimeDelta::Zero()), + drift_buf_(kMaxDriftJumpCount, TimeDelta::Zero()) { + Reset(); +} + +void RttFilter::Reset() { + got_non_zero_update_ = false; + avg_rtt_ = TimeDelta::Zero(); + var_rtt_ = 0; + max_rtt_ = TimeDelta::Zero(); + filt_fact_count_ = 1; + absl::c_fill(jump_buf_, TimeDelta::Zero()); + absl::c_fill(drift_buf_, TimeDelta::Zero()); +} + +void RttFilter::Update(TimeDelta rtt) { + if (!got_non_zero_update_) { + if (rtt.IsZero()) { + return; + } + got_non_zero_update_ = true; + } + + // Sanity check + if (rtt > kMaxRtt) { + rtt = kMaxRtt; + } + + double filt_factor = 0; + if (filt_fact_count_ > 1) { + filt_factor = static_cast<double>(filt_fact_count_ - 1) / filt_fact_count_; + } + filt_fact_count_++; + if (filt_fact_count_ > kFilterFactorMax) { + // This prevents filt_factor from going above + // (_filt_fact_max - 1) / filt_fact_max_, + // e.g., filt_fact_max_ = 50 => filt_factor = 49/50 = 0.98 + filt_fact_count_ = kFilterFactorMax; + } + TimeDelta old_avg = avg_rtt_; + int64_t old_var = var_rtt_; + avg_rtt_ = filt_factor * avg_rtt_ + (1 - filt_factor) * rtt; + int64_t delta_ms = (rtt - avg_rtt_).ms(); + var_rtt_ = filt_factor * var_rtt_ + (1 - filt_factor) * (delta_ms * delta_ms); + max_rtt_ = std::max(rtt, max_rtt_); + if (!JumpDetection(rtt) || !DriftDetection(rtt)) { + // In some cases we don't want to update the statistics + avg_rtt_ = old_avg; + var_rtt_ = old_var; + } +} + +bool RttFilter::JumpDetection(TimeDelta rtt) { + TimeDelta diff_from_avg = avg_rtt_ - rtt; + // Unit of var_rtt_ is ms^2. + TimeDelta jump_threshold = TimeDelta::Millis(kJumpStddev * sqrt(var_rtt_)); + if (diff_from_avg.Abs() > jump_threshold) { + bool positive_diff = diff_from_avg >= TimeDelta::Zero(); + if (!jump_buf_.empty() && positive_diff != last_jump_positive_) { + // Since the signs differ the samples currently + // in the buffer is useless as they represent a + // jump in a different direction. + jump_buf_.clear(); + } + if (jump_buf_.size() < kMaxDriftJumpCount) { + // Update the buffer used for the short time statistics. + // The sign of the diff is used for updating the counter since + // we want to use the same buffer for keeping track of when + // the RTT jumps down and up. + jump_buf_.push_back(rtt); + last_jump_positive_ = positive_diff; + } + if (jump_buf_.size() >= kMaxDriftJumpCount) { + // Detected an RTT jump + ShortRttFilter(jump_buf_); + filt_fact_count_ = kMaxDriftJumpCount + 1; + jump_buf_.clear(); + } else { + return false; + } + } else { + jump_buf_.clear(); + } + return true; +} + +bool RttFilter::DriftDetection(TimeDelta rtt) { + // Unit of sqrt of var_rtt_ is ms. + TimeDelta drift_threshold = TimeDelta::Millis(kDriftStdDev * sqrt(var_rtt_)); + if (max_rtt_ - avg_rtt_ > drift_threshold) { + if (drift_buf_.size() < kMaxDriftJumpCount) { + // Update the buffer used for the short time statistics. + drift_buf_.push_back(rtt); + } + if (drift_buf_.size() >= kMaxDriftJumpCount) { + // Detected an RTT drift + ShortRttFilter(drift_buf_); + filt_fact_count_ = kMaxDriftJumpCount + 1; + drift_buf_.clear(); + } + } else { + drift_buf_.clear(); + } + return true; +} + +void RttFilter::ShortRttFilter(const BufferList& buf) { + RTC_DCHECK_EQ(buf.size(), kMaxDriftJumpCount); + max_rtt_ = TimeDelta::Zero(); + avg_rtt_ = TimeDelta::Zero(); + for (const TimeDelta& rtt : buf) { + if (rtt > max_rtt_) { + max_rtt_ = rtt; + } + avg_rtt_ += rtt; + } + avg_rtt_ = avg_rtt_ / static_cast<double>(buf.size()); +} + +TimeDelta RttFilter::Rtt() const { + return max_rtt_; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.h b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.h new file mode 100644 index 0000000000..b8700b23ee --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_TIMING_RTT_FILTER_H_ +#define MODULES_VIDEO_CODING_TIMING_RTT_FILTER_H_ + +#include <stdint.h> + +#include "absl/container/inlined_vector.h" +#include "api/units/time_delta.h" + +namespace webrtc { + +class RttFilter { + public: + RttFilter(); + RttFilter(const RttFilter&) = delete; + RttFilter& operator=(const RttFilter&) = delete; + + // Resets the filter. + void Reset(); + // Updates the filter with a new sample. + void Update(TimeDelta rtt); + // A getter function for the current RTT level. + TimeDelta Rtt() const; + + private: + // The size of the drift and jump memory buffers + // and thus also the detection threshold for these + // detectors in number of samples. + static constexpr int kMaxDriftJumpCount = 5; + using BufferList = absl::InlinedVector<TimeDelta, kMaxDriftJumpCount>; + + // Detects RTT jumps by comparing the difference between + // samples and average to the standard deviation. + // Returns true if the long time statistics should be updated + // and false otherwise + bool JumpDetection(TimeDelta rtt); + + // Detects RTT drifts by comparing the difference between + // max and average to the standard deviation. + // Returns true if the long time statistics should be updated + // and false otherwise + bool DriftDetection(TimeDelta rtt); + + // Computes the short time average and maximum of the vector buf. + void ShortRttFilter(const BufferList& buf); + + bool got_non_zero_update_; + TimeDelta avg_rtt_; + // Variance units are TimeDelta^2. Store as ms^2. + int64_t var_rtt_; + TimeDelta max_rtt_; + uint32_t filt_fact_count_; + bool last_jump_positive_ = false; + BufferList jump_buf_; + BufferList drift_buf_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_TIMING_RTT_FILTER_H_ diff --git a/third_party/libwebrtc/modules/video_coding/timing/rtt_filter_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter_gn/moz.build new file mode 100644 index 0000000000..54c90f4fee --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter_gn/moz.build @@ -0,0 +1,201 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +Library("rtt_filter_gn") diff --git a/third_party/libwebrtc/modules/video_coding/timing/rtt_filter_unittest.cc b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter_unittest.cc new file mode 100644 index 0000000000..05502e6f5b --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter_unittest.cc @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/rtt_filter.h" + +#include "api/units/time_delta.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(RttFilterTest, RttIsCapped) { + RttFilter rtt_filter; + rtt_filter.Update(TimeDelta::Seconds(500)); + + EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Seconds(3)); +} + +// If the difference between samples is more than away 2.5 stddev from the mean +// then this is considered a jump. After more than 5 data points at the new +// level, the RTT is reset to the new level. +TEST(RttFilterTest, PositiveJumpDetection) { + RttFilter rtt_filter; + + rtt_filter.Update(TimeDelta::Millis(200)); + rtt_filter.Update(TimeDelta::Millis(200)); + rtt_filter.Update(TimeDelta::Millis(200)); + + // Trigger 5 jumps. + rtt_filter.Update(TimeDelta::Millis(1400)); + rtt_filter.Update(TimeDelta::Millis(1500)); + rtt_filter.Update(TimeDelta::Millis(1600)); + rtt_filter.Update(TimeDelta::Millis(1600)); + + EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Millis(1600)); + + rtt_filter.Update(TimeDelta::Millis(1600)); + EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Millis(1600)); +} + +TEST(RttFilterTest, NegativeJumpDetection) { + RttFilter rtt_filter; + + for (int i = 0; i < 10; ++i) + rtt_filter.Update(TimeDelta::Millis(1500)); + + // Trigger 5 negative data points that jump rtt down. + rtt_filter.Update(TimeDelta::Millis(200)); + rtt_filter.Update(TimeDelta::Millis(200)); + rtt_filter.Update(TimeDelta::Millis(200)); + rtt_filter.Update(TimeDelta::Millis(200)); + // Before 5 data points at the new level, max RTT is still 1500. + EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Millis(1500)); + + rtt_filter.Update(TimeDelta::Millis(300)); + EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Millis(300)); +} + +TEST(RttFilterTest, JumpsResetByDirectionShift) { + RttFilter rtt_filter; + for (int i = 0; i < 10; ++i) + rtt_filter.Update(TimeDelta::Millis(1500)); + + // Trigger 4 negative jumps, then a positive one. This resets the jump + // detection. + rtt_filter.Update(TimeDelta::Millis(200)); + rtt_filter.Update(TimeDelta::Millis(200)); + rtt_filter.Update(TimeDelta::Millis(200)); + rtt_filter.Update(TimeDelta::Millis(200)); + rtt_filter.Update(TimeDelta::Millis(2000)); + EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Millis(2000)); + + rtt_filter.Update(TimeDelta::Millis(300)); + EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Millis(2000)); +} + +// If the difference between the max and average is more than 3.5 stddevs away +// then a drift is detected, and a short filter is applied to find a new max +// rtt. +TEST(RttFilterTest, DriftDetection) { + RttFilter rtt_filter; + + // Descend RTT by 30ms and settle at 700ms RTT. A drift is detected after rtt + // of 700ms is reported around 50 times for these targets. + constexpr TimeDelta kStartRtt = TimeDelta::Millis(1000); + constexpr TimeDelta kDriftTarget = TimeDelta::Millis(700); + constexpr TimeDelta kDelta = TimeDelta::Millis(30); + for (TimeDelta rtt = kStartRtt; rtt >= kDriftTarget; rtt -= kDelta) + rtt_filter.Update(rtt); + + EXPECT_EQ(rtt_filter.Rtt(), kStartRtt); + + for (int i = 0; i < 50; ++i) + rtt_filter.Update(kDriftTarget); + EXPECT_EQ(rtt_filter.Rtt(), kDriftTarget); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/timing.cc b/third_party/libwebrtc/modules/video_coding/timing/timing.cc new file mode 100644 index 0000000000..37dc825bed --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/timing.cc @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/timing.h" + +#include <algorithm> + +#include "api/units/time_delta.h" +#include "rtc_base/experiments/field_trial_parser.h" +#include "rtc_base/logging.h" +#include "rtc_base/time/timestamp_extrapolator.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { +namespace { + +// Default pacing that is used for the low-latency renderer path. +constexpr TimeDelta kZeroPlayoutDelayDefaultMinPacing = TimeDelta::Millis(8); +constexpr TimeDelta kLowLatencyStreamMaxPlayoutDelayThreshold = + TimeDelta::Millis(500); + +void CheckDelaysValid(TimeDelta min_delay, TimeDelta max_delay) { + if (min_delay > max_delay) { + RTC_LOG(LS_ERROR) + << "Playout delays set incorrectly: min playout delay (" << min_delay + << ") > max playout delay (" << max_delay + << "). This is undefined behaviour. Application writers should " + "ensure that the min delay is always less than or equals max " + "delay. If trying to use the playout delay header extensions " + "described in " + "https://webrtc.googlesource.com/src/+/refs/heads/main/docs/" + "native-code/rtp-hdrext/playout-delay/, be careful that a playout " + "delay hint or A/V sync settings may have caused this conflict."; + } +} + +} // namespace + +VCMTiming::VCMTiming(Clock* clock, const FieldTrialsView& field_trials) + : clock_(clock), + ts_extrapolator_( + std::make_unique<TimestampExtrapolator>(clock_->CurrentTime())), + codec_timer_(std::make_unique<CodecTimer>()), + render_delay_(kDefaultRenderDelay), + min_playout_delay_(TimeDelta::Zero()), + max_playout_delay_(TimeDelta::Seconds(10)), + jitter_delay_(TimeDelta::Zero()), + current_delay_(TimeDelta::Zero()), + prev_frame_timestamp_(0), + num_decoded_frames_(0), + zero_playout_delay_min_pacing_("min_pacing", + kZeroPlayoutDelayDefaultMinPacing), + last_decode_scheduled_(Timestamp::Zero()) { + ParseFieldTrial({&zero_playout_delay_min_pacing_}, + field_trials.Lookup("WebRTC-ZeroPlayoutDelay")); +} + +void VCMTiming::Reset() { + MutexLock lock(&mutex_); + ts_extrapolator_->Reset(clock_->CurrentTime()); + codec_timer_ = std::make_unique<CodecTimer>(); + render_delay_ = kDefaultRenderDelay; + min_playout_delay_ = TimeDelta::Zero(); + jitter_delay_ = TimeDelta::Zero(); + current_delay_ = TimeDelta::Zero(); + prev_frame_timestamp_ = 0; +} + +void VCMTiming::set_render_delay(TimeDelta render_delay) { + MutexLock lock(&mutex_); + render_delay_ = render_delay; +} + +TimeDelta VCMTiming::min_playout_delay() const { + MutexLock lock(&mutex_); + return min_playout_delay_; +} + +void VCMTiming::set_min_playout_delay(TimeDelta min_playout_delay) { + MutexLock lock(&mutex_); + if (min_playout_delay_ != min_playout_delay) { + CheckDelaysValid(min_playout_delay, max_playout_delay_); + min_playout_delay_ = min_playout_delay; + } +} + +void VCMTiming::set_max_playout_delay(TimeDelta max_playout_delay) { + MutexLock lock(&mutex_); + if (max_playout_delay_ != max_playout_delay) { + CheckDelaysValid(min_playout_delay_, max_playout_delay); + max_playout_delay_ = max_playout_delay; + } +} + +void VCMTiming::SetJitterDelay(TimeDelta jitter_delay) { + MutexLock lock(&mutex_); + if (jitter_delay != jitter_delay_) { + jitter_delay_ = jitter_delay; + // When in initial state, set current delay to minimum delay. + if (current_delay_.IsZero()) { + current_delay_ = jitter_delay_; + } + } +} + +void VCMTiming::UpdateCurrentDelay(uint32_t frame_timestamp) { + MutexLock lock(&mutex_); + TimeDelta target_delay = TargetDelayInternal(); + + if (current_delay_.IsZero()) { + // Not initialized, set current delay to target. + current_delay_ = target_delay; + } else if (target_delay != current_delay_) { + TimeDelta delay_diff = target_delay - current_delay_; + // Never change the delay with more than 100 ms every second. If we're + // changing the delay in too large steps we will get noticeable freezes. By + // limiting the change we can increase the delay in smaller steps, which + // will be experienced as the video is played in slow motion. When lowering + // the delay the video will be played at a faster pace. + TimeDelta max_change = TimeDelta::Zero(); + if (frame_timestamp < 0x0000ffff && prev_frame_timestamp_ > 0xffff0000) { + // wrap + max_change = + TimeDelta::Millis(kDelayMaxChangeMsPerS * + (frame_timestamp + (static_cast<int64_t>(1) << 32) - + prev_frame_timestamp_) / + 90000); + } else { + max_change = + TimeDelta::Millis(kDelayMaxChangeMsPerS * + (frame_timestamp - prev_frame_timestamp_) / 90000); + } + + if (max_change <= TimeDelta::Zero()) { + // Any changes less than 1 ms are truncated and will be postponed. + // Negative change will be due to reordering and should be ignored. + return; + } + delay_diff = std::max(delay_diff, -max_change); + delay_diff = std::min(delay_diff, max_change); + + current_delay_ = current_delay_ + delay_diff; + } + prev_frame_timestamp_ = frame_timestamp; +} + +void VCMTiming::UpdateCurrentDelay(Timestamp render_time, + Timestamp actual_decode_time) { + MutexLock lock(&mutex_); + TimeDelta target_delay = TargetDelayInternal(); + TimeDelta delayed = + (actual_decode_time - render_time) + RequiredDecodeTime() + render_delay_; + + // Only consider `delayed` as negative by more than a few microseconds. + if (delayed.ms() < 0) { + return; + } + if (current_delay_ + delayed <= target_delay) { + current_delay_ += delayed; + } else { + current_delay_ = target_delay; + } +} + +void VCMTiming::StopDecodeTimer(TimeDelta decode_time, Timestamp now) { + MutexLock lock(&mutex_); + codec_timer_->AddTiming(decode_time.ms(), now.ms()); + RTC_DCHECK_GE(decode_time, TimeDelta::Zero()); + ++num_decoded_frames_; +} + +void VCMTiming::IncomingTimestamp(uint32_t rtp_timestamp, Timestamp now) { + MutexLock lock(&mutex_); + ts_extrapolator_->Update(now, rtp_timestamp); +} + +Timestamp VCMTiming::RenderTime(uint32_t frame_timestamp, Timestamp now) const { + MutexLock lock(&mutex_); + return RenderTimeInternal(frame_timestamp, now); +} + +void VCMTiming::SetLastDecodeScheduledTimestamp( + Timestamp last_decode_scheduled) { + MutexLock lock(&mutex_); + last_decode_scheduled_ = last_decode_scheduled; +} + +Timestamp VCMTiming::RenderTimeInternal(uint32_t frame_timestamp, + Timestamp now) const { + if (UseLowLatencyRendering()) { + // Render as soon as possible or with low-latency renderer algorithm. + return Timestamp::Zero(); + } + // Note that TimestampExtrapolator::ExtrapolateLocalTime is not a const + // method; it mutates the object's wraparound state. + Timestamp estimated_complete_time = + ts_extrapolator_->ExtrapolateLocalTime(frame_timestamp).value_or(now); + + // Make sure the actual delay stays in the range of `min_playout_delay_` + // and `max_playout_delay_`. + TimeDelta actual_delay = + current_delay_.Clamped(min_playout_delay_, max_playout_delay_); + return estimated_complete_time + actual_delay; +} + +TimeDelta VCMTiming::RequiredDecodeTime() const { + const int decode_time_ms = codec_timer_->RequiredDecodeTimeMs(); + RTC_DCHECK_GE(decode_time_ms, 0); + return TimeDelta::Millis(decode_time_ms); +} + +TimeDelta VCMTiming::MaxWaitingTime(Timestamp render_time, + Timestamp now, + bool too_many_frames_queued) const { + MutexLock lock(&mutex_); + + if (render_time.IsZero() && zero_playout_delay_min_pacing_->us() > 0 && + min_playout_delay_.IsZero() && max_playout_delay_ > TimeDelta::Zero()) { + // `render_time` == 0 indicates that the frame should be decoded and + // rendered as soon as possible. However, the decoder can be choked if too + // many frames are sent at once. Therefore, limit the interframe delay to + // |zero_playout_delay_min_pacing_| unless too many frames are queued in + // which case the frames are sent to the decoder at once. + if (too_many_frames_queued) { + return TimeDelta::Zero(); + } + Timestamp earliest_next_decode_start_time = + last_decode_scheduled_ + zero_playout_delay_min_pacing_; + TimeDelta max_wait_time = now >= earliest_next_decode_start_time + ? TimeDelta::Zero() + : earliest_next_decode_start_time - now; + return max_wait_time; + } + return render_time - now - RequiredDecodeTime() - render_delay_; +} + +TimeDelta VCMTiming::TargetVideoDelay() const { + MutexLock lock(&mutex_); + return TargetDelayInternal(); +} + +TimeDelta VCMTiming::TargetDelayInternal() const { + return std::max(min_playout_delay_, + jitter_delay_ + RequiredDecodeTime() + render_delay_); +} + +VideoFrame::RenderParameters VCMTiming::RenderParameters() const { + MutexLock lock(&mutex_); + return {.use_low_latency_rendering = UseLowLatencyRendering(), + .max_composition_delay_in_frames = max_composition_delay_in_frames_}; +} + +bool VCMTiming::UseLowLatencyRendering() const { + // min_playout_delay_==0, + // max_playout_delay_<=kLowLatencyStreamMaxPlayoutDelayThreshold indicates + // that the low-latency path should be used, which means that frames should be + // decoded and rendered as soon as possible. + return min_playout_delay_.IsZero() && + max_playout_delay_ <= kLowLatencyStreamMaxPlayoutDelayThreshold; +} + +VCMTiming::VideoDelayTimings VCMTiming::GetTimings() const { + MutexLock lock(&mutex_); + return VideoDelayTimings{.max_decode_duration = RequiredDecodeTime(), + .current_delay = current_delay_, + .target_delay = TargetDelayInternal(), + .jitter_buffer_delay = jitter_delay_, + .min_playout_delay = min_playout_delay_, + .max_playout_delay = max_playout_delay_, + .render_delay = render_delay_, + .num_decoded_frames = num_decoded_frames_}; +} + +void VCMTiming::SetTimingFrameInfo(const TimingFrameInfo& info) { + MutexLock lock(&mutex_); + timing_frame_info_.emplace(info); +} + +absl::optional<TimingFrameInfo> VCMTiming::GetTimingFrameInfo() { + MutexLock lock(&mutex_); + return timing_frame_info_; +} + +void VCMTiming::SetMaxCompositionDelayInFrames( + absl::optional<int> max_composition_delay_in_frames) { + MutexLock lock(&mutex_); + max_composition_delay_in_frames_ = max_composition_delay_in_frames; +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/video_coding/timing/timing.h b/third_party/libwebrtc/modules/video_coding/timing/timing.h new file mode 100644 index 0000000000..6ee1cf4d6f --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/timing.h @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_TIMING_TIMING_H_ +#define MODULES_VIDEO_CODING_TIMING_TIMING_H_ + +#include <memory> + +#include "absl/types/optional.h" +#include "api/field_trials_view.h" +#include "api/units/time_delta.h" +#include "api/video/video_frame.h" +#include "api/video/video_timing.h" +#include "modules/video_coding/timing/codec_timer.h" +#include "rtc_base/experiments/field_trial_parser.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" +#include "rtc_base/time/timestamp_extrapolator.h" + +namespace webrtc { + +class Clock; +class TimestampExtrapolator; + +class VCMTiming { + public: + static constexpr auto kDefaultRenderDelay = TimeDelta::Millis(10); + static constexpr auto kDelayMaxChangeMsPerS = 100; + + VCMTiming(Clock* clock, const FieldTrialsView& field_trials); + virtual ~VCMTiming() = default; + + // Resets the timing to the initial state. + void Reset(); + + // Set the amount of time needed to render an image. Defaults to 10 ms. + void set_render_delay(TimeDelta render_delay); + + // Set the minimum time the video must be delayed on the receiver to + // get the desired jitter buffer level. + void SetJitterDelay(TimeDelta required_delay); + + // Set/get the minimum playout delay from capture to render. + TimeDelta min_playout_delay() const; + void set_min_playout_delay(TimeDelta min_playout_delay); + + // Set/get the maximum playout delay from capture to render in ms. + void set_max_playout_delay(TimeDelta max_playout_delay); + + // Increases or decreases the current delay to get closer to the target delay. + // Calculates how long it has been since the previous call to this function, + // and increases/decreases the delay in proportion to the time difference. + void UpdateCurrentDelay(uint32_t frame_timestamp); + + // Increases or decreases the current delay to get closer to the target delay. + // Given the actual decode time in ms and the render time in ms for a frame, + // this function calculates how late the frame is and increases the delay + // accordingly. + void UpdateCurrentDelay(Timestamp render_time, Timestamp actual_decode_time); + + // Stops the decoder timer, should be called when the decoder returns a frame + // or when the decoded frame callback is called. + void StopDecodeTimer(TimeDelta decode_time, Timestamp now); + + // Used to report that a frame is passed to decoding. Updates the timestamp + // filter which is used to map between timestamps and receiver system time. + void IncomingTimestamp(uint32_t rtp_timestamp, Timestamp last_packet_time); + + // Returns the receiver system time when the frame with timestamp + // `frame_timestamp` should be rendered, assuming that the system time + // currently is `now`. + virtual Timestamp RenderTime(uint32_t frame_timestamp, Timestamp now) const; + + // Returns the maximum time in ms that we can wait for a frame to become + // complete before we must pass it to the decoder. render_time==0 indicates + // that the frames should be processed as quickly as possible, with possibly + // only a small delay added to make sure that the decoder is not overloaded. + // In this case, the parameter too_many_frames_queued is used to signal that + // the decode queue is full and that the frame should be decoded as soon as + // possible. + virtual TimeDelta MaxWaitingTime(Timestamp render_time, + Timestamp now, + bool too_many_frames_queued) const; + + // Returns the current target delay which is required delay + decode time + + // render delay. + TimeDelta TargetVideoDelay() const; + + // Return current timing information. Returns true if the first frame has been + // decoded, false otherwise. + struct VideoDelayTimings { + TimeDelta max_decode_duration; + TimeDelta current_delay; + TimeDelta target_delay; + TimeDelta jitter_buffer_delay; + TimeDelta min_playout_delay; + TimeDelta max_playout_delay; + TimeDelta render_delay; + size_t num_decoded_frames; + }; + VideoDelayTimings GetTimings() const; + + void SetTimingFrameInfo(const TimingFrameInfo& info); + absl::optional<TimingFrameInfo> GetTimingFrameInfo(); + + void SetMaxCompositionDelayInFrames( + absl::optional<int> max_composition_delay_in_frames); + + VideoFrame::RenderParameters RenderParameters() const; + + // Updates the last time a frame was scheduled for decoding. + void SetLastDecodeScheduledTimestamp(Timestamp last_decode_scheduled); + + protected: + TimeDelta RequiredDecodeTime() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + Timestamp RenderTimeInternal(uint32_t frame_timestamp, Timestamp now) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + TimeDelta TargetDelayInternal() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + bool UseLowLatencyRendering() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + private: + mutable Mutex mutex_; + Clock* const clock_; + const std::unique_ptr<TimestampExtrapolator> ts_extrapolator_ + RTC_PT_GUARDED_BY(mutex_); + std::unique_ptr<CodecTimer> codec_timer_ RTC_GUARDED_BY(mutex_) + RTC_PT_GUARDED_BY(mutex_); + TimeDelta render_delay_ RTC_GUARDED_BY(mutex_); + // Best-effort playout delay range for frames from capture to render. + // The receiver tries to keep the delay between `min_playout_delay_ms_` + // and `max_playout_delay_ms_` taking the network jitter into account. + // A special case is where min_playout_delay_ms_ = max_playout_delay_ms_ = 0, + // in which case the receiver tries to play the frames as they arrive. + TimeDelta min_playout_delay_ RTC_GUARDED_BY(mutex_); + TimeDelta max_playout_delay_ RTC_GUARDED_BY(mutex_); + TimeDelta jitter_delay_ RTC_GUARDED_BY(mutex_); + TimeDelta current_delay_ RTC_GUARDED_BY(mutex_); + uint32_t prev_frame_timestamp_ RTC_GUARDED_BY(mutex_); + absl::optional<TimingFrameInfo> timing_frame_info_ RTC_GUARDED_BY(mutex_); + size_t num_decoded_frames_ RTC_GUARDED_BY(mutex_); + absl::optional<int> max_composition_delay_in_frames_ RTC_GUARDED_BY(mutex_); + // Set by the field trial WebRTC-ZeroPlayoutDelay. The parameter min_pacing + // determines the minimum delay between frames scheduled for decoding that is + // used when min playout delay=0 and max playout delay>=0. + FieldTrialParameter<TimeDelta> zero_playout_delay_min_pacing_ + RTC_GUARDED_BY(mutex_); + // Timestamp at which the last frame was scheduled to be sent to the decoder. + // Used only when the RTP header extension playout delay is set to min=0 ms + // which is indicated by a render time set to 0. + Timestamp last_decode_scheduled_ RTC_GUARDED_BY(mutex_); +}; +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_TIMING_TIMING_H_ diff --git a/third_party/libwebrtc/modules/video_coding/timing/timing_module_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/timing_module_gn/moz.build new file mode 100644 index 0000000000..7f4e361630 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/timing_module_gn/moz.build @@ -0,0 +1,212 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + + ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ### + ### DO NOT edit it by hand. ### + +COMPILE_FLAGS["OS_INCLUDES"] = [] +AllowCompilerWarnings() + +DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1" +DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True +DEFINES["RTC_ENABLE_VP9"] = True +DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0" +DEFINES["WEBRTC_LIBRARY_IMPL"] = True +DEFINES["WEBRTC_MOZILLA_BUILD"] = True +DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0" + +FINAL_LIBRARY = "webrtc" + + +LOCAL_INCLUDES += [ + "!/ipc/ipdl/_ipdlheaders", + "/ipc/chromium/src", + "/third_party/libwebrtc/", + "/third_party/libwebrtc/third_party/abseil-cpp/", + "/tools/profiler/public" +] + +UNIFIED_SOURCES += [ + "/third_party/libwebrtc/modules/video_coding/timing/timing.cc" +] + +if not CONFIG["MOZ_DEBUG"]: + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0" + DEFINES["NDEBUG"] = True + DEFINES["NVALGRIND"] = True + +if CONFIG["MOZ_DEBUG"] == "1": + + DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1" + +if CONFIG["OS_TARGET"] == "Android": + + DEFINES["ANDROID"] = True + DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1" + DEFINES["HAVE_SYS_UIO_H"] = True + DEFINES["WEBRTC_ANDROID"] = True + DEFINES["WEBRTC_ANDROID_OPENSLES"] = True + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_GNU_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "log" + ] + +if CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_MAC"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True + DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0" + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_AURA"] = "1" + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_NSS_CERTS"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_UDEV"] = True + DEFINES["WEBRTC_LINUX"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + + OS_LIBS += [ + "rt" + ] + +if CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["USE_GLIB"] = "1" + DEFINES["USE_OZONE"] = "1" + DEFINES["USE_X11"] = "1" + DEFINES["WEBRTC_BSD"] = True + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_POSIX"] = True + DEFINES["_FILE_OFFSET_BITS"] = "64" + DEFINES["_LARGEFILE64_SOURCE"] = True + DEFINES["_LARGEFILE_SOURCE"] = True + DEFINES["__STDC_CONSTANT_MACROS"] = True + DEFINES["__STDC_FORMAT_MACROS"] = True + +if CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True + DEFINES["NOMINMAX"] = True + DEFINES["NTDDI_VERSION"] = "0x0A000000" + DEFINES["PSAPI_VERSION"] = "2" + DEFINES["UNICODE"] = True + DEFINES["USE_AURA"] = "1" + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["WEBRTC_WIN"] = True + DEFINES["WIN32"] = True + DEFINES["WIN32_LEAN_AND_MEAN"] = True + DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP" + DEFINES["WINVER"] = "0x0A00" + DEFINES["_ATL_NO_OPENGL"] = True + DEFINES["_CRT_RAND_S"] = True + DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True + DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True + DEFINES["_HAS_EXCEPTIONS"] = "0" + DEFINES["_HAS_NODISCARD"] = True + DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True + DEFINES["_SECURE_ATL"] = True + DEFINES["_UNICODE"] = True + DEFINES["_WIN32_WINNT"] = "0x0A00" + DEFINES["_WINDOWS"] = True + DEFINES["__STD_C"] = True + + OS_LIBS += [ + "crypt32", + "iphlpapi", + "secur32", + "winmm" + ] + +if CONFIG["CPU_ARCH"] == "aarch64": + + DEFINES["WEBRTC_ARCH_ARM64"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["CPU_ARCH"] == "arm": + + CXXFLAGS += [ + "-mfpu=neon" + ] + + DEFINES["WEBRTC_ARCH_ARM"] = True + DEFINES["WEBRTC_ARCH_ARM_V7"] = True + DEFINES["WEBRTC_HAS_NEON"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD": + + DEFINES["_DEBUG"] = True + +if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT": + + DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0" + +if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["USE_X11"] = "1" + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support", + "unwind" + ] + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android": + + OS_LIBS += [ + "android_support" + ] + +if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux": + + DEFINES["WEBRTC_ENABLE_AVX2"] = True + DEFINES["_GNU_SOURCE"] = True + +Library("timing_module_gn") diff --git a/third_party/libwebrtc/modules/video_coding/timing/timing_unittest.cc b/third_party/libwebrtc/modules/video_coding/timing/timing_unittest.cc new file mode 100644 index 0000000000..8633c0de39 --- /dev/null +++ b/third_party/libwebrtc/modules/video_coding/timing/timing_unittest.cc @@ -0,0 +1,339 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/timing/timing.h" + +#include "api/units/frequency.h" +#include "api/units/time_delta.h" +#include "system_wrappers/include/clock.h" +#include "test/gtest.h" +#include "test/scoped_key_value_config.h" + +namespace webrtc { +namespace { + +constexpr Frequency k25Fps = Frequency::Hertz(25); +constexpr Frequency k90kHz = Frequency::KiloHertz(90); + +} // namespace + +TEST(ReceiverTimingTest, JitterDelay) { + test::ScopedKeyValueConfig field_trials; + SimulatedClock clock(0); + VCMTiming timing(&clock, field_trials); + timing.Reset(); + + uint32_t timestamp = 0; + timing.UpdateCurrentDelay(timestamp); + + timing.Reset(); + + timing.IncomingTimestamp(timestamp, clock.CurrentTime()); + TimeDelta jitter_delay = TimeDelta::Millis(20); + timing.SetJitterDelay(jitter_delay); + timing.UpdateCurrentDelay(timestamp); + timing.set_render_delay(TimeDelta::Zero()); + auto wait_time = timing.MaxWaitingTime( + timing.RenderTime(timestamp, clock.CurrentTime()), clock.CurrentTime(), + /*too_many_frames_queued=*/false); + // First update initializes the render time. Since we have no decode delay + // we get wait_time = renderTime - now - renderDelay = jitter. + EXPECT_EQ(jitter_delay, wait_time); + + jitter_delay += TimeDelta::Millis(VCMTiming::kDelayMaxChangeMsPerS + 10); + timestamp += 90000; + clock.AdvanceTimeMilliseconds(1000); + timing.SetJitterDelay(jitter_delay); + timing.UpdateCurrentDelay(timestamp); + wait_time = timing.MaxWaitingTime( + timing.RenderTime(timestamp, clock.CurrentTime()), clock.CurrentTime(), + /*too_many_frames_queued=*/false); + // Since we gradually increase the delay we only get 100 ms every second. + EXPECT_EQ(jitter_delay - TimeDelta::Millis(10), wait_time); + + timestamp += 90000; + clock.AdvanceTimeMilliseconds(1000); + timing.UpdateCurrentDelay(timestamp); + wait_time = timing.MaxWaitingTime( + timing.RenderTime(timestamp, clock.CurrentTime()), clock.CurrentTime(), + /*too_many_frames_queued=*/false); + EXPECT_EQ(jitter_delay, wait_time); + + // Insert frames without jitter, verify that this gives the exact wait time. + const int kNumFrames = 300; + for (int i = 0; i < kNumFrames; i++) { + clock.AdvanceTime(1 / k25Fps); + timestamp += k90kHz / k25Fps; + timing.IncomingTimestamp(timestamp, clock.CurrentTime()); + } + timing.UpdateCurrentDelay(timestamp); + wait_time = timing.MaxWaitingTime( + timing.RenderTime(timestamp, clock.CurrentTime()), clock.CurrentTime(), + /*too_many_frames_queued=*/false); + EXPECT_EQ(jitter_delay, wait_time); + + // Add decode time estimates for 1 second. + const TimeDelta kDecodeTime = TimeDelta::Millis(10); + for (int i = 0; i < k25Fps.hertz(); i++) { + clock.AdvanceTime(kDecodeTime); + timing.StopDecodeTimer(kDecodeTime, clock.CurrentTime()); + timestamp += k90kHz / k25Fps; + clock.AdvanceTime(1 / k25Fps - kDecodeTime); + timing.IncomingTimestamp(timestamp, clock.CurrentTime()); + } + timing.UpdateCurrentDelay(timestamp); + wait_time = timing.MaxWaitingTime( + timing.RenderTime(timestamp, clock.CurrentTime()), clock.CurrentTime(), + /*too_many_frames_queued=*/false); + EXPECT_EQ(jitter_delay, wait_time); + + const TimeDelta kMinTotalDelay = TimeDelta::Millis(200); + timing.set_min_playout_delay(kMinTotalDelay); + clock.AdvanceTimeMilliseconds(5000); + timestamp += 5 * 90000; + timing.UpdateCurrentDelay(timestamp); + const TimeDelta kRenderDelay = TimeDelta::Millis(10); + timing.set_render_delay(kRenderDelay); + wait_time = timing.MaxWaitingTime( + timing.RenderTime(timestamp, clock.CurrentTime()), clock.CurrentTime(), + /*too_many_frames_queued=*/false); + // We should at least have kMinTotalDelayMs - decodeTime (10) - renderTime + // (10) to wait. + EXPECT_EQ(kMinTotalDelay - kDecodeTime - kRenderDelay, wait_time); + // The total video delay should be equal to the min total delay. + EXPECT_EQ(kMinTotalDelay, timing.TargetVideoDelay()); + + // Reset playout delay. + timing.set_min_playout_delay(TimeDelta::Zero()); + clock.AdvanceTimeMilliseconds(5000); + timestamp += 5 * 90000; + timing.UpdateCurrentDelay(timestamp); +} + +TEST(ReceiverTimingTest, TimestampWrapAround) { + constexpr auto kStartTime = Timestamp::Millis(1337); + test::ScopedKeyValueConfig field_trials; + SimulatedClock clock(kStartTime); + VCMTiming timing(&clock, field_trials); + + // Provoke a wrap-around. The fifth frame will have wrapped at 25 fps. + constexpr uint32_t kRtpTicksPerFrame = k90kHz / k25Fps; + uint32_t timestamp = 0xFFFFFFFFu - 3 * kRtpTicksPerFrame; + for (int i = 0; i < 5; ++i) { + timing.IncomingTimestamp(timestamp, clock.CurrentTime()); + clock.AdvanceTime(1 / k25Fps); + timestamp += kRtpTicksPerFrame; + EXPECT_EQ(kStartTime + 3 / k25Fps, + timing.RenderTime(0xFFFFFFFFu, clock.CurrentTime())); + // One ms later in 90 kHz. + EXPECT_EQ(kStartTime + 3 / k25Fps + TimeDelta::Millis(1), + timing.RenderTime(89u, clock.CurrentTime())); + } +} + +TEST(ReceiverTimingTest, UseLowLatencyRenderer) { + test::ScopedKeyValueConfig field_trials; + SimulatedClock clock(0); + VCMTiming timing(&clock, field_trials); + timing.Reset(); + // Default is false. + EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering); + // False if min playout delay > 0. + timing.set_min_playout_delay(TimeDelta::Millis(10)); + timing.set_max_playout_delay(TimeDelta::Millis(20)); + EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering); + // True if min==0, max > 0. + timing.set_min_playout_delay(TimeDelta::Zero()); + EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering); + // True if min==max==0. + timing.set_max_playout_delay(TimeDelta::Zero()); + EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering); + // True also for max playout delay==500 ms. + timing.set_max_playout_delay(TimeDelta::Millis(500)); + EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering); + // False if max playout delay > 500 ms. + timing.set_max_playout_delay(TimeDelta::Millis(501)); + EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering); +} + +TEST(ReceiverTimingTest, MaxWaitingTimeIsZeroForZeroRenderTime) { + // This is the default path when the RTP playout delay header extension is set + // to min==0 and max==0. + constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. + constexpr TimeDelta kTimeDelta = 1 / Frequency::Hertz(60); + constexpr Timestamp kZeroRenderTime = Timestamp::Zero(); + SimulatedClock clock(kStartTimeUs); + test::ScopedKeyValueConfig field_trials; + VCMTiming timing(&clock, field_trials); + timing.Reset(); + timing.set_max_playout_delay(TimeDelta::Zero()); + for (int i = 0; i < 10; ++i) { + clock.AdvanceTime(kTimeDelta); + Timestamp now = clock.CurrentTime(); + EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + } + // Another frame submitted at the same time also returns a negative max + // waiting time. + Timestamp now = clock.CurrentTime(); + EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + // MaxWaitingTime should be less than zero even if there's a burst of frames. + EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); +} + +TEST(ReceiverTimingTest, MaxWaitingTimeZeroDelayPacingExperiment) { + // The minimum pacing is enabled by a field trial and active if the RTP + // playout delay header extension is set to min==0. + constexpr TimeDelta kMinPacing = TimeDelta::Millis(3); + test::ScopedKeyValueConfig field_trials( + "WebRTC-ZeroPlayoutDelay/min_pacing:3ms/"); + constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. + constexpr TimeDelta kTimeDelta = 1 / Frequency::Hertz(60); + constexpr auto kZeroRenderTime = Timestamp::Zero(); + SimulatedClock clock(kStartTimeUs); + VCMTiming timing(&clock, field_trials); + timing.Reset(); + // MaxWaitingTime() returns zero for evenly spaced video frames. + for (int i = 0; i < 10; ++i) { + clock.AdvanceTime(kTimeDelta); + Timestamp now = clock.CurrentTime(); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + timing.SetLastDecodeScheduledTimestamp(now); + } + // Another frame submitted at the same time is paced according to the field + // trial setting. + auto now = clock.CurrentTime(); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + kMinPacing); + // If there's a burst of frames, the wait time is calculated based on next + // decode time. + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + kMinPacing); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + kMinPacing); + // Allow a few ms to pass, this should be subtracted from the MaxWaitingTime. + constexpr TimeDelta kTwoMs = TimeDelta::Millis(2); + clock.AdvanceTime(kTwoMs); + now = clock.CurrentTime(); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + kMinPacing - kTwoMs); + // A frame is decoded at the current time, the wait time should be restored to + // pacing delay. + timing.SetLastDecodeScheduledTimestamp(now); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + kMinPacing); +} + +TEST(ReceiverTimingTest, DefaultMaxWaitingTimeUnaffectedByPacingExperiment) { + // The minimum pacing is enabled by a field trial but should not have any + // effect if render_time_ms is greater than 0; + test::ScopedKeyValueConfig field_trials( + "WebRTC-ZeroPlayoutDelay/min_pacing:3ms/"); + constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. + const TimeDelta kTimeDelta = TimeDelta::Millis(1000.0 / 60.0); + SimulatedClock clock(kStartTimeUs); + VCMTiming timing(&clock, field_trials); + timing.Reset(); + clock.AdvanceTime(kTimeDelta); + auto now = clock.CurrentTime(); + Timestamp render_time = now + TimeDelta::Millis(30); + // Estimate the internal processing delay from the first frame. + TimeDelta estimated_processing_delay = + (render_time - now) - + timing.MaxWaitingTime(render_time, now, + /*too_many_frames_queued=*/false); + EXPECT_GT(estimated_processing_delay, TimeDelta::Zero()); + + // Any other frame submitted at the same time should be scheduled according to + // its render time. + for (int i = 0; i < 5; ++i) { + render_time += kTimeDelta; + EXPECT_EQ(timing.MaxWaitingTime(render_time, now, + /*too_many_frames_queued=*/false), + render_time - now - estimated_processing_delay); + } +} + +TEST(ReceiverTimingTest, MaxWaitingTimeReturnsZeroIfTooManyFramesQueuedIsTrue) { + // The minimum pacing is enabled by a field trial and active if the RTP + // playout delay header extension is set to min==0. + constexpr TimeDelta kMinPacing = TimeDelta::Millis(3); + test::ScopedKeyValueConfig field_trials( + "WebRTC-ZeroPlayoutDelay/min_pacing:3ms/"); + constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us. + const TimeDelta kTimeDelta = TimeDelta::Millis(1000.0 / 60.0); + constexpr auto kZeroRenderTime = Timestamp::Zero(); + SimulatedClock clock(kStartTimeUs); + VCMTiming timing(&clock, field_trials); + timing.Reset(); + // MaxWaitingTime() returns zero for evenly spaced video frames. + for (int i = 0; i < 10; ++i) { + clock.AdvanceTime(kTimeDelta); + auto now = clock.CurrentTime(); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now, + /*too_many_frames_queued=*/false), + TimeDelta::Zero()); + timing.SetLastDecodeScheduledTimestamp(now); + } + // Another frame submitted at the same time is paced according to the field + // trial setting. + auto now_ms = clock.CurrentTime(); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now_ms, + /*too_many_frames_queued=*/false), + kMinPacing); + // MaxWaitingTime returns 0 even if there's a burst of frames if + // too_many_frames_queued is set to true. + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now_ms, + /*too_many_frames_queued=*/true), + TimeDelta::Zero()); + EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now_ms, + /*too_many_frames_queued=*/true), + TimeDelta::Zero()); +} + +TEST(ReceiverTimingTest, UpdateCurrentDelayCapsWhenOffByMicroseconds) { + test::ScopedKeyValueConfig field_trials; + SimulatedClock clock(0); + VCMTiming timing(&clock, field_trials); + timing.Reset(); + + // Set larger initial current delay. + timing.set_min_playout_delay(TimeDelta::Millis(200)); + timing.UpdateCurrentDelay(Timestamp::Millis(900), Timestamp::Millis(1000)); + + // Add a few microseconds to ensure that the delta of decode time is 0 after + // rounding, and should reset to the target delay. + timing.set_min_playout_delay(TimeDelta::Millis(50)); + Timestamp decode_time = Timestamp::Millis(1337); + Timestamp render_time = + decode_time + TimeDelta::Millis(10) + TimeDelta::Micros(37); + timing.UpdateCurrentDelay(render_time, decode_time); + EXPECT_EQ(timing.GetTimings().current_delay, timing.TargetVideoDelay()); +} + +} // namespace webrtc |