summaryrefslogtreecommitdiffstats
path: root/third_party/libwebrtc/modules/video_coding/timing
diff options
context:
space:
mode:
Diffstat (limited to 'third_party/libwebrtc/modules/video_coding/timing')
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/BUILD.gn153
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/codec_timer.cc58
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/codec_timer.h50
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/codec_timer_gn/moz.build221
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter.cc148
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter.h106
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter_gn/moz.build221
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter_unittest.cc115
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.cc71
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.h46
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_gn/moz.build221
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_unittest.cc190
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.cc476
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.h218
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_gn/moz.build232
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_unittest.cc305
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/rtt_filter.cc161
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/rtt_filter.h69
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/rtt_filter_gn/moz.build221
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/rtt_filter_unittest.cc105
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator.cc169
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator.h48
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator_gn/moz.build221
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator_unittest.cc221
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/timing.cc297
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/timing.h160
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/timing_module_gn/moz.build232
-rw-r--r--third_party/libwebrtc/modules/video_coding/timing/timing_unittest.cc339
28 files changed, 5074 insertions, 0 deletions
diff --git a/third_party/libwebrtc/modules/video_coding/timing/BUILD.gn b/third_party/libwebrtc/modules/video_coding/timing/BUILD.gn
new file mode 100644
index 0000000000..38348e6967
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/BUILD.gn
@@ -0,0 +1,153 @@
+# Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import("../../../webrtc.gni")
+
+rtc_library("codec_timer") {
+ sources = [
+ "codec_timer.cc",
+ "codec_timer.h",
+ ]
+ deps = [ "../../../rtc_base:rtc_numerics" ]
+}
+
+rtc_library("inter_frame_delay") {
+ sources = [
+ "inter_frame_delay.cc",
+ "inter_frame_delay.h",
+ ]
+ deps = [
+ "../..:module_api_public",
+ "../../../api/units:frequency",
+ "../../../api/units:time_delta",
+ "../../../api/units:timestamp",
+ "../../../rtc_base:rtc_numerics",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+}
+
+rtc_library("frame_delay_variation_kalman_filter") {
+ sources = [
+ "frame_delay_variation_kalman_filter.cc",
+ "frame_delay_variation_kalman_filter.h",
+ ]
+ deps = [
+ "../../../api/units:data_size",
+ "../../../api/units:time_delta",
+ ]
+ visibility = [
+ ":jitter_estimator",
+ ":timing_unittests",
+ ]
+}
+
+rtc_library("jitter_estimator") {
+ sources = [
+ "jitter_estimator.cc",
+ "jitter_estimator.h",
+ ]
+ deps = [
+ ":frame_delay_variation_kalman_filter",
+ ":rtt_filter",
+ "../../../api:field_trials_view",
+ "../../../api/units:data_size",
+ "../../../api/units:frequency",
+ "../../../api/units:time_delta",
+ "../../../api/units:timestamp",
+ "../../../rtc_base:checks",
+ "../../../rtc_base:logging",
+ "../../../rtc_base:rolling_accumulator",
+ "../../../rtc_base:rtc_numerics",
+ "../../../rtc_base:safe_conversions",
+ "../../../rtc_base/experiments:field_trial_parser",
+ "../../../system_wrappers",
+ ]
+ absl_deps = [
+ "//third_party/abseil-cpp/absl/strings",
+ "//third_party/abseil-cpp/absl/types:optional",
+ ]
+}
+
+rtc_library("rtt_filter") {
+ sources = [
+ "rtt_filter.cc",
+ "rtt_filter.h",
+ ]
+ deps = [ "../../../api/units:time_delta" ]
+ absl_deps = [
+ "//third_party/abseil-cpp/absl/algorithm:container",
+ "//third_party/abseil-cpp/absl/container:inlined_vector",
+ ]
+}
+
+rtc_library("timestamp_extrapolator") {
+ sources = [
+ "timestamp_extrapolator.cc",
+ "timestamp_extrapolator.h",
+ ]
+ deps = [
+ "../../../api/units:timestamp",
+ "../../../modules:module_api_public",
+ "../../../rtc_base:rtc_numerics",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+}
+
+rtc_library("timing_module") {
+ sources = [
+ "timing.cc",
+ "timing.h",
+ ]
+ deps = [
+ ":codec_timer",
+ ":timestamp_extrapolator",
+ "../../../api:field_trials_view",
+ "../../../api/units:time_delta",
+ "../../../api/video:video_frame",
+ "../../../api/video:video_rtp_headers",
+ "../../../rtc_base:logging",
+ "../../../rtc_base:macromagic",
+ "../../../rtc_base:rtc_numerics",
+ "../../../rtc_base/experiments:field_trial_parser",
+ "../../../rtc_base/synchronization:mutex",
+ "../../../system_wrappers",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+}
+
+rtc_library("timing_unittests") {
+ testonly = true
+ sources = [
+ "frame_delay_variation_kalman_filter_unittest.cc",
+ "inter_frame_delay_unittest.cc",
+ "jitter_estimator_unittest.cc",
+ "rtt_filter_unittest.cc",
+ "timestamp_extrapolator_unittest.cc",
+ "timing_unittest.cc",
+ ]
+ deps = [
+ ":frame_delay_variation_kalman_filter",
+ ":inter_frame_delay",
+ ":jitter_estimator",
+ ":rtt_filter",
+ ":timestamp_extrapolator",
+ ":timing_module",
+ "../../../api:array_view",
+ "../../../api:field_trials",
+ "../../../api/units:data_size",
+ "../../../api/units:frequency",
+ "../../../api/units:time_delta",
+ "../../../api/units:timestamp",
+ "../../../rtc_base:histogram_percentile_counter",
+ "../../../rtc_base:timeutils",
+ "../../../system_wrappers:system_wrappers",
+ "../../../test:scoped_key_value_config",
+ "../../../test:test_support",
+ ]
+ absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+}
diff --git a/third_party/libwebrtc/modules/video_coding/timing/codec_timer.cc b/third_party/libwebrtc/modules/video_coding/timing/codec_timer.cc
new file mode 100644
index 0000000000..f57d42d40a
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/codec_timer.cc
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/timing/codec_timer.h"
+
+#include <cstdint>
+
+namespace webrtc {
+
+namespace {
+
+// The first kIgnoredSampleCount samples will be ignored.
+const int kIgnoredSampleCount = 5;
+// Return the `kPercentile` value in RequiredDecodeTimeMs().
+const float kPercentile = 0.95f;
+// The window size in ms.
+const int64_t kTimeLimitMs = 10000;
+
+} // anonymous namespace
+
+CodecTimer::CodecTimer() : ignored_sample_count_(0), filter_(kPercentile) {}
+CodecTimer::~CodecTimer() = default;
+
+void CodecTimer::AddTiming(int64_t decode_time_ms, int64_t now_ms) {
+ // Ignore the first `kIgnoredSampleCount` samples.
+ if (ignored_sample_count_ < kIgnoredSampleCount) {
+ ++ignored_sample_count_;
+ return;
+ }
+
+ // Insert new decode time value.
+ filter_.Insert(decode_time_ms);
+ history_.emplace(decode_time_ms, now_ms);
+
+ // Pop old decode time values.
+ while (!history_.empty() &&
+ now_ms - history_.front().sample_time_ms > kTimeLimitMs) {
+ filter_.Erase(history_.front().decode_time_ms);
+ history_.pop();
+ }
+}
+
+// Get the 95th percentile observed decode time within a time window.
+int64_t CodecTimer::RequiredDecodeTimeMs() const {
+ return filter_.GetPercentileValue();
+}
+
+CodecTimer::Sample::Sample(int64_t decode_time_ms, int64_t sample_time_ms)
+ : decode_time_ms(decode_time_ms), sample_time_ms(sample_time_ms) {}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/timing/codec_timer.h b/third_party/libwebrtc/modules/video_coding/timing/codec_timer.h
new file mode 100644
index 0000000000..9f12d82e98
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/codec_timer.h
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_TIMING_CODEC_TIMER_H_
+#define MODULES_VIDEO_CODING_TIMING_CODEC_TIMER_H_
+
+#include <queue>
+
+#include "rtc_base/numerics/percentile_filter.h"
+
+namespace webrtc {
+
+class CodecTimer {
+ public:
+ CodecTimer();
+ ~CodecTimer();
+
+ // Add a new decode time to the filter.
+ void AddTiming(int64_t new_decode_time_ms, int64_t now_ms);
+
+ // Get the required decode time in ms. It is the 95th percentile observed
+ // decode time within a time window.
+ int64_t RequiredDecodeTimeMs() const;
+
+ private:
+ struct Sample {
+ Sample(int64_t decode_time_ms, int64_t sample_time_ms);
+ int64_t decode_time_ms;
+ int64_t sample_time_ms;
+ };
+
+ // The number of samples ignored so far.
+ int ignored_sample_count_;
+ // Queue with history of latest decode time values.
+ std::queue<Sample> history_;
+ // `filter_` contains the same values as `history_`, but in a data structure
+ // that allows efficient retrieval of the percentile value.
+ PercentileFilter<int64_t> filter_;
+};
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_TIMING_CODEC_TIMER_H_
diff --git a/third_party/libwebrtc/modules/video_coding/timing/codec_timer_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/codec_timer_gn/moz.build
new file mode 100644
index 0000000000..59498feb91
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/codec_timer_gn/moz.build
@@ -0,0 +1,221 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+ ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ###
+ ### DO NOT edit it by hand. ###
+
+COMPILE_FLAGS["OS_INCLUDES"] = []
+AllowCompilerWarnings()
+
+DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1"
+DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True
+DEFINES["RTC_ENABLE_VP9"] = True
+DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0"
+DEFINES["WEBRTC_LIBRARY_IMPL"] = True
+DEFINES["WEBRTC_MOZILLA_BUILD"] = True
+DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0"
+DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0"
+
+FINAL_LIBRARY = "webrtc"
+
+
+LOCAL_INCLUDES += [
+ "!/ipc/ipdl/_ipdlheaders",
+ "!/third_party/libwebrtc/gen",
+ "/ipc/chromium/src",
+ "/third_party/libwebrtc/",
+ "/third_party/libwebrtc/third_party/abseil-cpp/",
+ "/tools/profiler/public"
+]
+
+UNIFIED_SOURCES += [
+ "/third_party/libwebrtc/modules/video_coding/timing/codec_timer.cc"
+]
+
+if not CONFIG["MOZ_DEBUG"]:
+
+ DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0"
+ DEFINES["NDEBUG"] = True
+ DEFINES["NVALGRIND"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1":
+
+ DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1"
+
+if CONFIG["OS_TARGET"] == "Android":
+
+ DEFINES["ANDROID"] = True
+ DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1"
+ DEFINES["HAVE_SYS_UIO_H"] = True
+ DEFINES["WEBRTC_ANDROID"] = True
+ DEFINES["WEBRTC_ANDROID_OPENSLES"] = True
+ DEFINES["WEBRTC_LINUX"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_GNU_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+ OS_LIBS += [
+ "log"
+ ]
+
+if CONFIG["OS_TARGET"] == "Darwin":
+
+ DEFINES["WEBRTC_MAC"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True
+ DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0"
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["USE_AURA"] = "1"
+ DEFINES["USE_GLIB"] = "1"
+ DEFINES["USE_NSS_CERTS"] = "1"
+ DEFINES["USE_OZONE"] = "1"
+ DEFINES["USE_UDEV"] = True
+ DEFINES["WEBRTC_LINUX"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_FILE_OFFSET_BITS"] = "64"
+ DEFINES["_LARGEFILE64_SOURCE"] = True
+ DEFINES["_LARGEFILE_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "OpenBSD":
+
+ DEFINES["USE_GLIB"] = "1"
+ DEFINES["USE_OZONE"] = "1"
+ DEFINES["USE_X11"] = "1"
+ DEFINES["WEBRTC_BSD"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_FILE_OFFSET_BITS"] = "64"
+ DEFINES["_LARGEFILE64_SOURCE"] = True
+ DEFINES["_LARGEFILE_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "WINNT":
+
+ DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True
+ DEFINES["NOMINMAX"] = True
+ DEFINES["NTDDI_VERSION"] = "0x0A000000"
+ DEFINES["PSAPI_VERSION"] = "2"
+ DEFINES["UNICODE"] = True
+ DEFINES["USE_AURA"] = "1"
+ DEFINES["WEBRTC_WIN"] = True
+ DEFINES["WIN32"] = True
+ DEFINES["WIN32_LEAN_AND_MEAN"] = True
+ DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP"
+ DEFINES["WINVER"] = "0x0A00"
+ DEFINES["_ATL_NO_OPENGL"] = True
+ DEFINES["_CRT_RAND_S"] = True
+ DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True
+ DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True
+ DEFINES["_HAS_EXCEPTIONS"] = "0"
+ DEFINES["_HAS_NODISCARD"] = True
+ DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True
+ DEFINES["_SECURE_ATL"] = True
+ DEFINES["_UNICODE"] = True
+ DEFINES["_WIN32_WINNT"] = "0x0A00"
+ DEFINES["_WINDOWS"] = True
+ DEFINES["__STD_C"] = True
+
+if CONFIG["CPU_ARCH"] == "aarch64":
+
+ DEFINES["WEBRTC_ARCH_ARM64"] = True
+ DEFINES["WEBRTC_HAS_NEON"] = True
+
+if CONFIG["CPU_ARCH"] == "arm":
+
+ CXXFLAGS += [
+ "-mfpu=neon"
+ ]
+
+ DEFINES["WEBRTC_ARCH_ARM"] = True
+ DEFINES["WEBRTC_ARCH_ARM_V7"] = True
+ DEFINES["WEBRTC_HAS_NEON"] = True
+
+if CONFIG["CPU_ARCH"] == "mips32":
+
+ DEFINES["MIPS32_LE"] = True
+ DEFINES["MIPS_FPU_LE"] = True
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "mips64":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86":
+
+ DEFINES["WEBRTC_ENABLE_AVX2"] = True
+
+if CONFIG["CPU_ARCH"] == "x86_64":
+
+ DEFINES["WEBRTC_ENABLE_AVX2"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT":
+
+ DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0"
+
+if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["USE_X11"] = "1"
+
+if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android":
+
+ OS_LIBS += [
+ "android_support",
+ "unwind"
+ ]
+
+if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android":
+
+ CXXFLAGS += [
+ "-msse2"
+ ]
+
+ OS_LIBS += [
+ "android_support"
+ ]
+
+if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux":
+
+ CXXFLAGS += [
+ "-msse2"
+ ]
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+Library("codec_timer_gn")
diff --git a/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter.cc b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter.cc
new file mode 100644
index 0000000000..ec6aa3445a
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter.cc
@@ -0,0 +1,148 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/timing/frame_delay_variation_kalman_filter.h"
+
+#include "api/units/data_size.h"
+#include "api/units/time_delta.h"
+
+namespace webrtc {
+
+namespace {
+// TODO(brandtr): The value below corresponds to 8 Gbps. Is that reasonable?
+constexpr double kMaxBandwidth = 0.000001; // Unit: [1 / bytes per ms].
+} // namespace
+
+FrameDelayVariationKalmanFilter::FrameDelayVariationKalmanFilter() {
+ // TODO(brandtr): Is there a factor 1000 missing here?
+ estimate_[0] = 1 / (512e3 / 8); // Unit: [1 / bytes per ms]
+ estimate_[1] = 0; // Unit: [ms]
+
+ // Initial estimate covariance.
+ estimate_cov_[0][0] = 1e-4; // Unit: [(1 / bytes per ms)^2]
+ estimate_cov_[1][1] = 1e2; // Unit: [ms^2]
+ estimate_cov_[0][1] = estimate_cov_[1][0] = 0;
+
+ // Process noise covariance.
+ process_noise_cov_diag_[0] = 2.5e-10; // Unit: [(1 / bytes per ms)^2]
+ process_noise_cov_diag_[1] = 1e-10; // Unit: [ms^2]
+}
+
+void FrameDelayVariationKalmanFilter::PredictAndUpdate(
+ double frame_delay_variation_ms,
+ double frame_size_variation_bytes,
+ double max_frame_size_bytes,
+ double var_noise) {
+ // Sanity checks.
+ if (max_frame_size_bytes < 1) {
+ return;
+ }
+ if (var_noise <= 0.0) {
+ return;
+ }
+
+ // This member function follows the data flow in
+ // https://en.wikipedia.org/wiki/Kalman_filter#Details.
+
+ // 1) Estimate prediction: `x = F*x`.
+ // For this model, there is no need to explicitly predict the estimate, since
+ // the state transition matrix is the identity.
+
+ // 2) Estimate covariance prediction: `P = F*P*F' + Q`.
+ // Again, since the state transition matrix is the identity, this update
+ // is performed by simply adding the process noise covariance.
+ estimate_cov_[0][0] += process_noise_cov_diag_[0];
+ estimate_cov_[1][1] += process_noise_cov_diag_[1];
+
+ // 3) Innovation: `y = z - H*x`.
+ // This is the part of the measurement that cannot be explained by the current
+ // estimate.
+ double innovation =
+ frame_delay_variation_ms -
+ GetFrameDelayVariationEstimateTotal(frame_size_variation_bytes);
+
+ // 4) Innovation variance: `s = H*P*H' + r`.
+ double estim_cov_times_obs[2];
+ estim_cov_times_obs[0] =
+ estimate_cov_[0][0] * frame_size_variation_bytes + estimate_cov_[0][1];
+ estim_cov_times_obs[1] =
+ estimate_cov_[1][0] * frame_size_variation_bytes + estimate_cov_[1][1];
+ double observation_noise_stddev =
+ (300.0 * exp(-fabs(frame_size_variation_bytes) /
+ (1e0 * max_frame_size_bytes)) +
+ 1) *
+ sqrt(var_noise);
+ if (observation_noise_stddev < 1.0) {
+ observation_noise_stddev = 1.0;
+ }
+ // TODO(brandtr): Shouldn't we add observation_noise_stddev^2 here? Otherwise,
+ // the dimensional analysis fails.
+ double innovation_var = frame_size_variation_bytes * estim_cov_times_obs[0] +
+ estim_cov_times_obs[1] + observation_noise_stddev;
+ if ((innovation_var < 1e-9 && innovation_var >= 0) ||
+ (innovation_var > -1e-9 && innovation_var <= 0)) {
+ RTC_DCHECK_NOTREACHED();
+ return;
+ }
+
+ // 5) Optimal Kalman gain: `K = P*H'/s`.
+ // How much to trust the model vs. how much to trust the measurement.
+ double kalman_gain[2];
+ kalman_gain[0] = estim_cov_times_obs[0] / innovation_var;
+ kalman_gain[1] = estim_cov_times_obs[1] / innovation_var;
+
+ // 6) Estimate update: `x = x + K*y`.
+ // Optimally weight the new information in the innovation and add it to the
+ // old estimate.
+ estimate_[0] += kalman_gain[0] * innovation;
+ estimate_[1] += kalman_gain[1] * innovation;
+
+ // (This clamping is not part of the linear Kalman filter.)
+ if (estimate_[0] < kMaxBandwidth) {
+ estimate_[0] = kMaxBandwidth;
+ }
+
+ // 7) Estimate covariance update: `P = (I - K*H)*P`
+ double t00 = estimate_cov_[0][0];
+ double t01 = estimate_cov_[0][1];
+ estimate_cov_[0][0] =
+ (1 - kalman_gain[0] * frame_size_variation_bytes) * t00 -
+ kalman_gain[0] * estimate_cov_[1][0];
+ estimate_cov_[0][1] =
+ (1 - kalman_gain[0] * frame_size_variation_bytes) * t01 -
+ kalman_gain[0] * estimate_cov_[1][1];
+ estimate_cov_[1][0] = estimate_cov_[1][0] * (1 - kalman_gain[1]) -
+ kalman_gain[1] * frame_size_variation_bytes * t00;
+ estimate_cov_[1][1] = estimate_cov_[1][1] * (1 - kalman_gain[1]) -
+ kalman_gain[1] * frame_size_variation_bytes * t01;
+
+ // Covariance matrix, must be positive semi-definite.
+ RTC_DCHECK(estimate_cov_[0][0] + estimate_cov_[1][1] >= 0 &&
+ estimate_cov_[0][0] * estimate_cov_[1][1] -
+ estimate_cov_[0][1] * estimate_cov_[1][0] >=
+ 0 &&
+ estimate_cov_[0][0] >= 0);
+}
+
+double FrameDelayVariationKalmanFilter::GetFrameDelayVariationEstimateSizeBased(
+ double frame_size_variation_bytes) const {
+ // Unit: [1 / bytes per millisecond] * [bytes] = [milliseconds].
+ return estimate_[0] * frame_size_variation_bytes;
+}
+
+double FrameDelayVariationKalmanFilter::GetFrameDelayVariationEstimateTotal(
+ double frame_size_variation_bytes) const {
+ double frame_transmission_delay_ms =
+ GetFrameDelayVariationEstimateSizeBased(frame_size_variation_bytes);
+ double link_queuing_delay_ms = estimate_[1];
+ return frame_transmission_delay_ms + link_queuing_delay_ms;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter.h b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter.h
new file mode 100644
index 0000000000..a65ceefa10
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter.h
@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_TIMING_FRAME_DELAY_VARIATION_KALMAN_FILTER_H_
+#define MODULES_VIDEO_CODING_TIMING_FRAME_DELAY_VARIATION_KALMAN_FILTER_H_
+
+#include "api/units/data_size.h"
+#include "api/units/time_delta.h"
+
+namespace webrtc {
+
+// This class uses a linear Kalman filter (see
+// https://en.wikipedia.org/wiki/Kalman_filter) to estimate the frame delay
+// variation (i.e., the difference in transmission time between a frame and the
+// prior frame) for a frame, given its size variation in bytes (i.e., the
+// difference in size between a frame and the prior frame). The idea is that,
+// given a fixed link bandwidth, a larger frame (in bytes) would take
+// proportionally longer to arrive than a correspondingly smaller frame. Using
+// the variations of frame delay and frame size, the underlying bandwidth and
+// queuing delay variation of the network link can be estimated.
+//
+// The filter takes as input the frame delay variation, the difference between
+// the actual inter-frame arrival time and the expected inter-frame arrival time
+// (based on RTP timestamp), and frame size variation, the inter-frame size
+// delta for a single frame. The frame delay variation is seen as the
+// measurement and the frame size variation is used in the observation model.
+// The hidden state of the filter is the link bandwidth and queuing delay
+// buildup. The estimated state can be used to get the expected frame delay
+// variation for a frame, given its frame size variation. This information can
+// then be used to estimate the frame delay variation coming from network
+// jitter.
+//
+// Mathematical details:
+// * The state (`x` in Wikipedia notation) is a 2x1 vector comprising the
+// reciprocal of link bandwidth [1 / bytes per ms] and the
+// link queuing delay buildup [ms].
+// * The state transition matrix (`F`) is the 2x2 identity matrix, meaning that
+// link bandwidth and link queuing delay buildup are modeled as independent.
+// * The measurement (`z`) is the (scalar) frame delay variation [ms].
+// * The observation matrix (`H`) is a 1x2 vector set as
+// `{frame_size_variation [bytes], 1.0}`.
+// * The state estimate covariance (`P`) is a symmetric 2x2 matrix.
+// * The process noise covariance (`Q`) is a constant 2x2 diagonal matrix
+// [(1 / bytes per ms)^2, ms^2].
+// * The observation noise covariance (`r`) is a scalar [ms^2] that is
+// determined externally to this class.
+class FrameDelayVariationKalmanFilter {
+ public:
+ FrameDelayVariationKalmanFilter();
+ ~FrameDelayVariationKalmanFilter() = default;
+
+ // Predicts and updates the filter, given a new pair of frame delay variation
+ // and frame size variation.
+ //
+ // Inputs:
+ // `frame_delay_variation_ms`:
+ // Frame delay variation as calculated by the `InterFrameDelay` estimator.
+ //
+ // `frame_size_variation_bytes`:
+ // Frame size variation, i.e., the current frame size minus the previous
+ // frame size (in bytes). Note that this quantity may be negative.
+ //
+ // `max_frame_size_bytes`:
+ // Filtered largest frame size received since the last reset.
+ //
+ // `var_noise`:
+ // Variance of the estimated random jitter.
+ //
+ // TODO(bugs.webrtc.org/14381): For now use doubles as input parameters as
+ // units defined in api/units have insufficient underlying precision for
+ // jitter estimation.
+ void PredictAndUpdate(double frame_delay_variation_ms,
+ double frame_size_variation_bytes,
+ double max_frame_size_bytes,
+ double var_noise);
+
+ // Given a frame size variation, returns the estimated frame delay variation
+ // explained by the link bandwidth alone.
+ double GetFrameDelayVariationEstimateSizeBased(
+ double frame_size_variation_bytes) const;
+
+ // Given a frame size variation, returns the estimated frame delay variation
+ // explained by both link bandwidth and link queuing delay buildup.
+ double GetFrameDelayVariationEstimateTotal(
+ double frame_size_variation_bytes) const;
+
+ private:
+ // State estimate (bandwidth [1 / bytes per ms], queue buildup [ms]).
+ double estimate_[2];
+ double estimate_cov_[2][2]; // Estimate covariance.
+
+ // Process noise covariance. This is a diagonal matrix, so we only store the
+ // diagonal entries.
+ double process_noise_cov_diag_[2];
+};
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_TIMING_FRAME_DELAY_VARIATION_KALMAN_FILTER_H_
diff --git a/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter_gn/moz.build
new file mode 100644
index 0000000000..ecdcfc93e1
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter_gn/moz.build
@@ -0,0 +1,221 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+ ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ###
+ ### DO NOT edit it by hand. ###
+
+COMPILE_FLAGS["OS_INCLUDES"] = []
+AllowCompilerWarnings()
+
+DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1"
+DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True
+DEFINES["RTC_ENABLE_VP9"] = True
+DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0"
+DEFINES["WEBRTC_LIBRARY_IMPL"] = True
+DEFINES["WEBRTC_MOZILLA_BUILD"] = True
+DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0"
+DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0"
+
+FINAL_LIBRARY = "webrtc"
+
+
+LOCAL_INCLUDES += [
+ "!/ipc/ipdl/_ipdlheaders",
+ "!/third_party/libwebrtc/gen",
+ "/ipc/chromium/src",
+ "/third_party/libwebrtc/",
+ "/third_party/libwebrtc/third_party/abseil-cpp/",
+ "/tools/profiler/public"
+]
+
+UNIFIED_SOURCES += [
+ "/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter.cc"
+]
+
+if not CONFIG["MOZ_DEBUG"]:
+
+ DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0"
+ DEFINES["NDEBUG"] = True
+ DEFINES["NVALGRIND"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1":
+
+ DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1"
+
+if CONFIG["OS_TARGET"] == "Android":
+
+ DEFINES["ANDROID"] = True
+ DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1"
+ DEFINES["HAVE_SYS_UIO_H"] = True
+ DEFINES["WEBRTC_ANDROID"] = True
+ DEFINES["WEBRTC_ANDROID_OPENSLES"] = True
+ DEFINES["WEBRTC_LINUX"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_GNU_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+ OS_LIBS += [
+ "log"
+ ]
+
+if CONFIG["OS_TARGET"] == "Darwin":
+
+ DEFINES["WEBRTC_MAC"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True
+ DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0"
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["USE_AURA"] = "1"
+ DEFINES["USE_GLIB"] = "1"
+ DEFINES["USE_NSS_CERTS"] = "1"
+ DEFINES["USE_OZONE"] = "1"
+ DEFINES["USE_UDEV"] = True
+ DEFINES["WEBRTC_LINUX"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_FILE_OFFSET_BITS"] = "64"
+ DEFINES["_LARGEFILE64_SOURCE"] = True
+ DEFINES["_LARGEFILE_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "OpenBSD":
+
+ DEFINES["USE_GLIB"] = "1"
+ DEFINES["USE_OZONE"] = "1"
+ DEFINES["USE_X11"] = "1"
+ DEFINES["WEBRTC_BSD"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_FILE_OFFSET_BITS"] = "64"
+ DEFINES["_LARGEFILE64_SOURCE"] = True
+ DEFINES["_LARGEFILE_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "WINNT":
+
+ DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True
+ DEFINES["NOMINMAX"] = True
+ DEFINES["NTDDI_VERSION"] = "0x0A000000"
+ DEFINES["PSAPI_VERSION"] = "2"
+ DEFINES["UNICODE"] = True
+ DEFINES["USE_AURA"] = "1"
+ DEFINES["WEBRTC_WIN"] = True
+ DEFINES["WIN32"] = True
+ DEFINES["WIN32_LEAN_AND_MEAN"] = True
+ DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP"
+ DEFINES["WINVER"] = "0x0A00"
+ DEFINES["_ATL_NO_OPENGL"] = True
+ DEFINES["_CRT_RAND_S"] = True
+ DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True
+ DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True
+ DEFINES["_HAS_EXCEPTIONS"] = "0"
+ DEFINES["_HAS_NODISCARD"] = True
+ DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True
+ DEFINES["_SECURE_ATL"] = True
+ DEFINES["_UNICODE"] = True
+ DEFINES["_WIN32_WINNT"] = "0x0A00"
+ DEFINES["_WINDOWS"] = True
+ DEFINES["__STD_C"] = True
+
+if CONFIG["CPU_ARCH"] == "aarch64":
+
+ DEFINES["WEBRTC_ARCH_ARM64"] = True
+ DEFINES["WEBRTC_HAS_NEON"] = True
+
+if CONFIG["CPU_ARCH"] == "arm":
+
+ CXXFLAGS += [
+ "-mfpu=neon"
+ ]
+
+ DEFINES["WEBRTC_ARCH_ARM"] = True
+ DEFINES["WEBRTC_ARCH_ARM_V7"] = True
+ DEFINES["WEBRTC_HAS_NEON"] = True
+
+if CONFIG["CPU_ARCH"] == "mips32":
+
+ DEFINES["MIPS32_LE"] = True
+ DEFINES["MIPS_FPU_LE"] = True
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "mips64":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86":
+
+ DEFINES["WEBRTC_ENABLE_AVX2"] = True
+
+if CONFIG["CPU_ARCH"] == "x86_64":
+
+ DEFINES["WEBRTC_ENABLE_AVX2"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT":
+
+ DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0"
+
+if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["USE_X11"] = "1"
+
+if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android":
+
+ OS_LIBS += [
+ "android_support",
+ "unwind"
+ ]
+
+if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android":
+
+ CXXFLAGS += [
+ "-msse2"
+ ]
+
+ OS_LIBS += [
+ "android_support"
+ ]
+
+if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux":
+
+ CXXFLAGS += [
+ "-msse2"
+ ]
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+Library("frame_delay_variation_kalman_filter_gn")
diff --git a/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter_unittest.cc b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter_unittest.cc
new file mode 100644
index 0000000000..6103f3a1bc
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/frame_delay_variation_kalman_filter_unittest.cc
@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/timing/frame_delay_variation_kalman_filter.h"
+
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace {
+
+// This test verifies that the initial filter state (link bandwidth, link
+// propagation delay) is such that a frame of size zero would take no time to
+// propagate.
+TEST(FrameDelayVariationKalmanFilterTest,
+ InitializedFilterWithZeroSizeFrameTakesNoTimeToPropagate) {
+ FrameDelayVariationKalmanFilter filter;
+
+ // A zero-sized frame...
+ double frame_size_variation_bytes = 0.0;
+
+ // ...should take no time to propagate due to it's size...
+ EXPECT_EQ(filter.GetFrameDelayVariationEstimateSizeBased(
+ frame_size_variation_bytes),
+ 0.0);
+
+ // ...and no time due to the initial link propagation delay being zero.
+ EXPECT_EQ(
+ filter.GetFrameDelayVariationEstimateTotal(frame_size_variation_bytes),
+ 0.0);
+}
+
+// TODO(brandtr): Look into if there is a factor 1000 missing here? It seems
+// unreasonable to have an initial link bandwidth of 512 _mega_bits per second?
+TEST(FrameDelayVariationKalmanFilterTest,
+ InitializedFilterWithSmallSizeFrameTakesFixedTimeToPropagate) {
+ FrameDelayVariationKalmanFilter filter;
+
+ // A 1000-byte frame...
+ double frame_size_variation_bytes = 1000.0;
+ // ...should take around `1000.0 / (512e3 / 8.0) = 0.015625 ms` to transmit.
+ double expected_frame_delay_variation_estimate_ms = 1000.0 / (512e3 / 8.0);
+
+ EXPECT_EQ(filter.GetFrameDelayVariationEstimateSizeBased(
+ frame_size_variation_bytes),
+ expected_frame_delay_variation_estimate_ms);
+ EXPECT_EQ(
+ filter.GetFrameDelayVariationEstimateTotal(frame_size_variation_bytes),
+ expected_frame_delay_variation_estimate_ms);
+}
+
+TEST(FrameDelayVariationKalmanFilterTest,
+ NegativeNoiseVarianceDoesNotUpdateFilter) {
+ FrameDelayVariationKalmanFilter filter;
+
+ // Negative variance...
+ double var_noise = -0.1;
+ filter.PredictAndUpdate(/*frame_delay_variation_ms=*/3,
+ /*frame_size_variation_bytes=*/200.0,
+ /*max_frame_size_bytes=*/2000, var_noise);
+
+ // ...does _not_ update the filter.
+ EXPECT_EQ(filter.GetFrameDelayVariationEstimateTotal(
+ /*frame_size_variation_bytes=*/0.0),
+ 0.0);
+
+ // Positive variance...
+ var_noise = 0.1;
+ filter.PredictAndUpdate(/*frame_delay_variation_ms=*/3,
+ /*frame_size_variation_bytes=*/200.0,
+ /*max_frame_size_bytes=*/2000, var_noise);
+
+ // ...does update the filter.
+ EXPECT_GT(filter.GetFrameDelayVariationEstimateTotal(
+ /*frame_size_variation_bytes=*/0.0),
+ 0.0);
+}
+
+TEST(FrameDelayVariationKalmanFilterTest,
+ VerifyConvergenceWithAlternatingDeviations) {
+ FrameDelayVariationKalmanFilter filter;
+
+ // One frame every 33 ms.
+ int framerate_fps = 30;
+ // Let's assume approximately 10% delay variation.
+ double frame_delay_variation_ms = 3;
+ // With a bitrate of 512 kbps, each frame will be around 2000 bytes.
+ double max_frame_size_bytes = 2000;
+ // And again, let's assume 10% size deviation.
+ double frame_size_variation_bytes = 200;
+ double var_noise = 0.1;
+ int test_duration_s = 60;
+
+ for (int i = 0; i < test_duration_s * framerate_fps; ++i) {
+ // For simplicity, assume alternating variations.
+ double sign = (i % 2 == 0) ? 1.0 : -1.0;
+ filter.PredictAndUpdate(sign * frame_delay_variation_ms,
+ sign * frame_size_variation_bytes,
+ max_frame_size_bytes, var_noise);
+ }
+
+ // Verify that the filter has converged within a margin of 0.1 ms.
+ EXPECT_NEAR(
+ filter.GetFrameDelayVariationEstimateTotal(frame_size_variation_bytes),
+ frame_delay_variation_ms, 0.1);
+}
+
+} // namespace
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.cc b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.cc
new file mode 100644
index 0000000000..bed9f875ee
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.cc
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/timing/inter_frame_delay.h"
+
+#include "absl/types/optional.h"
+#include "api/units/frequency.h"
+#include "api/units/time_delta.h"
+#include "modules/include/module_common_types_public.h"
+
+namespace webrtc {
+
+namespace {
+constexpr Frequency k90kHz = Frequency::KiloHertz(90);
+}
+
+InterFrameDelay::InterFrameDelay() {
+ Reset();
+}
+
+// Resets the delay estimate.
+void InterFrameDelay::Reset() {
+ prev_wall_clock_ = absl::nullopt;
+ prev_rtp_timestamp_unwrapped_ = 0;
+}
+
+// Calculates the delay of a frame with the given timestamp.
+// This method is called when the frame is complete.
+absl::optional<TimeDelta> InterFrameDelay::CalculateDelay(
+ uint32_t rtp_timestamp,
+ Timestamp now) {
+ int64_t rtp_timestamp_unwrapped = unwrapper_.Unwrap(rtp_timestamp);
+ if (!prev_wall_clock_) {
+ // First set of data, initialization, wait for next frame.
+ prev_wall_clock_ = now;
+ prev_rtp_timestamp_unwrapped_ = rtp_timestamp_unwrapped;
+ return TimeDelta::Zero();
+ }
+
+ // Account for reordering in jitter variance estimate in the future?
+ // Note that this also captures incomplete frames which are grabbed for
+ // decoding after a later frame has been complete, i.e. real packet losses.
+ uint32_t cropped_last = static_cast<uint32_t>(prev_rtp_timestamp_unwrapped_);
+ if (rtp_timestamp_unwrapped < prev_rtp_timestamp_unwrapped_ ||
+ !IsNewerTimestamp(rtp_timestamp, cropped_last)) {
+ return absl::nullopt;
+ }
+
+ // Compute the compensated timestamp difference.
+ int64_t d_rtp_ticks = rtp_timestamp_unwrapped - prev_rtp_timestamp_unwrapped_;
+ TimeDelta dts = d_rtp_ticks / k90kHz;
+ TimeDelta dt = now - *prev_wall_clock_;
+
+ // frameDelay is the difference of dT and dTS -- i.e. the difference of the
+ // wall clock time difference and the timestamp difference between two
+ // following frames.
+ TimeDelta delay = dt - dts;
+
+ prev_rtp_timestamp_unwrapped_ = rtp_timestamp_unwrapped;
+ prev_wall_clock_ = now;
+ return delay;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.h b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.h
new file mode 100644
index 0000000000..03b5f78cc1
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_TIMING_INTER_FRAME_DELAY_H_
+#define MODULES_VIDEO_CODING_TIMING_INTER_FRAME_DELAY_H_
+
+#include <stdint.h>
+
+#include "absl/types/optional.h"
+#include "api/units/time_delta.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/numerics/sequence_number_unwrapper.h"
+
+namespace webrtc {
+
+class InterFrameDelay {
+ public:
+ InterFrameDelay();
+
+ // Resets the estimate. Zeros are given as parameters.
+ void Reset();
+
+ // Calculates the delay of a frame with the given timestamp.
+ // This method is called when the frame is complete.
+ absl::optional<TimeDelta> CalculateDelay(uint32_t rtp_timestamp,
+ Timestamp now);
+
+ private:
+ // The previous rtp timestamp passed to the delay estimate
+ int64_t prev_rtp_timestamp_unwrapped_;
+ RtpTimestampUnwrapper unwrapper_;
+
+ // The previous wall clock timestamp used by the delay estimate
+ absl::optional<Timestamp> prev_wall_clock_;
+};
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_TIMING_INTER_FRAME_DELAY_H_
diff --git a/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_gn/moz.build
new file mode 100644
index 0000000000..765d42aade
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_gn/moz.build
@@ -0,0 +1,221 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+ ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ###
+ ### DO NOT edit it by hand. ###
+
+COMPILE_FLAGS["OS_INCLUDES"] = []
+AllowCompilerWarnings()
+
+DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1"
+DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True
+DEFINES["RTC_ENABLE_VP9"] = True
+DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0"
+DEFINES["WEBRTC_LIBRARY_IMPL"] = True
+DEFINES["WEBRTC_MOZILLA_BUILD"] = True
+DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0"
+DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0"
+
+FINAL_LIBRARY = "webrtc"
+
+
+LOCAL_INCLUDES += [
+ "!/ipc/ipdl/_ipdlheaders",
+ "!/third_party/libwebrtc/gen",
+ "/ipc/chromium/src",
+ "/third_party/libwebrtc/",
+ "/third_party/libwebrtc/third_party/abseil-cpp/",
+ "/tools/profiler/public"
+]
+
+UNIFIED_SOURCES += [
+ "/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay.cc"
+]
+
+if not CONFIG["MOZ_DEBUG"]:
+
+ DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0"
+ DEFINES["NDEBUG"] = True
+ DEFINES["NVALGRIND"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1":
+
+ DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1"
+
+if CONFIG["OS_TARGET"] == "Android":
+
+ DEFINES["ANDROID"] = True
+ DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1"
+ DEFINES["HAVE_SYS_UIO_H"] = True
+ DEFINES["WEBRTC_ANDROID"] = True
+ DEFINES["WEBRTC_ANDROID_OPENSLES"] = True
+ DEFINES["WEBRTC_LINUX"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_GNU_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+ OS_LIBS += [
+ "log"
+ ]
+
+if CONFIG["OS_TARGET"] == "Darwin":
+
+ DEFINES["WEBRTC_MAC"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True
+ DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0"
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["USE_AURA"] = "1"
+ DEFINES["USE_GLIB"] = "1"
+ DEFINES["USE_NSS_CERTS"] = "1"
+ DEFINES["USE_OZONE"] = "1"
+ DEFINES["USE_UDEV"] = True
+ DEFINES["WEBRTC_LINUX"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_FILE_OFFSET_BITS"] = "64"
+ DEFINES["_LARGEFILE64_SOURCE"] = True
+ DEFINES["_LARGEFILE_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "OpenBSD":
+
+ DEFINES["USE_GLIB"] = "1"
+ DEFINES["USE_OZONE"] = "1"
+ DEFINES["USE_X11"] = "1"
+ DEFINES["WEBRTC_BSD"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_FILE_OFFSET_BITS"] = "64"
+ DEFINES["_LARGEFILE64_SOURCE"] = True
+ DEFINES["_LARGEFILE_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "WINNT":
+
+ DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True
+ DEFINES["NOMINMAX"] = True
+ DEFINES["NTDDI_VERSION"] = "0x0A000000"
+ DEFINES["PSAPI_VERSION"] = "2"
+ DEFINES["UNICODE"] = True
+ DEFINES["USE_AURA"] = "1"
+ DEFINES["WEBRTC_WIN"] = True
+ DEFINES["WIN32"] = True
+ DEFINES["WIN32_LEAN_AND_MEAN"] = True
+ DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP"
+ DEFINES["WINVER"] = "0x0A00"
+ DEFINES["_ATL_NO_OPENGL"] = True
+ DEFINES["_CRT_RAND_S"] = True
+ DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True
+ DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True
+ DEFINES["_HAS_EXCEPTIONS"] = "0"
+ DEFINES["_HAS_NODISCARD"] = True
+ DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True
+ DEFINES["_SECURE_ATL"] = True
+ DEFINES["_UNICODE"] = True
+ DEFINES["_WIN32_WINNT"] = "0x0A00"
+ DEFINES["_WINDOWS"] = True
+ DEFINES["__STD_C"] = True
+
+if CONFIG["CPU_ARCH"] == "aarch64":
+
+ DEFINES["WEBRTC_ARCH_ARM64"] = True
+ DEFINES["WEBRTC_HAS_NEON"] = True
+
+if CONFIG["CPU_ARCH"] == "arm":
+
+ CXXFLAGS += [
+ "-mfpu=neon"
+ ]
+
+ DEFINES["WEBRTC_ARCH_ARM"] = True
+ DEFINES["WEBRTC_ARCH_ARM_V7"] = True
+ DEFINES["WEBRTC_HAS_NEON"] = True
+
+if CONFIG["CPU_ARCH"] == "mips32":
+
+ DEFINES["MIPS32_LE"] = True
+ DEFINES["MIPS_FPU_LE"] = True
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "mips64":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86":
+
+ DEFINES["WEBRTC_ENABLE_AVX2"] = True
+
+if CONFIG["CPU_ARCH"] == "x86_64":
+
+ DEFINES["WEBRTC_ENABLE_AVX2"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT":
+
+ DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0"
+
+if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["USE_X11"] = "1"
+
+if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android":
+
+ OS_LIBS += [
+ "android_support",
+ "unwind"
+ ]
+
+if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android":
+
+ CXXFLAGS += [
+ "-msse2"
+ ]
+
+ OS_LIBS += [
+ "android_support"
+ ]
+
+if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux":
+
+ CXXFLAGS += [
+ "-msse2"
+ ]
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+Library("inter_frame_delay_gn")
diff --git a/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_unittest.cc b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_unittest.cc
new file mode 100644
index 0000000000..183b378ced
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/inter_frame_delay_unittest.cc
@@ -0,0 +1,190 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/timing/inter_frame_delay.h"
+
+#include <limits>
+
+#include "absl/types/optional.h"
+#include "api/units/frequency.h"
+#include "api/units/time_delta.h"
+#include "api/units/timestamp.h"
+#include "system_wrappers/include/clock.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+
+namespace {
+
+// Test is for frames at 30fps. At 30fps, RTP timestamps will increase by
+// 90000 / 30 = 3000 ticks per frame.
+constexpr Frequency k30Fps = Frequency::Hertz(30);
+constexpr TimeDelta kFrameDelay = 1 / k30Fps;
+constexpr uint32_t kRtpTicksPerFrame = Frequency::KiloHertz(90) / k30Fps;
+constexpr Timestamp kStartTime = Timestamp::Millis(1337);
+
+} // namespace
+
+using ::testing::Eq;
+using ::testing::Optional;
+
+TEST(InterFrameDelayTest, OldRtpTimestamp) {
+ InterFrameDelay inter_frame_delay;
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(180000, kStartTime),
+ Optional(TimeDelta::Zero()));
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(90000, kStartTime),
+ Eq(absl::nullopt));
+}
+
+TEST(InterFrameDelayTest, NegativeWrapAroundIsSameAsOldRtpTimestamp) {
+ InterFrameDelay inter_frame_delay;
+ uint32_t rtp = 1500;
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, kStartTime),
+ Optional(TimeDelta::Zero()));
+ // RTP has wrapped around backwards.
+ rtp -= 3000;
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, kStartTime),
+ Eq(absl::nullopt));
+}
+
+TEST(InterFrameDelayTest, CorrectDelayForFrames) {
+ InterFrameDelay inter_frame_delay;
+ // Use a fake clock to simplify time keeping.
+ SimulatedClock clock(kStartTime);
+
+ // First frame is always delay 0.
+ uint32_t rtp = 90000;
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()),
+ Optional(TimeDelta::Zero()));
+
+ // Perfectly timed frame has 0 delay.
+ clock.AdvanceTime(kFrameDelay);
+ rtp += kRtpTicksPerFrame;
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()),
+ Optional(TimeDelta::Zero()));
+
+ // Slightly early frame will have a negative delay.
+ clock.AdvanceTime(kFrameDelay - TimeDelta::Millis(3));
+ rtp += kRtpTicksPerFrame;
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()),
+ Optional(-TimeDelta::Millis(3)));
+
+ // Slightly late frame will have positive delay.
+ clock.AdvanceTime(kFrameDelay + TimeDelta::Micros(5125));
+ rtp += kRtpTicksPerFrame;
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()),
+ Optional(TimeDelta::Micros(5125)));
+
+ // Simulate faster frame RTP at the same clock delay. The frame arrives late,
+ // since the RTP timestamp is faster than the delay, and thus is positive.
+ clock.AdvanceTime(kFrameDelay);
+ rtp += kRtpTicksPerFrame / 2;
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()),
+ Optional(kFrameDelay / 2.0));
+
+ // Simulate slower frame RTP at the same clock delay. The frame is early,
+ // since the RTP timestamp advanced more than the delay, and thus is negative.
+ clock.AdvanceTime(kFrameDelay);
+ rtp += 1.5 * kRtpTicksPerFrame;
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()),
+ Optional(-kFrameDelay / 2.0));
+}
+
+TEST(InterFrameDelayTest, PositiveWrapAround) {
+ InterFrameDelay inter_frame_delay;
+ // Use a fake clock to simplify time keeping.
+ SimulatedClock clock(kStartTime);
+
+ // First frame is behind the max RTP by 1500.
+ uint32_t rtp = std::numeric_limits<uint32_t>::max() - 1500;
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()),
+ Optional(TimeDelta::Zero()));
+
+ // Rtp wraps around, now 1499.
+ rtp += kRtpTicksPerFrame;
+
+ // Frame delay should be as normal, in this case simulated as 1ms late.
+ clock.AdvanceTime(kFrameDelay + TimeDelta::Millis(1));
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()),
+ Optional(TimeDelta::Millis(1)));
+}
+
+TEST(InterFrameDelayTest, MultipleWrapArounds) {
+ // Simulate a long pauses which cause wrap arounds multiple times.
+ constexpr Frequency k90Khz = Frequency::KiloHertz(90);
+ constexpr uint32_t kHalfRtp = std::numeric_limits<uint32_t>::max() / 2;
+ constexpr TimeDelta kWrapAroundDelay = kHalfRtp / k90Khz;
+
+ InterFrameDelay inter_frame_delay;
+ // Use a fake clock to simplify time keeping.
+ SimulatedClock clock(kStartTime);
+ uint32_t rtp = 0;
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()),
+ Optional(TimeDelta::Zero()));
+
+ rtp += kHalfRtp;
+ clock.AdvanceTime(kWrapAroundDelay);
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()),
+ Optional(TimeDelta::Zero()));
+ // 1st wrap around.
+ rtp += kHalfRtp + 1;
+ clock.AdvanceTime(kWrapAroundDelay + TimeDelta::Millis(1));
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()),
+ Optional(TimeDelta::Millis(1) - (1 / k90Khz)));
+
+ rtp += kHalfRtp;
+ clock.AdvanceTime(kWrapAroundDelay);
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()),
+ Optional(TimeDelta::Zero()));
+ // 2nd wrap arounds.
+ rtp += kHalfRtp + 1;
+ clock.AdvanceTime(kWrapAroundDelay - TimeDelta::Millis(1));
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()),
+ Optional(-TimeDelta::Millis(1) - (1 / k90Khz)));
+
+ // Ensure short delay (large RTP delay) between wrap-arounds has correct
+ // jitter.
+ rtp += kHalfRtp;
+ clock.AdvanceTime(TimeDelta::Millis(10));
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()),
+ Optional(-(kWrapAroundDelay - TimeDelta::Millis(10))));
+ // 3nd wrap arounds, this time with large RTP delay.
+ rtp += kHalfRtp + 1;
+ clock.AdvanceTime(TimeDelta::Millis(10));
+ EXPECT_THAT(
+ inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()),
+ Optional(-(kWrapAroundDelay - TimeDelta::Millis(10) + (1 / k90Khz))));
+}
+
+TEST(InterFrameDelayTest, NegativeWrapAroundAfterPositiveWrapAround) {
+ InterFrameDelay inter_frame_delay;
+ // Use a fake clock to simplify time keeping.
+ SimulatedClock clock(kStartTime);
+ uint32_t rtp = std::numeric_limits<uint32_t>::max() - 1500;
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()),
+ Optional(TimeDelta::Zero()));
+
+ // Rtp wraps around, now 1499.
+ rtp += kRtpTicksPerFrame;
+ // Frame delay should be as normal, in this case simulated as 1ms late.
+ clock.AdvanceTime(kFrameDelay);
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()),
+ Optional(TimeDelta::Zero()));
+
+ // Wrap back.
+ rtp -= kRtpTicksPerFrame;
+ // Frame delay should be as normal, in this case simulated as 1ms late.
+ clock.AdvanceTime(kFrameDelay);
+ EXPECT_THAT(inter_frame_delay.CalculateDelay(rtp, clock.CurrentTime()),
+ Eq(absl::nullopt));
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.cc b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.cc
new file mode 100644
index 0000000000..62757787a1
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.cc
@@ -0,0 +1,476 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/timing/jitter_estimator.h"
+
+#include <math.h>
+#include <string.h>
+
+#include <algorithm>
+#include <cstdint>
+
+#include "absl/types/optional.h"
+#include "api/field_trials_view.h"
+#include "api/units/data_size.h"
+#include "api/units/frequency.h"
+#include "api/units/time_delta.h"
+#include "api/units/timestamp.h"
+#include "modules/video_coding/timing/rtt_filter.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/numerics/safe_conversions.h"
+#include "system_wrappers/include/clock.h"
+
+namespace webrtc {
+namespace {
+
+// Number of frames to wait for before post processing estimate. Also used in
+// the frame rate estimator ramp-up.
+constexpr size_t kFrameProcessingStartupCount = 30;
+
+// Number of frames to wait for before enabling the frame size filters.
+constexpr size_t kFramesUntilSizeFiltering = 5;
+
+// Initial value for frame size filters.
+constexpr double kInitialAvgAndMaxFrameSizeBytes = 500.0;
+
+// Time constant for average frame size filter.
+constexpr double kPhi = 0.97;
+// Time constant for max frame size filter.
+constexpr double kPsi = 0.9999;
+// Default constants for percentile frame size filter.
+constexpr double kDefaultMaxFrameSizePercentile = 0.95;
+constexpr int kDefaultFrameSizeWindow = 30 * 10;
+
+// Outlier rejection constants.
+constexpr double kNumStdDevDelayClamp = 3.5;
+constexpr double kNumStdDevDelayOutlier = 15.0;
+constexpr double kNumStdDevSizeOutlier = 3.0;
+constexpr double kCongestionRejectionFactor = -0.25;
+
+// Rampup constant for deviation noise filters.
+constexpr size_t kAlphaCountMax = 400;
+
+// Noise threshold constants.
+// ~Less than 1% chance (look up in normal distribution table)...
+constexpr double kNoiseStdDevs = 2.33;
+// ...of getting 30 ms freezes
+constexpr double kNoiseStdDevOffset = 30.0;
+
+// Jitter estimate clamping limits.
+constexpr TimeDelta kMinJitterEstimate = TimeDelta::Millis(1);
+constexpr TimeDelta kMaxJitterEstimate = TimeDelta::Seconds(10);
+
+// A constant describing the delay from the jitter buffer to the delay on the
+// receiving side which is not accounted for by the jitter buffer nor the
+// decoding delay estimate.
+constexpr TimeDelta OPERATING_SYSTEM_JITTER = TimeDelta::Millis(10);
+
+// Time constant for reseting the NACK count.
+constexpr TimeDelta kNackCountTimeout = TimeDelta::Seconds(60);
+
+// RTT mult activation.
+constexpr size_t kNackLimit = 3;
+
+// Frame rate estimate clamping limit.
+constexpr Frequency kMaxFramerateEstimate = Frequency::Hertz(200);
+
+} // namespace
+
+constexpr char JitterEstimator::Config::kFieldTrialsKey[];
+
+JitterEstimator::Config JitterEstimator::Config::ParseAndValidate(
+ absl::string_view field_trial) {
+ Config config;
+ config.Parser()->Parse(field_trial);
+
+ // The `MovingPercentileFilter` RTC_CHECKs on the validity of the
+ // percentile and window length, so we'd better validate the field trial
+ // provided values here.
+ if (config.max_frame_size_percentile) {
+ double original = *config.max_frame_size_percentile;
+ config.max_frame_size_percentile = std::min(std::max(0.0, original), 1.0);
+ if (config.max_frame_size_percentile != original) {
+ RTC_LOG(LS_ERROR) << "Skipping invalid max_frame_size_percentile="
+ << original;
+ }
+ }
+ if (config.frame_size_window && config.frame_size_window < 1) {
+ RTC_LOG(LS_ERROR) << "Skipping invalid frame_size_window="
+ << *config.frame_size_window;
+ config.frame_size_window = 1;
+ }
+
+ // General sanity checks.
+ if (config.num_stddev_delay_clamp && config.num_stddev_delay_clamp < 0.0) {
+ RTC_LOG(LS_ERROR) << "Skipping invalid num_stddev_delay_clamp="
+ << *config.num_stddev_delay_clamp;
+ config.num_stddev_delay_clamp = 0.0;
+ }
+ if (config.num_stddev_delay_outlier &&
+ config.num_stddev_delay_outlier < 0.0) {
+ RTC_LOG(LS_ERROR) << "Skipping invalid num_stddev_delay_outlier="
+ << *config.num_stddev_delay_outlier;
+ config.num_stddev_delay_outlier = 0.0;
+ }
+ if (config.num_stddev_size_outlier && config.num_stddev_size_outlier < 0.0) {
+ RTC_LOG(LS_ERROR) << "Skipping invalid num_stddev_size_outlier="
+ << *config.num_stddev_size_outlier;
+ config.num_stddev_size_outlier = 0.0;
+ }
+
+ return config;
+}
+
+JitterEstimator::JitterEstimator(Clock* clock,
+ const FieldTrialsView& field_trials)
+ : config_(Config::ParseAndValidate(
+ field_trials.Lookup(Config::kFieldTrialsKey))),
+ avg_frame_size_median_bytes_(static_cast<size_t>(
+ config_.frame_size_window.value_or(kDefaultFrameSizeWindow))),
+ max_frame_size_bytes_percentile_(
+ config_.max_frame_size_percentile.value_or(
+ kDefaultMaxFrameSizePercentile),
+ static_cast<size_t>(
+ config_.frame_size_window.value_or(kDefaultFrameSizeWindow))),
+ fps_counter_(30), // TODO(sprang): Use an estimator with limit based
+ // on time, rather than number of samples.
+ clock_(clock) {
+ Reset();
+}
+
+JitterEstimator::~JitterEstimator() = default;
+
+// Resets the JitterEstimate.
+void JitterEstimator::Reset() {
+ avg_frame_size_bytes_ = kInitialAvgAndMaxFrameSizeBytes;
+ max_frame_size_bytes_ = kInitialAvgAndMaxFrameSizeBytes;
+ var_frame_size_bytes2_ = 100;
+ avg_frame_size_median_bytes_.Reset();
+ max_frame_size_bytes_percentile_.Reset();
+ last_update_time_ = absl::nullopt;
+ prev_estimate_ = absl::nullopt;
+ prev_frame_size_ = absl::nullopt;
+ avg_noise_ms_ = 0.0;
+ var_noise_ms2_ = 4.0;
+ alpha_count_ = 1;
+ filter_jitter_estimate_ = TimeDelta::Zero();
+ latest_nack_ = Timestamp::Zero();
+ nack_count_ = 0;
+ startup_frame_size_sum_bytes_ = 0;
+ startup_frame_size_count_ = 0;
+ startup_count_ = 0;
+ rtt_filter_.Reset();
+ fps_counter_.Reset();
+
+ kalman_filter_ = FrameDelayVariationKalmanFilter();
+}
+
+// Updates the estimates with the new measurements.
+void JitterEstimator::UpdateEstimate(TimeDelta frame_delay,
+ DataSize frame_size) {
+ if (frame_size.IsZero()) {
+ return;
+ }
+ // Can't use DataSize since this can be negative.
+ double delta_frame_bytes =
+ frame_size.bytes() - prev_frame_size_.value_or(DataSize::Zero()).bytes();
+ if (startup_frame_size_count_ < kFramesUntilSizeFiltering) {
+ startup_frame_size_sum_bytes_ += frame_size.bytes();
+ startup_frame_size_count_++;
+ } else if (startup_frame_size_count_ == kFramesUntilSizeFiltering) {
+ // Give the frame size filter.
+ avg_frame_size_bytes_ = startup_frame_size_sum_bytes_ /
+ static_cast<double>(startup_frame_size_count_);
+ startup_frame_size_count_++;
+ }
+
+ double avg_frame_size_bytes =
+ kPhi * avg_frame_size_bytes_ + (1 - kPhi) * frame_size.bytes();
+ double deviation_size_bytes = 2 * sqrt(var_frame_size_bytes2_);
+ if (frame_size.bytes() < avg_frame_size_bytes_ + deviation_size_bytes) {
+ // Only update the average frame size if this sample wasn't a key frame.
+ avg_frame_size_bytes_ = avg_frame_size_bytes;
+ }
+
+ double delta_bytes = frame_size.bytes() - avg_frame_size_bytes;
+ var_frame_size_bytes2_ = std::max(
+ kPhi * var_frame_size_bytes2_ + (1 - kPhi) * (delta_bytes * delta_bytes),
+ 1.0);
+
+ // Update non-linear IIR estimate of max frame size.
+ max_frame_size_bytes_ =
+ std::max<double>(kPsi * max_frame_size_bytes_, frame_size.bytes());
+
+ // Maybe update percentile estimates of frame sizes.
+ if (config_.avg_frame_size_median) {
+ avg_frame_size_median_bytes_.Insert(frame_size.bytes());
+ }
+ if (config_.MaxFrameSizePercentileEnabled()) {
+ max_frame_size_bytes_percentile_.Insert(frame_size.bytes());
+ }
+
+ if (!prev_frame_size_) {
+ prev_frame_size_ = frame_size;
+ return;
+ }
+ prev_frame_size_ = frame_size;
+
+ // Cap frame_delay based on the current time deviation noise.
+ double num_stddev_delay_clamp =
+ config_.num_stddev_delay_clamp.value_or(kNumStdDevDelayClamp);
+ TimeDelta max_time_deviation =
+ TimeDelta::Millis(num_stddev_delay_clamp * sqrt(var_noise_ms2_) + 0.5);
+ frame_delay.Clamp(-max_time_deviation, max_time_deviation);
+
+ double delay_deviation_ms =
+ frame_delay.ms() -
+ kalman_filter_.GetFrameDelayVariationEstimateTotal(delta_frame_bytes);
+
+ // Outlier rejection: these conditions depend on filtered versions of the
+ // delay and frame size _means_, respectively, together with a configurable
+ // number of standard deviations. If a sample is large with respect to the
+ // corresponding mean and dispersion (defined by the number of
+ // standard deviations and the sample standard deviation), it is deemed an
+ // outlier. This "empirical rule" is further described in
+ // https://en.wikipedia.org/wiki/68-95-99.7_rule. Note that neither of the
+ // estimated means are true sample means, which implies that they are possibly
+ // not normally distributed. Hence, this rejection method is just a heuristic.
+ double num_stddev_delay_outlier =
+ config_.num_stddev_delay_outlier.value_or(kNumStdDevDelayOutlier);
+ // Delay outlier rejection is two-sided.
+ bool abs_delay_is_not_outlier =
+ fabs(delay_deviation_ms) <
+ num_stddev_delay_outlier * sqrt(var_noise_ms2_);
+ // The reasoning above means, in particular, that we should use the sample
+ // mean-style `avg_frame_size_bytes_` estimate, as opposed to the
+ // median-filtered version, even if configured to use latter for the
+ // calculation in `CalculateEstimate()`.
+ // Size outlier rejection is one-sided.
+ double num_stddev_size_outlier =
+ config_.num_stddev_size_outlier.value_or(kNumStdDevSizeOutlier);
+ bool size_is_positive_outlier =
+ frame_size.bytes() >
+ avg_frame_size_bytes_ +
+ num_stddev_size_outlier * sqrt(var_frame_size_bytes2_);
+
+ // Only update the Kalman filter if the sample is not considered an extreme
+ // outlier. Even if it is an extreme outlier from a delay point of view, if
+ // the frame size also is large the deviation is probably due to an incorrect
+ // line slope.
+ if (abs_delay_is_not_outlier || size_is_positive_outlier) {
+ // Prevent updating with frames which have been congested by a large frame,
+ // and therefore arrives almost at the same time as that frame.
+ // This can occur when we receive a large frame (key frame) which has been
+ // delayed. The next frame is of normal size (delta frame), and thus deltaFS
+ // will be << 0. This removes all frame samples which arrives after a key
+ // frame.
+ double congestion_rejection_factor =
+ config_.congestion_rejection_factor.value_or(
+ kCongestionRejectionFactor);
+ double filtered_max_frame_size_bytes =
+ config_.MaxFrameSizePercentileEnabled()
+ ? max_frame_size_bytes_percentile_.GetFilteredValue()
+ : max_frame_size_bytes_;
+ bool is_not_congested =
+ delta_frame_bytes >
+ congestion_rejection_factor * filtered_max_frame_size_bytes;
+
+ if (is_not_congested || config_.estimate_noise_when_congested) {
+ // Update the variance of the deviation from the line given by the Kalman
+ // filter.
+ EstimateRandomJitter(delay_deviation_ms);
+ }
+ if (is_not_congested) {
+ // Neither a delay outlier nor a congested frame, so we can safely update
+ // the Kalman filter with the sample.
+ kalman_filter_.PredictAndUpdate(frame_delay.ms(), delta_frame_bytes,
+ filtered_max_frame_size_bytes,
+ var_noise_ms2_);
+ }
+ } else {
+ // Delay outliers affect the noise estimate through a value equal to the
+ // outlier rejection threshold.
+ double num_stddev = (delay_deviation_ms >= 0) ? num_stddev_delay_outlier
+ : -num_stddev_delay_outlier;
+ EstimateRandomJitter(num_stddev * sqrt(var_noise_ms2_));
+ }
+ // Post process the total estimated jitter
+ if (startup_count_ >= kFrameProcessingStartupCount) {
+ PostProcessEstimate();
+ } else {
+ startup_count_++;
+ }
+}
+
+// Updates the nack/packet ratio.
+void JitterEstimator::FrameNacked() {
+ if (nack_count_ < kNackLimit) {
+ nack_count_++;
+ }
+ latest_nack_ = clock_->CurrentTime();
+}
+
+void JitterEstimator::UpdateRtt(TimeDelta rtt) {
+ rtt_filter_.Update(rtt);
+}
+
+JitterEstimator::Config JitterEstimator::GetConfigForTest() const {
+ return config_;
+}
+
+// Estimates the random jitter by calculating the variance of the sample
+// distance from the line given by the Kalman filter.
+void JitterEstimator::EstimateRandomJitter(double d_dT) {
+ Timestamp now = clock_->CurrentTime();
+ if (last_update_time_.has_value()) {
+ fps_counter_.AddSample((now - *last_update_time_).us());
+ }
+ last_update_time_ = now;
+
+ if (alpha_count_ == 0) {
+ RTC_DCHECK_NOTREACHED();
+ return;
+ }
+ double alpha =
+ static_cast<double>(alpha_count_ - 1) / static_cast<double>(alpha_count_);
+ alpha_count_++;
+ if (alpha_count_ > kAlphaCountMax)
+ alpha_count_ = kAlphaCountMax;
+
+ // In order to avoid a low frame rate stream to react slower to changes,
+ // scale the alpha weight relative a 30 fps stream.
+ Frequency fps = GetFrameRate();
+ if (fps > Frequency::Zero()) {
+ constexpr Frequency k30Fps = Frequency::Hertz(30);
+ double rate_scale = k30Fps / fps;
+ // At startup, there can be a lot of noise in the fps estimate.
+ // Interpolate rate_scale linearly, from 1.0 at sample #1, to 30.0 / fps
+ // at sample #kFrameProcessingStartupCount.
+ if (alpha_count_ < kFrameProcessingStartupCount) {
+ rate_scale = (alpha_count_ * rate_scale +
+ (kFrameProcessingStartupCount - alpha_count_)) /
+ kFrameProcessingStartupCount;
+ }
+ alpha = pow(alpha, rate_scale);
+ }
+
+ double avg_noise_ms = alpha * avg_noise_ms_ + (1 - alpha) * d_dT;
+ double var_noise_ms2 = alpha * var_noise_ms2_ + (1 - alpha) *
+ (d_dT - avg_noise_ms_) *
+ (d_dT - avg_noise_ms_);
+ avg_noise_ms_ = avg_noise_ms;
+ var_noise_ms2_ = var_noise_ms2;
+ if (var_noise_ms2_ < 1.0) {
+ // The variance should never be zero, since we might get stuck and consider
+ // all samples as outliers.
+ var_noise_ms2_ = 1.0;
+ }
+}
+
+double JitterEstimator::NoiseThreshold() const {
+ double noise_threshold_ms =
+ kNoiseStdDevs * sqrt(var_noise_ms2_) - kNoiseStdDevOffset;
+ if (noise_threshold_ms < 1.0) {
+ noise_threshold_ms = 1.0;
+ }
+ return noise_threshold_ms;
+}
+
+// Calculates the current jitter estimate from the filtered estimates.
+TimeDelta JitterEstimator::CalculateEstimate() {
+ // Using median- and percentile-filtered versions of the frame sizes may be
+ // more robust than using sample mean-style estimates.
+ double filtered_avg_frame_size_bytes =
+ config_.avg_frame_size_median
+ ? avg_frame_size_median_bytes_.GetFilteredValue()
+ : avg_frame_size_bytes_;
+ double filtered_max_frame_size_bytes =
+ config_.MaxFrameSizePercentileEnabled()
+ ? max_frame_size_bytes_percentile_.GetFilteredValue()
+ : max_frame_size_bytes_;
+ double worst_case_frame_size_deviation_bytes =
+ filtered_max_frame_size_bytes - filtered_avg_frame_size_bytes;
+ double ret_ms = kalman_filter_.GetFrameDelayVariationEstimateSizeBased(
+ worst_case_frame_size_deviation_bytes) +
+ NoiseThreshold();
+ TimeDelta ret = TimeDelta::Millis(ret_ms);
+
+ // A very low estimate (or negative) is neglected.
+ if (ret < kMinJitterEstimate) {
+ ret = prev_estimate_.value_or(kMinJitterEstimate);
+ // Sanity check to make sure that no other method has set `prev_estimate_`
+ // to a value lower than `kMinJitterEstimate`.
+ RTC_DCHECK_GE(ret, kMinJitterEstimate);
+ } else if (ret > kMaxJitterEstimate) { // Sanity
+ ret = kMaxJitterEstimate;
+ }
+ prev_estimate_ = ret;
+ return ret;
+}
+
+void JitterEstimator::PostProcessEstimate() {
+ filter_jitter_estimate_ = CalculateEstimate();
+}
+
+// Returns the current filtered estimate if available,
+// otherwise tries to calculate an estimate.
+TimeDelta JitterEstimator::GetJitterEstimate(
+ double rtt_multiplier,
+ absl::optional<TimeDelta> rtt_mult_add_cap) {
+ TimeDelta jitter = CalculateEstimate() + OPERATING_SYSTEM_JITTER;
+ Timestamp now = clock_->CurrentTime();
+
+ if (now - latest_nack_ > kNackCountTimeout)
+ nack_count_ = 0;
+
+ if (filter_jitter_estimate_ > jitter)
+ jitter = filter_jitter_estimate_;
+ if (nack_count_ >= kNackLimit) {
+ if (rtt_mult_add_cap.has_value()) {
+ jitter += std::min(rtt_filter_.Rtt() * rtt_multiplier,
+ rtt_mult_add_cap.value());
+ } else {
+ jitter += rtt_filter_.Rtt() * rtt_multiplier;
+ }
+ }
+
+ static const Frequency kJitterScaleLowThreshold = Frequency::Hertz(5);
+ static const Frequency kJitterScaleHighThreshold = Frequency::Hertz(10);
+ Frequency fps = GetFrameRate();
+ // Ignore jitter for very low fps streams.
+ if (fps < kJitterScaleLowThreshold) {
+ if (fps.IsZero()) {
+ return std::max(TimeDelta::Zero(), jitter);
+ }
+ return TimeDelta::Zero();
+ }
+
+ // Semi-low frame rate; scale by factor linearly interpolated from 0.0 at
+ // kJitterScaleLowThreshold to 1.0 at kJitterScaleHighThreshold.
+ if (fps < kJitterScaleHighThreshold) {
+ jitter = (1.0 / (kJitterScaleHighThreshold - kJitterScaleLowThreshold)) *
+ (fps - kJitterScaleLowThreshold) * jitter;
+ }
+
+ return std::max(TimeDelta::Zero(), jitter);
+}
+
+Frequency JitterEstimator::GetFrameRate() const {
+ TimeDelta mean_frame_period = TimeDelta::Micros(fps_counter_.ComputeMean());
+ if (mean_frame_period <= TimeDelta::Zero())
+ return Frequency::Zero();
+
+ Frequency fps = 1 / mean_frame_period;
+ // Sanity check.
+ RTC_DCHECK_GE(fps, Frequency::Zero());
+ return std::min(fps, kMaxFramerateEstimate);
+}
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.h b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.h
new file mode 100644
index 0000000000..a89a4bf1fd
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.h
@@ -0,0 +1,218 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_TIMING_JITTER_ESTIMATOR_H_
+#define MODULES_VIDEO_CODING_TIMING_JITTER_ESTIMATOR_H_
+
+#include <algorithm>
+#include <memory>
+#include <queue>
+
+#include "absl/strings/string_view.h"
+#include "absl/types/optional.h"
+#include "api/field_trials_view.h"
+#include "api/units/data_size.h"
+#include "api/units/frequency.h"
+#include "api/units/time_delta.h"
+#include "api/units/timestamp.h"
+#include "modules/video_coding/timing/frame_delay_variation_kalman_filter.h"
+#include "modules/video_coding/timing/rtt_filter.h"
+#include "rtc_base/experiments/struct_parameters_parser.h"
+#include "rtc_base/numerics/moving_percentile_filter.h"
+#include "rtc_base/rolling_accumulator.h"
+
+namespace webrtc {
+
+class Clock;
+
+class JitterEstimator {
+ public:
+ // Configuration struct for statically overriding some constants and
+ // behaviour, configurable through field trials.
+ struct Config {
+ static constexpr char kFieldTrialsKey[] = "WebRTC-JitterEstimatorConfig";
+
+ // Parses a field trial string and validates the values.
+ static Config ParseAndValidate(absl::string_view field_trial);
+
+ std::unique_ptr<StructParametersParser> Parser() {
+ // clang-format off
+ return StructParametersParser::Create(
+ "avg_frame_size_median", &avg_frame_size_median,
+ "max_frame_size_percentile", &max_frame_size_percentile,
+ "frame_size_window", &frame_size_window,
+ "num_stddev_delay_clamp", &num_stddev_delay_clamp,
+ "num_stddev_delay_outlier", &num_stddev_delay_outlier,
+ "num_stddev_size_outlier", &num_stddev_size_outlier,
+ "congestion_rejection_factor", &congestion_rejection_factor,
+ "estimate_noise_when_congested", &estimate_noise_when_congested);
+ // clang-format on
+ }
+
+ bool MaxFrameSizePercentileEnabled() const {
+ return max_frame_size_percentile.has_value();
+ }
+
+ // If true, the "avg" frame size is calculated as the median over a window
+ // of recent frame sizes.
+ bool avg_frame_size_median = false;
+
+ // If set, the "max" frame size is calculated as this percentile over a
+ // window of recent frame sizes.
+ absl::optional<double> max_frame_size_percentile = absl::nullopt;
+
+ // The length of the percentile filters' window, in number of frames.
+ absl::optional<int> frame_size_window = absl::nullopt;
+
+ // The incoming frame delay variation samples are clamped to be at most
+ // this number of standard deviations away from zero.
+ //
+ // Increasing this value clamps fewer samples.
+ absl::optional<double> num_stddev_delay_clamp = absl::nullopt;
+
+ // A (relative) frame delay variation sample is an outlier if its absolute
+ // deviation from the Kalman filter model falls outside this number of
+ // sample standard deviations.
+ //
+ // Increasing this value rejects fewer samples.
+ absl::optional<double> num_stddev_delay_outlier = absl::nullopt;
+
+ // An (absolute) frame size sample is an outlier if its positive deviation
+ // from the estimated average frame size falls outside this number of sample
+ // standard deviations.
+ //
+ // Increasing this value rejects fewer samples.
+ absl::optional<double> num_stddev_size_outlier = absl::nullopt;
+
+ // A (relative) frame size variation sample is deemed "congested", and is
+ // thus rejected, if its value is less than this factor times the estimated
+ // max frame size.
+ //
+ // Decreasing this value rejects fewer samples.
+ absl::optional<double> congestion_rejection_factor = absl::nullopt;
+
+ // If true, the noise estimate will be updated for congestion rejected
+ // frames. This is currently enabled by default, but that may not be optimal
+ // since congested frames typically are not spread around the line with
+ // Gaussian noise. (This is the whole reason for the congestion rejection!)
+ bool estimate_noise_when_congested = true;
+ };
+
+ JitterEstimator(Clock* clock, const FieldTrialsView& field_trials);
+ JitterEstimator(const JitterEstimator&) = delete;
+ JitterEstimator& operator=(const JitterEstimator&) = delete;
+ ~JitterEstimator();
+
+ // Resets the estimate to the initial state.
+ void Reset();
+
+ // Updates the jitter estimate with the new data.
+ //
+ // Input:
+ // - frame_delay : Delay-delta calculated by UTILDelayEstimate.
+ // - frame_size : Frame size of the current frame.
+ void UpdateEstimate(TimeDelta frame_delay, DataSize frame_size);
+
+ // Returns the current jitter estimate and adds an RTT dependent term in cases
+ // of retransmission.
+ // Input:
+ // - rtt_multiplier : RTT param multiplier (when applicable).
+ // - rtt_mult_add_cap : Multiplier cap from the RTTMultExperiment.
+ //
+ // Return value : Jitter estimate.
+ TimeDelta GetJitterEstimate(double rtt_multiplier,
+ absl::optional<TimeDelta> rtt_mult_add_cap);
+
+ // Updates the nack counter.
+ void FrameNacked();
+
+ // Updates the RTT filter.
+ //
+ // Input:
+ // - rtt : Round trip time.
+ void UpdateRtt(TimeDelta rtt);
+
+ // Returns the configuration. Only to be used by unit tests.
+ Config GetConfigForTest() const;
+
+ private:
+ // Updates the random jitter estimate, i.e. the variance of the time
+ // deviations from the line given by the Kalman filter.
+ //
+ // Input:
+ // - d_dT : The deviation from the kalman estimate.
+ void EstimateRandomJitter(double d_dT);
+
+ double NoiseThreshold() const;
+
+ // Calculates the current jitter estimate.
+ //
+ // Return value : The current jitter estimate.
+ TimeDelta CalculateEstimate();
+
+ // Post process the calculated estimate.
+ void PostProcessEstimate();
+
+ // Returns the estimated incoming frame rate.
+ Frequency GetFrameRate() const;
+
+ // Configuration that may override some internals.
+ const Config config_;
+
+ // Filters the {frame_delay_delta, frame_size_delta} measurements through
+ // a linear Kalman filter.
+ FrameDelayVariationKalmanFilter kalman_filter_;
+
+ // TODO(bugs.webrtc.org/14381): Update `avg_frame_size_bytes_` to DataSize
+ // when api/units have sufficient precision.
+ double avg_frame_size_bytes_; // Average frame size
+ double var_frame_size_bytes2_; // Frame size variance. Unit is bytes^2.
+ // Largest frame size received (descending with a factor kPsi).
+ // Used by default.
+ // TODO(bugs.webrtc.org/14381): Update `max_frame_size_bytes_` to DataSize
+ // when api/units have sufficient precision.
+ double max_frame_size_bytes_;
+ // Percentile frame sized received (over a window). Only used if configured.
+ MovingMedianFilter<int64_t> avg_frame_size_median_bytes_;
+ MovingPercentileFilter<int64_t> max_frame_size_bytes_percentile_;
+ // TODO(bugs.webrtc.org/14381): Update `startup_frame_size_sum_bytes_` to
+ // DataSize when api/units have sufficient precision.
+ double startup_frame_size_sum_bytes_;
+ size_t startup_frame_size_count_;
+
+ absl::optional<Timestamp> last_update_time_;
+ // The previously returned jitter estimate
+ absl::optional<TimeDelta> prev_estimate_;
+ // Frame size of the previous frame
+ absl::optional<DataSize> prev_frame_size_;
+ // Average of the random jitter. Unit is milliseconds.
+ double avg_noise_ms_;
+ // Variance of the time-deviation from the line. Unit is milliseconds^2.
+ double var_noise_ms2_;
+ size_t alpha_count_;
+ // The filtered sum of jitter estimates
+ TimeDelta filter_jitter_estimate_ = TimeDelta::Zero();
+
+ size_t startup_count_;
+ // Time when the latest nack was seen
+ Timestamp latest_nack_ = Timestamp::Zero();
+ // Keeps track of the number of nacks received, but never goes above
+ // kNackLimit.
+ size_t nack_count_;
+ RttFilter rtt_filter_;
+
+ // Tracks frame rates in microseconds.
+ rtc::RollingAccumulator<uint64_t> fps_counter_;
+ Clock* clock_;
+};
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_TIMING_JITTER_ESTIMATOR_H_
diff --git a/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_gn/moz.build
new file mode 100644
index 0000000000..a0d6154d78
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_gn/moz.build
@@ -0,0 +1,232 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+ ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ###
+ ### DO NOT edit it by hand. ###
+
+COMPILE_FLAGS["OS_INCLUDES"] = []
+AllowCompilerWarnings()
+
+DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1"
+DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True
+DEFINES["RTC_ENABLE_VP9"] = True
+DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0"
+DEFINES["WEBRTC_LIBRARY_IMPL"] = True
+DEFINES["WEBRTC_MOZILLA_BUILD"] = True
+DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0"
+DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0"
+
+FINAL_LIBRARY = "webrtc"
+
+
+LOCAL_INCLUDES += [
+ "!/ipc/ipdl/_ipdlheaders",
+ "!/third_party/libwebrtc/gen",
+ "/ipc/chromium/src",
+ "/third_party/libwebrtc/",
+ "/third_party/libwebrtc/third_party/abseil-cpp/",
+ "/tools/profiler/public"
+]
+
+UNIFIED_SOURCES += [
+ "/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator.cc"
+]
+
+if not CONFIG["MOZ_DEBUG"]:
+
+ DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0"
+ DEFINES["NDEBUG"] = True
+ DEFINES["NVALGRIND"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1":
+
+ DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1"
+
+if CONFIG["OS_TARGET"] == "Android":
+
+ DEFINES["ANDROID"] = True
+ DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1"
+ DEFINES["HAVE_SYS_UIO_H"] = True
+ DEFINES["WEBRTC_ANDROID"] = True
+ DEFINES["WEBRTC_ANDROID_OPENSLES"] = True
+ DEFINES["WEBRTC_LINUX"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_GNU_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+ OS_LIBS += [
+ "log"
+ ]
+
+if CONFIG["OS_TARGET"] == "Darwin":
+
+ DEFINES["WEBRTC_MAC"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True
+ DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0"
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["USE_AURA"] = "1"
+ DEFINES["USE_GLIB"] = "1"
+ DEFINES["USE_NSS_CERTS"] = "1"
+ DEFINES["USE_OZONE"] = "1"
+ DEFINES["USE_UDEV"] = True
+ DEFINES["WEBRTC_LINUX"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_FILE_OFFSET_BITS"] = "64"
+ DEFINES["_LARGEFILE64_SOURCE"] = True
+ DEFINES["_LARGEFILE_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+ OS_LIBS += [
+ "rt"
+ ]
+
+if CONFIG["OS_TARGET"] == "OpenBSD":
+
+ DEFINES["USE_GLIB"] = "1"
+ DEFINES["USE_OZONE"] = "1"
+ DEFINES["USE_X11"] = "1"
+ DEFINES["WEBRTC_BSD"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_FILE_OFFSET_BITS"] = "64"
+ DEFINES["_LARGEFILE64_SOURCE"] = True
+ DEFINES["_LARGEFILE_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "WINNT":
+
+ DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True
+ DEFINES["NOMINMAX"] = True
+ DEFINES["NTDDI_VERSION"] = "0x0A000000"
+ DEFINES["PSAPI_VERSION"] = "2"
+ DEFINES["UNICODE"] = True
+ DEFINES["USE_AURA"] = "1"
+ DEFINES["WEBRTC_WIN"] = True
+ DEFINES["WIN32"] = True
+ DEFINES["WIN32_LEAN_AND_MEAN"] = True
+ DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP"
+ DEFINES["WINVER"] = "0x0A00"
+ DEFINES["_ATL_NO_OPENGL"] = True
+ DEFINES["_CRT_RAND_S"] = True
+ DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True
+ DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True
+ DEFINES["_HAS_EXCEPTIONS"] = "0"
+ DEFINES["_HAS_NODISCARD"] = True
+ DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True
+ DEFINES["_SECURE_ATL"] = True
+ DEFINES["_UNICODE"] = True
+ DEFINES["_WIN32_WINNT"] = "0x0A00"
+ DEFINES["_WINDOWS"] = True
+ DEFINES["__STD_C"] = True
+
+ OS_LIBS += [
+ "crypt32",
+ "iphlpapi",
+ "secur32",
+ "winmm"
+ ]
+
+if CONFIG["CPU_ARCH"] == "aarch64":
+
+ DEFINES["WEBRTC_ARCH_ARM64"] = True
+ DEFINES["WEBRTC_HAS_NEON"] = True
+
+if CONFIG["CPU_ARCH"] == "arm":
+
+ CXXFLAGS += [
+ "-mfpu=neon"
+ ]
+
+ DEFINES["WEBRTC_ARCH_ARM"] = True
+ DEFINES["WEBRTC_ARCH_ARM_V7"] = True
+ DEFINES["WEBRTC_HAS_NEON"] = True
+
+if CONFIG["CPU_ARCH"] == "mips32":
+
+ DEFINES["MIPS32_LE"] = True
+ DEFINES["MIPS_FPU_LE"] = True
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "mips64":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86":
+
+ DEFINES["WEBRTC_ENABLE_AVX2"] = True
+
+if CONFIG["CPU_ARCH"] == "x86_64":
+
+ DEFINES["WEBRTC_ENABLE_AVX2"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT":
+
+ DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0"
+
+if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["USE_X11"] = "1"
+
+if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android":
+
+ OS_LIBS += [
+ "android_support",
+ "unwind"
+ ]
+
+if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android":
+
+ CXXFLAGS += [
+ "-msse2"
+ ]
+
+ OS_LIBS += [
+ "android_support"
+ ]
+
+if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux":
+
+ CXXFLAGS += [
+ "-msse2"
+ ]
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+Library("jitter_estimator_gn")
diff --git a/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_unittest.cc b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_unittest.cc
new file mode 100644
index 0000000000..8e0c01587f
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/jitter_estimator_unittest.cc
@@ -0,0 +1,305 @@
+/* Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/timing/jitter_estimator.h"
+
+#include <stdint.h>
+
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/array_view.h"
+#include "api/field_trials.h"
+#include "api/units/data_size.h"
+#include "api/units/frequency.h"
+#include "api/units/time_delta.h"
+#include "rtc_base/numerics/histogram_percentile_counter.h"
+#include "rtc_base/time_utils.h"
+#include "system_wrappers/include/clock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace {
+
+// Generates some simple test data in the form of a sawtooth wave.
+class ValueGenerator {
+ public:
+ explicit ValueGenerator(int32_t amplitude)
+ : amplitude_(amplitude), counter_(0) {}
+
+ virtual ~ValueGenerator() = default;
+
+ TimeDelta Delay() const {
+ return TimeDelta::Millis((counter_ % 11) - 5) * amplitude_;
+ }
+
+ DataSize FrameSize() const {
+ return DataSize::Bytes(1000 + Delay().ms() / 5);
+ }
+
+ void Advance() { ++counter_; }
+
+ private:
+ const int32_t amplitude_;
+ int64_t counter_;
+};
+
+class JitterEstimatorTest : public ::testing::Test {
+ protected:
+ explicit JitterEstimatorTest(const std::string& field_trials)
+ : fake_clock_(0),
+ field_trials_(FieldTrials::CreateNoGlobal(field_trials)),
+ estimator_(&fake_clock_, *field_trials_) {}
+ JitterEstimatorTest() : JitterEstimatorTest("") {}
+ virtual ~JitterEstimatorTest() {}
+
+ void Run(int duration_s, int framerate_fps, ValueGenerator& gen) {
+ TimeDelta tick = 1 / Frequency::Hertz(framerate_fps);
+ for (int i = 0; i < duration_s * framerate_fps; ++i) {
+ estimator_.UpdateEstimate(gen.Delay(), gen.FrameSize());
+ fake_clock_.AdvanceTime(tick);
+ gen.Advance();
+ }
+ }
+
+ SimulatedClock fake_clock_;
+ std::unique_ptr<FieldTrials> field_trials_;
+ JitterEstimator estimator_;
+};
+
+TEST_F(JitterEstimatorTest, SteadyStateConvergence) {
+ ValueGenerator gen(10);
+ Run(/*duration_s=*/60, /*framerate_fps=*/30, gen);
+ EXPECT_EQ(estimator_.GetJitterEstimate(0, absl::nullopt).ms(), 54);
+}
+
+TEST_F(JitterEstimatorTest,
+ SizeOutlierIsNotRejectedAndIncreasesJitterEstimate) {
+ ValueGenerator gen(10);
+
+ // Steady state.
+ Run(/*duration_s=*/60, /*framerate_fps=*/30, gen);
+ TimeDelta steady_state_jitter =
+ estimator_.GetJitterEstimate(0, absl::nullopt);
+
+ // A single outlier frame size...
+ estimator_.UpdateEstimate(gen.Delay(), 10 * gen.FrameSize());
+ TimeDelta outlier_jitter = estimator_.GetJitterEstimate(0, absl::nullopt);
+
+ // ...changes the estimate.
+ EXPECT_GT(outlier_jitter.ms(), 1.25 * steady_state_jitter.ms());
+}
+
+TEST_F(JitterEstimatorTest, LowFramerateDisablesJitterEstimator) {
+ ValueGenerator gen(10);
+ // At 5 fps, we disable jitter delay altogether.
+ TimeDelta time_delta = 1 / Frequency::Hertz(5);
+ for (int i = 0; i < 60; ++i) {
+ estimator_.UpdateEstimate(gen.Delay(), gen.FrameSize());
+ fake_clock_.AdvanceTime(time_delta);
+ if (i > 2)
+ EXPECT_EQ(estimator_.GetJitterEstimate(0, absl::nullopt),
+ TimeDelta::Zero());
+ gen.Advance();
+ }
+}
+
+TEST_F(JitterEstimatorTest, RttMultAddCap) {
+ std::vector<std::pair<TimeDelta, rtc::HistogramPercentileCounter>>
+ jitter_by_rtt_mult_cap;
+ jitter_by_rtt_mult_cap.emplace_back(
+ /*rtt_mult_add_cap=*/TimeDelta::Millis(10), /*long_tail_boundary=*/1000);
+ jitter_by_rtt_mult_cap.emplace_back(
+ /*rtt_mult_add_cap=*/TimeDelta::Millis(200), /*long_tail_boundary=*/1000);
+
+ for (auto& [rtt_mult_add_cap, jitter] : jitter_by_rtt_mult_cap) {
+ estimator_.Reset();
+
+ ValueGenerator gen(50);
+ TimeDelta time_delta = 1 / Frequency::Hertz(30);
+ constexpr TimeDelta kRtt = TimeDelta::Millis(250);
+ for (int i = 0; i < 100; ++i) {
+ estimator_.UpdateEstimate(gen.Delay(), gen.FrameSize());
+ fake_clock_.AdvanceTime(time_delta);
+ estimator_.FrameNacked();
+ estimator_.UpdateRtt(kRtt);
+ jitter.Add(
+ estimator_.GetJitterEstimate(/*rtt_mult=*/1.0, rtt_mult_add_cap)
+ .ms());
+ gen.Advance();
+ }
+ }
+
+ // 200ms cap should result in at least 25% higher max compared to 10ms.
+ EXPECT_GT(*jitter_by_rtt_mult_cap[1].second.GetPercentile(1.0),
+ *jitter_by_rtt_mult_cap[0].second.GetPercentile(1.0) * 1.25);
+}
+
+// By default, the `JitterEstimator` is not robust against single large frames.
+TEST_F(JitterEstimatorTest, Single2xFrameSizeImpactsJitterEstimate) {
+ ValueGenerator gen(10);
+
+ // Steady state.
+ Run(/*duration_s=*/60, /*framerate_fps=*/30, gen);
+ TimeDelta steady_state_jitter =
+ estimator_.GetJitterEstimate(0, absl::nullopt);
+
+ // A single outlier frame size...
+ estimator_.UpdateEstimate(gen.Delay(), 2 * gen.FrameSize());
+ TimeDelta outlier_jitter = estimator_.GetJitterEstimate(0, absl::nullopt);
+
+ // ...impacts the estimate.
+ EXPECT_GT(outlier_jitter.ms(), steady_state_jitter.ms());
+}
+
+// Under the default config, congested frames are used when calculating the
+// noise variance, meaning that they will impact the final jitter estimate.
+TEST_F(JitterEstimatorTest, CongestedFrameImpactsJitterEstimate) {
+ ValueGenerator gen(10);
+
+ // Steady state.
+ Run(/*duration_s=*/10, /*framerate_fps=*/30, gen);
+ TimeDelta steady_state_jitter =
+ estimator_.GetJitterEstimate(0, absl::nullopt);
+
+ // Congested frame...
+ estimator_.UpdateEstimate(-10 * gen.Delay(), 0.1 * gen.FrameSize());
+ TimeDelta outlier_jitter = estimator_.GetJitterEstimate(0, absl::nullopt);
+
+ // ...impacts the estimate.
+ EXPECT_GT(outlier_jitter.ms(), steady_state_jitter.ms());
+}
+
+TEST_F(JitterEstimatorTest, EmptyFieldTrialsParsesToUnsetConfig) {
+ JitterEstimator::Config config = estimator_.GetConfigForTest();
+ EXPECT_FALSE(config.avg_frame_size_median);
+ EXPECT_FALSE(config.max_frame_size_percentile.has_value());
+ EXPECT_FALSE(config.frame_size_window.has_value());
+ EXPECT_FALSE(config.num_stddev_delay_clamp.has_value());
+ EXPECT_FALSE(config.num_stddev_delay_outlier.has_value());
+ EXPECT_FALSE(config.num_stddev_size_outlier.has_value());
+ EXPECT_FALSE(config.congestion_rejection_factor.has_value());
+ EXPECT_TRUE(config.estimate_noise_when_congested);
+}
+
+class FieldTrialsOverriddenJitterEstimatorTest : public JitterEstimatorTest {
+ protected:
+ FieldTrialsOverriddenJitterEstimatorTest()
+ : JitterEstimatorTest(
+ "WebRTC-JitterEstimatorConfig/"
+ "avg_frame_size_median:true,"
+ "max_frame_size_percentile:0.9,"
+ "frame_size_window:30,"
+ "num_stddev_delay_clamp:1.1,"
+ "num_stddev_delay_outlier:2,"
+ "num_stddev_size_outlier:3.1,"
+ "congestion_rejection_factor:-1.55,"
+ "estimate_noise_when_congested:false/") {}
+ ~FieldTrialsOverriddenJitterEstimatorTest() {}
+};
+
+TEST_F(FieldTrialsOverriddenJitterEstimatorTest, FieldTrialsParsesCorrectly) {
+ JitterEstimator::Config config = estimator_.GetConfigForTest();
+ EXPECT_TRUE(config.avg_frame_size_median);
+ EXPECT_EQ(*config.max_frame_size_percentile, 0.9);
+ EXPECT_EQ(*config.frame_size_window, 30);
+ EXPECT_EQ(*config.num_stddev_delay_clamp, 1.1);
+ EXPECT_EQ(*config.num_stddev_delay_outlier, 2.0);
+ EXPECT_EQ(*config.num_stddev_size_outlier, 3.1);
+ EXPECT_EQ(*config.congestion_rejection_factor, -1.55);
+ EXPECT_FALSE(config.estimate_noise_when_congested);
+}
+
+TEST_F(FieldTrialsOverriddenJitterEstimatorTest,
+ DelayOutlierIsRejectedAndMaintainsJitterEstimate) {
+ ValueGenerator gen(10);
+
+ // Steady state.
+ Run(/*duration_s=*/60, /*framerate_fps=*/30, gen);
+ TimeDelta steady_state_jitter =
+ estimator_.GetJitterEstimate(0, absl::nullopt);
+
+ // A single outlier frame size...
+ estimator_.UpdateEstimate(10 * gen.Delay(), gen.FrameSize());
+ TimeDelta outlier_jitter = estimator_.GetJitterEstimate(0, absl::nullopt);
+
+ // ...does not change the estimate.
+ EXPECT_EQ(outlier_jitter.ms(), steady_state_jitter.ms());
+}
+
+// The field trial is configured to be robust against the `(1 - 0.9) = 10%`
+// largest frames over a window of length `30`.
+TEST_F(FieldTrialsOverriddenJitterEstimatorTest,
+ Four2xFrameSizesImpactJitterEstimate) {
+ ValueGenerator gen(10);
+
+ // Steady state.
+ Run(/*duration_s=*/60, /*framerate_fps=*/30, gen);
+ TimeDelta steady_state_jitter =
+ estimator_.GetJitterEstimate(0, absl::nullopt);
+
+ // Three outlier frames do not impact the jitter estimate.
+ for (int i = 0; i < 3; ++i) {
+ estimator_.UpdateEstimate(gen.Delay(), 2 * gen.FrameSize());
+ }
+ TimeDelta outlier_jitter_3x = estimator_.GetJitterEstimate(0, absl::nullopt);
+ EXPECT_EQ(outlier_jitter_3x.ms(), steady_state_jitter.ms());
+
+ // Four outlier frames do impact the jitter estimate.
+ estimator_.UpdateEstimate(gen.Delay(), 2 * gen.FrameSize());
+ TimeDelta outlier_jitter_4x = estimator_.GetJitterEstimate(0, absl::nullopt);
+ EXPECT_GT(outlier_jitter_4x.ms(), outlier_jitter_3x.ms());
+}
+
+// When so configured, congested frames are NOT used when calculating the
+// noise variance, meaning that they will NOT impact the final jitter estimate.
+TEST_F(FieldTrialsOverriddenJitterEstimatorTest,
+ CongestedFrameDoesNotImpactJitterEstimate) {
+ ValueGenerator gen(10);
+
+ // Steady state.
+ Run(/*duration_s=*/10, /*framerate_fps=*/30, gen);
+ TimeDelta steady_state_jitter =
+ estimator_.GetJitterEstimate(0, absl::nullopt);
+
+ // Congested frame...
+ estimator_.UpdateEstimate(-10 * gen.Delay(), 0.1 * gen.FrameSize());
+ TimeDelta outlier_jitter = estimator_.GetJitterEstimate(0, absl::nullopt);
+
+ // ...does not impact the estimate.
+ EXPECT_EQ(outlier_jitter.ms(), steady_state_jitter.ms());
+}
+
+class MisconfiguredFieldTrialsJitterEstimatorTest : public JitterEstimatorTest {
+ protected:
+ MisconfiguredFieldTrialsJitterEstimatorTest()
+ : JitterEstimatorTest(
+ "WebRTC-JitterEstimatorConfig/"
+ "max_frame_size_percentile:-0.9,"
+ "frame_size_window:-1,"
+ "num_stddev_delay_clamp:-1.9,"
+ "num_stddev_delay_outlier:-2,"
+ "num_stddev_size_outlier:-23.1/") {}
+ ~MisconfiguredFieldTrialsJitterEstimatorTest() {}
+};
+
+TEST_F(MisconfiguredFieldTrialsJitterEstimatorTest, FieldTrialsAreValidated) {
+ JitterEstimator::Config config = estimator_.GetConfigForTest();
+ EXPECT_EQ(*config.max_frame_size_percentile, 0.0);
+ EXPECT_EQ(*config.frame_size_window, 1);
+ EXPECT_EQ(*config.num_stddev_delay_clamp, 0.0);
+ EXPECT_EQ(*config.num_stddev_delay_outlier, 0.0);
+ EXPECT_EQ(*config.num_stddev_size_outlier, 0.0);
+}
+
+} // namespace
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.cc b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.cc
new file mode 100644
index 0000000000..6962224d61
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.cc
@@ -0,0 +1,161 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/timing/rtt_filter.h"
+
+#include <math.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <algorithm>
+
+#include "absl/algorithm/container.h"
+#include "absl/container/inlined_vector.h"
+#include "api/units/time_delta.h"
+
+namespace webrtc {
+
+namespace {
+
+constexpr TimeDelta kMaxRtt = TimeDelta::Seconds(3);
+constexpr uint32_t kFilterFactorMax = 35;
+constexpr double kJumpStddev = 2.5;
+constexpr double kDriftStdDev = 3.5;
+
+} // namespace
+
+RttFilter::RttFilter()
+ : avg_rtt_(TimeDelta::Zero()),
+ var_rtt_(0),
+ max_rtt_(TimeDelta::Zero()),
+ jump_buf_(kMaxDriftJumpCount, TimeDelta::Zero()),
+ drift_buf_(kMaxDriftJumpCount, TimeDelta::Zero()) {
+ Reset();
+}
+
+void RttFilter::Reset() {
+ got_non_zero_update_ = false;
+ avg_rtt_ = TimeDelta::Zero();
+ var_rtt_ = 0;
+ max_rtt_ = TimeDelta::Zero();
+ filt_fact_count_ = 1;
+ absl::c_fill(jump_buf_, TimeDelta::Zero());
+ absl::c_fill(drift_buf_, TimeDelta::Zero());
+}
+
+void RttFilter::Update(TimeDelta rtt) {
+ if (!got_non_zero_update_) {
+ if (rtt.IsZero()) {
+ return;
+ }
+ got_non_zero_update_ = true;
+ }
+
+ // Sanity check
+ if (rtt > kMaxRtt) {
+ rtt = kMaxRtt;
+ }
+
+ double filt_factor = 0;
+ if (filt_fact_count_ > 1) {
+ filt_factor = static_cast<double>(filt_fact_count_ - 1) / filt_fact_count_;
+ }
+ filt_fact_count_++;
+ if (filt_fact_count_ > kFilterFactorMax) {
+ // This prevents filt_factor from going above
+ // (_filt_fact_max - 1) / filt_fact_max_,
+ // e.g., filt_fact_max_ = 50 => filt_factor = 49/50 = 0.98
+ filt_fact_count_ = kFilterFactorMax;
+ }
+ TimeDelta old_avg = avg_rtt_;
+ int64_t old_var = var_rtt_;
+ avg_rtt_ = filt_factor * avg_rtt_ + (1 - filt_factor) * rtt;
+ int64_t delta_ms = (rtt - avg_rtt_).ms();
+ var_rtt_ = filt_factor * var_rtt_ + (1 - filt_factor) * (delta_ms * delta_ms);
+ max_rtt_ = std::max(rtt, max_rtt_);
+ if (!JumpDetection(rtt) || !DriftDetection(rtt)) {
+ // In some cases we don't want to update the statistics
+ avg_rtt_ = old_avg;
+ var_rtt_ = old_var;
+ }
+}
+
+bool RttFilter::JumpDetection(TimeDelta rtt) {
+ TimeDelta diff_from_avg = avg_rtt_ - rtt;
+ // Unit of var_rtt_ is ms^2.
+ TimeDelta jump_threshold = TimeDelta::Millis(kJumpStddev * sqrt(var_rtt_));
+ if (diff_from_avg.Abs() > jump_threshold) {
+ bool positive_diff = diff_from_avg >= TimeDelta::Zero();
+ if (!jump_buf_.empty() && positive_diff != last_jump_positive_) {
+ // Since the signs differ the samples currently
+ // in the buffer is useless as they represent a
+ // jump in a different direction.
+ jump_buf_.clear();
+ }
+ if (jump_buf_.size() < kMaxDriftJumpCount) {
+ // Update the buffer used for the short time statistics.
+ // The sign of the diff is used for updating the counter since
+ // we want to use the same buffer for keeping track of when
+ // the RTT jumps down and up.
+ jump_buf_.push_back(rtt);
+ last_jump_positive_ = positive_diff;
+ }
+ if (jump_buf_.size() >= kMaxDriftJumpCount) {
+ // Detected an RTT jump
+ ShortRttFilter(jump_buf_);
+ filt_fact_count_ = kMaxDriftJumpCount + 1;
+ jump_buf_.clear();
+ } else {
+ return false;
+ }
+ } else {
+ jump_buf_.clear();
+ }
+ return true;
+}
+
+bool RttFilter::DriftDetection(TimeDelta rtt) {
+ // Unit of sqrt of var_rtt_ is ms.
+ TimeDelta drift_threshold = TimeDelta::Millis(kDriftStdDev * sqrt(var_rtt_));
+ if (max_rtt_ - avg_rtt_ > drift_threshold) {
+ if (drift_buf_.size() < kMaxDriftJumpCount) {
+ // Update the buffer used for the short time statistics.
+ drift_buf_.push_back(rtt);
+ }
+ if (drift_buf_.size() >= kMaxDriftJumpCount) {
+ // Detected an RTT drift
+ ShortRttFilter(drift_buf_);
+ filt_fact_count_ = kMaxDriftJumpCount + 1;
+ drift_buf_.clear();
+ }
+ } else {
+ drift_buf_.clear();
+ }
+ return true;
+}
+
+void RttFilter::ShortRttFilter(const BufferList& buf) {
+ RTC_DCHECK_EQ(buf.size(), kMaxDriftJumpCount);
+ max_rtt_ = TimeDelta::Zero();
+ avg_rtt_ = TimeDelta::Zero();
+ for (const TimeDelta& rtt : buf) {
+ if (rtt > max_rtt_) {
+ max_rtt_ = rtt;
+ }
+ avg_rtt_ += rtt;
+ }
+ avg_rtt_ = avg_rtt_ / static_cast<double>(buf.size());
+}
+
+TimeDelta RttFilter::Rtt() const {
+ return max_rtt_;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.h b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.h
new file mode 100644
index 0000000000..b8700b23ee
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.h
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_TIMING_RTT_FILTER_H_
+#define MODULES_VIDEO_CODING_TIMING_RTT_FILTER_H_
+
+#include <stdint.h>
+
+#include "absl/container/inlined_vector.h"
+#include "api/units/time_delta.h"
+
+namespace webrtc {
+
+class RttFilter {
+ public:
+ RttFilter();
+ RttFilter(const RttFilter&) = delete;
+ RttFilter& operator=(const RttFilter&) = delete;
+
+ // Resets the filter.
+ void Reset();
+ // Updates the filter with a new sample.
+ void Update(TimeDelta rtt);
+ // A getter function for the current RTT level.
+ TimeDelta Rtt() const;
+
+ private:
+ // The size of the drift and jump memory buffers
+ // and thus also the detection threshold for these
+ // detectors in number of samples.
+ static constexpr int kMaxDriftJumpCount = 5;
+ using BufferList = absl::InlinedVector<TimeDelta, kMaxDriftJumpCount>;
+
+ // Detects RTT jumps by comparing the difference between
+ // samples and average to the standard deviation.
+ // Returns true if the long time statistics should be updated
+ // and false otherwise
+ bool JumpDetection(TimeDelta rtt);
+
+ // Detects RTT drifts by comparing the difference between
+ // max and average to the standard deviation.
+ // Returns true if the long time statistics should be updated
+ // and false otherwise
+ bool DriftDetection(TimeDelta rtt);
+
+ // Computes the short time average and maximum of the vector buf.
+ void ShortRttFilter(const BufferList& buf);
+
+ bool got_non_zero_update_;
+ TimeDelta avg_rtt_;
+ // Variance units are TimeDelta^2. Store as ms^2.
+ int64_t var_rtt_;
+ TimeDelta max_rtt_;
+ uint32_t filt_fact_count_;
+ bool last_jump_positive_ = false;
+ BufferList jump_buf_;
+ BufferList drift_buf_;
+};
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_TIMING_RTT_FILTER_H_
diff --git a/third_party/libwebrtc/modules/video_coding/timing/rtt_filter_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter_gn/moz.build
new file mode 100644
index 0000000000..488a49d9c2
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter_gn/moz.build
@@ -0,0 +1,221 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+ ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ###
+ ### DO NOT edit it by hand. ###
+
+COMPILE_FLAGS["OS_INCLUDES"] = []
+AllowCompilerWarnings()
+
+DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1"
+DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True
+DEFINES["RTC_ENABLE_VP9"] = True
+DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0"
+DEFINES["WEBRTC_LIBRARY_IMPL"] = True
+DEFINES["WEBRTC_MOZILLA_BUILD"] = True
+DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0"
+DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0"
+
+FINAL_LIBRARY = "webrtc"
+
+
+LOCAL_INCLUDES += [
+ "!/ipc/ipdl/_ipdlheaders",
+ "!/third_party/libwebrtc/gen",
+ "/ipc/chromium/src",
+ "/third_party/libwebrtc/",
+ "/third_party/libwebrtc/third_party/abseil-cpp/",
+ "/tools/profiler/public"
+]
+
+UNIFIED_SOURCES += [
+ "/third_party/libwebrtc/modules/video_coding/timing/rtt_filter.cc"
+]
+
+if not CONFIG["MOZ_DEBUG"]:
+
+ DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0"
+ DEFINES["NDEBUG"] = True
+ DEFINES["NVALGRIND"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1":
+
+ DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1"
+
+if CONFIG["OS_TARGET"] == "Android":
+
+ DEFINES["ANDROID"] = True
+ DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1"
+ DEFINES["HAVE_SYS_UIO_H"] = True
+ DEFINES["WEBRTC_ANDROID"] = True
+ DEFINES["WEBRTC_ANDROID_OPENSLES"] = True
+ DEFINES["WEBRTC_LINUX"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_GNU_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+ OS_LIBS += [
+ "log"
+ ]
+
+if CONFIG["OS_TARGET"] == "Darwin":
+
+ DEFINES["WEBRTC_MAC"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True
+ DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0"
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["USE_AURA"] = "1"
+ DEFINES["USE_GLIB"] = "1"
+ DEFINES["USE_NSS_CERTS"] = "1"
+ DEFINES["USE_OZONE"] = "1"
+ DEFINES["USE_UDEV"] = True
+ DEFINES["WEBRTC_LINUX"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_FILE_OFFSET_BITS"] = "64"
+ DEFINES["_LARGEFILE64_SOURCE"] = True
+ DEFINES["_LARGEFILE_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "OpenBSD":
+
+ DEFINES["USE_GLIB"] = "1"
+ DEFINES["USE_OZONE"] = "1"
+ DEFINES["USE_X11"] = "1"
+ DEFINES["WEBRTC_BSD"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_FILE_OFFSET_BITS"] = "64"
+ DEFINES["_LARGEFILE64_SOURCE"] = True
+ DEFINES["_LARGEFILE_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "WINNT":
+
+ DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True
+ DEFINES["NOMINMAX"] = True
+ DEFINES["NTDDI_VERSION"] = "0x0A000000"
+ DEFINES["PSAPI_VERSION"] = "2"
+ DEFINES["UNICODE"] = True
+ DEFINES["USE_AURA"] = "1"
+ DEFINES["WEBRTC_WIN"] = True
+ DEFINES["WIN32"] = True
+ DEFINES["WIN32_LEAN_AND_MEAN"] = True
+ DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP"
+ DEFINES["WINVER"] = "0x0A00"
+ DEFINES["_ATL_NO_OPENGL"] = True
+ DEFINES["_CRT_RAND_S"] = True
+ DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True
+ DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True
+ DEFINES["_HAS_EXCEPTIONS"] = "0"
+ DEFINES["_HAS_NODISCARD"] = True
+ DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True
+ DEFINES["_SECURE_ATL"] = True
+ DEFINES["_UNICODE"] = True
+ DEFINES["_WIN32_WINNT"] = "0x0A00"
+ DEFINES["_WINDOWS"] = True
+ DEFINES["__STD_C"] = True
+
+if CONFIG["CPU_ARCH"] == "aarch64":
+
+ DEFINES["WEBRTC_ARCH_ARM64"] = True
+ DEFINES["WEBRTC_HAS_NEON"] = True
+
+if CONFIG["CPU_ARCH"] == "arm":
+
+ CXXFLAGS += [
+ "-mfpu=neon"
+ ]
+
+ DEFINES["WEBRTC_ARCH_ARM"] = True
+ DEFINES["WEBRTC_ARCH_ARM_V7"] = True
+ DEFINES["WEBRTC_HAS_NEON"] = True
+
+if CONFIG["CPU_ARCH"] == "mips32":
+
+ DEFINES["MIPS32_LE"] = True
+ DEFINES["MIPS_FPU_LE"] = True
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "mips64":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86":
+
+ DEFINES["WEBRTC_ENABLE_AVX2"] = True
+
+if CONFIG["CPU_ARCH"] == "x86_64":
+
+ DEFINES["WEBRTC_ENABLE_AVX2"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT":
+
+ DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0"
+
+if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["USE_X11"] = "1"
+
+if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android":
+
+ OS_LIBS += [
+ "android_support",
+ "unwind"
+ ]
+
+if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android":
+
+ CXXFLAGS += [
+ "-msse2"
+ ]
+
+ OS_LIBS += [
+ "android_support"
+ ]
+
+if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux":
+
+ CXXFLAGS += [
+ "-msse2"
+ ]
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+Library("rtt_filter_gn")
diff --git a/third_party/libwebrtc/modules/video_coding/timing/rtt_filter_unittest.cc b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter_unittest.cc
new file mode 100644
index 0000000000..05502e6f5b
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/rtt_filter_unittest.cc
@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/timing/rtt_filter.h"
+
+#include "api/units/time_delta.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+
+TEST(RttFilterTest, RttIsCapped) {
+ RttFilter rtt_filter;
+ rtt_filter.Update(TimeDelta::Seconds(500));
+
+ EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Seconds(3));
+}
+
+// If the difference between samples is more than away 2.5 stddev from the mean
+// then this is considered a jump. After more than 5 data points at the new
+// level, the RTT is reset to the new level.
+TEST(RttFilterTest, PositiveJumpDetection) {
+ RttFilter rtt_filter;
+
+ rtt_filter.Update(TimeDelta::Millis(200));
+ rtt_filter.Update(TimeDelta::Millis(200));
+ rtt_filter.Update(TimeDelta::Millis(200));
+
+ // Trigger 5 jumps.
+ rtt_filter.Update(TimeDelta::Millis(1400));
+ rtt_filter.Update(TimeDelta::Millis(1500));
+ rtt_filter.Update(TimeDelta::Millis(1600));
+ rtt_filter.Update(TimeDelta::Millis(1600));
+
+ EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Millis(1600));
+
+ rtt_filter.Update(TimeDelta::Millis(1600));
+ EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Millis(1600));
+}
+
+TEST(RttFilterTest, NegativeJumpDetection) {
+ RttFilter rtt_filter;
+
+ for (int i = 0; i < 10; ++i)
+ rtt_filter.Update(TimeDelta::Millis(1500));
+
+ // Trigger 5 negative data points that jump rtt down.
+ rtt_filter.Update(TimeDelta::Millis(200));
+ rtt_filter.Update(TimeDelta::Millis(200));
+ rtt_filter.Update(TimeDelta::Millis(200));
+ rtt_filter.Update(TimeDelta::Millis(200));
+ // Before 5 data points at the new level, max RTT is still 1500.
+ EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Millis(1500));
+
+ rtt_filter.Update(TimeDelta::Millis(300));
+ EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Millis(300));
+}
+
+TEST(RttFilterTest, JumpsResetByDirectionShift) {
+ RttFilter rtt_filter;
+ for (int i = 0; i < 10; ++i)
+ rtt_filter.Update(TimeDelta::Millis(1500));
+
+ // Trigger 4 negative jumps, then a positive one. This resets the jump
+ // detection.
+ rtt_filter.Update(TimeDelta::Millis(200));
+ rtt_filter.Update(TimeDelta::Millis(200));
+ rtt_filter.Update(TimeDelta::Millis(200));
+ rtt_filter.Update(TimeDelta::Millis(200));
+ rtt_filter.Update(TimeDelta::Millis(2000));
+ EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Millis(2000));
+
+ rtt_filter.Update(TimeDelta::Millis(300));
+ EXPECT_EQ(rtt_filter.Rtt(), TimeDelta::Millis(2000));
+}
+
+// If the difference between the max and average is more than 3.5 stddevs away
+// then a drift is detected, and a short filter is applied to find a new max
+// rtt.
+TEST(RttFilterTest, DriftDetection) {
+ RttFilter rtt_filter;
+
+ // Descend RTT by 30ms and settle at 700ms RTT. A drift is detected after rtt
+ // of 700ms is reported around 50 times for these targets.
+ constexpr TimeDelta kStartRtt = TimeDelta::Millis(1000);
+ constexpr TimeDelta kDriftTarget = TimeDelta::Millis(700);
+ constexpr TimeDelta kDelta = TimeDelta::Millis(30);
+ for (TimeDelta rtt = kStartRtt; rtt >= kDriftTarget; rtt -= kDelta)
+ rtt_filter.Update(rtt);
+
+ EXPECT_EQ(rtt_filter.Rtt(), kStartRtt);
+
+ for (int i = 0; i < 50; ++i)
+ rtt_filter.Update(kDriftTarget);
+ EXPECT_EQ(rtt_filter.Rtt(), kDriftTarget);
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator.cc b/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator.cc
new file mode 100644
index 0000000000..dc62ac674a
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator.cc
@@ -0,0 +1,169 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/timing/timestamp_extrapolator.h"
+
+#include <algorithm>
+
+#include "absl/types/optional.h"
+#include "rtc_base/numerics/sequence_number_unwrapper.h"
+
+namespace webrtc {
+
+namespace {
+
+constexpr double kLambda = 1;
+constexpr uint32_t kStartUpFilterDelayInPackets = 2;
+constexpr double kAlarmThreshold = 60e3;
+// in timestamp ticks, i.e. 15 ms
+constexpr double kAccDrift = 6600;
+constexpr double kAccMaxError = 7000;
+constexpr double kP11 = 1e10;
+
+} // namespace
+
+TimestampExtrapolator::TimestampExtrapolator(Timestamp start)
+ : start_(Timestamp::Zero()),
+ prev_(Timestamp::Zero()),
+ packet_count_(0),
+ detector_accumulator_pos_(0),
+ detector_accumulator_neg_(0) {
+ Reset(start);
+}
+
+void TimestampExtrapolator::Reset(Timestamp start) {
+ start_ = start;
+ prev_ = start_;
+ first_unwrapped_timestamp_ = absl::nullopt;
+ w_[0] = 90.0;
+ w_[1] = 0;
+ p_[0][0] = 1;
+ p_[1][1] = kP11;
+ p_[0][1] = p_[1][0] = 0;
+ unwrapper_ = RtpTimestampUnwrapper();
+ packet_count_ = 0;
+ detector_accumulator_pos_ = 0;
+ detector_accumulator_neg_ = 0;
+}
+
+void TimestampExtrapolator::Update(Timestamp now, uint32_t ts90khz) {
+ if (now - prev_ > TimeDelta::Seconds(10)) {
+ // Ten seconds without a complete frame.
+ // Reset the extrapolator
+ Reset(now);
+ } else {
+ prev_ = now;
+ }
+
+ // Remove offset to prevent badly scaled matrices
+ const TimeDelta offset = now - start_;
+ double t_ms = offset.ms();
+
+ int64_t unwrapped_ts90khz = unwrapper_.Unwrap(ts90khz);
+
+ if (!first_unwrapped_timestamp_) {
+ // Make an initial guess of the offset,
+ // should be almost correct since t_ms - start
+ // should about zero at this time.
+ w_[1] = -w_[0] * t_ms;
+ first_unwrapped_timestamp_ = unwrapped_ts90khz;
+ }
+
+ double residual =
+ (static_cast<double>(unwrapped_ts90khz) - *first_unwrapped_timestamp_) -
+ t_ms * w_[0] - w_[1];
+ if (DelayChangeDetection(residual) &&
+ packet_count_ >= kStartUpFilterDelayInPackets) {
+ // A sudden change of average network delay has been detected.
+ // Force the filter to adjust its offset parameter by changing
+ // the offset uncertainty. Don't do this during startup.
+ p_[1][1] = kP11;
+ }
+
+ if (prev_unwrapped_timestamp_ &&
+ unwrapped_ts90khz < prev_unwrapped_timestamp_) {
+ // Drop reordered frames.
+ return;
+ }
+
+ // T = [t(k) 1]';
+ // that = T'*w;
+ // K = P*T/(lambda + T'*P*T);
+ double K[2];
+ K[0] = p_[0][0] * t_ms + p_[0][1];
+ K[1] = p_[1][0] * t_ms + p_[1][1];
+ double TPT = kLambda + t_ms * K[0] + K[1];
+ K[0] /= TPT;
+ K[1] /= TPT;
+ // w = w + K*(ts(k) - that);
+ w_[0] = w_[0] + K[0] * residual;
+ w_[1] = w_[1] + K[1] * residual;
+ // P = 1/lambda*(P - K*T'*P);
+ double p00 =
+ 1 / kLambda * (p_[0][0] - (K[0] * t_ms * p_[0][0] + K[0] * p_[1][0]));
+ double p01 =
+ 1 / kLambda * (p_[0][1] - (K[0] * t_ms * p_[0][1] + K[0] * p_[1][1]));
+ p_[1][0] =
+ 1 / kLambda * (p_[1][0] - (K[1] * t_ms * p_[0][0] + K[1] * p_[1][0]));
+ p_[1][1] =
+ 1 / kLambda * (p_[1][1] - (K[1] * t_ms * p_[0][1] + K[1] * p_[1][1]));
+ p_[0][0] = p00;
+ p_[0][1] = p01;
+ prev_unwrapped_timestamp_ = unwrapped_ts90khz;
+ if (packet_count_ < kStartUpFilterDelayInPackets) {
+ packet_count_++;
+ }
+}
+
+absl::optional<Timestamp> TimestampExtrapolator::ExtrapolateLocalTime(
+ uint32_t timestamp90khz) const {
+ int64_t unwrapped_ts90khz = unwrapper_.PeekUnwrap(timestamp90khz);
+ RTC_DCHECK_GE(unwrapped_ts90khz, 0);
+
+ if (!first_unwrapped_timestamp_) {
+ return absl::nullopt;
+ } else if (packet_count_ < kStartUpFilterDelayInPackets) {
+ constexpr double kRtpTicksPerMs = 90;
+ TimeDelta diff = TimeDelta::Millis(
+ (unwrapped_ts90khz - *prev_unwrapped_timestamp_) / kRtpTicksPerMs);
+ if (diff.ms() < 0) {
+ RTC_DCHECK_GE(prev_.ms(), -diff.ms());
+ }
+ return prev_ + diff;
+ } else if (w_[0] < 1e-3) {
+ return start_;
+ } else {
+ double timestampDiff = unwrapped_ts90khz - *first_unwrapped_timestamp_;
+ auto diff_ms = static_cast<int64_t>((timestampDiff - w_[1]) / w_[0] + 0.5);
+ if (diff_ms < 0) {
+ RTC_DCHECK_GE(start_.ms(), -diff_ms);
+ }
+ return start_ + TimeDelta::Millis(diff_ms);
+ }
+}
+
+bool TimestampExtrapolator::DelayChangeDetection(double error) {
+ // CUSUM detection of sudden delay changes
+ error = (error > 0) ? std::min(error, kAccMaxError)
+ : std::max(error, -kAccMaxError);
+ detector_accumulator_pos_ =
+ std::max(detector_accumulator_pos_ + error - kAccDrift, double{0});
+ detector_accumulator_neg_ =
+ std::min(detector_accumulator_neg_ + error + kAccDrift, double{0});
+ if (detector_accumulator_pos_ > kAlarmThreshold ||
+ detector_accumulator_neg_ < -kAlarmThreshold) {
+ // Alarm
+ detector_accumulator_pos_ = detector_accumulator_neg_ = 0;
+ return true;
+ }
+ return false;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator.h b/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator.h
new file mode 100644
index 0000000000..6a9763943e
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_TIMING_TIMESTAMP_EXTRAPOLATOR_H_
+#define MODULES_VIDEO_CODING_TIMING_TIMESTAMP_EXTRAPOLATOR_H_
+
+#include <stdint.h>
+
+#include "absl/types/optional.h"
+#include "api/units/timestamp.h"
+#include "rtc_base/numerics/sequence_number_unwrapper.h"
+
+namespace webrtc {
+
+// Not thread safe.
+class TimestampExtrapolator {
+ public:
+ explicit TimestampExtrapolator(Timestamp start);
+ void Update(Timestamp now, uint32_t ts90khz);
+ absl::optional<Timestamp> ExtrapolateLocalTime(uint32_t timestamp90khz) const;
+ void Reset(Timestamp start);
+
+ private:
+ void CheckForWrapArounds(uint32_t ts90khz);
+ bool DelayChangeDetection(double error);
+
+ double w_[2];
+ double p_[2][2];
+ Timestamp start_;
+ Timestamp prev_;
+ absl::optional<int64_t> first_unwrapped_timestamp_;
+ RtpTimestampUnwrapper unwrapper_;
+ absl::optional<int64_t> prev_unwrapped_timestamp_;
+ uint32_t packet_count_;
+ double detector_accumulator_pos_;
+ double detector_accumulator_neg_;
+};
+
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_TIMING_TIMESTAMP_EXTRAPOLATOR_H_
diff --git a/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator_gn/moz.build
new file mode 100644
index 0000000000..4cd085dd09
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator_gn/moz.build
@@ -0,0 +1,221 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+ ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ###
+ ### DO NOT edit it by hand. ###
+
+COMPILE_FLAGS["OS_INCLUDES"] = []
+AllowCompilerWarnings()
+
+DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1"
+DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True
+DEFINES["RTC_ENABLE_VP9"] = True
+DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0"
+DEFINES["WEBRTC_LIBRARY_IMPL"] = True
+DEFINES["WEBRTC_MOZILLA_BUILD"] = True
+DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0"
+DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0"
+
+FINAL_LIBRARY = "webrtc"
+
+
+LOCAL_INCLUDES += [
+ "!/ipc/ipdl/_ipdlheaders",
+ "!/third_party/libwebrtc/gen",
+ "/ipc/chromium/src",
+ "/third_party/libwebrtc/",
+ "/third_party/libwebrtc/third_party/abseil-cpp/",
+ "/tools/profiler/public"
+]
+
+UNIFIED_SOURCES += [
+ "/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator.cc"
+]
+
+if not CONFIG["MOZ_DEBUG"]:
+
+ DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0"
+ DEFINES["NDEBUG"] = True
+ DEFINES["NVALGRIND"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1":
+
+ DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1"
+
+if CONFIG["OS_TARGET"] == "Android":
+
+ DEFINES["ANDROID"] = True
+ DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1"
+ DEFINES["HAVE_SYS_UIO_H"] = True
+ DEFINES["WEBRTC_ANDROID"] = True
+ DEFINES["WEBRTC_ANDROID_OPENSLES"] = True
+ DEFINES["WEBRTC_LINUX"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_GNU_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+ OS_LIBS += [
+ "log"
+ ]
+
+if CONFIG["OS_TARGET"] == "Darwin":
+
+ DEFINES["WEBRTC_MAC"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True
+ DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0"
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["USE_AURA"] = "1"
+ DEFINES["USE_GLIB"] = "1"
+ DEFINES["USE_NSS_CERTS"] = "1"
+ DEFINES["USE_OZONE"] = "1"
+ DEFINES["USE_UDEV"] = True
+ DEFINES["WEBRTC_LINUX"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_FILE_OFFSET_BITS"] = "64"
+ DEFINES["_LARGEFILE64_SOURCE"] = True
+ DEFINES["_LARGEFILE_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "OpenBSD":
+
+ DEFINES["USE_GLIB"] = "1"
+ DEFINES["USE_OZONE"] = "1"
+ DEFINES["USE_X11"] = "1"
+ DEFINES["WEBRTC_BSD"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_FILE_OFFSET_BITS"] = "64"
+ DEFINES["_LARGEFILE64_SOURCE"] = True
+ DEFINES["_LARGEFILE_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "WINNT":
+
+ DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True
+ DEFINES["NOMINMAX"] = True
+ DEFINES["NTDDI_VERSION"] = "0x0A000000"
+ DEFINES["PSAPI_VERSION"] = "2"
+ DEFINES["UNICODE"] = True
+ DEFINES["USE_AURA"] = "1"
+ DEFINES["WEBRTC_WIN"] = True
+ DEFINES["WIN32"] = True
+ DEFINES["WIN32_LEAN_AND_MEAN"] = True
+ DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP"
+ DEFINES["WINVER"] = "0x0A00"
+ DEFINES["_ATL_NO_OPENGL"] = True
+ DEFINES["_CRT_RAND_S"] = True
+ DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True
+ DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True
+ DEFINES["_HAS_EXCEPTIONS"] = "0"
+ DEFINES["_HAS_NODISCARD"] = True
+ DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True
+ DEFINES["_SECURE_ATL"] = True
+ DEFINES["_UNICODE"] = True
+ DEFINES["_WIN32_WINNT"] = "0x0A00"
+ DEFINES["_WINDOWS"] = True
+ DEFINES["__STD_C"] = True
+
+if CONFIG["CPU_ARCH"] == "aarch64":
+
+ DEFINES["WEBRTC_ARCH_ARM64"] = True
+ DEFINES["WEBRTC_HAS_NEON"] = True
+
+if CONFIG["CPU_ARCH"] == "arm":
+
+ CXXFLAGS += [
+ "-mfpu=neon"
+ ]
+
+ DEFINES["WEBRTC_ARCH_ARM"] = True
+ DEFINES["WEBRTC_ARCH_ARM_V7"] = True
+ DEFINES["WEBRTC_HAS_NEON"] = True
+
+if CONFIG["CPU_ARCH"] == "mips32":
+
+ DEFINES["MIPS32_LE"] = True
+ DEFINES["MIPS_FPU_LE"] = True
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "mips64":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86":
+
+ DEFINES["WEBRTC_ENABLE_AVX2"] = True
+
+if CONFIG["CPU_ARCH"] == "x86_64":
+
+ DEFINES["WEBRTC_ENABLE_AVX2"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT":
+
+ DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0"
+
+if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["USE_X11"] = "1"
+
+if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android":
+
+ OS_LIBS += [
+ "android_support",
+ "unwind"
+ ]
+
+if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android":
+
+ CXXFLAGS += [
+ "-msse2"
+ ]
+
+ OS_LIBS += [
+ "android_support"
+ ]
+
+if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux":
+
+ CXXFLAGS += [
+ "-msse2"
+ ]
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+Library("timestamp_extrapolator_gn")
diff --git a/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator_unittest.cc b/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator_unittest.cc
new file mode 100644
index 0000000000..0b5fd74a8e
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/timestamp_extrapolator_unittest.cc
@@ -0,0 +1,221 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/timing/timestamp_extrapolator.h"
+
+#include <stdint.h>
+
+#include <limits>
+
+#include "absl/types/optional.h"
+#include "api/units/frequency.h"
+#include "api/units/time_delta.h"
+#include "api/units/timestamp.h"
+#include "system_wrappers/include/clock.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+
+using ::testing::Eq;
+using ::testing::Optional;
+
+namespace {
+
+constexpr Frequency kRtpHz = Frequency::KiloHertz(90);
+constexpr Frequency k25Fps = Frequency::Hertz(25);
+constexpr TimeDelta k25FpsDelay = 1 / k25Fps;
+
+} // namespace
+
+TEST(TimestampExtrapolatorTest, ExtrapolationOccursAfter2Packets) {
+ SimulatedClock clock(Timestamp::Millis(1337));
+ TimestampExtrapolator ts_extrapolator(clock.CurrentTime());
+
+ // No packets so no timestamp.
+ EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(90000), Eq(absl::nullopt));
+
+ uint32_t rtp = 90000;
+ clock.AdvanceTime(k25FpsDelay);
+ // First result is a bit confusing since it is based off the "start" time,
+ // which is arbitrary.
+ ts_extrapolator.Update(clock.CurrentTime(), rtp);
+ EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+ Optional(clock.CurrentTime()));
+
+ rtp += kRtpHz / k25Fps;
+ clock.AdvanceTime(k25FpsDelay);
+ ts_extrapolator.Update(clock.CurrentTime(), rtp);
+ EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+ Optional(clock.CurrentTime()));
+ EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp + 90000),
+ Optional(clock.CurrentTime() + TimeDelta::Seconds(1)));
+}
+
+TEST(TimestampExtrapolatorTest, ResetsAfter10SecondPause) {
+ SimulatedClock clock(Timestamp::Millis(1337));
+ TimestampExtrapolator ts_extrapolator(clock.CurrentTime());
+
+ uint32_t rtp = 90000;
+ ts_extrapolator.Update(clock.CurrentTime(), rtp);
+ EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+ Optional(clock.CurrentTime()));
+
+ rtp += kRtpHz / k25Fps;
+ clock.AdvanceTime(k25FpsDelay);
+ ts_extrapolator.Update(clock.CurrentTime(), rtp);
+ EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+ Optional(clock.CurrentTime()));
+
+ rtp += 10 * kRtpHz.hertz();
+ clock.AdvanceTime(TimeDelta::Seconds(10) + TimeDelta::Micros(1));
+ ts_extrapolator.Update(clock.CurrentTime(), rtp);
+ EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+ Optional(clock.CurrentTime()));
+}
+
+TEST(TimestampExtrapolatorTest, TimestampExtrapolatesMultipleRtpWrapArounds) {
+ SimulatedClock clock(Timestamp::Millis(1337));
+ TimestampExtrapolator ts_extrapolator(clock.CurrentTime());
+
+ uint32_t rtp = std::numeric_limits<uint32_t>::max();
+ ts_extrapolator.Update(clock.CurrentTime(), rtp);
+ EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+ Optional(clock.CurrentTime()));
+
+ // One overflow. Static cast to avoid undefined behaviour with +=.
+ rtp += static_cast<uint32_t>(kRtpHz / k25Fps);
+ clock.AdvanceTime(k25FpsDelay);
+ ts_extrapolator.Update(clock.CurrentTime(), rtp);
+ EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+ Optional(clock.CurrentTime()));
+
+ // Assert that extrapolation works across the boundary as expected.
+ EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp + 90000),
+ Optional(clock.CurrentTime() + TimeDelta::Seconds(1)));
+ // This is not quite 1s since the math always rounds up.
+ EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp - 90000),
+ Optional(clock.CurrentTime() - TimeDelta::Millis(999)));
+
+ // In order to avoid a wrap arounds reset, add a packet every 10s until we
+ // overflow twice.
+ constexpr TimeDelta kRtpOverflowDelay =
+ std::numeric_limits<uint32_t>::max() / kRtpHz;
+ const Timestamp overflow_time = clock.CurrentTime() + kRtpOverflowDelay * 2;
+
+ while (clock.CurrentTime() < overflow_time) {
+ clock.AdvanceTime(TimeDelta::Seconds(10));
+ // Static-cast before += to avoid undefined behaviour of overflow.
+ rtp += static_cast<uint32_t>(kRtpHz * TimeDelta::Seconds(10));
+ ts_extrapolator.Update(clock.CurrentTime(), rtp);
+ EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+ Optional(clock.CurrentTime()));
+ }
+}
+
+TEST(TimestampExtrapolatorTest, NegativeRtpTimestampWrapAround) {
+ SimulatedClock clock(Timestamp::Millis(1337));
+ TimestampExtrapolator ts_extrapolator(clock.CurrentTime());
+ uint32_t rtp = 0;
+ ts_extrapolator.Update(clock.CurrentTime(), rtp);
+ EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+ Optional(clock.CurrentTime()));
+ // Go backwards!
+ rtp -= kRtpHz.hertz();
+ EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+ Optional(clock.CurrentTime() - TimeDelta::Seconds(1)));
+}
+
+TEST(TimestampExtrapolatorTest, Slow90KHzClock) {
+ // This simulates a slow camera, which produces frames at 24Hz instead of
+ // 25Hz. The extrapolator should be able to resolve this with enough data.
+ SimulatedClock clock(Timestamp::Millis(1337));
+ TimestampExtrapolator ts_extrapolator(clock.CurrentTime());
+
+ constexpr TimeDelta k24FpsDelay = 1 / Frequency::Hertz(24);
+ uint32_t rtp = 90000;
+ ts_extrapolator.Update(clock.CurrentTime(), rtp);
+
+ // Slow camera will increment RTP at 25 FPS rate even though its producing at
+ // 24 FPS. After 25 frames the extrapolator should settle at this rate.
+ for (int i = 0; i < 25; ++i) {
+ rtp += kRtpHz / k25Fps;
+ clock.AdvanceTime(k24FpsDelay);
+ ts_extrapolator.Update(clock.CurrentTime(), rtp);
+ }
+
+ // The camera would normally produce 25 frames in 90K ticks, but is slow
+ // so takes 1s + k24FpsDelay for 90K ticks.
+ constexpr Frequency kSlowRtpHz = 90000 / (25 * k24FpsDelay);
+ // The extrapolator will be predicting that time at millisecond precision.
+ auto ts = ts_extrapolator.ExtrapolateLocalTime(rtp + kSlowRtpHz.hertz());
+ ASSERT_TRUE(ts.has_value());
+ EXPECT_EQ(ts->ms(), clock.TimeInMilliseconds() + 1000);
+}
+
+TEST(TimestampExtrapolatorTest, Fast90KHzClock) {
+ // This simulates a fast camera, which produces frames at 26Hz instead of
+ // 25Hz. The extrapolator should be able to resolve this with enough data.
+ SimulatedClock clock(Timestamp::Millis(1337));
+ TimestampExtrapolator ts_extrapolator(clock.CurrentTime());
+
+ constexpr TimeDelta k26FpsDelay = 1 / Frequency::Hertz(26);
+ uint32_t rtp = 90000;
+ ts_extrapolator.Update(clock.CurrentTime(), rtp);
+
+ // Fast camera will increment RTP at 25 FPS rate even though its producing at
+ // 26 FPS. After 25 frames the extrapolator should settle at this rate.
+ for (int i = 0; i < 25; ++i) {
+ rtp += kRtpHz / k25Fps;
+ clock.AdvanceTime(k26FpsDelay);
+ ts_extrapolator.Update(clock.CurrentTime(), rtp);
+ }
+
+ // The camera would normally produce 25 frames in 90K ticks, but is slow
+ // so takes 1s + k24FpsDelay for 90K ticks.
+ constexpr Frequency kSlowRtpHz = 90000 / (25 * k26FpsDelay);
+ // The extrapolator will be predicting that time at millisecond precision.
+ auto ts = ts_extrapolator.ExtrapolateLocalTime(rtp + kSlowRtpHz.hertz());
+ ASSERT_TRUE(ts.has_value());
+ EXPECT_EQ(ts->ms(), clock.TimeInMilliseconds() + 1000);
+}
+
+TEST(TimestampExtrapolatorTest, TimestampJump) {
+ // This simulates a jump in RTP timestamp, which could occur if a camera was
+ // swapped for example.
+ SimulatedClock clock(Timestamp::Millis(1337));
+ TimestampExtrapolator ts_extrapolator(clock.CurrentTime());
+
+ uint32_t rtp = 90000;
+ clock.AdvanceTime(k25FpsDelay);
+ ts_extrapolator.Update(clock.CurrentTime(), rtp);
+ rtp += kRtpHz / k25Fps;
+ clock.AdvanceTime(k25FpsDelay);
+ ts_extrapolator.Update(clock.CurrentTime(), rtp);
+ rtp += kRtpHz / k25Fps;
+ clock.AdvanceTime(k25FpsDelay);
+ ts_extrapolator.Update(clock.CurrentTime(), rtp);
+ EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+ Optional(clock.CurrentTime()));
+ EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp + 90000),
+ Optional(clock.CurrentTime() + TimeDelta::Seconds(1)));
+
+ // Jump RTP.
+ uint32_t new_rtp = 1337 * 90000;
+ clock.AdvanceTime(k25FpsDelay);
+ ts_extrapolator.Update(clock.CurrentTime(), new_rtp);
+ new_rtp += kRtpHz / k25Fps;
+ clock.AdvanceTime(k25FpsDelay);
+ ts_extrapolator.Update(clock.CurrentTime(), new_rtp);
+ EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(new_rtp),
+ Optional(clock.CurrentTime()));
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/timing/timing.cc b/third_party/libwebrtc/modules/video_coding/timing/timing.cc
new file mode 100644
index 0000000000..0b61d5a35e
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/timing.cc
@@ -0,0 +1,297 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/timing/timing.h"
+
+#include <algorithm>
+
+#include "api/units/time_delta.h"
+#include "modules/video_coding/timing/timestamp_extrapolator.h"
+#include "rtc_base/experiments/field_trial_parser.h"
+#include "rtc_base/logging.h"
+#include "system_wrappers/include/clock.h"
+
+namespace webrtc {
+namespace {
+
+// Default pacing that is used for the low-latency renderer path.
+constexpr TimeDelta kZeroPlayoutDelayDefaultMinPacing = TimeDelta::Millis(8);
+constexpr TimeDelta kLowLatencyStreamMaxPlayoutDelayThreshold =
+ TimeDelta::Millis(500);
+
+void CheckDelaysValid(TimeDelta min_delay, TimeDelta max_delay) {
+ if (min_delay > max_delay) {
+ RTC_LOG(LS_ERROR)
+ << "Playout delays set incorrectly: min playout delay (" << min_delay
+ << ") > max playout delay (" << max_delay
+ << "). This is undefined behaviour. Application writers should "
+ "ensure that the min delay is always less than or equals max "
+ "delay. If trying to use the playout delay header extensions "
+ "described in "
+ "https://webrtc.googlesource.com/src/+/refs/heads/main/docs/"
+ "native-code/rtp-hdrext/playout-delay/, be careful that a playout "
+ "delay hint or A/V sync settings may have caused this conflict.";
+ }
+}
+
+} // namespace
+
+VCMTiming::VCMTiming(Clock* clock, const FieldTrialsView& field_trials)
+ : clock_(clock),
+ ts_extrapolator_(
+ std::make_unique<TimestampExtrapolator>(clock_->CurrentTime())),
+ codec_timer_(std::make_unique<CodecTimer>()),
+ render_delay_(kDefaultRenderDelay),
+ min_playout_delay_(TimeDelta::Zero()),
+ max_playout_delay_(TimeDelta::Seconds(10)),
+ jitter_delay_(TimeDelta::Zero()),
+ current_delay_(TimeDelta::Zero()),
+ prev_frame_timestamp_(0),
+ num_decoded_frames_(0),
+ zero_playout_delay_min_pacing_("min_pacing",
+ kZeroPlayoutDelayDefaultMinPacing),
+ last_decode_scheduled_(Timestamp::Zero()) {
+ ParseFieldTrial({&zero_playout_delay_min_pacing_},
+ field_trials.Lookup("WebRTC-ZeroPlayoutDelay"));
+}
+
+void VCMTiming::Reset() {
+ MutexLock lock(&mutex_);
+ ts_extrapolator_->Reset(clock_->CurrentTime());
+ codec_timer_ = std::make_unique<CodecTimer>();
+ render_delay_ = kDefaultRenderDelay;
+ min_playout_delay_ = TimeDelta::Zero();
+ jitter_delay_ = TimeDelta::Zero();
+ current_delay_ = TimeDelta::Zero();
+ prev_frame_timestamp_ = 0;
+}
+
+void VCMTiming::set_render_delay(TimeDelta render_delay) {
+ MutexLock lock(&mutex_);
+ render_delay_ = render_delay;
+}
+
+TimeDelta VCMTiming::min_playout_delay() const {
+ MutexLock lock(&mutex_);
+ return min_playout_delay_;
+}
+
+void VCMTiming::set_min_playout_delay(TimeDelta min_playout_delay) {
+ MutexLock lock(&mutex_);
+ if (min_playout_delay_ != min_playout_delay) {
+ CheckDelaysValid(min_playout_delay, max_playout_delay_);
+ min_playout_delay_ = min_playout_delay;
+ }
+}
+
+void VCMTiming::set_max_playout_delay(TimeDelta max_playout_delay) {
+ MutexLock lock(&mutex_);
+ if (max_playout_delay_ != max_playout_delay) {
+ CheckDelaysValid(min_playout_delay_, max_playout_delay);
+ max_playout_delay_ = max_playout_delay;
+ }
+}
+
+void VCMTiming::SetJitterDelay(TimeDelta jitter_delay) {
+ MutexLock lock(&mutex_);
+ if (jitter_delay != jitter_delay_) {
+ jitter_delay_ = jitter_delay;
+ // When in initial state, set current delay to minimum delay.
+ if (current_delay_.IsZero()) {
+ current_delay_ = jitter_delay_;
+ }
+ }
+}
+
+void VCMTiming::UpdateCurrentDelay(uint32_t frame_timestamp) {
+ MutexLock lock(&mutex_);
+ TimeDelta target_delay = TargetDelayInternal();
+
+ if (current_delay_.IsZero()) {
+ // Not initialized, set current delay to target.
+ current_delay_ = target_delay;
+ } else if (target_delay != current_delay_) {
+ TimeDelta delay_diff = target_delay - current_delay_;
+ // Never change the delay with more than 100 ms every second. If we're
+ // changing the delay in too large steps we will get noticeable freezes. By
+ // limiting the change we can increase the delay in smaller steps, which
+ // will be experienced as the video is played in slow motion. When lowering
+ // the delay the video will be played at a faster pace.
+ TimeDelta max_change = TimeDelta::Zero();
+ if (frame_timestamp < 0x0000ffff && prev_frame_timestamp_ > 0xffff0000) {
+ // wrap
+ max_change =
+ TimeDelta::Millis(kDelayMaxChangeMsPerS *
+ (frame_timestamp + (static_cast<int64_t>(1) << 32) -
+ prev_frame_timestamp_) /
+ 90000);
+ } else {
+ max_change =
+ TimeDelta::Millis(kDelayMaxChangeMsPerS *
+ (frame_timestamp - prev_frame_timestamp_) / 90000);
+ }
+
+ if (max_change <= TimeDelta::Zero()) {
+ // Any changes less than 1 ms are truncated and will be postponed.
+ // Negative change will be due to reordering and should be ignored.
+ return;
+ }
+ delay_diff = std::max(delay_diff, -max_change);
+ delay_diff = std::min(delay_diff, max_change);
+
+ current_delay_ = current_delay_ + delay_diff;
+ }
+ prev_frame_timestamp_ = frame_timestamp;
+}
+
+void VCMTiming::UpdateCurrentDelay(Timestamp render_time,
+ Timestamp actual_decode_time) {
+ MutexLock lock(&mutex_);
+ TimeDelta target_delay = TargetDelayInternal();
+ TimeDelta delayed =
+ (actual_decode_time - render_time) + RequiredDecodeTime() + render_delay_;
+
+ // Only consider `delayed` as negative by more than a few microseconds.
+ if (delayed.ms() < 0) {
+ return;
+ }
+ if (current_delay_ + delayed <= target_delay) {
+ current_delay_ += delayed;
+ } else {
+ current_delay_ = target_delay;
+ }
+}
+
+void VCMTiming::StopDecodeTimer(TimeDelta decode_time, Timestamp now) {
+ MutexLock lock(&mutex_);
+ codec_timer_->AddTiming(decode_time.ms(), now.ms());
+ RTC_DCHECK_GE(decode_time, TimeDelta::Zero());
+ ++num_decoded_frames_;
+}
+
+void VCMTiming::IncomingTimestamp(uint32_t rtp_timestamp, Timestamp now) {
+ MutexLock lock(&mutex_);
+ ts_extrapolator_->Update(now, rtp_timestamp);
+}
+
+Timestamp VCMTiming::RenderTime(uint32_t frame_timestamp, Timestamp now) const {
+ MutexLock lock(&mutex_);
+ return RenderTimeInternal(frame_timestamp, now);
+}
+
+void VCMTiming::SetLastDecodeScheduledTimestamp(
+ Timestamp last_decode_scheduled) {
+ MutexLock lock(&mutex_);
+ last_decode_scheduled_ = last_decode_scheduled;
+}
+
+Timestamp VCMTiming::RenderTimeInternal(uint32_t frame_timestamp,
+ Timestamp now) const {
+ if (UseLowLatencyRendering()) {
+ // Render as soon as possible or with low-latency renderer algorithm.
+ return Timestamp::Zero();
+ }
+ // Note that TimestampExtrapolator::ExtrapolateLocalTime is not a const
+ // method; it mutates the object's wraparound state.
+ Timestamp estimated_complete_time =
+ ts_extrapolator_->ExtrapolateLocalTime(frame_timestamp).value_or(now);
+
+ // Make sure the actual delay stays in the range of `min_playout_delay_`
+ // and `max_playout_delay_`.
+ TimeDelta actual_delay =
+ current_delay_.Clamped(min_playout_delay_, max_playout_delay_);
+ return estimated_complete_time + actual_delay;
+}
+
+TimeDelta VCMTiming::RequiredDecodeTime() const {
+ const int decode_time_ms = codec_timer_->RequiredDecodeTimeMs();
+ RTC_DCHECK_GE(decode_time_ms, 0);
+ return TimeDelta::Millis(decode_time_ms);
+}
+
+TimeDelta VCMTiming::MaxWaitingTime(Timestamp render_time,
+ Timestamp now,
+ bool too_many_frames_queued) const {
+ MutexLock lock(&mutex_);
+
+ if (render_time.IsZero() && zero_playout_delay_min_pacing_->us() > 0 &&
+ min_playout_delay_.IsZero() && max_playout_delay_ > TimeDelta::Zero()) {
+ // `render_time` == 0 indicates that the frame should be decoded and
+ // rendered as soon as possible. However, the decoder can be choked if too
+ // many frames are sent at once. Therefore, limit the interframe delay to
+ // |zero_playout_delay_min_pacing_| unless too many frames are queued in
+ // which case the frames are sent to the decoder at once.
+ if (too_many_frames_queued) {
+ return TimeDelta::Zero();
+ }
+ Timestamp earliest_next_decode_start_time =
+ last_decode_scheduled_ + zero_playout_delay_min_pacing_;
+ TimeDelta max_wait_time = now >= earliest_next_decode_start_time
+ ? TimeDelta::Zero()
+ : earliest_next_decode_start_time - now;
+ return max_wait_time;
+ }
+ return render_time - now - RequiredDecodeTime() - render_delay_;
+}
+
+TimeDelta VCMTiming::TargetVideoDelay() const {
+ MutexLock lock(&mutex_);
+ return TargetDelayInternal();
+}
+
+TimeDelta VCMTiming::TargetDelayInternal() const {
+ return std::max(min_playout_delay_,
+ jitter_delay_ + RequiredDecodeTime() + render_delay_);
+}
+
+VideoFrame::RenderParameters VCMTiming::RenderParameters() const {
+ MutexLock lock(&mutex_);
+ return {.use_low_latency_rendering = UseLowLatencyRendering(),
+ .max_composition_delay_in_frames = max_composition_delay_in_frames_};
+}
+
+bool VCMTiming::UseLowLatencyRendering() const {
+ // min_playout_delay_==0,
+ // max_playout_delay_<=kLowLatencyStreamMaxPlayoutDelayThreshold indicates
+ // that the low-latency path should be used, which means that frames should be
+ // decoded and rendered as soon as possible.
+ return min_playout_delay_.IsZero() &&
+ max_playout_delay_ <= kLowLatencyStreamMaxPlayoutDelayThreshold;
+}
+
+VCMTiming::VideoDelayTimings VCMTiming::GetTimings() const {
+ MutexLock lock(&mutex_);
+ return VideoDelayTimings{.max_decode_duration = RequiredDecodeTime(),
+ .current_delay = current_delay_,
+ .target_delay = TargetDelayInternal(),
+ .jitter_buffer_delay = jitter_delay_,
+ .min_playout_delay = min_playout_delay_,
+ .max_playout_delay = max_playout_delay_,
+ .render_delay = render_delay_,
+ .num_decoded_frames = num_decoded_frames_};
+}
+
+void VCMTiming::SetTimingFrameInfo(const TimingFrameInfo& info) {
+ MutexLock lock(&mutex_);
+ timing_frame_info_.emplace(info);
+}
+
+absl::optional<TimingFrameInfo> VCMTiming::GetTimingFrameInfo() {
+ MutexLock lock(&mutex_);
+ return timing_frame_info_;
+}
+
+void VCMTiming::SetMaxCompositionDelayInFrames(
+ absl::optional<int> max_composition_delay_in_frames) {
+ MutexLock lock(&mutex_);
+ max_composition_delay_in_frames_ = max_composition_delay_in_frames;
+}
+
+} // namespace webrtc
diff --git a/third_party/libwebrtc/modules/video_coding/timing/timing.h b/third_party/libwebrtc/modules/video_coding/timing/timing.h
new file mode 100644
index 0000000000..727527f009
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/timing.h
@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_TIMING_TIMING_H_
+#define MODULES_VIDEO_CODING_TIMING_TIMING_H_
+
+#include <memory>
+
+#include "absl/types/optional.h"
+#include "api/field_trials_view.h"
+#include "api/units/time_delta.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_timing.h"
+#include "modules/video_coding/timing/codec_timer.h"
+#include "modules/video_coding/timing/timestamp_extrapolator.h"
+#include "rtc_base/experiments/field_trial_parser.h"
+#include "rtc_base/synchronization/mutex.h"
+#include "rtc_base/thread_annotations.h"
+#include "system_wrappers/include/clock.h"
+
+namespace webrtc {
+
+class VCMTiming {
+ public:
+ static constexpr auto kDefaultRenderDelay = TimeDelta::Millis(10);
+ static constexpr auto kDelayMaxChangeMsPerS = 100;
+
+ VCMTiming(Clock* clock, const FieldTrialsView& field_trials);
+ virtual ~VCMTiming() = default;
+
+ // Resets the timing to the initial state.
+ void Reset();
+
+ // Set the amount of time needed to render an image. Defaults to 10 ms.
+ void set_render_delay(TimeDelta render_delay);
+
+ // Set the minimum time the video must be delayed on the receiver to
+ // get the desired jitter buffer level.
+ void SetJitterDelay(TimeDelta required_delay);
+
+ // Set/get the minimum playout delay from capture to render.
+ TimeDelta min_playout_delay() const;
+ void set_min_playout_delay(TimeDelta min_playout_delay);
+
+ // Set/get the maximum playout delay from capture to render in ms.
+ void set_max_playout_delay(TimeDelta max_playout_delay);
+
+ // Increases or decreases the current delay to get closer to the target delay.
+ // Calculates how long it has been since the previous call to this function,
+ // and increases/decreases the delay in proportion to the time difference.
+ void UpdateCurrentDelay(uint32_t frame_timestamp);
+
+ // Increases or decreases the current delay to get closer to the target delay.
+ // Given the actual decode time in ms and the render time in ms for a frame,
+ // this function calculates how late the frame is and increases the delay
+ // accordingly.
+ void UpdateCurrentDelay(Timestamp render_time, Timestamp actual_decode_time);
+
+ // Stops the decoder timer, should be called when the decoder returns a frame
+ // or when the decoded frame callback is called.
+ void StopDecodeTimer(TimeDelta decode_time, Timestamp now);
+
+ // Used to report that a frame is passed to decoding. Updates the timestamp
+ // filter which is used to map between timestamps and receiver system time.
+ virtual void IncomingTimestamp(uint32_t rtp_timestamp,
+ Timestamp last_packet_time);
+
+ // Returns the receiver system time when the frame with timestamp
+ // `frame_timestamp` should be rendered, assuming that the system time
+ // currently is `now`.
+ virtual Timestamp RenderTime(uint32_t frame_timestamp, Timestamp now) const;
+
+ // Returns the maximum time in ms that we can wait for a frame to become
+ // complete before we must pass it to the decoder. render_time==0 indicates
+ // that the frames should be processed as quickly as possible, with possibly
+ // only a small delay added to make sure that the decoder is not overloaded.
+ // In this case, the parameter too_many_frames_queued is used to signal that
+ // the decode queue is full and that the frame should be decoded as soon as
+ // possible.
+ virtual TimeDelta MaxWaitingTime(Timestamp render_time,
+ Timestamp now,
+ bool too_many_frames_queued) const;
+
+ // Returns the current target delay which is required delay + decode time +
+ // render delay.
+ TimeDelta TargetVideoDelay() const;
+
+ // Return current timing information. Returns true if the first frame has been
+ // decoded, false otherwise.
+ struct VideoDelayTimings {
+ TimeDelta max_decode_duration;
+ TimeDelta current_delay;
+ TimeDelta target_delay;
+ TimeDelta jitter_buffer_delay;
+ TimeDelta min_playout_delay;
+ TimeDelta max_playout_delay;
+ TimeDelta render_delay;
+ size_t num_decoded_frames;
+ };
+ VideoDelayTimings GetTimings() const;
+
+ void SetTimingFrameInfo(const TimingFrameInfo& info);
+ absl::optional<TimingFrameInfo> GetTimingFrameInfo();
+
+ void SetMaxCompositionDelayInFrames(
+ absl::optional<int> max_composition_delay_in_frames);
+
+ VideoFrame::RenderParameters RenderParameters() const;
+
+ // Updates the last time a frame was scheduled for decoding.
+ void SetLastDecodeScheduledTimestamp(Timestamp last_decode_scheduled);
+
+ protected:
+ TimeDelta RequiredDecodeTime() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+ Timestamp RenderTimeInternal(uint32_t frame_timestamp, Timestamp now) const
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+ TimeDelta TargetDelayInternal() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+ bool UseLowLatencyRendering() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+
+ private:
+ mutable Mutex mutex_;
+ Clock* const clock_;
+ const std::unique_ptr<TimestampExtrapolator> ts_extrapolator_
+ RTC_PT_GUARDED_BY(mutex_);
+ std::unique_ptr<CodecTimer> codec_timer_ RTC_GUARDED_BY(mutex_)
+ RTC_PT_GUARDED_BY(mutex_);
+ TimeDelta render_delay_ RTC_GUARDED_BY(mutex_);
+ // Best-effort playout delay range for frames from capture to render.
+ // The receiver tries to keep the delay between `min_playout_delay_ms_`
+ // and `max_playout_delay_ms_` taking the network jitter into account.
+ // A special case is where min_playout_delay_ms_ = max_playout_delay_ms_ = 0,
+ // in which case the receiver tries to play the frames as they arrive.
+ TimeDelta min_playout_delay_ RTC_GUARDED_BY(mutex_);
+ TimeDelta max_playout_delay_ RTC_GUARDED_BY(mutex_);
+ TimeDelta jitter_delay_ RTC_GUARDED_BY(mutex_);
+ TimeDelta current_delay_ RTC_GUARDED_BY(mutex_);
+ uint32_t prev_frame_timestamp_ RTC_GUARDED_BY(mutex_);
+ absl::optional<TimingFrameInfo> timing_frame_info_ RTC_GUARDED_BY(mutex_);
+ size_t num_decoded_frames_ RTC_GUARDED_BY(mutex_);
+ absl::optional<int> max_composition_delay_in_frames_ RTC_GUARDED_BY(mutex_);
+ // Set by the field trial WebRTC-ZeroPlayoutDelay. The parameter min_pacing
+ // determines the minimum delay between frames scheduled for decoding that is
+ // used when min playout delay=0 and max playout delay>=0.
+ FieldTrialParameter<TimeDelta> zero_playout_delay_min_pacing_
+ RTC_GUARDED_BY(mutex_);
+ // Timestamp at which the last frame was scheduled to be sent to the decoder.
+ // Used only when the RTP header extension playout delay is set to min=0 ms
+ // which is indicated by a render time set to 0.
+ Timestamp last_decode_scheduled_ RTC_GUARDED_BY(mutex_);
+};
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_TIMING_TIMING_H_
diff --git a/third_party/libwebrtc/modules/video_coding/timing/timing_module_gn/moz.build b/third_party/libwebrtc/modules/video_coding/timing/timing_module_gn/moz.build
new file mode 100644
index 0000000000..8ab43fb748
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/timing_module_gn/moz.build
@@ -0,0 +1,232 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+ ### This moz.build was AUTOMATICALLY GENERATED from a GN config, ###
+ ### DO NOT edit it by hand. ###
+
+COMPILE_FLAGS["OS_INCLUDES"] = []
+AllowCompilerWarnings()
+
+DEFINES["ABSL_ALLOCATOR_NOTHROW"] = "1"
+DEFINES["RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY"] = True
+DEFINES["RTC_ENABLE_VP9"] = True
+DEFINES["WEBRTC_ENABLE_PROTOBUF"] = "0"
+DEFINES["WEBRTC_LIBRARY_IMPL"] = True
+DEFINES["WEBRTC_MOZILLA_BUILD"] = True
+DEFINES["WEBRTC_NON_STATIC_TRACE_EVENT_HANDLERS"] = "0"
+DEFINES["WEBRTC_STRICT_FIELD_TRIALS"] = "0"
+
+FINAL_LIBRARY = "webrtc"
+
+
+LOCAL_INCLUDES += [
+ "!/ipc/ipdl/_ipdlheaders",
+ "!/third_party/libwebrtc/gen",
+ "/ipc/chromium/src",
+ "/third_party/libwebrtc/",
+ "/third_party/libwebrtc/third_party/abseil-cpp/",
+ "/tools/profiler/public"
+]
+
+UNIFIED_SOURCES += [
+ "/third_party/libwebrtc/modules/video_coding/timing/timing.cc"
+]
+
+if not CONFIG["MOZ_DEBUG"]:
+
+ DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "0"
+ DEFINES["NDEBUG"] = True
+ DEFINES["NVALGRIND"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1":
+
+ DEFINES["DYNAMIC_ANNOTATIONS_ENABLED"] = "1"
+
+if CONFIG["OS_TARGET"] == "Android":
+
+ DEFINES["ANDROID"] = True
+ DEFINES["ANDROID_NDK_VERSION_ROLL"] = "r22_1"
+ DEFINES["HAVE_SYS_UIO_H"] = True
+ DEFINES["WEBRTC_ANDROID"] = True
+ DEFINES["WEBRTC_ANDROID_OPENSLES"] = True
+ DEFINES["WEBRTC_LINUX"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_GNU_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+ OS_LIBS += [
+ "log"
+ ]
+
+if CONFIG["OS_TARGET"] == "Darwin":
+
+ DEFINES["WEBRTC_MAC"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_LIBCPP_HAS_NO_ALIGNED_ALLOCATION"] = True
+ DEFINES["__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES"] = "0"
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["USE_AURA"] = "1"
+ DEFINES["USE_GLIB"] = "1"
+ DEFINES["USE_NSS_CERTS"] = "1"
+ DEFINES["USE_OZONE"] = "1"
+ DEFINES["USE_UDEV"] = True
+ DEFINES["WEBRTC_LINUX"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_FILE_OFFSET_BITS"] = "64"
+ DEFINES["_LARGEFILE64_SOURCE"] = True
+ DEFINES["_LARGEFILE_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+ OS_LIBS += [
+ "rt"
+ ]
+
+if CONFIG["OS_TARGET"] == "OpenBSD":
+
+ DEFINES["USE_GLIB"] = "1"
+ DEFINES["USE_OZONE"] = "1"
+ DEFINES["USE_X11"] = "1"
+ DEFINES["WEBRTC_BSD"] = True
+ DEFINES["WEBRTC_POSIX"] = True
+ DEFINES["_FILE_OFFSET_BITS"] = "64"
+ DEFINES["_LARGEFILE64_SOURCE"] = True
+ DEFINES["_LARGEFILE_SOURCE"] = True
+ DEFINES["__STDC_CONSTANT_MACROS"] = True
+ DEFINES["__STDC_FORMAT_MACROS"] = True
+
+if CONFIG["OS_TARGET"] == "WINNT":
+
+ DEFINES["CERT_CHAIN_PARA_HAS_EXTRA_FIELDS"] = True
+ DEFINES["NOMINMAX"] = True
+ DEFINES["NTDDI_VERSION"] = "0x0A000000"
+ DEFINES["PSAPI_VERSION"] = "2"
+ DEFINES["UNICODE"] = True
+ DEFINES["USE_AURA"] = "1"
+ DEFINES["WEBRTC_WIN"] = True
+ DEFINES["WIN32"] = True
+ DEFINES["WIN32_LEAN_AND_MEAN"] = True
+ DEFINES["WINAPI_FAMILY"] = "WINAPI_FAMILY_DESKTOP_APP"
+ DEFINES["WINVER"] = "0x0A00"
+ DEFINES["_ATL_NO_OPENGL"] = True
+ DEFINES["_CRT_RAND_S"] = True
+ DEFINES["_CRT_SECURE_NO_DEPRECATE"] = True
+ DEFINES["_ENABLE_EXTENDED_ALIGNED_STORAGE"] = True
+ DEFINES["_HAS_EXCEPTIONS"] = "0"
+ DEFINES["_HAS_NODISCARD"] = True
+ DEFINES["_SCL_SECURE_NO_DEPRECATE"] = True
+ DEFINES["_SECURE_ATL"] = True
+ DEFINES["_UNICODE"] = True
+ DEFINES["_WIN32_WINNT"] = "0x0A00"
+ DEFINES["_WINDOWS"] = True
+ DEFINES["__STD_C"] = True
+
+ OS_LIBS += [
+ "crypt32",
+ "iphlpapi",
+ "secur32",
+ "winmm"
+ ]
+
+if CONFIG["CPU_ARCH"] == "aarch64":
+
+ DEFINES["WEBRTC_ARCH_ARM64"] = True
+ DEFINES["WEBRTC_HAS_NEON"] = True
+
+if CONFIG["CPU_ARCH"] == "arm":
+
+ CXXFLAGS += [
+ "-mfpu=neon"
+ ]
+
+ DEFINES["WEBRTC_ARCH_ARM"] = True
+ DEFINES["WEBRTC_ARCH_ARM_V7"] = True
+ DEFINES["WEBRTC_HAS_NEON"] = True
+
+if CONFIG["CPU_ARCH"] == "mips32":
+
+ DEFINES["MIPS32_LE"] = True
+ DEFINES["MIPS_FPU_LE"] = True
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "mips64":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86":
+
+ DEFINES["WEBRTC_ENABLE_AVX2"] = True
+
+if CONFIG["CPU_ARCH"] == "x86_64":
+
+ DEFINES["WEBRTC_ENABLE_AVX2"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Android":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Darwin":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "OpenBSD":
+
+ DEFINES["_DEBUG"] = True
+
+if CONFIG["MOZ_DEBUG"] == "1" and CONFIG["OS_TARGET"] == "WINNT":
+
+ DEFINES["_HAS_ITERATOR_DEBUGGING"] = "0"
+
+if CONFIG["MOZ_X11"] == "1" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["USE_X11"] = "1"
+
+if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Android":
+
+ OS_LIBS += [
+ "android_support",
+ "unwind"
+ ]
+
+if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Android":
+
+ CXXFLAGS += [
+ "-msse2"
+ ]
+
+ OS_LIBS += [
+ "android_support"
+ ]
+
+if CONFIG["CPU_ARCH"] == "aarch64" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "arm" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86" and CONFIG["OS_TARGET"] == "Linux":
+
+ CXXFLAGS += [
+ "-msse2"
+ ]
+
+ DEFINES["_GNU_SOURCE"] = True
+
+if CONFIG["CPU_ARCH"] == "x86_64" and CONFIG["OS_TARGET"] == "Linux":
+
+ DEFINES["_GNU_SOURCE"] = True
+
+Library("timing_module_gn")
diff --git a/third_party/libwebrtc/modules/video_coding/timing/timing_unittest.cc b/third_party/libwebrtc/modules/video_coding/timing/timing_unittest.cc
new file mode 100644
index 0000000000..8633c0de39
--- /dev/null
+++ b/third_party/libwebrtc/modules/video_coding/timing/timing_unittest.cc
@@ -0,0 +1,339 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/timing/timing.h"
+
+#include "api/units/frequency.h"
+#include "api/units/time_delta.h"
+#include "system_wrappers/include/clock.h"
+#include "test/gtest.h"
+#include "test/scoped_key_value_config.h"
+
+namespace webrtc {
+namespace {
+
+constexpr Frequency k25Fps = Frequency::Hertz(25);
+constexpr Frequency k90kHz = Frequency::KiloHertz(90);
+
+} // namespace
+
+TEST(ReceiverTimingTest, JitterDelay) {
+ test::ScopedKeyValueConfig field_trials;
+ SimulatedClock clock(0);
+ VCMTiming timing(&clock, field_trials);
+ timing.Reset();
+
+ uint32_t timestamp = 0;
+ timing.UpdateCurrentDelay(timestamp);
+
+ timing.Reset();
+
+ timing.IncomingTimestamp(timestamp, clock.CurrentTime());
+ TimeDelta jitter_delay = TimeDelta::Millis(20);
+ timing.SetJitterDelay(jitter_delay);
+ timing.UpdateCurrentDelay(timestamp);
+ timing.set_render_delay(TimeDelta::Zero());
+ auto wait_time = timing.MaxWaitingTime(
+ timing.RenderTime(timestamp, clock.CurrentTime()), clock.CurrentTime(),
+ /*too_many_frames_queued=*/false);
+ // First update initializes the render time. Since we have no decode delay
+ // we get wait_time = renderTime - now - renderDelay = jitter.
+ EXPECT_EQ(jitter_delay, wait_time);
+
+ jitter_delay += TimeDelta::Millis(VCMTiming::kDelayMaxChangeMsPerS + 10);
+ timestamp += 90000;
+ clock.AdvanceTimeMilliseconds(1000);
+ timing.SetJitterDelay(jitter_delay);
+ timing.UpdateCurrentDelay(timestamp);
+ wait_time = timing.MaxWaitingTime(
+ timing.RenderTime(timestamp, clock.CurrentTime()), clock.CurrentTime(),
+ /*too_many_frames_queued=*/false);
+ // Since we gradually increase the delay we only get 100 ms every second.
+ EXPECT_EQ(jitter_delay - TimeDelta::Millis(10), wait_time);
+
+ timestamp += 90000;
+ clock.AdvanceTimeMilliseconds(1000);
+ timing.UpdateCurrentDelay(timestamp);
+ wait_time = timing.MaxWaitingTime(
+ timing.RenderTime(timestamp, clock.CurrentTime()), clock.CurrentTime(),
+ /*too_many_frames_queued=*/false);
+ EXPECT_EQ(jitter_delay, wait_time);
+
+ // Insert frames without jitter, verify that this gives the exact wait time.
+ const int kNumFrames = 300;
+ for (int i = 0; i < kNumFrames; i++) {
+ clock.AdvanceTime(1 / k25Fps);
+ timestamp += k90kHz / k25Fps;
+ timing.IncomingTimestamp(timestamp, clock.CurrentTime());
+ }
+ timing.UpdateCurrentDelay(timestamp);
+ wait_time = timing.MaxWaitingTime(
+ timing.RenderTime(timestamp, clock.CurrentTime()), clock.CurrentTime(),
+ /*too_many_frames_queued=*/false);
+ EXPECT_EQ(jitter_delay, wait_time);
+
+ // Add decode time estimates for 1 second.
+ const TimeDelta kDecodeTime = TimeDelta::Millis(10);
+ for (int i = 0; i < k25Fps.hertz(); i++) {
+ clock.AdvanceTime(kDecodeTime);
+ timing.StopDecodeTimer(kDecodeTime, clock.CurrentTime());
+ timestamp += k90kHz / k25Fps;
+ clock.AdvanceTime(1 / k25Fps - kDecodeTime);
+ timing.IncomingTimestamp(timestamp, clock.CurrentTime());
+ }
+ timing.UpdateCurrentDelay(timestamp);
+ wait_time = timing.MaxWaitingTime(
+ timing.RenderTime(timestamp, clock.CurrentTime()), clock.CurrentTime(),
+ /*too_many_frames_queued=*/false);
+ EXPECT_EQ(jitter_delay, wait_time);
+
+ const TimeDelta kMinTotalDelay = TimeDelta::Millis(200);
+ timing.set_min_playout_delay(kMinTotalDelay);
+ clock.AdvanceTimeMilliseconds(5000);
+ timestamp += 5 * 90000;
+ timing.UpdateCurrentDelay(timestamp);
+ const TimeDelta kRenderDelay = TimeDelta::Millis(10);
+ timing.set_render_delay(kRenderDelay);
+ wait_time = timing.MaxWaitingTime(
+ timing.RenderTime(timestamp, clock.CurrentTime()), clock.CurrentTime(),
+ /*too_many_frames_queued=*/false);
+ // We should at least have kMinTotalDelayMs - decodeTime (10) - renderTime
+ // (10) to wait.
+ EXPECT_EQ(kMinTotalDelay - kDecodeTime - kRenderDelay, wait_time);
+ // The total video delay should be equal to the min total delay.
+ EXPECT_EQ(kMinTotalDelay, timing.TargetVideoDelay());
+
+ // Reset playout delay.
+ timing.set_min_playout_delay(TimeDelta::Zero());
+ clock.AdvanceTimeMilliseconds(5000);
+ timestamp += 5 * 90000;
+ timing.UpdateCurrentDelay(timestamp);
+}
+
+TEST(ReceiverTimingTest, TimestampWrapAround) {
+ constexpr auto kStartTime = Timestamp::Millis(1337);
+ test::ScopedKeyValueConfig field_trials;
+ SimulatedClock clock(kStartTime);
+ VCMTiming timing(&clock, field_trials);
+
+ // Provoke a wrap-around. The fifth frame will have wrapped at 25 fps.
+ constexpr uint32_t kRtpTicksPerFrame = k90kHz / k25Fps;
+ uint32_t timestamp = 0xFFFFFFFFu - 3 * kRtpTicksPerFrame;
+ for (int i = 0; i < 5; ++i) {
+ timing.IncomingTimestamp(timestamp, clock.CurrentTime());
+ clock.AdvanceTime(1 / k25Fps);
+ timestamp += kRtpTicksPerFrame;
+ EXPECT_EQ(kStartTime + 3 / k25Fps,
+ timing.RenderTime(0xFFFFFFFFu, clock.CurrentTime()));
+ // One ms later in 90 kHz.
+ EXPECT_EQ(kStartTime + 3 / k25Fps + TimeDelta::Millis(1),
+ timing.RenderTime(89u, clock.CurrentTime()));
+ }
+}
+
+TEST(ReceiverTimingTest, UseLowLatencyRenderer) {
+ test::ScopedKeyValueConfig field_trials;
+ SimulatedClock clock(0);
+ VCMTiming timing(&clock, field_trials);
+ timing.Reset();
+ // Default is false.
+ EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering);
+ // False if min playout delay > 0.
+ timing.set_min_playout_delay(TimeDelta::Millis(10));
+ timing.set_max_playout_delay(TimeDelta::Millis(20));
+ EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering);
+ // True if min==0, max > 0.
+ timing.set_min_playout_delay(TimeDelta::Zero());
+ EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering);
+ // True if min==max==0.
+ timing.set_max_playout_delay(TimeDelta::Zero());
+ EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering);
+ // True also for max playout delay==500 ms.
+ timing.set_max_playout_delay(TimeDelta::Millis(500));
+ EXPECT_TRUE(timing.RenderParameters().use_low_latency_rendering);
+ // False if max playout delay > 500 ms.
+ timing.set_max_playout_delay(TimeDelta::Millis(501));
+ EXPECT_FALSE(timing.RenderParameters().use_low_latency_rendering);
+}
+
+TEST(ReceiverTimingTest, MaxWaitingTimeIsZeroForZeroRenderTime) {
+ // This is the default path when the RTP playout delay header extension is set
+ // to min==0 and max==0.
+ constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us.
+ constexpr TimeDelta kTimeDelta = 1 / Frequency::Hertz(60);
+ constexpr Timestamp kZeroRenderTime = Timestamp::Zero();
+ SimulatedClock clock(kStartTimeUs);
+ test::ScopedKeyValueConfig field_trials;
+ VCMTiming timing(&clock, field_trials);
+ timing.Reset();
+ timing.set_max_playout_delay(TimeDelta::Zero());
+ for (int i = 0; i < 10; ++i) {
+ clock.AdvanceTime(kTimeDelta);
+ Timestamp now = clock.CurrentTime();
+ EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now,
+ /*too_many_frames_queued=*/false),
+ TimeDelta::Zero());
+ }
+ // Another frame submitted at the same time also returns a negative max
+ // waiting time.
+ Timestamp now = clock.CurrentTime();
+ EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now,
+ /*too_many_frames_queued=*/false),
+ TimeDelta::Zero());
+ // MaxWaitingTime should be less than zero even if there's a burst of frames.
+ EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now,
+ /*too_many_frames_queued=*/false),
+ TimeDelta::Zero());
+ EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now,
+ /*too_many_frames_queued=*/false),
+ TimeDelta::Zero());
+ EXPECT_LT(timing.MaxWaitingTime(kZeroRenderTime, now,
+ /*too_many_frames_queued=*/false),
+ TimeDelta::Zero());
+}
+
+TEST(ReceiverTimingTest, MaxWaitingTimeZeroDelayPacingExperiment) {
+ // The minimum pacing is enabled by a field trial and active if the RTP
+ // playout delay header extension is set to min==0.
+ constexpr TimeDelta kMinPacing = TimeDelta::Millis(3);
+ test::ScopedKeyValueConfig field_trials(
+ "WebRTC-ZeroPlayoutDelay/min_pacing:3ms/");
+ constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us.
+ constexpr TimeDelta kTimeDelta = 1 / Frequency::Hertz(60);
+ constexpr auto kZeroRenderTime = Timestamp::Zero();
+ SimulatedClock clock(kStartTimeUs);
+ VCMTiming timing(&clock, field_trials);
+ timing.Reset();
+ // MaxWaitingTime() returns zero for evenly spaced video frames.
+ for (int i = 0; i < 10; ++i) {
+ clock.AdvanceTime(kTimeDelta);
+ Timestamp now = clock.CurrentTime();
+ EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now,
+ /*too_many_frames_queued=*/false),
+ TimeDelta::Zero());
+ timing.SetLastDecodeScheduledTimestamp(now);
+ }
+ // Another frame submitted at the same time is paced according to the field
+ // trial setting.
+ auto now = clock.CurrentTime();
+ EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now,
+ /*too_many_frames_queued=*/false),
+ kMinPacing);
+ // If there's a burst of frames, the wait time is calculated based on next
+ // decode time.
+ EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now,
+ /*too_many_frames_queued=*/false),
+ kMinPacing);
+ EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now,
+ /*too_many_frames_queued=*/false),
+ kMinPacing);
+ // Allow a few ms to pass, this should be subtracted from the MaxWaitingTime.
+ constexpr TimeDelta kTwoMs = TimeDelta::Millis(2);
+ clock.AdvanceTime(kTwoMs);
+ now = clock.CurrentTime();
+ EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now,
+ /*too_many_frames_queued=*/false),
+ kMinPacing - kTwoMs);
+ // A frame is decoded at the current time, the wait time should be restored to
+ // pacing delay.
+ timing.SetLastDecodeScheduledTimestamp(now);
+ EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now,
+ /*too_many_frames_queued=*/false),
+ kMinPacing);
+}
+
+TEST(ReceiverTimingTest, DefaultMaxWaitingTimeUnaffectedByPacingExperiment) {
+ // The minimum pacing is enabled by a field trial but should not have any
+ // effect if render_time_ms is greater than 0;
+ test::ScopedKeyValueConfig field_trials(
+ "WebRTC-ZeroPlayoutDelay/min_pacing:3ms/");
+ constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us.
+ const TimeDelta kTimeDelta = TimeDelta::Millis(1000.0 / 60.0);
+ SimulatedClock clock(kStartTimeUs);
+ VCMTiming timing(&clock, field_trials);
+ timing.Reset();
+ clock.AdvanceTime(kTimeDelta);
+ auto now = clock.CurrentTime();
+ Timestamp render_time = now + TimeDelta::Millis(30);
+ // Estimate the internal processing delay from the first frame.
+ TimeDelta estimated_processing_delay =
+ (render_time - now) -
+ timing.MaxWaitingTime(render_time, now,
+ /*too_many_frames_queued=*/false);
+ EXPECT_GT(estimated_processing_delay, TimeDelta::Zero());
+
+ // Any other frame submitted at the same time should be scheduled according to
+ // its render time.
+ for (int i = 0; i < 5; ++i) {
+ render_time += kTimeDelta;
+ EXPECT_EQ(timing.MaxWaitingTime(render_time, now,
+ /*too_many_frames_queued=*/false),
+ render_time - now - estimated_processing_delay);
+ }
+}
+
+TEST(ReceiverTimingTest, MaxWaitingTimeReturnsZeroIfTooManyFramesQueuedIsTrue) {
+ // The minimum pacing is enabled by a field trial and active if the RTP
+ // playout delay header extension is set to min==0.
+ constexpr TimeDelta kMinPacing = TimeDelta::Millis(3);
+ test::ScopedKeyValueConfig field_trials(
+ "WebRTC-ZeroPlayoutDelay/min_pacing:3ms/");
+ constexpr int64_t kStartTimeUs = 3.15e13; // About one year in us.
+ const TimeDelta kTimeDelta = TimeDelta::Millis(1000.0 / 60.0);
+ constexpr auto kZeroRenderTime = Timestamp::Zero();
+ SimulatedClock clock(kStartTimeUs);
+ VCMTiming timing(&clock, field_trials);
+ timing.Reset();
+ // MaxWaitingTime() returns zero for evenly spaced video frames.
+ for (int i = 0; i < 10; ++i) {
+ clock.AdvanceTime(kTimeDelta);
+ auto now = clock.CurrentTime();
+ EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now,
+ /*too_many_frames_queued=*/false),
+ TimeDelta::Zero());
+ timing.SetLastDecodeScheduledTimestamp(now);
+ }
+ // Another frame submitted at the same time is paced according to the field
+ // trial setting.
+ auto now_ms = clock.CurrentTime();
+ EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now_ms,
+ /*too_many_frames_queued=*/false),
+ kMinPacing);
+ // MaxWaitingTime returns 0 even if there's a burst of frames if
+ // too_many_frames_queued is set to true.
+ EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now_ms,
+ /*too_many_frames_queued=*/true),
+ TimeDelta::Zero());
+ EXPECT_EQ(timing.MaxWaitingTime(kZeroRenderTime, now_ms,
+ /*too_many_frames_queued=*/true),
+ TimeDelta::Zero());
+}
+
+TEST(ReceiverTimingTest, UpdateCurrentDelayCapsWhenOffByMicroseconds) {
+ test::ScopedKeyValueConfig field_trials;
+ SimulatedClock clock(0);
+ VCMTiming timing(&clock, field_trials);
+ timing.Reset();
+
+ // Set larger initial current delay.
+ timing.set_min_playout_delay(TimeDelta::Millis(200));
+ timing.UpdateCurrentDelay(Timestamp::Millis(900), Timestamp::Millis(1000));
+
+ // Add a few microseconds to ensure that the delta of decode time is 0 after
+ // rounding, and should reset to the target delay.
+ timing.set_min_playout_delay(TimeDelta::Millis(50));
+ Timestamp decode_time = Timestamp::Millis(1337);
+ Timestamp render_time =
+ decode_time + TimeDelta::Millis(10) + TimeDelta::Micros(37);
+ timing.UpdateCurrentDelay(render_time, decode_time);
+ EXPECT_EQ(timing.GetTimings().current_delay, timing.TargetVideoDelay());
+}
+
+} // namespace webrtc